[clang-tools-extra] [compiler-rt] [llvm] [clang] [IRPGO][ValueProfile] Instrument virtual table address that could be used to do virtual table address comparision for indirect-call-promotion. (PR #66825)
Mingming Liu via cfe-commits
cfe-commits at lists.llvm.org
Tue Nov 7 11:14:21 PST 2023
https://github.com/minglotus-6 updated https://github.com/llvm/llvm-project/pull/66825
>From 361f8f6f884a6026e5ee65ceeea428d3db0b085b Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Sat, 16 Sep 2023 23:15:15 -0700
Subject: [PATCH 01/10] [IRPGO][ValueProfile] Instrument virtual table address
that could be used to do virtual table address comparision for
indirect-call-promotion.
The changes include:
1) Insert value profile intrinsics and lowering them.
- Introduced INSTR_PROF_VTABLE_DATA to record per-vtable data.
- Modified LLVM_PROF_RAW_HEADER to record the metadata for vtable profiles.
- Test case in llvm/test/Transforms/PGOProfile/vtable_profile.ll
2) Tooling support in llvm-profdata to show the added vtable information
- Changes are made in {raw,text,indexed} prof reader and/or writer to read/write vtable profile data.
- Test cases added in llvm/test/tools/llvm-profdata
---
compiler-rt/include/profile/InstrProfData.inc | 32 +++-
compiler-rt/lib/profile/InstrProfiling.h | 20 +-
.../lib/profile/InstrProfilingBuffer.c | 60 +++++-
.../lib/profile/InstrProfilingInternal.h | 11 +-
compiler-rt/lib/profile/InstrProfilingMerge.c | 20 +-
.../lib/profile/InstrProfilingPlatformLinux.c | 20 ++
.../lib/profile/InstrProfilingWriter.c | 45 ++++-
.../Linux/instrprof-value-prof-warn.test | 2 +-
.../profile/instrprof-write-buffer-internal.c | 10 +-
.../llvm/Analysis/IndirectCallVisitor.h | 22 ++-
llvm/include/llvm/ProfileData/InstrProf.h | 125 ++++++++++++-
.../llvm/ProfileData/InstrProfData.inc | 31 +++-
.../llvm/ProfileData/InstrProfReader.h | 59 +++++-
.../llvm/ProfileData/InstrProfWriter.h | 4 +
.../Instrumentation/InstrProfiling.h | 12 ++
llvm/lib/ProfileData/InstrProf.cpp | 172 +++++++++++++++---
llvm/lib/ProfileData/InstrProfReader.cpp | 79 +++++++-
llvm/lib/ProfileData/InstrProfWriter.cpp | 92 +++++++++-
.../Instrumentation/InstrProfiling.cpp | 165 +++++++++++++++++
.../Instrumentation/PGOInstrumentation.cpp | 1 +
.../Instrumentation/ValueProfilePlugins.inc | 26 ++-
.../Inputs/update_vtable_value_prof_inputs.sh | 84 +++++++++
.../Transforms/PGOProfile/comdat_internal.ll | 4 +-
.../indirect_call_profile_funclet.ll | 5 +-
.../Transforms/PGOProfile/vtable_profile.ll | 139 ++++++++++++++
.../llvm-profdata/Inputs/c-general.profraw | Bin 1800 -> 1912 bytes
.../llvm-profdata/Inputs/vtable-prof.proftext | 73 ++++++++
.../llvm-profdata/Inputs/vtable_prof.profraw | Bin 0 -> 816 bytes
.../llvm-profdata/binary-ids-padding.test | 10 +-
.../llvm-profdata/large-binary-id-size.test | 4 +-
...alformed-not-space-for-another-header.test | 7 +-
.../malformed-num-counters-zero.test | 7 +-
.../malformed-ptr-to-counter-array.test | 7 +-
.../misaligned-binary-ids-size.test | 4 +-
.../mismatched-raw-profile-header.test | 2 +
.../tools/llvm-profdata/raw-32-bits-be.test | 7 +-
.../tools/llvm-profdata/raw-32-bits-le.test | 6 +-
.../tools/llvm-profdata/raw-64-bits-be.test | 13 +-
.../tools/llvm-profdata/raw-64-bits-le.test | 29 ++-
.../tools/llvm-profdata/raw-two-profiles.test | 10 +-
.../tools/llvm-profdata/vtable-prof.proftext | 16 ++
.../vtable-value-prof-basic.test | 100 ++++++++++
llvm/tools/llvm-profdata/llvm-profdata.cpp | 37 +++-
43 files changed, 1472 insertions(+), 100 deletions(-)
create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
create mode 100644 llvm/test/Transforms/PGOProfile/vtable_profile.ll
create mode 100644 llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
create mode 100644 llvm/test/tools/llvm-profdata/Inputs/vtable_prof.profraw
create mode 100644 llvm/test/tools/llvm-profdata/vtable-prof.proftext
create mode 100644 llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index 8ba7e186d4fb1a6..5609870a5a8e5b4 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -92,6 +92,19 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
/* INSTR_PROF_DATA end. */
+#ifndef INSTR_PROF_VTABLE_DATA
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_VTABLE_DATA_DEFINED
+#endif
+INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, \
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(VTableName)))
+INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), VTablePointer, VTableAddr)
+INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), VTableSizeVal))
+#undef INSTR_PROF_VTABLE_DATA
+/* INSTR_PROF_VTABLE_DATA end. */
+
/* This is an internal data structure used by value profiler. It
* is defined here to allow serialization code sharing by LLVM
* to be used in unit test.
@@ -136,6 +149,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
(uintptr_t)CountersBegin - (uintptr_t)DataBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
#undef INSTR_PROF_RAW_HEADER
/* INSTR_PROF_RAW_HEADER end */
@@ -177,13 +192,14 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
+VALUE_PROF_KIND(IPVK_VTableTarget, 2, "vtable target")
/* These two kinds must be the last to be
* declared. This is to make sure the string
* array created with the template can be
* indexed with the kind value.
*/
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
-VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
+VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")
#undef VALUE_PROF_KIND
/* VALUE_PROF_KIND end */
@@ -270,12 +286,18 @@ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
INSTR_PROF_SECT_ENTRY(IPSK_name, \
INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
INSTR_PROF_NAME_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vname, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
+ INSTR_PROF_VNAME_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vals, \
INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
INSTR_PROF_VALS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
INSTR_PROF_VNODES_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
+ INSTR_PROF_VTAB_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
@@ -645,9 +667,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 8
+#define INSTR_PROF_RAW_VERSION 9
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 10
+#define INSTR_PROF_INDEX_VERSION 11
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 5
@@ -685,9 +707,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
+#define INSTR_PROF_VNAME_COMMON __llvm_prf_vnames
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
+#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
@@ -696,9 +720,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
*/
#define INSTR_PROF_DATA_COFF ".lprfd$M"
#define INSTR_PROF_NAME_COFF ".lprfn$M"
+#define INSTR_PROF_VNAME_COFF ".lprfn$M"
#define INSTR_PROF_CNTS_COFF ".lprfc$M"
#define INSTR_PROF_VALS_COFF ".lprfv$M"
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
+#define INSTR_PROF_VTAB_COFF ".lprfvt$M"
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index 4433d7bd48871fc..f3afa694e02c27d 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -38,6 +38,12 @@ typedef struct ValueProfNode {
#include "profile/InstrProfData.inc"
} ValueProfNode;
+typedef void *IntPtrT;
+typedef struct VTableProfData {
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) Type Name;
+#include "profile/InstrProfData.inc"
+} VTableProfData;
+
/*!
* \brief Return 1 if profile counters are continuously synced to the raw
* profile via an mmap(). This is in contrast to the default mode, in which
@@ -86,10 +92,14 @@ const __llvm_profile_data *__llvm_profile_begin_data(void);
const __llvm_profile_data *__llvm_profile_end_data(void);
const char *__llvm_profile_begin_names(void);
const char *__llvm_profile_end_names(void);
+const char *__llvm_profile_begin_vnames(void);
+const char *__llvm_profile_end_vnames(void);
char *__llvm_profile_begin_counters(void);
char *__llvm_profile_end_counters(void);
ValueProfNode *__llvm_profile_begin_vnodes();
ValueProfNode *__llvm_profile_end_vnodes();
+VTableProfData *__llvm_profile_begin_vtables();
+VTableProfData *__llvm_profile_end_vtables();
uint32_t *__llvm_profile_begin_orderfile();
/*!
@@ -276,6 +286,12 @@ uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End);
/*! \brief Get the size of the profile counters section in bytes. */
uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End);
+uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
+ const VTableProfData *End);
+
+uint64_t __llvm_profile_get_vtable_size(const VTableProfData *Begin,
+ const VTableProfData *End);
+
/* ! \brief Given the sizes of the data and counter information, return the
* number of padding bytes before and after the counters, and after the names,
* in the raw profile.
@@ -287,8 +303,10 @@ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End);
*/
void __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize,
+ uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
- uint64_t *PaddingBytesAfterNames);
+ uint64_t *PaddingBytesAfterNames, uint64_t *PaddingBytesAfterVTable,
+ uint64_t *PaddingBytesAfterVNames);
/*!
* \brief Set the flag that profile data has been dumped to the file.
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index 61ac5d9c0285002..0c36e40444c7344 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -9,6 +9,8 @@
// Note: This is linked into the Darwin kernel, and must remain compatible
// with freestanding compilation. See `darwin_add_builtin_libraries`.
+#include <assert.h>
+
#include "InstrProfiling.h"
#include "InstrProfilingInternal.h"
#include "InstrProfilingPort.h"
@@ -45,9 +47,14 @@ uint64_t __llvm_profile_get_size_for_buffer(void) {
const char *CountersEnd = __llvm_profile_end_counters();
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
+ const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
+ const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
+ const char *VNamesBegin = __llvm_profile_begin_vnames();
+ const char *VNamesEnd = __llvm_profile_end_vnames();
return __llvm_profile_get_size_for_buffer_internal(
- DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd);
+ DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd,
+ VTableBegin, VTableEnd, VNamesBegin, VNamesEnd);
}
COMPILER_RT_VISIBILITY
@@ -63,6 +70,18 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
const __llvm_profile_data *End) {
return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);
}
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
+ const VTableProfData *End) {
+ intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin;
+ return (EndI + sizeof(VTableProfData) - 1 - BeginI) / sizeof(VTableProfData);
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_vtable_size(const VTableProfData *Begin,
+ const VTableProfData *End) {
+ return __llvm_profile_get_num_vtable(Begin, End) * sizeof(VTableProfData);
+}
COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) {
if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE)
@@ -103,46 +122,68 @@ static int needsCounterPadding(void) {
COMPILER_RT_VISIBILITY
void __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize,
+ uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
- uint64_t *PaddingBytesAfterNames) {
+ uint64_t *PaddingBytesAfterNames, uint64_t *PaddingBytesAfterVTable,
+ uint64_t *PaddingBytesAfterVName) {
+ // Counter padding is needed only if continuous mode is enabled.
if (!needsCounterPadding()) {
*PaddingBytesBeforeCounters = 0;
*PaddingBytesAfterCounters =
__llvm_profile_get_num_padding_bytes(CountersSize);
*PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
+ *PaddingBytesAfterVTable = __llvm_profile_get_num_padding_bytes(VTableSize);
+ *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);
return;
}
+ // Value profiling not supported in continuous mode at profile-write time
+ // according to
+ // https://github.com/llvm/llvm-project/blob/e6a007f6b51a661ed3dd8b0210b734b3e9b4354f/compiler-rt/lib/profile/InstrProfilingWriter.c#L328
+ assert(VTableSize == 0 && VNameSize == 0 &&
+ "Value profile not supported for continuous mode");
// In continuous mode, the file offsets for headers and for the start of
// counter sections need to be page-aligned.
*PaddingBytesBeforeCounters =
calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize);
*PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize);
*PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
+ // Set these two variables to zero to avoid uninitialized variables
+ // even if VTableSize and VNameSize are asserted to be zero.
+ *PaddingBytesAfterVTable = 0;
+ *PaddingBytesAfterVName = 0;
}
COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
- const char *NamesEnd) {
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd) {
/* Match logic in __llvm_profile_write_buffer(). */
const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
uint64_t CountersSize =
__llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+ uint64_t VTableSize = __llvm_profile_get_vtable_size(VTableBegin, VTableEnd);
+ uint64_t VNameSize = (VNamesEnd - VNamesBegin) * sizeof(char);
/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterNames;
+ PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
__llvm_profile_get_padding_sizes_for_counters(
- DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
- &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
+ DataSize, CountersSize, NamesSize, VTableSize, VNameSize,
+ &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
+ &PaddingBytesAfterNames, &PaddingBytesAfterVTable,
+ &PaddingBytesAfterVNames);
return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
DataSize + PaddingBytesBeforeCounters + CountersSize +
- PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames;
+ PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames +
+ VTableSize + PaddingBytesAfterVTable + VNameSize +
+ PaddingBytesAfterVNames;
}
COMPILER_RT_VISIBILITY
@@ -163,6 +204,9 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
const char *CountersEnd, const char *NamesBegin, const char *NamesEnd) {
ProfDataWriter BufferWriter;
initBufferWriter(&BufferWriter, Buffer);
+ // Set virtual table arguments to NULL since they are not supported yet.
return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin,
- CountersEnd, 0, NamesBegin, NamesEnd, 0);
+ CountersEnd, 0, NamesBegin, NamesEnd,
+ NULL /* VTableBegin */, NULL /* VTableEnd */,
+ NULL /* VNamesBegin */, NULL /* VNamesEnd */, 0);
}
diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h
index 360165e32ab3fe2..bce333f933f0ffe 100644
--- a/compiler-rt/lib/profile/InstrProfilingInternal.h
+++ b/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -18,11 +18,16 @@
* pointers to the live data in memory. This function is probably not what you
* want. Use __llvm_profile_get_size_for_buffer instead. Use this function if
* your program has a custom memory layout.
+ * NOTE: The change of function signature requires modifying c source code
+ * as demonstrated by the existing tests. If this is causing backward
+ * compatible issues, considering adding another function for new use cases.
*/
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
- const char *NamesEnd);
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd);
/*!
* \brief Write instrumentation data to the given buffer, given explicit
@@ -154,7 +159,9 @@ int lprofWriteDataImpl(ProfDataWriter *Writer,
const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd,
VPDataReaderType *VPDataReader, const char *NamesBegin,
- const char *NamesEnd, int SkipNameDataWrite);
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd, int SkipNameDataWrite);
/* Merge value profile data pointed to by SrcValueProfData into
* in-memory profile counters pointed by to DstData. */
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 9cf12f251f7262d..2ef6227599ff139 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -124,9 +124,27 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
SrcCountersEnd = SrcCountersStart +
Header->NumCounters * __llvm_profile_counter_entry_size();
SrcNameStart = SrcCountersEnd;
- SrcValueProfDataStart =
+ // This is to assume counter size is a multiple of 8 bytes.
+ // uint64_t NamesSize = Header->NamesSize;
+ // uint64_t PaddingBytesAfterNames =
+ // __llvm_profile_get_num_padding_bytes(Header->NamesSize);
+ // First, skip rather than merge them
+ uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData);
+ uint64_t PaddingBytesAfterVTableSection =
+ __llvm_profile_get_num_padding_bytes(VTableSectionSize);
+ uint64_t VNamesSize = Header->VNamesSize;
+ uint64_t PaddingBytesAfterVNamesSize =
+ __llvm_profile_get_num_padding_bytes(VNamesSize);
+
+ uint64_t VTableProfDataOffset =
SrcNameStart + Header->NamesSize +
__llvm_profile_get_num_padding_bytes(Header->NamesSize);
+
+ uint64_t VTableNamesOffset =
+ VTableProfDataOffset + VTableSectionSize + PaddingBytesAfterVTableSection;
+
+ SrcValueProfDataStart =
+ VTableNamesOffset + VNamesSize + PaddingBytesAfterVNamesSize;
if (SrcNameStart < SrcCountersStart)
return 1;
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
index 2cce0a4b2c48d35..dc861632271ce79 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -33,8 +33,12 @@
#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON)
#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON)
#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON)
+#define PROF_VNAME_START INSTR_PROF_SECT_START(INSTR_PROF_VNAME_COMMON)
+#define PROF_VNAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNAME_COMMON)
#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON)
#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON)
+#define PROF_VTABLE_START INSTR_PROF_SECT_START(INSTR_PROF_VTAB_COMMON)
+#define PROF_VTABLE_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VTAB_COMMON)
#define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON)
#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON)
#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON)
@@ -48,6 +52,10 @@ extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY
COMPILER_RT_WEAK;
extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern VTableProfData PROF_VTABLE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern VTableProfData PROF_VTABLE_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_VNAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_VNAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
@@ -68,6 +76,18 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) {
COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) {
return &PROF_NAME_STOP;
}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vnames(void) {
+ return &PROF_VNAME_START;
+}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vnames(void) {
+ return &PROF_VNAME_STOP;
+}
+COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_begin_vtables(void) {
+ return &PROF_VTABLE_START;
+}
+COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_end_vtables(void) {
+ return &PROF_VTABLE_STOP;
+}
COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) {
return &PROF_CNTS_START;
}
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 1e22398a4c0f64a..ed5dbdb6ee4383f 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -248,8 +248,13 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
const char *CountersEnd = __llvm_profile_end_counters();
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
+ const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
+ const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
+ const char *VNamesBegin = __llvm_profile_begin_vnames();
+ const char *VNamesEnd = __llvm_profile_end_vnames();
return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
CountersEnd, VPDataReader, NamesBegin, NamesEnd,
+ VTableBegin, VTableEnd, VNamesBegin, VNamesEnd,
SkipNameDataWrite);
}
@@ -258,7 +263,9 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd,
VPDataReaderType *VPDataReader, const char *NamesBegin,
- const char *NamesEnd, int SkipNameDataWrite) {
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd, int SkipNameDataWrite) {
int DebugInfoCorrelate =
(__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL;
@@ -272,6 +279,13 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const uint64_t NumCounters =
__llvm_profile_get_num_counters(CountersBegin, CountersEnd);
const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin;
+ const uint64_t NumVTables =
+ __llvm_profile_get_num_vtable(VTableBegin, VTableEnd);
+ const uint64_t VTableSectionSize =
+ __llvm_profile_get_vtable_size(VTableBegin, VTableEnd);
+ // Note, in reality, vtable profiling is not supported when DebugInfoCorrelate
+ // is true.
+ const uint64_t VNamesSize = DebugInfoCorrelate ? 0 : VNamesEnd - VNamesBegin;
/* Create the header. */
__llvm_profile_header Header;
@@ -279,11 +293,12 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterNames;
+ PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
__llvm_profile_get_padding_sizes_for_counters(
- DataSectionSize, CountersSectionSize, NamesSize,
- &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
- &PaddingBytesAfterNames);
+ DataSectionSize, CountersSectionSize, NamesSize, VTableSectionSize,
+ VNamesSize, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
+ &PaddingBytesAfterNames, &PaddingBytesAfterVTable,
+ &PaddingBytesAfterVNames);
{
/* Initialize header structure. */
@@ -305,13 +320,24 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
/* Write the profile header. */
ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
+ // printf("Size of profile header is %d\n",
+ // (int)(sizeof(__llvm_profile_header)));
if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
return -1;
+ // printf("Completed profile header\n");
+
/* Write the binary id lengths and data. */
- if (__llvm_write_binary_ids(Writer) == -1)
+ int binary_id_size = __llvm_write_binary_ids(Writer);
+ if (binary_id_size == -1)
return -1;
+ // Might be needed for debugging. Clean up before commit.
+ // uint64_t VTableProfDataOffset =
+ // sizeof(__llvm_profile_header) + binary_id_size + DataSectionSize +
+ // PaddingBytesBeforeCounters + CountersSectionSize +
+ // PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames;
+
/* Write the profile data. */
ProfDataIOVec IOVecData[] = {
{DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize,
@@ -321,7 +347,12 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
{NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
{(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin,
sizeof(uint8_t), NamesSize, 0},
- {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
+ {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1},
+ {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1},
+ {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : VNamesBegin,
+ sizeof(uint8_t), VNamesSize, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}};
if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
return -1;
diff --git a/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test b/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
index 991fd2b69b5b303..89787e79c20e9d1 100644
--- a/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
+++ b/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
@@ -1,7 +1,7 @@
RUN: %clang_pgogen -O2 -fuse-ld=bfd -mllvm -disable-vp=false -mllvm -vp-static-alloc=true -DSTRESS=1 -o %t.ir.warn %S/../Inputs/instrprof-value-prof-real.c
RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=255 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=WARNING %s
# Test that enough static counters have been allocated
-RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=130 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=NOWARNING --allow-empty %s
+RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=80 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=NOWARNING --allow-empty %s
# WARNING: LLVM Profile Warning:
# NOWARNING-NOT: LLVM Profile Warning:
diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
index 7b96c6d91c33f5a..97dfff8d7fe71fb 100644
--- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c
+++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
@@ -28,7 +28,9 @@ char *__llvm_profile_end_counters(void);
uint64_t __llvm_profile_get_size_for_buffer_internal(
const void *DataBegin, const void *DataEnd, const char *CountersBegin,
- const char *CountersEnd, const char *NamesBegin, const char *NamesEnd);
+ const char *CountersEnd, const char *NamesBegin, const char *NamesEnd,
+ const char *VTableBegin, const char *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd);
int __llvm_profile_write_buffer_internal(char *Buffer, const void *DataBegin,
const void *DataEnd,
@@ -43,7 +45,11 @@ int main(int argc, const char *argv[]) {
uint64_t bufsize = __llvm_profile_get_size_for_buffer_internal(
__llvm_profile_begin_data(), __llvm_profile_end_data(),
__llvm_profile_begin_counters(), __llvm_profile_end_counters(),
- __llvm_profile_begin_names(), __llvm_profile_end_names());
+ __llvm_profile_begin_names(), __llvm_profile_end_names(), NULL, NULL,
+ NULL, NULL);
+
+ // printf("buffer size is %lld\n", bufsize);
+ //uint64_t aligned_bufsize = ((bufsize + 32) >> 6) << 6;
char *buf = malloc(bufsize);
int ret = __llvm_profile_write_buffer_internal(buf,
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 0825e19ecd2d240..52d4ff04d3d4ecc 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -19,20 +19,38 @@ namespace llvm {
// Visitor class that finds all indirect call.
struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
std::vector<CallBase *> IndirectCalls;
+ std::vector<Instruction *> VTableAddrs;
PGOIndirectCallVisitor() = default;
void visitCallBase(CallBase &Call) {
if (Call.isIndirectCall())
- IndirectCalls.push_back(&Call);
+ if (Call.isIndirectCall()) {
+ IndirectCalls.push_back(&Call);
+
+ LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
+ if (LI != nullptr) {
+ Value *Ptr = LI->getPointerOperand();
+ Value *VTablePtr = Ptr->stripInBoundsConstantOffsets();
+ if (VTablePtr != nullptr && isa<Instruction>(VTablePtr)) {
+ VTableAddrs.push_back(cast<Instruction>(VTablePtr));
+ }
+ }
+ }
}
};
-// Helper function that finds all indirect call sites.
inline std::vector<CallBase *> findIndirectCalls(Function &F) {
PGOIndirectCallVisitor ICV;
ICV.visit(F);
return ICV.IndirectCalls;
}
+
+inline std::vector<Instruction *> findVTableAddrs(Function &F) {
+ PGOIndirectCallVisitor ICV;
+ ICV.visit(F);
+ return ICV.VTableAddrs;
+}
+
} // namespace llvm
#endif
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index e968f8ffd5075fd..845ea1d0ab0b3af 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -90,6 +90,9 @@ inline StringRef getInstrProfValueProfMemOpFuncName() {
/// Return the name prefix of variables containing instrumented function names.
inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
+/// Return the name prefix of variables containing virtual table profile data.
+inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; }
+
/// Return the name prefix of variables containing per-function control data.
inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
@@ -108,6 +111,8 @@ inline StringRef getInstrProfNamesVarName() {
return "__llvm_prf_nm";
}
+inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; }
+
/// Return the name of a covarage mapping variable (internal linkage)
/// for each instrumented source module. Such variables are allocated
/// in the __llvm_covmap section.
@@ -240,11 +245,16 @@ Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
std::string &Result, bool doCompression = true);
+Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
+ std::string &Result, bool doCompression);
+
/// \c NameStrings is a string composed of one of more sub-strings encoded in
/// the format described above. The substrings are separated by 0 or more zero
/// bytes. This method decodes the string and populates the \c Symtab.
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);
+Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab);
+
/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
/// set in IR PGO compilation.
bool isIRPGOFlagSet(const Module *M);
@@ -294,7 +304,7 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
/// Check if we can use Comdat for profile variables. This will eliminate
/// the duplicated profile variables for Comdat functions.
-bool needsComdatForCounter(const Function &F, const Module &M);
+bool needsComdatForCounter(const GlobalValue &GV, const Module &M);
/// An enum describing the attributes of an instrumented profile.
enum class InstrProfKind {
@@ -432,14 +442,24 @@ class InstrProfSymtab {
uint64_t Address = 0;
// Unique name strings.
StringSet<> NameTab;
+ // Unique virtual table names.
+ StringSet<> VTableNames;
// A map from MD5 keys to function name strings.
std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
+ // A map from MD5 keys to virtual table definitions. Only populated when
+ // building the Symtab from a module.
+ std::vector<std::pair<uint64_t, GlobalVariable *>> MD5VTableMap;
// A map from MD5 keys to function define. We only populate this map
// when build the Symtab from a Module.
std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
// A map from function runtime address to function name MD5 hash.
// This map is only populated and used by raw instr profile reader.
AddrHashMap AddrToMD5Map;
+ // A map from virtual table runtime address to function name MD5 hash.
+ // This map is only populated and used by raw instr profile reader.
+ // This is a different map from 'AddrToMD5Map' for readability and
+ // debuggability.
+ AddrHashMap VTableAddrToMD5Map;
bool Sorted = false;
static StringRef getExternalSymbol() {
@@ -474,6 +494,8 @@ class InstrProfSymtab {
/// This method is a wrapper to \c readPGOFuncNameStrings method.
inline Error create(StringRef NameStrings);
+ inline Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
+
/// A wrapper interface to populate the PGO symtab with functions
/// decls from module \c M. This interface is used by transformation
/// passes such as indirect function call promotion. Variable \c InLTO
@@ -484,6 +506,13 @@ class InstrProfSymtab {
/// \p IterRange. This interface is used by IndexedProfReader.
template <typename NameIterRange> Error create(const NameIterRange &IterRange);
+ /// Create InstrProfSymtab from a set of function names and vtable
+ /// names iteratable from \p IterRange. This interface is used by
+ /// IndexedProfReader.
+ template <typename FuncNameIterRange, typename VTableNameIterRange>
+ Error create(const FuncNameIterRange &FuncIterRange,
+ const VTableNameIterRange &VTableIterRange);
+
/// Update the symtab by adding \p FuncName to the table. This interface
/// is used by the raw and text profile readers.
Error addFuncName(StringRef FuncName) {
@@ -499,15 +528,49 @@ class InstrProfSymtab {
return Error::success();
}
+ Error addVTableName(StringRef VTableName) {
+ if (VTableName.empty())
+ return make_error<InstrProfError>(instrprof_error::malformed,
+ "invalid input: VTableName is empty");
+ // Insert into NameTab.
+ auto Ins = NameTab.insert(VTableName);
+
+ // Insert into VTableNames.
+ VTableNames.insert(VTableName);
+
+ // If this is newly added, update MD5NameMap.
+ if (Ins.second) {
+ // printf("VTableName %s\n", VTableName.str().c_str());
+ // printf("AddVTableName hash %"PRIu64" to %s\n",
+ // IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey());
+ MD5NameMap.push_back(std::make_pair(
+ IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey()));
+ Sorted = false;
+ }
+ return Error::success();
+ }
+
+ const StringSet<> &getVTableNames() const { return VTableNames; }
+
/// Map a function address to its name's MD5 hash. This interface
/// is only used by the raw profiler reader.
void mapAddress(uint64_t Addr, uint64_t MD5Val) {
AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
}
+ // Map the start and end address of a variable to its names' MD5 hash.
+ // This interface is only used by the raw profile header.
+ void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
+ VTableAddrToMD5Map.push_back(std::make_pair(StartAddr, MD5Val));
+ VTableAddrToMD5Map.push_back(std::make_pair(EndAddr, MD5Val));
+ }
+
/// Return a function's hash, or 0, if the function isn't in this SymTab.
uint64_t getFunctionHashFromAddress(uint64_t Address);
+ /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
+ uint64_t getVTableHashFromAddress(uint64_t Address);
+
/// Return function's PGO name from the function name's symbol
/// address in the object file. If an error occurs, return
/// an empty string.
@@ -522,6 +585,11 @@ class InstrProfSymtab {
/// this symbol table.
inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash);
+ /// Just like getFuncName, except that it will return a non-empty StringRef
+ /// if the function is external to this symbol table. All such cases
+ /// will be represented using the same StringRef value.
+ // inline StringRef getVTableNameOrExternalSymbol(uint64_t VTableMD5Hash);
+
/// True if Symbol is the value used to represent external symbols.
static bool isExternalSymbol(const StringRef &Symbol) {
return Symbol == InstrProfSymtab::getExternalSymbol();
@@ -529,6 +597,8 @@ class InstrProfSymtab {
/// Return function from the name's md5 hash. Return nullptr if not found.
inline Function *getFunction(uint64_t FuncMD5Hash);
+ // Return vtable from the name's MD5 hash. Return nullptr if not found.
+ inline GlobalVariable *getGlobalVariable(uint64_t GlobalVariableMD5Hash);
/// Return the name section data.
inline StringRef getNameData() const { return Data; }
@@ -547,6 +617,16 @@ Error InstrProfSymtab::create(StringRef NameStrings) {
return readPGOFuncNameStrings(NameStrings, *this);
}
+Error InstrProfSymtab::create(StringRef FuncNameStrings,
+ StringRef VTableNameStrings) {
+ if (Error E = readPGOFuncNameStrings(FuncNameStrings, *this))
+ return E;
+
+ // FIXME: Add test coverage that this returns success when VTableNameStrings
+ // is empty.
+ return readVTableNames(VTableNameStrings, *this);
+}
+
template <typename NameIterRange>
Error InstrProfSymtab::create(const NameIterRange &IterRange) {
for (auto Name : IterRange)
@@ -557,6 +637,23 @@ Error InstrProfSymtab::create(const NameIterRange &IterRange) {
return Error::success();
}
+template <typename FuncNameIterRange, typename VTableNameIterRange>
+Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange,
+ const VTableNameIterRange &VTableIterRange) {
+ for (auto Name : FuncIterRange)
+ if (Error E = addFuncName(Name))
+ return E;
+
+ for (auto VTableName : VTableIterRange) {
+ if (Error E = addVTableName(VTableName)) {
+ return E;
+ }
+ }
+
+ finalizeSymtab();
+ return Error::success();
+}
+
void InstrProfSymtab::finalizeSymtab() {
if (Sorted)
return;
@@ -595,6 +692,19 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
return nullptr;
}
+GlobalVariable *
+InstrProfSymtab::getGlobalVariable(uint64_t GlobalVariableMD5Hash) {
+ finalizeSymtab();
+ auto Result =
+ llvm::lower_bound(MD5VTableMap, GlobalVariableMD5Hash,
+ [](const std::pair<uint64_t, GlobalVariable *> &LHS,
+ uint64_t RHS) { return LHS.first < RHS; });
+
+ if (Result != MD5VTableMap.end() && Result->first == GlobalVariableMD5Hash)
+ return Result->second;
+ return nullptr;
+}
+
// To store the sums of profile count values, or the percentage of
// the sums of the total count values.
struct CountSumOrPercent {
@@ -816,6 +926,7 @@ struct InstrProfRecord {
struct ValueProfData {
std::vector<InstrProfValueSiteRecord> IndirectCallSites;
std::vector<InstrProfValueSiteRecord> MemOPSizes;
+ std::vector<InstrProfValueSiteRecord> VTableTargets;
};
std::unique_ptr<ValueProfData> ValueData;
@@ -838,6 +949,8 @@ struct InstrProfRecord {
return ValueData->IndirectCallSites;
case IPVK_MemOPSize:
return ValueData->MemOPSizes;
+ case IPVK_VTableTarget:
+ return ValueData->VTableTargets;
default:
llvm_unreachable("Unknown value kind!");
}
@@ -852,6 +965,8 @@ struct InstrProfRecord {
return ValueData->IndirectCallSites;
case IPVK_MemOPSize:
return ValueData->MemOPSizes;
+ case IPVK_VTableTarget:
+ return ValueData->VTableTargets;
default:
llvm_unreachable("Unknown value kind!");
}
@@ -1014,6 +1129,8 @@ enum ProfVersion {
Version9 = 9,
// An additional (optional) temporal profile traces section is added.
Version10 = 10,
+ // VTable profiling,
+ Version11 = 11,
// The current version is 10.
CurrentVersion = INSTR_PROF_INDEX_VERSION
};
@@ -1034,6 +1151,7 @@ struct Header {
uint64_t MemProfOffset;
uint64_t BinaryIdOffset;
uint64_t TemporalProfTracesOffset;
+ uint64_t VTableNamesOffset; // Organize virtual table names.
// New fields should only be added at the end to ensure that the size
// computation is correct. The methods below need to be updated to ensure that
// the new field is read correctly.
@@ -1173,6 +1291,11 @@ template <class IntPtrT> struct alignas(8) ProfileData {
#include "llvm/ProfileData/InstrProfData.inc"
};
+template <class IntPtrT> struct alignas(8) VTableProfileData {
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name;
+#include "llvm/ProfileData/InstrProfData.inc"
+};
+
// File header structure of the LLVM profile data in raw format.
// The definition should match the header referenced in
// compiler-rt/lib/profile/InstrProfilingFile.c and
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index 13be2753e514efe..f1e7691afdc4bf0 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -91,6 +91,18 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
#undef INSTR_PROF_DATA
/* INSTR_PROF_DATA end. */
+#ifndef INSTR_PROF_VTABLE_DATA
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_VTABLE_DATA_DEFINED
+#endif
+INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, \
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(VTableName)))
+INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), VTablePointer, VTableAddr)
+INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), VTableSizeVal))
+#undef INSTR_PROF_VTABLE_DATA
+/* INSTR_PROF_VTABLE_DATA end. */
/* This is an internal data structure used by value profiler. It
* is defined here to allow serialization code sharing by LLVM
@@ -136,6 +148,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
(uintptr_t)CountersBegin - (uintptr_t)DataBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
#undef INSTR_PROF_RAW_HEADER
/* INSTR_PROF_RAW_HEADER end */
@@ -177,13 +191,14 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
+VALUE_PROF_KIND(IPVK_VTableTarget, 2, "vtable target")
/* These two kinds must be the last to be
* declared. This is to make sure the string
* array created with the template can be
* indexed with the kind value.
*/
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
-VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
+VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")
#undef VALUE_PROF_KIND
/* VALUE_PROF_KIND end */
@@ -270,12 +285,18 @@ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
INSTR_PROF_SECT_ENTRY(IPSK_name, \
INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
INSTR_PROF_NAME_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vname, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
+ INSTR_PROF_VNAME_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vals, \
INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
INSTR_PROF_VALS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
INSTR_PROF_VNODES_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
+ INSTR_PROF_VTAB_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
@@ -645,9 +666,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 8
+#define INSTR_PROF_RAW_VERSION 9
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 10
+#define INSTR_PROF_INDEX_VERSION 11
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 5
@@ -685,9 +706,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
+#define INSTR_PROF_VNAME_COMMON __llvm_prf_vnames
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
+#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
@@ -696,9 +719,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
*/
#define INSTR_PROF_DATA_COFF ".lprfd$M"
#define INSTR_PROF_NAME_COFF ".lprfn$M"
+#define INSTR_PROF_VNAME_COFF ".lprfvn$M"
#define INSTR_PROF_CNTS_COFF ".lprfc$M"
#define INSTR_PROF_VALS_COFF ".lprfv$M"
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
+#define INSTR_PROF_VTAB_COFF ".lprfvt$M"
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 5f54cbeb1b01eda..d7009bbe9dc3f93 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -326,10 +326,14 @@ class RawInstrProfReader : public InstrProfReader {
uint64_t NamesDelta;
const RawInstrProf::ProfileData<IntPtrT> *Data;
const RawInstrProf::ProfileData<IntPtrT> *DataEnd;
+ const RawInstrProf::VTableProfileData<IntPtrT> *VTableBegin = nullptr;
+ const RawInstrProf::VTableProfileData<IntPtrT> *VTableEnd = nullptr;
const char *CountersStart;
const char *CountersEnd;
const char *NamesStart;
const char *NamesEnd;
+ const char *VNamesStart = nullptr;
+ const char *VNamesEnd = nullptr;
// After value profile is all read, this pointer points to
// the header of next profile data (if exists)
const uint8_t *ValueDataStart;
@@ -468,6 +472,46 @@ enum class HashT : uint32_t;
} // end namespace IndexedInstrProf
+class InstrProfVTableLookupTrait {
+ char val;
+ IndexedInstrProf::HashT HashType;
+ unsigned FormatVersion;
+
+public:
+ InstrProfVTableLookupTrait(IndexedInstrProf::HashT HashType,
+ unsigned FormatVersion)
+ : HashType(HashType), FormatVersion(FormatVersion) {}
+
+ using data_type = char;
+
+ using internal_key_type = StringRef;
+ using external_key_type = StringRef;
+
+ using hash_value_type = uint64_t;
+ using offset_type = uint64_t;
+
+ static bool EqualKey(StringRef A, StringRef B) { return A == B; }
+ static StringRef GetInternalKey(StringRef K) { return K; }
+ static StringRef GetExternalKey(StringRef K) { return K; }
+
+ hash_value_type ComputeHash(StringRef K);
+
+ static std::pair<offset_type, offset_type>
+ ReadKeyDataLength(const unsigned char *&D) {
+ using namespace support;
+
+ offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D);
+ offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D);
+ return std::make_pair(KeyLen, DataLen);
+ }
+
+ StringRef ReadKey(const unsigned char *D, offset_type N) {
+ return StringRef((const char *)D, N);
+ }
+
+ data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
+};
+
/// Trait for lookups into the on-disk hash table for the binary instrprof
/// format.
class InstrProfLookupTrait {
@@ -521,6 +565,9 @@ class InstrProfLookupTrait {
}
};
+using VirtualTableNamesHashTable =
+ OnDiskIterableChainedHashTable<InstrProfVTableLookupTrait>;
+
struct InstrProfReaderIndexBase {
virtual ~InstrProfReaderIndexBase() = default;
@@ -543,7 +590,10 @@ struct InstrProfReaderIndexBase {
virtual bool hasMemoryProfile() const = 0;
virtual bool hasTemporalProfile() const = 0;
virtual InstrProfKind getProfileKind() const = 0;
- virtual Error populateSymtab(InstrProfSymtab &) = 0;
+ // The pointer VirtualTableIndex is not owned.
+ virtual Error
+ populateSymtab(InstrProfSymtab &,
+ VirtualTableNamesHashTable *VirtualTableIndex) = 0;
};
using OnDiskHashTableImplV3 =
@@ -618,7 +668,10 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase {
InstrProfKind getProfileKind() const override;
- Error populateSymtab(InstrProfSymtab &Symtab) override {
+ Error populateSymtab(InstrProfSymtab &Symtab,
+ VirtualTableNamesHashTable *VirtualTableIndex) override {
+ if (VirtualTableIndex != nullptr)
+ return Symtab.create(HashTable->keys(), VirtualTableIndex->keys());
return Symtab.create(HashTable->keys());
}
};
@@ -653,6 +706,8 @@ class IndexedInstrProfReader : public InstrProfReader {
std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable;
/// MemProf frame profile data on-disk indexed via frame id.
std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable;
+ /// Virtual table profile data indexed .
+ std::unique_ptr<VirtualTableNamesHashTable> VirtualTableIndex = nullptr;
/// Total size of binary ids.
uint64_t BinaryIdsSize{0};
/// Start address of binary id length and data pairs.
diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 311c84a2d52bc78..768a0dd2785f400 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -64,6 +64,9 @@ class InstrProfWriter {
// List of binary ids.
std::vector<llvm::object::BuildID> BinaryIds;
+ // Read the vtable names from raw instr profile reader.
+ StringSet<> VTableNames;
+
// An enum describing the attributes of the profile.
InstrProfKind ProfileKind = InstrProfKind::Unknown;
// Use raw pointer here for the incomplete type object.
@@ -85,6 +88,7 @@ class InstrProfWriter {
void addRecord(NamedInstrProfRecord &&I, function_ref<void(Error)> Warn) {
addRecord(std::move(I), 1, Warn);
}
+ void addVTableName(StringRef VTableName) { VTableNames.insert(VTableName); }
/// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the
/// total number of temporal profiling traces the source has seen.
diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
index cb0c055dcb74ae8..9af090c0c2e6825 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
@@ -56,12 +56,18 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
}
};
DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
+ // Key is virtual table variable, value is 'VTableProfData' in the form of
+ // GlobalVariable.
+ DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;
/// If runtime relocation is enabled, this maps functions to the load
/// instruction that produces the profile relocation bias.
DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
std::vector<GlobalValue *> CompilerUsedVars;
std::vector<GlobalValue *> UsedVars;
std::vector<GlobalVariable *> ReferencedNames;
+ // The list of virtual table variables of which the VTableProfData is
+ // collected.
+ std::vector<GlobalVariable *> ReferencedVTableNames;
GlobalVariable *NamesVar;
size_t NamesSize;
@@ -115,6 +121,9 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
/// referring to them will also be created.
GlobalVariable *getOrCreateRegionCounters(InstrProfInstBase *Inc);
+ /// Get the counters for virtual table values, creating them if necessary.
+ void getOrCreateVTableProfData(GlobalVariable *GV);
+
/// Create the region counters.
GlobalVariable *createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
GlobalValue::LinkageTypes Linkage);
@@ -122,6 +131,9 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
/// Emit the section with compressed function names.
void emitNameData();
+ /// Emit the section with compressed vtable names.
+ void emitVTableNames();
+
/// Emit value nodes section for value profiling.
void emitVNodes();
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 0cb296b3bde6c5d..31d29a6c4bd8b08 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -447,6 +447,18 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) {
if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO)))
return E;
}
+
+ SmallVector<MDNode *, 2> Types;
+ for (GlobalVariable &G : M.globals()) {
+ if (!G.hasName())
+ continue;
+ Types.clear();
+ G.getMetadata(LLVMContext::MD_type, Types);
+ if (!Types.empty()) {
+ // errs() << "Insert " << G.getGUID() << "\t into MD5VTableMap\n";
+ MD5VTableMap.emplace_back(G.getGUID(), &G);
+ }
+ }
Sorted = false;
finalizeSymtab();
return Error::success();
@@ -484,6 +496,26 @@ Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
return Error::success();
}
+uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
+ finalizeSymtab();
+ // printf("look up key 0x%llx\n", Address);
+ // for (auto iter = VTableAddrToMD5Map.begin(); iter !=
+ // VTableAddrToMD5Map.end(); iter++) {
+ // printf("<key, val> is <0x%llx, %"PRIu64"\n", iter->first, iter->second);
+ // }
+ auto It =
+ partition_point(VTableAddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
+ return A.first < Address;
+ });
+ // FIXME: Does the raw function pointers point apply here?
+ if (It != VTableAddrToMD5Map.end()) {
+ // printf("InstrProfSymtab::getVTableHashFromAddress map addr 0x%llx to hash
+ // value %"PRIu64"\n", Address, (uint64_t)It->second);
+ return (uint64_t)It->second;
+ }
+ return 0;
+}
+
uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
finalizeSymtab();
auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
@@ -560,42 +592,112 @@ Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
NameStrs, compression::zlib::isAvailable() && doCompression, Result);
}
+Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
+ std::string &Result, bool doCompression) {
+ std::vector<std::string> VTableNameStrs;
+ for (auto *VTable : VTables) {
+ // printf("VTable name %s added\n", VTable->getName().str().c_str());
+ VTableNameStrs.push_back(std::string(VTable->getName()));
+ }
+ return collectPGOFuncNameStrings(
+ VTableNameStrs, compression::zlib::isAvailable() && doCompression,
+ Result);
+}
+
+instrprof_error decodeAndSplitStrings(
+ const uint8_t *Input, SmallVector<uint8_t, 128> &UncompressedNameStrings,
+ StringRef &NameStrings, uint32_t &Dist, bool &isCompressed) {
+ Dist = 0;
+ const uint8_t *Start = Input;
+ uint32_t UncompressedSizeLen = 0;
+ uint64_t UncompressedSize = decodeULEB128(Start, &UncompressedSizeLen);
+ Start += UncompressedSizeLen;
+ Dist += UncompressedSizeLen;
+ uint32_t CompressedSizeLen = 0;
+ uint64_t CompressedSize = decodeULEB128(Start, &CompressedSizeLen);
+ Start += CompressedSizeLen;
+ Dist += CompressedSizeLen;
+ isCompressed = (CompressedSize != 0);
+ if (isCompressed) {
+ if (!llvm::compression::zlib::isAvailable())
+ return instrprof_error::zlib_unavailable;
+
+ if (Error E = compression::zlib::decompress(ArrayRef(Start, CompressedSize),
+ UncompressedNameStrings,
+ UncompressedSize)) {
+ consumeError(std::move(E));
+ return instrprof_error::uncompress_failed;
+ }
+ Dist += CompressedSize;
+ } else {
+ NameStrings =
+ StringRef(reinterpret_cast<const char *>(Start), UncompressedSize);
+ Dist += UncompressedSize;
+ }
+
+ return instrprof_error::success;
+}
+
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
const uint8_t *P = NameStrings.bytes_begin();
const uint8_t *EndP = NameStrings.bytes_end();
while (P < EndP) {
- uint32_t N;
- uint64_t UncompressedSize = decodeULEB128(P, &N);
- P += N;
- uint64_t CompressedSize = decodeULEB128(P, &N);
- P += N;
- bool isCompressed = (CompressedSize != 0);
- SmallVector<uint8_t, 128> UncompressedNameStrings;
+ // Now parse the name strings.
+ uint32_t Dist = 0;
StringRef NameStrings;
+ SmallVector<uint8_t, 128> UncompressedNameStrings;
+ SmallVector<StringRef, 0> Names;
+ bool isCompressed = false;
+ instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
+ NameStrings, Dist, isCompressed);
+ if (E != instrprof_error::success)
+ return make_error<InstrProfError>(E);
+
if (isCompressed) {
- if (!llvm::compression::zlib::isAvailable())
- return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
-
- if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
- UncompressedNameStrings,
- UncompressedSize)) {
- consumeError(std::move(E));
- return make_error<InstrProfError>(instrprof_error::uncompress_failed);
- }
- P += CompressedSize;
NameStrings = toStringRef(UncompressedNameStrings);
- } else {
- NameStrings =
- StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
- P += UncompressedSize;
}
+
+ NameStrings.split(Names, getInstrProfNameSeparator());
+ for (StringRef &Name : Names) {
+ if (Error E = Symtab.addFuncName(Name))
+ return E;
+ }
+
+ P += Dist;
+ // Skip padding?
+ while (P < EndP && *P == 0)
+ P++;
+ }
+ return Error::success();
+}
+
+Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab) {
+ const uint8_t *P = NameStrings.bytes_begin();
+ const uint8_t *EndP = NameStrings.bytes_end();
+ while (P < EndP) {
// Now parse the name strings.
+ uint32_t Dist = 0;
+ StringRef NameStrings;
+ SmallVector<uint8_t, 128> UncompressedNameStrings;
SmallVector<StringRef, 0> Names;
+ bool isCompressed = false;
+ instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
+ NameStrings, Dist, isCompressed);
+ if (E != instrprof_error::success)
+ return make_error<InstrProfError>(E);
+
+ if (isCompressed) {
+ NameStrings = toStringRef(UncompressedNameStrings);
+ }
NameStrings.split(Names, getInstrProfNameSeparator());
- for (StringRef &Name : Names)
- if (Error E = Symtab.addFuncName(Name))
+ for (StringRef &Name : Names) {
+ // printf("Read back vtable name %s\n", Name.str().c_str());
+ if (Error E = Symtab.addVTableName(Name))
return E;
+ }
+ P += Dist;
+ // Skip padding?
while (P < EndP && *P == 0)
P++;
}
@@ -852,6 +954,11 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
if (ValueKind == IPVK_IndirectCallTarget)
return SymTab->getFunctionHashFromAddress(Value);
+ if (ValueKind == IPVK_VTableTarget) {
+ uint64_t VTableHash = SymTab->getVTableHashFromAddress(Value);
+ return VTableHash;
+ }
+
return Value;
}
@@ -1242,8 +1349,8 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
F.setMetadata(getPGOFuncNameMetadataName(), N);
}
-bool needsComdatForCounter(const Function &F, const Module &M) {
- if (F.hasComdat())
+bool needsComdatForCounter(const GlobalValue &GV, const Module &M) {
+ if (GV.hasComdat())
return true;
if (!Triple(M.getTargetTriple()).supportsCOMDAT())
@@ -1259,7 +1366,7 @@ bool needsComdatForCounter(const Function &F, const Module &M) {
// available_externally functions will end up being duplicated in raw profile
// data. This can result in distorted profile as the counts of those dups
// will be accumulated by the profile merger.
- GlobalValue::LinkageTypes Linkage = F.getLinkage();
+ GlobalValue::LinkageTypes Linkage = GV.getLinkage();
if (Linkage != GlobalValue::ExternalWeakLinkage &&
Linkage != GlobalValue::AvailableExternallyLinkage)
return false;
@@ -1423,6 +1530,9 @@ void OverlapStats::dump(raw_fd_ostream &OS) const {
case IPVK_MemOPSize:
strncpy(ProfileKindName, "MemOP", 19);
break;
+ case IPVK_VTableTarget:
+ strncpy(ProfileKindName, "VTable", 19);
+ break;
default:
snprintf(ProfileKindName, 19, "VP[%d]", I);
break;
@@ -1487,9 +1597,12 @@ Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
// When a new field is added in the header add a case statement here to
// populate it.
static_assert(
- IndexedInstrProf::ProfVersion::CurrentVersion == Version10,
+ IndexedInstrProf::ProfVersion::CurrentVersion == Version11,
"Please update the reading code below if a new field has been added, "
"if not add a case statement to fall through to the latest version.");
+ case 11ull:
+ H.VTableNamesOffset = read(Buffer, offsetOf(&Header::VTableNamesOffset));
+ [[fallthrough]];
case 10ull:
H.TemporalProfTracesOffset =
read(Buffer, offsetOf(&Header::TemporalProfTracesOffset));
@@ -1513,10 +1626,13 @@ size_t Header::size() const {
// When a new field is added to the header add a case statement here to
// compute the size as offset of the new field + size of the new field. This
// relies on the field being added to the end of the list.
- static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version10,
+ static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11,
"Please update the size computation below if a new field has "
"been added to the header, if not add a case statement to "
"fall through to the latest version.");
+ case 11ull:
+ return offsetOf(&Header::VTableNamesOffset) +
+ sizeof(Header::VTableNamesOffset);
case 10ull:
return offsetOf(&Header::TemporalProfTracesOffset) +
sizeof(Header::TemporalProfTracesOffset);
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index a920a31d0a4b229..59e7ef946686320 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -373,6 +373,14 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
return E;
Value = IndexedInstrProf::ComputeHash(VD.first);
}
+ } else if (ValueKind == IPVK_VTableTarget) {
+ if (InstrProfSymtab::isExternalSymbol(VD.first)) {
+ Value = 0;
+ } else {
+ if (Error E = Symtab->addVTableName(VD.first))
+ return E;
+ Value = IndexedInstrProf::ComputeHash(VD.first);
+ }
} else {
READ_NUM(VD.first, Value);
}
@@ -517,14 +525,31 @@ Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
template <class IntPtrT>
Error RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {
- if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart)))
+ if (Error E =
+ Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart),
+ StringRef(VNamesStart, VNamesEnd - VNamesStart))) {
return error(std::move(E));
+ }
for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
const IntPtrT FPtr = swap(I->FunctionPointer);
if (!FPtr)
continue;
Symtab.mapAddress(FPtr, I->NameRef);
}
+
+ if (VTableBegin != nullptr && VTableEnd != nullptr) {
+ for (const RawInstrProf::VTableProfileData<IntPtrT> *I = VTableBegin;
+ I != VTableEnd; ++I) {
+ const IntPtrT VPtr = I->VTablePointer;
+ if (!VPtr)
+ continue;
+ // Map both begin and end address to the name hash, since the instrumented
+ // address could be somewhere in the middle.
+ // VPtr is of type uint32_t or uint64_t so 'VPtr + I->VTableSize' marks
+ // the end of vtable address.
+ Symtab.mapVTableAddress(VPtr, VPtr + I->VTableSize, I->VTableNameHash);
+ }
+ }
return success();
}
@@ -556,17 +581,29 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize();
auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters);
auto NamesSize = swap(Header.NamesSize);
+ auto VTableNameSize = Header.VNamesSize;
+ auto NumVTables = Header.NumVTables;
ValueKindLast = swap(Header.ValueKindLast);
auto DataSize = NumData * sizeof(RawInstrProf::ProfileData<IntPtrT>);
- auto PaddingSize = getNumPaddingBytes(NamesSize);
+ auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);
+ auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);
+
+ auto VTableSectionSize =
+ Header.NumVTables * sizeof(RawInstrProf::VTableProfileData<IntPtrT>);
+ auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);
// Profile data starts after profile header and binary ids if exist.
ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize;
ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
ptrdiff_t NamesOffset =
CountersOffset + CountersSize + PaddingBytesAfterCounters;
- ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
+ ptrdiff_t VTableProfDataOffset =
+ NamesOffset + NamesSize + PaddingBytesAfterNames;
+ ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
+ PaddingBytesAfterVTableProfData;
+ ptrdiff_t ValueDataOffset =
+ VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;
auto *Start = reinterpret_cast<const char *>(&Header);
if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
@@ -585,8 +622,14 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
Start + DataOffset);
DataEnd = Data + NumData;
+ VTableBegin =
+ reinterpret_cast<const RawInstrProf::VTableProfileData<IntPtrT> *>(
+ Start + VTableProfDataOffset);
+ VTableEnd = VTableBegin + NumVTables;
NamesStart = Start + NamesOffset;
NamesEnd = NamesStart + NamesSize;
+ VNamesStart = Start + VTableNameOffset;
+ VNamesEnd = VNamesStart + VTableNameSize;
}
// Binary ids start just after the header.
@@ -835,6 +878,19 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
return DataBuffer;
}
+InstrProfVTableLookupTrait::hash_value_type
+InstrProfVTableLookupTrait::ComputeHash(StringRef K) {
+ return IndexedInstrProf::ComputeHash(HashType, K);
+}
+
+InstrProfVTableLookupTrait::data_type
+InstrProfVTableLookupTrait::ReadData(StringRef K, const unsigned char *D,
+ offset_type N) {
+ char v =
+ support::endian::readNext<char, support::little, support::unaligned>(D);
+ return v;
+}
+
template <typename HashTableImpl>
Error InstrProfReaderIndex<HashTableImpl>::getRecords(
StringRef FuncName, ArrayRef<NamedInstrProfRecord> &Data) {
@@ -1162,6 +1218,21 @@ Error IndexedInstrProfReader::readHeader() {
"corrupted binary ids");
}
+ if (GET_VERSION(Header->formatVersion()) >= 11) {
+ uint64_t VTableNamesOffset =
+ endian::byte_swap<uint64_t, little>(Header->VTableNamesOffset);
+ const unsigned char *Ptr = Start + VTableNamesOffset;
+
+ const uint64_t HashTableMetadataOffset =
+ support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+
+ VirtualTableIndex.reset(VirtualTableNamesHashTable::Create(
+ /*Bucket=*/Start + HashTableMetadataOffset,
+ /*Payload=*/Ptr,
+ /*Base=*/Start,
+ InstrProfVTableLookupTrait(HashType, Header->formatVersion())));
+ }
+
if (GET_VERSION(Header->formatVersion()) >= 10 &&
Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) {
uint64_t TemporalProfTracesOffset =
@@ -1222,7 +1293,7 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
return *Symtab;
std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
- if (Error E = Index->populateSymtab(*NewSymtab)) {
+ if (Error E = Index->populateSymtab(*NewSymtab, VirtualTableIndex.get())) {
auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
consumeError(error(ErrCode, Msg));
}
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 6892654b00ea46c..0094cf8c029fca1 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -171,6 +171,47 @@ class InstrProfRecordWriterTrait {
}
};
+class InstrProfRecordVTableTrait {
+public:
+ using key_type = StringRef;
+ using key_type_ref = StringRef;
+
+ using data_type = char;
+ using data_type_ref = char;
+
+ using hash_value_type = uint64_t;
+ using offset_type = uint64_t;
+
+ InstrProfRecordVTableTrait() = default;
+
+ static hash_value_type ComputeHash(key_type_ref K) {
+ return IndexedInstrProf::ComputeHash(K);
+ }
+
+ static std::pair<offset_type, offset_type>
+ EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
+ using namespace support;
+
+ endian::Writer LE(Out, little);
+
+ offset_type N = K.size();
+ LE.write<offset_type>(N);
+
+ offset_type M = 1;
+ LE.write<offset_type>(M);
+
+ return std::make_pair(N, M);
+ }
+
+ void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
+ Out.write(K.data(), N);
+ }
+
+ void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
+ Out.write(&V, 1);
+ }
+};
+
} // end namespace llvm
InstrProfWriter::InstrProfWriter(bool Sparse,
@@ -447,12 +488,13 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
Header.MemProfOffset = 0;
Header.BinaryIdOffset = 0;
Header.TemporalProfTracesOffset = 0;
+ Header.VTableNamesOffset = 0;
int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
// Only write out all the fields except 'HashOffset', 'MemProfOffset',
- // 'BinaryIdOffset' and `TemporalProfTracesOffset`. We need to remember the
- // offset of these fields to allow back patching later.
- for (int I = 0; I < N - 4; I++)
+ // 'BinaryIdOffset', `TemporalProfTracesOffset` and `VTableNamesOffset`. We
+ // need to remember the offset of these fields to allow back patching later.
+ for (int I = 0; I < N - 5; I++)
OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
// Save the location of Header.HashOffset field in \c OS.
@@ -476,6 +518,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
uint64_t TemporalProfTracesOffset = OS.tell();
OS.write(0);
+ uint64_t VTableNamesOffset = OS.tell();
+ OS.write(0);
+
// Reserve space to write profile summary data.
uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
@@ -589,6 +634,36 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
OS.writeByte(0);
}
+ // if version >= the version with vtable profile metadata
+ // Intentionally put vtable names before temporal profile section.
+ uint64_t VTableNamesSectionStart = 0;
+ if (IndexedInstrProf::ProfVersion::CurrentVersion >= 11) {
+ VTableNamesSectionStart = OS.tell();
+
+ // Reserve space for vtable record table offset.
+ OS.write(0ULL);
+
+ OnDiskChainedHashTableGenerator<llvm::InstrProfRecordVTableTrait>
+ VTableNamesGenerator;
+ for (const auto &kv : VTableNames) {
+ // printf("InstrProfWriter.cpp key is %s\n", kv.getKey().str().c_str());
+ VTableNamesGenerator.insert(kv.getKey(), '0');
+ }
+
+ auto VTableNamesWriter =
+ std::make_unique<llvm::InstrProfRecordVTableTrait>();
+
+ uint64_t VTableNamesTableOffset =
+ VTableNamesGenerator.Emit(OS.OS, *VTableNamesWriter);
+
+ // printf("InstrProfWriter.cpp:VTableNamesSectionStart is %"PRIu64"\n",
+ // VTableNamesSectionStart); printf("\tVTableNamesTableOffset is
+ // %"PRIu64"\n", VTableNamesTableOffset);
+ PatchItem PatchItems[] = {
+ {VTableNamesSectionStart, &VTableNamesTableOffset, 1}};
+ OS.patch(PatchItems, 1);
+ }
+
uint64_t TemporalProfTracesSectionStart = 0;
if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
TemporalProfTracesSectionStart = OS.tell();
@@ -632,6 +707,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
// Patch the Header.TemporalProfTracesOffset (=0 for profiles without
// traces).
{TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
+ {VTableNamesOffset, &VTableNamesSectionStart, 1},
// Patch the summary data.
{SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
(int)(SummarySize / sizeof(uint64_t))},
@@ -684,7 +760,8 @@ Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
DenseSet<uint64_t> SeenValues;
for (uint32_t I = 0; I < ND; I++)
- if ((VK != IPVK_IndirectCallTarget) && !SeenValues.insert(VD[I].Value).second)
+ if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) &&
+ !SeenValues.insert(VD[I].Value).second)
return make_error<InstrProfError>(instrprof_error::invalid_prof);
}
}
@@ -721,7 +798,7 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
OS << ND << "\n";
std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
for (uint32_t I = 0; I < ND; I++) {
- if (VK == IPVK_IndirectCallTarget)
+ if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
OS << Symtab.getFuncOrVarNameIfDefined(VD[I].Value) << ":"
<< VD[I].Count << "\n";
else
@@ -758,6 +835,11 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
}
}
+ for (const auto &VTableName : VTableNames) {
+ if (Error E = Symtab.addVTableName(VTableName.getKey()))
+ return E;
+ }
+
if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
writeTextTemporalProfTraceData(OS, Symtab);
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 57fcfd53836911b..844b5397574b51b 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -561,6 +561,13 @@ bool InstrProfiling::run(
static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
}
+ for (GlobalVariable &GV : M.globals()) {
+ // Global variables with type metadata are virtual table variables.
+ if (GV.hasMetadata(LLVMContext::MD_type)) {
+ getOrCreateVTableProfData(&GV);
+ }
+ }
+
for (Function &F : M)
MadeChange |= lowerIntrinsics(&F);
@@ -574,6 +581,7 @@ bool InstrProfiling::run(
emitVNodes();
emitNameData();
+ emitVTableNames();
// Emit runtime hook for the cases where the target does not unconditionally
// require pulling in profile runtime, and coverage is enabled on code that is
@@ -951,6 +959,135 @@ InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
return GV;
}
+static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
+ if (!profDataReferencedByCode(*GV->getParent()))
+ return false;
+
+ if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
+ !GV->hasAvailableExternallyLinkage())
+ return true;
+
+ // This avoids the profile data from referencing internal symbols in
+ // COMDAT.
+ if (GV->hasLocalLinkage() && GV->hasComdat())
+ return false;
+
+ return true;
+}
+
+// FIXME: Does symbollic relocation from 'getFuncAddrForProfData' matter here?
+static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
+ // Store a nullptr in __profvt_ if a real address shouldn't be used.
+ auto *Int8PtrTy = Type::getInt8PtrTy(GV->getContext());
+
+ if (!shouldRecordVTableAddr(GV))
+ return ConstantPointerNull::get(Int8PtrTy);
+
+ return ConstantExpr::getBitCast(GV, Int8PtrTy);
+}
+
+/// Get the name of a profiling variable for a particular variable.
+static std::string getVarName(GlobalVariable *GV, StringRef Prefix) {
+ StringRef Name = GV->getName();
+ return (Prefix + Name).str();
+}
+
+void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
+ assert(!DebugInfoCorrelate &&
+ "Value profiling is not supported with lightweight instrumentation");
+ if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
+ return;
+
+ if (GV->getName().starts_with("llvm.") ||
+ GV->getName().starts_with("__llvm") ||
+ GV->getName().starts_with("__prof"))
+ return;
+
+ // VTableProfData already created
+ auto It = VTableDataMap.find(GV);
+ if (It != VTableDataMap.end() && It->second)
+ return;
+
+ GlobalValue::LinkageTypes Linkage = GV->getLinkage();
+ GlobalValue::VisibilityTypes Visibility = GV->getVisibility();
+
+ // This is to keep consistent with per-function profile data
+ // for correctness.
+ if (TT.isOSBinFormatXCOFF()) {
+ Linkage = GlobalValue::InternalLinkage;
+ Visibility = GlobalValue::DefaultVisibility;
+ }
+
+ LLVMContext &Ctx = M->getContext();
+ Type *DataTypes[] = {
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
+#include "llvm/ProfileData/InstrProfData.inc"
+ };
+
+ auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
+
+ // Used by INSTR_PROF_VTABLE_DATA MACRO
+ Constant *VTableAddr = getVTableAddrForProfData(GV);
+ StringRef VTableName = GV->getName();
+ // Record the length of the vtable. This is needed since vtable pointers
+ // loaded from C++ objects might be from the middle of a vtable definition.
+ uint32_t VTableSizeVal =
+ M->getDataLayout().getTypeAllocSize(GV->getValueType());
+
+ Constant *DataVals[] = {
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
+#include "llvm/ProfileData/InstrProfData.inc"
+ };
+
+ auto *Data =
+ new GlobalVariable(*M, DataTy, false /* constant */, Linkage,
+ ConstantStruct::get(DataTy, DataVals),
+ getVarName(GV, getInstrProfVTableVarPrefix()));
+
+ Data->setVisibility(Visibility);
+ Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat()));
+ Data->setAlignment(Align(8));
+
+ const bool NeedComdat = needsComdatForCounter(*GV, *M);
+
+ // GV is the data structure to record vtable information.
+ // Place the global variable for per-vtable profile data in a comdat group
+ // if the associated vtable definition is a COMDAT. This makes sure only one
+ // copy of the variable for the vtable will be emitted after linking.
+ auto MaybeSetComdat = [&](GlobalVariable *GV, StringRef GroupName) {
+ bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
+ if (UseComdat) {
+ // Create a new comdat group using the name of the global variable as
+ // opposed to using the comdat group of the vtable.
+ Comdat *C = M->getOrInsertComdat(GroupName);
+ // For ELF, when not using COMDAT, put the vtable profile data into a
+ // nodeduplicate COMDAT which is lowered to a zero-flag zero group.
+ // This allows -z -start-top-gc to discard the entire group when the
+ // vtable def is discarded.
+ if (!NeedComdat)
+ C->setSelectionKind(Comdat::NoDeduplicate);
+ GV->setComdat(C);
+ // COFF doesn't allow the comdat group leader to have private linkage, so
+ // upgrade private linkage to internal linkage to produce a symbol table
+ // entry.
+ if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) {
+ GV->setLinkage(GlobalValue::InternalLinkage);
+ }
+ return;
+ }
+ };
+
+ MaybeSetComdat(Data, Data->getName());
+
+ VTableDataMap[GV] = Data;
+
+ ReferencedVTableNames.push_back(GV);
+
+ // VTable <Hash, Addr> is used by runtime but not referenced by other
+ // sections. Conservatively mark it linker retained.
+ UsedVars.push_back(Data);
+}
+
GlobalVariable *
InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
GlobalVariable *NamePtr = Inc->getName();
@@ -1238,6 +1375,34 @@ void InstrProfiling::emitNameData() {
NamePtr->eraseFromParent();
}
+void InstrProfiling::emitVTableNames() {
+ if (ReferencedVTableNames.empty())
+ return;
+
+ // Collect VTable
+ std::string CompressedVTableNames;
+ if (Error E =
+ collectVTableStrings(ReferencedVTableNames, CompressedVTableNames,
+ DoInstrProfNameCompression)) {
+ report_fatal_error(Twine(toString(std::move(E))), false);
+ }
+
+ auto &Ctx = M->getContext();
+ auto *VTableNamesVal = ConstantDataArray::getString(
+ Ctx, StringRef(CompressedVTableNames), false /* AddNull */);
+ GlobalVariable *VTableNamesVar =
+ new GlobalVariable(*M, VTableNamesVal->getType(), true /* constant */,
+ GlobalValue::PrivateLinkage, VTableNamesVal,
+ getInstrProfVTableNamesVarName());
+ VTableNamesVar->setSection(
+ getInstrProfSectionName(IPSK_vname, TT.getObjectFormat()));
+ VTableNamesVar->setAlignment(Align(1));
+ // Make VTableNames linker retained.
+ UsedVars.push_back(VTableNamesVar);
+
+ // FIXME: Why emitNames call erase method?
+}
+
void InstrProfiling::emitRegistration() {
if (!needsRuntimeRegistrationOfSectionRange(TT))
return;
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 7ad1c9bc54f3780..75c57d4d559e5df 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -584,6 +584,7 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
NumOfPGOBB += MST.BBInfos.size();
ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget);
+ ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
} else {
NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
index 3a129de1acd02d6..59d51a59c48b735 100644
--- a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
+++ b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
@@ -90,9 +90,33 @@ public:
}
};
+///------------------------ VirtualTableValueProfilingPlugin ------------------------
+class VTableProfilingPlugin {
+ Function &F;
+
+public:
+ static constexpr InstrProfValueKind Kind = IPVK_VTableTarget;
+
+ VTableProfilingPlugin(Function &Fn, TargetLibraryInfo &TLI) : F(Fn) {}
+
+ void run(std::vector<CandidateInfo> &Candidates) {
+ std::vector<Instruction *> Result = findVTableAddrs(F);
+ for (Instruction* I : Result) {
+ Instruction* InsertPt = I->getNextNonDebugInstruction();
+ while(InsertPt && dyn_cast<PHINode>(InsertPt))
+ InsertPt = InsertPt->getNextNonDebugInstruction();
+ assert(InsertPt);
+
+ Instruction *AnnotatedInst = I;
+ Candidates.emplace_back(CandidateInfo{I, InsertPt, AnnotatedInst});
+ }
+ }
+};
+
///----------------------- Registration of the plugins -------------------------
/// For now, registering a plugin with the ValueProfileCollector is done by
/// adding the plugin type to the VP_PLUGIN_LIST macro.
#define VP_PLUGIN_LIST \
MemIntrinsicPlugin, \
- IndirectCallPromotionPlugin
+ IndirectCallPromotionPlugin, \
+ VTableProfilingPlugin
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh b/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
new file mode 100644
index 000000000000000..1b35ea0303d15d2
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+if [ $# -lt 2 ]; then
+ echo "Path to clang++ and llvm-profdata required!"
+ echo "Usage: update_vtable_value_prof_inputs.sh /path/to/updated/clang++ /path/to/updated/llvm-profdata"
+ exit 1
+else
+ CLANG=$1
+ LLVMPROFDATA=$2
+fi
+
+OUTDIR=$(dirname $(realpath -s $0))
+
+echo "Outdir is $OUTDIR"
+
+cat > ${OUTDIR}/vtable_prof.cc << EOF
+#include <cstdlib>
+#include <cstdio>
+
+class Base {
+ public:
+ virtual int func1(int a, int b) = 0;
+ virtual int func2(int a, int b) = 0;
+};
+
+class Derived1 : public Base {
+ public:
+ __attribute__((noinline))
+ int func1(int a, int b) override
+ {
+ return a + b;
+ }
+
+ __attribute__((noinline))
+ int func2(int a, int b) override {
+ return a * b;
+ }
+};
+
+class Derived2 : public Base {
+ public:
+
+ __attribute__((noinline))
+ int func1(int a, int b) override {
+ return a - b;
+ }
+
+ __attribute__((noinline))
+ int func2(int a, int b) override {
+ return a * (a - b);
+ }
+};
+
+__attribute__((noinline)) Base* createType(int a) {
+ Base* base = nullptr;
+ if (a % 4 == 0)
+ base = new Derived1();
+ else
+ base = new Derived2();
+ return base;
+}
+
+
+int main(int argc, char** argv) {
+ int sum = 0;
+ for (int i = 0; i < 1000; i++) {
+ int a = rand();
+ int b = rand();
+ Base* ptr = createType(i);
+ sum += ptr->func1(a, b) + ptr->func2(b, a);
+ }
+ printf("sum is %d\n", sum);
+ return 0;
+}
+EOF
+
+FLAGS="-fuse-ld=lld -O2 -g -fprofile-generate=. -flto=thin -Xclang -fwhole-program-vtables -Wl,--lto-whole-program-visibility"
+
+${CLANG} ${FLAGS} ${OUTDIR}/vtable_prof.cc -o ${OUTDIR}/vtable_prof
+env LLVM_PROFILE_FILE=${OUTDIR}/vtable_prof.profraw ${OUTDIR}/vtable_prof
+
+rm ${OUTDIR}/vtable_prof
+rm ${OUTDIR}/vtable_prof.cc
+
diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
index 1c44a274f3c0471..990b246ce07a8c1 100644
--- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll
+++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
@@ -13,9 +13,9 @@ $foo = comdat any
; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat
; CHECK-NOT: __profn__stdin__foo
; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
-; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [2 x i16] } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null
+; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [3 x i16] } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null
; CHECK-NOT: @foo
-; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8
+; CHECK-SAME: , ptr null, i32 1, [3 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8
; CHECK: @__llvm_prf_nm
; CHECK: @llvm.compiler.used
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
index 42018e4d5e54dd2..63173ab59b28ba2 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
@@ -54,11 +54,12 @@ try.cont: ; preds = %catch, %entry
}
; GEN: catch:
-; GEN: call void @llvm.instrprof.value.profile(
+; GEN: call void @llvm.instrprof.value.profile({{.*}} i32 2, i32 0)
+; GEN: call void @llvm.instrprof.value.profile({{.*}} i32 0, i32 0)
; GEN-SAME: [ "funclet"(token %tmp1) ]
; LOWER: catch:
-; LOWER: call void @__llvm_profile_instrument_target(
+; LOWER: call void @__llvm_profile_instrument_target({{.*}} i32 0)
; LOWER-SAME: [ "funclet"(token %tmp1) ]
declare dso_local void @"?may_throw@@YAXH at Z"(i32)
diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
new file mode 100644
index 000000000000000..0c6bd3c064d35fc
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
@@ -0,0 +1,139 @@
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$_ZTV7Derived = comdat any
+
+ at _ZTV7Derived = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
+ at _ZTVN10__cxxabiv121__vmi_class_type_infoE = external global [0 x ptr]
+ at _ZTS7Derived = constant [9 x i8] c"7Derived\00", align 1
+ at _ZTI5Base1 = external constant ptr
+ at _ZTI5Base2 = external constant ptr
+ at _ZTI7Derived = constant { ptr, ptr, i32, i32, ptr, i64, ptr, i64 } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), ptr @_ZTS7Derived, i32 0, i32 2, ptr @_ZTI5Base1, i64 2, ptr @_ZTI5Base2, i64 2050 }, align 8
+ at _ZTV5Base1 = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base1, ptr @_ZN5Base15func1Eii] }, align 8, !type !0, !type !1
+ at _ZTV5Base2 = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base2, ptr @_ZN5Base25func2Eii] }, align 8, !type !9, !type !4
+ at llvm.compiler.used = appending global [2 x ptr] [ptr @_ZTV5Base1, ptr @_ZTV5Base2], section "llvm.metadata"
+
+declare ptr @_Z10createTypei(i32)
+declare i32 @_ZN5Base15func1Eii(ptr, i32, i32)
+declare i32 @_ZN5Base25func2Eii(ptr, i32, i32)
+
+; GEN: @__llvm_profile_raw_version = hidden constant i64 72057594037927945, comdat
+; GEN: @__profn_test_vtable_value_profiling = private constant [27 x i8] c"test_vtable_value_profiling"
+
+; LOWER: $__profvt__ZTV7Derived = comdat any
+; LOWER: $__profvt__ZTV5Base1 = comdat nodeduplicate
+; LOWER: $__profvt__ZTV5Base2 = comdat nodeduplicate
+; LOWER: @__llvm_profile_raw_version = hidden constant i64 72057594037927945, comdat
+; LOWER: @__profc_test_vtable_value_profiling = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
+; LOWER: @__profvp_test_vtable_value_profiling = private global [4 x i64] zeroinitializer, section "__llvm_prf_vals", comdat($__profc_test_vtable_value_profiling), align 8
+; LOWER: @__profd_test_vtable_value_profiling = private global { i64, i64, i64, ptr, ptr, i32, [3 x i16] } { i64 1593873508557585901, i64 567090795815895039, i64 sub (i64 ptrtoint (ptr @__profc_test_vtable_value_profiling to i64), i64 ptrtoint (ptr @__profd_test_vtable_value_profiling to i64)), ptr @test_vtable_value_profiling.local, ptr @__profvp_test_vtable_value_profiling, i32 1, [3 x i16] [i16 2, i16 0, i16 2] }, section "__llvm_prf_data", comdat($__profc_test_vtable_value_profiling), align 8
+; LOWER: @__profvt__ZTV7Derived = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @__profvt__ZTV5Base1 = global { i64, ptr, i32 } { i64 3215870116411581797, ptr @_ZTV5Base1, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @__profvt__ZTV5Base2 = global { i64, ptr, i32 } { i64 8378219803387680050, ptr @_ZTV5Base2, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @__llvm_prf_vnodes = private global [10 x { i64, i64, ptr }] zeroinitializer, section "__llvm_prf_vnds", align 8
+; LOWER: @__llvm_prf_nm = private constant [37 x i8] c"\1B#x\DA+I-.\89/+IL\CAI\8D/K\CC)M\8D/(\CAO\CB\CC\C9\CCK\07\00\9Ea\0BC", section "__llvm_prf_names", align 1
+; LOWER: @__llvm_prf_vnm = private constant [34 x i8] c"\22 x\DA\8B\8F\0A\093wI-\CA,KMa\8C\07rL\9D\12\8BS\0D\11L#\00\C3\A2\0A\E9", section "__llvm_prf_vnames", align 1
+; LOWER: @llvm.used = appending global [6 x ptr] [ptr @__profvt__ZTV7Derived, ptr @__profvt__ZTV5Base1, ptr @__profvt__ZTV5Base2, ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata"
+
+define i32 @test_vtable_value_profiling(i32 %a, i32 %b, i32 %c) {
+; GEN-LABEL: define i32 @test_vtable_value_profiling(
+; GEN-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) {
+; GEN-NEXT: entry:
+; GEN-NEXT: call void @llvm.instrprof.increment(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i32 1, i32 0)
+; GEN-NEXT: [[CALL:%.*]] = tail call ptr @_Z10createTypei(i32 [[C]])
+; GEN-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 8
+; GEN-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
+; GEN-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[VTABLE]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP0]], i32 2, i32 0)
+; GEN-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
+; GEN-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[VFUNC]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP1]], i32 0, i32 0)
+; GEN-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
+; GEN-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
+; GEN-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP2]], i32 2, i32 1)
+; GEN-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
+; GEN-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP3]], i32 0, i32 1)
+; GEN-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
+; GEN-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
+; GEN-NEXT: ret i32 [[ADD]]
+;
+; LOWER-LABEL: define i32 @test_vtable_value_profiling(
+; LOWER-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) {
+; LOWER-NEXT: entry:
+; LOWER-NEXT: [[PGOCOUNT:%.*]] = load i64, ptr @__profc_test_vtable_value_profiling, align 8
+; LOWER-NEXT: [[TMP0:%.*]] = add i64 [[PGOCOUNT]], 1
+; LOWER-NEXT: store i64 [[TMP0]], ptr @__profc_test_vtable_value_profiling, align 8
+; LOWER-NEXT: [[CALL:%.*]] = tail call ptr @_Z10createTypei(i32 [[C]])
+; LOWER-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 8
+; LOWER-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
+; LOWER-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[VTABLE]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP1]], ptr @__profd_test_vtable_value_profiling, i32 2)
+; LOWER-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
+; LOWER-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VFUNC]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP2]], ptr @__profd_test_vtable_value_profiling, i32 0)
+; LOWER-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
+; LOWER-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
+; LOWER-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP3]], ptr @__profd_test_vtable_value_profiling, i32 3)
+; LOWER-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
+; LOWER-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP4]], ptr @__profd_test_vtable_value_profiling, i32 1)
+; LOWER-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
+; LOWER-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
+; LOWER-NEXT: ret i32 [[ADD]]
+;
+entry:
+ %call = tail call ptr @_Z10createTypei(i32 %c)
+ %add.ptr = getelementptr inbounds i8, ptr %call, i64 8
+ %vtable = load ptr, ptr %add.ptr, align 8
+ %vfunc = load ptr, ptr %vtable, align 8
+ %call1 = tail call i32 %vfunc(ptr %add.ptr, i32 %a, i32 %b)
+ %vtable2 = load ptr, ptr %call, align 8
+ %vfunc2 = load ptr, ptr %vtable2, align 8
+ %call4 = tail call i32 %vfunc2(ptr %call, i32 %b, i32 %a)
+ %add = add nsw i32 %call4, %call1
+ ret i32 %add
+}
+
+!0 = !{i64 16, !"_ZTS5Base1"}
+!1 = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
+!2 = !{i64 40, !"_ZTSM5Base1FiiiE.virtual"}
+!3 = !{i64 40, !"_ZTS5Base2"}
+!4 = !{i64 16, !"_ZTSM5Base2FiiiE.virtual"}
+!5 = !{i64 40, !"_ZTSM5Base2FiiiE.virtual"}
+!6 = !{i64 16, !"_ZTS7Derived"}
+!7 = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"}
+!8 = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
+!9 = !{i64 16, !"_ZTS5Base2"}
+;.
+; GEN: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+;.
+; LOWER: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+;.
+; GEN: [[META0:![0-9]+]] = !{i64 16, !"_ZTS5Base1"}
+; GEN: [[META1:![0-9]+]] = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
+; GEN: [[META2:![0-9]+]] = !{i64 40, !"_ZTSM5Base1FiiiE.virtual"}
+; GEN: [[META3:![0-9]+]] = !{i64 40, !"_ZTS5Base2"}
+; GEN: [[META4:![0-9]+]] = !{i64 16, !"_ZTSM5Base2FiiiE.virtual"}
+; GEN: [[META5:![0-9]+]] = !{i64 40, !"_ZTSM5Base2FiiiE.virtual"}
+; GEN: [[META6:![0-9]+]] = !{i64 16, !"_ZTS7Derived"}
+; GEN: [[META7:![0-9]+]] = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"}
+; GEN: [[META8:![0-9]+]] = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
+; GEN: [[META9:![0-9]+]] = !{i64 16, !"_ZTS5Base2"}
+;.
+; LOWER: [[META0:![0-9]+]] = !{i64 16, !"_ZTS5Base1"}
+; LOWER: [[META1:![0-9]+]] = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
+; LOWER: [[META2:![0-9]+]] = !{i64 40, !"_ZTSM5Base1FiiiE.virtual"}
+; LOWER: [[META3:![0-9]+]] = !{i64 40, !"_ZTS5Base2"}
+; LOWER: [[META4:![0-9]+]] = !{i64 16, !"_ZTSM5Base2FiiiE.virtual"}
+; LOWER: [[META5:![0-9]+]] = !{i64 40, !"_ZTSM5Base2FiiiE.virtual"}
+; LOWER: [[META6:![0-9]+]] = !{i64 16, !"_ZTS7Derived"}
+; LOWER: [[META7:![0-9]+]] = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"}
+; LOWER: [[META8:![0-9]+]] = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
+; LOWER: [[META9:![0-9]+]] = !{i64 16, !"_ZTS5Base2"}
+;.
diff --git a/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw b/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw
index bc8fc5db1cb154d98ca962e84313463e3298cb92..a065c4dd7b9ce20f631c1228147c97dc46dae8bb 100644
GIT binary patch
delta 184
zcmeC+`@zTASd^AuRFLzZb0Vib$GLNibNIp-7$!F`Nlc6|nB2g^F|k8{QDNdtc}9kb
z8^sv|CY}Vb9*Q$2O#BICO%`MnXS^|4lM%#r1hOqAM}pafKz7IEPB42Rqd3!nzY}N4
z0o7b&6zBZ#_dgggOg_jgKbeC`fa${DiCJ<W+a_Bu32^R!%NGLWGnfQ`CRH-YPmW*`
K*t~!#g9QKz%0Qq1
delta 226
zcmeyt*TKixSd^AuRFLzZV<M+LCwGqn0|+oqOf(m4Udy`lqjO(I&}GvIF&jY_jforO
zC*BZXOqjS)d=evvVCa+zP1gCUanf@<V@;ADUZ|L?$S6PAgHeF-2T;Z20-zFmUzxmw
zcE{>#|HF2)MP+VFm^=}vWCu{m4j at Z>@&llnuTyPVyaQr|<1ZS8=|7iS^J6k2ll){2
zCIKdcf0M14<b-NpJU>z}?X}aSS0831wSTCP_&2diZt at JE0*-%^S24*6MxEwya$o3I
Va?uU!c!md)A2P{r=3p*h0RV;ZQW5|F
diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
new file mode 100644
index 000000000000000..bf0ad6beb9825b8
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
@@ -0,0 +1,73 @@
+# IR level Instrumentation Flag
+:ir
+_Z10createTypei
+# Func Hash:
+146835647075900052
+# Num Counters:
+2
+# Counter Values:
+750
+250
+
+_ZN8Derived15func1Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+250
+
+_ZN8Derived15func2Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+250
+
+_ZN8Derived25func1Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+750
+
+_ZN8Derived25func2Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+750
+
+main
+# Func Hash:
+1124236338992350536
+# Num Counters:
+2
+# Counter Values:
+1000
+1
+# Num Value Kinds:
+2
+# ValueKind = IPVK_IndirectCallTarget:
+0
+# NumValueSites:
+2
+2
+_ZN8Derived25func1Eii:750
+_ZN8Derived15func1Eii:250
+2
+_ZN8Derived25func2Eii:750
+_ZN8Derived15func2Eii:250
+# ValueKind = IPVK_VTableTarget:
+2
+# NumValueSites:
+2
+2
+_ZTV8Derived2:750
+_ZTV8Derived1:250
+2
+_ZTV8Derived2:750
+_ZTV8Derived1:250
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.profraw b/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.profraw
new file mode 100644
index 0000000000000000000000000000000000000000..bc52463abfb388a4bb7873ebc503726d0ef78ccd
GIT binary patch
literal 816
zcmZoHO3N=Q$obF700xW at ih+R*#(>fsXncDp|G<9;XkF}No*c at +AOjU=LZhQ+^;o+Z
z?TnjpE6Bob6(<u+Ux0=)SRc%Im_8W)*VW$VW+_u|d$d~S{u7(UKLcvt1em_f5d9#1
zFbvY$H&IVgWp>j9pz`;ZxqP__VEQys?Sp$5YT-hLgPJ$_ at 7;vybAakwKtSKa@@n4Z
zZ>;?=eF9K at 8wlu|{As3#rP1CVn7#-9AQ}%4(Dx1+P`{w=e!&c3!2AOfhtV+gFgnM!
z;#PORk%@<%m#>Z|?@AqyvngkM&zwo_>HoC3z*F~>j<>I&iLdrqZ?lrKXO4)NItiN|
zT6K+MofyNli7Go7j<4KY|LlfI(+?X+cq%~MF- at h$YkkyXR|i{EacL=#eO#Pyvr~H9
zA4xJY)N64ng0!vxVp!aR;}Oc401XG2KVabr^9L+mVDhkVgUN%<0+|gpj|1v1nEC)H
JAEus=JOHD*$_M}e
literal 0
HcmV?d00001
diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test
index 67db5c98ef323a7..891aac0be4c6033 100644
--- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test
+++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test
@@ -8,10 +8,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
// There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes.
// 2 * 8 binary ID sizes
// + 2 * 20 binary IDs (of size 20)
@@ -27,6 +29,8 @@ RUN: printf '\20\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Binary IDs - There are only two in this case that are 20 bytes.
RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw
@@ -52,13 +56,15 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\067\265\035\031\112\165\023\344' >> %t.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
-RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t.profraw
+RUN: printf '\xd0\xff\3\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\067\0\0\0\0\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/large-binary-id-size.test b/llvm/test/tools/llvm-profdata/large-binary-id-size.test
index 2394431e94de482..3362186e1011583 100644
--- a/llvm/test/tools/llvm-profdata/large-binary-id-size.test
+++ b/llvm/test/tools/llvm-profdata/large-binary-id-size.test
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -9,6 +9,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Check for a corrupted size being too large past the end of the file.
RUN: printf '\7\7\7\7\7\7\7\7' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test
index 06f418d0235d260..bfcf91a36f9403d 100644
--- a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test
+++ b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test
@@ -8,10 +8,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -21,6 +23,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Data Section
//
@@ -36,6 +40,7 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\3\0foo\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test
index b718cf0fd8e9723..70ddca3dfd7de07 100644
--- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test
+++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test
@@ -8,10 +8,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -21,6 +23,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Data Section
//
@@ -37,6 +41,7 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Make NumCounters = 0 so that we get "number of counters is zero" error message
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\3\0foo\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test
index 38e40334a6a690d..54f861e08970da1 100644
--- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test
+++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test
@@ -8,10 +8,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -21,6 +23,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Data Section
//
@@ -42,6 +46,7 @@ RUN: printf '\11\0\6\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Counter Section
diff --git a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test
index 171b5cc60878f4c..489e06468ac7d82 100644
--- a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test
+++ b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
// We should fail on this because the binary IDs is not a multiple of 8 bytes.
RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -10,6 +10,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Binary IDs - There are only two in this case that are 20 bytes.
RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test
index c0072bcbde1b387..c3f1ce1ac70d98b 100644
--- a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test
+++ b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test
@@ -12,6 +12,8 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t
RUN: printf '\0\0\0\1\0\4\0\0' >> %t
RUN: printf '\0\0\0\2\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: not llvm-profdata show %t -o /dev/null 2>&1 | FileCheck %s
diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
index c8e862009ef0284..f7ded4b2ed82497 100644
--- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
+++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
@@ -1,5 +1,6 @@
+// Header
RUN: printf '\377lprofR\201' > %t
-RUN: printf '\0\0\0\0\0\0\0\10' >> %t
+RUN: printf '\0\0\0\0\0\0\0\11' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
@@ -9,6 +10,10 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t
RUN: printf '\0\0\0\0\1\0\0\0' >> %t
RUN: printf '\0\0\0\0\2\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumVTables
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// ValueKindLast
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\134\370\302\114\333\030\275\254' >> %t
RUN: printf '\0\0\0\0\0\0\0\1' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test
index 523ff1ceb4807a3..2d95f5ba9cbd979 100644
--- a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test
+++ b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test
@@ -1,5 +1,5 @@
RUN: printf '\201Rforpl\377' > %t
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\2\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
@@ -9,6 +9,10 @@ RUN: printf '\20\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\1\0\0\0\0' >> %t
RUN: printf '\0\0\0\2\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumVTables
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// ValueKindLast
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\254\275\030\333\114\302\370\134' >> %t
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test
index b2b8b31dafbf5ac..05ca0cf98d7beb2 100644
--- a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test
+++ b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test
@@ -1,5 +1,6 @@
+// Header
RUN: printf '\377lprofr\201' > %t
-RUN: printf '\0\0\0\0\0\0\0\10' >> %t
+RUN: printf '\0\0\0\0\0\0\0\11' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
@@ -9,20 +10,28 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t
RUN: printf '\0\0\0\1\0\4\0\0' >> %t
RUN: printf '\0\0\0\2\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumVTables
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// ValueKindLast
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// InstrProfData
RUN: printf '\134\370\302\114\333\030\275\254' >> %t
RUN: printf '\0\0\0\0\0\0\0\1' >> %t
RUN: printf '\0\0\0\1\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\1\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// InstrProfData
RUN: printf '\344\023\165\112\031\035\265\067' >> %t
RUN: printf '\0\0\0\0\0\0\0\02' >> %t
-RUN: printf '\0\0\0\1\0\3\xff\xd8' >> %t
+RUN: printf '\0\0\0\1\0\3\xff\xd0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\02\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\023' >> %t
RUN: printf '\0\0\0\0\0\0\0\067' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
index 4e95798bc0afbda..d8e12ef705a2582 100644
--- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
+++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
@@ -1,28 +1,53 @@
RUN: printf '\201rforpl\377' > %t
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t
+// Version
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t
+// BinaryIdsSize
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumData
RUN: printf '\2\0\0\0\0\0\0\0' >> %t
+// PaddingBytesBeforeCounters
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumCounters
RUN: printf '\3\0\0\0\0\0\0\0' >> %t
+// PaddingBytesAfterCounters
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NamesSize
RUN: printf '\20\0\0\0\0\0\0\0' >> %t
+// CountersDelta
RUN: printf '\0\0\4\0\1\0\0\0' >> %t
+// NamesDelta
RUN: printf '\0\0\4\0\2\0\0\0' >> %t
+// VNamesSize
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumVTables
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// ValueKindLast
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// InstrProfData
+// NameRef
RUN: printf '\254\275\030\333\114\302\370\134' >> %t
+// FuncHash
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
+// RelativeCounterPtr
RUN: printf '\0\0\4\0\1\0\0\0' >> %t
+// FunctionPointer
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// Values
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumCounters + Int16ArrayTy
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
+// Int16ArrayTy + padding
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// InstrProfData
RUN: printf '\067\265\035\031\112\165\023\344' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
-RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t
+RUN: printf '\xd0\xff\3\0\1\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\023\0\0\0\0\0\0\0' >> %t
RUN: printf '\067\0\0\0\0\0\0\0' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-two-profiles.test b/llvm/test/tools/llvm-profdata/raw-two-profiles.test
index 8d46c91e2732cd9..5038b6af9232d39 100644
--- a/llvm/test/tools/llvm-profdata/raw-two-profiles.test
+++ b/llvm/test/tools/llvm-profdata/raw-two-profiles.test
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t-foo.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
@@ -9,6 +9,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\254\275\030\333\114\302\370\134' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
@@ -16,12 +18,13 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw
RUN: printf '\201rforpl\377' > %t-bar.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
@@ -31,6 +34,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\067\265\035\031\112\165\023\344' >> %t-bar.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw
@@ -38,6 +43,7 @@ RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\067\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\101\0\0\0\0\0\0\0' >> %t-bar.profraw
diff --git a/llvm/test/tools/llvm-profdata/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
new file mode 100644
index 000000000000000..6ebe9b44266b6f2
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
@@ -0,0 +1,16 @@
+RUN: llvm-profdata show --function=main --show-vtables %p/Inputs/vtable-prof.proftext | FileCheck %s --check-prefix=VTABLE
+
+VTABLE: Counters:
+VTABLE: main:
+VTABLE: Hash: 0x0f9a16fe6d398548
+VTABLE: Counters: 2
+VTABLE: VTable Results:
+VTABLE: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+VTABLE: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+VTABLE: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+VTABLE: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+VTABLE: Instrumentation level: IR entry_first = 0
+VTABLE: Functions shown: 1
+VTABLE: Total functions: 6
+VTABLE: Maximum function count: 1000
+VTABLE: Maximum internal block count: 250
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
new file mode 100644
index 000000000000000..9dd7fa288969075
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -0,0 +1,100 @@
+To update the inputs used below, run Inputs/update_vtable_value_prof_inputs.sh /path/to/updated/clang++ /path/to/updated/llvm-profdata
+
+Show profile data from raw profiles.
+RUN: llvm-profdata show --function=main --ic-targets --show-vtables %p/Inputs/vtable_prof.profraw | FileCheck %s --check-prefix=RAW
+
+Generate indexed profile from raw profile and show the data.
+RUN: llvm-profdata merge %p/Inputs/vtable_prof.profraw -o %t_indexed.profdata
+RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t_indexed.profdata | FileCheck %s --check-prefix=INDEXED
+
+Generate text profile from raw profile and show the data.
+RUN: llvm-profdata merge --text %p/Inputs/vtable_prof.profraw -o %t.proftext
+RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t.proftext | FileCheck %s --check-prefix=ICTEXT
+
+RAW: Counters:
+RAW-NEXT: main:
+RAW-NEXT: Hash: 0x0f9a16fe6d398548
+RAW-NEXT: Counters: 2
+RAW-NEXT: Indirect Call Site Count: 2
+RAW-NEXT: VTable Results:
+RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: Indirect Target Results:
+RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+RAW-NEXT: Instrumentation level: IR entry_first = 0
+RAW-NEXT: Functions shown: 1
+RAW-NEXT: Total functions: 6
+RAW-NEXT: Maximum function count: 1000
+RAW-NEXT: Maximum internal block count: 250
+RAW-NEXT: Statistics for indirect call sites profile:
+RAW-NEXT: Total number of sites: 2
+RAW-NEXT: Total number of sites with values: 2
+RAW-NEXT: Total number of profiled values: 4
+RAW-NEXT: Value sites histogram:
+RAW-NEXT: NumTargets, SiteCount
+RAW-NEXT: 2, 2
+
+INDEXED: Counters:
+INDEXED-NEXT: main:
+INDEXED-NEXT: Hash: 0x0f9a16fe6d398548
+INDEXED-NEXT: Counters: 2
+INDEXED-NEXT: Indirect Call Site Count: 2
+INDEXED-NEXT: VTable Results:
+INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: Indirect Target Results:
+INDEXED-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
+INDEXED-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
+INDEXED-NEXT: Instrumentation level: IR entry_first = 0
+INDEXED-NEXT: Functions shown: 1
+INDEXED-NEXT: Total functions: 6
+INDEXED-NEXT: Maximum function count: 1000
+INDEXED-NEXT: Maximum internal block count: 250
+INDEXED-NEXT: Statistics for indirect call sites profile:
+INDEXED-NEXT: Total number of sites: 2
+INDEXED-NEXT: Total number of sites with values: 2
+INDEXED-NEXT: Total number of profiled values: 4
+INDEXED-NEXT: Value sites histogram:
+INDEXED-NEXT: NumTargets, SiteCount
+INDEXED-NEXT: 2, 2
+
+ICTEXT: :ir
+ICTEXT: main
+ICTEXT: # Func Hash:
+ICTEXT: 1124236338992350536
+ICTEXT: # Num Counters:
+ICTEXT: 2
+ICTEXT: # Counter Values:
+ICTEXT: 1000
+ICTEXT: 1
+ICTEXT: # Num Value Kinds:
+ICTEXT: 2
+ICTEXT: # ValueKind = IPVK_IndirectCallTarget:
+ICTEXT: 0
+ICTEXT: # NumValueSites:
+ICTEXT: 2
+ICTEXT: 2
+ICTEXT: _ZN8Derived25func1Eii:750
+ICTEXT: _ZN8Derived15func1Eii:250
+ICTEXT: 2
+ICTEXT: _ZN8Derived25func2Eii:750
+ICTEXT: _ZN8Derived15func2Eii:250
+ICTEXT: # ValueKind = IPVK_VTableTarget:
+ICTEXT: 2
+ICTEXT: # NumValueSites:
+ICTEXT: 2
+ICTEXT: 2
+ICTEXT: _ZTV8Derived2:750
+ICTEXT: _ZTV8Derived1:250
+ICTEXT: 2
+ICTEXT: _ZTV8Derived2:750
+ICTEXT: _ZTV8Derived1:250
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 7d665a8005b0d62..205b3b325284e94 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -355,6 +355,13 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
});
}
+ const InstrProfSymtab &symtab = Reader->getSymtab();
+ const auto &VTableNames = symtab.getVTableNames();
+
+ for (const auto &kv : VTableNames) {
+ WC->Writer.addVTableName(kv.getKey());
+ }
+
if (Reader->hasTemporalProfile()) {
auto &Traces = Reader->getTemporalProfTraces(Input.Weight);
if (!Traces.empty())
@@ -2422,12 +2429,12 @@ static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK,
static int showInstrProfile(
const std::string &Filename, bool ShowCounts, uint32_t TopN,
- bool ShowIndirectCallTargets, bool ShowMemOPSizes, bool ShowDetailedSummary,
- std::vector<uint32_t> DetailedSummaryCutoffs, bool ShowAllFunctions,
- bool ShowCS, uint64_t ValueCutoff, bool OnlyListBelow,
- const std::string &ShowFunction, bool TextFormat, bool ShowBinaryIds,
- bool ShowCovered, bool ShowProfileVersion, bool ShowTemporalProfTraces,
- ShowFormat SFormat, raw_fd_ostream &OS) {
+ bool ShowIndirectCallTargets, bool ShowMemOPSizes, bool ShowVTables,
+ bool ShowDetailedSummary, std::vector<uint32_t> DetailedSummaryCutoffs,
+ bool ShowAllFunctions, bool ShowCS, uint64_t ValueCutoff,
+ bool OnlyListBelow, const std::string &ShowFunction, bool TextFormat,
+ bool ShowBinaryIds, bool ShowCovered, bool ShowProfileVersion,
+ bool ShowTemporalProfTraces, ShowFormat SFormat, raw_fd_ostream &OS) {
if (SFormat == ShowFormat::Json)
exitWithError("JSON output is not supported for instr profiles");
if (SFormat == ShowFormat::Yaml)
@@ -2563,6 +2570,13 @@ static int showInstrProfile(
OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls
<< "\n";
+ if (ShowVTables) {
+ OS << " VTable Results:\n";
+ traverseAllValueSites(Func, IPVK_VTableTarget,
+ VPStats[IPVK_VTableTarget], OS,
+ &(Reader->getSymtab()));
+ }
+
if (ShowCounts) {
OS << " Block counts: [";
size_t Start = (IsIRInstr ? 0 : 1);
@@ -2943,6 +2957,9 @@ static int show_main(int argc, const char *argv[]) {
cl::opt<bool> ShowIndirectCallTargets(
"ic-targets", cl::init(false),
cl::desc("Show indirect call site target values for shown functions"));
+ cl::opt<bool> ShowVTables(
+ "show-vtables", cl::init(false),
+ cl::desc("Show virtual table target values for shown functions"));
cl::opt<bool> ShowMemOPSizes(
"memop-sizes", cl::init(false),
cl::desc("Show the profiled sizes of the memory intrinsic calls "
@@ -3045,10 +3062,10 @@ static int show_main(int argc, const char *argv[]) {
if (ProfileKind == instr)
return showInstrProfile(
Filename, ShowCounts, TopNFunctions, ShowIndirectCallTargets,
- ShowMemOPSizes, ShowDetailedSummary, DetailedSummaryCutoffs,
- ShowAllFunctions, ShowCS, ValueCutoff, OnlyListBelow, ShowFunction,
- TextFormat, ShowBinaryIds, ShowCovered, ShowProfileVersion,
- ShowTemporalProfTraces, SFormat, OS);
+ ShowMemOPSizes, ShowVTables, ShowDetailedSummary,
+ DetailedSummaryCutoffs, ShowAllFunctions, ShowCS, ValueCutoff,
+ OnlyListBelow, ShowFunction, TextFormat, ShowBinaryIds, ShowCovered,
+ ShowProfileVersion, ShowTemporalProfTraces, SFormat, OS);
if (ProfileKind == sample)
return showSampleProfile(Filename, ShowCounts, TopNFunctions,
ShowAllFunctions, ShowDetailedSummary,
>From cb0246ffea45ee6805d26a76e0d25b091be9ac3d Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Fri, 29 Sep 2023 14:42:19 -0700
Subject: [PATCH 02/10] Address feebacks on the code. Questions will be
followed up shortly. - In IndirectCallVisitor, instrument vtables and skip
the rest (e.g., function pointers, etc). - Add brief comments for helper
functions, remove debugging logs and rename vtable name section to vtabnames.
---
compiler-rt/include/profile/InstrProfData.inc | 15 +-
compiler-rt/lib/profile/InstrProfiling.h | 10 +-
.../lib/profile/InstrProfilingBuffer.c | 11 +-
compiler-rt/lib/profile/InstrProfilingMerge.c | 4 +-
.../lib/profile/InstrProfilingPlatformLinux.c | 4 +-
.../lib/profile/InstrProfilingWriter.c | 13 +-
.../profile/instrprof-write-buffer-internal.c | 8 --
.../llvm/Analysis/IndirectCallVisitor.h | 51 +++++--
llvm/include/llvm/ProfileData/InstrProf.h | 17 ++-
.../llvm/ProfileData/InstrProfData.inc | 2 +-
llvm/lib/ProfileData/InstrProf.cpp | 16 +--
llvm/lib/ProfileData/InstrProfWriter.cpp | 14 +-
.../indirect_call_profile_funclet.ll | 4 +
.../Transforms/PGOProfile/vtable_profile.ll | 133 +++++++++++++-----
14 files changed, 202 insertions(+), 100 deletions(-)
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index 5609870a5a8e5b4..64d9bad24524d6b 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -92,6 +92,9 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
/* INSTR_PROF_DATA end. */
+/* For a virtual table object, record the name hash to associate profiled addresses
+ * with global variables, and record {starting address, size in bytes} to map the profiled virtual table (which usually have an offset from the starting address)
+ * back to a virtual table object. */
#ifndef INSTR_PROF_VTABLE_DATA
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
#else
@@ -192,7 +195,15 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
-VALUE_PROF_KIND(IPVK_VTableTarget, 2, "vtable target")
+/* For virtual table address profiling, the addresses of the virtual table (i.e., the address contained in objects pointing to a virtual table) are
+ * profiled. Note this may not be the address of the per C++ class virtual table object (i.e., there is an offset).
+ *
+ * The profiled addresses are stored in raw profile, together with the following two types of information.
+ * 1. The (beginning and ending) addresses of per C++ class virtual table objects.
+ * 2. The (compressed) virtual table object names.
+ * RawInstrProfReader converts profiled virtual table addresses to virtual table objects' MD5 hash.
+ */
+VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The address of the compatible vtable (i.e., there is an offset from this address to per C++ class virtual table global variable.)")
/* These two kinds must be the last to be
* declared. This is to make sure the string
* array created with the template can be
@@ -707,7 +718,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
-#define INSTR_PROF_VNAME_COMMON __llvm_prf_vnames
+#define INSTR_PROF_VNAME_COMMON __llvm_prf_vtabnames
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index f3afa694e02c27d..c3aa72e411f7991 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -92,8 +92,8 @@ const __llvm_profile_data *__llvm_profile_begin_data(void);
const __llvm_profile_data *__llvm_profile_end_data(void);
const char *__llvm_profile_begin_names(void);
const char *__llvm_profile_end_names(void);
-const char *__llvm_profile_begin_vnames(void);
-const char *__llvm_profile_end_vnames(void);
+const char *__llvm_profile_begin_vtabnames(void);
+const char *__llvm_profile_end_vtabnames(void);
char *__llvm_profile_begin_counters(void);
char *__llvm_profile_end_counters(void);
ValueProfNode *__llvm_profile_begin_vnodes();
@@ -286,11 +286,13 @@ uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End);
/*! \brief Get the size of the profile counters section in bytes. */
uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End);
+/*! \brief Get the number of virtual table profile data entries */
uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
const VTableProfData *End);
-uint64_t __llvm_profile_get_vtable_size(const VTableProfData *Begin,
- const VTableProfData *End);
+/*! \brief Get the size of virtual table profile data in bytes. */
+uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,
+ const VTableProfData *End);
/* ! \brief Given the sizes of the data and counter information, return the
* number of padding bytes before and after the counters, and after the names,
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index 0c36e40444c7344..ad1643d48908df2 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -49,8 +49,8 @@ uint64_t __llvm_profile_get_size_for_buffer(void) {
const char *NamesEnd = __llvm_profile_end_names();
const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
- const char *VNamesBegin = __llvm_profile_begin_vnames();
- const char *VNamesEnd = __llvm_profile_end_vnames();
+ const char *VNamesBegin = __llvm_profile_begin_vtabnames();
+ const char *VNamesEnd = __llvm_profile_end_vtabnames();
return __llvm_profile_get_size_for_buffer_internal(
DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd,
@@ -78,8 +78,8 @@ uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
}
COMPILER_RT_VISIBILITY
-uint64_t __llvm_profile_get_vtable_size(const VTableProfData *Begin,
- const VTableProfData *End) {
+uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,
+ const VTableProfData *End) {
return __llvm_profile_get_num_vtable(Begin, End) * sizeof(VTableProfData);
}
@@ -166,7 +166,8 @@ uint64_t __llvm_profile_get_size_for_buffer_internal(
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
uint64_t CountersSize =
__llvm_profile_get_counters_size(CountersBegin, CountersEnd);
- uint64_t VTableSize = __llvm_profile_get_vtable_size(VTableBegin, VTableEnd);
+ uint64_t VTableSize =
+ __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
uint64_t VNameSize = (VNamesEnd - VNamesBegin) * sizeof(char);
/* Determine how much padding is needed before/after the counters and after
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 2ef6227599ff139..629d502cdde3127 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -125,9 +125,7 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
Header->NumCounters * __llvm_profile_counter_entry_size();
SrcNameStart = SrcCountersEnd;
// This is to assume counter size is a multiple of 8 bytes.
- // uint64_t NamesSize = Header->NamesSize;
- // uint64_t PaddingBytesAfterNames =
- // __llvm_profile_get_num_padding_bytes(Header->NamesSize);
+
// First, skip rather than merge them
uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData);
uint64_t PaddingBytesAfterVTableSection =
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
index dc861632271ce79..c681e253bb46a4a 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -76,10 +76,10 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) {
COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) {
return &PROF_NAME_STOP;
}
-COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vnames(void) {
+COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vtabnames(void) {
return &PROF_VNAME_START;
}
-COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vnames(void) {
+COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vtabnames(void) {
return &PROF_VNAME_STOP;
}
COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_begin_vtables(void) {
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index ed5dbdb6ee4383f..b998618aad7896f 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -250,8 +250,8 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
const char *NamesEnd = __llvm_profile_end_names();
const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
- const char *VNamesBegin = __llvm_profile_begin_vnames();
- const char *VNamesEnd = __llvm_profile_end_vnames();
+ const char *VNamesBegin = __llvm_profile_begin_vtabnames();
+ const char *VNamesEnd = __llvm_profile_end_vtabnames();
return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
CountersEnd, VPDataReader, NamesBegin, NamesEnd,
VTableBegin, VTableEnd, VNamesBegin, VNamesEnd,
@@ -282,9 +282,8 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const uint64_t NumVTables =
__llvm_profile_get_num_vtable(VTableBegin, VTableEnd);
const uint64_t VTableSectionSize =
- __llvm_profile_get_vtable_size(VTableBegin, VTableEnd);
- // Note, in reality, vtable profiling is not supported when DebugInfoCorrelate
- // is true.
+ __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
+ // Note vtable profiling is not supported when DebugInfoCorrelate is true.
const uint64_t VNamesSize = DebugInfoCorrelate ? 0 : VNamesEnd - VNamesBegin;
/* Create the header. */
@@ -320,13 +319,9 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
/* Write the profile header. */
ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
- // printf("Size of profile header is %d\n",
- // (int)(sizeof(__llvm_profile_header)));
if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
return -1;
- // printf("Completed profile header\n");
-
/* Write the binary id lengths and data. */
int binary_id_size = __llvm_write_binary_ids(Writer);
if (binary_id_size == -1)
diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
index 97dfff8d7fe71fb..484aa1f4f2d0eaf 100644
--- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c
+++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
@@ -48,20 +48,12 @@ int main(int argc, const char *argv[]) {
__llvm_profile_begin_names(), __llvm_profile_end_names(), NULL, NULL,
NULL, NULL);
- // printf("buffer size is %lld\n", bufsize);
- //uint64_t aligned_bufsize = ((bufsize + 32) >> 6) << 6;
-
char *buf = malloc(bufsize);
int ret = __llvm_profile_write_buffer_internal(buf,
__llvm_profile_begin_data(), __llvm_profile_end_data(),
__llvm_profile_begin_counters(), __llvm_profile_end_counters(),
__llvm_profile_begin_names(), __llvm_profile_end_names());
- if (ret != 0) {
- fprintf(stderr, "failed to write buffer");
- return ret;
- }
-
FILE *f = fopen(argv[1], "w");
fwrite(buf, bufsize, 1, f);
fclose(f);
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 52d4ff04d3d4ecc..23168b88a6988ce 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -12,6 +12,7 @@
#ifndef LLVM_ANALYSIS_INDIRECTCALLVISITOR_H
#define LLVM_ANALYSIS_INDIRECTCALLVISITOR_H
+#include "llvm/ADT/SetVector.h"
#include "llvm/IR/InstVisitor.h"
#include <vector>
@@ -19,24 +20,58 @@ namespace llvm {
// Visitor class that finds all indirect call.
struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
std::vector<CallBase *> IndirectCalls;
- std::vector<Instruction *> VTableAddrs;
+ SetVector<Instruction *, std::vector<Instruction *>> VTableAddrs;
PGOIndirectCallVisitor() = default;
void visitCallBase(CallBase &Call) {
- if (Call.isIndirectCall())
+ const CallInst *CI = dyn_cast<CallInst>(&Call);
+ if (CI && CI->getCalledFunction()) {
+ switch (CI->getCalledFunction()->getIntrinsicID()) {
+ case Intrinsic::type_test:
+ case Intrinsic::public_type_test:
+ case Intrinsic::type_checked_load_relative:
+ case Intrinsic::type_checked_load: {
+ Value *VTablePtr = CI->getArgOperand(0)->stripPointerCasts();
+
+ if (PtrTestedByTypeIntrinsics.count(VTablePtr) == 0) {
+ Instruction *I = dyn_cast_or_null<Instruction>(VTablePtr);
+ // This is the first type intrinsic where VTablePtr is used.
+ // Assert that the VTablePtr is not found as a type profiling
+ // candidate yet. Note nullptr won't be inserted into VTableAddrs in
+ // the first place, so this assertion works even if 'VTablePtr' is not
+ // an instruction.
+ assert(VTableAddrs.count(I) == 0 &&
+ "Expect type intrinsic to record VTablePtr before virtual "
+ "functions are loaded to find vtables that should be "
+ "instrumented");
+
+ PtrTestedByTypeIntrinsics.insert(VTablePtr);
+ }
+ } break;
+ }
+ }
if (Call.isIndirectCall()) {
IndirectCalls.push_back(&Call);
-
LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
if (LI != nullptr) {
- Value *Ptr = LI->getPointerOperand();
- Value *VTablePtr = Ptr->stripInBoundsConstantOffsets();
- if (VTablePtr != nullptr && isa<Instruction>(VTablePtr)) {
- VTableAddrs.push_back(cast<Instruction>(VTablePtr));
+ Value *MaybeVTablePtr =
+ LI->getPointerOperand()->stripInBoundsConstantOffsets();
+ Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
+ // If not used by any type intrinsic, this is not a vtable.
+ // Inst visitor should see the very first type intrinsic using a
+ // vtable before the very first virtual function load from this
+ // vtable. This condition is asserted above.
+ if (VTableInstr && PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
+ VTableAddrs.insert(VTableInstr);
}
}
}
}
+
+private:
+ // Keeps track of the pointers that are tested by llvm type intrinsics for
+ // look up.
+ SmallPtrSet<Value *, 4> PtrTestedByTypeIntrinsics;
};
inline std::vector<CallBase *> findIndirectCalls(Function &F) {
@@ -48,7 +83,7 @@ inline std::vector<CallBase *> findIndirectCalls(Function &F) {
inline std::vector<Instruction *> findVTableAddrs(Function &F) {
PGOIndirectCallVisitor ICV;
ICV.visit(F);
- return ICV.VTableAddrs;
+ return ICV.VTableAddrs.takeVector();
}
} // namespace llvm
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 845ea1d0ab0b3af..41b882863f1d40e 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -440,9 +440,13 @@ class InstrProfSymtab {
private:
StringRef Data;
uint64_t Address = 0;
- // Unique name strings.
+ // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's
+ // going to be sorted) has unique MD5 keys in the first place.
StringSet<> NameTab;
- // Unique virtual table names.
+ // Records the unique virtual table names. This is used by InstrProfWriter to
+ // write out an on-disk chained hash table of virtual table names.
+ // InstrProfWriter stores per function profile data (keyed by function names)
+ // so it doesn't use a StringSet for function names.
StringSet<> VTableNames;
// A map from MD5 keys to function name strings.
std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
@@ -532,17 +536,16 @@ class InstrProfSymtab {
if (VTableName.empty())
return make_error<InstrProfError>(instrprof_error::malformed,
"invalid input: VTableName is empty");
- // Insert into NameTab.
+ // Insert into NameTab so that MD5NameMap (a vector that is going to be
+ // sorted) won't have duplicated entries in the first place.
auto Ins = NameTab.insert(VTableName);
- // Insert into VTableNames.
+ // Record VTableName. InstrProfWriter uses this map. The comment around
+ // class member explains why.
VTableNames.insert(VTableName);
// If this is newly added, update MD5NameMap.
if (Ins.second) {
- // printf("VTableName %s\n", VTableName.str().c_str());
- // printf("AddVTableName hash %"PRIu64" to %s\n",
- // IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey());
MD5NameMap.push_back(std::make_pair(
IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey()));
Sorted = false;
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index f1e7691afdc4bf0..4c8f1b31c5e8dc5 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -706,7 +706,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
-#define INSTR_PROF_VNAME_COMMON __llvm_prf_vnames
+#define INSTR_PROF_VNAME_COMMON __llvm_prf_vtabnames
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 31d29a6c4bd8b08..c981e09ed3151c2 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -498,20 +498,15 @@ Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
finalizeSymtab();
- // printf("look up key 0x%llx\n", Address);
- // for (auto iter = VTableAddrToMD5Map.begin(); iter !=
- // VTableAddrToMD5Map.end(); iter++) {
- // printf("<key, val> is <0x%llx, %"PRIu64"\n", iter->first, iter->second);
- // }
auto It =
partition_point(VTableAddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
return A.first < Address;
});
- // FIXME: Does the raw function pointers point apply here?
+ // The virtual table address collected from value profiler could be defined
+ // in another module that is not instrumented. Force the value to be 0 in
+ // this case.
if (It != VTableAddrToMD5Map.end()) {
- // printf("InstrProfSymtab::getVTableHashFromAddress map addr 0x%llx to hash
- // value %"PRIu64"\n", Address, (uint64_t)It->second);
- return (uint64_t)It->second;
+ return It->second;
}
return 0;
}
@@ -596,7 +591,6 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
std::string &Result, bool doCompression) {
std::vector<std::string> VTableNameStrs;
for (auto *VTable : VTables) {
- // printf("VTable name %s added\n", VTable->getName().str().c_str());
VTableNameStrs.push_back(std::string(VTable->getName()));
}
return collectPGOFuncNameStrings(
@@ -691,13 +685,11 @@ Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab) {
}
NameStrings.split(Names, getInstrProfNameSeparator());
for (StringRef &Name : Names) {
- // printf("Read back vtable name %s\n", Name.str().c_str());
if (Error E = Symtab.addVTableName(Name))
return E;
}
P += Dist;
- // Skip padding?
while (P < EndP && *P == 0)
P++;
}
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 0094cf8c029fca1..0713864c21d973c 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -634,19 +634,22 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
OS.writeByte(0);
}
- // if version >= the version with vtable profile metadata
- // Intentionally put vtable names before temporal profile section.
+ // if version >= the version with vtable profile metadata.
uint64_t VTableNamesSectionStart = 0;
if (IndexedInstrProf::ProfVersion::CurrentVersion >= 11) {
VTableNamesSectionStart = OS.tell();
- // Reserve space for vtable record table offset.
+ // Reserve space for vtable records offset.
OS.write(0ULL);
OnDiskChainedHashTableGenerator<llvm::InstrProfRecordVTableTrait>
VTableNamesGenerator;
for (const auto &kv : VTableNames) {
- // printf("InstrProfWriter.cpp key is %s\n", kv.getKey().str().c_str());
+ // Use a char '0' as value placeholder, only keys (vtable names)
+ // are used.
+ // FIXME: It might make sense to have a OnDiskChainedHashSetGenerator if
+ // there are more use cases. Use a hash table for now, with one unused
+ // 'char' per entry.
VTableNamesGenerator.insert(kv.getKey(), '0');
}
@@ -656,9 +659,6 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
uint64_t VTableNamesTableOffset =
VTableNamesGenerator.Emit(OS.OS, *VTableNamesWriter);
- // printf("InstrProfWriter.cpp:VTableNamesSectionStart is %"PRIu64"\n",
- // VTableNamesSectionStart); printf("\tVTableNamesTableOffset is
- // %"PRIu64"\n", VTableNamesTableOffset);
PatchItem PatchItems[] = {
{VTableNamesSectionStart, &VTableNamesTableOffset, 1}};
OS.patch(PatchItems, 1);
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
index 63173ab59b28ba2..988a30865a7fee6 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
@@ -45,6 +45,8 @@ catch.dispatch: ; preds = %entry
catch: ; preds = %catch.dispatch
%tmp1 = catchpad within %tmp [ptr null, i32 64, ptr null]
%vtable = load ptr, ptr %b, align 8
+ %0 = tail call i1 @llvm.type.test(ptr %vtable, metadata !"_ZTS4base")
+ tail call void @llvm.assume(i1 %0)
%tmp3 = load ptr, ptr %vtable, align 8
call void %tmp3(ptr %b) [ "funclet"(token %tmp1) ]
catchret from %tmp1 to label %try.cont
@@ -64,3 +66,5 @@ try.cont: ; preds = %catch, %entry
declare dso_local void @"?may_throw@@YAXH at Z"(i32)
declare dso_local i32 @__CxxFrameHandler3(...)
+declare i1 @llvm.type.test(ptr, metadata)
+declare void @llvm.assume(i1 noundef)
diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
index 0c6bd3c064d35fc..059552a7634dc3c 100644
--- a/llvm/test/Transforms/PGOProfile/vtable_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
@@ -1,9 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 3
; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
+; The test IR is generated based on the following C++ program.
+; class Derived uses multiple inheritance so its virtual table
+; global variable contains two vtables. func1 is loaded from
+; the vtable compatible with class Base1, and func2 is loaded
+; from the vtable compatible with class Base2.
+; class Base1 {
+; public:
+; virtual int func1(int a, int b) ;
+; };
+;
+; class Base2 {
+; public:
+; virtual int func2(int a, int b);
+; };
+;
+; class Derived : public Base1, public Base2 {
+; public:
+; Derived(int c) : v(c) {}
+; private:
+; int v;
+; };
+;
+; Derived* createType(int c);
+; int func(int a, int b, int c) {
+; Derived* d = createType(c);
+;
+; return d->func2(a, b) + d->func1(b, a);
+; }
+
$_ZTV7Derived = comdat any
@_ZTV7Derived = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
@@ -19,25 +49,44 @@ $_ZTV7Derived = comdat any
declare ptr @_Z10createTypei(i32)
declare i32 @_ZN5Base15func1Eii(ptr, i32, i32)
declare i32 @_ZN5Base25func2Eii(ptr, i32, i32)
+declare i1 @llvm.public.type.test(ptr, metadata)
+declare void @llvm.assume(i1 noundef)
-; GEN: @__llvm_profile_raw_version = hidden constant i64 72057594037927945, comdat
-; GEN: @__profn_test_vtable_value_profiling = private constant [27 x i8] c"test_vtable_value_profiling"
-
-; LOWER: $__profvt__ZTV7Derived = comdat any
-; LOWER: $__profvt__ZTV5Base1 = comdat nodeduplicate
-; LOWER: $__profvt__ZTV5Base2 = comdat nodeduplicate
-; LOWER: @__llvm_profile_raw_version = hidden constant i64 72057594037927945, comdat
-; LOWER: @__profc_test_vtable_value_profiling = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
-; LOWER: @__profvp_test_vtable_value_profiling = private global [4 x i64] zeroinitializer, section "__llvm_prf_vals", comdat($__profc_test_vtable_value_profiling), align 8
-; LOWER: @__profd_test_vtable_value_profiling = private global { i64, i64, i64, ptr, ptr, i32, [3 x i16] } { i64 1593873508557585901, i64 567090795815895039, i64 sub (i64 ptrtoint (ptr @__profc_test_vtable_value_profiling to i64), i64 ptrtoint (ptr @__profd_test_vtable_value_profiling to i64)), ptr @test_vtable_value_profiling.local, ptr @__profvp_test_vtable_value_profiling, i32 1, [3 x i16] [i16 2, i16 0, i16 2] }, section "__llvm_prf_data", comdat($__profc_test_vtable_value_profiling), align 8
-; LOWER: @__profvt__ZTV7Derived = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8
-; LOWER: @__profvt__ZTV5Base1 = global { i64, ptr, i32 } { i64 3215870116411581797, ptr @_ZTV5Base1, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
-; LOWER: @__profvt__ZTV5Base2 = global { i64, ptr, i32 } { i64 8378219803387680050, ptr @_ZTV5Base2, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
-; LOWER: @__llvm_prf_vnodes = private global [10 x { i64, i64, ptr }] zeroinitializer, section "__llvm_prf_vnds", align 8
-; LOWER: @__llvm_prf_nm = private constant [37 x i8] c"\1B#x\DA+I-.\89/+IL\CAI\8D/K\CC)M\8D/(\CAO\CB\CC\C9\CCK\07\00\9Ea\0BC", section "__llvm_prf_names", align 1
-; LOWER: @__llvm_prf_vnm = private constant [34 x i8] c"\22 x\DA\8B\8F\0A\093wI-\CA,KMa\8C\07rL\9D\12\8BS\0D\11L#\00\C3\A2\0A\E9", section "__llvm_prf_vnames", align 1
-; LOWER: @llvm.used = appending global [6 x ptr] [ptr @__profvt__ZTV7Derived, ptr @__profvt__ZTV5Base1, ptr @__profvt__ZTV5Base2, ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata"
-
+;.
+; GEN: @[[_ZTV7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
+; GEN: @[[_ZTVN10__CXXABIV121__VMI_CLASS_TYPE_INFOE:[a-zA-Z0-9_$"\\.-]+]] = external global [0 x ptr]
+; GEN: @[[_ZTS7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant [9 x i8] c"7Derived\00", align 1
+; GEN: @[[_ZTI5BASE1:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
+; GEN: @[[_ZTI5BASE2:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
+; GEN: @[[_ZTI7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { ptr, ptr, i32, i32, ptr, i64, ptr, i64 } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), ptr @_ZTS7Derived, i32 0, i32 2, ptr @_ZTI5Base1, i64 2, ptr @_ZTI5Base2, i64 2050 }, align 8
+; GEN: @[[_ZTV5BASE1:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base1, ptr @_ZN5Base15func1Eii] }, align 8, !type !0, !type !1
+; GEN: @[[_ZTV5BASE2:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base2, ptr @_ZN5Base25func2Eii] }, align 8, !type !9, !type !4
+; GEN: @[[LLVM_COMPILER_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [2 x ptr] [ptr @_ZTV5Base1, ptr @_ZTV5Base2], section "llvm.metadata"
+; GEN: @[[__LLVM_PROFILE_RAW_VERSION:[a-zA-Z0-9_$"\\.-]+]] = hidden constant i64 72057594037927945, comdat
+; GEN: @[[__PROFN_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private constant [27 x i8] c"test_vtable_value_profiling"
+;.
+; LOWER: @[[_ZTV7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
+; LOWER: @[[_ZTVN10__CXXABIV121__VMI_CLASS_TYPE_INFOE:[a-zA-Z0-9_$"\\.-]+]] = external global [0 x ptr]
+; LOWER: @[[_ZTS7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant [9 x i8] c"7Derived\00", align 1
+; LOWER: @[[_ZTI5BASE1:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
+; LOWER: @[[_ZTI5BASE2:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
+; LOWER: @[[_ZTI7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { ptr, ptr, i32, i32, ptr, i64, ptr, i64 } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), ptr @_ZTS7Derived, i32 0, i32 2, ptr @_ZTI5Base1, i64 2, ptr @_ZTI5Base2, i64 2050 }, align 8
+; LOWER: @[[_ZTV5BASE1:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base1, ptr @_ZN5Base15func1Eii] }, align 8, !type !0, !type !1
+; LOWER: @[[_ZTV5BASE2:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base2, ptr @_ZN5Base25func2Eii] }, align 8, !type !9, !type !4
+; LOWER: @[[__LLVM_PROFILE_RAW_VERSION:[a-zA-Z0-9_$"\\.-]+]] = hidden constant i64 72057594037927945, comdat
+; LOWER: @[[__PROFC_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
+; LOWER: @[[__PROFVP_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private global [4 x i64] zeroinitializer, section "__llvm_prf_vals", comdat($__profc_test_vtable_value_profiling), align 8
+; LOWER: @[[__PROFD_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private global { i64, i64, i64, ptr, ptr, i32, [3 x i16] } { i64 1593873508557585901, i64 567090795815895039, i64 sub (i64 ptrtoint (ptr @__profc_test_vtable_value_profiling to i64), i64 ptrtoint (ptr @__profd_test_vtable_value_profiling to i64)), ptr @test_vtable_value_profiling.local, ptr @__profvp_test_vtable_value_profiling, i32 1, [3 x i16] [i16 2, i16 0, i16 2] }, section "__llvm_prf_data", comdat($__profc_test_vtable_value_profiling), align 8
+; LOWER: @[[__PROFVT__ZTV7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @[[__PROFVT__ZTV5BASE1:[a-zA-Z0-9_$"\\.-]+]] = global { i64, ptr, i32 } { i64 3215870116411581797, ptr @_ZTV5Base1, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @[[__PROFVT__ZTV5BASE2:[a-zA-Z0-9_$"\\.-]+]] = global { i64, ptr, i32 } { i64 8378219803387680050, ptr @_ZTV5Base2, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @[[__LLVM_PRF_VNODES:[a-zA-Z0-9_$"\\.-]+]] = private global [10 x { i64, i64, ptr }] zeroinitializer, section "__llvm_prf_vnds", align 8
+; LOWER: @[[__LLVM_PRF_NM:[a-zA-Z0-9_$"\\.-]+]] = private constant [37 x i8] c"\1B#x\DA+I-.\89/+IL\CAI\8D/K\CC)M\8D/(\CAO\CB\CC\C9\CCK\07\00\9Ea\0BC", section "__llvm_prf_names", align 1
+; LOWER: @[[__LLVM_PRF_VNM:[a-zA-Z0-9_$"\\.-]+]] = private constant [34 x i8] c"\22 x\DA\8B\8F\0A\093wI-\CA,KMa\8C\07rL\9D\12\8BS\0D\11L#\00\C3\A2\0A\E9", section "__llvm_prf_vtabnames", align 1
+; LOWER: @[[LLVM_COMPILER_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [3 x ptr] [ptr @_ZTV5Base1, ptr @_ZTV5Base2, ptr @__profd_test_vtable_value_profiling], section "llvm.metadata"
+; LOWER: @[[LLVM_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [6 x ptr] [ptr @__profvt__ZTV7Derived, ptr @__profvt__ZTV5Base1, ptr @__profvt__ZTV5Base2, ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata"
+; LOWER: @[[TEST_VTABLE_VALUE_PROFILING_LOCAL:[a-zA-Z0-9_$"\\.-]+]] = private alias i32 (i32, i32, i32), ptr @test_vtable_value_profiling
+;.
define i32 @test_vtable_value_profiling(i32 %a, i32 %b, i32 %c) {
; GEN-LABEL: define i32 @test_vtable_value_profiling(
; GEN-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) {
@@ -48,16 +97,20 @@ define i32 @test_vtable_value_profiling(i32 %a, i32 %b, i32 %c) {
; GEN-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
; GEN-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[VTABLE]] to i64
; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP0]], i32 2, i32 0)
+; GEN-NEXT: [[TMP1:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE]], metadata !"_ZTS5Base2")
+; GEN-NEXT: tail call void @llvm.assume(i1 [[TMP1]])
; GEN-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
-; GEN-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[VFUNC]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP1]], i32 0, i32 0)
+; GEN-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VFUNC]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP2]], i32 0, i32 0)
; GEN-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
; GEN-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
-; GEN-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP2]], i32 2, i32 1)
+; GEN-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP3]], i32 2, i32 1)
+; GEN-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE2]], metadata !"_ZTS5Base1")
+; GEN-NEXT: tail call void @llvm.assume(i1 [[TMP4]])
; GEN-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
-; GEN-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP3]], i32 0, i32 1)
+; GEN-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP5]], i32 0, i32 1)
; GEN-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
; GEN-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
; GEN-NEXT: ret i32 [[ADD]]
@@ -73,27 +126,39 @@ define i32 @test_vtable_value_profiling(i32 %a, i32 %b, i32 %c) {
; LOWER-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
; LOWER-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[VTABLE]] to i64
; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP1]], ptr @__profd_test_vtable_value_profiling, i32 2)
+; LOWER-NEXT: [[TMP2:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE]], metadata !"_ZTS5Base2")
+; LOWER-NEXT: tail call void @llvm.assume(i1 [[TMP2]])
; LOWER-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
-; LOWER-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VFUNC]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP2]], ptr @__profd_test_vtable_value_profiling, i32 0)
+; LOWER-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VFUNC]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP3]], ptr @__profd_test_vtable_value_profiling, i32 0)
; LOWER-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
; LOWER-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
-; LOWER-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP3]], ptr @__profd_test_vtable_value_profiling, i32 3)
+; LOWER-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP4]], ptr @__profd_test_vtable_value_profiling, i32 3)
+; LOWER-NEXT: [[TMP5:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE2]], metadata !"_ZTS5Base1")
+; LOWER-NEXT: tail call void @llvm.assume(i1 [[TMP5]])
; LOWER-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
-; LOWER-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP4]], ptr @__profd_test_vtable_value_profiling, i32 1)
+; LOWER-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP6]], ptr @__profd_test_vtable_value_profiling, i32 1)
; LOWER-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
; LOWER-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
; LOWER-NEXT: ret i32 [[ADD]]
;
entry:
%call = tail call ptr @_Z10createTypei(i32 %c)
+ ; The first indirect call is 'func2' so func ptr is
+ ; loaded from vtable compatible with Base2.
%add.ptr = getelementptr inbounds i8, ptr %call, i64 8
%vtable = load ptr, ptr %add.ptr, align 8
+ %0 = tail call i1 @llvm.public.type.test(ptr %vtable, metadata !"_ZTS5Base2")
+ tail call void @llvm.assume(i1 %0)
%vfunc = load ptr, ptr %vtable, align 8
%call1 = tail call i32 %vfunc(ptr %add.ptr, i32 %a, i32 %b)
+ ; The second indirect call is 'func1' so func ptr is
+ ; loaded from vtable compatible with Base1.
%vtable2 = load ptr, ptr %call, align 8
+ %1 = tail call i1 @llvm.public.type.test(ptr %vtable2, metadata !"_ZTS5Base1")
+ tail call void @llvm.assume(i1 %1)
%vfunc2 = load ptr, ptr %vtable2, align 8
%call4 = tail call i32 %vfunc2(ptr %call, i32 %b, i32 %a)
%add = add nsw i32 %call4, %call1
@@ -111,9 +176,13 @@ entry:
!8 = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
!9 = !{i64 16, !"_ZTS5Base2"}
;.
-; GEN: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+; GEN: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; GEN: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+; GEN: attributes #[[ATTR2:[0-9]+]] = { nounwind }
;.
-; LOWER: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+; LOWER: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; LOWER: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+; LOWER: attributes #[[ATTR2:[0-9]+]] = { nounwind }
;.
; GEN: [[META0:![0-9]+]] = !{i64 16, !"_ZTS5Base1"}
; GEN: [[META1:![0-9]+]] = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
>From 3c6ca047be0069012ecf65c6d50c1fa7992fc3c3 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Mon, 2 Oct 2023 13:46:09 -0700
Subject: [PATCH 03/10] In InstrProf.cpp, add option
-enable-vtable-type-profiling to flag control vtable instrumentation, and
record address range of vtables (in VTableAddRangeToMD5Map). - The new option
is used in PGOInstrumentation.cpp and InstrProfiling.cpp to flag-control
the instrumentation of static and runtime information. Static information
includes vtable address range and name md5 hash, runtime information are
types of a vtable value. - Before this commit, VTableAddrToMD5Map records
start and end of vtable address individually. After this commit, the map
records a range to return MD5 iff the address is within the range. This is
more accurate when runtime address is collected in one module but static
vtable information is not recorded.
---
.../llvm/Analysis/IndirectCallVisitor.h | 43 ++++++++++++-------
llvm/include/llvm/ProfileData/InstrProf.h | 12 +++---
llvm/lib/ProfileData/InstrProf.cpp | 23 +++++++---
.../Instrumentation/InstrProfiling.cpp | 16 ++++---
.../Instrumentation/PGOInstrumentation.cpp | 9 +++-
.../Inputs/update_vtable_value_prof_inputs.sh | 5 +--
6 files changed, 71 insertions(+), 37 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 23168b88a6988ce..347f72bc625cb85 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -17,15 +17,20 @@
#include <vector>
namespace llvm {
-// Visitor class that finds all indirect call.
+// Visitor class that finds indirect calls or instructions that gives vtable
+// value, depending on Type.
struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
+ enum class InstructionType {
+ kIndirectCall = 0,
+ kVTableVal = 1,
+ };
std::vector<CallBase *> IndirectCalls;
SetVector<Instruction *, std::vector<Instruction *>> VTableAddrs;
- PGOIndirectCallVisitor() = default;
+ PGOIndirectCallVisitor(InstructionType Type) : Type(Type) {}
void visitCallBase(CallBase &Call) {
const CallInst *CI = dyn_cast<CallInst>(&Call);
- if (CI && CI->getCalledFunction()) {
+ if (Type == InstructionType::kVTableVal && CI && CI->getCalledFunction()) {
switch (CI->getCalledFunction()->getIntrinsicID()) {
case Intrinsic::type_test:
case Intrinsic::public_type_test:
@@ -52,17 +57,20 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
}
if (Call.isIndirectCall()) {
IndirectCalls.push_back(&Call);
- LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
- if (LI != nullptr) {
- Value *MaybeVTablePtr =
- LI->getPointerOperand()->stripInBoundsConstantOffsets();
- Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
- // If not used by any type intrinsic, this is not a vtable.
- // Inst visitor should see the very first type intrinsic using a
- // vtable before the very first virtual function load from this
- // vtable. This condition is asserted above.
- if (VTableInstr && PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
- VTableAddrs.insert(VTableInstr);
+ if (Type == InstructionType::kVTableVal) {
+ LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
+ if (LI != nullptr) {
+ Value *MaybeVTablePtr =
+ LI->getPointerOperand()->stripInBoundsConstantOffsets();
+ Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
+ // If not used by any type intrinsic, this is not a vtable.
+ // Inst visitor should see the very first type intrinsic using a
+ // vtable before the very first virtual function load from this
+ // vtable. This condition is asserted above.
+ if (VTableInstr &&
+ PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
+ VTableAddrs.insert(VTableInstr);
+ }
}
}
}
@@ -72,16 +80,19 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
// Keeps track of the pointers that are tested by llvm type intrinsics for
// look up.
SmallPtrSet<Value *, 4> PtrTestedByTypeIntrinsics;
+ InstructionType Type;
};
inline std::vector<CallBase *> findIndirectCalls(Function &F) {
- PGOIndirectCallVisitor ICV;
+ PGOIndirectCallVisitor ICV(
+ PGOIndirectCallVisitor::InstructionType::kIndirectCall);
ICV.visit(F);
return ICV.IndirectCalls;
}
inline std::vector<Instruction *> findVTableAddrs(Function &F) {
- PGOIndirectCallVisitor ICV;
+ PGOIndirectCallVisitor ICV(
+ PGOIndirectCallVisitor::InstructionType::kVTableVal);
ICV.visit(F);
return ICV.VTableAddrs.takeVector();
}
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 41b882863f1d40e..0c7cce9b7355e46 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -436,6 +436,8 @@ uint64_t ComputeHash(StringRef K);
class InstrProfSymtab {
public:
using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
+ using RangeHashMap =
+ std::vector<std::pair<std::pair<uint64_t, uint64_t>, uint64_t>>;
private:
StringRef Data;
@@ -463,7 +465,7 @@ class InstrProfSymtab {
// This map is only populated and used by raw instr profile reader.
// This is a different map from 'AddrToMD5Map' for readability and
// debuggability.
- AddrHashMap VTableAddrToMD5Map;
+ RangeHashMap VTableAddrRangeToMD5Map;
bool Sorted = false;
static StringRef getExternalSymbol() {
@@ -561,11 +563,11 @@ class InstrProfSymtab {
AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
}
- // Map the start and end address of a variable to its names' MD5 hash.
- // This interface is only used by the raw profile header.
+ // Map the address range (i.e., [start_address, end_address]) of a variable to
+ // its names' MD5 hash. This interface is only used by the raw profile header.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
- VTableAddrToMD5Map.push_back(std::make_pair(StartAddr, MD5Val));
- VTableAddrToMD5Map.push_back(std::make_pair(EndAddr, MD5Val));
+ VTableAddrRangeToMD5Map.push_back(
+ std::make_pair(std::make_pair(StartAddr, EndAddr), MD5Val));
}
/// Return a function's hash, or 0, if the function isn't in this SymTab.
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index c981e09ed3151c2..54b9e397862b258 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -215,6 +215,13 @@ cl::opt<bool> DoInstrProfNameCompression(
"enable-name-compression",
cl::desc("Enable name/filename string compression"), cl::init(true));
+cl::opt<bool> EnableVTableValueProfiling(
+ "enable-vtable-value-profiling", cl::init(true),
+ cl::desc("If true, the virtual table address will be instrumented to know "
+ "the types of a C++ pointer. The information could be used in "
+ "indirect-call-promotion to do selective vtable-based comparison "
+ "and interprocedural type propagation."));
+
std::string getInstrProfSectionName(InstrProfSectKind IPSK,
Triple::ObjectFormatType OF,
bool AddSegmentInfo) {
@@ -498,16 +505,18 @@ Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
finalizeSymtab();
- auto It =
- partition_point(VTableAddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
- return A.first < Address;
- });
+ auto It = lower_bound(
+ VTableAddrRangeToMD5Map, Address,
+ [](std::pair<std::pair<uint64_t, uint64_t>, uint64_t> VTableRangeAddr,
+ uint64_t Addr) { return VTableRangeAddr.first.second < Addr; });
+
+ // Returns the MD5 hash if Address is within the address range of an entry.
+ if (It != VTableAddrRangeToMD5Map.end() && It->first.first <= Address) {
+ return It->second;
+ }
// The virtual table address collected from value profiler could be defined
// in another module that is not instrumented. Force the value to be 0 in
// this case.
- if (It != VTableAddrToMD5Map.end()) {
- return It->second;
- }
return 0;
}
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 844b5397574b51b..a405a63fe5d43df 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -64,6 +64,10 @@ cl::opt<bool>
DebugInfoCorrelate("debug-info-correlate",
cl::desc("Use debug info to correlate profiles."),
cl::init(false));
+
+// Command line option to enable vtable value profiling. Defined in
+// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
+extern cl::opt<bool> EnableVTableValueProfiling;
} // namespace llvm
namespace {
@@ -561,10 +565,12 @@ bool InstrProfiling::run(
static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
}
- for (GlobalVariable &GV : M.globals()) {
- // Global variables with type metadata are virtual table variables.
- if (GV.hasMetadata(LLVMContext::MD_type)) {
- getOrCreateVTableProfData(&GV);
+ if (EnableVTableValueProfiling) {
+ for (GlobalVariable &GV : M.globals()) {
+ // Global variables with type metadata are virtual table variables.
+ if (GV.hasMetadata(LLVMContext::MD_type)) {
+ getOrCreateVTableProfData(&GV);
+ }
}
}
@@ -1376,7 +1382,7 @@ void InstrProfiling::emitNameData() {
}
void InstrProfiling::emitVTableNames() {
- if (ReferencedVTableNames.empty())
+ if (!EnableVTableValueProfiling || ReferencedVTableNames.empty())
return;
// Collect VTable
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 75c57d4d559e5df..810dff726a05975 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -327,6 +327,11 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
extern cl::opt<std::string> ViewBlockFreqFuncName;
+extern cl::opt<bool> DebugInfoCorrelate;
+
+// Command line option to enable vtable value profiling. Defined in
+// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
+extern cl::opt<bool> EnableVTableValueProfiling;
} // namespace llvm
static cl::opt<bool>
@@ -584,7 +589,9 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
NumOfPGOBB += MST.BBInfos.size();
ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget);
- ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
+ if (EnableVTableValueProfiling) {
+ ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
+ }
} else {
NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh b/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
index 1b35ea0303d15d2..5c594e10e7370f0 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
+++ b/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
@@ -1,12 +1,11 @@
#!/bin/bash
if [ $# -lt 2 ]; then
- echo "Path to clang++ and llvm-profdata required!"
- echo "Usage: update_vtable_value_prof_inputs.sh /path/to/updated/clang++ /path/to/updated/llvm-profdata"
+ echo "Path to clang++ required!"
+ echo "Usage: update_vtable_value_prof_inputs.sh /path/to/updated/clang++"
exit 1
else
CLANG=$1
- LLVMPROFDATA=$2
fi
OUTDIR=$(dirname $(realpath -s $0))
>From 3780bd988618ac3586882f14c1d8a04a9ae9af6f Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Mon, 2 Oct 2023 16:27:49 -0700
Subject: [PATCH 04/10] A few fixes: 1. Format the code indentation in
IndirectCallVisitor.h, and add a FIXME to do more efficient vtable
instrumentation. 2. In InstrProfSymtab::finalizeSymtab, sort and uniquify
VTableAddrRangeToMD5Map (somehow forgot this when preparing this patch,
the full prototype includes profile-use did this already.)
---
.../llvm/Analysis/IndirectCallVisitor.h | 37 +++++++++++--------
llvm/include/llvm/ProfileData/InstrProf.h | 7 ++++
2 files changed, 28 insertions(+), 16 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 347f72bc625cb85..7fe9cba88410113 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -55,25 +55,30 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
} break;
}
}
- if (Call.isIndirectCall()) {
- IndirectCalls.push_back(&Call);
- if (Type == InstructionType::kVTableVal) {
- LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
- if (LI != nullptr) {
- Value *MaybeVTablePtr =
- LI->getPointerOperand()->stripInBoundsConstantOffsets();
- Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
- // If not used by any type intrinsic, this is not a vtable.
- // Inst visitor should see the very first type intrinsic using a
- // vtable before the very first virtual function load from this
- // vtable. This condition is asserted above.
- if (VTableInstr &&
- PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
- VTableAddrs.insert(VTableInstr);
- }
+ if (Call.isIndirectCall()) {
+ IndirectCalls.push_back(&Call);
+ if (Type == InstructionType::kVTableVal) {
+ // Note without -fstrict-vtable-pointers, vtable pointers of the same
+ // objects are loaded multiple times, and current implementation
+ // instruments each load once.
+ // FIXME: For more efficient instrumentation, analyze load invariant
+ // vtable values (e.g., from the same pointer in C++) and instrument
+ // them once.
+ LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
+ if (LI != nullptr) {
+ Value *MaybeVTablePtr =
+ LI->getPointerOperand()->stripInBoundsConstantOffsets();
+ Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
+ // If not used by any type intrinsic, this is not a vtable.
+ // Inst visitor should see the very first type intrinsic using a
+ // vtable before the very first virtual function load from this
+ // vtable. This condition is asserted above.
+ if (VTableInstr && PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
+ VTableAddrs.insert(VTableInstr);
}
}
}
+ }
}
private:
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 0c7cce9b7355e46..ae6828ac7e68dc2 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -667,6 +667,13 @@ void InstrProfSymtab::finalizeSymtab() {
llvm::sort(AddrToMD5Map, less_first());
AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
AddrToMD5Map.end());
+ // GlobalVariable address ranges should not overlap; so sort by either
+ // beginning address or end address is fine.
+ llvm::sort(VTableAddrRangeToMD5Map, less_first());
+ // std::unique uses == operator for std::pair.
+ VTableAddrRangeToMD5Map.erase(std::unique(VTableAddrRangeToMD5Map.begin(),
+ VTableAddrRangeToMD5Map.end()),
+ VTableAddrRangeToMD5Map.end());
Sorted = true;
}
>From 925586756cbc31eb199b42858de128951c5d5415 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Tue, 3 Oct 2023 22:59:21 -0700
Subject: [PATCH 05/10] resolve feedbacks on code
---
compiler-rt/lib/profile/InstrProfilingMerge.c | 6 ++++--
compiler-rt/lib/profile/InstrProfilingWriter.c | 6 ------
compiler-rt/test/profile/instrprof-write-buffer-internal.c | 5 +++++
llvm/include/llvm/ProfileData/InstrProf.h | 5 -----
llvm/lib/ProfileData/InstrProf.cpp | 5 +++--
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp | 2 --
6 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 629d502cdde3127..5f67e25b1edc993 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -124,9 +124,11 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
SrcCountersEnd = SrcCountersStart +
Header->NumCounters * __llvm_profile_counter_entry_size();
SrcNameStart = SrcCountersEnd;
- // This is to assume counter size is a multiple of 8 bytes.
- // First, skip rather than merge them
+ // Skip vtable profile data section and vtable names sections for runtime
+ // profile merge. To merge runtime addresses from multiple profiles, the
+ // same instrumented binary should run with ASLR disabled -> in this set-up
+ // these two sections remain unchanged.
uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData);
uint64_t PaddingBytesAfterVTableSection =
__llvm_profile_get_num_padding_bytes(VTableSectionSize);
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index b998618aad7896f..18affce48153f73 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -327,12 +327,6 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
if (binary_id_size == -1)
return -1;
- // Might be needed for debugging. Clean up before commit.
- // uint64_t VTableProfDataOffset =
- // sizeof(__llvm_profile_header) + binary_id_size + DataSectionSize +
- // PaddingBytesBeforeCounters + CountersSectionSize +
- // PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames;
-
/* Write the profile data. */
ProfDataIOVec IOVecData[] = {
{DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize,
diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
index 484aa1f4f2d0eaf..7ac65cdd62c982e 100644
--- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c
+++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
@@ -54,6 +54,11 @@ int main(int argc, const char *argv[]) {
__llvm_profile_begin_counters(), __llvm_profile_end_counters(),
__llvm_profile_begin_names(), __llvm_profile_end_names());
+ if (ret != 0) {
+ fprintf(stderr, "failed to write buffer");
+ return ret;
+ }
+
FILE *f = fopen(argv[1], "w");
fwrite(buf, bufsize, 1, f);
fclose(f);
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index ae6828ac7e68dc2..953fd5263918347 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -590,11 +590,6 @@ class InstrProfSymtab {
/// this symbol table.
inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash);
- /// Just like getFuncName, except that it will return a non-empty StringRef
- /// if the function is external to this symbol table. All such cases
- /// will be represented using the same StringRef value.
- // inline StringRef getVTableNameOrExternalSymbol(uint64_t VTableMD5Hash);
-
/// True if Symbol is the value used to represent external symbols.
static bool isExternalSymbol(const StringRef &Symbol) {
return Symbol == InstrProfSymtab::getExternalSymbol();
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 54b9e397862b258..311892b068bf17e 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -220,7 +220,9 @@ cl::opt<bool> EnableVTableValueProfiling(
cl::desc("If true, the virtual table address will be instrumented to know "
"the types of a C++ pointer. The information could be used in "
"indirect-call-promotion to do selective vtable-based comparison "
- "and interprocedural type propagation."));
+ "and interprocedural type propagation. Requires type metadata and "
+ "type intrinsics (https://llvm.org/docs/TypeMetadata.html) to use"
+ " this option."));
std::string getInstrProfSectionName(InstrProfSectKind IPSK,
Triple::ObjectFormatType OF,
@@ -462,7 +464,6 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) {
Types.clear();
G.getMetadata(LLVMContext::MD_type, Types);
if (!Types.empty()) {
- // errs() << "Insert " << G.getGUID() << "\t into MD5VTableMap\n";
MD5VTableMap.emplace_back(G.getGUID(), &G);
}
}
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index a405a63fe5d43df..ae73e0edaf5d744 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1405,8 +1405,6 @@ void InstrProfiling::emitVTableNames() {
VTableNamesVar->setAlignment(Align(1));
// Make VTableNames linker retained.
UsedVars.push_back(VTableNamesVar);
-
- // FIXME: Why emitNames call erase method?
}
void InstrProfiling::emitRegistration() {
>From 8d8a45a2e51e89a8b3053d32e7a92ad4c2758b54 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Fri, 27 Oct 2023 17:31:05 -0700
Subject: [PATCH 06/10] Rebase onto main (a fresh commit on Oct27) and add the
following changes - Set option '-enable-vtable-value-profiling' to false.
This option should make this change no-op. - For indexed profile format,
store vtable names in one section by compressing them and storing blobs
strings. - Before this commit, they are stored as OnDiskIterableHashTable
but hash table is not necessary; vtable names are stored in indexed format
profiles for llvm-profdata usage. Function profiles are on-disk hash table so
compiler could look up profiles faster. Clean up lookuptrait and writer trait
as a result. - In index profile reader, track {VTableNamePtr,
CompressedVTableNameLen} as metadata so llvm-profdata could construct symtab
by decompressing vtable names. Meanwhile, when compiler reads indexed
profiles, no need to decompress vtables since symtab could be constructed
from module IR. - Use the PGOName of vtables when computing compressed
strings and MD5 hash in instrumentation stage. - Previously, the vtable
name is used. The PGOName and name of vtables are the same for global
symbols. For local symbols, PGOName has the format 'filename:varname' - In
IndirectCallVisitor.h, use a heuristc to find address feeding instructions.
Currently clang doesn't emit type intrinsics without
'-fwhole-program-vtables'. '-fwhole-program-vtables' requires '-flto', and
'-flto' might not be enabled at instrumentation phase. - In
ValueProfilePlugins.inc, skip landingpad and catchpad instructions (to keep
them the first non-phi) when finding an insertion point for value profile
intrinsic. - In InstrProf.cpp, simplify string decompression code by adding
helper functions. - In llvm-profdata.cpp, print the number of instrumented
vtable sites and vpstats.
---
compiler-rt/include/profile/InstrProfData.inc | 2 +-
.../lib/profile/InstrProfilingBuffer.c | 39 ++++-----
.../lib/profile/InstrProfilingInternal.h | 4 +-
compiler-rt/lib/profile/InstrProfilingMerge.c | 39 +++++----
.../lib/profile/InstrProfilingWriter.c | 2 +-
.../profile/instrprof-write-buffer-internal.c | 7 +-
.../llvm/Analysis/IndirectCallVisitor.h | 80 +++++++----------
llvm/include/llvm/ProfileData/InstrProf.h | 70 ++++++++-------
.../llvm/ProfileData/InstrProfData.inc | 2 +-
.../llvm/ProfileData/InstrProfReader.h | 72 ++++-----------
.../Instrumentation/InstrProfiling.h | 2 +-
llvm/lib/ProfileData/InstrProf.cpp | 78 +++++++----------
llvm/lib/ProfileData/InstrProfReader.cpp | 37 ++++----
llvm/lib/ProfileData/InstrProfWriter.cpp | 87 ++++++-------------
.../Instrumentation/InstrProfiling.cpp | 12 +--
.../Instrumentation/ValueProfilePlugins.inc | 8 +-
.../indirect_call_profile_funclet.ll | 9 +-
.../Transforms/PGOProfile/vtable_profile.ll | 4 +-
.../vtable-value-prof-basic.test | 39 ++++++---
llvm/tools/llvm-profdata/llvm-profdata.cpp | 23 +++--
20 files changed, 264 insertions(+), 352 deletions(-)
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index 64d9bad24524d6b..2753ad66ffe4c91 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -101,7 +101,7 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
#define INSTR_PROF_VTABLE_DATA_DEFINED
#endif
INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, \
- ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(VTableName)))
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(PGOVTableName)))
INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), VTablePointer, VTableAddr)
INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), VTableSizeVal))
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index ad1643d48908df2..34007cbf726ddeb 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -47,14 +47,9 @@ uint64_t __llvm_profile_get_size_for_buffer(void) {
const char *CountersEnd = __llvm_profile_end_counters();
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
- const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
- const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
- const char *VNamesBegin = __llvm_profile_begin_vtabnames();
- const char *VNamesEnd = __llvm_profile_end_vtabnames();
return __llvm_profile_get_size_for_buffer_internal(
- DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd,
- VTableBegin, VTableEnd, VNamesBegin, VNamesEnd);
+ DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd);
}
COMPILER_RT_VISIBILITY
@@ -132,8 +127,11 @@ void __llvm_profile_get_padding_sizes_for_counters(
*PaddingBytesAfterCounters =
__llvm_profile_get_num_padding_bytes(CountersSize);
*PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
- *PaddingBytesAfterVTable = __llvm_profile_get_num_padding_bytes(VTableSize);
- *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);
+ if (PaddingBytesAfterVTable != NULL)
+ *PaddingBytesAfterVTable =
+ __llvm_profile_get_num_padding_bytes(VTableSize);
+ if (PaddingBytesAfterVName != NULL)
+ *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);
return;
}
@@ -150,41 +148,36 @@ void __llvm_profile_get_padding_sizes_for_counters(
*PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
// Set these two variables to zero to avoid uninitialized variables
// even if VTableSize and VNameSize are asserted to be zero.
- *PaddingBytesAfterVTable = 0;
- *PaddingBytesAfterVName = 0;
+ if (PaddingBytesAfterVTable != NULL)
+ *PaddingBytesAfterVTable = 0;
+ if (PaddingBytesAfterVName != NULL)
+ *PaddingBytesAfterVName = 0;
}
COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
- const char *NamesEnd, const VTableProfData *VTableBegin,
- const VTableProfData *VTableEnd, const char *VNamesBegin,
- const char *VNamesEnd) {
+ const char *NamesEnd) {
/* Match logic in __llvm_profile_write_buffer(). */
const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
uint64_t CountersSize =
__llvm_profile_get_counters_size(CountersBegin, CountersEnd);
- uint64_t VTableSize =
- __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
- uint64_t VNameSize = (VNamesEnd - VNamesBegin) * sizeof(char);
/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
+ PaddingBytesAfterNames;
__llvm_profile_get_padding_sizes_for_counters(
- DataSize, CountersSize, NamesSize, VTableSize, VNameSize,
+ DataSize, CountersSize, NamesSize, 0 /* VTableSize */, 0 /* VNameSize */,
&PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
- &PaddingBytesAfterNames, &PaddingBytesAfterVTable,
- &PaddingBytesAfterVNames);
+ &PaddingBytesAfterNames, NULL /* PaddingBytesAfterVTable */,
+ NULL /* PaddingBytesAfterVNames */);
return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
DataSize + PaddingBytesBeforeCounters + CountersSize +
- PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames +
- VTableSize + PaddingBytesAfterVTable + VNameSize +
- PaddingBytesAfterVNames;
+ PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames;
}
COMPILER_RT_VISIBILITY
diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h
index bce333f933f0ffe..09a77acef97b22b 100644
--- a/compiler-rt/lib/profile/InstrProfilingInternal.h
+++ b/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -25,9 +25,7 @@
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
- const char *NamesEnd, const VTableProfData *VTableBegin,
- const VTableProfData *VTableEnd, const char *VNamesBegin,
- const char *VNamesEnd);
+ const char *NamesEnd);
/*!
* \brief Write instrumentation data to the given buffer, given explicit
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 5f67e25b1edc993..32b3058535ede73 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -98,6 +98,25 @@ static uintptr_t signextIfWin64(void *V) {
#endif
}
+static uint64_t
+getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
+ // Skip names section, vtable profile data section and vtable names section
+ // for runtime profile merge. To merge runtime addresses from multiple
+ // profiles, the same instrumented binary should run with ASLR disabled -> in
+ // this set-up these three sections remain unchanged.
+ const uint64_t VTableSectionSize =
+ Header->NumVTables * sizeof(VTableProfData);
+ const uint64_t PaddingBytesAfterVTableSection =
+ __llvm_profile_get_num_padding_bytes(VTableSectionSize);
+ const uint64_t VNamesSize = Header->VNamesSize;
+ const uint64_t PaddingBytesAfterVNamesSize =
+ __llvm_profile_get_num_padding_bytes(VNamesSize);
+ return Header->NamesSize +
+ __llvm_profile_get_num_padding_bytes(Header->NamesSize) +
+ VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize +
+ PaddingBytesAfterVNamesSize;
+}
+
COMPILER_RT_VISIBILITY
int __llvm_profile_merge_from_buffer(const char *ProfileData,
uint64_t ProfileSize) {
@@ -125,26 +144,8 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
Header->NumCounters * __llvm_profile_counter_entry_size();
SrcNameStart = SrcCountersEnd;
- // Skip vtable profile data section and vtable names sections for runtime
- // profile merge. To merge runtime addresses from multiple profiles, the
- // same instrumented binary should run with ASLR disabled -> in this set-up
- // these two sections remain unchanged.
- uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData);
- uint64_t PaddingBytesAfterVTableSection =
- __llvm_profile_get_num_padding_bytes(VTableSectionSize);
- uint64_t VNamesSize = Header->VNamesSize;
- uint64_t PaddingBytesAfterVNamesSize =
- __llvm_profile_get_num_padding_bytes(VNamesSize);
-
- uint64_t VTableProfDataOffset =
- SrcNameStart + Header->NamesSize +
- __llvm_profile_get_num_padding_bytes(Header->NamesSize);
-
- uint64_t VTableNamesOffset =
- VTableProfDataOffset + VTableSectionSize + PaddingBytesAfterVTableSection;
-
SrcValueProfDataStart =
- VTableNamesOffset + VNamesSize + PaddingBytesAfterVNamesSize;
+ SrcNameStart + getDistanceFromCounterToValueProf(Header);
if (SrcNameStart < SrcCountersStart)
return 1;
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 18affce48153f73..c96d3f17a32697f 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -283,7 +283,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
__llvm_profile_get_num_vtable(VTableBegin, VTableEnd);
const uint64_t VTableSectionSize =
__llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
- // Note vtable profiling is not supported when DebugInfoCorrelate is true.
+ // Value profiling is not supported when DebugInfoCorrelate is true.
const uint64_t VNamesSize = DebugInfoCorrelate ? 0 : VNamesEnd - VNamesBegin;
/* Create the header. */
diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
index 7ac65cdd62c982e..7b96c6d91c33f5a 100644
--- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c
+++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
@@ -28,9 +28,7 @@ char *__llvm_profile_end_counters(void);
uint64_t __llvm_profile_get_size_for_buffer_internal(
const void *DataBegin, const void *DataEnd, const char *CountersBegin,
- const char *CountersEnd, const char *NamesBegin, const char *NamesEnd,
- const char *VTableBegin, const char *VTableEnd, const char *VNamesBegin,
- const char *VNamesEnd);
+ const char *CountersEnd, const char *NamesBegin, const char *NamesEnd);
int __llvm_profile_write_buffer_internal(char *Buffer, const void *DataBegin,
const void *DataEnd,
@@ -45,8 +43,7 @@ int main(int argc, const char *argv[]) {
uint64_t bufsize = __llvm_profile_get_size_for_buffer_internal(
__llvm_profile_begin_data(), __llvm_profile_end_data(),
__llvm_profile_begin_counters(), __llvm_profile_end_counters(),
- __llvm_profile_begin_names(), __llvm_profile_end_names(), NULL, NULL,
- NULL, NULL);
+ __llvm_profile_begin_names(), __llvm_profile_end_names());
char *buf = malloc(bufsize);
int ret = __llvm_profile_write_buffer_internal(buf,
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 7fe9cba88410113..4a3e5b61e14f37e 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -25,66 +25,44 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
kVTableVal = 1,
};
std::vector<CallBase *> IndirectCalls;
- SetVector<Instruction *, std::vector<Instruction *>> VTableAddrs;
+ std::vector<Instruction *> ProfiledAddresses;
PGOIndirectCallVisitor(InstructionType Type) : Type(Type) {}
void visitCallBase(CallBase &Call) {
- const CallInst *CI = dyn_cast<CallInst>(&Call);
- if (Type == InstructionType::kVTableVal && CI && CI->getCalledFunction()) {
- switch (CI->getCalledFunction()->getIntrinsicID()) {
- case Intrinsic::type_test:
- case Intrinsic::public_type_test:
- case Intrinsic::type_checked_load_relative:
- case Intrinsic::type_checked_load: {
- Value *VTablePtr = CI->getArgOperand(0)->stripPointerCasts();
-
- if (PtrTestedByTypeIntrinsics.count(VTablePtr) == 0) {
- Instruction *I = dyn_cast_or_null<Instruction>(VTablePtr);
- // This is the first type intrinsic where VTablePtr is used.
- // Assert that the VTablePtr is not found as a type profiling
- // candidate yet. Note nullptr won't be inserted into VTableAddrs in
- // the first place, so this assertion works even if 'VTablePtr' is not
- // an instruction.
- assert(VTableAddrs.count(I) == 0 &&
- "Expect type intrinsic to record VTablePtr before virtual "
- "functions are loaded to find vtables that should be "
- "instrumented");
-
- PtrTestedByTypeIntrinsics.insert(VTablePtr);
- }
- } break;
- }
- }
if (Call.isIndirectCall()) {
IndirectCalls.push_back(&Call);
- if (Type == InstructionType::kVTableVal) {
- // Note without -fstrict-vtable-pointers, vtable pointers of the same
- // objects are loaded multiple times, and current implementation
- // instruments each load once.
- // FIXME: For more efficient instrumentation, analyze load invariant
- // vtable values (e.g., from the same pointer in C++) and instrument
- // them once.
- LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
- if (LI != nullptr) {
- Value *MaybeVTablePtr =
- LI->getPointerOperand()->stripInBoundsConstantOffsets();
- Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
- // If not used by any type intrinsic, this is not a vtable.
- // Inst visitor should see the very first type intrinsic using a
- // vtable before the very first virtual function load from this
- // vtable. This condition is asserted above.
- if (VTableInstr && PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
- VTableAddrs.insert(VTableInstr);
- }
+
+ if (Type != InstructionType::kVTableVal)
+ return;
+
+ LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
+ // The code pattern to look for
+ //
+ // %vtable = load ptr, ptr %b
+ // %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1
+ // %2 = load ptr, ptr %vfn
+ // %call = tail call i32 %2(ptr %b)
+ //
+ // %vtable is the vtable address value to profile, and
+ // %2 is the indirect call target address to profile.
+ if (LI != nullptr) {
+ Value *Ptr = LI->getPointerOperand();
+ Value *VTablePtr = Ptr->stripInBoundsConstantOffsets();
+ // This is a heuristic to find address feeding instructions.
+ // FIXME: Add support in the frontend so LLVM type intrinsics are
+ // emitted without LTO. This way, added intrinsics could filter
+ // non-vtable instructions and reduce instrumentation overhead.
+ // Note a profiled address will be dropped if it's not within
+ // the address range of vtable objects, so there are no correctness
+ // or performance issues at profile use time.
+ if (VTablePtr != nullptr && isa<Instruction>(VTablePtr)) {
+ ProfiledAddresses.push_back(cast<Instruction>(VTablePtr));
}
}
}
}
private:
- // Keeps track of the pointers that are tested by llvm type intrinsics for
- // look up.
- SmallPtrSet<Value *, 4> PtrTestedByTypeIntrinsics;
InstructionType Type;
};
@@ -99,9 +77,9 @@ inline std::vector<Instruction *> findVTableAddrs(Function &F) {
PGOIndirectCallVisitor ICV(
PGOIndirectCallVisitor::InstructionType::kVTableVal);
ICV.visit(F);
- return ICV.VTableAddrs.takeVector();
+ return ICV.ProfiledAddresses;
}
} // namespace llvm
-#endif
+#endif
\ No newline at end of file
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 953fd5263918347..d289a4b63337901 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -250,10 +250,10 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
/// \c NameStrings is a string composed of one of more sub-strings encoded in
/// the format described above. The substrings are separated by 0 or more zero
-/// bytes. This method decodes the string and populates the \c Symtab.
-Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);
-
-Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab);
+/// bytes. This method decodes the string and calls `NameCallback` for each
+/// substring.
+Error readPGOFuncNameStrings(StringRef NameStrings,
+ std::function<Error(StringRef)> NameCallback);
/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
/// set in IR PGO compilation.
@@ -297,6 +297,8 @@ inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
/// Return the PGOFuncName meta data associated with a function.
MDNode *getPGOFuncNameMetadata(const Function &F);
+std::string getPGOName(const GlobalVariable &V, bool InLTO = false);
+
/// Create the PGOFuncName meta data if PGOFuncName is different from
/// function's raw name. This should only apply to internal linkage functions
/// declared by users only.
@@ -502,6 +504,9 @@ class InstrProfSymtab {
inline Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
+ inline Error
+ initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
+
/// A wrapper interface to populate the PGO symtab with functions
/// decls from module \c M. This interface is used by transformation
/// passes such as indirect function call promotion. Variable \c InLTO
@@ -519,39 +524,33 @@ class InstrProfSymtab {
Error create(const FuncNameIterRange &FuncIterRange,
const VTableNameIterRange &VTableIterRange);
- /// Update the symtab by adding \p FuncName to the table. This interface
- /// is used by the raw and text profile readers.
- Error addFuncName(StringRef FuncName) {
- if (FuncName.empty())
+ Error addSymbolName(StringRef SymbolName) {
+ if (SymbolName.empty())
return make_error<InstrProfError>(instrprof_error::malformed,
- "function name is empty");
- auto Ins = NameTab.insert(FuncName);
+ "symbol name is empty");
+
+ // Insert into NameTab so that MD5NameMap (a vector that will be sorted)
+ // won't have duplicated entries in the first place.
+ auto Ins = NameTab.insert(SymbolName);
if (Ins.second) {
MD5NameMap.push_back(std::make_pair(
- IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey()));
+ IndexedInstrProf::ComputeHash(SymbolName), Ins.first->getKey()));
Sorted = false;
}
return Error::success();
}
+ /// The method name is kept since there are many callers.
+ // It just forwards to 'addSymbolName'.
+ Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); }
+
Error addVTableName(StringRef VTableName) {
- if (VTableName.empty())
- return make_error<InstrProfError>(instrprof_error::malformed,
- "invalid input: VTableName is empty");
- // Insert into NameTab so that MD5NameMap (a vector that is going to be
- // sorted) won't have duplicated entries in the first place.
- auto Ins = NameTab.insert(VTableName);
+ if (Error E = addSymbolName(VTableName))
+ return E;
// Record VTableName. InstrProfWriter uses this map. The comment around
// class member explains why.
VTableNames.insert(VTableName);
-
- // If this is newly added, update MD5NameMap.
- if (Ins.second) {
- MD5NameMap.push_back(std::make_pair(
- IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey()));
- Sorted = false;
- }
return Error::success();
}
@@ -613,18 +612,31 @@ Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
return Error::success();
}
+// FIXME: Move 'InstrProfSymtab::create' definition into cpp,
+// and move 'readPGOFuncNameStrings' inside cpp.
Error InstrProfSymtab::create(StringRef NameStrings) {
- return readPGOFuncNameStrings(NameStrings, *this);
+ return readPGOFuncNameStrings(
+ NameStrings,
+ std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
}
Error InstrProfSymtab::create(StringRef FuncNameStrings,
StringRef VTableNameStrings) {
- if (Error E = readPGOFuncNameStrings(FuncNameStrings, *this))
+ if (Error E = readPGOFuncNameStrings(FuncNameStrings,
+ std::bind(&InstrProfSymtab::addFuncName,
+ this, std::placeholders::_1)))
return E;
- // FIXME: Add test coverage that this returns success when VTableNameStrings
- // is empty.
- return readVTableNames(VTableNameStrings, *this);
+ return readPGOFuncNameStrings(
+ VTableNameStrings,
+ std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+}
+
+Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
+ StringRef CompressedVTableStrings) {
+ return readPGOFuncNameStrings(
+ CompressedVTableStrings,
+ std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
}
template <typename NameIterRange>
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index 4c8f1b31c5e8dc5..5785f2c437fb94a 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -97,7 +97,7 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
#define INSTR_PROF_VTABLE_DATA_DEFINED
#endif
INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, \
- ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(VTableName)))
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(PGOVTableName)))
INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), VTablePointer, VTableAddr)
INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), VTableSizeVal))
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index d7009bbe9dc3f93..0e150b1ce011d0c 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -472,46 +472,6 @@ enum class HashT : uint32_t;
} // end namespace IndexedInstrProf
-class InstrProfVTableLookupTrait {
- char val;
- IndexedInstrProf::HashT HashType;
- unsigned FormatVersion;
-
-public:
- InstrProfVTableLookupTrait(IndexedInstrProf::HashT HashType,
- unsigned FormatVersion)
- : HashType(HashType), FormatVersion(FormatVersion) {}
-
- using data_type = char;
-
- using internal_key_type = StringRef;
- using external_key_type = StringRef;
-
- using hash_value_type = uint64_t;
- using offset_type = uint64_t;
-
- static bool EqualKey(StringRef A, StringRef B) { return A == B; }
- static StringRef GetInternalKey(StringRef K) { return K; }
- static StringRef GetExternalKey(StringRef K) { return K; }
-
- hash_value_type ComputeHash(StringRef K);
-
- static std::pair<offset_type, offset_type>
- ReadKeyDataLength(const unsigned char *&D) {
- using namespace support;
-
- offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D);
- offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D);
- return std::make_pair(KeyLen, DataLen);
- }
-
- StringRef ReadKey(const unsigned char *D, offset_type N) {
- return StringRef((const char *)D, N);
- }
-
- data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
-};
-
/// Trait for lookups into the on-disk hash table for the binary instrprof
/// format.
class InstrProfLookupTrait {
@@ -565,9 +525,6 @@ class InstrProfLookupTrait {
}
};
-using VirtualTableNamesHashTable =
- OnDiskIterableChainedHashTable<InstrProfVTableLookupTrait>;
-
struct InstrProfReaderIndexBase {
virtual ~InstrProfReaderIndexBase() = default;
@@ -590,10 +547,7 @@ struct InstrProfReaderIndexBase {
virtual bool hasMemoryProfile() const = 0;
virtual bool hasTemporalProfile() const = 0;
virtual InstrProfKind getProfileKind() const = 0;
- // The pointer VirtualTableIndex is not owned.
- virtual Error
- populateSymtab(InstrProfSymtab &,
- VirtualTableNamesHashTable *VirtualTableIndex) = 0;
+ virtual Error populateSymtab(InstrProfSymtab &) = 0;
};
using OnDiskHashTableImplV3 =
@@ -668,10 +622,13 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase {
InstrProfKind getProfileKind() const override;
- Error populateSymtab(InstrProfSymtab &Symtab,
- VirtualTableNamesHashTable *VirtualTableIndex) override {
- if (VirtualTableIndex != nullptr)
- return Symtab.create(HashTable->keys(), VirtualTableIndex->keys());
+ Error populateSymtab(InstrProfSymtab &Symtab) override {
+ // FIXME: the create method calls 'finalizeSymtab' and sorts a bunch of
+ // arrays/maps. Since there are other data sources other than 'HashTable' to
+ // populate a symtab, it might make sense to have something like this
+ // 1. Let each data source populate Symtab and init the arrays/maps without
+ // calling 'finalizeSymtab'
+ // 2. Call 'finalizeSymtab' once to get all arrays/maps sorted if needed.
return Symtab.create(HashTable->keys());
}
};
@@ -706,8 +663,17 @@ class IndexedInstrProfReader : public InstrProfReader {
std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable;
/// MemProf frame profile data on-disk indexed via frame id.
std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable;
- /// Virtual table profile data indexed .
- std::unique_ptr<VirtualTableNamesHashTable> VirtualTableIndex = nullptr;
+ /// The reader itself doesn't decompress vtable names.
+ /// A compiler that reads indexed profiles could construct
+ /// symtab from module IR so it doesn't need the decompressed
+ /// names.
+ /// When a symtab is constructed (from llvm-profdata),
+ /// the symtab decompress the list of names based on
+ /// `VTableNamePtr` and `CompressedVTableNamesLen`.
+ /// Points to the beginning of compressed vtable names.
+ const char *VTableNamePtr = nullptr;
+ /// The length of compressed vtable names.
+ uint64_t CompressedVTableNamesLen = 0;
/// Total size of binary ids.
uint64_t BinaryIdsSize{0};
/// Start address of binary id length and data pairs.
diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
index 9af090c0c2e6825..180dd23dd82a204 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
@@ -67,7 +67,7 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
std::vector<GlobalVariable *> ReferencedNames;
// The list of virtual table variables of which the VTableProfData is
// collected.
- std::vector<GlobalVariable *> ReferencedVTableNames;
+ std::vector<GlobalVariable *> ReferencedVTables;
GlobalVariable *NamesVar;
size_t NamesSize;
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 311892b068bf17e..8774f2f796df083 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -216,7 +216,7 @@ cl::opt<bool> DoInstrProfNameCompression(
cl::desc("Enable name/filename string compression"), cl::init(true));
cl::opt<bool> EnableVTableValueProfiling(
- "enable-vtable-value-profiling", cl::init(true),
+ "enable-vtable-value-profiling", cl::init(false),
cl::desc("If true, the virtual table address will be instrumented to know "
"the types of a C++ pointer. The information could be used in "
"indirect-call-promotion to do selective vtable-based comparison "
@@ -340,8 +340,8 @@ static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO,
}
// In LTO mode (when InLTO is true), first check if there is a meta data.
- if (auto IRPGOFuncName = lookupPGONameFromMetadata(PGONameMetadata))
- return *IRPGOFuncName;
+ if (auto IRPGOName = lookupPGONameFromMetadata(PGONameMetadata))
+ return *IRPGOName;
// If there is no meta data, the function must be a global before the value
// profile annotation pass. Its current linkage may be internal if it is
@@ -355,24 +355,36 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO) {
return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F));
}
-// This is similar to `getIRPGOFuncName` except that this function calls
-// 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls
-// 'getIRPGONameForGlobalObject'. See the difference between two callees in the
-// comments of `getIRPGONameForGlobalObject`.
-std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
+static std::string getPGOObjectName(const GlobalObject &GO, bool InLTO,
+ MDNode *PGONameMetadata) {
if (!InLTO) {
- auto FileName = getStrippedSourceFileName(F);
- return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
+ auto FileName = getStrippedSourceFileName(GO);
+ return getPGOFuncName(GO.getName(), GO.getLinkage(), FileName);
}
// In LTO mode (when InLTO is true), first check if there is a meta data.
- if (auto PGOFuncName = lookupPGONameFromMetadata(getPGOFuncNameMetadata(F)))
- return *PGOFuncName;
+ if (auto PGOName = lookupPGONameFromMetadata(PGONameMetadata))
+ return *PGOName;
// If there is no meta data, the function must be a global before the value
// profile annotation pass. Its current linkage may be internal if it is
// internalized in LTO mode.
- return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
+ return getPGOFuncName(GO.getName(), GlobalValue::ExternalLinkage, "");
+}
+
+// This is similar to `getIRPGOFuncName` except that this function calls
+// 'getPGOObjectcName' to get a name and `getIRPGOFuncName` calls
+// 'getIRPGONameForGlobalObject'. See the difference between two callees in the
+// comments of `getIRPGONameForGlobalObject`.
+std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
+ return getPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F));
+}
+
+std::string getPGOName(const GlobalVariable &V, bool InLTO) {
+ // PGONameMetadata should be set by compiler at profile use time
+ // and read by symtab creation to look up symbols corresponding to
+ // a MD5 hash.
+ return getPGOObjectName(V, InLTO, nullptr /* PGONameMetadata */);
}
// See getIRPGOFuncName() for a discription of the format.
@@ -601,9 +613,9 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
std::string &Result, bool doCompression) {
std::vector<std::string> VTableNameStrs;
for (auto *VTable : VTables) {
- VTableNameStrs.push_back(std::string(VTable->getName()));
+ VTableNameStrs.push_back(getPGOName(*VTable));
}
- return collectPGOFuncNameStrings(
+ return collectGlobalObjectNameStrings(
VTableNameStrs, compression::zlib::isAvailable() && doCompression,
Result);
}
@@ -642,7 +654,8 @@ instrprof_error decodeAndSplitStrings(
return instrprof_error::success;
}
-Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
+Error readPGOFuncNameStrings(StringRef NameStrings,
+ std::function<Error(StringRef)> NameCallback) {
const uint8_t *P = NameStrings.bytes_begin();
const uint8_t *EndP = NameStrings.bytes_end();
while (P < EndP) {
@@ -663,7 +676,7 @@ Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
NameStrings.split(Names, getInstrProfNameSeparator());
for (StringRef &Name : Names) {
- if (Error E = Symtab.addFuncName(Name))
+ if (Error E = NameCallback(Name))
return E;
}
@@ -675,37 +688,6 @@ Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
return Error::success();
}
-Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab) {
- const uint8_t *P = NameStrings.bytes_begin();
- const uint8_t *EndP = NameStrings.bytes_end();
- while (P < EndP) {
- // Now parse the name strings.
- uint32_t Dist = 0;
- StringRef NameStrings;
- SmallVector<uint8_t, 128> UncompressedNameStrings;
- SmallVector<StringRef, 0> Names;
- bool isCompressed = false;
- instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
- NameStrings, Dist, isCompressed);
- if (E != instrprof_error::success)
- return make_error<InstrProfError>(E);
-
- if (isCompressed) {
- NameStrings = toStringRef(UncompressedNameStrings);
- }
- NameStrings.split(Names, getInstrProfNameSeparator());
- for (StringRef &Name : Names) {
- if (Error E = Symtab.addVTableName(Name))
- return E;
- }
-
- P += Dist;
- while (P < EndP && *P == 0)
- P++;
- }
- return Error::success();
-}
-
void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
uint64_t FuncSum = 0;
Sum.NumEntries += Counts.size();
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 59e7ef946686320..e1ea44cb127fcc4 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -878,19 +878,6 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
return DataBuffer;
}
-InstrProfVTableLookupTrait::hash_value_type
-InstrProfVTableLookupTrait::ComputeHash(StringRef K) {
- return IndexedInstrProf::ComputeHash(HashType, K);
-}
-
-InstrProfVTableLookupTrait::data_type
-InstrProfVTableLookupTrait::ReadData(StringRef K, const unsigned char *D,
- offset_type N) {
- char v =
- support::endian::readNext<char, support::little, support::unaligned>(D);
- return v;
-}
-
template <typename HashTableImpl>
Error InstrProfReaderIndex<HashTableImpl>::getRecords(
StringRef FuncName, ArrayRef<NamedInstrProfRecord> &Data) {
@@ -1220,17 +1207,15 @@ Error IndexedInstrProfReader::readHeader() {
if (GET_VERSION(Header->formatVersion()) >= 11) {
uint64_t VTableNamesOffset =
- endian::byte_swap<uint64_t, little>(Header->VTableNamesOffset);
+ endian::byte_swap<uint64_t, llvm::endianness::little>(
+ Header->VTableNamesOffset);
const unsigned char *Ptr = Start + VTableNamesOffset;
- const uint64_t HashTableMetadataOffset =
- support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+ CompressedVTableNamesLen =
+ support::endian::readNext<uint64_t, llvm::endianness::little,
+ unaligned>(Ptr);
- VirtualTableIndex.reset(VirtualTableNamesHashTable::Create(
- /*Bucket=*/Start + HashTableMetadataOffset,
- /*Payload=*/Ptr,
- /*Base=*/Start,
- InstrProfVTableLookupTrait(HashType, Header->formatVersion())));
+ VTableNamePtr = (const char *)Ptr;
}
if (GET_VERSION(Header->formatVersion()) >= 10 &&
@@ -1293,7 +1278,15 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
return *Symtab;
std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
- if (Error E = Index->populateSymtab(*NewSymtab, VirtualTableIndex.get())) {
+
+ if (Error E = NewSymtab->initVTableNamesFromCompressedStrings(
+ StringRef(VTableNamePtr, CompressedVTableNamesLen))) {
+ auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
+ consumeError(error(ErrCode, Msg));
+ }
+
+ // finalizeSymtab is called inside populateSymtab.
+ if (Error E = Index->populateSymtab(*NewSymtab)) {
auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
consumeError(error(ErrCode, Msg));
}
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 0713864c21d973c..183c858448437b4 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -19,6 +19,7 @@
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/MemProf.h"
#include "llvm/ProfileData/ProfileCommon.h"
+#include "llvm/Support/Compression.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/EndianStream.h"
#include "llvm/Support/Error.h"
@@ -171,47 +172,6 @@ class InstrProfRecordWriterTrait {
}
};
-class InstrProfRecordVTableTrait {
-public:
- using key_type = StringRef;
- using key_type_ref = StringRef;
-
- using data_type = char;
- using data_type_ref = char;
-
- using hash_value_type = uint64_t;
- using offset_type = uint64_t;
-
- InstrProfRecordVTableTrait() = default;
-
- static hash_value_type ComputeHash(key_type_ref K) {
- return IndexedInstrProf::ComputeHash(K);
- }
-
- static std::pair<offset_type, offset_type>
- EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
- using namespace support;
-
- endian::Writer LE(Out, little);
-
- offset_type N = K.size();
- LE.write<offset_type>(N);
-
- offset_type M = 1;
- LE.write<offset_type>(M);
-
- return std::make_pair(N, M);
- }
-
- void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
- Out.write(K.data(), N);
- }
-
- void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
- Out.write(&V, 1);
- }
-};
-
} // end namespace llvm
InstrProfWriter::InstrProfWriter(bool Sparse,
@@ -639,29 +599,36 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
if (IndexedInstrProf::ProfVersion::CurrentVersion >= 11) {
VTableNamesSectionStart = OS.tell();
- // Reserve space for vtable records offset.
- OS.write(0ULL);
-
- OnDiskChainedHashTableGenerator<llvm::InstrProfRecordVTableTrait>
- VTableNamesGenerator;
- for (const auto &kv : VTableNames) {
- // Use a char '0' as value placeholder, only keys (vtable names)
- // are used.
- // FIXME: It might make sense to have a OnDiskChainedHashSetGenerator if
- // there are more use cases. Use a hash table for now, with one unused
- // 'char' per entry.
- VTableNamesGenerator.insert(kv.getKey(), '0');
+ std::string CompressedVTableNames;
+
+ std::vector<std::string> VTableNameStrs;
+ for (const auto &VTableName : VTableNames.keys()) {
+ VTableNameStrs.push_back(VTableName.str());
}
- auto VTableNamesWriter =
- std::make_unique<llvm::InstrProfRecordVTableTrait>();
+ if (!VTableNameStrs.empty()) {
+ if (Error E = collectGlobalObjectNameStrings(
+ VTableNameStrs, compression::zlib::isAvailable(),
+ CompressedVTableNames))
+ return E;
+ }
- uint64_t VTableNamesTableOffset =
- VTableNamesGenerator.Emit(OS.OS, *VTableNamesWriter);
+ uint64_t CompressedStringLen = CompressedVTableNames.length();
- PatchItem PatchItems[] = {
- {VTableNamesSectionStart, &VTableNamesTableOffset, 1}};
- OS.patch(PatchItems, 1);
+ // Record the length of compressed string.
+ OS.write(CompressedStringLen);
+
+ // Write the chars in compressed strings.
+ for (auto &c : CompressedVTableNames)
+ OS.writeByte(static_cast<uint8_t>(c));
+
+ // Pad up to a multiple of 8.
+ // InstrProfReader could read bytes according to 'CompressedStringLen'.
+ uint64_t PaddedLength = alignTo(CompressedStringLen, 8);
+
+ for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) {
+ OS.writeByte(0);
+ }
}
uint64_t TemporalProfTracesSectionStart = 0;
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index ae73e0edaf5d744..b11df94f7138b05 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1035,6 +1035,7 @@ void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
// Used by INSTR_PROF_VTABLE_DATA MACRO
Constant *VTableAddr = getVTableAddrForProfData(GV);
StringRef VTableName = GV->getName();
+ StringRef PGOVTableName = getPGOName(*GV);
// Record the length of the vtable. This is needed since vtable pointers
// loaded from C++ objects might be from the middle of a vtable definition.
uint32_t VTableSizeVal =
@@ -1087,7 +1088,7 @@ void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
VTableDataMap[GV] = Data;
- ReferencedVTableNames.push_back(GV);
+ ReferencedVTables.push_back(GV);
// VTable <Hash, Addr> is used by runtime but not referenced by other
// sections. Conservatively mark it linker retained.
@@ -1382,14 +1383,13 @@ void InstrProfiling::emitNameData() {
}
void InstrProfiling::emitVTableNames() {
- if (!EnableVTableValueProfiling || ReferencedVTableNames.empty())
+ if (!EnableVTableValueProfiling || ReferencedVTables.empty())
return;
- // Collect VTable
+ // Collect the PGO names of referenced vtables and compress them.
std::string CompressedVTableNames;
- if (Error E =
- collectVTableStrings(ReferencedVTableNames, CompressedVTableNames,
- DoInstrProfNameCompression)) {
+ if (Error E = collectVTableStrings(ReferencedVTables, CompressedVTableNames,
+ DoInstrProfNameCompression)) {
report_fatal_error(Twine(toString(std::move(E))), false);
}
diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
index 59d51a59c48b735..fea122adc1fad16 100644
--- a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
+++ b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
@@ -103,9 +103,13 @@ public:
std::vector<Instruction *> Result = findVTableAddrs(F);
for (Instruction* I : Result) {
Instruction* InsertPt = I->getNextNonDebugInstruction();
- while(InsertPt && dyn_cast<PHINode>(InsertPt))
+ // When finding an insertion point, keep PHI, landing pad and catch pad before vp intrinsics.
+ while(InsertPt && (dyn_cast<PHINode>(InsertPt) || dyn_cast<LandingPadInst>(InsertPt) || dyn_cast<CatchPadInst>(InsertPt)))
InsertPt = InsertPt->getNextNonDebugInstruction();
- assert(InsertPt);
+ // FIXME: Set InsertPt to the end of basic block if
+ // InsertPt is the last instruction.
+ if (InsertPt == nullptr)
+ continue;
Instruction *AnnotatedInst = I;
Candidates.emplace_back(CandidateInfo{I, InsertPt, AnnotatedInst});
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
index 988a30865a7fee6..42018e4d5e54dd2 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
@@ -45,8 +45,6 @@ catch.dispatch: ; preds = %entry
catch: ; preds = %catch.dispatch
%tmp1 = catchpad within %tmp [ptr null, i32 64, ptr null]
%vtable = load ptr, ptr %b, align 8
- %0 = tail call i1 @llvm.type.test(ptr %vtable, metadata !"_ZTS4base")
- tail call void @llvm.assume(i1 %0)
%tmp3 = load ptr, ptr %vtable, align 8
call void %tmp3(ptr %b) [ "funclet"(token %tmp1) ]
catchret from %tmp1 to label %try.cont
@@ -56,15 +54,12 @@ try.cont: ; preds = %catch, %entry
}
; GEN: catch:
-; GEN: call void @llvm.instrprof.value.profile({{.*}} i32 2, i32 0)
-; GEN: call void @llvm.instrprof.value.profile({{.*}} i32 0, i32 0)
+; GEN: call void @llvm.instrprof.value.profile(
; GEN-SAME: [ "funclet"(token %tmp1) ]
; LOWER: catch:
-; LOWER: call void @__llvm_profile_instrument_target({{.*}} i32 0)
+; LOWER: call void @__llvm_profile_instrument_target(
; LOWER-SAME: [ "funclet"(token %tmp1) ]
declare dso_local void @"?may_throw@@YAXH at Z"(i32)
declare dso_local i32 @__CxxFrameHandler3(...)
-declare i1 @llvm.type.test(ptr, metadata)
-declare void @llvm.assume(i1 noundef)
diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
index 059552a7634dc3c..762c2228110f826 100644
--- a/llvm/test/Transforms/PGOProfile/vtable_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 3
-; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
+; RUN: opt < %s -passes=pgo-instr-gen -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=LOWER
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
index 9dd7fa288969075..aa04b881e2ec926 100644
--- a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -1,4 +1,4 @@
-To update the inputs used below, run Inputs/update_vtable_value_prof_inputs.sh /path/to/updated/clang++ /path/to/updated/llvm-profdata
+To update the inputs used below, run Inputs/update_vtable_value_prof_inputs.sh /path/to/updated/clang++
Show profile data from raw profiles.
RUN: llvm-profdata show --function=main --ic-targets --show-vtables %p/Inputs/vtable_prof.profraw | FileCheck %s --check-prefix=RAW
@@ -16,16 +16,17 @@ RAW-NEXT: main:
RAW-NEXT: Hash: 0x0f9a16fe6d398548
RAW-NEXT: Counters: 2
RAW-NEXT: Indirect Call Site Count: 2
-RAW-NEXT: VTable Results:
-RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
-RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: Number of instrumented vtables: 2
RAW-NEXT: Indirect Target Results:
RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
RAW-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
RAW-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+RAW-NEXT: VTable Results:
+RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
RAW-NEXT: Instrumentation level: IR entry_first = 0
RAW-NEXT: Functions shown: 1
RAW-NEXT: Total functions: 6
@@ -38,22 +39,31 @@ RAW-NEXT: Total number of profiled values: 4
RAW-NEXT: Value sites histogram:
RAW-NEXT: NumTargets, SiteCount
RAW-NEXT: 2, 2
+RAW-NEXT: Statistics for vtable profile:
+RAW-NEXT: Total number of sites: 2
+RAW-NEXT: Total number of sites with values: 2
+RAW-NEXT: Total number of profiled values: 4
+RAW-NEXT: Value sites histogram:
+RAW-NEXT: NumTargets, SiteCount
+RAW-NEXT: 2, 2
+
INDEXED: Counters:
INDEXED-NEXT: main:
INDEXED-NEXT: Hash: 0x0f9a16fe6d398548
INDEXED-NEXT: Counters: 2
INDEXED-NEXT: Indirect Call Site Count: 2
-INDEXED-NEXT: VTable Results:
-INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
-INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
-INDEXED-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
-INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: Number of instrumented vtables: 2
INDEXED-NEXT: Indirect Target Results:
INDEXED-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
INDEXED-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
+INDEXED-NEXT: VTable Results:
+INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
INDEXED-NEXT: Instrumentation level: IR entry_first = 0
INDEXED-NEXT: Functions shown: 1
INDEXED-NEXT: Total functions: 6
@@ -66,6 +76,13 @@ INDEXED-NEXT: Total number of profiled values: 4
INDEXED-NEXT: Value sites histogram:
INDEXED-NEXT: NumTargets, SiteCount
INDEXED-NEXT: 2, 2
+INDEXED-NEXT: Statistics for vtable profile:
+INDEXED-NEXT: Total number of sites: 2
+INDEXED-NEXT: Total number of sites with values: 2
+INDEXED-NEXT: Total number of profiled values: 4
+INDEXED-NEXT: Value sites histogram:
+INDEXED-NEXT: NumTargets, SiteCount
+INDEXED-NEXT: 2, 2
ICTEXT: :ir
ICTEXT: main
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 205b3b325284e94..479ac181f46cb16 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -2565,18 +2565,15 @@ static int showInstrProfile(
OS << " Indirect Call Site Count: "
<< Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n";
+ if (ShowVTables)
+ OS << " Number of instrumented vtables: "
+ << Func.getNumValueSites(IPVK_VTableTarget) << "\n";
+
uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize);
if (ShowMemOPSizes && NumMemOPCalls > 0)
OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls
<< "\n";
- if (ShowVTables) {
- OS << " VTable Results:\n";
- traverseAllValueSites(Func, IPVK_VTableTarget,
- VPStats[IPVK_VTableTarget], OS,
- &(Reader->getSymtab()));
- }
-
if (ShowCounts) {
OS << " Block counts: [";
size_t Start = (IsIRInstr ? 0 : 1);
@@ -2593,6 +2590,13 @@ static int showInstrProfile(
&(Reader->getSymtab()));
}
+ if (ShowVTables) {
+ OS << " VTable Results:\n";
+ traverseAllValueSites(Func, IPVK_VTableTarget,
+ VPStats[IPVK_VTableTarget], OS,
+ &(Reader->getSymtab()));
+ }
+
if (ShowMemOPSizes && NumMemOPCalls > 0) {
OS << " Memory Intrinsic Size Results:\n";
traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS,
@@ -2641,6 +2645,11 @@ static int showInstrProfile(
VPStats[IPVK_IndirectCallTarget]);
}
+ if (ShownFunctions && ShowVTables) {
+ OS << "Statistics for vtable profile:\n";
+ showValueSitesStats(OS, IPVK_VTableTarget, VPStats[IPVK_VTableTarget]);
+ }
+
if (ShownFunctions && ShowMemOPSizes) {
OS << "Statistics for memory intrinsic calls sizes profile:\n";
showValueSitesStats(OS, IPVK_MemOPSize, VPStats[IPVK_MemOPSize]);
>From 82fac8e25798772862c52a6b31ef09f6a2be4757 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Wed, 1 Nov 2023 09:47:36 -0700
Subject: [PATCH 07/10] address feedback
---
clang/include/clang/Basic/CodeGenOptions.h | 10 +++++
clang/lib/CodeGen/CGVTables.cpp | 2 +-
clang/lib/CodeGen/MicrosoftCXXABI.cpp | 3 +-
compiler-rt/lib/profile/InstrProfilingMerge.c | 6 ++-
.../lib/profile/InstrProfilingWriter.c | 3 +-
.../llvm/Analysis/IndirectCallVisitor.h | 10 +++--
llvm/include/llvm/ProfileData/InstrProf.h | 10 +++--
.../llvm/ProfileData/InstrProfReader.h | 15 +++----
llvm/lib/ProfileData/InstrProf.cpp | 42 +++++++++---------
.../PGOProfile/Inputs/vtable_prof.profraw | Bin 0 -> 656 bytes
.../Inputs/update_vtable_value_prof_inputs.sh | 4 +-
.../vtable-value-prof-basic.test | 4 +-
12 files changed, 62 insertions(+), 47 deletions(-)
create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/vtable_prof.profraw
rename llvm/test/{Transforms/PGOProfile => tools/llvm-profdata}/Inputs/update_vtable_value_prof_inputs.sh (91%)
mode change 100644 => 100755
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index 12be4e0025a7054..fe013a49ff515cc 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -488,6 +488,16 @@ class CodeGenOptions : public CodeGenOptionsBase {
return getProfileInstr() == ProfileClangInstr;
}
+ // Returns true if IR level profile instrumentation or use is on;
+ // returns true for CS IR instrumentation or profile use as well.
+ bool hasProfileIRInstrOrUse() const {
+ // For CS IR PGO, hasProfileIRUse is true for both instrumentation
+ // and profile use. `-fprofile-use` points to IR PGO profiles
+ // in the former case and points to CS IR PGO profiles in the latter
+ // case.
+ return hasProfileIRInstr() || hasProfileIRUse();
+ }
+
/// Check if IR level profile instrumentation is on.
bool hasProfileIRInstr() const {
return getProfileInstr() == ProfileIRInstr;
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 25e4b1c27932026..5b4d17926a12915 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -1312,7 +1312,7 @@ llvm::GlobalObject::VCallVisibility CodeGenModule::GetVCallVisibilityLevel(
void CodeGenModule::EmitVTableTypeMetadata(const CXXRecordDecl *RD,
llvm::GlobalVariable *VTable,
const VTableLayout &VTLayout) {
- if (!getCodeGenOpts().LTOUnit)
+ if (!getCodeGenOpts().LTOUnit &&!getCodeGenOpts().hasProfileIRInstrOrUse())
return;
CharUnits ComponentWidth = GetTargetTypeStoreSize(getVTableComponentType());
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index b6f941052abee85..14a2146bbdcd0bd 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -1673,7 +1673,8 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
void MicrosoftCXXABI::emitVTableTypeMetadata(const VPtrInfo &Info,
const CXXRecordDecl *RD,
llvm::GlobalVariable *VTable) {
- if (!CGM.getCodeGenOpts().LTOUnit)
+ if (!CGM.getCodeGenOpts().LTOUnit &&
+ !CGM.getCodeGenOpts().hasProfileIRInstrOrUse())
return;
// TODO: Should VirtualFunctionElimination also be supported here?
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 32b3058535ede73..7b6cf775e6c6730 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -102,8 +102,10 @@ static uint64_t
getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
// Skip names section, vtable profile data section and vtable names section
// for runtime profile merge. To merge runtime addresses from multiple
- // profiles, the same instrumented binary should run with ASLR disabled -> in
- // this set-up these three sections remain unchanged.
+ // profiles collected from the same instrumented binary, the binary should be
+ // loaded at fixed base address (e.g., build with -no-pie, or run with ASLR
+ // disabled).
+ // In this set-up these three sections remain unchanged.
const uint64_t VTableSectionSize =
Header->NumVTables * sizeof(VTableProfData);
const uint64_t PaddingBytesAfterVTableSection =
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index c96d3f17a32697f..5ba1d8d7ad26e47 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -323,8 +323,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
return -1;
/* Write the binary id lengths and data. */
- int binary_id_size = __llvm_write_binary_ids(Writer);
- if (binary_id_size == -1)
+ if (__llvm_write_binary_ids(Writer) == -1)
return -1;
/* Write the profile data. */
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 4a3e5b61e14f37e..3cf23b29214a255 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -52,9 +52,13 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
// FIXME: Add support in the frontend so LLVM type intrinsics are
// emitted without LTO. This way, added intrinsics could filter
// non-vtable instructions and reduce instrumentation overhead.
- // Note a profiled address will be dropped if it's not within
- // the address range of vtable objects, so there are no correctness
- // or performance issues at profile use time.
+ // Since a non-vtable profiled address is not within the address
+ // range of vtable objects, it's stored as zero in indexed profiles.
+ // A pass that looks up symbol with an zero hash will (almost) always
+ // find nullptr and skip the actual transformation (e.g., comparison
+ // of symbols). So the performance overhead from non-vtable profiled
+ // address is negligible if exists at all. Comparing loaded address
+ // with symbol address guarantees correctness.
if (VTablePtr != nullptr && isa<Instruction>(VTablePtr)) {
ProfiledAddresses.push_back(cast<Instruction>(VTablePtr));
}
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index d289a4b63337901..d8c716625c90393 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -541,9 +541,11 @@ class InstrProfSymtab {
}
/// The method name is kept since there are many callers.
- // It just forwards to 'addSymbolName'.
+ /// It just forwards to 'addSymbolName'.
Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); }
+ /// Adds VTableName as a known symbol, and inserts it to a map that
+ /// tracks all vtable names.
Error addVTableName(StringRef VTableName) {
if (Error E = addSymbolName(VTableName))
return E;
@@ -562,8 +564,8 @@ class InstrProfSymtab {
AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
}
- // Map the address range (i.e., [start_address, end_address]) of a variable to
- // its names' MD5 hash. This interface is only used by the raw profile header.
+ /// Map the address range (i.e., [start_address, end_address]) of a variable to
+ /// its names' MD5 hash. This interface is only used by the raw profile header.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
VTableAddrRangeToMD5Map.push_back(
std::make_pair(std::make_pair(StartAddr, EndAddr), MD5Val));
@@ -674,7 +676,7 @@ void InstrProfSymtab::finalizeSymtab() {
llvm::sort(AddrToMD5Map, less_first());
AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
AddrToMD5Map.end());
- // GlobalVariable address ranges should not overlap; so sort by either
+ // VTable object address ranges should not overlap; so sort by either
// beginning address or end address is fine.
llvm::sort(VTableAddrRangeToMD5Map, less_first());
// std::unique uses == operator for std::pair.
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 0e150b1ce011d0c..d54fa76e40a6fdb 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -663,14 +663,13 @@ class IndexedInstrProfReader : public InstrProfReader {
std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable;
/// MemProf frame profile data on-disk indexed via frame id.
std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable;
- /// The reader itself doesn't decompress vtable names.
- /// A compiler that reads indexed profiles could construct
- /// symtab from module IR so it doesn't need the decompressed
- /// names.
- /// When a symtab is constructed (from llvm-profdata),
- /// the symtab decompress the list of names based on
- /// `VTableNamePtr` and `CompressedVTableNamesLen`.
- /// Points to the beginning of compressed vtable names.
+ /// The reader itself doesn't decompress vtable names. A compiler that reads
+ /// indexed profiles could construct symtab from module IR so it doesn't need
+ /// the decompressed names.
+ /// When a symtab is constructed from profiles by llvm-profdata, the list of
+ /// names could be decompressed based on `VTableNamePtr` and
+ /// `CompressedVTableNamesLen`.
+ /// VTableNamePtr points to the beginning of compressed vtable names.
const char *VTableNamePtr = nullptr;
/// The length of compressed vtable names.
uint64_t CompressedVTableNamesLen = 0;
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 8774f2f796df083..a180b030eff7516 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -218,11 +218,8 @@ cl::opt<bool> DoInstrProfNameCompression(
cl::opt<bool> EnableVTableValueProfiling(
"enable-vtable-value-profiling", cl::init(false),
cl::desc("If true, the virtual table address will be instrumented to know "
- "the types of a C++ pointer. The information could be used in "
- "indirect-call-promotion to do selective vtable-based comparison "
- "and interprocedural type propagation. Requires type metadata and "
- "type intrinsics (https://llvm.org/docs/TypeMetadata.html) to use"
- " this option."));
+ "the types of a C++ pointer. The information is used in indirect "
+ "call promotion to do selective vtable-based comparison."));
std::string getInstrProfSectionName(InstrProfSectKind IPSK,
Triple::ObjectFormatType OF,
@@ -622,7 +619,7 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
instrprof_error decodeAndSplitStrings(
const uint8_t *Input, SmallVector<uint8_t, 128> &UncompressedNameStrings,
- StringRef &NameStrings, uint32_t &Dist, bool &isCompressed) {
+ StringRef &NameStrings, uint32_t &Dist, bool &IsCompressed) {
Dist = 0;
const uint8_t *Start = Input;
uint32_t UncompressedSizeLen = 0;
@@ -633,23 +630,24 @@ instrprof_error decodeAndSplitStrings(
uint64_t CompressedSize = decodeULEB128(Start, &CompressedSizeLen);
Start += CompressedSizeLen;
Dist += CompressedSizeLen;
- isCompressed = (CompressedSize != 0);
- if (isCompressed) {
- if (!llvm::compression::zlib::isAvailable())
- return instrprof_error::zlib_unavailable;
-
- if (Error E = compression::zlib::decompress(ArrayRef(Start, CompressedSize),
- UncompressedNameStrings,
- UncompressedSize)) {
- consumeError(std::move(E));
- return instrprof_error::uncompress_failed;
- }
- Dist += CompressedSize;
- } else {
+ IsCompressed = (CompressedSize != 0);
+ if (!IsCompressed) {
NameStrings =
StringRef(reinterpret_cast<const char *>(Start), UncompressedSize);
Dist += UncompressedSize;
+ return instrprof_error::success;
+ }
+
+ if (!llvm::compression::zlib::isAvailable())
+ return instrprof_error::zlib_unavailable;
+
+ if (Error E = compression::zlib::decompress(ArrayRef(Start, CompressedSize),
+ UncompressedNameStrings,
+ UncompressedSize)) {
+ consumeError(std::move(E));
+ return instrprof_error::uncompress_failed;
}
+ Dist += CompressedSize;
return instrprof_error::success;
}
@@ -664,13 +662,13 @@ Error readPGOFuncNameStrings(StringRef NameStrings,
StringRef NameStrings;
SmallVector<uint8_t, 128> UncompressedNameStrings;
SmallVector<StringRef, 0> Names;
- bool isCompressed = false;
+ bool IsCompressed = false;
instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
- NameStrings, Dist, isCompressed);
+ NameStrings, Dist, IsCompressed);
if (E != instrprof_error::success)
return make_error<InstrProfError>(E);
- if (isCompressed) {
+ if (IsCompressed) {
NameStrings = toStringRef(UncompressedNameStrings);
}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/vtable_prof.profraw b/llvm/test/Transforms/PGOProfile/Inputs/vtable_prof.profraw
new file mode 100644
index 0000000000000000000000000000000000000000..5adeb774cddd6462bd2d3779a96d9ad5a06d5e23
GIT binary patch
literal 656
zcmZoHO3N=Q$obF700xW at ih+R*#(>fsXncDp|G<9;NPf(`(<%&25s=FS6^fqKW9??N
zGj7VQAPc)yoJ=r%1$-<h`e5o|CjGkF``j#L>TQo!%iMorv-oE~?b`s=moX8dAEXai
zZ{I{cNtM}66M)L!U*_`VDuC*1;77F&?qR5f3mFb--sHb`6Q<7rs&4`TeGkj4d7Hnn
z_QUiEK=mykpl|Z0nI4u#dwXE|9{hu7+(1CzJE&)WLEZg=8Nz`12PO`qVd`OYj%~%Q
z?tUW^4?Qnm9Z%ksIv!_J&iI}=libt)X>)<6?kOE_UqcgL?X%uyC1=kZ5ixZVHa)cJ
q8pk>@hHDd5b}&Q$t%KPG4tuD3VBrh17v_JMy|8eE$;12!lLr8f`l*Eg
literal 0
HcmV?d00001
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh b/llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh
old mode 100644
new mode 100755
similarity index 91%
rename from llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
rename to llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh
index 5c594e10e7370f0..dead295a5a72fb8
--- a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
+++ b/llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-if [ $# -lt 2 ]; then
+if [ $# -lt 1 ]; then
echo "Path to clang++ required!"
echo "Usage: update_vtable_value_prof_inputs.sh /path/to/updated/clang++"
exit 1
@@ -73,7 +73,7 @@ int main(int argc, char** argv) {
}
EOF
-FLAGS="-fuse-ld=lld -O2 -g -fprofile-generate=. -flto=thin -Xclang -fwhole-program-vtables -Wl,--lto-whole-program-visibility"
+FLAGS="-fuse-ld=lld -O2 -g -fprofile-generate=. "
${CLANG} ${FLAGS} ${OUTDIR}/vtable_prof.cc -o ${OUTDIR}/vtable_prof
env LLVM_PROFILE_FILE=${OUTDIR}/vtable_prof.profraw ${OUTDIR}/vtable_prof
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
index aa04b881e2ec926..733ff8d7f1eac32 100644
--- a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -60,8 +60,8 @@ INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
INDEXED-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
INDEXED-NEXT: VTable Results:
-INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
-INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
INDEXED-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
INDEXED-NEXT: Instrumentation level: IR entry_first = 0
>From d54d0594518a5d87e433e38732d5d7e6e6014c4c Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Wed, 1 Nov 2023 15:53:01 -0700
Subject: [PATCH 08/10] small changes: - (Hopefully) fix indentation in
llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test - 'git diff' and
'vim' looks okay. - Undo the change in
compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test This is not
necessary since -enable-vtable-value-profiling is false by default.
---
.../Linux/instrprof-value-prof-warn.test | 2 +-
.../llvm-profdata/Inputs/vtable-prof.proftext | 2 +-
.../tools/llvm-profdata/vtable-prof.proftext | 2 +-
.../vtable-value-prof-basic.test | 22 +++++++++----------
4 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test b/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
index 89787e79c20e9d1..991fd2b69b5b303 100644
--- a/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
+++ b/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
@@ -1,7 +1,7 @@
RUN: %clang_pgogen -O2 -fuse-ld=bfd -mllvm -disable-vp=false -mllvm -vp-static-alloc=true -DSTRESS=1 -o %t.ir.warn %S/../Inputs/instrprof-value-prof-real.c
RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=255 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=WARNING %s
# Test that enough static counters have been allocated
-RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=80 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=NOWARNING --allow-empty %s
+RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=130 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=NOWARNING --allow-empty %s
# WARNING: LLVM Profile Warning:
# NOWARNING-NOT: LLVM Profile Warning:
diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
index bf0ad6beb9825b8..0f5d93d26c18443 100644
--- a/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
+++ b/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
@@ -70,4 +70,4 @@ _ZTV8Derived2:750
_ZTV8Derived1:250
2
_ZTV8Derived2:750
-_ZTV8Derived1:250
\ No newline at end of file
+_ZTV8Derived1:250
diff --git a/llvm/test/tools/llvm-profdata/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
index 6ebe9b44266b6f2..b533c7b8a7f5184 100644
--- a/llvm/test/tools/llvm-profdata/vtable-prof.proftext
+++ b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
@@ -13,4 +13,4 @@ VTABLE: Instrumentation level: IR entry_first = 0
VTABLE: Functions shown: 1
VTABLE: Total functions: 6
VTABLE: Maximum function count: 1000
-VTABLE: Maximum internal block count: 250
\ No newline at end of file
+VTABLE: Maximum internal block count: 250
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
index 733ff8d7f1eac32..e853516832cb875 100644
--- a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -18,15 +18,15 @@ RAW-NEXT: Counters: 2
RAW-NEXT: Indirect Call Site Count: 2
RAW-NEXT: Number of instrumented vtables: 2
RAW-NEXT: Indirect Target Results:
-RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
-RAW-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
-RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
-RAW-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
RAW-NEXT: VTable Results:
-RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
-RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
RAW-NEXT: Instrumentation level: IR entry_first = 0
RAW-NEXT: Functions shown: 1
RAW-NEXT: Total functions: 6
@@ -44,7 +44,7 @@ RAW-NEXT: Total number of sites: 2
RAW-NEXT: Total number of sites with values: 2
RAW-NEXT: Total number of profiled values: 4
RAW-NEXT: Value sites histogram:
-RAW-NEXT: NumTargets, SiteCount
+RAW-NEXT: NumTargets, SiteCount
RAW-NEXT: 2, 2
@@ -81,8 +81,8 @@ INDEXED-NEXT: Total number of sites: 2
INDEXED-NEXT: Total number of sites with values: 2
INDEXED-NEXT: Total number of profiled values: 4
INDEXED-NEXT: Value sites histogram:
-INDEXED-NEXT: NumTargets, SiteCount
-INDEXED-NEXT: 2, 2
+INDEXED-NEXT: NumTargets, SiteCount
+INDEXED-NEXT: 2, 2
ICTEXT: :ir
ICTEXT: main
>From 52143e9afa3ed26544ba322098e9d0eff7bc9d31 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Mon, 6 Nov 2023 22:53:09 -0800
Subject: [PATCH 09/10] update clang-format version to 117.0.4 released binary
and run clang-format
---
compiler-rt/lib/profile/InstrProfiling.h | 12 ++++------
.../lib/profile/InstrProfilingBuffer.c | 23 ++++++++++---------
.../lib/profile/InstrProfilingWriter.c | 22 +++++++++---------
3 files changed, 28 insertions(+), 29 deletions(-)
diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index f25c195f0ea58fb..04ca7da8a4a5fc0 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -32,7 +32,7 @@ typedef struct __llvm_profile_header {
#include "profile/InstrProfData.inc"
} __llvm_profile_header;
-typedef struct ValueProfNode * PtrToNodeT;
+typedef struct ValueProfNode *PtrToNodeT;
typedef struct ValueProfNode {
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) Type Name;
#include "profile/InstrProfData.inc"
@@ -128,8 +128,7 @@ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size);
* structurally the in-process counters and bitmaps. If the profile data in
* buffer is not compatible, the interface returns 1 (failure).
*/
-int __llvm_profile_check_compatibility(const char *Profile,
- uint64_t Size);
+int __llvm_profile_check_compatibility(const char *Profile, uint64_t Size);
/*!
* \brief Counts the number of times a target value is seen.
@@ -142,7 +141,7 @@ int __llvm_profile_check_compatibility(const char *Profile,
void INSTR_PROF_VALUE_PROF_FUNC(
#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) ArgType ArgName
#include "profile/InstrProfData.inc"
- );
+);
void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data,
uint32_t CounterIndex,
@@ -319,9 +318,8 @@ void __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
- uint64_t *PaddingBytesAfterBitmap,
- uint64_t *PaddingBytesAfterNames, uint64_t *PaddingBytesAfterVTable,
- uint64_t *PaddingBytesAfterVNames);
+ uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames,
+ uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVNames);
/*!
* \brief Set the flag that profile data has been dumped to the file.
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index 9a990c3d33b5320..8cbad7c569c3613 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -136,12 +136,11 @@ static int needsCounterPadding(void) {
COMPILER_RT_VISIBILITY
void __llvm_profile_get_padding_sizes_for_counters(
- uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, uint64_t NamesSize,
- uint64_t VTableSize, uint64_t VNameSize,
+ uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
+ uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
- uint64_t* PaddingBytesAfterBitmapBytes,
- uint64_t *PaddingBytesAfterNames, uint64_t *PaddingBytesAfterVTable,
- uint64_t *PaddingBytesAfterVName) {
+ uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames,
+ uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) {
// Counter padding is needed only if continuous mode is enabled.
if (!needsCounterPadding()) {
*PaddingBytesBeforeCounters = 0;
@@ -196,8 +195,9 @@ uint64_t __llvm_profile_get_size_for_buffer_internal(
PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes;
__llvm_profile_get_padding_sizes_for_counters(
DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */,
- 0 /* VNameSize */, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
- &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames, NULL /* PaddingBytesAfterVTable */,
+ 0 /* VNameSize */, &PaddingBytesBeforeCounters,
+ &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
+ &PaddingBytesAfterNames, NULL /* PaddingBytesAfterVTable */,
NULL /* PaddingbytesAfterVNames */);
return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
@@ -226,8 +226,9 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
ProfDataWriter BufferWriter;
initBufferWriter(&BufferWriter, Buffer);
// Set virtual table arguments to NULL since they are not supported yet.
- return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin,
- CountersEnd, BitmapBegin, BitmapEnd, 0 /* VPDataReader */, NamesBegin, NamesEnd,
- NULL /* VTableBegin */, NULL /* VTableEnd */,
- NULL /* VNamesBegin */, NULL /* VNamesEnd */, 0 /* SkipNameDataWrite */);
+ return lprofWriteDataImpl(
+ &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd,
+ BitmapBegin, BitmapEnd, 0 /* VPDataReader */, NamesBegin, NamesEnd,
+ NULL /* VTableBegin */, NULL /* VTableEnd */, NULL /* VNamesBegin */,
+ NULL /* VNamesEnd */, 0 /* SkipNameDataWrite */);
}
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 2a95243956dafaf..39ff7fd1041f90f 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -255,10 +255,9 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
const char *VNamesBegin = __llvm_profile_begin_vtabnames();
const char *VNamesEnd = __llvm_profile_end_vtabnames();
return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
- CountersEnd, BitmapBegin, BitmapEnd,
- VPDataReader, NamesBegin, NamesEnd,
- VTableBegin, VTableEnd, VNamesBegin,
- VNamesEnd, SkipNameDataWrite);
+ CountersEnd, BitmapBegin, BitmapEnd, VPDataReader,
+ NamesBegin, NamesEnd, VTableBegin, VTableEnd,
+ VNamesBegin, VNamesEnd, SkipNameDataWrite);
}
COMPILER_RT_VISIBILITY int
@@ -288,20 +287,21 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const uint64_t VTableSectionSize =
__llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
// Value profiling is not supported when DebugInfoCorrelate is true.
- const uint64_t VNamesSize = __llvm_profile_get_name_size(VNamesBegin, VNamesEnd);
-
+ const uint64_t VNamesSize =
+ __llvm_profile_get_name_size(VNamesBegin, VNamesEnd);
+
/* Create the header. */
__llvm_profile_header Header;
/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterBitmapBytes,
- PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
+ PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames,
+ PaddingBytesAfterVTable, PaddingBytesAfterVNames;
__llvm_profile_get_padding_sizes_for_counters(
- DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, VTableSectionSize,
- VNamesSize, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
- &PaddingBytesAfterBitmapBytes,
+ DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize,
+ VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters,
+ &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
&PaddingBytesAfterNames, &PaddingBytesAfterVTable,
&PaddingBytesAfterVNames);
>From 2350d7f90f85abbb49c1cd2f2d9615a99dc54018 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Tue, 7 Nov 2023 10:44:54 -0800
Subject: [PATCH 10/10] The changes: 1. Primarily, use vtable PGO name in
compressed strings in helper function getVarName (around line 1115 in
InstrProfiling.cpp) 2. Use version 12 in {InstrProfReader.cpp around line
1312, InstrProfWriter.cpp around line 607}. - Using version 11 is an
overlook in 'git merge main' (a change in upstream main takes version 11
already). 3. In InstrProf.cpp and its header, rename 'readPGOFuncNameStrings'
to a generic name, and make it a static function. pr/71566 is an NFC doing
essentially the same thing. 4. A few code cleaning (e.g., delete debugging
comment in raw-64-bits*.test)
---
clang/include/clang/Basic/CodeGenOptions.h | 10 --
compiler-rt/include/profile/InstrProfData.inc | 19 ++-
.../lib/profile/InstrProfilingInternal.h | 3 -
.../llvm/Analysis/IndirectCallVisitor.h | 2 +-
llvm/include/llvm/ProfileData/InstrProf.h | 51 ++----
llvm/lib/ProfileData/InstrProf.cpp | 149 +++++++++---------
llvm/lib/ProfileData/InstrProfReader.cpp | 2 +-
llvm/lib/ProfileData/InstrProfWriter.cpp | 2 +-
.../Instrumentation/InstrProfiling.cpp | 4 +-
.../Instrumentation/PGOInstrumentation.cpp | 3 +-
.../tools/llvm-profdata/raw-32-bits-be.test | 2 -
.../tools/llvm-profdata/raw-64-bits-le.test | 9 --
12 files changed, 104 insertions(+), 152 deletions(-)
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index fe013a49ff515cc..12be4e0025a7054 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -488,16 +488,6 @@ class CodeGenOptions : public CodeGenOptionsBase {
return getProfileInstr() == ProfileClangInstr;
}
- // Returns true if IR level profile instrumentation or use is on;
- // returns true for CS IR instrumentation or profile use as well.
- bool hasProfileIRInstrOrUse() const {
- // For CS IR PGO, hasProfileIRUse is true for both instrumentation
- // and profile use. `-fprofile-use` points to IR PGO profiles
- // in the former case and points to CS IR PGO profiles in the latter
- // case.
- return hasProfileIRInstr() || hasProfileIRUse();
- }
-
/// Check if IR level profile instrumentation is on.
bool hasProfileIRInstr() const {
return getProfileInstr() == ProfileIRInstr;
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index b649cc75783fe3d..a8fc5fe1b870b22 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -95,9 +95,10 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
/* INSTR_PROF_DATA end. */
-/* For a virtual table object, record the name hash to associate profiled addresses
- * with global variables, and record {starting address, size in bytes} to map the profiled virtual table (which usually have an offset from the starting address)
- * back to a virtual table object. */
+/* For a virtual table object, record the name hash to associate profiled
+ * addresses with global variables, and record {starting address, size in bytes}
+ * to map the profiled virtual table (which usually have an offset from the
+ * starting address) back to a virtual table object. */
#ifndef INSTR_PROF_VTABLE_DATA
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
#else
@@ -202,13 +203,17 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
-/* For virtual table address profiling, the addresses of the virtual table (i.e., the address contained in objects pointing to a virtual table) are
- * profiled. Note this may not be the address of the per C++ class virtual table object (i.e., there is an offset).
+/* For virtual table address profiling, the addresses of the virtual table
+ * (i.e., the address contained in objects pointing to a virtual table) are
+ * profiled. Note this may not be the address of the per C++ class virtual table
+ * object (i.e., there is an offset).
*
- * The profiled addresses are stored in raw profile, together with the following two types of information.
+ * The profiled addresses are stored in raw profile, together with the following
+ * two types of information.
* 1. The (beginning and ending) addresses of per C++ class virtual table objects.
* 2. The (compressed) virtual table object names.
- * RawInstrProfReader converts profiled virtual table addresses to virtual table objects' MD5 hash.
+ * RawInstrProfReader converts profiled virtual table addresses to virtual table
+ * objects' MD5 hash.
*/
VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The address of the compatible vtable (i.e., there is an offset from this address to per C++ class virtual table global variable.)")
/* These two kinds must be the last to be
diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h
index b8a798f443ef615..38159b668a1dfdc 100644
--- a/compiler-rt/lib/profile/InstrProfilingInternal.h
+++ b/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -18,9 +18,6 @@
* pointers to the live data in memory. This function is probably not what you
* want. Use __llvm_profile_get_size_for_buffer instead. Use this function if
* your program has a custom memory layout.
- * NOTE: The change of function signature requires modifying c source code
- * as demonstrated by the existing tests. If this is causing backward
- * compatible issues, considering adding another function for new use cases.
*/
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 3cf23b29214a255..c8429e52bee966d 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -86,4 +86,4 @@ inline std::vector<Instruction *> findVTableAddrs(Function &F) {
} // namespace llvm
-#endif
\ No newline at end of file
+#endif
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 78ec01617ee2f43..c27881d8144e137 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -251,13 +251,6 @@ Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
std::string &Result, bool doCompression);
-/// \c NameStrings is a string composed of one of more sub-strings encoded in
-/// the format described above. The substrings are separated by 0 or more zero
-/// bytes. This method decodes the string and calls `NameCallback` for each
-/// substring.
-Error readPGOFuncNameStrings(StringRef NameStrings,
- std::function<Error(StringRef)> NameCallback);
-
/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
/// set in IR PGO compilation.
bool isIRPGOFlagSet(const Module *M);
@@ -502,15 +495,20 @@ class InstrProfSymtab {
/// format.
inline Error create(StringRef D, uint64_t BaseAddr);
- /// \c NameStrings is a string composed of one of more sub-strings
+ /// \c NameStrings is a string composed of one or more sub-strings
/// encoded in the format described in \c collectPGOFuncNameStrings.
- /// This method is a wrapper to \c readPGOFuncNameStrings method.
- inline Error create(StringRef NameStrings);
+ /// This method is a wrapper to \c readAndDecodeStrings method.
+ Error create(StringRef NameStrings);
- inline Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
+ /// \c FuncNameStrings is a string composed of one or more encoded function
+ /// name strings, and \c VTableNameStrings composes of one or more encoded
+ /// vtable names. This function is a wrapper to \c readAndDecodeStrings
+ /// method.
+ Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
- inline Error
- initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
+ /// Initialize 'this' with the set of vtable names encoded in
+ /// \c CompressedVTableNames.
+ Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
/// A wrapper interface to populate the PGO symtab with functions
/// decls from module \c M. This interface is used by transformation
@@ -619,33 +617,6 @@ Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
return Error::success();
}
-// FIXME: Move 'InstrProfSymtab::create' definition into cpp,
-// and move 'readPGOFuncNameStrings' inside cpp.
-Error InstrProfSymtab::create(StringRef NameStrings) {
- return readPGOFuncNameStrings(
- NameStrings,
- std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
-}
-
-Error InstrProfSymtab::create(StringRef FuncNameStrings,
- StringRef VTableNameStrings) {
- if (Error E = readPGOFuncNameStrings(FuncNameStrings,
- std::bind(&InstrProfSymtab::addFuncName,
- this, std::placeholders::_1)))
- return E;
-
- return readPGOFuncNameStrings(
- VTableNameStrings,
- std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
-}
-
-Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
- StringRef CompressedVTableStrings) {
- return readPGOFuncNameStrings(
- CompressedVTableStrings,
- std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
-}
-
template <typename NameIterRange>
Error InstrProfSymtab::create(const NameIterRange &IterRange) {
for (auto Name : IterRange)
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index c91453009f02b05..116be406cb5d4c3 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -487,6 +487,78 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) {
return Error::success();
}
+/// \c NameStrings is a string composed of one of more possibly encoded
+/// sub-strings. The substrings are separated by 0 or more zero bytes. This
+/// method decodes the string and calls `NameCallback` for each substring.
+static Error
+readAndDecodeStrings(StringRef NameStrings,
+ std::function<Error(StringRef)> NameCallback) {
+ const uint8_t *P = NameStrings.bytes_begin();
+ const uint8_t *EndP = NameStrings.bytes_end();
+ while (P < EndP) {
+ uint32_t N;
+ uint64_t UncompressedSize = decodeULEB128(P, &N);
+ P += N;
+ uint64_t CompressedSize = decodeULEB128(P, &N);
+ P += N;
+ bool isCompressed = (CompressedSize != 0);
+ SmallVector<uint8_t, 128> UncompressedNameStrings;
+ StringRef NameStrings;
+ if (isCompressed) {
+ if (!llvm::compression::zlib::isAvailable())
+ return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
+
+ if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
+ UncompressedNameStrings,
+ UncompressedSize)) {
+ consumeError(std::move(E));
+ return make_error<InstrProfError>(instrprof_error::uncompress_failed);
+ }
+ P += CompressedSize;
+ NameStrings = toStringRef(UncompressedNameStrings);
+ } else {
+ NameStrings =
+ StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
+ P += UncompressedSize;
+ }
+ // Now parse the name strings.
+ SmallVector<StringRef, 0> Names;
+ NameStrings.split(Names, getInstrProfNameSeparator());
+ for (StringRef &Name : Names)
+ if (Error E = NameCallback(Name))
+ return E;
+
+ while (P < EndP && *P == 0)
+ P++;
+ }
+ return Error::success();
+}
+
+Error InstrProfSymtab::create(StringRef NameStrings) {
+ return readAndDecodeStrings(
+ NameStrings,
+ std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
+}
+
+Error InstrProfSymtab::create(StringRef FuncNameStrings,
+ StringRef VTableNameStrings) {
+ if (Error E = readAndDecodeStrings(FuncNameStrings,
+ std::bind(&InstrProfSymtab::addFuncName,
+ this, std::placeholders::_1)))
+ return E;
+
+ return readAndDecodeStrings(
+ VTableNameStrings,
+ std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+}
+
+Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
+ StringRef CompressedVTableStrings) {
+ return readAndDecodeStrings(
+ CompressedVTableStrings,
+ std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+}
+
Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
if (Error E = addFuncName(PGOFuncName))
return E;
@@ -623,75 +695,6 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
Result);
}
-instrprof_error decodeAndSplitStrings(
- const uint8_t *Input, SmallVector<uint8_t, 128> &UncompressedNameStrings,
- StringRef &NameStrings, uint32_t &Dist, bool &IsCompressed) {
- Dist = 0;
- const uint8_t *Start = Input;
- uint32_t UncompressedSizeLen = 0;
- uint64_t UncompressedSize = decodeULEB128(Start, &UncompressedSizeLen);
- Start += UncompressedSizeLen;
- Dist += UncompressedSizeLen;
- uint32_t CompressedSizeLen = 0;
- uint64_t CompressedSize = decodeULEB128(Start, &CompressedSizeLen);
- Start += CompressedSizeLen;
- Dist += CompressedSizeLen;
- IsCompressed = (CompressedSize != 0);
- if (!IsCompressed) {
- NameStrings =
- StringRef(reinterpret_cast<const char *>(Start), UncompressedSize);
- Dist += UncompressedSize;
- return instrprof_error::success;
- }
-
- if (!llvm::compression::zlib::isAvailable())
- return instrprof_error::zlib_unavailable;
-
- if (Error E = compression::zlib::decompress(ArrayRef(Start, CompressedSize),
- UncompressedNameStrings,
- UncompressedSize)) {
- consumeError(std::move(E));
- return instrprof_error::uncompress_failed;
- }
- Dist += CompressedSize;
-
- return instrprof_error::success;
-}
-
-Error readPGOFuncNameStrings(StringRef NameStrings,
- std::function<Error(StringRef)> NameCallback) {
- const uint8_t *P = NameStrings.bytes_begin();
- const uint8_t *EndP = NameStrings.bytes_end();
- while (P < EndP) {
- // Now parse the name strings.
- uint32_t Dist = 0;
- StringRef NameStrings;
- SmallVector<uint8_t, 128> UncompressedNameStrings;
- SmallVector<StringRef, 0> Names;
- bool IsCompressed = false;
- instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
- NameStrings, Dist, IsCompressed);
- if (E != instrprof_error::success)
- return make_error<InstrProfError>(E);
-
- if (IsCompressed) {
- NameStrings = toStringRef(UncompressedNameStrings);
- }
-
- NameStrings.split(Names, getInstrProfNameSeparator());
- for (StringRef &Name : Names) {
- if (Error E = NameCallback(Name))
- return E;
- }
-
- P += Dist;
- // Skip padding?
- while (P < EndP && *P == 0)
- P++;
- }
- return Error::success();
-}
-
void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
uint64_t FuncSum = 0;
Sum.NumEntries += Counts.size();
@@ -954,10 +957,8 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
if (ValueKind == IPVK_IndirectCallTarget)
return SymTab->getFunctionHashFromAddress(Value);
- if (ValueKind == IPVK_VTableTarget) {
- uint64_t VTableHash = SymTab->getVTableHashFromAddress(Value);
- return VTableHash;
- }
+ if (ValueKind == IPVK_VTableTarget)
+ return SymTab->getVTableHashFromAddress(Value);
return Value;
}
@@ -1531,7 +1532,7 @@ void OverlapStats::dump(raw_fd_ostream &OS) const {
strncpy(ProfileKindName, "MemOP", 19);
break;
case IPVK_VTableTarget:
- strncpy(ProfileKindName, "VTable", 19);
+ strncpy(ProfileKindName, "VTable", 6);
break;
default:
snprintf(ProfileKindName, 19, "VP[%d]", I);
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index ee17e879cd57e9a..d5f247a8521fbdb 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -1309,7 +1309,7 @@ Error IndexedInstrProfReader::readHeader() {
"corrupted binary ids");
}
- if (GET_VERSION(Header->formatVersion()) >= 11) {
+ if (GET_VERSION(Header->formatVersion()) >= 12) {
uint64_t VTableNamesOffset =
endian::byte_swap<uint64_t, llvm::endianness::little>(
Header->VTableNamesOffset);
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index ef54efe449ed480..491158e03aa8ee7 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -604,7 +604,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
// if version >= the version with vtable profile metadata.
uint64_t VTableNamesSectionStart = 0;
- if (IndexedInstrProf::ProfVersion::CurrentVersion >= 11) {
+ if (IndexedInstrProf::ProfVersion::CurrentVersion >= 12) {
VTableNamesSectionStart = OS.tell();
std::string CompressedVTableNames;
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index c90830338ea4587..bb06670d34c3b99 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1102,9 +1102,9 @@ static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
// FIXME: Does symbollic relocation from 'getFuncAddrForProfData' matter here?
static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
- // Store a nullptr in __profvt_ if a real address shouldn't be used.
auto *Int8PtrTy = Type::getInt8PtrTy(GV->getContext());
+ // Store a nullptr in __profvt_ if a real address shouldn't be used.
if (!shouldRecordVTableAddr(GV))
return ConstantPointerNull::get(Int8PtrTy);
@@ -1113,7 +1113,7 @@ static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
/// Get the name of a profiling variable for a particular variable.
static std::string getVarName(GlobalVariable *GV, StringRef Prefix) {
- StringRef Name = GV->getName();
+ StringRef Name = getPGOName(*GV);
return (Prefix + Name).str();
}
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index f81e877ca1d2aa9..560969198695510 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -589,9 +589,8 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
NumOfPGOBB += MST.BBInfos.size();
ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget);
- if (EnableVTableValueProfiling) {
+ if (EnableVTableValueProfiling)
ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
- }
} else {
NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
index 26fa2504ebdada4..63782c8b94d4a50 100644
--- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
+++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
@@ -13,9 +13,7 @@ RUN: printf '\0\0\0\0\1\0\0\0' >> %t
RUN: printf '\0\0\0\0\3\0\0\0' >> %t
RUN: printf '\0\0\0\0\2\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// NumVTables
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// ValueKindLast
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\134\370\302\114\333\030\275\254' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
index fd231b4ab119cba..8f61cffe2132005 100644
--- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
+++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
@@ -15,24 +15,15 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// InstrProfData
-// NameRef
RUN: printf '\254\275\030\333\114\302\370\134' >> %t
-// FuncHash
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
-// RelativeCounterPtr
RUN: printf '\0\0\4\0\1\0\0\0' >> %t
-// RelativeBitmapPtr
RUN: printf '\0\0\4\0\3\0\0\0' >> %t
-// FunctionPointer
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// Values
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// NumCounters + Int16ArrayTy
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\3\0\0\0' >> %t
-// InstrProfData
RUN: printf '\067\265\035\031\112\165\023\344' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t
More information about the cfe-commits
mailing list