[libcxx-commits] [clang] [mlir] [clang-tools-extra] [llvm] [libc] [compiler-rt] [libunwind] [libcxx] [lldb] [lld] [flang] [IRPGO][ValueProfile] Instrument virtual table address that could be used to do virtual table address comparision for indirect-call-promotion. (PR #66825)
Mingming Liu via libcxx-commits
libcxx-commits at lists.llvm.org
Tue Dec 5 23:30:14 PST 2023
https://github.com/minglotus-6 updated https://github.com/llvm/llvm-project/pull/66825
>From 361f8f6f884a6026e5ee65ceeea428d3db0b085b Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Sat, 16 Sep 2023 23:15:15 -0700
Subject: [PATCH 01/15] [IRPGO][ValueProfile] Instrument virtual table address
that could be used to do virtual table address comparision for
indirect-call-promotion.
The changes include:
1) Insert value profile intrinsics and lowering them.
- Introduced INSTR_PROF_VTABLE_DATA to record per-vtable data.
- Modified LLVM_PROF_RAW_HEADER to record the metadata for vtable profiles.
- Test case in llvm/test/Transforms/PGOProfile/vtable_profile.ll
2) Tooling support in llvm-profdata to show the added vtable information
- Changes are made in {raw,text,indexed} prof reader and/or writer to read/write vtable profile data.
- Test cases added in llvm/test/tools/llvm-profdata
---
compiler-rt/include/profile/InstrProfData.inc | 32 +++-
compiler-rt/lib/profile/InstrProfiling.h | 20 +-
.../lib/profile/InstrProfilingBuffer.c | 60 +++++-
.../lib/profile/InstrProfilingInternal.h | 11 +-
compiler-rt/lib/profile/InstrProfilingMerge.c | 20 +-
.../lib/profile/InstrProfilingPlatformLinux.c | 20 ++
.../lib/profile/InstrProfilingWriter.c | 45 ++++-
.../Linux/instrprof-value-prof-warn.test | 2 +-
.../profile/instrprof-write-buffer-internal.c | 10 +-
.../llvm/Analysis/IndirectCallVisitor.h | 22 ++-
llvm/include/llvm/ProfileData/InstrProf.h | 125 ++++++++++++-
.../llvm/ProfileData/InstrProfData.inc | 31 +++-
.../llvm/ProfileData/InstrProfReader.h | 59 +++++-
.../llvm/ProfileData/InstrProfWriter.h | 4 +
.../Instrumentation/InstrProfiling.h | 12 ++
llvm/lib/ProfileData/InstrProf.cpp | 172 +++++++++++++++---
llvm/lib/ProfileData/InstrProfReader.cpp | 79 +++++++-
llvm/lib/ProfileData/InstrProfWriter.cpp | 92 +++++++++-
.../Instrumentation/InstrProfiling.cpp | 165 +++++++++++++++++
.../Instrumentation/PGOInstrumentation.cpp | 1 +
.../Instrumentation/ValueProfilePlugins.inc | 26 ++-
.../Inputs/update_vtable_value_prof_inputs.sh | 84 +++++++++
.../Transforms/PGOProfile/comdat_internal.ll | 4 +-
.../indirect_call_profile_funclet.ll | 5 +-
.../Transforms/PGOProfile/vtable_profile.ll | 139 ++++++++++++++
.../llvm-profdata/Inputs/c-general.profraw | Bin 1800 -> 1912 bytes
.../llvm-profdata/Inputs/vtable-prof.proftext | 73 ++++++++
.../llvm-profdata/Inputs/vtable_prof.profraw | Bin 0 -> 816 bytes
.../llvm-profdata/binary-ids-padding.test | 10 +-
.../llvm-profdata/large-binary-id-size.test | 4 +-
...alformed-not-space-for-another-header.test | 7 +-
.../malformed-num-counters-zero.test | 7 +-
.../malformed-ptr-to-counter-array.test | 7 +-
.../misaligned-binary-ids-size.test | 4 +-
.../mismatched-raw-profile-header.test | 2 +
.../tools/llvm-profdata/raw-32-bits-be.test | 7 +-
.../tools/llvm-profdata/raw-32-bits-le.test | 6 +-
.../tools/llvm-profdata/raw-64-bits-be.test | 13 +-
.../tools/llvm-profdata/raw-64-bits-le.test | 29 ++-
.../tools/llvm-profdata/raw-two-profiles.test | 10 +-
.../tools/llvm-profdata/vtable-prof.proftext | 16 ++
.../vtable-value-prof-basic.test | 100 ++++++++++
llvm/tools/llvm-profdata/llvm-profdata.cpp | 37 +++-
43 files changed, 1472 insertions(+), 100 deletions(-)
create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
create mode 100644 llvm/test/Transforms/PGOProfile/vtable_profile.ll
create mode 100644 llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
create mode 100644 llvm/test/tools/llvm-profdata/Inputs/vtable_prof.profraw
create mode 100644 llvm/test/tools/llvm-profdata/vtable-prof.proftext
create mode 100644 llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index 8ba7e186d4fb1..5609870a5a8e5 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -92,6 +92,19 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
/* INSTR_PROF_DATA end. */
+#ifndef INSTR_PROF_VTABLE_DATA
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_VTABLE_DATA_DEFINED
+#endif
+INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, \
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(VTableName)))
+INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), VTablePointer, VTableAddr)
+INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), VTableSizeVal))
+#undef INSTR_PROF_VTABLE_DATA
+/* INSTR_PROF_VTABLE_DATA end. */
+
/* This is an internal data structure used by value profiler. It
* is defined here to allow serialization code sharing by LLVM
* to be used in unit test.
@@ -136,6 +149,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
(uintptr_t)CountersBegin - (uintptr_t)DataBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
#undef INSTR_PROF_RAW_HEADER
/* INSTR_PROF_RAW_HEADER end */
@@ -177,13 +192,14 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
+VALUE_PROF_KIND(IPVK_VTableTarget, 2, "vtable target")
/* These two kinds must be the last to be
* declared. This is to make sure the string
* array created with the template can be
* indexed with the kind value.
*/
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
-VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
+VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")
#undef VALUE_PROF_KIND
/* VALUE_PROF_KIND end */
@@ -270,12 +286,18 @@ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
INSTR_PROF_SECT_ENTRY(IPSK_name, \
INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
INSTR_PROF_NAME_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vname, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
+ INSTR_PROF_VNAME_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vals, \
INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
INSTR_PROF_VALS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
INSTR_PROF_VNODES_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
+ INSTR_PROF_VTAB_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
@@ -645,9 +667,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 8
+#define INSTR_PROF_RAW_VERSION 9
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 10
+#define INSTR_PROF_INDEX_VERSION 11
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 5
@@ -685,9 +707,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
+#define INSTR_PROF_VNAME_COMMON __llvm_prf_vnames
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
+#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
@@ -696,9 +720,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
*/
#define INSTR_PROF_DATA_COFF ".lprfd$M"
#define INSTR_PROF_NAME_COFF ".lprfn$M"
+#define INSTR_PROF_VNAME_COFF ".lprfn$M"
#define INSTR_PROF_CNTS_COFF ".lprfc$M"
#define INSTR_PROF_VALS_COFF ".lprfv$M"
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
+#define INSTR_PROF_VTAB_COFF ".lprfvt$M"
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index 4433d7bd48871..f3afa694e02c2 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -38,6 +38,12 @@ typedef struct ValueProfNode {
#include "profile/InstrProfData.inc"
} ValueProfNode;
+typedef void *IntPtrT;
+typedef struct VTableProfData {
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) Type Name;
+#include "profile/InstrProfData.inc"
+} VTableProfData;
+
/*!
* \brief Return 1 if profile counters are continuously synced to the raw
* profile via an mmap(). This is in contrast to the default mode, in which
@@ -86,10 +92,14 @@ const __llvm_profile_data *__llvm_profile_begin_data(void);
const __llvm_profile_data *__llvm_profile_end_data(void);
const char *__llvm_profile_begin_names(void);
const char *__llvm_profile_end_names(void);
+const char *__llvm_profile_begin_vnames(void);
+const char *__llvm_profile_end_vnames(void);
char *__llvm_profile_begin_counters(void);
char *__llvm_profile_end_counters(void);
ValueProfNode *__llvm_profile_begin_vnodes();
ValueProfNode *__llvm_profile_end_vnodes();
+VTableProfData *__llvm_profile_begin_vtables();
+VTableProfData *__llvm_profile_end_vtables();
uint32_t *__llvm_profile_begin_orderfile();
/*!
@@ -276,6 +286,12 @@ uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End);
/*! \brief Get the size of the profile counters section in bytes. */
uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End);
+uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
+ const VTableProfData *End);
+
+uint64_t __llvm_profile_get_vtable_size(const VTableProfData *Begin,
+ const VTableProfData *End);
+
/* ! \brief Given the sizes of the data and counter information, return the
* number of padding bytes before and after the counters, and after the names,
* in the raw profile.
@@ -287,8 +303,10 @@ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End);
*/
void __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize,
+ uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
- uint64_t *PaddingBytesAfterNames);
+ uint64_t *PaddingBytesAfterNames, uint64_t *PaddingBytesAfterVTable,
+ uint64_t *PaddingBytesAfterVNames);
/*!
* \brief Set the flag that profile data has been dumped to the file.
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index 61ac5d9c02850..0c36e40444c73 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -9,6 +9,8 @@
// Note: This is linked into the Darwin kernel, and must remain compatible
// with freestanding compilation. See `darwin_add_builtin_libraries`.
+#include <assert.h>
+
#include "InstrProfiling.h"
#include "InstrProfilingInternal.h"
#include "InstrProfilingPort.h"
@@ -45,9 +47,14 @@ uint64_t __llvm_profile_get_size_for_buffer(void) {
const char *CountersEnd = __llvm_profile_end_counters();
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
+ const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
+ const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
+ const char *VNamesBegin = __llvm_profile_begin_vnames();
+ const char *VNamesEnd = __llvm_profile_end_vnames();
return __llvm_profile_get_size_for_buffer_internal(
- DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd);
+ DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd,
+ VTableBegin, VTableEnd, VNamesBegin, VNamesEnd);
}
COMPILER_RT_VISIBILITY
@@ -63,6 +70,18 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
const __llvm_profile_data *End) {
return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);
}
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
+ const VTableProfData *End) {
+ intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin;
+ return (EndI + sizeof(VTableProfData) - 1 - BeginI) / sizeof(VTableProfData);
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_vtable_size(const VTableProfData *Begin,
+ const VTableProfData *End) {
+ return __llvm_profile_get_num_vtable(Begin, End) * sizeof(VTableProfData);
+}
COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) {
if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE)
@@ -103,46 +122,68 @@ static int needsCounterPadding(void) {
COMPILER_RT_VISIBILITY
void __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize,
+ uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
- uint64_t *PaddingBytesAfterNames) {
+ uint64_t *PaddingBytesAfterNames, uint64_t *PaddingBytesAfterVTable,
+ uint64_t *PaddingBytesAfterVName) {
+ // Counter padding is needed only if continuous mode is enabled.
if (!needsCounterPadding()) {
*PaddingBytesBeforeCounters = 0;
*PaddingBytesAfterCounters =
__llvm_profile_get_num_padding_bytes(CountersSize);
*PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
+ *PaddingBytesAfterVTable = __llvm_profile_get_num_padding_bytes(VTableSize);
+ *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);
return;
}
+ // Value profiling not supported in continuous mode at profile-write time
+ // according to
+ // https://github.com/llvm/llvm-project/blob/e6a007f6b51a661ed3dd8b0210b734b3e9b4354f/compiler-rt/lib/profile/InstrProfilingWriter.c#L328
+ assert(VTableSize == 0 && VNameSize == 0 &&
+ "Value profile not supported for continuous mode");
// In continuous mode, the file offsets for headers and for the start of
// counter sections need to be page-aligned.
*PaddingBytesBeforeCounters =
calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize);
*PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize);
*PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
+ // Set these two variables to zero to avoid uninitialized variables
+ // even if VTableSize and VNameSize are asserted to be zero.
+ *PaddingBytesAfterVTable = 0;
+ *PaddingBytesAfterVName = 0;
}
COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
- const char *NamesEnd) {
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd) {
/* Match logic in __llvm_profile_write_buffer(). */
const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
uint64_t CountersSize =
__llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+ uint64_t VTableSize = __llvm_profile_get_vtable_size(VTableBegin, VTableEnd);
+ uint64_t VNameSize = (VNamesEnd - VNamesBegin) * sizeof(char);
/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterNames;
+ PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
__llvm_profile_get_padding_sizes_for_counters(
- DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
- &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
+ DataSize, CountersSize, NamesSize, VTableSize, VNameSize,
+ &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
+ &PaddingBytesAfterNames, &PaddingBytesAfterVTable,
+ &PaddingBytesAfterVNames);
return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
DataSize + PaddingBytesBeforeCounters + CountersSize +
- PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames;
+ PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames +
+ VTableSize + PaddingBytesAfterVTable + VNameSize +
+ PaddingBytesAfterVNames;
}
COMPILER_RT_VISIBILITY
@@ -163,6 +204,9 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
const char *CountersEnd, const char *NamesBegin, const char *NamesEnd) {
ProfDataWriter BufferWriter;
initBufferWriter(&BufferWriter, Buffer);
+ // Set virtual table arguments to NULL since they are not supported yet.
return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin,
- CountersEnd, 0, NamesBegin, NamesEnd, 0);
+ CountersEnd, 0, NamesBegin, NamesEnd,
+ NULL /* VTableBegin */, NULL /* VTableEnd */,
+ NULL /* VNamesBegin */, NULL /* VNamesEnd */, 0);
}
diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h
index 360165e32ab3f..bce333f933f0f 100644
--- a/compiler-rt/lib/profile/InstrProfilingInternal.h
+++ b/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -18,11 +18,16 @@
* pointers to the live data in memory. This function is probably not what you
* want. Use __llvm_profile_get_size_for_buffer instead. Use this function if
* your program has a custom memory layout.
+ * NOTE: The change of function signature requires modifying c source code
+ * as demonstrated by the existing tests. If this is causing backward
+ * compatible issues, considering adding another function for new use cases.
*/
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
- const char *NamesEnd);
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd);
/*!
* \brief Write instrumentation data to the given buffer, given explicit
@@ -154,7 +159,9 @@ int lprofWriteDataImpl(ProfDataWriter *Writer,
const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd,
VPDataReaderType *VPDataReader, const char *NamesBegin,
- const char *NamesEnd, int SkipNameDataWrite);
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd, int SkipNameDataWrite);
/* Merge value profile data pointed to by SrcValueProfData into
* in-memory profile counters pointed by to DstData. */
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 9cf12f251f726..2ef6227599ff1 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -124,9 +124,27 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
SrcCountersEnd = SrcCountersStart +
Header->NumCounters * __llvm_profile_counter_entry_size();
SrcNameStart = SrcCountersEnd;
- SrcValueProfDataStart =
+ // This is to assume counter size is a multiple of 8 bytes.
+ // uint64_t NamesSize = Header->NamesSize;
+ // uint64_t PaddingBytesAfterNames =
+ // __llvm_profile_get_num_padding_bytes(Header->NamesSize);
+ // First, skip rather than merge them
+ uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData);
+ uint64_t PaddingBytesAfterVTableSection =
+ __llvm_profile_get_num_padding_bytes(VTableSectionSize);
+ uint64_t VNamesSize = Header->VNamesSize;
+ uint64_t PaddingBytesAfterVNamesSize =
+ __llvm_profile_get_num_padding_bytes(VNamesSize);
+
+ uint64_t VTableProfDataOffset =
SrcNameStart + Header->NamesSize +
__llvm_profile_get_num_padding_bytes(Header->NamesSize);
+
+ uint64_t VTableNamesOffset =
+ VTableProfDataOffset + VTableSectionSize + PaddingBytesAfterVTableSection;
+
+ SrcValueProfDataStart =
+ VTableNamesOffset + VNamesSize + PaddingBytesAfterVNamesSize;
if (SrcNameStart < SrcCountersStart)
return 1;
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
index 2cce0a4b2c48d..dc861632271ce 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -33,8 +33,12 @@
#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON)
#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON)
#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON)
+#define PROF_VNAME_START INSTR_PROF_SECT_START(INSTR_PROF_VNAME_COMMON)
+#define PROF_VNAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNAME_COMMON)
#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON)
#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON)
+#define PROF_VTABLE_START INSTR_PROF_SECT_START(INSTR_PROF_VTAB_COMMON)
+#define PROF_VTABLE_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VTAB_COMMON)
#define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON)
#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON)
#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON)
@@ -48,6 +52,10 @@ extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY
COMPILER_RT_WEAK;
extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern VTableProfData PROF_VTABLE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern VTableProfData PROF_VTABLE_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_VNAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_VNAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
@@ -68,6 +76,18 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) {
COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) {
return &PROF_NAME_STOP;
}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vnames(void) {
+ return &PROF_VNAME_START;
+}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vnames(void) {
+ return &PROF_VNAME_STOP;
+}
+COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_begin_vtables(void) {
+ return &PROF_VTABLE_START;
+}
+COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_end_vtables(void) {
+ return &PROF_VTABLE_STOP;
+}
COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) {
return &PROF_CNTS_START;
}
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 1e22398a4c0f6..ed5dbdb6ee438 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -248,8 +248,13 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
const char *CountersEnd = __llvm_profile_end_counters();
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
+ const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
+ const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
+ const char *VNamesBegin = __llvm_profile_begin_vnames();
+ const char *VNamesEnd = __llvm_profile_end_vnames();
return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
CountersEnd, VPDataReader, NamesBegin, NamesEnd,
+ VTableBegin, VTableEnd, VNamesBegin, VNamesEnd,
SkipNameDataWrite);
}
@@ -258,7 +263,9 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd,
VPDataReaderType *VPDataReader, const char *NamesBegin,
- const char *NamesEnd, int SkipNameDataWrite) {
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd, int SkipNameDataWrite) {
int DebugInfoCorrelate =
(__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL;
@@ -272,6 +279,13 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const uint64_t NumCounters =
__llvm_profile_get_num_counters(CountersBegin, CountersEnd);
const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin;
+ const uint64_t NumVTables =
+ __llvm_profile_get_num_vtable(VTableBegin, VTableEnd);
+ const uint64_t VTableSectionSize =
+ __llvm_profile_get_vtable_size(VTableBegin, VTableEnd);
+ // Note, in reality, vtable profiling is not supported when DebugInfoCorrelate
+ // is true.
+ const uint64_t VNamesSize = DebugInfoCorrelate ? 0 : VNamesEnd - VNamesBegin;
/* Create the header. */
__llvm_profile_header Header;
@@ -279,11 +293,12 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterNames;
+ PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
__llvm_profile_get_padding_sizes_for_counters(
- DataSectionSize, CountersSectionSize, NamesSize,
- &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
- &PaddingBytesAfterNames);
+ DataSectionSize, CountersSectionSize, NamesSize, VTableSectionSize,
+ VNamesSize, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
+ &PaddingBytesAfterNames, &PaddingBytesAfterVTable,
+ &PaddingBytesAfterVNames);
{
/* Initialize header structure. */
@@ -305,13 +320,24 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
/* Write the profile header. */
ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
+ // printf("Size of profile header is %d\n",
+ // (int)(sizeof(__llvm_profile_header)));
if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
return -1;
+ // printf("Completed profile header\n");
+
/* Write the binary id lengths and data. */
- if (__llvm_write_binary_ids(Writer) == -1)
+ int binary_id_size = __llvm_write_binary_ids(Writer);
+ if (binary_id_size == -1)
return -1;
+ // Might be needed for debugging. Clean up before commit.
+ // uint64_t VTableProfDataOffset =
+ // sizeof(__llvm_profile_header) + binary_id_size + DataSectionSize +
+ // PaddingBytesBeforeCounters + CountersSectionSize +
+ // PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames;
+
/* Write the profile data. */
ProfDataIOVec IOVecData[] = {
{DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize,
@@ -321,7 +347,12 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
{NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
{(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin,
sizeof(uint8_t), NamesSize, 0},
- {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
+ {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1},
+ {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1},
+ {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : VNamesBegin,
+ sizeof(uint8_t), VNamesSize, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}};
if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
return -1;
diff --git a/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test b/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
index 991fd2b69b5b3..89787e79c20e9 100644
--- a/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
+++ b/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
@@ -1,7 +1,7 @@
RUN: %clang_pgogen -O2 -fuse-ld=bfd -mllvm -disable-vp=false -mllvm -vp-static-alloc=true -DSTRESS=1 -o %t.ir.warn %S/../Inputs/instrprof-value-prof-real.c
RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=255 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=WARNING %s
# Test that enough static counters have been allocated
-RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=130 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=NOWARNING --allow-empty %s
+RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=80 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=NOWARNING --allow-empty %s
# WARNING: LLVM Profile Warning:
# NOWARNING-NOT: LLVM Profile Warning:
diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
index 7b96c6d91c33f..97dfff8d7fe71 100644
--- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c
+++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
@@ -28,7 +28,9 @@ char *__llvm_profile_end_counters(void);
uint64_t __llvm_profile_get_size_for_buffer_internal(
const void *DataBegin, const void *DataEnd, const char *CountersBegin,
- const char *CountersEnd, const char *NamesBegin, const char *NamesEnd);
+ const char *CountersEnd, const char *NamesBegin, const char *NamesEnd,
+ const char *VTableBegin, const char *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd);
int __llvm_profile_write_buffer_internal(char *Buffer, const void *DataBegin,
const void *DataEnd,
@@ -43,7 +45,11 @@ int main(int argc, const char *argv[]) {
uint64_t bufsize = __llvm_profile_get_size_for_buffer_internal(
__llvm_profile_begin_data(), __llvm_profile_end_data(),
__llvm_profile_begin_counters(), __llvm_profile_end_counters(),
- __llvm_profile_begin_names(), __llvm_profile_end_names());
+ __llvm_profile_begin_names(), __llvm_profile_end_names(), NULL, NULL,
+ NULL, NULL);
+
+ // printf("buffer size is %lld\n", bufsize);
+ //uint64_t aligned_bufsize = ((bufsize + 32) >> 6) << 6;
char *buf = malloc(bufsize);
int ret = __llvm_profile_write_buffer_internal(buf,
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 0825e19ecd2d2..52d4ff04d3d4e 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -19,20 +19,38 @@ namespace llvm {
// Visitor class that finds all indirect call.
struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
std::vector<CallBase *> IndirectCalls;
+ std::vector<Instruction *> VTableAddrs;
PGOIndirectCallVisitor() = default;
void visitCallBase(CallBase &Call) {
if (Call.isIndirectCall())
- IndirectCalls.push_back(&Call);
+ if (Call.isIndirectCall()) {
+ IndirectCalls.push_back(&Call);
+
+ LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
+ if (LI != nullptr) {
+ Value *Ptr = LI->getPointerOperand();
+ Value *VTablePtr = Ptr->stripInBoundsConstantOffsets();
+ if (VTablePtr != nullptr && isa<Instruction>(VTablePtr)) {
+ VTableAddrs.push_back(cast<Instruction>(VTablePtr));
+ }
+ }
+ }
}
};
-// Helper function that finds all indirect call sites.
inline std::vector<CallBase *> findIndirectCalls(Function &F) {
PGOIndirectCallVisitor ICV;
ICV.visit(F);
return ICV.IndirectCalls;
}
+
+inline std::vector<Instruction *> findVTableAddrs(Function &F) {
+ PGOIndirectCallVisitor ICV;
+ ICV.visit(F);
+ return ICV.VTableAddrs;
+}
+
} // namespace llvm
#endif
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index e968f8ffd5075..845ea1d0ab0b3 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -90,6 +90,9 @@ inline StringRef getInstrProfValueProfMemOpFuncName() {
/// Return the name prefix of variables containing instrumented function names.
inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
+/// Return the name prefix of variables containing virtual table profile data.
+inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; }
+
/// Return the name prefix of variables containing per-function control data.
inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
@@ -108,6 +111,8 @@ inline StringRef getInstrProfNamesVarName() {
return "__llvm_prf_nm";
}
+inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; }
+
/// Return the name of a covarage mapping variable (internal linkage)
/// for each instrumented source module. Such variables are allocated
/// in the __llvm_covmap section.
@@ -240,11 +245,16 @@ Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
std::string &Result, bool doCompression = true);
+Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
+ std::string &Result, bool doCompression);
+
/// \c NameStrings is a string composed of one of more sub-strings encoded in
/// the format described above. The substrings are separated by 0 or more zero
/// bytes. This method decodes the string and populates the \c Symtab.
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);
+Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab);
+
/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
/// set in IR PGO compilation.
bool isIRPGOFlagSet(const Module *M);
@@ -294,7 +304,7 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
/// Check if we can use Comdat for profile variables. This will eliminate
/// the duplicated profile variables for Comdat functions.
-bool needsComdatForCounter(const Function &F, const Module &M);
+bool needsComdatForCounter(const GlobalValue &GV, const Module &M);
/// An enum describing the attributes of an instrumented profile.
enum class InstrProfKind {
@@ -432,14 +442,24 @@ class InstrProfSymtab {
uint64_t Address = 0;
// Unique name strings.
StringSet<> NameTab;
+ // Unique virtual table names.
+ StringSet<> VTableNames;
// A map from MD5 keys to function name strings.
std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
+ // A map from MD5 keys to virtual table definitions. Only populated when
+ // building the Symtab from a module.
+ std::vector<std::pair<uint64_t, GlobalVariable *>> MD5VTableMap;
// A map from MD5 keys to function define. We only populate this map
// when build the Symtab from a Module.
std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
// A map from function runtime address to function name MD5 hash.
// This map is only populated and used by raw instr profile reader.
AddrHashMap AddrToMD5Map;
+ // A map from virtual table runtime address to function name MD5 hash.
+ // This map is only populated and used by raw instr profile reader.
+ // This is a different map from 'AddrToMD5Map' for readability and
+ // debuggability.
+ AddrHashMap VTableAddrToMD5Map;
bool Sorted = false;
static StringRef getExternalSymbol() {
@@ -474,6 +494,8 @@ class InstrProfSymtab {
/// This method is a wrapper to \c readPGOFuncNameStrings method.
inline Error create(StringRef NameStrings);
+ inline Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
+
/// A wrapper interface to populate the PGO symtab with functions
/// decls from module \c M. This interface is used by transformation
/// passes such as indirect function call promotion. Variable \c InLTO
@@ -484,6 +506,13 @@ class InstrProfSymtab {
/// \p IterRange. This interface is used by IndexedProfReader.
template <typename NameIterRange> Error create(const NameIterRange &IterRange);
+ /// Create InstrProfSymtab from a set of function names and vtable
+ /// names iteratable from \p IterRange. This interface is used by
+ /// IndexedProfReader.
+ template <typename FuncNameIterRange, typename VTableNameIterRange>
+ Error create(const FuncNameIterRange &FuncIterRange,
+ const VTableNameIterRange &VTableIterRange);
+
/// Update the symtab by adding \p FuncName to the table. This interface
/// is used by the raw and text profile readers.
Error addFuncName(StringRef FuncName) {
@@ -499,15 +528,49 @@ class InstrProfSymtab {
return Error::success();
}
+ Error addVTableName(StringRef VTableName) {
+ if (VTableName.empty())
+ return make_error<InstrProfError>(instrprof_error::malformed,
+ "invalid input: VTableName is empty");
+ // Insert into NameTab.
+ auto Ins = NameTab.insert(VTableName);
+
+ // Insert into VTableNames.
+ VTableNames.insert(VTableName);
+
+ // If this is newly added, update MD5NameMap.
+ if (Ins.second) {
+ // printf("VTableName %s\n", VTableName.str().c_str());
+ // printf("AddVTableName hash %"PRIu64" to %s\n",
+ // IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey());
+ MD5NameMap.push_back(std::make_pair(
+ IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey()));
+ Sorted = false;
+ }
+ return Error::success();
+ }
+
+ const StringSet<> &getVTableNames() const { return VTableNames; }
+
/// Map a function address to its name's MD5 hash. This interface
/// is only used by the raw profiler reader.
void mapAddress(uint64_t Addr, uint64_t MD5Val) {
AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
}
+ // Map the start and end address of a variable to its names' MD5 hash.
+ // This interface is only used by the raw profile header.
+ void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
+ VTableAddrToMD5Map.push_back(std::make_pair(StartAddr, MD5Val));
+ VTableAddrToMD5Map.push_back(std::make_pair(EndAddr, MD5Val));
+ }
+
/// Return a function's hash, or 0, if the function isn't in this SymTab.
uint64_t getFunctionHashFromAddress(uint64_t Address);
+ /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
+ uint64_t getVTableHashFromAddress(uint64_t Address);
+
/// Return function's PGO name from the function name's symbol
/// address in the object file. If an error occurs, return
/// an empty string.
@@ -522,6 +585,11 @@ class InstrProfSymtab {
/// this symbol table.
inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash);
+ /// Just like getFuncName, except that it will return a non-empty StringRef
+ /// if the function is external to this symbol table. All such cases
+ /// will be represented using the same StringRef value.
+ // inline StringRef getVTableNameOrExternalSymbol(uint64_t VTableMD5Hash);
+
/// True if Symbol is the value used to represent external symbols.
static bool isExternalSymbol(const StringRef &Symbol) {
return Symbol == InstrProfSymtab::getExternalSymbol();
@@ -529,6 +597,8 @@ class InstrProfSymtab {
/// Return function from the name's md5 hash. Return nullptr if not found.
inline Function *getFunction(uint64_t FuncMD5Hash);
+ // Return vtable from the name's MD5 hash. Return nullptr if not found.
+ inline GlobalVariable *getGlobalVariable(uint64_t GlobalVariableMD5Hash);
/// Return the name section data.
inline StringRef getNameData() const { return Data; }
@@ -547,6 +617,16 @@ Error InstrProfSymtab::create(StringRef NameStrings) {
return readPGOFuncNameStrings(NameStrings, *this);
}
+Error InstrProfSymtab::create(StringRef FuncNameStrings,
+ StringRef VTableNameStrings) {
+ if (Error E = readPGOFuncNameStrings(FuncNameStrings, *this))
+ return E;
+
+ // FIXME: Add test coverage that this returns success when VTableNameStrings
+ // is empty.
+ return readVTableNames(VTableNameStrings, *this);
+}
+
template <typename NameIterRange>
Error InstrProfSymtab::create(const NameIterRange &IterRange) {
for (auto Name : IterRange)
@@ -557,6 +637,23 @@ Error InstrProfSymtab::create(const NameIterRange &IterRange) {
return Error::success();
}
+template <typename FuncNameIterRange, typename VTableNameIterRange>
+Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange,
+ const VTableNameIterRange &VTableIterRange) {
+ for (auto Name : FuncIterRange)
+ if (Error E = addFuncName(Name))
+ return E;
+
+ for (auto VTableName : VTableIterRange) {
+ if (Error E = addVTableName(VTableName)) {
+ return E;
+ }
+ }
+
+ finalizeSymtab();
+ return Error::success();
+}
+
void InstrProfSymtab::finalizeSymtab() {
if (Sorted)
return;
@@ -595,6 +692,19 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
return nullptr;
}
+GlobalVariable *
+InstrProfSymtab::getGlobalVariable(uint64_t GlobalVariableMD5Hash) {
+ finalizeSymtab();
+ auto Result =
+ llvm::lower_bound(MD5VTableMap, GlobalVariableMD5Hash,
+ [](const std::pair<uint64_t, GlobalVariable *> &LHS,
+ uint64_t RHS) { return LHS.first < RHS; });
+
+ if (Result != MD5VTableMap.end() && Result->first == GlobalVariableMD5Hash)
+ return Result->second;
+ return nullptr;
+}
+
// To store the sums of profile count values, or the percentage of
// the sums of the total count values.
struct CountSumOrPercent {
@@ -816,6 +926,7 @@ struct InstrProfRecord {
struct ValueProfData {
std::vector<InstrProfValueSiteRecord> IndirectCallSites;
std::vector<InstrProfValueSiteRecord> MemOPSizes;
+ std::vector<InstrProfValueSiteRecord> VTableTargets;
};
std::unique_ptr<ValueProfData> ValueData;
@@ -838,6 +949,8 @@ struct InstrProfRecord {
return ValueData->IndirectCallSites;
case IPVK_MemOPSize:
return ValueData->MemOPSizes;
+ case IPVK_VTableTarget:
+ return ValueData->VTableTargets;
default:
llvm_unreachable("Unknown value kind!");
}
@@ -852,6 +965,8 @@ struct InstrProfRecord {
return ValueData->IndirectCallSites;
case IPVK_MemOPSize:
return ValueData->MemOPSizes;
+ case IPVK_VTableTarget:
+ return ValueData->VTableTargets;
default:
llvm_unreachable("Unknown value kind!");
}
@@ -1014,6 +1129,8 @@ enum ProfVersion {
Version9 = 9,
// An additional (optional) temporal profile traces section is added.
Version10 = 10,
+ // VTable profiling,
+ Version11 = 11,
// The current version is 10.
CurrentVersion = INSTR_PROF_INDEX_VERSION
};
@@ -1034,6 +1151,7 @@ struct Header {
uint64_t MemProfOffset;
uint64_t BinaryIdOffset;
uint64_t TemporalProfTracesOffset;
+ uint64_t VTableNamesOffset; // Organize virtual table names.
// New fields should only be added at the end to ensure that the size
// computation is correct. The methods below need to be updated to ensure that
// the new field is read correctly.
@@ -1173,6 +1291,11 @@ template <class IntPtrT> struct alignas(8) ProfileData {
#include "llvm/ProfileData/InstrProfData.inc"
};
+template <class IntPtrT> struct alignas(8) VTableProfileData {
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name;
+#include "llvm/ProfileData/InstrProfData.inc"
+};
+
// File header structure of the LLVM profile data in raw format.
// The definition should match the header referenced in
// compiler-rt/lib/profile/InstrProfilingFile.c and
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index 13be2753e514e..f1e7691afdc4b 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -91,6 +91,18 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
#undef INSTR_PROF_DATA
/* INSTR_PROF_DATA end. */
+#ifndef INSTR_PROF_VTABLE_DATA
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_VTABLE_DATA_DEFINED
+#endif
+INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, \
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(VTableName)))
+INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), VTablePointer, VTableAddr)
+INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), VTableSizeVal))
+#undef INSTR_PROF_VTABLE_DATA
+/* INSTR_PROF_VTABLE_DATA end. */
/* This is an internal data structure used by value profiler. It
* is defined here to allow serialization code sharing by LLVM
@@ -136,6 +148,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
(uintptr_t)CountersBegin - (uintptr_t)DataBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
#undef INSTR_PROF_RAW_HEADER
/* INSTR_PROF_RAW_HEADER end */
@@ -177,13 +191,14 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
+VALUE_PROF_KIND(IPVK_VTableTarget, 2, "vtable target")
/* These two kinds must be the last to be
* declared. This is to make sure the string
* array created with the template can be
* indexed with the kind value.
*/
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
-VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
+VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")
#undef VALUE_PROF_KIND
/* VALUE_PROF_KIND end */
@@ -270,12 +285,18 @@ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
INSTR_PROF_SECT_ENTRY(IPSK_name, \
INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
INSTR_PROF_NAME_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vname, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
+ INSTR_PROF_VNAME_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vals, \
INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
INSTR_PROF_VALS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
INSTR_PROF_VNODES_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
+ INSTR_PROF_VTAB_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
@@ -645,9 +666,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 8
+#define INSTR_PROF_RAW_VERSION 9
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 10
+#define INSTR_PROF_INDEX_VERSION 11
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 5
@@ -685,9 +706,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
+#define INSTR_PROF_VNAME_COMMON __llvm_prf_vnames
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
+#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
@@ -696,9 +719,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
*/
#define INSTR_PROF_DATA_COFF ".lprfd$M"
#define INSTR_PROF_NAME_COFF ".lprfn$M"
+#define INSTR_PROF_VNAME_COFF ".lprfvn$M"
#define INSTR_PROF_CNTS_COFF ".lprfc$M"
#define INSTR_PROF_VALS_COFF ".lprfv$M"
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
+#define INSTR_PROF_VTAB_COFF ".lprfvt$M"
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 5f54cbeb1b01e..d7009bbe9dc3f 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -326,10 +326,14 @@ class RawInstrProfReader : public InstrProfReader {
uint64_t NamesDelta;
const RawInstrProf::ProfileData<IntPtrT> *Data;
const RawInstrProf::ProfileData<IntPtrT> *DataEnd;
+ const RawInstrProf::VTableProfileData<IntPtrT> *VTableBegin = nullptr;
+ const RawInstrProf::VTableProfileData<IntPtrT> *VTableEnd = nullptr;
const char *CountersStart;
const char *CountersEnd;
const char *NamesStart;
const char *NamesEnd;
+ const char *VNamesStart = nullptr;
+ const char *VNamesEnd = nullptr;
// After value profile is all read, this pointer points to
// the header of next profile data (if exists)
const uint8_t *ValueDataStart;
@@ -468,6 +472,46 @@ enum class HashT : uint32_t;
} // end namespace IndexedInstrProf
+class InstrProfVTableLookupTrait {
+ char val;
+ IndexedInstrProf::HashT HashType;
+ unsigned FormatVersion;
+
+public:
+ InstrProfVTableLookupTrait(IndexedInstrProf::HashT HashType,
+ unsigned FormatVersion)
+ : HashType(HashType), FormatVersion(FormatVersion) {}
+
+ using data_type = char;
+
+ using internal_key_type = StringRef;
+ using external_key_type = StringRef;
+
+ using hash_value_type = uint64_t;
+ using offset_type = uint64_t;
+
+ static bool EqualKey(StringRef A, StringRef B) { return A == B; }
+ static StringRef GetInternalKey(StringRef K) { return K; }
+ static StringRef GetExternalKey(StringRef K) { return K; }
+
+ hash_value_type ComputeHash(StringRef K);
+
+ static std::pair<offset_type, offset_type>
+ ReadKeyDataLength(const unsigned char *&D) {
+ using namespace support;
+
+ offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D);
+ offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D);
+ return std::make_pair(KeyLen, DataLen);
+ }
+
+ StringRef ReadKey(const unsigned char *D, offset_type N) {
+ return StringRef((const char *)D, N);
+ }
+
+ data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
+};
+
/// Trait for lookups into the on-disk hash table for the binary instrprof
/// format.
class InstrProfLookupTrait {
@@ -521,6 +565,9 @@ class InstrProfLookupTrait {
}
};
+using VirtualTableNamesHashTable =
+ OnDiskIterableChainedHashTable<InstrProfVTableLookupTrait>;
+
struct InstrProfReaderIndexBase {
virtual ~InstrProfReaderIndexBase() = default;
@@ -543,7 +590,10 @@ struct InstrProfReaderIndexBase {
virtual bool hasMemoryProfile() const = 0;
virtual bool hasTemporalProfile() const = 0;
virtual InstrProfKind getProfileKind() const = 0;
- virtual Error populateSymtab(InstrProfSymtab &) = 0;
+ // The pointer VirtualTableIndex is not owned.
+ virtual Error
+ populateSymtab(InstrProfSymtab &,
+ VirtualTableNamesHashTable *VirtualTableIndex) = 0;
};
using OnDiskHashTableImplV3 =
@@ -618,7 +668,10 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase {
InstrProfKind getProfileKind() const override;
- Error populateSymtab(InstrProfSymtab &Symtab) override {
+ Error populateSymtab(InstrProfSymtab &Symtab,
+ VirtualTableNamesHashTable *VirtualTableIndex) override {
+ if (VirtualTableIndex != nullptr)
+ return Symtab.create(HashTable->keys(), VirtualTableIndex->keys());
return Symtab.create(HashTable->keys());
}
};
@@ -653,6 +706,8 @@ class IndexedInstrProfReader : public InstrProfReader {
std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable;
/// MemProf frame profile data on-disk indexed via frame id.
std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable;
+ /// Virtual table profile data indexed .
+ std::unique_ptr<VirtualTableNamesHashTable> VirtualTableIndex = nullptr;
/// Total size of binary ids.
uint64_t BinaryIdsSize{0};
/// Start address of binary id length and data pairs.
diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 311c84a2d52bc..768a0dd2785f4 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -64,6 +64,9 @@ class InstrProfWriter {
// List of binary ids.
std::vector<llvm::object::BuildID> BinaryIds;
+ // Read the vtable names from raw instr profile reader.
+ StringSet<> VTableNames;
+
// An enum describing the attributes of the profile.
InstrProfKind ProfileKind = InstrProfKind::Unknown;
// Use raw pointer here for the incomplete type object.
@@ -85,6 +88,7 @@ class InstrProfWriter {
void addRecord(NamedInstrProfRecord &&I, function_ref<void(Error)> Warn) {
addRecord(std::move(I), 1, Warn);
}
+ void addVTableName(StringRef VTableName) { VTableNames.insert(VTableName); }
/// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the
/// total number of temporal profiling traces the source has seen.
diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
index cb0c055dcb74a..9af090c0c2e68 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
@@ -56,12 +56,18 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
}
};
DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
+ // Key is virtual table variable, value is 'VTableProfData' in the form of
+ // GlobalVariable.
+ DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;
/// If runtime relocation is enabled, this maps functions to the load
/// instruction that produces the profile relocation bias.
DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
std::vector<GlobalValue *> CompilerUsedVars;
std::vector<GlobalValue *> UsedVars;
std::vector<GlobalVariable *> ReferencedNames;
+ // The list of virtual table variables of which the VTableProfData is
+ // collected.
+ std::vector<GlobalVariable *> ReferencedVTableNames;
GlobalVariable *NamesVar;
size_t NamesSize;
@@ -115,6 +121,9 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
/// referring to them will also be created.
GlobalVariable *getOrCreateRegionCounters(InstrProfInstBase *Inc);
+ /// Get the counters for virtual table values, creating them if necessary.
+ void getOrCreateVTableProfData(GlobalVariable *GV);
+
/// Create the region counters.
GlobalVariable *createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
GlobalValue::LinkageTypes Linkage);
@@ -122,6 +131,9 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
/// Emit the section with compressed function names.
void emitNameData();
+ /// Emit the section with compressed vtable names.
+ void emitVTableNames();
+
/// Emit value nodes section for value profiling.
void emitVNodes();
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 0cb296b3bde6c..31d29a6c4bd8b 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -447,6 +447,18 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) {
if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO)))
return E;
}
+
+ SmallVector<MDNode *, 2> Types;
+ for (GlobalVariable &G : M.globals()) {
+ if (!G.hasName())
+ continue;
+ Types.clear();
+ G.getMetadata(LLVMContext::MD_type, Types);
+ if (!Types.empty()) {
+ // errs() << "Insert " << G.getGUID() << "\t into MD5VTableMap\n";
+ MD5VTableMap.emplace_back(G.getGUID(), &G);
+ }
+ }
Sorted = false;
finalizeSymtab();
return Error::success();
@@ -484,6 +496,26 @@ Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
return Error::success();
}
+uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
+ finalizeSymtab();
+ // printf("look up key 0x%llx\n", Address);
+ // for (auto iter = VTableAddrToMD5Map.begin(); iter !=
+ // VTableAddrToMD5Map.end(); iter++) {
+ // printf("<key, val> is <0x%llx, %"PRIu64"\n", iter->first, iter->second);
+ // }
+ auto It =
+ partition_point(VTableAddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
+ return A.first < Address;
+ });
+ // FIXME: Does the raw function pointers point apply here?
+ if (It != VTableAddrToMD5Map.end()) {
+ // printf("InstrProfSymtab::getVTableHashFromAddress map addr 0x%llx to hash
+ // value %"PRIu64"\n", Address, (uint64_t)It->second);
+ return (uint64_t)It->second;
+ }
+ return 0;
+}
+
uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
finalizeSymtab();
auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
@@ -560,42 +592,112 @@ Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
NameStrs, compression::zlib::isAvailable() && doCompression, Result);
}
+Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
+ std::string &Result, bool doCompression) {
+ std::vector<std::string> VTableNameStrs;
+ for (auto *VTable : VTables) {
+ // printf("VTable name %s added\n", VTable->getName().str().c_str());
+ VTableNameStrs.push_back(std::string(VTable->getName()));
+ }
+ return collectPGOFuncNameStrings(
+ VTableNameStrs, compression::zlib::isAvailable() && doCompression,
+ Result);
+}
+
+instrprof_error decodeAndSplitStrings(
+ const uint8_t *Input, SmallVector<uint8_t, 128> &UncompressedNameStrings,
+ StringRef &NameStrings, uint32_t &Dist, bool &isCompressed) {
+ Dist = 0;
+ const uint8_t *Start = Input;
+ uint32_t UncompressedSizeLen = 0;
+ uint64_t UncompressedSize = decodeULEB128(Start, &UncompressedSizeLen);
+ Start += UncompressedSizeLen;
+ Dist += UncompressedSizeLen;
+ uint32_t CompressedSizeLen = 0;
+ uint64_t CompressedSize = decodeULEB128(Start, &CompressedSizeLen);
+ Start += CompressedSizeLen;
+ Dist += CompressedSizeLen;
+ isCompressed = (CompressedSize != 0);
+ if (isCompressed) {
+ if (!llvm::compression::zlib::isAvailable())
+ return instrprof_error::zlib_unavailable;
+
+ if (Error E = compression::zlib::decompress(ArrayRef(Start, CompressedSize),
+ UncompressedNameStrings,
+ UncompressedSize)) {
+ consumeError(std::move(E));
+ return instrprof_error::uncompress_failed;
+ }
+ Dist += CompressedSize;
+ } else {
+ NameStrings =
+ StringRef(reinterpret_cast<const char *>(Start), UncompressedSize);
+ Dist += UncompressedSize;
+ }
+
+ return instrprof_error::success;
+}
+
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
const uint8_t *P = NameStrings.bytes_begin();
const uint8_t *EndP = NameStrings.bytes_end();
while (P < EndP) {
- uint32_t N;
- uint64_t UncompressedSize = decodeULEB128(P, &N);
- P += N;
- uint64_t CompressedSize = decodeULEB128(P, &N);
- P += N;
- bool isCompressed = (CompressedSize != 0);
- SmallVector<uint8_t, 128> UncompressedNameStrings;
+ // Now parse the name strings.
+ uint32_t Dist = 0;
StringRef NameStrings;
+ SmallVector<uint8_t, 128> UncompressedNameStrings;
+ SmallVector<StringRef, 0> Names;
+ bool isCompressed = false;
+ instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
+ NameStrings, Dist, isCompressed);
+ if (E != instrprof_error::success)
+ return make_error<InstrProfError>(E);
+
if (isCompressed) {
- if (!llvm::compression::zlib::isAvailable())
- return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
-
- if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
- UncompressedNameStrings,
- UncompressedSize)) {
- consumeError(std::move(E));
- return make_error<InstrProfError>(instrprof_error::uncompress_failed);
- }
- P += CompressedSize;
NameStrings = toStringRef(UncompressedNameStrings);
- } else {
- NameStrings =
- StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
- P += UncompressedSize;
}
+
+ NameStrings.split(Names, getInstrProfNameSeparator());
+ for (StringRef &Name : Names) {
+ if (Error E = Symtab.addFuncName(Name))
+ return E;
+ }
+
+ P += Dist;
+ // Skip padding?
+ while (P < EndP && *P == 0)
+ P++;
+ }
+ return Error::success();
+}
+
+Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab) {
+ const uint8_t *P = NameStrings.bytes_begin();
+ const uint8_t *EndP = NameStrings.bytes_end();
+ while (P < EndP) {
// Now parse the name strings.
+ uint32_t Dist = 0;
+ StringRef NameStrings;
+ SmallVector<uint8_t, 128> UncompressedNameStrings;
SmallVector<StringRef, 0> Names;
+ bool isCompressed = false;
+ instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
+ NameStrings, Dist, isCompressed);
+ if (E != instrprof_error::success)
+ return make_error<InstrProfError>(E);
+
+ if (isCompressed) {
+ NameStrings = toStringRef(UncompressedNameStrings);
+ }
NameStrings.split(Names, getInstrProfNameSeparator());
- for (StringRef &Name : Names)
- if (Error E = Symtab.addFuncName(Name))
+ for (StringRef &Name : Names) {
+ // printf("Read back vtable name %s\n", Name.str().c_str());
+ if (Error E = Symtab.addVTableName(Name))
return E;
+ }
+ P += Dist;
+ // Skip padding?
while (P < EndP && *P == 0)
P++;
}
@@ -852,6 +954,11 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
if (ValueKind == IPVK_IndirectCallTarget)
return SymTab->getFunctionHashFromAddress(Value);
+ if (ValueKind == IPVK_VTableTarget) {
+ uint64_t VTableHash = SymTab->getVTableHashFromAddress(Value);
+ return VTableHash;
+ }
+
return Value;
}
@@ -1242,8 +1349,8 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
F.setMetadata(getPGOFuncNameMetadataName(), N);
}
-bool needsComdatForCounter(const Function &F, const Module &M) {
- if (F.hasComdat())
+bool needsComdatForCounter(const GlobalValue &GV, const Module &M) {
+ if (GV.hasComdat())
return true;
if (!Triple(M.getTargetTriple()).supportsCOMDAT())
@@ -1259,7 +1366,7 @@ bool needsComdatForCounter(const Function &F, const Module &M) {
// available_externally functions will end up being duplicated in raw profile
// data. This can result in distorted profile as the counts of those dups
// will be accumulated by the profile merger.
- GlobalValue::LinkageTypes Linkage = F.getLinkage();
+ GlobalValue::LinkageTypes Linkage = GV.getLinkage();
if (Linkage != GlobalValue::ExternalWeakLinkage &&
Linkage != GlobalValue::AvailableExternallyLinkage)
return false;
@@ -1423,6 +1530,9 @@ void OverlapStats::dump(raw_fd_ostream &OS) const {
case IPVK_MemOPSize:
strncpy(ProfileKindName, "MemOP", 19);
break;
+ case IPVK_VTableTarget:
+ strncpy(ProfileKindName, "VTable", 19);
+ break;
default:
snprintf(ProfileKindName, 19, "VP[%d]", I);
break;
@@ -1487,9 +1597,12 @@ Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
// When a new field is added in the header add a case statement here to
// populate it.
static_assert(
- IndexedInstrProf::ProfVersion::CurrentVersion == Version10,
+ IndexedInstrProf::ProfVersion::CurrentVersion == Version11,
"Please update the reading code below if a new field has been added, "
"if not add a case statement to fall through to the latest version.");
+ case 11ull:
+ H.VTableNamesOffset = read(Buffer, offsetOf(&Header::VTableNamesOffset));
+ [[fallthrough]];
case 10ull:
H.TemporalProfTracesOffset =
read(Buffer, offsetOf(&Header::TemporalProfTracesOffset));
@@ -1513,10 +1626,13 @@ size_t Header::size() const {
// When a new field is added to the header add a case statement here to
// compute the size as offset of the new field + size of the new field. This
// relies on the field being added to the end of the list.
- static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version10,
+ static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11,
"Please update the size computation below if a new field has "
"been added to the header, if not add a case statement to "
"fall through to the latest version.");
+ case 11ull:
+ return offsetOf(&Header::VTableNamesOffset) +
+ sizeof(Header::VTableNamesOffset);
case 10ull:
return offsetOf(&Header::TemporalProfTracesOffset) +
sizeof(Header::TemporalProfTracesOffset);
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index a920a31d0a4b2..59e7ef9466863 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -373,6 +373,14 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
return E;
Value = IndexedInstrProf::ComputeHash(VD.first);
}
+ } else if (ValueKind == IPVK_VTableTarget) {
+ if (InstrProfSymtab::isExternalSymbol(VD.first)) {
+ Value = 0;
+ } else {
+ if (Error E = Symtab->addVTableName(VD.first))
+ return E;
+ Value = IndexedInstrProf::ComputeHash(VD.first);
+ }
} else {
READ_NUM(VD.first, Value);
}
@@ -517,14 +525,31 @@ Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
template <class IntPtrT>
Error RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {
- if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart)))
+ if (Error E =
+ Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart),
+ StringRef(VNamesStart, VNamesEnd - VNamesStart))) {
return error(std::move(E));
+ }
for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
const IntPtrT FPtr = swap(I->FunctionPointer);
if (!FPtr)
continue;
Symtab.mapAddress(FPtr, I->NameRef);
}
+
+ if (VTableBegin != nullptr && VTableEnd != nullptr) {
+ for (const RawInstrProf::VTableProfileData<IntPtrT> *I = VTableBegin;
+ I != VTableEnd; ++I) {
+ const IntPtrT VPtr = I->VTablePointer;
+ if (!VPtr)
+ continue;
+ // Map both begin and end address to the name hash, since the instrumented
+ // address could be somewhere in the middle.
+ // VPtr is of type uint32_t or uint64_t so 'VPtr + I->VTableSize' marks
+ // the end of vtable address.
+ Symtab.mapVTableAddress(VPtr, VPtr + I->VTableSize, I->VTableNameHash);
+ }
+ }
return success();
}
@@ -556,17 +581,29 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize();
auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters);
auto NamesSize = swap(Header.NamesSize);
+ auto VTableNameSize = Header.VNamesSize;
+ auto NumVTables = Header.NumVTables;
ValueKindLast = swap(Header.ValueKindLast);
auto DataSize = NumData * sizeof(RawInstrProf::ProfileData<IntPtrT>);
- auto PaddingSize = getNumPaddingBytes(NamesSize);
+ auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);
+ auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);
+
+ auto VTableSectionSize =
+ Header.NumVTables * sizeof(RawInstrProf::VTableProfileData<IntPtrT>);
+ auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);
// Profile data starts after profile header and binary ids if exist.
ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize;
ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
ptrdiff_t NamesOffset =
CountersOffset + CountersSize + PaddingBytesAfterCounters;
- ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
+ ptrdiff_t VTableProfDataOffset =
+ NamesOffset + NamesSize + PaddingBytesAfterNames;
+ ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
+ PaddingBytesAfterVTableProfData;
+ ptrdiff_t ValueDataOffset =
+ VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;
auto *Start = reinterpret_cast<const char *>(&Header);
if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
@@ -585,8 +622,14 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
Start + DataOffset);
DataEnd = Data + NumData;
+ VTableBegin =
+ reinterpret_cast<const RawInstrProf::VTableProfileData<IntPtrT> *>(
+ Start + VTableProfDataOffset);
+ VTableEnd = VTableBegin + NumVTables;
NamesStart = Start + NamesOffset;
NamesEnd = NamesStart + NamesSize;
+ VNamesStart = Start + VTableNameOffset;
+ VNamesEnd = VNamesStart + VTableNameSize;
}
// Binary ids start just after the header.
@@ -835,6 +878,19 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
return DataBuffer;
}
+InstrProfVTableLookupTrait::hash_value_type
+InstrProfVTableLookupTrait::ComputeHash(StringRef K) {
+ return IndexedInstrProf::ComputeHash(HashType, K);
+}
+
+InstrProfVTableLookupTrait::data_type
+InstrProfVTableLookupTrait::ReadData(StringRef K, const unsigned char *D,
+ offset_type N) {
+ char v =
+ support::endian::readNext<char, support::little, support::unaligned>(D);
+ return v;
+}
+
template <typename HashTableImpl>
Error InstrProfReaderIndex<HashTableImpl>::getRecords(
StringRef FuncName, ArrayRef<NamedInstrProfRecord> &Data) {
@@ -1162,6 +1218,21 @@ Error IndexedInstrProfReader::readHeader() {
"corrupted binary ids");
}
+ if (GET_VERSION(Header->formatVersion()) >= 11) {
+ uint64_t VTableNamesOffset =
+ endian::byte_swap<uint64_t, little>(Header->VTableNamesOffset);
+ const unsigned char *Ptr = Start + VTableNamesOffset;
+
+ const uint64_t HashTableMetadataOffset =
+ support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+
+ VirtualTableIndex.reset(VirtualTableNamesHashTable::Create(
+ /*Bucket=*/Start + HashTableMetadataOffset,
+ /*Payload=*/Ptr,
+ /*Base=*/Start,
+ InstrProfVTableLookupTrait(HashType, Header->formatVersion())));
+ }
+
if (GET_VERSION(Header->formatVersion()) >= 10 &&
Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) {
uint64_t TemporalProfTracesOffset =
@@ -1222,7 +1293,7 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
return *Symtab;
std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
- if (Error E = Index->populateSymtab(*NewSymtab)) {
+ if (Error E = Index->populateSymtab(*NewSymtab, VirtualTableIndex.get())) {
auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
consumeError(error(ErrCode, Msg));
}
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 6892654b00ea4..0094cf8c029fc 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -171,6 +171,47 @@ class InstrProfRecordWriterTrait {
}
};
+class InstrProfRecordVTableTrait {
+public:
+ using key_type = StringRef;
+ using key_type_ref = StringRef;
+
+ using data_type = char;
+ using data_type_ref = char;
+
+ using hash_value_type = uint64_t;
+ using offset_type = uint64_t;
+
+ InstrProfRecordVTableTrait() = default;
+
+ static hash_value_type ComputeHash(key_type_ref K) {
+ return IndexedInstrProf::ComputeHash(K);
+ }
+
+ static std::pair<offset_type, offset_type>
+ EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
+ using namespace support;
+
+ endian::Writer LE(Out, little);
+
+ offset_type N = K.size();
+ LE.write<offset_type>(N);
+
+ offset_type M = 1;
+ LE.write<offset_type>(M);
+
+ return std::make_pair(N, M);
+ }
+
+ void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
+ Out.write(K.data(), N);
+ }
+
+ void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
+ Out.write(&V, 1);
+ }
+};
+
} // end namespace llvm
InstrProfWriter::InstrProfWriter(bool Sparse,
@@ -447,12 +488,13 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
Header.MemProfOffset = 0;
Header.BinaryIdOffset = 0;
Header.TemporalProfTracesOffset = 0;
+ Header.VTableNamesOffset = 0;
int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
// Only write out all the fields except 'HashOffset', 'MemProfOffset',
- // 'BinaryIdOffset' and `TemporalProfTracesOffset`. We need to remember the
- // offset of these fields to allow back patching later.
- for (int I = 0; I < N - 4; I++)
+ // 'BinaryIdOffset', `TemporalProfTracesOffset` and `VTableNamesOffset`. We
+ // need to remember the offset of these fields to allow back patching later.
+ for (int I = 0; I < N - 5; I++)
OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
// Save the location of Header.HashOffset field in \c OS.
@@ -476,6 +518,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
uint64_t TemporalProfTracesOffset = OS.tell();
OS.write(0);
+ uint64_t VTableNamesOffset = OS.tell();
+ OS.write(0);
+
// Reserve space to write profile summary data.
uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
@@ -589,6 +634,36 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
OS.writeByte(0);
}
+ // if version >= the version with vtable profile metadata
+ // Intentionally put vtable names before temporal profile section.
+ uint64_t VTableNamesSectionStart = 0;
+ if (IndexedInstrProf::ProfVersion::CurrentVersion >= 11) {
+ VTableNamesSectionStart = OS.tell();
+
+ // Reserve space for vtable record table offset.
+ OS.write(0ULL);
+
+ OnDiskChainedHashTableGenerator<llvm::InstrProfRecordVTableTrait>
+ VTableNamesGenerator;
+ for (const auto &kv : VTableNames) {
+ // printf("InstrProfWriter.cpp key is %s\n", kv.getKey().str().c_str());
+ VTableNamesGenerator.insert(kv.getKey(), '0');
+ }
+
+ auto VTableNamesWriter =
+ std::make_unique<llvm::InstrProfRecordVTableTrait>();
+
+ uint64_t VTableNamesTableOffset =
+ VTableNamesGenerator.Emit(OS.OS, *VTableNamesWriter);
+
+ // printf("InstrProfWriter.cpp:VTableNamesSectionStart is %"PRIu64"\n",
+ // VTableNamesSectionStart); printf("\tVTableNamesTableOffset is
+ // %"PRIu64"\n", VTableNamesTableOffset);
+ PatchItem PatchItems[] = {
+ {VTableNamesSectionStart, &VTableNamesTableOffset, 1}};
+ OS.patch(PatchItems, 1);
+ }
+
uint64_t TemporalProfTracesSectionStart = 0;
if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
TemporalProfTracesSectionStart = OS.tell();
@@ -632,6 +707,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
// Patch the Header.TemporalProfTracesOffset (=0 for profiles without
// traces).
{TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
+ {VTableNamesOffset, &VTableNamesSectionStart, 1},
// Patch the summary data.
{SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
(int)(SummarySize / sizeof(uint64_t))},
@@ -684,7 +760,8 @@ Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
DenseSet<uint64_t> SeenValues;
for (uint32_t I = 0; I < ND; I++)
- if ((VK != IPVK_IndirectCallTarget) && !SeenValues.insert(VD[I].Value).second)
+ if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) &&
+ !SeenValues.insert(VD[I].Value).second)
return make_error<InstrProfError>(instrprof_error::invalid_prof);
}
}
@@ -721,7 +798,7 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
OS << ND << "\n";
std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
for (uint32_t I = 0; I < ND; I++) {
- if (VK == IPVK_IndirectCallTarget)
+ if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
OS << Symtab.getFuncOrVarNameIfDefined(VD[I].Value) << ":"
<< VD[I].Count << "\n";
else
@@ -758,6 +835,11 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
}
}
+ for (const auto &VTableName : VTableNames) {
+ if (Error E = Symtab.addVTableName(VTableName.getKey()))
+ return E;
+ }
+
if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
writeTextTemporalProfTraceData(OS, Symtab);
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 57fcfd5383691..844b5397574b5 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -561,6 +561,13 @@ bool InstrProfiling::run(
static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
}
+ for (GlobalVariable &GV : M.globals()) {
+ // Global variables with type metadata are virtual table variables.
+ if (GV.hasMetadata(LLVMContext::MD_type)) {
+ getOrCreateVTableProfData(&GV);
+ }
+ }
+
for (Function &F : M)
MadeChange |= lowerIntrinsics(&F);
@@ -574,6 +581,7 @@ bool InstrProfiling::run(
emitVNodes();
emitNameData();
+ emitVTableNames();
// Emit runtime hook for the cases where the target does not unconditionally
// require pulling in profile runtime, and coverage is enabled on code that is
@@ -951,6 +959,135 @@ InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
return GV;
}
+static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
+ if (!profDataReferencedByCode(*GV->getParent()))
+ return false;
+
+ if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
+ !GV->hasAvailableExternallyLinkage())
+ return true;
+
+ // This avoids the profile data from referencing internal symbols in
+ // COMDAT.
+ if (GV->hasLocalLinkage() && GV->hasComdat())
+ return false;
+
+ return true;
+}
+
+// FIXME: Does symbollic relocation from 'getFuncAddrForProfData' matter here?
+static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
+ // Store a nullptr in __profvt_ if a real address shouldn't be used.
+ auto *Int8PtrTy = Type::getInt8PtrTy(GV->getContext());
+
+ if (!shouldRecordVTableAddr(GV))
+ return ConstantPointerNull::get(Int8PtrTy);
+
+ return ConstantExpr::getBitCast(GV, Int8PtrTy);
+}
+
+/// Get the name of a profiling variable for a particular variable.
+static std::string getVarName(GlobalVariable *GV, StringRef Prefix) {
+ StringRef Name = GV->getName();
+ return (Prefix + Name).str();
+}
+
+void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
+ assert(!DebugInfoCorrelate &&
+ "Value profiling is not supported with lightweight instrumentation");
+ if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
+ return;
+
+ if (GV->getName().starts_with("llvm.") ||
+ GV->getName().starts_with("__llvm") ||
+ GV->getName().starts_with("__prof"))
+ return;
+
+ // VTableProfData already created
+ auto It = VTableDataMap.find(GV);
+ if (It != VTableDataMap.end() && It->second)
+ return;
+
+ GlobalValue::LinkageTypes Linkage = GV->getLinkage();
+ GlobalValue::VisibilityTypes Visibility = GV->getVisibility();
+
+ // This is to keep consistent with per-function profile data
+ // for correctness.
+ if (TT.isOSBinFormatXCOFF()) {
+ Linkage = GlobalValue::InternalLinkage;
+ Visibility = GlobalValue::DefaultVisibility;
+ }
+
+ LLVMContext &Ctx = M->getContext();
+ Type *DataTypes[] = {
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
+#include "llvm/ProfileData/InstrProfData.inc"
+ };
+
+ auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
+
+ // Used by INSTR_PROF_VTABLE_DATA MACRO
+ Constant *VTableAddr = getVTableAddrForProfData(GV);
+ StringRef VTableName = GV->getName();
+ // Record the length of the vtable. This is needed since vtable pointers
+ // loaded from C++ objects might be from the middle of a vtable definition.
+ uint32_t VTableSizeVal =
+ M->getDataLayout().getTypeAllocSize(GV->getValueType());
+
+ Constant *DataVals[] = {
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
+#include "llvm/ProfileData/InstrProfData.inc"
+ };
+
+ auto *Data =
+ new GlobalVariable(*M, DataTy, false /* constant */, Linkage,
+ ConstantStruct::get(DataTy, DataVals),
+ getVarName(GV, getInstrProfVTableVarPrefix()));
+
+ Data->setVisibility(Visibility);
+ Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat()));
+ Data->setAlignment(Align(8));
+
+ const bool NeedComdat = needsComdatForCounter(*GV, *M);
+
+ // GV is the data structure to record vtable information.
+ // Place the global variable for per-vtable profile data in a comdat group
+ // if the associated vtable definition is a COMDAT. This makes sure only one
+ // copy of the variable for the vtable will be emitted after linking.
+ auto MaybeSetComdat = [&](GlobalVariable *GV, StringRef GroupName) {
+ bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
+ if (UseComdat) {
+ // Create a new comdat group using the name of the global variable as
+ // opposed to using the comdat group of the vtable.
+ Comdat *C = M->getOrInsertComdat(GroupName);
+ // For ELF, when not using COMDAT, put the vtable profile data into a
+ // nodeduplicate COMDAT which is lowered to a zero-flag zero group.
+ // This allows -z -start-top-gc to discard the entire group when the
+ // vtable def is discarded.
+ if (!NeedComdat)
+ C->setSelectionKind(Comdat::NoDeduplicate);
+ GV->setComdat(C);
+ // COFF doesn't allow the comdat group leader to have private linkage, so
+ // upgrade private linkage to internal linkage to produce a symbol table
+ // entry.
+ if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) {
+ GV->setLinkage(GlobalValue::InternalLinkage);
+ }
+ return;
+ }
+ };
+
+ MaybeSetComdat(Data, Data->getName());
+
+ VTableDataMap[GV] = Data;
+
+ ReferencedVTableNames.push_back(GV);
+
+ // VTable <Hash, Addr> is used by runtime but not referenced by other
+ // sections. Conservatively mark it linker retained.
+ UsedVars.push_back(Data);
+}
+
GlobalVariable *
InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
GlobalVariable *NamePtr = Inc->getName();
@@ -1238,6 +1375,34 @@ void InstrProfiling::emitNameData() {
NamePtr->eraseFromParent();
}
+void InstrProfiling::emitVTableNames() {
+ if (ReferencedVTableNames.empty())
+ return;
+
+ // Collect VTable
+ std::string CompressedVTableNames;
+ if (Error E =
+ collectVTableStrings(ReferencedVTableNames, CompressedVTableNames,
+ DoInstrProfNameCompression)) {
+ report_fatal_error(Twine(toString(std::move(E))), false);
+ }
+
+ auto &Ctx = M->getContext();
+ auto *VTableNamesVal = ConstantDataArray::getString(
+ Ctx, StringRef(CompressedVTableNames), false /* AddNull */);
+ GlobalVariable *VTableNamesVar =
+ new GlobalVariable(*M, VTableNamesVal->getType(), true /* constant */,
+ GlobalValue::PrivateLinkage, VTableNamesVal,
+ getInstrProfVTableNamesVarName());
+ VTableNamesVar->setSection(
+ getInstrProfSectionName(IPSK_vname, TT.getObjectFormat()));
+ VTableNamesVar->setAlignment(Align(1));
+ // Make VTableNames linker retained.
+ UsedVars.push_back(VTableNamesVar);
+
+ // FIXME: Why emitNames call erase method?
+}
+
void InstrProfiling::emitRegistration() {
if (!needsRuntimeRegistrationOfSectionRange(TT))
return;
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 7ad1c9bc54f37..75c57d4d559e5 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -584,6 +584,7 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
NumOfPGOBB += MST.BBInfos.size();
ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget);
+ ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
} else {
NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
index 3a129de1acd02..59d51a59c48b7 100644
--- a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
+++ b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
@@ -90,9 +90,33 @@ public:
}
};
+///------------------------ VirtualTableValueProfilingPlugin ------------------------
+class VTableProfilingPlugin {
+ Function &F;
+
+public:
+ static constexpr InstrProfValueKind Kind = IPVK_VTableTarget;
+
+ VTableProfilingPlugin(Function &Fn, TargetLibraryInfo &TLI) : F(Fn) {}
+
+ void run(std::vector<CandidateInfo> &Candidates) {
+ std::vector<Instruction *> Result = findVTableAddrs(F);
+ for (Instruction* I : Result) {
+ Instruction* InsertPt = I->getNextNonDebugInstruction();
+ while(InsertPt && dyn_cast<PHINode>(InsertPt))
+ InsertPt = InsertPt->getNextNonDebugInstruction();
+ assert(InsertPt);
+
+ Instruction *AnnotatedInst = I;
+ Candidates.emplace_back(CandidateInfo{I, InsertPt, AnnotatedInst});
+ }
+ }
+};
+
///----------------------- Registration of the plugins -------------------------
/// For now, registering a plugin with the ValueProfileCollector is done by
/// adding the plugin type to the VP_PLUGIN_LIST macro.
#define VP_PLUGIN_LIST \
MemIntrinsicPlugin, \
- IndirectCallPromotionPlugin
+ IndirectCallPromotionPlugin, \
+ VTableProfilingPlugin
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh b/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
new file mode 100644
index 0000000000000..1b35ea0303d15
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+if [ $# -lt 2 ]; then
+ echo "Path to clang++ and llvm-profdata required!"
+ echo "Usage: update_vtable_value_prof_inputs.sh /path/to/updated/clang++ /path/to/updated/llvm-profdata"
+ exit 1
+else
+ CLANG=$1
+ LLVMPROFDATA=$2
+fi
+
+OUTDIR=$(dirname $(realpath -s $0))
+
+echo "Outdir is $OUTDIR"
+
+cat > ${OUTDIR}/vtable_prof.cc << EOF
+#include <cstdlib>
+#include <cstdio>
+
+class Base {
+ public:
+ virtual int func1(int a, int b) = 0;
+ virtual int func2(int a, int b) = 0;
+};
+
+class Derived1 : public Base {
+ public:
+ __attribute__((noinline))
+ int func1(int a, int b) override
+ {
+ return a + b;
+ }
+
+ __attribute__((noinline))
+ int func2(int a, int b) override {
+ return a * b;
+ }
+};
+
+class Derived2 : public Base {
+ public:
+
+ __attribute__((noinline))
+ int func1(int a, int b) override {
+ return a - b;
+ }
+
+ __attribute__((noinline))
+ int func2(int a, int b) override {
+ return a * (a - b);
+ }
+};
+
+__attribute__((noinline)) Base* createType(int a) {
+ Base* base = nullptr;
+ if (a % 4 == 0)
+ base = new Derived1();
+ else
+ base = new Derived2();
+ return base;
+}
+
+
+int main(int argc, char** argv) {
+ int sum = 0;
+ for (int i = 0; i < 1000; i++) {
+ int a = rand();
+ int b = rand();
+ Base* ptr = createType(i);
+ sum += ptr->func1(a, b) + ptr->func2(b, a);
+ }
+ printf("sum is %d\n", sum);
+ return 0;
+}
+EOF
+
+FLAGS="-fuse-ld=lld -O2 -g -fprofile-generate=. -flto=thin -Xclang -fwhole-program-vtables -Wl,--lto-whole-program-visibility"
+
+${CLANG} ${FLAGS} ${OUTDIR}/vtable_prof.cc -o ${OUTDIR}/vtable_prof
+env LLVM_PROFILE_FILE=${OUTDIR}/vtable_prof.profraw ${OUTDIR}/vtable_prof
+
+rm ${OUTDIR}/vtable_prof
+rm ${OUTDIR}/vtable_prof.cc
+
diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
index 1c44a274f3c04..990b246ce07a8 100644
--- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll
+++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
@@ -13,9 +13,9 @@ $foo = comdat any
; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat
; CHECK-NOT: __profn__stdin__foo
; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
-; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [2 x i16] } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null
+; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [3 x i16] } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null
; CHECK-NOT: @foo
-; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8
+; CHECK-SAME: , ptr null, i32 1, [3 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8
; CHECK: @__llvm_prf_nm
; CHECK: @llvm.compiler.used
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
index 42018e4d5e54d..63173ab59b28b 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
@@ -54,11 +54,12 @@ try.cont: ; preds = %catch, %entry
}
; GEN: catch:
-; GEN: call void @llvm.instrprof.value.profile(
+; GEN: call void @llvm.instrprof.value.profile({{.*}} i32 2, i32 0)
+; GEN: call void @llvm.instrprof.value.profile({{.*}} i32 0, i32 0)
; GEN-SAME: [ "funclet"(token %tmp1) ]
; LOWER: catch:
-; LOWER: call void @__llvm_profile_instrument_target(
+; LOWER: call void @__llvm_profile_instrument_target({{.*}} i32 0)
; LOWER-SAME: [ "funclet"(token %tmp1) ]
declare dso_local void @"?may_throw@@YAXH at Z"(i32)
diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
new file mode 100644
index 0000000000000..0c6bd3c064d35
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
@@ -0,0 +1,139 @@
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$_ZTV7Derived = comdat any
+
+ at _ZTV7Derived = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
+ at _ZTVN10__cxxabiv121__vmi_class_type_infoE = external global [0 x ptr]
+ at _ZTS7Derived = constant [9 x i8] c"7Derived\00", align 1
+ at _ZTI5Base1 = external constant ptr
+ at _ZTI5Base2 = external constant ptr
+ at _ZTI7Derived = constant { ptr, ptr, i32, i32, ptr, i64, ptr, i64 } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), ptr @_ZTS7Derived, i32 0, i32 2, ptr @_ZTI5Base1, i64 2, ptr @_ZTI5Base2, i64 2050 }, align 8
+ at _ZTV5Base1 = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base1, ptr @_ZN5Base15func1Eii] }, align 8, !type !0, !type !1
+ at _ZTV5Base2 = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base2, ptr @_ZN5Base25func2Eii] }, align 8, !type !9, !type !4
+ at llvm.compiler.used = appending global [2 x ptr] [ptr @_ZTV5Base1, ptr @_ZTV5Base2], section "llvm.metadata"
+
+declare ptr @_Z10createTypei(i32)
+declare i32 @_ZN5Base15func1Eii(ptr, i32, i32)
+declare i32 @_ZN5Base25func2Eii(ptr, i32, i32)
+
+; GEN: @__llvm_profile_raw_version = hidden constant i64 72057594037927945, comdat
+; GEN: @__profn_test_vtable_value_profiling = private constant [27 x i8] c"test_vtable_value_profiling"
+
+; LOWER: $__profvt__ZTV7Derived = comdat any
+; LOWER: $__profvt__ZTV5Base1 = comdat nodeduplicate
+; LOWER: $__profvt__ZTV5Base2 = comdat nodeduplicate
+; LOWER: @__llvm_profile_raw_version = hidden constant i64 72057594037927945, comdat
+; LOWER: @__profc_test_vtable_value_profiling = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
+; LOWER: @__profvp_test_vtable_value_profiling = private global [4 x i64] zeroinitializer, section "__llvm_prf_vals", comdat($__profc_test_vtable_value_profiling), align 8
+; LOWER: @__profd_test_vtable_value_profiling = private global { i64, i64, i64, ptr, ptr, i32, [3 x i16] } { i64 1593873508557585901, i64 567090795815895039, i64 sub (i64 ptrtoint (ptr @__profc_test_vtable_value_profiling to i64), i64 ptrtoint (ptr @__profd_test_vtable_value_profiling to i64)), ptr @test_vtable_value_profiling.local, ptr @__profvp_test_vtable_value_profiling, i32 1, [3 x i16] [i16 2, i16 0, i16 2] }, section "__llvm_prf_data", comdat($__profc_test_vtable_value_profiling), align 8
+; LOWER: @__profvt__ZTV7Derived = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @__profvt__ZTV5Base1 = global { i64, ptr, i32 } { i64 3215870116411581797, ptr @_ZTV5Base1, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @__profvt__ZTV5Base2 = global { i64, ptr, i32 } { i64 8378219803387680050, ptr @_ZTV5Base2, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @__llvm_prf_vnodes = private global [10 x { i64, i64, ptr }] zeroinitializer, section "__llvm_prf_vnds", align 8
+; LOWER: @__llvm_prf_nm = private constant [37 x i8] c"\1B#x\DA+I-.\89/+IL\CAI\8D/K\CC)M\8D/(\CAO\CB\CC\C9\CCK\07\00\9Ea\0BC", section "__llvm_prf_names", align 1
+; LOWER: @__llvm_prf_vnm = private constant [34 x i8] c"\22 x\DA\8B\8F\0A\093wI-\CA,KMa\8C\07rL\9D\12\8BS\0D\11L#\00\C3\A2\0A\E9", section "__llvm_prf_vnames", align 1
+; LOWER: @llvm.used = appending global [6 x ptr] [ptr @__profvt__ZTV7Derived, ptr @__profvt__ZTV5Base1, ptr @__profvt__ZTV5Base2, ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata"
+
+define i32 @test_vtable_value_profiling(i32 %a, i32 %b, i32 %c) {
+; GEN-LABEL: define i32 @test_vtable_value_profiling(
+; GEN-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) {
+; GEN-NEXT: entry:
+; GEN-NEXT: call void @llvm.instrprof.increment(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i32 1, i32 0)
+; GEN-NEXT: [[CALL:%.*]] = tail call ptr @_Z10createTypei(i32 [[C]])
+; GEN-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 8
+; GEN-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
+; GEN-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[VTABLE]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP0]], i32 2, i32 0)
+; GEN-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
+; GEN-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[VFUNC]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP1]], i32 0, i32 0)
+; GEN-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
+; GEN-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
+; GEN-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP2]], i32 2, i32 1)
+; GEN-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
+; GEN-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP3]], i32 0, i32 1)
+; GEN-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
+; GEN-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
+; GEN-NEXT: ret i32 [[ADD]]
+;
+; LOWER-LABEL: define i32 @test_vtable_value_profiling(
+; LOWER-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) {
+; LOWER-NEXT: entry:
+; LOWER-NEXT: [[PGOCOUNT:%.*]] = load i64, ptr @__profc_test_vtable_value_profiling, align 8
+; LOWER-NEXT: [[TMP0:%.*]] = add i64 [[PGOCOUNT]], 1
+; LOWER-NEXT: store i64 [[TMP0]], ptr @__profc_test_vtable_value_profiling, align 8
+; LOWER-NEXT: [[CALL:%.*]] = tail call ptr @_Z10createTypei(i32 [[C]])
+; LOWER-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 8
+; LOWER-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
+; LOWER-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[VTABLE]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP1]], ptr @__profd_test_vtable_value_profiling, i32 2)
+; LOWER-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
+; LOWER-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VFUNC]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP2]], ptr @__profd_test_vtable_value_profiling, i32 0)
+; LOWER-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
+; LOWER-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
+; LOWER-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP3]], ptr @__profd_test_vtable_value_profiling, i32 3)
+; LOWER-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
+; LOWER-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP4]], ptr @__profd_test_vtable_value_profiling, i32 1)
+; LOWER-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
+; LOWER-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
+; LOWER-NEXT: ret i32 [[ADD]]
+;
+entry:
+ %call = tail call ptr @_Z10createTypei(i32 %c)
+ %add.ptr = getelementptr inbounds i8, ptr %call, i64 8
+ %vtable = load ptr, ptr %add.ptr, align 8
+ %vfunc = load ptr, ptr %vtable, align 8
+ %call1 = tail call i32 %vfunc(ptr %add.ptr, i32 %a, i32 %b)
+ %vtable2 = load ptr, ptr %call, align 8
+ %vfunc2 = load ptr, ptr %vtable2, align 8
+ %call4 = tail call i32 %vfunc2(ptr %call, i32 %b, i32 %a)
+ %add = add nsw i32 %call4, %call1
+ ret i32 %add
+}
+
+!0 = !{i64 16, !"_ZTS5Base1"}
+!1 = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
+!2 = !{i64 40, !"_ZTSM5Base1FiiiE.virtual"}
+!3 = !{i64 40, !"_ZTS5Base2"}
+!4 = !{i64 16, !"_ZTSM5Base2FiiiE.virtual"}
+!5 = !{i64 40, !"_ZTSM5Base2FiiiE.virtual"}
+!6 = !{i64 16, !"_ZTS7Derived"}
+!7 = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"}
+!8 = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
+!9 = !{i64 16, !"_ZTS5Base2"}
+;.
+; GEN: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+;.
+; LOWER: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+;.
+; GEN: [[META0:![0-9]+]] = !{i64 16, !"_ZTS5Base1"}
+; GEN: [[META1:![0-9]+]] = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
+; GEN: [[META2:![0-9]+]] = !{i64 40, !"_ZTSM5Base1FiiiE.virtual"}
+; GEN: [[META3:![0-9]+]] = !{i64 40, !"_ZTS5Base2"}
+; GEN: [[META4:![0-9]+]] = !{i64 16, !"_ZTSM5Base2FiiiE.virtual"}
+; GEN: [[META5:![0-9]+]] = !{i64 40, !"_ZTSM5Base2FiiiE.virtual"}
+; GEN: [[META6:![0-9]+]] = !{i64 16, !"_ZTS7Derived"}
+; GEN: [[META7:![0-9]+]] = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"}
+; GEN: [[META8:![0-9]+]] = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
+; GEN: [[META9:![0-9]+]] = !{i64 16, !"_ZTS5Base2"}
+;.
+; LOWER: [[META0:![0-9]+]] = !{i64 16, !"_ZTS5Base1"}
+; LOWER: [[META1:![0-9]+]] = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
+; LOWER: [[META2:![0-9]+]] = !{i64 40, !"_ZTSM5Base1FiiiE.virtual"}
+; LOWER: [[META3:![0-9]+]] = !{i64 40, !"_ZTS5Base2"}
+; LOWER: [[META4:![0-9]+]] = !{i64 16, !"_ZTSM5Base2FiiiE.virtual"}
+; LOWER: [[META5:![0-9]+]] = !{i64 40, !"_ZTSM5Base2FiiiE.virtual"}
+; LOWER: [[META6:![0-9]+]] = !{i64 16, !"_ZTS7Derived"}
+; LOWER: [[META7:![0-9]+]] = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"}
+; LOWER: [[META8:![0-9]+]] = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
+; LOWER: [[META9:![0-9]+]] = !{i64 16, !"_ZTS5Base2"}
+;.
diff --git a/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw b/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw
index bc8fc5db1cb154d98ca962e84313463e3298cb92..a065c4dd7b9ce20f631c1228147c97dc46dae8bb 100644
GIT binary patch
delta 184
zcmeC+`@zTASd^AuRFLzZb0Vib$GLNibNIp-7$!F`Nlc6|nB2g^F|k8{QDNdtc}9kb
z8^sv|CY}Vb9*Q$2O#BICO%`MnXS^|4lM%#r1hOqAM}pafKz7IEPB42Rqd3!nzY}N4
z0o7b&6zBZ#_dgggOg_jgKbeC`fa${DiCJ<W+a_Bu32^R!%NGLWGnfQ`CRH-YPmW*`
K*t~!#g9QKz%0Qq1
delta 226
zcmeyt*TKixSd^AuRFLzZV<M+LCwGqn0|+oqOf(m4Udy`lqjO(I&}GvIF&jY_jforO
zC*BZXOqjS)d=evvVCa+zP1gCUanf@<V@;ADUZ|L?$S6PAgHeF-2T;Z20-zFmUzxmw
zcE{>#|HF2)MP+VFm^=}vWCu{m4j at Z>@&llnuTyPVyaQr|<1ZS8=|7iS^J6k2ll){2
zCIKdcf0M14<b-NpJU>z}?X}aSS0831wSTCP_&2diZt at JE0*-%^S24*6MxEwya$o3I
Va?uU!c!md)A2P{r=3p*h0RV;ZQW5|F
diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
new file mode 100644
index 0000000000000..bf0ad6beb9825
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
@@ -0,0 +1,73 @@
+# IR level Instrumentation Flag
+:ir
+_Z10createTypei
+# Func Hash:
+146835647075900052
+# Num Counters:
+2
+# Counter Values:
+750
+250
+
+_ZN8Derived15func1Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+250
+
+_ZN8Derived15func2Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+250
+
+_ZN8Derived25func1Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+750
+
+_ZN8Derived25func2Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+750
+
+main
+# Func Hash:
+1124236338992350536
+# Num Counters:
+2
+# Counter Values:
+1000
+1
+# Num Value Kinds:
+2
+# ValueKind = IPVK_IndirectCallTarget:
+0
+# NumValueSites:
+2
+2
+_ZN8Derived25func1Eii:750
+_ZN8Derived15func1Eii:250
+2
+_ZN8Derived25func2Eii:750
+_ZN8Derived15func2Eii:250
+# ValueKind = IPVK_VTableTarget:
+2
+# NumValueSites:
+2
+2
+_ZTV8Derived2:750
+_ZTV8Derived1:250
+2
+_ZTV8Derived2:750
+_ZTV8Derived1:250
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.profraw b/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.profraw
new file mode 100644
index 0000000000000000000000000000000000000000..bc52463abfb388a4bb7873ebc503726d0ef78ccd
GIT binary patch
literal 816
zcmZoHO3N=Q$obF700xW at ih+R*#(>fsXncDp|G<9;XkF}No*c at +AOjU=LZhQ+^;o+Z
z?TnjpE6Bob6(<u+Ux0=)SRc%Im_8W)*VW$VW+_u|d$d~S{u7(UKLcvt1em_f5d9#1
zFbvY$H&IVgWp>j9pz`;ZxqP__VEQys?Sp$5YT-hLgPJ$_ at 7;vybAakwKtSKa@@n4Z
zZ>;?=eF9K at 8wlu|{As3#rP1CVn7#-9AQ}%4(Dx1+P`{w=e!&c3!2AOfhtV+gFgnM!
z;#PORk%@<%m#>Z|?@AqyvngkM&zwo_>HoC3z*F~>j<>I&iLdrqZ?lrKXO4)NItiN|
zT6K+MofyNli7Go7j<4KY|LlfI(+?X+cq%~MF- at h$YkkyXR|i{EacL=#eO#Pyvr~H9
zA4xJY)N64ng0!vxVp!aR;}Oc401XG2KVabr^9L+mVDhkVgUN%<0+|gpj|1v1nEC)H
JAEus=JOHD*$_M}e
literal 0
HcmV?d00001
diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test
index 67db5c98ef323..891aac0be4c60 100644
--- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test
+++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test
@@ -8,10 +8,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
// There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes.
// 2 * 8 binary ID sizes
// + 2 * 20 binary IDs (of size 20)
@@ -27,6 +29,8 @@ RUN: printf '\20\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Binary IDs - There are only two in this case that are 20 bytes.
RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw
@@ -52,13 +56,15 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\067\265\035\031\112\165\023\344' >> %t.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
-RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t.profraw
+RUN: printf '\xd0\xff\3\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\067\0\0\0\0\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/large-binary-id-size.test b/llvm/test/tools/llvm-profdata/large-binary-id-size.test
index 2394431e94de4..3362186e10115 100644
--- a/llvm/test/tools/llvm-profdata/large-binary-id-size.test
+++ b/llvm/test/tools/llvm-profdata/large-binary-id-size.test
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -9,6 +9,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Check for a corrupted size being too large past the end of the file.
RUN: printf '\7\7\7\7\7\7\7\7' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test
index 06f418d0235d2..bfcf91a36f940 100644
--- a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test
+++ b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test
@@ -8,10 +8,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -21,6 +23,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Data Section
//
@@ -36,6 +40,7 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\3\0foo\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test
index b718cf0fd8e97..70ddca3dfd7de 100644
--- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test
+++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test
@@ -8,10 +8,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -21,6 +23,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Data Section
//
@@ -37,6 +41,7 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Make NumCounters = 0 so that we get "number of counters is zero" error message
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\3\0foo\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test
index 38e40334a6a69..54f861e08970d 100644
--- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test
+++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test
@@ -8,10 +8,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -21,6 +23,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Data Section
//
@@ -42,6 +46,7 @@ RUN: printf '\11\0\6\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Counter Section
diff --git a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test
index 171b5cc60878f..489e06468ac7d 100644
--- a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test
+++ b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
// We should fail on this because the binary IDs is not a multiple of 8 bytes.
RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -10,6 +10,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// Binary IDs - There are only two in this case that are 20 bytes.
RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test
index c0072bcbde1b3..c3f1ce1ac70d9 100644
--- a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test
+++ b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test
@@ -12,6 +12,8 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t
RUN: printf '\0\0\0\1\0\4\0\0' >> %t
RUN: printf '\0\0\0\2\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: not llvm-profdata show %t -o /dev/null 2>&1 | FileCheck %s
diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
index c8e862009ef02..f7ded4b2ed824 100644
--- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
+++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
@@ -1,5 +1,6 @@
+// Header
RUN: printf '\377lprofR\201' > %t
-RUN: printf '\0\0\0\0\0\0\0\10' >> %t
+RUN: printf '\0\0\0\0\0\0\0\11' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
@@ -9,6 +10,10 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t
RUN: printf '\0\0\0\0\1\0\0\0' >> %t
RUN: printf '\0\0\0\0\2\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumVTables
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// ValueKindLast
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\134\370\302\114\333\030\275\254' >> %t
RUN: printf '\0\0\0\0\0\0\0\1' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test
index 523ff1ceb4807..2d95f5ba9cbd9 100644
--- a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test
+++ b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test
@@ -1,5 +1,5 @@
RUN: printf '\201Rforpl\377' > %t
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\2\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
@@ -9,6 +9,10 @@ RUN: printf '\20\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\1\0\0\0\0' >> %t
RUN: printf '\0\0\0\2\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumVTables
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// ValueKindLast
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\254\275\030\333\114\302\370\134' >> %t
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test
index b2b8b31dafbf5..05ca0cf98d7be 100644
--- a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test
+++ b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test
@@ -1,5 +1,6 @@
+// Header
RUN: printf '\377lprofr\201' > %t
-RUN: printf '\0\0\0\0\0\0\0\10' >> %t
+RUN: printf '\0\0\0\0\0\0\0\11' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
@@ -9,20 +10,28 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t
RUN: printf '\0\0\0\1\0\4\0\0' >> %t
RUN: printf '\0\0\0\2\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumVTables
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// ValueKindLast
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// InstrProfData
RUN: printf '\134\370\302\114\333\030\275\254' >> %t
RUN: printf '\0\0\0\0\0\0\0\1' >> %t
RUN: printf '\0\0\0\1\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\1\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// InstrProfData
RUN: printf '\344\023\165\112\031\035\265\067' >> %t
RUN: printf '\0\0\0\0\0\0\0\02' >> %t
-RUN: printf '\0\0\0\1\0\3\xff\xd8' >> %t
+RUN: printf '\0\0\0\1\0\3\xff\xd0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\02\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\023' >> %t
RUN: printf '\0\0\0\0\0\0\0\067' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
index 4e95798bc0afb..d8e12ef705a25 100644
--- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
+++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
@@ -1,28 +1,53 @@
RUN: printf '\201rforpl\377' > %t
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t
+// Version
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t
+// BinaryIdsSize
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumData
RUN: printf '\2\0\0\0\0\0\0\0' >> %t
+// PaddingBytesBeforeCounters
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumCounters
RUN: printf '\3\0\0\0\0\0\0\0' >> %t
+// PaddingBytesAfterCounters
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NamesSize
RUN: printf '\20\0\0\0\0\0\0\0' >> %t
+// CountersDelta
RUN: printf '\0\0\4\0\1\0\0\0' >> %t
+// NamesDelta
RUN: printf '\0\0\4\0\2\0\0\0' >> %t
+// VNamesSize
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumVTables
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// ValueKindLast
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// InstrProfData
+// NameRef
RUN: printf '\254\275\030\333\114\302\370\134' >> %t
+// FuncHash
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
+// RelativeCounterPtr
RUN: printf '\0\0\4\0\1\0\0\0' >> %t
+// FunctionPointer
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// Values
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// NumCounters + Int16ArrayTy
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
+// Int16ArrayTy + padding
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+// InstrProfData
RUN: printf '\067\265\035\031\112\165\023\344' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
-RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t
+RUN: printf '\xd0\xff\3\0\1\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\023\0\0\0\0\0\0\0' >> %t
RUN: printf '\067\0\0\0\0\0\0\0' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-two-profiles.test b/llvm/test/tools/llvm-profdata/raw-two-profiles.test
index 8d46c91e2732c..5038b6af9232d 100644
--- a/llvm/test/tools/llvm-profdata/raw-two-profiles.test
+++ b/llvm/test/tools/llvm-profdata/raw-two-profiles.test
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t-foo.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
@@ -9,6 +9,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\254\275\030\333\114\302\370\134' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
@@ -16,12 +18,13 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw
RUN: printf '\201rforpl\377' > %t-bar.profraw
-RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\11\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
@@ -31,6 +34,8 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\067\265\035\031\112\165\023\344' >> %t-bar.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw
@@ -38,6 +43,7 @@ RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\067\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\101\0\0\0\0\0\0\0' >> %t-bar.profraw
diff --git a/llvm/test/tools/llvm-profdata/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
new file mode 100644
index 0000000000000..6ebe9b44266b6
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
@@ -0,0 +1,16 @@
+RUN: llvm-profdata show --function=main --show-vtables %p/Inputs/vtable-prof.proftext | FileCheck %s --check-prefix=VTABLE
+
+VTABLE: Counters:
+VTABLE: main:
+VTABLE: Hash: 0x0f9a16fe6d398548
+VTABLE: Counters: 2
+VTABLE: VTable Results:
+VTABLE: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+VTABLE: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+VTABLE: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+VTABLE: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+VTABLE: Instrumentation level: IR entry_first = 0
+VTABLE: Functions shown: 1
+VTABLE: Total functions: 6
+VTABLE: Maximum function count: 1000
+VTABLE: Maximum internal block count: 250
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
new file mode 100644
index 0000000000000..9dd7fa2889690
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -0,0 +1,100 @@
+To update the inputs used below, run Inputs/update_vtable_value_prof_inputs.sh /path/to/updated/clang++ /path/to/updated/llvm-profdata
+
+Show profile data from raw profiles.
+RUN: llvm-profdata show --function=main --ic-targets --show-vtables %p/Inputs/vtable_prof.profraw | FileCheck %s --check-prefix=RAW
+
+Generate indexed profile from raw profile and show the data.
+RUN: llvm-profdata merge %p/Inputs/vtable_prof.profraw -o %t_indexed.profdata
+RUN: llvm-profdata show --function=main --ic-targets --show-vtables %t_indexed.profdata | FileCheck %s --check-prefix=INDEXED
+
+Generate text profile from raw profile and show the data.
+RUN: llvm-profdata merge --text %p/Inputs/vtable_prof.profraw -o %t.proftext
+RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text %t.proftext | FileCheck %s --check-prefix=ICTEXT
+
+RAW: Counters:
+RAW-NEXT: main:
+RAW-NEXT: Hash: 0x0f9a16fe6d398548
+RAW-NEXT: Counters: 2
+RAW-NEXT: Indirect Call Site Count: 2
+RAW-NEXT: VTable Results:
+RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: Indirect Target Results:
+RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+RAW-NEXT: Instrumentation level: IR entry_first = 0
+RAW-NEXT: Functions shown: 1
+RAW-NEXT: Total functions: 6
+RAW-NEXT: Maximum function count: 1000
+RAW-NEXT: Maximum internal block count: 250
+RAW-NEXT: Statistics for indirect call sites profile:
+RAW-NEXT: Total number of sites: 2
+RAW-NEXT: Total number of sites with values: 2
+RAW-NEXT: Total number of profiled values: 4
+RAW-NEXT: Value sites histogram:
+RAW-NEXT: NumTargets, SiteCount
+RAW-NEXT: 2, 2
+
+INDEXED: Counters:
+INDEXED-NEXT: main:
+INDEXED-NEXT: Hash: 0x0f9a16fe6d398548
+INDEXED-NEXT: Counters: 2
+INDEXED-NEXT: Indirect Call Site Count: 2
+INDEXED-NEXT: VTable Results:
+INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: Indirect Target Results:
+INDEXED-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
+INDEXED-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
+INDEXED-NEXT: Instrumentation level: IR entry_first = 0
+INDEXED-NEXT: Functions shown: 1
+INDEXED-NEXT: Total functions: 6
+INDEXED-NEXT: Maximum function count: 1000
+INDEXED-NEXT: Maximum internal block count: 250
+INDEXED-NEXT: Statistics for indirect call sites profile:
+INDEXED-NEXT: Total number of sites: 2
+INDEXED-NEXT: Total number of sites with values: 2
+INDEXED-NEXT: Total number of profiled values: 4
+INDEXED-NEXT: Value sites histogram:
+INDEXED-NEXT: NumTargets, SiteCount
+INDEXED-NEXT: 2, 2
+
+ICTEXT: :ir
+ICTEXT: main
+ICTEXT: # Func Hash:
+ICTEXT: 1124236338992350536
+ICTEXT: # Num Counters:
+ICTEXT: 2
+ICTEXT: # Counter Values:
+ICTEXT: 1000
+ICTEXT: 1
+ICTEXT: # Num Value Kinds:
+ICTEXT: 2
+ICTEXT: # ValueKind = IPVK_IndirectCallTarget:
+ICTEXT: 0
+ICTEXT: # NumValueSites:
+ICTEXT: 2
+ICTEXT: 2
+ICTEXT: _ZN8Derived25func1Eii:750
+ICTEXT: _ZN8Derived15func1Eii:250
+ICTEXT: 2
+ICTEXT: _ZN8Derived25func2Eii:750
+ICTEXT: _ZN8Derived15func2Eii:250
+ICTEXT: # ValueKind = IPVK_VTableTarget:
+ICTEXT: 2
+ICTEXT: # NumValueSites:
+ICTEXT: 2
+ICTEXT: 2
+ICTEXT: _ZTV8Derived2:750
+ICTEXT: _ZTV8Derived1:250
+ICTEXT: 2
+ICTEXT: _ZTV8Derived2:750
+ICTEXT: _ZTV8Derived1:250
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 7d665a8005b0d..205b3b325284e 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -355,6 +355,13 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
});
}
+ const InstrProfSymtab &symtab = Reader->getSymtab();
+ const auto &VTableNames = symtab.getVTableNames();
+
+ for (const auto &kv : VTableNames) {
+ WC->Writer.addVTableName(kv.getKey());
+ }
+
if (Reader->hasTemporalProfile()) {
auto &Traces = Reader->getTemporalProfTraces(Input.Weight);
if (!Traces.empty())
@@ -2422,12 +2429,12 @@ static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK,
static int showInstrProfile(
const std::string &Filename, bool ShowCounts, uint32_t TopN,
- bool ShowIndirectCallTargets, bool ShowMemOPSizes, bool ShowDetailedSummary,
- std::vector<uint32_t> DetailedSummaryCutoffs, bool ShowAllFunctions,
- bool ShowCS, uint64_t ValueCutoff, bool OnlyListBelow,
- const std::string &ShowFunction, bool TextFormat, bool ShowBinaryIds,
- bool ShowCovered, bool ShowProfileVersion, bool ShowTemporalProfTraces,
- ShowFormat SFormat, raw_fd_ostream &OS) {
+ bool ShowIndirectCallTargets, bool ShowMemOPSizes, bool ShowVTables,
+ bool ShowDetailedSummary, std::vector<uint32_t> DetailedSummaryCutoffs,
+ bool ShowAllFunctions, bool ShowCS, uint64_t ValueCutoff,
+ bool OnlyListBelow, const std::string &ShowFunction, bool TextFormat,
+ bool ShowBinaryIds, bool ShowCovered, bool ShowProfileVersion,
+ bool ShowTemporalProfTraces, ShowFormat SFormat, raw_fd_ostream &OS) {
if (SFormat == ShowFormat::Json)
exitWithError("JSON output is not supported for instr profiles");
if (SFormat == ShowFormat::Yaml)
@@ -2563,6 +2570,13 @@ static int showInstrProfile(
OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls
<< "\n";
+ if (ShowVTables) {
+ OS << " VTable Results:\n";
+ traverseAllValueSites(Func, IPVK_VTableTarget,
+ VPStats[IPVK_VTableTarget], OS,
+ &(Reader->getSymtab()));
+ }
+
if (ShowCounts) {
OS << " Block counts: [";
size_t Start = (IsIRInstr ? 0 : 1);
@@ -2943,6 +2957,9 @@ static int show_main(int argc, const char *argv[]) {
cl::opt<bool> ShowIndirectCallTargets(
"ic-targets", cl::init(false),
cl::desc("Show indirect call site target values for shown functions"));
+ cl::opt<bool> ShowVTables(
+ "show-vtables", cl::init(false),
+ cl::desc("Show virtual table target values for shown functions"));
cl::opt<bool> ShowMemOPSizes(
"memop-sizes", cl::init(false),
cl::desc("Show the profiled sizes of the memory intrinsic calls "
@@ -3045,10 +3062,10 @@ static int show_main(int argc, const char *argv[]) {
if (ProfileKind == instr)
return showInstrProfile(
Filename, ShowCounts, TopNFunctions, ShowIndirectCallTargets,
- ShowMemOPSizes, ShowDetailedSummary, DetailedSummaryCutoffs,
- ShowAllFunctions, ShowCS, ValueCutoff, OnlyListBelow, ShowFunction,
- TextFormat, ShowBinaryIds, ShowCovered, ShowProfileVersion,
- ShowTemporalProfTraces, SFormat, OS);
+ ShowMemOPSizes, ShowVTables, ShowDetailedSummary,
+ DetailedSummaryCutoffs, ShowAllFunctions, ShowCS, ValueCutoff,
+ OnlyListBelow, ShowFunction, TextFormat, ShowBinaryIds, ShowCovered,
+ ShowProfileVersion, ShowTemporalProfTraces, SFormat, OS);
if (ProfileKind == sample)
return showSampleProfile(Filename, ShowCounts, TopNFunctions,
ShowAllFunctions, ShowDetailedSummary,
>From cb0246ffea45ee6805d26a76e0d25b091be9ac3d Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Fri, 29 Sep 2023 14:42:19 -0700
Subject: [PATCH 02/15] Address feebacks on the code. Questions will be
followed up shortly. - In IndirectCallVisitor, instrument vtables and skip
the rest (e.g., function pointers, etc). - Add brief comments for helper
functions, remove debugging logs and rename vtable name section to vtabnames.
---
compiler-rt/include/profile/InstrProfData.inc | 15 +-
compiler-rt/lib/profile/InstrProfiling.h | 10 +-
.../lib/profile/InstrProfilingBuffer.c | 11 +-
compiler-rt/lib/profile/InstrProfilingMerge.c | 4 +-
.../lib/profile/InstrProfilingPlatformLinux.c | 4 +-
.../lib/profile/InstrProfilingWriter.c | 13 +-
.../profile/instrprof-write-buffer-internal.c | 8 --
.../llvm/Analysis/IndirectCallVisitor.h | 51 +++++--
llvm/include/llvm/ProfileData/InstrProf.h | 17 ++-
.../llvm/ProfileData/InstrProfData.inc | 2 +-
llvm/lib/ProfileData/InstrProf.cpp | 16 +--
llvm/lib/ProfileData/InstrProfWriter.cpp | 14 +-
.../indirect_call_profile_funclet.ll | 4 +
.../Transforms/PGOProfile/vtable_profile.ll | 133 +++++++++++++-----
14 files changed, 202 insertions(+), 100 deletions(-)
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index 5609870a5a8e5..64d9bad24524d 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -92,6 +92,9 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
/* INSTR_PROF_DATA end. */
+/* For a virtual table object, record the name hash to associate profiled addresses
+ * with global variables, and record {starting address, size in bytes} to map the profiled virtual table (which usually have an offset from the starting address)
+ * back to a virtual table object. */
#ifndef INSTR_PROF_VTABLE_DATA
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
#else
@@ -192,7 +195,15 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
-VALUE_PROF_KIND(IPVK_VTableTarget, 2, "vtable target")
+/* For virtual table address profiling, the addresses of the virtual table (i.e., the address contained in objects pointing to a virtual table) are
+ * profiled. Note this may not be the address of the per C++ class virtual table object (i.e., there is an offset).
+ *
+ * The profiled addresses are stored in raw profile, together with the following two types of information.
+ * 1. The (beginning and ending) addresses of per C++ class virtual table objects.
+ * 2. The (compressed) virtual table object names.
+ * RawInstrProfReader converts profiled virtual table addresses to virtual table objects' MD5 hash.
+ */
+VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The address of the compatible vtable (i.e., there is an offset from this address to per C++ class virtual table global variable.)")
/* These two kinds must be the last to be
* declared. This is to make sure the string
* array created with the template can be
@@ -707,7 +718,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
-#define INSTR_PROF_VNAME_COMMON __llvm_prf_vnames
+#define INSTR_PROF_VNAME_COMMON __llvm_prf_vtabnames
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index f3afa694e02c2..c3aa72e411f79 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -92,8 +92,8 @@ const __llvm_profile_data *__llvm_profile_begin_data(void);
const __llvm_profile_data *__llvm_profile_end_data(void);
const char *__llvm_profile_begin_names(void);
const char *__llvm_profile_end_names(void);
-const char *__llvm_profile_begin_vnames(void);
-const char *__llvm_profile_end_vnames(void);
+const char *__llvm_profile_begin_vtabnames(void);
+const char *__llvm_profile_end_vtabnames(void);
char *__llvm_profile_begin_counters(void);
char *__llvm_profile_end_counters(void);
ValueProfNode *__llvm_profile_begin_vnodes();
@@ -286,11 +286,13 @@ uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End);
/*! \brief Get the size of the profile counters section in bytes. */
uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End);
+/*! \brief Get the number of virtual table profile data entries */
uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
const VTableProfData *End);
-uint64_t __llvm_profile_get_vtable_size(const VTableProfData *Begin,
- const VTableProfData *End);
+/*! \brief Get the size of virtual table profile data in bytes. */
+uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,
+ const VTableProfData *End);
/* ! \brief Given the sizes of the data and counter information, return the
* number of padding bytes before and after the counters, and after the names,
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index 0c36e40444c73..ad1643d48908d 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -49,8 +49,8 @@ uint64_t __llvm_profile_get_size_for_buffer(void) {
const char *NamesEnd = __llvm_profile_end_names();
const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
- const char *VNamesBegin = __llvm_profile_begin_vnames();
- const char *VNamesEnd = __llvm_profile_end_vnames();
+ const char *VNamesBegin = __llvm_profile_begin_vtabnames();
+ const char *VNamesEnd = __llvm_profile_end_vtabnames();
return __llvm_profile_get_size_for_buffer_internal(
DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd,
@@ -78,8 +78,8 @@ uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
}
COMPILER_RT_VISIBILITY
-uint64_t __llvm_profile_get_vtable_size(const VTableProfData *Begin,
- const VTableProfData *End) {
+uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,
+ const VTableProfData *End) {
return __llvm_profile_get_num_vtable(Begin, End) * sizeof(VTableProfData);
}
@@ -166,7 +166,8 @@ uint64_t __llvm_profile_get_size_for_buffer_internal(
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
uint64_t CountersSize =
__llvm_profile_get_counters_size(CountersBegin, CountersEnd);
- uint64_t VTableSize = __llvm_profile_get_vtable_size(VTableBegin, VTableEnd);
+ uint64_t VTableSize =
+ __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
uint64_t VNameSize = (VNamesEnd - VNamesBegin) * sizeof(char);
/* Determine how much padding is needed before/after the counters and after
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 2ef6227599ff1..629d502cdde31 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -125,9 +125,7 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
Header->NumCounters * __llvm_profile_counter_entry_size();
SrcNameStart = SrcCountersEnd;
// This is to assume counter size is a multiple of 8 bytes.
- // uint64_t NamesSize = Header->NamesSize;
- // uint64_t PaddingBytesAfterNames =
- // __llvm_profile_get_num_padding_bytes(Header->NamesSize);
+
// First, skip rather than merge them
uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData);
uint64_t PaddingBytesAfterVTableSection =
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
index dc861632271ce..c681e253bb46a 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -76,10 +76,10 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) {
COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) {
return &PROF_NAME_STOP;
}
-COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vnames(void) {
+COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vtabnames(void) {
return &PROF_VNAME_START;
}
-COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vnames(void) {
+COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vtabnames(void) {
return &PROF_VNAME_STOP;
}
COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_begin_vtables(void) {
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index ed5dbdb6ee438..b998618aad789 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -250,8 +250,8 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
const char *NamesEnd = __llvm_profile_end_names();
const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
- const char *VNamesBegin = __llvm_profile_begin_vnames();
- const char *VNamesEnd = __llvm_profile_end_vnames();
+ const char *VNamesBegin = __llvm_profile_begin_vtabnames();
+ const char *VNamesEnd = __llvm_profile_end_vtabnames();
return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
CountersEnd, VPDataReader, NamesBegin, NamesEnd,
VTableBegin, VTableEnd, VNamesBegin, VNamesEnd,
@@ -282,9 +282,8 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const uint64_t NumVTables =
__llvm_profile_get_num_vtable(VTableBegin, VTableEnd);
const uint64_t VTableSectionSize =
- __llvm_profile_get_vtable_size(VTableBegin, VTableEnd);
- // Note, in reality, vtable profiling is not supported when DebugInfoCorrelate
- // is true.
+ __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
+ // Note vtable profiling is not supported when DebugInfoCorrelate is true.
const uint64_t VNamesSize = DebugInfoCorrelate ? 0 : VNamesEnd - VNamesBegin;
/* Create the header. */
@@ -320,13 +319,9 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
/* Write the profile header. */
ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
- // printf("Size of profile header is %d\n",
- // (int)(sizeof(__llvm_profile_header)));
if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
return -1;
- // printf("Completed profile header\n");
-
/* Write the binary id lengths and data. */
int binary_id_size = __llvm_write_binary_ids(Writer);
if (binary_id_size == -1)
diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
index 97dfff8d7fe71..484aa1f4f2d0e 100644
--- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c
+++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
@@ -48,20 +48,12 @@ int main(int argc, const char *argv[]) {
__llvm_profile_begin_names(), __llvm_profile_end_names(), NULL, NULL,
NULL, NULL);
- // printf("buffer size is %lld\n", bufsize);
- //uint64_t aligned_bufsize = ((bufsize + 32) >> 6) << 6;
-
char *buf = malloc(bufsize);
int ret = __llvm_profile_write_buffer_internal(buf,
__llvm_profile_begin_data(), __llvm_profile_end_data(),
__llvm_profile_begin_counters(), __llvm_profile_end_counters(),
__llvm_profile_begin_names(), __llvm_profile_end_names());
- if (ret != 0) {
- fprintf(stderr, "failed to write buffer");
- return ret;
- }
-
FILE *f = fopen(argv[1], "w");
fwrite(buf, bufsize, 1, f);
fclose(f);
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 52d4ff04d3d4e..23168b88a6988 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -12,6 +12,7 @@
#ifndef LLVM_ANALYSIS_INDIRECTCALLVISITOR_H
#define LLVM_ANALYSIS_INDIRECTCALLVISITOR_H
+#include "llvm/ADT/SetVector.h"
#include "llvm/IR/InstVisitor.h"
#include <vector>
@@ -19,24 +20,58 @@ namespace llvm {
// Visitor class that finds all indirect call.
struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
std::vector<CallBase *> IndirectCalls;
- std::vector<Instruction *> VTableAddrs;
+ SetVector<Instruction *, std::vector<Instruction *>> VTableAddrs;
PGOIndirectCallVisitor() = default;
void visitCallBase(CallBase &Call) {
- if (Call.isIndirectCall())
+ const CallInst *CI = dyn_cast<CallInst>(&Call);
+ if (CI && CI->getCalledFunction()) {
+ switch (CI->getCalledFunction()->getIntrinsicID()) {
+ case Intrinsic::type_test:
+ case Intrinsic::public_type_test:
+ case Intrinsic::type_checked_load_relative:
+ case Intrinsic::type_checked_load: {
+ Value *VTablePtr = CI->getArgOperand(0)->stripPointerCasts();
+
+ if (PtrTestedByTypeIntrinsics.count(VTablePtr) == 0) {
+ Instruction *I = dyn_cast_or_null<Instruction>(VTablePtr);
+ // This is the first type intrinsic where VTablePtr is used.
+ // Assert that the VTablePtr is not found as a type profiling
+ // candidate yet. Note nullptr won't be inserted into VTableAddrs in
+ // the first place, so this assertion works even if 'VTablePtr' is not
+ // an instruction.
+ assert(VTableAddrs.count(I) == 0 &&
+ "Expect type intrinsic to record VTablePtr before virtual "
+ "functions are loaded to find vtables that should be "
+ "instrumented");
+
+ PtrTestedByTypeIntrinsics.insert(VTablePtr);
+ }
+ } break;
+ }
+ }
if (Call.isIndirectCall()) {
IndirectCalls.push_back(&Call);
-
LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
if (LI != nullptr) {
- Value *Ptr = LI->getPointerOperand();
- Value *VTablePtr = Ptr->stripInBoundsConstantOffsets();
- if (VTablePtr != nullptr && isa<Instruction>(VTablePtr)) {
- VTableAddrs.push_back(cast<Instruction>(VTablePtr));
+ Value *MaybeVTablePtr =
+ LI->getPointerOperand()->stripInBoundsConstantOffsets();
+ Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
+ // If not used by any type intrinsic, this is not a vtable.
+ // Inst visitor should see the very first type intrinsic using a
+ // vtable before the very first virtual function load from this
+ // vtable. This condition is asserted above.
+ if (VTableInstr && PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
+ VTableAddrs.insert(VTableInstr);
}
}
}
}
+
+private:
+ // Keeps track of the pointers that are tested by llvm type intrinsics for
+ // look up.
+ SmallPtrSet<Value *, 4> PtrTestedByTypeIntrinsics;
};
inline std::vector<CallBase *> findIndirectCalls(Function &F) {
@@ -48,7 +83,7 @@ inline std::vector<CallBase *> findIndirectCalls(Function &F) {
inline std::vector<Instruction *> findVTableAddrs(Function &F) {
PGOIndirectCallVisitor ICV;
ICV.visit(F);
- return ICV.VTableAddrs;
+ return ICV.VTableAddrs.takeVector();
}
} // namespace llvm
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 845ea1d0ab0b3..41b882863f1d4 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -440,9 +440,13 @@ class InstrProfSymtab {
private:
StringRef Data;
uint64_t Address = 0;
- // Unique name strings.
+ // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's
+ // going to be sorted) has unique MD5 keys in the first place.
StringSet<> NameTab;
- // Unique virtual table names.
+ // Records the unique virtual table names. This is used by InstrProfWriter to
+ // write out an on-disk chained hash table of virtual table names.
+ // InstrProfWriter stores per function profile data (keyed by function names)
+ // so it doesn't use a StringSet for function names.
StringSet<> VTableNames;
// A map from MD5 keys to function name strings.
std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
@@ -532,17 +536,16 @@ class InstrProfSymtab {
if (VTableName.empty())
return make_error<InstrProfError>(instrprof_error::malformed,
"invalid input: VTableName is empty");
- // Insert into NameTab.
+ // Insert into NameTab so that MD5NameMap (a vector that is going to be
+ // sorted) won't have duplicated entries in the first place.
auto Ins = NameTab.insert(VTableName);
- // Insert into VTableNames.
+ // Record VTableName. InstrProfWriter uses this map. The comment around
+ // class member explains why.
VTableNames.insert(VTableName);
// If this is newly added, update MD5NameMap.
if (Ins.second) {
- // printf("VTableName %s\n", VTableName.str().c_str());
- // printf("AddVTableName hash %"PRIu64" to %s\n",
- // IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey());
MD5NameMap.push_back(std::make_pair(
IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey()));
Sorted = false;
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index f1e7691afdc4b..4c8f1b31c5e8d 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -706,7 +706,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
-#define INSTR_PROF_VNAME_COMMON __llvm_prf_vnames
+#define INSTR_PROF_VNAME_COMMON __llvm_prf_vtabnames
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 31d29a6c4bd8b..c981e09ed3151 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -498,20 +498,15 @@ Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
finalizeSymtab();
- // printf("look up key 0x%llx\n", Address);
- // for (auto iter = VTableAddrToMD5Map.begin(); iter !=
- // VTableAddrToMD5Map.end(); iter++) {
- // printf("<key, val> is <0x%llx, %"PRIu64"\n", iter->first, iter->second);
- // }
auto It =
partition_point(VTableAddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
return A.first < Address;
});
- // FIXME: Does the raw function pointers point apply here?
+ // The virtual table address collected from value profiler could be defined
+ // in another module that is not instrumented. Force the value to be 0 in
+ // this case.
if (It != VTableAddrToMD5Map.end()) {
- // printf("InstrProfSymtab::getVTableHashFromAddress map addr 0x%llx to hash
- // value %"PRIu64"\n", Address, (uint64_t)It->second);
- return (uint64_t)It->second;
+ return It->second;
}
return 0;
}
@@ -596,7 +591,6 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
std::string &Result, bool doCompression) {
std::vector<std::string> VTableNameStrs;
for (auto *VTable : VTables) {
- // printf("VTable name %s added\n", VTable->getName().str().c_str());
VTableNameStrs.push_back(std::string(VTable->getName()));
}
return collectPGOFuncNameStrings(
@@ -691,13 +685,11 @@ Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab) {
}
NameStrings.split(Names, getInstrProfNameSeparator());
for (StringRef &Name : Names) {
- // printf("Read back vtable name %s\n", Name.str().c_str());
if (Error E = Symtab.addVTableName(Name))
return E;
}
P += Dist;
- // Skip padding?
while (P < EndP && *P == 0)
P++;
}
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 0094cf8c029fc..0713864c21d97 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -634,19 +634,22 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
OS.writeByte(0);
}
- // if version >= the version with vtable profile metadata
- // Intentionally put vtable names before temporal profile section.
+ // if version >= the version with vtable profile metadata.
uint64_t VTableNamesSectionStart = 0;
if (IndexedInstrProf::ProfVersion::CurrentVersion >= 11) {
VTableNamesSectionStart = OS.tell();
- // Reserve space for vtable record table offset.
+ // Reserve space for vtable records offset.
OS.write(0ULL);
OnDiskChainedHashTableGenerator<llvm::InstrProfRecordVTableTrait>
VTableNamesGenerator;
for (const auto &kv : VTableNames) {
- // printf("InstrProfWriter.cpp key is %s\n", kv.getKey().str().c_str());
+ // Use a char '0' as value placeholder, only keys (vtable names)
+ // are used.
+ // FIXME: It might make sense to have a OnDiskChainedHashSetGenerator if
+ // there are more use cases. Use a hash table for now, with one unused
+ // 'char' per entry.
VTableNamesGenerator.insert(kv.getKey(), '0');
}
@@ -656,9 +659,6 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
uint64_t VTableNamesTableOffset =
VTableNamesGenerator.Emit(OS.OS, *VTableNamesWriter);
- // printf("InstrProfWriter.cpp:VTableNamesSectionStart is %"PRIu64"\n",
- // VTableNamesSectionStart); printf("\tVTableNamesTableOffset is
- // %"PRIu64"\n", VTableNamesTableOffset);
PatchItem PatchItems[] = {
{VTableNamesSectionStart, &VTableNamesTableOffset, 1}};
OS.patch(PatchItems, 1);
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
index 63173ab59b28b..988a30865a7fe 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
@@ -45,6 +45,8 @@ catch.dispatch: ; preds = %entry
catch: ; preds = %catch.dispatch
%tmp1 = catchpad within %tmp [ptr null, i32 64, ptr null]
%vtable = load ptr, ptr %b, align 8
+ %0 = tail call i1 @llvm.type.test(ptr %vtable, metadata !"_ZTS4base")
+ tail call void @llvm.assume(i1 %0)
%tmp3 = load ptr, ptr %vtable, align 8
call void %tmp3(ptr %b) [ "funclet"(token %tmp1) ]
catchret from %tmp1 to label %try.cont
@@ -64,3 +66,5 @@ try.cont: ; preds = %catch, %entry
declare dso_local void @"?may_throw@@YAXH at Z"(i32)
declare dso_local i32 @__CxxFrameHandler3(...)
+declare i1 @llvm.type.test(ptr, metadata)
+declare void @llvm.assume(i1 noundef)
diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
index 0c6bd3c064d35..059552a7634dc 100644
--- a/llvm/test/Transforms/PGOProfile/vtable_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
@@ -1,9 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 3
; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
+; The test IR is generated based on the following C++ program.
+; class Derived uses multiple inheritance so its virtual table
+; global variable contains two vtables. func1 is loaded from
+; the vtable compatible with class Base1, and func2 is loaded
+; from the vtable compatible with class Base2.
+; class Base1 {
+; public:
+; virtual int func1(int a, int b) ;
+; };
+;
+; class Base2 {
+; public:
+; virtual int func2(int a, int b);
+; };
+;
+; class Derived : public Base1, public Base2 {
+; public:
+; Derived(int c) : v(c) {}
+; private:
+; int v;
+; };
+;
+; Derived* createType(int c);
+; int func(int a, int b, int c) {
+; Derived* d = createType(c);
+;
+; return d->func2(a, b) + d->func1(b, a);
+; }
+
$_ZTV7Derived = comdat any
@_ZTV7Derived = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
@@ -19,25 +49,44 @@ $_ZTV7Derived = comdat any
declare ptr @_Z10createTypei(i32)
declare i32 @_ZN5Base15func1Eii(ptr, i32, i32)
declare i32 @_ZN5Base25func2Eii(ptr, i32, i32)
+declare i1 @llvm.public.type.test(ptr, metadata)
+declare void @llvm.assume(i1 noundef)
-; GEN: @__llvm_profile_raw_version = hidden constant i64 72057594037927945, comdat
-; GEN: @__profn_test_vtable_value_profiling = private constant [27 x i8] c"test_vtable_value_profiling"
-
-; LOWER: $__profvt__ZTV7Derived = comdat any
-; LOWER: $__profvt__ZTV5Base1 = comdat nodeduplicate
-; LOWER: $__profvt__ZTV5Base2 = comdat nodeduplicate
-; LOWER: @__llvm_profile_raw_version = hidden constant i64 72057594037927945, comdat
-; LOWER: @__profc_test_vtable_value_profiling = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
-; LOWER: @__profvp_test_vtable_value_profiling = private global [4 x i64] zeroinitializer, section "__llvm_prf_vals", comdat($__profc_test_vtable_value_profiling), align 8
-; LOWER: @__profd_test_vtable_value_profiling = private global { i64, i64, i64, ptr, ptr, i32, [3 x i16] } { i64 1593873508557585901, i64 567090795815895039, i64 sub (i64 ptrtoint (ptr @__profc_test_vtable_value_profiling to i64), i64 ptrtoint (ptr @__profd_test_vtable_value_profiling to i64)), ptr @test_vtable_value_profiling.local, ptr @__profvp_test_vtable_value_profiling, i32 1, [3 x i16] [i16 2, i16 0, i16 2] }, section "__llvm_prf_data", comdat($__profc_test_vtable_value_profiling), align 8
-; LOWER: @__profvt__ZTV7Derived = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8
-; LOWER: @__profvt__ZTV5Base1 = global { i64, ptr, i32 } { i64 3215870116411581797, ptr @_ZTV5Base1, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
-; LOWER: @__profvt__ZTV5Base2 = global { i64, ptr, i32 } { i64 8378219803387680050, ptr @_ZTV5Base2, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
-; LOWER: @__llvm_prf_vnodes = private global [10 x { i64, i64, ptr }] zeroinitializer, section "__llvm_prf_vnds", align 8
-; LOWER: @__llvm_prf_nm = private constant [37 x i8] c"\1B#x\DA+I-.\89/+IL\CAI\8D/K\CC)M\8D/(\CAO\CB\CC\C9\CCK\07\00\9Ea\0BC", section "__llvm_prf_names", align 1
-; LOWER: @__llvm_prf_vnm = private constant [34 x i8] c"\22 x\DA\8B\8F\0A\093wI-\CA,KMa\8C\07rL\9D\12\8BS\0D\11L#\00\C3\A2\0A\E9", section "__llvm_prf_vnames", align 1
-; LOWER: @llvm.used = appending global [6 x ptr] [ptr @__profvt__ZTV7Derived, ptr @__profvt__ZTV5Base1, ptr @__profvt__ZTV5Base2, ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata"
-
+;.
+; GEN: @[[_ZTV7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
+; GEN: @[[_ZTVN10__CXXABIV121__VMI_CLASS_TYPE_INFOE:[a-zA-Z0-9_$"\\.-]+]] = external global [0 x ptr]
+; GEN: @[[_ZTS7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant [9 x i8] c"7Derived\00", align 1
+; GEN: @[[_ZTI5BASE1:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
+; GEN: @[[_ZTI5BASE2:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
+; GEN: @[[_ZTI7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { ptr, ptr, i32, i32, ptr, i64, ptr, i64 } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), ptr @_ZTS7Derived, i32 0, i32 2, ptr @_ZTI5Base1, i64 2, ptr @_ZTI5Base2, i64 2050 }, align 8
+; GEN: @[[_ZTV5BASE1:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base1, ptr @_ZN5Base15func1Eii] }, align 8, !type !0, !type !1
+; GEN: @[[_ZTV5BASE2:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base2, ptr @_ZN5Base25func2Eii] }, align 8, !type !9, !type !4
+; GEN: @[[LLVM_COMPILER_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [2 x ptr] [ptr @_ZTV5Base1, ptr @_ZTV5Base2], section "llvm.metadata"
+; GEN: @[[__LLVM_PROFILE_RAW_VERSION:[a-zA-Z0-9_$"\\.-]+]] = hidden constant i64 72057594037927945, comdat
+; GEN: @[[__PROFN_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private constant [27 x i8] c"test_vtable_value_profiling"
+;.
+; LOWER: @[[_ZTV7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
+; LOWER: @[[_ZTVN10__CXXABIV121__VMI_CLASS_TYPE_INFOE:[a-zA-Z0-9_$"\\.-]+]] = external global [0 x ptr]
+; LOWER: @[[_ZTS7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant [9 x i8] c"7Derived\00", align 1
+; LOWER: @[[_ZTI5BASE1:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
+; LOWER: @[[_ZTI5BASE2:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
+; LOWER: @[[_ZTI7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { ptr, ptr, i32, i32, ptr, i64, ptr, i64 } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), ptr @_ZTS7Derived, i32 0, i32 2, ptr @_ZTI5Base1, i64 2, ptr @_ZTI5Base2, i64 2050 }, align 8
+; LOWER: @[[_ZTV5BASE1:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base1, ptr @_ZN5Base15func1Eii] }, align 8, !type !0, !type !1
+; LOWER: @[[_ZTV5BASE2:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base2, ptr @_ZN5Base25func2Eii] }, align 8, !type !9, !type !4
+; LOWER: @[[__LLVM_PROFILE_RAW_VERSION:[a-zA-Z0-9_$"\\.-]+]] = hidden constant i64 72057594037927945, comdat
+; LOWER: @[[__PROFC_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
+; LOWER: @[[__PROFVP_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private global [4 x i64] zeroinitializer, section "__llvm_prf_vals", comdat($__profc_test_vtable_value_profiling), align 8
+; LOWER: @[[__PROFD_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private global { i64, i64, i64, ptr, ptr, i32, [3 x i16] } { i64 1593873508557585901, i64 567090795815895039, i64 sub (i64 ptrtoint (ptr @__profc_test_vtable_value_profiling to i64), i64 ptrtoint (ptr @__profd_test_vtable_value_profiling to i64)), ptr @test_vtable_value_profiling.local, ptr @__profvp_test_vtable_value_profiling, i32 1, [3 x i16] [i16 2, i16 0, i16 2] }, section "__llvm_prf_data", comdat($__profc_test_vtable_value_profiling), align 8
+; LOWER: @[[__PROFVT__ZTV7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @[[__PROFVT__ZTV5BASE1:[a-zA-Z0-9_$"\\.-]+]] = global { i64, ptr, i32 } { i64 3215870116411581797, ptr @_ZTV5Base1, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @[[__PROFVT__ZTV5BASE2:[a-zA-Z0-9_$"\\.-]+]] = global { i64, ptr, i32 } { i64 8378219803387680050, ptr @_ZTV5Base2, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @[[__LLVM_PRF_VNODES:[a-zA-Z0-9_$"\\.-]+]] = private global [10 x { i64, i64, ptr }] zeroinitializer, section "__llvm_prf_vnds", align 8
+; LOWER: @[[__LLVM_PRF_NM:[a-zA-Z0-9_$"\\.-]+]] = private constant [37 x i8] c"\1B#x\DA+I-.\89/+IL\CAI\8D/K\CC)M\8D/(\CAO\CB\CC\C9\CCK\07\00\9Ea\0BC", section "__llvm_prf_names", align 1
+; LOWER: @[[__LLVM_PRF_VNM:[a-zA-Z0-9_$"\\.-]+]] = private constant [34 x i8] c"\22 x\DA\8B\8F\0A\093wI-\CA,KMa\8C\07rL\9D\12\8BS\0D\11L#\00\C3\A2\0A\E9", section "__llvm_prf_vtabnames", align 1
+; LOWER: @[[LLVM_COMPILER_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [3 x ptr] [ptr @_ZTV5Base1, ptr @_ZTV5Base2, ptr @__profd_test_vtable_value_profiling], section "llvm.metadata"
+; LOWER: @[[LLVM_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [6 x ptr] [ptr @__profvt__ZTV7Derived, ptr @__profvt__ZTV5Base1, ptr @__profvt__ZTV5Base2, ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata"
+; LOWER: @[[TEST_VTABLE_VALUE_PROFILING_LOCAL:[a-zA-Z0-9_$"\\.-]+]] = private alias i32 (i32, i32, i32), ptr @test_vtable_value_profiling
+;.
define i32 @test_vtable_value_profiling(i32 %a, i32 %b, i32 %c) {
; GEN-LABEL: define i32 @test_vtable_value_profiling(
; GEN-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) {
@@ -48,16 +97,20 @@ define i32 @test_vtable_value_profiling(i32 %a, i32 %b, i32 %c) {
; GEN-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
; GEN-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[VTABLE]] to i64
; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP0]], i32 2, i32 0)
+; GEN-NEXT: [[TMP1:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE]], metadata !"_ZTS5Base2")
+; GEN-NEXT: tail call void @llvm.assume(i1 [[TMP1]])
; GEN-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
-; GEN-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[VFUNC]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP1]], i32 0, i32 0)
+; GEN-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VFUNC]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP2]], i32 0, i32 0)
; GEN-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
; GEN-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
-; GEN-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP2]], i32 2, i32 1)
+; GEN-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP3]], i32 2, i32 1)
+; GEN-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE2]], metadata !"_ZTS5Base1")
+; GEN-NEXT: tail call void @llvm.assume(i1 [[TMP4]])
; GEN-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
-; GEN-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP3]], i32 0, i32 1)
+; GEN-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP5]], i32 0, i32 1)
; GEN-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
; GEN-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
; GEN-NEXT: ret i32 [[ADD]]
@@ -73,27 +126,39 @@ define i32 @test_vtable_value_profiling(i32 %a, i32 %b, i32 %c) {
; LOWER-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
; LOWER-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[VTABLE]] to i64
; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP1]], ptr @__profd_test_vtable_value_profiling, i32 2)
+; LOWER-NEXT: [[TMP2:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE]], metadata !"_ZTS5Base2")
+; LOWER-NEXT: tail call void @llvm.assume(i1 [[TMP2]])
; LOWER-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
-; LOWER-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VFUNC]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP2]], ptr @__profd_test_vtable_value_profiling, i32 0)
+; LOWER-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VFUNC]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP3]], ptr @__profd_test_vtable_value_profiling, i32 0)
; LOWER-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
; LOWER-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
-; LOWER-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP3]], ptr @__profd_test_vtable_value_profiling, i32 3)
+; LOWER-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP4]], ptr @__profd_test_vtable_value_profiling, i32 3)
+; LOWER-NEXT: [[TMP5:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE2]], metadata !"_ZTS5Base1")
+; LOWER-NEXT: tail call void @llvm.assume(i1 [[TMP5]])
; LOWER-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
-; LOWER-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP4]], ptr @__profd_test_vtable_value_profiling, i32 1)
+; LOWER-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
+; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP6]], ptr @__profd_test_vtable_value_profiling, i32 1)
; LOWER-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
; LOWER-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
; LOWER-NEXT: ret i32 [[ADD]]
;
entry:
%call = tail call ptr @_Z10createTypei(i32 %c)
+ ; The first indirect call is 'func2' so func ptr is
+ ; loaded from vtable compatible with Base2.
%add.ptr = getelementptr inbounds i8, ptr %call, i64 8
%vtable = load ptr, ptr %add.ptr, align 8
+ %0 = tail call i1 @llvm.public.type.test(ptr %vtable, metadata !"_ZTS5Base2")
+ tail call void @llvm.assume(i1 %0)
%vfunc = load ptr, ptr %vtable, align 8
%call1 = tail call i32 %vfunc(ptr %add.ptr, i32 %a, i32 %b)
+ ; The second indirect call is 'func1' so func ptr is
+ ; loaded from vtable compatible with Base1.
%vtable2 = load ptr, ptr %call, align 8
+ %1 = tail call i1 @llvm.public.type.test(ptr %vtable2, metadata !"_ZTS5Base1")
+ tail call void @llvm.assume(i1 %1)
%vfunc2 = load ptr, ptr %vtable2, align 8
%call4 = tail call i32 %vfunc2(ptr %call, i32 %b, i32 %a)
%add = add nsw i32 %call4, %call1
@@ -111,9 +176,13 @@ entry:
!8 = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
!9 = !{i64 16, !"_ZTS5Base2"}
;.
-; GEN: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+; GEN: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; GEN: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+; GEN: attributes #[[ATTR2:[0-9]+]] = { nounwind }
;.
-; LOWER: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+; LOWER: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; LOWER: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+; LOWER: attributes #[[ATTR2:[0-9]+]] = { nounwind }
;.
; GEN: [[META0:![0-9]+]] = !{i64 16, !"_ZTS5Base1"}
; GEN: [[META1:![0-9]+]] = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
>From 3c6ca047be0069012ecf65c6d50c1fa7992fc3c3 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Mon, 2 Oct 2023 13:46:09 -0700
Subject: [PATCH 03/15] In InstrProf.cpp, add option
-enable-vtable-type-profiling to flag control vtable instrumentation, and
record address range of vtables (in VTableAddRangeToMD5Map). - The new option
is used in PGOInstrumentation.cpp and InstrProfiling.cpp to flag-control
the instrumentation of static and runtime information. Static information
includes vtable address range and name md5 hash, runtime information are
types of a vtable value. - Before this commit, VTableAddrToMD5Map records
start and end of vtable address individually. After this commit, the map
records a range to return MD5 iff the address is within the range. This is
more accurate when runtime address is collected in one module but static
vtable information is not recorded.
---
.../llvm/Analysis/IndirectCallVisitor.h | 43 ++++++++++++-------
llvm/include/llvm/ProfileData/InstrProf.h | 12 +++---
llvm/lib/ProfileData/InstrProf.cpp | 23 +++++++---
.../Instrumentation/InstrProfiling.cpp | 16 ++++---
.../Instrumentation/PGOInstrumentation.cpp | 9 +++-
.../Inputs/update_vtable_value_prof_inputs.sh | 5 +--
6 files changed, 71 insertions(+), 37 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 23168b88a6988..347f72bc625cb 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -17,15 +17,20 @@
#include <vector>
namespace llvm {
-// Visitor class that finds all indirect call.
+// Visitor class that finds indirect calls or instructions that gives vtable
+// value, depending on Type.
struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
+ enum class InstructionType {
+ kIndirectCall = 0,
+ kVTableVal = 1,
+ };
std::vector<CallBase *> IndirectCalls;
SetVector<Instruction *, std::vector<Instruction *>> VTableAddrs;
- PGOIndirectCallVisitor() = default;
+ PGOIndirectCallVisitor(InstructionType Type) : Type(Type) {}
void visitCallBase(CallBase &Call) {
const CallInst *CI = dyn_cast<CallInst>(&Call);
- if (CI && CI->getCalledFunction()) {
+ if (Type == InstructionType::kVTableVal && CI && CI->getCalledFunction()) {
switch (CI->getCalledFunction()->getIntrinsicID()) {
case Intrinsic::type_test:
case Intrinsic::public_type_test:
@@ -52,17 +57,20 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
}
if (Call.isIndirectCall()) {
IndirectCalls.push_back(&Call);
- LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
- if (LI != nullptr) {
- Value *MaybeVTablePtr =
- LI->getPointerOperand()->stripInBoundsConstantOffsets();
- Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
- // If not used by any type intrinsic, this is not a vtable.
- // Inst visitor should see the very first type intrinsic using a
- // vtable before the very first virtual function load from this
- // vtable. This condition is asserted above.
- if (VTableInstr && PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
- VTableAddrs.insert(VTableInstr);
+ if (Type == InstructionType::kVTableVal) {
+ LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
+ if (LI != nullptr) {
+ Value *MaybeVTablePtr =
+ LI->getPointerOperand()->stripInBoundsConstantOffsets();
+ Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
+ // If not used by any type intrinsic, this is not a vtable.
+ // Inst visitor should see the very first type intrinsic using a
+ // vtable before the very first virtual function load from this
+ // vtable. This condition is asserted above.
+ if (VTableInstr &&
+ PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
+ VTableAddrs.insert(VTableInstr);
+ }
}
}
}
@@ -72,16 +80,19 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
// Keeps track of the pointers that are tested by llvm type intrinsics for
// look up.
SmallPtrSet<Value *, 4> PtrTestedByTypeIntrinsics;
+ InstructionType Type;
};
inline std::vector<CallBase *> findIndirectCalls(Function &F) {
- PGOIndirectCallVisitor ICV;
+ PGOIndirectCallVisitor ICV(
+ PGOIndirectCallVisitor::InstructionType::kIndirectCall);
ICV.visit(F);
return ICV.IndirectCalls;
}
inline std::vector<Instruction *> findVTableAddrs(Function &F) {
- PGOIndirectCallVisitor ICV;
+ PGOIndirectCallVisitor ICV(
+ PGOIndirectCallVisitor::InstructionType::kVTableVal);
ICV.visit(F);
return ICV.VTableAddrs.takeVector();
}
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 41b882863f1d4..0c7cce9b7355e 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -436,6 +436,8 @@ uint64_t ComputeHash(StringRef K);
class InstrProfSymtab {
public:
using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
+ using RangeHashMap =
+ std::vector<std::pair<std::pair<uint64_t, uint64_t>, uint64_t>>;
private:
StringRef Data;
@@ -463,7 +465,7 @@ class InstrProfSymtab {
// This map is only populated and used by raw instr profile reader.
// This is a different map from 'AddrToMD5Map' for readability and
// debuggability.
- AddrHashMap VTableAddrToMD5Map;
+ RangeHashMap VTableAddrRangeToMD5Map;
bool Sorted = false;
static StringRef getExternalSymbol() {
@@ -561,11 +563,11 @@ class InstrProfSymtab {
AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
}
- // Map the start and end address of a variable to its names' MD5 hash.
- // This interface is only used by the raw profile header.
+ // Map the address range (i.e., [start_address, end_address]) of a variable to
+ // its names' MD5 hash. This interface is only used by the raw profile header.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
- VTableAddrToMD5Map.push_back(std::make_pair(StartAddr, MD5Val));
- VTableAddrToMD5Map.push_back(std::make_pair(EndAddr, MD5Val));
+ VTableAddrRangeToMD5Map.push_back(
+ std::make_pair(std::make_pair(StartAddr, EndAddr), MD5Val));
}
/// Return a function's hash, or 0, if the function isn't in this SymTab.
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index c981e09ed3151..54b9e397862b2 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -215,6 +215,13 @@ cl::opt<bool> DoInstrProfNameCompression(
"enable-name-compression",
cl::desc("Enable name/filename string compression"), cl::init(true));
+cl::opt<bool> EnableVTableValueProfiling(
+ "enable-vtable-value-profiling", cl::init(true),
+ cl::desc("If true, the virtual table address will be instrumented to know "
+ "the types of a C++ pointer. The information could be used in "
+ "indirect-call-promotion to do selective vtable-based comparison "
+ "and interprocedural type propagation."));
+
std::string getInstrProfSectionName(InstrProfSectKind IPSK,
Triple::ObjectFormatType OF,
bool AddSegmentInfo) {
@@ -498,16 +505,18 @@ Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
finalizeSymtab();
- auto It =
- partition_point(VTableAddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
- return A.first < Address;
- });
+ auto It = lower_bound(
+ VTableAddrRangeToMD5Map, Address,
+ [](std::pair<std::pair<uint64_t, uint64_t>, uint64_t> VTableRangeAddr,
+ uint64_t Addr) { return VTableRangeAddr.first.second < Addr; });
+
+ // Returns the MD5 hash if Address is within the address range of an entry.
+ if (It != VTableAddrRangeToMD5Map.end() && It->first.first <= Address) {
+ return It->second;
+ }
// The virtual table address collected from value profiler could be defined
// in another module that is not instrumented. Force the value to be 0 in
// this case.
- if (It != VTableAddrToMD5Map.end()) {
- return It->second;
- }
return 0;
}
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 844b5397574b5..a405a63fe5d43 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -64,6 +64,10 @@ cl::opt<bool>
DebugInfoCorrelate("debug-info-correlate",
cl::desc("Use debug info to correlate profiles."),
cl::init(false));
+
+// Command line option to enable vtable value profiling. Defined in
+// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
+extern cl::opt<bool> EnableVTableValueProfiling;
} // namespace llvm
namespace {
@@ -561,10 +565,12 @@ bool InstrProfiling::run(
static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
}
- for (GlobalVariable &GV : M.globals()) {
- // Global variables with type metadata are virtual table variables.
- if (GV.hasMetadata(LLVMContext::MD_type)) {
- getOrCreateVTableProfData(&GV);
+ if (EnableVTableValueProfiling) {
+ for (GlobalVariable &GV : M.globals()) {
+ // Global variables with type metadata are virtual table variables.
+ if (GV.hasMetadata(LLVMContext::MD_type)) {
+ getOrCreateVTableProfData(&GV);
+ }
}
}
@@ -1376,7 +1382,7 @@ void InstrProfiling::emitNameData() {
}
void InstrProfiling::emitVTableNames() {
- if (ReferencedVTableNames.empty())
+ if (!EnableVTableValueProfiling || ReferencedVTableNames.empty())
return;
// Collect VTable
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 75c57d4d559e5..810dff726a059 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -327,6 +327,11 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
extern cl::opt<std::string> ViewBlockFreqFuncName;
+extern cl::opt<bool> DebugInfoCorrelate;
+
+// Command line option to enable vtable value profiling. Defined in
+// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
+extern cl::opt<bool> EnableVTableValueProfiling;
} // namespace llvm
static cl::opt<bool>
@@ -584,7 +589,9 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
NumOfPGOBB += MST.BBInfos.size();
ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget);
- ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
+ if (EnableVTableValueProfiling) {
+ ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
+ }
} else {
NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh b/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
index 1b35ea0303d15..5c594e10e7370 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
+++ b/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
@@ -1,12 +1,11 @@
#!/bin/bash
if [ $# -lt 2 ]; then
- echo "Path to clang++ and llvm-profdata required!"
- echo "Usage: update_vtable_value_prof_inputs.sh /path/to/updated/clang++ /path/to/updated/llvm-profdata"
+ echo "Path to clang++ required!"
+ echo "Usage: update_vtable_value_prof_inputs.sh /path/to/updated/clang++"
exit 1
else
CLANG=$1
- LLVMPROFDATA=$2
fi
OUTDIR=$(dirname $(realpath -s $0))
>From 3780bd988618ac3586882f14c1d8a04a9ae9af6f Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Mon, 2 Oct 2023 16:27:49 -0700
Subject: [PATCH 04/15] A few fixes: 1. Format the code indentation in
IndirectCallVisitor.h, and add a FIXME to do more efficient vtable
instrumentation. 2. In InstrProfSymtab::finalizeSymtab, sort and uniquify
VTableAddrRangeToMD5Map (somehow forgot this when preparing this patch,
the full prototype includes profile-use did this already.)
---
.../llvm/Analysis/IndirectCallVisitor.h | 37 +++++++++++--------
llvm/include/llvm/ProfileData/InstrProf.h | 7 ++++
2 files changed, 28 insertions(+), 16 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 347f72bc625cb..7fe9cba884101 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -55,25 +55,30 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
} break;
}
}
- if (Call.isIndirectCall()) {
- IndirectCalls.push_back(&Call);
- if (Type == InstructionType::kVTableVal) {
- LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
- if (LI != nullptr) {
- Value *MaybeVTablePtr =
- LI->getPointerOperand()->stripInBoundsConstantOffsets();
- Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
- // If not used by any type intrinsic, this is not a vtable.
- // Inst visitor should see the very first type intrinsic using a
- // vtable before the very first virtual function load from this
- // vtable. This condition is asserted above.
- if (VTableInstr &&
- PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
- VTableAddrs.insert(VTableInstr);
- }
+ if (Call.isIndirectCall()) {
+ IndirectCalls.push_back(&Call);
+ if (Type == InstructionType::kVTableVal) {
+ // Note without -fstrict-vtable-pointers, vtable pointers of the same
+ // objects are loaded multiple times, and current implementation
+ // instruments each load once.
+ // FIXME: For more efficient instrumentation, analyze load invariant
+ // vtable values (e.g., from the same pointer in C++) and instrument
+ // them once.
+ LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
+ if (LI != nullptr) {
+ Value *MaybeVTablePtr =
+ LI->getPointerOperand()->stripInBoundsConstantOffsets();
+ Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
+ // If not used by any type intrinsic, this is not a vtable.
+ // Inst visitor should see the very first type intrinsic using a
+ // vtable before the very first virtual function load from this
+ // vtable. This condition is asserted above.
+ if (VTableInstr && PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
+ VTableAddrs.insert(VTableInstr);
}
}
}
+ }
}
private:
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 0c7cce9b7355e..ae6828ac7e68d 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -667,6 +667,13 @@ void InstrProfSymtab::finalizeSymtab() {
llvm::sort(AddrToMD5Map, less_first());
AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
AddrToMD5Map.end());
+ // GlobalVariable address ranges should not overlap; so sort by either
+ // beginning address or end address is fine.
+ llvm::sort(VTableAddrRangeToMD5Map, less_first());
+ // std::unique uses == operator for std::pair.
+ VTableAddrRangeToMD5Map.erase(std::unique(VTableAddrRangeToMD5Map.begin(),
+ VTableAddrRangeToMD5Map.end()),
+ VTableAddrRangeToMD5Map.end());
Sorted = true;
}
>From 925586756cbc31eb199b42858de128951c5d5415 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Tue, 3 Oct 2023 22:59:21 -0700
Subject: [PATCH 05/15] resolve feedbacks on code
---
compiler-rt/lib/profile/InstrProfilingMerge.c | 6 ++++--
compiler-rt/lib/profile/InstrProfilingWriter.c | 6 ------
compiler-rt/test/profile/instrprof-write-buffer-internal.c | 5 +++++
llvm/include/llvm/ProfileData/InstrProf.h | 5 -----
llvm/lib/ProfileData/InstrProf.cpp | 5 +++--
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp | 2 --
6 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 629d502cdde31..5f67e25b1edc9 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -124,9 +124,11 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
SrcCountersEnd = SrcCountersStart +
Header->NumCounters * __llvm_profile_counter_entry_size();
SrcNameStart = SrcCountersEnd;
- // This is to assume counter size is a multiple of 8 bytes.
- // First, skip rather than merge them
+ // Skip vtable profile data section and vtable names sections for runtime
+ // profile merge. To merge runtime addresses from multiple profiles, the
+ // same instrumented binary should run with ASLR disabled -> in this set-up
+ // these two sections remain unchanged.
uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData);
uint64_t PaddingBytesAfterVTableSection =
__llvm_profile_get_num_padding_bytes(VTableSectionSize);
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index b998618aad789..18affce48153f 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -327,12 +327,6 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
if (binary_id_size == -1)
return -1;
- // Might be needed for debugging. Clean up before commit.
- // uint64_t VTableProfDataOffset =
- // sizeof(__llvm_profile_header) + binary_id_size + DataSectionSize +
- // PaddingBytesBeforeCounters + CountersSectionSize +
- // PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames;
-
/* Write the profile data. */
ProfDataIOVec IOVecData[] = {
{DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize,
diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
index 484aa1f4f2d0e..7ac65cdd62c98 100644
--- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c
+++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
@@ -54,6 +54,11 @@ int main(int argc, const char *argv[]) {
__llvm_profile_begin_counters(), __llvm_profile_end_counters(),
__llvm_profile_begin_names(), __llvm_profile_end_names());
+ if (ret != 0) {
+ fprintf(stderr, "failed to write buffer");
+ return ret;
+ }
+
FILE *f = fopen(argv[1], "w");
fwrite(buf, bufsize, 1, f);
fclose(f);
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index ae6828ac7e68d..953fd52639183 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -590,11 +590,6 @@ class InstrProfSymtab {
/// this symbol table.
inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash);
- /// Just like getFuncName, except that it will return a non-empty StringRef
- /// if the function is external to this symbol table. All such cases
- /// will be represented using the same StringRef value.
- // inline StringRef getVTableNameOrExternalSymbol(uint64_t VTableMD5Hash);
-
/// True if Symbol is the value used to represent external symbols.
static bool isExternalSymbol(const StringRef &Symbol) {
return Symbol == InstrProfSymtab::getExternalSymbol();
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 54b9e397862b2..311892b068bf1 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -220,7 +220,9 @@ cl::opt<bool> EnableVTableValueProfiling(
cl::desc("If true, the virtual table address will be instrumented to know "
"the types of a C++ pointer. The information could be used in "
"indirect-call-promotion to do selective vtable-based comparison "
- "and interprocedural type propagation."));
+ "and interprocedural type propagation. Requires type metadata and "
+ "type intrinsics (https://llvm.org/docs/TypeMetadata.html) to use"
+ " this option."));
std::string getInstrProfSectionName(InstrProfSectKind IPSK,
Triple::ObjectFormatType OF,
@@ -462,7 +464,6 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) {
Types.clear();
G.getMetadata(LLVMContext::MD_type, Types);
if (!Types.empty()) {
- // errs() << "Insert " << G.getGUID() << "\t into MD5VTableMap\n";
MD5VTableMap.emplace_back(G.getGUID(), &G);
}
}
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index a405a63fe5d43..ae73e0edaf5d7 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1405,8 +1405,6 @@ void InstrProfiling::emitVTableNames() {
VTableNamesVar->setAlignment(Align(1));
// Make VTableNames linker retained.
UsedVars.push_back(VTableNamesVar);
-
- // FIXME: Why emitNames call erase method?
}
void InstrProfiling::emitRegistration() {
>From 8d8a45a2e51e89a8b3053d32e7a92ad4c2758b54 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Fri, 27 Oct 2023 17:31:05 -0700
Subject: [PATCH 06/15] Rebase onto main (a fresh commit on Oct27) and add the
following changes - Set option '-enable-vtable-value-profiling' to false.
This option should make this change no-op. - For indexed profile format,
store vtable names in one section by compressing them and storing blobs
strings. - Before this commit, they are stored as OnDiskIterableHashTable
but hash table is not necessary; vtable names are stored in indexed format
profiles for llvm-profdata usage. Function profiles are on-disk hash table so
compiler could look up profiles faster. Clean up lookuptrait and writer trait
as a result. - In index profile reader, track {VTableNamePtr,
CompressedVTableNameLen} as metadata so llvm-profdata could construct symtab
by decompressing vtable names. Meanwhile, when compiler reads indexed
profiles, no need to decompress vtables since symtab could be constructed
from module IR. - Use the PGOName of vtables when computing compressed
strings and MD5 hash in instrumentation stage. - Previously, the vtable
name is used. The PGOName and name of vtables are the same for global
symbols. For local symbols, PGOName has the format 'filename:varname' - In
IndirectCallVisitor.h, use a heuristc to find address feeding instructions.
Currently clang doesn't emit type intrinsics without
'-fwhole-program-vtables'. '-fwhole-program-vtables' requires '-flto', and
'-flto' might not be enabled at instrumentation phase. - In
ValueProfilePlugins.inc, skip landingpad and catchpad instructions (to keep
them the first non-phi) when finding an insertion point for value profile
intrinsic. - In InstrProf.cpp, simplify string decompression code by adding
helper functions. - In llvm-profdata.cpp, print the number of instrumented
vtable sites and vpstats.
---
compiler-rt/include/profile/InstrProfData.inc | 2 +-
.../lib/profile/InstrProfilingBuffer.c | 39 ++++-----
.../lib/profile/InstrProfilingInternal.h | 4 +-
compiler-rt/lib/profile/InstrProfilingMerge.c | 39 +++++----
.../lib/profile/InstrProfilingWriter.c | 2 +-
.../profile/instrprof-write-buffer-internal.c | 7 +-
.../llvm/Analysis/IndirectCallVisitor.h | 80 +++++++----------
llvm/include/llvm/ProfileData/InstrProf.h | 70 ++++++++-------
.../llvm/ProfileData/InstrProfData.inc | 2 +-
.../llvm/ProfileData/InstrProfReader.h | 72 ++++-----------
.../Instrumentation/InstrProfiling.h | 2 +-
llvm/lib/ProfileData/InstrProf.cpp | 78 +++++++----------
llvm/lib/ProfileData/InstrProfReader.cpp | 37 ++++----
llvm/lib/ProfileData/InstrProfWriter.cpp | 87 ++++++-------------
.../Instrumentation/InstrProfiling.cpp | 12 +--
.../Instrumentation/ValueProfilePlugins.inc | 8 +-
.../indirect_call_profile_funclet.ll | 9 +-
.../Transforms/PGOProfile/vtable_profile.ll | 4 +-
.../vtable-value-prof-basic.test | 39 ++++++---
llvm/tools/llvm-profdata/llvm-profdata.cpp | 23 +++--
20 files changed, 264 insertions(+), 352 deletions(-)
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index 64d9bad24524d..2753ad66ffe4c 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -101,7 +101,7 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
#define INSTR_PROF_VTABLE_DATA_DEFINED
#endif
INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, \
- ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(VTableName)))
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(PGOVTableName)))
INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), VTablePointer, VTableAddr)
INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), VTableSizeVal))
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index ad1643d48908d..34007cbf726dd 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -47,14 +47,9 @@ uint64_t __llvm_profile_get_size_for_buffer(void) {
const char *CountersEnd = __llvm_profile_end_counters();
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
- const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
- const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
- const char *VNamesBegin = __llvm_profile_begin_vtabnames();
- const char *VNamesEnd = __llvm_profile_end_vtabnames();
return __llvm_profile_get_size_for_buffer_internal(
- DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd,
- VTableBegin, VTableEnd, VNamesBegin, VNamesEnd);
+ DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd);
}
COMPILER_RT_VISIBILITY
@@ -132,8 +127,11 @@ void __llvm_profile_get_padding_sizes_for_counters(
*PaddingBytesAfterCounters =
__llvm_profile_get_num_padding_bytes(CountersSize);
*PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
- *PaddingBytesAfterVTable = __llvm_profile_get_num_padding_bytes(VTableSize);
- *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);
+ if (PaddingBytesAfterVTable != NULL)
+ *PaddingBytesAfterVTable =
+ __llvm_profile_get_num_padding_bytes(VTableSize);
+ if (PaddingBytesAfterVName != NULL)
+ *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);
return;
}
@@ -150,41 +148,36 @@ void __llvm_profile_get_padding_sizes_for_counters(
*PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
// Set these two variables to zero to avoid uninitialized variables
// even if VTableSize and VNameSize are asserted to be zero.
- *PaddingBytesAfterVTable = 0;
- *PaddingBytesAfterVName = 0;
+ if (PaddingBytesAfterVTable != NULL)
+ *PaddingBytesAfterVTable = 0;
+ if (PaddingBytesAfterVName != NULL)
+ *PaddingBytesAfterVName = 0;
}
COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
- const char *NamesEnd, const VTableProfData *VTableBegin,
- const VTableProfData *VTableEnd, const char *VNamesBegin,
- const char *VNamesEnd) {
+ const char *NamesEnd) {
/* Match logic in __llvm_profile_write_buffer(). */
const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
uint64_t CountersSize =
__llvm_profile_get_counters_size(CountersBegin, CountersEnd);
- uint64_t VTableSize =
- __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
- uint64_t VNameSize = (VNamesEnd - VNamesBegin) * sizeof(char);
/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
+ PaddingBytesAfterNames;
__llvm_profile_get_padding_sizes_for_counters(
- DataSize, CountersSize, NamesSize, VTableSize, VNameSize,
+ DataSize, CountersSize, NamesSize, 0 /* VTableSize */, 0 /* VNameSize */,
&PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
- &PaddingBytesAfterNames, &PaddingBytesAfterVTable,
- &PaddingBytesAfterVNames);
+ &PaddingBytesAfterNames, NULL /* PaddingBytesAfterVTable */,
+ NULL /* PaddingBytesAfterVNames */);
return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
DataSize + PaddingBytesBeforeCounters + CountersSize +
- PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames +
- VTableSize + PaddingBytesAfterVTable + VNameSize +
- PaddingBytesAfterVNames;
+ PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames;
}
COMPILER_RT_VISIBILITY
diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h
index bce333f933f0f..09a77acef97b2 100644
--- a/compiler-rt/lib/profile/InstrProfilingInternal.h
+++ b/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -25,9 +25,7 @@
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
- const char *NamesEnd, const VTableProfData *VTableBegin,
- const VTableProfData *VTableEnd, const char *VNamesBegin,
- const char *VNamesEnd);
+ const char *NamesEnd);
/*!
* \brief Write instrumentation data to the given buffer, given explicit
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 5f67e25b1edc9..32b3058535ede 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -98,6 +98,25 @@ static uintptr_t signextIfWin64(void *V) {
#endif
}
+static uint64_t
+getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
+ // Skip names section, vtable profile data section and vtable names section
+ // for runtime profile merge. To merge runtime addresses from multiple
+ // profiles, the same instrumented binary should run with ASLR disabled -> in
+ // this set-up these three sections remain unchanged.
+ const uint64_t VTableSectionSize =
+ Header->NumVTables * sizeof(VTableProfData);
+ const uint64_t PaddingBytesAfterVTableSection =
+ __llvm_profile_get_num_padding_bytes(VTableSectionSize);
+ const uint64_t VNamesSize = Header->VNamesSize;
+ const uint64_t PaddingBytesAfterVNamesSize =
+ __llvm_profile_get_num_padding_bytes(VNamesSize);
+ return Header->NamesSize +
+ __llvm_profile_get_num_padding_bytes(Header->NamesSize) +
+ VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize +
+ PaddingBytesAfterVNamesSize;
+}
+
COMPILER_RT_VISIBILITY
int __llvm_profile_merge_from_buffer(const char *ProfileData,
uint64_t ProfileSize) {
@@ -125,26 +144,8 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
Header->NumCounters * __llvm_profile_counter_entry_size();
SrcNameStart = SrcCountersEnd;
- // Skip vtable profile data section and vtable names sections for runtime
- // profile merge. To merge runtime addresses from multiple profiles, the
- // same instrumented binary should run with ASLR disabled -> in this set-up
- // these two sections remain unchanged.
- uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData);
- uint64_t PaddingBytesAfterVTableSection =
- __llvm_profile_get_num_padding_bytes(VTableSectionSize);
- uint64_t VNamesSize = Header->VNamesSize;
- uint64_t PaddingBytesAfterVNamesSize =
- __llvm_profile_get_num_padding_bytes(VNamesSize);
-
- uint64_t VTableProfDataOffset =
- SrcNameStart + Header->NamesSize +
- __llvm_profile_get_num_padding_bytes(Header->NamesSize);
-
- uint64_t VTableNamesOffset =
- VTableProfDataOffset + VTableSectionSize + PaddingBytesAfterVTableSection;
-
SrcValueProfDataStart =
- VTableNamesOffset + VNamesSize + PaddingBytesAfterVNamesSize;
+ SrcNameStart + getDistanceFromCounterToValueProf(Header);
if (SrcNameStart < SrcCountersStart)
return 1;
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 18affce48153f..c96d3f17a3269 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -283,7 +283,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
__llvm_profile_get_num_vtable(VTableBegin, VTableEnd);
const uint64_t VTableSectionSize =
__llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
- // Note vtable profiling is not supported when DebugInfoCorrelate is true.
+ // Value profiling is not supported when DebugInfoCorrelate is true.
const uint64_t VNamesSize = DebugInfoCorrelate ? 0 : VNamesEnd - VNamesBegin;
/* Create the header. */
diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
index 7ac65cdd62c98..7b96c6d91c33f 100644
--- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c
+++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c
@@ -28,9 +28,7 @@ char *__llvm_profile_end_counters(void);
uint64_t __llvm_profile_get_size_for_buffer_internal(
const void *DataBegin, const void *DataEnd, const char *CountersBegin,
- const char *CountersEnd, const char *NamesBegin, const char *NamesEnd,
- const char *VTableBegin, const char *VTableEnd, const char *VNamesBegin,
- const char *VNamesEnd);
+ const char *CountersEnd, const char *NamesBegin, const char *NamesEnd);
int __llvm_profile_write_buffer_internal(char *Buffer, const void *DataBegin,
const void *DataEnd,
@@ -45,8 +43,7 @@ int main(int argc, const char *argv[]) {
uint64_t bufsize = __llvm_profile_get_size_for_buffer_internal(
__llvm_profile_begin_data(), __llvm_profile_end_data(),
__llvm_profile_begin_counters(), __llvm_profile_end_counters(),
- __llvm_profile_begin_names(), __llvm_profile_end_names(), NULL, NULL,
- NULL, NULL);
+ __llvm_profile_begin_names(), __llvm_profile_end_names());
char *buf = malloc(bufsize);
int ret = __llvm_profile_write_buffer_internal(buf,
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 7fe9cba884101..4a3e5b61e14f3 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -25,66 +25,44 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
kVTableVal = 1,
};
std::vector<CallBase *> IndirectCalls;
- SetVector<Instruction *, std::vector<Instruction *>> VTableAddrs;
+ std::vector<Instruction *> ProfiledAddresses;
PGOIndirectCallVisitor(InstructionType Type) : Type(Type) {}
void visitCallBase(CallBase &Call) {
- const CallInst *CI = dyn_cast<CallInst>(&Call);
- if (Type == InstructionType::kVTableVal && CI && CI->getCalledFunction()) {
- switch (CI->getCalledFunction()->getIntrinsicID()) {
- case Intrinsic::type_test:
- case Intrinsic::public_type_test:
- case Intrinsic::type_checked_load_relative:
- case Intrinsic::type_checked_load: {
- Value *VTablePtr = CI->getArgOperand(0)->stripPointerCasts();
-
- if (PtrTestedByTypeIntrinsics.count(VTablePtr) == 0) {
- Instruction *I = dyn_cast_or_null<Instruction>(VTablePtr);
- // This is the first type intrinsic where VTablePtr is used.
- // Assert that the VTablePtr is not found as a type profiling
- // candidate yet. Note nullptr won't be inserted into VTableAddrs in
- // the first place, so this assertion works even if 'VTablePtr' is not
- // an instruction.
- assert(VTableAddrs.count(I) == 0 &&
- "Expect type intrinsic to record VTablePtr before virtual "
- "functions are loaded to find vtables that should be "
- "instrumented");
-
- PtrTestedByTypeIntrinsics.insert(VTablePtr);
- }
- } break;
- }
- }
if (Call.isIndirectCall()) {
IndirectCalls.push_back(&Call);
- if (Type == InstructionType::kVTableVal) {
- // Note without -fstrict-vtable-pointers, vtable pointers of the same
- // objects are loaded multiple times, and current implementation
- // instruments each load once.
- // FIXME: For more efficient instrumentation, analyze load invariant
- // vtable values (e.g., from the same pointer in C++) and instrument
- // them once.
- LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
- if (LI != nullptr) {
- Value *MaybeVTablePtr =
- LI->getPointerOperand()->stripInBoundsConstantOffsets();
- Instruction *VTableInstr = dyn_cast<Instruction>(MaybeVTablePtr);
- // If not used by any type intrinsic, this is not a vtable.
- // Inst visitor should see the very first type intrinsic using a
- // vtable before the very first virtual function load from this
- // vtable. This condition is asserted above.
- if (VTableInstr && PtrTestedByTypeIntrinsics.count(MaybeVTablePtr)) {
- VTableAddrs.insert(VTableInstr);
- }
+
+ if (Type != InstructionType::kVTableVal)
+ return;
+
+ LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
+ // The code pattern to look for
+ //
+ // %vtable = load ptr, ptr %b
+ // %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1
+ // %2 = load ptr, ptr %vfn
+ // %call = tail call i32 %2(ptr %b)
+ //
+ // %vtable is the vtable address value to profile, and
+ // %2 is the indirect call target address to profile.
+ if (LI != nullptr) {
+ Value *Ptr = LI->getPointerOperand();
+ Value *VTablePtr = Ptr->stripInBoundsConstantOffsets();
+ // This is a heuristic to find address feeding instructions.
+ // FIXME: Add support in the frontend so LLVM type intrinsics are
+ // emitted without LTO. This way, added intrinsics could filter
+ // non-vtable instructions and reduce instrumentation overhead.
+ // Note a profiled address will be dropped if it's not within
+ // the address range of vtable objects, so there are no correctness
+ // or performance issues at profile use time.
+ if (VTablePtr != nullptr && isa<Instruction>(VTablePtr)) {
+ ProfiledAddresses.push_back(cast<Instruction>(VTablePtr));
}
}
}
}
private:
- // Keeps track of the pointers that are tested by llvm type intrinsics for
- // look up.
- SmallPtrSet<Value *, 4> PtrTestedByTypeIntrinsics;
InstructionType Type;
};
@@ -99,9 +77,9 @@ inline std::vector<Instruction *> findVTableAddrs(Function &F) {
PGOIndirectCallVisitor ICV(
PGOIndirectCallVisitor::InstructionType::kVTableVal);
ICV.visit(F);
- return ICV.VTableAddrs.takeVector();
+ return ICV.ProfiledAddresses;
}
} // namespace llvm
-#endif
+#endif
\ No newline at end of file
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 953fd52639183..d289a4b633379 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -250,10 +250,10 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
/// \c NameStrings is a string composed of one of more sub-strings encoded in
/// the format described above. The substrings are separated by 0 or more zero
-/// bytes. This method decodes the string and populates the \c Symtab.
-Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);
-
-Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab);
+/// bytes. This method decodes the string and calls `NameCallback` for each
+/// substring.
+Error readPGOFuncNameStrings(StringRef NameStrings,
+ std::function<Error(StringRef)> NameCallback);
/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
/// set in IR PGO compilation.
@@ -297,6 +297,8 @@ inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
/// Return the PGOFuncName meta data associated with a function.
MDNode *getPGOFuncNameMetadata(const Function &F);
+std::string getPGOName(const GlobalVariable &V, bool InLTO = false);
+
/// Create the PGOFuncName meta data if PGOFuncName is different from
/// function's raw name. This should only apply to internal linkage functions
/// declared by users only.
@@ -502,6 +504,9 @@ class InstrProfSymtab {
inline Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
+ inline Error
+ initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
+
/// A wrapper interface to populate the PGO symtab with functions
/// decls from module \c M. This interface is used by transformation
/// passes such as indirect function call promotion. Variable \c InLTO
@@ -519,39 +524,33 @@ class InstrProfSymtab {
Error create(const FuncNameIterRange &FuncIterRange,
const VTableNameIterRange &VTableIterRange);
- /// Update the symtab by adding \p FuncName to the table. This interface
- /// is used by the raw and text profile readers.
- Error addFuncName(StringRef FuncName) {
- if (FuncName.empty())
+ Error addSymbolName(StringRef SymbolName) {
+ if (SymbolName.empty())
return make_error<InstrProfError>(instrprof_error::malformed,
- "function name is empty");
- auto Ins = NameTab.insert(FuncName);
+ "symbol name is empty");
+
+ // Insert into NameTab so that MD5NameMap (a vector that will be sorted)
+ // won't have duplicated entries in the first place.
+ auto Ins = NameTab.insert(SymbolName);
if (Ins.second) {
MD5NameMap.push_back(std::make_pair(
- IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey()));
+ IndexedInstrProf::ComputeHash(SymbolName), Ins.first->getKey()));
Sorted = false;
}
return Error::success();
}
+ /// The method name is kept since there are many callers.
+ // It just forwards to 'addSymbolName'.
+ Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); }
+
Error addVTableName(StringRef VTableName) {
- if (VTableName.empty())
- return make_error<InstrProfError>(instrprof_error::malformed,
- "invalid input: VTableName is empty");
- // Insert into NameTab so that MD5NameMap (a vector that is going to be
- // sorted) won't have duplicated entries in the first place.
- auto Ins = NameTab.insert(VTableName);
+ if (Error E = addSymbolName(VTableName))
+ return E;
// Record VTableName. InstrProfWriter uses this map. The comment around
// class member explains why.
VTableNames.insert(VTableName);
-
- // If this is newly added, update MD5NameMap.
- if (Ins.second) {
- MD5NameMap.push_back(std::make_pair(
- IndexedInstrProf::ComputeHash(VTableName), Ins.first->getKey()));
- Sorted = false;
- }
return Error::success();
}
@@ -613,18 +612,31 @@ Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
return Error::success();
}
+// FIXME: Move 'InstrProfSymtab::create' definition into cpp,
+// and move 'readPGOFuncNameStrings' inside cpp.
Error InstrProfSymtab::create(StringRef NameStrings) {
- return readPGOFuncNameStrings(NameStrings, *this);
+ return readPGOFuncNameStrings(
+ NameStrings,
+ std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
}
Error InstrProfSymtab::create(StringRef FuncNameStrings,
StringRef VTableNameStrings) {
- if (Error E = readPGOFuncNameStrings(FuncNameStrings, *this))
+ if (Error E = readPGOFuncNameStrings(FuncNameStrings,
+ std::bind(&InstrProfSymtab::addFuncName,
+ this, std::placeholders::_1)))
return E;
- // FIXME: Add test coverage that this returns success when VTableNameStrings
- // is empty.
- return readVTableNames(VTableNameStrings, *this);
+ return readPGOFuncNameStrings(
+ VTableNameStrings,
+ std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+}
+
+Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
+ StringRef CompressedVTableStrings) {
+ return readPGOFuncNameStrings(
+ CompressedVTableStrings,
+ std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
}
template <typename NameIterRange>
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index 4c8f1b31c5e8d..5785f2c437fb9 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -97,7 +97,7 @@ INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
#define INSTR_PROF_VTABLE_DATA_DEFINED
#endif
INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, \
- ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(VTableName)))
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), IndexedInstrProf::ComputeHash(PGOVTableName)))
INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), VTablePointer, VTableAddr)
INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), VTableSizeVal))
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index d7009bbe9dc3f..0e150b1ce011d 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -472,46 +472,6 @@ enum class HashT : uint32_t;
} // end namespace IndexedInstrProf
-class InstrProfVTableLookupTrait {
- char val;
- IndexedInstrProf::HashT HashType;
- unsigned FormatVersion;
-
-public:
- InstrProfVTableLookupTrait(IndexedInstrProf::HashT HashType,
- unsigned FormatVersion)
- : HashType(HashType), FormatVersion(FormatVersion) {}
-
- using data_type = char;
-
- using internal_key_type = StringRef;
- using external_key_type = StringRef;
-
- using hash_value_type = uint64_t;
- using offset_type = uint64_t;
-
- static bool EqualKey(StringRef A, StringRef B) { return A == B; }
- static StringRef GetInternalKey(StringRef K) { return K; }
- static StringRef GetExternalKey(StringRef K) { return K; }
-
- hash_value_type ComputeHash(StringRef K);
-
- static std::pair<offset_type, offset_type>
- ReadKeyDataLength(const unsigned char *&D) {
- using namespace support;
-
- offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D);
- offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D);
- return std::make_pair(KeyLen, DataLen);
- }
-
- StringRef ReadKey(const unsigned char *D, offset_type N) {
- return StringRef((const char *)D, N);
- }
-
- data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
-};
-
/// Trait for lookups into the on-disk hash table for the binary instrprof
/// format.
class InstrProfLookupTrait {
@@ -565,9 +525,6 @@ class InstrProfLookupTrait {
}
};
-using VirtualTableNamesHashTable =
- OnDiskIterableChainedHashTable<InstrProfVTableLookupTrait>;
-
struct InstrProfReaderIndexBase {
virtual ~InstrProfReaderIndexBase() = default;
@@ -590,10 +547,7 @@ struct InstrProfReaderIndexBase {
virtual bool hasMemoryProfile() const = 0;
virtual bool hasTemporalProfile() const = 0;
virtual InstrProfKind getProfileKind() const = 0;
- // The pointer VirtualTableIndex is not owned.
- virtual Error
- populateSymtab(InstrProfSymtab &,
- VirtualTableNamesHashTable *VirtualTableIndex) = 0;
+ virtual Error populateSymtab(InstrProfSymtab &) = 0;
};
using OnDiskHashTableImplV3 =
@@ -668,10 +622,13 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase {
InstrProfKind getProfileKind() const override;
- Error populateSymtab(InstrProfSymtab &Symtab,
- VirtualTableNamesHashTable *VirtualTableIndex) override {
- if (VirtualTableIndex != nullptr)
- return Symtab.create(HashTable->keys(), VirtualTableIndex->keys());
+ Error populateSymtab(InstrProfSymtab &Symtab) override {
+ // FIXME: the create method calls 'finalizeSymtab' and sorts a bunch of
+ // arrays/maps. Since there are other data sources other than 'HashTable' to
+ // populate a symtab, it might make sense to have something like this
+ // 1. Let each data source populate Symtab and init the arrays/maps without
+ // calling 'finalizeSymtab'
+ // 2. Call 'finalizeSymtab' once to get all arrays/maps sorted if needed.
return Symtab.create(HashTable->keys());
}
};
@@ -706,8 +663,17 @@ class IndexedInstrProfReader : public InstrProfReader {
std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable;
/// MemProf frame profile data on-disk indexed via frame id.
std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable;
- /// Virtual table profile data indexed .
- std::unique_ptr<VirtualTableNamesHashTable> VirtualTableIndex = nullptr;
+ /// The reader itself doesn't decompress vtable names.
+ /// A compiler that reads indexed profiles could construct
+ /// symtab from module IR so it doesn't need the decompressed
+ /// names.
+ /// When a symtab is constructed (from llvm-profdata),
+ /// the symtab decompress the list of names based on
+ /// `VTableNamePtr` and `CompressedVTableNamesLen`.
+ /// Points to the beginning of compressed vtable names.
+ const char *VTableNamePtr = nullptr;
+ /// The length of compressed vtable names.
+ uint64_t CompressedVTableNamesLen = 0;
/// Total size of binary ids.
uint64_t BinaryIdsSize{0};
/// Start address of binary id length and data pairs.
diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
index 9af090c0c2e68..180dd23dd82a2 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
@@ -67,7 +67,7 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
std::vector<GlobalVariable *> ReferencedNames;
// The list of virtual table variables of which the VTableProfData is
// collected.
- std::vector<GlobalVariable *> ReferencedVTableNames;
+ std::vector<GlobalVariable *> ReferencedVTables;
GlobalVariable *NamesVar;
size_t NamesSize;
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 311892b068bf1..8774f2f796df0 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -216,7 +216,7 @@ cl::opt<bool> DoInstrProfNameCompression(
cl::desc("Enable name/filename string compression"), cl::init(true));
cl::opt<bool> EnableVTableValueProfiling(
- "enable-vtable-value-profiling", cl::init(true),
+ "enable-vtable-value-profiling", cl::init(false),
cl::desc("If true, the virtual table address will be instrumented to know "
"the types of a C++ pointer. The information could be used in "
"indirect-call-promotion to do selective vtable-based comparison "
@@ -340,8 +340,8 @@ static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO,
}
// In LTO mode (when InLTO is true), first check if there is a meta data.
- if (auto IRPGOFuncName = lookupPGONameFromMetadata(PGONameMetadata))
- return *IRPGOFuncName;
+ if (auto IRPGOName = lookupPGONameFromMetadata(PGONameMetadata))
+ return *IRPGOName;
// If there is no meta data, the function must be a global before the value
// profile annotation pass. Its current linkage may be internal if it is
@@ -355,24 +355,36 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO) {
return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F));
}
-// This is similar to `getIRPGOFuncName` except that this function calls
-// 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls
-// 'getIRPGONameForGlobalObject'. See the difference between two callees in the
-// comments of `getIRPGONameForGlobalObject`.
-std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
+static std::string getPGOObjectName(const GlobalObject &GO, bool InLTO,
+ MDNode *PGONameMetadata) {
if (!InLTO) {
- auto FileName = getStrippedSourceFileName(F);
- return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
+ auto FileName = getStrippedSourceFileName(GO);
+ return getPGOFuncName(GO.getName(), GO.getLinkage(), FileName);
}
// In LTO mode (when InLTO is true), first check if there is a meta data.
- if (auto PGOFuncName = lookupPGONameFromMetadata(getPGOFuncNameMetadata(F)))
- return *PGOFuncName;
+ if (auto PGOName = lookupPGONameFromMetadata(PGONameMetadata))
+ return *PGOName;
// If there is no meta data, the function must be a global before the value
// profile annotation pass. Its current linkage may be internal if it is
// internalized in LTO mode.
- return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
+ return getPGOFuncName(GO.getName(), GlobalValue::ExternalLinkage, "");
+}
+
+// This is similar to `getIRPGOFuncName` except that this function calls
+// 'getPGOObjectcName' to get a name and `getIRPGOFuncName` calls
+// 'getIRPGONameForGlobalObject'. See the difference between two callees in the
+// comments of `getIRPGONameForGlobalObject`.
+std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
+ return getPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F));
+}
+
+std::string getPGOName(const GlobalVariable &V, bool InLTO) {
+ // PGONameMetadata should be set by compiler at profile use time
+ // and read by symtab creation to look up symbols corresponding to
+ // a MD5 hash.
+ return getPGOObjectName(V, InLTO, nullptr /* PGONameMetadata */);
}
// See getIRPGOFuncName() for a discription of the format.
@@ -601,9 +613,9 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
std::string &Result, bool doCompression) {
std::vector<std::string> VTableNameStrs;
for (auto *VTable : VTables) {
- VTableNameStrs.push_back(std::string(VTable->getName()));
+ VTableNameStrs.push_back(getPGOName(*VTable));
}
- return collectPGOFuncNameStrings(
+ return collectGlobalObjectNameStrings(
VTableNameStrs, compression::zlib::isAvailable() && doCompression,
Result);
}
@@ -642,7 +654,8 @@ instrprof_error decodeAndSplitStrings(
return instrprof_error::success;
}
-Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
+Error readPGOFuncNameStrings(StringRef NameStrings,
+ std::function<Error(StringRef)> NameCallback) {
const uint8_t *P = NameStrings.bytes_begin();
const uint8_t *EndP = NameStrings.bytes_end();
while (P < EndP) {
@@ -663,7 +676,7 @@ Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
NameStrings.split(Names, getInstrProfNameSeparator());
for (StringRef &Name : Names) {
- if (Error E = Symtab.addFuncName(Name))
+ if (Error E = NameCallback(Name))
return E;
}
@@ -675,37 +688,6 @@ Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
return Error::success();
}
-Error readVTableNames(StringRef NameStrings, InstrProfSymtab &Symtab) {
- const uint8_t *P = NameStrings.bytes_begin();
- const uint8_t *EndP = NameStrings.bytes_end();
- while (P < EndP) {
- // Now parse the name strings.
- uint32_t Dist = 0;
- StringRef NameStrings;
- SmallVector<uint8_t, 128> UncompressedNameStrings;
- SmallVector<StringRef, 0> Names;
- bool isCompressed = false;
- instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
- NameStrings, Dist, isCompressed);
- if (E != instrprof_error::success)
- return make_error<InstrProfError>(E);
-
- if (isCompressed) {
- NameStrings = toStringRef(UncompressedNameStrings);
- }
- NameStrings.split(Names, getInstrProfNameSeparator());
- for (StringRef &Name : Names) {
- if (Error E = Symtab.addVTableName(Name))
- return E;
- }
-
- P += Dist;
- while (P < EndP && *P == 0)
- P++;
- }
- return Error::success();
-}
-
void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
uint64_t FuncSum = 0;
Sum.NumEntries += Counts.size();
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 59e7ef9466863..e1ea44cb127fc 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -878,19 +878,6 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
return DataBuffer;
}
-InstrProfVTableLookupTrait::hash_value_type
-InstrProfVTableLookupTrait::ComputeHash(StringRef K) {
- return IndexedInstrProf::ComputeHash(HashType, K);
-}
-
-InstrProfVTableLookupTrait::data_type
-InstrProfVTableLookupTrait::ReadData(StringRef K, const unsigned char *D,
- offset_type N) {
- char v =
- support::endian::readNext<char, support::little, support::unaligned>(D);
- return v;
-}
-
template <typename HashTableImpl>
Error InstrProfReaderIndex<HashTableImpl>::getRecords(
StringRef FuncName, ArrayRef<NamedInstrProfRecord> &Data) {
@@ -1220,17 +1207,15 @@ Error IndexedInstrProfReader::readHeader() {
if (GET_VERSION(Header->formatVersion()) >= 11) {
uint64_t VTableNamesOffset =
- endian::byte_swap<uint64_t, little>(Header->VTableNamesOffset);
+ endian::byte_swap<uint64_t, llvm::endianness::little>(
+ Header->VTableNamesOffset);
const unsigned char *Ptr = Start + VTableNamesOffset;
- const uint64_t HashTableMetadataOffset =
- support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+ CompressedVTableNamesLen =
+ support::endian::readNext<uint64_t, llvm::endianness::little,
+ unaligned>(Ptr);
- VirtualTableIndex.reset(VirtualTableNamesHashTable::Create(
- /*Bucket=*/Start + HashTableMetadataOffset,
- /*Payload=*/Ptr,
- /*Base=*/Start,
- InstrProfVTableLookupTrait(HashType, Header->formatVersion())));
+ VTableNamePtr = (const char *)Ptr;
}
if (GET_VERSION(Header->formatVersion()) >= 10 &&
@@ -1293,7 +1278,15 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
return *Symtab;
std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
- if (Error E = Index->populateSymtab(*NewSymtab, VirtualTableIndex.get())) {
+
+ if (Error E = NewSymtab->initVTableNamesFromCompressedStrings(
+ StringRef(VTableNamePtr, CompressedVTableNamesLen))) {
+ auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
+ consumeError(error(ErrCode, Msg));
+ }
+
+ // finalizeSymtab is called inside populateSymtab.
+ if (Error E = Index->populateSymtab(*NewSymtab)) {
auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
consumeError(error(ErrCode, Msg));
}
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 0713864c21d97..183c858448437 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -19,6 +19,7 @@
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/MemProf.h"
#include "llvm/ProfileData/ProfileCommon.h"
+#include "llvm/Support/Compression.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/EndianStream.h"
#include "llvm/Support/Error.h"
@@ -171,47 +172,6 @@ class InstrProfRecordWriterTrait {
}
};
-class InstrProfRecordVTableTrait {
-public:
- using key_type = StringRef;
- using key_type_ref = StringRef;
-
- using data_type = char;
- using data_type_ref = char;
-
- using hash_value_type = uint64_t;
- using offset_type = uint64_t;
-
- InstrProfRecordVTableTrait() = default;
-
- static hash_value_type ComputeHash(key_type_ref K) {
- return IndexedInstrProf::ComputeHash(K);
- }
-
- static std::pair<offset_type, offset_type>
- EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
- using namespace support;
-
- endian::Writer LE(Out, little);
-
- offset_type N = K.size();
- LE.write<offset_type>(N);
-
- offset_type M = 1;
- LE.write<offset_type>(M);
-
- return std::make_pair(N, M);
- }
-
- void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
- Out.write(K.data(), N);
- }
-
- void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
- Out.write(&V, 1);
- }
-};
-
} // end namespace llvm
InstrProfWriter::InstrProfWriter(bool Sparse,
@@ -639,29 +599,36 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
if (IndexedInstrProf::ProfVersion::CurrentVersion >= 11) {
VTableNamesSectionStart = OS.tell();
- // Reserve space for vtable records offset.
- OS.write(0ULL);
-
- OnDiskChainedHashTableGenerator<llvm::InstrProfRecordVTableTrait>
- VTableNamesGenerator;
- for (const auto &kv : VTableNames) {
- // Use a char '0' as value placeholder, only keys (vtable names)
- // are used.
- // FIXME: It might make sense to have a OnDiskChainedHashSetGenerator if
- // there are more use cases. Use a hash table for now, with one unused
- // 'char' per entry.
- VTableNamesGenerator.insert(kv.getKey(), '0');
+ std::string CompressedVTableNames;
+
+ std::vector<std::string> VTableNameStrs;
+ for (const auto &VTableName : VTableNames.keys()) {
+ VTableNameStrs.push_back(VTableName.str());
}
- auto VTableNamesWriter =
- std::make_unique<llvm::InstrProfRecordVTableTrait>();
+ if (!VTableNameStrs.empty()) {
+ if (Error E = collectGlobalObjectNameStrings(
+ VTableNameStrs, compression::zlib::isAvailable(),
+ CompressedVTableNames))
+ return E;
+ }
- uint64_t VTableNamesTableOffset =
- VTableNamesGenerator.Emit(OS.OS, *VTableNamesWriter);
+ uint64_t CompressedStringLen = CompressedVTableNames.length();
- PatchItem PatchItems[] = {
- {VTableNamesSectionStart, &VTableNamesTableOffset, 1}};
- OS.patch(PatchItems, 1);
+ // Record the length of compressed string.
+ OS.write(CompressedStringLen);
+
+ // Write the chars in compressed strings.
+ for (auto &c : CompressedVTableNames)
+ OS.writeByte(static_cast<uint8_t>(c));
+
+ // Pad up to a multiple of 8.
+ // InstrProfReader could read bytes according to 'CompressedStringLen'.
+ uint64_t PaddedLength = alignTo(CompressedStringLen, 8);
+
+ for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) {
+ OS.writeByte(0);
+ }
}
uint64_t TemporalProfTracesSectionStart = 0;
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index ae73e0edaf5d7..b11df94f7138b 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1035,6 +1035,7 @@ void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
// Used by INSTR_PROF_VTABLE_DATA MACRO
Constant *VTableAddr = getVTableAddrForProfData(GV);
StringRef VTableName = GV->getName();
+ StringRef PGOVTableName = getPGOName(*GV);
// Record the length of the vtable. This is needed since vtable pointers
// loaded from C++ objects might be from the middle of a vtable definition.
uint32_t VTableSizeVal =
@@ -1087,7 +1088,7 @@ void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
VTableDataMap[GV] = Data;
- ReferencedVTableNames.push_back(GV);
+ ReferencedVTables.push_back(GV);
// VTable <Hash, Addr> is used by runtime but not referenced by other
// sections. Conservatively mark it linker retained.
@@ -1382,14 +1383,13 @@ void InstrProfiling::emitNameData() {
}
void InstrProfiling::emitVTableNames() {
- if (!EnableVTableValueProfiling || ReferencedVTableNames.empty())
+ if (!EnableVTableValueProfiling || ReferencedVTables.empty())
return;
- // Collect VTable
+ // Collect the PGO names of referenced vtables and compress them.
std::string CompressedVTableNames;
- if (Error E =
- collectVTableStrings(ReferencedVTableNames, CompressedVTableNames,
- DoInstrProfNameCompression)) {
+ if (Error E = collectVTableStrings(ReferencedVTables, CompressedVTableNames,
+ DoInstrProfNameCompression)) {
report_fatal_error(Twine(toString(std::move(E))), false);
}
diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
index 59d51a59c48b7..fea122adc1fad 100644
--- a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
+++ b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
@@ -103,9 +103,13 @@ public:
std::vector<Instruction *> Result = findVTableAddrs(F);
for (Instruction* I : Result) {
Instruction* InsertPt = I->getNextNonDebugInstruction();
- while(InsertPt && dyn_cast<PHINode>(InsertPt))
+ // When finding an insertion point, keep PHI, landing pad and catch pad before vp intrinsics.
+ while(InsertPt && (dyn_cast<PHINode>(InsertPt) || dyn_cast<LandingPadInst>(InsertPt) || dyn_cast<CatchPadInst>(InsertPt)))
InsertPt = InsertPt->getNextNonDebugInstruction();
- assert(InsertPt);
+ // FIXME: Set InsertPt to the end of basic block if
+ // InsertPt is the last instruction.
+ if (InsertPt == nullptr)
+ continue;
Instruction *AnnotatedInst = I;
Candidates.emplace_back(CandidateInfo{I, InsertPt, AnnotatedInst});
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
index 988a30865a7fe..42018e4d5e54d 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_profile_funclet.ll
@@ -45,8 +45,6 @@ catch.dispatch: ; preds = %entry
catch: ; preds = %catch.dispatch
%tmp1 = catchpad within %tmp [ptr null, i32 64, ptr null]
%vtable = load ptr, ptr %b, align 8
- %0 = tail call i1 @llvm.type.test(ptr %vtable, metadata !"_ZTS4base")
- tail call void @llvm.assume(i1 %0)
%tmp3 = load ptr, ptr %vtable, align 8
call void %tmp3(ptr %b) [ "funclet"(token %tmp1) ]
catchret from %tmp1 to label %try.cont
@@ -56,15 +54,12 @@ try.cont: ; preds = %catch, %entry
}
; GEN: catch:
-; GEN: call void @llvm.instrprof.value.profile({{.*}} i32 2, i32 0)
-; GEN: call void @llvm.instrprof.value.profile({{.*}} i32 0, i32 0)
+; GEN: call void @llvm.instrprof.value.profile(
; GEN-SAME: [ "funclet"(token %tmp1) ]
; LOWER: catch:
-; LOWER: call void @__llvm_profile_instrument_target({{.*}} i32 0)
+; LOWER: call void @__llvm_profile_instrument_target(
; LOWER-SAME: [ "funclet"(token %tmp1) ]
declare dso_local void @"?may_throw@@YAXH at Z"(i32)
declare dso_local i32 @__CxxFrameHandler3(...)
-declare i1 @llvm.type.test(ptr, metadata)
-declare void @llvm.assume(i1 noundef)
diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
index 059552a7634dc..762c2228110f8 100644
--- a/llvm/test/Transforms/PGOProfile/vtable_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 3
-; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
+; RUN: opt < %s -passes=pgo-instr-gen -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=LOWER
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
index 9dd7fa2889690..aa04b881e2ec9 100644
--- a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -1,4 +1,4 @@
-To update the inputs used below, run Inputs/update_vtable_value_prof_inputs.sh /path/to/updated/clang++ /path/to/updated/llvm-profdata
+To update the inputs used below, run Inputs/update_vtable_value_prof_inputs.sh /path/to/updated/clang++
Show profile data from raw profiles.
RUN: llvm-profdata show --function=main --ic-targets --show-vtables %p/Inputs/vtable_prof.profraw | FileCheck %s --check-prefix=RAW
@@ -16,16 +16,17 @@ RAW-NEXT: main:
RAW-NEXT: Hash: 0x0f9a16fe6d398548
RAW-NEXT: Counters: 2
RAW-NEXT: Indirect Call Site Count: 2
-RAW-NEXT: VTable Results:
-RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
-RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: Number of instrumented vtables: 2
RAW-NEXT: Indirect Target Results:
RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
RAW-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
RAW-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+RAW-NEXT: VTable Results:
+RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
RAW-NEXT: Instrumentation level: IR entry_first = 0
RAW-NEXT: Functions shown: 1
RAW-NEXT: Total functions: 6
@@ -38,22 +39,31 @@ RAW-NEXT: Total number of profiled values: 4
RAW-NEXT: Value sites histogram:
RAW-NEXT: NumTargets, SiteCount
RAW-NEXT: 2, 2
+RAW-NEXT: Statistics for vtable profile:
+RAW-NEXT: Total number of sites: 2
+RAW-NEXT: Total number of sites with values: 2
+RAW-NEXT: Total number of profiled values: 4
+RAW-NEXT: Value sites histogram:
+RAW-NEXT: NumTargets, SiteCount
+RAW-NEXT: 2, 2
+
INDEXED: Counters:
INDEXED-NEXT: main:
INDEXED-NEXT: Hash: 0x0f9a16fe6d398548
INDEXED-NEXT: Counters: 2
INDEXED-NEXT: Indirect Call Site Count: 2
-INDEXED-NEXT: VTable Results:
-INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
-INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
-INDEXED-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
-INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: Number of instrumented vtables: 2
INDEXED-NEXT: Indirect Target Results:
INDEXED-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
INDEXED-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
+INDEXED-NEXT: VTable Results:
+INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
INDEXED-NEXT: Instrumentation level: IR entry_first = 0
INDEXED-NEXT: Functions shown: 1
INDEXED-NEXT: Total functions: 6
@@ -66,6 +76,13 @@ INDEXED-NEXT: Total number of profiled values: 4
INDEXED-NEXT: Value sites histogram:
INDEXED-NEXT: NumTargets, SiteCount
INDEXED-NEXT: 2, 2
+INDEXED-NEXT: Statistics for vtable profile:
+INDEXED-NEXT: Total number of sites: 2
+INDEXED-NEXT: Total number of sites with values: 2
+INDEXED-NEXT: Total number of profiled values: 4
+INDEXED-NEXT: Value sites histogram:
+INDEXED-NEXT: NumTargets, SiteCount
+INDEXED-NEXT: 2, 2
ICTEXT: :ir
ICTEXT: main
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 205b3b325284e..479ac181f46cb 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -2565,18 +2565,15 @@ static int showInstrProfile(
OS << " Indirect Call Site Count: "
<< Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n";
+ if (ShowVTables)
+ OS << " Number of instrumented vtables: "
+ << Func.getNumValueSites(IPVK_VTableTarget) << "\n";
+
uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize);
if (ShowMemOPSizes && NumMemOPCalls > 0)
OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls
<< "\n";
- if (ShowVTables) {
- OS << " VTable Results:\n";
- traverseAllValueSites(Func, IPVK_VTableTarget,
- VPStats[IPVK_VTableTarget], OS,
- &(Reader->getSymtab()));
- }
-
if (ShowCounts) {
OS << " Block counts: [";
size_t Start = (IsIRInstr ? 0 : 1);
@@ -2593,6 +2590,13 @@ static int showInstrProfile(
&(Reader->getSymtab()));
}
+ if (ShowVTables) {
+ OS << " VTable Results:\n";
+ traverseAllValueSites(Func, IPVK_VTableTarget,
+ VPStats[IPVK_VTableTarget], OS,
+ &(Reader->getSymtab()));
+ }
+
if (ShowMemOPSizes && NumMemOPCalls > 0) {
OS << " Memory Intrinsic Size Results:\n";
traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS,
@@ -2641,6 +2645,11 @@ static int showInstrProfile(
VPStats[IPVK_IndirectCallTarget]);
}
+ if (ShownFunctions && ShowVTables) {
+ OS << "Statistics for vtable profile:\n";
+ showValueSitesStats(OS, IPVK_VTableTarget, VPStats[IPVK_VTableTarget]);
+ }
+
if (ShownFunctions && ShowMemOPSizes) {
OS << "Statistics for memory intrinsic calls sizes profile:\n";
showValueSitesStats(OS, IPVK_MemOPSize, VPStats[IPVK_MemOPSize]);
>From 82fac8e25798772862c52a6b31ef09f6a2be4757 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Wed, 1 Nov 2023 09:47:36 -0700
Subject: [PATCH 07/15] address feedback
---
clang/include/clang/Basic/CodeGenOptions.h | 10 +++++
clang/lib/CodeGen/CGVTables.cpp | 2 +-
clang/lib/CodeGen/MicrosoftCXXABI.cpp | 3 +-
compiler-rt/lib/profile/InstrProfilingMerge.c | 6 ++-
.../lib/profile/InstrProfilingWriter.c | 3 +-
.../llvm/Analysis/IndirectCallVisitor.h | 10 +++--
llvm/include/llvm/ProfileData/InstrProf.h | 10 +++--
.../llvm/ProfileData/InstrProfReader.h | 15 +++----
llvm/lib/ProfileData/InstrProf.cpp | 42 +++++++++---------
.../PGOProfile/Inputs/vtable_prof.profraw | Bin 0 -> 656 bytes
.../Inputs/update_vtable_value_prof_inputs.sh | 4 +-
.../vtable-value-prof-basic.test | 4 +-
12 files changed, 62 insertions(+), 47 deletions(-)
create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/vtable_prof.profraw
rename llvm/test/{Transforms/PGOProfile => tools/llvm-profdata}/Inputs/update_vtable_value_prof_inputs.sh (91%)
mode change 100644 => 100755
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index 12be4e0025a70..fe013a49ff515 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -488,6 +488,16 @@ class CodeGenOptions : public CodeGenOptionsBase {
return getProfileInstr() == ProfileClangInstr;
}
+ // Returns true if IR level profile instrumentation or use is on;
+ // returns true for CS IR instrumentation or profile use as well.
+ bool hasProfileIRInstrOrUse() const {
+ // For CS IR PGO, hasProfileIRUse is true for both instrumentation
+ // and profile use. `-fprofile-use` points to IR PGO profiles
+ // in the former case and points to CS IR PGO profiles in the latter
+ // case.
+ return hasProfileIRInstr() || hasProfileIRUse();
+ }
+
/// Check if IR level profile instrumentation is on.
bool hasProfileIRInstr() const {
return getProfileInstr() == ProfileIRInstr;
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 25e4b1c279320..5b4d17926a129 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -1312,7 +1312,7 @@ llvm::GlobalObject::VCallVisibility CodeGenModule::GetVCallVisibilityLevel(
void CodeGenModule::EmitVTableTypeMetadata(const CXXRecordDecl *RD,
llvm::GlobalVariable *VTable,
const VTableLayout &VTLayout) {
- if (!getCodeGenOpts().LTOUnit)
+ if (!getCodeGenOpts().LTOUnit &&!getCodeGenOpts().hasProfileIRInstrOrUse())
return;
CharUnits ComponentWidth = GetTargetTypeStoreSize(getVTableComponentType());
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index b6f941052abee..14a2146bbdcd0 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -1673,7 +1673,8 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
void MicrosoftCXXABI::emitVTableTypeMetadata(const VPtrInfo &Info,
const CXXRecordDecl *RD,
llvm::GlobalVariable *VTable) {
- if (!CGM.getCodeGenOpts().LTOUnit)
+ if (!CGM.getCodeGenOpts().LTOUnit &&
+ !CGM.getCodeGenOpts().hasProfileIRInstrOrUse())
return;
// TODO: Should VirtualFunctionElimination also be supported here?
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 32b3058535ede..7b6cf775e6c67 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -102,8 +102,10 @@ static uint64_t
getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
// Skip names section, vtable profile data section and vtable names section
// for runtime profile merge. To merge runtime addresses from multiple
- // profiles, the same instrumented binary should run with ASLR disabled -> in
- // this set-up these three sections remain unchanged.
+ // profiles collected from the same instrumented binary, the binary should be
+ // loaded at fixed base address (e.g., build with -no-pie, or run with ASLR
+ // disabled).
+ // In this set-up these three sections remain unchanged.
const uint64_t VTableSectionSize =
Header->NumVTables * sizeof(VTableProfData);
const uint64_t PaddingBytesAfterVTableSection =
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index c96d3f17a3269..5ba1d8d7ad26e 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -323,8 +323,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
return -1;
/* Write the binary id lengths and data. */
- int binary_id_size = __llvm_write_binary_ids(Writer);
- if (binary_id_size == -1)
+ if (__llvm_write_binary_ids(Writer) == -1)
return -1;
/* Write the profile data. */
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 4a3e5b61e14f3..3cf23b29214a2 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -52,9 +52,13 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
// FIXME: Add support in the frontend so LLVM type intrinsics are
// emitted without LTO. This way, added intrinsics could filter
// non-vtable instructions and reduce instrumentation overhead.
- // Note a profiled address will be dropped if it's not within
- // the address range of vtable objects, so there are no correctness
- // or performance issues at profile use time.
+ // Since a non-vtable profiled address is not within the address
+ // range of vtable objects, it's stored as zero in indexed profiles.
+ // A pass that looks up symbol with an zero hash will (almost) always
+ // find nullptr and skip the actual transformation (e.g., comparison
+ // of symbols). So the performance overhead from non-vtable profiled
+ // address is negligible if exists at all. Comparing loaded address
+ // with symbol address guarantees correctness.
if (VTablePtr != nullptr && isa<Instruction>(VTablePtr)) {
ProfiledAddresses.push_back(cast<Instruction>(VTablePtr));
}
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index d289a4b633379..d8c716625c903 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -541,9 +541,11 @@ class InstrProfSymtab {
}
/// The method name is kept since there are many callers.
- // It just forwards to 'addSymbolName'.
+ /// It just forwards to 'addSymbolName'.
Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); }
+ /// Adds VTableName as a known symbol, and inserts it to a map that
+ /// tracks all vtable names.
Error addVTableName(StringRef VTableName) {
if (Error E = addSymbolName(VTableName))
return E;
@@ -562,8 +564,8 @@ class InstrProfSymtab {
AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
}
- // Map the address range (i.e., [start_address, end_address]) of a variable to
- // its names' MD5 hash. This interface is only used by the raw profile header.
+ /// Map the address range (i.e., [start_address, end_address]) of a variable to
+ /// its names' MD5 hash. This interface is only used by the raw profile header.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
VTableAddrRangeToMD5Map.push_back(
std::make_pair(std::make_pair(StartAddr, EndAddr), MD5Val));
@@ -674,7 +676,7 @@ void InstrProfSymtab::finalizeSymtab() {
llvm::sort(AddrToMD5Map, less_first());
AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
AddrToMD5Map.end());
- // GlobalVariable address ranges should not overlap; so sort by either
+ // VTable object address ranges should not overlap; so sort by either
// beginning address or end address is fine.
llvm::sort(VTableAddrRangeToMD5Map, less_first());
// std::unique uses == operator for std::pair.
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 0e150b1ce011d..d54fa76e40a6f 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -663,14 +663,13 @@ class IndexedInstrProfReader : public InstrProfReader {
std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable;
/// MemProf frame profile data on-disk indexed via frame id.
std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable;
- /// The reader itself doesn't decompress vtable names.
- /// A compiler that reads indexed profiles could construct
- /// symtab from module IR so it doesn't need the decompressed
- /// names.
- /// When a symtab is constructed (from llvm-profdata),
- /// the symtab decompress the list of names based on
- /// `VTableNamePtr` and `CompressedVTableNamesLen`.
- /// Points to the beginning of compressed vtable names.
+ /// The reader itself doesn't decompress vtable names. A compiler that reads
+ /// indexed profiles could construct symtab from module IR so it doesn't need
+ /// the decompressed names.
+ /// When a symtab is constructed from profiles by llvm-profdata, the list of
+ /// names could be decompressed based on `VTableNamePtr` and
+ /// `CompressedVTableNamesLen`.
+ /// VTableNamePtr points to the beginning of compressed vtable names.
const char *VTableNamePtr = nullptr;
/// The length of compressed vtable names.
uint64_t CompressedVTableNamesLen = 0;
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 8774f2f796df0..a180b030eff75 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -218,11 +218,8 @@ cl::opt<bool> DoInstrProfNameCompression(
cl::opt<bool> EnableVTableValueProfiling(
"enable-vtable-value-profiling", cl::init(false),
cl::desc("If true, the virtual table address will be instrumented to know "
- "the types of a C++ pointer. The information could be used in "
- "indirect-call-promotion to do selective vtable-based comparison "
- "and interprocedural type propagation. Requires type metadata and "
- "type intrinsics (https://llvm.org/docs/TypeMetadata.html) to use"
- " this option."));
+ "the types of a C++ pointer. The information is used in indirect "
+ "call promotion to do selective vtable-based comparison."));
std::string getInstrProfSectionName(InstrProfSectKind IPSK,
Triple::ObjectFormatType OF,
@@ -622,7 +619,7 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
instrprof_error decodeAndSplitStrings(
const uint8_t *Input, SmallVector<uint8_t, 128> &UncompressedNameStrings,
- StringRef &NameStrings, uint32_t &Dist, bool &isCompressed) {
+ StringRef &NameStrings, uint32_t &Dist, bool &IsCompressed) {
Dist = 0;
const uint8_t *Start = Input;
uint32_t UncompressedSizeLen = 0;
@@ -633,23 +630,24 @@ instrprof_error decodeAndSplitStrings(
uint64_t CompressedSize = decodeULEB128(Start, &CompressedSizeLen);
Start += CompressedSizeLen;
Dist += CompressedSizeLen;
- isCompressed = (CompressedSize != 0);
- if (isCompressed) {
- if (!llvm::compression::zlib::isAvailable())
- return instrprof_error::zlib_unavailable;
-
- if (Error E = compression::zlib::decompress(ArrayRef(Start, CompressedSize),
- UncompressedNameStrings,
- UncompressedSize)) {
- consumeError(std::move(E));
- return instrprof_error::uncompress_failed;
- }
- Dist += CompressedSize;
- } else {
+ IsCompressed = (CompressedSize != 0);
+ if (!IsCompressed) {
NameStrings =
StringRef(reinterpret_cast<const char *>(Start), UncompressedSize);
Dist += UncompressedSize;
+ return instrprof_error::success;
+ }
+
+ if (!llvm::compression::zlib::isAvailable())
+ return instrprof_error::zlib_unavailable;
+
+ if (Error E = compression::zlib::decompress(ArrayRef(Start, CompressedSize),
+ UncompressedNameStrings,
+ UncompressedSize)) {
+ consumeError(std::move(E));
+ return instrprof_error::uncompress_failed;
}
+ Dist += CompressedSize;
return instrprof_error::success;
}
@@ -664,13 +662,13 @@ Error readPGOFuncNameStrings(StringRef NameStrings,
StringRef NameStrings;
SmallVector<uint8_t, 128> UncompressedNameStrings;
SmallVector<StringRef, 0> Names;
- bool isCompressed = false;
+ bool IsCompressed = false;
instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
- NameStrings, Dist, isCompressed);
+ NameStrings, Dist, IsCompressed);
if (E != instrprof_error::success)
return make_error<InstrProfError>(E);
- if (isCompressed) {
+ if (IsCompressed) {
NameStrings = toStringRef(UncompressedNameStrings);
}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/vtable_prof.profraw b/llvm/test/Transforms/PGOProfile/Inputs/vtable_prof.profraw
new file mode 100644
index 0000000000000000000000000000000000000000..5adeb774cddd6462bd2d3779a96d9ad5a06d5e23
GIT binary patch
literal 656
zcmZoHO3N=Q$obF700xW at ih+R*#(>fsXncDp|G<9;NPf(`(<%&25s=FS6^fqKW9??N
zGj7VQAPc)yoJ=r%1$-<h`e5o|CjGkF``j#L>TQo!%iMorv-oE~?b`s=moX8dAEXai
zZ{I{cNtM}66M)L!U*_`VDuC*1;77F&?qR5f3mFb--sHb`6Q<7rs&4`TeGkj4d7Hnn
z_QUiEK=mykpl|Z0nI4u#dwXE|9{hu7+(1CzJE&)WLEZg=8Nz`12PO`qVd`OYj%~%Q
z?tUW^4?Qnm9Z%ksIv!_J&iI}=libt)X>)<6?kOE_UqcgL?X%uyC1=kZ5ixZVHa)cJ
q8pk>@hHDd5b}&Q$t%KPG4tuD3VBrh17v_JMy|8eE$;12!lLr8f`l*Eg
literal 0
HcmV?d00001
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh b/llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh
old mode 100644
new mode 100755
similarity index 91%
rename from llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
rename to llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh
index 5c594e10e7370..dead295a5a72f
--- a/llvm/test/Transforms/PGOProfile/Inputs/update_vtable_value_prof_inputs.sh
+++ b/llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-if [ $# -lt 2 ]; then
+if [ $# -lt 1 ]; then
echo "Path to clang++ required!"
echo "Usage: update_vtable_value_prof_inputs.sh /path/to/updated/clang++"
exit 1
@@ -73,7 +73,7 @@ int main(int argc, char** argv) {
}
EOF
-FLAGS="-fuse-ld=lld -O2 -g -fprofile-generate=. -flto=thin -Xclang -fwhole-program-vtables -Wl,--lto-whole-program-visibility"
+FLAGS="-fuse-ld=lld -O2 -g -fprofile-generate=. "
${CLANG} ${FLAGS} ${OUTDIR}/vtable_prof.cc -o ${OUTDIR}/vtable_prof
env LLVM_PROFILE_FILE=${OUTDIR}/vtable_prof.profraw ${OUTDIR}/vtable_prof
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
index aa04b881e2ec9..733ff8d7f1eac 100644
--- a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -60,8 +60,8 @@ INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
INDEXED-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
INDEXED-NEXT: VTable Results:
-INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
-INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
INDEXED-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
INDEXED-NEXT: Instrumentation level: IR entry_first = 0
>From d54d0594518a5d87e433e38732d5d7e6e6014c4c Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Wed, 1 Nov 2023 15:53:01 -0700
Subject: [PATCH 08/15] small changes: - (Hopefully) fix indentation in
llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test - 'git diff' and
'vim' looks okay. - Undo the change in
compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test This is not
necessary since -enable-vtable-value-profiling is false by default.
---
.../Linux/instrprof-value-prof-warn.test | 2 +-
.../llvm-profdata/Inputs/vtable-prof.proftext | 2 +-
.../tools/llvm-profdata/vtable-prof.proftext | 2 +-
.../vtable-value-prof-basic.test | 22 +++++++++----------
4 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test b/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
index 89787e79c20e9..991fd2b69b5b3 100644
--- a/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
+++ b/compiler-rt/test/profile/Linux/instrprof-value-prof-warn.test
@@ -1,7 +1,7 @@
RUN: %clang_pgogen -O2 -fuse-ld=bfd -mllvm -disable-vp=false -mllvm -vp-static-alloc=true -DSTRESS=1 -o %t.ir.warn %S/../Inputs/instrprof-value-prof-real.c
RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=255 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=WARNING %s
# Test that enough static counters have been allocated
-RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=80 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=NOWARNING --allow-empty %s
+RUN: env LLVM_PROFILE_FILE=%t.ir.profraw LLVM_VP_MAX_NUM_VALS_PER_SITE=130 %run %t.ir.warn 2>&1 |FileCheck --check-prefix=NOWARNING --allow-empty %s
# WARNING: LLVM Profile Warning:
# NOWARNING-NOT: LLVM Profile Warning:
diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
index bf0ad6beb9825..0f5d93d26c184 100644
--- a/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
+++ b/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
@@ -70,4 +70,4 @@ _ZTV8Derived2:750
_ZTV8Derived1:250
2
_ZTV8Derived2:750
-_ZTV8Derived1:250
\ No newline at end of file
+_ZTV8Derived1:250
diff --git a/llvm/test/tools/llvm-profdata/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
index 6ebe9b44266b6..b533c7b8a7f51 100644
--- a/llvm/test/tools/llvm-profdata/vtable-prof.proftext
+++ b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
@@ -13,4 +13,4 @@ VTABLE: Instrumentation level: IR entry_first = 0
VTABLE: Functions shown: 1
VTABLE: Total functions: 6
VTABLE: Maximum function count: 1000
-VTABLE: Maximum internal block count: 250
\ No newline at end of file
+VTABLE: Maximum internal block count: 250
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
index 733ff8d7f1eac..e853516832cb8 100644
--- a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -18,15 +18,15 @@ RAW-NEXT: Counters: 2
RAW-NEXT: Indirect Call Site Count: 2
RAW-NEXT: Number of instrumented vtables: 2
RAW-NEXT: Indirect Target Results:
-RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
-RAW-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
-RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
-RAW-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
RAW-NEXT: VTable Results:
-RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
-RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
+RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
RAW-NEXT: Instrumentation level: IR entry_first = 0
RAW-NEXT: Functions shown: 1
RAW-NEXT: Total functions: 6
@@ -44,7 +44,7 @@ RAW-NEXT: Total number of sites: 2
RAW-NEXT: Total number of sites with values: 2
RAW-NEXT: Total number of profiled values: 4
RAW-NEXT: Value sites histogram:
-RAW-NEXT: NumTargets, SiteCount
+RAW-NEXT: NumTargets, SiteCount
RAW-NEXT: 2, 2
@@ -81,8 +81,8 @@ INDEXED-NEXT: Total number of sites: 2
INDEXED-NEXT: Total number of sites with values: 2
INDEXED-NEXT: Total number of profiled values: 4
INDEXED-NEXT: Value sites histogram:
-INDEXED-NEXT: NumTargets, SiteCount
-INDEXED-NEXT: 2, 2
+INDEXED-NEXT: NumTargets, SiteCount
+INDEXED-NEXT: 2, 2
ICTEXT: :ir
ICTEXT: main
>From 52143e9afa3ed26544ba322098e9d0eff7bc9d31 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Mon, 6 Nov 2023 22:53:09 -0800
Subject: [PATCH 09/15] update clang-format version to 117.0.4 released binary
and run clang-format
---
compiler-rt/lib/profile/InstrProfiling.h | 12 ++++------
.../lib/profile/InstrProfilingBuffer.c | 23 ++++++++++---------
.../lib/profile/InstrProfilingWriter.c | 22 +++++++++---------
3 files changed, 28 insertions(+), 29 deletions(-)
diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index f25c195f0ea58..04ca7da8a4a5f 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -32,7 +32,7 @@ typedef struct __llvm_profile_header {
#include "profile/InstrProfData.inc"
} __llvm_profile_header;
-typedef struct ValueProfNode * PtrToNodeT;
+typedef struct ValueProfNode *PtrToNodeT;
typedef struct ValueProfNode {
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) Type Name;
#include "profile/InstrProfData.inc"
@@ -128,8 +128,7 @@ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size);
* structurally the in-process counters and bitmaps. If the profile data in
* buffer is not compatible, the interface returns 1 (failure).
*/
-int __llvm_profile_check_compatibility(const char *Profile,
- uint64_t Size);
+int __llvm_profile_check_compatibility(const char *Profile, uint64_t Size);
/*!
* \brief Counts the number of times a target value is seen.
@@ -142,7 +141,7 @@ int __llvm_profile_check_compatibility(const char *Profile,
void INSTR_PROF_VALUE_PROF_FUNC(
#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) ArgType ArgName
#include "profile/InstrProfData.inc"
- );
+);
void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data,
uint32_t CounterIndex,
@@ -319,9 +318,8 @@ void __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
- uint64_t *PaddingBytesAfterBitmap,
- uint64_t *PaddingBytesAfterNames, uint64_t *PaddingBytesAfterVTable,
- uint64_t *PaddingBytesAfterVNames);
+ uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames,
+ uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVNames);
/*!
* \brief Set the flag that profile data has been dumped to the file.
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index 9a990c3d33b53..8cbad7c569c36 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -136,12 +136,11 @@ static int needsCounterPadding(void) {
COMPILER_RT_VISIBILITY
void __llvm_profile_get_padding_sizes_for_counters(
- uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, uint64_t NamesSize,
- uint64_t VTableSize, uint64_t VNameSize,
+ uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
+ uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
- uint64_t* PaddingBytesAfterBitmapBytes,
- uint64_t *PaddingBytesAfterNames, uint64_t *PaddingBytesAfterVTable,
- uint64_t *PaddingBytesAfterVName) {
+ uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames,
+ uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) {
// Counter padding is needed only if continuous mode is enabled.
if (!needsCounterPadding()) {
*PaddingBytesBeforeCounters = 0;
@@ -196,8 +195,9 @@ uint64_t __llvm_profile_get_size_for_buffer_internal(
PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes;
__llvm_profile_get_padding_sizes_for_counters(
DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */,
- 0 /* VNameSize */, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
- &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames, NULL /* PaddingBytesAfterVTable */,
+ 0 /* VNameSize */, &PaddingBytesBeforeCounters,
+ &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
+ &PaddingBytesAfterNames, NULL /* PaddingBytesAfterVTable */,
NULL /* PaddingbytesAfterVNames */);
return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
@@ -226,8 +226,9 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
ProfDataWriter BufferWriter;
initBufferWriter(&BufferWriter, Buffer);
// Set virtual table arguments to NULL since they are not supported yet.
- return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin,
- CountersEnd, BitmapBegin, BitmapEnd, 0 /* VPDataReader */, NamesBegin, NamesEnd,
- NULL /* VTableBegin */, NULL /* VTableEnd */,
- NULL /* VNamesBegin */, NULL /* VNamesEnd */, 0 /* SkipNameDataWrite */);
+ return lprofWriteDataImpl(
+ &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd,
+ BitmapBegin, BitmapEnd, 0 /* VPDataReader */, NamesBegin, NamesEnd,
+ NULL /* VTableBegin */, NULL /* VTableEnd */, NULL /* VNamesBegin */,
+ NULL /* VNamesEnd */, 0 /* SkipNameDataWrite */);
}
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 2a95243956daf..39ff7fd1041f9 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -255,10 +255,9 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
const char *VNamesBegin = __llvm_profile_begin_vtabnames();
const char *VNamesEnd = __llvm_profile_end_vtabnames();
return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
- CountersEnd, BitmapBegin, BitmapEnd,
- VPDataReader, NamesBegin, NamesEnd,
- VTableBegin, VTableEnd, VNamesBegin,
- VNamesEnd, SkipNameDataWrite);
+ CountersEnd, BitmapBegin, BitmapEnd, VPDataReader,
+ NamesBegin, NamesEnd, VTableBegin, VTableEnd,
+ VNamesBegin, VNamesEnd, SkipNameDataWrite);
}
COMPILER_RT_VISIBILITY int
@@ -288,20 +287,21 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const uint64_t VTableSectionSize =
__llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
// Value profiling is not supported when DebugInfoCorrelate is true.
- const uint64_t VNamesSize = __llvm_profile_get_name_size(VNamesBegin, VNamesEnd);
-
+ const uint64_t VNamesSize =
+ __llvm_profile_get_name_size(VNamesBegin, VNamesEnd);
+
/* Create the header. */
__llvm_profile_header Header;
/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterBitmapBytes,
- PaddingBytesAfterNames, PaddingBytesAfterVTable, PaddingBytesAfterVNames;
+ PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames,
+ PaddingBytesAfterVTable, PaddingBytesAfterVNames;
__llvm_profile_get_padding_sizes_for_counters(
- DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, VTableSectionSize,
- VNamesSize, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
- &PaddingBytesAfterBitmapBytes,
+ DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize,
+ VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters,
+ &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
&PaddingBytesAfterNames, &PaddingBytesAfterVTable,
&PaddingBytesAfterVNames);
>From 2350d7f90f85abbb49c1cd2f2d9615a99dc54018 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Tue, 7 Nov 2023 10:44:54 -0800
Subject: [PATCH 10/15] The changes: 1. Primarily, use vtable PGO name in
compressed strings in helper function getVarName (around line 1115 in
InstrProfiling.cpp) 2. Use version 12 in {InstrProfReader.cpp around line
1312, InstrProfWriter.cpp around line 607}. - Using version 11 is an
overlook in 'git merge main' (a change in upstream main takes version 11
already). 3. In InstrProf.cpp and its header, rename 'readPGOFuncNameStrings'
to a generic name, and make it a static function. pr/71566 is an NFC doing
essentially the same thing. 4. A few code cleaning (e.g., delete debugging
comment in raw-64-bits*.test)
---
clang/include/clang/Basic/CodeGenOptions.h | 10 --
compiler-rt/include/profile/InstrProfData.inc | 19 ++-
.../lib/profile/InstrProfilingInternal.h | 3 -
.../llvm/Analysis/IndirectCallVisitor.h | 2 +-
llvm/include/llvm/ProfileData/InstrProf.h | 51 ++----
llvm/lib/ProfileData/InstrProf.cpp | 149 +++++++++---------
llvm/lib/ProfileData/InstrProfReader.cpp | 2 +-
llvm/lib/ProfileData/InstrProfWriter.cpp | 2 +-
.../Instrumentation/InstrProfiling.cpp | 4 +-
.../Instrumentation/PGOInstrumentation.cpp | 3 +-
.../tools/llvm-profdata/raw-32-bits-be.test | 2 -
.../tools/llvm-profdata/raw-64-bits-le.test | 9 --
12 files changed, 104 insertions(+), 152 deletions(-)
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index fe013a49ff515..12be4e0025a70 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -488,16 +488,6 @@ class CodeGenOptions : public CodeGenOptionsBase {
return getProfileInstr() == ProfileClangInstr;
}
- // Returns true if IR level profile instrumentation or use is on;
- // returns true for CS IR instrumentation or profile use as well.
- bool hasProfileIRInstrOrUse() const {
- // For CS IR PGO, hasProfileIRUse is true for both instrumentation
- // and profile use. `-fprofile-use` points to IR PGO profiles
- // in the former case and points to CS IR PGO profiles in the latter
- // case.
- return hasProfileIRInstr() || hasProfileIRUse();
- }
-
/// Check if IR level profile instrumentation is on.
bool hasProfileIRInstr() const {
return getProfileInstr() == ProfileIRInstr;
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index b649cc75783fe..a8fc5fe1b870b 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -95,9 +95,10 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
/* INSTR_PROF_DATA end. */
-/* For a virtual table object, record the name hash to associate profiled addresses
- * with global variables, and record {starting address, size in bytes} to map the profiled virtual table (which usually have an offset from the starting address)
- * back to a virtual table object. */
+/* For a virtual table object, record the name hash to associate profiled
+ * addresses with global variables, and record {starting address, size in bytes}
+ * to map the profiled virtual table (which usually have an offset from the
+ * starting address) back to a virtual table object. */
#ifndef INSTR_PROF_VTABLE_DATA
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
#else
@@ -202,13 +203,17 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
-/* For virtual table address profiling, the addresses of the virtual table (i.e., the address contained in objects pointing to a virtual table) are
- * profiled. Note this may not be the address of the per C++ class virtual table object (i.e., there is an offset).
+/* For virtual table address profiling, the addresses of the virtual table
+ * (i.e., the address contained in objects pointing to a virtual table) are
+ * profiled. Note this may not be the address of the per C++ class virtual table
+ * object (i.e., there is an offset).
*
- * The profiled addresses are stored in raw profile, together with the following two types of information.
+ * The profiled addresses are stored in raw profile, together with the following
+ * two types of information.
* 1. The (beginning and ending) addresses of per C++ class virtual table objects.
* 2. The (compressed) virtual table object names.
- * RawInstrProfReader converts profiled virtual table addresses to virtual table objects' MD5 hash.
+ * RawInstrProfReader converts profiled virtual table addresses to virtual table
+ * objects' MD5 hash.
*/
VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The address of the compatible vtable (i.e., there is an offset from this address to per C++ class virtual table global variable.)")
/* These two kinds must be the last to be
diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h
index b8a798f443ef6..38159b668a1df 100644
--- a/compiler-rt/lib/profile/InstrProfilingInternal.h
+++ b/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -18,9 +18,6 @@
* pointers to the live data in memory. This function is probably not what you
* want. Use __llvm_profile_get_size_for_buffer instead. Use this function if
* your program has a custom memory layout.
- * NOTE: The change of function signature requires modifying c source code
- * as demonstrated by the existing tests. If this is causing backward
- * compatible issues, considering adding another function for new use cases.
*/
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 3cf23b29214a2..c8429e52bee96 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -86,4 +86,4 @@ inline std::vector<Instruction *> findVTableAddrs(Function &F) {
} // namespace llvm
-#endif
\ No newline at end of file
+#endif
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 78ec01617ee2f..c27881d8144e1 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -251,13 +251,6 @@ Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
std::string &Result, bool doCompression);
-/// \c NameStrings is a string composed of one of more sub-strings encoded in
-/// the format described above. The substrings are separated by 0 or more zero
-/// bytes. This method decodes the string and calls `NameCallback` for each
-/// substring.
-Error readPGOFuncNameStrings(StringRef NameStrings,
- std::function<Error(StringRef)> NameCallback);
-
/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
/// set in IR PGO compilation.
bool isIRPGOFlagSet(const Module *M);
@@ -502,15 +495,20 @@ class InstrProfSymtab {
/// format.
inline Error create(StringRef D, uint64_t BaseAddr);
- /// \c NameStrings is a string composed of one of more sub-strings
+ /// \c NameStrings is a string composed of one or more sub-strings
/// encoded in the format described in \c collectPGOFuncNameStrings.
- /// This method is a wrapper to \c readPGOFuncNameStrings method.
- inline Error create(StringRef NameStrings);
+ /// This method is a wrapper to \c readAndDecodeStrings method.
+ Error create(StringRef NameStrings);
- inline Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
+ /// \c FuncNameStrings is a string composed of one or more encoded function
+ /// name strings, and \c VTableNameStrings composes of one or more encoded
+ /// vtable names. This function is a wrapper to \c readAndDecodeStrings
+ /// method.
+ Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
- inline Error
- initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
+ /// Initialize 'this' with the set of vtable names encoded in
+ /// \c CompressedVTableNames.
+ Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
/// A wrapper interface to populate the PGO symtab with functions
/// decls from module \c M. This interface is used by transformation
@@ -619,33 +617,6 @@ Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
return Error::success();
}
-// FIXME: Move 'InstrProfSymtab::create' definition into cpp,
-// and move 'readPGOFuncNameStrings' inside cpp.
-Error InstrProfSymtab::create(StringRef NameStrings) {
- return readPGOFuncNameStrings(
- NameStrings,
- std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
-}
-
-Error InstrProfSymtab::create(StringRef FuncNameStrings,
- StringRef VTableNameStrings) {
- if (Error E = readPGOFuncNameStrings(FuncNameStrings,
- std::bind(&InstrProfSymtab::addFuncName,
- this, std::placeholders::_1)))
- return E;
-
- return readPGOFuncNameStrings(
- VTableNameStrings,
- std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
-}
-
-Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
- StringRef CompressedVTableStrings) {
- return readPGOFuncNameStrings(
- CompressedVTableStrings,
- std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
-}
-
template <typename NameIterRange>
Error InstrProfSymtab::create(const NameIterRange &IterRange) {
for (auto Name : IterRange)
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index c91453009f02b..116be406cb5d4 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -487,6 +487,78 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) {
return Error::success();
}
+/// \c NameStrings is a string composed of one of more possibly encoded
+/// sub-strings. The substrings are separated by 0 or more zero bytes. This
+/// method decodes the string and calls `NameCallback` for each substring.
+static Error
+readAndDecodeStrings(StringRef NameStrings,
+ std::function<Error(StringRef)> NameCallback) {
+ const uint8_t *P = NameStrings.bytes_begin();
+ const uint8_t *EndP = NameStrings.bytes_end();
+ while (P < EndP) {
+ uint32_t N;
+ uint64_t UncompressedSize = decodeULEB128(P, &N);
+ P += N;
+ uint64_t CompressedSize = decodeULEB128(P, &N);
+ P += N;
+ bool isCompressed = (CompressedSize != 0);
+ SmallVector<uint8_t, 128> UncompressedNameStrings;
+ StringRef NameStrings;
+ if (isCompressed) {
+ if (!llvm::compression::zlib::isAvailable())
+ return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
+
+ if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
+ UncompressedNameStrings,
+ UncompressedSize)) {
+ consumeError(std::move(E));
+ return make_error<InstrProfError>(instrprof_error::uncompress_failed);
+ }
+ P += CompressedSize;
+ NameStrings = toStringRef(UncompressedNameStrings);
+ } else {
+ NameStrings =
+ StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
+ P += UncompressedSize;
+ }
+ // Now parse the name strings.
+ SmallVector<StringRef, 0> Names;
+ NameStrings.split(Names, getInstrProfNameSeparator());
+ for (StringRef &Name : Names)
+ if (Error E = NameCallback(Name))
+ return E;
+
+ while (P < EndP && *P == 0)
+ P++;
+ }
+ return Error::success();
+}
+
+Error InstrProfSymtab::create(StringRef NameStrings) {
+ return readAndDecodeStrings(
+ NameStrings,
+ std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
+}
+
+Error InstrProfSymtab::create(StringRef FuncNameStrings,
+ StringRef VTableNameStrings) {
+ if (Error E = readAndDecodeStrings(FuncNameStrings,
+ std::bind(&InstrProfSymtab::addFuncName,
+ this, std::placeholders::_1)))
+ return E;
+
+ return readAndDecodeStrings(
+ VTableNameStrings,
+ std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+}
+
+Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
+ StringRef CompressedVTableStrings) {
+ return readAndDecodeStrings(
+ CompressedVTableStrings,
+ std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+}
+
Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
if (Error E = addFuncName(PGOFuncName))
return E;
@@ -623,75 +695,6 @@ Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
Result);
}
-instrprof_error decodeAndSplitStrings(
- const uint8_t *Input, SmallVector<uint8_t, 128> &UncompressedNameStrings,
- StringRef &NameStrings, uint32_t &Dist, bool &IsCompressed) {
- Dist = 0;
- const uint8_t *Start = Input;
- uint32_t UncompressedSizeLen = 0;
- uint64_t UncompressedSize = decodeULEB128(Start, &UncompressedSizeLen);
- Start += UncompressedSizeLen;
- Dist += UncompressedSizeLen;
- uint32_t CompressedSizeLen = 0;
- uint64_t CompressedSize = decodeULEB128(Start, &CompressedSizeLen);
- Start += CompressedSizeLen;
- Dist += CompressedSizeLen;
- IsCompressed = (CompressedSize != 0);
- if (!IsCompressed) {
- NameStrings =
- StringRef(reinterpret_cast<const char *>(Start), UncompressedSize);
- Dist += UncompressedSize;
- return instrprof_error::success;
- }
-
- if (!llvm::compression::zlib::isAvailable())
- return instrprof_error::zlib_unavailable;
-
- if (Error E = compression::zlib::decompress(ArrayRef(Start, CompressedSize),
- UncompressedNameStrings,
- UncompressedSize)) {
- consumeError(std::move(E));
- return instrprof_error::uncompress_failed;
- }
- Dist += CompressedSize;
-
- return instrprof_error::success;
-}
-
-Error readPGOFuncNameStrings(StringRef NameStrings,
- std::function<Error(StringRef)> NameCallback) {
- const uint8_t *P = NameStrings.bytes_begin();
- const uint8_t *EndP = NameStrings.bytes_end();
- while (P < EndP) {
- // Now parse the name strings.
- uint32_t Dist = 0;
- StringRef NameStrings;
- SmallVector<uint8_t, 128> UncompressedNameStrings;
- SmallVector<StringRef, 0> Names;
- bool IsCompressed = false;
- instrprof_error E = decodeAndSplitStrings(P, UncompressedNameStrings,
- NameStrings, Dist, IsCompressed);
- if (E != instrprof_error::success)
- return make_error<InstrProfError>(E);
-
- if (IsCompressed) {
- NameStrings = toStringRef(UncompressedNameStrings);
- }
-
- NameStrings.split(Names, getInstrProfNameSeparator());
- for (StringRef &Name : Names) {
- if (Error E = NameCallback(Name))
- return E;
- }
-
- P += Dist;
- // Skip padding?
- while (P < EndP && *P == 0)
- P++;
- }
- return Error::success();
-}
-
void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
uint64_t FuncSum = 0;
Sum.NumEntries += Counts.size();
@@ -954,10 +957,8 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
if (ValueKind == IPVK_IndirectCallTarget)
return SymTab->getFunctionHashFromAddress(Value);
- if (ValueKind == IPVK_VTableTarget) {
- uint64_t VTableHash = SymTab->getVTableHashFromAddress(Value);
- return VTableHash;
- }
+ if (ValueKind == IPVK_VTableTarget)
+ return SymTab->getVTableHashFromAddress(Value);
return Value;
}
@@ -1531,7 +1532,7 @@ void OverlapStats::dump(raw_fd_ostream &OS) const {
strncpy(ProfileKindName, "MemOP", 19);
break;
case IPVK_VTableTarget:
- strncpy(ProfileKindName, "VTable", 19);
+ strncpy(ProfileKindName, "VTable", 6);
break;
default:
snprintf(ProfileKindName, 19, "VP[%d]", I);
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index ee17e879cd57e..d5f247a8521fb 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -1309,7 +1309,7 @@ Error IndexedInstrProfReader::readHeader() {
"corrupted binary ids");
}
- if (GET_VERSION(Header->formatVersion()) >= 11) {
+ if (GET_VERSION(Header->formatVersion()) >= 12) {
uint64_t VTableNamesOffset =
endian::byte_swap<uint64_t, llvm::endianness::little>(
Header->VTableNamesOffset);
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index ef54efe449ed4..491158e03aa8e 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -604,7 +604,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
// if version >= the version with vtable profile metadata.
uint64_t VTableNamesSectionStart = 0;
- if (IndexedInstrProf::ProfVersion::CurrentVersion >= 11) {
+ if (IndexedInstrProf::ProfVersion::CurrentVersion >= 12) {
VTableNamesSectionStart = OS.tell();
std::string CompressedVTableNames;
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index c90830338ea45..bb06670d34c3b 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1102,9 +1102,9 @@ static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
// FIXME: Does symbollic relocation from 'getFuncAddrForProfData' matter here?
static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
- // Store a nullptr in __profvt_ if a real address shouldn't be used.
auto *Int8PtrTy = Type::getInt8PtrTy(GV->getContext());
+ // Store a nullptr in __profvt_ if a real address shouldn't be used.
if (!shouldRecordVTableAddr(GV))
return ConstantPointerNull::get(Int8PtrTy);
@@ -1113,7 +1113,7 @@ static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
/// Get the name of a profiling variable for a particular variable.
static std::string getVarName(GlobalVariable *GV, StringRef Prefix) {
- StringRef Name = GV->getName();
+ StringRef Name = getPGOName(*GV);
return (Prefix + Name).str();
}
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index f81e877ca1d2a..5609691986955 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -589,9 +589,8 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
NumOfPGOBB += MST.BBInfos.size();
ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget);
- if (EnableVTableValueProfiling) {
+ if (EnableVTableValueProfiling)
ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
- }
} else {
NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
index 26fa2504ebdad..63782c8b94d4a 100644
--- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
+++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
@@ -13,9 +13,7 @@ RUN: printf '\0\0\0\0\1\0\0\0' >> %t
RUN: printf '\0\0\0\0\3\0\0\0' >> %t
RUN: printf '\0\0\0\0\2\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// NumVTables
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// ValueKindLast
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\134\370\302\114\333\030\275\254' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
index fd231b4ab119c..8f61cffe21320 100644
--- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
+++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
@@ -15,24 +15,15 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// InstrProfData
-// NameRef
RUN: printf '\254\275\030\333\114\302\370\134' >> %t
-// FuncHash
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
-// RelativeCounterPtr
RUN: printf '\0\0\4\0\1\0\0\0' >> %t
-// RelativeBitmapPtr
RUN: printf '\0\0\4\0\3\0\0\0' >> %t
-// FunctionPointer
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// Values
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-// NumCounters + Int16ArrayTy
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\3\0\0\0' >> %t
-// InstrProfData
RUN: printf '\067\265\035\031\112\165\023\344' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t
>From fcf92af7c1af5da76d17a36f014812c22d8c1fe9 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Sun, 12 Nov 2023 23:57:55 -0800
Subject: [PATCH 11/15] resolve feedback
---
compiler-rt/lib/profile/InstrProfiling.h | 10 +-
.../lib/profile/InstrProfilingBuffer.c | 15 +-
.../lib/profile/InstrProfilingWriter.c | 6 +-
llvm/include/llvm/ProfileData/InstrProf.h | 5 +-
llvm/lib/ProfileData/InstrProf.cpp | 11 +-
.../Instrumentation/InstrProfiling.cpp | 5 +-
llvm/unittests/ProfileData/InstrProfTest.cpp | 802 +++++++++++-------
7 files changed, 509 insertions(+), 345 deletions(-)
diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index 04ca7da8a4a5f..b049f504562dc 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -305,16 +305,18 @@ uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,
const VTableProfData *End);
-/* ! \brief Given the sizes of the data and counter information, return the
- * number of padding bytes before and after the counters, and after the names,
- * in the raw profile.
+/* ! \brief Given the sizes of the data and counter information, computes the
+ * number of padding bytes before and after the counter section, as well as the
+ * number of padding bytes after other setions in the raw profile.
+ * Returns -1 upon errors and 0 upon success. Output parameters should be used
+ * iff return value is 0.
*
* Note: When mmap() mode is disabled, no padding bytes before/after counters
* are needed. However, in mmap() mode, the counter section in the raw profile
* must be page-aligned: this API computes the number of padding bytes
* needed to achieve that.
*/
-void __llvm_profile_get_padding_sizes_for_counters(
+int __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index 8cbad7c569c36..bae8c881fdc11 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -9,8 +9,6 @@
// Note: This is linked into the Darwin kernel, and must remain compatible
// with freestanding compilation. See `darwin_add_builtin_libraries`.
-#include <assert.h>
-
#include "InstrProfiling.h"
#include "InstrProfilingInternal.h"
#include "InstrProfilingPort.h"
@@ -135,7 +133,7 @@ static int needsCounterPadding(void) {
}
COMPILER_RT_VISIBILITY
-void __llvm_profile_get_padding_sizes_for_counters(
+int __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
@@ -154,12 +152,14 @@ void __llvm_profile_get_padding_sizes_for_counters(
__llvm_profile_get_num_padding_bytes(VTableSize);
if (PaddingBytesAfterVName != NULL)
*PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);
- return;
+ return 0;
}
// Value profiling not supported in continuous mode at profile-write time.
- assert(VTableSize == 0 && VNameSize == 0 &&
- "Value profile not supported for continuous mode");
+ // Return -1 to alert the incompatibility.
+ if (VTableSize != 0 || VNameSize != 0)
+ return -1;
+
// In continuous mode, the file offsets for headers and for the start of
// counter sections need to be page-aligned.
*PaddingBytesBeforeCounters =
@@ -169,11 +169,12 @@ void __llvm_profile_get_padding_sizes_for_counters(
calculateBytesNeededToPageAlign(NumBitmapBytes);
*PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
// Set these two variables to zero to avoid uninitialized variables
- // even if VTableSize and VNameSize are asserted to be zero.
+ // even if VTableSize and VNameSize are known to be zero.
if (PaddingBytesAfterVTable != NULL)
*PaddingBytesAfterVTable = 0;
if (PaddingBytesAfterVName != NULL)
*PaddingBytesAfterVName = 0;
+ return 0;
}
COMPILER_RT_VISIBILITY
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 39ff7fd1041f9..beafc9f1248fc 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -286,7 +286,6 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
__llvm_profile_get_num_vtable(VTableBegin, VTableEnd);
const uint64_t VTableSectionSize =
__llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
- // Value profiling is not supported when DebugInfoCorrelate is true.
const uint64_t VNamesSize =
__llvm_profile_get_name_size(VNamesBegin, VNamesEnd);
@@ -298,12 +297,13 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames,
PaddingBytesAfterVTable, PaddingBytesAfterVNames;
- __llvm_profile_get_padding_sizes_for_counters(
+ if (__llvm_profile_get_padding_sizes_for_counters(
DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize,
VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters,
&PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
&PaddingBytesAfterNames, &PaddingBytesAfterVTable,
- &PaddingBytesAfterVNames);
+ &PaddingBytesAfterVNames) == -1)
+ return -1;
{
/* Initialize header structure. */
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index bc8434ccc0030..7e1541fc21ccc 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -566,8 +566,9 @@ class InstrProfSymtab {
AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
}
- /// Map the address range (i.e., [start_address, end_address]) of a variable to
- /// its names' MD5 hash. This interface is only used by the raw profile header.
+ /// Map the address range (i.e., [start_address, end_address]) of a variable
+ /// to its names' MD5 hash. This interface is only used by the raw profile
+ /// reader.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
VTableAddrRangeToMD5Map.push_back(
std::make_pair(std::make_pair(StartAddr, EndAddr), MD5Val));
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 116be406cb5d4..150ae66d3e653 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -596,7 +596,12 @@ uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
auto It = lower_bound(
VTableAddrRangeToMD5Map, Address,
[](std::pair<std::pair<uint64_t, uint64_t>, uint64_t> VTableRangeAddr,
- uint64_t Addr) { return VTableRangeAddr.first.second < Addr; });
+ uint64_t Addr) {
+ // Find the first address range of which end address is larger than
+ // `Addr`. Smaller-than-or-equal-to is used because the profiled address
+ // within a vtable should be [start-address, end-address).
+ return VTableRangeAddr.first.second <= Addr;
+ });
// Returns the MD5 hash if Address is within the address range of an entry.
if (It != VTableAddrRangeToMD5Map.end() && It->first.first <= Address) {
@@ -1523,7 +1528,7 @@ void OverlapStats::dump(raw_fd_ostream &OS) const {
for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
continue;
- char ProfileKindName[20];
+ char ProfileKindName[20] = {0};
switch (I) {
case IPVK_IndirectCallTarget:
strncpy(ProfileKindName, "IndirectCall", 19);
@@ -1532,7 +1537,7 @@ void OverlapStats::dump(raw_fd_ostream &OS) const {
strncpy(ProfileKindName, "MemOP", 19);
break;
case IPVK_VTableTarget:
- strncpy(ProfileKindName, "VTable", 6);
+ strncpy(ProfileKindName, "VTable", 19);
break;
default:
snprintf(ProfileKindName, 19, "VP[%d]", I);
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 9910b210acf8d..472fc11ca738b 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1100,7 +1100,7 @@ static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
return true;
}
-// FIXME: Does symbollic relocation from 'getFuncAddrForProfData' matter here?
+// FIXME: Does symbolic relocation from 'getFuncAddrForProfData' matter here?
static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
auto *Int8PtrTy = PointerType::getUnqual(GV->getContext());
@@ -1153,7 +1153,6 @@ void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
// Used by INSTR_PROF_VTABLE_DATA MACRO
Constant *VTableAddr = getVTableAddrForProfData(GV);
- StringRef VTableName = GV->getName();
StringRef PGOVTableName = getPGOName(*GV);
// Record the length of the vtable. This is needed since vtable pointers
// loaded from C++ objects might be from the middle of a vtable definition.
@@ -1188,7 +1187,7 @@ void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
Comdat *C = M->getOrInsertComdat(GroupName);
// For ELF, when not using COMDAT, put the vtable profile data into a
// nodeduplicate COMDAT which is lowered to a zero-flag zero group.
- // This allows -z -start-top-gc to discard the entire group when the
+ // This allows -z -start-stop-gc to discard the entire group when the
// vtable def is discarded.
if (!NeedComdat)
C->setSelectionKind(Comdat::NoDeduplicate);
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index f26f244afc537..1ad5778256465 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -15,6 +15,7 @@
#include "llvm/ProfileData/MemProf.h"
#include "llvm/ProfileData/MemProfData.inc"
#include "llvm/Support/Compression.h"
+#include "llvm/Support/MD5.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Testing/Support/Error.h"
#include "gtest/gtest.h"
@@ -64,13 +65,209 @@ struct InstrProfTest : ::testing::Test {
}
};
+// callee1 to callee6 are from vtable1 to vtable6 respectively.
+// callee7 and callee8 are not virtual functions and not from any vtables.
+static const char callee1[] = "callee1";
+static const char callee2[] = "callee2";
+static const char callee3[] = "callee3";
+static const char callee4[] = "callee4";
+static const char callee5[] = "callee5";
+static const char callee6[] = "callee6";
+static const char callee7[] = "callee7";
+static const char callee8[] = "callee8";
+// 'callee' is primarily used to create multiple-element vtables.
+static const char callee[] = "callee";
+static const uint64_t vtable1[] = {uint64_t(callee), uint64_t(callee1)};
+static const uint64_t vtable2[] = {uint64_t(callee2), uint64_t(callee)};
+static const uint64_t vtable3[] = {
+ uint64_t(callee),
+ uint64_t(callee3),
+};
+static const uint64_t vtable4[] = {uint64_t(callee4), uint64_t(callee)};
+static const uint64_t vtable5[] = {uint64_t(callee5), uint64_t(callee)};
+static const uint64_t vtable6[] = {uint64_t(callee6), uint64_t(callee)};
+
+// Returns the address of callee with a numbered suffix in vtable.
+static uint64_t getCalleeAddress(const uint64_t *vtableAddr) {
+ uint64_t CalleeAddr;
+ // Callee with a numbered suffix is the 2nd element in vtable1 and vtable3,
+ // and the 1st element in the rest of vtables.
+ if (vtableAddr == vtable1 || vtableAddr == vtable3)
+ CalleeAddr = uint64_t(vtableAddr) + 8;
+ else
+ CalleeAddr = uint64_t(vtableAddr);
+ return CalleeAddr;
+}
+
+typedef std::vector<std::vector<InstrProfValueData>> VDArray;
+
struct SparseInstrProfTest : public InstrProfTest {
void SetUp() override { Writer.setOutputSparse(true); }
};
+// 'ValueDataArray' should be a non-const reference, and the array element is
+// a vector of InstrProfRecord. This is mainly because method
+// `InstrProfRecord::addValueData` might modify the underlying C array.
+static void addValueProfData(InstrProfRecord &Record, uint32_t ValueKind,
+ VDArray &ValueDataArray) {
+ Record.reserveSites(ValueKind, ValueDataArray.size());
+ for (long unsigned int i = 0; i < ValueDataArray.size(); i++) {
+ // The state of data() is not specified when the vector is empty. So, the
+ // state is valid but unspecified state.
+ Record.addValueData(ValueKind, i,
+ ValueDataArray[i].empty() ? nullptr
+ : ValueDataArray[i].data(),
+ ValueDataArray[i].size(), nullptr);
+ }
+}
+
+static void warn(Twine Message, std::string Whence = "",
+ std::string Hint = "") {
+ if (!Whence.empty())
+ errs() << Whence << ": ";
+ errs() << Message << "\n";
+}
+
+static const auto Err = [](Error E) {
+ if (E.isA<InstrProfError>()) {
+ handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
+ warn(IPE.message(), std::string(""), std::string(""));
+ });
+ }
+ consumeError(std::move(E));
+ FAIL();
+};
+
+static const auto Warn = [](InstrProfError E) {
+ Err(make_error<InstrProfError>(E));
+};
+
struct MaybeSparseInstrProfTest : public InstrProfTest,
public ::testing::WithParamInterface<bool> {
void SetUp() override { Writer.setOutputSparse(GetParam()); }
+ void TearDown() override {
+ // Restore little endianness after each test case.
+ Writer.setValueProfDataEndianness(llvm::endianness::little);
+ if (Reader)
+ Reader->setValueProfDataEndianness(llvm::endianness::little);
+ }
+
+private:
+ // Returns a copy rather than a reference so callers can modify the result.
+ static VDArray getFuncVDArrayForReaderWriterTest() {
+ static std::vector<InstrProfValueData> FuncVD0 = {
+ {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
+ static std::vector<InstrProfValueData> FuncVD2 = {{uint64_t(callee1), 1},
+ {uint64_t(callee2), 2}};
+ static std::vector<InstrProfValueData> FuncVD3 = {{uint64_t(callee7), 1}};
+ VDArray FuncVD = {FuncVD0, {}, FuncVD2, FuncVD3};
+ return FuncVD;
+ }
+
+ // Returns a copy rather than a reference so callers can modify the result.
+ static VDArray getVTableVDArrayForReaderWriterTest() {
+ static std::vector<InstrProfValueData> VTableVD0 = {
+ {getCalleeAddress(vtable1), 1},
+ {getCalleeAddress(vtable2), 2},
+ {getCalleeAddress(vtable3), 3},
+ };
+ static std::vector<InstrProfValueData> VTableVD2 = {
+ {getCalleeAddress(vtable1), 1},
+ {getCalleeAddress(vtable2), 2},
+ };
+ VDArray VTableVD = {VTableVD0, VTableVD2};
+ return VTableVD;
+ }
+
+public:
+ void testValueProfileMergeSaturation(uint32_t ValueKind);
+ void testValueProfileMergeTrunc(uint32_t ValueKind);
+
+ // Tests that value profiles in Record has the same content as (possibly
+ // weighted and sorted) InputVDs for each value kind. ValueProfSorted is true
+ // iff the value profiles of Record are already sorted by count.
+ // InputVDs is a non-const reference since it might need a in-place sort.
+ void testValueDataArray(
+ std::vector<std::pair<uint32_t /* ValueKind */, VDArray &>> &InputVDs,
+ InstrProfRecord &Record, bool ValueProfSorted = false,
+ uint64_t ProfWeight = 1) {
+ for (auto &[ValueKind, InputVD] : InputVDs) {
+ // Tests that the numbers of instrumented value sites are the same.
+ EXPECT_EQ(InputVD.size(), Record.getNumValueSites(ValueKind));
+ for (unsigned i = 0; i < InputVD.size(); i++) {
+ ASSERT_EQ(InputVD[i].size(),
+ Record.getNumValueDataForSite(ValueKind, i));
+
+ uint64_t WantTotalC = 0, GetTotalC = 0;
+ std::unique_ptr<InstrProfValueData[]> OutputVD =
+ Record.getValueForSite(ValueKind, i, &GetTotalC);
+
+ // If value profile elements of the same instrumented site are sorted in
+ // Record, sort the input value data array for comparison.
+ if (ValueProfSorted) {
+ llvm::stable_sort(InputVD[i], [](const InstrProfValueData &lhs,
+ const InstrProfValueData &rhs) {
+ return lhs.Count > rhs.Count;
+ });
+ }
+
+ // The previous ASSERT_EQ already tests the number of value data is
+ // InputVD[i].size(), so there won't be out-of-bound access.
+ for (unsigned j = 0; j < InputVD[i].size(); j++) {
+ EXPECT_EQ(InputVD[i][j].Count * ProfWeight, OutputVD[j].Count);
+ EXPECT_EQ(InputVD[i][j].Value, OutputVD[j].Value);
+ WantTotalC += InputVD[i][j].Count;
+ }
+ EXPECT_EQ(WantTotalC * ProfWeight, GetTotalC);
+ }
+ }
+ }
+
+ void testICallAndVTableDataReadWrite(
+ uint64_t ProfWeight,
+ llvm::endianness Endianness = llvm::endianness::little) {
+ NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
+
+ // 4 function value sites.
+ VDArray FuncVD =
+ MaybeSparseInstrProfTest::getFuncVDArrayForReaderWriterTest();
+ addValueProfData(Record1, IPVK_IndirectCallTarget, FuncVD);
+
+ // 2 vtable sites.
+ VDArray VTableVD =
+ MaybeSparseInstrProfTest::getVTableVDArrayForReaderWriterTest();
+ addValueProfData(Record1, IPVK_VTableTarget, VTableVD);
+
+ if (ProfWeight == 1U) {
+ Writer.addRecord(std::move(Record1), Err);
+ } else {
+ Writer.addRecord(std::move(Record1), ProfWeight, Err);
+ }
+ Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
+ Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
+ Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
+ Writer.addRecord({"callee7", 0x1235, {3, 4}}, Err);
+ Writer.addVTableName("vtable1");
+ Writer.addVTableName("vtable2");
+ Writer.addVTableName("vtable3");
+
+ // Set writer endianness.
+ Writer.setValueProfDataEndianness(Endianness);
+
+ auto Profile = Writer.writeBuffer();
+ readProfile(std::move(Profile));
+
+ // Set reader endianness.
+ Reader->setValueProfDataEndianness(Endianness);
+
+ Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
+ EXPECT_THAT_ERROR(R.takeError(), Succeeded());
+
+ std::vector<std::pair<uint32_t, VDArray &>> InputVDs = {
+ std::pair<uint32_t, VDArray &>{IPVK_IndirectCallTarget, FuncVD},
+ {IPVK_VTableTarget, VTableVD}};
+ testValueDataArray(InputVDs, R.get(), true, ProfWeight);
+ }
};
TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) {
@@ -79,11 +276,6 @@ TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) {
ASSERT_TRUE(Reader->begin() == Reader->end());
}
-static const auto Err = [](Error E) {
- consumeError(std::move(E));
- FAIL();
-};
-
TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) {
Writer.addRecord({"foo", 0x1234, {1, 2, 3, 4}}, Err);
auto Profile = Writer.writeBuffer();
@@ -633,55 +825,8 @@ TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) {
Succeeded());
}
-static const char callee1[] = "callee1";
-static const char callee2[] = "callee2";
-static const char callee3[] = "callee3";
-static const char callee4[] = "callee4";
-static const char callee5[] = "callee5";
-static const char callee6[] = "callee6";
-
-TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) {
- NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
-
- // 4 value sites.
- Record1.reserveSites(IPVK_IndirectCallTarget, 4);
- InstrProfValueData VD0[] = {
- {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
- Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
- // No value profile data at the second site.
- Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
- InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
- Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
- InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
- Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
-
- Writer.addRecord(std::move(Record1), Err);
- Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
- Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
- Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
- auto Profile = Writer.writeBuffer();
- readProfile(std::move(Profile));
-
- Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
- EXPECT_THAT_ERROR(R.takeError(), Succeeded());
- ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
- ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
- ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
- ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
- ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
-
- uint64_t TotalC;
- std::unique_ptr<InstrProfValueData[]> VD =
- R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
-
- ASSERT_EQ(3U, VD[0].Count);
- ASSERT_EQ(2U, VD[1].Count);
- ASSERT_EQ(1U, VD[2].Count);
- ASSERT_EQ(6U, TotalC);
-
- ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
- ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
- ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
+TEST_P(MaybeSparseInstrProfTest, get_icall_and_vtable_data_read_write) {
+ testICallAndVTableDataReadWrite(1 /* ProfWeight */);
}
TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
@@ -780,142 +925,91 @@ TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
ASSERT_EQ(1U, ValueData[3].Count);
}
-TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_with_weight) {
- NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
-
- // 4 value sites.
- Record1.reserveSites(IPVK_IndirectCallTarget, 4);
- InstrProfValueData VD0[] = {
- {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
- Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
- // No value profile data at the second site.
- Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
- InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
- Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
- InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
- Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
-
- Writer.addRecord(std::move(Record1), 10, Err);
- Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
- Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
- Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
- auto Profile = Writer.writeBuffer();
- readProfile(std::move(Profile));
-
- Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
- EXPECT_THAT_ERROR(R.takeError(), Succeeded());
- ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
- ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
- ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
- ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
- ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
-
- uint64_t TotalC;
- std::unique_ptr<InstrProfValueData[]> VD =
- R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
- ASSERT_EQ(30U, VD[0].Count);
- ASSERT_EQ(20U, VD[1].Count);
- ASSERT_EQ(10U, VD[2].Count);
- ASSERT_EQ(60U, TotalC);
-
- ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
- ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
- ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
+TEST_P(MaybeSparseInstrProfTest,
+ get_icall_and_vtable_data_read_write_with_weight) {
+ testICallAndVTableDataReadWrite(10 /* ProfWeight */);
}
-TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) {
- NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
-
- // 4 value sites.
- Record1.reserveSites(IPVK_IndirectCallTarget, 4);
- InstrProfValueData VD0[] = {
- {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
- Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
- // No value profile data at the second site.
- Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
- InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
- Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
- InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
- Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
-
- Writer.addRecord(std::move(Record1), Err);
- Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
- Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
- Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
-
- // Set big endian output.
- Writer.setValueProfDataEndianness(llvm::endianness::big);
-
- auto Profile = Writer.writeBuffer();
- readProfile(std::move(Profile));
-
- // Set big endian input.
- Reader->setValueProfDataEndianness(llvm::endianness::big);
-
- Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
- EXPECT_THAT_ERROR(R.takeError(), Succeeded());
- ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
- ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
- ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
- ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
- ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
-
- std::unique_ptr<InstrProfValueData[]> VD =
- R->getValueForSite(IPVK_IndirectCallTarget, 0);
- ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
- ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
- ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
+TEST_P(MaybeSparseInstrProfTest,
+ get_icall_and_vtable_data_read_write_big_endian) {
+ testICallAndVTableDataReadWrite(1 /* ProfWeight */, llvm::endianness::big);
+}
- // Restore little endian default:
- Writer.setValueProfDataEndianness(llvm::endianness::little);
+// A helper function to get the merged value profiles from inputs.
+static VDArray getMergedValueProfiles(const VDArray &FirstVDArray,
+ const VDArray &SecondVDArray) {
+ // Caller should merge value profiles only if the number of value profiling
+ // sitess is the same.
+ assert(FirstVDArray.size() == SecondVDArray.size());
+ InstrProfValueSiteRecord First, Second;
+ VDArray MergedVDArray;
+ for (unsigned i = 0; i < FirstVDArray.size(); i++) {
+ First.ValueData.clear();
+ Second.ValueData.clear();
+ // InstrProfValueSiteRecord stores value data in std::list.
+ std::copy(FirstVDArray[i].begin(), FirstVDArray[i].end(),
+ std::back_inserter(First.ValueData));
+ std::copy(SecondVDArray[i].begin(), SecondVDArray[i].end(),
+ std::back_inserter(Second.ValueData));
+ First.merge(Second, 1 /* ProfWeight */, Warn);
+ MergedVDArray.push_back({First.ValueData.begin(), First.ValueData.end()});
+ }
+ return MergedVDArray;
}
-TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) {
+TEST_P(MaybeSparseInstrProfTest, get_icall_and_vtable_value_profile_merge) {
static const char caller[] = "caller";
NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
- NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
-
- // 5 value sites.
- Record11.reserveSites(IPVK_IndirectCallTarget, 5);
- InstrProfValueData VD0[] = {{uint64_t(callee1), 1},
- {uint64_t(callee2), 2},
- {uint64_t(callee3), 3},
- {uint64_t(callee4), 4}};
- Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr);
- // No value profile data at the second site.
- Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
-
- InstrProfValueData VD2[] = {
+ // Record1
+ // 6 value sites for indirect calls.
+ std::vector<InstrProfValueData> VD0 = {{uint64_t(callee1), 1},
+ {uint64_t(callee2), 2},
+ {uint64_t(callee3), 3},
+ {uint64_t(callee4), 4}};
+ std::vector<InstrProfValueData> VD2 = {
{uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
- Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
-
- InstrProfValueData VD3[] = {{uint64_t(callee1), 1}};
- Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
-
- InstrProfValueData VD4[] = {{uint64_t(callee1), 1},
- {uint64_t(callee2), 2},
- {uint64_t(callee3), 3}};
- Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr);
-
- // A different record for the same caller.
- Record12.reserveSites(IPVK_IndirectCallTarget, 5);
- InstrProfValueData VD02[] = {{uint64_t(callee2), 5}, {uint64_t(callee3), 3}};
- Record12.addValueData(IPVK_IndirectCallTarget, 0, VD02, 2, nullptr);
-
- // No value profile data at the second site.
- Record12.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
+ std::vector<InstrProfValueData> VD3 = {{uint64_t(callee1), 1}};
+ std::vector<InstrProfValueData> VD4 = {
+ {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
+ std::vector<InstrProfValueData> VD5 = {{uint64_t(callee5), 3},
+ {uint64_t(callee6), 3}};
+ // No function value profile data at the second site.
+ VDArray FuncVD0 = {VD0, {}, VD2, VD3, VD4, VD5};
+ addValueProfData(Record11, IPVK_IndirectCallTarget, FuncVD0);
+
+ // 4 value sites for vtables.
+ std::vector<InstrProfValueData> VTVD0 = {{getCalleeAddress(vtable1), 6},
+ {getCalleeAddress(vtable2), 4}};
+ std::vector<InstrProfValueData> VTVD2 = {{getCalleeAddress(vtable1), 6}};
+ std::vector<InstrProfValueData> VTVD3 = {{getCalleeAddress(vtable1), 1}};
+ std::vector<InstrProfValueData> VTVD4 = {{getCalleeAddress(vtable1), 6}};
+ // No vtable value profile data at the second site.
+ VDArray VTableVD0 = {VTVD0, {}, VTVD2, VTVD3, VTVD4};
+ addValueProfData(Record11, IPVK_VTableTarget, VTableVD0);
+
+ // Record2 is a different record for the same caller.
+ NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
- InstrProfValueData VD22[] = {
+ // Record2 should have the same number of value profiling sites as record1.
+ std::vector<InstrProfValueData> VD02 = {{uint64_t(callee2), 5},
+ {uint64_t(callee3), 3}};
+ std::vector<InstrProfValueData> VD22 = {
{uint64_t(callee2), 1}, {uint64_t(callee3), 3}, {uint64_t(callee4), 4}};
- Record12.addValueData(IPVK_IndirectCallTarget, 2, VD22, 3, nullptr);
-
- Record12.addValueData(IPVK_IndirectCallTarget, 3, nullptr, 0, nullptr);
-
- InstrProfValueData VD42[] = {{uint64_t(callee1), 1},
- {uint64_t(callee2), 2},
- {uint64_t(callee3), 3}};
- Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, nullptr);
+ std::vector<InstrProfValueData> VD42 = {
+ {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
+ std::vector<InstrProfValueData> VD52 = {{uint64_t(callee5), 5},
+ {uint64_t(callee6), 3}};
+ // No function value profile data at the second and fourth site.
+ VDArray FuncVD1 = {VD02, {}, VD22, {}, VD42, VD52};
+ addValueProfData(Record12, IPVK_IndirectCallTarget, FuncVD1);
+
+ std::vector<InstrProfValueData> VTVD02 = {{getCalleeAddress(vtable1), 8}};
+ std::vector<InstrProfValueData> VTVD22 = {{getCalleeAddress(vtable1), 4},
+ {getCalleeAddress(vtable2), 4}};
+ std::vector<InstrProfValueData> VTVD42 = {{getCalleeAddress(vtable1), 6}};
+ VDArray VTableVD1 = {VTVD02, {}, VTVD22, {}, VTVD42};
+ addValueProfData(Record12, IPVK_VTableTarget, VTableVD1);
Writer.addRecord(std::move(Record11), Err);
// Merge profile data.
@@ -924,63 +1018,47 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) {
Writer.addRecord({callee1, 0x1235, {3, 4}}, Err);
Writer.addRecord({callee2, 0x1235, {3, 4}}, Err);
Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
- Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
Writer.addRecord({callee4, 0x1235, {3, 5}}, Err);
+ Writer.addRecord({callee5, 0x1235, {3, 5}}, Err);
+ Writer.addRecord({callee6, 0x1235, {3, 5}}, Err);
auto Profile = Writer.writeBuffer();
readProfile(std::move(Profile));
Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
EXPECT_THAT_ERROR(R.takeError(), Succeeded());
- ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget));
- ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
- ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
- ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
- ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
- ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
- std::unique_ptr<InstrProfValueData[]> VD =
- R->getValueForSite(IPVK_IndirectCallTarget, 0);
- ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2"));
- ASSERT_EQ(7U, VD[0].Count);
- ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3"));
- ASSERT_EQ(6U, VD[1].Count);
- ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4"));
- ASSERT_EQ(4U, VD[2].Count);
- ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1"));
- ASSERT_EQ(1U, VD[3].Count);
-
- std::unique_ptr<InstrProfValueData[]> VD_2(
- R->getValueForSite(IPVK_IndirectCallTarget, 2));
- ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3"));
- ASSERT_EQ(6U, VD_2[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4"));
- ASSERT_EQ(4U, VD_2[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2"));
- ASSERT_EQ(3U, VD_2[2].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1"));
- ASSERT_EQ(1U, VD_2[3].Count);
-
- std::unique_ptr<InstrProfValueData[]> VD_3(
- R->getValueForSite(IPVK_IndirectCallTarget, 3));
- ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1"));
- ASSERT_EQ(1U, VD_3[0].Count);
-
- std::unique_ptr<InstrProfValueData[]> VD_4(
- R->getValueForSite(IPVK_IndirectCallTarget, 4));
- ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3"));
- ASSERT_EQ(6U, VD_4[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2"));
- ASSERT_EQ(4U, VD_4[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1"));
- ASSERT_EQ(2U, VD_4[2].Count);
+ // Test that merged value profiles in `Record` is the same as merged input
+ // value profiles.
+ VDArray MergedFuncVD = getMergedValueProfiles(FuncVD0, FuncVD1);
+ VDArray MergedVTableVD = getMergedValueProfiles(VTableVD0, VTableVD1);
+ std::vector<std::pair<uint32_t, VDArray &>> InputVDs = {
+ std::pair<uint32_t, VDArray &>{IPVK_IndirectCallTarget, MergedFuncVD},
+ {IPVK_VTableTarget, MergedVTableVD}};
+ testValueDataArray(InputVDs, R.get(), true /* ValueProfSorted */);
}
-TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) {
+void MaybeSparseInstrProfTest::testValueProfileMergeSaturation(
+ uint32_t ValueKind) {
+ assert(ValueKind == IPVK_IndirectCallTarget ||
+ ValueKind == IPVK_VTableTarget);
static const char bar[] = "bar";
+ static const uint64_t vtable[] = {uint64_t(bar)};
+
+ auto getProfiledAddr = [](uint32_t ValueKind) {
+ if (ValueKind == IPVK_IndirectCallTarget)
+ return uint64_t(bar);
+ return uint64_t(vtable);
+ };
+
+ const uint64_t ProfiledValue = getProfiledAddr(ValueKind);
+
+ InstrProfValueData VDWithCounterOne[] = {{ProfiledValue, 1}};
const uint64_t MaxValCount = std::numeric_limits<uint64_t>::max();
const uint64_t MaxEdgeCount = getInstrMaxCountValue();
+ InstrProfValueData VDWithMaxValCount[] = {{ProfiledValue, MaxValCount}};
+
instrprof_error Result;
auto Err = [&](Error E) {
Result = std::get<0>(InstrProfError::take(std::move(E)));
@@ -999,18 +1077,18 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) {
ASSERT_EQ(Result, instrprof_error::success);
NamedInstrProfRecord Record4("baz", 0x5678, {3, 4});
- Record4.reserveSites(IPVK_IndirectCallTarget, 1);
- InstrProfValueData VD4[] = {{uint64_t(bar), 1}};
- Record4.addValueData(IPVK_IndirectCallTarget, 0, VD4, 1, nullptr);
+ Record4.reserveSites(ValueKind, 1);
+
+ Record4.addValueData(ValueKind, 0, VDWithCounterOne, 1, nullptr);
Result = instrprof_error::success;
Writer.addRecord(std::move(Record4), Err);
ASSERT_EQ(Result, instrprof_error::success);
// Verify value data counter overflow.
NamedInstrProfRecord Record5("baz", 0x5678, {5, 6});
- Record5.reserveSites(IPVK_IndirectCallTarget, 1);
- InstrProfValueData VD5[] = {{uint64_t(bar), MaxValCount}};
- Record5.addValueData(IPVK_IndirectCallTarget, 0, VD5, 1, nullptr);
+ Record5.reserveSites(ValueKind, 1);
+
+ Record5.addValueData(ValueKind, 0, VDWithMaxValCount, 1, nullptr);
Result = instrprof_error::success;
Writer.addRecord(std::move(Record5), Err);
ASSERT_EQ(Result, instrprof_error::counter_overflow);
@@ -1027,42 +1105,47 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) {
Expected<InstrProfRecord> ReadRecord2 =
Reader->getInstrProfRecord("baz", 0x5678);
ASSERT_TRUE(bool(ReadRecord2));
- ASSERT_EQ(1U, ReadRecord2->getNumValueSites(IPVK_IndirectCallTarget));
+ ASSERT_EQ(1U, ReadRecord2->getNumValueSites(ValueKind));
std::unique_ptr<InstrProfValueData[]> VD =
- ReadRecord2->getValueForSite(IPVK_IndirectCallTarget, 0);
- ASSERT_EQ(StringRef("bar"), StringRef((const char *)VD[0].Value, 3));
+ ReadRecord2->getValueForSite(ValueKind, 0);
+ ASSERT_EQ(ProfiledValue, VD[0].Value);
ASSERT_EQ(MaxValCount, VD[0].Count);
}
-// This test tests that when there are too many values
-// for a given site, the merged results are properly
-// truncated.
-TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) {
+TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_saturation) {
+ testValueProfileMergeSaturation(IPVK_IndirectCallTarget);
+}
+
+TEST_P(MaybeSparseInstrProfTest, get_vtable_data_merge_saturation) {
+ testValueProfileMergeSaturation(IPVK_VTableTarget);
+}
+
+void MaybeSparseInstrProfTest::testValueProfileMergeTrunc(uint32_t ValueKind) {
static const char caller[] = "caller";
NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
// 2 value sites.
- Record11.reserveSites(IPVK_IndirectCallTarget, 2);
+ Record11.reserveSites(ValueKind, 2);
InstrProfValueData VD0[255];
for (int I = 0; I < 255; I++) {
VD0[I].Value = 2 * I;
VD0[I].Count = 2 * I + 1000;
}
- Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 255, nullptr);
- Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
+ Record11.addValueData(ValueKind, 0, VD0, 255, nullptr);
+ Record11.addValueData(ValueKind, 1, nullptr, 0, nullptr);
- Record12.reserveSites(IPVK_IndirectCallTarget, 2);
+ Record12.reserveSites(ValueKind, 2);
InstrProfValueData VD1[255];
for (int I = 0; I < 255; I++) {
VD1[I].Value = 2 * I + 1;
VD1[I].Count = 2 * I + 1001;
}
- Record12.addValueData(IPVK_IndirectCallTarget, 0, VD1, 255, nullptr);
- Record12.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
+ Record12.addValueData(ValueKind, 0, VD1, 255, nullptr);
+ Record12.addValueData(ValueKind, 1, nullptr, 0, nullptr);
Writer.addRecord(std::move(Record11), Err);
// Merge profile data.
@@ -1073,108 +1156,104 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) {
Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
EXPECT_THAT_ERROR(R.takeError(), Succeeded());
- std::unique_ptr<InstrProfValueData[]> VD(
- R->getValueForSite(IPVK_IndirectCallTarget, 0));
- ASSERT_EQ(2U, R->getNumValueSites(IPVK_IndirectCallTarget));
- ASSERT_EQ(255U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
+ ASSERT_EQ(2U, R->getNumValueSites(ValueKind));
+ std::unique_ptr<InstrProfValueData[]> VD(R->getValueForSite(ValueKind, 0));
+ ASSERT_EQ(255U, R->getNumValueDataForSite(ValueKind, 0));
for (unsigned I = 0; I < 255; I++) {
ASSERT_EQ(VD[I].Value, 509 - I);
ASSERT_EQ(VD[I].Count, 1509 - I);
}
}
-static void addValueProfData(InstrProfRecord &Record) {
- Record.reserveSites(IPVK_IndirectCallTarget, 5);
- InstrProfValueData VD0[] = {{uint64_t(callee1), 400},
- {uint64_t(callee2), 1000},
- {uint64_t(callee3), 500},
- {uint64_t(callee4), 300},
- {uint64_t(callee5), 100}};
- Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr);
- InstrProfValueData VD1[] = {{uint64_t(callee5), 800},
- {uint64_t(callee3), 1000},
- {uint64_t(callee2), 2500},
- {uint64_t(callee1), 1300}};
- Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr);
- InstrProfValueData VD2[] = {{uint64_t(callee6), 800},
- {uint64_t(callee3), 1000},
- {uint64_t(callee4), 5500}};
- Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
- InstrProfValueData VD3[] = {{uint64_t(callee2), 1800},
- {uint64_t(callee3), 2000}};
- Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr);
- Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr);
+// Test that when there are too many values for a given indirect call site, the
+// merged results are properly truncated.
+TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) {
+ testValueProfileMergeTrunc(IPVK_IndirectCallTarget);
+}
+
+// Test that when there are too many values for a given vtable site, the merged
+// results are properly truncated.
+TEST_P(MaybeSparseInstrProfTest, get_vtable_data_merge_site_trunc) {
+ testValueProfileMergeTrunc(IPVK_VTableTarget);
+}
+
+static VDArray getFuncVDArrayForSerializationTest() {
+ static std::vector<InstrProfValueData> VD0 = {{uint64_t(callee1), 400},
+ {uint64_t(callee2), 1000},
+ {uint64_t(callee3), 500},
+ {uint64_t(callee4), 300},
+ {uint64_t(callee5), 100}};
+
+ static std::vector<InstrProfValueData> VD1 = {{uint64_t(callee5), 800},
+ {uint64_t(callee3), 1000},
+ {uint64_t(callee2), 2500},
+ {uint64_t(callee1), 1300}};
+
+ static std::vector<InstrProfValueData> VD2 = {{uint64_t(callee6), 800},
+ {uint64_t(callee3), 1000},
+ {uint64_t(callee4), 5500}};
+
+ static std::vector<InstrProfValueData> VD3 = {{uint64_t(callee7), 1800},
+ {uint64_t(callee8), 2000}};
+ // No value profile at the last site.
+ return VDArray{VD0, VD1, VD2, VD3, {}};
+}
+
+static VDArray getVTableVDArrayForSerializationTest() {
+ static std::vector<InstrProfValueData> VD0 = {
+ {getCalleeAddress(vtable1), 400},
+ {getCalleeAddress(vtable2), 1000},
+ {getCalleeAddress(vtable3), 500},
+ {getCalleeAddress(vtable4), 300},
+ {getCalleeAddress(vtable5), 100}};
+
+ static std::vector<InstrProfValueData> VD1 = {
+ {getCalleeAddress(vtable5), 800},
+ {getCalleeAddress(vtable3), 1000},
+ {getCalleeAddress(vtable2), 2500},
+ {getCalleeAddress(vtable1), 1300}};
+
+ static std::vector<InstrProfValueData> VD2 = {
+ {getCalleeAddress(vtable6), 800},
+ {getCalleeAddress(vtable3), 1000},
+ {getCalleeAddress(vtable4), 5500},
+ };
+ // No value profile at the fourth and last site.
+ return VDArray{VD0, VD1, VD2, {}, {}};
}
+// Tests 'serializeFrom' and 'deserializeTo' methods of ValueProfData.
TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) {
InstrProfRecord SrcRecord({1ULL << 31, 2});
- addValueProfData(SrcRecord);
+ VDArray FuncVDArray = getFuncVDArrayForSerializationTest();
+ VDArray VTableVDArray = getVTableVDArrayForSerializationTest();
+ addValueProfData(SrcRecord, IPVK_IndirectCallTarget, FuncVDArray);
+ addValueProfData(SrcRecord, IPVK_VTableTarget, VTableVDArray);
std::unique_ptr<ValueProfData> VPData =
ValueProfData::serializeFrom(SrcRecord);
InstrProfRecord Record({1ULL << 31, 2});
- VPData->deserializeTo(Record, nullptr);
+ // Symtab not provided, so each profied value is mapped to itself.
+ VPData->deserializeTo(Record, nullptr /* Symtab */);
- // Now read data from Record and sanity check the data
- ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
- ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
- ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
- ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
- ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
- ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
-
- auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
- return VD1.Count > VD2.Count;
+ std::vector<std::pair<uint32_t /* ValueKind */, VDArray &>> InputVDs = {
+ std::pair<uint32_t, VDArray &>{IPVK_IndirectCallTarget, FuncVDArray},
+ {IPVK_VTableTarget, VTableVDArray},
};
- std::unique_ptr<InstrProfValueData[]> VD_0(
- Record.getValueForSite(IPVK_IndirectCallTarget, 0));
- llvm::sort(&VD_0[0], &VD_0[5], Cmp);
- ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2"));
- ASSERT_EQ(1000U, VD_0[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3"));
- ASSERT_EQ(500U, VD_0[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1"));
- ASSERT_EQ(400U, VD_0[2].Count);
- ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4"));
- ASSERT_EQ(300U, VD_0[3].Count);
- ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5"));
- ASSERT_EQ(100U, VD_0[4].Count);
-
- std::unique_ptr<InstrProfValueData[]> VD_1(
- Record.getValueForSite(IPVK_IndirectCallTarget, 1));
- llvm::sort(&VD_1[0], &VD_1[4], Cmp);
- ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2"));
- ASSERT_EQ(2500U, VD_1[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1"));
- ASSERT_EQ(1300U, VD_1[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3"));
- ASSERT_EQ(1000U, VD_1[2].Count);
- ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5"));
- ASSERT_EQ(800U, VD_1[3].Count);
-
- std::unique_ptr<InstrProfValueData[]> VD_2(
- Record.getValueForSite(IPVK_IndirectCallTarget, 2));
- llvm::sort(&VD_2[0], &VD_2[3], Cmp);
- ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4"));
- ASSERT_EQ(5500U, VD_2[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3"));
- ASSERT_EQ(1000U, VD_2[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6"));
- ASSERT_EQ(800U, VD_2[2].Count);
-
- std::unique_ptr<InstrProfValueData[]> VD_3(
- Record.getValueForSite(IPVK_IndirectCallTarget, 3));
- llvm::sort(&VD_3[0], &VD_3[2], Cmp);
- ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3"));
- ASSERT_EQ(2000U, VD_3[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2"));
- ASSERT_EQ(1800U, VD_3[1].Count);
+ // Now read data from Record and sanity check the data. Value profiles are
+ // not sorted by 'serializeFrom' or 'deserializeTo'.
+ testValueDataArray(InputVDs, Record, false /* ValueProfSorted */, 1);
}
+// Tests that Symtab remaps the profiled values in 'deserializeTo' method.
TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
-
NamedInstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});
- addValueProfData(SrcRecord);
+ VDArray FuncVDArray = getFuncVDArrayForSerializationTest();
+ addValueProfData(SrcRecord, IPVK_IndirectCallTarget, FuncVDArray);
+
+ VDArray VTableVDArray = getVTableVDArrayForSerializationTest();
+ addValueProfData(SrcRecord, IPVK_VTableTarget, VTableVDArray);
+
std::unique_ptr<ValueProfData> VPData =
ValueProfData::serializeFrom(SrcRecord);
@@ -1186,6 +1265,23 @@ TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
Symtab.mapAddress(uint64_t(callee4), 0x4000ULL);
// Missing mapping for callee5
+ auto getVTableStartAddr = [](const uint64_t *vtable) -> uint64_t {
+ return uint64_t(vtable);
+ };
+ auto getVTableEndAddr = [](const uint64_t *vtable) -> uint64_t {
+ return uint64_t(vtable) + 16;
+ };
+ // vtable1, vtable2, vtable3, vtable4 get mapped; vtable5, vtable6 are not
+ // mapped.
+ Symtab.mapVTableAddress(getVTableStartAddr(vtable1),
+ getVTableEndAddr(vtable1), MD5Hash("vtable1"));
+ Symtab.mapVTableAddress(getVTableStartAddr(vtable2),
+ getVTableEndAddr(vtable2), MD5Hash("vtable2"));
+ Symtab.mapVTableAddress(getVTableStartAddr(vtable3),
+ getVTableEndAddr(vtable3), MD5Hash("vtable3"));
+ Symtab.mapVTableAddress(getVTableStartAddr(vtable4),
+ getVTableEndAddr(vtable4), MD5Hash("vtable4"));
+
VPData->deserializeTo(Record, &Symtab);
// Now read data from Record and sanity check the data
@@ -1207,6 +1303,66 @@ TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
// callee5 does not have a mapped value -- default to 0.
ASSERT_EQ(VD_0[4].Value, 0ULL);
+
+ // Sanity check the vtable value data
+ ASSERT_EQ(5U, Record.getNumValueSites(IPVK_VTableTarget));
+
+ // The fourth and last site has no vtable profile record.
+ EXPECT_EQ(0, Record.getNumValueDataForSite(IPVK_VTableTarget, 3));
+ EXPECT_EQ(nullptr, Record.getValueForSite(IPVK_VTableTarget, 3));
+ EXPECT_EQ(0, Record.getNumValueDataForSite(IPVK_VTableTarget, 4));
+ EXPECT_EQ(nullptr, Record.getValueForSite(IPVK_VTableTarget, 4));
+ {
+ // The first vtable site.
+ std::unique_ptr<InstrProfValueData[]> VD(
+ Record.getValueForSite(IPVK_VTableTarget, 0));
+ ASSERT_EQ(5, Record.getNumValueDataForSite(IPVK_VTableTarget, 0));
+ llvm::sort(&VD[0], &VD[5], Cmp);
+ EXPECT_EQ(1000U, VD[0].Count);
+ EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));
+ EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));
+ EXPECT_EQ(500U, VD[1].Count);
+ EXPECT_EQ(VD[2].Value, MD5Hash("vtable1"));
+ EXPECT_EQ(400U, VD[2].Count);
+ EXPECT_EQ(VD[3].Value, MD5Hash("vtable4"));
+ EXPECT_EQ(300U, VD[3].Count);
+
+ // vtable5 isn't mapped -- default to 0.
+ EXPECT_EQ(VD[4].Value, 0ULL);
+ EXPECT_EQ(VD[4].Count, 100ULL);
+ }
+
+ {
+ // The second vtable site.
+ std::unique_ptr<InstrProfValueData[]> VD(
+ Record.getValueForSite(IPVK_VTableTarget, 1));
+ ASSERT_EQ(4, Record.getNumValueDataForSite(IPVK_VTableTarget, 1));
+ llvm::sort(&VD[0], &VD[4], Cmp);
+ EXPECT_EQ(2500, VD[0].Count);
+ EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));
+ EXPECT_EQ(VD[1].Value, MD5Hash("vtable1"));
+ EXPECT_EQ(1300, VD[1].Count);
+ EXPECT_EQ(VD[2].Value, MD5Hash("vtable3"));
+ EXPECT_EQ(1000U, VD[2].Count);
+ // vtable5 isn't mapped -- default to 0.
+ EXPECT_EQ(VD[3].Value,0);
+ EXPECT_EQ(800U, VD[3].Count);
+ }
+
+ {
+ // The second vtable site.
+ std::unique_ptr<InstrProfValueData[]> VD(
+ Record.getValueForSite(IPVK_VTableTarget, 2));
+ ASSERT_EQ(3, Record.getNumValueDataForSite(IPVK_VTableTarget, 2));
+ llvm::sort(&VD[0], &VD[3], Cmp);
+ EXPECT_EQ(5500, VD[0].Count);
+ EXPECT_EQ(VD[0].Value, MD5Hash("vtable4"));
+ EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));
+ EXPECT_EQ(1000U, VD[1].Count);
+ // vtable6 isn't mapped -- default to 0.
+ EXPECT_EQ(VD[2].Value,0);
+ EXPECT_EQ(800U, VD[2].Count);
+ }
}
TEST_P(MaybeSparseInstrProfTest, get_max_function_count) {
>From c000e64d3270ece830c9a23c70d2e80c4ffb974a Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl at google.com>
Date: Thu, 16 Nov 2023 11:00:00 -0800
Subject: [PATCH 12/15] run clang-format over modified files
---
.../lib/profile/InstrProfilingInternal.h | 10 ++---
compiler-rt/lib/profile/InstrProfilingMerge.c | 2 +-
.../lib/profile/InstrProfilingPlatformLinux.c | 7 ++--
llvm/include/llvm/ProfileData/InstrProf.h | 37 +++++++------------
.../llvm/ProfileData/InstrProfReader.h | 12 +++---
llvm/lib/ProfileData/InstrProf.cpp | 28 +++++++-------
llvm/lib/ProfileData/InstrProfReader.cpp | 26 ++++++-------
llvm/lib/ProfileData/InstrProfWriter.cpp | 4 +-
.../Instrumentation/InstrProfiling.cpp | 4 +-
llvm/tools/llvm-profdata/llvm-profdata.cpp | 24 +++++-------
llvm/unittests/ProfileData/InstrProfTest.cpp | 26 ++++++-------
11 files changed, 80 insertions(+), 100 deletions(-)
diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h
index 38159b668a1df..0fd0e114c8b34 100644
--- a/compiler-rt/lib/profile/InstrProfilingInternal.h
+++ b/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -126,14 +126,14 @@ typedef struct VPDataReaderType {
uint8_t *SiteCountArray[]);
/* Function pointer to getValueProfRecordHeader method. */
uint32_t (*GetValueProfRecordHeaderSize)(uint32_t NumSites);
- /* Function pointer to getFristValueProfRecord method. */
+ /* Function pointer to getFristValueProfRecord method. */
struct ValueProfRecord *(*GetFirstValueProfRecord)(struct ValueProfData *);
/* Return the number of value data for site \p Site. */
uint32_t (*GetNumValueDataForSite)(uint32_t VK, uint32_t Site);
- /* Return the total size of the value profile data of the
+ /* Return the total size of the value profile data of the
* current function. */
uint32_t (*GetValueProfDataSize)(void);
- /*!
+ /*!
* Read the next \p N value data for site \p Site and store the data
* in \p Dst. \p StartNode is the first value node to start with if
* it is not null. The function returns the pointer to the value
@@ -167,7 +167,7 @@ void lprofMergeValueProfData(struct ValueProfData *SrcValueProfData,
VPDataReaderType *lprofGetVPDataReader();
-/* Internal interface used by test to reset the max number of
+/* Internal interface used by test to reset the max number of
* tracked values per value site to be \p MaxVals.
*/
void lprofSetMaxValsPerSite(uint32_t MaxVals);
@@ -180,7 +180,7 @@ void lprofSetupValueProfiler();
* to dump merged profile data into its own profile file. */
uint64_t lprofGetLoadModuleSignature();
-/*
+/*
* Return non zero value if the profile data has already been
* dumped to the file.
*/
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 53df1279ef961..12d305b249bad 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -83,7 +83,7 @@ int __llvm_profile_check_compatibility(const char *ProfileData,
return 1;
for (SrcData = SrcDataStart,
- DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
+ DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
SrcData < SrcDataEnd; ++SrcData, ++DstData) {
if (SrcData->NameRef != DstData->NameRef ||
SrcData->FuncHash != DstData->FuncHash ||
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
index dabe0e900465e..759926b676faf 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -6,8 +6,8 @@
|*
\*===----------------------------------------------------------------------===*/
-#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
- (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
+ (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \
defined(_AIX)
#if !defined(_AIX)
@@ -108,8 +108,7 @@ COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) {
return &PROF_ORDERFILE_START;
}
-COMPILER_RT_VISIBILITY ValueProfNode *
-__llvm_profile_begin_vnodes(void) {
+COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_begin_vnodes(void) {
return &PROF_VNODES_START;
}
COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_end_vnodes(void) {
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 7e1541fc21ccc..1e3728590a1e8 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -109,9 +109,7 @@ inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
/// Return the name of the variable holding the strings (possibly compressed)
/// of all function's PGO names.
-inline StringRef getInstrProfNamesVarName() {
- return "__llvm_prf_nm";
-}
+inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm"; }
inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; }
@@ -145,7 +143,8 @@ inline StringRef getInstrProfRegFuncName() {
return "__llvm_profile_register_function";
}
-/// Return the name of the runtime interface that registers the PGO name strings.
+/// Return the name of the runtime interface that registers the PGO name
+/// strings.
inline StringRef getInstrProfNamesRegFuncName() {
return "__llvm_profile_register_names_function";
}
@@ -467,9 +466,7 @@ class InstrProfSymtab {
RangeHashMap VTableAddrRangeToMD5Map;
bool Sorted = false;
- static StringRef getExternalSymbol() {
- return "** External Symbol **";
- }
+ static StringRef getExternalSymbol() { return "** External Symbol **"; }
Error addFuncWithName(Function &F, StringRef PGOFuncName);
@@ -517,7 +514,8 @@ class InstrProfSymtab {
/// Create InstrProfSymtab from a set of names iteratable from
/// \p IterRange. This interface is used by IndexedProfReader.
- template <typename NameIterRange> Error create(const NameIterRange &IterRange);
+ template <typename NameIterRange>
+ Error create(const NameIterRange &IterRange);
/// Create InstrProfSymtab from a set of function names and vtable
/// names iteratable from \p IterRange. This interface is used by
@@ -679,7 +677,7 @@ StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) {
return StringRef();
}
-Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
+Function *InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
finalizeSymtab();
auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash,
[](const std::pair<uint64_t, Function *> &LHS,
@@ -939,8 +937,8 @@ struct InstrProfRecord {
// cast away the constness from the result.
auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
ValueKind);
- return MutableArrayRef(
- const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
+ return MutableArrayRef(const_cast<InstrProfValueSiteRecord *>(AR.data()),
+ AR.size());
}
ArrayRef<InstrProfValueSiteRecord>
getValueSitesForKind(uint32_t ValueKind) const {
@@ -1091,10 +1089,7 @@ void InstrProfValueSiteRecord::sortByCount() {
namespace IndexedInstrProf {
-enum class HashT : uint32_t {
- MD5,
- Last = MD5
-};
+enum class HashT : uint32_t { MD5, Last = MD5 };
inline uint64_t ComputeHash(HashT Type, StringRef K) {
switch (Type) {
@@ -1240,13 +1235,9 @@ struct Summary {
return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
}
- uint64_t get(SummaryFieldKind K) const {
- return getSummaryDataBase()[K];
- }
+ uint64_t get(SummaryFieldKind K) const { return getSummaryDataBase()[K]; }
- void set(SummaryFieldKind K, uint64_t V) {
- getSummaryDataBase()[K] = V;
- }
+ void set(SummaryFieldKind K, uint64_t V) { getSummaryDataBase()[K] = V; }
const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
@@ -1297,8 +1288,8 @@ template <> inline uint64_t getMagic<uint32_t>() {
// It should also match the synthesized type in
// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
template <class IntPtrT> struct alignas(8) ProfileData {
- #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
- #include "llvm/ProfileData/InstrProfData.inc"
+#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
+#include "llvm/ProfileData/InstrProfData.inc"
};
template <class IntPtrT> struct alignas(8) VTableProfileData {
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 4e5f270001a5b..fbdb612d04ee0 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -309,8 +309,7 @@ class TextInstrProfReader : public InstrProfReader {
///
/// Templated on the unsigned type whose size matches pointers on the platform
/// that wrote the profile.
-template <class IntPtrT>
-class RawInstrProfReader : public InstrProfReader {
+template <class IntPtrT> class RawInstrProfReader : public InstrProfReader {
private:
/// The profile data file contents.
std::unique_ptr<MemoryBuffer> DataBuffer;
@@ -466,8 +465,8 @@ class RawInstrProfReader : public InstrProfReader {
}
const char *getNextHeaderPos() const {
- assert(atEnd());
- return (const char *)ValueDataStart;
+ assert(atEnd());
+ return (const char *)ValueDataStart;
}
StringRef getName(uint64_t NameRef) const {
@@ -550,7 +549,7 @@ struct InstrProfReaderIndexBase {
// Read all the profile records with the key equal to FuncName
virtual Error getRecords(StringRef FuncName,
- ArrayRef<NamedInstrProfRecord> &Data) = 0;
+ ArrayRef<NamedInstrProfRecord> &Data) = 0;
virtual void advanceToNextKey() = 0;
virtual bool atEnd() const = 0;
virtual void setValueProfDataEndianness(llvm::endianness Endianness) = 0;
@@ -574,8 +573,7 @@ using MemProfRecordHashTable =
using MemProfFrameHashTable =
OnDiskIterableChainedHashTable<memprof::FrameLookupTrait>;
-template <typename HashTableImpl>
-class InstrProfReaderItaniumRemapper;
+template <typename HashTableImpl> class InstrProfReaderItaniumRemapper;
template <typename HashTableImpl>
class InstrProfReaderIndex : public InstrProfReaderIndexBase {
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 150ae66d3e653..d9d6c8f4d42fc 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -265,7 +265,7 @@ std::string getPGOFuncName(StringRef RawFuncName,
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
uint32_t Count = NumPrefix;
uint32_t Pos = 0, LastPos = 0;
- for (auto & CI : PathNameStr) {
+ for (auto &CI : PathNameStr) {
++Pos;
if (llvm::sys::path::is_separator(CI)) {
LastPos = Pos;
@@ -642,9 +642,10 @@ Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
std::string UncompressedNameStrings =
join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
- assert(StringRef(UncompressedNameStrings)
- .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
- "PGO name is invalid (contains separator token)");
+ assert(
+ StringRef(UncompressedNameStrings).count(getInstrProfNameSeparator()) ==
+ (NameStrs.size() - 1) &&
+ "PGO name is invalid (contains separator token)");
unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
P += EncLen;
@@ -1037,19 +1038,19 @@ uint32_t getNumValueKindsInstrProf(const void *Record) {
}
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
- return reinterpret_cast<const InstrProfRecord *>(Record)
- ->getNumValueSites(VKind);
+ return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueSites(
+ VKind);
}
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
- return reinterpret_cast<const InstrProfRecord *>(Record)
- ->getNumValueData(VKind);
+ return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueData(
+ VKind);
}
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
uint32_t S) {
- return reinterpret_cast<const InstrProfRecord *>(R)
- ->getNumValueDataForSite(VK, S);
+ return reinterpret_cast<const InstrProfRecord *>(R)->getNumValueDataForSite(
+ VK, S);
}
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
@@ -1255,9 +1256,8 @@ void annotateValueSite(Module &M, Instruction &Inst,
}
void annotateValueSite(Module &M, Instruction &Inst,
- ArrayRef<InstrProfValueData> VDs,
- uint64_t Sum, InstrProfValueKind ValueKind,
- uint32_t MaxMDCount) {
+ ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
+ InstrProfValueKind ValueKind, uint32_t MaxMDCount) {
LLVMContext &Ctx = M.getContext();
MDBuilder MDHelper(Ctx);
SmallVector<Metadata *, 3> Vals;
@@ -1346,7 +1346,7 @@ MDNode *getPGOFuncNameMetadata(const Function &F) {
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
// Only for internal linkage functions.
if (PGOFuncName == F.getName())
- return;
+ return;
// Don't create duplicated meta-data.
if (getPGOFuncNameMetadata(F))
return;
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index d5f247a8521fb..1c68bb8ef76f0 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -322,8 +322,7 @@ Error TextInstrProfReader::readTemporalProfTraceData() {
return success();
}
-Error
-TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
+Error TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
#define CHECK_LINE_END(Line) \
if (Line.is_at_end()) \
@@ -504,13 +503,12 @@ bool RawInstrProfReader<IntPtrT>::hasFormat(const MemoryBuffer &DataBuffer) {
if (DataBuffer.getBufferSize() < sizeof(uint64_t))
return false;
uint64_t Magic =
- *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
+ *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
return RawInstrProf::getMagic<IntPtrT>() == Magic ||
llvm::byteswap(RawInstrProf::getMagic<IntPtrT>()) == Magic;
}
-template <class IntPtrT>
-Error RawInstrProfReader<IntPtrT>::readHeader() {
+template <class IntPtrT> Error RawInstrProfReader<IntPtrT>::readHeader() {
if (!hasFormat(*DataBuffer))
return error(instrprof_error::bad_magic);
if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header))
@@ -676,7 +674,8 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
if (BinaryIdsStart + BinaryIdsSize > BufferEnd)
return error(instrprof_error::bad_header);
- std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
+ std::unique_ptr<InstrProfSymtab> NewSymtab =
+ std::make_unique<InstrProfSymtab>();
if (Error E = createSymtab(*NewSymtab))
return E;
@@ -697,8 +696,7 @@ Error RawInstrProfReader<IntPtrT>::readFuncHash(NamedInstrProfRecord &Record) {
}
template <class IntPtrT>
-Error RawInstrProfReader<IntPtrT>::readRawCounts(
- InstrProfRecord &Record) {
+Error RawInstrProfReader<IntPtrT>::readRawCounts(InstrProfRecord &Record) {
uint32_t NumCounters = swap(Data->NumCounters);
if (NumCounters == 0)
return error(instrprof_error::malformed, "number of counters is zero");
@@ -837,7 +835,8 @@ Error RawInstrProfReader<IntPtrT>::readValueProfilingData(
}
template <class IntPtrT>
-Error RawInstrProfReader<IntPtrT>::readNextRecord(NamedInstrProfRecord &Record) {
+Error RawInstrProfReader<IntPtrT>::readNextRecord(
+ NamedInstrProfRecord &Record) {
// Keep reading profiles that consist of only headers and no profile data and
// counters.
while (atEnd())
@@ -1047,14 +1046,12 @@ class InstrProfReaderNullRemapper : public InstrProfReaderRemapper {
/// A remapper that applies remappings based on a symbol remapping file.
template <typename HashTableImpl>
-class llvm::InstrProfReaderItaniumRemapper
- : public InstrProfReaderRemapper {
+class llvm::InstrProfReaderItaniumRemapper : public InstrProfReaderRemapper {
public:
InstrProfReaderItaniumRemapper(
std::unique_ptr<MemoryBuffer> RemapBuffer,
InstrProfReaderIndex<HashTableImpl> &Underlying)
- : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {
- }
+ : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {}
/// Extract the original function name from a PGO function name.
static StringRef extractName(StringRef Name) {
@@ -1381,7 +1378,8 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
if (Symtab)
return *Symtab;
- std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
+ std::unique_ptr<InstrProfSymtab> NewSymtab =
+ std::make_unique<InstrProfSymtab>();
if (Error E = NewSymtab->initVTableNamesFromCompressedStrings(
StringRef(VTableNamePtr, CompressedVTableNamesLen))) {
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 491158e03aa8e..45400b14b3b90 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -194,9 +194,7 @@ void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {
InfoObj->ValueProfDataEndianness = Endianness;
}
-void InstrProfWriter::setOutputSparse(bool Sparse) {
- this->Sparse = Sparse;
-}
+void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; }
void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
function_ref<void(Error)> Warn) {
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 5db278f46af0e..ec33accbae020 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1207,8 +1207,8 @@ void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
UsedVars.push_back(Data);
}
-GlobalVariable *
-InstrProfiling::setupProfileSection(InstrProfInstBase *Inc, InstrProfSectKind IPSK) {
+GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc,
+ InstrProfSectKind IPSK) {
GlobalVariable *NamePtr = Inc->getName();
// Match the linkage and visibility of the name global.
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 6a3431369fcd1..ff481ff0033f5 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -333,10 +333,9 @@ cl::opt<bool> ShowIndirectCallTargets(
"ic-targets", cl::init(false),
cl::desc("Show indirect call site target values for shown functions"),
cl::sub(ShowSubcommand));
-cl::opt<bool> ShowVTables(
- "show-vtables", cl::init(false),
- cl::desc("Show vtable names for shown functions"),
- cl::sub(ShowSubcommand));
+cl::opt<bool> ShowVTables("show-vtables", cl::init(false),
+ cl::desc("Show vtable names for shown functions"),
+ cl::sub(ShowSubcommand));
cl::opt<bool> ShowMemOPSizes(
"memop-sizes", cl::init(false),
cl::desc("Show the profiled sizes of the memory intrinsic calls "
@@ -552,7 +551,7 @@ class SymbolRemapper {
return New.empty() ? Name : FunctionId(New);
}
};
-}
+} // namespace
struct WeightedFile {
std::string Filename;
@@ -1280,8 +1279,8 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
BodySample.second.getSamples());
for (const auto &Target : BodySample.second.getCallTargets()) {
Result.addCalledTargetSamples(BodySample.first.LineOffset,
- MaskedDiscriminator,
- Remapper(Target.first), Target.second);
+ MaskedDiscriminator, Remapper(Target.first),
+ Target.second);
}
}
for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
@@ -1297,12 +1296,8 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
}
static sampleprof::SampleProfileFormat FormatMap[] = {
- sampleprof::SPF_None,
- sampleprof::SPF_Text,
- sampleprof::SPF_None,
- sampleprof::SPF_Ext_Binary,
- sampleprof::SPF_GCC,
- sampleprof::SPF_Binary};
+ sampleprof::SPF_None, sampleprof::SPF_Text, sampleprof::SPF_None,
+ sampleprof::SPF_Ext_Binary, sampleprof::SPF_GCC, sampleprof::SPF_Binary};
static std::unique_ptr<MemoryBuffer>
getInputFileBuf(const StringRef &InputFile) {
@@ -3135,7 +3130,8 @@ static int show_main(int argc, const char *argv[]) {
exitWithErrorCode(EC, OutputFilename);
if (ShowAllFunctions && !FuncNameFilter.empty())
- WithColor::warning() << "-function argument ignored: showing all functions\n";
+ WithColor::warning()
+ << "-function argument ignored: showing all functions\n";
if (!DebugInfoFilename.empty())
return showDebugInfoCorrelation(DebugInfoFilename, SFormat, OS);
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index 1ad5778256465..2349dc6fe2864 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -832,8 +832,8 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_and_vtable_data_read_write) {
TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
NamedInstrProfRecord Record("caller", 0x1234, {1, 2});
Record.reserveSites(IPVK_IndirectCallTarget, 1);
- InstrProfValueData VD0[] = {{1000, 1}, {2000, 2}, {3000, 3}, {5000, 5},
- {4000, 4}, {6000, 6}};
+ InstrProfValueData VD0[] = {{1000, 1}, {2000, 2}, {3000, 3},
+ {5000, 5}, {4000, 4}, {6000, 6}};
Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 6, nullptr);
Writer.addRecord(std::move(Record), Err);
auto Profile = Writer.writeBuffer();
@@ -887,8 +887,8 @@ TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
Inst->setMetadata(LLVMContext::MD_prof, 0);
// Annotate 5 records this time.
annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0, 5);
- Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
- ValueData, N, T);
+ Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5, ValueData,
+ N, T);
ASSERT_TRUE(Res);
ASSERT_EQ(5U, N);
ASSERT_EQ(21U, T);
@@ -906,12 +906,12 @@ TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
// Remove the MD_prof metadata
Inst->setMetadata(LLVMContext::MD_prof, 0);
// Annotate with 4 records.
- InstrProfValueData VD0Sorted[] = {{1000, 6}, {2000, 5}, {3000, 4}, {4000, 3},
- {5000, 2}, {6000, 1}};
+ InstrProfValueData VD0Sorted[] = {{1000, 6}, {2000, 5}, {3000, 4},
+ {4000, 3}, {5000, 2}, {6000, 1}};
annotateValueSite(*M, *Inst, ArrayRef(VD0Sorted).slice(2), 10,
IPVK_IndirectCallTarget, 5);
- Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
- ValueData, N, T);
+ Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5, ValueData,
+ N, T);
ASSERT_TRUE(Res);
ASSERT_EQ(4U, N);
ASSERT_EQ(10U, T);
@@ -1306,13 +1306,13 @@ TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
// Sanity check the vtable value data
ASSERT_EQ(5U, Record.getNumValueSites(IPVK_VTableTarget));
-
+
// The fourth and last site has no vtable profile record.
EXPECT_EQ(0, Record.getNumValueDataForSite(IPVK_VTableTarget, 3));
EXPECT_EQ(nullptr, Record.getValueForSite(IPVK_VTableTarget, 3));
EXPECT_EQ(0, Record.getNumValueDataForSite(IPVK_VTableTarget, 4));
EXPECT_EQ(nullptr, Record.getValueForSite(IPVK_VTableTarget, 4));
- {
+ {
// The first vtable site.
std::unique_ptr<InstrProfValueData[]> VD(
Record.getValueForSite(IPVK_VTableTarget, 0));
@@ -1345,11 +1345,11 @@ TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
EXPECT_EQ(VD[2].Value, MD5Hash("vtable3"));
EXPECT_EQ(1000U, VD[2].Count);
// vtable5 isn't mapped -- default to 0.
- EXPECT_EQ(VD[3].Value,0);
+ EXPECT_EQ(VD[3].Value, 0);
EXPECT_EQ(800U, VD[3].Count);
}
- {
+ {
// The second vtable site.
std::unique_ptr<InstrProfValueData[]> VD(
Record.getValueForSite(IPVK_VTableTarget, 2));
@@ -1360,7 +1360,7 @@ TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));
EXPECT_EQ(1000U, VD[1].Count);
// vtable6 isn't mapped -- default to 0.
- EXPECT_EQ(VD[2].Value,0);
+ EXPECT_EQ(VD[2].Value, 0);
EXPECT_EQ(800U, VD[2].Count);
}
}
>From 313eb103b22dbed5134be34dd50d3ca8cdd24070 Mon Sep 17 00:00:00 2001
From: mingmingl <mingmingl at google.com>
Date: Sun, 26 Nov 2023 09:48:02 -0800
Subject: [PATCH 13/15] undo clang-format on unchanged lines
---
compiler-rt/lib/profile/InstrProfiling.h | 7 +++--
.../lib/profile/InstrProfilingInternal.h | 10 +++----
compiler-rt/lib/profile/InstrProfilingMerge.c | 2 +-
.../lib/profile/InstrProfilingPlatformLinux.c | 7 +++--
llvm/include/llvm/ProfileData/InstrProf.h | 30 ++++++++++++-------
.../llvm/ProfileData/InstrProfReader.h | 12 ++++----
llvm/lib/ProfileData/InstrProf.cpp | 28 ++++++++---------
llvm/lib/ProfileData/InstrProfReader.cpp | 26 ++++++++--------
llvm/lib/ProfileData/InstrProfWriter.cpp | 4 ++-
llvm/tools/llvm-profdata/llvm-profdata.cpp | 13 ++++----
10 files changed, 80 insertions(+), 59 deletions(-)
diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index 4fd35a07c67e0..04076590753ef 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -32,7 +32,7 @@ typedef struct __llvm_profile_header {
#include "profile/InstrProfData.inc"
} __llvm_profile_header;
-typedef struct ValueProfNode *PtrToNodeT;
+typedef struct ValueProfNode * PtrToNodeT;
typedef struct ValueProfNode {
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) Type Name;
#include "profile/InstrProfData.inc"
@@ -128,7 +128,8 @@ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size);
* structurally the in-process counters and bitmaps. If the profile data in
* buffer is not compatible, the interface returns 1 (failure).
*/
-int __llvm_profile_check_compatibility(const char *Profile, uint64_t Size);
+int __llvm_profile_check_compatibility(const char *Profile,
+ uint64_t Size);
/*!
* \brief Counts the number of times a target value is seen.
@@ -141,7 +142,7 @@ int __llvm_profile_check_compatibility(const char *Profile, uint64_t Size);
void INSTR_PROF_VALUE_PROF_FUNC(
#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) ArgType ArgName
#include "profile/InstrProfData.inc"
-);
+ );
void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data,
uint32_t CounterIndex,
diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h
index 0fd0e114c8b34..38159b668a1df 100644
--- a/compiler-rt/lib/profile/InstrProfilingInternal.h
+++ b/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -126,14 +126,14 @@ typedef struct VPDataReaderType {
uint8_t *SiteCountArray[]);
/* Function pointer to getValueProfRecordHeader method. */
uint32_t (*GetValueProfRecordHeaderSize)(uint32_t NumSites);
- /* Function pointer to getFristValueProfRecord method. */
+ /* Function pointer to getFristValueProfRecord method. */
struct ValueProfRecord *(*GetFirstValueProfRecord)(struct ValueProfData *);
/* Return the number of value data for site \p Site. */
uint32_t (*GetNumValueDataForSite)(uint32_t VK, uint32_t Site);
- /* Return the total size of the value profile data of the
+ /* Return the total size of the value profile data of the
* current function. */
uint32_t (*GetValueProfDataSize)(void);
- /*!
+ /*!
* Read the next \p N value data for site \p Site and store the data
* in \p Dst. \p StartNode is the first value node to start with if
* it is not null. The function returns the pointer to the value
@@ -167,7 +167,7 @@ void lprofMergeValueProfData(struct ValueProfData *SrcValueProfData,
VPDataReaderType *lprofGetVPDataReader();
-/* Internal interface used by test to reset the max number of
+/* Internal interface used by test to reset the max number of
* tracked values per value site to be \p MaxVals.
*/
void lprofSetMaxValsPerSite(uint32_t MaxVals);
@@ -180,7 +180,7 @@ void lprofSetupValueProfiler();
* to dump merged profile data into its own profile file. */
uint64_t lprofGetLoadModuleSignature();
-/*
+/*
* Return non zero value if the profile data has already been
* dumped to the file.
*/
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 12d305b249bad..53df1279ef961 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -83,7 +83,7 @@ int __llvm_profile_check_compatibility(const char *ProfileData,
return 1;
for (SrcData = SrcDataStart,
- DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
+ DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
SrcData < SrcDataEnd; ++SrcData, ++DstData) {
if (SrcData->NameRef != DstData->NameRef ||
SrcData->FuncHash != DstData->FuncHash ||
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
index 759926b676faf..dabe0e900465e 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -6,8 +6,8 @@
|*
\*===----------------------------------------------------------------------===*/
-#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
- (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
+ (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \
defined(_AIX)
#if !defined(_AIX)
@@ -108,7 +108,8 @@ COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) {
return &PROF_ORDERFILE_START;
}
-COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_begin_vnodes(void) {
+COMPILER_RT_VISIBILITY ValueProfNode *
+__llvm_profile_begin_vnodes(void) {
return &PROF_VNODES_START;
}
COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_end_vnodes(void) {
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 1e3728590a1e8..19c691a13c28e 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -109,7 +109,9 @@ inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
/// Return the name of the variable holding the strings (possibly compressed)
/// of all function's PGO names.
-inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm"; }
+inline StringRef getInstrProfNamesVarName() {
+ return "__llvm_prf_nm";
+}
inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; }
@@ -143,8 +145,7 @@ inline StringRef getInstrProfRegFuncName() {
return "__llvm_profile_register_function";
}
-/// Return the name of the runtime interface that registers the PGO name
-/// strings.
+/// Return the name of the runtime interface that registers the PGO name strings.
inline StringRef getInstrProfNamesRegFuncName() {
return "__llvm_profile_register_names_function";
}
@@ -466,7 +467,9 @@ class InstrProfSymtab {
RangeHashMap VTableAddrRangeToMD5Map;
bool Sorted = false;
- static StringRef getExternalSymbol() { return "** External Symbol **"; }
+ static StringRef getExternalSymbol() {
+ return "** External Symbol **";
+ }
Error addFuncWithName(Function &F, StringRef PGOFuncName);
@@ -677,7 +680,7 @@ StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) {
return StringRef();
}
-Function *InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
+Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
finalizeSymtab();
auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash,
[](const std::pair<uint64_t, Function *> &LHS,
@@ -937,8 +940,8 @@ struct InstrProfRecord {
// cast away the constness from the result.
auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
ValueKind);
- return MutableArrayRef(const_cast<InstrProfValueSiteRecord *>(AR.data()),
- AR.size());
+ return MutableArrayRef(
+ const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
}
ArrayRef<InstrProfValueSiteRecord>
getValueSitesForKind(uint32_t ValueKind) const {
@@ -1089,7 +1092,10 @@ void InstrProfValueSiteRecord::sortByCount() {
namespace IndexedInstrProf {
-enum class HashT : uint32_t { MD5, Last = MD5 };
+enum class HashT : uint32_t {
+ MD5,
+ Last = MD5
+};
inline uint64_t ComputeHash(HashT Type, StringRef K) {
switch (Type) {
@@ -1235,9 +1241,13 @@ struct Summary {
return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
}
- uint64_t get(SummaryFieldKind K) const { return getSummaryDataBase()[K]; }
+ uint64_t get(SummaryFieldKind K) const {
+ return getSummaryDataBase()[K];
+ }
- void set(SummaryFieldKind K, uint64_t V) { getSummaryDataBase()[K] = V; }
+ void set(SummaryFieldKind K, uint64_t V) {
+ getSummaryDataBase()[K] = V;
+ }
const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index f0d2c108780c3..f10802bc764f4 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -309,7 +309,8 @@ class TextInstrProfReader : public InstrProfReader {
///
/// Templated on the unsigned type whose size matches pointers on the platform
/// that wrote the profile.
-template <class IntPtrT> class RawInstrProfReader : public InstrProfReader {
+template <class IntPtrT>
+class RawInstrProfReader : public InstrProfReader {
private:
/// The profile data file contents.
std::unique_ptr<MemoryBuffer> DataBuffer;
@@ -461,8 +462,8 @@ template <class IntPtrT> class RawInstrProfReader : public InstrProfReader {
}
const char *getNextHeaderPos() const {
- assert(atEnd());
- return (const char *)ValueDataStart;
+ assert(atEnd());
+ return (const char *)ValueDataStart;
}
StringRef getName(uint64_t NameRef) const {
@@ -545,7 +546,7 @@ struct InstrProfReaderIndexBase {
// Read all the profile records with the key equal to FuncName
virtual Error getRecords(StringRef FuncName,
- ArrayRef<NamedInstrProfRecord> &Data) = 0;
+ ArrayRef<NamedInstrProfRecord> &Data) = 0;
virtual void advanceToNextKey() = 0;
virtual bool atEnd() const = 0;
virtual void setValueProfDataEndianness(llvm::endianness Endianness) = 0;
@@ -569,7 +570,8 @@ using MemProfRecordHashTable =
using MemProfFrameHashTable =
OnDiskIterableChainedHashTable<memprof::FrameLookupTrait>;
-template <typename HashTableImpl> class InstrProfReaderItaniumRemapper;
+template <typename HashTableImpl>
+class InstrProfReaderItaniumRemapper;
template <typename HashTableImpl>
class InstrProfReaderIndex : public InstrProfReaderIndexBase {
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index d9d6c8f4d42fc..150ae66d3e653 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -265,7 +265,7 @@ std::string getPGOFuncName(StringRef RawFuncName,
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
uint32_t Count = NumPrefix;
uint32_t Pos = 0, LastPos = 0;
- for (auto &CI : PathNameStr) {
+ for (auto & CI : PathNameStr) {
++Pos;
if (llvm::sys::path::is_separator(CI)) {
LastPos = Pos;
@@ -642,10 +642,9 @@ Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
std::string UncompressedNameStrings =
join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
- assert(
- StringRef(UncompressedNameStrings).count(getInstrProfNameSeparator()) ==
- (NameStrs.size() - 1) &&
- "PGO name is invalid (contains separator token)");
+ assert(StringRef(UncompressedNameStrings)
+ .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
+ "PGO name is invalid (contains separator token)");
unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
P += EncLen;
@@ -1038,19 +1037,19 @@ uint32_t getNumValueKindsInstrProf(const void *Record) {
}
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
- return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueSites(
- VKind);
+ return reinterpret_cast<const InstrProfRecord *>(Record)
+ ->getNumValueSites(VKind);
}
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
- return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueData(
- VKind);
+ return reinterpret_cast<const InstrProfRecord *>(Record)
+ ->getNumValueData(VKind);
}
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
uint32_t S) {
- return reinterpret_cast<const InstrProfRecord *>(R)->getNumValueDataForSite(
- VK, S);
+ return reinterpret_cast<const InstrProfRecord *>(R)
+ ->getNumValueDataForSite(VK, S);
}
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
@@ -1256,8 +1255,9 @@ void annotateValueSite(Module &M, Instruction &Inst,
}
void annotateValueSite(Module &M, Instruction &Inst,
- ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
- InstrProfValueKind ValueKind, uint32_t MaxMDCount) {
+ ArrayRef<InstrProfValueData> VDs,
+ uint64_t Sum, InstrProfValueKind ValueKind,
+ uint32_t MaxMDCount) {
LLVMContext &Ctx = M.getContext();
MDBuilder MDHelper(Ctx);
SmallVector<Metadata *, 3> Vals;
@@ -1346,7 +1346,7 @@ MDNode *getPGOFuncNameMetadata(const Function &F) {
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
// Only for internal linkage functions.
if (PGOFuncName == F.getName())
- return;
+ return;
// Don't create duplicated meta-data.
if (getPGOFuncNameMetadata(F))
return;
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 8b8b4cc09fe30..7f599aca12f39 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -312,7 +312,8 @@ Error TextInstrProfReader::readTemporalProfTraceData() {
return success();
}
-Error TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
+Error
+TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
#define CHECK_LINE_END(Line) \
if (Line.is_at_end()) \
@@ -493,12 +494,13 @@ bool RawInstrProfReader<IntPtrT>::hasFormat(const MemoryBuffer &DataBuffer) {
if (DataBuffer.getBufferSize() < sizeof(uint64_t))
return false;
uint64_t Magic =
- *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
+ *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
return RawInstrProf::getMagic<IntPtrT>() == Magic ||
llvm::byteswap(RawInstrProf::getMagic<IntPtrT>()) == Magic;
}
-template <class IntPtrT> Error RawInstrProfReader<IntPtrT>::readHeader() {
+template <class IntPtrT>
+Error RawInstrProfReader<IntPtrT>::readHeader() {
if (!hasFormat(*DataBuffer))
return error(instrprof_error::bad_magic);
if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header))
@@ -541,9 +543,8 @@ template <class IntPtrT>
Error RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {
if (Error E =
Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart),
- StringRef(VNamesStart, VNamesEnd - VNamesStart))) {
+ StringRef(VNamesStart, VNamesEnd - VNamesStart)))
return error(std::move(E));
- }
for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
const IntPtrT FPtr = swap(I->FunctionPointer);
if (!FPtr)
@@ -668,8 +669,7 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
BitmapEnd = BitmapStart + NumBitmapBytes;
ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
- std::unique_ptr<InstrProfSymtab> NewSymtab =
- std::make_unique<InstrProfSymtab>();
+ std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
if (Error E = createSymtab(*NewSymtab))
return E;
@@ -690,7 +690,8 @@ Error RawInstrProfReader<IntPtrT>::readFuncHash(NamedInstrProfRecord &Record) {
}
template <class IntPtrT>
-Error RawInstrProfReader<IntPtrT>::readRawCounts(InstrProfRecord &Record) {
+Error RawInstrProfReader<IntPtrT>::readRawCounts(
+ InstrProfRecord &Record) {
uint32_t NumCounters = swap(Data->NumCounters);
if (NumCounters == 0)
return error(instrprof_error::malformed, "number of counters is zero");
@@ -829,8 +830,7 @@ Error RawInstrProfReader<IntPtrT>::readValueProfilingData(
}
template <class IntPtrT>
-Error RawInstrProfReader<IntPtrT>::readNextRecord(
- NamedInstrProfRecord &Record) {
+Error RawInstrProfReader<IntPtrT>::readNextRecord(NamedInstrProfRecord &Record) {
// Keep reading profiles that consist of only headers and no profile data and
// counters.
while (atEnd())
@@ -1042,12 +1042,14 @@ class InstrProfReaderNullRemapper : public InstrProfReaderRemapper {
/// A remapper that applies remappings based on a symbol remapping file.
template <typename HashTableImpl>
-class llvm::InstrProfReaderItaniumRemapper : public InstrProfReaderRemapper {
+class llvm::InstrProfReaderItaniumRemapper
+ : public InstrProfReaderRemapper {
public:
InstrProfReaderItaniumRemapper(
std::unique_ptr<MemoryBuffer> RemapBuffer,
InstrProfReaderIndex<HashTableImpl> &Underlying)
- : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {}
+ : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {
+ }
/// Extract the original function name from a PGO function name.
static StringRef extractName(StringRef Name) {
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 5c79d6db8c097..85ee1a42d56f7 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -194,7 +194,9 @@ void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {
InfoObj->ValueProfDataEndianness = Endianness;
}
-void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; }
+void InstrProfWriter::setOutputSparse(bool Sparse) {
+ this->Sparse = Sparse;
+}
void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
function_ref<void(Error)> Warn) {
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index ff481ff0033f5..f40cc14f0f806 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -551,7 +551,7 @@ class SymbolRemapper {
return New.empty() ? Name : FunctionId(New);
}
};
-} // namespace
+}
struct WeightedFile {
std::string Filename;
@@ -1296,8 +1296,12 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
}
static sampleprof::SampleProfileFormat FormatMap[] = {
- sampleprof::SPF_None, sampleprof::SPF_Text, sampleprof::SPF_None,
- sampleprof::SPF_Ext_Binary, sampleprof::SPF_GCC, sampleprof::SPF_Binary};
+ sampleprof::SPF_None,
+ sampleprof::SPF_Text,
+ sampleprof::SPF_None,
+ sampleprof::SPF_Ext_Binary,
+ sampleprof::SPF_GCC,
+ sampleprof::SPF_Binary};
static std::unique_ptr<MemoryBuffer>
getInputFileBuf(const StringRef &InputFile) {
@@ -3130,8 +3134,7 @@ static int show_main(int argc, const char *argv[]) {
exitWithErrorCode(EC, OutputFilename);
if (ShowAllFunctions && !FuncNameFilter.empty())
- WithColor::warning()
- << "-function argument ignored: showing all functions\n";
+ WithColor::warning() << "-function argument ignored: showing all functions\n";
if (!DebugInfoFilename.empty())
return showDebugInfoCorrelation(DebugInfoFilename, SFormat, OS);
>From abcbc6d4ad23426e2dbd183ada26964bbdd971b6 Mon Sep 17 00:00:00 2001
From: mingmingl <mingmingl at google.com>
Date: Sat, 2 Dec 2023 18:38:12 -0800
Subject: [PATCH 14/15] Changes: 1. In InstrProfiling.cpp, use the vtable's PGO
name. The previous commits uses mangled name. This follows how function
PGO name is stored in raw profiles. a) For unit test, add test coverage
for new value profile kind in InstrProfTest.cpp b) For regression
tests, 1) In test/tools/llvm-profdata, add test case for local-linkage
vtables. 2) In test/Transforms/PGOProfile, add test coverage for
local-linkage vtables and simplify the GEN/LOWER tested lines. 2. In
ValueProfilePlugins.inc, skip all EHPad instructions when finding an
insertion point.
---
llvm/lib/ProfileData/InstrProf.cpp | 31 +-
.../Instrumentation/InstrProfiling.cpp | 12 +-
.../Instrumentation/ValueProfilePlugins.inc | 11 +-
.../Transforms/PGOProfile/vtable_profile.ll | 234 ++-----
.../Inputs/update_vtable_value_prof_inputs.sh | 5 +-
.../llvm-profdata/Inputs/vtable-prof.proftext | 73 ---
.../llvm-profdata/Inputs/vtable_prof.profraw | Bin 888 -> 1096 bytes
.../llvm-profdata/Inputs/vtable_prof.proftext | 74 +++
.../llvm-profdata/binary-ids-padding.test | 10 -
.../tools/llvm-profdata/vtable-prof.proftext | 6 +-
.../vtable-value-prof-basic.test | 24 +-
llvm/unittests/ProfileData/InstrProfTest.cpp | 604 ++++++++++++++----
12 files changed, 647 insertions(+), 437 deletions(-)
delete mode 100644 llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
create mode 100644 llvm/test/tools/llvm-profdata/Inputs/vtable_prof.proftext
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 150ae66d3e653..5a07f0400e408 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -343,8 +343,8 @@ static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO,
}
// In LTO mode (when InLTO is true), first check if there is a meta data.
- if (auto IRPGOName = lookupPGONameFromMetadata(PGONameMetadata))
- return *IRPGOName;
+ if (auto IRPGOFuncName = lookupPGONameFromMetadata(PGONameMetadata))
+ return *IRPGOFuncName;
// If there is no meta data, the function must be a global before the value
// profile annotation pass. Its current linkage may be internal if it is
@@ -358,36 +358,31 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO) {
return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F));
}
-static std::string getPGOObjectName(const GlobalObject &GO, bool InLTO,
- MDNode *PGONameMetadata) {
+// This is similar to `getIRPGOFuncName` except that this function calls
+// 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls
+// 'getIRPGONameForGlobalObject'. See the difference between two callees in the
+// comments of `getIRPGONameForGlobalObject`.
+std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
if (!InLTO) {
- auto FileName = getStrippedSourceFileName(GO);
- return getPGOFuncName(GO.getName(), GO.getLinkage(), FileName);
+ auto FileName = getStrippedSourceFileName(F);
+ return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
}
// In LTO mode (when InLTO is true), first check if there is a meta data.
- if (auto PGOName = lookupPGONameFromMetadata(PGONameMetadata))
- return *PGOName;
+ if (auto PGOFuncName = lookupPGONameFromMetadata(getPGOFuncNameMetadata(F)))
+ return *PGOFuncName;
// If there is no meta data, the function must be a global before the value
// profile annotation pass. Its current linkage may be internal if it is
// internalized in LTO mode.
- return getPGOFuncName(GO.getName(), GlobalValue::ExternalLinkage, "");
-}
-
-// This is similar to `getIRPGOFuncName` except that this function calls
-// 'getPGOObjectcName' to get a name and `getIRPGOFuncName` calls
-// 'getIRPGONameForGlobalObject'. See the difference between two callees in the
-// comments of `getIRPGONameForGlobalObject`.
-std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
- return getPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F));
+ return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
}
std::string getPGOName(const GlobalVariable &V, bool InLTO) {
// PGONameMetadata should be set by compiler at profile use time
// and read by symtab creation to look up symbols corresponding to
// a MD5 hash.
- return getPGOObjectName(V, InLTO, nullptr /* PGONameMetadata */);
+ return getIRPGOObjectName(V, InLTO, nullptr /* PGONameMetadata */);
}
// See getIRPGOFuncName() for a discription of the format.
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index cb585c76ef6f4..d737a5ee87390 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1101,12 +1101,6 @@ static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
return ConstantExpr::getBitCast(GV, Int8PtrTy);
}
-/// Get the name of a profiling variable for a particular variable.
-static std::string getVarName(GlobalVariable *GV, StringRef Prefix) {
- StringRef Name = getPGOName(*GV);
- return (Prefix + Name).str();
-}
-
void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
assert(!DebugInfoCorrelate &&
"Value profiling is not supported with lightweight instrumentation");
@@ -1143,7 +1137,7 @@ void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
// Used by INSTR_PROF_VTABLE_DATA MACRO
Constant *VTableAddr = getVTableAddrForProfData(GV);
- StringRef PGOVTableName = getPGOName(*GV);
+ const std::string PGOVTableName = getPGOName(*GV);
// Record the length of the vtable. This is needed since vtable pointers
// loaded from C++ objects might be from the middle of a vtable definition.
uint32_t VTableSizeVal =
@@ -1154,10 +1148,10 @@ void InstrProfiling::getOrCreateVTableProfData(GlobalVariable *GV) {
#include "llvm/ProfileData/InstrProfData.inc"
};
+ std::string VarName = getInstrProfVTableVarPrefix().str() + PGOVTableName;
auto *Data =
new GlobalVariable(*M, DataTy, false /* constant */, Linkage,
- ConstantStruct::get(DataTy, DataVals),
- getVarName(GV, getInstrProfVTableVarPrefix()));
+ ConstantStruct::get(DataTy, DataVals), VarName);
Data->setVisibility(Visibility);
Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat()));
diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
index fea122adc1fad..be6e554abbe00 100644
--- a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
+++ b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
@@ -103,11 +103,14 @@ public:
std::vector<Instruction *> Result = findVTableAddrs(F);
for (Instruction* I : Result) {
Instruction* InsertPt = I->getNextNonDebugInstruction();
- // When finding an insertion point, keep PHI, landing pad and catch pad before vp intrinsics.
- while(InsertPt && (dyn_cast<PHINode>(InsertPt) || dyn_cast<LandingPadInst>(InsertPt) || dyn_cast<CatchPadInst>(InsertPt)))
+ // When finding an insertion point, keep PHI and EH pad instructions
+ // before vp intrinsics. This is similar to
+ // `BasicBlock::getFirstInsertionPt`.
+ while (InsertPt && (dyn_cast<PHINode>(InsertPt) || InsertPt->isEHPad()))
InsertPt = InsertPt->getNextNonDebugInstruction();
- // FIXME: Set InsertPt to the end of basic block if
- // InsertPt is the last instruction.
+ // Skip instrumentating the value if InsertPt is the last instruction.
+ // FIXME: Set InsertPt to the end of basic block to instrument the value
+ // if InsertPt is the last instruction.
if (InsertPt == nullptr)
continue;
diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
index 4bdec2a1b643c..5c50ea207928a 100644
--- a/llvm/test/Transforms/PGOProfile/vtable_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
@@ -2,207 +2,95 @@
; RUN: opt < %s -passes=pgo-instr-gen -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=GEN
; RUN: opt < %s -passes=pgo-instr-gen,instrprof -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=LOWER
+source_filename = "vtable_local.ll"
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
; The test IR is generated based on the following C++ program.
+; Base1 has external linkage and Base2 has local linkage.
; class Derived uses multiple inheritance so its virtual table
; global variable contains two vtables. func1 is loaded from
; the vtable compatible with class Base1, and func2 is loaded
; from the vtable compatible with class Base2.
+
; class Base1 {
; public:
-; virtual int func1(int a, int b) ;
+; virtual int func1(int a) ;
; };
;
+; namespace {
; class Base2 {
; public:
-; virtual int func2(int a, int b);
+; __attribute__((noinline)) virtual int func2(int a) {
+; return a;
+; }
; };
-;
+; }
+
; class Derived : public Base1, public Base2 {
; public:
-; Derived(int c) : v(c) {}
+; Derived(int c) : v(c) {}
; private:
-; int v;
+; int v;
; };
;
-; Derived* createType(int c);
-; int func(int a, int b, int c) {
-; Derived* d = createType(c);
-;
-; return d->func2(a, b) + d->func1(b, a);
+; Derived* createType();
+
+; int func(int a) {
+; Derived* d = createType();
+; return d->func2(a) + d->func1(a);
; }
-$_ZTV7Derived = comdat any
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
- at _ZTV7Derived = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
- at _ZTVN10__cxxabiv121__vmi_class_type_infoE = external global [0 x ptr]
- at _ZTS7Derived = constant [9 x i8] c"7Derived\00", align 1
- at _ZTI5Base1 = external constant ptr
- at _ZTI5Base2 = external constant ptr
- at _ZTI7Derived = constant { ptr, ptr, i32, i32, ptr, i64, ptr, i64 } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), ptr @_ZTS7Derived, i32 0, i32 2, ptr @_ZTI5Base1, i64 2, ptr @_ZTI5Base2, i64 2050 }, align 8
- at _ZTV5Base1 = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base1, ptr @_ZN5Base15func1Eii] }, align 8, !type !0, !type !1
- at _ZTV5Base2 = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base2, ptr @_ZN5Base25func2Eii] }, align 8, !type !9, !type !4
- at llvm.compiler.used = appending global [2 x ptr] [ptr @_ZTV5Base1, ptr @_ZTV5Base2], section "llvm.metadata"
+ at _ZTV7Derived = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func1Ei], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr @_ZN12_GLOBAL__N_15Base25func2Ei] }, !type !0, !type !3, !type !6, !type !8, !type !10
+ at _ZTV5Base1 = available_externally constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func1Ei] }, !type !0
+ at _ZTVN12_GLOBAL__N_15Base2E = internal constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN12_GLOBAL__N_15Base25func2Ei] }, !type !11, !type !8; !vcall_visibility !12
+ at llvm.compiler.used = appending global [1 x ptr] [ptr @_ZTV5Base1], section "llvm.metadata"
-declare ptr @_Z10createTypei(i32)
-declare i32 @_ZN5Base15func1Eii(ptr, i32, i32)
-declare i32 @_ZN5Base25func2Eii(ptr, i32, i32)
-declare i1 @llvm.public.type.test(ptr, metadata)
-declare void @llvm.assume(i1 noundef)
+; GEN: __llvm_profile_raw_version = comdat any
+; GEN: __llvm_profile_raw_version = hidden constant i64 72057594037927946, comdat
+; GEN: __profn__Z4funci = private constant [8 x i8] c"_Z4funci"
-;.
-; GEN: @[[_ZTV7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
-; GEN: @[[_ZTVN10__CXXABIV121__VMI_CLASS_TYPE_INFOE:[a-zA-Z0-9_$"\\.-]+]] = external global [0 x ptr]
-; GEN: @[[_ZTS7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant [9 x i8] c"7Derived\00", align 1
-; GEN: @[[_ZTI5BASE1:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
-; GEN: @[[_ZTI5BASE2:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
-; GEN: @[[_ZTI7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { ptr, ptr, i32, i32, ptr, i64, ptr, i64 } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), ptr @_ZTS7Derived, i32 0, i32 2, ptr @_ZTI5Base1, i64 2, ptr @_ZTI5Base2, i64 2050 }, align 8
-; GEN: @[[_ZTV5BASE1:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base1, ptr @_ZN5Base15func1Eii] }, align 8, !type !0, !type !1
-; GEN: @[[_ZTV5BASE2:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base2, ptr @_ZN5Base25func2Eii] }, align 8, !type !9, !type !4
-; GEN: @[[LLVM_COMPILER_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [2 x ptr] [ptr @_ZTV5Base1, ptr @_ZTV5Base2], section "llvm.metadata"
-; GEN: @[[__LLVM_PROFILE_RAW_VERSION:[a-zA-Z0-9_$"\\.-]+]] = hidden constant i64 72057594037927946, comdat
-; GEN: @[[__PROFN_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private constant [27 x i8] c"test_vtable_value_profiling"
-;.
-; LOWER: @[[_ZTV7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI7Derived, ptr @_ZN5Base15func1Eii], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr @_ZTI7Derived, ptr @_ZN5Base25func2Eii] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8
-; LOWER: @[[_ZTVN10__CXXABIV121__VMI_CLASS_TYPE_INFOE:[a-zA-Z0-9_$"\\.-]+]] = external global [0 x ptr]
-; LOWER: @[[_ZTS7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant [9 x i8] c"7Derived\00", align 1
-; LOWER: @[[_ZTI5BASE1:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
-; LOWER: @[[_ZTI5BASE2:[a-zA-Z0-9_$"\\.-]+]] = external constant ptr
-; LOWER: @[[_ZTI7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = constant { ptr, ptr, i32, i32, ptr, i64, ptr, i64 } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2), ptr @_ZTS7Derived, i32 0, i32 2, ptr @_ZTI5Base1, i64 2, ptr @_ZTI5Base2, i64 2050 }, align 8
-; LOWER: @[[_ZTV5BASE1:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base1, ptr @_ZN5Base15func1Eii] }, align 8, !type !0, !type !1
-; LOWER: @[[_ZTV5BASE2:[a-zA-Z0-9_$"\\.-]+]] = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI5Base2, ptr @_ZN5Base25func2Eii] }, align 8, !type !9, !type !4
-; LOWER: @[[__LLVM_PROFILE_RAW_VERSION:[a-zA-Z0-9_$"\\.-]+]] = hidden constant i64 72057594037927946, comdat
-; LOWER: @[[__PROFC_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
-; LOWER: @[[__PROFVP_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private global [4 x i64] zeroinitializer, section "__llvm_prf_vals", comdat($__profc_test_vtable_value_profiling), align 8
-; LOWER: @[[__PROFD_TEST_VTABLE_VALUE_PROFILING:[a-zA-Z0-9_$"\\.-]+]] = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 1593873508557585901, i64 567090795815895039, i64 sub (i64 ptrtoint (ptr @__profc_test_vtable_value_profiling to i64), i64 ptrtoint (ptr @__profd_test_vtable_value_profiling to i64)), i64 0, ptr @test_vtable_value_profiling.local, ptr @__profvp_test_vtable_value_profiling, i32 1, [3 x i16] [i16 2, i16 0, i16 2], i32 0 }, section "__llvm_prf_data", comdat($__profc_test_vtable_value_profiling), align 8
-; LOWER: @[[__PROFVT__ZTV7DERIVED:[a-zA-Z0-9_$"\\.-]+]] = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8
-; LOWER: @[[__PROFVT__ZTV5BASE1:[a-zA-Z0-9_$"\\.-]+]] = global { i64, ptr, i32 } { i64 3215870116411581797, ptr @_ZTV5Base1, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
-; LOWER: @[[__PROFVT__ZTV5BASE2:[a-zA-Z0-9_$"\\.-]+]] = global { i64, ptr, i32 } { i64 8378219803387680050, ptr @_ZTV5Base2, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
-; LOWER: @[[__LLVM_PRF_VNODES:[a-zA-Z0-9_$"\\.-]+]] = private global [10 x { i64, i64, ptr }] zeroinitializer, section "__llvm_prf_vnds", align 8
-; LOWER: @[[__LLVM_PRF_NM:[a-zA-Z0-9_$"\\.-]+]] = private constant [37 x i8] c"\1B#x\DA+I-.\89/+IL\CAI\8D/K\CC)M\8D/(\CAO\CB\CC\C9\CCK\07\00\9Ea\0BC", section "__llvm_prf_names", align 1
-; LOWER: @[[__LLVM_PRF_VNM:[a-zA-Z0-9_$"\\.-]+]] = private constant [34 x i8] c"\22 x\DA\8B\8F\0A\093wI-\CA,KMa\8C\07rL\9D\12\8BS\0D\11L#\00\C3\A2\0A\E9", section "__llvm_prf_vtabnames", align 1
-; LOWER: @[[LLVM_COMPILER_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [3 x ptr] [ptr @_ZTV5Base1, ptr @_ZTV5Base2, ptr @__profd_test_vtable_value_profiling], section "llvm.metadata"
-; LOWER: @[[LLVM_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [6 x ptr] [ptr @__profvt__ZTV7Derived, ptr @__profvt__ZTV5Base1, ptr @__profvt__ZTV5Base2, ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata"
-; LOWER: @[[TEST_VTABLE_VALUE_PROFILING_LOCAL:[a-zA-Z0-9_$"\\.-]+]] = private alias i32 (i32, i32, i32), ptr @test_vtable_value_profiling
-;.
-define i32 @test_vtable_value_profiling(i32 %a, i32 %b, i32 %c) {
-; GEN-LABEL: define i32 @test_vtable_value_profiling(
-; GEN-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) {
-; GEN-NEXT: entry:
-; GEN-NEXT: call void @llvm.instrprof.increment(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i32 1, i32 0)
-; GEN-NEXT: [[CALL:%.*]] = tail call ptr @_Z10createTypei(i32 [[C]])
-; GEN-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 8
-; GEN-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
-; GEN-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[VTABLE]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP0]], i32 2, i32 0)
-; GEN-NEXT: [[TMP1:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE]], metadata !"_ZTS5Base2")
-; GEN-NEXT: tail call void @llvm.assume(i1 [[TMP1]])
-; GEN-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
-; GEN-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[VFUNC]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP2]], i32 0, i32 0)
-; GEN-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
-; GEN-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
-; GEN-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP3]], i32 2, i32 1)
-; GEN-NEXT: [[TMP4:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE2]], metadata !"_ZTS5Base1")
-; GEN-NEXT: tail call void @llvm.assume(i1 [[TMP4]])
-; GEN-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
-; GEN-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
-; GEN-NEXT: call void @llvm.instrprof.value.profile(ptr @__profn_test_vtable_value_profiling, i64 567090795815895039, i64 [[TMP5]], i32 0, i32 1)
-; GEN-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
-; GEN-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
-; GEN-NEXT: ret i32 [[ADD]]
-;
-; LOWER-LABEL: define i32 @test_vtable_value_profiling(
-; LOWER-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) {
-; LOWER-NEXT: entry:
-; LOWER-NEXT: [[PGOCOUNT:%.*]] = load i64, ptr @__profc_test_vtable_value_profiling, align 8
-; LOWER-NEXT: [[TMP0:%.*]] = add i64 [[PGOCOUNT]], 1
-; LOWER-NEXT: store i64 [[TMP0]], ptr @__profc_test_vtable_value_profiling, align 8
-; LOWER-NEXT: [[CALL:%.*]] = tail call ptr @_Z10createTypei(i32 [[C]])
-; LOWER-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 8
-; LOWER-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[ADD_PTR]], align 8
-; LOWER-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[VTABLE]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP1]], ptr @__profd_test_vtable_value_profiling, i32 2)
-; LOWER-NEXT: [[TMP2:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE]], metadata !"_ZTS5Base2")
-; LOWER-NEXT: tail call void @llvm.assume(i1 [[TMP2]])
-; LOWER-NEXT: [[VFUNC:%.*]] = load ptr, ptr [[VTABLE]], align 8
-; LOWER-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[VFUNC]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP3]], ptr @__profd_test_vtable_value_profiling, i32 0)
-; LOWER-NEXT: [[CALL1:%.*]] = tail call i32 [[VFUNC]](ptr [[ADD_PTR]], i32 [[A]], i32 [[B]])
-; LOWER-NEXT: [[VTABLE2:%.*]] = load ptr, ptr [[CALL]], align 8
-; LOWER-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[VTABLE2]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP4]], ptr @__profd_test_vtable_value_profiling, i32 3)
-; LOWER-NEXT: [[TMP5:%.*]] = tail call i1 @llvm.public.type.test(ptr [[VTABLE2]], metadata !"_ZTS5Base1")
-; LOWER-NEXT: tail call void @llvm.assume(i1 [[TMP5]])
-; LOWER-NEXT: [[VFUNC2:%.*]] = load ptr, ptr [[VTABLE2]], align 8
-; LOWER-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[VFUNC2]] to i64
-; LOWER-NEXT: call void @__llvm_profile_instrument_target(i64 [[TMP6]], ptr @__profd_test_vtable_value_profiling, i32 1)
-; LOWER-NEXT: [[CALL4:%.*]] = tail call i32 [[VFUNC2]](ptr [[CALL]], i32 [[B]], i32 [[A]])
-; LOWER-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL4]], [[CALL1]]
-; LOWER-NEXT: ret i32 [[ADD]]
-;
+; LOWER: $__profvt__ZTV7Derived = comdat nodeduplicate
+; LOWER: $"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = comdat nodeduplicate
+; LOWER: @__profvt__ZTV7Derived = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = internal global { i64, ptr, i32 } { i64 1419990121885302679, ptr @_ZTVN12_GLOBAL__N_15Base2E, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
+; LOWER: @__llvm_prf_vnm = private constant [64 x i8] c"7>x\DA\8B\8F\0A\093wI-\CA,KMa,+IL\CAI\8D\CF\C9ON\CC\D1\CB\C9\B1\8E\07J\FA\19\1A\C5\BB\FB\F8;9\FA\C4\C7\FB\C5\1B\9A:%\16\A7\1A\B9\02\00\19:\12o", section "__llvm_prf_vtabnames", align 1
+; LOWER: @llvm.used = appending global [5 x ptr] [ptr @__profvt__ZTV7Derived, ptr @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E", ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata"
+
+define i32 @_Z4funci(i32 %a) {
entry:
- %call = tail call ptr @_Z10createTypei(i32 %c)
- ; The first indirect call is 'func2' so func ptr is
- ; loaded from vtable compatible with Base2.
+ %call = call ptr @_Z10createTypev()
%add.ptr = getelementptr inbounds i8, ptr %call, i64 8
- %vtable = load ptr, ptr %add.ptr, align 8
- %0 = tail call i1 @llvm.public.type.test(ptr %vtable, metadata !"_ZTS5Base2")
- tail call void @llvm.assume(i1 %0)
- %vfunc = load ptr, ptr %vtable, align 8
- %call1 = tail call i32 %vfunc(ptr %add.ptr, i32 %a, i32 %b)
- ; The second indirect call is 'func1' so func ptr is
- ; loaded from vtable compatible with Base1.
- %vtable2 = load ptr, ptr %call, align 8
- %1 = tail call i1 @llvm.public.type.test(ptr %vtable2, metadata !"_ZTS5Base1")
- tail call void @llvm.assume(i1 %1)
- %vfunc2 = load ptr, ptr %vtable2, align 8
- %call4 = tail call i32 %vfunc2(ptr %call, i32 %b, i32 %a)
- %add = add nsw i32 %call4, %call1
+ %vtable = load ptr, ptr %add.ptr
+; GEN: [[P1:%[0-9]+]] = ptrtoint ptr %vtable to i64
+; GEN: call void @llvm.instrprof.value.profile(ptr @__profn__Z4funci, i64 [[CFGHash:[0-9]+]], i64 [[P1]], i32 2, i32 0)
+; LOWER: [[P1:%[0-9]+]] = ptrtoint ptr %vtable to i64
+; LOWER: call void @__llvm_profile_instrument_target(i64 [[P1]], ptr @__profd__Z4funci, i32 2)
+ %vfunc1 = load ptr, ptr %vtable
+ %call1 = call i32 %vfunc1(ptr %add.ptr, i32 %a)
+ %vtable2 = load ptr, ptr %call
+; GEN: [[P2:%[0-9]+]] = ptrtoint ptr %vtable2 to i64
+; GEN: call void @llvm.instrprof.value.profile(ptr @__profn__Z4funci, i64 [[CFGHash]], i64 [[P2]], i32 2, i32 1)
+; LOWER: [[P2:%[0-9]+]] = ptrtoint ptr %vtable2 to i64
+; LOWER: call void @__llvm_profile_instrument_target(i64 [[P2]], ptr @__profd__Z4funci, i32 3)
+ %vfunc2 = load ptr, ptr %vtable2
+ %call4 = call i32 %vfunc2(ptr %call, i32 %a)
+ %add = add nsw i32 %call1, %call4
ret i32 %add
}
+declare ptr @_Z10createTypev()
+declare i32 @_ZN12_GLOBAL__N_15Base25func2Ei(ptr %this, i32 %a)
+declare i32 @_ZN5Base15func1Ei(ptr, i32)
+
!0 = !{i64 16, !"_ZTS5Base1"}
-!1 = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
-!2 = !{i64 40, !"_ZTSM5Base1FiiiE.virtual"}
-!3 = !{i64 40, !"_ZTS5Base2"}
-!4 = !{i64 16, !"_ZTSM5Base2FiiiE.virtual"}
-!5 = !{i64 40, !"_ZTSM5Base2FiiiE.virtual"}
-!6 = !{i64 16, !"_ZTS7Derived"}
-!7 = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"}
-!8 = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
-!9 = !{i64 16, !"_ZTS5Base2"}
-;.
-; GEN: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; GEN: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
-; GEN: attributes #[[ATTR2:[0-9]+]] = { nounwind }
-;.
-; LOWER: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; LOWER: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
-; LOWER: attributes #[[ATTR2:[0-9]+]] = { nounwind }
-;.
-; GEN: [[META0:![0-9]+]] = !{i64 16, !"_ZTS5Base1"}
-; GEN: [[META1:![0-9]+]] = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
-; GEN: [[META2:![0-9]+]] = !{i64 40, !"_ZTSM5Base1FiiiE.virtual"}
-; GEN: [[META3:![0-9]+]] = !{i64 40, !"_ZTS5Base2"}
-; GEN: [[META4:![0-9]+]] = !{i64 16, !"_ZTSM5Base2FiiiE.virtual"}
-; GEN: [[META5:![0-9]+]] = !{i64 40, !"_ZTSM5Base2FiiiE.virtual"}
-; GEN: [[META6:![0-9]+]] = !{i64 16, !"_ZTS7Derived"}
-; GEN: [[META7:![0-9]+]] = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"}
-; GEN: [[META8:![0-9]+]] = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
-; GEN: [[META9:![0-9]+]] = !{i64 16, !"_ZTS5Base2"}
-;.
-; LOWER: [[META0:![0-9]+]] = !{i64 16, !"_ZTS5Base1"}
-; LOWER: [[META1:![0-9]+]] = !{i64 16, !"_ZTSM5Base1FiiiE.virtual"}
-; LOWER: [[META2:![0-9]+]] = !{i64 40, !"_ZTSM5Base1FiiiE.virtual"}
-; LOWER: [[META3:![0-9]+]] = !{i64 40, !"_ZTS5Base2"}
-; LOWER: [[META4:![0-9]+]] = !{i64 16, !"_ZTSM5Base2FiiiE.virtual"}
-; LOWER: [[META5:![0-9]+]] = !{i64 40, !"_ZTSM5Base2FiiiE.virtual"}
-; LOWER: [[META6:![0-9]+]] = !{i64 16, !"_ZTS7Derived"}
-; LOWER: [[META7:![0-9]+]] = !{i64 16, !"_ZTSM7DerivedFiiiE.virtual"}
-; LOWER: [[META8:![0-9]+]] = !{i64 40, !"_ZTSM7DerivedFiiiE.virtual"}
-; LOWER: [[META9:![0-9]+]] = !{i64 16, !"_ZTS5Base2"}
-;.
+!3 = !{i64 16, !"_ZTS7Derived"}
+!6 = !{i64 40, !7}
+!7 = distinct !{}
+!8 = !{i64 16, !9}
+!9 = distinct !{}
+!10 = !{i64 40, !9}
+!11 = !{i64 16, !7}
diff --git a/llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh b/llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh
index 40409707cca8e..490d10a210555 100755
--- a/llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh
+++ b/llvm/test/tools/llvm-profdata/Inputs/update_vtable_value_prof_inputs.sh
@@ -10,8 +10,6 @@ fi
OUTDIR=$(dirname $(realpath -s $0))
-echo "Outdir is $OUTDIR"
-
cat > ${OUTDIR}/vtable_prof.cc << EOF
#include <cstdlib>
#include <cstdio>
@@ -36,9 +34,9 @@ class Derived1 : public Base {
}
};
+namespace {
class Derived2 : public Base {
public:
-
__attribute__((noinline))
int func1(int a, int b) override {
return a - b;
@@ -49,6 +47,7 @@ class Derived2 : public Base {
return a * (a - b);
}
};
+} // namespace
__attribute__((noinline)) Base* createType(int a) {
Base* base = nullptr;
diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
deleted file mode 100644
index 0f5d93d26c184..0000000000000
--- a/llvm/test/tools/llvm-profdata/Inputs/vtable-prof.proftext
+++ /dev/null
@@ -1,73 +0,0 @@
-# IR level Instrumentation Flag
-:ir
-_Z10createTypei
-# Func Hash:
-146835647075900052
-# Num Counters:
-2
-# Counter Values:
-750
-250
-
-_ZN8Derived15func1Eii
-# Func Hash:
-742261418966908927
-# Num Counters:
-1
-# Counter Values:
-250
-
-_ZN8Derived15func2Eii
-# Func Hash:
-742261418966908927
-# Num Counters:
-1
-# Counter Values:
-250
-
-_ZN8Derived25func1Eii
-# Func Hash:
-742261418966908927
-# Num Counters:
-1
-# Counter Values:
-750
-
-_ZN8Derived25func2Eii
-# Func Hash:
-742261418966908927
-# Num Counters:
-1
-# Counter Values:
-750
-
-main
-# Func Hash:
-1124236338992350536
-# Num Counters:
-2
-# Counter Values:
-1000
-1
-# Num Value Kinds:
-2
-# ValueKind = IPVK_IndirectCallTarget:
-0
-# NumValueSites:
-2
-2
-_ZN8Derived25func1Eii:750
-_ZN8Derived15func1Eii:250
-2
-_ZN8Derived25func2Eii:750
-_ZN8Derived15func2Eii:250
-# ValueKind = IPVK_VTableTarget:
-2
-# NumValueSites:
-2
-2
-_ZTV8Derived2:750
-_ZTV8Derived1:250
-2
-_ZTV8Derived2:750
-_ZTV8Derived1:250
diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.profraw b/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.profraw
index 28ee930755615031fefe6badf9e42ad8acb1827e..092a2c1a72e67822e7ed0538f64e062195af18c4 100644
GIT binary patch
literal 1096
zcmZoHO3N=Q$obF300xW at ih+R*#(>fsXnb_;GN{0T{}7PD!y>wN<^TW1PQPCTg)uO+
zLFJjy=;&EJ)^0{S<EGpSvanml$pq66GhZMdq94YG>4))uUG05tmNNCWN2_J-Ke1W-
z6QK6P<O87kCB8z;0qKWfkea at UdXg%$n?Ty$U*_`VN<h<JP=IPbJb<7UEMz#Sd6WO%
zO_+WSH2n<(^k4aN_t9PNABSN2Km3Cj05zT=fPnrg;gY5O+Y*~#`gfq|PavTG9W+3H
zK`ng23}L|h2@{9WF!eC{0n;MJid$zVIC32_5O96jSmNU-5Tn_}u{1BHR5;0M?M2UJ
zOD;EUzpnA>5cdv=AOHWJO)goQX0}Ms=AKo_Omn at IU0UMOJ$Xl>&dyeh*joB-`m at eM
z7u*%LwJk`^<9v8V?%?hV%aj$Q{w{e~QM**{6YE+VZjNtEhkX8~=VU3feOoBKq&@IV
zXMnoR*0P at l0Xy%Bwtro9Vb_&cZztF5{y(Aq_c`C3tMzpZ498b)u77sJr0K+cNPH?l
z{TncS{i=NV+lvGqpo&jtYy$erw(lUwUoIb;OFaFX)LjJ^Uh#f;@U+n7+%<Pgyf>Zw
zo-Jz2b7qg_nFnk0wz+rJF1xlS!A~$^PU;7xd7N at W98UJXTzSq|uCeYseBC#2-jgrF
z4w{-PIF{Gn+#04Eb1d^3OXS<RoZ@>5^V;7(Jtb11x4yl$Gh*RSSxrCvNfzef3=AuP
y?u4azaJq$ZVCfPTFR*k1ix*hBgUQ3v0Zblj7RYR<c{iZp0#mO54G)-lLh=ASxm?}=
literal 888
zcmZoHO3N=Q$obF300xW at ih+R*#(>fsXnb_8JyhVpe+cN9?P{2@>i_ at d%{$Lc4`pDG
zfyy(X(b2Pdtlf-u#!a~uWMQ|8lL at 9DW`4nah<+F!rXR-tb+z}oS<2Mg9<7$S|HNkT
zPk`DFlivW<|08u5*c~AKFbq=DH&IVgWp)!t+xyF0zFZ0D`X8X$4-X)y1q&GtYTo3(
zcN3;x15N({0{S18SMxT1W9^6O|L_lD0MvMf3k38}{xs9W(r9lFO#cov{SOG}e+LcF
zUr-BQFhdwHf5OCJG)z5=&athy)!lDo;-Tl|tK-SLQpe+L${F7?XOesRKW#4X)IFu+
z?Q3Y_t9{nntmN#OBO<0w!ls8-UE^3M#&B(-$_|F(D>v6ayJ6C_BLEUV3Q%`UQ>pP<
zANAOkArMtuS_)(z7bo28lpgm-l8g-XTAYd?tt)^SmPWww4CQQqh6BtWuyBR>0~Sv(
dd04o?<iTcv%!ZnG0_t9vdI6|AVd at FV0{}?`@8<vj
diff --git a/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.proftext b/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.proftext
new file mode 100644
index 0000000000000..0fb2b448233b1
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.proftext
@@ -0,0 +1,74 @@
+# IR level Instrumentation Flag
+:ir
+/usr/local/google/home/mingmingl/llvm-fork/llvm-project/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+750
+
+/usr/local/google/home/mingmingl/llvm-fork/llvm-project/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+750
+
+_Z10createTypei
+# Func Hash:
+146835647075900052
+# Num Counters:
+2
+# Counter Values:
+750
+250
+
+_ZN8Derived15func1Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+250
+
+_ZN8Derived15func2Eii
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+250
+
+main
+# Func Hash:
+1124236338992350536
+# Num Counters:
+2
+# Counter Values:
+1000
+1
+# Num Value Kinds:
+2
+# ValueKind = IPVK_IndirectCallTarget:
+0
+# NumValueSites:
+2
+2
+/usr/local/google/home/mingmingl/llvm-fork/llvm-project/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii:750
+_ZN8Derived15func1Eii:250
+2
+/usr/local/google/home/mingmingl/llvm-fork/llvm-project/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii:750
+_ZN8Derived15func2Eii:250
+# ValueKind = IPVK_VTableTarget:
+2
+# NumValueSites:
+2
+2
+/usr/local/google/home/mingmingl/llvm-fork/llvm-project/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750
+_ZTV8Derived1:250
+2
+/usr/local/google/home/mingmingl/llvm-fork/llvm-project/llvm/test/tools/llvm-profdata/Inputs/vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750
+_ZTV8Derived1:250
+
diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test
index 7bbf397d380a4..61881b69cfd5c 100644
--- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test
+++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test
@@ -55,26 +55,16 @@ RUN: printf '\3\3\3\3\0\0\0\0' >> %t.profraw
// #include "llvm/ProfileData/InstrProfData.inc"
// };
-// NameRef
RUN: printf '\254\275\030\333\114\302\370\134' >> %t.profraw
-// FuncHash
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
-// RelativeCounterPtr
RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
-// bit map ptr
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
-// function pointer
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
-// Values
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
-// Num Counters, one vp kind
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
-// Two vp kind
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
-// NameRef
RUN: printf '\067\265\035\031\112\165\023\344' >> %t.profraw
-// FuncHash
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/vtable-prof.proftext b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
index b533c7b8a7f51..a128d2b171571 100644
--- a/llvm/test/tools/llvm-profdata/vtable-prof.proftext
+++ b/llvm/test/tools/llvm-profdata/vtable-prof.proftext
@@ -1,13 +1,13 @@
-RUN: llvm-profdata show --function=main --show-vtables %p/Inputs/vtable-prof.proftext | FileCheck %s --check-prefix=VTABLE
+RUN: llvm-profdata show --function=main --show-vtables %p/Inputs/vtable_prof.proftext | FileCheck %s --check-prefix=VTABLE
VTABLE: Counters:
VTABLE: main:
VTABLE: Hash: 0x0f9a16fe6d398548
VTABLE: Counters: 2
VTABLE: VTable Results:
-VTABLE: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+VTABLE: [ 0, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
VTABLE: [ 0, _ZTV8Derived1, 250 ] (25.00%)
-VTABLE: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+VTABLE: [ 1, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
VTABLE: [ 1, _ZTV8Derived1, 250 ] (25.00%)
VTABLE: Instrumentation level: IR entry_first = 0
VTABLE: Functions shown: 1
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
index e853516832cb8..02b5ebc948eb8 100644
--- a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -19,14 +19,14 @@ RAW-NEXT: Indirect Call Site Count: 2
RAW-NEXT: Number of instrumented vtables: 2
RAW-NEXT: Indirect Target Results:
RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
-RAW-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
+RAW-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%)
RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
-RAW-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+RAW-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%)
RAW-NEXT: VTable Results:
RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
-RAW-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+RAW-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
RAW-NEXT: Instrumentation level: IR entry_first = 0
RAW-NEXT: Functions shown: 1
RAW-NEXT: Total functions: 6
@@ -55,14 +55,14 @@ INDEXED-NEXT: Counters: 2
INDEXED-NEXT: Indirect Call Site Count: 2
INDEXED-NEXT: Number of instrumented vtables: 2
INDEXED-NEXT: Indirect Target Results:
-INDEXED-NEXT: [ 0, _ZN8Derived25func1Eii, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
-INDEXED-NEXT: [ 1, _ZN8Derived25func2Eii, 750 ] (75.00%)
+INDEXED-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
INDEXED-NEXT: VTable Results:
-INDEXED-NEXT: [ 0, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
-INDEXED-NEXT: [ 1, _ZTV8Derived2, 750 ] (75.00%)
+INDEXED-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
INDEXED-NEXT: Instrumentation level: IR entry_first = 0
INDEXED-NEXT: Functions shown: 1
@@ -100,18 +100,18 @@ ICTEXT: 0
ICTEXT: # NumValueSites:
ICTEXT: 2
ICTEXT: 2
-ICTEXT: _ZN8Derived25func1Eii:750
+ICTEXT: {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii:750
ICTEXT: _ZN8Derived15func1Eii:250
ICTEXT: 2
-ICTEXT: _ZN8Derived25func2Eii:750
+ICTEXT: {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii:750
ICTEXT: _ZN8Derived15func2Eii:250
ICTEXT: # ValueKind = IPVK_VTableTarget:
ICTEXT: 2
ICTEXT: # NumValueSites:
ICTEXT: 2
ICTEXT: 2
-ICTEXT: _ZTV8Derived2:750
+ICTEXT: {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750
ICTEXT: _ZTV8Derived1:250
ICTEXT: 2
-ICTEXT: _ZTV8Derived2:750
+ICTEXT: {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750
ICTEXT: _ZTV8Derived1:250
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index e6613a90dc7c5..a3a2cfc1dee80 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -652,32 +652,78 @@ TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) {
Succeeded());
}
+// callee1 to callee6 are from vtable1 to vtable6 respectively.
static const char callee1[] = "callee1";
static const char callee2[] = "callee2";
static const char callee3[] = "callee3";
static const char callee4[] = "callee4";
static const char callee5[] = "callee5";
static const char callee6[] = "callee6";
+// callee7 and callee8 are not from any vtables.
+static const char callee7[] = "callee7";
+static const char callee8[] = "callee8";
+// 'callee' is primarily used to create multiple-element vtables.
+static const char callee[] = "callee";
+static const uint64_t vtable1[] = {uint64_t(callee), uint64_t(callee1)};
+static const uint64_t vtable2[] = {uint64_t(callee2), uint64_t(callee)};
+static const uint64_t vtable3[] = {
+ uint64_t(callee),
+ uint64_t(callee3),
+};
+static const uint64_t vtable4[] = {uint64_t(callee4), uint64_t(callee)};
+static const uint64_t vtable5[] = {uint64_t(callee5), uint64_t(callee)};
+static const uint64_t vtable6[] = {uint64_t(callee6), uint64_t(callee)};
+
+// Returns the address of callee with a numbered suffix in vtable.
+static uint64_t getCalleeAddress(const uint64_t *vtableAddr) {
+ uint64_t CalleeAddr;
+ // Callee with a numbered suffix is the 2nd element in vtable1 and vtable3,
+ // and the 1st element in the rest of vtables.
+ if (vtableAddr == vtable1 || vtableAddr == vtable3)
+ CalleeAddr = uint64_t(vtableAddr) + 8;
+ else
+ CalleeAddr = uint64_t(vtableAddr);
+ return CalleeAddr;
+}
-TEST_P(InstrProfReaderWriterTest, icall_data_read_write) {
+TEST_P(InstrProfReaderWriterTest, icall_and_vtable_data_read_write) {
NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
- // 4 value sites.
- Record1.reserveSites(IPVK_IndirectCallTarget, 4);
- InstrProfValueData VD0[] = {
- {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
- Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
- // No value profile data at the second site.
- Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
- InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
- Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
- InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
- Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
+ // 4 indirect call value sites.
+ {
+ Record1.reserveSites(IPVK_IndirectCallTarget, 4);
+ InstrProfValueData VD0[] = {
+ {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
+ Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
+ // No value profile data at the second site.
+ Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
+ InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
+ Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
+ InstrProfValueData VD3[] = {{(uint64_t)callee7, 1}, {(uint64_t)callee8, 2}};
+ Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr);
+ }
+
+ // 2 vtable value sites.
+ {
+ InstrProfValueData VD0[] = {
+ {getCalleeAddress(vtable1), 1},
+ {getCalleeAddress(vtable2), 2},
+ {getCalleeAddress(vtable3), 3},
+ };
+ InstrProfValueData VD2[] = {
+ {getCalleeAddress(vtable1), 1},
+ {getCalleeAddress(vtable2), 2},
+ };
+ Record1.addValueData(IPVK_VTableTarget, 0, VD0, 3, nullptr);
+ Record1.addValueData(IPVK_VTableTarget, 2, VD2, 2, nullptr);
+ }
Writer.addRecord(std::move(Record1), getProfWeight(), Err);
Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
+ Writer.addRecord({"callee7", 0x1235, {3, 4}}, Err);
+ Writer.addRecord({"callee8", 0x1235, {3, 4}}, Err);
// Set writer value prof data endianness.
Writer.setValueProfDataEndianness(getEndianness());
@@ -690,24 +736,66 @@ TEST_P(InstrProfReaderWriterTest, icall_data_read_write) {
Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
ASSERT_THAT_ERROR(R.takeError(), Succeeded());
+
+ // Test the number of instrumented indirect call sites and the number of
+ // profiled values at each site.
ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
EXPECT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
EXPECT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
- EXPECT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
+ EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
+
+ // Test the number of instrumented vtable sites and the number of profiled
+ // values at each site.
+ ASSERT_EQ(2U, R->getNumValueSites(IPVK_VTableTarget));
+ EXPECT_EQ(3U, R->getNumValueDataForSite(IPVK_VTableTarget, 0));
+ EXPECT_EQ(2U, R->getNumValueDataForSite(IPVK_VTableTarget, 1));
+
+ // First indirect site.
+ {
+ uint64_t TotalC;
+ std::unique_ptr<InstrProfValueData[]> VD =
+ R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
+
+ EXPECT_EQ(3U * getProfWeight(), VD[0].Count);
+ EXPECT_EQ(2U * getProfWeight(), VD[1].Count);
+ EXPECT_EQ(1U * getProfWeight(), VD[2].Count);
+ EXPECT_EQ(6U * getProfWeight(), TotalC);
+
+ EXPECT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
+ EXPECT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
+ EXPECT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
+ }
- uint64_t TotalC;
- std::unique_ptr<InstrProfValueData[]> VD =
- R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
+ // First vtable site.
+ {
+ uint64_t TotalC;
+ std::unique_ptr<InstrProfValueData[]> VD =
+ R->getValueForSite(IPVK_VTableTarget, 0, &TotalC);
+
+ EXPECT_EQ(3U * getProfWeight(), VD[0].Count);
+ EXPECT_EQ(2U * getProfWeight(), VD[1].Count);
+ EXPECT_EQ(1U * getProfWeight(), VD[2].Count);
+ EXPECT_EQ(6U * getProfWeight(), TotalC);
- EXPECT_EQ(3U * getProfWeight(), VD[0].Count);
- EXPECT_EQ(2U * getProfWeight(), VD[1].Count);
- EXPECT_EQ(1U * getProfWeight(), VD[2].Count);
- EXPECT_EQ(6U * getProfWeight(), TotalC);
+ EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable3));
+ EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable2));
+ EXPECT_EQ(VD[2].Value, getCalleeAddress(vtable1));
+ }
- EXPECT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
- EXPECT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
- EXPECT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
+ // Second vtable site.
+ {
+ uint64_t TotalC;
+ std::unique_ptr<InstrProfValueData[]> VD =
+ R->getValueForSite(IPVK_VTableTarget, 1, &TotalC);
+
+ EXPECT_EQ(2U * getProfWeight(), VD[0].Count);
+ EXPECT_EQ(1U * getProfWeight(), VD[1].Count);
+ EXPECT_EQ(3U * getProfWeight(), TotalC);
+
+ EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable2));
+ EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable1));
+ }
}
INSTANTIATE_TEST_SUITE_P(
@@ -815,33 +903,53 @@ TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
ASSERT_EQ(1U, ValueData[3].Count);
}
-TEST_P(MaybeSparseInstrProfTest, icall_data_merge) {
+TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) {
static const char caller[] = "caller";
NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
- // 5 value sites.
- Record11.reserveSites(IPVK_IndirectCallTarget, 5);
- InstrProfValueData VD0[] = {{uint64_t(callee1), 1},
- {uint64_t(callee2), 2},
- {uint64_t(callee3), 3},
- {uint64_t(callee4), 4}};
- Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr);
+ // 5 value sites for indirect calls.
+ {
+ Record11.reserveSites(IPVK_IndirectCallTarget, 5);
+ InstrProfValueData VD0[] = {{uint64_t(callee1), 1},
+ {uint64_t(callee2), 2},
+ {uint64_t(callee3), 3},
+ {uint64_t(callee4), 4}};
+ Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr);
- // No value profile data at the second site.
- Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
+ // No value profile data at the second site.
+ Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
- InstrProfValueData VD2[] = {
- {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
- Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
+ InstrProfValueData VD2[] = {
+ {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
+ Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
- InstrProfValueData VD3[] = {{uint64_t(callee1), 1}};
- Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
+ InstrProfValueData VD3[] = {{uint64_t(callee7), 1}, {uint64_t(callee8), 2}};
+ Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr);
- InstrProfValueData VD4[] = {{uint64_t(callee1), 1},
- {uint64_t(callee2), 2},
- {uint64_t(callee3), 3}};
- Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr);
+ InstrProfValueData VD4[] = {
+ {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
+ Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr);
+ }
+ // 3 value sites for vtables.
+ {
+ Record11.reserveSites(IPVK_VTableTarget, 3);
+ InstrProfValueData VD0[] = {{getCalleeAddress(vtable1), 1},
+ {getCalleeAddress(vtable2), 2},
+ {getCalleeAddress(vtable3), 3},
+ {getCalleeAddress(vtable4), 4}};
+ Record11.addValueData(IPVK_VTableTarget, 0, VD0, 4, nullptr);
+
+ InstrProfValueData VD2[] = {{getCalleeAddress(vtable1), 1},
+ {getCalleeAddress(vtable2), 2},
+ {getCalleeAddress(vtable3), 3}};
+ Record11.addValueData(IPVK_VTableTarget, 1, VD2, 3, nullptr);
+
+ InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1},
+ {getCalleeAddress(vtable2), 2},
+ {getCalleeAddress(vtable3), 3}};
+ Record11.addValueData(IPVK_VTableTarget, 3, VD4, 3, nullptr);
+ }
// A different record for the same caller.
Record12.reserveSites(IPVK_IndirectCallTarget, 5);
@@ -857,11 +965,28 @@ TEST_P(MaybeSparseInstrProfTest, icall_data_merge) {
Record12.addValueData(IPVK_IndirectCallTarget, 3, nullptr, 0, nullptr);
- InstrProfValueData VD42[] = {{uint64_t(callee1), 1},
- {uint64_t(callee2), 2},
- {uint64_t(callee3), 3}};
+ InstrProfValueData VD42[] = {
+ {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, nullptr);
+ // 3 value sites for vtables.
+ {
+ Record12.reserveSites(IPVK_VTableTarget, 3);
+ InstrProfValueData VD0[] = {{getCalleeAddress(vtable2), 5},
+ {getCalleeAddress(vtable3), 3}};
+ Record12.addValueData(IPVK_VTableTarget, 0, VD0, 2, nullptr);
+
+ InstrProfValueData VD2[] = {{getCalleeAddress(vtable2), 1},
+ {getCalleeAddress(vtable3), 3},
+ {getCalleeAddress(vtable4), 4}};
+ Record12.addValueData(IPVK_VTableTarget, 1, VD2, 3, nullptr);
+
+ InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1},
+ {getCalleeAddress(vtable2), 2},
+ {getCalleeAddress(vtable3), 3}};
+ Record12.addValueData(IPVK_VTableTarget, 3, VD4, 3, nullptr);
+ }
+
Writer.addRecord(std::move(Record11), Err);
// Merge profile data.
Writer.addRecord(std::move(Record12), Err);
@@ -871,53 +996,99 @@ TEST_P(MaybeSparseInstrProfTest, icall_data_merge) {
Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
Writer.addRecord({callee4, 0x1235, {3, 5}}, Err);
+ Writer.addRecord({callee7, 0x1235, {3, 5}}, Err);
+ Writer.addRecord({callee8, 0x1235, {3, 5}}, Err);
auto Profile = Writer.writeBuffer();
readProfile(std::move(Profile));
+ // Test the number of instrumented value sites and the number of profiled
+ // values for each site.
Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
EXPECT_THAT_ERROR(R.takeError(), Succeeded());
+ // For indirect calls.
ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget));
ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
- ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
+ ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
+ // For vtables.
+ ASSERT_EQ(3U, R->getNumValueSites(IPVK_VTableTarget));
+ ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_VTableTarget, 0));
+ ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_VTableTarget, 1));
+ ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_VTableTarget, 2));
+
+ // Test the merged values for indirect calls.
+ {
+ std::unique_ptr<InstrProfValueData[]> VD =
+ R->getValueForSite(IPVK_IndirectCallTarget, 0);
+ EXPECT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2"));
+ EXPECT_EQ(7U, VD[0].Count);
+ EXPECT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3"));
+ EXPECT_EQ(6U, VD[1].Count);
+ EXPECT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4"));
+ EXPECT_EQ(4U, VD[2].Count);
+ EXPECT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1"));
+ EXPECT_EQ(1U, VD[3].Count);
+
+ std::unique_ptr<InstrProfValueData[]> VD_2(
+ R->getValueForSite(IPVK_IndirectCallTarget, 2));
+ EXPECT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3"));
+ EXPECT_EQ(6U, VD_2[0].Count);
+ EXPECT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4"));
+ EXPECT_EQ(4U, VD_2[1].Count);
+ EXPECT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2"));
+ EXPECT_EQ(3U, VD_2[2].Count);
+ EXPECT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1"));
+ EXPECT_EQ(1U, VD_2[3].Count);
+
+ std::unique_ptr<InstrProfValueData[]> VD_3(
+ R->getValueForSite(IPVK_IndirectCallTarget, 3));
+ EXPECT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee8"));
+ EXPECT_EQ(2U, VD_3[0].Count);
+ EXPECT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee7"));
+ EXPECT_EQ(1U, VD_3[1].Count);
+
+ std::unique_ptr<InstrProfValueData[]> VD_4(
+ R->getValueForSite(IPVK_IndirectCallTarget, 4));
+ EXPECT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3"));
+ EXPECT_EQ(6U, VD_4[0].Count);
+ EXPECT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2"));
+ EXPECT_EQ(4U, VD_4[1].Count);
+ EXPECT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1"));
+ EXPECT_EQ(2U, VD_4[2].Count);
+ }
- std::unique_ptr<InstrProfValueData[]> VD =
- R->getValueForSite(IPVK_IndirectCallTarget, 0);
- ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2"));
- ASSERT_EQ(7U, VD[0].Count);
- ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3"));
- ASSERT_EQ(6U, VD[1].Count);
- ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4"));
- ASSERT_EQ(4U, VD[2].Count);
- ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1"));
- ASSERT_EQ(1U, VD[3].Count);
-
- std::unique_ptr<InstrProfValueData[]> VD_2(
- R->getValueForSite(IPVK_IndirectCallTarget, 2));
- ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3"));
- ASSERT_EQ(6U, VD_2[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4"));
- ASSERT_EQ(4U, VD_2[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2"));
- ASSERT_EQ(3U, VD_2[2].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1"));
- ASSERT_EQ(1U, VD_2[3].Count);
-
- std::unique_ptr<InstrProfValueData[]> VD_3(
- R->getValueForSite(IPVK_IndirectCallTarget, 3));
- ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1"));
- ASSERT_EQ(1U, VD_3[0].Count);
-
- std::unique_ptr<InstrProfValueData[]> VD_4(
- R->getValueForSite(IPVK_IndirectCallTarget, 4));
- ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3"));
- ASSERT_EQ(6U, VD_4[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2"));
- ASSERT_EQ(4U, VD_4[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1"));
- ASSERT_EQ(2U, VD_4[2].Count);
+ // Test the merged values for vtables
+ {
+ auto VD0 = R->getValueForSite(IPVK_VTableTarget, 0);
+ EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2));
+ EXPECT_EQ(VD0[0].Count, 7U);
+ EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3));
+ EXPECT_EQ(VD0[1].Count, 6U);
+ EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable4));
+ EXPECT_EQ(VD0[2].Count, 4U);
+ EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable1));
+ EXPECT_EQ(VD0[3].Count, 1U);
+
+ auto VD1 = R->getValueForSite(IPVK_VTableTarget, 1);
+ EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable3));
+ EXPECT_EQ(VD1[0].Count, 6U);
+ EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable4));
+ EXPECT_EQ(VD1[1].Count, 4U);
+ EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable2));
+ EXPECT_EQ(VD1[2].Count, 3U);
+ EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable1));
+ EXPECT_EQ(VD1[3].Count, 1U);
+
+ auto VD2 = R->getValueForSite(IPVK_VTableTarget, 2);
+ EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable3));
+ EXPECT_EQ(VD2[0].Count, 6U);
+ EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable2));
+ EXPECT_EQ(VD2[1].Count, 4U);
+ EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable1));
+ EXPECT_EQ(VD2[2].Count, 2U);
+ }
}
struct ValueProfileMergeEdgeCaseTest
@@ -1041,30 +1212,62 @@ INSTANTIATE_TEST_SUITE_P(
EdgeCaseTest, ValueProfileMergeEdgeCaseTest,
::testing::Combine(::testing::Bool(), /* Sparse */
::testing::Values(IPVK_IndirectCallTarget,
- IPVK_MemOPSize) /* ValueKind */
+ IPVK_MemOPSize,
+ IPVK_VTableTarget) /* ValueKind */
));
static void addValueProfData(InstrProfRecord &Record) {
- Record.reserveSites(IPVK_IndirectCallTarget, 5);
- InstrProfValueData VD0[] = {{uint64_t(callee1), 400},
- {uint64_t(callee2), 1000},
- {uint64_t(callee3), 500},
- {uint64_t(callee4), 300},
- {uint64_t(callee5), 100}};
- Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr);
- InstrProfValueData VD1[] = {{uint64_t(callee5), 800},
- {uint64_t(callee3), 1000},
- {uint64_t(callee2), 2500},
- {uint64_t(callee1), 1300}};
- Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr);
- InstrProfValueData VD2[] = {{uint64_t(callee6), 800},
- {uint64_t(callee3), 1000},
- {uint64_t(callee4), 5500}};
- Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
- InstrProfValueData VD3[] = {{uint64_t(callee2), 1800},
- {uint64_t(callee3), 2000}};
- Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr);
- Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr);
+ // Add test data for indirect calls.
+ {
+ Record.reserveSites(IPVK_IndirectCallTarget, 6);
+ InstrProfValueData VD0[] = {{uint64_t(callee1), 400},
+ {uint64_t(callee2), 1000},
+ {uint64_t(callee3), 500},
+ {uint64_t(callee4), 300},
+ {uint64_t(callee5), 100}};
+ Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr);
+ InstrProfValueData VD1[] = {{uint64_t(callee5), 800},
+ {uint64_t(callee3), 1000},
+ {uint64_t(callee2), 2500},
+ {uint64_t(callee1), 1300}};
+ Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr);
+ InstrProfValueData VD2[] = {{uint64_t(callee6), 800},
+ {uint64_t(callee3), 1000},
+ {uint64_t(callee4), 5500}};
+ Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
+ InstrProfValueData VD3[] = {{uint64_t(callee2), 1800},
+ {uint64_t(callee3), 2000}};
+ Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr);
+ Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr);
+ InstrProfValueData VD5[] = {{uint64_t(callee7), 1234},
+ {uint64_t(callee8), 5678}};
+ Record.addValueData(IPVK_IndirectCallTarget, 5, VD5, 2, nullptr);
+ }
+
+ // Add test data for vtables
+ {
+ Record.reserveSites(IPVK_VTableTarget, 4);
+ InstrProfValueData VD0[] = {
+ {getCalleeAddress(vtable1), 400}, {getCalleeAddress(vtable2), 1000},
+ {getCalleeAddress(vtable3), 500}, {getCalleeAddress(vtable4), 300},
+ {getCalleeAddress(vtable5), 100},
+ };
+ InstrProfValueData VD1[] = {{getCalleeAddress(vtable5), 800},
+ {getCalleeAddress(vtable3), 1000},
+ {getCalleeAddress(vtable2), 2500},
+ {getCalleeAddress(vtable1), 1300}};
+ InstrProfValueData VD2[] = {
+ {getCalleeAddress(vtable6), 800},
+ {getCalleeAddress(vtable3), 1000},
+ {getCalleeAddress(vtable4), 5500},
+ };
+ InstrProfValueData VD3[] = {{getCalleeAddress(vtable2), 1800},
+ {getCalleeAddress(vtable3), 2000}};
+ Record.addValueData(IPVK_VTableTarget, 0, VD0, 5, nullptr);
+ Record.addValueData(IPVK_VTableTarget, 1, VD1, 4, nullptr);
+ Record.addValueData(IPVK_VTableTarget, 2, VD2, 3, nullptr);
+ Record.addValueData(IPVK_VTableTarget, 3, VD3, 2, nullptr);
+ }
}
TEST(ValueProfileReadWriteTest, value_prof_data_read_write) {
@@ -1077,59 +1280,111 @@ TEST(ValueProfileReadWriteTest, value_prof_data_read_write) {
VPData->deserializeTo(Record, nullptr);
// Now read data from Record and sanity check the data
- ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
+ ASSERT_EQ(6U, Record.getNumValueSites(IPVK_IndirectCallTarget));
ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
+ ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 5));
auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
return VD1.Count > VD2.Count;
};
+
std::unique_ptr<InstrProfValueData[]> VD_0(
Record.getValueForSite(IPVK_IndirectCallTarget, 0));
llvm::sort(&VD_0[0], &VD_0[5], Cmp);
- ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2"));
- ASSERT_EQ(1000U, VD_0[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3"));
- ASSERT_EQ(500U, VD_0[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1"));
- ASSERT_EQ(400U, VD_0[2].Count);
- ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4"));
- ASSERT_EQ(300U, VD_0[3].Count);
- ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5"));
- ASSERT_EQ(100U, VD_0[4].Count);
+ EXPECT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2"));
+ EXPECT_EQ(1000U, VD_0[0].Count);
+ EXPECT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3"));
+ EXPECT_EQ(500U, VD_0[1].Count);
+ EXPECT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1"));
+ EXPECT_EQ(400U, VD_0[2].Count);
+ EXPECT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4"));
+ EXPECT_EQ(300U, VD_0[3].Count);
+ EXPECT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5"));
+ EXPECT_EQ(100U, VD_0[4].Count);
std::unique_ptr<InstrProfValueData[]> VD_1(
Record.getValueForSite(IPVK_IndirectCallTarget, 1));
llvm::sort(&VD_1[0], &VD_1[4], Cmp);
- ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2"));
- ASSERT_EQ(2500U, VD_1[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1"));
- ASSERT_EQ(1300U, VD_1[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3"));
- ASSERT_EQ(1000U, VD_1[2].Count);
- ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5"));
- ASSERT_EQ(800U, VD_1[3].Count);
+ EXPECT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2"));
+ EXPECT_EQ(2500U, VD_1[0].Count);
+ EXPECT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1"));
+ EXPECT_EQ(1300U, VD_1[1].Count);
+ EXPECT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3"));
+ EXPECT_EQ(1000U, VD_1[2].Count);
+ EXPECT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5"));
+ EXPECT_EQ(800U, VD_1[3].Count);
std::unique_ptr<InstrProfValueData[]> VD_2(
Record.getValueForSite(IPVK_IndirectCallTarget, 2));
llvm::sort(&VD_2[0], &VD_2[3], Cmp);
- ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4"));
- ASSERT_EQ(5500U, VD_2[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3"));
- ASSERT_EQ(1000U, VD_2[1].Count);
- ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6"));
- ASSERT_EQ(800U, VD_2[2].Count);
+ EXPECT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4"));
+ EXPECT_EQ(5500U, VD_2[0].Count);
+ EXPECT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3"));
+ EXPECT_EQ(1000U, VD_2[1].Count);
+ EXPECT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6"));
+ EXPECT_EQ(800U, VD_2[2].Count);
std::unique_ptr<InstrProfValueData[]> VD_3(
Record.getValueForSite(IPVK_IndirectCallTarget, 3));
llvm::sort(&VD_3[0], &VD_3[2], Cmp);
- ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3"));
- ASSERT_EQ(2000U, VD_3[0].Count);
- ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2"));
- ASSERT_EQ(1800U, VD_3[1].Count);
+ EXPECT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3"));
+ EXPECT_EQ(2000U, VD_3[0].Count);
+ EXPECT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2"));
+ EXPECT_EQ(1800U, VD_3[1].Count);
+
+ ASSERT_EQ(4U, Record.getNumValueSites(IPVK_VTableTarget));
+ ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_VTableTarget, 0));
+ ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_VTableTarget, 1));
+ ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_VTableTarget, 2));
+ ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_VTableTarget, 3));
+
+ std::unique_ptr<InstrProfValueData[]> VD0(
+ Record.getValueForSite(IPVK_VTableTarget, 0));
+ llvm::sort(&VD0[0], &VD0[5], Cmp);
+ EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2));
+ EXPECT_EQ(VD0[0].Count, 1000U);
+ EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3));
+ EXPECT_EQ(VD0[1].Count, 500U);
+ EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable1));
+ EXPECT_EQ(VD0[2].Count, 400U);
+ EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable4));
+ EXPECT_EQ(VD0[3].Count, 300U);
+ EXPECT_EQ(VD0[4].Value, getCalleeAddress(vtable5));
+ EXPECT_EQ(VD0[4].Count, 100U);
+
+ std::unique_ptr<InstrProfValueData[]> VD1(
+ Record.getValueForSite(IPVK_VTableTarget, 1));
+ llvm::sort(&VD1[0], &VD1[4], Cmp);
+ EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable2));
+ EXPECT_EQ(VD1[0].Count, 2500U);
+ EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable1));
+ EXPECT_EQ(VD1[1].Count, 1300U);
+ EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable3));
+ EXPECT_EQ(VD1[2].Count, 1000U);
+ EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable5));
+ EXPECT_EQ(VD1[3].Count, 800U);
+
+ std::unique_ptr<InstrProfValueData[]> VD2(
+ Record.getValueForSite(IPVK_VTableTarget, 2));
+ llvm::sort(&VD2[0], &VD2[3], Cmp);
+ EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable4));
+ EXPECT_EQ(VD2[0].Count, 5500U);
+ EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable3));
+ EXPECT_EQ(VD2[1].Count, 1000U);
+ EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable6));
+ EXPECT_EQ(VD2[2].Count, 800U);
+
+ std::unique_ptr<InstrProfValueData[]> VD3(
+ Record.getValueForSite(IPVK_VTableTarget, 3));
+ llvm::sort(&VD3[0], &VD3[2], Cmp);
+ EXPECT_EQ(VD3[0].Value, getCalleeAddress(vtable3));
+ EXPECT_EQ(VD3[0].Count, 2000U);
+ EXPECT_EQ(VD3[1].Value, getCalleeAddress(vtable2));
+ EXPECT_EQ(VD3[1].Count, 1800U);
}
TEST(ValueProfileReadWriteTest, symtab_mapping) {
@@ -1146,10 +1401,27 @@ TEST(ValueProfileReadWriteTest, symtab_mapping) {
Symtab.mapAddress(uint64_t(callee4), 0x4000ULL);
// Missing mapping for callee5
+ auto getVTableStartAddr = [](const uint64_t *vtable) -> uint64_t {
+ return uint64_t(vtable);
+ };
+ auto getVTableEndAddr = [](const uint64_t *vtable) -> uint64_t {
+ return uint64_t(vtable) + 16;
+ };
+ // vtable1, vtable2, vtable3, vtable4 get mapped; vtable5, vtable6 are not
+ // mapped.
+ Symtab.mapVTableAddress(getVTableStartAddr(vtable1),
+ getVTableEndAddr(vtable1), MD5Hash("vtable1"));
+ Symtab.mapVTableAddress(getVTableStartAddr(vtable2),
+ getVTableEndAddr(vtable2), MD5Hash("vtable2"));
+ Symtab.mapVTableAddress(getVTableStartAddr(vtable3),
+ getVTableEndAddr(vtable3), MD5Hash("vtable3"));
+ Symtab.mapVTableAddress(getVTableStartAddr(vtable4),
+ getVTableEndAddr(vtable4), MD5Hash("vtable4"));
+
VPData->deserializeTo(Record, &Symtab);
// Now read data from Record and sanity check the data
- ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
+ ASSERT_EQ(6U, Record.getNumValueSites(IPVK_IndirectCallTarget));
ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
@@ -1167,6 +1439,74 @@ TEST(ValueProfileReadWriteTest, symtab_mapping) {
// callee5 does not have a mapped value -- default to 0.
ASSERT_EQ(VD_0[4].Value, 0ULL);
+
+ // Sanity check the vtable value data
+ ASSERT_EQ(4U, Record.getNumValueSites(IPVK_VTableTarget));
+
+ {
+ // The first vtable site.
+ std::unique_ptr<InstrProfValueData[]> VD(
+ Record.getValueForSite(IPVK_VTableTarget, 0));
+ ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_VTableTarget, 0));
+ llvm::sort(&VD[0], &VD[5], Cmp);
+ EXPECT_EQ(1000U, VD[0].Count);
+ EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));
+ EXPECT_EQ(500U, VD[1].Count);
+ EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));
+ EXPECT_EQ(VD[2].Value, MD5Hash("vtable1"));
+ EXPECT_EQ(400U, VD[2].Count);
+ EXPECT_EQ(VD[3].Value, MD5Hash("vtable4"));
+ EXPECT_EQ(300U, VD[3].Count);
+
+ // vtable5 isn't mapped -- default to 0.
+ EXPECT_EQ(VD[4].Value, 0U);
+ EXPECT_EQ(VD[4].Count, 100U);
+ }
+
+ {
+ // The second vtable site.
+ std::unique_ptr<InstrProfValueData[]> VD(
+ Record.getValueForSite(IPVK_VTableTarget, 1));
+ ASSERT_EQ(4, Record.getNumValueDataForSite(IPVK_VTableTarget, 1));
+ llvm::sort(&VD[0], &VD[4], Cmp);
+ EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));
+ EXPECT_EQ(2500U, VD[0].Count);
+ EXPECT_EQ(VD[1].Value, MD5Hash("vtable1"));
+ EXPECT_EQ(1300U, VD[1].Count);
+
+ EXPECT_EQ(VD[2].Value, MD5Hash("vtable3"));
+ EXPECT_EQ(1000U, VD[2].Count);
+ // vtable5 isn't mapped -- default to 0.
+ EXPECT_EQ(VD[3].Value, 0U);
+ EXPECT_EQ(800U, VD[3].Count);
+ }
+
+ {
+ // The third vtable site.
+ std::unique_ptr<InstrProfValueData[]> VD(
+ Record.getValueForSite(IPVK_VTableTarget, 2));
+ ASSERT_EQ(3, Record.getNumValueDataForSite(IPVK_VTableTarget, 2));
+ llvm::sort(&VD[0], &VD[3], Cmp);
+ EXPECT_EQ(5500U, VD[0].Count);
+ EXPECT_EQ(VD[0].Value, MD5Hash("vtable4"));
+ EXPECT_EQ(1000U, VD[1].Count);
+ EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));
+ // vtable6 isn't mapped -- default to 0.
+ EXPECT_EQ(VD[2].Value, 0U);
+ EXPECT_EQ(800U, VD[2].Count);
+ }
+
+ {
+ // The fourth vtable site.
+ std::unique_ptr<InstrProfValueData[]> VD(
+ Record.getValueForSite(IPVK_VTableTarget, 3));
+ ASSERT_EQ(2, Record.getNumValueDataForSite(IPVK_VTableTarget, 3));
+ llvm::sort(&VD[0], &VD[2], Cmp);
+ EXPECT_EQ(2000U, VD[0].Count);
+ EXPECT_EQ(VD[0].Value, MD5Hash("vtable3"));
+ EXPECT_EQ(1800U, VD[1].Count);
+ EXPECT_EQ(VD[1].Value, MD5Hash("vtable2"));
+ }
}
TEST_P(MaybeSparseInstrProfTest, get_max_function_count) {
>From 8c725efaaaa1f0ceffb47c5b339f7b582a2beb3f Mon Sep 17 00:00:00 2001
From: mingmingl <mingmingl at google.com>
Date: Tue, 5 Dec 2023 23:28:48 -0800
Subject: [PATCH 15/15] fix three test failures related with zlib usage
---
.../Transforms/PGOProfile/vtable_profile.ll | 3 +++
.../llvm-profdata/Inputs/compressed.profraw | Bin 1968 -> 1984 bytes
.../vtable-value-prof-basic.test | 4 ++++
3 files changed, 7 insertions(+)
diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
index 5c50ea207928a..d61d55db7b85d 100644
--- a/llvm/test/Transforms/PGOProfile/vtable_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
@@ -2,6 +2,9 @@
; RUN: opt < %s -passes=pgo-instr-gen -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=GEN
; RUN: opt < %s -passes=pgo-instr-gen,instrprof -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=LOWER
+; __llvm_prf_vnm stores zlib-compressed vtable names.
+; REQUIRES: zlib
+
source_filename = "vtable_local.ll"
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw b/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw
index 9966729d92ddc33bf89eeb3fee87215bbabbbef1..4d36ffcf5e05b084cf0d1e04fe3933f80b0b1749 100644
GIT binary patch
delta 40
ycmV+ at 0N4Mp55NxzfpTVVa&T<_3Xus<4&eFQuj8rz|DDYvP#jj1P6HaTa6kus4H8fQ
delta 39
vcmX at Wzk#2#u_!ISs37M*=R{6_L5r?)Gq*SV|KArNnC4N>z`(e%(w!XuMI#TR
diff --git a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
index 02b5ebc948eb8..61545c7d4b878 100644
--- a/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
+++ b/llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
@@ -1,5 +1,9 @@
To update the inputs used below, run Inputs/update_vtable_value_prof_inputs.sh /path/to/updated/clang++
+; Raw profiles stores zlib-compressed vtable names. Raw profile reader needs
+; to decompress them.
+; REQUIRES: zlib
+
Show profile data from raw profiles.
RUN: llvm-profdata show --function=main --ic-targets --show-vtables %p/Inputs/vtable_prof.profraw | FileCheck %s --check-prefix=RAW
More information about the libcxx-commits
mailing list