[compiler-rt] r256543 - [PGO]: Do not update Data->Value field during profile write.
Xinliang David Li via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 28 23:13:59 PST 2015
Author: davidxl
Date: Tue Dec 29 01:13:59 2015
New Revision: 256543
URL: http://llvm.org/viewvc/llvm-project?rev=256543&view=rev
Log:
[PGO]: Do not update Data->Value field during profile write.
The profile reader no longer depends on this field to be updated and point
to owning func's vp data. The VP data also no longer needs to be allocated
in a contiguous memory space.
Differential Revision: http://reviews.llvm.org/D15258
Modified:
compiler-rt/trunk/lib/profile/InstrProfiling.h
compiler-rt/trunk/lib/profile/InstrProfilingFile.c
compiler-rt/trunk/lib/profile/InstrProfilingInternal.h
compiler-rt/trunk/lib/profile/InstrProfilingValue.c
compiler-rt/trunk/lib/profile/InstrProfilingWriter.c
compiler-rt/trunk/test/profile/instrprof-value-prof.c
Modified: compiler-rt/trunk/lib/profile/InstrProfiling.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/profile/InstrProfiling.h?rev=256543&r1=256542&r2=256543&view=diff
==============================================================================
--- compiler-rt/trunk/lib/profile/InstrProfiling.h (original)
+++ compiler-rt/trunk/lib/profile/InstrProfiling.h Tue Dec 29 01:13:59 2015
@@ -78,15 +78,16 @@ void INSTR_PROF_VALUE_PROF_FUNC(
/*!
* \brief Prepares the value profiling data for output.
*
- * Prepares a single __llvm_profile_value_data array out of the many
- * ValueProfNode trees (one per instrumented function).
+ * Returns an array of pointers to value profile data.
*/
-uint64_t __llvm_profile_gather_value_data(uint8_t **DataArray);
+struct ValueProfData;
+struct ValueProfData **__llvm_profile_gather_value_data(uint64_t *Size);
/*!
* \brief Write instrumentation data to the current file.
*
- * Writes to the file with the last name given to \a __llvm_profile_set_filename(),
+ * Writes to the file with the last name given to \a *
+ * __llvm_profile_set_filename(),
* or if it hasn't been called, the \c LLVM_PROFILE_FILE environment variable,
* or if that's not set, the last name given to
* \a __llvm_profile_override_default_filename(), or if that's not set,
Modified: compiler-rt/trunk/lib/profile/InstrProfilingFile.c
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/profile/InstrProfilingFile.c?rev=256543&r1=256542&r2=256543&view=diff
==============================================================================
--- compiler-rt/trunk/lib/profile/InstrProfilingFile.c (original)
+++ compiler-rt/trunk/lib/profile/InstrProfilingFile.c Tue Dec 29 01:13:59 2015
@@ -31,12 +31,16 @@ static uint32_t fileWriter(ProfDataIOVec
}
static int writeFile(FILE *File) {
- uint8_t *ValueDataBegin = NULL;
- const uint64_t ValueDataSize =
- __llvm_profile_gather_value_data(&ValueDataBegin);
- int r = llvmWriteProfData(fileWriter, File, ValueDataBegin, ValueDataSize);
- free(ValueDataBegin);
- return r;
+ const char *BufferSzStr = 0;
+ uint64_t ValueDataSize = 0;
+ struct ValueProfData **ValueDataArray =
+ __llvm_profile_gather_value_data(&ValueDataSize);
+ FreeHook = &free;
+ CallocHook = &calloc;
+ BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
+ if (BufferSzStr && BufferSzStr[0])
+ VPBufferSize = atoi(BufferSzStr);
+ return llvmWriteProfData(fileWriter, File, ValueDataArray, ValueDataSize);
}
static int writeFileWithName(const char *OutputName) {
Modified: compiler-rt/trunk/lib/profile/InstrProfilingInternal.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/profile/InstrProfilingInternal.h?rev=256543&r1=256542&r2=256543&view=diff
==============================================================================
--- compiler-rt/trunk/lib/profile/InstrProfilingInternal.h (original)
+++ compiler-rt/trunk/lib/profile/InstrProfilingInternal.h Tue Dec 29 01:13:59 2015
@@ -52,17 +52,20 @@ typedef uint32_t (*WriterCallback)(ProfD
uint32_t llvmBufferWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs,
void **WriterCtx);
int llvmWriteProfData(WriterCallback Writer, void *WriterCtx,
- const uint8_t *ValueDataBegin,
+ struct ValueProfData **ValueDataArray,
const uint64_t ValueDataSize);
int llvmWriteProfDataImpl(WriterCallback Writer, void *WriterCtx,
const __llvm_profile_data *DataBegin,
const __llvm_profile_data *DataEnd,
const uint64_t *CountersBegin,
const uint64_t *CountersEnd,
- const uint8_t *ValueDataBegin,
+ struct ValueProfData **ValueDataBeginArray,
const uint64_t ValueDataSize, const char *NamesBegin,
const char *NamesEnd);
extern char *(*GetEnvHook)(const char *);
+extern void (*FreeHook)(void *);
+extern void* (*CallocHook)(size_t, size_t);
+extern uint32_t VPBufferSize;
#endif
Modified: compiler-rt/trunk/lib/profile/InstrProfilingValue.c
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/profile/InstrProfilingValue.c?rev=256543&r1=256542&r2=256543&view=diff
==============================================================================
--- compiler-rt/trunk/lib/profile/InstrProfilingValue.c (original)
+++ compiler-rt/trunk/lib/profile/InstrProfilingValue.c Tue Dec 29 01:13:59 2015
@@ -21,7 +21,8 @@
#define PROF_OOM_RETURN(Msg) \
{ \
PROF_OOM(Msg) \
- return 0; \
+ free(ValueDataArray); \
+ return NULL; \
}
#if COMPILER_RT_HAS_ATOMICS != 1
@@ -78,24 +79,6 @@ static int allocateValueProfileCounters(
return 1;
}
-static void deallocateValueProfileCounters(__llvm_profile_data *Data) {
- uint64_t NumVSites = 0, I;
- uint32_t VKI;
- if (!Data->Values)
- return;
- for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
- NumVSites += Data->NumValueSites[VKI];
- for (I = 0; I < NumVSites; I++) {
- ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I];
- while (Node) {
- ValueProfNode *Next = Node->Next;
- free(Node);
- Node = Next;
- }
- }
- free(Data->Values);
-}
-
COMPILER_RT_VISIBILITY void
__llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
uint32_t CounterIndex) {
@@ -147,92 +130,51 @@ __llvm_profile_instrument_target(uint64_
}
}
-/* For multi-threaded programs, while the profile is being dumped, other
- threads may still be updating the value profile data and creating new
- value entries. To accommadate this, we need to add extra bytes to the
- data buffer. The size of the extra space is controlled by an environment
- variable. */
-static unsigned getVprofExtraBytes() {
- const char *ExtraStr =
- GetEnvHook ? GetEnvHook("LLVM_VALUE_PROF_BUFFER_EXTRA") : 0;
- if (!ExtraStr || !ExtraStr[0])
- return 1024;
- return (unsigned)atoi(ExtraStr);
-}
-
-/* Extract the value profile data info from the runtime. */
-#define DEF_VALUE_RECORD(R, NS, V) \
- ValueProfRuntimeRecord R; \
- if (initializeValueProfRuntimeRecord(&R, NS, V)) \
- PROF_OOM_RETURN("Failed to write value profile data ");
-
-#define DTOR_VALUE_RECORD(R) finalizeValueProfRuntimeRecord(&R);
-
-COMPILER_RT_VISIBILITY uint64_t
-__llvm_profile_gather_value_data(uint8_t **VDataArray) {
- size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0;
+COMPILER_RT_VISIBILITY ValueProfData **
+__llvm_profile_gather_value_data(uint64_t *ValueDataSize) {
+ size_t S = 0;
__llvm_profile_data *I;
- if (!VDataArray)
- PROF_OOM_RETURN("Failed to write value profile data ");
+ ValueProfData **ValueDataArray;
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ if (!ValueDataSize)
+ return NULL;
+
+ ValueDataArray =
+ (ValueProfData **)calloc(DataEnd - DataBegin, sizeof(void *));
+ if (!ValueDataArray)
+ PROF_OOM_RETURN("Failed to write value profile data ");
+
/*
* Compute the total Size of the buffer to hold ValueProfData
* structures for functions with value profile data.
*/
for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
-
- DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
+ ValueProfRuntimeRecord R;
+ if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values))
+ PROF_OOM_RETURN("Failed to write value profile data ");
/* Compute the size of ValueProfData from this runtime record. */
- if (getNumValueKindsRT(&R) != 0)
- S += getValueProfDataSizeRT(&R);
-
- DTOR_VALUE_RECORD(R);
- }
- /* No value sites or no value profile data is collected. */
- if (!S)
- return 0;
-
- Extra = getVprofExtraBytes();
- BufferCapacity = S + Extra;
- *VDataArray = calloc(BufferCapacity, sizeof(uint8_t));
- if (!*VDataArray)
- PROF_OOM_RETURN("Failed to write value profile data ");
-
- ValueProfData *VD = (ValueProfData *)(*VDataArray);
- /*
- * Extract value profile data and write into ValueProfData structure
- * one by one. Note that new value profile data added to any value
- * site (from another thread) after the ValueProfRuntimeRecord is
- * initialized (when the profile data snapshot is taken) won't be
- * collected. This is not a problem as those dropped value will have
- * very low taken count.
- */
- for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
- DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
- if (getNumValueKindsRT(&R) == 0)
- continue;
-
- /* Record R has taken a snapshot of the VP data at this point. Newly
- added VP data for this function will be dropped. */
- /* Check if there is enough space. */
- if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) {
- PROF_ERR("Value profile data is dropped :%s \n",
- "Out of buffer space. Use environment "
- " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more");
- I->Values = 0;
+ if (getNumValueKindsRT(&R) != 0) {
+ ValueProfData *VD = NULL;
+ uint32_t VS = getValueProfDataSizeRT(&R);
+ VD = (ValueProfData *)calloc(VS, sizeof(uint8_t));
+ if (!VD)
+ PROF_OOM_RETURN("Failed to write value profile data ");
+ serializeValueProfDataFromRT(&R, VD);
+ ValueDataArray[I - DataBegin] = VD;
+ S += VS;
}
+ finalizeValueProfRuntimeRecord(&R);
+ }
- serializeValueProfDataFromRT(&R, VD);
- deallocateValueProfileCounters(I);
- I->Values = VD;
- RealSize += VD->TotalSize;
- VD = (ValueProfData *)((char *)VD + VD->TotalSize);
- DTOR_VALUE_RECORD(R);
+ if (!S) {
+ free(ValueDataArray);
+ ValueDataArray = NULL;
}
- return RealSize;
+ *ValueDataSize = S;
+ return ValueDataArray;
}
Modified: compiler-rt/trunk/lib/profile/InstrProfilingWriter.c
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/profile/InstrProfilingWriter.c?rev=256543&r1=256542&r2=256543&view=diff
==============================================================================
--- compiler-rt/trunk/lib/profile/InstrProfilingWriter.c (original)
+++ compiler-rt/trunk/lib/profile/InstrProfilingWriter.c Tue Dec 29 01:13:59 2015
@@ -11,6 +11,12 @@
#include "InstrProfilingInternal.h"
#include <string.h>
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "InstrProfData.inc"
+void (*FreeHook)(void *) = NULL;
+void* (*CallocHook)(size_t, size_t) = NULL;
+uint32_t VPBufferSize = 0;
+
/* The buffer writer is reponsponsible in keeping writer state
* across the call.
*/
@@ -29,7 +35,7 @@ COMPILER_RT_VISIBILITY uint32_t llvmBuff
COMPILER_RT_VISIBILITY int llvmWriteProfData(WriterCallback Writer,
void *WriterCtx,
- const uint8_t *ValueDataBegin,
+ ValueProfData **ValueDataArray,
const uint64_t ValueDataSize) {
/* Match logic in __llvm_profile_write_buffer(). */
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
@@ -39,15 +45,80 @@ COMPILER_RT_VISIBILITY int llvmWriteProf
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
return llvmWriteProfDataImpl(Writer, WriterCtx, DataBegin, DataEnd,
- CountersBegin, CountersEnd, ValueDataBegin,
+ CountersBegin, CountersEnd, ValueDataArray,
ValueDataSize, NamesBegin, NamesEnd);
}
+#define VP_BUFFER_SIZE 8 * 1024
+static int writeValueProfData(WriterCallback Writer, void *WriterCtx,
+ ValueProfData **ValueDataBegin,
+ uint64_t NumVData) {
+ ValueProfData **ValueDataArray = ValueDataBegin;
+ char *BufferStart = 0, *Buffer;
+ ValueProfData *CurVData;
+ uint32_t I = 0, BufferSz;
+
+ if (!ValueDataBegin)
+ return 0;
+
+ BufferSz = VPBufferSize ? VPBufferSize : VP_BUFFER_SIZE;
+ BufferStart = (char *)CallocHook(BufferSz, sizeof(uint8_t));
+ if (!BufferStart)
+ return -1;
+
+ uint32_t WriteSize = 0;
+ Buffer = BufferStart;
+ do {
+ CurVData = ValueDataArray[I];
+ if (!CurVData) {
+ I++;
+ continue;
+ }
+
+ /* Buffer is full or not large enough, it is time to flush. */
+ if (CurVData->TotalSize + WriteSize > BufferSz) {
+ if (WriteSize) {
+ ProfDataIOVec IO[] = {{BufferStart, sizeof(uint8_t), WriteSize}};
+ if (Writer(IO, 1, &WriterCtx))
+ return -1;
+ WriteSize = 0;
+ Buffer = BufferStart;
+ }
+ /* Special case, bypass the buffer completely. */
+ if (CurVData->TotalSize > BufferSz) {
+ ProfDataIOVec IO[] = {{CurVData, sizeof(uint8_t), CurVData->TotalSize}};
+ if (Writer(IO, 1, &WriterCtx))
+ return -1;
+ FreeHook(ValueDataArray[I]);
+ I++;
+ }
+ } else {
+ /* Write the data to buffer */
+ ProfDataIOVec IO[] = {{CurVData, sizeof(uint8_t), CurVData->TotalSize}};
+ llvmBufferWriter(IO, 1, (void **)&Buffer);
+ WriteSize += CurVData->TotalSize;
+ FreeHook(ValueDataArray[I]);
+ I++;
+ }
+ } while (I < NumVData);
+
+ /* Final flush. */
+ if (WriteSize) {
+ ProfDataIOVec IO[] = {{BufferStart, sizeof(uint8_t), WriteSize}};
+ if (Writer(IO, 1, &WriterCtx))
+ return -1;
+ }
+
+ FreeHook(ValueDataBegin);
+ FreeHook(BufferStart);
+ return 0;
+}
+
COMPILER_RT_VISIBILITY int llvmWriteProfDataImpl(
WriterCallback Writer, void *WriterCtx,
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const uint64_t *CountersBegin, const uint64_t *CountersEnd,
- const uint8_t *ValueDataBegin, const uint64_t ValueDataSize,
+ ValueProfData **ValueDataBegin, const uint64_t ValueDataSize,
const char *NamesBegin, const char *NamesEnd) {
/* Calculate size of sections. */
@@ -70,18 +141,13 @@ COMPILER_RT_VISIBILITY int llvmWriteProf
#include "InstrProfData.inc"
/* Write the data. */
- ProfDataIOVec IOVec[] = {
- {&Header, sizeof(__llvm_profile_header), 1},
- {DataBegin, sizeof(__llvm_profile_data), DataSize},
- {CountersBegin, sizeof(uint64_t), CountersSize},
- {NamesBegin, sizeof(char), NamesSize},
- {Zeroes, sizeof(char), Padding}};
+ ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1},
+ {DataBegin, sizeof(__llvm_profile_data), DataSize},
+ {CountersBegin, sizeof(uint64_t), CountersSize},
+ {NamesBegin, sizeof(uint8_t), NamesSize},
+ {Zeroes, sizeof(uint8_t), Padding}};
if (Writer(IOVec, sizeof(IOVec) / sizeof(*IOVec), &WriterCtx))
return -1;
- if (ValueDataBegin) {
- ProfDataIOVec IOVec2[] = {{ValueDataBegin, sizeof(char), ValueDataSize}};
- if (Writer(IOVec2, sizeof(IOVec2) / sizeof(*IOVec2), &WriterCtx))
- return -1;
- }
- return 0;
+
+ return writeValueProfData(Writer, WriterCtx, ValueDataBegin, DataSize);
}
Modified: compiler-rt/trunk/test/profile/instrprof-value-prof.c
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/profile/instrprof-value-prof.c?rev=256543&r1=256542&r2=256543&view=diff
==============================================================================
--- compiler-rt/trunk/test/profile/instrprof-value-prof.c (original)
+++ compiler-rt/trunk/test/profile/instrprof-value-prof.c Tue Dec 29 01:13:59 2015
@@ -9,6 +9,22 @@
// value profile merging current do sorting based on target values -- this will destroy the order of the target
// in the list leading to comparison problem. For now just check a small subset of output.
// RUN: llvm-profdata show --all-functions -ic-targets %t-merged.profdata | FileCheck %s -check-prefix=MERGE
+//
+// RUN: env LLVM_PROFILE_FILE=%t-3.profraw LLVM_VP_BUFFER_SIZE=1 %run %t 1
+// RUN: env LLVM_PROFILE_FILE=%t-4.profraw LLVM_VP_BUFFER_SIZE=8 %run %t 1
+// RUN: env LLVM_PROFILE_FILE=%t-5.profraw LLVM_VP_BUFFER_SIZE=128 %run %t 1
+// RUN: env LLVM_PROFILE_FILE=%t-6.profraw LLVM_VP_BUFFER_SIZE=1024 %run %t 1
+// RUN: env LLVM_PROFILE_FILE=%t-7.profraw LLVM_VP_BUFFER_SIZE=102400 %run %t 1
+// RUN: llvm-profdata merge -o %t-3.profdata %t-3.profraw
+// RUN: llvm-profdata merge -o %t-4.profdata %t-4.profraw
+// RUN: llvm-profdata merge -o %t-5.profdata %t-5.profraw
+// RUN: llvm-profdata merge -o %t-6.profdata %t-6.profraw
+// RUN: llvm-profdata merge -o %t-7.profdata %t-7.profraw
+// RUN: llvm-profdata show --all-functions -ic-targets %t-3.profdata | FileCheck %s
+// RUN: llvm-profdata show --all-functions -ic-targets %t-4.profdata | FileCheck %s
+// RUN: llvm-profdata show --all-functions -ic-targets %t-5.profdata | FileCheck %s
+// RUN: llvm-profdata show --all-functions -ic-targets %t-6.profdata | FileCheck %s
+// RUN: llvm-profdata show --all-functions -ic-targets %t-7.profdata | FileCheck %s
#include <stdint.h>
#include <stdio.h>
More information about the llvm-commits
mailing list