[compiler-rt] r272227 - [profile] in-process merging support part-3
Xinliang David Li via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 9 14:20:56 PDT 2016
What is the portable way to do force remove?
I can move this test to be linux only for now.
David
On Thu, Jun 9, 2016 at 2:18 PM, Michael Spencer via llvm-commits <
llvm-commits at lists.llvm.org> wrote:
> On Wed, Jun 8, 2016 at 4:43 PM, Xinliang David Li via llvm-commits
> <llvm-commits at lists.llvm.org> wrote:
> > Author: davidxl
> > Date: Wed Jun 8 18:43:56 2016
> > New Revision: 272227
> >
> > URL: http://llvm.org/viewvc/llvm-project?rev=272227&view=rev
> > Log:
> > [profile] in-process merging support part-3
> >
> > Differential Revision: http://reviews.llvm.org/D21056
> >
> >
> > Modified:
> > compiler-rt/trunk/lib/profile/InstrProfilingFile.c
> > compiler-rt/trunk/lib/profile/InstrProfilingInternal.h
> > compiler-rt/trunk/lib/profile/InstrProfilingMerge.c
> > compiler-rt/trunk/lib/profile/InstrProfilingPort.h
> > compiler-rt/trunk/test/profile/instrprof-basic.c
> >
> > Modified: compiler-rt/trunk/lib/profile/InstrProfilingFile.c
> > URL:
> http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/profile/InstrProfilingFile.c?rev=272227&r1=272226&r2=272227&view=diff
> >
> ==============================================================================
> > --- compiler-rt/trunk/lib/profile/InstrProfilingFile.c (original)
> > +++ compiler-rt/trunk/lib/profile/InstrProfilingFile.c Wed Jun 8
> 18:43:56 2016
> > @@ -18,6 +18,18 @@
> > /* For _alloca. */
> > #include <malloc.h>
> > #endif
> > +#if defined(_WIN32)
> > +#include "WindowsMMap.h"
> > +/* For _chsize_s */
> > +#include <io.h>
> > +#else
> > +#include <sys/file.h>
> > +#include <sys/mman.h>
> > +#include <unistd.h>
> > +#if defined(__linux__)
> > +#include <sys/types.h>
> > +#endif
> > +#endif
> >
> > #define MAX_PID_SIZE 16
> > /* Data structure holding the result of parsed filename pattern. */
> > @@ -28,13 +40,22 @@ typedef struct lprofFilename {
> > char Hostname[COMPILER_RT_MAX_HOSTLEN];
> > unsigned NumPids;
> > unsigned NumHosts;
> > + /* When in-process merging is enabled, this parameter specifies
> > + * the total number of profile data files shared by all the processes
> > + * spawned from the same binary. By default the value is 1. If merging
> > + * is not enabled, its value should be 0. This parameter is specified
> > + * by the %[0-9]m specifier. For instance %2m enables merging using
> > + * 2 profile data files. %1m is equivalent to %m. Also %m specifier
> > + * can only appear once at the end of the name pattern. */
> > + unsigned MergePoolSize;
> > } lprofFilename;
> >
> > -lprofFilename lprofCurFilename = {0, {0}, {0}, 0, 0};
> > +lprofFilename lprofCurFilename = {0, {0}, {0}, 0, 0, 0};
> >
> > int getpid(void);
> > static int getCurFilenameLength();
> > static const char *getCurFilename(char *FilenameBuf);
> > +static unsigned doMerging() { return lprofCurFilename.MergePoolSize; }
> >
> > /* Return 1 if there is an error, otherwise return 0. */
> > static uint32_t fileWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs,
> > @@ -66,13 +87,96 @@ static void setupIOBuffer() {
> > }
> > }
> >
> > +/* Read profile data in \c ProfileFile and merge with in-memory
> > + profile counters. Returns -1 if there is fatal error, otheriwse
> > + 0 is returned.
> > +*/
> > +static int doProfileMerging(FILE *ProfileFile) {
> > + uint64_t ProfileFileSize;
> > + char *ProfileBuffer;
> > +
> > + if (fseek(ProfileFile, 0L, SEEK_END) == -1) {
> > + PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
> > + strerror(errno));
> > + return -1;
> > + }
> > + ProfileFileSize = ftell(ProfileFile);
> > +
> > + /* Restore file offset. */
> > + if (fseek(ProfileFile, 0L, SEEK_SET) == -1) {
> > + PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
> > + strerror(errno));
> > + return -1;
> > + }
> > +
> > + /* Nothing to merge. */
> > + if (ProfileFileSize < sizeof(__llvm_profile_header)) {
> > + if (ProfileFileSize)
> > + PROF_WARN("Unable to merge profile data: %s\n",
> > + "source profile file is too small.");
> > + return 0;
> > + }
> > +
> > + ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED |
> MAP_FILE,
> > + fileno(ProfileFile), 0);
> > + if (ProfileBuffer == MAP_FAILED) {
> > + PROF_ERR("Unable to merge profile data, mmap failed: %s\n",
> > + strerror(errno));
> > + return -1;
> > + }
> > +
> > + if (__llvm_profile_check_compatibility(ProfileBuffer,
> ProfileFileSize)) {
> > + (void)munmap(ProfileBuffer, ProfileFileSize);
> > + PROF_WARN("Unable to merge profile data: %s\n",
> > + "source profile file is not compatible.");
> > + return 0;
> > + }
> > +
> > + /* Now start merging */
> > + __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize);
> > + (void)munmap(ProfileBuffer, ProfileFileSize);
> > +
> > + return 0;
> > +}
> > +
> > +/* Open the profile data for merging. It opens the file in r+b mode with
> > + * file locking. If the file has content which is compatible with the
> > + * current process, it also reads in the profile data in the file and
> merge
> > + * it with in-memory counters. After the profile data is merged in
> memory,
> > + * the original profile data is truncated and gets ready for the profile
> > + * dumper. With profile merging enabled, each executable as well as any
> of
> > + * its instrumented shared libraries dump profile data into their own
> data file.
> > +*/
> > +static FILE *openFileForMerging(const char *ProfileFileName) {
> > + FILE *ProfileFile;
> > + int rc;
> > +
> > + ProfileFile = lprofOpenFileEx(ProfileFileName);
> > + if (!ProfileFile)
> > + return NULL;
> > +
> > + rc = doProfileMerging(ProfileFile);
> > + if (rc || COMPILER_RT_FTRUNCATE(ProfileFile, 0L) ||
> > + fseek(ProfileFile, 0L, SEEK_SET) == -1) {
> > + PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName,
> > + strerror(errno));
> > + fclose(ProfileFile);
> > + return NULL;
> > + }
> > + fseek(ProfileFile, 0L, SEEK_SET);
> > + return ProfileFile;
> > +}
> > +
> > /* Write profile data to file \c OutputName. */
> > static int writeFile(const char *OutputName) {
> > int RetVal;
> > FILE *OutputFile;
> >
> > - /* Append to the file to support profiling multiple shared objects. */
> > - OutputFile = fopen(OutputName, "ab");
> > + if (!doMerging())
> > + OutputFile = fopen(OutputName, "ab");
> > + else
> > + OutputFile = openFileForMerging(OutputName);
> > +
> > if (!OutputFile)
> > return -1;
> >
> > @@ -115,13 +219,21 @@ static void resetFilenameToDefault(void)
> > lprofCurFilename.FilenamePat = "default.profraw";
> > }
> >
> > -/* Parses the pattern string \p FilenamePat and store the result to
> > - * lprofcurFilename structure. */
> > +static int containsMergeSpecifier(const char *FilenamePat, int I) {
> > + return (FilenamePat[I] == 'm' ||
> > + (FilenamePat[I] >= '1' && FilenamePat[I] <= '9' &&
> > + /* If FilenamePat[I] is not '\0', the next byte is guaranteed
> > + * to be in-bound as the string is null terminated. */
> > + FilenamePat[I + 1] == 'm'));
> > +}
> >
> > +/* Parses the pattern string \p FilenamePat and stores the result to
> > + * lprofcurFilename structure. */
> > static int parseFilenamePattern(const char *FilenamePat) {
> > int NumPids = 0, NumHosts = 0, I;
> > char *PidChars = &lprofCurFilename.PidChars[0];
> > char *Hostname = &lprofCurFilename.Hostname[0];
> > + int MergingEnabled = 0;
> >
> > lprofCurFilename.FilenamePat = FilenamePat;
> > /* Check the filename for "%p", which indicates a pid-substitution. */
> > @@ -144,6 +256,20 @@ static int parseFilenamePattern(const ch
> > FilenamePat);
> > return -1;
> > }
> > + } else if (containsMergeSpecifier(FilenamePat, I)) {
> > + if (MergingEnabled) {
> > + PROF_WARN(
> > + "%%m specifier can only be specified once at the end of
> %s.\n",
> > + FilenamePat);
> > + return -1;
> > + }
> > + MergingEnabled = 1;
> > + if (FilenamePat[I] == 'm')
> > + lprofCurFilename.MergePoolSize = 1;
> > + else {
> > + lprofCurFilename.MergePoolSize = FilenamePat[I] - '0';
> > + I++; /* advance to 'm' */
> > + }
> > }
> > }
> >
> > @@ -162,22 +288,29 @@ static void parseAndSetFilename(const ch
> > NewFile =
> > !OldFilenamePat || (strcmp(OldFilenamePat,
> lprofCurFilename.FilenamePat));
> >
> > - if (NewFile)
> > + if (NewFile && !lprofCurFilename.MergePoolSize)
> > truncateCurrentFile();
> > }
> >
> > /* Return buffer length that is required to store the current profile
> > * filename with PID and hostname substitutions. */
> > +/* The length to hold uint64_t followed by 2 digit pool id including
> '_' */
> > +#define SIGLEN 24
> > static int getCurFilenameLength() {
> > + int Len;
> > if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
> > return 0;
> >
> > - if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts))
> > + if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
> > + lprofCurFilename.MergePoolSize))
> > return strlen(lprofCurFilename.FilenamePat);
> >
> > - return strlen(lprofCurFilename.FilenamePat) +
> > - lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars)
> - 2) +
> > - lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname)
> - 2);
> > + Len = strlen(lprofCurFilename.FilenamePat) +
> > + lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) -
> 2) +
> > + lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname)
> - 2);
> > + if (lprofCurFilename.MergePoolSize)
> > + Len += SIGLEN;
> > + return Len;
> > }
> >
> > /* Return the pointer to the current profile file name (after
> substituting
> > @@ -191,7 +324,8 @@ static const char *getCurFilename(char *
> > if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
> > return 0;
> >
> > - if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts))
> > + if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
> > + lprofCurFilename.MergePoolSize))
> > return lprofCurFilename.FilenamePat;
> >
> > PidLength = strlen(lprofCurFilename.PidChars);
> > @@ -205,6 +339,18 @@ static const char *getCurFilename(char *
> > } else if (FilenamePat[I] == 'h') {
> > memcpy(FilenameBuf + J, lprofCurFilename.Hostname,
> HostNameLength);
> > J += HostNameLength;
> > + } else if (containsMergeSpecifier(FilenamePat, I)) {
> > + char LoadModuleSignature[SIGLEN];
> > + int S;
> > + int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize;
> > + S = snprintf(LoadModuleSignature, SIGLEN, "%" PRIu64 "_%d",
> > + lprofGetLoadModuleSignature(), ProfilePoolId);
> > + if (S == -1 || S > SIGLEN)
> > + S = SIGLEN;
> > + memcpy(FilenameBuf + J, LoadModuleSignature, S);
> > + J += S;
> > + if (FilenamePat[I] != 'm')
> > + I++;
> > }
> > /* Drop any unknown substitutions. */
> > } else
> >
> > Modified: compiler-rt/trunk/lib/profile/InstrProfilingInternal.h
> > URL:
> http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/profile/InstrProfilingInternal.h?rev=272227&r1=272226&r2=272227&view=diff
> >
> ==============================================================================
> > --- compiler-rt/trunk/lib/profile/InstrProfilingInternal.h (original)
> > +++ compiler-rt/trunk/lib/profile/InstrProfilingInternal.h Wed Jun 8
> 18:43:56 2016
> > @@ -156,6 +156,13 @@ VPDataReaderType *lprofGetVPDataReader()
> > void lprofSetMaxValsPerSite(uint32_t MaxVals);
> > void lprofSetupValueProfiler();
> >
> > +/* Return the profile header 'signature' value associated with the
> current
> > + * executable or shared library. The signature value can be used to for
> > + * a profile name that is unique to this load module so that it does not
> > + * collide with profiles from other binaries. It also allows shared
> libraries
> > + * to dump merged profile data into its own profile file. */
> > +uint64_t lprofGetLoadModuleSignature();
> > +
> > COMPILER_RT_VISIBILITY extern char *(*GetEnvHook)(const char *);
> > COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *);
> > COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer;
> >
> > Modified: compiler-rt/trunk/lib/profile/InstrProfilingMerge.c
> > URL:
> http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/profile/InstrProfilingMerge.c?rev=272227&r1=272226&r2=272227&view=diff
> >
> ==============================================================================
> > --- compiler-rt/trunk/lib/profile/InstrProfilingMerge.c (original)
> > +++ compiler-rt/trunk/lib/profile/InstrProfilingMerge.c Wed Jun 8
> 18:43:56 2016
> > @@ -19,6 +19,22 @@
> >
> > COMPILER_RT_WEAK void (*VPMergeHook)(ValueProfData *,
> > __llvm_profile_data *) = NULL;
> > +COMPILER_RT_VISIBILITY
> > +uint64_t lprofGetLoadModuleSignature() {
> > + /* A very fast way to compute a module signature. */
> > + uint64_t CounterSize = (uint64_t)(__llvm_profile_end_counters() -
> > + __llvm_profile_begin_counters());
> > + uint64_t DataSize =
> __llvm_profile_get_data_size(__llvm_profile_begin_data(),
> > +
> __llvm_profile_end_data());
> > + uint64_t NamesSize =
> > + (uint64_t)(__llvm_profile_end_names() -
> __llvm_profile_begin_names());
> > + uint64_t NumVnodes =
> > + (uint64_t)(__llvm_profile_end_vnodes() -
> __llvm_profile_begin_vnodes());
> > + const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
> > +
> > + return (NamesSize << 40) + (CounterSize << 30) + (DataSize << 20) +
> > + (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0);
> > +}
> >
> > /* Returns 1 if profile is not structurally compatible. */
> > COMPILER_RT_VISIBILITY
> > @@ -31,6 +47,9 @@ int __llvm_profile_check_compatibility(c
> > (__llvm_profile_data *)(ProfileData +
> sizeof(__llvm_profile_header));
> > SrcDataEnd = SrcDataStart + Header->DataSize;
> >
> > + if (ProfileSize < sizeof(__llvm_profile_header))
> > + return 1;
> > +
> > /* Check the header first. */
> > if (Header->Magic != __llvm_profile_get_magic() ||
> > Header->Version != __llvm_profile_get_version() ||
> >
> > Modified: compiler-rt/trunk/lib/profile/InstrProfilingPort.h
> > URL:
> http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/profile/InstrProfilingPort.h?rev=272227&r1=272226&r2=272227&view=diff
> >
> ==============================================================================
> > --- compiler-rt/trunk/lib/profile/InstrProfilingPort.h (original)
> > +++ compiler-rt/trunk/lib/profile/InstrProfilingPort.h Wed Jun 8
> 18:43:56 2016
> > @@ -14,12 +14,16 @@
> > #define COMPILER_RT_ALIGNAS(x) __declspec(align(x))
> > #define COMPILER_RT_VISIBILITY
> > #define COMPILER_RT_WEAK __declspec(selectany)
> > +/* Need to include <windows.h> */
> > #define COMPILER_RT_ALLOCA _alloca
> > +/* Need to include <stdio.h> and <io.h> */
> > +#define COMPILER_RT_FTRUNCATE(f,l) _chsize(_fileno(f),l)
> > #elif __GNUC__
> > #define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x)))
> > #define COMPILER_RT_VISIBILITY __attribute__((visibility("hidden")))
> > #define COMPILER_RT_WEAK __attribute__((weak))
> > #define COMPILER_RT_ALLOCA __builtin_alloca
> > +#define COMPILER_RT_FTRUNCATE(f,l) ftruncate(fileno(f),l)
> > #endif
> >
> > #if defined(__APPLE__)
> >
> > Modified: compiler-rt/trunk/test/profile/instrprof-basic.c
> > URL:
> http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/profile/instrprof-basic.c?rev=272227&r1=272226&r2=272227&view=diff
> >
> ==============================================================================
> > --- compiler-rt/trunk/test/profile/instrprof-basic.c (original)
> > +++ compiler-rt/trunk/test/profile/instrprof-basic.c Wed Jun 8 18:43:56
> 2016
> > @@ -1,17 +1,30 @@
> > // RUN: %clang_profgen -o %t -O3 %s
> > // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t
> > // RUN: llvm-profdata merge -o %t.profdata %t.profraw
> > -// RUN: %clang_profuse=%t.profdata -o - -S -emit-llvm %s | FileCheck %s
> > +// RUN: %clang_profuse=%t.profdata -o - -S -emit-llvm %s | FileCheck %s
> --check-prefix=COMMON --check-prefix=ORIG
> > +//
> > +// RUN: rm -f %t.profraw_e_*
> > +// RUN: env LLVM_PROFILE_FILE=%t.profraw_e_%1m %run %t
> > +// RUN: env LLVM_PROFILE_FILE=%t.profraw_e_%1m %run %t
> > +// RUN: llvm-profdata merge -o %t.em.profdata %t.profraw_e_*
> > +// RUN: %clang_profuse=%t.em.profdata -o - -S -emit-llvm %s | FileCheck
> %s --check-prefix=COMMON --check-prefix=MERGE
> > +//
> > +// RUN: %clang -o %t.merge -fprofile-instr-generate=%t.%m.profraw -O3 %s
> > +// RUN: rm -f %t.*.profraw*
> > +// RUN: %run %t.merge
> > +// RUN: %run %t.merge
> > +// RUN: llvm-profdata merge -o %t.m.profdata %t.*.profraw
> > +// RUN: %clang_profuse=%t.m.profdata -o - -S -emit-llvm %s | FileCheck
> %s --check-prefix=COMMON --check-prefix=MERGE
> >
> > int begin(int i) {
> > - // CHECK: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof
> ![[PD1:[0-9]+]]
> > + // COMMON: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof
> ![[PD1:[0-9]+]]
> > if (i)
> > return 0;
> > return 1;
> > }
> >
> > int end(int i) {
> > - // CHECK: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof
> ![[PD2:[0-9]+]]
> > + // COMMON: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof
> ![[PD2:[0-9]+]]
> > if (i)
> > return 0;
> > return 1;
> > @@ -21,11 +34,13 @@ int main(int argc, const char *argv[]) {
> > begin(0);
> > end(1);
> >
> > - // CHECK: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof
> ![[PD2:[0-9]+]]
> > + // COMMON: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof
> ![[PD2:[0-9]+]]
> > if (argc)
> > return 0;
> > return 1;
> > }
> >
> > -// CHECK: ![[PD1]] = !{!"branch_weights", i32 1, i32 2}
> > -// CHECK: ![[PD2]] = !{!"branch_weights", i32 2, i32 1}
> > +// ORIG: ![[PD1]] = !{!"branch_weights", i32 1, i32 2}
> > +// ORIG: ![[PD2]] = !{!"branch_weights", i32 2, i32 1}
> > +// MERGE: ![[PD1]] = !{!"branch_weights", i32 1, i32 3}
> > +// MERGE: ![[PD2]] = !{!"branch_weights", i32 3, i32 1}
> >
> >
> > _______________________________________________
> > llvm-commits mailing list
> > llvm-commits at lists.llvm.org
> > http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
>
> This breaks on Windows:
>
> $ "FileCheck"
> "C:\self-support.src\compiler-rt\test\profile\instrprof-basic.c"
> "--check-prefix=COMMON" "--check-prefix=ORIG"
> $ "rm" "-f"
> "C:\self-support-compilerrt.obj\test\profile\Profile-x86_64\Output\instrprof-basic.c.tmp.profraw_e_*"
> # command stderr:
> rm: cannot remove
>
> `C:\\self-support-compilerrt.obj\\test\\profile\\Profile-x86_64\\Output\\instrprof-basic.c.tmp.profraw_e_*':
> Invalid argument
>
>
> error: command failed with exit status: 1
>
> - Michael Spencer
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20160609/0f4ba48e/attachment-0001.html>
More information about the llvm-commits
mailing list