[llvm] r204496 - InstrProf: Read raw binary profile in llvm-profdata

Hal Finkel hfinkel at anl.gov
Sat Mar 22 22:53:05 PDT 2014


----- Original Message -----
> From: "Duncan P. N. Exon Smith" <dexonsmith at apple.com>
> To: llvm-commits at cs.uiuc.edu
> Sent: Friday, March 21, 2014 1:26:06 PM
> Subject: [llvm] r204496 - InstrProf: Read raw binary profile in llvm-profdata
> 
> Author: dexonsmith
> Date: Fri Mar 21 13:26:05 2014
> New Revision: 204496
> 
> URL: http://llvm.org/viewvc/llvm-project?rev=204496&view=rev
> Log:
> InstrProf: Read raw binary profile in llvm-profdata
> 
> Read a raw binary profile that corresponds to a memory dump from the
> runtime profile.
> 
> The test is a binary file generated from
> cfe/trunk/test/Profile/c-general.c with the new compiler-rt runtime
> and
> the matching text version of the input.  It includes instructions on
> how
> to regenerate.
> 
> <rdar://problem/15950346>
> 
> Added:
>     llvm/trunk/test/tools/llvm-profdata/Inputs/binary-compare.profdata
>     llvm/trunk/test/tools/llvm-profdata/Inputs/binary.profdata
>     llvm/trunk/test/tools/llvm-profdata/binary.test
> Modified:
>     llvm/trunk/include/llvm/ProfileData/InstrProfReader.h
>     llvm/trunk/lib/ProfileData/InstrProfReader.cpp
> 
> Modified: llvm/trunk/include/llvm/ProfileData/InstrProfReader.h
> URL:
> http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/ProfileData/InstrProfReader.h?rev=204496&r1=204495&r2=204496&view=diff
> ==============================================================================
> --- llvm/trunk/include/llvm/ProfileData/InstrProfReader.h (original)
> +++ llvm/trunk/include/llvm/ProfileData/InstrProfReader.h Fri Mar 21
> 13:26:05 2014
> @@ -19,6 +19,7 @@
>  #include "llvm/ProfileData/InstrProf.h"
>  #include "llvm/Support/LineIterator.h"
>  #include "llvm/Support/MemoryBuffer.h"
> +#include "llvm/Support/Endian.h"
>  
>  #include <iterator>
>  
> @@ -59,20 +60,25 @@ public:
>    InstrProfReader() : LastError(instrprof_error::success) {}
>    virtual ~InstrProfReader() {}
>  
> +  /// Read the header.  Required before reading first record.
> +  virtual error_code readHeader() = 0;
>    /// Read a single record.
>    virtual error_code readNextRecord(InstrProfRecord &Record) = 0;
>    /// Iterator over profile data.
>    InstrProfIterator begin() { return InstrProfIterator(this); }
>    InstrProfIterator end() { return InstrProfIterator(); }
>  
> +protected:
>    /// Set the current error_code and return same.
>    error_code error(error_code EC) {
>      LastError = EC;
>      return EC;
>    }
> +
>    /// Clear the current error code and return a successful one.
>    error_code success() { return error(instrprof_error::success); }
>  
> +public:
>    /// Return true if the reader has finished reading the profile
>    data.
>    bool isEOF() { return LastError == instrprof_error::eof; }
>    /// Return true if the reader encountered an error reading
>    profiling data.
> @@ -110,10 +116,72 @@ public:
>    TextInstrProfReader(std::unique_ptr<MemoryBuffer> &DataBuffer_)
>        : DataBuffer(DataBuffer_.release()), Line(*DataBuffer, '#') {}
>  
> +  /// Read the header.
> +  error_code readHeader() override { return success(); }
>    /// Read a single record.
>    error_code readNextRecord(InstrProfRecord &Record) override;
>  };
>  
> +/// Reader for the raw instrprof binary format from runtime.
> +///
> +/// This format is a raw memory dump of the instrumentation-baed
> profiling data
> +/// from the runtime.  It has no index.
> +class RawInstrProfReader : public InstrProfReader {
> +private:
> +  /// The profile data file contents.
> +  std::unique_ptr<MemoryBuffer> DataBuffer;
> +  /// The current set of counter values.
> +  std::vector<uint64_t> Counts;
> +  struct ProfileData {
> +    const uint32_t NameSize;
> +    const uint32_t NumCounters;
> +    const uint64_t FuncHash;
> +    const uint64_t NamePtr;
> +    const uint64_t CounterPtr;
> +  };
> +  struct RawHeader {
> +    const uint64_t Magic;
> +    const uint64_t Version;
> +    const uint64_t DataSize;
> +    const uint64_t CountersSize;
> +    const uint64_t NamesSize;
> +    const uint64_t CountersDelta;
> +    const uint64_t NamesDelta;
> +  };
> +
> +  bool ShouldSwapBytes;
> +  uint64_t CountersDelta;
> +  uint64_t NamesDelta;
> +  const ProfileData *Data;
> +  const ProfileData *DataEnd;
> +  const uint64_t *CountersStart;
> +  const char *NamesStart;
> +
> +  RawInstrProfReader(const TextInstrProfReader &)
> LLVM_DELETED_FUNCTION;
> +  RawInstrProfReader &operator=(const TextInstrProfReader &)
> +    LLVM_DELETED_FUNCTION;
> +public:
> +  RawInstrProfReader(std::unique_ptr<MemoryBuffer> &DataBuffer);
> +
> +  error_code readHeader() override;
> +  error_code readNextRecord(InstrProfRecord &Record) override;
> +
> +private:
> +  error_code readHeader(const RawHeader &Header);
> +  template <class IntT>
> +  IntT swap(IntT Int) const {
> +    return ShouldSwapBytes ? sys::SwapByteOrder(Int) : Int;
> +  }
> +  const uint64_t *getCounter(uint64_t CounterPtr) const {
> +    ptrdiff_t Offset = (swap(CounterPtr) - CountersDelta) /
> sizeof(uint64_t);
> +    return CountersStart + Offset;
> +  }
> +  const char *getName(uint64_t NamePtr) const {
> +    ptrdiff_t Offset = (swap(NamePtr) - NamesDelta) / sizeof(char);
> +    return NamesStart + Offset;
> +  }
> +};
> +
>  } // end namespace llvm
>  
>  #endif // LLVM_PROFILEDATA_INSTRPROF_READER_H__
> 
> Modified: llvm/trunk/lib/ProfileData/InstrProfReader.cpp
> URL:
> http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/ProfileData/InstrProfReader.cpp?rev=204496&r1=204495&r2=204496&view=diff
> ==============================================================================
> --- llvm/trunk/lib/ProfileData/InstrProfReader.cpp (original)
> +++ llvm/trunk/lib/ProfileData/InstrProfReader.cpp Fri Mar 21
> 13:26:05 2014
> @@ -14,12 +14,23 @@
>  
>  #include "llvm/ProfileData/InstrProfReader.h"
>  #include "llvm/ProfileData/InstrProf.h"
> -#include "llvm/Support/Endian.h"
>  
>  #include <cassert>
>  
>  using namespace llvm;
>  
> +static uint64_t getRawMagic() {
> +  return
> +    uint64_t('l') << 56 |
> +    uint64_t('p') << 48 |
> +    uint64_t('r') << 40 |
> +    uint64_t('o') << 32 |
> +    uint64_t('f') << 24 |
> +    uint64_t('r') << 16 |
> +    uint64_t('a') <<  8 |
> +    uint64_t('w');
> +}
> +
>  error_code InstrProfReader::create(std::string Path,
>                                     std::unique_ptr<InstrProfReader>
>                                     &Result) {
>    std::unique_ptr<MemoryBuffer> Buffer;
> @@ -30,10 +41,19 @@ error_code InstrProfReader::create(std::
>    if (Buffer->getBufferSize() >
>    std::numeric_limits<unsigned>::max())
>      return instrprof_error::too_large;
>  
> -  // FIXME: This needs to determine which format the file is and
> construct the
> -  // correct subclass.
> -  Result.reset(new TextInstrProfReader(Buffer));
> +  if (Buffer->getBufferSize() < sizeof(uint64_t)) {
> +    Result.reset(new TextInstrProfReader(Buffer));
> +    Result->readHeader();
> +    return instrprof_error::success;
> +  }
>  
> +  uint64_t Magic = *(uint64_t *)Buffer->getBufferStart();
> +  uint64_t SwappedMagic = sys::SwapByteOrder(Magic);
> +  if (Magic == getRawMagic() || SwappedMagic == getRawMagic())
> +    Result.reset(new RawInstrProfReader(Buffer));
> +  else
> +    Result.reset(new TextInstrProfReader(Buffer));
> +  Result->readHeader();
>    return instrprof_error::success;
>  }
>  
> @@ -82,3 +102,85 @@ error_code TextInstrProfReader::readNext
>  
>    return success();
>  }
> +
> +static uint64_t getRawVersion() {
> +  return 1;
> +}
> +namespace {
> +}
> +RawInstrProfReader::RawInstrProfReader(std::unique_ptr<MemoryBuffer>
> &DataBuffer)
> +    : DataBuffer(DataBuffer.release()) { }
> +
> +error_code RawInstrProfReader::readHeader() {
> +  if (DataBuffer->getBufferSize() < sizeof(RawHeader))
> +    return error(instrprof_error::malformed);
> +  const RawHeader *Header = (RawHeader
> *)DataBuffer->getBufferStart();
> +  if (Header->Magic == getRawMagic())
> +    ShouldSwapBytes = false;
> +  else {
> +    if (sys::SwapByteOrder(Header->Magic) != getRawMagic())
> +      return error(instrprof_error::malformed);
> +
> +    ShouldSwapBytes = true;
> +  }
> +  return readHeader(*Header);
> +}
> +
> +error_code RawInstrProfReader::readHeader(const RawHeader &Header) {
> +  if (swap(Header.Version) != getRawVersion())
> +    return error(instrprof_error::unsupported_version);
> +
> +  CountersDelta = swap(Header.CountersDelta);
> +  NamesDelta = swap(Header.NamesDelta);
> +  auto DataSize = swap(Header.DataSize);
> +  auto CountersSize = swap(Header.CountersSize);
> +  auto NamesSize = swap(Header.NamesSize);
> +
> +  ptrdiff_t DataOffset = sizeof(RawHeader);
> +  ptrdiff_t CountersOffset = DataOffset + sizeof(ProfileData) *
> DataSize;
> +  ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) *
> CountersSize;
> +  size_t FileSize = NamesOffset + sizeof(char) * NamesSize;
> +
> +  if (FileSize != DataBuffer->getBufferSize())
> +    return error(instrprof_error::malformed);
> +
> +  Data = (ProfileData *)(DataBuffer->getBufferStart() + DataOffset);

gcc 4.8.0 complains here:

/src/llvm-trunk-writable/lib/ProfileData/InstrProfReader.cpp: In instantiation of ‘llvm::error_code llvm::RawInstrProfReader<IntPtrT>::readHeader(const llvm::RawInstrProfReader<IntPtrT>::RawHeader&) [with IntPtrT = unsigned int]’:
/src/llvm-trunk-writable/lib/ProfileData/InstrProfReader.cpp:206:16:   required from here
/src/llvm-trunk-writable/lib/ProfileData/InstrProfReader.cpp:162:8: warning: cast from type ‘const char*’ to type ‘llvm::RawInstrProfReader<unsigned int>::ProfileData*’ casts away qualifiers [-Wcast-qual]
   Data = (ProfileData *)(DataBuffer->getBufferStart() + DataOffset);

> +  DataEnd = Data + DataSize;
> +  CountersStart = (uint64_t *)(DataBuffer->getBufferStart() +
> CountersOffset);

And also complains here:

/src/llvm-trunk-writable/lib/ProfileData/InstrProfReader.cpp:164:17: warning: cast from type ‘const char*’ to type ‘uint64_t* {aka long unsigned int*}’ casts away qualifiers [-Wcast-qual]
   CountersStart = (uint64_t *)(DataBuffer->getBufferStart() + CountersOffset);

> +  NamesStart = DataBuffer->getBufferStart() + NamesOffset;
> +
> +  return success();
> +}
> +
> +error_code RawInstrProfReader::readNextRecord(InstrProfRecord
> &Record) {
> +  if (Data == DataEnd)
> +    return error(instrprof_error::eof);
> +
> +  // Get the raw data.
> +  StringRef RawName(getName(Data->NamePtr), swap(Data->NameSize));
> +  auto RawCounts = makeArrayRef(getCounter(Data->CounterPtr),
> +                                swap(Data->NumCounters));
> +
> +  // Check bounds.
> +  if (RawName.data() < NamesStart ||
> +      RawName.data() + RawName.size() > DataBuffer->getBufferEnd()
> ||
> +      RawCounts.data() < CountersStart ||
> +      RawCounts.data() + RawCounts.size() > (uint64_t *)NamesStart)

And also every time you do (uint64_t *)NamesStart:

   warning: cast from type ‘const char*’ to type ‘uint64_t* {aka long unsigned int*}’ casts away qualifiers [-Wcast-qual]

 -Hal

> +    return error(instrprof_error::malformed);
> +
> +  // Store the data in Record, byte-swapping as necessary.
> +  Record.Hash = swap(Data->FuncHash);
> +  Record.Name = RawName;
> +  if (ShouldSwapBytes) {
> +    Counts.clear();
> +    Counts.reserve(RawCounts.size());
> +    for (uint64_t Count : RawCounts)
> +      Counts.push_back(swap(Count));
> +    Record.Counts = Counts;
> +  } else
> +    Record.Counts = RawCounts;
> +
> +  // Iterate.
> +  ++Data;
> +  return success();
> +}
> 
> Added:
> llvm/trunk/test/tools/llvm-profdata/Inputs/binary-compare.profdata
> URL:
> http://llvm.org/viewvc/llvm-project/llvm/trunk/test/tools/llvm-profdata/Inputs/binary-compare.profdata?rev=204496&view=auto
> ==============================================================================
> ---
> llvm/trunk/test/tools/llvm-profdata/Inputs/binary-compare.profdata
> (added)
> +++
> llvm/trunk/test/tools/llvm-profdata/Inputs/binary-compare.profdata
> Fri Mar 21 13:26:05 2014
> @@ -0,0 +1,150 @@
> +simple_loops
> +4
> +4
> +1
> +100
> +100
> +75
> +
> +conditionals
> +11
> +11
> +1
> +100
> +50
> +50
> +33
> +33
> +16
> +99
> +100
> +99
> +100
> +
> +early_exits
> +9
> +9
> +1
> +0
> +51
> +1
> +25
> +1
> +25
> +1
> +0
> +
> +jumps
> +22
> +22
> +1
> +1
> +0
> +1
> +0
> +0
> +1
> +0
> +1
> +2
> +3
> +2
> +0
> +3
> +0
> +1
> +1
> +1
> +10
> +0
> +10
> +9
> +
> +switches
> +19
> +19
> +1
> +1
> +1
> +15
> +7
> +1
> +0
> +2
> +2
> +3
> +3
> +4
> +4
> +0
> +4
> +4
> +5
> +1
> +0
> +
> +big_switch
> +17
> +17
> +1
> +32
> +32
> +1
> +0
> +1
> +1
> +11
> +11
> +1
> +1
> +15
> +15
> +1
> +1
> +2
> +2
> +
> +boolean_operators
> +8
> +8
> +1
> +100
> +34
> +66
> +17
> +34
> +33
> +50
> +
> +boolop_loops
> +9
> +9
> +1
> +50
> +51
> +50
> +26
> +50
> +51
> +50
> +26
> +
> +do_fallthrough
> +4
> +4
> +1
> +10
> +2
> +8
> +
> +main
> +1
> +1
> +1
> +
> +c-general.c:static_func
> +2
> +2
> +1
> +10
> +
> 
> Added: llvm/trunk/test/tools/llvm-profdata/Inputs/binary.profdata
> URL:
> http://llvm.org/viewvc/llvm-project/llvm/trunk/test/tools/llvm-profdata/Inputs/binary.profdata?rev=204496&view=auto
> ==============================================================================
> Binary files
> llvm/trunk/test/tools/llvm-profdata/Inputs/binary.profdata (added)
> and llvm/trunk/test/tools/llvm-profdata/Inputs/binary.profdata Fri
> Mar 21 13:26:05 2014 differ
> 
> Added: llvm/trunk/test/tools/llvm-profdata/binary.test
> URL:
> http://llvm.org/viewvc/llvm-project/llvm/trunk/test/tools/llvm-profdata/binary.test?rev=204496&view=auto
> ==============================================================================
> --- llvm/trunk/test/tools/llvm-profdata/binary.test (added)
> +++ llvm/trunk/test/tools/llvm-profdata/binary.test Fri Mar 21
> 13:26:05 2014
> @@ -0,0 +1,15 @@
> +REGENERATE: You need a checkout of clang with compiler-rt to
> generate the
> +REGENERATE: binary file here.  These shell commands can be used to
> regenerate
> +REGENERATE: it.
> +REGENERATE:
> +REGENERATE: $ SRC=path/to/llvm
> +REGENERATE: $ CFE=$SRC/tools/clang
> +REGENERATE: $ TESTDIR=$SRC/test/tools/llvm-profdata
> +REGENERATE: $ CFE_TESTDIR=$CFE/test/Profile
> +REGENERATE: $ clang -o a.out -fprofile-instr-generate
> $CFE_TESTDIR/test/Profile/c-general.c
> +REGENERATE: $ LLVM_PROFILE_FILE=$TESTDIR/Inputs/binary.profdata
> ./a.out
> +REGENERATE: $ cp $CFE_TESTDIR/Inputs/c-general.profdata
> $TESTDIR/Inputs/binary-compare.profdata
> +
> +RUN: llvm-profdata show %p/Inputs/binary.profdata -o %t1
> +RUN: llvm-profdata show %p/Inputs/binary-compare.profdata -o %t2
> +RUN: diff -up %t1 %t2
> 
> 
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits
> 

-- 
Hal Finkel
Assistant Computational Scientist
Leadership Computing Facility
Argonne National Laboratory




More information about the llvm-commits mailing list