[compiler-rt] [llvm] [MC/DC][Coverage] Enable profile correlation for MC/DC (PR #136437)
Roman Beliaev via llvm-commits
llvm-commits at lists.llvm.org
Sat Apr 19 09:19:02 PDT 2025
https://github.com/belyaevrd created https://github.com/llvm/llvm-project/pull/136437
When using the `-fcoverage-mcdc` option in profile correlation mode MC/DC coverage is not actually collected.
In binary profile correlation mode this completes elements of a vector of per-function profile data structures called `Data` with BitmapPtr and NumBitmapBytes values that are taken from profile data section in the same way as it is done for Counters.
In debug info correlation mode this adds new `Profile Bitmap Type` DIEs to DWARFContext. These entries contain FunctionName and NumBitmapBits for functions. They are used by `correlateProfileDataImpl()` function to obtain BitmapPtr and NumBitmapBytes values to complete the corresponding elements of the vector of per-function profile data structures called `Data`. Creating and reading these new DIEs occur in the same way as it is done for DIEs of the type `Profile Data Type`.
Fixes #97385
>From ab275784a98095810b9c2ac6b86dea04fdf28599 Mon Sep 17 00:00:00 2001
From: Roman Beliaev <r.beliaev at ispras.ru>
Date: Sat, 19 Apr 2025 14:54:29 +0300
Subject: [PATCH] [MC/DC][Coverage] Enable profile correlation for MC/DC
When using the `-fcoverage-mcdc` option in profile correlation mode MC/DC
coverage is not actually collected.
In binary profile correlation mode this completes elements of a vector of
per-function profile data structures called `Data` with BitmapPtr and
NumBitmapBytes values that are taken from profile data section in the same way
as it is done for Counters.
In debug info correlation mode this adds new `Profile Bitmap Type` DIEs to
DWARFContext. These entries contain FunctionName and NumBitmapBits for
functions. They are used by `correlateProfileDataImpl()` function to obtain
BitmapPtr and NumBitmapBytes values to complete the corresponding elements of
the vector of per-function profile data structures called `Data`. Creating and
reading these new DIEs occur in the same way as it is done for DIEs of the type
`Profile Data Type`.
Fixes #97385
---
.../lib/profile/InstrProfilingWriter.c | 1 +
.../Inputs/instrprof-mcdc-correlation.cpp | 14 +
.../test/profile/instrprof-mcdc-correlation.c | 21 ++
.../llvm/ProfileData/InstrProfCorrelator.h | 16 +-
.../llvm/ProfileData/InstrProfReader.h | 3 +-
llvm/lib/ProfileData/InstrProfCorrelator.cpp | 309 +++++++++++++-----
llvm/lib/ProfileData/InstrProfReader.cpp | 2 +-
.../Instrumentation/InstrProfiling.cpp | 34 ++
8 files changed, 304 insertions(+), 96 deletions(-)
create mode 100644 compiler-rt/test/profile/Inputs/instrprof-mcdc-correlation.cpp
create mode 100644 compiler-rt/test/profile/instrprof-mcdc-correlation.c
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 633fdb9661162..4d453bc8d74b4 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -320,6 +320,7 @@ COMPILER_RT_VISIBILITY int lprofWriteDataImpl(
/* The data and names sections are omitted in lightweight mode. */
if (NumData == 0 && NamesSize == 0) {
Header.CountersDelta = 0;
+ Header.BitmapDelta = 0;
Header.NamesDelta = 0;
}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-mcdc-correlation.cpp b/compiler-rt/test/profile/Inputs/instrprof-mcdc-correlation.cpp
new file mode 100644
index 0000000000000..acc63f0375cbd
--- /dev/null
+++ b/compiler-rt/test/profile/Inputs/instrprof-mcdc-correlation.cpp
@@ -0,0 +1,14 @@
+void test(bool a, bool b, bool c, bool d) {
+ if ((a && b) || (c && d))
+ ;
+ if (b && c)
+ ;
+}
+
+int main() {
+ test(true, true, true, true);
+ test(true, true, false, true);
+ test(true, false, true, true);
+ (void)0;
+ return 0;
+}
diff --git a/compiler-rt/test/profile/instrprof-mcdc-correlation.c b/compiler-rt/test/profile/instrprof-mcdc-correlation.c
new file mode 100644
index 0000000000000..e915f27c7b80a
--- /dev/null
+++ b/compiler-rt/test/profile/instrprof-mcdc-correlation.c
@@ -0,0 +1,21 @@
+// REQUIRES: linux || windows
+// Default
+// RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping -fcoverage-mcdc %S/Inputs/instrprof-mcdc-correlation.cpp
+// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal
+// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw
+// RUN: llvm-profdata show --all-functions --counts --text %t.normal.profdata > %t.normal.profdata.show
+
+// With -profile-correlate=binary flag
+// RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -fcoverage-mcdc -mllvm -profile-correlate=binary %S/Inputs/instrprof-mcdc-correlation.cpp
+// RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe
+// RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw
+// RUN: llvm-profdata show --all-functions --counts --text %t-1.profdata > %t-1.profdata.show
+
+// With -profile-correlate=debug-info flag
+// RUN: %clang -o %t-2.exe -fprofile-instr-generate -fcoverage-mapping -fcoverage-mcdc -mllvm -profile-correlate=debug-info -g %S/Inputs/instrprof-mcdc-correlation.cpp
+// RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe
+// RUN: llvm-profdata merge -o %t-2.profdata --debug-info=%t-2.exe %t-2.profraw
+// RUN: llvm-profdata show --all-functions --counts --text %t-2.profdata > %t-2.profdata.show
+
+// RUN: diff %t.normal.profdata.show %t-1.profdata.show
+// RUN: diff %t.normal.profdata.show %t-2.profdata.show
diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
index ed8642495cd74..1c800ae22cd3c 100644
--- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
+++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
@@ -68,6 +68,7 @@ class InstrProfCorrelator {
static const char *FunctionNameAttributeName;
static const char *CFGHashAttributeName;
static const char *NumCountersAttributeName;
+ static const char *NumBitmapBitsAttributeName;
enum InstrProfCorrelatorKind { CK_32Bit, CK_64Bit };
InstrProfCorrelatorKind getKind() const { return Kind; }
@@ -82,6 +83,9 @@ class InstrProfCorrelator {
/// The address range of the __llvm_prf_cnts section.
uint64_t CountersSectionStart;
uint64_t CountersSectionEnd;
+ /// The address range of the __llvm_prf_bits section.
+ uint64_t BitmapSectionStart;
+ uint64_t BitmapSectionEnd;
/// The pointer points to start/end of profile data/name sections if
/// FileKind is Binary.
const char *DataStart;
@@ -104,7 +108,9 @@ class InstrProfCorrelator {
std::optional<std::string> LinkageName;
yaml::Hex64 CFGHash;
yaml::Hex64 CounterOffset;
+ yaml::Hex64 BitmapOffset;
uint32_t NumCounters;
+ uint32_t NumBitmapBytes;
std::optional<std::string> FilePath;
std::optional<int> LineNumber;
};
@@ -158,8 +164,9 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator {
Error dumpYaml(int MaxWarnings, raw_ostream &OS) override;
void addDataProbe(uint64_t FunctionName, uint64_t CFGHash,
- IntPtrT CounterOffset, IntPtrT FunctionPtr,
- uint32_t NumCounters);
+ IntPtrT CounterOffset, IntPtrT BitmapOffset,
+ IntPtrT FunctionPtr, uint32_t NumCounters,
+ uint32_t NumBitmapBytes);
// Byte-swap the value if necessary.
template <class T> T maybeSwap(T Value) const {
@@ -171,6 +178,7 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator {
std::unique_ptr<InstrProfCorrelator::Context> Ctx)
: InstrProfCorrelator(Kind, std::move(Ctx)){};
llvm::DenseSet<IntPtrT> CounterOffsets;
+ llvm::DenseSet<IntPtrT> BitmapOffsets;
};
/// DwarfInstrProfCorrelator - A child of InstrProfCorrelatorImpl that takes
@@ -190,8 +198,8 @@ class DwarfInstrProfCorrelator : public InstrProfCorrelatorImpl<IntPtrT> {
std::optional<uint64_t> getLocation(const DWARFDie &Die) const;
/// Returns true if the provided DIE symbolizes an instrumentation probe
- /// symbol.
- static bool isDIEOfProbe(const DWARFDie &Die);
+ /// symbol of the necessary type.
+ static bool isDIEOfProbe(const DWARFDie &Die, const StringRef &Prefix);
/// Iterate over DWARF DIEs to find those that symbolize instrumentation
/// probes and construct the ProfileData vector and Names string.
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index f1010b312ee56..02f49d08cf29d 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -470,7 +470,8 @@ class RawInstrProfReader : public InstrProfReader {
bool atEnd() const { return Data == DataEnd; }
void advanceData() {
- // `CountersDelta` is a constant zero when using debug info correlation.
+ // `CountersDelta` and `BitmapDelta` are constant zero when using debug info
+ // correlation.
if (!Correlator && !BIDFetcherCorrelator) {
// The initial CountersDelta is the in-memory address difference between
// the data and counts sections:
diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
index d92107f93dc56..7d6d320f92b16 100644
--- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp
+++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
@@ -51,6 +51,7 @@ Expected<object::SectionRef> getInstrProfSection(const object::ObjectFile &Obj,
const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
+const char *InstrProfCorrelator::NumBitmapBitsAttributeName = "Num BitmapBits";
llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
@@ -81,6 +82,17 @@ InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
C->Buffer = std::move(Buffer);
C->CountersSectionStart = CountersSection->getAddress();
C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
+
+ auto BitmapSection = getInstrProfSection(Obj, IPSK_bitmap);
+ if (auto E = BitmapSection.takeError()) {
+ // It is not an error if NumBitmapBytes of each function is zero.
+ consumeError(std::move(E));
+ C->BitmapSectionStart = 0;
+ C->BitmapSectionEnd = 0;
+ } else {
+ C->BitmapSectionStart = BitmapSection->getAddress();
+ C->BitmapSectionEnd = C->BitmapSectionStart + BitmapSection->getSize();
+ }
// In COFF object file, there's a null byte at the beginning of the counter
// section which doesn't exist in raw profile.
if (Obj.getTripleObjectFormat() == Triple::COFF)
@@ -236,6 +248,7 @@ Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData(int MaxWarnings) {
"could not find any profile data metadata in correlated file");
Error Result = correlateProfileNameImpl();
this->CounterOffsets.clear();
+ this->BitmapOffsets.clear();
this->NamesVec.clear();
return Result;
}
@@ -253,7 +266,9 @@ template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
io.mapOptional("Linkage Name", P.LinkageName);
io.mapRequired("CFG Hash", P.CFGHash);
io.mapRequired("Counter Offset", P.CounterOffset);
+ io.mapRequired("Bitmap Offset", P.BitmapOffset);
io.mapRequired("Num Counters", P.NumCounters);
+ io.mapRequired("Num BitmapBytes", P.NumBitmapBytes);
io.mapOptional("File", P.FilePath);
io.mapOptional("Line", P.LineNumber);
}
@@ -281,8 +296,10 @@ template <class IntPtrT>
void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(uint64_t NameRef,
uint64_t CFGHash,
IntPtrT CounterOffset,
+ IntPtrT BitmapOffset,
IntPtrT FunctionPtr,
- uint32_t NumCounters) {
+ uint32_t NumCounters,
+ uint32_t NumBitmapBytes) {
// Check if a probe was already added for this counter offset.
if (!CounterOffsets.insert(CounterOffset).second)
return;
@@ -292,15 +309,13 @@ void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(uint64_t NameRef,
// In this mode, CounterPtr actually stores the section relative address
// of the counter.
maybeSwap<IntPtrT>(CounterOffset),
- // TODO: MC/DC is not yet supported.
- /*BitmapOffset=*/maybeSwap<IntPtrT>(0),
+ maybeSwap<IntPtrT>(BitmapOffset),
maybeSwap<IntPtrT>(FunctionPtr),
// TODO: Value profiling is not yet supported.
/*ValuesPtr=*/maybeSwap<IntPtrT>(0),
maybeSwap<uint32_t>(NumCounters),
/*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
- // TODO: MC/DC is not yet supported.
- /*NumBitmapBytes=*/maybeSwap<uint32_t>(0),
+ maybeSwap<uint32_t>(NumBitmapBytes),
});
}
@@ -331,7 +346,8 @@ DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
}
template <class IntPtrT>
-bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
+bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die,
+ const StringRef &Prefix) {
const auto &ParentDie = Die.getParent();
if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
return false;
@@ -342,106 +358,199 @@ bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
if (!Die.hasChildren())
return false;
if (const char *Name = Die.getName(DINameKind::ShortName))
- return StringRef(Name).starts_with(getInstrProfCountersVarPrefix());
+ return StringRef(Name).starts_with(Prefix);
return false;
}
template <class IntPtrT>
void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) {
+ using RawProfData = RawInstrProf::ProfileData<IntPtrT>;
bool UnlimitedWarnings = (MaxWarnings == 0);
// -N suppressed warnings means we can emit up to N (unsuppressed) warnings
int NumSuppressedWarnings = -MaxWarnings;
- auto maybeAddProbe = [&](DWARFDie Die) {
- if (!isDIEOfProbe(Die))
+ auto maybeAddProbe = [&](DWARFDie Die, const StringRef &Prefix) {
+ if (!isDIEOfProbe(Die, Prefix))
return;
- std::optional<const char *> FunctionName;
- std::optional<uint64_t> CFGHash;
- std::optional<uint64_t> CounterPtr = getLocation(Die);
- auto FnDie = Die.getParent();
- auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
- std::optional<uint64_t> NumCounters;
- for (const DWARFDie &Child : Die.children()) {
- if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
- continue;
- auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
- auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
- if (!AnnotationFormName || !AnnotationFormValue)
- continue;
- auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
- if (auto Err = AnnotationNameOrErr.takeError()) {
- consumeError(std::move(Err));
- continue;
+ if (Prefix == getInstrProfCountersVarPrefix()) {
+ std::optional<const char *> FunctionName;
+ std::optional<uint64_t> CFGHash;
+ std::optional<uint64_t> CounterPtr = getLocation(Die);
+ auto FnDie = Die.getParent();
+ auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
+ std::optional<uint64_t> NumCounters;
+ for (const DWARFDie &Child : Die.children()) {
+ if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
+ continue;
+ auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
+ auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
+ if (!AnnotationFormName || !AnnotationFormValue)
+ continue;
+ auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
+ if (auto Err = AnnotationNameOrErr.takeError()) {
+ consumeError(std::move(Err));
+ continue;
+ }
+ StringRef AnnotationName = *AnnotationNameOrErr;
+ if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {
+ if (auto EC =
+ AnnotationFormValue->getAsCString().moveInto(FunctionName))
+ consumeError(std::move(EC));
+ } else if (AnnotationName ==
+ InstrProfCorrelator::CFGHashAttributeName) {
+ CFGHash = AnnotationFormValue->getAsUnsignedConstant();
+ } else if (AnnotationName ==
+ InstrProfCorrelator::NumCountersAttributeName) {
+ NumCounters = AnnotationFormValue->getAsUnsignedConstant();
+ }
}
- StringRef AnnotationName = *AnnotationNameOrErr;
- if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {
- if (auto EC =
- AnnotationFormValue->getAsCString().moveInto(FunctionName))
- consumeError(std::move(EC));
- } else if (AnnotationName == InstrProfCorrelator::CFGHashAttributeName) {
- CFGHash = AnnotationFormValue->getAsUnsignedConstant();
- } else if (AnnotationName ==
- InstrProfCorrelator::NumCountersAttributeName) {
- NumCounters = AnnotationFormValue->getAsUnsignedConstant();
+ if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning()
+ << "Incomplete DIE for function " << FunctionName
+ << ": CFGHash=" << CFGHash << " CounterPtr=" << CounterPtr
+ << " NumCounters=" << NumCounters << "\n";
+ LLVM_DEBUG(Die.dump(dbgs()));
+ }
+ return;
}
- }
- if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
- if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
- WithColor::warning()
- << "Incomplete DIE for function " << FunctionName
- << ": CFGHash=" << CFGHash << " CounterPtr=" << CounterPtr
- << " NumCounters=" << NumCounters << "\n";
- LLVM_DEBUG(Die.dump(dbgs()));
+ uint64_t CountersStart = this->Ctx->CountersSectionStart;
+ uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
+ if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning()
+ << format("CounterPtr out of range for function %s: Actual=0x%x "
+ "Expected=[0x%x, 0x%x)\n",
+ *FunctionName, *CounterPtr, CountersStart, CountersEnd);
+ LLVM_DEBUG(Die.dump(dbgs()));
+ }
+ return;
}
- return;
- }
- uint64_t CountersStart = this->Ctx->CountersSectionStart;
- uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
- if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
- if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {
WithColor::warning()
- << format("CounterPtr out of range for function %s: Actual=0x%x "
- "Expected=[0x%x, 0x%x)\n",
- *FunctionName, *CounterPtr, CountersStart, CountersEnd);
+ << format("Could not find address of function %s\n", *FunctionName);
LLVM_DEBUG(Die.dump(dbgs()));
}
- return;
- }
- if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {
- WithColor::warning() << format("Could not find address of function %s\n",
- *FunctionName);
- LLVM_DEBUG(Die.dump(dbgs()));
- }
- // In debug info correlation mode, the CounterPtr is an absolute address of
- // the counter, but it's expected to be relative later when iterating Data.
- IntPtrT CounterOffset = *CounterPtr - CountersStart;
- if (Data) {
- InstrProfCorrelator::Probe P;
- P.FunctionName = *FunctionName;
- if (auto Name = FnDie.getName(DINameKind::LinkageName))
- P.LinkageName = Name;
- P.CFGHash = *CFGHash;
- P.CounterOffset = CounterOffset;
- P.NumCounters = *NumCounters;
- auto FilePath = FnDie.getDeclFile(
- DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
- if (!FilePath.empty())
- P.FilePath = FilePath;
- if (auto LineNumber = FnDie.getDeclLine())
- P.LineNumber = LineNumber;
- Data->Probes.push_back(P);
- } else {
- this->addDataProbe(IndexedInstrProf::ComputeHash(*FunctionName), *CFGHash,
- CounterOffset, FunctionPtr.value_or(0), *NumCounters);
- this->NamesVec.push_back(*FunctionName);
+ // In debug info correlation mode, the CounterPtr is an absolute address
+ // of the counter, but it's expected to be relative later when iterating
+ // Data.
+ IntPtrT CounterOffset = *CounterPtr - CountersStart;
+ if (Data) {
+ InstrProfCorrelator::Probe P;
+ P.FunctionName = *FunctionName;
+ if (auto Name = FnDie.getName(DINameKind::LinkageName))
+ P.LinkageName = Name;
+ P.CFGHash = *CFGHash;
+ P.CounterOffset = CounterOffset;
+ P.NumCounters = *NumCounters;
+ auto FilePath = FnDie.getDeclFile(
+ DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
+ if (!FilePath.empty())
+ P.FilePath = FilePath;
+ if (auto LineNumber = FnDie.getDeclLine())
+ P.LineNumber = LineNumber;
+ Data->Probes.push_back(P);
+ } else {
+ this->addDataProbe(IndexedInstrProf::ComputeHash(*FunctionName),
+ *CFGHash, CounterOffset, 0, FunctionPtr.value_or(0),
+ *NumCounters, 0);
+ this->NamesVec.push_back(*FunctionName);
+ }
+ } else if (Prefix == getInstrProfBitmapVarPrefix()) {
+ std::optional<const char *> FunctionName;
+ std::optional<uint64_t> BitmapPtr = getLocation(Die);
+ uint64_t NumBitmapBytes;
+ for (const DWARFDie &Child : Die.children()) {
+ if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
+ continue;
+ auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
+ auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
+ if (!AnnotationFormName || !AnnotationFormValue)
+ continue;
+ auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
+ if (auto Err = AnnotationNameOrErr.takeError()) {
+ consumeError(std::move(Err));
+ continue;
+ }
+ StringRef AnnotationName = *AnnotationNameOrErr;
+ if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {
+ if (auto EC =
+ AnnotationFormValue->getAsCString().moveInto(FunctionName))
+ consumeError(std::move(EC));
+ } else if (AnnotationName ==
+ InstrProfCorrelator::NumBitmapBitsAttributeName) {
+ std::optional<uint64_t> NumBitmapBits =
+ AnnotationFormValue->getAsUnsignedConstant();
+ NumBitmapBytes = alignTo(*NumBitmapBits, CHAR_BIT) / CHAR_BIT;
+ }
+ }
+ if (!FunctionName || !BitmapPtr || !NumBitmapBytes) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning() << "Incomplete DIE for function " << FunctionName
+ << " BitmapPtr=" << BitmapPtr
+ << " NumBitmapBytes=" << NumBitmapBytes << "\n";
+ LLVM_DEBUG(Die.dump(dbgs()));
+ }
+ return;
+ }
+ uint64_t BitmapStart = this->Ctx->BitmapSectionStart;
+ uint64_t BitmapEnd = this->Ctx->BitmapSectionEnd;
+ if (!BitmapStart && !BitmapEnd && NumBitmapBytes) {
+ auto E = make_error<InstrProfError>(
+ instrprof_error::unable_to_correlate_profile,
+ "could not find profile bitmap section in correlated file");
+ return;
+ }
+ if (*BitmapPtr < BitmapStart ||
+ (*BitmapPtr >= BitmapEnd && NumBitmapBytes)) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning()
+ << format("BitmapPtr out of range for function %s: Actual=0x%x "
+ "Expected=[0x%x, 0x%x)\n",
+ *FunctionName, *BitmapPtr, BitmapStart, BitmapEnd);
+ LLVM_DEBUG(Die.dump(dbgs()));
+ }
+ return;
+ }
+ // In debug info correlation mode, the BitmapPtr is an absolute address of
+ // the bitmap, but it's expected to be relative later when iterating Data.
+ IntPtrT BitmapOffset = *BitmapPtr - BitmapStart;
+ // Add bitmap information to the corresponding profile data entry.
+ if (Data) {
+ for (auto P : Data->Probes) {
+ if (P.FunctionName == *FunctionName) {
+ P.BitmapOffset = BitmapOffset;
+ P.NumBitmapBytes = NumBitmapBytes;
+ return;
+ }
+ }
+ } else {
+ for (const RawProfData &I : this->Data) {
+ if (I.NameRef == IndexedInstrProf::ComputeHash(*FunctionName)) {
+ const_cast<IntPtrT &>(I.BitmapPtr) = BitmapOffset;
+ const_cast<uint32_t &>(I.NumBitmapBytes) = NumBitmapBytes;
+ return;
+ }
+ }
+ }
+ WithColor::warning() << format(
+ "Could not find Counters for function %s\n", *FunctionName);
}
};
- for (auto &CU : DICtx->normal_units())
+ for (auto &CU : DICtx->normal_units()) {
for (const auto &Entry : CU->dies())
- maybeAddProbe(DWARFDie(CU.get(), &Entry));
- for (auto &CU : DICtx->dwo_units())
+ maybeAddProbe(DWARFDie(CU.get(), &Entry),
+ getInstrProfCountersVarPrefix());
for (const auto &Entry : CU->dies())
- maybeAddProbe(DWARFDie(CU.get(), &Entry));
+ maybeAddProbe(DWARFDie(CU.get(), &Entry), getInstrProfBitmapVarPrefix());
+ }
+ for (auto &CU : DICtx->dwo_units()) {
+ for (const auto &Entry : CU->dies())
+ maybeAddProbe(DWARFDie(CU.get(), &Entry),
+ getInstrProfCountersVarPrefix());
+ for (const auto &Entry : CU->dies())
+ maybeAddProbe(DWARFDie(CU.get(), &Entry), getInstrProfBitmapVarPrefix());
+ }
if (!UnlimitedWarnings && NumSuppressedWarnings > 0)
WithColor::warning() << format("Suppressed %d additional warnings\n",
@@ -485,11 +594,31 @@ void BinaryInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
(I - DataStart) * sizeof(RawProfData));
}
}
- // In binary correlation mode, the CounterPtr is an absolute address of the
- // counter, but it's expected to be relative later when iterating Data.
+ uint64_t BitmapPtr = this->template maybeSwap<IntPtrT>(I->BitmapPtr);
+ uint64_t BitmapStart = this->Ctx->BitmapSectionStart;
+ uint64_t BitmapEnd = this->Ctx->BitmapSectionEnd;
+ if (!BitmapStart && !BitmapEnd && I->NumBitmapBytes) {
+ auto E = make_error<InstrProfError>(
+ instrprof_error::unable_to_correlate_profile,
+ "could not find profile bitmap section in correlated file");
+ return;
+ }
+ if (BitmapPtr < BitmapStart ||
+ (BitmapPtr >= BitmapEnd && I->NumBitmapBytes)) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning()
+ << format("BitmapPtr out of range for function: Actual=0x%x "
+ "Expected=[0x%x, 0x%x) at data offset=0x%x\n",
+ BitmapPtr, BitmapStart, BitmapEnd,
+ (I - DataStart) * sizeof(RawProfData));
+ }
+ }
+ // In binary correlation mode, CounterPtr and BitmapPtr are absolute
+ // addresses, but they're expected to be relative later when iterating Data.
IntPtrT CounterOffset = CounterPtr - CountersStart;
- this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset,
- I->FunctionPointer, I->NumCounters);
+ IntPtrT BitmapOffset = BitmapPtr - BitmapStart;
+ this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset, BitmapOffset,
+ I->FunctionPointer, I->NumCounters, I->NumBitmapBytes);
}
}
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 4075b513c218d..412242a369dad 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -661,7 +661,7 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
// These sizes in the raw file are zero because we constructed them in the
// Correlator.
if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
- NamesDelta == 0))
+ BitmapDelta == 0 && NamesDelta == 0))
return error(instrprof_error::unexpected_correlation_info);
Data = Correlator->getDataPointer();
DataEnd = Data + Correlator->getDataSize();
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 84bf4c62c7aad..38882ad1ec4d7 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1652,6 +1652,40 @@ InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
PD.RegionBitmaps = BitmapPtr;
PD.NumBitmapBytes = Inc->getNumBitmapBytes();
+
+ if (PD.NumBitmapBytes &&
+ (DebugInfoCorrelate ||
+ ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)) {
+ LLVMContext &Ctx = M.getContext();
+ Function *Fn = Inc->getParent()->getParent();
+ if (auto *SP = Fn->getSubprogram()) {
+ DIBuilder DB(M, true, SP->getUnit());
+ Metadata *FunctionNameAnnotation[] = {
+ MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName),
+ MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)),
+ };
+ Metadata *NumBitmapBitsAnnotation[] = {
+ MDString::get(Ctx, InstrProfCorrelator::NumBitmapBitsAttributeName),
+ ConstantAsMetadata::get(Inc->getNumBitmapBits()),
+ };
+ auto Annotations = DB.getOrCreateArray({
+ MDNode::get(Ctx, FunctionNameAnnotation),
+ MDNode::get(Ctx, NumBitmapBitsAnnotation),
+ });
+ auto *DICounter = DB.createGlobalVariableExpression(
+ SP, BitmapPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
+ /*LineNo=*/0, DB.createUnspecifiedType("Profile Bitmap Type"),
+ BitmapPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
+ /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
+ Annotations);
+ BitmapPtr->addDebugInfo(DICounter);
+ DB.finalize();
+ }
+
+ // Mark the bitmap variable as used so that it isn't optimized out.
+ CompilerUsedVars.push_back(PD.RegionBitmaps);
+ }
+
return PD.RegionBitmaps;
}
More information about the llvm-commits
mailing list