[compiler-rt] [llvm] [MC/DC][Coverage] Enable profile correlation for MC/DC (PR #136437)
Roman Beliaev via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 4 16:32:12 PST 2025
https://github.com/belyaevrd updated https://github.com/llvm/llvm-project/pull/136437
>From 776bfcd076b0c4ee89e3ea77d507496978832970 Mon Sep 17 00:00:00 2001
From: Roman Beliaev <r.beliaev at ispras.ru>
Date: Fri, 5 Dec 2025 02:02:33 +0300
Subject: [PATCH] [MC/DC][Coverage] Enable profile correlation for MC/DC
When using the `-fcoverage-mcdc` option in profile correlation mode MC/DC
coverage is not actually collected.
In binary profile correlation mode this completes elements of a vector of
per-function profile data structures called `Data` with BitmapPtr and
NumBitmapBytes values that are taken from profile data section in the same way
as it is done for Counters.
In debug info correlation mode this adds new `Profile Bitmap Type` DIEs to
DWARFContext. These entries contain FunctionName and NumBitmapBits for
functions. They are used by `correlateProfileDataImpl()` function to obtain
BitmapPtr and NumBitmapBytes values to complete the corresponding elements of
the vector of per-function profile data structures called `Data`. Creating and
reading these new DIEs occur in the same way as it is done for DIEs of the type
`Profile Data Type`.
Fixes #97385
---
.../lib/profile/InstrProfilingWriter.c | 1 +
.../Inputs/instrprof-mcdc-correlation.cpp | 14 +
.../test/profile/instrprof-mcdc-correlation.c | 21 +
.../llvm/ProfileData/InstrProfCorrelator.h | 24 +-
.../llvm/ProfileData/InstrProfReader.h | 3 +-
llvm/lib/ProfileData/InstrProfCorrelator.cpp | 378 +++++++++++++-----
llvm/lib/ProfileData/InstrProfReader.cpp | 2 +-
.../Instrumentation/InstrProfiling.cpp | 33 ++
8 files changed, 365 insertions(+), 111 deletions(-)
create mode 100644 compiler-rt/test/profile/Inputs/instrprof-mcdc-correlation.cpp
create mode 100644 compiler-rt/test/profile/instrprof-mcdc-correlation.c
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 633fdb9661162..4d453bc8d74b4 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -320,6 +320,7 @@ COMPILER_RT_VISIBILITY int lprofWriteDataImpl(
/* The data and names sections are omitted in lightweight mode. */
if (NumData == 0 && NamesSize == 0) {
Header.CountersDelta = 0;
+ Header.BitmapDelta = 0;
Header.NamesDelta = 0;
}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-mcdc-correlation.cpp b/compiler-rt/test/profile/Inputs/instrprof-mcdc-correlation.cpp
new file mode 100644
index 0000000000000..acc63f0375cbd
--- /dev/null
+++ b/compiler-rt/test/profile/Inputs/instrprof-mcdc-correlation.cpp
@@ -0,0 +1,14 @@
+void test(bool a, bool b, bool c, bool d) {
+ if ((a && b) || (c && d))
+ ;
+ if (b && c)
+ ;
+}
+
+int main() {
+ test(true, true, true, true);
+ test(true, true, false, true);
+ test(true, false, true, true);
+ (void)0;
+ return 0;
+}
diff --git a/compiler-rt/test/profile/instrprof-mcdc-correlation.c b/compiler-rt/test/profile/instrprof-mcdc-correlation.c
new file mode 100644
index 0000000000000..e915f27c7b80a
--- /dev/null
+++ b/compiler-rt/test/profile/instrprof-mcdc-correlation.c
@@ -0,0 +1,21 @@
+// REQUIRES: linux || windows
+// Default
+// RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping -fcoverage-mcdc %S/Inputs/instrprof-mcdc-correlation.cpp
+// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal
+// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw
+// RUN: llvm-profdata show --all-functions --counts --text %t.normal.profdata > %t.normal.profdata.show
+
+// With -profile-correlate=binary flag
+// RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -fcoverage-mcdc -mllvm -profile-correlate=binary %S/Inputs/instrprof-mcdc-correlation.cpp
+// RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe
+// RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw
+// RUN: llvm-profdata show --all-functions --counts --text %t-1.profdata > %t-1.profdata.show
+
+// With -profile-correlate=debug-info flag
+// RUN: %clang -o %t-2.exe -fprofile-instr-generate -fcoverage-mapping -fcoverage-mcdc -mllvm -profile-correlate=debug-info -g %S/Inputs/instrprof-mcdc-correlation.cpp
+// RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe
+// RUN: llvm-profdata merge -o %t-2.profdata --debug-info=%t-2.exe %t-2.profraw
+// RUN: llvm-profdata show --all-functions --counts --text %t-2.profdata > %t-2.profdata.show
+
+// RUN: diff %t.normal.profdata.show %t-1.profdata.show
+// RUN: diff %t.normal.profdata.show %t-2.profdata.show
diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
index 1617ae782307c..1310a19d88a8b 100644
--- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
+++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
@@ -69,6 +69,7 @@ class InstrProfCorrelator {
LLVM_ABI static const char *FunctionNameAttributeName;
LLVM_ABI static const char *CFGHashAttributeName;
LLVM_ABI static const char *NumCountersAttributeName;
+ LLVM_ABI static const char *NumBitmapBitsAttributeName;
enum InstrProfCorrelatorKind { CK_32Bit, CK_64Bit };
InstrProfCorrelatorKind getKind() const { return Kind; }
@@ -83,6 +84,9 @@ class InstrProfCorrelator {
/// The address range of the __llvm_prf_cnts section.
uint64_t CountersSectionStart;
uint64_t CountersSectionEnd;
+ /// The address range of the __llvm_prf_bits section.
+ uint64_t BitmapSectionStart;
+ uint64_t BitmapSectionEnd;
/// The pointer points to start/end of profile data/name sections if
/// FileKind is Binary.
const char *DataStart;
@@ -105,7 +109,9 @@ class InstrProfCorrelator {
std::optional<std::string> LinkageName;
yaml::Hex64 CFGHash;
yaml::Hex64 CounterOffset;
+ yaml::Hex64 BitmapOffset;
uint32_t NumCounters;
+ uint32_t NumBitmapBytes;
std::optional<std::string> FilePath;
std::optional<int> LineNumber;
};
@@ -159,8 +165,9 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator {
Error dumpYaml(int MaxWarnings, raw_ostream &OS) override;
void addDataProbe(uint64_t FunctionName, uint64_t CFGHash,
- IntPtrT CounterOffset, IntPtrT FunctionPtr,
- uint32_t NumCounters);
+ IntPtrT CounterOffset, IntPtrT BitmapOffset,
+ IntPtrT FunctionPtr, uint32_t NumCounters,
+ uint32_t NumBitmapBytes);
// Byte-swap the value if necessary.
template <class T> T maybeSwap(T Value) const {
@@ -172,6 +179,7 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator {
std::unique_ptr<InstrProfCorrelator::Context> Ctx)
: InstrProfCorrelator(Kind, std::move(Ctx)){};
llvm::DenseSet<IntPtrT> CounterOffsets;
+ llvm::DenseSet<IntPtrT> BitmapOffsets;
};
/// DwarfInstrProfCorrelator - A child of InstrProfCorrelatorImpl that takes
@@ -191,8 +199,16 @@ class DwarfInstrProfCorrelator : public InstrProfCorrelatorImpl<IntPtrT> {
std::optional<uint64_t> getLocation(const DWARFDie &Die) const;
/// Returns true if the provided DIE symbolizes an instrumentation probe
- /// symbol.
- static bool isDIEOfProbe(const DWARFDie &Die);
+ /// symbol of the necessary type.
+ static bool isDIEOfProbe(const DWARFDie &Die, StringRef Prefix);
+
+ void addCountersToDataProbe(InstrProfCorrelator::CorrelationData *Data,
+ const DWARFDie &Die, const bool UnlimitedWarnings,
+ int &NumSuppressedWarnings);
+
+ void addBitmapToDataProbe(InstrProfCorrelator::CorrelationData *Data,
+ const DWARFDie &Die, const bool UnlimitedWarnings,
+ int &NumSuppressedWarnings);
/// Iterate over DWARF DIEs to find those that symbolize instrumentation
/// probes and construct the ProfileData vector and Names string.
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 134195059f9e6..8fcb455b9ac2f 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -473,7 +473,8 @@ class RawInstrProfReader : public InstrProfReader {
bool atEnd() const { return Data == DataEnd; }
void advanceData() {
- // `CountersDelta` is a constant zero when using debug info correlation.
+ // `CountersDelta` and `BitmapDelta` are constant zero when using debug info
+ // correlation.
if (!Correlator && !BIDFetcherCorrelator) {
// The initial CountersDelta is the in-memory address difference between
// the data and counts sections:
diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
index 65fd5ba1c5ad2..2bd23e741f385 100644
--- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp
+++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
@@ -51,6 +51,7 @@ getInstrProfSection(const object::ObjectFile &Obj, InstrProfSectKind IPSK) {
const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
+const char *InstrProfCorrelator::NumBitmapBitsAttributeName = "Num BitmapBits";
llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
@@ -81,6 +82,17 @@ InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
C->Buffer = std::move(Buffer);
C->CountersSectionStart = CountersSection->getAddress();
C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
+
+ auto BitmapSection = getInstrProfSection(Obj, IPSK_bitmap);
+ if (auto E = BitmapSection.takeError()) {
+ // It is not an error if NumBitmapBytes of each function is zero.
+ consumeError(std::move(E));
+ C->BitmapSectionStart = 0;
+ C->BitmapSectionEnd = 0;
+ } else {
+ C->BitmapSectionStart = BitmapSection->getAddress();
+ C->BitmapSectionEnd = C->BitmapSectionStart + BitmapSection->getSize();
+ }
// In COFF object file, there's a null byte at the beginning of the counter
// section which doesn't exist in raw profile.
if (Obj.getTripleObjectFormat() == Triple::COFF)
@@ -235,6 +247,7 @@ Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData(int MaxWarnings) {
"could not find any profile data metadata in correlated file");
Error Result = correlateProfileNameImpl();
this->CounterOffsets.clear();
+ this->BitmapOffsets.clear();
this->NamesVec.clear();
return Result;
}
@@ -252,7 +265,9 @@ template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
io.mapOptional("Linkage Name", P.LinkageName);
io.mapRequired("CFG Hash", P.CFGHash);
io.mapRequired("Counter Offset", P.CounterOffset);
+ io.mapRequired("Bitmap Offset", P.BitmapOffset);
io.mapRequired("Num Counters", P.NumCounters);
+ io.mapRequired("Num BitmapBytes", P.NumBitmapBytes);
io.mapOptional("File", P.FilePath);
io.mapOptional("Line", P.LineNumber);
}
@@ -277,13 +292,12 @@ Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(int MaxWarnings,
}
template <class IntPtrT>
-void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(uint64_t NameRef,
- uint64_t CFGHash,
- IntPtrT CounterOffset,
- IntPtrT FunctionPtr,
- uint32_t NumCounters) {
+void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(
+ uint64_t NameRef, uint64_t CFGHash, IntPtrT CounterOffset,
+ IntPtrT BitmapOffset, IntPtrT FunctionPtr, uint32_t NumCounters,
+ uint32_t NumBitmapBytes) {
// Check if a probe was already added for this counter offset.
- if (!CounterOffsets.insert(CounterOffset).second)
+ if (CounterOffset && !CounterOffsets.insert(CounterOffset).second)
return;
Data.push_back({
maybeSwap<uint64_t>(NameRef),
@@ -291,15 +305,13 @@ void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(uint64_t NameRef,
// In this mode, CounterPtr actually stores the section relative address
// of the counter.
maybeSwap<IntPtrT>(CounterOffset),
- // TODO: MC/DC is not yet supported.
- /*BitmapOffset=*/maybeSwap<IntPtrT>(0),
+ maybeSwap<IntPtrT>(BitmapOffset),
maybeSwap<IntPtrT>(FunctionPtr),
// TODO: Value profiling is not yet supported.
/*ValuesPtr=*/maybeSwap<IntPtrT>(0),
maybeSwap<uint32_t>(NumCounters),
/*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
- // TODO: MC/DC is not yet supported.
- /*NumBitmapBytes=*/maybeSwap<uint32_t>(0),
+ maybeSwap<uint32_t>(NumBitmapBytes),
});
}
@@ -330,7 +342,8 @@ DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
}
template <class IntPtrT>
-bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
+bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die,
+ StringRef Prefix) {
const auto &ParentDie = Die.getParent();
if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
return false;
@@ -341,109 +354,244 @@ bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
if (!Die.hasChildren())
return false;
if (const char *Name = Die.getName(DINameKind::ShortName))
- return StringRef(Name).starts_with(getInstrProfCountersVarPrefix());
+ return StringRef(Name).starts_with(Prefix);
return false;
}
template <class IntPtrT>
-void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
- int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) {
- bool UnlimitedWarnings = (MaxWarnings == 0);
- // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
- int NumSuppressedWarnings = -MaxWarnings;
- auto MaybeAddProbe = [&](DWARFDie Die) {
- if (!isDIEOfProbe(Die))
- return;
- std::optional<const char *> FunctionName;
- std::optional<uint64_t> CFGHash;
- std::optional<uint64_t> CounterPtr = getLocation(Die);
- auto FnDie = Die.getParent();
- auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
- std::optional<uint64_t> NumCounters;
- for (const DWARFDie &Child : Die.children()) {
- if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
- continue;
- auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
- auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
- if (!AnnotationFormName || !AnnotationFormValue)
- continue;
- auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
- if (auto Err = AnnotationNameOrErr.takeError()) {
- consumeError(std::move(Err));
- continue;
- }
- StringRef AnnotationName = *AnnotationNameOrErr;
- if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {
- if (auto EC =
- AnnotationFormValue->getAsCString().moveInto(FunctionName))
- consumeError(std::move(EC));
- } else if (AnnotationName == InstrProfCorrelator::CFGHashAttributeName) {
- CFGHash = AnnotationFormValue->getAsUnsignedConstant();
- } else if (AnnotationName ==
- InstrProfCorrelator::NumCountersAttributeName) {
- NumCounters = AnnotationFormValue->getAsUnsignedConstant();
- }
+void DwarfInstrProfCorrelator<IntPtrT>::addCountersToDataProbe(
+ InstrProfCorrelator::CorrelationData *Data, const DWARFDie &Die,
+ const bool UnlimitedWarnings, int &NumSuppressedWarnings) {
+ using RawProfData = RawInstrProf::ProfileData<IntPtrT>;
+ std::optional<const char *> FunctionName;
+ std::optional<uint64_t> CFGHash;
+ std::optional<uint64_t> CounterPtr = getLocation(Die);
+ auto FnDie = Die.getParent();
+ auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
+ std::optional<uint64_t> NumCounters;
+ for (const DWARFDie &Child : Die.children()) {
+ if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
+ continue;
+ auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
+ auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
+ if (!AnnotationFormName || !AnnotationFormValue)
+ continue;
+ auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
+ if (auto Err = AnnotationNameOrErr.takeError()) {
+ consumeError(std::move(Err));
+ continue;
}
- // If there is no function and no counter, assume it was dead-stripped
- if (!FunctionPtr && !CounterPtr)
- return;
- if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
- if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
- WithColor::warning()
- << "Incomplete DIE for function " << FunctionName
- << ": CFGHash=" << CFGHash << " CounterPtr=" << CounterPtr
- << " NumCounters=" << NumCounters << "\n";
- LLVM_DEBUG(Die.dump(dbgs()));
+ StringRef AnnotationName = *AnnotationNameOrErr;
+ if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {
+ if (auto EC = AnnotationFormValue->getAsCString().moveInto(FunctionName))
+ consumeError(std::move(EC));
+ } else if (AnnotationName == InstrProfCorrelator::CFGHashAttributeName) {
+ CFGHash = AnnotationFormValue->getAsUnsignedConstant();
+ } else if (AnnotationName ==
+ InstrProfCorrelator::NumCountersAttributeName) {
+ NumCounters = AnnotationFormValue->getAsUnsignedConstant();
+ }
+ }
+ // If there is no function and no counter, assume it was dead-stripped
+ if (!FunctionPtr && !CounterPtr)
+ return;
+ if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning() << "Incomplete DIE for function " << FunctionName
+ << ": CFGHash=" << CFGHash
+ << " CounterPtr=" << CounterPtr
+ << " NumCounters=" << NumCounters << "\n";
+ LLVM_DEBUG(Die.dump(dbgs()));
+ }
+ return;
+ }
+ uint64_t CountersStart = this->Ctx->CountersSectionStart;
+ uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
+ if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning() << format(
+ "CounterPtr out of range for function %s: Actual=0x%x "
+ "Expected=[0x%x, 0x%x)\n",
+ *FunctionName, *CounterPtr, CountersStart, CountersEnd);
+ LLVM_DEBUG(Die.dump(dbgs()));
+ }
+ return;
+ }
+ if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {
+ WithColor::warning() << format("Could not find address of function %s\n",
+ *FunctionName);
+ LLVM_DEBUG(Die.dump(dbgs()));
+ }
+ // In debug info correlation mode, the CounterPtr is an absolute address
+ // of the counter, but it's expected to be relative later when iterating
+ // Data.
+ IntPtrT CounterOffset = *CounterPtr - CountersStart;
+ if (Data) {
+ InstrProfCorrelator::Probe P;
+ P.FunctionName = *FunctionName;
+ if (const char *Name = FnDie.getName(DINameKind::LinkageName))
+ P.LinkageName = Name;
+ P.CFGHash = *CFGHash;
+ P.CounterOffset = CounterOffset;
+ P.BitmapOffset = 0;
+ P.NumCounters = *NumCounters;
+ P.NumBitmapBytes = 0;
+ auto FilePath = FnDie.getDeclFile(
+ DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
+ if (!FilePath.empty())
+ P.FilePath = FilePath;
+ if (auto LineNumber = FnDie.getDeclLine())
+ P.LineNumber = LineNumber;
+ // Try to find appropriate probe.
+ for (InstrProfCorrelator::Probe &Probe : Data->Probes) {
+ if (Probe.FunctionName == *FunctionName) {
+ Probe.LinkageName = P.LinkageName;
+ Probe.CFGHash = P.CFGHash;
+ Probe.CounterOffset = P.CounterOffset;
+ Probe.NumCounters = P.NumCounters;
+ Probe.FilePath = P.FilePath;
+ Probe.LineNumber = P.LineNumber;
+ return;
}
- return;
}
- uint64_t CountersStart = this->Ctx->CountersSectionStart;
- uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
- if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
- if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
- WithColor::warning()
- << format("CounterPtr out of range for function %s: Actual=0x%x "
- "Expected=[0x%x, 0x%x)\n",
- *FunctionName, *CounterPtr, CountersStart, CountersEnd);
- LLVM_DEBUG(Die.dump(dbgs()));
+ Data->Probes.push_back(P);
+ } else {
+ uint64_t NameRef = IndexedInstrProf::ComputeHash(*FunctionName);
+ for (const RawProfData &I : this->Data) {
+ if (I.NameRef == NameRef) {
+ const_cast<uint64_t &>(I.FuncHash) = *CFGHash;
+ const_cast<IntPtrT &>(I.CounterPtr) = CounterOffset;
+ const_cast<IntPtrT &>(I.FunctionPointer) = FunctionPtr.value_or(0);
+ const_cast<uint32_t &>(I.NumCounters) = *NumCounters;
+ return;
}
- return;
}
- if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {
- WithColor::warning() << format("Could not find address of function %s\n",
- *FunctionName);
+ this->addDataProbe(NameRef, *CFGHash, CounterOffset, 0,
+ FunctionPtr.value_or(0), *NumCounters, 0);
+ this->NamesVec.push_back(*FunctionName);
+ }
+}
+
+template <class IntPtrT>
+void DwarfInstrProfCorrelator<IntPtrT>::addBitmapToDataProbe(
+ InstrProfCorrelator::CorrelationData *Data, const DWARFDie &Die,
+ const bool UnlimitedWarnings, int &NumSuppressedWarnings) {
+ using RawProfData = RawInstrProf::ProfileData<IntPtrT>;
+ std::optional<const char *> FunctionName;
+ std::optional<uint64_t> BitmapPtr = getLocation(Die);
+ uint64_t NumBitmapBytes;
+ for (const DWARFDie &Child : Die.children()) {
+ if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
+ continue;
+ auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
+ auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
+ if (!AnnotationFormName || !AnnotationFormValue)
+ continue;
+ auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
+ if (auto Err = AnnotationNameOrErr.takeError()) {
+ consumeError(std::move(Err));
+ continue;
+ }
+ StringRef AnnotationName = *AnnotationNameOrErr;
+ if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {
+ if (auto EC = AnnotationFormValue->getAsCString().moveInto(FunctionName))
+ consumeError(std::move(EC));
+ } else if (AnnotationName ==
+ InstrProfCorrelator::NumBitmapBitsAttributeName) {
+ std::optional<uint64_t> NumBitmapBits =
+ AnnotationFormValue->getAsUnsignedConstant();
+ NumBitmapBytes = alignTo(*NumBitmapBits, CHAR_BIT) / CHAR_BIT;
+ }
+ }
+ if (!FunctionName || !BitmapPtr || !NumBitmapBytes) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning() << "Incomplete DIE for function " << FunctionName
+ << " BitmapPtr=" << BitmapPtr
+ << " NumBitmapBytes=" << NumBitmapBytes << "\n";
LLVM_DEBUG(Die.dump(dbgs()));
}
- // In debug info correlation mode, the CounterPtr is an absolute address of
- // the counter, but it's expected to be relative later when iterating Data.
- IntPtrT CounterOffset = *CounterPtr - CountersStart;
- if (Data) {
- InstrProfCorrelator::Probe P;
- P.FunctionName = *FunctionName;
- if (const char *Name = FnDie.getName(DINameKind::LinkageName))
- P.LinkageName = Name;
- P.CFGHash = *CFGHash;
- P.CounterOffset = CounterOffset;
- P.NumCounters = *NumCounters;
- auto FilePath = FnDie.getDeclFile(
- DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
- if (!FilePath.empty())
- P.FilePath = FilePath;
- if (auto LineNumber = FnDie.getDeclLine())
- P.LineNumber = LineNumber;
- Data->Probes.push_back(P);
- } else {
- this->addDataProbe(IndexedInstrProf::ComputeHash(*FunctionName), *CFGHash,
- CounterOffset, FunctionPtr.value_or(0), *NumCounters);
- this->NamesVec.push_back(*FunctionName);
+ return;
+ }
+ uint64_t BitmapStart = this->Ctx->BitmapSectionStart;
+ uint64_t BitmapEnd = this->Ctx->BitmapSectionEnd;
+ if (!BitmapStart && !BitmapEnd && NumBitmapBytes) {
+ auto E = make_error<InstrProfError>(
+ instrprof_error::unable_to_correlate_profile,
+ "could not find profile bitmap section in correlated file");
+ return;
+ }
+ if (*BitmapPtr < BitmapStart || (*BitmapPtr >= BitmapEnd && NumBitmapBytes)) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning() << format(
+ "BitmapPtr out of range for function %s: Actual=0x%x "
+ "Expected=[0x%x, 0x%x)\n",
+ *FunctionName, *BitmapPtr, BitmapStart, BitmapEnd);
+ LLVM_DEBUG(Die.dump(dbgs()));
}
+ return;
+ }
+ // In debug info correlation mode, the BitmapPtr is an absolute address of
+ // the bitmap, but it's expected to be relative later when iterating Data.
+ IntPtrT BitmapOffset = *BitmapPtr - BitmapStart;
+ // Add bitmap information to the corresponding profile data entry.
+ if (Data) {
+ InstrProfCorrelator::Probe P;
+ P.FunctionName = *FunctionName;
+ P.BitmapOffset = BitmapOffset;
+ P.NumBitmapBytes = NumBitmapBytes;
+ for (InstrProfCorrelator::Probe &Probe : Data->Probes) {
+ if (Probe.FunctionName == *FunctionName) {
+ Probe.BitmapOffset = P.BitmapOffset;
+ Probe.NumBitmapBytes = P.NumBitmapBytes;
+ return;
+ }
+ }
+ Data->Probes.push_back(P);
+ } else {
+ uint64_t NameRef = IndexedInstrProf::ComputeHash(*FunctionName);
+ for (const RawProfData &I : this->Data) {
+ if (I.NameRef == NameRef) {
+ const_cast<IntPtrT &>(I.BitmapPtr) = BitmapOffset;
+ const_cast<uint32_t &>(I.NumBitmapBytes) = NumBitmapBytes;
+ return;
+ }
+ }
+ this->addDataProbe(NameRef, 0, 0, BitmapOffset, 0, 0, NumBitmapBytes);
+ this->NamesVec.push_back(*FunctionName);
+ }
+}
+
+template <class IntPtrT>
+void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
+ int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) {
+ bool UnlimitedWarnings = (MaxWarnings == 0);
+ // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
+ int NumSuppressedWarnings = -MaxWarnings;
+ auto MaybeAddProbe = [&](DWARFDie Die, StringRef Prefix) {
+ if (!isDIEOfProbe(Die, Prefix))
+ return false;
+ if (Prefix == getInstrProfCountersVarPrefix()) {
+ addCountersToDataProbe(Data, Die, UnlimitedWarnings,
+ NumSuppressedWarnings);
+ } else if (Prefix == getInstrProfBitmapVarPrefix()) {
+ addBitmapToDataProbe(Data, Die, UnlimitedWarnings, NumSuppressedWarnings);
+ }
+ return true;
};
- for (auto &CU : DICtx->normal_units())
- for (const auto &Entry : CU->dies())
- MaybeAddProbe(DWARFDie(CU.get(), &Entry));
- for (auto &CU : DICtx->dwo_units())
- for (const auto &Entry : CU->dies())
- MaybeAddProbe(DWARFDie(CU.get(), &Entry));
+ for (auto &CU : DICtx->normal_units()) {
+ for (const auto &Entry : CU->dies()) {
+ DWARFDie Die = DWARFDie(CU.get(), &Entry);
+ if (!MaybeAddProbe(Die, getInstrProfCountersVarPrefix()))
+ MaybeAddProbe(Die, getInstrProfBitmapVarPrefix());
+ }
+ }
+ for (auto &CU : DICtx->dwo_units()) {
+ for (const auto &Entry : CU->dies()) {
+ DWARFDie Die = DWARFDie(CU.get(), &Entry);
+ if (!MaybeAddProbe(Die, getInstrProfCountersVarPrefix()))
+ MaybeAddProbe(Die, getInstrProfBitmapVarPrefix());
+ }
+ }
if (!UnlimitedWarnings && NumSuppressedWarnings > 0)
WithColor::warning() << format("Suppressed %d additional warnings\n",
@@ -487,11 +635,31 @@ void BinaryInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
(I - DataStart) * sizeof(RawProfData));
}
}
- // In binary correlation mode, the CounterPtr is an absolute address of the
- // counter, but it's expected to be relative later when iterating Data.
+ uint64_t BitmapPtr = this->template maybeSwap<IntPtrT>(I->BitmapPtr);
+ uint64_t BitmapStart = this->Ctx->BitmapSectionStart;
+ uint64_t BitmapEnd = this->Ctx->BitmapSectionEnd;
+ if (!BitmapStart && !BitmapEnd && I->NumBitmapBytes) {
+ auto E = make_error<InstrProfError>(
+ instrprof_error::unable_to_correlate_profile,
+ "could not find profile bitmap section in correlated file");
+ return;
+ }
+ if (BitmapPtr < BitmapStart ||
+ (BitmapPtr >= BitmapEnd && I->NumBitmapBytes)) {
+ if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
+ WithColor::warning()
+ << format("BitmapPtr out of range for function: Actual=0x%x "
+ "Expected=[0x%x, 0x%x) at data offset=0x%x\n",
+ BitmapPtr, BitmapStart, BitmapEnd,
+ (I - DataStart) * sizeof(RawProfData));
+ }
+ }
+ // In binary correlation mode, CounterPtr and BitmapPtr are absolute
+ // addresses, but they're expected to be relative later when iterating Data.
IntPtrT CounterOffset = CounterPtr - CountersStart;
- this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset,
- I->FunctionPointer, I->NumCounters);
+ IntPtrT BitmapOffset = BitmapPtr - BitmapStart;
+ this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset, BitmapOffset,
+ I->FunctionPointer, I->NumCounters, I->NumBitmapBytes);
}
}
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index d2ae4b5226ff6..0e7d074252e6d 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -662,7 +662,7 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
// These sizes in the raw file are zero because we constructed them in the
// Correlator.
if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
- NamesDelta == 0))
+ BitmapDelta == 0 && NamesDelta == 0))
return error(instrprof_error::unexpected_correlation_info);
Data = Correlator->getDataPointer();
DataEnd = Data + Correlator->getDataSize();
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 8c8d16a6e3d25..e5c612db5bb2c 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1642,6 +1642,39 @@ InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
PD.RegionBitmaps = BitmapPtr;
PD.NumBitmapBytes = Inc->getNumBitmapBytes();
+
+ if (PD.NumBitmapBytes &&
+ ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) {
+ LLVMContext &Ctx = M.getContext();
+ Function *Fn = Inc->getParent()->getParent();
+ if (auto *SP = Fn->getSubprogram()) {
+ DIBuilder DB(M, true, SP->getUnit());
+ Metadata *FunctionNameAnnotation[] = {
+ MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName),
+ MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)),
+ };
+ Metadata *NumBitmapBitsAnnotation[] = {
+ MDString::get(Ctx, InstrProfCorrelator::NumBitmapBitsAttributeName),
+ ConstantAsMetadata::get(Inc->getNumBitmapBits()),
+ };
+ auto Annotations = DB.getOrCreateArray({
+ MDNode::get(Ctx, FunctionNameAnnotation),
+ MDNode::get(Ctx, NumBitmapBitsAnnotation),
+ });
+ auto *DICounter = DB.createGlobalVariableExpression(
+ SP, BitmapPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
+ /*LineNo=*/0, DB.createUnspecifiedType("Profile Bitmap Type"),
+ BitmapPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
+ /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
+ Annotations);
+ BitmapPtr->addDebugInfo(DICounter);
+ DB.finalize();
+ }
+
+ // Mark the bitmap variable as used so that it isn't optimized out.
+ CompilerUsedVars.push_back(PD.RegionBitmaps);
+ }
+
return PD.RegionBitmaps;
}
More information about the llvm-commits
mailing list