[compiler-rt] [llvm] [MC/DC][Coverage] Introduce "Bitmap Bias" for continuous mode (PR #96126)
NAKAMURA Takumi via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 25 23:21:10 PDT 2024
https://github.com/chapuni updated https://github.com/llvm/llvm-project/pull/96126
>From 5c00b7a552fb956ab8a754e67179e8d813ac1093 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic at gmail.com>
Date: Thu, 20 Jun 2024 08:30:10 +0900
Subject: [PATCH 1/5] [MC/DC][Coverage] Introduce "Bitmap Bias" for continuous
mode
---
compiler-rt/include/profile/InstrProfData.inc | 1 +
compiler-rt/lib/profile/InstrProfilingFile.c | 37 +++++++++++++++++--
llvm/include/llvm/ProfileData/InstrProf.h | 4 ++
.../llvm/ProfileData/InstrProfData.inc | 1 +
.../Instrumentation/InstrProfiling.cpp | 23 +++++++-----
.../Instrumentation/InstrProfiling/mcdc.ll | 24 +++++++-----
6 files changed, 68 insertions(+), 22 deletions(-)
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index e9866d94b762c..6d2df2195c739 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -738,6 +738,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
+#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias
#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
/* The variable that holds the name of the profile data
diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c
index b88e0b4b0b2ab..805e152837a83 100644
--- a/compiler-rt/lib/profile/InstrProfilingFile.c
+++ b/compiler-rt/lib/profile/InstrProfilingFile.c
@@ -199,11 +199,15 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
COMPILER_RT_VISIBILITY intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
+#define INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR \
+ INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR, _default)
+COMPILER_RT_VISIBILITY intptr_t INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR = 0;
/* This variable is a weak external reference which could be used to detect
* whether or not the compiler defined this symbol. */
#if defined(_MSC_VER)
COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
+COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
#if defined(_M_IX86) || defined(__i386__)
#define WIN_SYM_PREFIX "_"
#else
@@ -214,10 +218,17 @@ COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
"/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
+#pragma comment( \
+ linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
+ INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \
+ INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))
#else
COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
__attribute__((weak, alias(INSTR_PROF_QUOTE(
INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
+COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR
+ __attribute__((weak, alias(INSTR_PROF_QUOTE(
+ INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))));
#endif
static const int ContinuousModeSupported = 1;
static const int UseBiasVar = 1;
@@ -228,6 +239,9 @@ static const char *FileOpenMode = "w+b";
* used and runtime provides a weak alias so we can check if it's defined. */
static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
+static void *BitmapBiasAddr = &INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
+static void *BitmapBiasDefaultAddr =
+ &INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR;
static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
/* Get the sizes of various profile data sections. Taken from
* __llvm_profile_get_size_for_buffer(). */
@@ -238,11 +252,18 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
const char *BitmapBegin = __llvm_profile_begin_bitmap();
const char *BitmapEnd = __llvm_profile_end_bitmap();
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+ uint64_t CountersSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+ uint64_t NumBitmapBytes =
+ __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
/* Get the file size. */
uint64_t FileSize = 0;
if (getProfileFileSizeForMerging(File, &FileSize))
return 1;
+ uint64_t PaddingBytesAfterCounters =
+ __llvm_profile_get_num_padding_bytes(CountersSize);
+
/* Map the profile. */
char *Profile = (char *)mmap(NULL, FileSize, PROT_READ | PROT_WRITE,
MAP_SHARED, fileno(File), 0);
@@ -259,7 +280,15 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
/* Return the memory allocated for counters to OS. */
lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
- /* BIAS MODE not supported yet for Bitmap (MCDC). */
+ if (NumBitmapBytes == 0)
+ return 0;
+
+ /* Update profbm_bias. */
+ uint64_t FileOffsetToBitmap =
+ CountersOffsetInBiasMode + CountersSize + PaddingBytesAfterCounters;
+ /* Update the profile fields based on the current mapping. */
+ INSTR_PROF_PROFILE_BITMAP_BIAS_VAR =
+ (intptr_t)Profile - (uintptr_t)BitmapBegin + FileOffsetToBitmap;
/* Return the memory allocated for counters to OS. */
lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd);
@@ -618,8 +647,10 @@ static void initializeProfileForContinuousMode(void) {
PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
return;
}
- if (UseBiasVar && BiasAddr == BiasDefaultAddr) {
- PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined");
+ if (UseBiasVar && BiasAddr == BiasDefaultAddr &&
+ BitmapBiasAddr == BitmapBiasDefaultAddr) {
+ PROF_ERR("%s\n", "Neither __llvm_profile_counter_bias nor "
+ "__llvm_profile_bitmap_bias is defined");
return;
}
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index df79073da5b50..a6461250f7f13 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -174,6 +174,10 @@ inline StringRef getInstrProfCounterBiasVarName() {
return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
}
+inline StringRef getInstrProfBitmapBiasVarName() {
+ return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR);
+}
+
/// Return the marker used to separate PGO names during serialization.
inline StringRef getInstrProfNameSeparator() { return "\01"; }
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index e9866d94b762c..6d2df2195c739 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -738,6 +738,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
+#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias
#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
/* The variable that holds the name of the profile data
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index b9f1fcdd9c233..9879fba86ae24 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -939,18 +939,21 @@ Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
auto *Bitmaps = getOrCreateRegionBitmaps(I);
- IRBuilder<> Builder(I);
+ if (!isRuntimeCounterRelocationEnabled())
+ return Bitmaps;
- if (isRuntimeCounterRelocationEnabled()) {
- LLVMContext &Ctx = M.getContext();
- Ctx.diagnose(DiagnosticInfoPGOProfile(
- M.getName().data(),
- Twine("Runtime counter relocation is presently not supported for MC/DC "
- "bitmaps."),
- DS_Warning));
- }
+ // Put BiasLI onto the entry block.
+ Type *Int64Ty = Type::getInt64Ty(M.getContext());
+ Function *Fn = I->getParent()->getParent();
+ IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
+ auto *Bias = getOrCreateBiasVar(getInstrProfBitmapBiasVarName());
+ auto *BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profbm_bias");
+ BiasLI->setMetadata(LLVMContext::MD_invariant_load,
+ MDNode::get(M.getContext(), std::nullopt));
- return Bitmaps;
+ // Add Bias to Bitmaps and put it before the intrinsic.
+ IRBuilder<> Builder(I);
+ return Builder.CreatePtrAdd(Bitmaps, BiasLI, "profbm_addr");
}
void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
diff --git a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll
index 4980b45f90c50..25e9ed90832b1 100644
--- a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll
@@ -1,21 +1,24 @@
; Check that MC/DC intrinsics are properly lowered
-; RUN: opt < %s -passes=instrprof -S | FileCheck %s
-; RUN: opt < %s -passes=instrprof -runtime-counter-relocation -S 2>&1 | FileCheck %s --check-prefix RELOC
-
-; RELOC: Runtime counter relocation is presently not supported for MC/DC bitmaps
+; RUN: opt < %s -passes=instrprof -S | FileCheck %s --check-prefixes=CHECK,BASIC
+; RUN: opt < %s -passes=instrprof -S -runtime-counter-relocation | FileCheck %s --check-prefixes=CHECK,RELOC
target triple = "x86_64-unknown-linux-gnu"
@__profn_test = private constant [4 x i8] c"test"
-; CHECK: @__profbm_test = private global [1 x i8] zeroinitializer, section "__llvm_prf_bits", comdat, align 1
+; BASIC: [[PROFBM_ADDR:@__profbm_test]] = private global [1 x i8] zeroinitializer, section "__llvm_prf_bits", comdat, align 1
define dso_local void @test(i32 noundef %A) {
entry:
+ ; RELOC: %profbm_bias = load i64, ptr @__llvm_profile_bitmap_bias, align 8, !invariant.load !0
+ ; RELOC: %profc_bias = load i64, ptr @__llvm_profile_counter_bias, align 8
%A.addr = alloca i32, align 4
%mcdc.addr = alloca i32, align 4
call void @llvm.instrprof.cover(ptr @__profn_test, i64 99278, i32 5, i32 0)
- ; CHECK: store i8 0, ptr @__profc_test, align 1
+ ; BASIC: store i8 0, ptr @__profc_test, align 1
+ ; RELOC: %[[PROFC_INTADDR:.+]] = add i64 ptrtoint (ptr @__profc_test to i64), %profc_bias
+ ; RELOC: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_INTADDR]] to ptr
+ ; RELOC: store i8 0, ptr %[[PROFC_ADDR]], align 1
call void @llvm.instrprof.mcdc.parameters(ptr @__profn_test, i64 99278, i32 1)
store i32 0, ptr %mcdc.addr, align 4
@@ -23,16 +26,19 @@ entry:
%tobool = icmp ne i32 %0, 0
call void @llvm.instrprof.mcdc.tvbitmap.update(ptr @__profn_test, i64 99278, i32 0, ptr %mcdc.addr)
+ ; RELOC: [[PROFBM_ADDR:%.+]] = getelementptr i8, ptr @__profbm_test, i64 %profbm_bias
; CHECK: %[[TEMP0:mcdc.*]] = load i32, ptr %mcdc.addr, align 4
; CHECK-NEXT: %[[TEMP:[0-9]+]] = add i32 %[[TEMP0]], 0
; CHECK-NEXT: %[[LAB4:[0-9]+]] = lshr i32 %[[TEMP]], 3
- ; CHECK-NEXT: %[[LAB7:[0-9]+]] = getelementptr inbounds i8, ptr @__profbm_test, i32 %[[LAB4]]
+ ; CHECK-NEXT: %[[LAB7:[0-9]+]] = getelementptr inbounds i8, ptr [[PROFBM_ADDR]], i32 %[[LAB4]]
; CHECK-NEXT: %[[LAB8:[0-9]+]] = and i32 %[[TEMP]], 7
; CHECK-NEXT: %[[LAB9:[0-9]+]] = trunc i32 %[[LAB8]] to i8
; CHECK-NEXT: %[[LAB10:[0-9]+]] = shl i8 1, %[[LAB9]]
; CHECK-NEXT: %[[BITS:mcdc.*]] = load i8, ptr %[[LAB7]], align 1
- ; CHECK-NEXT: %[[LAB11:[0-9]+]] = or i8 %[[BITS]], %[[LAB10]]
- ; CHECK-NEXT: store i8 %[[LAB11]], ptr %[[LAB7]], align 1
+ ; BASIC-NEXT: %[[LAB11:[0-9]+]] = or i8 %[[BITS]], %[[LAB10]]
+ ; RELOC-NEXT: %[[LAB11:[0-9]+]] = or i8 %[[BITS]], %[[LAB10]]
+ ; BASIC-NEXT: store i8 %[[LAB11]], ptr %[[LAB7]], align 1
+ ; RELOC-NEXT: store i8 %[[LAB11]], ptr %[[LAB7]], align 1
ret void
}
>From 63662fd06e243fd66025dffa37ee298798b62c91 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic at gmail.com>
Date: Thu, 20 Jun 2024 23:36:00 +0900
Subject: [PATCH 2/5] Amend __APPLE__
---
compiler-rt/lib/profile/InstrProfilingFile.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c
index c4f9b61221b26..aa329baf64c23 100644
--- a/compiler-rt/lib/profile/InstrProfilingFile.c
+++ b/compiler-rt/lib/profile/InstrProfilingFile.c
@@ -101,6 +101,8 @@ static const int UseBiasVar = 0;
static const char *FileOpenMode = "a+b";
static void *BiasAddr = NULL;
static void *BiasDefaultAddr = NULL;
+static void *BitmapBiasAddr = NULL;
+static void *BitmapBiasDefaultAddr = NULL;
static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
/* Get the sizes of various profile data sections. Taken from
* __llvm_profile_get_size_for_buffer(). */
>From cee86a5e0ac3ea06222403579a5a41b671b2e245 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic at gmail.com>
Date: Thu, 20 Jun 2024 23:39:14 +0900
Subject: [PATCH 3/5] Add comment
---
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 9879fba86ae24..0e4afe9bdd3b6 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -948,6 +948,7 @@ Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
auto *Bias = getOrCreateBiasVar(getInstrProfBitmapBiasVarName());
auto *BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profbm_bias");
+ // Assume BiasLI invariant (in the function at least)
BiasLI->setMetadata(LLVMContext::MD_invariant_load,
MDNode::get(M.getContext(), std::nullopt));
>From d33e2230b2bd2ae363994d461d300469cab6c572 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic at gmail.com>
Date: Fri, 21 Jun 2024 09:43:34 +0900
Subject: [PATCH 4/5] Fixup after the merge
---
compiler-rt/lib/profile/InstrProfilingFile.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c
index 5e6731500bafd..6d5d2ec154231 100644
--- a/compiler-rt/lib/profile/InstrProfilingFile.c
+++ b/compiler-rt/lib/profile/InstrProfilingFile.c
@@ -254,12 +254,18 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
const char *BitmapBegin = __llvm_profile_begin_bitmap();
const char *BitmapEnd = __llvm_profile_end_bitmap();
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+ uint64_t CountersSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+ uint64_t NumBitmapBytes =
+ __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
/* Get the file size. */
uint64_t FileSize = 0;
if (getProfileFileSizeForMerging(File, &FileSize))
return 1;
int Fileno = fileno(File);
+ uint64_t PaddingBytesAfterCounters =
+ __llvm_profile_get_num_padding_bytes(CountersSize);
uint64_t FileOffsetToCounters =
sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize;
>From 1b1067992f4f36cbd7acbfb938be2543b9694edc Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic at gmail.com>
Date: Wed, 26 Jun 2024 13:05:39 +0900
Subject: [PATCH 5/5] Reformat
---
compiler-rt/lib/profile/InstrProfilingFile.c | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c
index 6d5d2ec154231..66668c9bf57fd 100644
--- a/compiler-rt/lib/profile/InstrProfilingFile.c
+++ b/compiler-rt/lib/profile/InstrProfilingFile.c
@@ -216,14 +216,13 @@ COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
#define WIN_SYM_PREFIX
#endif
#pragma comment( \
- linker, \
- "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
- INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
- INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
+ linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
+ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
+ INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
#pragma comment( \
- linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
- INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \
- INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))
+ linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
+ INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \
+ INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))
#else
COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
__attribute__((weak, alias(INSTR_PROF_QUOTE(
More information about the llvm-commits
mailing list