[llvm-branch-commits] [compiler-rt] [llvm] [PGO] Collect GPU wave counts alongside lane counters (PR #225589)
Yaxun Liu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Nov 1 12:28:25 PST 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/225589
>From 52dadd9809727423d72bc0f8d3bb0594c6509756 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 28 Oct 2026 23:43:08 +0000
Subject: [PATCH] [PGO] Collect GPU wave counts alongside lane counters
Lane counts measure how many active lanes reach an instrumentation point.
They do not tell us how many times a wave executes it. For example, 32 lane
visits can come from two visits with 16 active lanes or 32 visits with one
active lane. Uniformity counters cannot distinguish these cases either:
neither has a fully active 32-lane wave.
Add a wave counter for each GPU instrumentation counter. The runtime adds
one per active wave visit, independently of the lane increment's step.
Append wave counters to the existing counter allocation and expose them
separately when reading a profile, so they do not enter lane-flow
reconstruction.
For ordinary AMDGPU IR instrumentation, also measure blocks without a
sparse counter. Preserve the sparse block and select indices, then append
zero-step counters for the remaining blocks that permit instrumentation.
Record this dense layout in the profile version flags so profile use can
reconstruct it without matching a command-line setting.
Extend the raw and indexed profile formats to carry the wave channel.
Preserve it in binary merging, weighting, and scaling; diagnose text or
older-version exports that would discard it. Require matching compiler
and runtime versions for the changed raw layout.
---
compiler-rt/include/profile/InstrProfData.inc | 11 +-
compiler-rt/lib/profile/InstrProfiling.h | 6 +-
compiler-rt/lib/profile/InstrProfilingMerge.c | 1 +
.../lib/profile/InstrProfilingPlatformGPU.c | 5 +-
.../GPU/instrprof-hip-variable-step.hip | 21 ++--
llvm/docs/InstrProfileFormat.md | 49 ++++++++
llvm/include/llvm/ProfileData/InstrProf.h | 17 ++-
.../llvm/ProfileData/InstrProfData.inc | 11 +-
.../llvm/ProfileData/InstrProfReader.h | 6 +
.../llvm/ProfileData/InstrProfWriter.h | 15 ++-
llvm/lib/ProfileData/InstrProf.cpp | 82 +++++++------
llvm/lib/ProfileData/InstrProfCorrelator.cpp | 1 +
llvm/lib/ProfileData/InstrProfReader.cpp | 33 +++++-
llvm/lib/ProfileData/InstrProfWriter.cpp | 40 +++++--
.../Instrumentation/InstrProfiling.cpp | 65 ++++++++---
.../Instrumentation/PGOInstrumentation.cpp | 68 +++++++++--
.../InstrProfiling/amdgpu-3d-grid.ll | 4 +-
.../amdgpu-contiguous-counters.ll | 8 +-
.../InstrProfiling/amdgpu-uniform-counters.ll | 4 +-
.../InstrProfiling/amdgpu-wave32.ll | 6 +-
.../InstrProfiling/amdgpu-wave64.ll | 6 +-
.../InstrProfiling/coverage.ll | 8 +-
.../InstrProfiling/gpu-wave-convergence.ll | 14 +++
.../InstrProfiling/gpu-wave-counts.ll | 38 ++++++
.../InstrProfiling/gpu-wave-inline.ll | 34 ++++++
.../InstrProfiling/gpu-wave-link.ll | 51 +++++++++
.../Transforms/PGOProfile/comdat_internal.ll | 4 +-
.../Transforms/PGOProfile/dense-wave-cfg.ll | 108 ++++++++++++++++++
.../PGOProfile/dense-wave-instrumentation.ll | 59 ++++++++++
.../PGOProfile/dense-wave-profile-use.ll | 82 +++++++++++++
.../Transforms/PGOProfile/gpu-wave-counts.ll | 38 ++++++
.../instrprof_burst_sampling_fast.ll | 2 +-
.../PGOProfile/uniformity-profile-use.ll | 12 +-
.../Transforms/PGOProfile/vtable_profile.ll | 2 +-
.../llvm-profdata/Inputs/c-general.profraw | Bin 2152 -> 2248 bytes
.../llvm-profdata/Inputs/compressed.profraw | Bin 2104 -> 2200 bytes
.../llvm-profdata/binary-ids-padding.test | 8 +-
.../llvm-profdata/dense-wave-layout.test | 43 +++++++
.../tools/llvm-profdata/gpu-wave-counts.test | 50 ++++++++
.../insufficient-binary-ids-size.test | 2 +-
.../llvm-profdata/large-binary-id-size.test | 2 +-
...alformed-not-space-for-another-header.test | 4 +-
.../malformed-num-counters-zero.test | 8 +-
.../malformed-ptr-to-counter-array.test | 4 +-
.../malformed-uniform-counter-array.test | 12 +-
.../misaligned-binary-ids-size.test | 2 +-
llvm/test/tools/llvm-profdata/nocompress.test | 17 ++-
.../tools/llvm-profdata/profile-version.test | 2 +-
.../tools/llvm-profdata/raw-32-bits-be.test | 2 +-
.../tools/llvm-profdata/raw-32-bits-le.test | 2 +-
.../tools/llvm-profdata/raw-64-bits-be.test | 10 +-
.../tools/llvm-profdata/raw-64-bits-le.test | 10 +-
.../tools/llvm-profdata/raw-two-profiles.test | 8 +-
.../llvm-profdata/truncated-profile.test | 6 +-
llvm/tools/llvm-profdata/llvm-profdata.cpp | 47 +++++---
llvm/unittests/ProfileData/InstrProfTest.cpp | 91 +++++++++++++++
56 files changed, 1049 insertions(+), 192 deletions(-)
create mode 100644 llvm/test/Instrumentation/InstrProfiling/gpu-wave-convergence.ll
create mode 100644 llvm/test/Instrumentation/InstrProfiling/gpu-wave-counts.ll
create mode 100644 llvm/test/Instrumentation/InstrProfiling/gpu-wave-inline.ll
create mode 100644 llvm/test/Instrumentation/InstrProfiling/gpu-wave-link.ll
create mode 100644 llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll
create mode 100644 llvm/test/Transforms/PGOProfile/dense-wave-instrumentation.ll
create mode 100644 llvm/test/Transforms/PGOProfile/dense-wave-profile-use.ll
create mode 100644 llvm/test/Transforms/PGOProfile/gpu-wave-counts.ll
create mode 100644 llvm/test/tools/llvm-profdata/dense-wave-layout.test
create mode 100644 llvm/test/tools/llvm-profdata/gpu-wave-counts.test
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index b64026b6798f45..fe8fe9ba1db309 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -99,6 +99,9 @@ INSTR_PROF_DATA(const uint16_t, llvm::Type::getInt16Ty(Ctx), \
OffloadDeviceWaveSizeVal))
INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))
+/* The last NumWaveCounters entries in CounterPtr count GPU wave visits. */
+INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumWaveCounters, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumWaveCounters))
#undef INSTR_PROF_DATA
/* INSTR_PROF_DATA end. */
@@ -773,9 +776,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 11
+#define INSTR_PROF_RAW_VERSION 12
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 14
+#define INSTR_PROF_INDEX_VERSION 15
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 6
@@ -784,7 +787,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
* (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
* generated profile, and 0 if this is a Clang FE generated profile.
* 1 in bit 57 indicates there are context-sensitive records in the profile.
- * The 54th bit indicates whether to always instrument loop entry blocks.
+ * The 54th bit indicates dense GPU wave instrumentation.
+ * The 55th bit indicates whether to always instrument loop entry blocks.
* The 58th bit indicates whether to always instrument function entry blocks.
* The 59th bit indicates whether to use debug info to correlate profiles.
* The 60th bit indicates single byte coverage instrumentation.
@@ -794,6 +798,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
*/
#define VARIANT_MASKS_ALL 0xffffffff00000000ULL
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
+#define VARIANT_MASK_DENSE_WAVE (0x1ULL << 54)
#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index 928b045ab87afc..a1159c6360d0cb 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -144,11 +144,11 @@ void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data,
* \brief Wave-cooperative counter increment for GPU targets.
*
* Reduces per-lane atomic contention by electing a single lane per wave to
- * perform the counter update. \c Uniform is an optional counter tracking the
- * number of uniform.
+ * perform the counter updates. \c Uniform optionally counts lane executions
+ * with a full active mask, and \c Wave counts wave executions.
*/
void INSTR_PROF_INSTRUMENT_GPU_FUNC(uint64_t *Counter, uint64_t *Uniform,
- uint64_t Step);
+ uint64_t Step, uint64_t *Wave);
/*!
* \brief Set the FILE object for writing instrumentation data. Return 0 if set
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index ff84cf8633cf03..58f40e7df509d7 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -90,6 +90,7 @@ int __llvm_profile_check_compatibility(const char *ProfileData,
if (SrcData->NameRef != DstData->NameRef ||
SrcData->FuncHash != DstData->FuncHash ||
SrcData->NumCounters != DstData->NumCounters ||
+ SrcData->NumWaveCounters != DstData->NumWaveCounters ||
SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
return 1;
}
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformGPU.c b/compiler-rt/lib/profile/InstrProfilingPlatformGPU.c
index 6991fe0f5fcfbb..af6d60c354d648 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformGPU.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformGPU.c
@@ -30,9 +30,11 @@ static int is_uniform(uint64_t mask) {
// Wave-cooperative counter increment. The instrumentation pass emits calls to
// this in place of the default non-atomic load/add/store or atomicrmw sequence.
// The optional uniform counter records contributions from full active waves.
+// The wave counter records every wave visit, including visits with zero steps.
COMPILER_RT_VISIBILITY void INSTR_PROF_INSTRUMENT_GPU_FUNC(uint64_t *counter,
uint64_t *uniform,
- uint64_t step) {
+ uint64_t step,
+ uint64_t *wave) {
// Compiler-generated counter steps are zero or one.
uint64_t mask = __gpu_lane_mask();
uint64_t sum = __builtin_popcountg(__gpu_ballot(mask, step));
@@ -42,6 +44,7 @@ COMPILER_RT_VISIBILITY void INSTR_PROF_INSTRUMENT_GPU_FUNC(uint64_t *counter,
if (uniform && is_uniform(mask))
__scoped_atomic_fetch_add(uniform, sum, __ATOMIC_RELAXED,
__MEMORY_SCOPE_DEVICE);
+ __scoped_atomic_fetch_add(wave, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE);
}
}
diff --git a/compiler-rt/test/profile/GPU/instrprof-hip-variable-step.hip b/compiler-rt/test/profile/GPU/instrprof-hip-variable-step.hip
index c8b96212031833..3f0e89b007482c 100644
--- a/compiler-rt/test/profile/GPU/instrprof-hip-variable-step.hip
+++ b/compiler-rt/test/profile/GPU/instrprof-hip-variable-step.hip
@@ -1,7 +1,8 @@
// NOTE: Do not autogenerate. Check runtime counts and application output.
// Exercise the installed GPU profiling helper with zero or one from each
-// active lane. Also check partial/sparse masks and both counter channels
-// across multiple waves. Uniform counters record contributions
+// active lane. Also check partial/sparse masks and all three counter channels
+// across multiple waves. Wave counters count nonempty masks even when all
+// steps are zero. Uniform counters record contributions
// made with a full active mask; they do not classify the step's uniformity.
// REQUIRES: hip, amdgpu
// RUN: rm -rf %t.dir && mkdir -p %t.dir
@@ -19,13 +20,13 @@
#include <hip/hip_runtime.h>
extern "C" __device__ void __llvm_profile_instrument_gpu(uint64_t *, uint64_t *,
- uint64_t);
+ uint64_t, uint64_t *);
__global__ void accumulate(uint64_t *counts, const uint64_t *steps,
const unsigned char *active) {
unsigned i = threadIdx.x;
if (active[i])
- __llvm_profile_instrument_gpu(counts, counts + 1, steps[i]);
+ __llvm_profile_instrument_gpu(counts, counts + 1, steps[i], counts + 2);
}
static void check(hipError_t error) {
@@ -73,14 +74,14 @@ int main() {
return 2;
uint64_t *counts, *steps;
unsigned char *active;
- check(hipMalloc(&counts, 2 * sizeof(uint64_t)));
+ check(hipMalloc(&counts, 3 * sizeof(uint64_t)));
check(hipMalloc(&steps, n * sizeof(uint64_t)));
check(hipMalloc(&active, n));
uint64_t values[256];
unsigned char mask[256];
unsigned errors = 0;
for (unsigned test = 0; test < 11; ++test) {
- uint64_t expected[2] = {};
+ uint64_t expected[3] = {};
for (unsigned wave = 0; wave < 4; ++wave) {
uint64_t sum = 0;
unsigned population = 0;
@@ -97,18 +98,20 @@ int main() {
}
}
expected[0] += sum;
+ if (population != 0)
+ ++expected[2];
if (population == width)
expected[1] += sum;
}
check(
hipMemcpy(steps, values, n * sizeof(uint64_t), hipMemcpyHostToDevice));
check(hipMemcpy(active, mask, n, hipMemcpyHostToDevice));
- check(hipMemset(counts, 0, 2 * sizeof(uint64_t)));
+ check(hipMemset(counts, 0, 3 * sizeof(uint64_t)));
accumulate<<<1, n>>>(counts, steps, active);
check(hipGetLastError());
- uint64_t actual[2];
+ uint64_t actual[3];
check(hipMemcpy(actual, counts, sizeof(actual), hipMemcpyDeviceToHost));
- for (unsigned channel = 0; channel < 2; ++channel) {
+ for (unsigned channel = 0; channel < 3; ++channel) {
if (actual[channel] != expected[channel]) {
++errors;
std::printf("case=%u channel=%u actual=%llu expected=%llu\n", test,
diff --git a/llvm/docs/InstrProfileFormat.md b/llvm/docs/InstrProfileFormat.md
index 3fa1d2c4494150..788953d43e615d 100644
--- a/llvm/docs/InstrProfileFormat.md
+++ b/llvm/docs/InstrProfileFormat.md
@@ -1,5 +1,54 @@
# Instrumentation Profile Format
+## Experimental GPU wave counters
+
+Raw format version 12 appends a `uint32_t NumWaveCounters` field to each
+profile data record. `NumCounters` is the total number of 64-bit lane and
+wave counters addressed by `CounterPtr`; its final `NumWaveCounters` entries
+are wave visits. The uniform-counter array still has only
+`NumCounters - NumWaveCounters` entries. A nonempty wave tail must leave at
+least one lane counter. This changes the raw record layout and requires a
+matching compiler and profiling runtime.
+
+Indexed format version 15 stores a little-endian 64-bit wave-counter count
+followed by that many 64-bit values after the padded uniformity vector and
+before value-profile data. Zero indicates no wave profile. Readers expose
+the tail separately from the lane-flow counter vector. Merging requires
+matching wave-vector lengths; weighted merging and scaling use saturating
+arithmetic, as for ordinary counters. Text and previous-version export of
+wave profiles is currently unsupported and diagnosed.
+
+GPU counter instrumentation always collects wave counts. Each existing
+instrumentation counter has a wave counter at the same index. The GPU runtime
+updates lane, uniformity, and wave counters in one call. Each active lane
+contributes its step to the lane count; one active lane increments the wave
+counter once, independently of those steps.
+Workgroup sampling controls the overhead of all three channels.
+
+Ordinary AMDGPU IR-PGO instrumentation also measures waves in blocks without
+an existing counter. The sparse block counters and select counters retain
+their original indices. Additional block counters follow them in function
+block order, using a zero lane step so that lane-flow reconstruction and
+uniformity annotations retain their original inputs. Blocks without a legal
+insertion point are left unmeasured. Context-sensitive, coverage and temporal
+instrumentation do not use this extension.
+
+`VARIANT_MASK_DENSE_WAVE` (bit 54 in the raw and indexed version fields)
+identifies this layout. The device module supplies the flag to the collector;
+profile readers and writers preserve it through merging. Profile use reads
+the flag to reconstruct the extra indices automatically. Sparse profiles in
+the supported indexed formats remain supported.
+Dense and sparse layouts cannot be merged. The hidden generation option
+`-pgo-instrument-dense-wave-counts=false` selects the legacy sparse layout for
+debugging; its value during profile use does not override the recorded layout.
+
+Wave counts observe the active groups that execute each instrumentation point.
+They do not require full waves or preserve lane grouping across compiler
+transformations. Both sides of a divergent branch may execute once per wave,
+so wave counts must not participate in scalar flow reconstruction. Existing
+lane counts, uniformity classification, and optimization consumers retain
+their previous meaning.
+
## Overview
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 4134f9ddf2b286..431b55bb492c33 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -383,7 +383,9 @@ enum class InstrProfKind {
TemporalProfile = 0x80,
// A profile with loop entry basic blocks instrumentation.
LoopEntriesInstrumentation = 0x100,
- LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/LoopEntriesInstrumentation)
+ // Append zero-step counters to measure waves in otherwise unmeasured blocks.
+ DenseWaveInstrumentation = 0x200,
+ LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/DenseWaveInstrumentation)
};
LLVM_ABI const std::error_category &instrprof_category();
@@ -892,6 +894,8 @@ struct InstrProfValueSiteRecord {
/// Profiling information for a single function.
struct InstrProfRecord {
std::vector<uint64_t> Counts;
+ /// GPU wave visits at each counter index, separate from lane-flow counts.
+ std::vector<uint64_t> WaveCounts;
std::vector<uint8_t> BitmapBytes;
/// For AMDGPU offload profiling: raw or merged uniform counters. One uint64_t
/// per instrumented block, tracking entries where all lanes were active.
@@ -909,8 +913,9 @@ struct InstrProfRecord {
: Counts(std::move(Counts)), BitmapBytes(std::move(BitmapBytes)) {}
InstrProfRecord(InstrProfRecord &&) = default;
InstrProfRecord(const InstrProfRecord &RHS)
- : Counts(RHS.Counts), BitmapBytes(RHS.BitmapBytes),
- UniformCounts(RHS.UniformCounts), UniformityBits(RHS.UniformityBits),
+ : Counts(RHS.Counts), WaveCounts(RHS.WaveCounts),
+ BitmapBytes(RHS.BitmapBytes), UniformCounts(RHS.UniformCounts),
+ UniformityBits(RHS.UniformityBits),
OffloadDeviceWaveSize(RHS.OffloadDeviceWaveSize),
ValueData(RHS.ValueData
? std::make_unique<ValueProfData>(*RHS.ValueData)
@@ -918,6 +923,7 @@ struct InstrProfRecord {
InstrProfRecord &operator=(InstrProfRecord &&) = default;
InstrProfRecord &operator=(const InstrProfRecord &RHS) {
Counts = RHS.Counts;
+ WaveCounts = RHS.WaveCounts;
BitmapBytes = RHS.BitmapBytes;
UniformCounts = RHS.UniformCounts;
UniformityBits = RHS.UniformityBits;
@@ -989,6 +995,7 @@ struct InstrProfRecord {
/// Clear value data entries, edge counters, and uniformity data.
void Clear() {
Counts.clear();
+ WaveCounts.clear();
UniformCounts.clear();
UniformityBits.clear();
OffloadDeviceWaveSize = 0;
@@ -1216,7 +1223,9 @@ enum ProfVersion {
Version13 = 13,
// UniformityBits added for AMDGPU offload profiling divergence detection.
Version14 = 14,
- // The current version is 14.
+ // GPU wave counts added to record data.
+ Version15 = 15,
+ // The current version is 15.
CurrentVersion = INSTR_PROF_INDEX_VERSION
};
const uint64_t Version = ProfVersion::CurrentVersion;
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index b64026b6798f45..fe8fe9ba1db309 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -99,6 +99,9 @@ INSTR_PROF_DATA(const uint16_t, llvm::Type::getInt16Ty(Ctx), \
OffloadDeviceWaveSizeVal))
INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))
+/* The last NumWaveCounters entries in CounterPtr count GPU wave visits. */
+INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumWaveCounters, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumWaveCounters))
#undef INSTR_PROF_DATA
/* INSTR_PROF_DATA end. */
@@ -773,9 +776,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 11
+#define INSTR_PROF_RAW_VERSION 12
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 14
+#define INSTR_PROF_INDEX_VERSION 15
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 6
@@ -784,7 +787,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
* (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
* generated profile, and 0 if this is a Clang FE generated profile.
* 1 in bit 57 indicates there are context-sensitive records in the profile.
- * The 54th bit indicates whether to always instrument loop entry blocks.
+ * The 54th bit indicates dense GPU wave instrumentation.
+ * The 55th bit indicates whether to always instrument loop entry blocks.
* The 58th bit indicates whether to always instrument function entry blocks.
* The 59th bit indicates whether to use debug info to correlate profiles.
* The 60th bit indicates single byte coverage instrumentation.
@@ -794,6 +798,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
*/
#define VARIANT_MASKS_ALL 0xffffffff00000000ULL
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
+#define VARIANT_MASK_DENSE_WAVE (0x1ULL << 54)
#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index a891ef623e24a3..854f34c845a77f 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -144,6 +144,12 @@ class InstrProfReader {
/// individual attributes prefer using the helpers above.
virtual InstrProfKind getProfileKind() const = 0;
+ /// Whether block wave measurements extend the sparse block/select layout.
+ bool hasDenseWaveProfile() const {
+ return static_cast<bool>(getProfileKind() &
+ InstrProfKind::DenseWaveInstrumentation);
+ }
+
/// Return the PGO symtab. There are three different readers:
/// Raw, Text, and Indexed profile readers. The first two types
/// of readers are used only by llvm-profdata tool, while the indexed
diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 73fb9432597cf0..faa6b8922f1c53 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -153,10 +153,10 @@ class InstrProfWriter {
LLVM_ABI Error validateRecord(const InstrProfRecord &Func);
/// Write \c Record in text format to \c OS
- LLVM_ABI static void writeRecordInText(StringRef Name, uint64_t Hash,
- const InstrProfRecord &Counters,
- InstrProfSymtab &Symtab,
- raw_fd_ostream &OS);
+ LLVM_ABI static Error writeRecordInText(StringRef Name, uint64_t Hash,
+ const InstrProfRecord &Counters,
+ InstrProfSymtab &Symtab,
+ raw_fd_ostream &OS);
/// Write the profile, returning the raw data. For testing.
LLVM_ABI std::unique_ptr<MemoryBuffer> writeBuffer();
@@ -201,6 +201,13 @@ class InstrProfWriter {
return make_error<InstrProfError>(
instrprof_error::coverage_count_mismatch);
}
+ if (static_cast<bool>(
+ (ProfileKind & InstrProfKind::DenseWaveInstrumentation) ^
+ (Other & InstrProfKind::DenseWaveInstrumentation))) {
+ return make_error<InstrProfError>(
+ instrprof_error::unsupported_version,
+ "cannot merge dense and sparse wave instrumentation layouts");
+ }
// Now we update the profile type with the bits that are set.
ProfileKind |= Other;
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 2b7dc198735da1..a576f49244c338 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -968,6 +968,23 @@ static void mergeUniformityBits(std::vector<uint8_t> &Dst,
Dst[I] &= Src[I];
}
+static void mergeCounts(MutableArrayRef<uint64_t> Dst, ArrayRef<uint64_t> Src,
+ uint64_t Weight,
+ function_ref<void(instrprof_error)> Warn) {
+ assert(Dst.size() == Src.size() && "counter sizes must match");
+ for (size_t I = 0, E = Src.size(); I < E; ++I) {
+ bool Overflowed;
+ uint64_t Value = SaturatingMultiplyAdd(Src[I], Weight, Dst[I], &Overflowed);
+ if (Value > getInstrMaxCountValue()) {
+ Value = getInstrMaxCountValue();
+ Overflowed = true;
+ }
+ Dst[I] = Value;
+ if (Overflowed)
+ Warn(instrprof_error::counter_overflow);
+ }
+}
+
void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
function_ref<void(instrprof_error)> Warn) {
// If the number of counters doesn't match we either have bad data
@@ -976,6 +993,11 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
Warn(instrprof_error::count_mismatch);
return;
}
+ // Reject partial wave profiles rather than mixing different populations.
+ if (WaveCounts.size() != Other.WaveCounts.size()) {
+ Warn(instrprof_error::count_mismatch);
+ return;
+ }
computeBlockUniformity();
Other.computeBlockUniformity();
@@ -1000,35 +1022,14 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
OffloadDeviceWaveSize = Other.OffloadDeviceWaveSize;
bool HasUniformCounts = !UniformCounts.empty();
bool OtherHasUniformCounts = !Other.UniformCounts.empty();
- for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
- bool Overflowed;
- uint64_t Value =
- SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
- if (Value > getInstrMaxCountValue()) {
- Value = getInstrMaxCountValue();
- Overflowed = true;
- }
- Counts[I] = Value;
- if (Overflowed)
- Warn(instrprof_error::counter_overflow);
- }
+ mergeCounts(Counts, Other.Counts, Weight, Warn);
if (HasUniformCounts && OtherHasUniformCounts) {
if (UniformCounts.size() != Other.UniformCounts.size()) {
UniformCounts.clear();
UniformityBits.clear();
} else {
- for (size_t I = 0, E = Other.UniformCounts.size(); I < E; ++I) {
- bool Overflowed;
- UniformCounts[I] = SaturatingMultiplyAdd(Other.UniformCounts[I], Weight,
- UniformCounts[I], &Overflowed);
- if (UniformCounts[I] > getInstrMaxCountValue()) {
- UniformCounts[I] = getInstrMaxCountValue();
- Overflowed = true;
- }
- if (Overflowed)
- Warn(instrprof_error::counter_overflow);
- }
+ mergeCounts(UniformCounts, Other.UniformCounts, Weight, Warn);
computeBlockUniformity();
}
} else {
@@ -1036,6 +1037,8 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
mergeUniformityBits(UniformityBits, Other.UniformityBits);
}
+ mergeCounts(WaveCounts, Other.WaveCounts, Weight, Warn);
+
// If the number of bitmap bytes doesn't match we either have bad data
// or a hash collision.
if (BitmapBytes.size() != Other.BitmapBytes.size()) {
@@ -1062,25 +1065,17 @@ void InstrProfRecord::scaleValueProfData(
void InstrProfRecord::scale(uint64_t N, uint64_t D,
function_ref<void(instrprof_error)> Warn) {
assert(D != 0 && "D cannot be 0");
- for (auto &Count : this->Counts) {
- bool Overflowed;
- Count = SaturatingMultiply(Count, N, &Overflowed) / D;
- if (Count > getInstrMaxCountValue()) {
- Count = getInstrMaxCountValue();
- Overflowed = true;
- }
- if (Overflowed)
- Warn(instrprof_error::counter_overflow);
- }
- for (auto &Count : this->UniformCounts) {
- bool Overflowed;
- Count = SaturatingMultiply(Count, N, &Overflowed) / D;
- if (Count > getInstrMaxCountValue()) {
- Count = getInstrMaxCountValue();
- Overflowed = true;
+ for (auto *Counters : {&Counts, &UniformCounts, &WaveCounts}) {
+ for (auto &Count : *Counters) {
+ bool Overflowed;
+ Count = SaturatingMultiply(Count, N, &Overflowed) / D;
+ if (Count > getInstrMaxCountValue()) {
+ Count = getInstrMaxCountValue();
+ Overflowed = true;
+ }
+ if (Overflowed)
+ Warn(instrprof_error::counter_overflow);
}
- if (Overflowed)
- Warn(instrprof_error::counter_overflow);
}
computeBlockUniformity();
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
@@ -1745,7 +1740,7 @@ Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
IndexedInstrProf::ProfVersion::CurrentVersion)
return make_error<InstrProfError>(instrprof_error::unsupported_version);
- static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version14,
+ static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version15,
"Please update the reader as needed when a new field is added "
"or when indexed profile version gets bumped.");
@@ -1778,10 +1773,11 @@ size_t Header::size() const {
// of the header, and byte offset of existing fields shouldn't change when
// indexed profile version gets incremented.
static_assert(
- IndexedInstrProf::ProfVersion::CurrentVersion == Version14,
+ IndexedInstrProf::ProfVersion::CurrentVersion == Version15,
"Please update the size computation below if a new field has "
"been added to the header; for a version bump without new "
"fields, add a case statement to fall through to the latest version.");
+ case 15ull: // Wave counts added in record data, no header change
case 14ull: // UniformityBits added in record data, no header change
case 13ull:
case 12ull:
diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
index 76281fcf4bfe9c..9b3d77b00c0867 100644
--- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp
+++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
@@ -339,6 +339,7 @@ void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(
/*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
/*OffloadDeviceWaveSize=*/maybeSwap<uint16_t>(0),
maybeSwap<uint32_t>(NumBitmapBytes),
+ /*NumWaveCounters=*/maybeSwap<uint32_t>(0),
});
}
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index db8e3c4fe695ba..99cd5ffd86c3f8 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -56,6 +56,9 @@ static InstrProfKind getProfileKindFromVersion(uint64_t Version) {
if (Version & VARIANT_MASK_INSTR_LOOP_ENTRIES) {
ProfileKind |= InstrProfKind::LoopEntriesInstrumentation;
}
+ if (Version & VARIANT_MASK_DENSE_WAVE) {
+ ProfileKind |= InstrProfKind::DenseWaveInstrumentation;
+ }
if (Version & VARIANT_MASK_BYTE_COVERAGE) {
ProfileKind |= InstrProfKind::SingleByteCoverage;
}
@@ -268,6 +271,8 @@ Error TextInstrProfReader::readHeader() {
ProfileKind &= ~InstrProfKind::FunctionEntryInstrumentation;
else if (Str.equals_insensitive("instrument_loop_entries"))
ProfileKind |= InstrProfKind::LoopEntriesInstrumentation;
+ else if (Str.equals_insensitive("dense_wave"))
+ ProfileKind |= InstrProfKind::DenseWaveInstrumentation;
else if (Str.equals_insensitive("single_byte_coverage"))
ProfileKind |= InstrProfKind::SingleByteCoverage;
else if (Str.equals_insensitive("temporal_prof_traces")) {
@@ -550,6 +555,9 @@ Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
// There's another profile to read, so we need to process the header.
auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos);
+ if ((Version ^ swap(Header->Version)) & VARIANT_MASK_DENSE_WAVE)
+ return error(instrprof_error::unsupported_version,
+ "cannot merge dense and sparse wave instrumentation layouts");
return readHeader(*Header);
}
@@ -878,7 +886,7 @@ Error RawInstrProfReader<IntPtrT>::readRawUniformCounters(
if (UniformCountersStart == UniformCountersEnd)
return success();
- uint32_t NumCounters = swap(Data->NumCounters);
+ uint32_t NumCounters = swap(Data->NumCounters) - swap(Data->NumWaveCounters);
ptrdiff_t UniformCounterOffset =
swap(Data->UniformCounterPtr) - UniformCountersDelta;
@@ -968,6 +976,14 @@ Error RawInstrProfReader<IntPtrT>::readNextRecord(NamedInstrProfRecord &Record)
if (Error E = readRawCounts(Record))
return error(std::move(E));
+ uint32_t NumWaveCounters = swap(Data->NumWaveCounters);
+ if (NumWaveCounters && NumWaveCounters >= Record.Counts.size())
+ return error(instrprof_error::malformed, "invalid number of wave counters");
+ size_t NumLaneCounters = Record.Counts.size() - NumWaveCounters;
+ Record.WaveCounts.assign(Record.Counts.begin() + NumLaneCounters,
+ Record.Counts.end());
+ Record.Counts.resize(NumLaneCounters);
+
// Read raw bitmap bytes and set Record.
if (Error E = readRawBitmapBytes(Record))
return error(std::move(E));
@@ -1119,6 +1135,21 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
std::move(BitmapByteBuffer),
std::move(UniformityBitsBuffer));
+ if (GET_VERSION(FormatVersion) >=
+ IndexedInstrProf::ProfVersion::Version15) {
+ if (End - D < ptrdiff_t(sizeof(uint64_t)))
+ return data_type();
+ uint64_t NumWaveCounters =
+ endian::readNext<uint64_t, llvm::endianness::little>(D);
+ if (NumWaveCounters > uint64_t(End - D) / sizeof(uint64_t))
+ return data_type();
+ auto &WaveCounts = DataBuffer.back().WaveCounts;
+ WaveCounts.reserve(NumWaveCounters);
+ for (uint64_t I = 0; I < NumWaveCounters; ++I)
+ WaveCounts.push_back(
+ endian::readNext<uint64_t, llvm::endianness::little>(D));
+ }
+
// Read value profiling data.
if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
!readValueProfilingData(D, End)) {
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index cd78265f5534f0..932c7e6633e208 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -83,6 +83,7 @@ class InstrProfRecordWriterTrait {
M += alignTo(ProfRecord.BitmapBytes.size(), sizeof(uint64_t));
M += sizeof(uint64_t); // The size of the UniformityBits vector
M += alignTo(ProfRecord.UniformityBits.size(), sizeof(uint64_t));
+ M += sizeof(uint64_t) * (1 + ProfRecord.WaveCounts.size());
}
// Value data
@@ -135,6 +136,10 @@ class InstrProfRecordWriterTrait {
I < alignTo(ProfRecord.UniformityBits.size(), sizeof(uint64_t));
++I)
LE.write<uint8_t>(0);
+
+ LE.write<uint64_t>(ProfRecord.WaveCounts.size());
+ for (uint64_t Count : ProfRecord.WaveCounts)
+ LE.write<uint64_t>(Count);
}
// Write value data
@@ -539,6 +544,16 @@ Error InstrProfWriter::writeVTableNames(ProfOStream &OS) {
}
Error InstrProfWriter::writeImpl(ProfOStream &OS) {
+ if (WritePrevVersion) {
+ for (const auto &Function : FunctionData) {
+ for (const auto &Record : Function.getValue()) {
+ if (!Record.second.WaveCounts.empty())
+ return make_error<InstrProfError>(
+ instrprof_error::unsupported_version,
+ "wave counts require indexed profile version 15");
+ }
+ }
+ }
using namespace IndexedInstrProf;
using namespace support;
@@ -567,7 +582,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
// The WritePrevVersion handling will either need to be removed or updated
// if the version is advanced beyond 12.
static_assert(IndexedInstrProf::ProfVersion::CurrentVersion ==
- IndexedInstrProf::ProfVersion::Version14);
+ IndexedInstrProf::ProfVersion::Version15);
if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
Header.Version |= VARIANT_MASK_IR_PROF;
if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
@@ -578,6 +593,8 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
if (static_cast<bool>(ProfileKind &
InstrProfKind::LoopEntriesInstrumentation))
Header.Version |= VARIANT_MASK_INSTR_LOOP_ENTRIES;
+ if (static_cast<bool>(ProfileKind & InstrProfKind::DenseWaveInstrumentation))
+ Header.Version |= VARIANT_MASK_DENSE_WAVE;
if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
@@ -730,10 +747,15 @@ Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
return Error::success();
}
-void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
- const InstrProfRecord &Func,
- InstrProfSymtab &Symtab,
- raw_fd_ostream &OS) {
+Error InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
+ const InstrProfRecord &Func,
+ InstrProfSymtab &Symtab,
+ raw_fd_ostream &OS) {
+ if (!Func.WaveCounts.empty())
+ return make_error<InstrProfError>(
+ instrprof_error::unsupported_version,
+ "text profiles do not support wave counts");
+
OS << Name << "\n";
OS << "# Func Hash:\n" << Hash << "\n";
OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
@@ -755,7 +777,7 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
uint32_t NumValueKinds = Func.getNumValueKinds();
if (!NumValueKinds) {
OS << "\n";
- return;
+ return Error::success();
}
OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
@@ -779,6 +801,7 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
}
OS << "\n";
+ return Error::success();
}
Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
@@ -797,6 +820,8 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
"blocks\n:instrument_loop_entries\n";
if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
OS << "# Instrument block coverage\n:single_byte_coverage\n";
+ if (static_cast<bool>(ProfileKind & InstrProfKind::DenseWaveInstrumentation))
+ OS << "# Dense GPU block wave instrumentation\n:dense_wave\n";
InstrProfSymtab Symtab;
using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
@@ -827,7 +852,8 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
for (const auto &record : OrderedFuncData) {
const StringRef &Name = record.first;
const FuncPair &Func = record.second;
- writeRecordInText(Name, Func.first, Func.second, Symtab, OS);
+ if (Error E = writeRecordInText(Name, Func.first, Func.second, Symtab, OS))
+ return E;
}
for (const auto &record : OrderedFuncData) {
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index fa9f2c1bcffe4b..c9b4a8bc560f73 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -178,8 +178,15 @@ class InstrLowerer final {
GlobalVariable *DataVar = nullptr;
GlobalVariable *RegionBitmaps = nullptr;
uint32_t NumBitmapBytes = 0;
+ // GPU region counters contain lane counts followed by one wave count for
+ // each lane-counter index. Other targets have no wave counters.
+ uint32_t NumWaveCounters = 0;
PerFunctionProfileData() = default;
+
+ uint64_t getNumRegionCounters() const {
+ return cast<ArrayType>(RegionCounters->getValueType())->getNumElements();
+ }
};
DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
// Key is virtual table variable, value is 'VTableProfData' in the form of
@@ -937,6 +944,10 @@ bool InstrLowerer::lower() {
InstrProfCntrInstBase *FirstProfInst = nullptr;
for (BasicBlock &BB : F) {
for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
+ if (isGPUProfTarget(M) &&
+ (isa<InstrProfCoverInst>(I) || isa<InstrProfTimestampInst>(I)))
+ report_fatal_error("wave counts require ordinary counter increments",
+ false);
if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
computeNumValueSiteCounts(Ind);
else {
@@ -1221,14 +1232,13 @@ void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
auto *PtrTy = PointerType::getUnqual(Context);
auto *Addr = getCounterAddress(Inc);
+ auto &PD = ProfileDataMap[Inc->getName()];
// Store the device wave/warp size into the profile data struct once per
// function. AMDGPU folds llvm.amdgcn.wavefrontsize to the subtarget's
// constant; other GPUs use their fixed warp size.
if (!Inv.WaveSizeStored) {
Inv.WaveSizeStored = true;
- GlobalVariable *NamePtr = Inc->getName();
- auto &PD = ProfileDataMap[NamePtr];
if (PD.DataVar) {
IRBuilder<> EntryBuilder(&*F->getEntryBlock().getFirstInsertionPt());
Value *WaveSize16 = nullptr;
@@ -1254,22 +1264,26 @@ void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
}
}
- GlobalVariable *UniformCounters = getOrCreateUniformCounters(Inc);
- Value *UniformAddrArg = ConstantPointerNull::get(PtrTy);
- if (UniformCounters) {
- Value *UniformIndices[] = {Builder.getInt32(0), Inc->getIndex()};
- Value *UniformAddr = Builder.CreateInBoundsGEP(
- UniformCounters->getValueType(), UniformCounters, UniformIndices,
- "unifctr.addr");
- UniformAddrArg =
- Builder.CreatePointerBitCastOrAddrSpaceCast(UniformAddr, PtrTy);
- }
+ // getCounterAddress creates the lane, uniform, and wave counters together.
+ GlobalVariable *UniformCounters = PD.UniformCounters;
+ Value *UniformIndices[] = {Builder.getInt32(0), Inc->getIndex()};
+ Value *UniformAddr = Builder.CreateInBoundsGEP(
+ UniformCounters->getValueType(), UniformCounters, UniformIndices,
+ "unifctr.addr");
+ Value *UniformAddrArg =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(UniformAddr, PtrTy);
Value *CastAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PtrTy);
Value *StepI64 =
Builder.CreateZExtOrTrunc(Inc->getStep(), Int64Ty, "step.i64");
+ uint32_t WaveOffset = PD.getNumRegionCounters() - PD.NumWaveCounters;
+ uint32_t WaveIndex = WaveOffset + Inc->getIndex()->getZExtValue();
+ Value *WaveAddr = Builder.CreateConstInBoundsGEP2_32(
+ PD.RegionCounters->getValueType(), PD.RegionCounters, 0, WaveIndex);
+ WaveAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(WaveAddr, PtrTy);
+
auto *CalleeTy = FunctionType::get(Type::getVoidTy(Context),
- {PtrTy, PtrTy, Int64Ty}, false);
+ {PtrTy, PtrTy, Int64Ty, PtrTy}, false);
FunctionCallee Callee =
M.getOrInsertFunction(RTLIB::RuntimeLibcallsInfo::getLibcallImplName(
RTLIB::impl___llvm_profile_instrument_gpu),
@@ -1286,10 +1300,11 @@ void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
HeadBuilder.CreateCondBr(Inv.Matched, ThenBB, ContBB);
IRBuilder<> ThenBuilder(ThenBB);
- ThenBuilder.CreateCall(Callee, {CastAddr, UniformAddrArg, StepI64});
+ ThenBuilder.CreateCall(Callee,
+ {CastAddr, UniformAddrArg, StepI64, WaveAddr});
ThenBuilder.CreateBr(ContBB);
} else {
- Builder.CreateCall(Callee, {CastAddr, UniformAddrArg, StepI64});
+ Builder.CreateCall(Callee, {CastAddr, UniformAddrArg, StepI64, WaveAddr});
}
Inc->eraseFromParent();
return;
@@ -1801,7 +1816,8 @@ InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
GlobalVariable *
InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
GlobalValue::LinkageTypes Linkage) {
- uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
+ uint64_t NumCounters = Inc->getNumCounters()->getZExtValue() +
+ ProfileDataMap[Inc->getName()].NumWaveCounters;
auto &Ctx = M.getContext();
GlobalVariable *GV;
if (isa<InstrProfCoverInst>(Inc)) {
@@ -1830,6 +1846,20 @@ InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
if (PD.RegionCounters)
return PD.RegionCounters;
+ if (isGPUProfTarget(M)) {
+ if (!isa<InstrProfIncrementInst>(Inc) || IsCS)
+ report_fatal_error("wave counts require ordinary counter increments",
+ false);
+ if (Opts.profile_correlate || isSamplingEnabled())
+ report_fatal_error("wave counts do not support profile correlation "
+ "or lane-level instrumentation sampling",
+ false);
+ uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
+ if (NumCounters > UINT32_MAX / 2)
+ report_fatal_error("too many wave profiling counters", false);
+ PD.NumWaveCounters = NumCounters;
+ }
+
// If RegionCounters doesn't already exist, create it by first setting up
// the corresponding profile section.
auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
@@ -1980,7 +2010,8 @@ void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
ValuesVar, PointerType::get(Fn->getContext(), 0));
}
- uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
+ uint32_t NumWaveCounters = PD.NumWaveCounters;
+ uint64_t NumCounters = PD.getNumRegionCounters();
Constant *CounterPtr = PD.RegionCounters;
Constant *UniformCounterPtr = PD.UniformCounters;
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 2a43b994c56223..0e4c3eac6b7682 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -173,6 +173,11 @@ cl::opt<bool>
cl::desc("Use this option to turn off/on "
"warnings about profile cfg mismatch."));
+static cl::opt<bool> PGOInstrumentDenseWaveCounts(
+ "pgo-instrument-dense-wave-counts", cl::Hidden, cl::init(true),
+ cl::desc("Measure wave counts in all eligible AMDGPU blocks during IR PGO "
+ "instrumentation"));
+
// Command line option to enable vtable value profiling. Defined in
// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
extern cl::opt<bool> EnableVTableValueProfiling;
@@ -275,6 +280,15 @@ static const char *ValueProfKindDescr[] = {
#include "llvm/ProfileData/InstrProfData.inc"
};
+static bool
+shouldInstrumentDenseWaves(const InstrumentationOptions &Opts, const Module &M,
+ PGOInstrumentationType InstrumentationType) {
+ return PGOInstrumentDenseWaveCounts && M.getTargetTriple().isAMDGPU() &&
+ InstrumentationType == PGOInstrumentationType::FDO &&
+ !Opts.pgo_function_entry_coverage && !Opts.pgo_block_coverage &&
+ !Opts.pgo_temporal_instrumentation;
+}
+
// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
// aware this is an ir_level profile so it can set the version flag.
static GlobalVariable *
@@ -290,6 +304,8 @@ createIRLevelProfileFlagVar(const InstrumentationOptions &Opts, Module &M,
ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
if (valueOr(Opts.pgo_instrument_loop_entries, false))
ProfileVersion |= VARIANT_MASK_INSTR_LOOP_ENTRIES;
+ if (shouldInstrumentDenseWaves(Opts, M, InstrumentationType))
+ ProfileVersion |= VARIANT_MASK_DENSE_WAVE;
if (Opts.profile_correlate == ProfCorrelatorKind::DebugInfo)
ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
if (Opts.pgo_function_entry_coverage)
@@ -763,6 +779,17 @@ populateEHOperandBundle(VPCandidateInfo &Cand,
}
}
+static SmallVector<BasicBlock *>
+getExtraWaveBlocks(Function &F, ArrayRef<BasicBlock *> InstrumentBBs) {
+ SmallVector<BasicBlock *> Extra;
+ SmallPtrSet<BasicBlock *, 32> Measured(InstrumentBBs.begin(),
+ InstrumentBBs.end());
+ for (BasicBlock &BB : F)
+ if (!Measured.contains(&BB) && BB.getFirstNonPHIOrAlloca() != BB.end())
+ Extra.push_back(&BB);
+ return Extra;
+}
+
// Visit all edge and instrument the edges not in MST, and do value profiling.
// Critical edges will be split.
void FunctionInstrumenter::instrument() {
@@ -799,8 +826,12 @@ void FunctionInstrumenter::instrument() {
std::vector<BasicBlock *> InstrumentBBs;
FuncInfo.getInstrumentBBs(InstrumentBBs);
- unsigned NumCounters =
- InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
+ SmallVector<BasicBlock *> ExtraWaveBBs;
+ if (shouldInstrumentDenseWaves(Opts, M, InstrumentationType))
+ ExtraWaveBBs = getExtraWaveBlocks(F, InstrumentBBs);
+ unsigned NumCounters = InstrumentBBs.size() +
+ FuncInfo.SIVisitor.getNumOfSelectInsts() +
+ ExtraWaveBBs.size();
if (IsCtxProf) {
StringSet<> SkipCSInstr(llvm::from_range,
@@ -873,6 +904,15 @@ void FunctionInstrumenter::instrument() {
// Now instrument select instructions:
FuncInfo.SIVisitor.instrumentSelects(&I, NumCounters, Name,
FuncInfo.FunctionHash);
+ // Preserve the sparse block/select prefix. A zero lane step still records
+ // one wave visit in the GPU runtime, without affecting lane-flow counts.
+ for (BasicBlock *BB : ExtraWaveBBs) {
+ IRBuilder<> Builder(BB->getFirstNonPHIOrAlloca());
+ Builder.CreateIntrinsic(Intrinsic::instrprof_increment_step,
+ {NormalizedNamePtr, CFGHash,
+ Builder.getInt32(NumCounters),
+ Builder.getInt32(I++), Builder.getInt64(0)});
+ }
assert(I == NumCounters);
if (isValueProfilingDisabled())
@@ -1003,12 +1043,15 @@ class PGOUseFunc {
BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFIin,
LoopInfo *LI, ProfileSummaryInfo *PSI, bool IsCS,
bool InstrumentFuncEntry, bool InstrumentLoopEntries,
- bool HasSingleByteCoverage)
+ bool HasSingleByteCoverage, bool HasDenseWaveProfile)
: Opts(Opts), F(Func), M(Modu), BFI(BFIin), PSI(PSI),
FuncInfo(Opts, Func, TLI, ComdatMembers, false, BPI, BFIin, LI, IsCS,
InstrumentFuncEntry, InstrumentLoopEntries,
HasSingleByteCoverage),
- FreqAttr(FFA_Normal), IsCS(IsCS), VPC(Func, TLI) {}
+ FreqAttr(FFA_Normal), IsCS(IsCS),
+ HasDenseWaveProfile(HasDenseWaveProfile && !IsCS &&
+ Modu->getTargetTriple().isAMDGPU()),
+ VPC(Func, TLI) {}
void handleInstrProfError(Error Err, uint64_t MismatchedFuncSum);
@@ -1099,6 +1142,9 @@ class PGOUseFunc {
// Is to use the context sensitive profile.
bool IsCS;
+ // The input profile owns its counter layout, independently of gen options.
+ bool HasDenseWaveProfile;
+
ValueProfileCollector VPC;
// Find the Instrumented BB and set the value. Return false on error.
@@ -1152,12 +1198,16 @@ bool PGOUseFunc::setInstrumentedCounts(
unsigned NumInstrumentedBBs = InstrumentBBs.size();
unsigned NumSelects = FuncInfo.SIVisitor.getNumOfSelectInsts();
unsigned NumCounters = NumInstrumentedBBs + NumSelects;
+ SmallVector<BasicBlock *> ExtraWaveBBs;
+ if (HasDenseWaveProfile)
+ ExtraWaveBBs = getExtraWaveBlocks(F, InstrumentBBs);
// The number of counters here should match the number of counters
// in profile. Return if they mismatch.
- if (NumCounters != CountFromProfile.size()) {
+ if (NumCounters + ExtraWaveBBs.size() != CountFromProfile.size()) {
LLVM_DEBUG({
dbgs() << "PGO COUNTER MISMATCH for function " << F.getName() << ":\n";
- dbgs() << " Expected counters: " << NumCounters << "\n";
+ dbgs() << " Expected counters: " << NumCounters + ExtraWaveBBs.size()
+ << "\n";
dbgs() << " - From instrumented edges: " << NumInstrumentedBBs << "\n";
for (size_t i = 0; i < InstrumentBBs.size(); ++i) {
dbgs() << " " << i << ": ";
@@ -1184,7 +1234,9 @@ bool PGOUseFunc::setInstrumentedCounts(
CountValue = 1;
Info.setBBInfoCount(CountValue);
}
- ProfileCountSize = CountFromProfile.size();
+ // The appended zero-step slots do not participate in lane-flow or select
+ // reconstruction. Uniformity also retains its original sparse indices.
+ ProfileCountSize = NumCounters;
CountPosition = I;
// Set the edge count and update the count of unknown edges for BBs.
@@ -2159,7 +2211,7 @@ static bool annotateAllFunctions(
}
PGOUseFunc Func(Opts, F, &M, TLI, ComdatMembers, BPI, BFI, LI, PSI, IsCS,
InstrumentFuncEntry, InstrumentLoopEntries,
- HasSingleByteCoverage);
+ HasSingleByteCoverage, PGOReader->hasDenseWaveProfile());
if (!Func.getRecord(PGOReader.get()))
continue;
if (HasSingleByteCoverage) {
diff --git a/llvm/test/Instrumentation/InstrProfiling/amdgpu-3d-grid.ll b/llvm/test/Instrumentation/InstrProfiling/amdgpu-3d-grid.ll
index 11f6c39ba3a3d0..eda2a0f5f65249 100644
--- a/llvm/test/Instrumentation/InstrProfiling/amdgpu-3d-grid.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/amdgpu-3d-grid.ll
@@ -15,9 +15,9 @@ define amdgpu_kernel void @kernel_3d() {
declare void @llvm.instrprof.increment(ptr, i64, i32, i32)
;; Per-function comdat counters (3D grid linearization is handled in the runtime library)
-; CHECK: @__profc_kernel_3d = linkonce_odr protected addrspace(1) global [1 x i64]
+; CHECK: @__profc_kernel_3d = linkonce_odr protected addrspace(1) global [2 x i64]
; CHECK: @__llvm_prf_unifcnt_kernel_3d = linkonce_odr protected addrspace(1) global [1 x i64]
;; Check sampling guard calls library function
; CHECK: call i32 @__llvm_profile_sampling_gpu(i32 3)
-; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) @__profc_kernel_3d to ptr), ptr addrspacecast (ptr addrspace(1) @__llvm_prf_unifcnt_kernel_3d to ptr), i64 1)
+; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) @__profc_kernel_3d to ptr), ptr addrspacecast (ptr addrspace(1) @__llvm_prf_unifcnt_kernel_3d to ptr), i64 1, ptr addrspacecast (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @__profc_kernel_3d, i64 8) to ptr))
diff --git a/llvm/test/Instrumentation/InstrProfiling/amdgpu-contiguous-counters.ll b/llvm/test/Instrumentation/InstrProfiling/amdgpu-contiguous-counters.ll
index eb503e35271cc4..dc4733153496f3 100644
--- a/llvm/test/Instrumentation/InstrProfiling/amdgpu-contiguous-counters.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/amdgpu-contiguous-counters.ll
@@ -1,4 +1,4 @@
-;; Per-kernel __profc_* arrays land in section __llvm_prf_cnts with one slot
+;; Per-kernel __profc_* arrays land in section __llvm_prf_cnts with lane and wave slots
;; per counter, and a parallel __llvm_prf_unifcnt_* array lands in
;; __llvm_prf_ucnts. Counter increments lower to __llvm_profile_instrument_gpu
;; calls whose first pointer argument is a GEP into the per-kernel counter
@@ -10,9 +10,9 @@
@__profn_kernel1 = private constant [7 x i8] c"kernel1"
@__profn_kernel2 = private constant [7 x i8] c"kernel2"
-; CHECK: @__profc_kernel1 = linkonce_odr protected addrspace(1) global [2 x i64] zeroinitializer, section "__llvm_prf_cnts"
+; CHECK: @__profc_kernel1 = linkonce_odr protected addrspace(1) global [4 x i64] zeroinitializer, section "__llvm_prf_cnts"
; CHECK: @__llvm_prf_unifcnt_kernel1 = linkonce_odr protected addrspace(1) global [2 x i64] zeroinitializer, section "__llvm_prf_ucnts"
-; CHECK: @__profc_kernel2 = linkonce_odr protected addrspace(1) global [1 x i64] zeroinitializer, section "__llvm_prf_cnts"
+; CHECK: @__profc_kernel2 = linkonce_odr protected addrspace(1) global [2 x i64] zeroinitializer, section "__llvm_prf_cnts"
define amdgpu_kernel void @kernel1() {
call void @llvm.instrprof.increment(ptr @__profn_kernel1, i64 12345, i32 2, i32 0)
@@ -29,4 +29,4 @@ declare void @llvm.instrprof.increment(ptr, i64, i32, i32)
;; Second counter slot uses a GEP into the per-kernel counter array and the
;; matching uniform-counter array.
-; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @__profc_kernel1, i64 8) to ptr), ptr addrspacecast (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @__llvm_prf_unifcnt_kernel1, i64 8) to ptr), i64 1)
+; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @__profc_kernel1, i64 8) to ptr), ptr addrspacecast (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @__llvm_prf_unifcnt_kernel1, i64 8) to ptr), i64 1, ptr addrspacecast (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @__profc_kernel1, i64 24) to ptr))
diff --git a/llvm/test/Instrumentation/InstrProfiling/amdgpu-uniform-counters.ll b/llvm/test/Instrumentation/InstrProfiling/amdgpu-uniform-counters.ll
index ae1164157c271e..063edeffa88bd6 100644
--- a/llvm/test/Instrumentation/InstrProfiling/amdgpu-uniform-counters.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/amdgpu-uniform-counters.ll
@@ -15,8 +15,8 @@ define amdgpu_kernel void @test_kernel() {
declare void @llvm.instrprof.increment(ptr, i64, i32, i32)
;; Per-function counter + uniform-counter globals (comdat)
-; CHECK: @__profc_test_kernel = linkonce_odr protected addrspace(1) global [1 x i64]
+; CHECK: @__profc_test_kernel = linkonce_odr protected addrspace(1) global [2 x i64]
; CHECK: @__llvm_prf_unifcnt_test_kernel = linkonce_odr protected addrspace(1) global [1 x i64]
;; __llvm_profile_instrument_gpu receives counter and uniform-counter bases
-; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) @__profc_test_kernel to ptr), ptr addrspacecast (ptr addrspace(1) @__llvm_prf_unifcnt_test_kernel to ptr), i64 1)
+; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) @__profc_test_kernel to ptr), ptr addrspacecast (ptr addrspace(1) @__llvm_prf_unifcnt_test_kernel to ptr), i64 1, ptr addrspacecast (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @__profc_test_kernel, i64 8) to ptr))
diff --git a/llvm/test/Instrumentation/InstrProfiling/amdgpu-wave32.ll b/llvm/test/Instrumentation/InstrProfiling/amdgpu-wave32.ll
index 3a1653ff46c484..e62864ee1d95aa 100644
--- a/llvm/test/Instrumentation/InstrProfiling/amdgpu-wave32.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/amdgpu-wave32.ll
@@ -17,9 +17,9 @@ declare void @llvm.instrprof.increment(ptr, i64, i32, i32)
attributes #0 = { "target-cpu"="gfx1100" }
;; Per-function comdat counters + uniform counters
-; CHECK: @__profc_kernel_w32 = linkonce_odr protected addrspace(1) global [1 x i64]
+; CHECK: @__profc_kernel_w32 = linkonce_odr protected addrspace(1) global [2 x i64]
; CHECK: @__llvm_prf_unifcnt_kernel_w32 = linkonce_odr protected addrspace(1) global [1 x i64]
-; CHECK: @__profd_kernel_w32 = linkonce_odr protected addrspace(1) global { {{.*}} i16 0, i32 0 }
+; CHECK: @__profd_kernel_w32 = linkonce_odr protected addrspace(1) global { {{.*}} i16 0, i32 0, i32 1 }
;; Check wave size stored via intrinsic
; CHECK: %wavesize.i16 = trunc i32 %{{.*}} to i16
@@ -32,4 +32,4 @@ attributes #0 = { "target-cpu"="gfx1100" }
;; Check library call
; CHECK: po_then:
-; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) @__profc_kernel_w32 to ptr), ptr addrspacecast (ptr addrspace(1) @__llvm_prf_unifcnt_kernel_w32 to ptr), i64 1)
+; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) @__profc_kernel_w32 to ptr), ptr addrspacecast (ptr addrspace(1) @__llvm_prf_unifcnt_kernel_w32 to ptr), i64 1, ptr addrspacecast (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @__profc_kernel_w32, i64 8) to ptr))
diff --git a/llvm/test/Instrumentation/InstrProfiling/amdgpu-wave64.ll b/llvm/test/Instrumentation/InstrProfiling/amdgpu-wave64.ll
index 3979653df1f40f..1650eaeaea3e12 100644
--- a/llvm/test/Instrumentation/InstrProfiling/amdgpu-wave64.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/amdgpu-wave64.ll
@@ -16,9 +16,9 @@ declare void @llvm.instrprof.increment(ptr, i64, i32, i32)
attributes #0 = { "target-cpu"="gfx908" }
;; Per-function comdat counters
-; CHECK: @__profc_kernel_w64 = linkonce_odr protected addrspace(1) global [1 x i64]
+; CHECK: @__profc_kernel_w64 = linkonce_odr protected addrspace(1) global [2 x i64]
; CHECK: @__llvm_prf_unifcnt_kernel_w64 = linkonce_odr protected addrspace(1) global [1 x i64]
-; CHECK: @__profd_kernel_w64 = linkonce_odr protected addrspace(1) global { {{.*}} i16 0, i32 0 }
+; CHECK: @__profd_kernel_w64 = linkonce_odr protected addrspace(1) global { {{.*}} i16 0, i32 0, i32 1 }
;; Check wave size stored via intrinsic
; CHECK: %wavesize.i16 = trunc i32 %{{.*}} to i16
@@ -31,4 +31,4 @@ attributes #0 = { "target-cpu"="gfx908" }
;; Check library call
; CHECK: po_then:
-; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) @__profc_kernel_w64 to ptr), ptr addrspacecast (ptr addrspace(1) @__llvm_prf_unifcnt_kernel_w64 to ptr), i64 1)
+; CHECK: call void @__llvm_profile_instrument_gpu(ptr addrspacecast (ptr addrspace(1) @__profc_kernel_w64 to ptr), ptr addrspacecast (ptr addrspace(1) @__llvm_prf_unifcnt_kernel_w64 to ptr), i64 1, ptr addrspacecast (ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @__profc_kernel_w64, i64 8) to ptr))
diff --git a/llvm/test/Instrumentation/InstrProfiling/coverage.ll b/llvm/test/Instrumentation/InstrProfiling/coverage.ll
index 695a8829fdf750..6bfb17b6565953 100644
--- a/llvm/test/Instrumentation/InstrProfiling/coverage.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/coverage.ll
@@ -5,12 +5,12 @@ target triple = "aarch64-unknown-linux-gnu"
@__profn_foo = private constant [3 x i8] c"foo"
; CHECK: @__profc_foo = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1
-; CHECK: @__profd_foo = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64)
-; BINARY: @__profd_foo = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64),
+; CHECK: @__profd_foo = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32, i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64)
+; BINARY: @__profd_foo = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32, i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64),
@__profn_bar = private constant [3 x i8] c"bar"
; CHECK: @__profc_bar = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1
-; CHECK: @__profd_bar = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64)
-; BINARY: @__profd_bar = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64),
+; CHECK: @__profd_bar = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32, i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64)
+; BINARY: @__profd_bar = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32, i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64),
; CHECK: @__llvm_prf_nm = {{.*}} section "__llvm_prf_names"
; BINARY: @__llvm_prf_nm ={{.*}} section "__llvm_covnames"
diff --git a/llvm/test/Instrumentation/InstrProfiling/gpu-wave-convergence.ll b/llvm/test/Instrumentation/InstrProfiling/gpu-wave-convergence.ll
new file mode 100644
index 00000000000000..d3919b5949ccbf
--- /dev/null
+++ b/llvm/test/Instrumentation/InstrProfiling/gpu-wave-convergence.ll
@@ -0,0 +1,14 @@
+; RUN: opt -passes='pgo-instr-gen,instrprof,verify' -offload-pgo-sampling=0 -S %s | FileCheck %s
+
+; Wave collection introduces no additional convergent operation.
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK-LABEL: define amdgpu_kernel void @controlled_kernel
+; CHECK: call void @__llvm_profile_instrument_gpu(
+; CHECK: call token @llvm.experimental.convergence.entry()
+; CHECK: ret void
+define amdgpu_kernel void @controlled_kernel() convergent {
+entry:
+ %token = call token @llvm.experimental.convergence.entry()
+ ret void
+}
diff --git a/llvm/test/Instrumentation/InstrProfiling/gpu-wave-counts.ll b/llvm/test/Instrumentation/InstrProfiling/gpu-wave-counts.ll
new file mode 100644
index 00000000000000..8cd63c1cac1e07
--- /dev/null
+++ b/llvm/test/Instrumentation/InstrProfiling/gpu-wave-counts.ll
@@ -0,0 +1,38 @@
+; RUN: opt -passes=instrprof -offload-pgo-sampling=0 -S %s | FileCheck %s
+; RUN: opt -passes=instrprof -offload-pgo-sampling=3 -S %s | FileCheck %s --check-prefix=SAMPLE
+; RUN: not opt -passes=instrprof -profile-correlate=debug-info -disable-output %s 2>&1 | FileCheck %s --check-prefix=ERROR
+; RUN: not opt -passes=instrprof -sampled-instrumentation -disable-output %s 2>&1 | FileCheck %s --check-prefix=ERROR
+; ERROR: wave counts do not support profile correlation or lane-level instrumentation sampling
+
+target triple = "amdgcn-amd-amdhsa"
+ at __profn_test = private constant [4 x i8] c"test"
+
+; CHECK: @__profc_test = {{.*}}[4 x i64] zeroinitializer
+; CHECK: @__llvm_prf_unifcnt_test = {{.*}}[2 x i64] zeroinitializer
+; CHECK: @__profd_test = {{.*}}i32 4, [3 x i16] zeroinitializer, i16 0, i32 0, i32 2 }
+; CHECK-LABEL: define amdgpu_kernel void @test
+; CHECK: call void @__llvm_profile_instrument_gpu(ptr {{.*}}, ptr {{.*}}, i64 1, ptr {{.*}}i64 16
+; CHECK-NEXT: br i1 %cond, label %a, label %b
+; CHECK: a:
+; CHECK-NEXT: call void @__llvm_profile_instrument_gpu(ptr {{.*}}, ptr {{.*}}, i64 1, ptr {{.*}}i64 24
+; CHECK-NEXT: br label %exit
+; CHECK: b:
+; CHECK-NEXT: br label %exit
+; CHECK: exit:
+; CHECK-NEXT: ret void
+; SAMPLE: call i32 @__llvm_profile_sampling_gpu(i32 3)
+; SAMPLE: br i1
+; SAMPLE: call void @__llvm_profile_instrument_gpu(ptr {{.*}}, ptr {{.*}}, i64 1, ptr {{.*}}i64 16
+; SAMPLE: call void @__llvm_profile_instrument_gpu(ptr {{.*}}, ptr {{.*}}, i64 1, ptr {{.*}}i64 24
+define amdgpu_kernel void @test(i1 %cond) {
+entry:
+ call void @llvm.instrprof.increment(ptr @__profn_test, i64 123, i32 2, i32 0)
+ br i1 %cond, label %a, label %b
+a:
+ call void @llvm.instrprof.increment(ptr @__profn_test, i64 123, i32 2, i32 1)
+ br label %exit
+b:
+ br label %exit
+exit:
+ ret void
+}
diff --git a/llvm/test/Instrumentation/InstrProfiling/gpu-wave-inline.ll b/llvm/test/Instrumentation/InstrProfiling/gpu-wave-inline.ll
new file mode 100644
index 00000000000000..1fc7d023da9650
--- /dev/null
+++ b/llvm/test/Instrumentation/InstrProfiling/gpu-wave-inline.ll
@@ -0,0 +1,34 @@
+; RUN: opt -passes='pgo-instr-gen,always-inline,instrprof,verify' -offload-pgo-sampling=0 -S %s | FileCheck %s
+
+; Inlining leaves several profile names in each caller. Each name needs its own
+; lane, uniform, and wave counters, even after the callee definition is removed.
+source_filename = "wave-inline.c"
+target triple = "amdgcn-amd-amdhsa"
+
+; CHECK-DAG: @__profc_caller = {{.*}}[2 x i64] zeroinitializer
+; CHECK-DAG: @__profc_other = {{.*}}[2 x i64] zeroinitializer
+; CHECK-DAG: @[[CALLEE:__profc_.*callee]] = {{.*}}[2 x i64] zeroinitializer
+; CHECK-DAG: @__llvm_prf_unifcnt_{{.*}}callee = {{.*}}[1 x i64] zeroinitializer
+; CHECK-DAG: @__profd_{{.*}}callee = {{.*}}i32 2, [3 x i16] zeroinitializer, i16 0, i32 0, i32 1 }
+
+; CHECK-LABEL: define i32 @caller(
+; CHECK: call void @__llvm_profile_instrument_gpu({{.*}}i64 1, ptr {{.*}}@__profc_caller{{.*}}i64 8
+; CHECK: call void @__llvm_profile_instrument_gpu({{.*}}i64 1, ptr {{.*}}@[[CALLEE]]{{.*}}i64 8
+define i32 @caller(i32 %x) {
+ %v = call i32 @callee(i32 %x)
+ ret i32 %v
+}
+
+; CHECK-LABEL: define i32 @other(
+; CHECK: call void @__llvm_profile_instrument_gpu({{.*}}i64 1, ptr {{.*}}@__profc_other{{.*}}i64 8
+; CHECK: call void @__llvm_profile_instrument_gpu({{.*}}i64 1, ptr {{.*}}@[[CALLEE]]{{.*}}i64 8
+; CHECK-NOT: define {{.*}}@callee(
+define i32 @other(i32 %x) {
+ %v = call i32 @callee(i32 %x)
+ ret i32 %v
+}
+
+define internal i32 @callee(i32 %x) alwaysinline {
+ %v = add i32 %x, 1
+ ret i32 %v
+}
diff --git a/llvm/test/Instrumentation/InstrProfiling/gpu-wave-link.ll b/llvm/test/Instrumentation/InstrProfiling/gpu-wave-link.ll
new file mode 100644
index 00000000000000..5ec73d99c2481d
--- /dev/null
+++ b/llvm/test/Instrumentation/InstrProfiling/gpu-wave-link.ll
@@ -0,0 +1,51 @@
+; RUN: split-file %s %t
+; RUN: opt -passes='pgo-instr-gen,instrprof,always-inline,globaldce,verify' -offload-pgo-sampling=0 %t/a.ll -o %t/a.bc
+; RUN: opt -passes='pgo-instr-gen,instrprof,always-inline,globaldce,verify' -offload-pgo-sampling=0 %t/b.ll -o %t/b.bc
+; RUN: llvm-link -S %t/a.bc %t/b.bc | FileCheck %s --check-prefixes=CHECK,AB
+; RUN: llvm-link -S %t/b.bc %t/a.bc | FileCheck %s --check-prefixes=CHECK,BA
+
+; Model an inline device function shared by two translation units. Both callers
+; must update the same lane/wave allocation after COMDAT selection, in either
+; link order. No collection option is needed to obtain the wave slots.
+; CHECK: @[[SHARED:__profc_shared[^ ]*]] = {{.*}}[2 x i64] zeroinitializer
+
+; AB-LABEL: define amdgpu_kernel void @caller_a(
+; AB: call void @__llvm_profile_instrument_gpu({{.*}}i64 1, ptr {{.*}}@[[SHARED]]{{.*}}i64 8
+; AB-LABEL: define amdgpu_kernel void @caller_b(
+; AB: call void @__llvm_profile_instrument_gpu({{.*}}i64 1, ptr {{.*}}@[[SHARED]]{{.*}}i64 8
+; BA-LABEL: define amdgpu_kernel void @caller_b(
+; BA: call void @__llvm_profile_instrument_gpu({{.*}}i64 1, ptr {{.*}}@[[SHARED]]{{.*}}i64 8
+; BA-LABEL: define amdgpu_kernel void @caller_a(
+; BA: call void @__llvm_profile_instrument_gpu({{.*}}i64 1, ptr {{.*}}@[[SHARED]]{{.*}}i64 8
+
+;--- a.ll
+source_filename = "a.cpp"
+target triple = "amdgcn-amd-amdhsa"
+$shared = comdat any
+
+define amdgpu_kernel void @caller_a(ptr addrspace(1) %out, i32 %x) {
+ %r = call i32 @shared(i32 %x)
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define linkonce_odr i32 @shared(i32 %x) alwaysinline comdat {
+ %r = add i32 %x, 1
+ ret i32 %r
+}
+
+;--- b.ll
+source_filename = "b.cpp"
+target triple = "amdgcn-amd-amdhsa"
+$shared = comdat any
+
+define amdgpu_kernel void @caller_b(ptr addrspace(1) %out, i32 %x) {
+ %r = call i32 @shared(i32 %x)
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define linkonce_odr i32 @shared(i32 %x) alwaysinline comdat {
+ %r = add i32 %x, 1
+ ret i32 %r
+}
diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
index c948ad84392a07..cc8ae70c1e2363 100644
--- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll
+++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
@@ -13,9 +13,9 @@ $foo = comdat any
; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat
; CHECK-NOT: __profn__stdin__foo
; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
-; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32 } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, i64 0, ptr null
+; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32, i32 } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, i64 0, ptr null
; CHECK-NOT: @foo
-; CHECK-SAME: , ptr null, i32 1, [3 x i16] zeroinitializer, i16 0, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8
+; CHECK-SAME: , ptr null, i32 1, [3 x i16] zeroinitializer, i16 0, i32 0, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8
; CHECK: @__llvm_prf_nm
; CHECK: @llvm.compiler.used
diff --git a/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll b/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll
new file mode 100644
index 00000000000000..b6f4924fc66320
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll
@@ -0,0 +1,108 @@
+; RUN: split-file %s %t
+; RUN: opt -passes=pgo-instr-gen,verify -pgo-instrument-entry -S %t/loop.ll | FileCheck %s --check-prefix=GEN --implicit-check-not='call void @llvm.instrprof.increment'
+; RUN: %python %t/raw.py > %t/loop.raw
+; RUN: llvm-profdata merge %t/loop.raw -o %t/loop.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/loop.profdata -S %t/loop.ll -o %t/use.ll
+; RUN: FileCheck %s --check-prefix=COUNTS < %t/use.ll
+; RUN: opt -passes=pgo-instr-gen,verify -pgo-instrument-entry -S %t/eh.ll | FileCheck %s --check-prefix=EH --implicit-check-not='call void @llvm.instrprof.increment'
+
+; Dense instrumentation preserves the entry and split backedge counters,
+; then appends the header and exit measurements.
+; GEN: @__llvm_profile_raw_version = {{.*}}constant i64 378302368699121676
+; GEN-LABEL: define void @loop(
+; GEN: entry:
+; GEN-NEXT: call void @llvm.instrprof.increment({{.*}}i64 [[HASH:784007056844089447]], i32 4, i32 0)
+; GEN-NEXT: br label %header
+; GEN: header:
+; GEN-NEXT: %i = phi i32 [ 0, %entry ], [ %next, %header.header_crit_edge ]
+; GEN-NEXT: call void @llvm.instrprof.increment.step({{.*}}i64 [[HASH]], i32 4, i32 2, i64 0)
+; GEN: header.header_crit_edge:
+; GEN-NEXT: call void @llvm.instrprof.increment({{.*}}i64 [[HASH]], i32 4, i32 1)
+; GEN-NEXT: br label %header
+; GEN: exit:
+; GEN-NEXT: call void @llvm.instrprof.increment.step({{.*}}i64 [[HASH]], i32 4, i32 3, i64 0)
+; GEN-NEXT: ret void
+
+; Profile use reconstructs counter placement from the recorded flags. The
+; header has four times the entry wave count; sparse data cannot supply it.
+; COUNTS-LABEL: define void @loop(
+; COUNTS-SAME: !prof ![[ENTRY_COUNT:[0-9]+]]
+; COUNTS: br i1 %done, label %exit, label %header.header_crit_edge, !prof ![[WEIGHTS:[0-9]+]]
+; COUNTS-DAG: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 192}
+; COUNTS-DAG: ![[WEIGHTS]] = !{!"branch_weights", i32 192, i32 576}
+
+; A catchswitch has no legal insertion point. Skip dispatch and append only
+; exit; the sparse catch counter must remain after its catchpad.
+; EH-LABEL: define void @eh(
+; EH: entry:
+; EH-NEXT: call void @llvm.instrprof.increment({{.*}}i64 [[EH_HASH:146835646621254984]], i32 3, i32 0)
+; EH: dispatch:
+; EH-NEXT: %cs = catchswitch within none [label %catch] unwind to caller
+; EH: catch:
+; EH-NEXT: %cp = catchpad within %cs [ptr null, i32 64, ptr null]
+; EH-NEXT: call void @llvm.instrprof.increment({{.*}}i64 [[EH_HASH]], i32 3, i32 1)
+; EH-NEXT: catchret from %cp to label %exit
+; EH: exit:
+; EH-NEXT: call void @llvm.instrprof.increment.step({{.*}}i64 [[EH_HASH]], i32 3, i32 2, i64 0)
+; EH-NEXT: ret void
+
+;--- loop.ll
+target triple = "amdgcn-amd-amdhsa"
+
+define void @loop(ptr %p, i32 %n) {
+entry:
+ br label %header
+header:
+ %i = phi i32 [0, %entry], [%next, %header]
+ store volatile i32 %i, ptr %p
+ %next = add i32 %i, 1
+ %done = icmp eq i32 %next, %n
+ br i1 %done, label %exit, label %header
+exit:
+ ret void
+}
+
+;--- eh.ll
+target triple = "amdgcn-amd-amdhsa"
+
+define void @eh() personality ptr @__CxxFrameHandler3 {
+entry:
+ invoke void @may_throw() to label %exit unwind label %dispatch
+dispatch:
+ %cs = catchswitch within none [label %catch] unwind to caller
+catch:
+ %cp = catchpad within %cs [ptr null, i32 64, ptr null]
+ catchret from %cp to label %exit
+exit:
+ ret void
+}
+
+declare void @may_throw()
+declare i32 @__CxxFrameHandler3(...)
+
+;--- raw.py
+import hashlib
+import struct
+import sys
+
+# Three full waves, each taking four loop iterations. Counter order is
+# entry, backedge, header, exit, as checked in the generation above.
+name = b"loop"
+name_ref = int.from_bytes(hashlib.md5(name).digest()[:8], "little")
+lanes = [192, 576, 0, 0]
+waves = [3, 9, 12, 3]
+version = 12 | (1 << 54) | (1 << 56) | (1 << 58)
+names = bytes([len(name), 0]) + name
+# Raw v12: 80-byte data record, eight i64 lane/wave counters, four uniform counters.
+counter_delta = 80
+uniform_delta = counter_delta + 8 * (len(lanes) + len(waves))
+names_delta = uniform_delta + 8 * len(lanes)
+header = [0xff6c70726f667281, version, 0, 1, 0, 8, 0,
+ 0, 0, 4, 0, uniform_delta, len(names), counter_delta,
+ uniform_delta, names_delta, 0, 0, 2]
+record = struct.pack("<7QI4HII4x", name_ref, 784007056844089447, counter_delta,
+ uniform_delta, 0, 0, 0, 8, 0, 0, 0, 64, 0, 4)
+counts = lanes + waves + lanes
+sys.stdout.buffer.write(struct.pack("<19Q", *header) + record +
+ struct.pack("<12Q", *counts) +
+ names + bytes((-len(names)) % 8))
diff --git a/llvm/test/Transforms/PGOProfile/dense-wave-instrumentation.ll b/llvm/test/Transforms/PGOProfile/dense-wave-instrumentation.ll
new file mode 100644
index 00000000000000..86c1cab9ce6008
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/dense-wave-instrumentation.ll
@@ -0,0 +1,59 @@
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -S %s | FileCheck %s --check-prefix=DENSE
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -pgo-instrument-dense-wave-counts=false -S %s | FileCheck %s --check-prefix=SPARSE
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -mtriple=x86_64-unknown-linux-gnu -S %s | FileCheck %s --check-prefix=SPARSE
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry=false -S %s | FileCheck %s --check-prefix=NOENTRY
+; RUN: opt -passes=pgo-instr-gen -pgo-function-entry-coverage -S %s | FileCheck %s --check-prefix=ENTRY-COV --implicit-check-not='i64 0)'
+; RUN: opt -passes=pgo-instr-gen -pgo-block-coverage -S %s | FileCheck %s --check-prefix=BLOCK-COV --implicit-check-not='i64 0)'
+; RUN: opt -passes=pgo-instr-gen -pgo-temporal-instrumentation -S %s | FileCheck %s --check-prefix=TEMPORAL --implicit-check-not='i64 0)'
+; RUN: not opt -passes='default<O2>' -cs-profilegen-file=dense-wave -cspgo-kind=cspgo-instr-gen-pipeline -pgo-instrument-entry -print-before=instrprof -disable-output %s 2>&1 | FileCheck %s --check-prefix=CS --implicit-check-not='i64 0)'
+
+target triple = "amdgcn-amd-amdhsa"
+
+; Dense generation keeps the original block/select prefix, then appends the
+; unmeasured blocks in function order. The zero steps measure waves only.
+; DENSE: @__llvm_profile_raw_version = {{.*}}constant i64 378302368699121676
+; SPARSE: @__llvm_profile_raw_version = {{.*}}constant i64 360287970189639692
+; NOENTRY: @__llvm_profile_raw_version = {{.*}}constant i64 90071992547409932
+; ENTRY-COV: @__llvm_profile_raw_version = {{.*}}constant i64 3530822107858468876
+; BLOCK-COV: @__llvm_profile_raw_version = {{.*}}constant i64 1224979098644774924
+; TEMPORAL: @__llvm_profile_raw_version = {{.*}}constant i64 -9151314442816847860
+; CS: @__llvm_profile_raw_version = {{.*}}constant i64 504403158265495564
+; CS: LLVM ERROR: wave counts require ordinary counter increments
+;
+; DENSE-LABEL: define void @diamond
+; DENSE: entry:
+; DENSE-NEXT: call void @llvm.instrprof.increment({{.*}}i64 [[HASH:942389667449461396]], i32 5, i32 0)
+; DENSE: a:
+; DENSE-NEXT: call void @llvm.instrprof.increment.step({{.*}}i64 [[HASH]], i32 5, i32 3, i64 0)
+; DENSE: b:
+; DENSE-NEXT: call void @llvm.instrprof.increment({{.*}}i64 [[HASH]], i32 5, i32 1)
+; DENSE: exit:
+; DENSE-NEXT: call void @llvm.instrprof.increment.step({{.*}}i64 [[HASH]], i32 5, i32 4, i64 0)
+; DENSE: call void @llvm.instrprof.increment.step({{.*}}i64 [[HASH]], i32 5, i32 2, i64 {{%.*}})
+;
+; SPARSE-LABEL: define void @diamond
+; SPARSE: call void @llvm.instrprof.increment({{.*}}i32 3, i32 0)
+; SPARSE: a:
+; SPARSE-NEXT: store volatile
+; SPARSE: call void @llvm.instrprof.increment({{.*}}i32 3, i32 1)
+; SPARSE: exit:
+; SPARSE: call void @llvm.instrprof.increment.step({{.*}}i32 3, i32 2, i64 {{%.*}})
+;
+; Even without forced entry instrumentation, the entry gets a direct wave site.
+; NOENTRY-LABEL: define void @diamond
+; NOENTRY: entry:
+; NOENTRY-NEXT: call void @llvm.instrprof.increment.step({{.*}}i64 942389667449461396, i32 5, i32 3, i64 0)
+define void @diamond(i1 %cond, i1 %select_cond, ptr %p) {
+entry:
+ br i1 %cond, label %a, label %b
+a:
+ store volatile i32 1, ptr %p
+ br label %exit
+b:
+ store volatile i32 2, ptr %p
+ br label %exit
+exit:
+ %value = select i1 %select_cond, i32 1, i32 2
+ store volatile i32 %value, ptr %p
+ ret void
+}
diff --git a/llvm/test/Transforms/PGOProfile/dense-wave-profile-use.ll b/llvm/test/Transforms/PGOProfile/dense-wave-profile-use.ll
new file mode 100644
index 00000000000000..6229d375f6fb15
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/dense-wave-profile-use.ll
@@ -0,0 +1,82 @@
+; RUN: split-file %s %t
+; RUN: %python %t/raw.py dense > %t/dense.raw
+; RUN: llvm-profdata merge %t/dense.raw -o %t/dense.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/dense.profdata -S %t/input.ll -o %t/dense.ll
+; RUN: FileCheck %s --check-prefix=COUNTS < %t/dense.ll
+; RUN: FileCheck %s --check-prefix=UNIFORM < %t/dense.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/dense.profdata -pgo-instrument-dense-wave-counts=false -S %t/input.ll -o %t/disabled-gen.ll
+; RUN: diff %t/dense.ll %t/disabled-gen.ll
+; RUN: %python %t/raw.py no-entry > %t/no-entry.raw
+; RUN: llvm-profdata merge %t/no-entry.raw -o %t/no-entry.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/no-entry.profdata -S %t/input.ll | FileCheck %s --check-prefix=COUNTS
+; RUN: %python %t/raw.py sparse > %t/sparse.raw
+; RUN: not llvm-profdata merge %t/dense.raw %t/sparse.raw -o %t/mixed 2>&1 | FileCheck %s --check-prefix=MIXED
+; RUN: not llvm-profdata merge %t/sparse.raw %t/dense.raw -o %t/mixed 2>&1 | FileCheck %s --check-prefix=MIXED
+; RUN: cat %t/dense.raw %t/sparse.raw > %t/mixed.raw
+; RUN: not llvm-profdata merge %t/mixed.raw -o %t/mixed 2>&1 | FileCheck %s --check-prefix=MIXED
+; RUN: cat %t/sparse.raw %t/dense.raw > %t/mixed.raw
+; RUN: not llvm-profdata merge %t/mixed.raw -o %t/mixed 2>&1 | FileCheck %s --check-prefix=MIXED
+
+; The profile flags select the layout independently of generation options.
+; Appended slots do not change scalar/select counts or uniformity coverage.
+; MIXED: cannot merge dense and sparse wave instrumentation layouts
+; COUNTS-DAG: !{!"function_entry_count", i64 6400}
+; COUNTS-DAG: !{!"branch_weights", i32 3200, i32 3200}
+; COUNTS-DAG: !{!"branch_weights", i32 1600, i32 4800}
+; UNIFORM: define void @diamond({{.*}}!uniformity.profile
+; UNIFORM: br i1 %cond, {{.*}}!block.uniformity.profile{{.*}}!branch.uniformity.profile
+; UNIFORM: b:
+; UNIFORM: br label %exit, {{.*}}!block.uniformity.profile
+
+;--- input.ll
+source_filename = "wave-profile-use.ll"
+target triple = "amdgcn-amd-amdhsa"
+define void @diamond(i1 %cond, i1 %select_cond, ptr %p) {
+entry:
+ br i1 %cond, label %a, label %b
+a:
+ store volatile i32 1, ptr %p
+ br label %exit
+b:
+ store volatile i32 2, ptr %p
+ br label %exit
+exit:
+ %value = select i1 %select_cond, i32 1, i32 2
+ store volatile i32 %value, ptr %p
+ ret void
+}
+
+;--- raw.py
+import hashlib
+import struct
+import sys
+
+mode = sys.argv[1]
+name = b"diamond"
+name_ref = int.from_bytes(hashlib.md5(name).digest()[:8], "little")
+# Preserve the sparse block/select prefix and append two zero lane steps.
+lanes = [6400, 3200, 1600]
+waves = [100, 50, 100]
+version = 12 | (1 << 56) | (1 << 58)
+if mode != "sparse":
+ version |= 1 << 54
+ lanes += [0, 0]
+ waves += [100, 100]
+if mode == "no-entry":
+ version &= ~(1 << 58)
+ lanes[0] = 3200
+num_counters = len(lanes) + len(waves)
+counter_delta = 80
+uniform_delta = counter_delta + num_counters * 8
+names_delta = uniform_delta + len(lanes) * 8
+names = bytes([len(name), 0]) + name
+header = [0xff6c70726f667281, version, 0, 1, 0, num_counters, 0,
+ 0, 0, len(lanes), 0, uniform_delta, len(names), counter_delta,
+ uniform_delta, names_delta, 0, 0, 2]
+record = struct.pack("<7QI4HII4x", name_ref, 942389667449461396, counter_delta,
+ uniform_delta, 0, 0, 0, num_counters, 0, 0, 0, 64, 0,
+ len(waves))
+counts = lanes + waves + lanes
+sys.stdout.buffer.write(struct.pack("<19Q", *header) + record +
+ struct.pack("<" + "Q" * len(counts), *counts) +
+ names + bytes((-len(names)) % 8))
diff --git a/llvm/test/Transforms/PGOProfile/gpu-wave-counts.ll b/llvm/test/Transforms/PGOProfile/gpu-wave-counts.ll
new file mode 100644
index 00000000000000..9d0ab9f7c46311
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/gpu-wave-counts.ll
@@ -0,0 +1,38 @@
+; RUN: opt -passes='pgo-instr-gen,instrprof,mem2reg,verify' -S %s | FileCheck %s
+; RUN: not opt -passes='pgo-instr-gen,instrprof' -pgo-block-coverage -disable-output %s 2>&1 | FileCheck %s --check-prefix=ERROR
+; RUN: not opt -passes='pgo-instr-gen,instrprof' -pgo-temporal-instrumentation -disable-output %s 2>&1 | FileCheck %s --check-prefix=ERROR
+; ERROR: wave counts require ordinary counter increments
+
+target triple = "amdgcn-amd-amdhsa"
+
+; Wave counts use the existing instrumentation sites and intrinsics.
+; CHECK-LABEL: define void @diamond
+; CHECK: call void @__llvm_profile_instrument_gpu(
+; CHECK: call void @__llvm_profile_instrument_gpu(
+define void @diamond(i1 %cond, ptr %p) {
+entry:
+ br i1 %cond, label %a, label %b
+a:
+ store volatile i32 1, ptr %p
+ br label %exit
+b:
+ store volatile i32 2, ptr %p
+ br label %exit
+exit:
+ ret void
+}
+
+; Sampling must leave static allocas in the entry so mem2reg can promote them.
+; CHECK-LABEL: define amdgpu_kernel void @alloca_kernel
+; CHECK-NOT: = alloca
+; CHECK: store i32 %value, ptr addrspace(1) %out
+; CHECK-NOT: = alloca
+; CHECK: ret void
+define amdgpu_kernel void @alloca_kernel(ptr addrspace(1) %out, i32 %value) {
+entry:
+ %slot = alloca i32, align 4, addrspace(5)
+ store i32 %value, ptr addrspace(5) %slot
+ %loaded = load i32, ptr addrspace(5) %slot
+ store i32 %loaded, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_fast.ll b/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_fast.ll
index 7261cc24b0dcff..087afc002efe82 100644
--- a/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_fast.ll
+++ b/llvm/test/Transforms/PGOProfile/instrprof_burst_sampling_fast.ll
@@ -14,7 +14,7 @@ $__llvm_profile_raw_version = comdat any
; SAMPLE-VAR: @__llvm_profile_sampling = thread_local global i16 0, comdat
; SAMPLE-VAR: @__profc_f = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
-; SAMPLE-VAR: @__profd_f = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32 } { i64 -3706093650706652785, i64 12884901887, i64 sub (i64 ptrtoint (ptr @__profc_f to i64), i64 ptrtoint (ptr @__profd_f to i64)), i64 0, i64 0, ptr @f.local, ptr null, i32 1, [3 x i16] zeroinitializer, i16 0, i32 0 }, section "__llvm_prf_data", comdat($__profc_f), align 8
+; SAMPLE-VAR: @__profd_f = private global { i64, i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i16, i32, i32 } { i64 -3706093650706652785, i64 12884901887, i64 sub (i64 ptrtoint (ptr @__profc_f to i64), i64 ptrtoint (ptr @__profd_f to i64)), i64 0, i64 0, ptr @f.local, ptr null, i32 1, [3 x i16] zeroinitializer, i16 0, i32 0, i32 0 }, section "__llvm_prf_data", comdat($__profc_f), align 8
; SAMPLE-VAR: @__llvm_prf_nm = private constant {{.*}}, section "__llvm_prf_names", align 1
; SAMPLE-VAR: @llvm.compiler.used = appending global [2 x ptr] [ptr @__llvm_profile_sampling, ptr @__profd_f], section "llvm.metadata"
; SAMPLE-VAR: @llvm.used = appending global [1 x ptr] [ptr @__llvm_prf_nm], section "llvm.metadata"
diff --git a/llvm/test/Transforms/PGOProfile/uniformity-profile-use.ll b/llvm/test/Transforms/PGOProfile/uniformity-profile-use.ll
index 0621c6ba57de62..a0576987473ce0 100644
--- a/llvm/test/Transforms/PGOProfile/uniformity-profile-use.ll
+++ b/llvm/test/Transforms/PGOProfile/uniformity-profile-use.ll
@@ -1,5 +1,5 @@
; RUN: split-file %s %t
-; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -S %t/input.ll | FileCheck %s --check-prefix=GEN
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -pgo-instrument-dense-wave-counts=false -S %t/input.ll | FileCheck %s --check-prefix=GEN
; RUN: %python %t/raw.py > %t/profile.raw
; RUN: llvm-profdata merge %t/profile.raw -o %t/profile
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/profile -S %t/input.ll -o %t/default.ll
@@ -65,7 +65,7 @@ import sys
mode = sys.argv[1] if len(sys.argv) > 1 else "uniform"
name = b"missing" if mode == "missing" else b"diamond"
name_ref = int.from_bytes(hashlib.md5(name).digest()[:8], "little")
-# Raw v11, IR entry-first layout; hash checked against pgo-instr-gen above.
+# Raw v12, sparse IR entry-first layout; hash checked against pgo-instr-gen above.
func_hash = 942389667449461396
if mode == "mismatch":
func_hash += 1
@@ -73,17 +73,17 @@ lanes = [6400, 3200, 1600]
if mode == "zero":
lanes = [0, 0, 0]
uniform = lanes[:]
-version = 11 | (1 << 56) | (1 << 58)
+version = 12 | (1 << 56) | (1 << 58)
num_counters = len(lanes)
-counter_delta = 72
+counter_delta = 80
uniform_delta = counter_delta + num_counters * 8
names_delta = uniform_delta + len(uniform) * 8
names = bytes([len(name), 0]) + name
header = [0xff6c70726f667281, version, 0, 1, 0, num_counters, 0,
0, 0, len(uniform), 0, uniform_delta, len(names), counter_delta,
uniform_delta, names_delta, 0, 0, 2]
-record = struct.pack("<7QI4HI", name_ref, func_hash, counter_delta,
- uniform_delta, 0, 0, 0, num_counters, 0, 0, 0, 64, 0)
+record = struct.pack("<7QI4HII4x", name_ref, func_hash, counter_delta,
+ uniform_delta, 0, 0, 0, num_counters, 0, 0, 0, 64, 0, 0)
counts = lanes + uniform
sys.stdout.buffer.write(struct.pack("<19Q", *header) + record +
struct.pack("<" + "Q" * len(counts), *counts) +
diff --git a/llvm/test/Transforms/PGOProfile/vtable_profile.ll b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
index 0c554db05cfb46..ea1c0e51f80f4e 100644
--- a/llvm/test/Transforms/PGOProfile/vtable_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/vtable_profile.ll
@@ -49,7 +49,7 @@ target triple = "x86_64-unknown-linux-gnu"
@llvm.compiler.used = appending global [1 x ptr] [ptr @_ZTV5Base1], section "llvm.metadata"
; GEN: __llvm_profile_raw_version = comdat any
-; GEN: __llvm_profile_raw_version = hidden constant i64 72057594037927947, comdat
+; GEN: __llvm_profile_raw_version = hidden constant i64 72057594037927948, comdat
; GEN: __profn__Z4funci = private constant [8 x i8] c"_Z4funci"
; LOWER: $__profvt__ZTV7Derived = comdat nodeduplicate
diff --git a/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw b/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw
index dec90151a79352bf898c8c9d1d2d23eaa4bf5eed..93191c38878057d64e71cb4a193cda9324740036 100644
GIT binary patch
delta 199
zcmaDMa6*u?u_!ISs37M*&qU62z5p%L2Rh6D|IeR$O8#=h#JZ%34jdC72rzO?{AfNo
zfSGf$CL=$i!o-j2lM@&PfO3-y7zG$5CO0yggE%{YoC=WS2e9NMCIQ9+lLeW;avng=
z50G2~6IkN}AYWqgLWtxIAjbg6QJ*Zp43<@3hNzx+0nE(+a!-IXKVXrVyaC8*n0%1g
NeDVhtflVwKEC9qDL~H;6
delta 262
zcmX>h_(FiQu_!ISs37M*_e9QgK8LRVl{;7d|1WFC(B>94u`X%i3jq#+KmWmCqM^cM
z1xA6%7K{QM21sHBK(P)qu>(M{8$hucNa_Tb1SV at R32?kX5=#JzRRF~_{vzzy02Dg`
z6bnES1KBIVEWlBKBo at FdFgXJ#=712JctK$D4xo?(LTK^_pfU~?0gf9;);h2V0L3{r
JC$MC&001zQg at 6D6
diff --git a/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw b/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw
index 778e80fce2691a35d9654748763b0aa8c233ec13..d358dfc21e8da60e6bf93824214a58040e83272f 100644
GIT binary patch
delta 198
zcmdlXFhh{Du_!ISs37M*&qU62UX5GfIg3{P|KIR-`Q+G%waF75I3_+2VC0zi(R^|M
zGv{PYMt(+xi67M`Col>C<t7&}3NT7cZe%nEadrSX6(GqEV97~L0*nVH3o?P_Jb;`Z
zAh`x6u*L~MzQp8(5Xl=rjscLPK3RYnEUUl_Q9bbjn41CQo&afnz#=hu1CY}&`5?3T
M<PR(Yn^-JZ0Mmj*ZU6uP
delta 261
zcmbOsxI=)mu_!ISs37M*_e9QgUWZ?ERC`wa|1Wzi=~nT?+T at 8Z1ULl#{0D=Hh6<Av
z7zHL<FbZ%OAc++K#X8W$4gke&0L5k?sS{uln5 at Ah!0`e}ECDE10Tk2ti?Cw at Q0xRy
zEC5LiWUmCX07n6mSOBxY<P4yg143-#1%b&sfI<=op~)YB$~agAIBp<W>%bxa6zABS
Iz+%Ay09e0;nE(I)
diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test
index 35eb92483603d6..fb2e46a0c9f1db 100644
--- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test
+++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test
@@ -21,7 +21,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.profraw
// There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes.
// 2 * 8 binary ID sizes
// + 2 * 20 binary IDs (of size 20)
@@ -72,17 +72,17 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\067\265\035\031\112\165\023\344' >> %t.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
-RUN: printf '\270\377\3\0\1\0\0\0' >> %t.profraw
+RUN: printf '\260\377\3\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\067\0\0\0\0\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/dense-wave-layout.test b/llvm/test/tools/llvm-profdata/dense-wave-layout.test
new file mode 100644
index 00000000000000..5788a8bf64e790
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/dense-wave-layout.test
@@ -0,0 +1,43 @@
+# RUN: split-file %s %t
+# RUN: llvm-profdata merge %t/dense.txt -o %t/dense.profdata
+# RUN: llvm-profdata show --profile-version %t/dense.profdata | FileCheck %s --check-prefix=DENSE
+# RUN: llvm-profdata merge --text %t/dense.profdata -o %t/roundtrip.txt
+# RUN: FileCheck %s --check-prefix=TEXT < %t/roundtrip.txt
+# RUN: llvm-profdata merge %t/roundtrip.txt -o %t/roundtrip.profdata
+# RUN: llvm-profdata show --profile-version %t/roundtrip.profdata | FileCheck %s --check-prefix=DENSE
+# RUN: llvm-profdata merge %t/sparse.txt -o %t/sparse.profdata
+# RUN: llvm-profdata show --profile-version %t/sparse.profdata | FileCheck %s --check-prefix=SPARSE
+# RUN: not llvm-profdata merge -j 1 %t/dense.profdata %t/sparse.profdata -o %t/mixed 2>&1 | FileCheck %s --check-prefix=ERROR
+# RUN: not llvm-profdata merge -j 1 %t/sparse.profdata %t/dense.profdata -o %t/mixed 2>&1 | FileCheck %s --check-prefix=ERROR
+# RUN: not llvm-profdata merge -j 2 %t/dense.profdata %t/sparse.profdata -o %t/mixed 2>&1 | FileCheck %s --check-prefix=ERROR
+# RUN: not llvm-profdata merge -j 2 %t/sparse.profdata %t/dense.profdata -o %t/mixed 2>&1 | FileCheck %s --check-prefix=ERROR
+# RUN: llvm-profdata merge -j 2 %t/dense.profdata %t/dense.profdata -o %t/double.profdata
+# RUN: llvm-profdata show --profile-version %t/double.profdata | FileCheck %s --check-prefix=DENSE
+
+# RUN: touch %t/empty.raw
+# RUN: llvm-profdata merge -j 2 %t/dense.profdata %t/empty.raw -o %t/empty-merge.profdata
+# RUN: llvm-profdata show --profile-version %t/empty-merge.profdata | FileCheck %s --check-prefix=DENSE
+# RUN: llvm-profdata merge -j 2 %t/empty.raw %t/dense.profdata -o %t/empty-merge.profdata
+# RUN: llvm-profdata show --profile-version %t/empty-merge.profdata | FileCheck %s --check-prefix=DENSE
+
+# Header round trips use a lane-only record; wave payload text export remains
+# unsupported. Equal-length records must still reject incompatible layouts.
+# DENSE: dense_wave = 1
+# SPARSE: dense_wave = 0
+# TEXT: :dense_wave
+# ERROR: cannot merge dense and sparse wave instrumentation layouts
+
+#--- dense.txt
+:ir
+:dense_wave
+same
+123
+1
+0
+
+#--- sparse.txt
+:ir
+same
+123
+1
+0
diff --git a/llvm/test/tools/llvm-profdata/gpu-wave-counts.test b/llvm/test/tools/llvm-profdata/gpu-wave-counts.test
new file mode 100644
index 00000000000000..ce87b5bd942a42
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/gpu-wave-counts.test
@@ -0,0 +1,50 @@
+# RUN: split-file %s %t
+# RUN: %python %t/raw.py little 2 > %t/little.raw
+# RUN: %python %t/raw.py big 2 > %t/big.raw
+# RUN: llvm-profdata merge %t/little.raw -o %t/one.profdata
+# RUN: llvm-profdata show --all-functions --counts %t/one.profdata | FileCheck %s --check-prefix=ONE
+# RUN: llvm-profdata merge %t/one.profdata --weighted-input=2,%t/big.raw -o %t/three.profdata
+# RUN: llvm-profdata show --all-functions --counts %t/three.profdata | FileCheck %s --check-prefix=THREE
+# RUN: not llvm-profdata merge --text %t/one.profdata -o %t/out.txt 2>&1 | FileCheck %s --check-prefix=TEXT
+# RUN: not llvm-profdata show --all-functions --text %t/one.profdata -o %t/show.txt 2>&1 | FileCheck %s --check-prefix=TEXT
+# RUN: not llvm-profdata show --all-functions --text %t/little.raw -o %t/show-raw.txt 2>&1 | FileCheck %s --check-prefix=TEXT
+# RUN: not llvm-profdata merge --write-prev-version %t/one.profdata -o %t/old.profdata 2>&1 | FileCheck %s --check-prefix=OLD
+# RUN: %python %t/raw.py little 5 > %t/bad.raw
+# RUN: not llvm-profdata merge %t/bad.raw -o %t/bad.profdata 2>&1 | FileCheck %s --check-prefix=BAD
+# RUN: %python %t/raw.py little 4 > %t/all-wave.raw
+# RUN: not llvm-profdata merge %t/all-wave.raw -o %t/bad.profdata 2>&1 | FileCheck %s --check-prefix=BAD
+
+# ONE: Block counts: [6400, 3200]
+# ONE: Block wave counts: [100, 100]
+# THREE: Block counts: [19200, 9600]
+# THREE: Block wave counts: [300, 300]
+# TEXT: text profiles do not support wave counts
+# OLD: wave counts require indexed profile version 15
+# BAD: invalid number of wave counters
+
+#--- raw.py
+import hashlib
+import struct
+import sys
+
+endian = '<' if sys.argv[1] == 'little' else '>'
+num_wave = int(sys.argv[2])
+name = b'wave'
+name_ref = int.from_bytes(hashlib.md5(name).digest()[:8], 'little')
+# Raw v12: 80-byte record, 4 counters (2 lane + 2 wave), 2 uniforms.
+record_size = 80
+counter_delta = record_size
+uniform_delta = counter_delta + 4 * 8
+names_delta = uniform_delta + 2 * 8
+names = bytes([len(name), 0]) + name
+header = [0xff6c70726f667281, 12 | (1 << 56), 0, 1, 0, 4, 0, 0, 0,
+ 2, 0, uniform_delta, len(names), counter_delta, uniform_delta,
+ names_delta, 0, 0, 2]
+record = struct.pack(endian + '7QI4HII4x', name_ref, 123,
+ counter_delta, uniform_delta, 0, 0, 0,
+ 4, 0, 0, 0, 64, 0, num_wave)
+assert len(record) == record_size
+sys.stdout.buffer.write(struct.pack(endian + '19Q', *header) + record +
+ struct.pack(endian + '6Q', 6400, 3200,
+ 100, 100, 6400, 0) +
+ names + bytes((-len(names)) % 8))
diff --git a/llvm/test/tools/llvm-profdata/insufficient-binary-ids-size.test b/llvm/test/tools/llvm-profdata/insufficient-binary-ids-size.test
index b6cdf282dd97ab..01b77d09ac90e3 100644
--- a/llvm/test/tools/llvm-profdata/insufficient-binary-ids-size.test
+++ b/llvm/test/tools/llvm-profdata/insufficient-binary-ids-size.test
@@ -3,7 +3,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-zos
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.profraw
// We should fail on this because the data buffer (profraw file) is not long
// enough to hold this binary IDs size. NOTE that this (combined with the 8-byte
// alignment requirement for binary IDs size) will ensure we can at least read one
diff --git a/llvm/test/tools/llvm-profdata/large-binary-id-size.test b/llvm/test/tools/llvm-profdata/large-binary-id-size.test
index f27bdf4287035c..7f77c49f67d702 100644
--- a/llvm/test/tools/llvm-profdata/large-binary-id-size.test
+++ b/llvm/test/tools/llvm-profdata/large-binary-id-size.test
@@ -5,7 +5,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test
index aec3323bd0fe9d..c454d43221a751 100644
--- a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test
+++ b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test
@@ -21,7 +21,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -56,7 +56,7 @@ RUN: printf '\0\0\4\0\3\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\3\0foo\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test
index 139d78341ba850..473a5f5a4231aa 100644
--- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test
+++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test
@@ -21,7 +21,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -62,7 +62,7 @@ RUN: cp %t.profraw %t-good.profraw
// Make NumCounters = 0 so that we get "number of counters is zero" error message
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\3\0foo\0\0\0' >> %t.profraw
@@ -74,6 +74,8 @@ ZERO: malformed instrumentation profile data: number of counters is zero
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bad.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bad.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bad.profraw
+// NumWaveCounters and tail padding.
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bad.profraw
// Counter value is 72057594037927937
RUN: printf '\1\0\0\0\0\0\0\1' >> %t-bad.profraw
RUN: printf '\3\0foo\0\0\0' >> %t-bad.profraw
@@ -81,6 +83,8 @@ RUN: printf '\3\0foo\0\0\0' >> %t-bad.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-good.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-good.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-good.profraw
+// NumWaveCounters and tail padding.
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-good.profraw
// Counter value is 72057594037927937
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-good.profraw
RUN: printf '\3\0foo\0\0\0' >> %t-good.profraw
diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test
index 9fafbeebaa735a..98e3c82f361100 100644
--- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test
+++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test
@@ -21,7 +21,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
@@ -61,7 +61,7 @@ RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t.profraw
// Counter Section
diff --git a/llvm/test/tools/llvm-profdata/malformed-uniform-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-uniform-counter-array.test
index 814c5298fc561f..d3963c2c7865cd 100644
--- a/llvm/test/tools/llvm-profdata/malformed-uniform-counter-array.test
+++ b/llvm/test/tools/llvm-profdata/malformed-uniform-counter-array.test
@@ -6,7 +6,7 @@ UNSUPPORTED: system-windows
// UniformCounterPtr is one byte before UniformCountersDelta.
RUN: printf '\201rforpl\377' > %t.neg.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.neg.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.neg.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.neg.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.neg.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.neg.profraw
@@ -32,7 +32,7 @@ RUN: printf '\130\0\0\0\0\0\0\0' >> %t.neg.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.neg.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.neg.profraw
RUN: printf '\2\0\0\0\0\0\0\0' >> %t.neg.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t.neg.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t.neg.profraw
RUN: printf '\067\0\0\0\0\0\0\0' >> %t.neg.profraw
RUN: printf '\101\0\0\0\0\0\0\0' >> %t.neg.profraw
RUN: printf '\5\0\0\0\0\0\0\0' >> %t.neg.profraw
@@ -44,7 +44,7 @@ NEG: error: no profile can be merged
// UniformCounterPtr points one byte past the end of the uniform-counter section.
RUN: printf '\201rforpl\377' > %t.past.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.past.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.past.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.past.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.past.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.past.profraw
@@ -70,7 +70,7 @@ RUN: printf '\130\0\0\0\0\0\0\0' >> %t.past.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.past.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.past.profraw
RUN: printf '\2\0\0\0\0\0\0\0' >> %t.past.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t.past.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t.past.profraw
RUN: printf '\067\0\0\0\0\0\0\0' >> %t.past.profraw
RUN: printf '\101\0\0\0\0\0\0\0' >> %t.past.profraw
RUN: printf '\5\0\0\0\0\0\0\0' >> %t.past.profraw
@@ -82,7 +82,7 @@ PAST: error: no profile can be merged
// UniformCounterPtr leaves only one uniform counter, but NumCounters is two.
RUN: printf '\201rforpl\377' > %t.too-many.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.too-many.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.too-many.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.too-many.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.too-many.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.too-many.profraw
@@ -108,7 +108,7 @@ RUN: printf '\130\0\0\0\0\0\0\0' >> %t.too-many.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.too-many.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.too-many.profraw
RUN: printf '\2\0\0\0\0\0\0\0' >> %t.too-many.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t.too-many.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t.too-many.profraw
RUN: printf '\067\0\0\0\0\0\0\0' >> %t.too-many.profraw
RUN: printf '\101\0\0\0\0\0\0\0' >> %t.too-many.profraw
RUN: printf '\5\0\0\0\0\0\0\0' >> %t.too-many.profraw
diff --git a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test
index fb864f9810d481..e454de15c4bc7b 100644
--- a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test
+++ b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test
@@ -5,7 +5,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.profraw
// We should fail on this because the binary IDs is not a multiple of 8 bytes.
RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
diff --git a/llvm/test/tools/llvm-profdata/nocompress.test b/llvm/test/tools/llvm-profdata/nocompress.test
index 23657060e05e81..fe53f4611a1811 100644
--- a/llvm/test/tools/llvm-profdata/nocompress.test
+++ b/llvm/test/tools/llvm-profdata/nocompress.test
@@ -1,12 +1,11 @@
-You need a checkout of clang with compiler-rt to generate the
-binary file here. These shell commands can be used to regenerate
-it.
-$ SRC=path/to/llvm
-$ CFE=$SRC/tools/clang
-$ TESTDIR=$SRC/test/tools/llvm-profdata
-$ CFE_TESTDIR=$CFE/test/Profile
-$ clang -o a.out -fprofile-instr-generate $CFE_TESTDIR/c-general.c
-$ LLVM_PROFILE_FILE=$TESTDIR/Inputs/compressed.profraw ./a.out
+Generate this fixture with a matching Clang and compiler-rt that support zlib.
+Use a local source filename to keep machine paths out of the profile names.
+
+$ SRC=/path/to/llvm-project
+$ cp "$SRC/clang/test/Profile/c-general.c" c-general.c
+$ clang -fprofile-instr-generate c-general.c -o c-general
+$ LLVM_PROFILE_FILE=compressed.profraw ./c-general
+$ cp compressed.profraw "$SRC/llvm/test/tools/llvm-profdata/Inputs/"
RUN: not llvm-profdata show %p/Inputs/compressed.profraw -o %t 2>&1 | FileCheck %s
diff --git a/llvm/test/tools/llvm-profdata/profile-version.test b/llvm/test/tools/llvm-profdata/profile-version.test
index 7f627cb89c7d0a..3898a04c428a8d 100644
--- a/llvm/test/tools/llvm-profdata/profile-version.test
+++ b/llvm/test/tools/llvm-profdata/profile-version.test
@@ -2,7 +2,7 @@ Test the profile version.
RUN: llvm-profdata merge -o %t.profdata %p/Inputs/basic.proftext
RUN: llvm-profdata show --profile-version %t.profdata | FileCheck %s
-CHECK: Profile version: 14
+CHECK: Profile version: 15
RUN: llvm-profdata merge -o %t.prev.profdata %p/Inputs/basic.proftext --write-prev-version
RUN: llvm-profdata show --profile-version %t.prev.profdata | FileCheck %s --check-prefix=PREV
diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
index 353989dcaa3153..938447f0ae4fd3 100644
--- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
+++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test
@@ -6,7 +6,7 @@
UNSUPPORTED: system-windows, system-zos
// Header
RUN: printf '\377lprofR\201' > %t
-RUN: printf '\0\0\0\0\0\0\0\13' >> %t
+RUN: printf '\0\0\0\0\0\0\0\14' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test
index c919850e2293b6..97b4f4ad1ada29 100644
--- a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test
+++ b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test
@@ -5,7 +5,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\201Rforpl\377' > %t
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\2\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test
index c4108e95ea36cc..af0d9402b1e5db 100644
--- a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test
+++ b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test
@@ -5,7 +5,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\377lprofr\201' > %t
-RUN: printf '\0\0\0\0\0\0\0\13' >> %t
+RUN: printf '\0\0\0\0\0\0\0\14' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
@@ -32,17 +32,17 @@ RUN: printf '\0\0\0\3\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\1\0\0\0\0' >> %t
-RUN: printf '\0\0\0\0\0\0\0\3' >> %t
+RUN: printf '\0\0\0\0\0\0\0\3\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\344\023\165\112\031\035\265\067' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
-RUN: printf '\0\0\0\1\0\3\377\300' >> %t
+RUN: printf '\0\0\0\1\0\3\377\270' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-RUN: printf '\0\0\0\3\0\3\377\273' >> %t
+RUN: printf '\0\0\0\3\0\3\377\263' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\2\0\0\0\0' >> %t
-RUN: printf '\0\0\0\0\0\0\0\1' >> %t
+RUN: printf '\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\023' >> %t
RUN: printf '\0\0\0\0\0\0\0\067' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
index 301c34074a8fad..328ac552f9a35e 100644
--- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
+++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test
@@ -5,7 +5,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\201rforpl\377' > %t
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\2\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
@@ -32,17 +32,17 @@ RUN: printf '\0\0\4\0\3\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
-RUN: printf '\0\0\0\0\3\0\0\0' >> %t
+RUN: printf '\0\0\0\0\3\0\0\0\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\067\265\035\031\112\165\023\344' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
-RUN: printf '\300\377\3\0\1\0\0\0' >> %t
+RUN: printf '\270\377\3\0\1\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
-RUN: printf '\273\377\3\0\3\0\0\0' >> %t
+RUN: printf '\263\377\3\0\3\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
-RUN: printf '\0\0\0\0\1\0\0\0' >> %t
+RUN: printf '\0\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\023\0\0\0\0\0\0\0' >> %t
RUN: printf '\067\0\0\0\0\0\0\0' >> %t
diff --git a/llvm/test/tools/llvm-profdata/raw-two-profiles.test b/llvm/test/tools/llvm-profdata/raw-two-profiles.test
index 80fbba33e8e015..08ab6eedafa400 100644
--- a/llvm/test/tools/llvm-profdata/raw-two-profiles.test
+++ b/llvm/test/tools/llvm-profdata/raw-two-profiles.test
@@ -5,7 +5,7 @@
// TODO: use a builtin version of printf
UNSUPPORTED: system-windows, system-zos
RUN: printf '\201rforpl\377' > %t-foo.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
@@ -32,13 +32,13 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw
RUN: printf '\201rforpl\377' > %t-bar.profraw
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
@@ -65,7 +65,7 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw
-RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\067\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\101\0\0\0\0\0\0\0' >> %t-bar.profraw
diff --git a/llvm/test/tools/llvm-profdata/truncated-profile.test b/llvm/test/tools/llvm-profdata/truncated-profile.test
index b49549f3a28282..e6418c15b31068 100644
--- a/llvm/test/tools/llvm-profdata/truncated-profile.test
+++ b/llvm/test/tools/llvm-profdata/truncated-profile.test
@@ -5,8 +5,8 @@ UNSUPPORTED: system-zos
// Magic
RUN: printf '\201rforpl\377' > %t.profraw
-// Version (11 = 0o13)
-RUN: printf '\13\0\0\0\0\0\0\0' >> %t.profraw
+// Version (12 = 0o14)
+RUN: printf '\14\0\0\0\0\0\0\0' >> %t.profraw
// BinaryIdsSize
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
// NumData
@@ -46,4 +46,4 @@ RUN: printf '\3\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0' >> %t.profraw
RUN: not llvm-profdata show %t.profraw 2>&1 | FileCheck %s
-CHECK: invalid instrumentation profile data (file is incomplete or header is corrupt): profile file size (156 bytes) smaller than expected (at least 248 bytes: 152(Header) + 0(BinaryIdSize) + 72(DataSize) + 8(CountersSize) + 0(NumBitmapBytes) + 0(UniformCountersSectionSize) + 16(NamesSize) + 0(VTableSectionSize) + 0(VTableNameSize) + 0(Padding))
+CHECK: invalid instrumentation profile data (file is incomplete or header is corrupt): profile file size (156 bytes) smaller than expected (at least 256 bytes: 152(Header) + 0(BinaryIdSize) + 80(DataSize) + 8(CountersSize) + 0(NumBitmapBytes) + 0(UniformCountersSectionSize) + 16(NamesSize) + 0(VTableSectionSize) + 0(VTableNameSize) + 0(Padding))
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 1335daaeced6a0..043a5e48ad03f5 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -936,8 +936,14 @@ static Error mergeWriterContexts(WriterContext *Dst, WriterContext *Src) {
Dst->Errors.push_back(std::move(ErrorPair));
Src->Errors.clear();
- if (Error E = Dst->Writer.mergeProfileKind(Src->Writer.getProfileKind()))
- return makeError(std::move(E));
+ // Empty input files have no layout to reconcile. In particular, a worker
+ // that only saw empty files must not turn a dense wave profile into a mixed
+ // dense/sparse merge.
+ if (Src->Writer.getProfileKind() != InstrProfKind::Unknown ||
+ !Src->Writer.getProfileData().empty()) {
+ if (Error E = Dst->Writer.mergeProfileKind(Src->Writer.getProfileKind()))
+ return makeError(std::move(E));
+ }
Dst->Writer.mergeRecordsFromWriter(std::move(Src->Writer), [&](Error E) {
auto [ErrorCode, Msg] = InstrProfError::take(std::move(E));
@@ -1013,15 +1019,21 @@ static Error writeInstrProfile(StringRef OutputFilename,
if (EC)
return makeError(EC, OutputFilename);
- if (OutputFormat == PF_Text) {
- if (Error E = Writer.writeText(Output))
- warn(std::move(E));
- } else {
- if (Output.is_displayed())
- return makeError(
- "cannot write a non-text format profile to the terminal");
- if (Error E = Writer.write(Output))
- warn(std::move(E));
+ if (OutputFormat != PF_Text && Output.is_displayed())
+ return makeError("cannot write a non-text format profile to the terminal");
+
+ if (Error E = OutputFormat == PF_Text ? Writer.writeText(Output)
+ : Writer.write(Output)) {
+ // Unsupported formats cannot represent the profile at all. Preserve the
+ // existing warning behavior for recoverable profile-data errors.
+ bool UnsupportedVersion = false;
+ E = handleErrors(std::move(E), [&](const InstrProfError &IPE) -> Error {
+ UnsupportedVersion = IPE.get() == instrprof_error::unsupported_version;
+ return make_error<InstrProfError>(IPE.get(), IPE.getMessage());
+ });
+ if (UnsupportedVersion)
+ return makeError(std::move(E), OutputFilename);
+ warn(std::move(E));
}
return Error::success();
}
@@ -3034,8 +3046,9 @@ static Error showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
if (doTextFormatDump) {
InstrProfSymtab &Symtab = Reader->getSymtab();
- InstrProfWriter::writeRecordInText(Func.Name, Func.Hash, Func, Symtab,
- OS);
+ if (Error E = InstrProfWriter::writeRecordInText(Func.Name, Func.Hash,
+ Func, Symtab, OS))
+ return makeError(std::move(E), Filename);
continue;
}
@@ -3121,6 +3134,13 @@ static Error showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
}
OS << "]\n";
+ if (!Func.WaveCounts.empty()) {
+ OS << " Block wave counts: [";
+ for (size_t I = 0, E = Func.WaveCounts.size(); I < E; ++I)
+ OS << (I ? ", " : "") << Func.WaveCounts[I];
+ OS << "]\n";
+ }
+
// Show uniformity bits if present
if (!Func.UniformityBits.empty()) {
OS << " Block uniformity: [";
@@ -3164,6 +3184,7 @@ static Error showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
if (IsIR) {
OS << " entry_first = " << Reader->instrEntryBBEnabled();
OS << " instrument_loop_entries = " << Reader->instrLoopEntriesEnabled();
+ OS << " dense_wave = " << Reader->hasDenseWaveProfile();
}
OS << "\n";
if (ShowAllFunctions || !FuncNameFilter.empty())
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index 0c09467169abe3..8a9464d45393ae 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -112,6 +112,97 @@ static const auto Err = [](Error E) {
FAIL();
};
+TEST_F(InstrProfTest, WaveCountsRoundTripAndMerge) {
+ NamedInstrProfRecord A("wave", 123, {6400, 3200});
+ A.WaveCounts = {100, 100};
+ Writer.addRecord(std::move(A), Err);
+ NamedInstrProfRecord B("wave", 123, {640, 320});
+ B.WaveCounts = {10, 10};
+ Writer.addRecord(std::move(B), 2, Err);
+ Writer.addRecord({"plain", 124, {42}}, Err);
+ readProfile(Writer.writeBuffer());
+ auto R = Reader->getInstrProfRecord("wave", 123);
+ ASSERT_THAT_ERROR(R.takeError(), Succeeded());
+ EXPECT_THAT(R->Counts, ElementsAre(7680, 3840));
+ EXPECT_THAT(R->WaveCounts, ElementsAre(120, 120));
+ auto Plain = Reader->getInstrProfRecord("plain", 124);
+ ASSERT_THAT_ERROR(Plain.takeError(), Succeeded());
+ EXPECT_TRUE(Plain->WaveCounts.empty());
+ InstrProfWriter Again;
+ for (auto &Record : *Reader)
+ Again.addRecord(NamedInstrProfRecord(Record), Err);
+ readProfile(Again.writeBuffer());
+ R = Reader->getInstrProfRecord("wave", 123);
+ ASSERT_THAT_ERROR(R.takeError(), Succeeded());
+ EXPECT_THAT(R->WaveCounts, ElementsAre(120, 120));
+}
+
+TEST_F(InstrProfTest, WaveCountsMismatchDoesNotMutate) {
+ InstrProfRecord A({64});
+ A.WaveCounts = {1};
+ InstrProfRecord B({128});
+ bool Warned = false;
+ A.merge(B, 1, [&](instrprof_error E) {
+ EXPECT_EQ(E, instrprof_error::count_mismatch);
+ Warned = true;
+ });
+ EXPECT_TRUE(Warned);
+ EXPECT_THAT(A.Counts, ElementsAre(64));
+ EXPECT_THAT(A.WaveCounts, ElementsAre(1));
+}
+
+TEST_F(InstrProfTest, DenseWaveLayoutRoundTrip) {
+ ASSERT_THAT_ERROR(
+ Writer.mergeProfileKind(InstrProfKind::IRInstrumentation |
+ InstrProfKind::DenseWaveInstrumentation),
+ Succeeded());
+ NamedInstrProfRecord R("wave", 123, {64, 0});
+ R.WaveCounts = {1, 2};
+ Writer.addRecord(std::move(R), Err);
+ readProfile(Writer.writeBuffer());
+ EXPECT_TRUE(Reader->hasDenseWaveProfile());
+ auto Record = Reader->getInstrProfRecord("wave", 123);
+ ASSERT_THAT_ERROR(Record.takeError(), Succeeded());
+ EXPECT_THAT(Record->Counts, ElementsAre(64, 0));
+ EXPECT_THAT(Record->WaveCounts, ElementsAre(1, 2));
+}
+
+TEST_F(InstrProfTest, DenseWaveLayoutMismatchDoesNotMutate) {
+ auto Sparse = InstrProfKind::IRInstrumentation;
+ auto Dense = Sparse | InstrProfKind::DenseWaveInstrumentation;
+ for (bool DenseFirst : {false, true}) {
+ InstrProfWriter W;
+ auto First = DenseFirst ? Dense : Sparse;
+ auto Second = DenseFirst ? Sparse : Dense;
+ ASSERT_THAT_ERROR(W.mergeProfileKind(First), Succeeded());
+ EXPECT_TRUE(ErrorEquals(instrprof_error::unsupported_version,
+ W.mergeProfileKind(Second)));
+ EXPECT_EQ(W.getProfileKind(), First);
+ EXPECT_THAT_ERROR(W.mergeProfileKind(First), Succeeded());
+ }
+}
+
+TEST_F(InstrProfTest, WaveCountsScaleCopyClearAndOverflow) {
+ InstrProfRecord A({64, 128});
+ A.WaveCounts = {2, 4};
+ A.scale(3, 2, [](instrprof_error) { FAIL(); });
+ EXPECT_THAT(A.WaveCounts, ElementsAre(3, 6));
+ InstrProfRecord B(A), C;
+ C = B;
+ EXPECT_EQ(C.WaveCounts, A.WaveCounts);
+ C.Clear();
+ EXPECT_TRUE(C.WaveCounts.empty());
+ A.WaveCounts = {getInstrMaxCountValue(), 1};
+ B.WaveCounts = {1, 1};
+ bool Overflowed = false;
+ A.merge(B, 1, [&](instrprof_error E) {
+ EXPECT_EQ(E, instrprof_error::counter_overflow);
+ Overflowed = true;
+ });
+ EXPECT_TRUE(Overflowed);
+ EXPECT_THAT(A.WaveCounts, ElementsAre(getInstrMaxCountValue(), 2));
+}
+
TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) {
Writer.addRecord({"foo", 0x1234, {1, 2, 3, 4}}, Err);
auto Profile = Writer.writeBuffer();
More information about the llvm-branch-commits
mailing list