[llvm-branch-commits] [llvm] [PGO] Load GPU wave counts into IR metadata (PR #225911)
Yaxun Liu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Sep 28 18:02:57 PDT 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/225911
>From e655a710e1e0349cb04fb4437cda891a20164ed0 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 23 Sep 2026 12:51:45 -0400
Subject: [PATCH 1/3] [PGO] Load GPU wave counts into IR metadata
GPU profiles contain wave counts alongside lane counts, but profile use
does not expose them to optimizations. Wave visits do not obey scalar
flow conservation, so unmeasured blocks cannot use counts reconstructed
from neighboring blocks or ordinary branch weights.
Map wave-counter indices to the blocks selected by PGO instrumentation,
after reproducing its critical-edge splits. Attach measured counts using
wave.profile metadata, retaining measured zeros and marking other blocks
unmeasured. Require a measured entry count for normalization and exclude
select-counter slots from the block mapping.
Validate the wave-counter layout against the accepted lane profile.
Clear old wave metadata when loading a replacement profile, including
when a function has no usable record. Do not emit wave metadata for
previously profiled functions: their branch weights may change counter
placement without changing the CFG hash. Keep ordinary lane-count
reconstruction and branch weights unchanged.
---
.../Instrumentation/PGOInstrumentation.cpp | 44 ++++
.../Transforms/PGOProfile/wave-profile-use.ll | 212 ++++++++++++++++++
2 files changed, 256 insertions(+)
create mode 100644 llvm/test/Transforms/PGOProfile/wave-profile-use.ll
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 516d6b7047856..550b92f5eb3d7 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -51,6 +51,7 @@
#include "ValueProfileCollector.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
@@ -1314,6 +1315,8 @@ class PGOUseFunc {
// Find the Instrumented BB and set the value. Return false on error.
bool setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
+ void setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs);
+
// Set the edge counter value for the unknown edge -- there should be only
// one unknown edge.
void setEdgeCount(DirectEdges &Edges, uint64_t Value);
@@ -1349,6 +1352,43 @@ static void setupBBInfoEdges(
}
}
+// Wave slots use the same indices as lane counters, including the trailing
+// select counters. Only block-counter slots describe block-entry events.
+void PGOUseFunc::setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs) {
+ if (ProfileRecord.WaveCounts.empty() || !isGPUProfTarget(*M) || IsCS)
+ return;
+ if (ProfileRecord.WaveCounts.size() != ProfileRecord.Counts.size()) {
+ F.getContext().diagnose(DiagnosticInfoPGOProfile(
+ M->getName().data(),
+ Twine("Inconsistent number of wave counts in ") + F.getName() +
+ "; ignoring wave profile",
+ DS_Warning));
+ return;
+ }
+
+ // Earlier profile annotations can change the instrumentation MST without
+ // changing the CFG hash. Do not reuse its counter indices for wave counts.
+ if (F.getMetadata(LLVMContext::MD_prof))
+ return;
+
+ DenseMap<const BasicBlock *, uint64_t> MeasuredCounts;
+ for (auto [Index, BB] : enumerate(InstrumentBBs))
+ MeasuredCounts.try_emplace(BB, ProfileRecord.WaveCounts[Index]);
+ // Lane-flow reconstruction cannot provide a missing wave normalization count.
+ if (!MeasuredCounts.contains(&F.getEntryBlock()))
+ return;
+
+ SmallVector<uint64_t> Counts;
+ BitVector HasCounts;
+ for (const BasicBlock &BB : F) {
+ auto It = MeasuredCounts.find(&BB);
+ bool HasCount = It != MeasuredCounts.end();
+ Counts.push_back(HasCount ? It->second : 0);
+ HasCounts.push_back(HasCount);
+ }
+ setBlockWaveCounts(F, Counts, HasCounts);
+}
+
// Visit all the edges and assign the count value for the instrumented
// edges and the BB. Return false on error.
bool PGOUseFunc::setInstrumentedCounts(
@@ -1384,6 +1424,7 @@ bool PGOUseFunc::setInstrumentedCounts(
});
return false;
}
+ setWaveCounts(InstrumentBBs);
auto *FuncEntry = &*F.begin();
// Set the profile count to the Instrumented BBs.
@@ -2345,6 +2386,9 @@ static bool annotateAllFunctions(
bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
for (auto &F : M) {
+ // A replacement profile must not leave an earlier wave mapping live when
+ // its record is absent, mismatched, or contains only lane counts.
+ clearBlockWaveCounts(F);
if (skipPGOUse(F))
continue;
TargetLibraryInfo &TLI = LookupTLI(F);
diff --git a/llvm/test/Transforms/PGOProfile/wave-profile-use.ll b/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
new file mode 100644
index 0000000000000..9728dd9a9a717
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
@@ -0,0 +1,212 @@
+; RUN: split-file %s %t
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -S %t/input.ll | FileCheck %s --check-prefix=GEN
+; RUN: %python %t/raw.py wave > %t/wave.raw
+; RUN: llvm-profdata merge %t/wave.raw -o %t/wave.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -S %t/input.ll -o %t/used.ll
+; RUN: FileCheck %s --check-prefixes=WAVE,COUNTS < %t/used.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -S %t/used.ll | FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py lane > %t/lane.raw
+; RUN: llvm-profdata merge %t/lane.raw -o %t/lane.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/lane.profdata -S %t/used.ll | FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py bad-wave > %t/bad-wave.raw
+; RUN: llvm-profdata merge %t/bad-wave.raw -o %t/bad-wave.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/bad-wave.profdata -S %t/used.ll -o %t/bad-wave.ll 2>&1 | FileCheck %s --check-prefix=WARN
+; RUN: FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile < %t/bad-wave.ll
+; RUN: %python %t/raw.py zero > %t/zero.raw
+; RUN: llvm-profdata merge %t/zero.raw -o %t/zero.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/zero.profdata -S %t/input.ll | FileCheck %s --check-prefix=ZERO
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/zero.profdata -S %t/used.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py no-entry > %t/no-entry.raw
+; RUN: llvm-profdata merge %t/no-entry.raw -o %t/no-entry.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/no-entry.profdata -S %t/input.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/no-entry.profdata -S %t/used.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py missing > %t/missing.raw
+; RUN: llvm-profdata merge %t/missing.raw -o %t/missing.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/missing.profdata -S %t/used.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py hash > %t/hash.raw
+; RUN: llvm-profdata merge %t/hash.raw -o %t/hash.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/hash.profdata -S %t/used.ll -o %t/hash.ll 2>&1 | FileCheck %s --check-prefix=MISMATCH
+; RUN: FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile < %t/hash.ll
+; RUN: %python %t/raw.py bad-lane > %t/bad-lane.raw
+; RUN: llvm-profdata merge %t/bad-lane.raw -o %t/bad-lane.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/bad-lane.profdata -S %t/used.ll -o %t/bad-lane.ll 2>&1 | FileCheck %s --check-prefix=BAD-LANE
+; RUN: FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile < %t/bad-lane.ll
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -S %t/used.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -S %t/critical.ll | FileCheck %s --check-prefix=CGEN
+; RUN: %python %t/raw.py critical > %t/critical.raw
+; RUN: llvm-profdata merge %t/critical.raw -o %t/critical.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/critical.profdata -S %t/critical.ll | FileCheck %s --check-prefix=CRIT
+
+; RUN: %python %t/raw.py skew > %t/skew.raw
+; RUN: llvm-profdata merge %t/skew.raw -o %t/skew.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/skew.profdata -S %t/input.ll -o %t/skew.ll
+; RUN: FileCheck %s --check-prefix=SKEW < %t/skew.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/skew.profdata -S %t/skew.ll -o %t/reapplied.ll
+; RUN: FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile < %t/reapplied.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/skew.profdata -S %t/reapplied.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -S %t/skew.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py skew-lane > %t/skew-lane.raw
+; RUN: llvm-profdata merge %t/skew-lane.raw -o %t/skew-lane.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/skew-lane.profdata -S %t/input.ll -o %t/skew-lane.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/skew.profdata -S %t/skew-lane.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
+
+; Wave slots follow instrumentation indices, not IR block order. Only entry and
+; b have block counters. The trailing select counter must not make exit measured.
+; Generation checks keep the raw fixture aligned with the actual producer.
+; GEN-LABEL: define void @diamond
+; GEN: entry:
+; GEN: call void @llvm.instrprof.increment({{.*}}i64 [[HASH:942389667449461396]], i32 3, i32 0)
+; GEN: b:
+; GEN: call void @llvm.instrprof.increment({{.*}}i64 [[HASH]], i32 3, i32 1)
+; GEN: exit:
+; GEN: call void @llvm.instrprof.increment.step({{.*}}i64 [[HASH]], i32 3, i32 2,
+
+; A measured zero is retained even when all lane counters are zero. Blocks with
+; no measurements remain unavailable. Replacing a profile clears stale metadata
+; for absent, incompatible, lane-only, and unsupported-target records.
+; NONE: define void @diamond
+; WAVE-LABEL: define void @diamond
+; WAVE-SAME: !wave.profile ![[WAVES:[0-9]+]]
+; WAVE: br i1 %cond, label %a, label %b, !prof ![[WEIGHTS:[0-9]+]]{{.*}}!wave.profile.block ![[ENTRY:[0-9]+]]
+; WAVE: br label %exit, !wave.profile.block ![[A:[0-9]+]]
+; WAVE: br label %exit, {{.*}}!wave.profile.block ![[B:[0-9]+]]
+; WAVE: ret void, !wave.profile.block ![[EXIT:[0-9]+]]
+; WAVE-DAG: ![[WAVES]] = distinct !{i64 2, i64 [[FID:-?[0-9]+]], i64 100, i64 0, i64 100, i64 0}
+; WAVE-DAG: ![[ENTRY]] = !{i64 2, i64 [[FID]], i64 0, i64 1, i64 1, i64 2}
+; WAVE-DAG: ![[A]] = !{i64 2, i64 [[FID]], i64 1, i64 0, i64 3}
+; WAVE-DAG: ![[B]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 3}
+; WAVE-DAG: ![[EXIT]] = !{i64 2, i64 [[FID]], i64 3, i64 0}
+; COUNTS-DAG: !{!"branch_weights", i32 3200, i32 3200}
+; COUNTS-DAG: !{!"branch_weights", i32 1600, i32 4800}
+; ZERO-LABEL: define void @diamond
+; ZERO-SAME: !wave.profile ![[WAVES:[0-9]+]]
+; ZERO: br i1 %cond, label %a, label %b, {{.*}}!wave.profile.block ![[ENTRY:[0-9]+]]
+; ZERO: br label %exit, !wave.profile.block ![[A:[0-9]+]]
+; ZERO: br label %exit, {{.*}}!wave.profile.block ![[B:[0-9]+]]
+; ZERO: ret void, !wave.profile.block ![[EXIT:[0-9]+]]
+; ZERO-DAG: ![[WAVES]] = distinct !{i64 2, i64 [[FID:-?[0-9]+]], i64 0, i64 0, i64 0, i64 0}
+; ZERO-DAG: ![[ENTRY]] = !{i64 2, i64 [[FID]], i64 0, i64 1, i64 1, i64 2}
+; ZERO-DAG: ![[A]] = !{i64 2, i64 [[FID]], i64 1, i64 0, i64 3}
+; ZERO-DAG: ![[B]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 3}
+; ZERO-DAG: ![[EXIT]] = !{i64 2, i64 [[FID]], i64 3, i64 0}
+; WARN: Inconsistent number of wave counts in diamond; ignoring wave profile
+; MISMATCH: function control flow change detected (hash mismatch) diamond
+; BAD-LANE: Inconsistent number of counts in diamond
+
+; Existing branch weights can change the instrumentation MST without changing
+; the CFG hash. Reapplying this skewed profile would move counter 1 from b to a.
+; Drop wave metadata on previously profiled functions, including after a lane-only
+; load, and do not recreate it on a subsequent load.
+; SKEW-LABEL: define void @diamond
+; SKEW-SAME: !wave.profile ![[WAVES:[0-9]+]]
+; SKEW: br i1 %cond, label %a, label %b, !prof ![[WEIGHTS:[0-9]+]]
+; SKEW: a:
+; SKEW: br label %exit, !wave.profile.block ![[A:[0-9]+]]
+; SKEW: b:
+; SKEW: br label %exit, {{.*}}!wave.profile.block ![[B:[0-9]+]]
+; SKEW-DAG: ![[WAVES]] = distinct !{i64 2, i64 [[FID:-?[0-9]+]], i64 100, i64 0, i64 90, i64 0}
+; SKEW-DAG: ![[WEIGHTS]] = !{!"branch_weights", i32 640, i32 5760}
+; SKEW-DAG: ![[A]] = !{i64 2, i64 [[FID]], i64 1, i64 0, i64 3}
+; SKEW-DAG: ![[B]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 3}
+
+; Critical edges are split before mapping counter indices to block identities.
+; CGEN-LABEL: define void @critical
+; CGEN: entry:
+; CGEN: call void @llvm.instrprof.increment({{.*}}i64 [[CHASH:784007059655560962]], i32 2, i32 0)
+; CGEN: entry.exit_crit_edge:
+; CGEN: call void @llvm.instrprof.increment({{.*}}i64 [[CHASH]], i32 2, i32 1)
+; CRIT-LABEL: define void @critical
+; CRIT-SAME: !wave.profile ![[CWAVES:[0-9]+]]
+; CRIT: br i1 %cond, label %a, label %entry.exit_crit_edge, {{.*}}!wave.profile.block ![[CENT:[0-9]+]]
+; CRIT: entry.exit_crit_edge:
+; CRIT: br label %exit, {{.*}}!wave.profile.block ![[CEDGE:[0-9]+]]
+; CRIT: a:
+; CRIT: br label %exit, !wave.profile.block ![[CA:[0-9]+]]
+; CRIT: ret void, !wave.profile.block ![[CEXIT:[0-9]+]]
+; CRIT-DAG: ![[CWAVES]] = distinct !{i64 2, i64 [[CFID:-?[0-9]+]], i64 100, i64 25, i64 0, i64 0}
+; CRIT-DAG: ![[CENT]] = !{i64 2, i64 [[CFID]], i64 0, i64 1, i64 2, i64 1}
+; CRIT-DAG: ![[CEDGE]] = !{i64 2, i64 [[CFID]], i64 1, i64 1, i64 3}
+; CRIT-DAG: ![[CA]] = !{i64 2, i64 [[CFID]], i64 2, i64 0, i64 3}
+; CRIT-DAG: ![[CEXIT]] = !{i64 2, i64 [[CFID]], i64 3, i64 0}
+
+;--- input.ll
+source_filename = "wave-profile-use.ll"
+target triple = "amdgcn-amd-amdhsa"
+define void @diamond(i1 %cond, i1 %select_cond, ptr %p) {
+entry:
+ br i1 %cond, label %a, label %b
+a:
+ store volatile i32 1, ptr %p
+ br label %exit
+b:
+ store volatile i32 2, ptr %p
+ br label %exit
+exit:
+ %value = select i1 %select_cond, i32 1, i32 2
+ store volatile i32 %value, ptr %p
+ ret void
+}
+
+;--- critical.ll
+source_filename = "wave-profile-critical.ll"
+target triple = "amdgcn-amd-amdhsa"
+define void @critical(i1 %cond, ptr %p) {
+entry:
+ br i1 %cond, label %a, label %exit, !prof !0
+a:
+ store volatile i32 1, ptr %p
+ br label %exit
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1000000, i32 1}
+
+;--- raw.py
+import hashlib
+import struct
+import sys
+
+mode = sys.argv[1]
+name = {"missing": b"missing", "critical": b"critical"}.get(mode, b"diamond")
+name_ref = int.from_bytes(hashlib.md5(name).digest()[:8], "little")
+# Hash emitted by pgo-instr-gen for input.ll; raw v12 has an 80-byte record.
+func_hash = 942389667449461396
+if mode == "hash":
+ func_hash += 1
+lanes = [6400, 3200, 1600]
+waves = [100, 100, 100]
+if mode == "critical":
+ func_hash = 784007059655560962
+ lanes = [6400, 800]
+ waves = [100, 25]
+if mode in ("skew", "skew-lane"):
+ lanes = [6400, 5760, 1600]
+ waves = [100, 90, 100]
+if mode in ("lane", "skew-lane"):
+ waves = []
+elif mode == "bad-wave":
+ waves.pop()
+elif mode == "bad-lane":
+ lanes.pop()
+elif mode == "zero":
+ lanes = [0, 0, 0]
+ waves = [0, 0, 0]
+version = 12 | (1 << 56)
+if mode != "no-entry":
+ version |= 1 << 58
+num_counters = len(lanes) + len(waves)
+counter_delta = 80
+uniform_delta = counter_delta + num_counters * 8
+names_delta = uniform_delta + len(lanes) * 8
+names = bytes([len(name), 0]) + name
+header = [0xff6c70726f667281, version, 0, 1, 0, num_counters, 0,
+ 0, 0, len(lanes), 0, uniform_delta, len(names), counter_delta,
+ uniform_delta, names_delta, 0, 0, 2]
+record = struct.pack("<7QI4HII4x", name_ref, func_hash, counter_delta,
+ uniform_delta, 0, 0, 0, num_counters, 0, 0, 0, 64, 0,
+ len(waves))
+counts = lanes + waves + [0] * len(lanes)
+sys.stdout.buffer.write(struct.pack("<19Q", *header) + record +
+ struct.pack("<" + "Q" * len(counts), *counts) +
+ names + bytes((-len(names)) % 8))
>From dcef0bbf0dd755c4d065a2201e0813650a2ae163 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 28 Sep 2026 11:00:28 -0400
Subject: [PATCH 2/3] [PGO] Add a debugging switch for wave profile metadata
Add the hidden pgo-wave-metadata option, enabled by default, for debugging,
performance comparisons and disabling wave annotations when investigating
regressions without turning off ordinary PGO or uniformity hints.
Gate wave metadata emission while retaining the existing clearing of stale
function and block annotations during profile use. Profile collection and
ordinary count reconstruction are unchanged.
Test default/explicit enablement, disabling, retained counts and uniformity
hints, and replacement profiles with missing, mismatched or zero counts.
---
.../Instrumentation/PGOInstrumentation.cpp | 7 +++-
.../Transforms/PGOProfile/wave-profile-use.ll | 35 ++++++++++++++++++-
2 files changed, 40 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 550b92f5eb3d7..c7dab41b7ed3d 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -177,6 +177,10 @@ static cl::opt<bool> DisableValueProfiling("disable-vp", cl::init(false),
cl::Hidden,
cl::desc("Disable Value Profiling"));
+static cl::opt<bool> PGOWaveMetadata(
+ "pgo-wave-metadata", cl::init(true), cl::Hidden,
+ cl::desc("Enable wave count profile metadata during PGO use"));
+
// Command line option to set the maximum number of VP annotations to write to
// the metadata for a single indirect call callsite.
static cl::opt<unsigned> MaxNumAnnotations(
@@ -1355,7 +1359,8 @@ static void setupBBInfoEdges(
// Wave slots use the same indices as lane counters, including the trailing
// select counters. Only block-counter slots describe block-entry events.
void PGOUseFunc::setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs) {
- if (ProfileRecord.WaveCounts.empty() || !isGPUProfTarget(*M) || IsCS)
+ if (!PGOWaveMetadata || ProfileRecord.WaveCounts.empty() ||
+ !isGPUProfTarget(*M) || IsCS)
return;
if (ProfileRecord.WaveCounts.size() != ProfileRecord.Counts.size()) {
F.getContext().diagnose(DiagnosticInfoPGOProfile(
diff --git a/llvm/test/Transforms/PGOProfile/wave-profile-use.ll b/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
index 9728dd9a9a717..0d9f1ea0d3408 100644
--- a/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
+++ b/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
@@ -4,6 +4,10 @@
; RUN: llvm-profdata merge %t/wave.raw -o %t/wave.profdata
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -S %t/input.ll -o %t/used.ll
; RUN: FileCheck %s --check-prefixes=WAVE,COUNTS < %t/used.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -pgo-wave-metadata=true -S %t/input.ll -o %t/enabled.ll
+; RUN: diff %t/used.ll %t/enabled.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -pgo-wave-metadata=false -S %t/input.ll | FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -pgo-wave-metadata=false -S %t/used.ll | FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -S %t/used.ll | FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
; RUN: %python %t/raw.py lane > %t/lane.raw
; RUN: llvm-profdata merge %t/lane.raw -o %t/lane.profdata
@@ -14,6 +18,8 @@
; RUN: FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile < %t/bad-wave.ll
; RUN: %python %t/raw.py zero > %t/zero.raw
; RUN: llvm-profdata merge %t/zero.raw -o %t/zero.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/zero.profdata -pgo-wave-metadata=false -S %t/input.ll | FileCheck %s --check-prefix=ZERO-OFF --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/zero.profdata -pgo-wave-metadata=false -S %t/used.ll | FileCheck %s --check-prefix=ZERO-OFF --implicit-check-not=wave.profile
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/zero.profdata -S %t/input.ll | FileCheck %s --check-prefix=ZERO
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/zero.profdata -S %t/used.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
; RUN: %python %t/raw.py no-entry > %t/no-entry.raw
@@ -22,9 +28,11 @@
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/no-entry.profdata -S %t/used.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
; RUN: %python %t/raw.py missing > %t/missing.raw
; RUN: llvm-profdata merge %t/missing.raw -o %t/missing.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/missing.profdata -pgo-wave-metadata=false -S %t/used.ll | FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/missing.profdata -S %t/used.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
; RUN: %python %t/raw.py hash > %t/hash.raw
; RUN: llvm-profdata merge %t/hash.raw -o %t/hash.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/hash.profdata -pgo-wave-metadata=false -S %t/used.ll | FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/hash.profdata -S %t/used.ll -o %t/hash.ll 2>&1 | FileCheck %s --check-prefix=MISMATCH
; RUN: FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile < %t/hash.ll
; RUN: %python %t/raw.py bad-lane > %t/bad-lane.raw
@@ -50,6 +58,15 @@
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/skew-lane.profdata -S %t/input.ll -o %t/skew-lane.ll
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/skew.profdata -S %t/skew-lane.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py uniform-wave > %t/uniform.raw
+; RUN: llvm-profdata merge %t/uniform.raw -o %t/uniform.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/uniform.profdata -S %t/input.ll -o %t/uniform.ll
+; RUN: FileCheck %s --check-prefixes=UNIFORM-WAVE,COUNTS < %t/uniform.ll
+; RUN: FileCheck %s --check-prefix=UNIFORM < %t/uniform.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/uniform.profdata -pgo-wave-metadata=false -S %t/input.ll -o %t/uniform-no-wave.ll
+; RUN: FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile < %t/uniform-no-wave.ll
+; RUN: FileCheck %s --check-prefix=UNIFORM < %t/uniform-no-wave.ll
+
; Wave slots follow instrumentation indices, not IR block order. Only entry and
; b have block counters. The trailing select counter must not make exit measured.
; Generation checks keep the raw fixture aligned with the actual producer.
@@ -65,6 +82,7 @@
; no measurements remain unavailable. Replacing a profile clears stale metadata
; for absent, incompatible, lane-only, and unsupported-target records.
; NONE: define void @diamond
+; UNIFORM-WAVE: define void @diamond{{.*}}!wave.profile
; WAVE-LABEL: define void @diamond
; WAVE-SAME: !wave.profile ![[WAVES:[0-9]+]]
; WAVE: br i1 %cond, label %a, label %b, !prof ![[WEIGHTS:[0-9]+]]{{.*}}!wave.profile.block ![[ENTRY:[0-9]+]]
@@ -76,6 +94,8 @@
; WAVE-DAG: ![[A]] = !{i64 2, i64 [[FID]], i64 1, i64 0, i64 3}
; WAVE-DAG: ![[B]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 3}
; WAVE-DAG: ![[EXIT]] = !{i64 2, i64 [[FID]], i64 3, i64 0}
+; UNIFORM-WAVE-DAG: !{i64 2, i64 {{-?[0-9]+}}, i64 100, i64 0, i64 50, i64 0}
+; COUNTS-DAG: !{!"function_entry_count", i64 6400}
; COUNTS-DAG: !{!"branch_weights", i32 3200, i32 3200}
; COUNTS-DAG: !{!"branch_weights", i32 1600, i32 4800}
; ZERO-LABEL: define void @diamond
@@ -89,6 +109,16 @@
; ZERO-DAG: ![[A]] = !{i64 2, i64 [[FID]], i64 1, i64 0, i64 3}
; ZERO-DAG: ![[B]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 3}
; ZERO-DAG: ![[EXIT]] = !{i64 2, i64 [[FID]], i64 3, i64 0}
+; ZERO-OFF: define void @diamond
+; ZERO-OFF: !{!"function_entry_count", i64 0}
+
+; The wave metadata switch leaves function, block and branch uniformity hints
+; available, together with the ordinary entry, branch and select counts.
+; UNIFORM: define void @diamond({{.*}}!uniformity.profile
+; UNIFORM: br i1 %cond, {{.*}}!block.uniformity.profile{{.*}}!branch.uniformity.profile
+; UNIFORM: b:
+; UNIFORM: br label %exit, {{.*}}!block.uniformity.profile
+
; WARN: Inconsistent number of wave counts in diamond; ignoring wave profile
; MISMATCH: function control flow change detected (hash mismatch) diamond
; BAD-LANE: Inconsistent number of counts in diamond
@@ -176,6 +206,8 @@ if mode == "hash":
func_hash += 1
lanes = [6400, 3200, 1600]
waves = [100, 100, 100]
+if mode == "uniform-wave":
+ waves[1] = 50
if mode == "critical":
func_hash = 784007059655560962
lanes = [6400, 800]
@@ -206,7 +238,8 @@ header = [0xff6c70726f667281, version, 0, 1, 0, num_counters, 0,
record = struct.pack("<7QI4HII4x", name_ref, func_hash, counter_delta,
uniform_delta, 0, 0, 0, num_counters, 0, 0, 0, 64, 0,
len(waves))
-counts = lanes + waves + [0] * len(lanes)
+uniform = lanes if mode == "uniform-wave" else [0] * len(lanes)
+counts = lanes + waves + uniform
sys.stdout.buffer.write(struct.pack("<19Q", *header) + record +
struct.pack("<" + "Q" * len(counts), *counts) +
names + bytes((-len(names)) % 8))
>From 5ef0a66c888dc2e46ffc2af9a67f2d78208e46e8 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 28 Sep 2026 20:47:36 -0400
Subject: [PATCH 3/3] [PGO] Load dense block wave counts from device profiles
Use the profile's dense layout flag to map appended wave-only slots after
the original block/select prefix. Reuse the producer's block selection so
eligible loop and reconvergence blocks retain their directly measured wave
frequencies. Keep select slots out of the block mapping.
Test dense and sparse profiles, measured zeros, generation and metadata
switches, and exact loop block identities after critical-edge splitting.
---
.../Instrumentation/PGOInstrumentation.cpp | 15 ++++--
.../Transforms/PGOProfile/dense-wave-cfg.ll | 16 +++++-
.../Transforms/PGOProfile/wave-profile-use.ll | 52 +++++++++++++++++--
3 files changed, 75 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index c7dab41b7ed3d..69abe8e218c30 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1319,7 +1319,8 @@ class PGOUseFunc {
// Find the Instrumented BB and set the value. Return false on error.
bool setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
- void setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs);
+ void setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs,
+ ArrayRef<BasicBlock *> ExtraWaveBBs);
// Set the edge counter value for the unknown edge -- there should be only
// one unknown edge.
@@ -1357,8 +1358,10 @@ static void setupBBInfoEdges(
}
// Wave slots use the same indices as lane counters, including the trailing
-// select counters. Only block-counter slots describe block-entry events.
-void PGOUseFunc::setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs) {
+// select counters, followed by any dense block measurements. Select slots do
+// not describe block-entry events.
+void PGOUseFunc::setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs,
+ ArrayRef<BasicBlock *> ExtraWaveBBs) {
if (!PGOWaveMetadata || ProfileRecord.WaveCounts.empty() ||
!isGPUProfTarget(*M) || IsCS)
return;
@@ -1379,6 +1382,10 @@ void PGOUseFunc::setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs) {
DenseMap<const BasicBlock *, uint64_t> MeasuredCounts;
for (auto [Index, BB] : enumerate(InstrumentBBs))
MeasuredCounts.try_emplace(BB, ProfileRecord.WaveCounts[Index]);
+ unsigned Index =
+ InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
+ for (BasicBlock *BB : ExtraWaveBBs)
+ MeasuredCounts.try_emplace(BB, ProfileRecord.WaveCounts[Index++]);
// Lane-flow reconstruction cannot provide a missing wave normalization count.
if (!MeasuredCounts.contains(&F.getEntryBlock()))
return;
@@ -1429,7 +1436,7 @@ bool PGOUseFunc::setInstrumentedCounts(
});
return false;
}
- setWaveCounts(InstrumentBBs);
+ setWaveCounts(InstrumentBBs, ExtraWaveBBs);
auto *FuncEntry = &*F.begin();
// Set the profile count to the Instrumented BBs.
diff --git a/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll b/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll
index b6f4924fc6632..6dc679e474278 100644
--- a/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll
+++ b/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll
@@ -3,7 +3,8 @@
; RUN: %python %t/raw.py > %t/loop.raw
; RUN: llvm-profdata merge %t/loop.raw -o %t/loop.profdata
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/loop.profdata -S %t/loop.ll -o %t/use.ll
-; RUN: FileCheck %s --check-prefix=COUNTS < %t/use.ll
+; RUN: FileCheck %s --check-prefixes=COUNTS,WAVE < %t/use.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/loop.profdata -pgo-wave-metadata=false -S %t/loop.ll | FileCheck %s --check-prefix=COUNTS --implicit-check-not=wave.profile
; RUN: opt -passes=pgo-instr-gen,verify -pgo-instrument-entry -S %t/eh.ll | FileCheck %s --check-prefix=EH --implicit-check-not='call void @llvm.instrprof.increment'
; Dense instrumentation preserves the entry and split backedge counters,
@@ -27,9 +28,22 @@
; header has four times the entry wave count; sparse data cannot supply it.
; COUNTS-LABEL: define void @loop(
; COUNTS-SAME: !prof ![[ENTRY_COUNT:[0-9]+]]
+; WAVE-SAME: !wave.profile ![[WAVES:[0-9]+]]
+; WAVE: entry:
+; WAVE-NEXT: br label %header, {{.*}}!wave.profile.block ![[ENTRY:[0-9]+]]
; COUNTS: br i1 %done, label %exit, label %header.header_crit_edge, !prof ![[WEIGHTS:[0-9]+]]
+; WAVE-SAME: !wave.profile.block ![[HEADER:[0-9]+]]
+; WAVE: header.header_crit_edge:
+; WAVE-NEXT: br label %header, {{.*}}!wave.profile.block ![[BACKEDGE:[0-9]+]]
+; WAVE: exit:
+; WAVE-NEXT: ret void, !wave.profile.block ![[EXIT:[0-9]+]]
; COUNTS-DAG: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 192}
; COUNTS-DAG: ![[WEIGHTS]] = !{!"branch_weights", i32 192, i32 576}
+; WAVE-DAG: ![[WAVES]] = distinct !{i64 2, i64 [[FID:-?[0-9]+]], i64 3, i64 12, i64 9, i64 3}
+; WAVE-DAG: ![[ENTRY]] = !{i64 2, i64 [[FID]], i64 0, i64 1, i64 1}
+; WAVE-DAG: ![[HEADER]] = !{i64 2, i64 [[FID]], i64 1, i64 1, i64 3, i64 2}
+; WAVE-DAG: ![[BACKEDGE]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 1}
+; WAVE-DAG: ![[EXIT]] = !{i64 2, i64 [[FID]], i64 3, i64 1}
; A catchswitch has no legal insertion point. Skip dispatch and append only
; exit; the sparse catch counter must remain after its catchpad.
diff --git a/llvm/test/Transforms/PGOProfile/wave-profile-use.ll b/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
index 0d9f1ea0d3408..cc56ae02614e7 100644
--- a/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
+++ b/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
@@ -1,5 +1,5 @@
; RUN: split-file %s %t
-; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -S %t/input.ll | FileCheck %s --check-prefix=GEN
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-dense-wave-counts=false -pgo-instrument-entry -S %t/input.ll | FileCheck %s --check-prefix=GEN
; RUN: %python %t/raw.py wave > %t/wave.raw
; RUN: llvm-profdata merge %t/wave.raw -o %t/wave.profdata
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -S %t/input.ll -o %t/used.ll
@@ -40,7 +40,7 @@
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/bad-lane.profdata -S %t/used.ll -o %t/bad-lane.ll 2>&1 | FileCheck %s --check-prefix=BAD-LANE
; RUN: FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile < %t/bad-lane.ll
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/wave.profdata -S %t/used.ll | FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile
-; RUN: opt -passes=pgo-instr-gen -pgo-instrument-entry -S %t/critical.ll | FileCheck %s --check-prefix=CGEN
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-dense-wave-counts=false -pgo-instrument-entry -S %t/critical.ll | FileCheck %s --check-prefix=CGEN
; RUN: %python %t/raw.py critical > %t/critical.raw
; RUN: llvm-profdata merge %t/critical.raw -o %t/critical.profdata
; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/critical.profdata -S %t/critical.ll | FileCheck %s --check-prefix=CRIT
@@ -67,6 +67,42 @@
; RUN: FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile < %t/uniform-no-wave.ll
; RUN: FileCheck %s --check-prefix=UNIFORM < %t/uniform-no-wave.ll
+; Dense layout is selected from the profile, even with generation disabled.
+; RUN: %python %t/raw.py dense > %t/dense.raw
+; RUN: llvm-profdata merge %t/dense.raw -o %t/dense.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/dense.profdata -S %t/input.ll -o %t/dense.ll
+; RUN: FileCheck %s --check-prefix=DENSE < %t/dense.ll
+; RUN: FileCheck %s --check-prefix=COUNTS < %t/dense.ll
+; RUN: FileCheck %s --check-prefix=UNIFORM < %t/dense.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/dense.profdata -pgo-instrument-dense-wave-counts=false -S %t/input.ll -o %t/dense-disabled-gen.ll
+; RUN: diff %t/dense.ll %t/dense-disabled-gen.ll
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/dense.profdata -pgo-wave-metadata=false -S %t/input.ll -o %t/dense-off.ll
+; RUN: FileCheck %s --check-prefix=COUNTS --implicit-check-not=wave.profile < %t/dense-off.ll
+; RUN: FileCheck %s --check-prefix=UNIFORM < %t/dense-off.ll
+; RUN: %python %t/raw.py dense-no-entry > %t/dense-no-entry.raw
+; RUN: llvm-profdata merge %t/dense-no-entry.raw -o %t/dense-no-entry.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/dense-no-entry.profdata -S %t/input.ll -o %t/dense-no-entry.ll
+; RUN: FileCheck %s --check-prefix=DENSE < %t/dense-no-entry.ll
+; RUN: FileCheck %s --check-prefix=COUNTS < %t/dense-no-entry.ll
+; RUN: %python %t/raw.py dense-zero > %t/dense-zero.raw
+; RUN: llvm-profdata merge %t/dense-zero.raw -o %t/dense-zero.profdata
+; RUN: opt -passes=pgo-instr-use,verify -pgo-test-profile-file=%t/dense-zero.profdata -S %t/input.ll | FileCheck %s --check-prefix=DENSE-ZERO
+
+; All four blocks are measured. Appended zero lane steps do not change select
+; counts or expand uniformity coverage beyond its sparse measurements.
+; DENSE-LABEL: define void @diamond
+; DENSE-SAME: !wave.profile ![[DW:[0-9]+]]
+; DENSE: a:
+; DENSE-NEXT: store volatile i32 1, ptr %p
+; DENSE-NEXT: br label %exit, !wave.profile.block ![[DA:[0-9]+]]
+; DENSE: ret void, !wave.profile.block ![[DX:[0-9]+]]
+; DENSE-DAG: ![[DW]] = distinct !{i64 2, i64 [[DFID:-?[0-9]+]], i64 100, i64 100, i64 50, i64 100}
+; DENSE-DAG: ![[DA]] = !{i64 2, i64 [[DFID]], i64 1, i64 1, i64 3}
+; DENSE-DAG: ![[DX]] = !{i64 2, i64 [[DFID]], i64 3, i64 1}
+; DENSE-ZERO: distinct !{i64 2, i64 [[ZFID:-?[0-9]+]], i64 0, i64 0, i64 0, i64 0}
+; DENSE-ZERO-DAG: !{i64 2, i64 [[ZFID]], i64 1, i64 1, i64 3}
+; DENSE-ZERO-DAG: !{i64 2, i64 [[ZFID]], i64 3, i64 1}
+
; Wave slots follow instrumentation indices, not IR block order. Only entry and
; b have block counters. The trailing select counter must not make exit measured.
; Generation checks keep the raw fixture aligned with the actual producer.
@@ -227,6 +263,16 @@ elif mode == "zero":
version = 12 | (1 << 56)
if mode != "no-entry":
version |= 1 << 58
+if mode.startswith("dense"):
+ version |= 1 << 54
+ lanes = [6400, 3200, 1600, 0, 0]
+ waves = [100, 50, 100, 100, 100]
+ if mode == "dense-no-entry":
+ version &= ~(1 << 58)
+ lanes[0] = 3200
+ if mode == "dense-zero":
+ lanes = [0] * 5
+ waves = [0] * 5
num_counters = len(lanes) + len(waves)
counter_delta = 80
uniform_delta = counter_delta + num_counters * 8
@@ -238,7 +284,7 @@ header = [0xff6c70726f667281, version, 0, 1, 0, num_counters, 0,
record = struct.pack("<7QI4HII4x", name_ref, func_hash, counter_delta,
uniform_delta, 0, 0, 0, num_counters, 0, 0, 0, 64, 0,
len(waves))
-uniform = lanes if mode == "uniform-wave" else [0] * len(lanes)
+uniform = lanes if mode in ("uniform-wave", "dense") else [0] * len(lanes)
counts = lanes + waves + uniform
sys.stdout.buffer.write(struct.pack("<19Q", *header) + record +
struct.pack("<" + "Q" * len(counts), *counts) +
More information about the llvm-branch-commits
mailing list