[llvm] [llvm-exegesis] Add option to specify the number of measurement repetitions (PR #74276)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Tue Dec 5 18:13:33 PST 2023
https://github.com/boomanaiden154 updated https://github.com/llvm/llvm-project/pull/74276
>From 7e59113e570449abf2e3b87defcc0eb2f2d24833 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Sun, 3 Dec 2023 21:00:09 -0800
Subject: [PATCH 1/2] [llvm-exegesis] Add option to specify the number of
latency repetitions
Currently, the llvm-exegesis LatencyBenchmarkRunner repeats the
benchmark several times (currently 30) and then aggregates the result
to deal with noise in the measurement process. With this patch, the
number of repetitions to perform is made configurable rather than left
as a static number. This allows for significantly faster execution in
situations where someone is performing a task like experimenting with
memory annotations where the exact cycle counts might not be useful, and
also allows for increased precision when desired.
---
.../llvm-exegesis/lib/LatencyBenchmarkRunner.cpp | 7 ++++---
.../tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h | 4 +++-
llvm/tools/llvm-exegesis/lib/Target.cpp | 12 ++++++++----
llvm/tools/llvm-exegesis/lib/Target.h | 6 ++++--
llvm/tools/llvm-exegesis/llvm-exegesis.cpp | 6 ++++++
5 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
index 753efed138163..7e560c2d64f27 100644
--- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
@@ -21,11 +21,13 @@ namespace exegesis {
LatencyBenchmarkRunner::LatencyBenchmarkRunner(
const LLVMState &State, Benchmark::ModeE Mode,
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
- Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode)
+ Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode,
+ unsigned BenchmarkRepeatCount)
: BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode) {
assert((Mode == Benchmark::Latency || Mode == Benchmark::InverseThroughput) &&
"invalid mode");
ResultAggMode = ResultAgg;
+ BenchmarkRepetitions = BenchmarkRepeatCount;
}
LatencyBenchmarkRunner::~LatencyBenchmarkRunner() = default;
@@ -68,13 +70,12 @@ Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
// Cycle measurements include some overhead from the kernel. Repeat the
// measure several times and return the aggregated value, as specified by
// ResultAggMode.
- constexpr const int NumMeasurements = 30;
llvm::SmallVector<int64_t, 4> AccumulatedValues;
double MinVariance = std::numeric_limits<double>::infinity();
const char *CounterName = State.getPfmCounters().CycleCounter;
// Values count for each run.
int ValuesCount = 0;
- for (size_t I = 0; I < NumMeasurements; ++I) {
+ for (size_t I = 0; I < BenchmarkRepetitions; ++I) {
auto ExpectedCounterValues = Executor.runAndSample(CounterName);
if (!ExpectedCounterValues)
return ExpectedCounterValues.takeError();
diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
index 34b912f0abded..02dedeca63850 100644
--- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
+++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
@@ -24,7 +24,8 @@ class LatencyBenchmarkRunner : public BenchmarkRunner {
LatencyBenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
Benchmark::ResultAggregationModeE ResultAggMode,
- ExecutionModeE ExecutionMode);
+ ExecutionModeE ExecutionMode,
+ unsigned BenchmarkRepeatCount);
~LatencyBenchmarkRunner() override;
private:
@@ -32,6 +33,7 @@ class LatencyBenchmarkRunner : public BenchmarkRunner {
runMeasurements(const FunctionExecutor &Executor) const override;
Benchmark::ResultAggregationModeE ResultAggMode;
+ unsigned BenchmarkRepetitions;
};
} // namespace exegesis
} // namespace llvm
diff --git a/llvm/tools/llvm-exegesis/lib/Target.cpp b/llvm/tools/llvm-exegesis/lib/Target.cpp
index aed093548f158..67eddb9dbb1db 100644
--- a/llvm/tools/llvm-exegesis/lib/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Target.cpp
@@ -79,7 +79,8 @@ ExegesisTarget::createBenchmarkRunner(
Benchmark::ModeE Mode, const LLVMState &State,
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
BenchmarkRunner::ExecutionModeE ExecutionMode,
- Benchmark::ResultAggregationModeE ResultAggMode) const {
+ Benchmark::ResultAggregationModeE ResultAggMode,
+ unsigned BenchmarkRepeatCount) const {
PfmCountersInfo PfmCounters = State.getPfmCounters();
switch (Mode) {
case Benchmark::Unknown:
@@ -101,7 +102,8 @@ ExegesisTarget::createBenchmarkRunner(
"the kernel for real event counts."));
}
return createLatencyBenchmarkRunner(State, Mode, BenchmarkPhaseSelector,
- ResultAggMode, ExecutionMode);
+ ResultAggMode, ExecutionMode,
+ BenchmarkRepeatCount);
case Benchmark::Uops:
if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure &&
!PfmCounters.UopsCounter && !PfmCounters.IssueCounters)
@@ -130,9 +132,11 @@ std::unique_ptr<BenchmarkRunner> ExegesisTarget::createLatencyBenchmarkRunner(
const LLVMState &State, Benchmark::ModeE Mode,
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
Benchmark::ResultAggregationModeE ResultAggMode,
- BenchmarkRunner::ExecutionModeE ExecutionMode) const {
+ BenchmarkRunner::ExecutionModeE ExecutionMode,
+ unsigned BenchmarkRepeatCount) const {
return std::make_unique<LatencyBenchmarkRunner>(
- State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode);
+ State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode,
+ BenchmarkRepeatCount);
}
std::unique_ptr<BenchmarkRunner> ExegesisTarget::createUopsBenchmarkRunner(
diff --git a/llvm/tools/llvm-exegesis/lib/Target.h b/llvm/tools/llvm-exegesis/lib/Target.h
index 6de5b3c1065f1..26f5f5403aa44 100644
--- a/llvm/tools/llvm-exegesis/lib/Target.h
+++ b/llvm/tools/llvm-exegesis/lib/Target.h
@@ -262,7 +262,8 @@ class ExegesisTarget {
Benchmark::ModeE Mode, const LLVMState &State,
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
BenchmarkRunner::ExecutionModeE ExecutionMode,
- Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min) const;
+ Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min,
+ unsigned BenchmarkRepeatCount = 30) const;
// Returns the ExegesisTarget for the given triple or nullptr if the target
// does not exist.
@@ -305,7 +306,8 @@ class ExegesisTarget {
const LLVMState &State, Benchmark::ModeE Mode,
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
Benchmark::ResultAggregationModeE ResultAggMode,
- BenchmarkRunner::ExecutionModeE ExecutionMode) const;
+ BenchmarkRunner::ExecutionModeE ExecutionMode,
+ unsigned BenchmarkRepeatCount) const;
std::unique_ptr<BenchmarkRunner> virtual createUopsBenchmarkRunner(
const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
Benchmark::ResultAggregationModeE ResultAggMode,
diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
index 261335a817d06..57330a8ab0fb7 100644
--- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
+++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
@@ -262,6 +262,12 @@ static cl::opt<BenchmarkRunner::ExecutionModeE> ExecutionMode(
"allows for the use of memory annotations")),
cl::init(BenchmarkRunner::ExecutionModeE::InProcess));
+static cl::opt<unsigned>
+ BenchmarkRepeatCount("benchmark-repeat-count",
+ cl::desc("The number of times to repeat the benchmark "
+ "before aggregating the results"),
+ cl::cat(BenchmarkOptions), cl::init(30));
+
static ExitOnError ExitOnErr("llvm-exegesis error: ");
// Helper function that logs the error(s) and exits.
>From 4541a1d899be5849825e7178681eba49d1c0d970 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <agrossman154 at yahoo.com>
Date: Tue, 5 Dec 2023 18:13:23 -0800
Subject: [PATCH 2/2] Address reviewer feedback
---
llvm/tools/llvm-exegesis/lib/Target.cpp | 4 ++--
llvm/tools/llvm-exegesis/lib/Target.h | 4 ++--
llvm/tools/llvm-exegesis/llvm-exegesis.cpp | 12 ++++++------
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/llvm/tools/llvm-exegesis/lib/Target.cpp b/llvm/tools/llvm-exegesis/lib/Target.cpp
index 67eddb9dbb1db..23c80e5b98953 100644
--- a/llvm/tools/llvm-exegesis/lib/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Target.cpp
@@ -79,8 +79,8 @@ ExegesisTarget::createBenchmarkRunner(
Benchmark::ModeE Mode, const LLVMState &State,
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
BenchmarkRunner::ExecutionModeE ExecutionMode,
- Benchmark::ResultAggregationModeE ResultAggMode,
- unsigned BenchmarkRepeatCount) const {
+ unsigned BenchmarkRepeatCount,
+ Benchmark::ResultAggregationModeE ResultAggMode) const {
PfmCountersInfo PfmCounters = State.getPfmCounters();
switch (Mode) {
case Benchmark::Unknown:
diff --git a/llvm/tools/llvm-exegesis/lib/Target.h b/llvm/tools/llvm-exegesis/lib/Target.h
index 26f5f5403aa44..c37dd8b708216 100644
--- a/llvm/tools/llvm-exegesis/lib/Target.h
+++ b/llvm/tools/llvm-exegesis/lib/Target.h
@@ -262,8 +262,8 @@ class ExegesisTarget {
Benchmark::ModeE Mode, const LLVMState &State,
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
BenchmarkRunner::ExecutionModeE ExecutionMode,
- Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min,
- unsigned BenchmarkRepeatCount = 30) const;
+ unsigned BenchmarkRepeatCount,
+ Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min) const;
// Returns the ExegesisTarget for the given triple or nullptr if the target
// does not exist.
diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
index 57330a8ab0fb7..5719dc7376e1e 100644
--- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
+++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
@@ -262,11 +262,11 @@ static cl::opt<BenchmarkRunner::ExecutionModeE> ExecutionMode(
"allows for the use of memory annotations")),
cl::init(BenchmarkRunner::ExecutionModeE::InProcess));
-static cl::opt<unsigned>
- BenchmarkRepeatCount("benchmark-repeat-count",
- cl::desc("The number of times to repeat the benchmark "
- "before aggregating the results"),
- cl::cat(BenchmarkOptions), cl::init(30));
+static cl::opt<unsigned> BenchmarkRepeatCount(
+ "benchmark-repeat-count",
+ cl::desc("The number of times to repeat measurements on the benchmark k "
+ "before aggregating the results"),
+ cl::cat(BenchmarkOptions), cl::init(30));
static ExitOnError ExitOnErr("llvm-exegesis error: ");
@@ -491,7 +491,7 @@ void benchmarkMain() {
const std::unique_ptr<BenchmarkRunner> Runner =
ExitOnErr(State.getExegesisTarget().createBenchmarkRunner(
BenchmarkMode, State, BenchmarkPhaseSelector, ExecutionMode,
- ResultAggMode));
+ BenchmarkRepeatCount, ResultAggMode));
if (!Runner) {
ExitWithError("cannot create benchmark runner");
}
More information about the llvm-commits
mailing list