[llvm] 355ad3a - Add JSON output option to llvm-remark-size-diff
Jessica Paquette via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 7 16:54:20 PST 2022
Author: Jessica Paquette
Date: 2022-03-07T16:53:27-08:00
New Revision: 355ad3a3cdb36fd3207b53d62775b9d8d28413f4
URL: https://github.com/llvm/llvm-project/commit/355ad3a3cdb36fd3207b53d62775b9d8d28413f4
DIFF: https://github.com/llvm/llvm-project/commit/355ad3a3cdb36fd3207b53d62775b9d8d28413f4.diff
LOG: Add JSON output option to llvm-remark-size-diff
This adds JSON output to llvm-remark-size-diff.
The goal here is to make it easy for external tools to consume output from
llvm-remark-size-diff. These tools could be used for automated size analysis.
(E.g. in CI).
To specify JSON output, use `--report_style=json`. JSON output can be
pretty-printed via `--pretty`.
With automation in mind, the schema looks like this:
```
"Files": {
"A": <filename_a>
"B": <filename_b>
},
"InBoth": [
{
"FunctionName": <function name>,
"InstCount": [
<count_in_a>,
<count_in_b>
],
"StackSize": [
<count_in_a>,
<count_in_b>
]
},
...
]
"OnlyInA": [
{
"FunctionName": <function name>,
"InstCount": [
<count_in_a>,
0
],
"StackSize": [
<count_in_a>,
0
]
},
...
]
"OnlyInB": [
{
"FunctionName": <function name>,
"InstCount": [
0,
<count_in_b>
],
"StackSize": [
0,
<count_in_b>
]
},
...
]
```
A few notes:
- Filenames are included, because tools may want to combine many outputs
together in some way (a big JSON file, a big CSV, or something.)
- Counts are represented as [a, b] so that a diff can be calculated via b - a.
The original counts may be useful for size analysis (e.g. was this function
extremely large before?) and so both are preserved.
- `OnlyInA` and `OnlyInB` have a 0 for one of the counts always. This is to
make it easier for tools to share code between `OnlyInA`, `OnlyInB`, and
`InBoth`.
Differential Revision: https://reviews.llvm.org/D121173
Added:
llvm/test/tools/llvm-remark-size-diff/json-add-remove-func.test
llvm/test/tools/llvm-remark-size-diff/json-increase-decrease-inst-count.test
llvm/test/tools/llvm-remark-size-diff/json-no-difference.test
Modified:
llvm/tools/llvm-remark-size-diff/RemarkSizeDiff.cpp
Removed:
################################################################################
diff --git a/llvm/test/tools/llvm-remark-size-
diff /json-add-remove-func.test b/llvm/test/tools/llvm-remark-size-
diff /json-add-remove-func.test
new file mode 100644
index 0000000000000..788a8487d2656
--- /dev/null
+++ b/llvm/test/tools/llvm-remark-size-
diff /json-add-remove-func.test
@@ -0,0 +1,57 @@
+RUN: llvm-remark-size-
diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/2-identical-func-1-instr-1-stack.yaml --parser=yaml --report_style=json --pretty | FileCheck -strict-whitespace %s --check-prefix=ADD
+RUN: llvm-remark-size-
diff %p/Inputs/2-identical-func-1-instr-1-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml --parser=yaml --report_style=json --pretty | FileCheck -strict-whitespace %s --check-prefix=REMOVE
+
+; The "two-identical-one-instr-funcs" file contains a single-instruction
+; function which does not appear in the other file.
+
+; ADD-LABEL: "Files":
+; ADD: "A":{{.*}}1-func-1-instr-1-stack.yaml
+; ADD-NEXT: "B":{{.*}}2-identical-func-1-instr-1-stack.yaml
+
+; ADD-LABEL: "InBoth": [
+; ADD: "FunctionName": "func0",
+; ADD-NEXT: "InstCount": [
+; ADD-NEXT: 1,
+; ADD-NEXT: 1
+; ADD-NEXT: ],
+; ADD-NEXT: "StackSize": [
+; ADD-NEXT: 1,
+; ADD-NEXT: 1
+
+; ADD-LABEL: "OnlyInA": [],
+
+; ADD-LABEL: "OnlyInB": [
+; ADD: "FunctionName": "func1",
+; ADD-NEXT: "InstCount": [
+; ADD-NEXT: 0,
+; ADD-NEXT: 1
+; ADD-NEXT: ],
+; ADD-NEXT: "StackSize": [
+; ADD-NEXT: 0,
+; ADD-NEXT: 1
+
+; REMOVE-LABEL: "Files":
+; REMOVE: "A":{{.*}}2-identical-func-1-instr-1-stack.yaml
+; REMOVE-NEXT: "B":{{.*}}1-func-1-instr-1-stack.yaml
+
+; REMOVE-LABEL: "InBoth": [
+; REMOVE: "FunctionName": "func0",
+; REMOVE-NEXT: "InstCount": [
+; REMOVE-NEXT: 1,
+; REMOVE-NEXT: 1
+; REMOVE-NEXT: ],
+; REMOVE-NEXT: "StackSize": [
+; REMOVE-NEXT: 1,
+; REMOVE-NEXT: 1
+
+; REMOVE-LABEL: "OnlyInA": [
+; REMOVE: "FunctionName": "func1",
+; REMOVE-NEXT: "InstCount": [
+; REMOVE-NEXT: 1,
+; REMOVE-NEXT: 0
+; REMOVE-NEXT: ],
+; REMOVE-NEXT: "StackSize": [
+; REMOVE-NEXT: 1,
+; REMOVE-NEXT: 0
+
+; REMOVE-LABEL: "OnlyInB": []
diff --git a/llvm/test/tools/llvm-remark-size-
diff /json-increase-decrease-inst-count.test b/llvm/test/tools/llvm-remark-size-
diff /json-increase-decrease-inst-count.test
new file mode 100644
index 0000000000000..e86767d869d3c
--- /dev/null
+++ b/llvm/test/tools/llvm-remark-size-
diff /json-increase-decrease-inst-count.test
@@ -0,0 +1,38 @@
+RUN: llvm-remark-size-
diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/1-func-2-instr-2-stack.yaml --parser=yaml --report_style=json --pretty | FileCheck -strict-whitespace %s --check-prefix=INCREASE
+RUN: llvm-remark-size-
diff %p/Inputs/1-func-2-instr-2-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml --parser=yaml --report_style=json --pretty | FileCheck -strict-whitespace %s --check-prefix=DECREASE
+
+; Test a size increase/decrease of one instruction + 1 stack byte.
+
+; INCREASE-LABEL: "Files":
+; INCREASE: "A":{{.*}}1-func-1-instr-1-stack.yaml
+; INCREASE-NEXT: "B":{{.*}}1-func-2-instr-2-stack.yaml
+
+; INCREASE-LABEL: "InBoth": [
+; INCREASE: "FunctionName": "func0"
+; INCREASE-NEXT: "InstCount":
+; INCREASE-NEXT: 1,
+; INCREASE-NEXT: 2
+; INCREASE-NEXT: ],
+; INCREASE-NEXT: "StackSize":
+; INCREASE-NEXT: 1,
+; INCREASE-NEXT: 2
+
+; INCREASE: "OnlyInA": [],
+; INCREASE: "OnlyInB": []
+
+; DECREASE-LABEL: "Files":
+; DECREASE: "A":{{.*}}1-func-2-instr-2-stack.yaml
+; DECREASE-NEXT: "B":{{.*}}1-func-1-instr-1-stack.yaml
+
+; DECREASE-LABEL: "InBoth": [
+; DECREASE: "FunctionName": "func0"
+; DECREASE-NEXT: "InstCount":
+; DECREASE-NEXT: 2,
+; DECREASE-NEXT: 1
+; DECREASE-NEXT: ],
+; DECREASE-NEXT: "StackSize":
+; DECREASE-NEXT: 2,
+; DECREASE-NEXT: 1
+
+; DECREASE: "OnlyInA": [],
+; DECREASE: "OnlyInB": []
diff --git a/llvm/test/tools/llvm-remark-size-
diff /json-no-
diff erence.test b/llvm/test/tools/llvm-remark-size-
diff /json-no-
diff erence.test
new file mode 100644
index 0000000000000..4650fa7238614
--- /dev/null
+++ b/llvm/test/tools/llvm-remark-size-
diff /json-no-
diff erence.test
@@ -0,0 +1,18 @@
+RUN: llvm-remark-size-
diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml --parser=yaml --report_style=json --pretty | FileCheck -strict-whitespace %s
+
+; CHECK-LABEL: "Files":
+; CHECK: "A":{{.*}}1-func-1-instr-1-stack.yaml
+; CHECK-NEXT: "B":{{.*}}1-func-1-instr-1-stack.yaml
+
+; CHECK-LABEL: "InBoth":
+; CHECK: "FunctionName": "func0",
+; CHECK-NEXT: "InstCount":
+; CHECK-NEXT: 1,
+; CHECK-NEXT: 1
+; CHECK-NEXT: ],
+; CHECK-NEXT: "StackSize": [
+; CHECK-NEXT: 1,
+; CHECK-NEXT: 1
+
+; CHECK: "OnlyInA": []
+; CHECK: "OnlyInB": []
diff --git a/llvm/tools/llvm-remark-size-
diff /RemarkSizeDiff.cpp b/llvm/tools/llvm-remark-size-
diff /RemarkSizeDiff.cpp
index 8faa573bad261..ab59820f38dbc 100644
--- a/llvm/tools/llvm-remark-size-
diff /RemarkSizeDiff.cpp
+++ b/llvm/tools/llvm-remark-size-
diff /RemarkSizeDiff.cpp
@@ -12,8 +12,6 @@
/// This is intended for use by compiler developers who want to see how their
/// changes impact program code size.
///
-/// TODO: Add structured output (JSON, or YAML, or something...)
-///
//===----------------------------------------------------------------------===//
#include "llvm-c/Remarks.h"
@@ -24,10 +22,12 @@
#include "llvm/Remarks/RemarkParser.h"
#include "llvm/Remarks/RemarkSerializer.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/JSON.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/ToolOutputFile.h"
#include "llvm/Support/WithColor.h"
@@ -36,6 +36,7 @@
using namespace llvm;
enum ParserFormatOptions { yaml, bitstream };
+enum ReportStyleOptions { human_output, json_output };
static cl::OptionCategory SizeDiffCategory("llvm-remark-size-
diff options");
static cl::opt<std::string> InputFileNameA(cl::Positional, cl::Required,
cl::cat(SizeDiffCategory),
@@ -52,6 +53,15 @@ static cl::opt<ParserFormatOptions>
cl::desc("Set the remark parser format:"),
cl::values(clEnumVal(yaml, "YAML format"),
clEnumVal(bitstream, "Bitstream format")));
+static cl::opt<ReportStyleOptions> ReportStyle(
+ "report_style", cl::cat(SizeDiffCategory),
+ cl::init(ReportStyleOptions::human_output),
+ cl::desc("Choose the report output format:"),
+ cl::values(clEnumValN(human_output, "human", "Human-readable format"),
+ clEnumValN(json_output, "json", "JSON format")));
+static cl::opt<bool> PrettyPrint("pretty", cl::cat(SizeDiffCategory),
+ cl::init(false),
+ cl::desc("Pretty-print JSON"));
/// Contains information from size remarks.
// This is a little nicer to read than a std::pair.
@@ -382,23 +392,109 @@ static ErrorOr<std::unique_ptr<ToolOutputFile>> getOutputStream() {
return EC;
}
-/// Output all
diff s in \p DiffsByFilesPresent.
+/// \return a json::Array representing all FunctionDiffs in \p FunctionDiffs.
+/// \p WhichFiles represents which files the functions in \p FunctionDiffs
+/// appeared in (A, B, or both).
+json::Array
+getFunctionDiffListAsJSON(const SmallVector<FunctionDiff> &FunctionDiffs,
+ const FilesPresent &WhichFiles) {
+ json::Array FunctionDiffsAsJSON;
+ int64_t InstCountA, InstCountB, StackSizeA, StackSizeB;
+ for (auto &Diff : FunctionDiffs) {
+ InstCountA = InstCountB = StackSizeA = StackSizeB = 0;
+ switch (WhichFiles) {
+ case BOTH:
+ LLVM_FALLTHROUGH;
+ case A:
+ InstCountA = Diff.getInstCountA();
+ StackSizeA = Diff.getStackSizeA();
+ if (WhichFiles != BOTH)
+ break;
+ LLVM_FALLTHROUGH;
+ case B:
+ InstCountB = Diff.getInstCountB();
+ StackSizeB = Diff.getStackSizeB();
+ break;
+ }
+ // Each metric we care about is represented like:
+ // "Val": [A, B]
+ // This allows any consumer of the JSON to calculate the
diff using B - A.
+ // This is somewhat wasteful for OnlyInA and OnlyInB (we only need A or B).
+ // However, this should make writing consuming tools easier, since the tool
+ // writer doesn't need to think about slightly
diff erent formats in each
+ // section.
+ json::Object FunctionObject({{"FunctionName", Diff.FuncName},
+ {"InstCount", {InstCountA, InstCountB}},
+ {"StackSize", {StackSizeA, StackSizeB}}});
+ FunctionDiffsAsJSON.push_back(std::move(FunctionObject));
+ }
+ return FunctionDiffsAsJSON;
+}
+
+/// Output all
diff s in \p DiffsByFilesPresent as a JSON report. This is
+/// intended for consumption by external tools.
+///
+/// \p InputFileNameA - File A used to produce the report.
+/// \p InputFileNameB - File B used ot produce the report.
+/// \p OS - Output stream.
+///
+/// JSON output includes:
+/// - \p InputFileNameA and \p InputFileNameB under "Files".
+/// - Functions present in both files under "InBoth".
+/// - Functions present only in A in "OnlyInA".
+/// - Functions present only in B in "OnlyInB".
+/// - Instruction count and stack size
diff erences for each function.
+///
+/// Differences are represented using [count_a, count_b]. The actual
diff erence
+/// can be computed via count_b - count_a.
+static void
+outputJSONForAllDiffs(StringRef InputFileNameA, StringRef InputFileNameB,
+ const DiffsCategorizedByFilesPresent &DiffsByFilesPresent,
+ llvm::raw_ostream &OS) {
+ json::Object Output;
+ // Include file names in the report.
+ json::Object Files(
+ {{"A", InputFileNameA.str()}, {"B", InputFileNameB.str()}});
+ Output["Files"] = std::move(Files);
+ Output["OnlyInA"] = getFunctionDiffListAsJSON(DiffsByFilesPresent.OnlyInA, A);
+ Output["OnlyInB"] = getFunctionDiffListAsJSON(DiffsByFilesPresent.OnlyInB, B);
+ Output["InBoth"] =
+ getFunctionDiffListAsJSON(DiffsByFilesPresent.InBoth, BOTH);
+ json::OStream JOS(OS, PrettyPrint ? 2 : 0);
+ JOS.value(std::move(Output));
+ OS << '\n';
+}
+
+/// Output all
diff s in \p DiffsByFilesPresent using the desired output style.
/// \returns Error::success() on success, and an Error otherwise.
+/// \p InputFileNameA - Name of input file A; may be used in the report.
+/// \p InputFileNameB - Name of input file B; may be used in the report.
static Error
-outputAllDiffs(DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
+outputAllDiffs(StringRef InputFileNameA, StringRef InputFileNameB,
+ DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
auto MaybeOF = getOutputStream();
if (std::error_code EC = MaybeOF.getError())
return errorCodeToError(EC);
std::unique_ptr<ToolOutputFile> OF = std::move(*MaybeOF);
- printDiffsCategorizedByFilesPresent(DiffsByFilesPresent, OF->os());
+ switch (ReportStyle) {
+ case human_output:
+ printDiffsCategorizedByFilesPresent(DiffsByFilesPresent, OF->os());
+ break;
+ case json_output:
+ outputJSONForAllDiffs(InputFileNameA, InputFileNameB, DiffsByFilesPresent,
+ OF->os());
+ break;
+ }
OF->keep();
return Error::success();
}
/// Boolean wrapper for outputDiff which handles errors.
static bool
-tryOutputAllDiffs(DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
- if (Error E = outputAllDiffs(DiffsByFilesPresent)) {
+tryOutputAllDiffs(StringRef InputFileNameA, StringRef InputFileNameB,
+ DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
+ if (Error E =
+ outputAllDiffs(InputFileNameA, InputFileNameB, DiffsByFilesPresent)) {
handleAllErrors(std::move(E), [&](const ErrorInfoBase &PE) {
PE.log(WithColor::error());
errs() << '\n';
@@ -421,6 +517,6 @@ int main(int argc, const char **argv) {
return 1;
DiffsCategorizedByFilesPresent DiffsByFilesPresent;
computeDiff(FuncNameToSizeInfoA, FuncNameToSizeInfoB, DiffsByFilesPresent);
- if (!tryOutputAllDiffs(DiffsByFilesPresent))
+ if (!tryOutputAllDiffs(InputFileNameA, InputFileNameB, DiffsByFilesPresent))
return 1;
}
More information about the llvm-commits
mailing list