[llvm] 8507dba - [llvm][llvm-lit] Add option to create unique result file names if results already exist (#112729)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 21 02:43:49 PDT 2024
Author: David Spickett
Date: 2024-10-21T10:43:45+01:00
New Revision: 8507dbaec3f644b8a0c6291f097800d82a4f4b16
URL: https://github.com/llvm/llvm-project/commit/8507dbaec3f644b8a0c6291f097800d82a4f4b16
DIFF: https://github.com/llvm/llvm-project/commit/8507dbaec3f644b8a0c6291f097800d82a4f4b16.diff
LOG: [llvm][llvm-lit] Add option to create unique result file names if results already exist (#112729)
When running a build like:
```
ninja check-clang check-llvm
```
Prior to my changes you ended up with one results file, in this specific case Junit XML:
```
results.xml
```
This would only include the last set of tests lit ran, which were for
llvm. To get around this, many CI systems will run one check target,
move the file away, then run another, somehow propgating the return code
as well.
```
rectode=0
for target in targets:
ninja target
retcode=$?
mv results.xml results-${target}.xml
<report the overall return code>
```
I want to use something like this Buildkite reporting plugin in CI, which needs to have all the results available:
https://buildkite.com/docs/agent/v3/cli-annotate#using-annotations-to-report-test-results
Modifying CI's build scripts for Windows and Linux is a lot of work. So
my changes instead make lit detect an existing result file and modify
the file name to find a new file to write to. Now you will get:
```
results.xml results.<tempfile generated value>.xml
```
This will work for all result file types since I'm doing it in the base
Report class. Now you've got separate files, it's easy to collect them
with `<path>/*.xml`.
Note that the `<tempfile generated value>` is not ordered.
Added:
llvm/utils/lit/tests/unique-output-file.py
Modified:
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/reports.py
Removed:
################################################################################
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 5ccae4be096796..c08c51b7b7a23e 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -175,6 +175,15 @@ def parse_args():
type=lit.reports.TimeTraceReport,
help="Write Chrome tracing compatible JSON to the specified file",
)
+ execution_group.add_argument(
+ "--use-unique-output-file-name",
+ help="When enabled, lit will not overwrite existing test report files. "
+ "Instead it will write to a new file named the same as the output file "
+ "name but with an extra part before the file extension. For example "
+ "if results.xml already exists, results.<something>.xml will be written "
+ "to. The <something> is not ordered in any way. [Default: Off]",
+ action="store_true",
+ )
execution_group.add_argument(
"--timeout",
dest="maxIndividualTestTime",
@@ -332,16 +341,21 @@ def parse_args():
else:
opts.shard = None
- opts.reports = filter(
- None,
- [
- opts.output,
- opts.xunit_xml_output,
- opts.resultdb_output,
- opts.time_trace_output,
- ],
+ opts.reports = list(
+ filter(
+ None,
+ [
+ opts.output,
+ opts.xunit_xml_output,
+ opts.resultdb_output,
+ opts.time_trace_output,
+ ],
+ )
)
+ for report in opts.reports:
+ report.use_unique_output_file_name = opts.use_unique_output_file_name
+
return opts
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index 8ec83d698ae86a..8312dcddc769ae 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -1,7 +1,10 @@
+import abc
import base64
import datetime
import itertools
import json
+import os
+import tempfile
from xml.sax.saxutils import quoteattr as quo
@@ -14,11 +17,34 @@ def by_suite_and_test_path(test):
return (test.suite.name, id(test.suite), test.path_in_suite)
-class JsonReport(object):
+class Report(object):
def __init__(self, output_file):
self.output_file = output_file
+ # Set by the option parser later.
+ self.use_unique_output_file_name = False
def write_results(self, tests, elapsed):
+ if self.use_unique_output_file_name:
+ filename, ext = os.path.splitext(os.path.basename(self.output_file))
+ fd, _ = tempfile.mkstemp(
+ suffix=ext, prefix=f"{filename}.", dir=os.path.dirname(self.output_file)
+ )
+ report_file = os.fdopen(fd, "w")
+ else:
+ # Overwrite if the results already exist.
+ report_file = open(self.output_file, "w")
+
+ with report_file:
+ self._write_results_to_file(tests, elapsed, report_file)
+
+ @abc.abstractmethod
+ def _write_results_to_file(self, tests, elapsed, file):
+ """Write test results to the file object "file"."""
+ pass
+
+
+class JsonReport(Report):
+ def _write_results_to_file(self, tests, elapsed, file):
unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
tests = [t for t in tests if t.result.code not in unexecuted_codes]
# Construct the data we will write.
@@ -67,9 +93,8 @@ def write_results(self, tests, elapsed):
tests_data.append(test_data)
- with open(self.output_file, "w") as file:
- json.dump(data, file, indent=2, sort_keys=True)
- file.write("\n")
+ json.dump(data, file, indent=2, sort_keys=True)
+ file.write("\n")
_invalid_xml_chars_dict = {
@@ -88,21 +113,18 @@ def remove_invalid_xml_chars(s):
return s.translate(_invalid_xml_chars_dict)
-class XunitReport(object):
- def __init__(self, output_file):
- self.output_file = output_file
- self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
+class XunitReport(Report):
+ skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
- def write_results(self, tests, elapsed):
+ def _write_results_to_file(self, tests, elapsed, file):
tests.sort(key=by_suite_and_test_path)
tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
- with open(self.output_file, "w") as file:
- file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
- file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
- for suite, test_iter in tests_by_suite:
- self._write_testsuite(file, suite, list(test_iter))
- file.write("</testsuites>\n")
+ file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+ file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
+ for suite, test_iter in tests_by_suite:
+ self._write_testsuite(file, suite, list(test_iter))
+ file.write("</testsuites>\n")
def _write_testsuite(self, file, suite, tests):
skipped = 0
@@ -206,11 +228,8 @@ def gen_resultdb_test_entry(
return test_data
-class ResultDBReport(object):
- def __init__(self, output_file):
- self.output_file = output_file
-
- def write_results(self, tests, elapsed):
+class ResultDBReport(Report):
+ def _write_results_to_file(self, tests, elapsed, file):
unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
tests = [t for t in tests if t.result.code not in unexecuted_codes]
data = {}
@@ -249,17 +268,14 @@ def write_results(self, tests, elapsed):
)
)
- with open(self.output_file, "w") as file:
- json.dump(data, file, indent=2, sort_keys=True)
- file.write("\n")
+ json.dump(data, file, indent=2, sort_keys=True)
+ file.write("\n")
-class TimeTraceReport(object):
- def __init__(self, output_file):
- self.output_file = output_file
- self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
+class TimeTraceReport(Report):
+ skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
- def write_results(self, tests, elapsed):
+ def _write_results_to_file(self, tests, elapsed, file):
# Find when first test started so we can make start times relative.
first_start_time = min([t.result.start for t in tests])
events = [
@@ -270,8 +286,7 @@ def write_results(self, tests, elapsed):
json_data = {"traceEvents": events}
- with open(self.output_file, "w") as time_trace_file:
- json.dump(json_data, time_trace_file, indent=2, sort_keys=True)
+ json.dump(json_data, time_trace_file, indent=2, sort_keys=True)
def _get_test_event(self, test, first_start_time):
test_name = test.getFullName()
diff --git a/llvm/utils/lit/tests/unique-output-file.py b/llvm/utils/lit/tests/unique-output-file.py
new file mode 100644
index 00000000000000..fea57682d9fdab
--- /dev/null
+++ b/llvm/utils/lit/tests/unique-output-file.py
@@ -0,0 +1,22 @@
+## Check that lit will not overwrite existing result files when given
+## --use-unique-output-file-name.
+
+## Files are overwritten without the option.
+# RUN: rm -f %t.xunit*.xml
+# RUN: echo "test" > %t.xunit.xml
+# RUN: not %{lit} --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# RUN: FileCheck < %t.xunit.xml %s --check-prefix=NEW
+# NEW: <?xml version="1.0" encoding="UTF-8"?>
+# NEW-NEXT: <testsuites time="{{[0-9.]+}}">
+## (other tests will check the contents of the whole file)
+
+# RUN: rm -f %t.xunit*.xml
+# RUN: echo "test" > %t.xunit.xml
+## Files should not be overwritten with the option.
+# RUN: not %{lit} --xunit-xml-output %t.xunit.xml --use-unique-output-file-name %{inputs}/xunit-output
+# RUN: FileCheck < %t.xunit.xml %s --check-prefix=EXISTING
+# EXISTING: test
+## Results in a new file with some discriminator added.
+# RUN: ls -l %t.xunit*.xml | wc -l | FileCheck %s --check-prefix=NUMFILES
+# NUMFILES: 2
+# RUN: FileCheck < %t.xunit.*.xml %s --check-prefix=NEW
More information about the llvm-commits
mailing list