[llvm] 867ff2d - [lit] Add a flag to disable lit time tests (#98270)

[via llvm-commits]

Author: Vincent Lee
Date: 2024-07-20T00:13:20-07:00
New Revision: 867ff2d4268ca7eed89a24f100b67b68f5329439

URL: https://github.com/llvm/llvm-project/commit/867ff2d4268ca7eed89a24f100b67b68f5329439
DIFF: https://github.com/llvm/llvm-project/commit/867ff2d4268ca7eed89a24f100b67b68f5329439.diff

LOG: [lit] Add a flag to disable lit time tests (#98270)

LLVM lit assumes control of the test parallelism when running a test
suite. This style of testing doesn't play nicely with build systems like
Buck or Bazel since it prefers finer grained actions on a per-test
level. In order for external build systems to control the test
parallelism, add an option to disable `.lit_test_times.txt` under the
`--skip-test-time-recording` flag, thus allowing other build systems 
to determine the parallelism and avoid race conditions when writing 
to that file. I went for `--skip-test-time-recording` instead of `--time-tests` in 
order to preserve the original functionality of writing to `.lit_test_times.txt`
as the default behavior and only opt-in for those who do _not_ want
`.lit_test_times.txt` file.

Added: 
    llvm/utils/lit/tests/Inputs/time-tests/a.txt
    llvm/utils/lit/tests/Inputs/time-tests/lit.cfg
    llvm/utils/lit/tests/time-tests.py

Modified: 
    llvm/docs/CommandGuide/lit.rst
    llvm/utils/lit/lit/cl_arguments.py
    llvm/utils/lit/lit/main.py

Removed: 
    


################################################################################
diff  --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 799ee34e9f9ff..c9d5baba3e2f4 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -151,6 +151,10 @@ EXECUTION OPTIONS
  feature that can be used to conditionally disable (or expect failure in)
  certain tests.
 
+.. option:: --skip-test-time-recording
+
+ Disable tracking the wall time individual tests take to execute.
+
 .. option:: --time-tests
 
  Track the wall time individual tests take to execute and includes the results

diff  --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index b9122d07afd8a..ed78256ee414b 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -154,11 +154,6 @@ def parse_args():
         action="append",
         default=[],
     )
-    execution_group.add_argument(
-        "--time-tests",
-        help="Track elapsed wall time for each test",
-        action="store_true",
-    )
     execution_group.add_argument(
         "--no-execute",
         dest="noExecute",
@@ -209,6 +204,17 @@ def parse_args():
         action="store_true",
         help="Exit with status zero even if some tests fail",
     )
+    execution_test_time_group = execution_group.add_mutually_exclusive_group()
+    execution_test_time_group.add_argument(
+        "--skip-test-time-recording",
+        help="Do not track elapsed wall time for each test",
+        action="store_true",
+    )
+    execution_test_time_group.add_argument(
+        "--time-tests",
+        help="Track elapsed wall time for each test printed in a histogram",
+        action="store_true",
+    )
 
     selection_group = parser.add_argument_group("Test Selection")
     selection_group.add_argument(

diff  --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index db9f24f748d9e..24ba804f0c363 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -124,7 +124,8 @@ def main(builtin_params={}):
     run_tests(selected_tests, lit_config, opts, len(discovered_tests))
     elapsed = time.time() - start
 
-    record_test_times(selected_tests, lit_config)
+    if not opts.skip_test_time_recording:
+        record_test_times(selected_tests, lit_config)
 
     selected_tests, discovered_tests = GoogleTest.post_process_shard_results(
         selected_tests, discovered_tests

diff  --git a/llvm/utils/lit/tests/Inputs/time-tests/a.txt b/llvm/utils/lit/tests/Inputs/time-tests/a.txt
new file mode 100644
index 0000000000000..b80b60b7a2794
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/time-tests/a.txt
@@ -0,0 +1 @@
+# RUN: true

diff  --git a/llvm/utils/lit/tests/Inputs/time-tests/lit.cfg b/llvm/utils/lit/tests/Inputs/time-tests/lit.cfg
new file mode 100644
index 0000000000000..e6ae41833874a
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/time-tests/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "time-tests"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None

diff  --git a/llvm/utils/lit/tests/time-tests.py b/llvm/utils/lit/tests/time-tests.py
new file mode 100644
index 0000000000000..20b83a64330f0
--- /dev/null
+++ b/llvm/utils/lit/tests/time-tests.py
@@ -0,0 +1,15 @@
+## Check that --skip-test-time-recording skips .lit_test_times.txt recording.
+
+# RUN: %{lit-no-order-opt} --skip-test-time-recording %{inputs}/time-tests
+# RUN: not ls %{inputs}/time-tests/.lit_test_times.txt
+
+## Check that --time-tests generates a printed histogram.
+
+# RUN: %{lit-no-order-opt} --time-tests %{inputs}/time-tests > %t.out
+# RUN: FileCheck < %t.out %s
+# RUN: rm %{inputs}/time-tests/.lit_test_times.txt
+
+# CHECK:      Tests Times:
+# CHECK-NEXT: --------------------------------------------------------------------------
+# CHECK-NEXT: [    Range    ] :: [               Percentage               ] :: [Count]
+# CHECK-NEXT: --------------------------------------------------------------------------