[Openmp-commits] [libcxx] [llvm] [openmp] [lit] Add a flag to disable lit time tests (PR #98270)
Vincent Lee via Openmp-commits
openmp-commits at lists.llvm.org
Thu Jul 11 22:43:22 PDT 2024
https://github.com/thevinster updated https://github.com/llvm/llvm-project/pull/98270
>From b948e54960de38985defaae6c2738be4f1b94754 Mon Sep 17 00:00:00 2001
From: Vincent Lee <leevince at fb.com>
Date: Tue, 9 Jul 2024 18:56:46 -0700
Subject: [PATCH 1/3] [lit] Add a flag to disable lit time tests
---
.ci/monolithic-linux.sh | 4 ++--
.ci/monolithic-windows.sh | 2 +-
libcxx/utils/ci/run-buildbot | 4 ++--
llvm/docs/CommandGuide/lit.rst | 7 ++++++-
llvm/utils/lit/lit/cl_arguments.py | 16 +++++++++++-----
llvm/utils/lit/lit/main.py | 5 +++--
offload/cmake/OpenMPTesting.cmake | 2 +-
openmp/cmake/OpenMPTesting.cmake | 2 +-
8 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index b78dc59432b65..04395621a8f27 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -47,7 +47,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
- -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
+ -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests-histogram" \
-D LLVM_ENABLE_LLD=ON \
-D CMAKE_CXX_FLAGS=-gmlt \
-D LLVM_CCACHE_BUILD=ON \
@@ -124,6 +124,6 @@ if [[ "${runtimes}" != "" ]]; then
-D LIBCXXABI_TEST_PARAMS="enable_modules=clang"
echo "--- ninja runtimes clang modules"
-
+
ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
fi
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index 91e719c52d436..b64856af044be 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -53,7 +53,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
- -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
+ -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests-histogram" \
-D COMPILER_RT_BUILD_ORC=OFF \
-D CMAKE_C_COMPILER_LAUNCHER=sccache \
-D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index f1c20b9d72190..a59c5930b5252 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -119,7 +119,7 @@ function generate-cmake-base() {
-DLIBCXX_ENABLE_WERROR=YES \
-DLIBCXXABI_ENABLE_WERROR=YES \
-DLIBUNWIND_ENABLE_WERROR=YES \
- -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests" \
+ -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests-histogram" \
"${@}"
}
@@ -381,7 +381,7 @@ bootstrapping-build)
-DLLVM_TARGETS_TO_BUILD="host" \
-DRUNTIMES_BUILD_ALLOW_DARWIN=ON \
-DLLVM_ENABLE_ASSERTIONS=ON \
- -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests"
+ -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests-histogram"
echo "+++ Running the LLDB libc++ data formatter tests"
${NINJA} -vC "${BUILD_DIR}" check-lldb-api-functionalities-data-formatter-data-formatter-stl-libcxx
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 799ee34e9f9ff..c63f679e74a4b 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -151,7 +151,12 @@ EXECUTION OPTIONS
feature that can be used to conditionally disable (or expect failure in)
certain tests.
-.. option:: --time-tests
+.. option:: --no-time-tests
+
+ Disable tracking the wall time individual tests take to execute. This is useful
+ for external build systems to orchestrate the scheduled tests.
+
+.. option:: --time-tests-histogram
Track the wall time individual tests take to execute and includes the results
in the summary output. This is useful for determining which tests in a test
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index b9122d07afd8a..8df76ef4b8c82 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -154,11 +154,6 @@ def parse_args():
action="append",
default=[],
)
- execution_group.add_argument(
- "--time-tests",
- help="Track elapsed wall time for each test",
- action="store_true",
- )
execution_group.add_argument(
"--no-execute",
dest="noExecute",
@@ -209,6 +204,17 @@ def parse_args():
action="store_true",
help="Exit with status zero even if some tests fail",
)
+ execution_test_time_group = execution_group.add_mutually_exclusive_group()
+ execution_test_time_group.add_argument(
+ "--no-time-tests",
+ help="Do not track elapsed wall time for each test",
+ action="store_true",
+ )
+ execution_test_time_group.add_argument(
+ "--time-tests-histogram",
+ help="Track elapsed wall time for each test in a histogram",
+ action="store_true",
+ )
selection_group = parser.add_argument_group("Test Selection")
selection_group.add_argument(
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index db9f24f748d9e..8cc784dbf6471 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -124,13 +124,14 @@ def main(builtin_params={}):
run_tests(selected_tests, lit_config, opts, len(discovered_tests))
elapsed = time.time() - start
- record_test_times(selected_tests, lit_config)
+ if not opts.no_time_tests or opts.time_tests_histogram:
+ record_test_times(selected_tests, lit_config)
selected_tests, discovered_tests = GoogleTest.post_process_shard_results(
selected_tests, discovered_tests
)
- if opts.time_tests:
+ if opts.time_tests_histogram:
print_histogram(discovered_tests)
print_results(discovered_tests, elapsed, opts)
diff --git a/offload/cmake/OpenMPTesting.cmake b/offload/cmake/OpenMPTesting.cmake
index 11eafeb764260..fa767bb291d8a 100644
--- a/offload/cmake/OpenMPTesting.cmake
+++ b/offload/cmake/OpenMPTesting.cmake
@@ -58,7 +58,7 @@ if (${OPENMP_STANDALONE_BUILD})
set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --no-progress-bar")
endif()
if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
- set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests --timeout=1800")
+ set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests-histogram --timeout=1800")
endif()
set(OPENMP_LIT_ARGS "${DEFAULT_LIT_ARGS}" CACHE STRING "Options for lit.")
separate_arguments(OPENMP_LIT_ARGS)
diff --git a/openmp/cmake/OpenMPTesting.cmake b/openmp/cmake/OpenMPTesting.cmake
index c67ad8b1cbd9c..fb58cad04c019 100644
--- a/openmp/cmake/OpenMPTesting.cmake
+++ b/openmp/cmake/OpenMPTesting.cmake
@@ -58,7 +58,7 @@ if (${OPENMP_STANDALONE_BUILD})
set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --no-progress-bar")
endif()
if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
- set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests --timeout=3000")
+ set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests-histogram --timeout=3000")
endif()
set(OPENMP_LIT_ARGS "${DEFAULT_LIT_ARGS}" CACHE STRING "Options for lit.")
separate_arguments(OPENMP_LIT_ARGS)
>From 9590d80cd5550d48277327810778595d1523cf60 Mon Sep 17 00:00:00 2001
From: Vincent Lee <leevince at fb.com>
Date: Wed, 10 Jul 2024 21:37:22 -0700
Subject: [PATCH 2/3] Fix wording, add release note changes
---
llvm/docs/CommandGuide/lit.rst | 4 ++--
llvm/docs/ReleaseNotes.rst | 10 +++++++---
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index c63f679e74a4b..be39a647cb883 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -153,8 +153,8 @@ EXECUTION OPTIONS
.. option:: --no-time-tests
- Disable tracking the wall time individual tests take to execute. This is useful
- for external build systems to orchestrate the scheduled tests.
+ Disable tracking the wall time individual tests take to execute. This allows
+ external build systems to orchestrate the scheduled tests.
.. option:: --time-tests-histogram
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 9e8a2b24e1a4e..78926df729570 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -114,7 +114,7 @@ Changes to the AArch64 Backend
Cortex-A725, Cortex-X925, Neoverse-N3, Neoverse-V3 and Neoverse-V3AE CPUs.
* ``-mbranch-protection=standard`` now enables FEAT_PAuth_LR by
- default when the feature is enabled. The new behaviour results
+ default when the feature is enabled. The new behaviour results
in ``standard`` being equal to ``bti+pac-ret+pc`` when ``+pauth-lr``
is passed as part of ``-mcpu=`` options.
@@ -370,11 +370,15 @@ Changes to the LLVM tools
<https://github.com/llvm/llvm-project/pull/92835>`_).
* llvm-cov now generates HTML report with JavaScript code to allow simple
- jumping between uncovered parts (lines/regions/branches) of code
- using buttons on top-right corner of the page or using keys (L/R/B or
+ jumping between uncovered parts (lines/regions/branches) of code
+ using buttons on top-right corner of the page or using keys (L/R/B or
jumping in reverse direction with shift+L/R/B). (`#95662
<https://github.com/llvm/llvm-project/pull/95662>`_).
+* llvm-lit's ``--time-tests`` has been renamed to ``--time-tests-histogram``.
+ Additionally, a new option, ``--no-time-tests`` is added as an option to
+ disable lit from creating ``.lit_test_times.txt``.
+
Changes to LLDB
---------------------------------
>From 486a265ba4f1ba00208ee81d7f8d365ef3644b93 Mon Sep 17 00:00:00 2001
From: Vincent Lee <leevince at fb.com>
Date: Thu, 11 Jul 2024 22:41:42 -0700
Subject: [PATCH 3/3] Remove breaking change and rename flag
---
.ci/monolithic-linux.sh | 4 ++--
.ci/monolithic-windows.sh | 2 +-
libcxx/utils/ci/run-buildbot | 4 ++--
llvm/docs/CommandGuide/lit.rst | 4 ++--
llvm/docs/ReleaseNotes.rst | 10 +++-------
llvm/utils/lit/lit/cl_arguments.py | 4 ++--
llvm/utils/lit/lit/main.py | 4 ++--
offload/cmake/OpenMPTesting.cmake | 2 +-
openmp/cmake/OpenMPTesting.cmake | 2 +-
9 files changed, 16 insertions(+), 20 deletions(-)
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 04395621a8f27..b78dc59432b65 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -47,7 +47,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
- -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests-histogram" \
+ -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
-D LLVM_ENABLE_LLD=ON \
-D CMAKE_CXX_FLAGS=-gmlt \
-D LLVM_CCACHE_BUILD=ON \
@@ -124,6 +124,6 @@ if [[ "${runtimes}" != "" ]]; then
-D LIBCXXABI_TEST_PARAMS="enable_modules=clang"
echo "--- ninja runtimes clang modules"
-
+
ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
fi
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index b64856af044be..91e719c52d436 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -53,7 +53,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
- -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests-histogram" \
+ -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
-D COMPILER_RT_BUILD_ORC=OFF \
-D CMAKE_C_COMPILER_LAUNCHER=sccache \
-D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index a59c5930b5252..f1c20b9d72190 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -119,7 +119,7 @@ function generate-cmake-base() {
-DLIBCXX_ENABLE_WERROR=YES \
-DLIBCXXABI_ENABLE_WERROR=YES \
-DLIBUNWIND_ENABLE_WERROR=YES \
- -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests-histogram" \
+ -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests" \
"${@}"
}
@@ -381,7 +381,7 @@ bootstrapping-build)
-DLLVM_TARGETS_TO_BUILD="host" \
-DRUNTIMES_BUILD_ALLOW_DARWIN=ON \
-DLLVM_ENABLE_ASSERTIONS=ON \
- -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests-histogram"
+ -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests"
echo "+++ Running the LLDB libc++ data formatter tests"
${NINJA} -vC "${BUILD_DIR}" check-lldb-api-functionalities-data-formatter-data-formatter-stl-libcxx
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index be39a647cb883..d6d8e2132e020 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -151,12 +151,12 @@ EXECUTION OPTIONS
feature that can be used to conditionally disable (or expect failure in)
certain tests.
-.. option:: --no-time-tests
+.. option:: --skip-test-recording
Disable tracking the wall time individual tests take to execute. This allows
external build systems to orchestrate the scheduled tests.
-.. option:: --time-tests-histogram
+.. option:: --time-tests
Track the wall time individual tests take to execute and includes the results
in the summary output. This is useful for determining which tests in a test
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 78926df729570..9e8a2b24e1a4e 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -114,7 +114,7 @@ Changes to the AArch64 Backend
Cortex-A725, Cortex-X925, Neoverse-N3, Neoverse-V3 and Neoverse-V3AE CPUs.
* ``-mbranch-protection=standard`` now enables FEAT_PAuth_LR by
- default when the feature is enabled. The new behaviour results
+ default when the feature is enabled. The new behaviour results
in ``standard`` being equal to ``bti+pac-ret+pc`` when ``+pauth-lr``
is passed as part of ``-mcpu=`` options.
@@ -370,15 +370,11 @@ Changes to the LLVM tools
<https://github.com/llvm/llvm-project/pull/92835>`_).
* llvm-cov now generates HTML report with JavaScript code to allow simple
- jumping between uncovered parts (lines/regions/branches) of code
- using buttons on top-right corner of the page or using keys (L/R/B or
+ jumping between uncovered parts (lines/regions/branches) of code
+ using buttons on top-right corner of the page or using keys (L/R/B or
jumping in reverse direction with shift+L/R/B). (`#95662
<https://github.com/llvm/llvm-project/pull/95662>`_).
-* llvm-lit's ``--time-tests`` has been renamed to ``--time-tests-histogram``.
- Additionally, a new option, ``--no-time-tests`` is added as an option to
- disable lit from creating ``.lit_test_times.txt``.
-
Changes to LLDB
---------------------------------
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 8df76ef4b8c82..621828e23bddf 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -206,12 +206,12 @@ def parse_args():
)
execution_test_time_group = execution_group.add_mutually_exclusive_group()
execution_test_time_group.add_argument(
- "--no-time-tests",
+ "--skip-test-recording",
help="Do not track elapsed wall time for each test",
action="store_true",
)
execution_test_time_group.add_argument(
- "--time-tests-histogram",
+ "--time-tests",
help="Track elapsed wall time for each test in a histogram",
action="store_true",
)
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 8cc784dbf6471..664a83be5767f 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -124,14 +124,14 @@ def main(builtin_params={}):
run_tests(selected_tests, lit_config, opts, len(discovered_tests))
elapsed = time.time() - start
- if not opts.no_time_tests or opts.time_tests_histogram:
+ if not opts.skip_test_recording or opts.time_tests:
record_test_times(selected_tests, lit_config)
selected_tests, discovered_tests = GoogleTest.post_process_shard_results(
selected_tests, discovered_tests
)
- if opts.time_tests_histogram:
+ if opts.time_tests:
print_histogram(discovered_tests)
print_results(discovered_tests, elapsed, opts)
diff --git a/offload/cmake/OpenMPTesting.cmake b/offload/cmake/OpenMPTesting.cmake
index fa767bb291d8a..11eafeb764260 100644
--- a/offload/cmake/OpenMPTesting.cmake
+++ b/offload/cmake/OpenMPTesting.cmake
@@ -58,7 +58,7 @@ if (${OPENMP_STANDALONE_BUILD})
set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --no-progress-bar")
endif()
if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
- set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests-histogram --timeout=1800")
+ set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests --timeout=1800")
endif()
set(OPENMP_LIT_ARGS "${DEFAULT_LIT_ARGS}" CACHE STRING "Options for lit.")
separate_arguments(OPENMP_LIT_ARGS)
diff --git a/openmp/cmake/OpenMPTesting.cmake b/openmp/cmake/OpenMPTesting.cmake
index fb58cad04c019..c67ad8b1cbd9c 100644
--- a/openmp/cmake/OpenMPTesting.cmake
+++ b/openmp/cmake/OpenMPTesting.cmake
@@ -58,7 +58,7 @@ if (${OPENMP_STANDALONE_BUILD})
set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --no-progress-bar")
endif()
if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
- set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests-histogram --timeout=3000")
+ set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests --timeout=3000")
endif()
set(OPENMP_LIT_ARGS "${DEFAULT_LIT_ARGS}" CACHE STRING "Options for lit.")
separate_arguments(OPENMP_LIT_ARGS)
More information about the Openmp-commits
mailing list