[llvm] 64d1954 - [LIT] Added an option to llvm-lit to emit the necessary test coverage data, divided per test case
Shivam Gupta via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 26 04:19:12 PDT 2023
Author: Shivam Gupta
Date: 2023-07-26T16:47:17+05:30
New Revision: 64d19542e78a43edb7ae26ea6762a2b1c360a916
URL: https://github.com/llvm/llvm-project/commit/64d19542e78a43edb7ae26ea6762a2b1c360a916
DIFF: https://github.com/llvm/llvm-project/commit/64d19542e78a43edb7ae26ea6762a2b1c360a916.diff
LOG: [LIT] Added an option to llvm-lit to emit the necessary test coverage data, divided per test case
This patch is the first part of https://llvm.org/OpenProjects.html#llvm_patch_coverage.
We have first define a new variable LLVM_TEST_COVERAGE which when set, pass --per-test-coverage option to
llvm-lit which will help in setting a unique value to LLVM_PROFILE_FILE for each RUN. So for example
coverage data for test case llvm/test/Analysis/AliasSet/memtransfer.ll will be emitted as
build/test/Analysis/AliasSet/memtransfer0.profraw
Reviewed By: hnrklssn
Differential Revision: https://reviews.llvm.org/D154280
Added:
llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/lit.cfg
llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py
llvm/utils/lit/tests/Inputs/per-test-coverage/lit.cfg
llvm/utils/lit/tests/Inputs/per-test-coverage/per-test-coverage.py
llvm/utils/lit/tests/per-test-coverage-by-lit-cfg.py
llvm/utils/lit/tests/per-test-coverage.py
Modified:
llvm/CMakeLists.txt
llvm/cmake/modules/HandleLLVMOptions.cmake
llvm/docs/CMake.rst
llvm/docs/CommandGuide/lit.rst
llvm/utils/lit/lit/LitConfig.py
llvm/utils/lit/lit/TestRunner.py
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/main.py
Removed:
################################################################################
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index f566a2ef43e3ab..6f0bbf99e86037 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -676,6 +676,9 @@ set(LIT_ARGS_DEFAULT "-sv")
if (MSVC_IDE OR XCODE)
set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --no-progress-bar")
endif()
+if(LLVM_INDIVIDUAL_TEST_COVERAGE)
+ set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --per-test-coverage")
+endif()
set(LLVM_LIT_ARGS "${LIT_ARGS_DEFAULT}" CACHE STRING "Default options for lit")
# On Win32 hosts, provide an option to specify the path to the GnuWin32 tools.
@@ -797,24 +800,25 @@ if (MSVC_IDE)
option(LLVM_ADD_NATIVE_VISUALIZERS_TO_SOLUTION "Configure project to use Visual Studio native visualizers" TRUE)
endif()
-if (LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE OR
- LLVM_ENABLE_IR_PGO)
- if(NOT LLVM_PROFILE_MERGE_POOL_SIZE)
- # A pool size of 1-2 is probably sufficient on a SSD. 3-4 should be fine
- # for spining disks. Anything higher may only help on slower mediums.
- set(LLVM_PROFILE_MERGE_POOL_SIZE "4")
- endif()
- if(NOT LLVM_PROFILE_FILE_PATTERN)
- if(NOT LLVM_PROFILE_DATA_DIR)
- file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/profiles" LLVM_PROFILE_DATA_DIR)
+if(NOT LLVM_INDIVIDUAL_TEST_COVERAGE)
+ if(LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE OR LLVM_ENABLE_IR_PGO)
+ if(NOT LLVM_PROFILE_MERGE_POOL_SIZE)
+ # A pool size of 1-2 is probably sufficient on an SSD. 3-4 should be fine
+ # for spinning disks. Anything higher may only help on slower mediums.
+ set(LLVM_PROFILE_MERGE_POOL_SIZE "4")
endif()
- file(TO_NATIVE_PATH "${LLVM_PROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_PROFILE_FILE_PATTERN)
- endif()
- if(NOT LLVM_CSPROFILE_FILE_PATTERN)
- if(NOT LLVM_CSPROFILE_DATA_DIR)
- file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/csprofiles" LLVM_CSPROFILE_DATA_DIR)
+ if(NOT LLVM_PROFILE_FILE_PATTERN)
+ if(NOT LLVM_PROFILE_DATA_DIR)
+ file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/profiles" LLVM_PROFILE_DATA_DIR)
+ endif()
+ file(TO_NATIVE_PATH "${LLVM_PROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_PROFILE_FILE_PATTERN)
+ endif()
+ if(NOT LLVM_CSPROFILE_FILE_PATTERN)
+ if(NOT LLVM_CSPROFILE_DATA_DIR)
+ file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/csprofiles" LLVM_CSPROFILE_DATA_DIR)
+ endif()
+ file(TO_NATIVE_PATH "${LLVM_CSPROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_CSPROFILE_FILE_PATTERN)
endif()
- file(TO_NATIVE_PATH "${LLVM_CSPROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_CSPROFILE_FILE_PATTERN)
endif()
endif()
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 19d6c58d991b7c..e8c8dff1f30c91 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -1160,6 +1160,7 @@ if(LLVM_PROFDATA_FILE AND EXISTS ${LLVM_PROFDATA_FILE})
endif()
option(LLVM_BUILD_INSTRUMENTED_COVERAGE "Build LLVM and tools with Code Coverage instrumentation" Off)
+option(LLVM_INDIVIDUAL_TEST_COVERAGE "Emit individual coverage file for each test case." OFF)
mark_as_advanced(LLVM_BUILD_INSTRUMENTED_COVERAGE)
append_if(LLVM_BUILD_INSTRUMENTED_COVERAGE "-fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\" -fcoverage-mapping"
CMAKE_CXX_FLAGS
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index c566b590a58438..d04feda0a771c1 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -375,6 +375,12 @@ enabled sub-projects. Nearly all of these variable names begin with
will limit code coverage summaries to just the listed directories. If unset,
coverage reports will include all sources identified by the tooling.
+ **LLVM_INDIVIDUAL_TEST_COVERAGE**: BOOL
+ Enable individual test case coverage. When set to ON, code coverage data for
+ each test case will be generated and stored in a separate directory under the
+ config.test_exec_root path. This feature allows code coverage analysis of each
+ individual test case. Defaults to OFF.
+
**LLVM_BUILD_LLVM_DYLIB**:BOOL
If enabled, the target for building the libLLVM shared library is added.
This library contains all of LLVM's components in a single shared library.
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 2f587c43585aa2..e82cfeb0695c1f 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -181,6 +181,12 @@ The timing data is stored in the `test_exec_root` in a file named
Run the tests in a random order, not failing/slowest first. Deprecated,
use :option:`--order` instead.
+.. option:: --per-test-coverage
+
+ Emit the necessary test coverage data, divided per test case (involves
+ setting a unique value to LLVM_PROFILE_FILE for each RUN). The coverage
+ data files will be emitted in the directory specified by `config.test_exec_root`.
+
.. option:: --max-failures N
Stop execution after the given number ``N`` of failures.
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index fcd6825f1d6da7..331c21d7b9dfcf 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -37,6 +37,7 @@ def __init__(
maxIndividualTestTime=0,
parallelism_groups={},
echo_all_commands=False,
+ per_test_coverage=False,
):
# The name of the test runner.
self.progname = progname
@@ -87,6 +88,7 @@ def __init__(
self.maxIndividualTestTime = maxIndividualTestTime
self.parallelism_groups = parallelism_groups
self.echo_all_commands = echo_all_commands
+ self.per_test_coverage = per_test_coverage
@property
def maxIndividualTestTime(self):
@@ -128,6 +130,22 @@ def maxIndividualTestTime(self, value):
elif self.maxIndividualTestTime < 0:
self.fatal("The timeout per test must be >= 0 seconds")
+ @property
+ def per_test_coverage(self):
+ """
+ Interface for getting the per_test_coverage value
+ """
+ return self._per_test_coverage
+
+ @per_test_coverage.setter
+ def per_test_coverage(self, value):
+ """
+ Interface for setting the per_test_coverage value
+ """
+ if not isinstance(value, bool):
+ self.fatal("per_test_coverage must set to a value of type bool.")
+ self._per_test_coverage = value
+
def load_config(self, config, path):
"""load_config(config, path) - Load a config object from an alternate
path."""
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 24670610e3a57c..5c639b76155eb1 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1067,10 +1067,25 @@ def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
def executeScript(test, litConfig, tmpBase, commands, cwd):
bashPath = litConfig.getBashPath()
isWin32CMDEXE = litConfig.isWindows and not bashPath
+ coverage_index = 0 # Counter for coverage file index
script = tmpBase + ".script"
if isWin32CMDEXE:
script += ".bat"
+ # Set unique LLVM_PROFILE_FILE for each run command
+ for j, ln in enumerate(commands):
+ match = re.match(kPdbgRegex, ln)
+ if match:
+ command = match.group(2)
+ commands[j] = match.expand(": '\\1'; \\2" if command else ": '\\1'")
+ if litConfig.per_test_coverage:
+ # Extract the test case name from the test object
+ test_case_name = test.path_in_suite[-1]
+ test_case_name = test_case_name.rsplit(".", 1)[0] # Remove the file extension
+ llvm_profile_file = f"{test_case_name}{coverage_index}.profraw"
+ commands[j] = f"export LLVM_PROFILE_FILE={llvm_profile_file} && {commands[j]}"
+ coverage_index += 1
+
# Write script file
mode = "w"
open_kwargs = {}
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 5c0dde5e883a5b..747824574dd67f 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -184,6 +184,12 @@ def parse_args():
help="Do not fail the run if all tests are filtered out",
action="store_true",
)
+ execution_group.add_argument(
+ "--per-test-coverage",
+ dest="per_test_coverage",
+ action="store_true",
+ help="Enable individual test case coverage",
+ )
execution_group.add_argument(
"--ignore-fail",
dest="ignoreFail",
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index cf9186d528cd23..6858961752a66f 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -41,6 +41,7 @@ def main(builtin_params={}):
params=params,
config_prefix=opts.configPrefix,
echo_all_commands=opts.echoAllCommands,
+ per_test_coverage=opts.per_test_coverage,
)
discovered_tests = lit.discovery.find_tests_for_inputs(
diff --git a/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/lit.cfg b/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/lit.cfg
new file mode 100644
index 00000000000000..186c81ebd17283
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/lit.cfg
@@ -0,0 +1,8 @@
+import lit.formats
+import os
+
+config.name = "per-test-coverage-by-lit-cfg"
+config.suffixes = [".py"]
+config.test_format = lit.formats.ShTest(execute_external=True)
+lit_config.per_test_coverage = True
+config.substitutions.append(("%{python}", '"%s"' % (sys.executable)))
diff --git a/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py b/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py
new file mode 100644
index 00000000000000..0abe00bdedb1f2
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py
@@ -0,0 +1,11 @@
+# Check that the environment variable is set correctly
+# RUN: %{python} %s | FileCheck %s
+
+# Python script to read the environment variable
+# and print its value
+import os
+
+llvm_profile_file = os.environ.get('LLVM_PROFILE_FILE')
+print(llvm_profile_file)
+
+# CHECK: per-test-coverage-by-lit-cfg0.profraw
diff --git a/llvm/utils/lit/tests/Inputs/per-test-coverage/lit.cfg b/llvm/utils/lit/tests/Inputs/per-test-coverage/lit.cfg
new file mode 100644
index 00000000000000..6e7f4c0e3bc97d
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/per-test-coverage/lit.cfg
@@ -0,0 +1,8 @@
+import lit.formats
+import os
+
+config.name = "per-test-coverage"
+config.suffixes = [".py"]
+config.test_format = lit.formats.ShTest(execute_external=True)
+config.substitutions.append(("%{python}", '"%s"' % (sys.executable)))
+
diff --git a/llvm/utils/lit/tests/Inputs/per-test-coverage/per-test-coverage.py b/llvm/utils/lit/tests/Inputs/per-test-coverage/per-test-coverage.py
new file mode 100644
index 00000000000000..65e4aa69bff079
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/per-test-coverage/per-test-coverage.py
@@ -0,0 +1,11 @@
+# Check that the environment variable is set correctly
+# RUN: %{python} %s | FileCheck %s
+
+# Python script to read the environment variable
+# and print its value
+import os
+
+llvm_profile_file = os.environ.get('LLVM_PROFILE_FILE')
+print(llvm_profile_file)
+
+# CHECK: per-test-coverage0.profraw
diff --git a/llvm/utils/lit/tests/per-test-coverage-by-lit-cfg.py b/llvm/utils/lit/tests/per-test-coverage-by-lit-cfg.py
new file mode 100644
index 00000000000000..1cc3927dc89041
--- /dev/null
+++ b/llvm/utils/lit/tests/per-test-coverage-by-lit-cfg.py
@@ -0,0 +1,6 @@
+# Test if lit_config.per_test_coverage in lit.cfg sets individual test case coverage.
+
+# RUN: %{lit} -a -v %{inputs}/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py \
+# RUN: | FileCheck -match-full-lines %s
+#
+# CHECK: PASS: per-test-coverage-by-lit-cfg :: per-test-coverage-by-lit-cfg.py ({{[^)]*}})
diff --git a/llvm/utils/lit/tests/per-test-coverage.py b/llvm/utils/lit/tests/per-test-coverage.py
new file mode 100644
index 00000000000000..428712e4792576
--- /dev/null
+++ b/llvm/utils/lit/tests/per-test-coverage.py
@@ -0,0 +1,6 @@
+# Test LLVM_PROFILE_FILE is set when --per-test-coverage is passed to command line.
+
+# RUN: %{lit} -a -v --per-test-coverage %{inputs}/per-test-coverage/per-test-coverage.py \
+# RUN: | FileCheck -match-full-lines %s
+#
+# CHECK: PASS: per-test-coverage :: per-test-coverage.py ({{[^)]*}})
More information about the llvm-commits
mailing list