[llvm] [Code Coverage] Add a tool to check test coverage of a patch (PR #71841)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 13 19:16:58 PST 2023
================
@@ -0,0 +1,722 @@
+#!/usr/bin/env python3
+#
+# ===- git-check-coverage - CheckCoverage Git Integration ---------*- python -*--===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===------------------------------------------------------------------------===#
+
+r"""
+code-coverage git integration
+============================
+This file provides a code-coverage integration for git. Put it in your
+llvm-project root directory and ensure that it is executable. Code
+coverage information will be provided for the last commit/HEAD by
+runing below command.
+Example uses -
+ git check-coverage -b build bin/opt llvm/test
+Here b is build directory (optional, default is build)
+next we have binray
+and then test suite path
+"""
+
+import argparse
+import logging
+import os
+import subprocess
+import re
+import sys
+from unidiff import PatchSet
+
+
+# Configure the logging module
+def configure_logging(build_dir):
+ logging.basicConfig(
+ filename=os.path.join(
+ build_dir, "patch_coverage.log"
+ ), # Specify the log file in the build directory
+ level=logging.INFO, # Set the logging level to INFO
+ format="%(message)s", # Specify the log message format
+ )
+
+
+# Define a custom print function that writes to both the log file and the terminal
+def custom_print(*args):
+ message = " ".join(map(str, args))
+ logging.info(message) # Write to the log file
+
+
+def create_patch_from_last_commit(output_path):
+ """Create a patch file from the last commit in the Git repository."""
+
+ try:
+ # Create the patch from the last commit
+ patch_cmd = ["git", "format-patch", "-1", "--stdout"]
+ patch_output = subprocess.check_output(patch_cmd).decode("utf-8", "ignore")
+
+ # Write the patch to the output file in binary mode
+ with open(output_path, "wb") as patch_file:
+ patch_file.write(patch_output.encode("utf-8"))
+
+ print("Patch file '{}' created successfully.".format(output_path))
+ print("")
+
+ except subprocess.CalledProcessError as e:
+ print("Error while creating the patch from the last commit:", e)
+ sys.exit(1)
+
+
+def extract_source_files_from_patch(patch_path):
+ """Read the patch file and extract the names of .cpp and .h files that
+ have been modified or added in the patch."""
+
+ try:
+ source_files = []
+ with open(patch_path, "rb") as patch_file:
+ patch_diff = patch_file.read().decode("utf-8", "ignore")
+
+ # Use regular expression to find .cpp files in the patch
+ source_file_matches = re.findall(r"\+{3} b/(\S+\.(?:cpp|c))", patch_diff)
+
+ # Filter out files with "test" in their directory path
+ source_files = [file for file in source_file_matches if "test" not in file]
+
+ print()
+ print("Source files in the patch (excluding test files):")
+ for source_file in source_files:
+ print(source_file)
+ print("")
+ return source_files
+
+ except Exception as ex:
+ print("Error while extracting .cpp files from patch:", ex)
+ sys.exit(1)
+
+
+def write_source_file_allowlist(source_files, output_path):
+ """Write a file containing the list of source files in the format"""
+ try:
+ # Get the absolute path of the current directory
+ current_directory = os.getcwd()
+ absolute_path = os.path.abspath(current_directory)
+
+ # Write the source file paths to the allowlist file in the specified format
+ with open(output_path, "w") as allowlist_file:
+ for source_file in source_files:
+ source_file = os.path.join(absolute_path, source_file)
+ allowlist_file.write("source:{}=allow\n".format(source_file))
+ allowlist_file.write("default:skip") # Specify default behavior
+
+ # Print a success message after writing the allowlist file
+ custom_print("Source file allowlist written to file '{}'.".format(output_path))
+ custom_print("")
+
+ except subprocess.CalledProcessError as e:
+ custom_print("Error while writing allow list for -fprofile-list:", e)
+ sys.exit(1)
+
+
+def extract_modified_source_lines_from_patch(patch_path, tests):
+ """Extract the modified source lines from the patch."""
+
+ source_lines = {} # Dictionary for modified lines in source code files
+
+ tests_relative = {os.path.relpath(file) for file in tests}
+
+ try:
+ # Parse the patch file using the unidiff library
+ patchset = PatchSet.from_filename(patch_path)
+ custom_print("All files in patch:")
+ for patched_file in patchset:
+ current_file = patched_file.target_file
+ # Check if the current file is not a test file
+ if os.path.relpath(current_file)[2:] not in tests:
+ custom_print(os.path.relpath(current_file)[2:])
+ # Initialize an empty list for modified lines in this file
+ source_lines[current_file] = []
+
+ for hunk in patched_file:
+ for line in hunk:
+ if line.is_added:
+ # Skip test file since we want only source file
+ if os.path.relpath(current_file)[2:] not in tests_relative:
+ # Append the modified line as a tuple (line number, line content)
+ source_lines[current_file].append(
+ (line.target_line_no, line.value)
+ )
+ custom_print("")
+
+ # Return dictionary of modified lines
+ return source_lines
+
+ except Exception as ex:
+ custom_print("Error while extracting modified lines from patch:", ex)
+ return {}
+
+
+def build_llvm(build_dir):
+ """Configure and build LLVM in the specified build directory."""
+
+ try:
+ cwd = os.getcwd()
+
+ # Change to the build directory
+ os.chdir(build_dir)
+
+ # Remove older profile files
+ command = 'find . -type f -name "*.profraw" -delete'
+ try:
+ subprocess.run(command, shell=True, check=True)
+ custom_print(
+ "Files in build directory with '.profraw' extension deleted successfully."
+ )
+ except subprocess.CalledProcessError as e:
+ custom_print(f"Error: {e}")
+ custom_print("")
+
+ # Run the cmake command to re-configure the LLVM build for coverage instrumentation
+ cmake_command = [
+ "cmake",
+ "-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON",
+ "-DLLVM_INDIVIDUAL_TEST_COVERAGE=ON",
+ f"-DCMAKE_C_FLAGS=-fprofile-list={os.path.abspath('fun.list')}",
+ f"-DCMAKE_CXX_FLAGS=-fprofile-list={os.path.abspath('fun.list')}",
+ ".",
+ ]
+
+ subprocess.check_call(cmake_command)
+
+ try:
+ # Run the ninja build command
+ print()
+ subprocess.check_call(["ninja"])
+ except subprocess.CalledProcessError as ninja_error:
+ custom_print(f"Error during Ninja build: {ninja_error}")
+ custom_print(
+ "Attempting to build with 'make' using the available processors."
+ )
+ # Get the number of processors on the system
+ num_processors = os.cpu_count() or 1
+ make_command = ["make", f"-j{num_processors}"]
+ subprocess.check_call(make_command)
+
+ os.chdir(cwd)
+
+ custom_print("LLVM build completed successfully.")
+ custom_print("")
+
+ except subprocess.CalledProcessError as e:
+ custom_print("Error during LLVM build:", e)
+ sys.exit(1)
+
+
+def run_single_test_with_coverage(llvm_lit_path, test_path):
+ """Run a single test case using llvm-lit with coverage."""
+
+ try:
+ # Run llvm-lit with --per-test-coverage
+ # https://llvm.org/docs/CommandGuide/lit.html#cmdoption-lit-per-test-coverage
+ lit_cmd = [llvm_lit_path, "--per-test-coverage", test_path]
+ subprocess.check_call(lit_cmd)
+
+ custom_print("Test case executed:", test_path)
+
+ except subprocess.CalledProcessError as e:
+ custom_print("Error while running test:", e)
+ sys.exit(1)
+
+ except Exception as ex:
+ custom_print("Error:", ex)
+ sys.exit(1)
+
+
+def run_modified_lit_tests(llvm_lit_path, patch_path, tests):
+ """Read the patch file, identify modified and added test cases, and
+ then execute each of these test cases."""
+
+ try:
+ # Get the list of modified and added test cases from the patch
+ with open(patch_path, "r") as patch_file:
+ patch_diff = patch_file.read()
+
+ modified_tests = []
+
+ # Use regular expressions to find modified test cases with ".ll|.c|.cpp" extension
+ for match in re.finditer(
+ r"^\+\+\+ [ab]/(.*\.(ll|mir|mlir|fir|c|cpp|f90|s|test))$",
+ patch_diff,
+ re.MULTILINE,
+ ):
+ test_file = match.group(1)
+
+ # Get the current working directory
+ cwd = os.getcwd()
+
+ # Build the full file path dynamically by going two steps back from cwd
+ full_test_file = os.path.join(
+ os.path.dirname(cwd), "llvm-project", test_file
+ )
+
+ if full_test_file in tests:
+ custom_print("Lit test file in the patch:", test_file)
+ custom_print("Full lit test file path:", full_test_file)
+
+ # Check if the file name is starts with +++
+ if match.group(0).startswith("+++"):
+ modified_tests.append(full_test_file)
+
+ if not modified_tests:
+ custom_print("No modified lit tests found in the patch.")
+ return
+
+ # Run each modified test case
+ custom_print("")
+ custom_print("Running modified test cases:")
+ for test_file in modified_tests:
+ run_single_test_with_coverage(llvm_lit_path, test_file)
+
+ except subprocess.CalledProcessError as e:
+ custom_print("Error while running modified tests:", e)
+ sys.exit(1)
+
+ except Exception as ex:
+ custom_print("Error:", ex)
+ sys.exit(1)
+
+
+def run_modified_unit_tests(build_dir, patch_path, tests):
+ """Read the patch file, identify modified and added test cases, and
+ then execute each of these test cases."""
+
+ try:
+ # Get the list of modified and added test cases from the patch
+ with open(patch_path, "r") as patch_file:
+ patch_diff = patch_file.read()
+
+ modified_tests = []
+
+ custom_print()
+ # Use regular expressions to find modified test cases with ".ll|.c|.cpp" extension
+ for match in re.finditer(
+ r"^\+\+\+ [ab]/(.*\.(c|cpp|f90))$",
----------------
boomanaiden154 wrote:
Are there any unit tests in `.f90`? Also, is there a more structured way to do this with some lit invocation?
https://github.com/llvm/llvm-project/pull/71841
More information about the llvm-commits
mailing list