[llvm] [Code Coverage] Add a tool to check test coverage of a patch (PR #71841)
Aiden Grossman via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 13 19:16:59 PST 2023
================
@@ -0,0 +1,722 @@
+#!/usr/bin/env python3
+#
+# ===- git-check-coverage - CheckCoverage Git Integration ---------*- python -*--===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===------------------------------------------------------------------------===#
+
+r"""
+code-coverage git integration
+============================
+This file provides a code-coverage integration for git. Put it in your
+llvm-project root directory and ensure that it is executable. Code
+coverage information will be provided for the last commit/HEAD by
+runing below command.
+Example uses -
+ git check-coverage -b build bin/opt llvm/test
+Here b is build directory (optional, default is build)
+next we have binray
+and then test suite path
+"""
+
+import argparse
+import logging
+import os
+import subprocess
+import re
+import sys
+from unidiff import PatchSet
+
+
+# Configure the logging module
+def configure_logging(build_dir):
+ logging.basicConfig(
+ filename=os.path.join(
+ build_dir, "patch_coverage.log"
+ ), # Specify the log file in the build directory
+ level=logging.INFO, # Set the logging level to INFO
+ format="%(message)s", # Specify the log message format
+ )
+
+
+# Define a custom print function that writes to both the log file and the terminal
+def custom_print(*args):
+ message = " ".join(map(str, args))
+ logging.info(message) # Write to the log file
+
+
+def create_patch_from_last_commit(output_path):
+ """Create a patch file from the last commit in the Git repository."""
+
+ try:
+ # Create the patch from the last commit
+ patch_cmd = ["git", "format-patch", "-1", "--stdout"]
+ patch_output = subprocess.check_output(patch_cmd).decode("utf-8", "ignore")
+
+ # Write the patch to the output file in binary mode
+ with open(output_path, "wb") as patch_file:
+ patch_file.write(patch_output.encode("utf-8"))
+
+ print("Patch file '{}' created successfully.".format(output_path))
+ print("")
+
+ except subprocess.CalledProcessError as e:
+ print("Error while creating the patch from the last commit:", e)
+ sys.exit(1)
+
+
+def extract_source_files_from_patch(patch_path):
+ """Read the patch file and extract the names of .cpp and .h files that
+ have been modified or added in the patch."""
+
+ try:
+ source_files = []
+ with open(patch_path, "rb") as patch_file:
+ patch_diff = patch_file.read().decode("utf-8", "ignore")
+
+ # Use regular expression to find .cpp files in the patch
+ source_file_matches = re.findall(r"\+{3} b/(\S+\.(?:cpp|c))", patch_diff)
+
+ # Filter out files with "test" in their directory path
+ source_files = [file for file in source_file_matches if "test" not in file]
+
+ print()
+ print("Source files in the patch (excluding test files):")
+ for source_file in source_files:
+ print(source_file)
+ print("")
+ return source_files
+
+ except Exception as ex:
+ print("Error while extracting .cpp files from patch:", ex)
+ sys.exit(1)
+
+
+def write_source_file_allowlist(source_files, output_path):
+ """Write a file containing the list of source files in the format"""
+ try:
+ # Get the absolute path of the current directory
+ current_directory = os.getcwd()
+ absolute_path = os.path.abspath(current_directory)
+
+ # Write the source file paths to the allowlist file in the specified format
+ with open(output_path, "w") as allowlist_file:
+ for source_file in source_files:
+ source_file = os.path.join(absolute_path, source_file)
+ allowlist_file.write("source:{}=allow\n".format(source_file))
+ allowlist_file.write("default:skip") # Specify default behavior
+
+ # Print a success message after writing the allowlist file
+ custom_print("Source file allowlist written to file '{}'.".format(output_path))
+ custom_print("")
+
+ except subprocess.CalledProcessError as e:
+ custom_print("Error while writing allow list for -fprofile-list:", e)
+ sys.exit(1)
+
+
+def extract_modified_source_lines_from_patch(patch_path, tests):
+ """Extract the modified source lines from the patch."""
+
+ source_lines = {} # Dictionary for modified lines in source code files
+
+ tests_relative = {os.path.relpath(file) for file in tests}
+
+ try:
+ # Parse the patch file using the unidiff library
+ patchset = PatchSet.from_filename(patch_path)
+ custom_print("All files in patch:")
+ for patched_file in patchset:
+ current_file = patched_file.target_file
+ # Check if the current file is not a test file
+ if os.path.relpath(current_file)[2:] not in tests:
+ custom_print(os.path.relpath(current_file)[2:])
+ # Initialize an empty list for modified lines in this file
+ source_lines[current_file] = []
+
+ for hunk in patched_file:
+ for line in hunk:
+ if line.is_added:
+ # Skip test file since we want only source file
+ if os.path.relpath(current_file)[2:] not in tests_relative:
+ # Append the modified line as a tuple (line number, line content)
+ source_lines[current_file].append(
+ (line.target_line_no, line.value)
+ )
+ custom_print("")
+
+ # Return dictionary of modified lines
+ return source_lines
+
+ except Exception as ex:
+ custom_print("Error while extracting modified lines from patch:", ex)
+ return {}
+
+
+def build_llvm(build_dir):
+ """Configure and build LLVM in the specified build directory."""
+
+ try:
+ cwd = os.getcwd()
+
+ # Change to the build directory
+ os.chdir(build_dir)
+
+ # Remove older profile files
+ command = 'find . -type f -name "*.profraw" -delete'
+ try:
+ subprocess.run(command, shell=True, check=True)
+ custom_print(
+ "Files in build directory with '.profraw' extension deleted successfully."
+ )
+ except subprocess.CalledProcessError as e:
+ custom_print(f"Error: {e}")
+ custom_print("")
+
+ # Run the cmake command to re-configure the LLVM build for coverage instrumentation
+ cmake_command = [
+ "cmake",
+ "-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON",
+ "-DLLVM_INDIVIDUAL_TEST_COVERAGE=ON",
+ f"-DCMAKE_C_FLAGS=-fprofile-list={os.path.abspath('fun.list')}",
+ f"-DCMAKE_CXX_FLAGS=-fprofile-list={os.path.abspath('fun.list')}",
+ ".",
+ ]
+
+ subprocess.check_call(cmake_command)
+
+ try:
+ # Run the ninja build command
+ print()
+ subprocess.check_call(["ninja"])
+ except subprocess.CalledProcessError as ninja_error:
+ custom_print(f"Error during Ninja build: {ninja_error}")
+ custom_print(
+ "Attempting to build with 'make' using the available processors."
+ )
+ # Get the number of processors on the system
+ num_processors = os.cpu_count() or 1
+ make_command = ["make", f"-j{num_processors}"]
+ subprocess.check_call(make_command)
+
+ os.chdir(cwd)
+
+ custom_print("LLVM build completed successfully.")
+ custom_print("")
+
+ except subprocess.CalledProcessError as e:
+ custom_print("Error during LLVM build:", e)
+ sys.exit(1)
+
+
+def run_single_test_with_coverage(llvm_lit_path, test_path):
+ """Run a single test case using llvm-lit with coverage."""
+
+ try:
+ # Run llvm-lit with --per-test-coverage
+ # https://llvm.org/docs/CommandGuide/lit.html#cmdoption-lit-per-test-coverage
+ lit_cmd = [llvm_lit_path, "--per-test-coverage", test_path]
+ subprocess.check_call(lit_cmd)
+
+ custom_print("Test case executed:", test_path)
+
+ except subprocess.CalledProcessError as e:
+ custom_print("Error while running test:", e)
+ sys.exit(1)
+
+ except Exception as ex:
+ custom_print("Error:", ex)
+ sys.exit(1)
+
+
+def run_modified_lit_tests(llvm_lit_path, patch_path, tests):
+ """Read the patch file, identify modified and added test cases, and
+ then execute each of these test cases."""
+
+ try:
+ # Get the list of modified and added test cases from the patch
+ with open(patch_path, "r") as patch_file:
+ patch_diff = patch_file.read()
+
+ modified_tests = []
+
+ # Use regular expressions to find modified test cases with ".ll|.c|.cpp" extension
+ for match in re.finditer(
+ r"^\+\+\+ [ab]/(.*\.(ll|mir|mlir|fir|c|cpp|f90|s|test))$",
----------------
boomanaiden154 wrote:
I think it would be better to just look at specific paths within the monorepo here rather than looking for file extensions.
Also, why not just look and see if the match is in `tests` like in `extract_modified_source_lines_from_patch`?
https://github.com/llvm/llvm-project/pull/71841
More information about the llvm-commits
mailing list