[llvm] [Utils] Add a python tool for test coverage of patches (PR #191721)
Shivam Gupta via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 21:58:23 PDT 2026
https://github.com/xgupta updated https://github.com/llvm/llvm-project/pull/191721
>From ff3b08878711e744219c1ada36d64856e4fc8b9b Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Thu, 9 Apr 2026 19:09:45 +0530
Subject: [PATCH] Add patch based code coverage tool
---
llvm/utils/patch-coverage/README.md | 82 ++++++++++
llvm/utils/patch-coverage/__init__.py | 0
llvm/utils/patch-coverage/build.py | 155 +++++++++++++++++++
llvm/utils/patch-coverage/git-patch-coverage | 2 +
llvm/utils/patch-coverage/lit.py | 46 ++++++
llvm/utils/patch-coverage/main.py | 130 ++++++++++++++++
llvm/utils/patch-coverage/patch.py | 126 +++++++++++++++
llvm/utils/patch-coverage/print.py | 65 ++++++++
llvm/utils/patch-coverage/process.py | 104 +++++++++++++
llvm/utils/patch-coverage/requirements.txt | 1 +
llvm/utils/patch-coverage/test.py | 126 +++++++++++++++
llvm/utils/patch-coverage/utils.py | 24 +++
12 files changed, 861 insertions(+)
create mode 100644 llvm/utils/patch-coverage/README.md
create mode 100644 llvm/utils/patch-coverage/__init__.py
create mode 100644 llvm/utils/patch-coverage/build.py
create mode 100755 llvm/utils/patch-coverage/git-patch-coverage
create mode 100644 llvm/utils/patch-coverage/lit.py
create mode 100644 llvm/utils/patch-coverage/main.py
create mode 100644 llvm/utils/patch-coverage/patch.py
create mode 100644 llvm/utils/patch-coverage/print.py
create mode 100644 llvm/utils/patch-coverage/process.py
create mode 100644 llvm/utils/patch-coverage/requirements.txt
create mode 100644 llvm/utils/patch-coverage/test.py
create mode 100644 llvm/utils/patch-coverage/utils.py
diff --git a/llvm/utils/patch-coverage/README.md b/llvm/utils/patch-coverage/README.md
new file mode 100644
index 0000000000000..b44cc38371c1f
--- /dev/null
+++ b/llvm/utils/patch-coverage/README.md
@@ -0,0 +1,82 @@
+# Patch based Code Coverage git integration
+
+This tool provides patch-level code coverage integration for Git workflows in the LLVM project.
+It helps developers and reviewers to understand which lines introduced by patch are
+covered by tests and which are not.
+
+It is designed to integrate with LLVM’s existing infrastructure (e.g., llvm-lit,
+coverage instrumentation, and CI), and can be used in both local system system and LLVM GitHub
+CI to provide patch coverage for pull requests.
+
+# Overview
+Given a set of recent commits:
+Extracts the patch (git diff)
+Identifies modified source files and lines
+Detects modified tests (lit-based)
+Rebuilds LLVM with coverage instrumentation
+Executes relevant tests using llvm-lit --per-test-coverage
+Collects coverage data using llvm-profdata and llvm-cov
+Reports:
+Covered lines introduced by the patch
+Uncovered lines introduced by the patch
+
+
+# Setup
+Build LLVM (standard workflow):
+```shell
+$ git clone https://github.com/llvm/llvm-project.git
+$ cd llvm-project
+$ mkdir build
+$ cmake -G Ninja -B build -S llvm \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_ENABLE_ASSERTIONS=ON
+ -DLLVM_INCLUDE_TESTS=ON
+ -DLLVM_BUILD_TESTS=ON
+
+$ ninja -C build
+```
+
+Add the tool into your path so that you can use it as git subcommand.
+```shell
+$ export PATH=$PATH:llvm/utils/patch-coverage/
+```
+
+# Usage
+Run patch coverage for recent commits:
+```shell
+$ git patch-coverage -b build opt build/test
+```
+Arguments
+Argument Description
+-b, --build-dir Path to build directory (default: build)
+<binary> Instrumented LLVM binary (e.g., bin/opt)
+<test-path> One or more test suite paths (default: build/test)
+
+Optional Arguments
+Argument Description
+-n, --num-commits Number of recent commits to analyze (default: 1)
+-i, --instrumented-build-dir Use pre-instrumented build directory
+
+# Example
+```shell
+$ git patch-coverage -b build -n 3 bin/opt llvm/test
+```
+
+This will:
+Generate a patch for the last 3 commits
+Identify modified source and test files
+Rebuild LLVM with coverage instrumentation (if needed)
+Run modified lit tests
+Produce per-line coverage for modified code
+
+# Output
+The tool reports:
+Modified files and lines
+Covered lines (executed by tests)
+Uncovered lines (not exercised)
+Example output:
+```
+Modified File: llvm/lib/IR/Example.cpp
+ Covered Line: 42 : foo();
+ Uncovered Line: 45 : bar();
+```
diff --git a/llvm/utils/patch-coverage/__init__.py b/llvm/utils/patch-coverage/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/patch-coverage/build.py b/llvm/utils/patch-coverage/build.py
new file mode 100644
index 0000000000000..88c967165bf9d
--- /dev/null
+++ b/llvm/utils/patch-coverage/build.py
@@ -0,0 +1,155 @@
+import os
+import sys
+import subprocess
+from utils import compute_patch_hash
+from utils import log
+
+
+def is_configured_build(build_dir):
+ return os.path.exists(os.path.join(build_dir, "build.ninja")) or os.path.exists(
+ os.path.join(build_dir, "CMakeCache.txt")
+ )
+
+
+def configure_llvm_build(build_dir):
+ print(f"[patch-coverage] Configuring LLVM build in {build_dir}...")
+
+ subprocess.check_call(
+ [
+ "cmake",
+ "-G",
+ "Ninja",
+ "-S",
+ "llvm",
+ "-B",
+ build_dir,
+ "-DCMAKE_BUILD_TYPE=Release",
+ "-DLLVM_ENABLE_ASSERTIONS=ON",
+ "-DLLVM_INCLUDE_TESTS=ON",
+ "-DLLVM_BUILD_TESTS=ON",
+ ]
+ )
+
+
+def ensure_llvm_tools(build_dir):
+ if not is_configured_build(build_dir):
+ configure_llvm_build(build_dir)
+
+ required_tools = [
+ "bin/llvm-lit",
+ "bin/FileCheck",
+ "bin/count",
+ "bin/not",
+ "bin/llvm-config",
+ ]
+
+ missing = [
+ tool
+ for tool in required_tools
+ if not os.path.exists(os.path.join(build_dir, tool))
+ ]
+
+ if not missing:
+ return
+
+ subprocess.check_call(
+ [
+ "ninja",
+ "-C",
+ build_dir,
+ "FileCheck",
+ "count",
+ "not",
+ "llvm-config",
+ ]
+ )
+
+
+def should_rebuild(build_dir, patch_path):
+ new_hash = compute_patch_hash(patch_path)
+ hash_file = os.path.join(build_dir, ".last_patch_hash")
+
+ if os.path.exists(hash_file):
+ old_hash = open(hash_file).read().strip()
+ if old_hash == new_hash:
+ return False
+
+ with open(hash_file, "w") as f:
+ f.write(new_hash)
+
+ return True
+
+
+def build_llvm(build_dir, binary):
+ try:
+
+ cwd = os.getcwd()
+
+ os.chdir(build_dir)
+
+ command = 'find . -type f -name "*.profraw" -delete'
+
+ try:
+ subprocess.run(command, shell=True, check=True)
+ log(
+ "Files in build directory with '.profraw' extension deleted successfully."
+ )
+ except subprocess.CalledProcessError as e:
+ log(f"Error: {e}")
+ log("")
+
+ cmake_command = [
+ "cmake",
+ "-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON",
+ "-DLLVM_INDIVIDUAL_TEST_COVERAGE=ON",
+ f"-DCMAKE_C_FLAGS=-fprofile-list={os.path.abspath('fun.list')}",
+ f"-DCMAKE_CXX_FLAGS=-fprofile-list={os.path.abspath('fun.list')}",
+ ".",
+ ]
+
+ subprocess.check_call(cmake_command)
+
+ target_name = os.path.basename(binary)
+ try:
+ print()
+ subprocess.check_call(
+ [
+ "ninja",
+ "FileCheck",
+ "count",
+ "not",
+ "llvm-config",
+ "llvm-cov",
+ "llvm-profdata",
+ "llvm-dwarfdump",
+ "llvm-readobj",
+ target_name,
+ ]
+ )
+
+ except subprocess.CalledProcessError as ninja_error:
+ log(f"Error during Ninja build: {ninja_error}")
+ log("Attempting to build with 'make' using the available processors.")
+ num_processors = os.cpu_count() or 1
+ make_command = ["make", f"-j{num_processors}"]
+ subprocess.check_call(
+ make_command,
+ "FileCheck",
+ "count",
+ "not",
+ "llvm-config",
+ "llvm-cov",
+ "llvm-profdata",
+ "llvm-dwarfdump",
+ "llvm-readobj",
+ target_name,
+ )
+
+ os.chdir(cwd)
+
+ log("LLVM build completed successfully.")
+ log("")
+
+ except subprocess.CalledProcessError as e:
+ log("Error during LLVM build:", e)
+ sys.exit(1)
diff --git a/llvm/utils/patch-coverage/git-patch-coverage b/llvm/utils/patch-coverage/git-patch-coverage
new file mode 100755
index 0000000000000..6dcf6b5611009
--- /dev/null
+++ b/llvm/utils/patch-coverage/git-patch-coverage
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 "$(dirname "$0")/main.py" "$@"
diff --git a/llvm/utils/patch-coverage/lit.py b/llvm/utils/patch-coverage/lit.py
new file mode 100644
index 0000000000000..48f69953be322
--- /dev/null
+++ b/llvm/utils/patch-coverage/lit.py
@@ -0,0 +1,46 @@
+import subprocess
+import os
+
+def parse_suite_info(s):
+ curr_suite = None
+ res = {}
+
+ for line in s.decode().splitlines():
+ leading_spaces = len(line) - len(line.lstrip(" "))
+
+ if leading_spaces == 2 and line.split():
+ curr_suite = line.split()[0]
+ elif curr_suite is not None and leading_spaces == 4 and "Source Root:" in line:
+ assert curr_suite not in res
+
+ res[curr_suite] = line.split()[-1]
+
+ return res
+
+
+def find_lit_tests(lit_path, test_paths):
+ suites_cmd = [lit_path, "--show-suites"] + test_paths
+ output = subprocess.check_output(suites_cmd)
+
+ test_suites = parse_suite_info(output)
+
+ tests_cmd = [lit_path, "--show-tests"] + test_paths
+ output = subprocess.check_output(tests_cmd)
+
+ lines = [line.decode() for line in output.splitlines()]
+
+ test_info = [line.split() for line in lines if "::" in line]
+
+ if test_info is not None and len(test_info) > 0:
+ if len(test_info[0]) == 3:
+ return [
+ os.path.join(test_suites[suite], test_case)
+ for (suite, sep, test_case) in test_info
+ ]
+ elif len(test_info[0]) == 4:
+ return [
+ os.path.join(test_suites[suite1], test_case)
+ for (suite1, suite2, sep, test_case) in test_info
+ ]
+ else:
+ return []
diff --git a/llvm/utils/patch-coverage/main.py b/llvm/utils/patch-coverage/main.py
new file mode 100644
index 0000000000000..d863eea1fe57f
--- /dev/null
+++ b/llvm/utils/patch-coverage/main.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+#
+# ===- git-check-coverage - CheckCoverage Git Integration ---------*- python -*--===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------------===#
+
+import argparse
+import logging
+import os
+import subprocess
+import re
+import sys
+
+from utils import configure_logging
+from build import ensure_llvm_tools
+from patch import create_patch_from_last_commits
+from lit import find_lit_tests
+from patch import extract_source_files_from_patch
+from patch import write_source_file_allowlist
+from patch import extract_modified_source_lines_from_patch
+from build import should_rebuild
+from build import build_llvm
+from test import run_modified_lit_tests
+from test import run_modified_unit_tests
+from process import process_coverage_data
+from utils import log
+from print import report_covered_and_uncovered_lines
+
+sys.path.append(os.path.dirname(__file__))
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument("-b", "--build-dir", dest="build_dir", default="build")
+
+ parser.add_argument(
+ "-i", "--instrumented-build-dir", dest="instrumented_build_dir", default=None
+ )
+
+ parser.add_argument(
+ "-n",
+ "--num-commits",
+ dest="num_commits",
+ default=1,
+ )
+
+ parser.add_argument("binary")
+
+ parser.add_argument("test_path", nargs="+")
+
+ args = parser.parse_args()
+
+ return (
+ args.build_dir,
+ args.instrumented_build_dir,
+ args.num_commits,
+ args.binary,
+ args.test_path,
+ )
+
+
+def main():
+ (
+ build_dir,
+ instrumented_build_dir,
+ num_commits,
+ binary,
+ test_paths,
+ ) = parse_args()
+
+ configure_logging(build_dir)
+
+ ensure_llvm_tools(build_dir)
+
+ patch_path = os.path.join(build_dir, "patch.diff")
+
+ create_patch_from_last_commits(patch_path, num_commits)
+
+ llvm_lit_path = os.path.join(build_dir, "bin/llvm-lit")
+
+ hash_file = os.path.join(build_dir, ".last_patch_hash")
+
+ tests = frozenset(find_lit_tests(llvm_lit_path, test_paths))
+
+ source_files = extract_source_files_from_patch(patch_path)
+
+ allowlist_path = os.path.join(build_dir, "fun.list")
+
+ write_source_file_allowlist(source_files, allowlist_path)
+
+ modified_lines = extract_modified_source_lines_from_patch(patch_path, tests)
+
+ for file, lines in modified_lines.items():
+ log(f"File: {os.path.relpath(file)[2:]}")
+ for line_number, line_content in lines:
+ cleaned_line_content = line_content.rstrip("\n")
+ log(f"Line {line_number}: {cleaned_line_content}")
+ log("")
+
+ if not instrumented_build_dir:
+ rebuild = should_rebuild(build_dir, patch_path)
+
+ if rebuild:
+ build_llvm(build_dir, binary)
+ else:
+ print("\n[patch-coverage] Skipping rebuild (no changes)")
+
+ run_modified_lit_tests(llvm_lit_path, patch_path, tests)
+
+ run_modified_unit_tests(build_dir, patch_path, tests)
+
+ coverage_files = process_coverage_data(source_files, build_dir, binary)
+
+ report_covered_and_uncovered_lines(coverage_files, modified_lines)
+
+ parent_dir = os.path.dirname(os.getcwd())
+ default_profraw_path = os.path.join(parent_dir, "default.profraw")
+ try:
+ os.remove(default_profraw_path)
+ except FileNotFoundError:
+ pass
+
+
+if __name__ == "__main__":
+ main()
diff --git a/llvm/utils/patch-coverage/patch.py b/llvm/utils/patch-coverage/patch.py
new file mode 100644
index 0000000000000..ac4492fd20096
--- /dev/null
+++ b/llvm/utils/patch-coverage/patch.py
@@ -0,0 +1,126 @@
+import subprocess
+import sys
+import re
+import os
+
+from utils import log
+from unidiff import PatchSet
+
+
+def create_patch_from_last_commits(output_path, num_commits):
+ try:
+ diff_cmd = ["git", "diff", f"HEAD~{num_commits}", "HEAD"]
+ diff_output = subprocess.check_output(diff_cmd).decode("utf-8", "ignore")
+
+ with open(output_path, "wb") as patch_file:
+ patch_file.write(diff_output.encode("utf-8"))
+
+ log(f"Patch file '{output_path}' created successfully.")
+ log("")
+
+ except subprocess.CalledProcessError as e:
+ print("Error while creating the patch from the last commits:", e)
+ sys.exit(1)
+
+
+def extract_source_files_from_patch(patch_path):
+ try:
+ source_files = []
+ test_files = []
+ with open(patch_path, "rb") as patch_file:
+ patch_diff = patch_file.read().decode("utf-8", "ignore")
+
+ file_matches = re.findall(r"\+{3} b/(\S+)", patch_diff)
+
+ for file in file_matches:
+ if any(keyword in file.lower() for keyword in ["test", "unittest"]):
+ test_files.append(file)
+ else:
+ repo_root = os.path.abspath(os.getcwd())
+ abs_path = os.path.abspath(os.path.join(repo_root, file))
+ source_files.append(abs_path)
+
+ if not source_files:
+ print("No source files found in the patch. Exiting.")
+ sys.exit(1)
+
+ if not test_files:
+ print("No test files found in the patch. Exiting.")
+ sys.exit(1)
+
+ print("\n[patch-coverage] Source files in the patch:")
+ for source_file in source_files:
+ print(source_file)
+
+ print("\n[patch-coverage] Test files in the patch:")
+ for test_file in test_files:
+ print(test_file)
+
+ return source_files
+
+ except Exception as ex:
+ print("Error while extracting files from patch:", ex)
+ sys.exit(1)
+
+
+def write_source_file_allowlist(source_files, allowlist_path):
+ try:
+ current_directory = os.getcwd()
+ absolute_path = os.path.abspath(current_directory)
+
+ with open(allowlist_path, "w") as allowlist_file:
+ for source_file in source_files:
+ source_file = os.path.join(absolute_path, source_file)
+ allowlist_file.write("source:{}=allow\n".format(source_file))
+ allowlist_file.write("default:skip") # Specify default behavior
+
+ log("Source file allowlist written to file '{}'.".format(allowlist_path))
+ log("")
+
+ except subprocess.CalledProcessError as e:
+ log("Error while writing allow list for -fprofile-list:", e)
+ sys.exit(1)
+
+
+def extract_modified_source_lines_from_patch(patch_path, tests):
+ source_lines = {}
+
+ tests_relative = {os.path.relpath(file) for file in tests}
+
+ try:
+ patchset = PatchSet.from_filename(patch_path)
+
+ for patched_file in patchset:
+
+ repo_root = os.path.abspath(os.getcwd())
+
+ target = patched_file.target_file
+ if target.startswith("b/"):
+ target = target[2:]
+
+ current_file = os.path.abspath(os.path.join(repo_root, target))
+
+ is_test_by_tests = any(test in current_file for test in tests)
+ is_test_by_relative = any(test in current_file for test in tests_relative)
+
+ if is_test_by_tests:
+ continue
+
+ source_lines[current_file] = []
+
+ for hunk in patched_file:
+
+ for line in hunk:
+ if not line.is_added:
+ continue
+
+ if is_test_by_relative:
+ continue
+
+ source_lines[current_file].append((line.target_line_no, line.value))
+
+ return source_lines
+
+ except Exception as ex:
+ print("[ERROR]", ex)
+ return {}
diff --git a/llvm/utils/patch-coverage/print.py b/llvm/utils/patch-coverage/print.py
new file mode 100644
index 0000000000000..eb422b85949af
--- /dev/null
+++ b/llvm/utils/patch-coverage/print.py
@@ -0,0 +1,65 @@
+import sys
+import os
+
+from utils import log
+from process import read_coverage_file
+
+
+def print_coverage_details(file, lines, uncovered_line_numbers, covered_line_numbers):
+ log(f"Modified File: {os.path.basename(file)}")
+
+ for line_number_source, line_content in lines:
+ if line_number_source in uncovered_line_numbers:
+ log(f" Uncovered Line: {line_number_source} : {line_content.strip()}")
+ elif line_number_source in covered_line_numbers:
+ log(f" Covered Line: {line_number_source} : {line_content.strip()}")
+
+
+def print_common_uncovered_lines(cpp_file, uncovered_lines, modified_lines):
+ print(f"\n[code-coverage] Common uncovered lines for {cpp_file} after all changes:")
+
+ for file, lines in modified_lines.items():
+ if os.path.normpath(file) == os.path.normpath(cpp_file):
+ for line_number_source, line_content in lines:
+ if line_number_source in uncovered_lines:
+ print(f" Line {line_number_source}: {line_content.strip()}")
+
+
+def report_covered_and_uncovered_lines(coverage_files, modified_lines):
+ try:
+ common_uncovered_line_numbers_for_each_file = {}
+
+ for cpp_file, coverage_files_list in coverage_files.items():
+ all_uncovered_lines = set()
+ all_covered_lines = set()
+
+ for coverage_file in coverage_files_list:
+ log(f"Coverage File: {coverage_file}")
+ covered_line_numbers, uncovered_line_numbers = read_coverage_file(
+ coverage_file
+ )
+
+ all_uncovered_lines |= uncovered_line_numbers
+ all_covered_lines |= covered_line_numbers
+
+ effective_uncovered_lines = all_uncovered_lines - all_covered_lines
+ common_uncovered_line_numbers_for_each_file[cpp_file] = (
+ effective_uncovered_lines
+ )
+
+ for file, lines in modified_lines.items():
+ if os.path.normpath(file) == os.path.normpath(cpp_file):
+ print_coverage_details(
+ file, lines, effective_uncovered_lines, all_covered_lines
+ )
+ log("")
+
+ for (
+ cpp_file,
+ uncovered_lines,
+ ) in common_uncovered_line_numbers_for_each_file.items():
+ print_common_uncovered_lines(cpp_file, uncovered_lines, modified_lines)
+
+ except Exception as ex:
+ log("Error while reporting covered and uncovered lines:", ex)
+ sys.exit(1)
diff --git a/llvm/utils/patch-coverage/process.py b/llvm/utils/patch-coverage/process.py
new file mode 100644
index 0000000000000..bdbc39068261b
--- /dev/null
+++ b/llvm/utils/patch-coverage/process.py
@@ -0,0 +1,104 @@
+import subprocess
+import sys
+import os
+from utils import log
+
+
+def process_coverage_data(cpp_files, build_dir, binary):
+ coverage_files = {}
+
+ try:
+ os.chdir(build_dir)
+
+ for root, dirs, files in os.walk("."):
+ for file in files:
+
+ if os.path.basename(file) == "default.profraw":
+ continue
+ if file.endswith(".profraw"):
+ profraw_file = os.path.abspath(os.path.join(root, file))
+ profdata_output = os.path.abspath(
+ os.path.splitext(profraw_file)[0] + ".profdata"
+ )
+
+ log("")
+ log("Profraw File:", profraw_file)
+ log("Profdata File:", profdata_output)
+
+ llvm_profdata_cmd = [
+ "./bin/llvm-profdata",
+ "merge",
+ "-o",
+ profdata_output,
+ profraw_file,
+ ]
+
+ subprocess.check_call(llvm_profdata_cmd)
+
+ log("Converted {} to {}".format(profraw_file, profdata_output))
+
+ for cpp_file in cpp_files:
+ cpp_file_original = cpp_file
+
+ output_file = (
+ os.path.splitext(profdata_output)[0]
+ + f"_{cpp_file_original.replace('/', '_')}.txt"
+ )
+
+ parent_directory = os.path.dirname(os.getcwd())
+ abs_cpp_file = os.path.abspath(
+ os.path.join(parent_directory, cpp_file_original)
+ )
+ # binary = "unittests/Support/SupportTests"
+ target = "bin/" + binary
+ llvm_cov_cmd = [
+ "./bin/llvm-cov",
+ "show",
+ "-instr-profile",
+ profdata_output,
+ target,
+ "--format=text",
+ abs_cpp_file,
+ ]
+
+ with open(output_file, "w") as output:
+ subprocess.check_call(llvm_cov_cmd, stdout=output)
+
+ log(f"Processed file saved as: {output_file}")
+ abs_cpp_key = os.path.abspath(
+ os.path.join(
+ os.path.dirname(os.getcwd()), cpp_file_original
+ )
+ )
+ coverage_files.setdefault(abs_cpp_key, []).append(output_file)
+
+ log("")
+ log("Conversion of profraw files to human-readable form is completed.")
+ log("")
+
+ return coverage_files
+
+ except subprocess.CalledProcessError as e:
+ log("Error during profraw to profdata conversion:", e)
+ sys.exit(1)
+
+
+def read_coverage_file(coverage_file):
+ uncovered_line_numbers = set()
+ covered_line_numbers = set()
+
+ with open(coverage_file, "r") as cov_file:
+ for line in cov_file:
+ parts = line.strip().split("|")
+ if len(parts) >= 3:
+ line_number_str = parts[0].strip()
+ execution_count = parts[1].strip()
+
+ if line_number_str.isdigit() and execution_count.isdigit():
+ line_number = int(line_number_str)
+ if int(execution_count) == 0:
+ uncovered_line_numbers.add(line_number)
+ elif int(execution_count) > 0:
+ covered_line_numbers.add(line_number)
+
+ return covered_line_numbers, uncovered_line_numbers
diff --git a/llvm/utils/patch-coverage/requirements.txt b/llvm/utils/patch-coverage/requirements.txt
new file mode 100644
index 0000000000000..da4b8fdb2bcba
--- /dev/null
+++ b/llvm/utils/patch-coverage/requirements.txt
@@ -0,0 +1 @@
+unidiff
diff --git a/llvm/utils/patch-coverage/test.py b/llvm/utils/patch-coverage/test.py
new file mode 100644
index 0000000000000..2887f17f805e2
--- /dev/null
+++ b/llvm/utils/patch-coverage/test.py
@@ -0,0 +1,126 @@
+import re
+import os
+import subprocess
+import sys
+from pathlib import Path
+from utils import log
+
+
+def run_single_test_with_coverage(llvm_lit_path, test_path):
+ try:
+ profile_dir = os.path.abspath("build/profiles")
+ os.makedirs(profile_dir, exist_ok=True)
+
+ test_env = os.environ.copy()
+ test_env["LLVM_PROFILE_FILE"] = os.path.join(profile_dir, "test-%p.profraw")
+
+ lit_cmd = [llvm_lit_path, test_path]
+ subprocess.check_call(lit_cmd, env=test_env)
+
+ log("Test case executed:", test_path)
+
+ except subprocess.CalledProcessError as e:
+ log("Error while running test:", e)
+ sys.exit(1)
+
+ except Exception as ex:
+ log("Error:", ex)
+ sys.exit(1)
+
+
+def run_modified_lit_tests(llvm_lit_path, patch_path, tests):
+ try:
+ with open(patch_path, "r") as patch_file:
+ patch_diff = patch_file.read()
+
+ modified_tests = []
+
+ for match in re.finditer(
+ r"^\+\+\+ [ab]/(.*\.(ll|mir|mlir|fir|c|cpp|f90|s|test))$",
+ patch_diff,
+ re.MULTILINE,
+ ):
+ test_file = match.group(1)
+
+ cwd = os.getcwd()
+
+ full_test_file = os.path.join(
+ os.path.dirname(cwd), "llvm-project", test_file
+ )
+
+ if full_test_file in tests:
+ log("Lit test file in the patch:", test_file)
+
+ if match.group(0).startswith("+++"):
+ modified_tests.append(full_test_file)
+
+ if not modified_tests:
+ log("No modified lit tests found in the patch.")
+ return
+
+ log("")
+ log("Running modified test cases:")
+ for test_file in modified_tests:
+ run_single_test_with_coverage(llvm_lit_path, test_file)
+
+ except subprocess.CalledProcessError as e:
+ log("Error while running modified tests:", e)
+ sys.exit(1)
+
+ except Exception as ex:
+ log("Error:", ex)
+ sys.exit(1)
+
+
+def run_modified_unit_tests(build_dir, patch_path, tests):
+ log("Starting unit test execution...")
+
+ try:
+ patch_diff = Path(patch_path).read_text()
+ modified_unit_tests = []
+
+ matches = re.findall(
+ r"^\+\+\+ [ab]/(.*unittests.*\.(?:cpp|c))$", patch_diff, re.MULTILINE
+ )
+ for test_file_rel in matches:
+ test_path_obj = Path(test_file_rel)
+ suite_name = test_path_obj.parent.name
+ ninja_target = f"{suite_name}Tests"
+ log("target", ninja_target)
+
+ match_in_build = next(
+ (t for t in tests if f"unittests/{suite_name}" in t), None
+ )
+
+ if not match_in_build:
+ continue
+
+ log(f"\nProcessing: {test_file_rel}")
+
+ binary_path = Path(match_in_build).resolve()
+ binary_path = Path(build_dir) / "unittests" / suite_name / ninja_target
+
+ profraw_path = binary_path.parent / f"{test_path_obj.stem}.profraw"
+
+ try:
+ subprocess.check_call(["ninja", "-C", build_dir, ninja_target])
+
+ env = os.environ.copy()
+ env["LLVM_PROFILE_FILE"] = str(profraw_path)
+
+ log(f"Executing: {ninja_target}")
+ subprocess.check_call([str(binary_path)], env=env)
+
+ modified_unit_tests.append(test_file_rel)
+ log(f"Success. Profile saved to: {profraw_path.name}")
+
+ except subprocess.CalledProcessError as e:
+ log(f"Failed executing {ninja_target}: {e}")
+ continue
+
+ if not modified_unit_tests:
+ log("No modified unit tests found. Running default suite...")
+
+ except Exception as e:
+ log(f"Critical Error: {e}")
+ sys.exit(1)
diff --git a/llvm/utils/patch-coverage/utils.py b/llvm/utils/patch-coverage/utils.py
new file mode 100644
index 0000000000000..21045abe37f7e
--- /dev/null
+++ b/llvm/utils/patch-coverage/utils.py
@@ -0,0 +1,24 @@
+import os
+import logging
+
+from unidiff import PatchSet
+import hashlib
+
+
+def compute_patch_hash(patch_path):
+ with open(patch_path, "rb") as f:
+ return hashlib.sha256(f.read()).hexdigest()
+
+
+def configure_logging(build_dir):
+ os.makedirs(build_dir, exist_ok=True)
+ logging.basicConfig(
+ filename=os.path.join(build_dir, "patch_coverage.log"),
+ level=logging.INFO,
+ format="%(message)s",
+ )
+
+
+def log(*args):
+ message = " ".join(map(str, args))
+ logging.info(message)
More information about the llvm-commits
mailing list