[llvm] [Utils] Add a python tool for test coverage of patches (PR #191721)
Shivam Gupta via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 22:44:12 PDT 2026
https://github.com/xgupta updated https://github.com/llvm/llvm-project/pull/191721
>From be33086ee91f6396b70f43bd645a58d12dc4b1d8 Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Thu, 9 Apr 2026 19:09:45 +0530
Subject: [PATCH 1/9] Add patch based code coverage tool
---
llvm/utils/patch-coverage/README.md | 137 ++++++++++++++++
llvm/utils/patch-coverage/__init__.py | 0
llvm/utils/patch-coverage/build.py | 116 ++++++++++++++
llvm/utils/patch-coverage/git-patch-coverage | 10 ++
llvm/utils/patch-coverage/lit.py | 53 +++++++
llvm/utils/patch-coverage/main.py | 159 +++++++++++++++++++
llvm/utils/patch-coverage/patch.py | 127 +++++++++++++++
llvm/utils/patch-coverage/print.py | 93 +++++++++++
llvm/utils/patch-coverage/process.py | 86 ++++++++++
llvm/utils/patch-coverage/requirements.txt | 1 +
llvm/utils/patch-coverage/test.py | 120 ++++++++++++++
llvm/utils/patch-coverage/utils.py | 105 ++++++++++++
12 files changed, 1007 insertions(+)
create mode 100644 llvm/utils/patch-coverage/README.md
create mode 100644 llvm/utils/patch-coverage/__init__.py
create mode 100644 llvm/utils/patch-coverage/build.py
create mode 100755 llvm/utils/patch-coverage/git-patch-coverage
create mode 100644 llvm/utils/patch-coverage/lit.py
create mode 100644 llvm/utils/patch-coverage/main.py
create mode 100644 llvm/utils/patch-coverage/patch.py
create mode 100644 llvm/utils/patch-coverage/print.py
create mode 100644 llvm/utils/patch-coverage/process.py
create mode 100644 llvm/utils/patch-coverage/requirements.txt
create mode 100644 llvm/utils/patch-coverage/test.py
create mode 100644 llvm/utils/patch-coverage/utils.py
diff --git a/llvm/utils/patch-coverage/README.md b/llvm/utils/patch-coverage/README.md
new file mode 100644
index 0000000000000..2c16f9279c5bd
--- /dev/null
+++ b/llvm/utils/patch-coverage/README.md
@@ -0,0 +1,137 @@
+# Patch-Based Code Coverage (LLVM Git Integration)
+
+This tool provides patch-level code coverage for LLVM development workflows.
+It helps developers and reviewers understand which lines introduced by a patch are covered by tests and which are not.
+
+It integrates with LLVM’s infrastructure (`llvm-lit`, coverage instrumentation, and CI) and works both locally and in CI environments.
+
+
+## Overview
+
+Given recent commits, the tool:
+
+1. Generates a patch (`git diff`)
+2. Identifies modified source files and lines
+3. Detects modified lit tests and unit tests
+4. Builds LLVM with coverage instrumentation
+5. Runs only modified tests
+6. Collects coverage data using `llvm-profdata` and `llvm-cov`
+7. Reports:
+
+ * Covered lines introduced by the patch
+ * Uncovered lines introduced by the patch
+
+
+## Key Features
+
+* Patch-aware coverage (only modified lines)
+* Selective test execution (only affected tests)
+* Supports:
+
+ * `llvm-lit` tests
+ * LLVM unit tests
+* Uses allowlist-based instrumentation for faster builds
+* Avoids unnecessary rebuilds via patch hashing
+
+
+## Setup
+
+### 1. Build LLVM (standard workflow)
+
+```bash
+git clone https://github.com/llvm/llvm-project.git
+cd llvm-project
+
+cmake -G Ninja -B build -S llvm \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_ENABLE_ASSERTIONS=ON \
+ -DLLVM_INCLUDE_TESTS=ON \
+ -DLLVM_BUILD_TESTS=ON
+
+ninja -C build
+```
+
+### 2. Install Python dependencies
+
+```bash
+pip install -r llvm/utils/patch-coverage/requirements.txt
+```
+
+### 3. Add tool to PATH
+
+```bash
+export PATH=$PATH:llvm/utils/patch-coverage/
+```
+
+This enables:
+
+```bash
+git patch-coverage ...
+```
+
+## Usage
+
+### Basic command
+
+```bash
+git patch-coverage -b <build-dir> <binary> <test-path>
+```
+
+### Example
+
+```bash
+git patch-coverage -b build -n 3 bin/opt llvm/test
+```
+
+## Arguments
+
+### Required
+
+| Argument | Description |
+| ----------------- | ---------------------------------------------------- |
+| `<binary>` | LLVM binary to analyze (e.g., `bin/opt`) |
+
+
+### Optional
+
+| Argument | Description |
+| ------------------------------ | ----------------------------------------------------------- |
+| `-n, --num-commits` | Number of recent commits to analyze (default: 1) |
+| `-i, --instrumented-build-dir` | Use pre-built instrumented directory (Deafult: `build_inst`)|
+| `-b, --build-dir` | Path to LLVM build directory (default: `build`) |
+| `<test-path>` | One or more test suite paths (default: `build/test`) |
+
+
+## How It Works
+
+To be written
+
+
+## Output
+
+### `<build>/patch_coverage.log`
+
+```
+Modified File: llvm/lib/IR/Example.cpp
+ Covered Line: 42 : foo();
+ Uncovered Line: 45 : bar();
+```
+
+### Summary on Standard Output
+
+```
+[code-coverage] Common uncovered lines for llvm/lib/IR/Example.cpp:
+ Line 45: bar();
+```
+
+## Troubleshooting
+
+### No coverage generated
+
+* Ensure instrumented build succeeded
+* Check `.profraw` files in build directory
+
+### Rebuild skipped unexpectedly
+
+* Delete `.last_patch_hash` to force rebuild
+
diff --git a/llvm/utils/patch-coverage/__init__.py b/llvm/utils/patch-coverage/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llvm/utils/patch-coverage/build.py b/llvm/utils/patch-coverage/build.py
new file mode 100644
index 0000000000000..2c5672767e0a8
--- /dev/null
+++ b/llvm/utils/patch-coverage/build.py
@@ -0,0 +1,116 @@
+import os
+import sys
+import subprocess
+
+from utils import log
+
+
+def is_configured_build(build_dir):
+ return os.path.exists(os.path.join(build_dir, "build.ninja"))
+
+
+def configure_llvm_build(build_dir):
+ print(f"[patch-coverage] Configuring LLVM build in {build_dir}...")
+ subprocess.check_call(
+ [
+ "cmake",
+ "-G",
+ "Ninja",
+ "-S",
+ "llvm",
+ "-B",
+ build_dir,
+ "-DCMAKE_BUILD_TYPE=Release",
+ "-DLLVM_ENABLE_ASSERTIONS=ON",
+ "-DLLVM_INCLUDE_TESTS=ON",
+ "-DLLVM_BUILD_TESTS=ON",
+ ]
+ )
+
+
+def ensure_llvm_tools(build_dir):
+ print("Making sure we have required tool...")
+ if not is_configured_build(build_dir):
+ configure_llvm_build(build_dir)
+
+ required_tools = [
+ "bin/llvm-lit",
+ "bin/FileCheck",
+ "bin/count",
+ "bin/not",
+ "bin/llvm-config",
+ ]
+
+ missing = [
+ tool
+ for tool in required_tools
+ if not os.path.exists(os.path.join(build_dir, tool))
+ ]
+
+ if not missing:
+ print("Yes, We have all the tools for fetching testsuite info.")
+ return
+
+ print("Building all required tools.")
+ subprocess.check_call(
+ [
+ "ninja",
+ "-C",
+ build_dir,
+ "FileCheck",
+ "count",
+ "not",
+ "llvm-config",
+ ]
+ )
+
+
+def build_llvm(inst_build_dir, binary, allowlist_path):
+ try:
+ if not is_configured_build(inst_build_dir):
+ print("Configuring the instrumented build for patch coverage.")
+ subprocess.check_call(
+ [
+ "cmake",
+ "-G",
+ "Ninja",
+ "-S",
+ "llvm",
+ "-B",
+ inst_build_dir,
+ "-DCMAKE_BUILD_TYPE=Release",
+ "-DLLVM_ENABLE_ASSERTIONS=ON",
+ "-DLLVM_INCLUDE_TESTS=ON",
+ "-DLLVM_BUILD_TESTS=ON",
+ "-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON",
+ "-DLLVM_INDIVIDUAL_TEST_COVERAGE=ON",
+ f"-DCMAKE_C_FLAGS=-fprofile-list={os.path.abspath(allowlist_path)}",
+ f"-DCMAKE_CXX_FLAGS=-fprofile-list={os.path.abspath(allowlist_path)}",
+ ]
+ )
+
+ target = [
+ "FileCheck",
+ "count",
+ "not",
+ "llvm-config",
+ "llvm-cov",
+ "llvm-profdata",
+ "llvm-dwarfdump",
+ "llvm-readobj",
+ binary,
+ ]
+
+ try:
+ print("Building the instrumented target")
+ subprocess.check_call(["ninja", "-C", inst_build_dir] + target)
+
+ except subprocess.CalledProcessError as ninja_error:
+ log(f"Error during Ninja build: {ninja_error}")
+ sys.exit(1)
+
+ log("LLVM build completed successfully.\n")
+
+ except subprocess.CalledProcessError as e:
+ log("Error during LLVM build:", e)
+ sys.exit(1)
diff --git a/llvm/utils/patch-coverage/git-patch-coverage b/llvm/utils/patch-coverage/git-patch-coverage
new file mode 100755
index 0000000000000..a63442cd3f782
--- /dev/null
+++ b/llvm/utils/patch-coverage/git-patch-coverage
@@ -0,0 +1,10 @@
+#!/bin/bash
+#
+# ===- git-patch-coverage - Patch-Based Code-Coverage Git Integration -----------===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------------===#
+python3 "$(dirname "$0")/main.py" "$@"
diff --git a/llvm/utils/patch-coverage/lit.py b/llvm/utils/patch-coverage/lit.py
new file mode 100644
index 0000000000000..85148fc60761e
--- /dev/null
+++ b/llvm/utils/patch-coverage/lit.py
@@ -0,0 +1,53 @@
+import subprocess
+import os
+
+
+def parse_suite_info(s):
+ curr_suite = None
+ res = {}
+
+ for line in s.decode().splitlines():
+ leading_spaces = len(line) - len(line.lstrip(" "))
+
+ if leading_spaces == 2 and line.split():
+ curr_suite = line.split()[0]
+ elif curr_suite is not None and leading_spaces == 4 and "Source Root:" in line:
+ if curr_suite in res:
+ raise RuntimeError(f"Duplicate suite detected: {curr_suite}")
+ res[curr_suite] = line.split()[-1]
+
+ return res
+
+
+def find_lit_tests(lit_path, test_paths):
+ suites_cmd = [lit_path, "--show-suites"] + test_paths
+ output = subprocess.check_output(suites_cmd)
+
+ # Above command give large output like this -
+ # LLVM - 61914 tests
+ # Source Root: /Users/<username>/llvm-project/llvm/test
+ # Parse it to construct following format -
+ # {'LLVM': '/Users/<username>/llvm-project/llvm/test'}
+ test_suites = parse_suite_info(output)
+
+ tests_cmd = [lit_path, "--show-tests"] + test_paths
+ output = subprocess.check_output(tests_cmd)
+
+ lines = [line.decode() for line in output.splitlines()]
+
+ test_info = [line.split() for line in lines if "::" in line]
+
+ # Construct absolute path of each test case of test suite.
+ if test_info is not None and len(test_info) > 0:
+ if len(test_info[0]) == 3:
+ return [
+ os.path.join(test_suites[suite], test_case)
+ for (suite, sep, test_case) in test_info
+ ]
+ elif len(test_info[0]) == 4:
+ return [
+ os.path.join(test_suites[suite1], test_case)
+ for (suite1, suite2, sep, test_case) in test_info
+ ]
+ else:
+ return []
diff --git a/llvm/utils/patch-coverage/main.py b/llvm/utils/patch-coverage/main.py
new file mode 100644
index 0000000000000..ef88db2b4591c
--- /dev/null
+++ b/llvm/utils/patch-coverage/main.py
@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+#
+# ===----------------------------------------------------------------------------===#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------------===#
+
+import argparse
+import logging
+import os
+import re
+import subprocess
+import sys
+
+from build import build_llvm
+from build import ensure_llvm_tools
+from lit import find_lit_tests
+from patch import extract_modified_source_lines_from_patch
+from patch import extract_source_files_from_patch
+from patch import create_patch_from_last_commits
+from patch import write_source_file_allowlist
+from print import report_covered_and_uncovered_lines
+from process import process_coverage_data
+from test import run_modified_lit_tests
+from test import run_modified_unit_tests
+from utils import configure_logging
+from utils import classify_tests
+from utils import delete_profraw
+from utils import log
+from utils import mark_build_success
+from utils import should_rebuild
+from utils import target_name
+
+sys.path.append(os.path.dirname(__file__))
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument("-b", "--build-dir", dest="build_dir", default="build")
+
+ parser.add_argument(
+ "-i",
+ "--instrumented-build-dir",
+ dest="inst_build_dir",
+ default="build_instrumented",
+ )
+
+ parser.add_argument(
+ "-n",
+ "--num-commits",
+ dest="num_commits",
+ default=1,
+ )
+
+ parser.add_argument("binary")
+
+ parser.add_argument("test_path", nargs="*")
+
+ args = parser.parse_args()
+
+ if not args.test_path:
+ args.test_path = [os.path.join(args.build_dir, "test")]
+
+ return (
+ args.build_dir,
+ args.inst_build_dir,
+ args.num_commits,
+ args.binary,
+ args.test_path,
+ )
+
+
+def main():
+ (
+ build_dir,
+ inst_build_dir,
+ num_commits,
+ binary,
+ test_paths,
+ ) = parse_args()
+
+ configure_logging(inst_build_dir)
+
+ # Ensure we have required tools to parse test suite info.
+ ensure_llvm_tools(build_dir)
+
+ # Create a diff file from the commit/s.
+ patch_path = os.path.join(build_dir, "patch.diff")
+ create_patch_from_last_commits(patch_path, num_commits)
+ source_files = extract_source_files_from_patch(patch_path)
+
+ # Get all the modified lines of patch, from both source files and test files.
+ llvm_lit_path = os.path.join(build_dir, "bin/llvm-lit")
+ tests = frozenset(find_lit_tests(llvm_lit_path, test_paths))
+ modified_lines = extract_modified_source_lines_from_patch(patch_path, tests)
+
+ # Use allow list feature to generate ".profraw" data for only source files in the patch.
+ os.makedirs(inst_build_dir, exist_ok=True)
+ allowlist_path = os.path.join(inst_build_dir, "fun.list")
+ write_source_file_allowlist(source_files, allowlist_path)
+
+ # Print all the modified lines of the patch.
+ for file, lines in modified_lines.items():
+ log(f"File: {os.path.relpath(file)[2:]}")
+ for line_number, line_content in lines:
+ cleaned_line_content = line_content.rstrip("\n")
+ log(f"Line {line_number}: {cleaned_line_content}")
+ log("")
+
+ # Contruct the absolute path of binary that we want to check coverage for.
+ unit_tests, lit_tests = classify_tests(patch_path)
+ unit_binary = None
+ lit_binary = None
+ if lit_tests:
+ lit_binary = os.path.abspath(os.path.join(inst_build_dir, "bin", binary))
+ if unit_tests:
+ target = target_name(patch_path, inst_build_dir)
+ unit_binary = os.path.abspath(os.path.join(inst_build_dir, target))
+
+ # Build the LLVM in instrumented directory using LLVM_BUILD_INSTRUMENTED_COVERAGE.
+ rebuild = should_rebuild(inst_build_dir, patch_path, lit_binary or unit_binary)
+ if rebuild:
+ delete_profraw(inst_build_dir)
+ build_llvm(inst_build_dir, binary, allowlist_path)
+ mark_build_success(inst_build_dir, patch_path)
+ else:
+ print("\n[patch-coverage] Skipping patch coverage (no changes)")
+ sys.exit(1)
+
+ # Run all the test cases of patch with instrumented binary.
+ inst_lit_path = os.path.join(inst_build_dir, "bin/llvm-lit")
+ patch_path = os.path.abspath(patch_path)
+ run_modified_lit_tests(inst_lit_path, patch_path, tests, inst_build_dir)
+ run_modified_unit_tests(build_dir, inst_build_dir, patch_path)
+
+ # Report covered and uncovered lines of each source file.
+ coverage_files = process_coverage_data(
+ source_files,
+ inst_build_dir,
+ lit_binary,
+ unit_binary,
+ patch_path,
+ )
+ report_covered_and_uncovered_lines(coverage_files, modified_lines)
+
+ # Remove redundant "default.profraw" generated in source root.
+ curr_dir = os.path.dirname(os.getcwd())
+ default_profraw_path = os.path.join(curr_dir, "default.profraw")
+ try:
+ os.remove(default_profraw_path)
+ except FileNotFoundError:
+ pass
+
+
+if __name__ == "__main__":
+ main()
diff --git a/llvm/utils/patch-coverage/patch.py b/llvm/utils/patch-coverage/patch.py
new file mode 100644
index 0000000000000..2dda93c727b5f
--- /dev/null
+++ b/llvm/utils/patch-coverage/patch.py
@@ -0,0 +1,127 @@
+import subprocess
+import sys
+import re
+import os
+
+from utils import log
+from unidiff import PatchSet
+
+
+def create_patch_from_last_commits(output_path, num_commits):
+ try:
+ diff_cmd = ["git", "diff", f"HEAD~{num_commits}", "HEAD"]
+ diff_output = subprocess.check_output(diff_cmd).decode("utf-8", "ignore")
+
+ with open(output_path, "wb") as patch_file:
+ patch_file.write(diff_output.encode("utf-8"))
+
+ log(f"Patch file '{output_path}' created successfully.")
+ log("")
+
+ except subprocess.CalledProcessError as e:
+ print("Error while creating the patch from the last commits:", e)
+ sys.exit(1)
+
+
+def extract_source_files_from_patch(patch_path):
+ try:
+ source_files = []
+ test_files = []
+ with open(patch_path, "rb") as patch_file:
+ patch_diff = patch_file.read().decode("utf-8", "ignore")
+
+ file_matches = re.findall(r"\+{3} b/(\S+)", patch_diff)
+
+ for file in file_matches:
+ if any(keyword in file.lower() for keyword in ["test", "unittest"]):
+ test_files.append(file)
+ else:
+ repo_root = os.path.abspath(os.getcwd())
+ full_path = os.path.join(repo_root, file)
+ source_files.append(full_path)
+
+ if not source_files:
+ print("No source files found in the patch. Exiting.")
+ sys.exit(1)
+
+ if not test_files:
+ print("No test files found in the patch. Exiting.")
+ sys.exit(1)
+
+ print("\n[patch-coverage] Source files in the patch:")
+ for source_file in source_files:
+ print(source_file)
+
+ print("\n[patch-coverage] Test files in the patch:")
+ for test_file in test_files:
+ print(test_file)
+ print("\n")
+
+ return source_files
+
+ except Exception as ex:
+ print("Error while extracting files from patch:", ex)
+ sys.exit(1)
+
+
+def extract_modified_source_lines_from_patch(patch_path, tests):
+ source_lines = {}
+
+ tests_set = {os.path.abspath(os.path.normpath(t)) for t in tests}
+ repo_root = os.path.abspath(os.getcwd())
+
+ try:
+ patchset = PatchSet.from_filename(patch_path)
+
+ for patched_file in patchset:
+ target_path = patched_file.path
+
+ current_file = os.path.abspath(
+ os.path.normpath(os.path.join(repo_root, target_path))
+ )
+
+ if current_file in tests_set or patched_file.is_removed_file:
+ continue
+
+ lines = []
+
+ for hunk in patched_file:
+ for line in hunk:
+ if line.is_added:
+ lines.append((line.target_line_no, line.value.rstrip("\n")))
+
+ if lines:
+ source_lines[current_file] = lines
+
+ return source_lines
+
+ except Exception as ex:
+ print(f"[patch-coverage] Failed to parse patch {patch_path}: {ex}")
+ return {}
+
+
+def write_source_file_allowlist(source_files, allowlist_path):
+ try:
+ lines = []
+ for source_file in source_files:
+ lines.append(f"source:{source_file}=allow\n")
+
+ lines.append("default:skip\n")
+
+ new_content = "".join(lines)
+
+ if os.path.exists(allowlist_path):
+ with open(allowlist_path) as f:
+ if f.read() == new_content:
+ log("Allowlist unchanged, skipping write.")
+ log("")
+ return
+
+ with open(allowlist_path, "w") as f:
+ f.write(new_content)
+
+ log(f"Source file allowlist written to '{allowlist_path}'.\n")
+
+ except Exception as e:
+ log(f"Error while writing allow list: {e}")
+ sys.exit(1)
diff --git a/llvm/utils/patch-coverage/print.py b/llvm/utils/patch-coverage/print.py
new file mode 100644
index 0000000000000..cbb33082d9fd6
--- /dev/null
+++ b/llvm/utils/patch-coverage/print.py
@@ -0,0 +1,93 @@
+import sys
+import os
+
+from utils import log
+
+
+def read_coverage_file(coverage_file):
+ uncovered_line_numbers = set()
+ covered_line_numbers = set()
+
+ try:
+ with open(coverage_file, "r") as cov_file:
+ for line in cov_file:
+ parts = line.strip().split("|")
+ if len(parts) < 3:
+ continue
+
+ try:
+ line_number = int(parts[0].strip())
+ execution_count = int(parts[1].strip())
+ except ValueError:
+ continue
+
+ if execution_count == 0:
+ uncovered_line_numbers.add(line_number)
+ else:
+ covered_line_numbers.add(line_number)
+
+ except OSError as e:
+ raise RuntimeError(f"Failed to read {coverage_file}: {e}")
+
+ return covered_line_numbers, uncovered_line_numbers
+
+
+def log_coverage_details(file, lines, uncovered_line_numbers, covered_line_numbers):
+ log(f"Modified File: {os.path.basename(file)}")
+
+ for line_number_source, line_content in lines:
+ if line_number_source in uncovered_line_numbers:
+ log(f" Uncovered Line: {line_number_source} : {line_content.strip()}")
+ elif line_number_source in covered_line_numbers:
+ log(f" Covered Line: {line_number_source} : {line_content.strip()}")
+
+
+def print_common_uncovered_lines(cpp_file, uncovered_lines, modified_lines):
+ if cpp_file not in modified_lines:
+ return
+
+ print(f"\n[code-coverage] Common uncovered lines for {cpp_file} after all changes:")
+ for line_number_source, line_content in modified_lines[cpp_file]:
+ if line_number_source in uncovered_lines:
+ print(f" Line {line_number_source}: {line_content.strip()}")
+
+
+def report_covered_and_uncovered_lines(coverage_files, modified_lines):
+ try:
+ # Normalize the paths.
+ norm_modified = {os.path.normpath(k): v for k, v in modified_lines.items()}
+
+ common_uncovered_results = {}
+
+ for cpp_file, coverage_files_list in coverage_files.items():
+ norm_cpp_path = os.path.normpath(cpp_file)
+ all_uncovered = set()
+ all_covered = set()
+
+ # Filter covered and uncovered lines from each coverage source file.
+ for coverage_file in coverage_files_list:
+ log(f"Coverage File: {coverage_file}")
+ covered, uncovered = read_coverage_file(coverage_file)
+ all_uncovered |= uncovered
+ all_covered |= covered
+
+ effective_uncovered = all_uncovered - all_covered
+ common_uncovered_results[norm_cpp_path] = effective_uncovered
+
+ # Log the covered and uncovered lines to <inst_build>/patch_coverage.log
+ if norm_cpp_path in norm_modified:
+ log_coverage_details(
+ cpp_file,
+ norm_modified[norm_cpp_path],
+ effective_uncovered,
+ all_covered,
+ )
+
+ # Print the common uncovered lines summary on stanadard output.
+ for cpp_path, uncovered_set in common_uncovered_results.items():
+ if cpp_path in norm_modified:
+ print_common_uncovered_lines(cpp_path, uncovered_set, norm_modified)
+
+ except Exception as ex:
+ log("Error while reporting covered and uncovered lines:", ex)
+ sys.exit(1)
diff --git a/llvm/utils/patch-coverage/process.py b/llvm/utils/patch-coverage/process.py
new file mode 100644
index 0000000000000..28f916a2fc1c2
--- /dev/null
+++ b/llvm/utils/patch-coverage/process.py
@@ -0,0 +1,86 @@
+import subprocess
+import sys
+import os
+
+from utils import classify_tests
+from utils import log
+from utils import target_name
+
+# TODO: Can we process parallelly
+
+
+def process_coverage_data(
+ source_files, inst_build_dir, lit_binary, unit_binary, patch_path
+):
+ coverage_files = {}
+
+ try:
+ os.chdir(inst_build_dir)
+
+ unit_tests, lit_tests = classify_tests(patch_path)
+ if not (unit_tests or lit_tests):
+ return coverage_map
+
+ binary = unit_binary if unit_tests else lit_binary
+
+ for root, dirs, files in os.walk("."):
+ for file in files:
+ if os.path.basename(file) == "default.profraw":
+ continue
+
+ if file.endswith(".profraw"):
+ profraw_file = os.path.abspath(os.path.join(root, file))
+ profdata_output = os.path.abspath(
+ os.path.splitext(profraw_file)[0] + ".profdata"
+ )
+
+ log("\nProfraw File:", profraw_file)
+ log("Profdata File:", profdata_output)
+
+ llvm_profdata_cmd = [
+ "./bin/llvm-profdata",
+ "merge",
+ "-o",
+ profdata_output,
+ profraw_file,
+ ]
+
+ subprocess.check_call(llvm_profdata_cmd)
+
+ log(f"Converted {profraw_file} to {profdata_output}")
+
+ for cpp_file in source_files:
+ output_file = (
+ os.path.splitext(profdata_output)[0]
+ + f"_{cpp_file.replace('/', '_')}.txt"
+ )
+
+ parent_directory = os.path.dirname(os.getcwd())
+ abs_cpp_file = os.path.abspath(
+ os.path.join(parent_directory, cpp_file)
+ )
+
+ llvm_cov_cmd = [
+ "./bin/llvm-cov",
+ "show",
+ "-instr-profile",
+ profdata_output,
+ binary,
+ "--format=text",
+ abs_cpp_file,
+ ]
+
+ with open(output_file, "w") as output:
+ subprocess.check_call(llvm_cov_cmd, stdout=output)
+
+ log(f"Processed file saved as: {output_file}")
+
+ coverage_files.setdefault(abs_cpp_file, []).append(output_file)
+
+ log("\nConversion of profraw files to human-readable form is completed.\n")
+
+ return coverage_files
+
+ except subprocess.CalledProcessError as e:
+ log("Error during profraw to profdata conversion:", e)
+ sys.exit(1)
diff --git a/llvm/utils/patch-coverage/requirements.txt b/llvm/utils/patch-coverage/requirements.txt
new file mode 100644
index 0000000000000..da4b8fdb2bcba
--- /dev/null
+++ b/llvm/utils/patch-coverage/requirements.txt
@@ -0,0 +1 @@
+unidiff
diff --git a/llvm/utils/patch-coverage/test.py b/llvm/utils/patch-coverage/test.py
new file mode 100644
index 0000000000000..a5279cb9ab48e
--- /dev/null
+++ b/llvm/utils/patch-coverage/test.py
@@ -0,0 +1,120 @@
+import re
+import os
+import subprocess
+import sys
+
+from pathlib import Path
+from utils import log
+from utils import target_name
+
+# TODO: Can we run llvm-lit in batch mode
+
+
+def run_single_test_with_coverage(llvm_lit_path, test_path, inst_build_dir):
+ try:
+ profile_dir = os.path.abspath(os.path.join(inst_build_dir, "profiles"))
+ os.makedirs(profile_dir, exist_ok=True)
+
+ test_env = os.environ.copy()
+ test_env["LLVM_PROFILE_FILE"] = os.path.join(profile_dir, "test-%p.profraw")
+
+ lit_cmd = [llvm_lit_path, "-v", test_path]
+ subprocess.check_call(lit_cmd, env=test_env)
+
+ log("Test case executed:", test_path)
+
+ except subprocess.CalledProcessError as e:
+ log("Error while running test:", e)
+ sys.exit(1)
+
+ except Exception as ex:
+ log("Error:", ex)
+ sys.exit(1)
+
+
+def run_modified_lit_tests(llvm_lit_path, patch_path, tests, inst_build_dir):
+ try:
+ with open(patch_path, "r") as patch_file:
+ patch_diff = patch_file.read()
+
+ tests_set = {os.path.normpath(t) for t in tests}
+
+ modified_tests = set()
+
+ for match in re.finditer(
+ r"^\+\+\+\s+(?:[ab]/)?(.+)$",
+ patch_diff,
+ re.MULTILINE,
+ ):
+ file_path = match.group(1).strip()
+
+ # Skip deleted files
+ if file_path == "/dev/null":
+ continue
+
+ if not file_path.endswith(
+ (".ll", ".mir", ".mlir", ".fir", ".test", ".s", ".c", ".cpp", ".f90")
+ ):
+ continue
+
+ abs_path = os.path.normpath(
+ os.path.abspath(os.path.join(os.getcwd(), file_path))
+ )
+
+ if abs_path in tests_set:
+ modified_tests.add(abs_path)
+ log("Lit test file in patch:", abs_path)
+
+ if not modified_tests:
+ log("No modified lit tests found in the patch.")
+ return
+
+ log("\nRunning modified test cases:")
+
+ failures = []
+ for test_file in sorted(modified_tests):
+ try:
+ run_single_test_with_coverage(llvm_lit_path, test_file, inst_build_dir)
+ except SystemExit:
+ failures.append(test_file)
+
+ if failures:
+ log("\nFailed tests:")
+ for f in failures:
+ log(" -", f)
+ sys.exit(1)
+
+ except Exception as ex:
+ log("Error:", ex)
+ sys.exit(1)
+
+
+def run_modified_unit_tests(build_dir, inst_build_dir, patch_path):
+ log("Starting unit test execution...")
+
+ try:
+ target = target_name(patch_path, inst_build_dir)
+
+ if not target:
+ log("No unit test target found.")
+ return
+
+ binary_path = os.path.join(inst_build_dir, target)
+ test_name = Path(target).name
+ profraw_path = Path(inst_build_dir) / f"{test_name}.profraw"
+
+ subprocess.check_call(["ninja", "-C", inst_build_dir, target])
+
+ env = os.environ.copy()
+ env["LLVM_PROFILE_FILE"] = str(profraw_path)
+
+ log(f"Executing modified unit test case target: {binary_path}")
+ subprocess.check_call([binary_path], env=env)
+
+ except subprocess.CalledProcessError as e:
+ log(f"Error while running unit test {target}: {e}")
+ sys.exit(1)
+
+ except Exception as ex:
+ log("Error:", ex)
+ sys.exit(1)
diff --git a/llvm/utils/patch-coverage/utils.py b/llvm/utils/patch-coverage/utils.py
new file mode 100644
index 0000000000000..70631895a4a59
--- /dev/null
+++ b/llvm/utils/patch-coverage/utils.py
@@ -0,0 +1,105 @@
+import hashlib
+import logging
+import os
+import subprocess
+import re
+
+from pathlib import Path
+from unidiff import PatchSet
+
+
+def compute_patch_hash(patch_path):
+ with open(patch_path, "rb") as f:
+ return hashlib.sha256(f.read()).hexdigest()
+
+
+def should_rebuild(build_dir, patch_path, binary_path):
+ new_hash = compute_patch_hash(patch_path)
+ hash_file = os.path.join(build_dir, ".last_patch_hash")
+
+ binary_path = os.path.abspath(binary_path)
+
+ if not os.path.exists(binary_path):
+ return True
+
+ if not os.path.exists(hash_file):
+ return True
+
+ with open(hash_file) as f:
+ if f.read().strip() == new_hash:
+ return False
+
+ return True
+
+
+def mark_build_success(build_dir, patch_path):
+ with open(os.path.join(build_dir, ".last_patch_hash"), "w") as f:
+ f.write(compute_patch_hash(patch_path))
+
+
+def delete_profraw(inst_build_dir):
+ cwd = os.getcwd()
+ os.chdir(inst_build_dir)
+
+ command = 'find . -type f -name "*.profraw" -delete'
+ os.chdir(cwd)
+
+ try:
+ subprocess.run(command, shell=True, check=True)
+ log("Older '.profraw' files are successfully deleted.")
+ except subprocess.CalledProcessError as e:
+ log(f"Error: {e}")
+
+
+def configure_logging(inst_build_dir):
+ os.makedirs(inst_build_dir, exist_ok=True)
+ logging.basicConfig(
+ filename=os.path.join(inst_build_dir, "patch_coverage.log"),
+ level=logging.INFO,
+ format="%(message)s",
+ )
+
+
+def log(*args):
+ message = " ".join(map(str, args))
+ logging.info(message)
+
+
+def classify_tests(patch_path):
+ patch_diff = Path(patch_path).read_text()
+
+ unit_tests = re.findall(
+ r"^\+\+\+ [ab]/(.*unittests.*\.(?:cpp|c))$",
+ patch_diff,
+ re.MULTILINE,
+ )
+
+ lit_tests = re.findall(
+ r"^\+\+\+ [ab]/(llvm/test/.*\.(?:ll|mir|mlir|fir|test|txt))$",
+ patch_diff,
+ re.MULTILINE,
+ )
+
+ return unit_tests, lit_tests
+
+
+def target_name(patch_path, inst_build_dir):
+ try:
+ patch_diff = Path(patch_path).read_text()
+
+ matches = re.findall(
+ r"^\+\+\+ [ab]/(.*unittests.*\.(?:cpp|c))$",
+ patch_diff,
+ re.MULTILINE,
+ )
+
+ if not matches:
+ return None
+
+ # just take the first match
+ suite_name = Path(matches[0]).parent.name
+ return f"unittests/{suite_name}/{suite_name}Tests"
+
+ except Exception as e:
+ log(f"Error finding target name: {e}")
+ sys.exit(1)
>From 22fac99912be60591b4d2fe52ff1cf2abccb3df2 Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Thu, 16 Apr 2026 01:35:43 +0530
Subject: [PATCH 2/9] make clang-tidy's patches work
---
llvm/utils/patch-coverage/build.py | 129 +++++++++++++++------------
llvm/utils/patch-coverage/main.py | 13 ++-
llvm/utils/patch-coverage/patch.py | 3 +
llvm/utils/patch-coverage/process.py | 2 +-
llvm/utils/patch-coverage/test.py | 1 -
llvm/utils/patch-coverage/utils.py | 24 ++++-
6 files changed, 112 insertions(+), 60 deletions(-)
diff --git a/llvm/utils/patch-coverage/build.py b/llvm/utils/patch-coverage/build.py
index 2c5672767e0a8..ada2c919ff6d5 100644
--- a/llvm/utils/patch-coverage/build.py
+++ b/llvm/utils/patch-coverage/build.py
@@ -3,35 +3,42 @@
import subprocess
from utils import log
+from utils import resolve_projects
def is_configured_build(build_dir):
return os.path.exists(os.path.join(build_dir, "build.ninja"))
-def configure_llvm_build(build_dir):
+def configure_llvm_build(build_dir, projects):
print(f"[patch-coverage] Configuring LLVM build in {build_dir}...")
- subprocess.check_call(
- [
- "cmake",
- "-G",
- "Ninja",
- "-S",
- "llvm",
- "-B",
- build_dir,
- "-DCMAKE_BUILD_TYPE=Release",
- "-DLLVM_ENABLE_ASSERTIONS=ON",
- "-DLLVM_INCLUDE_TESTS=ON",
- "-DLLVM_BUILD_TESTS=ON",
- ]
- )
+ cmake_cmd = [
+ "cmake",
+ "-G",
+ "Ninja",
+ "-S",
+ "llvm",
+ "-B",
+ build_dir,
+ "-DCMAKE_BUILD_TYPE=Release",
+ "-DLLVM_ENABLE_ASSERTIONS=ON",
+ "-DLLVM_INCLUDE_TESTS=ON",
+ "-DLLVM_BUILD_TESTS=ON",
+ ]
+
+ if projects:
+ cmake_cmd.append(f"-DLLVM_ENABLE_PROJECTS={projects}")
+
+ print("CMake cmd:", " ".join(cmake_cmd))
+ subprocess.check_call(cmake_cmd)
+
+
+def ensure_llvm_tools(build_dir, projects, binary):
+ print("Making sure we have required tools...")
-def ensure_llvm_tools(build_dir):
- print("Making sure we have required tool...")
if not is_configured_build(build_dir):
- configure_llvm_build(build_dir)
+ configure_llvm_build(build_dir, projects)
required_tools = [
"bin/llvm-lit",
@@ -41,53 +48,63 @@ def ensure_llvm_tools(build_dir):
"bin/llvm-config",
]
- missing = [
+ missing_tools = [
tool
for tool in required_tools
if not os.path.exists(os.path.join(build_dir, tool))
]
- if not missing:
- print("Yes, We have all the tools for fetching testsuite info.")
- return
+ if missing_tools:
+ print("Building missing core tools:", missing_tools)
+ subprocess.check_call(
+ ["ninja", "-C", build_dir, "FileCheck", "count", "not", "llvm-config"]
+ )
+ else:
+ print("Core tools already present.")
- print("Building all required tools.")
- subprocess.check_call(
- [
- "ninja",
- "-C",
- build_dir,
- "FileCheck",
- "count",
- "not",
- "llvm-config",
- ]
- )
+ # Handle binary-specific requirements
+ extra_targets = []
+ if binary == "clang-tidy":
+ clang_path = os.path.join(build_dir, "bin", "clang")
+ if not os.path.exists(clang_path):
+ extra_targets.append("clang")
-def build_llvm(inst_build_dir, binary, allowlist_path):
+ if extra_targets:
+ print("Building required targets to parse testsuite info:", extra_targets)
+ subprocess.check_call(["ninja", "-C", build_dir] + extra_targets)
+ else:
+ print("All targets already present to get testsuite info.")
+
+
+def build_llvm(inst_build_dir, build_dir, binary, projects, allowlist_path):
try:
if not is_configured_build(inst_build_dir):
print("Configuring the instrumented build for patch coverage.")
- subprocess.check_call(
- [
- "cmake",
- "-G",
- "Ninja",
- "-S",
- "llvm",
- "-B",
- inst_build_dir,
- "-DCMAKE_BUILD_TYPE=Release",
- "-DLLVM_ENABLE_ASSERTIONS=ON",
- "-DLLVM_INCLUDE_TESTS=ON",
- "-DLLVM_BUILD_TESTS=ON",
- "-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON",
- "-DLLVM_INDIVIDUAL_TEST_COVERAGE=ON",
- f"-DCMAKE_C_FLAGS=-fprofile-list={os.path.abspath(allowlist_path)}",
- f"-DCMAKE_CXX_FLAGS=-fprofile-list={os.path.abspath(allowlist_path)}",
- ]
- )
+
+ cmake_cmd = [
+ "cmake",
+ "-G",
+ "Ninja",
+ "-S",
+ "llvm",
+ "-B",
+ inst_build_dir,
+ "-DCMAKE_BUILD_TYPE=Release",
+ "-DLLVM_ENABLE_ASSERTIONS=ON",
+ "-DLLVM_INCLUDE_TESTS=ON",
+ "-DLLVM_BUILD_TESTS=ON",
+ "-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON",
+ "-DLLVM_INDIVIDUAL_TEST_COVERAGE=ON",
+ f"-DCMAKE_C_FLAGS=-fprofile-list={os.path.abspath(allowlist_path)}",
+ f"-DCMAKE_CXX_FLAGS=-fprofile-list={os.path.abspath(allowlist_path)}",
+ ]
+
+ if projects:
+ cmake_cmd.append(f"-DLLVM_ENABLE_PROJECTS={projects}")
+
+ print("CMake cmd:", " ".join(cmake_cmd))
+ subprocess.check_call(cmake_cmd)
target = [
"FileCheck",
@@ -101,6 +118,8 @@ def build_llvm(inst_build_dir, binary, allowlist_path):
binary,
]
+ if binary == "clang-tidy":
+ target.append("clang")
try:
print("Building the instrumented target")
subprocess.check_call(["ninja", "-C", inst_build_dir] + target)
diff --git a/llvm/utils/patch-coverage/main.py b/llvm/utils/patch-coverage/main.py
index ef88db2b4591c..0fc11378eff7a 100644
--- a/llvm/utils/patch-coverage/main.py
+++ b/llvm/utils/patch-coverage/main.py
@@ -30,6 +30,7 @@
from utils import delete_profraw
from utils import log
from utils import mark_build_success
+from utils import resolve_projects
from utils import should_rebuild
from utils import target_name
@@ -59,6 +60,11 @@ def parse_args():
parser.add_argument("test_path", nargs="*")
+ parser.add_argument(
+ "--projects",
+ help="LLVM projects to enable (semicolon-separated, e.g. clang;mlir)",
+ )
+
args = parser.parse_args()
if not args.test_path:
@@ -70,6 +76,7 @@ def parse_args():
args.num_commits,
args.binary,
args.test_path,
+ args.projects,
)
@@ -80,12 +87,14 @@ def main():
num_commits,
binary,
test_paths,
+ projects,
) = parse_args()
configure_logging(inst_build_dir)
+ projects = resolve_projects(projects, build_dir)
# Ensure we have required tools to parse test suite info.
- ensure_llvm_tools(build_dir)
+ ensure_llvm_tools(build_dir, projects, binary)
# Create a diff file from the commit/s.
patch_path = os.path.join(build_dir, "patch.diff")
@@ -124,7 +133,7 @@ def main():
rebuild = should_rebuild(inst_build_dir, patch_path, lit_binary or unit_binary)
if rebuild:
delete_profraw(inst_build_dir)
- build_llvm(inst_build_dir, binary, allowlist_path)
+ build_llvm(inst_build_dir, build_dir, binary, projects, allowlist_path)
mark_build_success(inst_build_dir, patch_path)
else:
print("\n[patch-coverage] Skipping patch coverage (no changes)")
diff --git a/llvm/utils/patch-coverage/patch.py b/llvm/utils/patch-coverage/patch.py
index 2dda93c727b5f..64fe35d93182c 100644
--- a/llvm/utils/patch-coverage/patch.py
+++ b/llvm/utils/patch-coverage/patch.py
@@ -31,8 +31,11 @@ def extract_source_files_from_patch(patch_path):
patch_diff = patch_file.read().decode("utf-8", "ignore")
file_matches = re.findall(r"\+{3} b/(\S+)", patch_diff)
+ known_source_file_extension = (".cpp", ".c")
for file in file_matches:
+ if not file.lower().endswith(known_source_file_extension):
+ continue
if any(keyword in file.lower() for keyword in ["test", "unittest"]):
test_files.append(file)
else:
diff --git a/llvm/utils/patch-coverage/process.py b/llvm/utils/patch-coverage/process.py
index 28f916a2fc1c2..9d2e2da812589 100644
--- a/llvm/utils/patch-coverage/process.py
+++ b/llvm/utils/patch-coverage/process.py
@@ -19,7 +19,7 @@ def process_coverage_data(
unit_tests, lit_tests = classify_tests(patch_path)
if not (unit_tests or lit_tests):
- return coverage_map
+ return coverage_files
binary = unit_binary if unit_tests else lit_binary
diff --git a/llvm/utils/patch-coverage/test.py b/llvm/utils/patch-coverage/test.py
index a5279cb9ab48e..4b1cd4bdba3ac 100644
--- a/llvm/utils/patch-coverage/test.py
+++ b/llvm/utils/patch-coverage/test.py
@@ -60,7 +60,6 @@ def run_modified_lit_tests(llvm_lit_path, patch_path, tests, inst_build_dir):
abs_path = os.path.normpath(
os.path.abspath(os.path.join(os.getcwd(), file_path))
)
-
if abs_path in tests_set:
modified_tests.add(abs_path)
log("Lit test file in patch:", abs_path)
diff --git a/llvm/utils/patch-coverage/utils.py b/llvm/utils/patch-coverage/utils.py
index 70631895a4a59..674af608c2818 100644
--- a/llvm/utils/patch-coverage/utils.py
+++ b/llvm/utils/patch-coverage/utils.py
@@ -17,6 +17,9 @@ def should_rebuild(build_dir, patch_path, binary_path):
new_hash = compute_patch_hash(patch_path)
hash_file = os.path.join(build_dir, ".last_patch_hash")
+ if not binary_path:
+ return True
+
binary_path = os.path.abspath(binary_path)
if not os.path.exists(binary_path):
@@ -75,7 +78,7 @@ def classify_tests(patch_path):
)
lit_tests = re.findall(
- r"^\+\+\+ [ab]/(llvm/test/.*\.(?:ll|mir|mlir|fir|test|txt))$",
+ r"^\+\+\+ [ab]/(.*test/.*\.(?:ll|mir|mlir|fir|test|txt|s|c|cpp|f90))$",
patch_diff,
re.MULTILINE,
)
@@ -103,3 +106,22 @@ def target_name(patch_path, inst_build_dir):
except Exception as e:
log(f"Error finding target name: {e}")
sys.exit(1)
+
+
+def get_projects_from_cache(cache_file):
+ with open(cache_file) as f:
+ for line in f:
+ if line.startswith("LLVM_ENABLE_PROJECTS"):
+ return line.split("=")[1].strip()
+ return ""
+
+
+def resolve_projects(projects, build_dir):
+ if projects:
+ return projects
+
+ cache_file = os.path.join(build_dir, "CMakeCache.txt")
+ if os.path.exists(cache_file):
+ return get_projects_from_cache(cache_file)
+
+ return ""
>From fce4687b62b89ffbf132b3cc65cbfaf3773bd093 Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Thu, 16 Apr 2026 12:10:52 +0530
Subject: [PATCH 3/9] fix .ll test cases recognition
---
llvm/utils/patch-coverage/patch.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/utils/patch-coverage/patch.py b/llvm/utils/patch-coverage/patch.py
index 64fe35d93182c..eac74cde88942 100644
--- a/llvm/utils/patch-coverage/patch.py
+++ b/llvm/utils/patch-coverage/patch.py
@@ -34,10 +34,11 @@ def extract_source_files_from_patch(patch_path):
known_source_file_extension = (".cpp", ".c")
for file in file_matches:
- if not file.lower().endswith(known_source_file_extension):
- continue
if any(keyword in file.lower() for keyword in ["test", "unittest"]):
test_files.append(file)
+ continue
+ if not file.lower().endswith(known_source_file_extension):
+ continue
else:
repo_root = os.path.abspath(os.getcwd())
full_path = os.path.join(repo_root, file)
>From 34acd5bbd1fc646c1be64e7804049f268214c908 Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Thu, 16 Apr 2026 17:13:50 +0530
Subject: [PATCH 4/9] some minor README.md updates
---
llvm/utils/patch-coverage/README.md | 23 ++++++++++++++++-------
llvm/utils/patch-coverage/main.py | 2 +-
2 files changed, 17 insertions(+), 8 deletions(-)
diff --git a/llvm/utils/patch-coverage/README.md b/llvm/utils/patch-coverage/README.md
index 2c16f9279c5bd..9dea5d42d9a03 100644
--- a/llvm/utils/patch-coverage/README.md
+++ b/llvm/utils/patch-coverage/README.md
@@ -56,6 +56,12 @@ ninja -C build
```bash
pip install -r llvm/utils/patch-coverage/requirements.txt
```
+or
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install unidiff
+```
### 3. Add tool to PATH
@@ -80,7 +86,7 @@ git patch-coverage -b <build-dir> <binary> <test-path>
### Example
```bash
-git patch-coverage -b build -n 3 bin/opt llvm/test
+git patch-coverage -b build -n 3 opt llvm/test
```
## Arguments
@@ -89,7 +95,7 @@ git patch-coverage -b build -n 3 bin/opt llvm/test
| Argument | Description |
| ----------------- | ---------------------------------------------------- |
-| `<binary>` | LLVM binary to analyze (e.g., `bin/opt`) |
+| `<binary>` | LLVM binary to analyze (e.g., `opt`) |
### Optional
@@ -97,19 +103,20 @@ git patch-coverage -b build -n 3 bin/opt llvm/test
| Argument | Description |
| ------------------------------ | ----------------------------------------------------------- |
| `-n, --num-commits` | Number of recent commits to analyze (default: 1) |
+| `--projects` | LLVM projects to enable (e.g. clang;mlir (default: None) |
| `-i, --instrumented-build-dir` | Use pre-built instrumented directory (Deafult: `build_inst`)|
| `-b, --build-dir` | Path to LLVM build directory (default: `build`) |
| `<test-path>` | One or more test suite paths (default: `build/test`) |
+
## How It Works
-To be written
## Output
-### `<build>/patch_coverage.log`
+### `<build_inst>/patch_coverage.log`
```
Modified File: llvm/lib/IR/Example.cpp
@@ -128,10 +135,12 @@ Modified File: llvm/lib/IR/Example.cpp
### No coverage generated
-* Ensure instrumented build succeeded
-* Check `.profraw` files in build directory
+* Ensure instrumented build succeeded and tests run
+* Check `.profraw` files in `build_inst` directory
### Rebuild skipped unexpectedly
-* Delete `.last_patch_hash` to force rebuild
+* Delete `binary` to force rebuild
+### For any other doubt
+* Check `<build_inst>/patch_coverage.log`. It has full logging of tool
diff --git a/llvm/utils/patch-coverage/main.py b/llvm/utils/patch-coverage/main.py
index 0fc11378eff7a..3c4c26901c623 100644
--- a/llvm/utils/patch-coverage/main.py
+++ b/llvm/utils/patch-coverage/main.py
@@ -46,7 +46,7 @@ def parse_args():
"-i",
"--instrumented-build-dir",
dest="inst_build_dir",
- default="build_instrumented",
+ default="build_inst",
)
parser.add_argument(
>From ccba7358f0c4f5957cf03bc13493de224a34651c Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Fri, 17 Apr 2026 03:04:54 +0530
Subject: [PATCH 5/9] print coverage summary in colored diff style
---
llvm/utils/patch-coverage/print.py | 73 ++++++++++++++++++++++++++----
llvm/utils/patch-coverage/utils.py | 16 +++++++
2 files changed, 81 insertions(+), 8 deletions(-)
diff --git a/llvm/utils/patch-coverage/print.py b/llvm/utils/patch-coverage/print.py
index cbb33082d9fd6..1ec8be0b7f999 100644
--- a/llvm/utils/patch-coverage/print.py
+++ b/llvm/utils/patch-coverage/print.py
@@ -1,7 +1,9 @@
-import sys
import os
+import re
+import sys
from utils import log
+from utils import group_contiguous_lines
def read_coverage_file(coverage_file):
@@ -42,14 +44,69 @@ def log_coverage_details(file, lines, uncovered_line_numbers, covered_line_numbe
log(f" Covered Line: {line_number_source} : {line_content.strip()}")
-def print_common_uncovered_lines(cpp_file, uncovered_lines, modified_lines):
+def coverage_percentage(cpp_file, uncovered_lines, modified_lines):
+ total = 0
+ covered_count = 0
+
+ for line_number_source, line_content in modified_lines[cpp_file]:
+ total += 1
+
+ content = line_content.strip()
+ if not content:
+ covered_count += 1
+
+ elif line_number_source not in uncovered_lines:
+ covered_count += 1
+
+ percent = (covered_count * 100 / total) if total else 0
+ print(
+ f"\n\033[33mPATCH COVERAGE: {percent:.1f}% ({covered_count}/{total} lines)\033[0m"
+ )
+
+
+def print_coverage_report(cpp_file, uncovered_lines, modified_lines):
if cpp_file not in modified_lines:
return
- print(f"\n[code-coverage] Common uncovered lines for {cpp_file} after all changes:")
- for line_number_source, line_content in modified_lines[cpp_file]:
- if line_number_source in uncovered_lines:
- print(f" Line {line_number_source}: {line_content.strip()}")
+ rela_path = re.sub(r"^(\.\./)+", "", os.path.relpath(cpp_file))
+ print(f"\n\033[1m[code-coverage] Coverage report for {rela_path}:\033[0m")
+
+ with open(cpp_file, "r") as f:
+ file_lines = f.readlines()
+
+ patch_lines = sorted(line for line, _ in modified_lines[cpp_file])
+ uncovered_lines = set(uncovered_lines)
+
+ groups = group_contiguous_lines(patch_lines)
+
+ # To make use we do not print overlapped context window lines
+ printed = set()
+
+ for group in groups:
+ start = group[0]
+ end = group[-1]
+
+ window_start = max(1, start - 1)
+ window_end = min(len(file_lines), end + 1)
+
+ for line in range(window_start, window_end + 1):
+ if line in printed:
+ continue
+ printed.add(line)
+
+ # Syncronise with 0 based indexing that file_lines have.
+ content = file_lines[line - 1].rstrip()
+
+ # red 31, green 32, white 0
+ if line in patch_lines:
+ if line in uncovered_lines:
+ print(f"\033[36m Line {line:<5}\033[0m: \033[31m{content}\033[0m")
+ else:
+ print(f"\033[36m Line {line:<5}\033[0m: \033[32m{content}\033[0m")
+ else:
+ print(f"\033[36m Line {line:<5}\033[0m: {content}")
+
+ coverage_percentage(cpp_file, uncovered_lines, modified_lines)
def report_covered_and_uncovered_lines(coverage_files, modified_lines):
@@ -83,10 +140,10 @@ def report_covered_and_uncovered_lines(coverage_files, modified_lines):
all_covered,
)
- # Print the common uncovered lines summary on stanadard output.
+ # Print the covered, uncovered and context lines on stanadard output in diff style.
for cpp_path, uncovered_set in common_uncovered_results.items():
if cpp_path in norm_modified:
- print_common_uncovered_lines(cpp_path, uncovered_set, norm_modified)
+ print_coverage_report(cpp_path, uncovered_set, norm_modified)
except Exception as ex:
log("Error while reporting covered and uncovered lines:", ex)
diff --git a/llvm/utils/patch-coverage/utils.py b/llvm/utils/patch-coverage/utils.py
index 674af608c2818..319f02a240e61 100644
--- a/llvm/utils/patch-coverage/utils.py
+++ b/llvm/utils/patch-coverage/utils.py
@@ -125,3 +125,19 @@ def resolve_projects(projects, build_dir):
return get_projects_from_cache(cache_file)
return ""
+
+
+def group_contiguous_lines(lines):
+ groups = []
+ current = [lines[0]]
+
+ for i in range(1, len(lines)):
+ if lines[i] == lines[i - 1] + 1:
+ current.append(lines[i])
+ else:
+ groups.append(current)
+ current = [lines[i]]
+
+ groups.append(current)
+ return groups
+
>From 783b0abf381f8aa517a02557fdc690dbd88971d1 Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Fri, 17 Apr 2026 20:01:51 +0530
Subject: [PATCH 6/9] print complete patch coverage percentage
---
llvm/utils/patch-coverage/lit.py | 3 ++-
llvm/utils/patch-coverage/print.py | 19 +++++++++++++++----
2 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/llvm/utils/patch-coverage/lit.py b/llvm/utils/patch-coverage/lit.py
index 85148fc60761e..30b013bd8a64d 100644
--- a/llvm/utils/patch-coverage/lit.py
+++ b/llvm/utils/patch-coverage/lit.py
@@ -23,9 +23,10 @@ def find_lit_tests(lit_path, test_paths):
suites_cmd = [lit_path, "--show-suites"] + test_paths
output = subprocess.check_output(suites_cmd)
- # Above command give large output like this -
+ # `--show-suites` produce output in the following form -
# LLVM - 61914 tests
# Source Root: /Users/<username>/llvm-project/llvm/test
+ #
# Parse it to construct following format -
# {'LLVM': '/Users/<username>/llvm-project/llvm/test'}
test_suites = parse_suite_info(output)
diff --git a/llvm/utils/patch-coverage/print.py b/llvm/utils/patch-coverage/print.py
index 1ec8be0b7f999..618d745fc1a56 100644
--- a/llvm/utils/patch-coverage/print.py
+++ b/llvm/utils/patch-coverage/print.py
@@ -60,8 +60,9 @@ def coverage_percentage(cpp_file, uncovered_lines, modified_lines):
percent = (covered_count * 100 / total) if total else 0
print(
- f"\n\033[33mPATCH COVERAGE: {percent:.1f}% ({covered_count}/{total} lines)\033[0m"
+ f"\n\033[33mFILE COVERAGE: {percent:.1f}% ({covered_count}/{total} lines)\033[0m"
)
+ return total, covered_count
def print_coverage_report(cpp_file, uncovered_lines, modified_lines):
@@ -106,7 +107,7 @@ def print_coverage_report(cpp_file, uncovered_lines, modified_lines):
else:
print(f"\033[36m Line {line:<5}\033[0m: {content}")
- coverage_percentage(cpp_file, uncovered_lines, modified_lines)
+ return coverage_percentage(cpp_file, uncovered_lines, modified_lines)
def report_covered_and_uncovered_lines(coverage_files, modified_lines):
@@ -140,10 +141,20 @@ def report_covered_and_uncovered_lines(coverage_files, modified_lines):
all_covered,
)
- # Print the covered, uncovered and context lines on stanadard output in diff style.
+ # Print the covered, uncovered and context lines on standard output in diff style.
+ total = 0
+ covered_count = 0
for cpp_path, uncovered_set in common_uncovered_results.items():
if cpp_path in norm_modified:
- print_coverage_report(cpp_path, uncovered_set, norm_modified)
+ temp1, temp2 = print_coverage_report(
+ cpp_path, uncovered_set, norm_modified
+ )
+ total += temp1
+ covered_count += temp2
+ percent = (covered_count * 100 / total) if total else 0
+ print(
+ f"\n\033[33mPATCH COVERAGE: {percent:.1f}% ({covered_count}/{total} lines)\033[0m"
+ )
except Exception as ex:
log("Error while reporting covered and uncovered lines:", ex)
>From bf31edc23a340aa065832bffd867e816369cf22d Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Fri, 17 Apr 2026 22:48:17 +0530
Subject: [PATCH 7/9] update help messages
---
llvm/utils/patch-coverage/:wq | 185 +++++++++++++++++++++++++++++
llvm/utils/patch-coverage/main.py | 31 ++++-
llvm/utils/patch-coverage/utils.py | 1 -
3 files changed, 211 insertions(+), 6 deletions(-)
create mode 100644 llvm/utils/patch-coverage/:wq
diff --git a/llvm/utils/patch-coverage/:wq b/llvm/utils/patch-coverage/:wq
new file mode 100644
index 0000000000000..d06b973abab14
--- /dev/null
+++ b/llvm/utils/patch-coverage/:wq
@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+#
+# ===----------------------------------------------------------------------------===#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------------===#
+
+import argparse
+import logging
+import os
+import re
+import subprocess
+import sys
+
+from build import build_llvm
+from build import ensure_llvm_tools
+from lit import find_lit_tests
+from patch import extract_modified_source_lines_from_patch
+from patch import extract_source_files_from_patch
+from patch import create_patch_from_last_commits
+from patch import write_source_file_allowlist
+from print import report_covered_and_uncovered_lines
+from process import process_coverage_data
+from test import run_modified_lit_tests
+from test import run_modified_unit_tests
+from utils import configure_logging
+from utils import classify_tests
+from utils import delete_profraw
+from utils import log
+from utils import mark_build_success
+from utils import resolve_projects
+from utils import should_rebuild
+from utils import target_name
+
+sys.path.append(os.path.dirname(__file__))
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ prog="git patch-coverage",
+ description="Patch-based code coverage tool for LLVM",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+
+ parser.add_argument(
+ "-b",
+ "--build-dir",
+ help="path to build directory if the target is already built",
+ dest="build_dir",
+ default="build",
+ )
+
+ parser.add_argument(
+ "-i",
+ "--instrumented-build-dir",
+ help="path to directory in which tool will build projects with intrumentation enabled",
+ dest="inst_build_dir",
+ default="build_inst",
+ )
+
+ parser.add_argument(
+ "-n",
+ "--num-commits",
+ help="number of commits to include in patch",
+ dest="num_commits",
+ default=1,
+ )
+
+ parser.add_argument(
+ "binary", help="target binary to generate coverage for (e.g. opt, clang)",
+ )
+
+ parser.add_argument(
+ "test_path", metavar="TEST_PATH", nargs="*", help="path to test suite/s"
+ )
+
+ parser.add_argument(
+ "--projects",
+ help="LLVM projects to enable (semicolon-separated, e.g. clang;mlir)",
+ default="llvm",
+ )
+
+ args = parser.parse_args()
+
+ if not args.test_path:
+ args.test_path = [os.path.join(args.build_dir, "test")]
+
+ return (
+ args.build_dir,
+ args.inst_build_dir,
+ args.num_commits,
+ args.binary,
+ args.test_path,
+ args.projects,
+ )
+
+
+def main():
+ (
+ build_dir,
+ inst_build_dir,
+ num_commits,
+ binary,
+ test_paths,
+ projects,
+ ) = parse_args()
+
+ configure_logging(inst_build_dir)
+ projects = resolve_projects(projects, build_dir)
+
+ # Ensure we have required tools to parse test suite info.
+ ensure_llvm_tools(build_dir, projects, binary)
+
+ # Create a diff file from the commit/s.
+ patch_path = os.path.join(build_dir, "patch.diff")
+ create_patch_from_last_commits(patch_path, num_commits)
+ source_files = extract_source_files_from_patch(patch_path)
+
+ # Get all the modified lines of patch, from both source files and test files.
+ llvm_lit_path = os.path.join(build_dir, "bin/llvm-lit")
+ tests = frozenset(find_lit_tests(llvm_lit_path, test_paths))
+ modified_lines = extract_modified_source_lines_from_patch(patch_path, tests)
+
+ # Use allow list feature to generate ".profraw" data for only source files in the patch.
+ os.makedirs(inst_build_dir, exist_ok=True)
+ allowlist_path = os.path.join(inst_build_dir, "fun.list")
+ write_source_file_allowlist(source_files, allowlist_path)
+
+ # Print all the modified lines of the patch.
+ for file, lines in modified_lines.items():
+ log(f"File: {os.path.relpath(file)[2:]}")
+ for line_number, line_content in lines:
+ cleaned_line_content = line_content.rstrip("\n")
+ log(f"Line {line_number}: {cleaned_line_content}")
+ log("")
+
+ # Contruct the absolute path of binary that we want to check coverage for.
+ unit_tests, lit_tests = classify_tests(patch_path)
+ unit_binary = None
+ lit_binary = None
+ if lit_tests:
+ lit_binary = os.path.abspath(os.path.join(inst_build_dir, "bin", binary))
+ if unit_tests:
+ target = target_name(patch_path, inst_build_dir)
+ unit_binary = os.path.abspath(os.path.join(inst_build_dir, target))
+
+ # Build the LLVM in instrumented directory using LLVM_BUILD_INSTRUMENTED_COVERAGE.
+ rebuild = should_rebuild(inst_build_dir, patch_path, lit_binary or unit_binary)
+ if rebuild:
+ delete_profraw(inst_build_dir)
+ build_llvm(inst_build_dir, build_dir, binary, projects, allowlist_path)
+ mark_build_success(inst_build_dir, patch_path)
+ else:
+ print("\n[patch-coverage] Skipping patch coverage (no changes)")
+ sys.exit(1)
+
+ # Run all the test cases of patch with instrumented binary.
+ inst_lit_path = os.path.join(inst_build_dir, "bin/llvm-lit")
+ patch_path = os.path.abspath(patch_path)
+ run_modified_lit_tests(inst_lit_path, patch_path, tests, inst_build_dir)
+ run_modified_unit_tests(build_dir, inst_build_dir, patch_path)
+
+ # Report covered and uncovered lines of each source file.
+ coverage_files = process_coverage_data(
+ source_files,
+ inst_build_dir,
+ lit_binary,
+ unit_binary,
+ patch_path,
+ )
+ report_covered_and_uncovered_lines(coverage_files, modified_lines)
+
+ # Remove redundant "default.profraw" generated in source root.
+ curr_dir = os.path.dirname(os.getcwd())
+ default_profraw_path = os.path.join(curr_dir, "default.profraw")
+ try:
+ os.remove(default_profraw_path)
+ except FileNotFoundError:
+ pass
+
+
+if __name__ == "__main__":
+ main()
diff --git a/llvm/utils/patch-coverage/main.py b/llvm/utils/patch-coverage/main.py
index 3c4c26901c623..5e5d1a811ad45 100644
--- a/llvm/utils/patch-coverage/main.py
+++ b/llvm/utils/patch-coverage/main.py
@@ -38,13 +38,24 @@
def parse_args():
- parser = argparse.ArgumentParser()
+ parser = argparse.ArgumentParser(
+ prog="git patch-coverage",
+ description="Patch-based code coverage tool for LLVM",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
- parser.add_argument("-b", "--build-dir", dest="build_dir", default="build")
+ parser.add_argument(
+ "-b",
+ "--build-dir",
+ help="path to build directory",
+ dest="build_dir",
+ default="build",
+ )
parser.add_argument(
"-i",
"--instrumented-build-dir",
+ help="path to directory in which the tool will build projects with intrumentation enabled",
dest="inst_build_dir",
default="build_inst",
)
@@ -52,22 +63,32 @@ def parse_args():
parser.add_argument(
"-n",
"--num-commits",
+ help="number of commits to include in patch",
dest="num_commits",
default=1,
)
- parser.add_argument("binary")
+ parser.add_argument(
+ "binary",
+ help="target binary to generate coverage for (e.g. opt, clang)",
+ )
- parser.add_argument("test_path", nargs="*")
+ parser.add_argument(
+ "test_path",
+ nargs="*",
+ help="path to test suite/s (default: <build-dir>/test)",
+ default=argparse.SUPPRESS,
+ )
parser.add_argument(
"--projects",
help="LLVM projects to enable (semicolon-separated, e.g. clang;mlir)",
+ default="llvm",
)
args = parser.parse_args()
- if not args.test_path:
+ if not hasattr(args, "test_path") or not args.test_path:
args.test_path = [os.path.join(args.build_dir, "test")]
return (
diff --git a/llvm/utils/patch-coverage/utils.py b/llvm/utils/patch-coverage/utils.py
index 319f02a240e61..08271302d6e92 100644
--- a/llvm/utils/patch-coverage/utils.py
+++ b/llvm/utils/patch-coverage/utils.py
@@ -140,4 +140,3 @@ def group_contiguous_lines(lines):
groups.append(current)
return groups
-
>From 0e8c7924fcab9df9c960579de4c854bb8bef24ae Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Fri, 17 Apr 2026 22:50:58 +0530
Subject: [PATCH 8/9] remove extra file
---
llvm/utils/patch-coverage/:wq | 185 ----------------------------------
1 file changed, 185 deletions(-)
delete mode 100644 llvm/utils/patch-coverage/:wq
diff --git a/llvm/utils/patch-coverage/:wq b/llvm/utils/patch-coverage/:wq
deleted file mode 100644
index d06b973abab14..0000000000000
--- a/llvm/utils/patch-coverage/:wq
+++ /dev/null
@@ -1,185 +0,0 @@
-#!/usr/bin/env python3
-#
-# ===----------------------------------------------------------------------------===#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# ===----------------------------------------------------------------------------===#
-
-import argparse
-import logging
-import os
-import re
-import subprocess
-import sys
-
-from build import build_llvm
-from build import ensure_llvm_tools
-from lit import find_lit_tests
-from patch import extract_modified_source_lines_from_patch
-from patch import extract_source_files_from_patch
-from patch import create_patch_from_last_commits
-from patch import write_source_file_allowlist
-from print import report_covered_and_uncovered_lines
-from process import process_coverage_data
-from test import run_modified_lit_tests
-from test import run_modified_unit_tests
-from utils import configure_logging
-from utils import classify_tests
-from utils import delete_profraw
-from utils import log
-from utils import mark_build_success
-from utils import resolve_projects
-from utils import should_rebuild
-from utils import target_name
-
-sys.path.append(os.path.dirname(__file__))
-
-
-def parse_args():
- parser = argparse.ArgumentParser(
- prog="git patch-coverage",
- description="Patch-based code coverage tool for LLVM",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
- )
-
- parser.add_argument(
- "-b",
- "--build-dir",
- help="path to build directory if the target is already built",
- dest="build_dir",
- default="build",
- )
-
- parser.add_argument(
- "-i",
- "--instrumented-build-dir",
- help="path to directory in which tool will build projects with intrumentation enabled",
- dest="inst_build_dir",
- default="build_inst",
- )
-
- parser.add_argument(
- "-n",
- "--num-commits",
- help="number of commits to include in patch",
- dest="num_commits",
- default=1,
- )
-
- parser.add_argument(
- "binary", help="target binary to generate coverage for (e.g. opt, clang)",
- )
-
- parser.add_argument(
- "test_path", metavar="TEST_PATH", nargs="*", help="path to test suite/s"
- )
-
- parser.add_argument(
- "--projects",
- help="LLVM projects to enable (semicolon-separated, e.g. clang;mlir)",
- default="llvm",
- )
-
- args = parser.parse_args()
-
- if not args.test_path:
- args.test_path = [os.path.join(args.build_dir, "test")]
-
- return (
- args.build_dir,
- args.inst_build_dir,
- args.num_commits,
- args.binary,
- args.test_path,
- args.projects,
- )
-
-
-def main():
- (
- build_dir,
- inst_build_dir,
- num_commits,
- binary,
- test_paths,
- projects,
- ) = parse_args()
-
- configure_logging(inst_build_dir)
- projects = resolve_projects(projects, build_dir)
-
- # Ensure we have required tools to parse test suite info.
- ensure_llvm_tools(build_dir, projects, binary)
-
- # Create a diff file from the commit/s.
- patch_path = os.path.join(build_dir, "patch.diff")
- create_patch_from_last_commits(patch_path, num_commits)
- source_files = extract_source_files_from_patch(patch_path)
-
- # Get all the modified lines of patch, from both source files and test files.
- llvm_lit_path = os.path.join(build_dir, "bin/llvm-lit")
- tests = frozenset(find_lit_tests(llvm_lit_path, test_paths))
- modified_lines = extract_modified_source_lines_from_patch(patch_path, tests)
-
- # Use allow list feature to generate ".profraw" data for only source files in the patch.
- os.makedirs(inst_build_dir, exist_ok=True)
- allowlist_path = os.path.join(inst_build_dir, "fun.list")
- write_source_file_allowlist(source_files, allowlist_path)
-
- # Print all the modified lines of the patch.
- for file, lines in modified_lines.items():
- log(f"File: {os.path.relpath(file)[2:]}")
- for line_number, line_content in lines:
- cleaned_line_content = line_content.rstrip("\n")
- log(f"Line {line_number}: {cleaned_line_content}")
- log("")
-
- # Contruct the absolute path of binary that we want to check coverage for.
- unit_tests, lit_tests = classify_tests(patch_path)
- unit_binary = None
- lit_binary = None
- if lit_tests:
- lit_binary = os.path.abspath(os.path.join(inst_build_dir, "bin", binary))
- if unit_tests:
- target = target_name(patch_path, inst_build_dir)
- unit_binary = os.path.abspath(os.path.join(inst_build_dir, target))
-
- # Build the LLVM in instrumented directory using LLVM_BUILD_INSTRUMENTED_COVERAGE.
- rebuild = should_rebuild(inst_build_dir, patch_path, lit_binary or unit_binary)
- if rebuild:
- delete_profraw(inst_build_dir)
- build_llvm(inst_build_dir, build_dir, binary, projects, allowlist_path)
- mark_build_success(inst_build_dir, patch_path)
- else:
- print("\n[patch-coverage] Skipping patch coverage (no changes)")
- sys.exit(1)
-
- # Run all the test cases of patch with instrumented binary.
- inst_lit_path = os.path.join(inst_build_dir, "bin/llvm-lit")
- patch_path = os.path.abspath(patch_path)
- run_modified_lit_tests(inst_lit_path, patch_path, tests, inst_build_dir)
- run_modified_unit_tests(build_dir, inst_build_dir, patch_path)
-
- # Report covered and uncovered lines of each source file.
- coverage_files = process_coverage_data(
- source_files,
- inst_build_dir,
- lit_binary,
- unit_binary,
- patch_path,
- )
- report_covered_and_uncovered_lines(coverage_files, modified_lines)
-
- # Remove redundant "default.profraw" generated in source root.
- curr_dir = os.path.dirname(os.getcwd())
- default_profraw_path = os.path.join(curr_dir, "default.profraw")
- try:
- os.remove(default_profraw_path)
- except FileNotFoundError:
- pass
-
-
-if __name__ == "__main__":
- main()
>From 12ae8b8f461ffc0f2065d5a25eac41665d16c3bb Mon Sep 17 00:00:00 2001
From: Shivam Gupta <shivam98.tkg at gmail.com>
Date: Sat, 18 Apr 2026 11:13:48 +0530
Subject: [PATCH 9/9] support lldb api tests
---
llvm/utils/patch-coverage/build.py | 18 ++++++++++++++++++
llvm/utils/patch-coverage/main.py | 1 -
llvm/utils/patch-coverage/patch.py | 2 +-
llvm/utils/patch-coverage/test.py | 13 ++++++++++++-
4 files changed, 31 insertions(+), 3 deletions(-)
diff --git a/llvm/utils/patch-coverage/build.py b/llvm/utils/patch-coverage/build.py
index ada2c919ff6d5..7fee0b36acabf 100644
--- a/llvm/utils/patch-coverage/build.py
+++ b/llvm/utils/patch-coverage/build.py
@@ -1,4 +1,5 @@
import os
+import shutil
import sys
import subprocess
@@ -30,6 +31,10 @@ def configure_llvm_build(build_dir, projects):
if projects:
cmake_cmd.append(f"-DLLVM_ENABLE_PROJECTS={projects}")
+ if projects and "lldb" in projects:
+ cmake_cmd.append(f"-DLLVM_ENABLE_RUNTIMES=libcxx")
+ cmake_cmd.append(f"-DPYTHON_EXECUTABLE={shutil.which('python3')}")
+
print("CMake cmd:", " ".join(cmake_cmd))
subprocess.check_call(cmake_cmd)
@@ -70,6 +75,11 @@ def ensure_llvm_tools(build_dir, projects, binary):
if not os.path.exists(clang_path):
extra_targets.append("clang")
+ if binary == "lldb":
+ lldb_path = os.path.join(build_dir, "bin", "lldb")
+ if not os.path.exists(lldb_path):
+ extra_targets.append("lldb")
+
if extra_targets:
print("Building required targets to parse testsuite info:", extra_targets)
subprocess.check_call(["ninja", "-C", build_dir] + extra_targets)
@@ -103,6 +113,10 @@ def build_llvm(inst_build_dir, build_dir, binary, projects, allowlist_path):
if projects:
cmake_cmd.append(f"-DLLVM_ENABLE_PROJECTS={projects}")
+ if projects and "lldb" in projects:
+ cmake_cmd.append(f"-DLLVM_ENABLE_RUNTIMES=libcxx")
+ cmake_cmd.append(f"-DPYTHON_EXECUTABLE={shutil.which('python3')}")
+
print("CMake cmd:", " ".join(cmake_cmd))
subprocess.check_call(cmake_cmd)
@@ -120,6 +134,10 @@ def build_llvm(inst_build_dir, build_dir, binary, projects, allowlist_path):
if binary == "clang-tidy":
target.append("clang")
+
+ if binary == "lldb":
+ target.extend(["clang", "lldb", "dsymutil"])
+
try:
print("Building the instrumented target")
subprocess.check_call(["ninja", "-C", inst_build_dir] + target)
diff --git a/llvm/utils/patch-coverage/main.py b/llvm/utils/patch-coverage/main.py
index 5e5d1a811ad45..14ebf3546b12f 100644
--- a/llvm/utils/patch-coverage/main.py
+++ b/llvm/utils/patch-coverage/main.py
@@ -83,7 +83,6 @@ def parse_args():
parser.add_argument(
"--projects",
help="LLVM projects to enable (semicolon-separated, e.g. clang;mlir)",
- default="llvm",
)
args = parser.parse_args()
diff --git a/llvm/utils/patch-coverage/patch.py b/llvm/utils/patch-coverage/patch.py
index eac74cde88942..dfa2a7c43f525 100644
--- a/llvm/utils/patch-coverage/patch.py
+++ b/llvm/utils/patch-coverage/patch.py
@@ -31,7 +31,7 @@ def extract_source_files_from_patch(patch_path):
patch_diff = patch_file.read().decode("utf-8", "ignore")
file_matches = re.findall(r"\+{3} b/(\S+)", patch_diff)
- known_source_file_extension = (".cpp", ".c")
+ known_source_file_extension = (".cpp", ".c", ".mm")
for file in file_matches:
if any(keyword in file.lower() for keyword in ["test", "unittest"]):
diff --git a/llvm/utils/patch-coverage/test.py b/llvm/utils/patch-coverage/test.py
index 4b1cd4bdba3ac..9789f07dc293d 100644
--- a/llvm/utils/patch-coverage/test.py
+++ b/llvm/utils/patch-coverage/test.py
@@ -53,7 +53,18 @@ def run_modified_lit_tests(llvm_lit_path, patch_path, tests, inst_build_dir):
continue
if not file_path.endswith(
- (".ll", ".mir", ".mlir", ".fir", ".test", ".s", ".c", ".cpp", ".f90")
+ (
+ ".ll",
+ ".mir",
+ ".mlir",
+ ".fir",
+ ".test",
+ ".s",
+ ".c",
+ ".cpp",
+ ".f90",
+ ".py",
+ )
):
continue
More information about the llvm-commits
mailing list