[libc-commits] [libc] [llvm] Implement Automated Line and MC/DC Coverage CI Workflows (PR #219165)
Tapiwa Gonga via libc-commits
libc-commits at lists.llvm.org
Tue Sep 8 06:09:57 PDT 2026
https://github.com/tapiwagonga updated https://github.com/llvm/llvm-project/pull/219165
>From 59cbaabacd637a7a7c220ecf3ede6eb7b2a3619a Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Tue, 25 Aug 2026 10:51:29 +0000
Subject: [PATCH 01/15] [libc][ci] Add automated code coverage workflows and
analysis tools
---
.github/workflows/libc-full-mcdc.yml | 149 ++++++
.github/workflows/libc-patch-mcdc.yml | 374 +++++++++++++++
libc/utils/coverage/full_report.py | 249 ++++++++++
libc/utils/coverage/patch_report.py | 500 +++++++++++++++++++++
libc/utils/coverage/test_coverage_tools.py | 310 +++++++++++++
5 files changed, 1582 insertions(+)
create mode 100644 .github/workflows/libc-full-mcdc.yml
create mode 100644 .github/workflows/libc-patch-mcdc.yml
create mode 100644 libc/utils/coverage/full_report.py
create mode 100644 libc/utils/coverage/patch_report.py
create mode 100644 libc/utils/coverage/test_coverage_tools.py
diff --git a/.github/workflows/libc-full-mcdc.yml b/.github/workflows/libc-full-mcdc.yml
new file mode 100644
index 0000000000000..091f4cdc65fe9
--- /dev/null
+++ b/.github/workflows/libc-full-mcdc.yml
@@ -0,0 +1,149 @@
+name: Libc Full Codebase MC/DC Coverage
+
+permissions:
+ contents: write
+
+env:
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
+
+on:
+ # Daily overnight run at 02:30 UTC (staggered by 30 mins from baseline sweep)
+ schedule:
+ - cron: '30 2 * * *'
+
+ # Allow manual on-demand execution from GitHub Actions UI ("Run workflow" button)
+ workflow_dispatch:
+
+ # Trigger on pushes to main / libc-mcdc-coverage / post-commit-bot
+ push:
+ branches:
+ - main
+ - libc-mcdc-coverage
+ - post-commit-bot
+ paths:
+ - 'libc/**'
+ - '.github/workflows/libc-full-mcdc.yml'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ full-mcdc-coverage:
+ timeout-minutes: 60
+ name: libc-full-mcdc-coverage
+ runs-on: ubuntu-24.04
+ container:
+ image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
+ options: --privileged
+ defaults:
+ run:
+ shell: bash
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout at v4
+ with:
+ fetch-depth: 1
+ persist-credentials: false
+
+ - name: Setup Compiler Cache (sccache)
+ uses: hendrikmuhs/ccache-action at v1.2.23
+ with:
+ max-size: 1G
+ key: libc_mcdc_coverage_unified_x86_64
+ variant: sccache
+
+ - name: Configure CMake with MC/DC
+ run: |
+ echo "=== [STAGE 1/4] Starting CMake Configuration (MC/DC) at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ START_TIME=$(date +%s)
+
+ export CMAKE_FLAGS="
+ -G Ninja
+ -S runtimes
+ -B build-cov
+ -DCMAKE_C_COMPILER=clang-23
+ -DCMAKE_CXX_COMPILER=clang++-23
+ -DCMAKE_BUILD_TYPE=Debug
+ -DCMAKE_C_COMPILER_LAUNCHER=sccache
+ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+ -DLLVM_USE_LINKER=lld-23
+ -DLLVM_ENABLE_RUNTIMES=libc
+ -DLLVM_LIBC_FULL_BUILD=ON
+ -DLLVM_LIBC_ENABLE_COVERAGE=ON
+ -DLIBC_ENABLE_MCDC=ON
+ -DLIBC_TEST_UNIT_TEST_ONLY=ON
+ -DLIBC_TEST_SKIP_DEATH_TESTS=ON
+ -DLIBC_TEST_SKIP_SHARED_TESTS=ON
+ "
+ cmake $CMAKE_FLAGS
+
+ END_TIME=$(date +%s)
+ echo "=== [STAGE 1/4] CMake Configuration Completed in $((END_TIME - START_TIME))s ==="
+
+ - name: Run Full Codebase Unit Tests
+ run: |
+ echo "=== [STAGE 2/4] Running Full Unit Test Suite at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ START_TIME=$(date +%s)
+
+ export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
+ ninja -k 0 -C build-cov libc-unit-tests || true
+
+ END_TIME=$(date +%s)
+ echo "=== [STAGE 2/4] libc-unit-tests Completed in $((END_TIME - START_TIME))s ==="
+
+ - name: Merge Profiles
+ run: |
+ echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ find build-cov -name "libc_cov_*.profraw" > profraw_list.txt
+ NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
+ echo "[LOG] Discovered $NUM_PROFS raw profile data files."
+ llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
+ echo "[LOG] Merged full codebase profile into libc_full.profdata."
+
+ - name: Generate Reports and Summary
+ env:
+ COMMIT_SHA: ${{ github.sha }}
+ BRANCH_REF: ${{ github.ref_name }}
+ run: |
+ echo "=== [STAGE 4/4] Generating Full Codebase MC/DC Coverage Reports at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
+ OBJECTS=("${EXECUTABLES[@]:1}")
+ OBJECTS=("${OBJECTS[@]/#/-object=}")
+
+ echo "[LOG] Exporting coverage data across ${#EXECUTABLES[@]} test binaries."
+
+ # 1. Generate JSON export for Full Coverage Analyzer (contains mcdc_records)
+ llvm-cov-23 export -format=text -instr-profile=libc_full.profdata "${EXECUTABLES[0]}" "${OBJECTS[@]}" > coverage.json
+
+ # 2. Generate HTML Coverage Report with directory hierarchy, branch tracking, and MC/DC truth tables
+ llvm-cov-23 show -format=html \
+ -output-dir=coverage_mcdc_html \
+ -instr-profile=libc_full.profdata \
+ "${EXECUTABLES[0]}" "${OBJECTS[@]}" \
+ --show-directory-coverage \
+ --show-branches=count \
+ --show-mcdc \
+ --show-mcdc-summary \
+ --compilation-dir=. \
+ --path-equivalence="$GITHUB_WORKSPACE,." \
+ -ignore-filename-regex=".*(test|utils).*"
+ touch coverage_mcdc_html/.nojekyll
+
+ # 3. Run Full Coverage Analyzer
+ python3 libc/utils/coverage/full_report.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
+ echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
+
+ - name: Deploy MC/DC Coverage Report to GitHub Pages
+ if: github.repository != 'llvm/llvm-project'
+ continue-on-error: true
+ uses: peaceiris/actions-gh-pages at v4
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ publish_dir: ./coverage_mcdc_html
+ destination_dir: mcdc
+ keep_files: true
+ enable_jekyll: false
+ force_orphan: false
+
+
diff --git a/.github/workflows/libc-patch-mcdc.yml b/.github/workflows/libc-patch-mcdc.yml
new file mode 100644
index 0000000000000..9be978b41272d
--- /dev/null
+++ b/.github/workflows/libc-patch-mcdc.yml
@@ -0,0 +1,374 @@
+name: Libc Patch MC/DC Coverage
+
+permissions:
+ contents: read
+ pull-requests: write # Required to post/update patch coverage comments
+
+env:
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
+
+on:
+ workflow_dispatch:
+ push:
+ branches:
+ - 'test-*'
+ - 'libc-mcdc-*'
+ - main
+ paths:
+ - 'libc/src/**'
+ - 'libc/include/**'
+ - 'libc/test/**'
+ - 'libc/CMakeLists.txt'
+ - '.github/workflows/libc-patch-mcdc.yml'
+ - '!libc/docs/**'
+ - '!libc/benchmarks/**'
+ - '!libc/fuzzing/**'
+ - '!libc/utils/**'
+ - '!**.md'
+ pull_request:
+ paths:
+ - 'libc/src/**'
+ - 'libc/include/**'
+ - 'libc/test/**'
+ - 'libc/CMakeLists.txt'
+ - '.github/workflows/libc-patch-mcdc.yml'
+ - '!libc/docs/**'
+ - '!libc/benchmarks/**'
+ - '!libc/fuzzing/**'
+ - '!libc/utils/**'
+ - '!**.md'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ pre-commit-mcdc-coverage:
+ timeout-minutes: 60
+ name: libc-pre-commit-mcdc-coverage
+ runs-on: ubuntu-24.04
+ container:
+ image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
+ options: >-
+ --privileged
+ defaults:
+ run:
+ shell: bash
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout at v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ fetch-depth: 2
+ persist-credentials: false
+
+ - name: Setup Compiler Cache (sccache)
+ uses: hendrikmuhs/ccache-action at v1.2.23
+ with:
+ max-size: 1G
+ key: libc_coverage_unified_v2_x86_64
+ variant: sccache
+
+ - name: Configure CMake
+ run: |
+ echo "=== [STAGE 1/4] Starting CMake Configuration at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ START_TIME=$(date +%s)
+
+ export CMAKE_FLAGS="
+ -G Ninja
+ -S runtimes
+ -B build-cov
+ -DCMAKE_C_COMPILER=clang-23
+ -DCMAKE_CXX_COMPILER=clang++-23
+ -DCMAKE_BUILD_TYPE=Debug
+ -DCMAKE_C_COMPILER_LAUNCHER=sccache
+ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+ -DLLVM_USE_LINKER=lld-23
+ -DLLVM_ENABLE_RUNTIMES=libc
+ -DLLVM_LIBC_FULL_BUILD=ON
+ -DLLVM_LIBC_ENABLE_COVERAGE=ON
+ -DLIBC_ENABLE_MCDC=ON
+ -DLIBC_TEST_UNIT_TEST_ONLY=ON
+ -DLIBC_TEST_SKIP_DEATH_TESTS=ON
+ -DLIBC_TEST_SKIP_SHARED_TESTS=ON
+ "
+ cmake $CMAKE_FLAGS
+
+ END_TIME=$(date +%s)
+ echo "=== [STAGE 1/4] CMake Configuration Completed in $((END_TIME - START_TIME))s ==="
+
+ - name: Build and Run Targeted Tests
+ env:
+ EVENT_NAME: ${{ github.event_name }}
+ PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
+ PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+ PR_BASE_REPO: ${{ github.event.pull_request.base.repo.full_name }}
+ PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+ COMMIT_SHA: ${{ github.sha }}
+ BRANCH_REF: ${{ github.ref_name }}
+ REPO_NAME: ${{ github.repository }}
+ run: |
+ echo "=== [STAGE 2/4] Starting Targeted Test Discovery at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ START_TIME=$(date +%s)
+
+ # Ensure upstream llvm-project remote is configured
+ git remote add upstream https://github.com/llvm/llvm-project.git 2>/dev/null || true
+
+ # 1. Deterministically resolve Base and Head references against upstream llvm-project
+ if [ "$EVENT_NAME" == "pull_request" ]; then
+ BASE_SHA="$PR_BASE_SHA"
+ HEAD_SHA="$PR_HEAD_SHA"
+ BASE_REF="$PR_BASE_REF"
+ HEAD_REF="$PR_HEAD_REF"
+ BASE_REPO="${PR_BASE_REPO:-llvm/llvm-project}"
+ HEAD_REPO="${PR_HEAD_REPO:-$REPO_NAME}"
+ git fetch upstream "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_REF" --depth=1 2>/dev/null || true
+ DIFF_BASE="$BASE_SHA"
+ elif [ "$BRANCH_REF" != "main" ]; then
+ # For feature branches, compare against main merge-base
+ git fetch origin main --depth=100 2>/dev/null || true
+ BASE_SHA=$(git merge-base origin/main HEAD 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || echo "HEAD~1")
+ HEAD_SHA="$COMMIT_SHA"
+ BASE_REF="main"
+ HEAD_REF="$BRANCH_REF"
+ BASE_REPO="llvm/llvm-project"
+ HEAD_REPO="$REPO_NAME"
+ DIFF_BASE="$BASE_SHA"
+ else
+ BASE_SHA=$(git rev-parse HEAD~1 2>/dev/null || echo "HEAD~1")
+ HEAD_SHA="$COMMIT_SHA"
+ BASE_REF="main"
+ HEAD_REF="$BRANCH_REF"
+ BASE_REPO="llvm/llvm-project"
+ HEAD_REPO="$REPO_NAME"
+ DIFF_BASE="HEAD~1"
+ fi
+
+ # Persist resolved commit references for all downstream steps
+ echo "DIFF_BASE=$DIFF_BASE" >> $GITHUB_ENV
+ echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV
+ echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_ENV
+ echo "BASE_REF=$BASE_REF" >> $GITHUB_ENV
+ echo "HEAD_REF=$HEAD_REF" >> $GITHUB_ENV
+ echo "BASE_REPO=$BASE_REPO" >> $GITHUB_ENV
+ echo "HEAD_REPO=$HEAD_REPO" >> $GITHUB_ENV
+
+ echo "[LOG] Resolved Base: ${BASE_REF} (${BASE_SHA:0:7}) in ${BASE_REPO}"
+ echo "[LOG] Resolved Head: ${HEAD_REF} (${HEAD_SHA:0:7}) in ${HEAD_REPO}"
+
+ MODIFIED_FILES=$(git diff --name-only "$DIFF_BASE" HEAD -- libc/src/ || true)
+ echo "[LOG] Modified files in libc/src/:"
+ echo "$MODIFIED_FILES"
+
+ if [ -z "$MODIFIED_FILES" ]; then
+ echo "[LOG] No source files modified in libc/src/. Exiting successfully."
+ echo "TARGETS=" >> $GITHUB_ENV
+
+ echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "" >> coverage_report.md
+ echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
+ echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
+ echo "---" >> coverage_report.md
+ echo "" >> coverage_report.md
+ echo "**Coverage Validated**" >> coverage_report.md
+ echo "No \`.cpp\` source files in \`libc/src/\` were modified in this patch." >> coverage_report.md
+
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+ exit 0
+ fi
+
+ # Get all valid ninja targets
+ ninja -C build-cov -t targets > all_targets.txt
+
+ TARGETS=""
+
+ for FILE in $MODIFIED_FILES; do
+ # Skip core support files from direct target mapping (tested transitively)
+ if [[ "$FILE" == *"__support/"* ]]; then
+ echo "[LOG] -> Support file ($FILE) will be tested transitively."
+ continue
+ fi
+
+ if [[ "$FILE" =~ ^libc/src/([^/]+)/(.*/)?([^/]+)\.cpp$ ]]; then
+ DIR="${BASH_REMATCH[1]}"
+ FUNC="${BASH_REMATCH[3]}"
+ ALL_MATCHES=$(grep -oE "^libc\.test\.src\.${DIR}\.([a-zA-Z0-9_]+\.)*${FUNC}_test\.__unit__" all_targets.txt || true)
+ FOUND_TARGET=$(echo "$ALL_MATCHES" | head -n 1 | tr -d '\r\n ' || true)
+ if [ -n "$FOUND_TARGET" ]; then
+ echo "[LOG] -> Matched $FILE -> $FOUND_TARGET"
+ TARGETS="$TARGETS $FOUND_TARGET"
+ else
+ echo "[LOG] -> Notice: No direct unit test target found for $FILE."
+ fi
+ fi
+ done
+
+ # Remove duplicate targets
+ TARGETS=$(echo "$TARGETS" | xargs -n1 | sort -u | xargs || true)
+ echo "[LOG] Executing Ninja targets: $TARGETS"
+
+ if [ -n "$TARGETS" ]; then
+ export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
+ ninja -C build-cov $TARGETS
+ echo "TARGETS=$TARGETS" >> $GITHUB_ENV
+ else
+ echo "[LOG] No standalone unit test targets to run."
+ echo "TARGETS=" >> $GITHUB_ENV
+
+ echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "" >> coverage_report.md
+ echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
+ echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
+ echo "---" >> coverage_report.md
+ echo "" >> coverage_report.md
+ echo "**Coverage Validated**" >> coverage_report.md
+ echo "Modified files do not have standalone unit test targets." >> coverage_report.md
+
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+ fi
+
+ END_TIME=$(date +%s)
+ echo "=== [STAGE 2/4] Targeted Tests Completed in $((END_TIME - START_TIME))s ==="
+
+ - name: Merge Profiles
+ if: env.TARGETS != ''
+ run: |
+ echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ find build-cov -name "libc_cov_*.profraw" > profraw_list.txt
+ NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
+ echo "[LOG] Discovered $NUM_PROFS raw profile data files."
+ if [ -s profraw_list.txt ]; then
+ llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
+ echo "[LOG] Successfully merged profile data into libc_full.profdata."
+ else
+ echo "[LOG] Warning: No profraw files found."
+ fi
+
+ - name: Extract Executables and Generate Summary
+ if: env.TARGETS != ''
+ run: |
+ echo "=== [STAGE 4/4] Generating Coverage Report at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
+ OBJECTS=("${EXECUTABLES[@]:1}")
+ OBJECTS=("${OBJECTS[@]/#/-object=}")
+
+ echo "[LOG] Discovered ${#EXECUTABLES[@]} instrumented binaries for llvm-cov export."
+
+ if [ ! -f libc_full.profdata ]; then
+ echo "[LOG] Notice: libc_full.profdata not found. Generating non-coverage summary."
+ echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "" >> coverage_report.md
+ echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
+ echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
+ echo "---" >> coverage_report.md
+ echo "" >> coverage_report.md
+ echo "**Coverage Validated**" >> coverage_report.md
+ echo "No executable statement profile data was collected for the modified lines in this patch." >> coverage_report.md
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+ exit 0
+ fi
+
+ # 1. Generate JSON report for Python script
+ llvm-cov-23 export -format=text -instr-profile=libc_full.profdata "${EXECUTABLES[0]}" "${OBJECTS[@]}" > coverage.json
+
+ # 2. Generate git diff for modified libc source files
+ git diff "$DIFF_BASE" HEAD -- libc/src/ > patch.diff
+
+ # 3. Run Patch Report Python Script using persisted commit metadata
+ python3 libc/utils/coverage/patch_report.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+ echo "[LOG] Coverage report successfully written to GITHUB_STEP_SUMMARY."
+
+ - name: Fallback Failure Summary
+ if: failure() && !hashFiles('coverage_report.md')
+ run: |
+ echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "" >> coverage_report.md
+ echo "- **Base Branch:** [\`${BASE_REF:-main}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO:-llvm/llvm-project}/commit/${BASE_SHA:-main})" >> coverage_report.md
+ echo "- **Head Commit:** [\`${HEAD_REF:-HEAD}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO:-llvm/llvm-project}/commit/${HEAD_SHA:-HEAD})" >> coverage_report.md
+ echo "---" >> coverage_report.md
+ echo "" >> coverage_report.md
+ echo "**Coverage Validated**" >> coverage_report.md
+ echo "The targeted build or test execution encountered an error before coverage data could be finalized. Inspect the job logs for details." >> coverage_report.md
+
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+
+ - name: Post or Update Sticky PR Comment
+ if: always() && github.event_name == 'pull_request'
+ uses: actions/github-script at v7
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const fs = require('fs');
+ if (!fs.existsSync('coverage_report.md')) return;
+
+ const reportContent = fs.readFileSync('coverage_report.md', 'utf8');
+ const identifier = '<!-- LLVM-LIBC-PRE-COMMIT-MCDC-COMMENT -->';
+ const issue_number = context.payload.pull_request.number;
+ const owner = context.repo.owner;
+ const repo = context.repo.repo;
+
+ try {
+ const comments = await github.rest.issues.listComments({
+ owner,
+ repo,
+ issue_number,
+ per_page: 100
+ });
+
+ const botComment = comments.data.find(c => c.body && c.body.includes(identifier));
+
+ const headSha = context.payload.pull_request.head.sha.substring(0, 7);
+ const dateStr = new Date().toISOString().replace('T', ' ').substring(0, 16);
+
+ const lineMatch = reportContent.match(/\|\s*\*\*Patch Line Coverage\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\|/);
+ const linePct = lineMatch ? lineMatch[1] + '%' : '100.00%';
+
+ const mcdcMatch = reportContent.match(/\|\s*\*\*MC\/DC Condition Independence\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\|/);
+ const mcdcPct = mcdcMatch ? mcdcMatch[1] + '%' : '100.00%';
+
+ const missMatch = reportContent.match(/\|\s*\*\*Unexecuted Lines\*\*\s*\|\s*\*\*([0-9]+)\*\*\s*\|/);
+ const missedLines = missMatch ? missMatch[1] : '0';
+
+ const newHistoryRow = `| [\`${headSha}\`](https://github.com/${owner}/${repo}/commit/${context.payload.pull_request.head.sha}) | ${dateStr} | **${mcdcPct}** | **${linePct}** | ${missedLines} |`;
+
+ let existingHistory = '';
+ if (botComment && botComment.body.includes('<!-- HISTORY_START -->')) {
+ const historyStart = botComment.body.indexOf('<!-- HISTORY_START -->') + '<!-- HISTORY_START -->'.length;
+ const historyEnd = botComment.body.indexOf('<!-- HISTORY_END -->');
+ if (historyStart !== -1 && historyEnd !== -1) {
+ existingHistory = botComment.body.substring(historyStart, historyEnd).trim();
+ }
+ }
+
+ let historyTable = existingHistory;
+ if (!historyTable) {
+ historyTable = `| Commit | Date (UTC) | MC/DC Coverage | Line Coverage | Missed Lines |\n| :---: | :---: | :---: | :---: | :---: |\n${newHistoryRow}`;
+ } else {
+ if (!historyTable.includes(headSha)) {
+ historyTable += `\n${newHistoryRow}`;
+ }
+ }
+
+ const fullBody = `${identifier}\n${reportContent}\n\n<details>\n<summary><b>View Coverage History for this PR</b></summary>\n\n<!-- HISTORY_START -->\n${historyTable}\n<!-- HISTORY_END -->\n\n</details>`;
+
+ if (botComment) {
+ await github.rest.issues.updateComment({
+ owner,
+ repo,
+ comment_id: botComment.id,
+ body: fullBody
+ });
+ } else {
+ await github.rest.issues.createComment({
+ owner,
+ repo,
+ issue_number,
+ body: fullBody
+ });
+ }
+ } catch (err) {
+ core.warning(`Could not post/update sticky PR comment: ${err.message}`);
+ }
diff --git a/libc/utils/coverage/full_report.py b/libc/utils/coverage/full_report.py
new file mode 100644
index 0000000000000..8977b51e85e9e
--- /dev/null
+++ b/libc/utils/coverage/full_report.py
@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+#
+# ====- Generate full codebase coverage reports ----------------*- python -*--==#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ==-------------------------------------------------------------------------==#
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+
+def render_full_report(cov_data: dict) -> None:
+ if "data" not in cov_data or not cov_data["data"]:
+ print("## LLVM-libc Full Codebase Coverage Report\n")
+ print("> [!WARNING]")
+ print("> ### No Coverage Data Detected")
+ print(
+ "> The test execution completed but no coverage profiles were exported."
+ )
+ return
+
+ subsystems: Dict[str, Dict[str, Any]] = {}
+
+ total_lines_cov = 0
+ total_lines_tot = 0
+ total_func_cov = 0
+ total_func_tot = 0
+ total_mcdc_cov = 0
+ total_mcdc_tot = 0
+ total_decisions_count = 0
+ total_decisions_full = 0
+
+ for item in cov_data["data"][0].get("files", []):
+ fpath = item.get("filename", "")
+ if "src/" not in fpath or "/test/" in fpath or "/utils/" in fpath:
+ continue
+
+ idx = fpath.find("src/")
+ if idx == -1:
+ continue
+ rel_path = fpath[idx:]
+
+ summary = item.get("summary", {})
+ lines_summary = summary.get("lines", {})
+ func_summary = summary.get("functions", {})
+ mcdc_summary = summary.get("mcdc", {})
+
+ line_tot = lines_summary.get("count", 0)
+ line_cov = lines_summary.get("covered", 0)
+ func_tot = func_summary.get("count", 0)
+ func_cov = func_summary.get("covered", 0)
+ mcdc_tot = mcdc_summary.get("count", 0)
+ mcdc_cov = mcdc_summary.get("covered", 0)
+
+ mcdc_records = item.get("mcdc_records", [])
+ file_decisions_tot = len(mcdc_records)
+ file_decisions_full = 0
+ for rec in mcdc_records:
+ if len(rec) >= 10 and isinstance(rec[9], list):
+ conds = rec[9]
+ if all(conds):
+ file_decisions_full += 1
+
+ if line_tot == 0:
+ continue
+
+ total_lines_cov += line_cov
+ total_lines_tot += line_tot
+ total_func_cov += func_cov
+ total_func_tot += func_tot
+ total_mcdc_cov += mcdc_cov
+ total_mcdc_tot += mcdc_tot
+ total_decisions_count += file_decisions_tot
+ total_decisions_full += file_decisions_full
+
+ parts = rel_path.split("/")
+ subsystem = "/".join(parts[:2]) if len(parts) >= 2 else parts[0]
+
+ if subsystem not in subsystems:
+ subsystems[subsystem] = {
+ "lines_cov": 0,
+ "lines_tot": 0,
+ "func_cov": 0,
+ "func_tot": 0,
+ "mcdc_cov": 0,
+ "mcdc_tot": 0,
+ "decisions_tot": 0,
+ "decisions_full": 0,
+ }
+
+ subsystems[subsystem]["lines_cov"] += line_cov
+ subsystems[subsystem]["lines_tot"] += line_tot
+ subsystems[subsystem]["func_cov"] += func_cov
+ subsystems[subsystem]["func_tot"] += func_tot
+ subsystems[subsystem]["mcdc_cov"] += mcdc_cov
+ subsystems[subsystem]["mcdc_tot"] += mcdc_tot
+ subsystems[subsystem]["decisions_tot"] += file_decisions_tot
+ subsystems[subsystem]["decisions_full"] += file_decisions_full
+
+ line_pct = (
+ (total_lines_cov / total_lines_tot * 100) if total_lines_tot > 0 else 0
+ )
+ func_pct = (
+ (total_func_cov / total_func_tot * 100) if total_func_tot > 0 else 0
+ )
+ has_mcdc = total_mcdc_tot > 0
+ mcdc_pct = (
+ (total_mcdc_cov / total_mcdc_tot * 100) if total_mcdc_tot > 0 else 0
+ )
+ decisions_pct = (
+ (total_decisions_full / total_decisions_count * 100)
+ if total_decisions_count > 0
+ else 0
+ )
+
+ pages_url = os.environ.get("COVERAGE_DASHBOARD_URL")
+ if not pages_url:
+ repo = os.environ.get("GITHUB_REPOSITORY", "llvm/llvm-project")
+ if "/" in repo:
+ owner, repo_name = repo.split("/", 1)
+ pages_url = f"https://{owner}.github.io/{repo_name}/"
+ else:
+ pages_url = f"https://{repo}.github.io/"
+
+ if has_mcdc and not pages_url.endswith("/mcdc/"):
+ mcdc_pages_url = pages_url.rstrip("/") + "/mcdc/"
+ else:
+ mcdc_pages_url = pages_url
+
+ print("## LLVM-libc Full Codebase Coverage Report\n")
+
+ print("> [!NOTE]")
+ if has_mcdc:
+ print(
+ f"> ### Overall Codebase Coverage: **{line_pct:.2f}% Line** | **{mcdc_pct:.2f}% MC/DC**"
+ )
+ print(
+ f"> Tested **{total_lines_cov:,} / {total_lines_tot:,}** executable lines and **{total_mcdc_cov:,} / {total_mcdc_tot:,}** boolean conditions across **{total_decisions_count:,}** decisions."
+ )
+ print("")
+ print(
+ f"- **Coverage Dashboard:** [{mcdc_pages_url}]({mcdc_pages_url})"
+ )
+ else:
+ print(f"> ### Overall Codebase Coverage: **{line_pct:.2f}%**")
+ print(
+ f"> Tested **{total_lines_cov:,} / {total_lines_tot:,}** executable lines across all LLVM-libc subsystems."
+ )
+ print("")
+ print(f"- **Coverage Dashboard:** [{pages_url}]({pages_url})")
+
+ print("\n---\n")
+
+ print("### Overall")
+ print("| Metric | Covered | Total | Coverage % |")
+ print("| :--- | :---: | :---: | :---: |")
+ if has_mcdc:
+ print(
+ f"| **MC/DC Condition Independence** | {total_mcdc_cov:,} | {total_mcdc_tot:,} | **{mcdc_pct:.2f}%** |"
+ )
+ print(
+ f"| **Fully Verified Decisions** | {total_decisions_full:,} | {total_decisions_count:,} | **{decisions_pct:.2f}%** |"
+ )
+ print(
+ f"| **Executable Lines** | {total_lines_cov:,} | {total_lines_tot:,} | **{line_pct:.2f}%** |"
+ )
+ print(
+ f"| **Functions** | {total_func_cov:,} | {total_func_tot:,} | **{func_pct:.2f}%** |"
+ )
+ print("")
+
+ print("### Coverage Breakdown")
+ if has_mcdc:
+ print(
+ "| Subsystem | MC/DC Conditions | Decisions (Verified / Total) | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
+ )
+ print("| :--- | :---: | :---: | :---: | :---: | :---: | :---: |")
+
+ sorted_subsystems = sorted(subsystems.keys())
+ else:
+ print(
+ "| Subsystem | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
+ )
+ print("| :--- | :---: | :---: | :---: | :---: |")
+ sorted_subsystems = sorted(subsystems.keys())
+
+ for sub in sorted_subsystems:
+ data = subsystems[sub]
+ s_line_pct = (
+ (data["lines_cov"] / data["lines_tot"] * 100)
+ if data["lines_tot"] > 0
+ else 0
+ )
+ s_func_pct = (
+ (data["func_cov"] / data["func_tot"] * 100)
+ if data["func_tot"] > 0
+ else 0
+ )
+ missed_lines = data["lines_tot"] - data["lines_cov"]
+
+ if has_mcdc:
+ s_mc_pct = (
+ (data["mcdc_cov"] / data["mcdc_tot"] * 100)
+ if data["mcdc_tot"] > 0
+ else 0
+ )
+ mc_cell = (
+ f"**{s_mc_pct:.1f}%** ({data['mcdc_cov']}/{data['mcdc_tot']})"
+ if data["mcdc_tot"] > 0
+ else "N/A"
+ )
+ dec_cell = (
+ f"{data['decisions_full']} / {data['decisions_tot']}"
+ if data["decisions_tot"] > 0
+ else "N/A"
+ )
+ print(
+ f"| `libc/{sub}` | {mc_cell} | {dec_cell} | **{s_line_pct:.2f}%** | {s_func_pct:.2f}% | {data['lines_tot']:,} | {missed_lines:,} |"
+ )
+ else:
+ print(
+ f"| `libc/{sub}` | **{s_line_pct:.2f}%** | {s_func_pct:.2f}% | {data['lines_tot']:,} | {missed_lines:,} |"
+ )
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="LLVM-libc Full Coverage Analyzer")
+ parser.add_argument("json_file", help="Path to llvm-cov export JSON file")
+
+ args, _ = parser.parse_known_args()
+
+ try:
+ with open(args.json_file, "r", encoding="utf-8") as f:
+ cov_data = json.load(f)
+ except Exception as e:
+ sys.stderr.write(f"Error: Failed to parse coverage JSON: {e}\n")
+ sys.exit(1)
+ render_full_report(cov_data)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libc/utils/coverage/patch_report.py b/libc/utils/coverage/patch_report.py
new file mode 100644
index 0000000000000..d2da34a469405
--- /dev/null
+++ b/libc/utils/coverage/patch_report.py
@@ -0,0 +1,500 @@
+#!/usr/bin/env python3
+#
+# ====- Generate patch coverage reports ------------------------*- python -*--==#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ==-------------------------------------------------------------------------==#
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+
+class DiffHunk:
+ def __init__(self, header: str):
+ self.header: str = header
+ self.lines: List[Tuple[str, str, int]] = [] # (prefix, text, line_number)
+
+
+class DiffParser:
+ @staticmethod
+ def parse(diff_source: str) -> Dict[str, List[DiffHunk]]:
+ files: Dict[str, List[DiffHunk]] = {}
+ current_file: Optional[str] = None
+ current_hunk: Optional[DiffHunk] = None
+ current_line_num: int = 0
+
+ # Support both file path and raw diff string
+ if os.path.isfile(diff_source):
+ with open(diff_source, "r", encoding="utf-8") as f:
+ lines = f.readlines()
+ else:
+ lines = diff_source.splitlines(keepends=True)
+
+ for line in lines:
+ line = line.rstrip("\n")
+
+ if line.startswith("+++ b/"):
+ current_file = line[6:]
+ files[current_file] = []
+ current_hunk = None
+ continue
+
+ if line.startswith("+++ /dev/null"):
+ current_file = None
+ current_hunk = None
+ continue
+
+ if current_file is None:
+ continue
+
+ if line.startswith("@@"):
+ match = re.search(r"\+([0-9]+)", line)
+ if match:
+ current_line_num = int(match.group(1))
+ current_hunk = DiffHunk(line)
+ files[current_file].append(current_hunk)
+ continue
+
+ if current_hunk is None:
+ continue
+
+ if line.startswith("-"):
+ continue
+ elif line.startswith("+"):
+ current_hunk.lines.append(("+", line[1:], current_line_num))
+ current_line_num += 1
+ elif line.startswith(" "):
+ current_hunk.lines.append((" ", line[1:], current_line_num))
+ current_line_num += 1
+
+ return files
+
+
+class CoverageJSONParser:
+ @staticmethod
+ def load(json_path: str) -> dict:
+ try:
+ with open(json_path, "r", encoding="utf-8") as f:
+ return json.load(f)
+ except Exception as e:
+ sys.stderr.write(f"Error: Failed to parse coverage JSON: {e}\n")
+ sys.exit(1)
+
+ @staticmethod
+ def extract_patch_matrix(
+ cov_data: dict, diff_files: Dict[str, List[DiffHunk]]
+ ) -> Dict[str, Dict[str, Any]]:
+ coverage_matrix: Dict[str, Dict[str, Any]] = {
+ fpath: {"covered": set(), "missed": set(), "mcdc_decisions": []}
+ for fpath in diff_files.keys()
+ }
+
+ if "data" not in cov_data or not cov_data["data"]:
+ return coverage_matrix
+
+ for item in cov_data["data"][0].get("files", []):
+ fpath = item.get("filename", "")
+ rel_path = next(
+ (rp for rp in diff_files.keys() if fpath.endswith(rp)), None
+ )
+ if not rel_path:
+ continue
+
+ # 1. Statement segments
+ segments = item.get("segments", [])
+ for i in range(len(segments) - 1):
+ current = segments[i]
+ nxt = segments[i + 1]
+
+ line_start = current[0]
+ line_end = nxt[0]
+ count = current[2]
+ has_count = current[3]
+
+ if has_count:
+ for line_num in range(line_start, line_end + 1):
+ if count > 0:
+ coverage_matrix[rel_path]["covered"].add(line_num)
+ else:
+ coverage_matrix[rel_path]["missed"].add(line_num)
+
+ # 2. MC/DC decision records (Clang 18+)
+ mcdc_records = item.get("mcdc_records", [])
+ for rec in mcdc_records:
+ if len(rec) >= 10 and isinstance(rec[9], list):
+ l_start = rec[0]
+ l_end = rec[2]
+ conds = rec[9]
+ cov_conds = sum(1 for c in conds if c)
+ coverage_matrix[rel_path]["mcdc_decisions"].append(
+ {
+ "line_start": l_start,
+ "line_end": l_end,
+ "conditions": conds,
+ "covered": cov_conds,
+ "total": len(conds),
+ }
+ )
+
+ return coverage_matrix
+
+
+def is_executable_line(text: str) -> bool:
+ s = text.strip()
+ if not s:
+ return False
+ # Comments
+ if (
+ s.startswith("//")
+ or s.startswith("/*")
+ or s.startswith("*")
+ or s.startswith("*/")
+ ):
+ return False
+ # Structural braces and colons
+ if s in ("{", "}", "};", "{};") or s.startswith(":"):
+ return False
+ # Preprocessor directives
+ if s.startswith("#"):
+ return False
+ # Declarations / keywords / attributes
+ if (
+ s.startswith("namespace ")
+ or s.startswith("extern ")
+ or s.startswith("using ")
+ or s.startswith("__attribute__")
+ or s.startswith("template")
+ or s.startswith("typedef ")
+ or s.startswith("struct ")
+ or s.startswith("class ")
+ or s.startswith("enum ")
+ ):
+ return False
+ return True
+
+
+def format_line_ranges(lines: Set[int]) -> str:
+ if not lines:
+ return "None"
+ sorted_lines = sorted(lines)
+ ranges = []
+ start = sorted_lines[0]
+ end = sorted_lines[0]
+ for n in sorted_lines[1:]:
+ if n == end + 1:
+ end = n
+ else:
+ ranges.append(f"`L{start}-L{end}`" if start != end else f"`L{start}`")
+ start = end = n
+ ranges.append(f"`L{start}-L{end}`" if start != end else f"`L{start}`")
+ return ", ".join(ranges)
+
+
+def render_patch_report(
+ diff_files: Dict[str, List[DiffHunk]],
+ coverage_matrix: Dict[str, Dict[str, Any]],
+ base_sha: Optional[str],
+ head_sha: Optional[str],
+ base_branch: Optional[str],
+ head_branch: Optional[str],
+ targets_str: Optional[str] = None,
+ base_repo: str = "llvm/llvm-project",
+ head_repo: str = "llvm/llvm-project",
+) -> None:
+ total_covered = 0
+ total_missed = 0
+ active_files = {}
+
+ total_mcdc_cov = 0
+ total_mcdc_tot = 0
+ total_decisions_count = 0
+ fully_verified_decisions = 0
+ file_mcdc_data: Dict[str, Tuple[int, int, List[str]]] = {}
+
+ for fpath, data in coverage_matrix.items():
+ added_lines: Set[int] = set()
+ for hunk in diff_files.get(fpath, []):
+ for l_type, text, l_num in hunk.lines:
+ if l_type == "+":
+ if not is_executable_line(text):
+ continue
+ added_lines.add(l_num)
+
+ if not added_lines:
+ continue
+
+ f_covered = added_lines.intersection(data["covered"])
+ f_missed = (added_lines.intersection(data["missed"])) - f_covered
+
+ if len(data["covered"]) > 0 or len(data["missed"]) > 0:
+ total_covered += len(f_covered)
+ total_missed += len(f_missed)
+ active_files[fpath] = (f_covered, f_missed, added_lines)
+ else:
+ total_missed += len(added_lines)
+ active_files[fpath] = (set(), added_lines, added_lines)
+
+ # Evaluate MC/DC decisions on modified lines
+ f_mcdc_cov = 0
+ f_mcdc_tot = 0
+ f_dec_ver = 0
+ f_dec_tot = 0
+ condition_diagnostics = []
+ unverified_decision_lines: Dict[int, List[str]] = {}
+
+ for decision in data.get("mcdc_decisions", []):
+ d_start = decision["line_start"]
+ d_end = decision["line_end"]
+ # Check if any modified line overlaps this decision range
+ if any(d_start <= l <= d_end for l in added_lines):
+ total_decisions_count += 1
+ f_dec_tot += 1
+ f_mcdc_cov += decision["covered"]
+ f_mcdc_tot += decision["total"]
+ if decision["covered"] == decision["total"]:
+ fully_verified_decisions += 1
+ f_dec_ver += 1
+ condition_diagnostics.append(
+ f"`L{d_start}`: {decision['covered']}/{decision['total']} verified"
+ )
+ else:
+ uncovered_idx = [
+ f"C{i+1}"
+ for i, is_cov in enumerate(decision["conditions"])
+ if not is_cov
+ ]
+ unverified_str = ", ".join(uncovered_idx)
+ condition_diagnostics.append(
+ f"`L{d_start}`: {decision['covered']}/{decision['total']} verified ({unverified_str} unverified)"
+ )
+ for l in range(d_start, d_end + 1):
+ if l in added_lines:
+ unverified_decision_lines[l] = uncovered_idx
+
+ if f_mcdc_tot > 0 or f_dec_tot > 0:
+ total_mcdc_cov += f_mcdc_cov
+ total_mcdc_tot += f_mcdc_tot
+ file_mcdc_data[fpath] = (
+ f_mcdc_cov,
+ f_mcdc_tot,
+ condition_diagnostics,
+ f_dec_ver,
+ f_dec_tot,
+ unverified_decision_lines,
+ )
+
+ total_lines = total_covered + total_missed
+ has_mcdc = total_mcdc_tot > 0
+
+ if has_mcdc:
+ print("## LLVM-libc MC/DC Patch Coverage Report\n")
+ else:
+ print("## LLVM-libc Patch Coverage Report\n")
+
+ if total_lines == 0 or not active_files:
+ if base_sha and head_sha and base_branch and head_branch:
+ print(
+ f"- **Base Branch:** [`{base_branch}` ({base_sha[:7]})](https://github.com/{base_repo}/commit/{base_sha})"
+ )
+ print(
+ f"- **Head Commit:** [`{head_branch}` ({head_sha[:7]})](https://github.com/{head_repo}/commit/{head_sha})\n"
+ )
+ print("---\n")
+ print("> [!NOTE]")
+ print("> ### Coverage Validated")
+ print("> No `.cpp` source files in `libc/src/` were modified in this patch.")
+ return
+
+ coverage_percent = (total_covered / total_lines) * 100
+ mcdc_percent = (total_mcdc_cov / total_mcdc_tot * 100) if has_mcdc else 0.0
+
+ if total_missed == 0:
+ if not has_mcdc:
+ print("> [!TIP]")
+ print(f"> ### Patch Coverage: **{coverage_percent:.2f}%**")
+ print(
+ f"> All **{total_lines}** newly added or modified executable lines are covered."
+ )
+ elif total_mcdc_cov == total_mcdc_tot:
+ print("> [!TIP]")
+ print(
+ f"> ### Patch Coverage: **{coverage_percent:.2f}% Line** | **100.00% MC/DC**"
+ )
+ print(
+ f"> All **{total_lines}** executable lines and **{total_mcdc_tot}** boolean conditions across **{total_decisions_count}** decisions are covered."
+ )
+ else:
+ print("> [!NOTE]")
+ print(
+ f"> ### Patch Coverage: **{coverage_percent:.2f}% Line** | **{mcdc_percent:.1f}% MC/DC**"
+ )
+ print(
+ f"> Executed **{total_covered} / {total_lines}** lines. **{total_mcdc_cov} / {total_mcdc_tot}** boolean conditions achieved independence across **{fully_verified_decisions} / {total_decisions_count}** decisions."
+ )
+ else:
+ print("> [!WARNING]")
+ print(
+ f"> ### Patch Coverage: **{coverage_percent:.2f}%** ({total_missed} Missed Lines)"
+ )
+ print(
+ f"> **{total_missed}** unexecuted lines detected in patch."
+ )
+ print("")
+
+ # Commit metadata and targets executed
+ if base_sha and head_sha and base_branch and head_branch:
+ print(
+ f"- **Base Branch:** [`{base_branch}` ({base_sha[:7]})](https://github.com/{base_repo}/commit/{base_sha})"
+ )
+ print(
+ f"- **Head Commit:** [`{head_branch}` ({head_sha[:7]})](https://github.com/{head_repo}/commit/{head_sha})"
+ )
+ if targets_str:
+ targets_formatted = ", ".join(
+ f"`{t.strip()}`" for t in targets_str.split() if t.strip()
+ )
+ print(f"- **Targeted Tests Executed:** {targets_formatted}")
+ print("\n---\n")
+
+ # Unified Coverage Breakdown Table
+ print("### Coverage Breakdown")
+ if has_mcdc:
+ print(
+ "| Modified Source File | Line Coverage | MC/DC Conditions | Decisions (Verified / Total) | Missed Lines | Unverified Conditions |"
+ )
+ print("| :--- | :---: | :---: | :---: | :---: | :--- |")
+ else:
+ print(
+ "| Modified Source File | Patch Coverage | Covered / Total | Missed Lines | Unexecuted Line Spans |"
+ )
+ print("| :--- | :---: | :---: | :---: | :---: |")
+
+ for fpath, (f_covered, f_missed, added_lines) in active_files.items():
+ f_total = len(f_covered) + len(f_missed)
+ f_pct = (len(f_covered) / f_total * 100) if f_total > 0 else 0.0
+ line_spans = format_line_ranges(f_missed)
+ file_link = f"[`{fpath}`](https://github.com/{head_repo}/blob/{head_sha or 'main'}/{fpath})"
+
+ if has_mcdc:
+ f_mc_data = file_mcdc_data.get(fpath, (0, 0, [], 0, 0, {}))
+ f_mc_cov, f_mc_tot, diag_list, f_dec_ver, f_dec_tot = (
+ f_mc_data[0],
+ f_mc_data[1],
+ f_mc_data[2],
+ f_mc_data[3],
+ f_mc_data[4],
+ )
+ if f_mc_tot > 0:
+ f_mc_pct = f_mc_cov / f_mc_tot * 100
+ mcdc_cell = f"**{f_mc_pct:.1f}%** ({f_mc_cov}/{f_mc_tot})"
+ else:
+ mcdc_cell = "N/A"
+
+ dec_cell = (
+ f"**{f_dec_ver} / {f_dec_tot}**" if f_dec_tot > 0 else "N/A"
+ )
+ diag_cell = "<br>".join(diag_list) if diag_list else "None"
+
+ print(
+ f"| {file_link} | **{f_pct:.2f}%** ({len(f_covered)}/{f_total}) | {mcdc_cell} | {dec_cell} | {len(f_missed)} | {diag_cell} |"
+ )
+ else:
+ print(
+ f"| {file_link} | **{f_pct:.2f}%** | {len(f_covered)} / {f_total} | {len(f_missed)} | {line_spans} |"
+ )
+
+ # Summary Row
+ if has_mcdc:
+ total_dec_cell = f"**{fully_verified_decisions} / {total_decisions_count}**"
+ print(
+ f"| **Total (Patch)** | **{coverage_percent:.2f}%** ({total_covered}/{total_lines}) | **{mcdc_percent:.1f}%** ({total_mcdc_cov}/{total_mcdc_tot}) | {total_dec_cell} | **{total_missed}** | - |"
+ )
+ else:
+ print(
+ f"| **Total (Patch)** | **{coverage_percent:.2f}%** | {total_covered} / {total_lines} | **{total_missed}** | - |"
+ )
+ print("")
+
+ # Collapsible Source Map Diff
+ print("<details>")
+ print("<summary><b>View Annotated Patch Diff (Source Map)</b></summary>\n")
+
+ for fpath, (f_covered, f_missed, added_lines) in active_files.items():
+ hunks = diff_files.get(fpath, [])
+ f_mc_data = file_mcdc_data.get(fpath, (0, 0, [], 0, 0, {}))
+ unverified_lines = f_mc_data[5] if len(f_mc_data) > 5 else {}
+
+ print(f"#### `{fpath}`")
+ print("```diff")
+ for hunk in hunks:
+ print(hunk.header)
+ for l_type, text, l_num in hunk.lines:
+ if l_type == "+":
+ if l_num in f_missed:
+ print(f"- {text} // [MISSED]")
+ elif l_num in unverified_lines:
+ unverified_conds = ", ".join(unverified_lines[l_num])
+ print(f"! {text} // [PARTIAL MC/DC: {unverified_conds} unverified]")
+ elif l_num in f_covered:
+ print(f"+ {text}")
+ else:
+ print(f" {text}")
+ elif l_type == " ":
+ print(f" {text}")
+ print("```\n")
+ print("</details>")
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="LLVM-libc Patch Coverage Analyzer")
+ parser.add_argument("diff_file", help="Path to unified diff file")
+ parser.add_argument("json_file", help="Path to llvm-cov export JSON file")
+ parser.add_argument("base_sha", nargs="?", help="Base commit SHA")
+ parser.add_argument("head_sha", nargs="?", help="Head commit SHA")
+ parser.add_argument("base_branch", nargs="?", help="Base branch name")
+ parser.add_argument("head_branch", nargs="?", help="Head branch name")
+ parser.add_argument(
+ "targets", nargs="?", help="Space-separated list of executed test targets"
+ )
+ parser.add_argument(
+ "base_repo",
+ nargs="?",
+ default="llvm/llvm-project",
+ help="Base repository (e.g. llvm/llvm-project)",
+ )
+ parser.add_argument(
+ "head_repo",
+ nargs="?",
+ default="llvm/llvm-project",
+ help="Head repository (e.g. contributor/llvm-project)",
+ )
+
+ args = parser.parse_args()
+
+ diff_files = DiffParser.parse(args.diff_file)
+ cov_data = CoverageJSONParser.load(args.json_file)
+ coverage_matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
+
+ render_patch_report(
+ diff_files,
+ coverage_matrix,
+ args.base_sha,
+ args.head_sha,
+ args.base_branch,
+ args.head_branch,
+ args.targets,
+ args.base_repo,
+ args.head_repo,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libc/utils/coverage/test_coverage_tools.py b/libc/utils/coverage/test_coverage_tools.py
new file mode 100644
index 0000000000000..a69be35cdd29e
--- /dev/null
+++ b/libc/utils/coverage/test_coverage_tools.py
@@ -0,0 +1,310 @@
+#!/usr/bin/env python3
+# ===-- test_coverage_tools.py - Unit tests for coverage utilities --------===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===#
+
+import io
+import json
+import os
+import sys
+import unittest
+from contextlib import redirect_stdout
+from unittest.mock import patch
+
+# Add parent directory to path to import coverage tools
+current_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, current_dir)
+
+from patch_report import (
+ DiffParser,
+ CoverageJSONParser,
+ is_executable_line,
+ format_line_ranges,
+ render_patch_report,
+)
+from full_report import render_full_report
+
+
+class TestDiffParser(unittest.TestCase):
+ def test_parse_unified_diff(self):
+ sample_diff = """diff --git a/libc/src/string/memchr.cpp b/libc/src/string/memchr.cpp
+index 1234567..89abcdef 100644
+--- a/libc/src/string/memchr.cpp
++++ b/libc/src/string/memchr.cpp
+@@ -10,6 +10,8 @@
+ void *memchr(const void *src, int c, size_t n) {
++ const unsigned char *p = (const unsigned char *)src;
++ // Check boundaries
++ if (n == 0) return nullptr;
+ return nullptr;
+ }
+"""
+ parsed = DiffParser.parse(sample_diff)
+ self.assertIn("libc/src/string/memchr.cpp", parsed)
+ hunks = parsed["libc/src/string/memchr.cpp"]
+ self.assertEqual(len(hunks), 1)
+
+ added_lines = [l_num for l_type, _, l_num in hunks[0].lines if l_type == "+"]
+ self.assertEqual(added_lines, [11, 12, 13])
+
+ def test_is_executable_line(self):
+ self.assertTrue(is_executable_line(" int x = 42;"))
+ self.assertTrue(is_executable_line(" return nullptr;"))
+ self.assertTrue(is_executable_line(" if (a > b) {"))
+
+ self.assertFalse(is_executable_line(" // Just a comment"))
+ self.assertFalse(is_executable_line(" /* Block comment */"))
+ self.assertFalse(is_executable_line(" #include <stddef.h>"))
+ self.assertFalse(is_executable_line(" {"))
+ self.assertFalse(is_executable_line(" }"))
+ self.assertFalse(is_executable_line(" "))
+
+
+class TestCoverageJSONParser(unittest.TestCase):
+ def test_extract_patch_matrix_standard(self):
+ sample_cov_data = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/llvm-project/libc/src/string/memchr.cpp",
+ "segments": [
+ [10, 1, 5, True, True, False],
+ [15, 1, 0, True, True, False],
+ [20, 1, 0, False, False, False],
+ ],
+ "summary": {
+ "lines": {"count": 20, "covered": 10, "percent": 50.0},
+ "functions": {"count": 1, "covered": 1, "percent": 100.0},
+ },
+ }
+ ]
+ }
+ ]
+ }
+
+ diff_files = {"libc/src/string/memchr.cpp": []}
+ matrix = CoverageJSONParser.extract_patch_matrix(sample_cov_data, diff_files)
+ self.assertIn("libc/src/string/memchr.cpp", matrix)
+ self.assertIn(10, matrix["libc/src/string/memchr.cpp"]["covered"])
+ self.assertIn(14, matrix["libc/src/string/memchr.cpp"]["covered"])
+ self.assertIn(15, matrix["libc/src/string/memchr.cpp"]["missed"])
+ self.assertIn(19, matrix["libc/src/string/memchr.cpp"]["missed"])
+
+ def test_extract_patch_matrix_with_mcdc(self):
+ sample_mcdc_data = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/llvm-project/libc/src/string/memchr.cpp",
+ "segments": [
+ [10, 1, 5, True, True, False],
+ [20, 1, 0, False, False, False],
+ ],
+ "mcdc_records": [
+ [12, 5, 12, 25, 1, 1, 0, 0, 5, [True, False, False]]
+ ],
+ "summary": {
+ "lines": {"count": 10, "covered": 10, "percent": 100.0},
+ "functions": {"count": 1, "covered": 1, "percent": 100.0},
+ "mcdc": {"count": 3, "covered": 1, "notcovered": 2, "percent": 33.33},
+ },
+ }
+ ]
+ }
+ ]
+ }
+
+ diff_files = {"libc/src/string/memchr.cpp": []}
+ matrix = CoverageJSONParser.extract_patch_matrix(sample_mcdc_data, diff_files)
+
+ decisions = matrix["libc/src/string/memchr.cpp"]["mcdc_decisions"]
+ self.assertEqual(len(decisions), 1)
+ self.assertEqual(decisions[0]["line_start"], 12)
+ self.assertEqual(decisions[0]["total"], 3)
+ self.assertEqual(decisions[0]["covered"], 1)
+
+
+class TestLineRangeFormatter(unittest.TestCase):
+ def test_format_contiguous_and_discrete_ranges(self):
+ self.assertEqual(format_line_ranges(set()), "None")
+ self.assertEqual(format_line_ranges({5}), "`L5`")
+ self.assertEqual(format_line_ranges({10, 11, 12, 13}), "`L10-L13`")
+ self.assertEqual(
+ format_line_ranges({1, 2, 3, 7, 10, 11, 15}),
+ "`L1-L3`, `L7`, `L10-L11`, `L15`",
+ )
+
+
+class TestPatchReportRendering(unittest.TestCase):
+ def test_render_linear_pass(self):
+ diff_text = """diff --git a/libc/src/string/memchr.cpp b/libc/src/string/memchr.cpp
+--- a/libc/src/string/memchr.cpp
++++ b/libc/src/string/memchr.cpp
+@@ -10,2 +10,2 @@
++ int a = 1;
++ int b = 2;
+"""
+ diff_files = DiffParser.parse(diff_text)
+ coverage_matrix = {
+ "libc/src/string/memchr.cpp": {
+ "covered": {10, 11},
+ "missed": set(),
+ "mcdc_decisions": [],
+ }
+ }
+
+ f = io.StringIO()
+ with redirect_stdout(f):
+ render_patch_report(
+ diff_files,
+ coverage_matrix,
+ base_sha="abc1234567",
+ head_sha="def8901234",
+ base_branch="main",
+ head_branch="patch-1",
+ targets_str="libc.test.src.string.memchr_test.__unit__",
+ )
+ output = f.getvalue()
+
+ self.assertIn("## LLVM-libc Patch Coverage Report", output)
+ self.assertIn("100.00%", output)
+
+ def test_render_full_mcdc(self):
+ diff_text = """diff --git a/libc/src/string/memchr.cpp b/libc/src/string/memchr.cpp
+--- a/libc/src/string/memchr.cpp
++++ b/libc/src/string/memchr.cpp
+@@ -10,2 +10,2 @@
++ if (a && b) {
++ return nullptr;
+"""
+ diff_files = DiffParser.parse(diff_text)
+ coverage_matrix = {
+ "libc/src/string/memchr.cpp": {
+ "covered": {10, 11},
+ "missed": set(),
+ "mcdc_decisions": [
+ {
+ "line_start": 10,
+ "line_end": 10,
+ "conditions": [True, True],
+ "covered": 2,
+ "total": 2,
+ }
+ ],
+ }
+ }
+
+ f = io.StringIO()
+ with redirect_stdout(f):
+ render_patch_report(
+ diff_files,
+ coverage_matrix,
+ base_sha="abc1234567",
+ head_sha="def8901234",
+ base_branch="main",
+ head_branch="patch-1",
+ targets_str="libc.test.src.string.memchr_test.__unit__",
+ )
+ output = f.getvalue()
+
+ self.assertIn("## LLVM-libc MC/DC Patch Coverage Report", output)
+ self.assertIn("100.00% MC/DC", output)
+ self.assertIn("Decisions (Verified / Total)", output)
+
+ def test_render_partial_mcdc(self):
+ diff_text = """diff --git a/libc/src/string/memchr.cpp b/libc/src/string/memchr.cpp
+--- a/libc/src/string/memchr.cpp
++++ b/libc/src/string/memchr.cpp
+@@ -10,2 +10,2 @@
++ if (a && (b || c)) {
++ return nullptr;
+"""
+ diff_files = DiffParser.parse(diff_text)
+ coverage_matrix = {
+ "libc/src/string/memchr.cpp": {
+ "covered": {10, 11},
+ "missed": set(),
+ "mcdc_decisions": [
+ {
+ "line_start": 10,
+ "line_end": 10,
+ "conditions": [True, False, False],
+ "covered": 1,
+ "total": 3,
+ }
+ ],
+ }
+ }
+
+ f = io.StringIO()
+ with redirect_stdout(f):
+ render_patch_report(
+ diff_files,
+ coverage_matrix,
+ base_sha="abc1234567",
+ head_sha="def8901234",
+ base_branch="main",
+ head_branch="patch-1",
+ targets_str="libc.test.src.string.memchr_test.__unit__",
+ )
+ output = f.getvalue()
+
+ self.assertIn("33.3% MC/DC", output)
+ self.assertIn("C2, C3 unverified", output)
+
+
+class TestFullReportRendering(unittest.TestCase):
+ @patch.dict(os.environ, {"GITHUB_REPOSITORY": "llvm/llvm-project"}, clear=True)
+ def test_render_full_report_streamlined(self):
+ cov_data = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/root/llvm-project/libc/src/string/memchr.cpp",
+ "mcdc_records": [
+ [10, 1, 10, 20, 1, 1, 0, 0, 5, [True, True]],
+ ],
+ "summary": {
+ "lines": {"count": 10, "covered": 10, "percent": 100.0},
+ "functions": {"count": 1, "covered": 1, "percent": 100.0},
+ "mcdc": {"count": 2, "covered": 2, "notcovered": 0, "percent": 100.0},
+ },
+ },
+ {
+ "filename": "/root/llvm-project/libc/src/math/sin.cpp",
+ "mcdc_records": [
+ [20, 1, 20, 30, 1, 1, 0, 0, 5, [False, False]],
+ ],
+ "summary": {
+ "lines": {"count": 20, "covered": 15, "percent": 75.0},
+ "functions": {"count": 1, "covered": 1, "percent": 100.0},
+ "mcdc": {"count": 2, "covered": 0, "notcovered": 2, "percent": 0.0},
+ },
+ }
+ ]
+ }
+ ]
+ }
+
+ f = io.StringIO()
+ with redirect_stdout(f):
+ render_full_report(cov_data)
+ output = f.getvalue()
+
+ self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
+ self.assertIn("Overall", output)
+ self.assertIn("Coverage Breakdown", output)
+ self.assertNotIn("Status", output)
+ self.assertNotIn("Health", output)
+ self.assertNotIn("Safety Priority", output)
+
+
+if __name__ == "__main__":
+ unittest.main()
>From 738d7b3db50f0cd143485c55907ece358395c188 Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Wed, 26 Aug 2026 13:36:25 +0000
Subject: [PATCH 02/15] [libc][ci] Add full and patch coverage bots and remove
developer test suite
---
.github/workflows/libc-full-coverage.yml | 143 ++++++++
.github/workflows/libc-patch-coverage.yml | 366 +++++++++++++++++++++
libc/utils/coverage/test_coverage_tools.py | 310 -----------------
3 files changed, 509 insertions(+), 310 deletions(-)
create mode 100644 .github/workflows/libc-full-coverage.yml
create mode 100644 .github/workflows/libc-patch-coverage.yml
delete mode 100644 libc/utils/coverage/test_coverage_tools.py
diff --git a/.github/workflows/libc-full-coverage.yml b/.github/workflows/libc-full-coverage.yml
new file mode 100644
index 0000000000000..3d7c2a9651efb
--- /dev/null
+++ b/.github/workflows/libc-full-coverage.yml
@@ -0,0 +1,143 @@
+name: Libc Full Codebase Coverage
+
+permissions:
+ contents: write # Required to publish live HTML dashboard to gh-pages branch
+
+env:
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
+
+on:
+ # Daily overnight run at 02:00 UTC
+ schedule:
+ - cron: '0 2 * * *'
+
+ # Allow manual on-demand execution from GitHub Actions UI
+ workflow_dispatch:
+
+ # Trigger on pushes to main / libc-coverage-ci-bots / post-commit-bot
+ push:
+ branches:
+ - main
+ - libc-coverage-ci-bots
+ - post-commit-bot
+ paths:
+ - 'libc/**'
+ - '.github/workflows/libc-full-coverage.yml'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ full-coverage:
+ timeout-minutes: 60
+ name: libc-full-coverage
+ runs-on: ubuntu-24.04
+ container:
+ image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
+ options: --privileged
+ defaults:
+ run:
+ shell: bash
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout at v4
+ with:
+ fetch-depth: 1
+ persist-credentials: false
+
+ - name: Setup Compiler Cache (sccache)
+ uses: hendrikmuhs/ccache-action at v1.2.23
+ with:
+ max-size: 1G
+ key: libc_coverage_unified_v2_x86_64
+ variant: sccache
+
+ - name: Configure CMake
+ run: |
+ echo "=== [STAGE 1/4] Starting CMake Configuration at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ START_TIME=$(date +%s)
+
+ export CMAKE_FLAGS="
+ -G Ninja
+ -S runtimes
+ -B build-cov
+ -DCMAKE_C_COMPILER=clang-23
+ -DCMAKE_CXX_COMPILER=clang++-23
+ -DCMAKE_BUILD_TYPE=Debug
+ -DCMAKE_C_COMPILER_LAUNCHER=sccache
+ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+ -DLLVM_USE_LINKER=lld-23
+ -DLLVM_ENABLE_RUNTIMES=libc
+ -DLLVM_LIBC_FULL_BUILD=ON
+ -DLLVM_LIBC_ENABLE_COVERAGE=ON
+ -DLIBC_TEST_UNIT_TEST_ONLY=ON
+ -DLIBC_TEST_SKIP_DEATH_TESTS=ON
+ -DLIBC_TEST_SKIP_SHARED_TESTS=ON
+ "
+ cmake $CMAKE_FLAGS
+
+ END_TIME=$(date +%s)
+ echo "=== [STAGE 1/4] CMake Configuration Completed in $((END_TIME - START_TIME))s ==="
+
+ - name: Run Full Codebase Unit Tests
+ run: |
+ echo "=== [STAGE 2/4] Running Full Unit Test Suite at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ START_TIME=$(date +%s)
+
+ export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
+ ninja -k 0 -C build-cov libc-unit-tests || true
+
+ END_TIME=$(date +%s)
+ echo "=== [STAGE 2/4] libc-unit-tests Completed in $((END_TIME - START_TIME))s ==="
+
+ - name: Merge Profiles
+ run: |
+ echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ find . build-cov -name "libc_cov_*.profraw" > profraw_list.txt
+ NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
+ echo "[LOG] Discovered $NUM_PROFS raw profile data files."
+ llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
+ echo "[LOG] Merged full codebase profile into libc_full.profdata."
+
+ - name: Generate Reports and Summary
+ env:
+ COMMIT_SHA: ${{ github.sha }}
+ BRANCH_REF: ${{ github.ref_name }}
+ run: |
+ echo "=== [STAGE 4/4] Generating Full Codebase Coverage Reports at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
+ OBJECTS=("${EXECUTABLES[@]:1}")
+ OBJECTS=("${OBJECTS[@]/#/-object=}")
+
+ echo "[LOG] Exporting coverage data across ${#EXECUTABLES[@]} test binaries."
+
+ # 1. Generate JSON export for Full Coverage Analyzer
+ llvm-cov-23 export -format=text -instr-profile=libc_full.profdata "${EXECUTABLES[0]}" "${OBJECTS[@]}" > coverage.json
+
+ # 2. Generate HTML Coverage Report with directory hierarchy and branch tracking
+ llvm-cov-23 show -format=html \
+ -output-dir=coverage_html \
+ -instr-profile=libc_full.profdata \
+ "${EXECUTABLES[0]}" "${OBJECTS[@]}" \
+ --show-directory-coverage \
+ --show-branches=count \
+ --compilation-dir=. \
+ --path-equivalence="$GITHUB_WORKSPACE,." \
+ -ignore-filename-regex=".*(test|utils).*"
+ touch coverage_html/.nojekyll
+
+ # 3. Run Full Coverage Analyzer
+ python3 libc/utils/coverage/full_report.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
+ echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
+
+ - name: Deploy Coverage Dashboard to GitHub Pages
+ if: github.repository != 'llvm/llvm-project'
+ continue-on-error: true
+ uses: peaceiris/actions-gh-pages at v4
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ publish_dir: ./coverage_html
+ keep_files: true
+ enable_jekyll: false
+ force_orphan: false
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
new file mode 100644
index 0000000000000..a81dbbbc3ea7b
--- /dev/null
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -0,0 +1,366 @@
+name: Libc Patch Code Coverage
+
+permissions:
+ contents: read
+ pull-requests: write # Required to post/update patch coverage comments
+
+env:
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
+
+on:
+ workflow_dispatch:
+ push:
+ branches:
+ - pre-commit-bot
+ - libc-coverage-ci-bots
+ - main
+ paths:
+ - 'libc/src/**'
+ - 'libc/include/**'
+ - 'libc/test/**'
+ - 'libc/CMakeLists.txt'
+ - '.github/workflows/libc-patch-coverage.yml'
+ - '!libc/docs/**'
+ - '!libc/benchmarks/**'
+ - '!libc/fuzzing/**'
+ - '!libc/utils/**'
+ - '!**.md'
+ pull_request:
+ paths:
+ - 'libc/src/**'
+ - 'libc/include/**'
+ - 'libc/test/**'
+ - 'libc/CMakeLists.txt'
+ - '.github/workflows/libc-patch-coverage.yml'
+ - '!libc/docs/**'
+ - '!libc/benchmarks/**'
+ - '!libc/fuzzing/**'
+ - '!libc/utils/**'
+ - '!**.md'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ pre-commit-coverage:
+ timeout-minutes: 60
+ name: libc-pre-commit-coverage
+ runs-on: ubuntu-24.04
+ container:
+ image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
+ options: >-
+ --privileged
+ defaults:
+ run:
+ shell: bash
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout at v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ fetch-depth: 2
+ persist-credentials: false
+
+ - name: Setup Compiler Cache (sccache)
+ uses: hendrikmuhs/ccache-action at v1.2.23
+ with:
+ max-size: 1G
+ key: libc_coverage_unified_v2_x86_64
+ variant: sccache
+
+ - name: Configure CMake
+ run: |
+ echo "=== [STAGE 1/4] Starting CMake Configuration at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ START_TIME=$(date +%s)
+
+ export CMAKE_FLAGS="
+ -G Ninja
+ -S runtimes
+ -B build-cov
+ -DCMAKE_C_COMPILER=clang-23
+ -DCMAKE_CXX_COMPILER=clang++-23
+ -DCMAKE_BUILD_TYPE=Debug
+ -DCMAKE_C_COMPILER_LAUNCHER=sccache
+ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+ -DLLVM_USE_LINKER=lld-23
+ -DLLVM_ENABLE_RUNTIMES=libc
+ -DLLVM_LIBC_FULL_BUILD=ON
+ -DLLVM_LIBC_ENABLE_COVERAGE=ON
+ -DLIBC_TEST_UNIT_TEST_ONLY=ON
+ -DLIBC_TEST_SKIP_DEATH_TESTS=ON
+ -DLIBC_TEST_SKIP_SHARED_TESTS=ON
+ "
+ cmake $CMAKE_FLAGS
+
+ END_TIME=$(date +%s)
+ echo "=== [STAGE 1/4] CMake Configuration Completed in $((END_TIME - START_TIME))s ==="
+
+ - name: Build and Run Targeted Tests
+ env:
+ EVENT_NAME: ${{ github.event_name }}
+ PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
+ PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+ PR_BASE_REPO: ${{ github.event.pull_request.base.repo.full_name }}
+ PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+ COMMIT_SHA: ${{ github.sha }}
+ BRANCH_REF: ${{ github.ref_name }}
+ REPO_NAME: ${{ github.repository }}
+ run: |
+ echo "=== [STAGE 2/4] Starting Targeted Test Discovery at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ START_TIME=$(date +%s)
+
+ # Ensure upstream llvm-project remote is configured
+ git remote add upstream https://github.com/llvm/llvm-project.git 2>/dev/null || true
+
+ # 1. Deterministically resolve Base and Head references against upstream llvm-project
+ if [ "$EVENT_NAME" == "pull_request" ]; then
+ BASE_SHA="$PR_BASE_SHA"
+ HEAD_SHA="$PR_HEAD_SHA"
+ BASE_REF="$PR_BASE_REF"
+ HEAD_REF="$PR_HEAD_REF"
+ BASE_REPO="${PR_BASE_REPO:-llvm/llvm-project}"
+ HEAD_REPO="${PR_HEAD_REPO:-$REPO_NAME}"
+ git fetch upstream "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_REF" --depth=1 2>/dev/null || true
+ DIFF_BASE="$BASE_SHA"
+ else
+ # For direct pushes, compare the pushed commit against its parent (HEAD~1)
+ BASE_SHA=$(git rev-parse HEAD~1 2>/dev/null || echo "HEAD~1")
+ HEAD_SHA="$COMMIT_SHA"
+ BASE_REF="main"
+ HEAD_REF="$BRANCH_REF"
+ BASE_REPO="llvm/llvm-project"
+ HEAD_REPO="$REPO_NAME"
+ DIFF_BASE="HEAD~1"
+ fi
+
+ # Persist resolved commit references for all downstream steps
+ echo "DIFF_BASE=$DIFF_BASE" >> $GITHUB_ENV
+ echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV
+ echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_ENV
+ echo "BASE_REF=$BASE_REF" >> $GITHUB_ENV
+ echo "HEAD_REF=$HEAD_REF" >> $GITHUB_ENV
+ echo "BASE_REPO=$BASE_REPO" >> $GITHUB_ENV
+ echo "HEAD_REPO=$HEAD_REPO" >> $GITHUB_ENV
+
+ echo "[LOG] Resolved Base: ${BASE_REF} (${BASE_SHA:0:7}) in ${BASE_REPO}"
+ echo "[LOG] Resolved Head: ${HEAD_REF} (${HEAD_SHA:0:7}) in ${HEAD_REPO}"
+
+ MODIFIED_FILES=$(git diff --name-only "$DIFF_BASE" HEAD -- libc/src/ || true)
+ echo "[LOG] Modified files in libc/src/:"
+ echo "$MODIFIED_FILES"
+
+ if [ -z "$MODIFIED_FILES" ]; then
+ echo "[LOG] No source files modified in libc/src/. Exiting successfully."
+ echo "TARGETS=" >> $GITHUB_ENV
+
+ echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "" >> coverage_report.md
+ echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
+ echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
+ echo "---" >> coverage_report.md
+ echo "" >> coverage_report.md
+ echo "**Coverage Validated**" >> coverage_report.md
+ echo "No \`.cpp\` source files in \`libc/src/\` were modified in this patch." >> coverage_report.md
+
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+ exit 0
+ fi
+
+ # Get all valid ninja targets
+ ninja -C build-cov -t targets > all_targets.txt
+
+ TARGETS=""
+
+ for FILE in $MODIFIED_FILES; do
+ # Skip core support files from direct target mapping (tested transitively)
+ if [[ "$FILE" == *"__support/"* ]]; then
+ echo "[LOG] -> Support file ($FILE) will be tested transitively."
+ continue
+ fi
+
+ if [[ "$FILE" =~ ^libc/src/([^/]+)/(.*/)?([^/]+)\.cpp$ ]]; then
+ DIR="${BASH_REMATCH[1]}"
+ FUNC="${BASH_REMATCH[3]}"
+ ALL_MATCHES=$(grep -oE "^libc\.test\.src\.${DIR}\.([a-zA-Z0-9_]+\.)*${FUNC}_test\.__unit__" all_targets.txt || true)
+ FOUND_TARGET=$(echo "$ALL_MATCHES" | head -n 1 | tr -d '\r\n ' || true)
+ if [ -n "$FOUND_TARGET" ]; then
+ echo "[LOG] -> Matched $FILE -> $FOUND_TARGET"
+ TARGETS="$TARGETS $FOUND_TARGET"
+ else
+ echo "[LOG] -> Notice: No direct unit test target found for $FILE."
+ fi
+ fi
+ done
+
+ # Remove duplicate targets
+ TARGETS=$(echo "$TARGETS" | xargs -n1 | sort -u | xargs || true)
+ echo "[LOG] Executing Ninja targets: $TARGETS"
+
+ if [ -n "$TARGETS" ]; then
+ export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
+ ninja -C build-cov $TARGETS
+ echo "TARGETS=$TARGETS" >> $GITHUB_ENV
+ else
+ echo "[LOG] No standalone unit test targets to run."
+ echo "TARGETS=" >> $GITHUB_ENV
+
+ echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "" >> coverage_report.md
+ echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
+ echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
+ echo "---" >> coverage_report.md
+ echo "" >> coverage_report.md
+ echo "**Coverage Validated**" >> coverage_report.md
+ echo "Modified files do not have standalone unit test targets." >> coverage_report.md
+
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+ fi
+
+ END_TIME=$(date +%s)
+ echo "=== [STAGE 2/4] Targeted Tests Completed in $((END_TIME - START_TIME))s ==="
+
+ - name: Merge Profiles
+ if: env.TARGETS != ''
+ run: |
+ echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ find build-cov -name "libc_cov_*.profraw" > profraw_list.txt
+ NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
+ echo "[LOG] Discovered $NUM_PROFS raw profile data files."
+ if [ -s profraw_list.txt ]; then
+ llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
+ echo "[LOG] Successfully merged profile data into libc_full.profdata."
+ else
+ echo "[LOG] Warning: No profraw files found."
+ fi
+
+ - name: Extract Executables and Generate Summary
+ if: env.TARGETS != ''
+ run: |
+ echo "=== [STAGE 4/4] Generating Coverage Report at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
+ EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
+ OBJECTS=("${EXECUTABLES[@]:1}")
+ OBJECTS=("${OBJECTS[@]/#/-object=}")
+
+ echo "[LOG] Discovered ${#EXECUTABLES[@]} instrumented binaries for llvm-cov export."
+
+ if [ ! -f libc_full.profdata ]; then
+ echo "[LOG] Notice: libc_full.profdata not found. Generating non-coverage summary."
+ echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "" >> coverage_report.md
+ echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
+ echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
+ echo "---" >> coverage_report.md
+ echo "" >> coverage_report.md
+ echo "**Coverage Validated**" >> coverage_report.md
+ echo "No executable statement profile data was collected for the modified lines in this patch." >> coverage_report.md
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+ exit 0
+ fi
+
+ # 1. Generate JSON report for Python script
+ llvm-cov-23 export -format=text -instr-profile=libc_full.profdata "${EXECUTABLES[0]}" "${OBJECTS[@]}" > coverage.json
+
+ # 2. Generate git diff for modified libc source files
+ git diff "$DIFF_BASE" HEAD -- libc/src/ > patch.diff
+
+ # 3. Run Patch Report Python Script using persisted commit metadata
+ python3 libc/utils/coverage/patch_report.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+ echo "[LOG] Coverage report successfully written to GITHUB_STEP_SUMMARY."
+
+ - name: Fallback Failure Summary
+ if: failure() && !hashFiles('coverage_report.md')
+ run: |
+ echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "" >> coverage_report.md
+ echo "- **Base Branch:** [\`${BASE_REF:-main}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO:-llvm/llvm-project}/commit/${BASE_SHA:-main})" >> coverage_report.md
+ echo "- **Head Commit:** [\`${HEAD_REF:-HEAD}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO:-llvm/llvm-project}/commit/${HEAD_SHA:-HEAD})" >> coverage_report.md
+ echo "---" >> coverage_report.md
+ echo "" >> coverage_report.md
+ echo "**Coverage Validated**" >> coverage_report.md
+ echo "The targeted build or test execution encountered an error before coverage data could be finalized. Inspect the job logs for details." >> coverage_report.md
+
+ cat coverage_report.md >> $GITHUB_STEP_SUMMARY
+
+ - name: Post or Update Sticky PR Comment
+ if: always() && github.event_name == 'pull_request'
+ uses: actions/github-script at v7
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const fs = require('fs');
+ if (!fs.existsSync('coverage_report.md')) return;
+
+ const reportContent = fs.readFileSync('coverage_report.md', 'utf8');
+ const identifier = '<!-- LLVM-LIBC-PRE-COMMIT-COVERAGE-COMMENT -->';
+ const issue_number = context.payload.pull_request.number;
+ const owner = context.repo.owner;
+ const repo = context.repo.repo;
+
+ try {
+ // Fetch existing PR comments
+ const comments = await github.rest.issues.listComments({
+ owner,
+ repo,
+ issue_number,
+ per_page: 100
+ });
+
+ const botComment = comments.data.find(c => c.body && c.body.includes(identifier));
+
+ // Parse metrics from reportContent for history table entry
+ const headSha = context.payload.pull_request.head.sha.substring(0, 7);
+ const dateStr = new Date().toISOString().replace('T', ' ').substring(0, 16);
+
+ const pctMatch = reportContent.match(/\|\s*\*\*Patch Line Coverage\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\|/);
+ const pct = pctMatch ? pctMatch[1] + '%' : '100.00%';
+
+ const lineMatch = reportContent.match(/\|\s*\*\*Executable Lines Evaluated\*\*\s*\|\s*\*\*([0-9]+)\*\*\s*\|/);
+ const totalLines = lineMatch ? lineMatch[1] : 'N/A';
+
+ const missMatch = reportContent.match(/\|\s*\*\*Unexecuted Lines\*\*\s*\|\s*\*\*([0-9]+)\*\*\s*\|/);
+ const missedLines = missMatch ? missMatch[1] : '0';
+
+ const newHistoryRow = `| [\`${headSha}\`](https://github.com/${owner}/${repo}/commit/${context.payload.pull_request.head.sha}) | ${dateStr} | ${totalLines} | **${pct}** | ${missedLines} |`;
+
+ let existingHistory = '';
+ if (botComment && botComment.body.includes('<!-- HISTORY_START -->')) {
+ const historyStart = botComment.body.indexOf('<!-- HISTORY_START -->') + '<!-- HISTORY_START -->'.length;
+ const historyEnd = botComment.body.indexOf('<!-- HISTORY_END -->');
+ if (historyStart !== -1 && historyEnd !== -1) {
+ existingHistory = botComment.body.substring(historyStart, historyEnd).trim();
+ }
+ }
+
+ let historyTable = existingHistory;
+ if (!historyTable) {
+ historyTable = `| Commit | Date (UTC) | Executable Lines | Patch Coverage | Missed Lines |\n| :---: | :---: | :---: | :---: | :---: |\n${newHistoryRow}`;
+ } else {
+ if (!historyTable.includes(headSha)) {
+ historyTable += `\n${newHistoryRow}`;
+ }
+ }
+
+ const fullBody = `${identifier}\n${reportContent}\n\n<details>\n<summary><b>View Coverage History for this PR</b></summary>\n\n<!-- HISTORY_START -->\n${historyTable}\n<!-- HISTORY_END -->\n\n</details>`;
+
+ if (botComment) {
+ await github.rest.issues.updateComment({
+ owner,
+ repo,
+ comment_id: botComment.id,
+ body: fullBody
+ });
+ } else {
+ await github.rest.issues.createComment({
+ owner,
+ repo,
+ issue_number,
+ body: fullBody
+ });
+ }
+ } catch (err) {
+ core.warning(`Could not post/update sticky PR comment: ${err.message}`);
+ }
diff --git a/libc/utils/coverage/test_coverage_tools.py b/libc/utils/coverage/test_coverage_tools.py
deleted file mode 100644
index a69be35cdd29e..0000000000000
--- a/libc/utils/coverage/test_coverage_tools.py
+++ /dev/null
@@ -1,310 +0,0 @@
-#!/usr/bin/env python3
-# ===-- test_coverage_tools.py - Unit tests for coverage utilities --------===#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# ===----------------------------------------------------------------------===#
-
-import io
-import json
-import os
-import sys
-import unittest
-from contextlib import redirect_stdout
-from unittest.mock import patch
-
-# Add parent directory to path to import coverage tools
-current_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, current_dir)
-
-from patch_report import (
- DiffParser,
- CoverageJSONParser,
- is_executable_line,
- format_line_ranges,
- render_patch_report,
-)
-from full_report import render_full_report
-
-
-class TestDiffParser(unittest.TestCase):
- def test_parse_unified_diff(self):
- sample_diff = """diff --git a/libc/src/string/memchr.cpp b/libc/src/string/memchr.cpp
-index 1234567..89abcdef 100644
---- a/libc/src/string/memchr.cpp
-+++ b/libc/src/string/memchr.cpp
-@@ -10,6 +10,8 @@
- void *memchr(const void *src, int c, size_t n) {
-+ const unsigned char *p = (const unsigned char *)src;
-+ // Check boundaries
-+ if (n == 0) return nullptr;
- return nullptr;
- }
-"""
- parsed = DiffParser.parse(sample_diff)
- self.assertIn("libc/src/string/memchr.cpp", parsed)
- hunks = parsed["libc/src/string/memchr.cpp"]
- self.assertEqual(len(hunks), 1)
-
- added_lines = [l_num for l_type, _, l_num in hunks[0].lines if l_type == "+"]
- self.assertEqual(added_lines, [11, 12, 13])
-
- def test_is_executable_line(self):
- self.assertTrue(is_executable_line(" int x = 42;"))
- self.assertTrue(is_executable_line(" return nullptr;"))
- self.assertTrue(is_executable_line(" if (a > b) {"))
-
- self.assertFalse(is_executable_line(" // Just a comment"))
- self.assertFalse(is_executable_line(" /* Block comment */"))
- self.assertFalse(is_executable_line(" #include <stddef.h>"))
- self.assertFalse(is_executable_line(" {"))
- self.assertFalse(is_executable_line(" }"))
- self.assertFalse(is_executable_line(" "))
-
-
-class TestCoverageJSONParser(unittest.TestCase):
- def test_extract_patch_matrix_standard(self):
- sample_cov_data = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/llvm-project/libc/src/string/memchr.cpp",
- "segments": [
- [10, 1, 5, True, True, False],
- [15, 1, 0, True, True, False],
- [20, 1, 0, False, False, False],
- ],
- "summary": {
- "lines": {"count": 20, "covered": 10, "percent": 50.0},
- "functions": {"count": 1, "covered": 1, "percent": 100.0},
- },
- }
- ]
- }
- ]
- }
-
- diff_files = {"libc/src/string/memchr.cpp": []}
- matrix = CoverageJSONParser.extract_patch_matrix(sample_cov_data, diff_files)
- self.assertIn("libc/src/string/memchr.cpp", matrix)
- self.assertIn(10, matrix["libc/src/string/memchr.cpp"]["covered"])
- self.assertIn(14, matrix["libc/src/string/memchr.cpp"]["covered"])
- self.assertIn(15, matrix["libc/src/string/memchr.cpp"]["missed"])
- self.assertIn(19, matrix["libc/src/string/memchr.cpp"]["missed"])
-
- def test_extract_patch_matrix_with_mcdc(self):
- sample_mcdc_data = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/llvm-project/libc/src/string/memchr.cpp",
- "segments": [
- [10, 1, 5, True, True, False],
- [20, 1, 0, False, False, False],
- ],
- "mcdc_records": [
- [12, 5, 12, 25, 1, 1, 0, 0, 5, [True, False, False]]
- ],
- "summary": {
- "lines": {"count": 10, "covered": 10, "percent": 100.0},
- "functions": {"count": 1, "covered": 1, "percent": 100.0},
- "mcdc": {"count": 3, "covered": 1, "notcovered": 2, "percent": 33.33},
- },
- }
- ]
- }
- ]
- }
-
- diff_files = {"libc/src/string/memchr.cpp": []}
- matrix = CoverageJSONParser.extract_patch_matrix(sample_mcdc_data, diff_files)
-
- decisions = matrix["libc/src/string/memchr.cpp"]["mcdc_decisions"]
- self.assertEqual(len(decisions), 1)
- self.assertEqual(decisions[0]["line_start"], 12)
- self.assertEqual(decisions[0]["total"], 3)
- self.assertEqual(decisions[0]["covered"], 1)
-
-
-class TestLineRangeFormatter(unittest.TestCase):
- def test_format_contiguous_and_discrete_ranges(self):
- self.assertEqual(format_line_ranges(set()), "None")
- self.assertEqual(format_line_ranges({5}), "`L5`")
- self.assertEqual(format_line_ranges({10, 11, 12, 13}), "`L10-L13`")
- self.assertEqual(
- format_line_ranges({1, 2, 3, 7, 10, 11, 15}),
- "`L1-L3`, `L7`, `L10-L11`, `L15`",
- )
-
-
-class TestPatchReportRendering(unittest.TestCase):
- def test_render_linear_pass(self):
- diff_text = """diff --git a/libc/src/string/memchr.cpp b/libc/src/string/memchr.cpp
---- a/libc/src/string/memchr.cpp
-+++ b/libc/src/string/memchr.cpp
-@@ -10,2 +10,2 @@
-+ int a = 1;
-+ int b = 2;
-"""
- diff_files = DiffParser.parse(diff_text)
- coverage_matrix = {
- "libc/src/string/memchr.cpp": {
- "covered": {10, 11},
- "missed": set(),
- "mcdc_decisions": [],
- }
- }
-
- f = io.StringIO()
- with redirect_stdout(f):
- render_patch_report(
- diff_files,
- coverage_matrix,
- base_sha="abc1234567",
- head_sha="def8901234",
- base_branch="main",
- head_branch="patch-1",
- targets_str="libc.test.src.string.memchr_test.__unit__",
- )
- output = f.getvalue()
-
- self.assertIn("## LLVM-libc Patch Coverage Report", output)
- self.assertIn("100.00%", output)
-
- def test_render_full_mcdc(self):
- diff_text = """diff --git a/libc/src/string/memchr.cpp b/libc/src/string/memchr.cpp
---- a/libc/src/string/memchr.cpp
-+++ b/libc/src/string/memchr.cpp
-@@ -10,2 +10,2 @@
-+ if (a && b) {
-+ return nullptr;
-"""
- diff_files = DiffParser.parse(diff_text)
- coverage_matrix = {
- "libc/src/string/memchr.cpp": {
- "covered": {10, 11},
- "missed": set(),
- "mcdc_decisions": [
- {
- "line_start": 10,
- "line_end": 10,
- "conditions": [True, True],
- "covered": 2,
- "total": 2,
- }
- ],
- }
- }
-
- f = io.StringIO()
- with redirect_stdout(f):
- render_patch_report(
- diff_files,
- coverage_matrix,
- base_sha="abc1234567",
- head_sha="def8901234",
- base_branch="main",
- head_branch="patch-1",
- targets_str="libc.test.src.string.memchr_test.__unit__",
- )
- output = f.getvalue()
-
- self.assertIn("## LLVM-libc MC/DC Patch Coverage Report", output)
- self.assertIn("100.00% MC/DC", output)
- self.assertIn("Decisions (Verified / Total)", output)
-
- def test_render_partial_mcdc(self):
- diff_text = """diff --git a/libc/src/string/memchr.cpp b/libc/src/string/memchr.cpp
---- a/libc/src/string/memchr.cpp
-+++ b/libc/src/string/memchr.cpp
-@@ -10,2 +10,2 @@
-+ if (a && (b || c)) {
-+ return nullptr;
-"""
- diff_files = DiffParser.parse(diff_text)
- coverage_matrix = {
- "libc/src/string/memchr.cpp": {
- "covered": {10, 11},
- "missed": set(),
- "mcdc_decisions": [
- {
- "line_start": 10,
- "line_end": 10,
- "conditions": [True, False, False],
- "covered": 1,
- "total": 3,
- }
- ],
- }
- }
-
- f = io.StringIO()
- with redirect_stdout(f):
- render_patch_report(
- diff_files,
- coverage_matrix,
- base_sha="abc1234567",
- head_sha="def8901234",
- base_branch="main",
- head_branch="patch-1",
- targets_str="libc.test.src.string.memchr_test.__unit__",
- )
- output = f.getvalue()
-
- self.assertIn("33.3% MC/DC", output)
- self.assertIn("C2, C3 unverified", output)
-
-
-class TestFullReportRendering(unittest.TestCase):
- @patch.dict(os.environ, {"GITHUB_REPOSITORY": "llvm/llvm-project"}, clear=True)
- def test_render_full_report_streamlined(self):
- cov_data = {
- "data": [
- {
- "files": [
- {
- "filename": "/root/llvm-project/libc/src/string/memchr.cpp",
- "mcdc_records": [
- [10, 1, 10, 20, 1, 1, 0, 0, 5, [True, True]],
- ],
- "summary": {
- "lines": {"count": 10, "covered": 10, "percent": 100.0},
- "functions": {"count": 1, "covered": 1, "percent": 100.0},
- "mcdc": {"count": 2, "covered": 2, "notcovered": 0, "percent": 100.0},
- },
- },
- {
- "filename": "/root/llvm-project/libc/src/math/sin.cpp",
- "mcdc_records": [
- [20, 1, 20, 30, 1, 1, 0, 0, 5, [False, False]],
- ],
- "summary": {
- "lines": {"count": 20, "covered": 15, "percent": 75.0},
- "functions": {"count": 1, "covered": 1, "percent": 100.0},
- "mcdc": {"count": 2, "covered": 0, "notcovered": 2, "percent": 0.0},
- },
- }
- ]
- }
- ]
- }
-
- f = io.StringIO()
- with redirect_stdout(f):
- render_full_report(cov_data)
- output = f.getvalue()
-
- self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
- self.assertIn("Overall", output)
- self.assertIn("Coverage Breakdown", output)
- self.assertNotIn("Status", output)
- self.assertNotIn("Health", output)
- self.assertNotIn("Safety Priority", output)
-
-
-if __name__ == "__main__":
- unittest.main()
>From 0c30ed291c046b1ef80541e7dc521ba885875c67 Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Wed, 26 Aug 2026 17:56:08 +0000
Subject: [PATCH 03/15] [libc][ci] Add automated coverage bots with modular
reporting utilities and unit tests
---
.github/workflows/libc-full-coverage.yml | 6 +-
.github/workflows/libc-full-mcdc.yml | 6 +-
.github/workflows/libc-patch-coverage.yml | 18 +-
.github/workflows/libc-patch-mcdc.yml | 18 +-
libc/utils/coverage/codebase_coverage.py | 329 +++++++++
libc/utils/coverage/diff_coverage.py | 643 ++++++++++++++++++
libc/utils/coverage/full_report.py | 249 -------
libc/utils/coverage/patch_report.py | 500 --------------
libc/utils/coverage/test_codebase_coverage.py | 236 +++++++
libc/utils/coverage/test_diff_coverage.py | 451 ++++++++++++
10 files changed, 1679 insertions(+), 777 deletions(-)
create mode 100644 libc/utils/coverage/codebase_coverage.py
create mode 100644 libc/utils/coverage/diff_coverage.py
delete mode 100644 libc/utils/coverage/full_report.py
delete mode 100644 libc/utils/coverage/patch_report.py
create mode 100644 libc/utils/coverage/test_codebase_coverage.py
create mode 100644 libc/utils/coverage/test_diff_coverage.py
diff --git a/.github/workflows/libc-full-coverage.yml b/.github/workflows/libc-full-coverage.yml
index 3d7c2a9651efb..bded8dbe28d5d 100644
--- a/.github/workflows/libc-full-coverage.yml
+++ b/.github/workflows/libc-full-coverage.yml
@@ -70,7 +70,7 @@ jobs:
-DLLVM_USE_LINKER=lld-23
-DLLVM_ENABLE_RUNTIMES=libc
-DLLVM_LIBC_FULL_BUILD=ON
- -DLLVM_LIBC_ENABLE_COVERAGE=ON
+ -DLIBC_ENABLE_COVERAGE=ON
-DLIBC_TEST_UNIT_TEST_ONLY=ON
-DLIBC_TEST_SKIP_DEATH_TESTS=ON
-DLIBC_TEST_SKIP_SHARED_TESTS=ON
@@ -127,8 +127,8 @@ jobs:
-ignore-filename-regex=".*(test|utils).*"
touch coverage_html/.nojekyll
- # 3. Run Full Coverage Analyzer
- python3 libc/utils/coverage/full_report.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
+ # 3. Run Codebase Coverage Analyzer
+ python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
- name: Deploy Coverage Dashboard to GitHub Pages
diff --git a/.github/workflows/libc-full-mcdc.yml b/.github/workflows/libc-full-mcdc.yml
index 091f4cdc65fe9..8aec845f0f301 100644
--- a/.github/workflows/libc-full-mcdc.yml
+++ b/.github/workflows/libc-full-mcdc.yml
@@ -70,7 +70,7 @@ jobs:
-DLLVM_USE_LINKER=lld-23
-DLLVM_ENABLE_RUNTIMES=libc
-DLLVM_LIBC_FULL_BUILD=ON
- -DLLVM_LIBC_ENABLE_COVERAGE=ON
+ -DLIBC_ENABLE_COVERAGE=ON
-DLIBC_ENABLE_MCDC=ON
-DLIBC_TEST_UNIT_TEST_ONLY=ON
-DLIBC_TEST_SKIP_DEATH_TESTS=ON
@@ -130,8 +130,8 @@ jobs:
-ignore-filename-regex=".*(test|utils).*"
touch coverage_mcdc_html/.nojekyll
- # 3. Run Full Coverage Analyzer
- python3 libc/utils/coverage/full_report.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
+ # 3. Run Codebase Coverage Analyzer
+ python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
- name: Deploy MC/DC Coverage Report to GitHub Pages
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
index a81dbbbc3ea7b..db5257c8a4797 100644
--- a/.github/workflows/libc-patch-coverage.yml
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -86,7 +86,7 @@ jobs:
-DLLVM_USE_LINKER=lld-23
-DLLVM_ENABLE_RUNTIMES=libc
-DLLVM_LIBC_FULL_BUILD=ON
- -DLLVM_LIBC_ENABLE_COVERAGE=ON
+ -DLIBC_ENABLE_COVERAGE=ON
-DLIBC_TEST_UNIT_TEST_ONLY=ON
-DLIBC_TEST_SKIP_DEATH_TESTS=ON
-DLIBC_TEST_SKIP_SHARED_TESTS=ON
@@ -266,8 +266,8 @@ jobs:
# 2. Generate git diff for modified libc source files
git diff "$DIFF_BASE" HEAD -- libc/src/ > patch.diff
- # 3. Run Patch Report Python Script using persisted commit metadata
- python3 libc/utils/coverage/patch_report.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
+ # 3. Run Diff Coverage Python Script using persisted commit metadata
+ python3 libc/utils/coverage/diff_coverage.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
echo "[LOG] Coverage report successfully written to GITHUB_STEP_SUMMARY."
@@ -315,14 +315,10 @@ jobs:
const headSha = context.payload.pull_request.head.sha.substring(0, 7);
const dateStr = new Date().toISOString().replace('T', ' ').substring(0, 16);
- const pctMatch = reportContent.match(/\|\s*\*\*Patch Line Coverage\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\|/);
- const pct = pctMatch ? pctMatch[1] + '%' : '100.00%';
-
- const lineMatch = reportContent.match(/\|\s*\*\*Executable Lines Evaluated\*\*\s*\|\s*\*\*([0-9]+)\*\*\s*\|/);
- const totalLines = lineMatch ? lineMatch[1] : 'N/A';
-
- const missMatch = reportContent.match(/\|\s*\*\*Unexecuted Lines\*\*\s*\|\s*\*\*([0-9]+)\*\*\s*\|/);
- const missedLines = missMatch ? missMatch[1] : '0';
+ const summaryMatch = reportContent.match(/\|\s*\*\*Total \(Patch\)\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\|\s*([0-9]+)\s*\/\s*([0-9]+)\s*\|\s*\*\*([0-9]+)\*\*/);
+ const pct = summaryMatch ? summaryMatch[1] + '%' : '100.00%';
+ const totalLines = summaryMatch ? summaryMatch[3] : 'N/A';
+ const missedLines = summaryMatch ? summaryMatch[4] : '0';
const newHistoryRow = `| [\`${headSha}\`](https://github.com/${owner}/${repo}/commit/${context.payload.pull_request.head.sha}) | ${dateStr} | ${totalLines} | **${pct}** | ${missedLines} |`;
diff --git a/.github/workflows/libc-patch-mcdc.yml b/.github/workflows/libc-patch-mcdc.yml
index 9be978b41272d..22e91fcda8902 100644
--- a/.github/workflows/libc-patch-mcdc.yml
+++ b/.github/workflows/libc-patch-mcdc.yml
@@ -86,7 +86,7 @@ jobs:
-DLLVM_USE_LINKER=lld-23
-DLLVM_ENABLE_RUNTIMES=libc
-DLLVM_LIBC_FULL_BUILD=ON
- -DLLVM_LIBC_ENABLE_COVERAGE=ON
+ -DLIBC_ENABLE_COVERAGE=ON
-DLIBC_ENABLE_MCDC=ON
-DLIBC_TEST_UNIT_TEST_ONLY=ON
-DLIBC_TEST_SKIP_DEATH_TESTS=ON
@@ -276,8 +276,8 @@ jobs:
# 2. Generate git diff for modified libc source files
git diff "$DIFF_BASE" HEAD -- libc/src/ > patch.diff
- # 3. Run Patch Report Python Script using persisted commit metadata
- python3 libc/utils/coverage/patch_report.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
+ # 3. Run Diff Coverage Python Script using persisted commit metadata
+ python3 libc/utils/coverage/diff_coverage.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
echo "[LOG] Coverage report successfully written to GITHUB_STEP_SUMMARY."
@@ -323,14 +323,10 @@ jobs:
const headSha = context.payload.pull_request.head.sha.substring(0, 7);
const dateStr = new Date().toISOString().replace('T', ' ').substring(0, 16);
- const lineMatch = reportContent.match(/\|\s*\*\*Patch Line Coverage\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\|/);
- const linePct = lineMatch ? lineMatch[1] + '%' : '100.00%';
-
- const mcdcMatch = reportContent.match(/\|\s*\*\*MC\/DC Condition Independence\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\|/);
- const mcdcPct = mcdcMatch ? mcdcMatch[1] + '%' : '100.00%';
-
- const missMatch = reportContent.match(/\|\s*\*\*Unexecuted Lines\*\*\s*\|\s*\*\*([0-9]+)\*\*\s*\|/);
- const missedLines = missMatch ? missMatch[1] : '0';
+ const summaryMatch = reportContent.match(/\|\s*\*\*Total \(Patch\)\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\(([0-9]+)\/([0-9]+)\)\s*\|\s*\*\*([0-9.]+)%\*\*\s*\(([0-9]+)\/([0-9]+)\)\s*\|.*?\|\s*\*\*([0-9]+)\*\*/);
+ const linePct = summaryMatch ? summaryMatch[1] + '%' : '100.00%';
+ const mcdcPct = summaryMatch ? summaryMatch[4] + '%' : '100.00%';
+ const missedLines = summaryMatch ? summaryMatch[7] : '0';
const newHistoryRow = `| [\`${headSha}\`](https://github.com/${owner}/${repo}/commit/${context.payload.pull_request.head.sha}) | ${dateStr} | **${mcdcPct}** | **${linePct}** | ${missedLines} |`;
diff --git a/libc/utils/coverage/codebase_coverage.py b/libc/utils/coverage/codebase_coverage.py
new file mode 100644
index 0000000000000..cb7b82f531145
--- /dev/null
+++ b/libc/utils/coverage/codebase_coverage.py
@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+#
+# ====- Generate codebase coverage reports ---------------------*- python -*--==#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ==-------------------------------------------------------------------------==#
+
+"""
+Standalone analyzer for generating whole-codebase statement, branch, and MC/DC coverage reports.
+
+This script parses full-codebase `llvm-cov export` JSON files, aggregates metrics
+across all top-level LLVM-libc subsystems (e.g. `src/ctype`, `src/math`, `src/string`),
+and outputs Markdown summary tables for CI step summaries.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+# -----------------------------------------------------------------------------
+# Constants & Configuration
+# -----------------------------------------------------------------------------
+
+DEFAULT_REPOSITORY = "llvm/llvm-project"
+
+
+# -----------------------------------------------------------------------------
+# Data Models
+# -----------------------------------------------------------------------------
+
+ at dataclass
+class SubsystemCoverageMetrics:
+ """Encapsulates coverage metrics and boolean decision counts for a subsystem or whole codebase."""
+
+ name: str = ""
+ lines_cov: int = 0
+ lines_tot: int = 0
+ func_cov: int = 0
+ func_tot: int = 0
+ mcdc_cov: int = 0
+ mcdc_tot: int = 0
+ decisions_tot: int = 0
+ decisions_full: int = 0
+
+ @property
+ def line_pct(self) -> float:
+ """Percentage of executed statements."""
+ return (self.lines_cov / self.lines_tot * 100.0) if self.lines_tot > 0 else 0.0
+
+ @property
+ def func_pct(self) -> float:
+ """Percentage of executed functions."""
+ return (self.func_cov / self.func_tot * 100.0) if self.func_tot > 0 else 0.0
+
+ @property
+ def mcdc_pct(self) -> float:
+ """Percentage of evaluated independent boolean conditions."""
+ return (self.mcdc_cov / self.mcdc_tot * 100.0) if self.mcdc_tot > 0 else 0.0
+
+ @property
+ def decisions_pct(self) -> float:
+ """Percentage of fully verified boolean decisions."""
+ return (self.decisions_full / self.decisions_tot * 100.0) if self.decisions_tot > 0 else 0.0
+
+ @property
+ def missed_lines(self) -> int:
+ """Count of unexecuted lines."""
+ return max(0, self.lines_tot - self.lines_cov)
+
+
+ at dataclass
+class FullCoverageSummary:
+ """Encapsulates global and subsystem-level coverage statistics across LLVM-libc."""
+
+ global_stats: SubsystemCoverageMetrics = field(default_factory=SubsystemCoverageMetrics)
+ subsystems: Dict[str, SubsystemCoverageMetrics] = field(default_factory=dict)
+ dashboard_url: str = ""
+
+ @property
+ def has_mcdc(self) -> bool:
+ """Returns True if any MC/DC condition data exists in the summary."""
+ return self.global_stats.mcdc_tot > 0
+
+
+# -----------------------------------------------------------------------------
+# Data Extraction & Aggregation
+# -----------------------------------------------------------------------------
+
+def resolve_dashboard_url(has_mcdc: bool) -> str:
+ """Resolves the live dashboard URL based on repository environment variables."""
+ pages_url = os.environ.get("COVERAGE_DASHBOARD_URL")
+ if not pages_url:
+ repo = os.environ.get("GITHUB_REPOSITORY", DEFAULT_REPOSITORY)
+ if "/" in repo:
+ owner, repo_name = repo.split("/", 1)
+ pages_url = f"https://{owner}.github.io/{repo_name}/"
+ else:
+ pages_url = f"https://{repo}.github.io/"
+
+ if has_mcdc and not pages_url.endswith("/mcdc/"):
+ return pages_url.rstrip("/") + "/mcdc/"
+ return pages_url
+
+
+def extract_full_coverage_statistics(cov_data: dict) -> Optional[FullCoverageSummary]:
+ """Extracts global and per-subsystem metrics from llvm-cov export JSON data."""
+ if "data" not in cov_data or not cov_data["data"]:
+ return None
+
+ global_m = SubsystemCoverageMetrics(name="global")
+ subsystems: Dict[str, SubsystemCoverageMetrics] = {}
+
+ for item in cov_data["data"][0].get("files", []):
+ fpath = item.get("filename", "")
+ if "src/" not in fpath or "/test/" in fpath or "/utils/" in fpath:
+ continue
+
+ idx = fpath.find("src/")
+ if idx == -1:
+ continue
+ rel_path = fpath[idx:]
+
+ summary = item.get("summary", {})
+ lines_summary = summary.get("lines", {})
+ func_summary = summary.get("functions", {})
+ mcdc_summary = summary.get("mcdc", {})
+
+ line_tot = lines_summary.get("count", 0)
+ line_cov = lines_summary.get("covered", 0)
+ func_tot = func_summary.get("count", 0)
+ func_cov = func_summary.get("covered", 0)
+ mcdc_tot = mcdc_summary.get("count", 0)
+ mcdc_cov = mcdc_summary.get("covered", 0)
+
+ if line_tot == 0:
+ continue
+
+ mcdc_records = item.get("mcdc_records", [])
+ file_decisions_tot = len(mcdc_records)
+ file_decisions_full = sum(
+ 1 for rec in mcdc_records if len(rec) >= 10 and isinstance(rec[9], list) and all(rec[9])
+ )
+
+ global_m.lines_cov += line_cov
+ global_m.lines_tot += line_tot
+ global_m.func_cov += func_cov
+ global_m.func_tot += func_tot
+ global_m.mcdc_cov += mcdc_cov
+ global_m.mcdc_tot += mcdc_tot
+ global_m.decisions_tot += file_decisions_tot
+ global_m.decisions_full += file_decisions_full
+
+ parts = rel_path.split("/")
+ subsystem_name = "/".join(parts[:2]) if len(parts) >= 2 else parts[0]
+
+ if subsystem_name not in subsystems:
+ subsystems[subsystem_name] = SubsystemCoverageMetrics(name=subsystem_name)
+
+ sub_m = subsystems[subsystem_name]
+ sub_m.lines_cov += line_cov
+ sub_m.lines_tot += line_tot
+ sub_m.func_cov += func_cov
+ sub_m.func_tot += func_tot
+ sub_m.mcdc_cov += mcdc_cov
+ sub_m.mcdc_tot += mcdc_tot
+ sub_m.decisions_tot += file_decisions_tot
+ sub_m.decisions_full += file_decisions_full
+
+ if global_m.lines_tot == 0:
+ return None
+
+ has_mcdc = global_m.mcdc_tot > 0
+ dashboard_url = resolve_dashboard_url(has_mcdc)
+
+ return FullCoverageSummary(
+ global_stats=global_m,
+ subsystems=subsystems,
+ dashboard_url=dashboard_url,
+ )
+
+
+# -----------------------------------------------------------------------------
+# Report Formatting
+# -----------------------------------------------------------------------------
+
+def format_overview_callout(summary: FullCoverageSummary) -> str:
+ """Generates the executive summary banner with dashboard link."""
+ g = summary.global_stats
+ lines: List[str] = ["> [!NOTE]"]
+
+ if summary.has_mcdc:
+ lines.append(
+ f"> ### Overall Codebase Coverage: **{g.line_pct:.2f}% Line** | **{g.mcdc_pct:.2f}% MC/DC**"
+ )
+ lines.append(
+ f"> Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines and **{g.mcdc_cov:,} / {g.mcdc_tot:,}** boolean conditions across **{g.decisions_tot:,}** decisions."
+ )
+ else:
+ lines.append(f"> ### Overall Codebase Coverage: **{g.line_pct:.2f}%**")
+ lines.append(
+ f"> Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines across all LLVM-libc subsystems."
+ )
+
+ lines.append("")
+ lines.append(f"- **Coverage Dashboard:** [{summary.dashboard_url}]({summary.dashboard_url})")
+ return "\n".join(lines)
+
+
+def format_global_summary_table(summary: FullCoverageSummary) -> str:
+ """Generates the top-level metric summary table."""
+ g = summary.global_stats
+ lines: List[str] = [
+ "### Overall",
+ "| Metric | Covered | Total | Coverage % |",
+ "| :--- | :---: | :---: | :---: |",
+ ]
+
+ if summary.has_mcdc:
+ lines.append(
+ f"| **MC/DC Condition Independence** | {g.mcdc_cov:,} | {g.mcdc_tot:,} | **{g.mcdc_pct:.2f}%** |"
+ )
+ lines.append(
+ f"| **Fully Verified Decisions** | {g.decisions_full:,} | {g.decisions_tot:,} | **{g.decisions_pct:.2f}%** |"
+ )
+
+ lines.append(
+ f"| **Executable Lines** | {g.lines_cov:,} | {g.lines_tot:,} | **{g.line_pct:.2f}%** |"
+ )
+ lines.append(
+ f"| **Functions** | {g.func_cov:,} | {g.func_tot:,} | **{g.func_pct:.2f}%** |"
+ )
+ return "\n".join(lines)
+
+
+def format_subsystem_breakdown_table(summary: FullCoverageSummary) -> str:
+ """Generates the subsystem breakdown table."""
+ lines: List[str] = ["### Coverage Breakdown"]
+ has_mcdc = summary.has_mcdc
+
+ if has_mcdc:
+ lines.append(
+ "| Subsystem | MC/DC Conditions | Decisions (Verified / Total) | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
+ )
+ lines.append("| :--- | :---: | :---: | :---: | :---: | :---: | :---: |")
+ else:
+ lines.append(
+ "| Subsystem | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
+ )
+ lines.append("| :--- | :---: | :---: | :---: | :---: |")
+
+ for sub_name in sorted(summary.subsystems.keys()):
+ data = summary.subsystems[sub_name]
+ if has_mcdc:
+ mc_cell = (
+ f"**{data.mcdc_pct:.1f}%** ({data.mcdc_cov}/{data.mcdc_tot})"
+ if data.mcdc_tot > 0
+ else "N/A"
+ )
+ dec_cell = (
+ f"{data.decisions_full} / {data.decisions_tot}"
+ if data.decisions_tot > 0
+ else "N/A"
+ )
+ lines.append(
+ f"| `libc/{sub_name}` | {mc_cell} | {dec_cell} | **{data.line_pct:.2f}%** | {data.func_pct:.2f}% | {data.lines_tot:,} | {data.missed_lines:,} |"
+ )
+ else:
+ lines.append(
+ f"| `libc/{sub_name}` | **{data.line_pct:.2f}%** | {data.func_pct:.2f}% | {data.lines_tot:,} | {data.missed_lines:,} |"
+ )
+
+ return "\n".join(lines)
+
+
+def render_full_report(cov_data: dict) -> None:
+ """Orchestrates extraction and renders the full Markdown report to stdout."""
+ summary = extract_full_coverage_statistics(cov_data)
+
+ print("## LLVM-libc Full Codebase Coverage Report\n")
+
+ if not summary:
+ print("> [!WARNING]")
+ print("> ### No Coverage Data Detected")
+ print("> The test execution completed but no coverage profiles were exported.")
+ return
+
+ # 1. Executive Callout Banner
+ print(format_overview_callout(summary))
+ print("\n---\n")
+
+ # 2. Global Metric Summary Table
+ print(format_global_summary_table(summary))
+ print("")
+
+ # 3. Subsystem Breakdown Table
+ print(format_subsystem_breakdown_table(summary))
+
+
+# -----------------------------------------------------------------------------
+# CLI Entry Point
+# -----------------------------------------------------------------------------
+
+def main() -> None:
+ """Parses command-line arguments and triggers report generation."""
+ parser = argparse.ArgumentParser(description="LLVM-libc Codebase Coverage Analyzer")
+ parser.add_argument("json_file", help="Path to llvm-cov export JSON file")
+
+ args, _ = parser.parse_known_args()
+
+ try:
+ with open(args.json_file, "r", encoding="utf-8") as f:
+ cov_data = json.load(f)
+ except Exception as err:
+ sys.stderr.write(f"Error: Failed to parse coverage JSON from '{args.json_file}': {err}\n")
+ sys.exit(1)
+
+ render_full_report(cov_data)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libc/utils/coverage/diff_coverage.py b/libc/utils/coverage/diff_coverage.py
new file mode 100644
index 0000000000000..7c95f90761a22
--- /dev/null
+++ b/libc/utils/coverage/diff_coverage.py
@@ -0,0 +1,643 @@
+#!/usr/bin/env python3
+#
+# ====- Generate diff coverage reports -------------------------*- python -*--==#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ==-------------------------------------------------------------------------==#
+
+"""
+Standalone analyzer for evaluating diff-level statement, branch, and MC/DC coverage.
+
+This script parses unified git diffs alongside `llvm-cov export` JSON summaries,
+correlates added/modified lines with execution counts and boolean decision records,
+and outputs formatted Markdown reports for CI job summaries and PR comments.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+# -----------------------------------------------------------------------------
+# Constants & Configuration
+# -----------------------------------------------------------------------------
+
+DEFAULT_BASE_REPO = "llvm/llvm-project"
+DEFAULT_HEAD_REPO = "llvm/llvm-project"
+
+COMMENT_PREFIXES = ("//", "/*", "*", "*/")
+STRUCTURAL_TOKENS = ("{", "}", "};", "{};")
+DECLARATION_PREFIXES = (
+ "namespace ",
+ "extern ",
+ "using ",
+ "__attribute__",
+ "template",
+ "typedef ",
+)
+
+
+# -----------------------------------------------------------------------------
+# Data Models
+# -----------------------------------------------------------------------------
+
+ at dataclass
+class DiffHunk:
+ """Represents a unified diff hunk with its header and line tokens."""
+
+ header: str
+ lines: List[Tuple[str, str, int]] = field(default_factory=list) # (prefix, text, line_number)
+
+
+ at dataclass
+class FilePatchMetrics:
+ """Encapsulates coverage metrics and decision records for a single modified file."""
+
+ fpath: str
+ covered_lines: Set[int] = field(default_factory=set)
+ missed_lines: Set[int] = field(default_factory=set)
+ added_lines: Set[int] = field(default_factory=set)
+ mcdc_cov: int = 0
+ mcdc_tot: int = 0
+ decisions_ver: int = 0
+ decisions_tot: int = 0
+ condition_diagnostics: List[str] = field(default_factory=list)
+ unverified_decision_lines: Dict[int, List[str]] = field(default_factory=dict)
+
+ @property
+ def total_lines(self) -> int:
+ """Total executable lines evaluated in this file."""
+ return len(self.covered_lines) + len(self.missed_lines)
+
+ @property
+ def line_coverage_pct(self) -> float:
+ """Percentage of executed patch lines."""
+ return (len(self.covered_lines) / self.total_lines * 100.0) if self.total_lines > 0 else 0.0
+
+ @property
+ def mcdc_coverage_pct(self) -> float:
+ """Percentage of independent boolean conditions evaluated."""
+ return (self.mcdc_cov / self.mcdc_tot * 100.0) if self.mcdc_tot > 0 else 0.0
+
+
+ at dataclass
+class PatchCoverageSummary:
+ """Aggregated coverage statistics across all modified files in the patch."""
+
+ files: Dict[str, FilePatchMetrics] = field(default_factory=dict)
+ total_covered_lines: int = 0
+ total_missed_lines: int = 0
+ total_mcdc_cov: int = 0
+ total_mcdc_tot: int = 0
+ total_decisions_count: int = 0
+ fully_verified_decisions: int = 0
+
+ @property
+ def total_lines(self) -> int:
+ """Total executable lines across all modified files in the patch."""
+ return self.total_covered_lines + self.total_missed_lines
+
+ @property
+ def line_coverage_pct(self) -> float:
+ """Aggregated patch line coverage percentage."""
+ return (self.total_covered_lines / self.total_lines * 100.0) if self.total_lines > 0 else 0.0
+
+ @property
+ def mcdc_coverage_pct(self) -> float:
+ """Aggregated patch MC/DC condition coverage percentage."""
+ return (self.total_mcdc_cov / self.total_mcdc_tot * 100.0) if self.total_mcdc_tot > 0 else 0.0
+
+ @property
+ def has_mcdc(self) -> bool:
+ """Returns True if any MC/DC decision records intersect the patch."""
+ return self.total_mcdc_tot > 0
+
+
+# -----------------------------------------------------------------------------
+# Parsing Utilities
+# -----------------------------------------------------------------------------
+
+class DiffParser:
+ """Parses unified diff outputs into structured file hunks with line numbers."""
+
+ @staticmethod
+ def parse(diff_source: str) -> Dict[str, List[DiffHunk]]:
+ """Parses a diff file path or raw diff string into a mapping of file path to hunks."""
+ files: Dict[str, List[DiffHunk]] = {}
+ current_file: Optional[str] = None
+ current_hunk: Optional[DiffHunk] = None
+ current_line_num: int = 0
+
+ if os.path.isfile(diff_source):
+ with open(diff_source, "r", encoding="utf-8") as f:
+ lines = f.readlines()
+ else:
+ lines = diff_source.splitlines(keepends=True)
+
+ for raw_line in lines:
+ line = raw_line.rstrip("\n")
+
+ if line.startswith("+++ b/"):
+ current_file = line[6:]
+ files[current_file] = []
+ current_hunk = None
+ continue
+
+ if line.startswith("+++ /dev/null"):
+ current_file = None
+ current_hunk = None
+ continue
+
+ if current_file is None:
+ continue
+
+ if line.startswith("@@"):
+ match = re.search(r"\+([0-9]+)", line)
+ if match:
+ current_line_num = int(match.group(1))
+ current_hunk = DiffHunk(header=line)
+ files[current_file].append(current_hunk)
+ continue
+
+ if current_hunk is None:
+ continue
+
+ if line.startswith("-"):
+ continue
+ elif line.startswith("+"):
+ current_hunk.lines.append(("+", line[1:], current_line_num))
+ current_line_num += 1
+ elif line.startswith(" "):
+ current_hunk.lines.append((" ", line[1:], current_line_num))
+ current_line_num += 1
+
+ return files
+
+
+class CoverageJSONParser:
+ """Parses and extracts statement segments and MC/DC records from llvm-cov JSON export."""
+
+ @staticmethod
+ def load(json_path: str) -> dict:
+ """Loads JSON file from disk with error reporting."""
+ try:
+ with open(json_path, "r", encoding="utf-8") as f:
+ return json.load(f)
+ except Exception as err:
+ sys.stderr.write(f"Error: Failed to parse coverage JSON from '{json_path}': {err}\n")
+ sys.exit(1)
+
+ @staticmethod
+ def extract_patch_matrix(
+ cov_data: dict, diff_files: Dict[str, List[DiffHunk]]
+ ) -> Dict[str, Dict[str, Any]]:
+ """Maps statement coverage segments and MC/DC decision records to modified files."""
+ coverage_matrix: Dict[str, Dict[str, Any]] = {
+ fpath: {"covered": set(), "missed": set(), "mcdc_decisions": []}
+ for fpath in diff_files.keys()
+ }
+
+ if "data" not in cov_data or not cov_data["data"]:
+ return coverage_matrix
+
+ for item in cov_data["data"][0].get("files", []):
+ fpath = item.get("filename", "")
+ rel_path = next(
+ (
+ rp
+ for rp in diff_files.keys()
+ if fpath == rp or fpath.endswith("/" + rp) or rp.endswith("/" + fpath)
+ ),
+ None,
+ )
+ if not rel_path:
+ continue
+
+ # 1. Process statement coverage segments
+ segments = item.get("segments", [])
+ for i, current in enumerate(segments):
+ line_start = current[0]
+ count = current[2]
+ has_count = current[3]
+
+ if not has_count:
+ continue
+
+ if i < len(segments) - 1:
+ nxt = segments[i + 1]
+ line_end = nxt[0]
+ end_range = line_end if line_end > line_start else line_start + 1
+ else:
+ end_range = line_start + 1
+
+ for line_num in range(line_start, end_range):
+ if count > 0:
+ coverage_matrix[rel_path]["covered"].add(line_num)
+ else:
+ coverage_matrix[rel_path]["missed"].add(line_num)
+
+ # 2. Process MC/DC decision records
+ mcdc_records = item.get("mcdc_records", [])
+ for rec in mcdc_records:
+ if len(rec) >= 10 and isinstance(rec[9], list):
+ l_start = rec[0]
+ l_end = rec[2]
+ conds = rec[9]
+ cov_conds = sum(1 for c in conds if c)
+ coverage_matrix[rel_path]["mcdc_decisions"].append(
+ {
+ "line_start": l_start,
+ "line_end": l_end,
+ "conditions": conds,
+ "covered": cov_conds,
+ "total": len(conds),
+ }
+ )
+
+ return coverage_matrix
+
+
+def is_executable_line(text: str) -> bool:
+ """Filters out non-executable code lines (comments, braces, pure declarations)."""
+ s = text.strip()
+ if not s:
+ return False
+ if any(s.startswith(prefix) for prefix in COMMENT_PREFIXES):
+ return False
+ if s in STRUCTURAL_TOKENS or s.startswith(":"):
+ return False
+ if s.startswith("#"):
+ return False
+ if any(s.startswith(prefix) for prefix in DECLARATION_PREFIXES):
+ return False
+ if s.startswith("struct ") or s.startswith("class ") or s.startswith("enum "):
+ if "{" in s or (s.endswith(";") and "=" not in s and "(" not in s):
+ return False
+ return True
+
+
+def format_line_ranges(lines: Set[int]) -> str:
+ """Formats an integer set of line numbers into concise span representations."""
+ if not lines:
+ return "None"
+ sorted_lines = sorted(lines)
+ ranges: List[str] = []
+ start = sorted_lines[0]
+ end = sorted_lines[0]
+ for n in sorted_lines[1:]:
+ if n == end + 1:
+ end = n
+ else:
+ ranges.append(f"`L{start}-L{end}`" if start != end else f"`L{start}`")
+ start = end = n
+ ranges.append(f"`L{start}-L{end}`" if start != end else f"`L{start}`")
+ return ", ".join(ranges)
+
+
+# -----------------------------------------------------------------------------
+# Statistics Calculation
+# -----------------------------------------------------------------------------
+
+def calculate_patch_statistics(
+ diff_files: Dict[str, List[DiffHunk]],
+ coverage_matrix: Dict[str, Dict[str, Any]],
+) -> PatchCoverageSummary:
+ """Calculates granular line, branch, and MC/DC statistics for all modified patch files."""
+ summary = PatchCoverageSummary()
+
+ for fpath, data in coverage_matrix.items():
+ added_lines: Set[int] = set()
+ for hunk in diff_files.get(fpath, []):
+ for l_type, text, l_num in hunk.lines:
+ if l_type == "+" and is_executable_line(text):
+ added_lines.add(l_num)
+
+ if not added_lines:
+ continue
+
+ f_covered = added_lines.intersection(data["covered"])
+ f_missed = (added_lines.intersection(data["missed"])) - f_covered
+
+ file_metric = FilePatchMetrics(
+ fpath=fpath,
+ added_lines=added_lines,
+ )
+
+ if len(data["covered"]) > 0 or len(data["missed"]) > 0:
+ file_metric.covered_lines = f_covered
+ file_metric.missed_lines = f_missed
+ summary.total_covered_lines += len(f_covered)
+ summary.total_missed_lines += len(f_missed)
+ else:
+ file_metric.missed_lines = added_lines
+ summary.total_missed_lines += len(added_lines)
+
+ # Evaluate MC/DC decision records intersecting modified lines
+ for decision in data.get("mcdc_decisions", []):
+ d_start = decision["line_start"]
+ d_end = decision["line_end"]
+ if any(d_start <= l <= d_end for l in added_lines):
+ summary.total_decisions_count += 1
+ file_metric.decisions_tot += 1
+ file_metric.mcdc_cov += decision["covered"]
+ file_metric.mcdc_tot += decision["total"]
+ summary.total_mcdc_cov += decision["covered"]
+ summary.total_mcdc_tot += decision["total"]
+
+ if decision["covered"] == decision["total"]:
+ summary.fully_verified_decisions += 1
+ file_metric.decisions_ver += 1
+ file_metric.condition_diagnostics.append(
+ f"`L{d_start}`: {decision['covered']}/{decision['total']} verified"
+ )
+ else:
+ uncovered_idx = [
+ f"C{i+1}"
+ for i, is_cov in enumerate(decision["conditions"])
+ if not is_cov
+ ]
+ unverified_str = ", ".join(uncovered_idx)
+ file_metric.condition_diagnostics.append(
+ f"`L{d_start}`: {decision['covered']}/{decision['total']} verified ({unverified_str} unverified)"
+ )
+ for l in range(d_start, d_end + 1):
+ if l in added_lines:
+ file_metric.unverified_decision_lines[l] = uncovered_idx
+
+ summary.files[fpath] = file_metric
+
+ return summary
+
+
+# -----------------------------------------------------------------------------
+# Report Formatting
+# -----------------------------------------------------------------------------
+
+def format_status_banner(summary: PatchCoverageSummary) -> str:
+ """Generates the executive summary callout block."""
+ lines: List[str] = []
+ if summary.total_missed_lines == 0:
+ if not summary.has_mcdc:
+ lines.append("> [!TIP]")
+ lines.append(f"> ### Patch Coverage: **{summary.line_coverage_pct:.2f}%**")
+ lines.append(
+ f"> All **{summary.total_lines}** newly added or modified executable lines are covered."
+ )
+ elif summary.total_mcdc_cov == summary.total_mcdc_tot:
+ lines.append("> [!TIP]")
+ lines.append(
+ f"> ### Patch Coverage: **{summary.line_coverage_pct:.2f}% Line** | **100.00% MC/DC**"
+ )
+ lines.append(
+ f"> All **{summary.total_lines}** executable lines and **{summary.total_mcdc_tot}** boolean conditions across **{summary.total_decisions_count}** decisions are covered."
+ )
+ else:
+ lines.append("> [!NOTE]")
+ lines.append(
+ f"> ### Patch Coverage: **{summary.line_coverage_pct:.2f}% Line** | **{summary.mcdc_coverage_pct:.1f}% MC/DC**"
+ )
+ lines.append(
+ f"> Executed **{summary.total_covered_lines} / {summary.total_lines}** lines. **{summary.total_mcdc_cov} / {summary.total_mcdc_tot}** boolean conditions achieved independence across **{summary.fully_verified_decisions} / {summary.total_decisions_count}** decisions."
+ )
+ else:
+ lines.append("> [!WARNING]")
+ lines.append(
+ f"> ### Patch Coverage: **{summary.line_coverage_pct:.2f}%** ({summary.total_missed_lines} Missed Lines)"
+ )
+ lines.append(
+ f"> **{summary.total_missed_lines}** unexecuted lines detected in patch."
+ )
+ return "\n".join(lines)
+
+
+def format_metadata_section(
+ base_sha: Optional[str],
+ head_sha: Optional[str],
+ base_branch: Optional[str],
+ head_branch: Optional[str],
+ targets_str: Optional[str] = None,
+ base_repo: str = DEFAULT_BASE_REPO,
+ head_repo: str = DEFAULT_HEAD_REPO,
+) -> str:
+ """Formats Git commit and target test metadata."""
+ lines: List[str] = []
+ if base_sha and head_sha and base_branch and head_branch:
+ lines.append(
+ f"- **Base Branch:** [`{base_branch}` ({base_sha[:7]})](https://github.com/{base_repo}/commit/{base_sha})"
+ )
+ lines.append(
+ f"- **Head Commit:** [`{head_branch}` ({head_sha[:7]})](https://github.com/{head_repo}/commit/{head_sha})"
+ )
+ if targets_str:
+ targets_formatted = ", ".join(
+ f"`{t.strip()}`" for t in targets_str.split() if t.strip()
+ )
+ lines.append(f"- **Targeted Tests Executed:** {targets_formatted}")
+ return "\n".join(lines)
+
+
+def format_breakdown_table(
+ summary: PatchCoverageSummary,
+ head_repo: str = DEFAULT_HEAD_REPO,
+ head_sha: Optional[str] = None,
+) -> str:
+ """Generates the Markdown table breaking down coverage per source file."""
+ lines: List[str] = ["### Coverage Breakdown"]
+ if summary.has_mcdc:
+ lines.append(
+ "| Modified Source File | Line Coverage | MC/DC Conditions | Decisions (Verified / Total) | Missed Lines | Unverified Conditions |"
+ )
+ lines.append("| :--- | :---: | :---: | :---: | :---: | :--- |")
+ else:
+ lines.append(
+ "| Modified Source File | Patch Coverage | Covered / Total | Missed Lines | Unexecuted Line Spans |"
+ )
+ lines.append("| :--- | :---: | :---: | :---: | :---: |")
+
+ for fpath, file_metric in summary.files.items():
+ file_link = f"[`{fpath}`](https://github.com/{head_repo}/blob/{head_sha or 'main'}/{fpath})"
+ f_missed = file_metric.missed_lines
+ f_cov_len = len(file_metric.covered_lines)
+ f_tot_len = file_metric.total_lines
+
+ if summary.has_mcdc:
+ mcdc_cell = (
+ f"**{file_metric.mcdc_coverage_pct:.1f}%** ({file_metric.mcdc_cov}/{file_metric.mcdc_tot})"
+ if file_metric.mcdc_tot > 0
+ else "N/A"
+ )
+ dec_cell = (
+ f"**{file_metric.decisions_ver} / {file_metric.decisions_tot}**"
+ if file_metric.decisions_tot > 0
+ else "N/A"
+ )
+ diag_cell = "<br>".join(file_metric.condition_diagnostics) if file_metric.condition_diagnostics else "None"
+ lines.append(
+ f"| {file_link} | **{file_metric.line_coverage_pct:.2f}%** ({f_cov_len}/{f_tot_len}) | {mcdc_cell} | {dec_cell} | {len(f_missed)} | {diag_cell} |"
+ )
+ else:
+ line_spans = format_line_ranges(f_missed)
+ lines.append(
+ f"| {file_link} | **{file_metric.line_coverage_pct:.2f}%** | {f_cov_len} / {f_tot_len} | {len(f_missed)} | {line_spans} |"
+ )
+
+ # Summary Row
+ if summary.has_mcdc:
+ total_dec_cell = f"**{summary.fully_verified_decisions} / {summary.total_decisions_count}**"
+ lines.append(
+ f"| **Total (Patch)** | **{summary.line_coverage_pct:.2f}%** ({summary.total_covered_lines}/{summary.total_lines}) | **{summary.mcdc_coverage_pct:.1f}%** ({summary.total_mcdc_cov}/{summary.total_mcdc_tot}) | {total_dec_cell} | **{summary.total_missed_lines}** | - |"
+ )
+ else:
+ lines.append(
+ f"| **Total (Patch)** | **{summary.line_coverage_pct:.2f}%** | {summary.total_covered_lines} / {summary.total_lines} | **{summary.total_missed_lines}** | - |"
+ )
+
+ return "\n".join(lines)
+
+
+def format_annotated_diff(
+ summary: PatchCoverageSummary,
+ diff_files: Dict[str, List[DiffHunk]],
+) -> str:
+ """Renders the collapsible source map diff with execution indicators."""
+ lines: List[str] = [
+ "<details>",
+ "<summary><b>View Annotated Patch Diff (Source Map)</b></summary>\n",
+ ]
+
+ for fpath, file_metric in summary.files.items():
+ hunks = diff_files.get(fpath, [])
+ unverified_lines = file_metric.unverified_decision_lines
+
+ lines.append(f"#### `{fpath}`")
+ lines.append("```diff")
+ for hunk in hunks:
+ lines.append(hunk.header)
+ for l_type, text, l_num in hunk.lines:
+ if l_type == "+":
+ if l_num in file_metric.missed_lines:
+ lines.append(f"- {text} // [MISSED]")
+ elif l_num in unverified_lines:
+ unverified_conds = ", ".join(unverified_lines[l_num])
+ lines.append(f"! {text} // [PARTIAL MC/DC: {unverified_conds} unverified]")
+ elif l_num in file_metric.covered_lines:
+ lines.append(f"+ {text}")
+ else:
+ lines.append(f" {text}")
+ elif l_type == " ":
+ lines.append(f" {text}")
+ lines.append("```\n")
+
+ lines.append("</details>")
+ return "\n".join(lines)
+
+
+def render_patch_report(
+ diff_files: Dict[str, List[DiffHunk]],
+ coverage_matrix: Dict[str, Dict[str, Any]],
+ base_sha: Optional[str],
+ head_sha: Optional[str],
+ base_branch: Optional[str],
+ head_branch: Optional[str],
+ targets_str: Optional[str] = None,
+ base_repo: str = DEFAULT_BASE_REPO,
+ head_repo: str = DEFAULT_HEAD_REPO,
+) -> None:
+ """Composes and outputs the full Markdown report."""
+ summary = calculate_patch_statistics(diff_files, coverage_matrix)
+
+ if summary.has_mcdc:
+ print("## LLVM-libc MC/DC Patch Coverage Report\n")
+ else:
+ print("## LLVM-libc Patch Coverage Report\n")
+
+ if summary.total_lines == 0 or not summary.files:
+ meta_str = format_metadata_section(
+ base_sha, head_sha, base_branch, head_branch, targets_str, base_repo, head_repo
+ )
+ if meta_str:
+ print(meta_str)
+ print("\n---\n")
+ print("> [!NOTE]")
+ print("> ### Coverage Validated")
+ print("> No `.cpp` source files in `libc/src/` were modified in this patch.")
+ return
+
+ # 1. Status Banner
+ print(format_status_banner(summary))
+ print("")
+
+ # 2. Metadata Section
+ meta_str = format_metadata_section(
+ base_sha, head_sha, base_branch, head_branch, targets_str, base_repo, head_repo
+ )
+ if meta_str:
+ print(meta_str)
+ print("\n---\n")
+
+ # 3. Breakdown Table
+ print(format_breakdown_table(summary, head_repo, head_sha))
+ print("")
+
+ # 4. Source Map Diff
+ print(format_annotated_diff(summary, diff_files))
+
+
+# -----------------------------------------------------------------------------
+# CLI Entry Point
+# -----------------------------------------------------------------------------
+
+def main() -> None:
+ """Parses command-line arguments and triggers report generation."""
+ parser = argparse.ArgumentParser(description="LLVM-libc Diff Coverage Analyzer")
+ parser.add_argument("diff_file", help="Path to unified diff file")
+ parser.add_argument("json_file", help="Path to llvm-cov export JSON file")
+ parser.add_argument("base_sha", nargs="?", help="Base commit SHA")
+ parser.add_argument("head_sha", nargs="?", help="Head commit SHA")
+ parser.add_argument("base_branch", nargs="?", help="Base branch name")
+ parser.add_argument("head_branch", nargs="?", help="Head branch name")
+ parser.add_argument(
+ "targets", nargs="?", help="Space-separated list of executed test targets"
+ )
+ parser.add_argument(
+ "base_repo",
+ nargs="?",
+ default=DEFAULT_BASE_REPO,
+ help=f"Base repository (default: {DEFAULT_BASE_REPO})",
+ )
+ parser.add_argument(
+ "head_repo",
+ nargs="?",
+ default=DEFAULT_HEAD_REPO,
+ help=f"Head repository (default: {DEFAULT_HEAD_REPO})",
+ )
+
+ args = parser.parse_args()
+
+ diff_files = DiffParser.parse(args.diff_file)
+ cov_data = CoverageJSONParser.load(args.json_file)
+ coverage_matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
+
+ render_patch_report(
+ diff_files,
+ coverage_matrix,
+ args.base_sha,
+ args.head_sha,
+ args.base_branch,
+ args.head_branch,
+ args.targets,
+ args.base_repo,
+ args.head_repo,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libc/utils/coverage/full_report.py b/libc/utils/coverage/full_report.py
deleted file mode 100644
index 8977b51e85e9e..0000000000000
--- a/libc/utils/coverage/full_report.py
+++ /dev/null
@@ -1,249 +0,0 @@
-#!/usr/bin/env python3
-#
-# ====- Generate full codebase coverage reports ----------------*- python -*--==#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# ==-------------------------------------------------------------------------==#
-
-import argparse
-import json
-import os
-import sys
-from pathlib import Path
-from typing import Any, Dict, List, Tuple
-
-
-def render_full_report(cov_data: dict) -> None:
- if "data" not in cov_data or not cov_data["data"]:
- print("## LLVM-libc Full Codebase Coverage Report\n")
- print("> [!WARNING]")
- print("> ### No Coverage Data Detected")
- print(
- "> The test execution completed but no coverage profiles were exported."
- )
- return
-
- subsystems: Dict[str, Dict[str, Any]] = {}
-
- total_lines_cov = 0
- total_lines_tot = 0
- total_func_cov = 0
- total_func_tot = 0
- total_mcdc_cov = 0
- total_mcdc_tot = 0
- total_decisions_count = 0
- total_decisions_full = 0
-
- for item in cov_data["data"][0].get("files", []):
- fpath = item.get("filename", "")
- if "src/" not in fpath or "/test/" in fpath or "/utils/" in fpath:
- continue
-
- idx = fpath.find("src/")
- if idx == -1:
- continue
- rel_path = fpath[idx:]
-
- summary = item.get("summary", {})
- lines_summary = summary.get("lines", {})
- func_summary = summary.get("functions", {})
- mcdc_summary = summary.get("mcdc", {})
-
- line_tot = lines_summary.get("count", 0)
- line_cov = lines_summary.get("covered", 0)
- func_tot = func_summary.get("count", 0)
- func_cov = func_summary.get("covered", 0)
- mcdc_tot = mcdc_summary.get("count", 0)
- mcdc_cov = mcdc_summary.get("covered", 0)
-
- mcdc_records = item.get("mcdc_records", [])
- file_decisions_tot = len(mcdc_records)
- file_decisions_full = 0
- for rec in mcdc_records:
- if len(rec) >= 10 and isinstance(rec[9], list):
- conds = rec[9]
- if all(conds):
- file_decisions_full += 1
-
- if line_tot == 0:
- continue
-
- total_lines_cov += line_cov
- total_lines_tot += line_tot
- total_func_cov += func_cov
- total_func_tot += func_tot
- total_mcdc_cov += mcdc_cov
- total_mcdc_tot += mcdc_tot
- total_decisions_count += file_decisions_tot
- total_decisions_full += file_decisions_full
-
- parts = rel_path.split("/")
- subsystem = "/".join(parts[:2]) if len(parts) >= 2 else parts[0]
-
- if subsystem not in subsystems:
- subsystems[subsystem] = {
- "lines_cov": 0,
- "lines_tot": 0,
- "func_cov": 0,
- "func_tot": 0,
- "mcdc_cov": 0,
- "mcdc_tot": 0,
- "decisions_tot": 0,
- "decisions_full": 0,
- }
-
- subsystems[subsystem]["lines_cov"] += line_cov
- subsystems[subsystem]["lines_tot"] += line_tot
- subsystems[subsystem]["func_cov"] += func_cov
- subsystems[subsystem]["func_tot"] += func_tot
- subsystems[subsystem]["mcdc_cov"] += mcdc_cov
- subsystems[subsystem]["mcdc_tot"] += mcdc_tot
- subsystems[subsystem]["decisions_tot"] += file_decisions_tot
- subsystems[subsystem]["decisions_full"] += file_decisions_full
-
- line_pct = (
- (total_lines_cov / total_lines_tot * 100) if total_lines_tot > 0 else 0
- )
- func_pct = (
- (total_func_cov / total_func_tot * 100) if total_func_tot > 0 else 0
- )
- has_mcdc = total_mcdc_tot > 0
- mcdc_pct = (
- (total_mcdc_cov / total_mcdc_tot * 100) if total_mcdc_tot > 0 else 0
- )
- decisions_pct = (
- (total_decisions_full / total_decisions_count * 100)
- if total_decisions_count > 0
- else 0
- )
-
- pages_url = os.environ.get("COVERAGE_DASHBOARD_URL")
- if not pages_url:
- repo = os.environ.get("GITHUB_REPOSITORY", "llvm/llvm-project")
- if "/" in repo:
- owner, repo_name = repo.split("/", 1)
- pages_url = f"https://{owner}.github.io/{repo_name}/"
- else:
- pages_url = f"https://{repo}.github.io/"
-
- if has_mcdc and not pages_url.endswith("/mcdc/"):
- mcdc_pages_url = pages_url.rstrip("/") + "/mcdc/"
- else:
- mcdc_pages_url = pages_url
-
- print("## LLVM-libc Full Codebase Coverage Report\n")
-
- print("> [!NOTE]")
- if has_mcdc:
- print(
- f"> ### Overall Codebase Coverage: **{line_pct:.2f}% Line** | **{mcdc_pct:.2f}% MC/DC**"
- )
- print(
- f"> Tested **{total_lines_cov:,} / {total_lines_tot:,}** executable lines and **{total_mcdc_cov:,} / {total_mcdc_tot:,}** boolean conditions across **{total_decisions_count:,}** decisions."
- )
- print("")
- print(
- f"- **Coverage Dashboard:** [{mcdc_pages_url}]({mcdc_pages_url})"
- )
- else:
- print(f"> ### Overall Codebase Coverage: **{line_pct:.2f}%**")
- print(
- f"> Tested **{total_lines_cov:,} / {total_lines_tot:,}** executable lines across all LLVM-libc subsystems."
- )
- print("")
- print(f"- **Coverage Dashboard:** [{pages_url}]({pages_url})")
-
- print("\n---\n")
-
- print("### Overall")
- print("| Metric | Covered | Total | Coverage % |")
- print("| :--- | :---: | :---: | :---: |")
- if has_mcdc:
- print(
- f"| **MC/DC Condition Independence** | {total_mcdc_cov:,} | {total_mcdc_tot:,} | **{mcdc_pct:.2f}%** |"
- )
- print(
- f"| **Fully Verified Decisions** | {total_decisions_full:,} | {total_decisions_count:,} | **{decisions_pct:.2f}%** |"
- )
- print(
- f"| **Executable Lines** | {total_lines_cov:,} | {total_lines_tot:,} | **{line_pct:.2f}%** |"
- )
- print(
- f"| **Functions** | {total_func_cov:,} | {total_func_tot:,} | **{func_pct:.2f}%** |"
- )
- print("")
-
- print("### Coverage Breakdown")
- if has_mcdc:
- print(
- "| Subsystem | MC/DC Conditions | Decisions (Verified / Total) | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
- )
- print("| :--- | :---: | :---: | :---: | :---: | :---: | :---: |")
-
- sorted_subsystems = sorted(subsystems.keys())
- else:
- print(
- "| Subsystem | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
- )
- print("| :--- | :---: | :---: | :---: | :---: |")
- sorted_subsystems = sorted(subsystems.keys())
-
- for sub in sorted_subsystems:
- data = subsystems[sub]
- s_line_pct = (
- (data["lines_cov"] / data["lines_tot"] * 100)
- if data["lines_tot"] > 0
- else 0
- )
- s_func_pct = (
- (data["func_cov"] / data["func_tot"] * 100)
- if data["func_tot"] > 0
- else 0
- )
- missed_lines = data["lines_tot"] - data["lines_cov"]
-
- if has_mcdc:
- s_mc_pct = (
- (data["mcdc_cov"] / data["mcdc_tot"] * 100)
- if data["mcdc_tot"] > 0
- else 0
- )
- mc_cell = (
- f"**{s_mc_pct:.1f}%** ({data['mcdc_cov']}/{data['mcdc_tot']})"
- if data["mcdc_tot"] > 0
- else "N/A"
- )
- dec_cell = (
- f"{data['decisions_full']} / {data['decisions_tot']}"
- if data["decisions_tot"] > 0
- else "N/A"
- )
- print(
- f"| `libc/{sub}` | {mc_cell} | {dec_cell} | **{s_line_pct:.2f}%** | {s_func_pct:.2f}% | {data['lines_tot']:,} | {missed_lines:,} |"
- )
- else:
- print(
- f"| `libc/{sub}` | **{s_line_pct:.2f}%** | {s_func_pct:.2f}% | {data['lines_tot']:,} | {missed_lines:,} |"
- )
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(description="LLVM-libc Full Coverage Analyzer")
- parser.add_argument("json_file", help="Path to llvm-cov export JSON file")
-
- args, _ = parser.parse_known_args()
-
- try:
- with open(args.json_file, "r", encoding="utf-8") as f:
- cov_data = json.load(f)
- except Exception as e:
- sys.stderr.write(f"Error: Failed to parse coverage JSON: {e}\n")
- sys.exit(1)
- render_full_report(cov_data)
-
-
-if __name__ == "__main__":
- main()
diff --git a/libc/utils/coverage/patch_report.py b/libc/utils/coverage/patch_report.py
deleted file mode 100644
index d2da34a469405..0000000000000
--- a/libc/utils/coverage/patch_report.py
+++ /dev/null
@@ -1,500 +0,0 @@
-#!/usr/bin/env python3
-#
-# ====- Generate patch coverage reports ------------------------*- python -*--==#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# ==-------------------------------------------------------------------------==#
-
-import argparse
-import json
-import os
-import re
-import sys
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Set, Tuple
-
-
-class DiffHunk:
- def __init__(self, header: str):
- self.header: str = header
- self.lines: List[Tuple[str, str, int]] = [] # (prefix, text, line_number)
-
-
-class DiffParser:
- @staticmethod
- def parse(diff_source: str) -> Dict[str, List[DiffHunk]]:
- files: Dict[str, List[DiffHunk]] = {}
- current_file: Optional[str] = None
- current_hunk: Optional[DiffHunk] = None
- current_line_num: int = 0
-
- # Support both file path and raw diff string
- if os.path.isfile(diff_source):
- with open(diff_source, "r", encoding="utf-8") as f:
- lines = f.readlines()
- else:
- lines = diff_source.splitlines(keepends=True)
-
- for line in lines:
- line = line.rstrip("\n")
-
- if line.startswith("+++ b/"):
- current_file = line[6:]
- files[current_file] = []
- current_hunk = None
- continue
-
- if line.startswith("+++ /dev/null"):
- current_file = None
- current_hunk = None
- continue
-
- if current_file is None:
- continue
-
- if line.startswith("@@"):
- match = re.search(r"\+([0-9]+)", line)
- if match:
- current_line_num = int(match.group(1))
- current_hunk = DiffHunk(line)
- files[current_file].append(current_hunk)
- continue
-
- if current_hunk is None:
- continue
-
- if line.startswith("-"):
- continue
- elif line.startswith("+"):
- current_hunk.lines.append(("+", line[1:], current_line_num))
- current_line_num += 1
- elif line.startswith(" "):
- current_hunk.lines.append((" ", line[1:], current_line_num))
- current_line_num += 1
-
- return files
-
-
-class CoverageJSONParser:
- @staticmethod
- def load(json_path: str) -> dict:
- try:
- with open(json_path, "r", encoding="utf-8") as f:
- return json.load(f)
- except Exception as e:
- sys.stderr.write(f"Error: Failed to parse coverage JSON: {e}\n")
- sys.exit(1)
-
- @staticmethod
- def extract_patch_matrix(
- cov_data: dict, diff_files: Dict[str, List[DiffHunk]]
- ) -> Dict[str, Dict[str, Any]]:
- coverage_matrix: Dict[str, Dict[str, Any]] = {
- fpath: {"covered": set(), "missed": set(), "mcdc_decisions": []}
- for fpath in diff_files.keys()
- }
-
- if "data" not in cov_data or not cov_data["data"]:
- return coverage_matrix
-
- for item in cov_data["data"][0].get("files", []):
- fpath = item.get("filename", "")
- rel_path = next(
- (rp for rp in diff_files.keys() if fpath.endswith(rp)), None
- )
- if not rel_path:
- continue
-
- # 1. Statement segments
- segments = item.get("segments", [])
- for i in range(len(segments) - 1):
- current = segments[i]
- nxt = segments[i + 1]
-
- line_start = current[0]
- line_end = nxt[0]
- count = current[2]
- has_count = current[3]
-
- if has_count:
- for line_num in range(line_start, line_end + 1):
- if count > 0:
- coverage_matrix[rel_path]["covered"].add(line_num)
- else:
- coverage_matrix[rel_path]["missed"].add(line_num)
-
- # 2. MC/DC decision records (Clang 18+)
- mcdc_records = item.get("mcdc_records", [])
- for rec in mcdc_records:
- if len(rec) >= 10 and isinstance(rec[9], list):
- l_start = rec[0]
- l_end = rec[2]
- conds = rec[9]
- cov_conds = sum(1 for c in conds if c)
- coverage_matrix[rel_path]["mcdc_decisions"].append(
- {
- "line_start": l_start,
- "line_end": l_end,
- "conditions": conds,
- "covered": cov_conds,
- "total": len(conds),
- }
- )
-
- return coverage_matrix
-
-
-def is_executable_line(text: str) -> bool:
- s = text.strip()
- if not s:
- return False
- # Comments
- if (
- s.startswith("//")
- or s.startswith("/*")
- or s.startswith("*")
- or s.startswith("*/")
- ):
- return False
- # Structural braces and colons
- if s in ("{", "}", "};", "{};") or s.startswith(":"):
- return False
- # Preprocessor directives
- if s.startswith("#"):
- return False
- # Declarations / keywords / attributes
- if (
- s.startswith("namespace ")
- or s.startswith("extern ")
- or s.startswith("using ")
- or s.startswith("__attribute__")
- or s.startswith("template")
- or s.startswith("typedef ")
- or s.startswith("struct ")
- or s.startswith("class ")
- or s.startswith("enum ")
- ):
- return False
- return True
-
-
-def format_line_ranges(lines: Set[int]) -> str:
- if not lines:
- return "None"
- sorted_lines = sorted(lines)
- ranges = []
- start = sorted_lines[0]
- end = sorted_lines[0]
- for n in sorted_lines[1:]:
- if n == end + 1:
- end = n
- else:
- ranges.append(f"`L{start}-L{end}`" if start != end else f"`L{start}`")
- start = end = n
- ranges.append(f"`L{start}-L{end}`" if start != end else f"`L{start}`")
- return ", ".join(ranges)
-
-
-def render_patch_report(
- diff_files: Dict[str, List[DiffHunk]],
- coverage_matrix: Dict[str, Dict[str, Any]],
- base_sha: Optional[str],
- head_sha: Optional[str],
- base_branch: Optional[str],
- head_branch: Optional[str],
- targets_str: Optional[str] = None,
- base_repo: str = "llvm/llvm-project",
- head_repo: str = "llvm/llvm-project",
-) -> None:
- total_covered = 0
- total_missed = 0
- active_files = {}
-
- total_mcdc_cov = 0
- total_mcdc_tot = 0
- total_decisions_count = 0
- fully_verified_decisions = 0
- file_mcdc_data: Dict[str, Tuple[int, int, List[str]]] = {}
-
- for fpath, data in coverage_matrix.items():
- added_lines: Set[int] = set()
- for hunk in diff_files.get(fpath, []):
- for l_type, text, l_num in hunk.lines:
- if l_type == "+":
- if not is_executable_line(text):
- continue
- added_lines.add(l_num)
-
- if not added_lines:
- continue
-
- f_covered = added_lines.intersection(data["covered"])
- f_missed = (added_lines.intersection(data["missed"])) - f_covered
-
- if len(data["covered"]) > 0 or len(data["missed"]) > 0:
- total_covered += len(f_covered)
- total_missed += len(f_missed)
- active_files[fpath] = (f_covered, f_missed, added_lines)
- else:
- total_missed += len(added_lines)
- active_files[fpath] = (set(), added_lines, added_lines)
-
- # Evaluate MC/DC decisions on modified lines
- f_mcdc_cov = 0
- f_mcdc_tot = 0
- f_dec_ver = 0
- f_dec_tot = 0
- condition_diagnostics = []
- unverified_decision_lines: Dict[int, List[str]] = {}
-
- for decision in data.get("mcdc_decisions", []):
- d_start = decision["line_start"]
- d_end = decision["line_end"]
- # Check if any modified line overlaps this decision range
- if any(d_start <= l <= d_end for l in added_lines):
- total_decisions_count += 1
- f_dec_tot += 1
- f_mcdc_cov += decision["covered"]
- f_mcdc_tot += decision["total"]
- if decision["covered"] == decision["total"]:
- fully_verified_decisions += 1
- f_dec_ver += 1
- condition_diagnostics.append(
- f"`L{d_start}`: {decision['covered']}/{decision['total']} verified"
- )
- else:
- uncovered_idx = [
- f"C{i+1}"
- for i, is_cov in enumerate(decision["conditions"])
- if not is_cov
- ]
- unverified_str = ", ".join(uncovered_idx)
- condition_diagnostics.append(
- f"`L{d_start}`: {decision['covered']}/{decision['total']} verified ({unverified_str} unverified)"
- )
- for l in range(d_start, d_end + 1):
- if l in added_lines:
- unverified_decision_lines[l] = uncovered_idx
-
- if f_mcdc_tot > 0 or f_dec_tot > 0:
- total_mcdc_cov += f_mcdc_cov
- total_mcdc_tot += f_mcdc_tot
- file_mcdc_data[fpath] = (
- f_mcdc_cov,
- f_mcdc_tot,
- condition_diagnostics,
- f_dec_ver,
- f_dec_tot,
- unverified_decision_lines,
- )
-
- total_lines = total_covered + total_missed
- has_mcdc = total_mcdc_tot > 0
-
- if has_mcdc:
- print("## LLVM-libc MC/DC Patch Coverage Report\n")
- else:
- print("## LLVM-libc Patch Coverage Report\n")
-
- if total_lines == 0 or not active_files:
- if base_sha and head_sha and base_branch and head_branch:
- print(
- f"- **Base Branch:** [`{base_branch}` ({base_sha[:7]})](https://github.com/{base_repo}/commit/{base_sha})"
- )
- print(
- f"- **Head Commit:** [`{head_branch}` ({head_sha[:7]})](https://github.com/{head_repo}/commit/{head_sha})\n"
- )
- print("---\n")
- print("> [!NOTE]")
- print("> ### Coverage Validated")
- print("> No `.cpp` source files in `libc/src/` were modified in this patch.")
- return
-
- coverage_percent = (total_covered / total_lines) * 100
- mcdc_percent = (total_mcdc_cov / total_mcdc_tot * 100) if has_mcdc else 0.0
-
- if total_missed == 0:
- if not has_mcdc:
- print("> [!TIP]")
- print(f"> ### Patch Coverage: **{coverage_percent:.2f}%**")
- print(
- f"> All **{total_lines}** newly added or modified executable lines are covered."
- )
- elif total_mcdc_cov == total_mcdc_tot:
- print("> [!TIP]")
- print(
- f"> ### Patch Coverage: **{coverage_percent:.2f}% Line** | **100.00% MC/DC**"
- )
- print(
- f"> All **{total_lines}** executable lines and **{total_mcdc_tot}** boolean conditions across **{total_decisions_count}** decisions are covered."
- )
- else:
- print("> [!NOTE]")
- print(
- f"> ### Patch Coverage: **{coverage_percent:.2f}% Line** | **{mcdc_percent:.1f}% MC/DC**"
- )
- print(
- f"> Executed **{total_covered} / {total_lines}** lines. **{total_mcdc_cov} / {total_mcdc_tot}** boolean conditions achieved independence across **{fully_verified_decisions} / {total_decisions_count}** decisions."
- )
- else:
- print("> [!WARNING]")
- print(
- f"> ### Patch Coverage: **{coverage_percent:.2f}%** ({total_missed} Missed Lines)"
- )
- print(
- f"> **{total_missed}** unexecuted lines detected in patch."
- )
- print("")
-
- # Commit metadata and targets executed
- if base_sha and head_sha and base_branch and head_branch:
- print(
- f"- **Base Branch:** [`{base_branch}` ({base_sha[:7]})](https://github.com/{base_repo}/commit/{base_sha})"
- )
- print(
- f"- **Head Commit:** [`{head_branch}` ({head_sha[:7]})](https://github.com/{head_repo}/commit/{head_sha})"
- )
- if targets_str:
- targets_formatted = ", ".join(
- f"`{t.strip()}`" for t in targets_str.split() if t.strip()
- )
- print(f"- **Targeted Tests Executed:** {targets_formatted}")
- print("\n---\n")
-
- # Unified Coverage Breakdown Table
- print("### Coverage Breakdown")
- if has_mcdc:
- print(
- "| Modified Source File | Line Coverage | MC/DC Conditions | Decisions (Verified / Total) | Missed Lines | Unverified Conditions |"
- )
- print("| :--- | :---: | :---: | :---: | :---: | :--- |")
- else:
- print(
- "| Modified Source File | Patch Coverage | Covered / Total | Missed Lines | Unexecuted Line Spans |"
- )
- print("| :--- | :---: | :---: | :---: | :---: |")
-
- for fpath, (f_covered, f_missed, added_lines) in active_files.items():
- f_total = len(f_covered) + len(f_missed)
- f_pct = (len(f_covered) / f_total * 100) if f_total > 0 else 0.0
- line_spans = format_line_ranges(f_missed)
- file_link = f"[`{fpath}`](https://github.com/{head_repo}/blob/{head_sha or 'main'}/{fpath})"
-
- if has_mcdc:
- f_mc_data = file_mcdc_data.get(fpath, (0, 0, [], 0, 0, {}))
- f_mc_cov, f_mc_tot, diag_list, f_dec_ver, f_dec_tot = (
- f_mc_data[0],
- f_mc_data[1],
- f_mc_data[2],
- f_mc_data[3],
- f_mc_data[4],
- )
- if f_mc_tot > 0:
- f_mc_pct = f_mc_cov / f_mc_tot * 100
- mcdc_cell = f"**{f_mc_pct:.1f}%** ({f_mc_cov}/{f_mc_tot})"
- else:
- mcdc_cell = "N/A"
-
- dec_cell = (
- f"**{f_dec_ver} / {f_dec_tot}**" if f_dec_tot > 0 else "N/A"
- )
- diag_cell = "<br>".join(diag_list) if diag_list else "None"
-
- print(
- f"| {file_link} | **{f_pct:.2f}%** ({len(f_covered)}/{f_total}) | {mcdc_cell} | {dec_cell} | {len(f_missed)} | {diag_cell} |"
- )
- else:
- print(
- f"| {file_link} | **{f_pct:.2f}%** | {len(f_covered)} / {f_total} | {len(f_missed)} | {line_spans} |"
- )
-
- # Summary Row
- if has_mcdc:
- total_dec_cell = f"**{fully_verified_decisions} / {total_decisions_count}**"
- print(
- f"| **Total (Patch)** | **{coverage_percent:.2f}%** ({total_covered}/{total_lines}) | **{mcdc_percent:.1f}%** ({total_mcdc_cov}/{total_mcdc_tot}) | {total_dec_cell} | **{total_missed}** | - |"
- )
- else:
- print(
- f"| **Total (Patch)** | **{coverage_percent:.2f}%** | {total_covered} / {total_lines} | **{total_missed}** | - |"
- )
- print("")
-
- # Collapsible Source Map Diff
- print("<details>")
- print("<summary><b>View Annotated Patch Diff (Source Map)</b></summary>\n")
-
- for fpath, (f_covered, f_missed, added_lines) in active_files.items():
- hunks = diff_files.get(fpath, [])
- f_mc_data = file_mcdc_data.get(fpath, (0, 0, [], 0, 0, {}))
- unverified_lines = f_mc_data[5] if len(f_mc_data) > 5 else {}
-
- print(f"#### `{fpath}`")
- print("```diff")
- for hunk in hunks:
- print(hunk.header)
- for l_type, text, l_num in hunk.lines:
- if l_type == "+":
- if l_num in f_missed:
- print(f"- {text} // [MISSED]")
- elif l_num in unverified_lines:
- unverified_conds = ", ".join(unverified_lines[l_num])
- print(f"! {text} // [PARTIAL MC/DC: {unverified_conds} unverified]")
- elif l_num in f_covered:
- print(f"+ {text}")
- else:
- print(f" {text}")
- elif l_type == " ":
- print(f" {text}")
- print("```\n")
- print("</details>")
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(description="LLVM-libc Patch Coverage Analyzer")
- parser.add_argument("diff_file", help="Path to unified diff file")
- parser.add_argument("json_file", help="Path to llvm-cov export JSON file")
- parser.add_argument("base_sha", nargs="?", help="Base commit SHA")
- parser.add_argument("head_sha", nargs="?", help="Head commit SHA")
- parser.add_argument("base_branch", nargs="?", help="Base branch name")
- parser.add_argument("head_branch", nargs="?", help="Head branch name")
- parser.add_argument(
- "targets", nargs="?", help="Space-separated list of executed test targets"
- )
- parser.add_argument(
- "base_repo",
- nargs="?",
- default="llvm/llvm-project",
- help="Base repository (e.g. llvm/llvm-project)",
- )
- parser.add_argument(
- "head_repo",
- nargs="?",
- default="llvm/llvm-project",
- help="Head repository (e.g. contributor/llvm-project)",
- )
-
- args = parser.parse_args()
-
- diff_files = DiffParser.parse(args.diff_file)
- cov_data = CoverageJSONParser.load(args.json_file)
- coverage_matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
-
- render_patch_report(
- diff_files,
- coverage_matrix,
- args.base_sha,
- args.head_sha,
- args.base_branch,
- args.head_branch,
- args.targets,
- args.base_repo,
- args.head_repo,
- )
-
-
-if __name__ == "__main__":
- main()
diff --git a/libc/utils/coverage/test_codebase_coverage.py b/libc/utils/coverage/test_codebase_coverage.py
new file mode 100644
index 0000000000000..de9820c0d05c5
--- /dev/null
+++ b/libc/utils/coverage/test_codebase_coverage.py
@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+#
+# ====- Unit tests for codebase coverage analyzer --------------*- python -*--==#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ==-------------------------------------------------------------------------==#
+
+import io
+import json
+import os
+import sys
+import unittest
+from contextlib import redirect_stdout
+from typing import Dict
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from codebase_coverage import (
+ FullCoverageSummary,
+ SubsystemCoverageMetrics,
+ extract_full_coverage_statistics,
+ format_global_summary_table,
+ format_overview_callout,
+ format_subsystem_breakdown_table,
+ render_full_report,
+ resolve_dashboard_url,
+)
+
+
+class TestSubsystemCoverageMetrics(unittest.TestCase):
+ """Tests for mathematical calculation and edge-case handling in metric models."""
+
+ def test_metrics_percentages(self) -> None:
+ metrics = SubsystemCoverageMetrics(
+ name="math",
+ lines_cov=850,
+ lines_tot=1000,
+ func_cov=45,
+ func_tot=50,
+ mcdc_cov=90,
+ mcdc_tot=100,
+ decisions_tot=40,
+ decisions_full=36,
+ )
+ self.assertEqual(metrics.line_pct, 85.0)
+ self.assertEqual(metrics.func_pct, 90.0)
+ self.assertEqual(metrics.mcdc_pct, 90.0)
+ self.assertEqual(metrics.decisions_pct, 90.0)
+ self.assertEqual(metrics.missed_lines, 150)
+
+ def test_zero_division_guard(self) -> None:
+ empty = SubsystemCoverageMetrics(name="empty")
+ self.assertEqual(empty.line_pct, 0.0)
+ self.assertEqual(empty.func_pct, 0.0)
+ self.assertEqual(empty.mcdc_pct, 0.0)
+ self.assertEqual(empty.decisions_pct, 0.0)
+ self.assertEqual(empty.missed_lines, 0)
+
+
+class TestDashboardURLResolution(unittest.TestCase):
+ """Tests for environment variable resolution and URL formatting."""
+
+ def test_custom_environment_variable(self) -> None:
+ os.environ["COVERAGE_DASHBOARD_URL"] = "https://custom-dashboard.internal/"
+ url = resolve_dashboard_url(has_mcdc=False)
+ self.assertEqual(url, "https://custom-dashboard.internal/")
+
+ url_mcdc = resolve_dashboard_url(has_mcdc=True)
+ self.assertEqual(url_mcdc, "https://custom-dashboard.internal/mcdc/")
+
+ def test_github_repository_fallback(self) -> None:
+ os.environ.pop("COVERAGE_DASHBOARD_URL", None)
+ os.environ["GITHUB_REPOSITORY"] = "tapiwagonga/llvm-project"
+
+ url = resolve_dashboard_url(has_mcdc=False)
+ self.assertEqual(url, "https://tapiwagonga.github.io/llvm-project/")
+
+ url_mcdc = resolve_dashboard_url(has_mcdc=True)
+ self.assertEqual(url_mcdc, "https://tapiwagonga.github.io/llvm-project/mcdc/")
+
+
+class TestDataExtraction(unittest.TestCase):
+ """Tests for extracting and aggregating metrics across multi-directory subsystems."""
+
+ def test_multi_subsystem_aggregation(self) -> None:
+ cov_data = {
+ "data": [
+ {
+ "files": [
+ # 1. ctype subsystem
+ {
+ "filename": "libc/src/ctype/isalpha.cpp",
+ "summary": {
+ "lines": {"count": 10, "covered": 10},
+ "functions": {"count": 1, "covered": 1},
+ "mcdc": {"count": 2, "covered": 2},
+ },
+ "mcdc_records": [
+ [10, 1, 10, 20, 2, 2, 2, 1, 1, [True, True]]
+ ],
+ },
+ # 2. math subsystem with nested directory
+ {
+ "filename": "libc/src/math/generic/sin.cpp",
+ "summary": {
+ "lines": {"count": 50, "covered": 40},
+ "functions": {"count": 2, "covered": 2},
+ "mcdc": {"count": 6, "covered": 4},
+ },
+ "mcdc_records": [
+ [15, 1, 15, 30, 2, 1, 2, 1, 1, [True, False]],
+ [25, 1, 25, 30, 2, 2, 2, 1, 1, [True, True]],
+ ],
+ },
+ # 3. support subsystem
+ {
+ "filename": "libc/src/__support/OSUtil/linux/syscall.cpp",
+ "summary": {
+ "lines": {"count": 100, "covered": 90},
+ "functions": {"count": 5, "covered": 5},
+ "mcdc": {"count": 0, "covered": 0},
+ },
+ "mcdc_records": [],
+ },
+ # 4. Ignored test and benchmark files
+ {
+ "filename": "libc/test/src/math/sin_test.cpp",
+ "summary": {"lines": {"count": 200, "covered": 200}},
+ },
+ {
+ "filename": "libc/utils/mathtools/ryu.py",
+ "summary": {"lines": {"count": 80, "covered": 80}},
+ },
+ ]
+ }
+ ]
+ }
+
+ summary = extract_full_coverage_statistics(cov_data)
+ self.assertIsNotNone(summary)
+ assert summary is not None
+
+ # Verify only 3 libc/src subsystems are tracked
+ self.assertEqual(len(summary.subsystems), 3)
+ self.assertIn("src/ctype", summary.subsystems)
+ self.assertIn("src/math", summary.subsystems)
+ self.assertIn("src/__support", summary.subsystems)
+
+ # Verify subsystem-specific metrics
+ math_m = summary.subsystems["src/math"]
+ self.assertEqual(math_m.lines_cov, 40)
+ self.assertEqual(math_m.lines_tot, 50)
+ self.assertEqual(math_m.line_pct, 80.0)
+ self.assertEqual(math_m.mcdc_cov, 4)
+ self.assertEqual(math_m.mcdc_tot, 6)
+ self.assertEqual(math_m.decisions_tot, 2)
+ self.assertEqual(math_m.decisions_full, 1)
+
+ # Verify global aggregate sums
+ self.assertEqual(summary.global_stats.lines_cov, 140) # 10 + 40 + 90
+ self.assertEqual(summary.global_stats.lines_tot, 160) # 10 + 50 + 100
+ self.assertEqual(summary.global_stats.func_cov, 8) # 1 + 2 + 5
+ self.assertEqual(summary.global_stats.mcdc_cov, 6) # 2 + 4 + 0
+ self.assertEqual(summary.global_stats.mcdc_tot, 8) # 2 + 6 + 0
+ self.assertEqual(summary.global_stats.decisions_tot, 3)# 1 + 2 + 0
+ self.assertEqual(summary.global_stats.decisions_full, 2)# 1 + 1 + 0
+ self.assertTrue(summary.has_mcdc)
+
+ def test_empty_or_malformed_json(self) -> None:
+ self.assertIsNone(extract_full_coverage_statistics({}))
+ self.assertIsNone(extract_full_coverage_statistics({"data": []}))
+ self.assertIsNone(extract_full_coverage_statistics({"data": [{"files": []}]}))
+
+
+class TestReportFormatting(unittest.TestCase):
+ """Tests for Markdown table generation, progress indicators, and banners."""
+
+ def test_format_overview_callout(self) -> None:
+ # Standard line mode
+ g_std = SubsystemCoverageMetrics(lines_cov=950, lines_tot=1000)
+ s_std = FullCoverageSummary(global_stats=g_std, dashboard_url="https://llvm.github.io/llvm-project/")
+ callout_std = format_overview_callout(s_std)
+ self.assertIn("95.00%", callout_std)
+ self.assertNotIn("MC/DC", callout_std)
+
+ # MC/DC mode
+ g_mcdc = SubsystemCoverageMetrics(lines_cov=950, lines_tot=1000, mcdc_cov=90, mcdc_tot=100, decisions_tot=40)
+ s_mcdc = FullCoverageSummary(global_stats=g_mcdc, dashboard_url="https://llvm.github.io/llvm-project/mcdc/")
+ callout_mcdc = format_overview_callout(s_mcdc)
+ self.assertIn("95.00% Line", callout_mcdc)
+ self.assertIn("90.00% MC/DC", callout_mcdc)
+ self.assertIn("https://llvm.github.io/llvm-project/mcdc/", callout_mcdc)
+
+ def test_format_global_summary_table(self) -> None:
+ g = SubsystemCoverageMetrics(
+ lines_cov=950, lines_tot=1000, func_cov=98, func_tot=100,
+ mcdc_cov=90, mcdc_tot=100, decisions_full=36, decisions_tot=40
+ )
+ summary = FullCoverageSummary(global_stats=g)
+ table = format_global_summary_table(summary)
+
+ self.assertIn("### Overall", table)
+ self.assertIn("| **MC/DC Condition Independence** | 90 | 100 | **90.00%** |", table)
+ self.assertIn("| **Fully Verified Decisions** | 36 | 40 | **90.00%** |", table)
+ self.assertIn("| **Executable Lines** | 950 | 1,000 | **95.00%** |", table)
+ self.assertIn("| **Functions** | 98 | 100 | **98.00%** |", table)
+
+ def test_format_subsystem_breakdown_table_sorted(self) -> None:
+ sub_metrics = {
+ "src/string": SubsystemCoverageMetrics(name="src/string", lines_cov=20, lines_tot=20),
+ "src/ctype": SubsystemCoverageMetrics(name="src/ctype", lines_cov=10, lines_tot=10),
+ "src/math": SubsystemCoverageMetrics(name="src/math", lines_cov=40, lines_tot=50),
+ }
+ summary = FullCoverageSummary(subsystems=sub_metrics)
+ table = format_subsystem_breakdown_table(summary)
+
+ # Must be alphabetically sorted: ctype, math, string
+ pos_ctype = table.find("`libc/src/ctype`")
+ pos_math = table.find("`libc/src/math`")
+ pos_string = table.find("`libc/src/string`")
+
+ self.assertTrue(pos_ctype < pos_math < pos_string)
+
+ def test_render_full_report_empty_data(self) -> None:
+ stdout_buf = io.StringIO()
+ with redirect_stdout(stdout_buf):
+ render_full_report({})
+ output = stdout_buf.getvalue()
+ self.assertIn("No Coverage Data Detected", output)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/libc/utils/coverage/test_diff_coverage.py b/libc/utils/coverage/test_diff_coverage.py
new file mode 100644
index 0000000000000..da7ed98e7ef28
--- /dev/null
+++ b/libc/utils/coverage/test_diff_coverage.py
@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+#
+# ====- Unit tests for diff coverage analyzer ------------------*- python -*--==#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ==-------------------------------------------------------------------------==#
+
+import io
+import json
+import os
+import sys
+import tempfile
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from typing import Dict, List, Set
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from diff_coverage import (
+ CoverageJSONParser,
+ DiffHunk,
+ DiffParser,
+ FilePatchMetrics,
+ PatchCoverageSummary,
+ calculate_patch_statistics,
+ format_annotated_diff,
+ format_breakdown_table,
+ format_line_ranges,
+ format_metadata_section,
+ format_status_banner,
+ is_executable_line,
+ render_patch_report,
+)
+
+
+class TestDiffParser(unittest.TestCase):
+ """Exhaustive unit tests for unified diff parsing."""
+
+ def test_single_file_single_hunk(self) -> None:
+ raw_diff = """diff --git a/libc/src/ctype/isalpha.cpp b/libc/src/ctype/isalpha.cpp
+--- a/libc/src/ctype/isalpha.cpp
++++ b/libc/src/ctype/isalpha.cpp
+@@ -10,3 +10,4 @@
+ int isalpha(int c) {
++ int x = c;
+ return x;
+ }
+"""
+ files = DiffParser.parse(raw_diff)
+ self.assertIn("libc/src/ctype/isalpha.cpp", files)
+ hunks = files["libc/src/ctype/isalpha.cpp"]
+ self.assertEqual(len(hunks), 1)
+ self.assertEqual(len(hunks[0].lines), 4)
+ self.assertEqual(hunks[0].lines[0], (" ", "int isalpha(int c) {", 10))
+ self.assertEqual(hunks[0].lines[1], ("+", " int x = c;", 11))
+ self.assertEqual(hunks[0].lines[2], (" ", " return x;", 12))
+ self.assertEqual(hunks[0].lines[3], (" ", "}", 13))
+
+ def test_multi_file_multi_hunk(self) -> None:
+ raw_diff = """diff --git a/libc/src/ctype/isalpha.cpp b/libc/src/ctype/isalpha.cpp
+--- a/libc/src/ctype/isalpha.cpp
++++ b/libc/src/ctype/isalpha.cpp
+@@ -5,2 +5,3 @@
++// Header comment
+ int isalpha(int c);
+@@ -20,2 +21,3 @@
++ int z = 1;
+ return z;
+diff --git a/libc/src/math/sin.cpp b/libc/src/math/sin.cpp
+--- a/libc/src/math/sin.cpp
++++ b/libc/src/math/sin.cpp
+@@ -1,3 +1,4 @@
++// Math file
+ double sin(double x) {
++ return x;
+ }
+"""
+ files = DiffParser.parse(raw_diff)
+ self.assertEqual(len(files), 2)
+ self.assertIn("libc/src/ctype/isalpha.cpp", files)
+ self.assertIn("libc/src/math/sin.cpp", files)
+ self.assertEqual(len(files["libc/src/ctype/isalpha.cpp"]), 2)
+ self.assertEqual(len(files["libc/src/math/sin.cpp"]), 1)
+
+ def test_deleted_and_renamed_files(self) -> None:
+ raw_diff = """diff --git a/libc/src/old.cpp b/libc/src/old.cpp
+deleted file mode 100644
+--- a/libc/src/old.cpp
++++ /dev/null
+@@ -1,3 +0,0 @@
+-int old_func();
+"""
+ files = DiffParser.parse(raw_diff)
+ self.assertEqual(len(files), 0)
+
+ def test_diff_from_temporary_file(self) -> None:
+ raw_diff = """diff --git a/libc/src/string/strlen.cpp b/libc/src/string/strlen.cpp
+--- a/libc/src/string/strlen.cpp
++++ b/libc/src/string/strlen.cpp
+@@ -1,2 +1,3 @@
+ size_t strlen(const char *s) {
++ return 0;
+ }
+"""
+ with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as f:
+ f.write(raw_diff)
+ f_path = f.name
+
+ try:
+ files = DiffParser.parse(f_path)
+ self.assertIn("libc/src/string/strlen.cpp", files)
+ self.assertEqual(len(files["libc/src/string/strlen.cpp"][0].lines), 3)
+ finally:
+ os.remove(f_path)
+
+ def test_diff_with_no_newline_warning(self) -> None:
+ raw_diff = """diff --git a/libc/src/stdio/puts.cpp b/libc/src/stdio/puts.cpp
+--- a/libc/src/stdio/puts.cpp
++++ b/libc/src/stdio/puts.cpp
+@@ -1,2 +1,2 @@
+-int puts(const char *s);
++int puts(const char *str);
+\\ No newline at end of file
+"""
+ files = DiffParser.parse(raw_diff)
+ self.assertIn("libc/src/stdio/puts.cpp", files)
+ lines = files["libc/src/stdio/puts.cpp"][0].lines
+ self.assertEqual(len(lines), 1)
+ self.assertEqual(lines[0], ("+", "int puts(const char *str);", 1))
+
+
+class TestExecutableLineFilter(unittest.TestCase):
+ """Exhaustive tests for filtering executable lines vs comments and declarations."""
+
+ def test_non_executable_comments_and_whitespace(self) -> None:
+ test_cases = [
+ "",
+ " ",
+ "\t\t",
+ "// Single line comment",
+ " // Indented comment",
+ "/* Multi-line block start",
+ " * Continuation line",
+ " */ End of comment block",
+ ]
+ for line in test_cases:
+ with self.subTest(line=line):
+ self.assertFalse(is_executable_line(line))
+
+ def test_non_executable_syntax_and_preprocessor(self) -> None:
+ test_cases = [
+ "{",
+ "}",
+ "};",
+ "{};",
+ ": m_val(0)",
+ "#include <stddef.h>",
+ "#define FOO 1",
+ "#ifdef LIBC_ENABLE_COVERAGE",
+ "#endif",
+ "namespace __llvm_libc {",
+ "extern \"C\" {",
+ "using size_t = unsigned long;",
+ "template <typename T>",
+ "typedef int (*func_ptr)(void);",
+ "__attribute__((noinline))",
+ ]
+ for line in test_cases:
+ with self.subTest(line=line):
+ self.assertFalse(is_executable_line(line))
+
+ def test_non_executable_type_definitions(self) -> None:
+ test_cases = [
+ "struct Foo {",
+ "struct Foo;",
+ "class Bar {",
+ "class Bar;",
+ "enum Color {",
+ "enum class Status : int {",
+ ]
+ for line in test_cases:
+ with self.subTest(line=line):
+ self.assertFalse(is_executable_line(line))
+
+ def test_executable_statements(self) -> None:
+ test_cases = [
+ "int x = 5;",
+ "return a + b;",
+ "struct Foo f = init_foo();",
+ "class Bar b(10);",
+ "if (c >= 'a' && c <= 'z')",
+ "for (int i = 0; i < 10; ++i) {",
+ "while (*s++) {",
+ "switch (op) {",
+ "case 1:",
+ "break;",
+ "continue;",
+ "goto cleanup;",
+ "foo(); // inline comment",
+ ]
+ for line in test_cases:
+ with self.subTest(line=line):
+ self.assertTrue(is_executable_line(line))
+
+
+class TestFormatLineRanges(unittest.TestCase):
+ """Tests for line number formatting into human-readable spans."""
+
+ def test_formatting_variations(self) -> None:
+ self.assertEqual(format_line_ranges(set()), "None")
+ self.assertEqual(format_line_ranges({42}), "`L42`")
+ self.assertEqual(format_line_ranges({10, 11, 12}), "`L10-L12`")
+ self.assertEqual(
+ format_line_ranges({5, 6, 7, 10, 15, 16, 20}),
+ "`L5-L7`, `L10`, `L15-L16`, `L20`",
+ )
+
+
+class TestCoverageJSONParser(unittest.TestCase):
+ """Tests for parsing llvm-cov JSON export structures and mapping to diffs."""
+
+ def test_invalid_json_handling(self) -> None:
+ with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as f:
+ f.write("{ invalid json")
+ f_path = f.name
+
+ try:
+ with redirect_stderr(io.StringIO()):
+ with self.assertRaises(SystemExit):
+ CoverageJSONParser.load(f_path)
+ finally:
+ os.remove(f_path)
+
+ def test_extract_patch_matrix_matching(self) -> None:
+ cov_data = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/runner/work/llvm-project/libc/src/math/cos.cpp",
+ "segments": [
+ [10, 1, 5, True, True, False],
+ [12, 1, 0, True, True, False],
+ [14, 1, 0, False, False, False],
+ ],
+ "mcdc_records": [
+ [10, 5, 10, 25, 2, 2, 2, 1, 1, [True, True]]
+ ],
+ }
+ ]
+ }
+ ]
+ }
+ diff_files = {"libc/src/math/cos.cpp": []}
+ matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
+
+ self.assertIn("libc/src/math/cos.cpp", matrix)
+ self.assertIn(10, matrix["libc/src/math/cos.cpp"]["covered"])
+ self.assertIn(11, matrix["libc/src/math/cos.cpp"]["covered"])
+ self.assertIn(12, matrix["libc/src/math/cos.cpp"]["missed"])
+ self.assertEqual(len(matrix["libc/src/math/cos.cpp"]["mcdc_decisions"]), 1)
+
+
+class TestStatisticsAndReporting(unittest.TestCase):
+ """Tests for patch coverage calculation, diagnostic generation, and Markdown rendering."""
+
+ def test_line_coverage_calculation(self) -> None:
+ diff_text = """diff --git a/libc/src/math/fabs.cpp b/libc/src/math/fabs.cpp
+--- a/libc/src/math/fabs.cpp
++++ b/libc/src/math/fabs.cpp
+@@ -10,3 +10,5 @@
+ double fabs(double x) {
++ if (x < 0)
++ return -x;
+ return x;
+ }
+"""
+ diff_files = DiffParser.parse(diff_text)
+ cov_data = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "libc/src/math/fabs.cpp",
+ "segments": [
+ [10, 1, 10, True, True, False],
+ [12, 1, 0, True, True, False],
+ [14, 1, 0, False, False, False],
+ ],
+ }
+ ]
+ }
+ ]
+ }
+ matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
+ stats = calculate_patch_statistics(diff_files, matrix)
+
+ self.assertEqual(stats.total_covered_lines, 1) # line 11 (if x < 0)
+ self.assertEqual(stats.total_missed_lines, 1) # line 12 (return -x)
+ self.assertEqual(stats.total_lines, 2)
+ self.assertEqual(stats.line_coverage_pct, 50.0)
+ self.assertFalse(stats.has_mcdc)
+
+ def test_mcdc_decision_diagnostics(self) -> None:
+ diff_text = """diff --git a/libc/src/ctype/isspace.cpp b/libc/src/ctype/isspace.cpp
+--- a/libc/src/ctype/isspace.cpp
++++ b/libc/src/ctype/isspace.cpp
+@@ -10,2 +10,3 @@
+ int isspace(int c) {
++ if (c == ' ' || c == '\t' || c == '\n')
+ return 1;
+"""
+ diff_files = DiffParser.parse(diff_text)
+ cov_data = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "libc/src/ctype/isspace.cpp",
+ "segments": [
+ [10, 1, 10, True, True, False],
+ [13, 1, 0, False, False, False],
+ ],
+ "mcdc_records": [
+ [11, 7, 11, 40, 3, 2, 3, 1, 1, [True, True, False]]
+ ],
+ }
+ ]
+ }
+ ]
+ }
+ matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
+ stats = calculate_patch_statistics(diff_files, matrix)
+
+ self.assertTrue(stats.has_mcdc)
+ self.assertEqual(stats.total_mcdc_cov, 2)
+ self.assertEqual(stats.total_mcdc_tot, 3)
+ self.assertEqual(stats.mcdc_coverage_pct, 66.66666666666666)
+ self.assertEqual(stats.total_decisions_count, 1)
+ self.assertEqual(stats.fully_verified_decisions, 0)
+
+ f_metrics = stats.files["libc/src/ctype/isspace.cpp"]
+ self.assertIn("C3 unverified", f_metrics.condition_diagnostics[0])
+ self.assertEqual(f_metrics.unverified_decision_lines[11], ["C3"])
+
+ def test_format_status_banner(self) -> None:
+ # Full statement & full MC/DC
+ s1 = PatchCoverageSummary(
+ total_covered_lines=10,
+ total_missed_lines=0,
+ total_mcdc_cov=4,
+ total_mcdc_tot=4,
+ total_decisions_count=2,
+ fully_verified_decisions=2,
+ )
+ banner1 = format_status_banner(s1)
+ self.assertIn("> [!TIP]", banner1)
+ self.assertIn("100.00% Line", banner1)
+ self.assertIn("100.00% MC/DC", banner1)
+
+ # Full statement, partial MC/DC
+ s2 = PatchCoverageSummary(
+ total_covered_lines=10,
+ total_missed_lines=0,
+ total_mcdc_cov=3,
+ total_mcdc_tot=4,
+ total_decisions_count=2,
+ fully_verified_decisions=1,
+ )
+ banner2 = format_status_banner(s2)
+ self.assertIn("> [!NOTE]", banner2)
+ self.assertIn("75.0% MC/DC", banner2)
+
+ # Warning when lines missed
+ s3 = PatchCoverageSummary(
+ total_covered_lines=8,
+ total_missed_lines=2,
+ )
+ banner3 = format_status_banner(s3)
+ self.assertIn("> [!WARNING]", banner3)
+ self.assertIn("80.00%", banner3)
+ self.assertIn("unexecuted lines detected in patch", banner3)
+
+ def test_format_metadata_section(self) -> None:
+ meta = format_metadata_section(
+ base_sha="abcdef1234567890",
+ head_sha="123456abcdef7890",
+ base_branch="main",
+ head_branch="my-pr",
+ targets_str="libc.test.src.math.sin_test libc.test.src.math.cos_test",
+ base_repo="llvm/llvm-project",
+ head_repo="user/llvm-project",
+ )
+ self.assertIn("abcdef1", meta)
+ self.assertIn("123456a", meta)
+ self.assertIn("`libc.test.src.math.sin_test`, `libc.test.src.math.cos_test`", meta)
+
+ def test_format_breakdown_table_standard_and_mcdc(self) -> None:
+ fm = FilePatchMetrics(
+ fpath="libc/src/math/tan.cpp",
+ covered_lines={10, 11},
+ missed_lines={12},
+ mcdc_cov=2,
+ mcdc_tot=2,
+ decisions_ver=1,
+ decisions_tot=1,
+ condition_diagnostics=["`L10`: 2/2 verified"],
+ )
+ summary_mcdc = PatchCoverageSummary(
+ files={"libc/src/math/tan.cpp": fm},
+ total_covered_lines=2,
+ total_missed_lines=1,
+ total_mcdc_cov=2,
+ total_mcdc_tot=2,
+ total_decisions_count=1,
+ fully_verified_decisions=1,
+ )
+ table_mcdc = format_breakdown_table(summary_mcdc, head_repo="llvm/llvm-project")
+ self.assertIn("MC/DC Conditions", table_mcdc)
+ self.assertIn("`L10`: 2/2 verified", table_mcdc)
+
+ summary_std = PatchCoverageSummary(
+ files={"libc/src/math/tan.cpp": fm},
+ total_covered_lines=2,
+ total_missed_lines=1,
+ )
+ table_std = format_breakdown_table(summary_std, head_repo="llvm/llvm-project")
+ self.assertIn("Unexecuted Line Spans", table_std)
+ self.assertIn("`L12`", table_std)
+
+ def test_render_patch_report_empty_diff(self) -> None:
+ stdout_buf = io.StringIO()
+ with redirect_stdout(stdout_buf):
+ render_patch_report(
+ diff_files={},
+ coverage_matrix={},
+ base_sha="base",
+ head_sha="head",
+ base_branch="main",
+ head_branch="patch",
+ )
+ output = stdout_buf.getvalue()
+ self.assertIn("Coverage Validated", output)
+ self.assertIn("No `.cpp` source files in `libc/src/` were modified", output)
+
+
+if __name__ == "__main__":
+ unittest.main()
>From ca8a02dca1d05e521c91177bfd07d2d74841fca4 Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Thu, 27 Aug 2026 09:18:02 +0000
Subject: [PATCH 04/15] [libc][ci] Refactor coverage reporting utilities and
test suites
---
libc/utils/coverage/codebase_coverage.py | 141 ++---
libc/utils/coverage/diff_coverage.py | 534 +++++++++---------
libc/utils/coverage/test_codebase_coverage.py | 58 +-
libc/utils/coverage/test_diff_coverage.py | 117 ++--
4 files changed, 422 insertions(+), 428 deletions(-)
diff --git a/libc/utils/coverage/codebase_coverage.py b/libc/utils/coverage/codebase_coverage.py
index cb7b82f531145..7876c8e40106e 100644
--- a/libc/utils/coverage/codebase_coverage.py
+++ b/libc/utils/coverage/codebase_coverage.py
@@ -1,18 +1,18 @@
#!/usr/bin/env python3
#
-# ====- Generate codebase coverage reports ---------------------*- python -*--==#
+# ===- Generate codebase coverage reports --------------------*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
-# ==-------------------------------------------------------------------------==#
+# ==------------------------------------------------------------------------==#
"""
-Standalone analyzer for generating whole-codebase statement, branch, and MC/DC coverage reports.
+Standalone file for generating whole-codebase statement, branch, and MC/DC coverage reports.
This script parses full-codebase `llvm-cov export` JSON files, aggregates metrics
-across all top-level LLVM-libc subsystems (e.g. `src/ctype`, `src/math`, `src/string`),
+across all top-level LLVM-libc directories (e.g. `src/ctype`, `src/math`, `src/string`),
and outputs Markdown summary tables for CI step summaries.
"""
@@ -25,20 +25,12 @@
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
-# -----------------------------------------------------------------------------
-# Constants & Configuration
-# -----------------------------------------------------------------------------
-
DEFAULT_REPOSITORY = "llvm/llvm-project"
-# -----------------------------------------------------------------------------
-# Data Models
-# -----------------------------------------------------------------------------
-
@dataclass
-class SubsystemCoverageMetrics:
- """Encapsulates coverage metrics and boolean decision counts for a subsystem or whole codebase."""
+class DirectoryCoverageMetrics:
+ """Encapsulates coverage metrics and boolean decision counts for a directory or whole codebase."""
name: str = ""
lines_cov: int = 0
@@ -78,10 +70,10 @@ def missed_lines(self) -> int:
@dataclass
class FullCoverageSummary:
- """Encapsulates global and subsystem-level coverage statistics across LLVM-libc."""
+ """Encapsulates global and directory-level coverage statistics across LLVM-libc."""
- global_stats: SubsystemCoverageMetrics = field(default_factory=SubsystemCoverageMetrics)
- subsystems: Dict[str, SubsystemCoverageMetrics] = field(default_factory=dict)
+ global_stats: DirectoryCoverageMetrics = field(default_factory=DirectoryCoverageMetrics)
+ directories: Dict[str, DirectoryCoverageMetrics] = field(default_factory=dict)
dashboard_url: str = ""
@property
@@ -90,10 +82,6 @@ def has_mcdc(self) -> bool:
return self.global_stats.mcdc_tot > 0
-# -----------------------------------------------------------------------------
-# Data Extraction & Aggregation
-# -----------------------------------------------------------------------------
-
def resolve_dashboard_url(has_mcdc: bool) -> str:
"""Resolves the live dashboard URL based on repository environment variables."""
pages_url = os.environ.get("COVERAGE_DASHBOARD_URL")
@@ -111,22 +99,22 @@ def resolve_dashboard_url(has_mcdc: bool) -> str:
def extract_full_coverage_statistics(cov_data: dict) -> Optional[FullCoverageSummary]:
- """Extracts global and per-subsystem metrics from llvm-cov export JSON data."""
+ """Extracts global and per-directory metrics from llvm-cov export JSON data."""
if "data" not in cov_data or not cov_data["data"]:
return None
- global_m = SubsystemCoverageMetrics(name="global")
- subsystems: Dict[str, SubsystemCoverageMetrics] = {}
+ global_metrics = DirectoryCoverageMetrics(name="global")
+ directories: Dict[str, DirectoryCoverageMetrics] = {}
for item in cov_data["data"][0].get("files", []):
- fpath = item.get("filename", "")
- if "src/" not in fpath or "/test/" in fpath or "/utils/" in fpath:
+ file_path = item.get("filename", "")
+ if "src/" not in file_path or "/test/" in file_path or "/utils/" in file_path:
continue
- idx = fpath.find("src/")
+ idx = file_path.find("src/")
if idx == -1:
continue
- rel_path = fpath[idx:]
+ rel_path = file_path[idx:]
summary = item.get("summary", {})
lines_summary = summary.get("lines", {})
@@ -149,64 +137,60 @@ def extract_full_coverage_statistics(cov_data: dict) -> Optional[FullCoverageSum
1 for rec in mcdc_records if len(rec) >= 10 and isinstance(rec[9], list) and all(rec[9])
)
- global_m.lines_cov += line_cov
- global_m.lines_tot += line_tot
- global_m.func_cov += func_cov
- global_m.func_tot += func_tot
- global_m.mcdc_cov += mcdc_cov
- global_m.mcdc_tot += mcdc_tot
- global_m.decisions_tot += file_decisions_tot
- global_m.decisions_full += file_decisions_full
+ global_metrics.lines_cov += line_cov
+ global_metrics.lines_tot += line_tot
+ global_metrics.func_cov += func_cov
+ global_metrics.func_tot += func_tot
+ global_metrics.mcdc_cov += mcdc_cov
+ global_metrics.mcdc_tot += mcdc_tot
+ global_metrics.decisions_tot += file_decisions_tot
+ global_metrics.decisions_full += file_decisions_full
parts = rel_path.split("/")
- subsystem_name = "/".join(parts[:2]) if len(parts) >= 2 else parts[0]
-
- if subsystem_name not in subsystems:
- subsystems[subsystem_name] = SubsystemCoverageMetrics(name=subsystem_name)
-
- sub_m = subsystems[subsystem_name]
- sub_m.lines_cov += line_cov
- sub_m.lines_tot += line_tot
- sub_m.func_cov += func_cov
- sub_m.func_tot += func_tot
- sub_m.mcdc_cov += mcdc_cov
- sub_m.mcdc_tot += mcdc_tot
- sub_m.decisions_tot += file_decisions_tot
- sub_m.decisions_full += file_decisions_full
-
- if global_m.lines_tot == 0:
+ directory_name = "/".join(parts[:2]) if len(parts) >= 2 else parts[0]
+
+ if directory_name not in directories:
+ directories[directory_name] = DirectoryCoverageMetrics(name=directory_name)
+
+ dir_metrics = directories[directory_name]
+ dir_metrics.lines_cov += line_cov
+ dir_metrics.lines_tot += line_tot
+ dir_metrics.func_cov += func_cov
+ dir_metrics.func_tot += func_tot
+ dir_metrics.mcdc_cov += mcdc_cov
+ dir_metrics.mcdc_tot += mcdc_tot
+ dir_metrics.decisions_tot += file_decisions_tot
+ dir_metrics.decisions_full += file_decisions_full
+
+ if global_metrics.lines_tot == 0:
return None
- has_mcdc = global_m.mcdc_tot > 0
+ has_mcdc = global_metrics.mcdc_tot > 0
dashboard_url = resolve_dashboard_url(has_mcdc)
return FullCoverageSummary(
- global_stats=global_m,
- subsystems=subsystems,
+ global_stats=global_metrics,
+ directories=directories,
dashboard_url=dashboard_url,
)
-# -----------------------------------------------------------------------------
-# Report Formatting
-# -----------------------------------------------------------------------------
-
def format_overview_callout(summary: FullCoverageSummary) -> str:
"""Generates the executive summary banner with dashboard link."""
g = summary.global_stats
- lines: List[str] = ["> [!NOTE]"]
+ lines: List[str] = []
if summary.has_mcdc:
lines.append(
- f"> ### Overall Codebase Coverage: **{g.line_pct:.2f}% Line** | **{g.mcdc_pct:.2f}% MC/DC**"
+ f"### Overall Codebase Coverage: **{g.line_pct:.2f}% Line** | **{g.mcdc_pct:.2f}% MC/DC**"
)
lines.append(
- f"> Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines and **{g.mcdc_cov:,} / {g.mcdc_tot:,}** boolean conditions across **{g.decisions_tot:,}** decisions."
+ f"Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines and **{g.mcdc_cov:,} / {g.mcdc_tot:,}** boolean conditions across **{g.decisions_tot:,}** decisions."
)
else:
- lines.append(f"> ### Overall Codebase Coverage: **{g.line_pct:.2f}%**")
+ lines.append(f"### Overall Codebase Coverage: **{g.line_pct:.2f}%**")
lines.append(
- f"> Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines across all LLVM-libc subsystems."
+ f"Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines across all LLVM-libc directories."
)
lines.append("")
@@ -240,24 +224,24 @@ def format_global_summary_table(summary: FullCoverageSummary) -> str:
return "\n".join(lines)
-def format_subsystem_breakdown_table(summary: FullCoverageSummary) -> str:
- """Generates the subsystem breakdown table."""
+def format_directory_breakdown_table(summary: FullCoverageSummary) -> str:
+ """Generates the directory breakdown table."""
lines: List[str] = ["### Coverage Breakdown"]
has_mcdc = summary.has_mcdc
if has_mcdc:
lines.append(
- "| Subsystem | MC/DC Conditions | Decisions (Verified / Total) | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
+ "| Directory | MC/DC Conditions | Decisions (Verified / Total) | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
)
lines.append("| :--- | :---: | :---: | :---: | :---: | :---: | :---: |")
else:
lines.append(
- "| Subsystem | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
+ "| Directory | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
)
lines.append("| :--- | :---: | :---: | :---: | :---: |")
- for sub_name in sorted(summary.subsystems.keys()):
- data = summary.subsystems[sub_name]
+ for dir_name in sorted(summary.directories.keys()):
+ data = summary.directories[dir_name]
if has_mcdc:
mc_cell = (
f"**{data.mcdc_pct:.1f}%** ({data.mcdc_cov}/{data.mcdc_tot})"
@@ -270,11 +254,11 @@ def format_subsystem_breakdown_table(summary: FullCoverageSummary) -> str:
else "N/A"
)
lines.append(
- f"| `libc/{sub_name}` | {mc_cell} | {dec_cell} | **{data.line_pct:.2f}%** | {data.func_pct:.2f}% | {data.lines_tot:,} | {data.missed_lines:,} |"
+ f"| `libc/{dir_name}` | {mc_cell} | {dec_cell} | **{data.line_pct:.2f}%** | {data.func_pct:.2f}% | {data.lines_tot:,} | {data.missed_lines:,} |"
)
else:
lines.append(
- f"| `libc/{sub_name}` | **{data.line_pct:.2f}%** | {data.func_pct:.2f}% | {data.lines_tot:,} | {data.missed_lines:,} |"
+ f"| `libc/{dir_name}` | **{data.line_pct:.2f}%** | {data.func_pct:.2f}% | {data.lines_tot:,} | {data.missed_lines:,} |"
)
return "\n".join(lines)
@@ -287,9 +271,8 @@ def render_full_report(cov_data: dict) -> None:
print("## LLVM-libc Full Codebase Coverage Report\n")
if not summary:
- print("> [!WARNING]")
- print("> ### No Coverage Data Detected")
- print("> The test execution completed but no coverage profiles were exported.")
+ print("### No Coverage Data Detected")
+ print("The test execution completed but no coverage profiles were exported.")
return
# 1. Executive Callout Banner
@@ -300,13 +283,9 @@ def render_full_report(cov_data: dict) -> None:
print(format_global_summary_table(summary))
print("")
- # 3. Subsystem Breakdown Table
- print(format_subsystem_breakdown_table(summary))
-
+ # 3. Directory Breakdown Table
+ print(format_directory_breakdown_table(summary))
-# -----------------------------------------------------------------------------
-# CLI Entry Point
-# -----------------------------------------------------------------------------
def main() -> None:
"""Parses command-line arguments and triggers report generation."""
diff --git a/libc/utils/coverage/diff_coverage.py b/libc/utils/coverage/diff_coverage.py
index 7c95f90761a22..38f043e45fa44 100644
--- a/libc/utils/coverage/diff_coverage.py
+++ b/libc/utils/coverage/diff_coverage.py
@@ -1,12 +1,12 @@
#!/usr/bin/env python3
#
-# ====- Generate diff coverage reports -------------------------*- python -*--==#
+# ===- Generate diff coverage reports ------------------------*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
-# ==-------------------------------------------------------------------------==#
+# ==------------------------------------------------------------------------==#
"""
Standalone analyzer for evaluating diff-level statement, branch, and MC/DC coverage.
@@ -26,12 +26,8 @@
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Set, Tuple
-# -----------------------------------------------------------------------------
-# Constants & Configuration
-# -----------------------------------------------------------------------------
-
-DEFAULT_BASE_REPO = "llvm/llvm-project"
-DEFAULT_HEAD_REPO = "llvm/llvm-project"
+DEFAULT_BASE_REPOSITORY = "llvm/llvm-project"
+DEFAULT_HEAD_REPOSITORY = "llvm/llvm-project"
COMMENT_PREFIXES = ("//", "/*", "*", "*/")
STRUCTURAL_TOKENS = ("{", "}", "};", "{};")
@@ -44,11 +40,6 @@
"typedef ",
)
-
-# -----------------------------------------------------------------------------
-# Data Models
-# -----------------------------------------------------------------------------
-
@dataclass
class DiffHunk:
"""Represents a unified diff hunk with its header and line tokens."""
@@ -56,19 +47,18 @@ class DiffHunk:
header: str
lines: List[Tuple[str, str, int]] = field(default_factory=list) # (prefix, text, line_number)
-
@dataclass
class FilePatchMetrics:
"""Encapsulates coverage metrics and decision records for a single modified file."""
- fpath: str
+ file_path: str
covered_lines: Set[int] = field(default_factory=set)
missed_lines: Set[int] = field(default_factory=set)
added_lines: Set[int] = field(default_factory=set)
- mcdc_cov: int = 0
- mcdc_tot: int = 0
- decisions_ver: int = 0
- decisions_tot: int = 0
+ mcdc_covered_conditions: int = 0
+ mcdc_total_conditions: int = 0
+ decisions_verified: int = 0
+ decisions_total: int = 0
condition_diagnostics: List[str] = field(default_factory=list)
unverified_decision_lines: Dict[int, List[str]] = field(default_factory=dict)
@@ -78,15 +68,22 @@ def total_lines(self) -> int:
return len(self.covered_lines) + len(self.missed_lines)
@property
- def line_coverage_pct(self) -> float:
+ def line_coverage_percentage(self) -> float:
"""Percentage of executed patch lines."""
- return (len(self.covered_lines) / self.total_lines * 100.0) if self.total_lines > 0 else 0.0
+ return (
+ (len(self.covered_lines) / self.total_lines * 100.0)
+ if self.total_lines > 0
+ else 0.0
+ )
@property
- def mcdc_coverage_pct(self) -> float:
+ def mcdc_coverage_percentage(self) -> float:
"""Percentage of independent boolean conditions evaluated."""
- return (self.mcdc_cov / self.mcdc_tot * 100.0) if self.mcdc_tot > 0 else 0.0
-
+ return (
+ (self.mcdc_covered_conditions / self.mcdc_total_conditions * 100.0)
+ if self.mcdc_total_conditions > 0
+ else 0.0
+ )
@dataclass
class PatchCoverageSummary:
@@ -95,8 +92,8 @@ class PatchCoverageSummary:
files: Dict[str, FilePatchMetrics] = field(default_factory=dict)
total_covered_lines: int = 0
total_missed_lines: int = 0
- total_mcdc_cov: int = 0
- total_mcdc_tot: int = 0
+ total_mcdc_covered_conditions: int = 0
+ total_mcdc_total_conditions: int = 0
total_decisions_count: int = 0
fully_verified_decisions: int = 0
@@ -106,25 +103,29 @@ def total_lines(self) -> int:
return self.total_covered_lines + self.total_missed_lines
@property
- def line_coverage_pct(self) -> float:
+ def line_coverage_percentage(self) -> float:
"""Aggregated patch line coverage percentage."""
- return (self.total_covered_lines / self.total_lines * 100.0) if self.total_lines > 0 else 0.0
+ return (
+ (self.total_covered_lines / self.total_lines * 100.0)
+ if self.total_lines > 0
+ else 0.0
+ )
@property
- def mcdc_coverage_pct(self) -> float:
+ def mcdc_coverage_percentage(self) -> float:
"""Aggregated patch MC/DC condition coverage percentage."""
- return (self.total_mcdc_cov / self.total_mcdc_tot * 100.0) if self.total_mcdc_tot > 0 else 0.0
+ return (
+ (self.total_mcdc_covered_conditions / self.total_mcdc_total_conditions * 100.0)
+ if self.total_mcdc_total_conditions > 0
+ else 0.0
+ )
@property
def has_mcdc(self) -> bool:
"""Returns True if any MC/DC decision records intersect the patch."""
- return self.total_mcdc_tot > 0
+ return self.total_mcdc_total_conditions > 0
-# -----------------------------------------------------------------------------
-# Parsing Utilities
-# -----------------------------------------------------------------------------
-
class DiffParser:
"""Parses unified diff outputs into structured file hunks with line numbers."""
@@ -134,11 +135,11 @@ def parse(diff_source: str) -> Dict[str, List[DiffHunk]]:
files: Dict[str, List[DiffHunk]] = {}
current_file: Optional[str] = None
current_hunk: Optional[DiffHunk] = None
- current_line_num: int = 0
+ current_line_number: int = 0
if os.path.isfile(diff_source):
- with open(diff_source, "r", encoding="utf-8") as f:
- lines = f.readlines()
+ with open(diff_source, "r", encoding="utf-8", errors="replace") as file_handle:
+ lines = file_handle.readlines()
else:
lines = diff_source.splitlines(keepends=True)
@@ -160,9 +161,9 @@ def parse(diff_source: str) -> Dict[str, List[DiffHunk]]:
continue
if line.startswith("@@"):
- match = re.search(r"\+([0-9]+)", line)
- if match:
- current_line_num = int(match.group(1))
+ hunk_match = re.search(r"\+([0-9]+)", line)
+ if hunk_match:
+ current_line_number = int(hunk_match.group(1))
current_hunk = DiffHunk(header=line)
files[current_file].append(current_hunk)
continue
@@ -173,11 +174,11 @@ def parse(diff_source: str) -> Dict[str, List[DiffHunk]]:
if line.startswith("-"):
continue
elif line.startswith("+"):
- current_hunk.lines.append(("+", line[1:], current_line_num))
- current_line_num += 1
+ current_hunk.lines.append(("+", line[1:], current_line_number))
+ current_line_number += 1
elif line.startswith(" "):
- current_hunk.lines.append((" ", line[1:], current_line_num))
- current_line_num += 1
+ current_hunk.lines.append((" ", line[1:], current_line_number))
+ current_line_number += 1
return files
@@ -189,122 +190,126 @@ class CoverageJSONParser:
def load(json_path: str) -> dict:
"""Loads JSON file from disk with error reporting."""
try:
- with open(json_path, "r", encoding="utf-8") as f:
- return json.load(f)
- except Exception as err:
- sys.stderr.write(f"Error: Failed to parse coverage JSON from '{json_path}': {err}\n")
+ with open(json_path, "r", encoding="utf-8") as file_handle:
+ return json.load(file_handle)
+ except Exception as error:
+ sys.stderr.write(
+ f"Error: Failed to parse coverage JSON from '{json_path}': {error}\n"
+ )
sys.exit(1)
@staticmethod
def extract_patch_matrix(
- cov_data: dict, diff_files: Dict[str, List[DiffHunk]]
+ coverage_data: dict, diff_files: Dict[str, List[DiffHunk]]
) -> Dict[str, Dict[str, Any]]:
"""Maps statement coverage segments and MC/DC decision records to modified files."""
coverage_matrix: Dict[str, Dict[str, Any]] = {
- fpath: {"covered": set(), "missed": set(), "mcdc_decisions": []}
- for fpath in diff_files.keys()
+ file_path: {"covered": set(), "missed": set(), "mcdc_decisions": []}
+ for file_path in diff_files.keys()
}
- if "data" not in cov_data or not cov_data["data"]:
+ if "data" not in coverage_data or not coverage_data["data"]:
return coverage_matrix
- for item in cov_data["data"][0].get("files", []):
- fpath = item.get("filename", "")
- rel_path = next(
+ for item in coverage_data["data"][0].get("files", []):
+ file_name = item.get("filename", "")
+ relative_file_path = next(
(
- rp
- for rp in diff_files.keys()
- if fpath == rp or fpath.endswith("/" + rp) or rp.endswith("/" + fpath)
+ target_path
+ for target_path in diff_files.keys()
+ if file_name == target_path
+ or file_name.endswith("/" + target_path)
+ or target_path.endswith("/" + file_name)
),
None,
)
- if not rel_path:
+ if not relative_file_path:
continue
# 1. Process statement coverage segments
segments = item.get("segments", [])
- for i, current in enumerate(segments):
- line_start = current[0]
- count = current[2]
- has_count = current[3]
+ for index, current_segment in enumerate(segments):
+ line_start = current_segment[0]
+ execution_count = current_segment[2]
+ has_execution_count = current_segment[3]
- if not has_count:
+ if not has_execution_count:
continue
- if i < len(segments) - 1:
- nxt = segments[i + 1]
- line_end = nxt[0]
- end_range = line_end if line_end > line_start else line_start + 1
+ if index < len(segments) - 1:
+ next_segment = segments[index + 1]
+ next_line = next_segment[0]
+ end_range = next_line if next_line > line_start else line_start + 1
else:
end_range = line_start + 1
- for line_num in range(line_start, end_range):
- if count > 0:
- coverage_matrix[rel_path]["covered"].add(line_num)
+ for line_number in range(line_start, end_range):
+ if execution_count > 0:
+ coverage_matrix[relative_file_path]["covered"].add(line_number)
else:
- coverage_matrix[rel_path]["missed"].add(line_num)
+ coverage_matrix[relative_file_path]["missed"].add(line_number)
# 2. Process MC/DC decision records
mcdc_records = item.get("mcdc_records", [])
- for rec in mcdc_records:
- if len(rec) >= 10 and isinstance(rec[9], list):
- l_start = rec[0]
- l_end = rec[2]
- conds = rec[9]
- cov_conds = sum(1 for c in conds if c)
- coverage_matrix[rel_path]["mcdc_decisions"].append(
+ for record in mcdc_records:
+ if len(record) >= 10 and isinstance(record[9], list):
+ decision_start_line = record[0]
+ decision_end_line = record[2]
+ boolean_conditions = record[9]
+ covered_conditions_count = sum(1 for condition in boolean_conditions if condition)
+ coverage_matrix[relative_file_path]["mcdc_decisions"].append(
{
- "line_start": l_start,
- "line_end": l_end,
- "conditions": conds,
- "covered": cov_conds,
- "total": len(conds),
+ "line_start": decision_start_line,
+ "line_end": decision_end_line,
+ "conditions": boolean_conditions,
+ "covered": covered_conditions_count,
+ "total": len(boolean_conditions),
}
)
return coverage_matrix
-def is_executable_line(text: str) -> bool:
+def is_executable_line(line_text: str) -> bool:
"""Filters out non-executable code lines (comments, braces, pure declarations)."""
- s = text.strip()
- if not s:
+ stripped_line = line_text.strip()
+ if not stripped_line:
return False
- if any(s.startswith(prefix) for prefix in COMMENT_PREFIXES):
+ if any(stripped_line.startswith(prefix) for prefix in COMMENT_PREFIXES):
return False
- if s in STRUCTURAL_TOKENS or s.startswith(":"):
+ if stripped_line in STRUCTURAL_TOKENS or stripped_line.startswith(":"):
return False
- if s.startswith("#"):
+ if stripped_line.startswith("#"):
return False
- if any(s.startswith(prefix) for prefix in DECLARATION_PREFIXES):
+ if any(stripped_line.startswith(prefix) for prefix in DECLARATION_PREFIXES):
return False
- if s.startswith("struct ") or s.startswith("class ") or s.startswith("enum "):
- if "{" in s or (s.endswith(";") and "=" not in s and "(" not in s):
+ if stripped_line.startswith("struct ") or stripped_line.startswith("class ") or stripped_line.startswith("enum "):
+ if "{" in stripped_line or (stripped_line.endswith(";") and "=" not in stripped_line and "(" not in stripped_line):
return False
return True
-def format_line_ranges(lines: Set[int]) -> str:
+def format_line_ranges(line_numbers: Set[int]) -> str:
"""Formats an integer set of line numbers into concise span representations."""
- if not lines:
+ if not line_numbers:
return "None"
- sorted_lines = sorted(lines)
- ranges: List[str] = []
- start = sorted_lines[0]
- end = sorted_lines[0]
- for n in sorted_lines[1:]:
- if n == end + 1:
- end = n
+ sorted_line_numbers = sorted(line_numbers)
+ formatted_ranges: List[str] = []
+ start_line = sorted_line_numbers[0]
+ end_line = sorted_line_numbers[0]
+ for current_number in sorted_line_numbers[1:]:
+ if current_number == end_line + 1:
+ end_line = current_number
else:
- ranges.append(f"`L{start}-L{end}`" if start != end else f"`L{start}`")
- start = end = n
- ranges.append(f"`L{start}-L{end}`" if start != end else f"`L{start}`")
- return ", ".join(ranges)
-
+ formatted_ranges.append(
+ f"`L{start_line}-L{end_line}`" if start_line != end_line else f"`L{start_line}`"
+ )
+ start_line = end_line = current_number
+ formatted_ranges.append(
+ f"`L{start_line}-L{end_line}`" if start_line != end_line else f"`L{start_line}`"
+ )
+ return ", ".join(formatted_ranges)
-# -----------------------------------------------------------------------------
-# Statistics Calculation
-# -----------------------------------------------------------------------------
def calculate_patch_statistics(
diff_files: Dict[str, List[DiffHunk]],
@@ -313,141 +318,143 @@ def calculate_patch_statistics(
"""Calculates granular line, branch, and MC/DC statistics for all modified patch files."""
summary = PatchCoverageSummary()
- for fpath, data in coverage_matrix.items():
+ for file_path, file_data in coverage_matrix.items():
added_lines: Set[int] = set()
- for hunk in diff_files.get(fpath, []):
- for l_type, text, l_num in hunk.lines:
- if l_type == "+" and is_executable_line(text):
- added_lines.add(l_num)
+ for hunk in diff_files.get(file_path, []):
+ for line_type, text, line_number in hunk.lines:
+ if line_type == "+" and is_executable_line(text):
+ added_lines.add(line_number)
if not added_lines:
continue
- f_covered = added_lines.intersection(data["covered"])
- f_missed = (added_lines.intersection(data["missed"])) - f_covered
+ file_covered_lines = added_lines.intersection(file_data["covered"])
+ file_missed_lines = (added_lines.intersection(file_data["missed"])) - file_covered_lines
- file_metric = FilePatchMetrics(
- fpath=fpath,
+ file_metrics = FilePatchMetrics(
+ file_path=file_path,
added_lines=added_lines,
)
- if len(data["covered"]) > 0 or len(data["missed"]) > 0:
- file_metric.covered_lines = f_covered
- file_metric.missed_lines = f_missed
- summary.total_covered_lines += len(f_covered)
- summary.total_missed_lines += len(f_missed)
+ if len(file_data["covered"]) > 0 or len(file_data["missed"]) > 0:
+ file_metrics.covered_lines = file_covered_lines
+ file_metrics.missed_lines = file_missed_lines
+ summary.total_covered_lines += len(file_covered_lines)
+ summary.total_missed_lines += len(file_missed_lines)
else:
- file_metric.missed_lines = added_lines
+ file_metrics.missed_lines = added_lines
summary.total_missed_lines += len(added_lines)
# Evaluate MC/DC decision records intersecting modified lines
- for decision in data.get("mcdc_decisions", []):
- d_start = decision["line_start"]
- d_end = decision["line_end"]
- if any(d_start <= l <= d_end for l in added_lines):
+ for decision in file_data.get("mcdc_decisions", []):
+ decision_start_line = decision["line_start"]
+ decision_end_line = decision["line_end"]
+ if any(decision_start_line <= line_number <= decision_end_line for line_number in added_lines):
summary.total_decisions_count += 1
- file_metric.decisions_tot += 1
- file_metric.mcdc_cov += decision["covered"]
- file_metric.mcdc_tot += decision["total"]
- summary.total_mcdc_cov += decision["covered"]
- summary.total_mcdc_tot += decision["total"]
+ file_metrics.decisions_total += 1
+ file_metrics.mcdc_covered_conditions += decision["covered"]
+ file_metrics.mcdc_total_conditions += decision["total"]
+ summary.total_mcdc_covered_conditions += decision["covered"]
+ summary.total_mcdc_total_conditions += decision["total"]
if decision["covered"] == decision["total"]:
summary.fully_verified_decisions += 1
- file_metric.decisions_ver += 1
- file_metric.condition_diagnostics.append(
- f"`L{d_start}`: {decision['covered']}/{decision['total']} verified"
+ file_metrics.decisions_verified += 1
+ file_metrics.condition_diagnostics.append(
+ f"`L{decision_start_line}`: {decision['covered']}/{decision['total']} verified"
)
else:
- uncovered_idx = [
- f"C{i+1}"
- for i, is_cov in enumerate(decision["conditions"])
- if not is_cov
+ uncovered_indices = [
+ f"C{condition_index + 1}"
+ for condition_index, is_covered in enumerate(decision["conditions"])
+ if not is_covered
]
- unverified_str = ", ".join(uncovered_idx)
- file_metric.condition_diagnostics.append(
- f"`L{d_start}`: {decision['covered']}/{decision['total']} verified ({unverified_str} unverified)"
+ unverified_conditions_string = ", ".join(uncovered_indices)
+ file_metrics.condition_diagnostics.append(
+ f"`L{decision_start_line}`: {decision['covered']}/{decision['total']} verified ({unverified_conditions_string} unverified)"
)
- for l in range(d_start, d_end + 1):
- if l in added_lines:
- file_metric.unverified_decision_lines[l] = uncovered_idx
+ for decision_line in range(decision_start_line, decision_end_line + 1):
+ if decision_line in added_lines:
+ file_metrics.unverified_decision_lines[decision_line] = uncovered_indices
- summary.files[fpath] = file_metric
+ summary.files[file_path] = file_metrics
return summary
-# -----------------------------------------------------------------------------
-# Report Formatting
-# -----------------------------------------------------------------------------
-
def format_status_banner(summary: PatchCoverageSummary) -> str:
- """Generates the executive summary callout block."""
+ """Generates the executive summary block."""
lines: List[str] = []
if summary.total_missed_lines == 0:
if not summary.has_mcdc:
- lines.append("> [!TIP]")
- lines.append(f"> ### Patch Coverage: **{summary.line_coverage_pct:.2f}%**")
lines.append(
- f"> All **{summary.total_lines}** newly added or modified executable lines are covered."
+ f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}%**"
+ )
+ lines.append(
+ f"All **{summary.total_lines}** newly added or modified executable lines are covered."
)
- elif summary.total_mcdc_cov == summary.total_mcdc_tot:
- lines.append("> [!TIP]")
+ elif summary.total_mcdc_covered_conditions == summary.total_mcdc_total_conditions:
lines.append(
- f"> ### Patch Coverage: **{summary.line_coverage_pct:.2f}% Line** | **100.00% MC/DC**"
+ f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}% Line** | **100.00% MC/DC**"
)
lines.append(
- f"> All **{summary.total_lines}** executable lines and **{summary.total_mcdc_tot}** boolean conditions across **{summary.total_decisions_count}** decisions are covered."
+ f"All **{summary.total_lines}** executable lines and **{summary.total_mcdc_total_conditions}** boolean conditions across **{summary.total_decisions_count}** decisions are covered."
)
else:
- lines.append("> [!NOTE]")
lines.append(
- f"> ### Patch Coverage: **{summary.line_coverage_pct:.2f}% Line** | **{summary.mcdc_coverage_pct:.1f}% MC/DC**"
+ f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}% Line** | **{summary.mcdc_coverage_percentage:.1f}% MC/DC**"
)
lines.append(
- f"> Executed **{summary.total_covered_lines} / {summary.total_lines}** lines. **{summary.total_mcdc_cov} / {summary.total_mcdc_tot}** boolean conditions achieved independence across **{summary.fully_verified_decisions} / {summary.total_decisions_count}** decisions."
+ f"Executed **{summary.total_covered_lines} / {summary.total_lines}** lines. **{summary.total_mcdc_covered_conditions} / {summary.total_mcdc_total_conditions}** boolean conditions achieved independence across **{summary.fully_verified_decisions} / {summary.total_decisions_count}** decisions."
)
else:
- lines.append("> [!WARNING]")
- lines.append(
- f"> ### Patch Coverage: **{summary.line_coverage_pct:.2f}%** ({summary.total_missed_lines} Missed Lines)"
- )
- lines.append(
- f"> **{summary.total_missed_lines}** unexecuted lines detected in patch."
- )
+ if not summary.has_mcdc:
+ lines.append(
+ f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}%** ({summary.total_missed_lines} Missed Lines)"
+ )
+ lines.append(
+ f"Executed **{summary.total_covered_lines} / {summary.total_lines}** lines (**{summary.total_missed_lines}** unexecuted lines detected in patch)."
+ )
+ else:
+ lines.append(
+ f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}% Line** | **{summary.mcdc_coverage_percentage:.1f}% MC/DC** ({summary.total_missed_lines} Missed Lines)"
+ )
+ lines.append(
+ f"Executed **{summary.total_covered_lines} / {summary.total_lines}** lines. **{summary.total_mcdc_covered_conditions} / {summary.total_mcdc_total_conditions}** boolean conditions achieved independence across **{summary.fully_verified_decisions} / {summary.total_decisions_count}** decisions (**{summary.total_missed_lines}** unexecuted lines detected in patch)."
+ )
return "\n".join(lines)
def format_metadata_section(
- base_sha: Optional[str],
- head_sha: Optional[str],
- base_branch: Optional[str],
- head_branch: Optional[str],
- targets_str: Optional[str] = None,
- base_repo: str = DEFAULT_BASE_REPO,
- head_repo: str = DEFAULT_HEAD_REPO,
+ base_commit_sha: Optional[str],
+ head_commit_sha: Optional[str],
+ base_branch_name: Optional[str],
+ head_branch_name: Optional[str],
+ targeted_tests_string: Optional[str] = None,
+ base_repository: str = DEFAULT_BASE_REPOSITORY,
+ head_repository: str = DEFAULT_HEAD_REPOSITORY,
) -> str:
"""Formats Git commit and target test metadata."""
lines: List[str] = []
- if base_sha and head_sha and base_branch and head_branch:
+ if base_commit_sha and head_commit_sha and base_branch_name and head_branch_name:
lines.append(
- f"- **Base Branch:** [`{base_branch}` ({base_sha[:7]})](https://github.com/{base_repo}/commit/{base_sha})"
+ f"- **Base Branch:** [`{base_branch_name}` ({base_commit_sha[:7]})](https://github.com/{base_repository}/commit/{base_commit_sha})"
)
lines.append(
- f"- **Head Commit:** [`{head_branch}` ({head_sha[:7]})](https://github.com/{head_repo}/commit/{head_sha})"
+ f"- **Head Commit:** [`{head_branch_name}` ({head_commit_sha[:7]})](https://github.com/{head_repository}/commit/{head_commit_sha})"
)
- if targets_str:
- targets_formatted = ", ".join(
- f"`{t.strip()}`" for t in targets_str.split() if t.strip()
+ if targeted_tests_string:
+ formatted_targets = ", ".join(
+ f"`{target.strip()}`" for target in targeted_tests_string.split() if target.strip()
)
- lines.append(f"- **Targeted Tests Executed:** {targets_formatted}")
+ lines.append(f"- **Targeted Tests Executed:** {formatted_targets}")
return "\n".join(lines)
def format_breakdown_table(
summary: PatchCoverageSummary,
- head_repo: str = DEFAULT_HEAD_REPO,
- head_sha: Optional[str] = None,
+ head_repository: str = DEFAULT_HEAD_REPOSITORY,
+ head_commit_sha: Optional[str] = None,
) -> str:
"""Generates the Markdown table breaking down coverage per source file."""
lines: List[str] = ["### Coverage Breakdown"]
@@ -462,42 +469,46 @@ def format_breakdown_table(
)
lines.append("| :--- | :---: | :---: | :---: | :---: |")
- for fpath, file_metric in summary.files.items():
- file_link = f"[`{fpath}`](https://github.com/{head_repo}/blob/{head_sha or 'main'}/{fpath})"
- f_missed = file_metric.missed_lines
- f_cov_len = len(file_metric.covered_lines)
- f_tot_len = file_metric.total_lines
+ for file_path, file_metric in summary.files.items():
+ file_link = f"[`{file_path}`](https://github.com/{head_repository}/blob/{head_commit_sha or 'main'}/{file_path})"
+ missed_lines = file_metric.missed_lines
+ covered_count = len(file_metric.covered_lines)
+ total_file_lines = file_metric.total_lines
if summary.has_mcdc:
mcdc_cell = (
- f"**{file_metric.mcdc_coverage_pct:.1f}%** ({file_metric.mcdc_cov}/{file_metric.mcdc_tot})"
- if file_metric.mcdc_tot > 0
+ f"**{file_metric.mcdc_coverage_percentage:.1f}%** ({file_metric.mcdc_covered_conditions}/{file_metric.mcdc_total_conditions})"
+ if file_metric.mcdc_total_conditions > 0
else "N/A"
)
- dec_cell = (
- f"**{file_metric.decisions_ver} / {file_metric.decisions_tot}**"
- if file_metric.decisions_tot > 0
+ decision_cell = (
+ f"**{file_metric.decisions_verified} / {file_metric.decisions_total}**"
+ if file_metric.decisions_total > 0
else "N/A"
)
- diag_cell = "<br>".join(file_metric.condition_diagnostics) if file_metric.condition_diagnostics else "None"
+ diagnostic_cell = (
+ "<br>".join(file_metric.condition_diagnostics)
+ if file_metric.condition_diagnostics
+ else "None"
+ )
lines.append(
- f"| {file_link} | **{file_metric.line_coverage_pct:.2f}%** ({f_cov_len}/{f_tot_len}) | {mcdc_cell} | {dec_cell} | {len(f_missed)} | {diag_cell} |"
+ f"| {file_link} | **{file_metric.line_coverage_percentage:.2f}%** ({covered_count}/{total_file_lines}) | {mcdc_cell} | {decision_cell} | {len(missed_lines)} | {diagnostic_cell} |"
)
else:
- line_spans = format_line_ranges(f_missed)
+ line_spans = format_line_ranges(missed_lines)
lines.append(
- f"| {file_link} | **{file_metric.line_coverage_pct:.2f}%** | {f_cov_len} / {f_tot_len} | {len(f_missed)} | {line_spans} |"
+ f"| {file_link} | **{file_metric.line_coverage_percentage:.2f}%** | {covered_count} / {total_file_lines} | {len(missed_lines)} | {line_spans} |"
)
# Summary Row
if summary.has_mcdc:
- total_dec_cell = f"**{summary.fully_verified_decisions} / {summary.total_decisions_count}**"
+ total_decision_cell = f"**{summary.fully_verified_decisions} / {summary.total_decisions_count}**"
lines.append(
- f"| **Total (Patch)** | **{summary.line_coverage_pct:.2f}%** ({summary.total_covered_lines}/{summary.total_lines}) | **{summary.mcdc_coverage_pct:.1f}%** ({summary.total_mcdc_cov}/{summary.total_mcdc_tot}) | {total_dec_cell} | **{summary.total_missed_lines}** | - |"
+ f"| **Total (Patch)** | **{summary.line_coverage_percentage:.2f}%** ({summary.total_covered_lines}/{summary.total_lines}) | **{summary.mcdc_coverage_percentage:.1f}%** ({summary.total_mcdc_covered_conditions}/{summary.total_mcdc_total_conditions}) | {total_decision_cell} | **{summary.total_missed_lines}** | - |"
)
else:
lines.append(
- f"| **Total (Patch)** | **{summary.line_coverage_pct:.2f}%** | {summary.total_covered_lines} / {summary.total_lines} | **{summary.total_missed_lines}** | - |"
+ f"| **Total (Patch)** | **{summary.line_coverage_percentage:.2f}%** | {summary.total_covered_lines} / {summary.total_lines} | **{summary.total_missed_lines}** | - |"
)
return "\n".join(lines)
@@ -513,27 +524,27 @@ def format_annotated_diff(
"<summary><b>View Annotated Patch Diff (Source Map)</b></summary>\n",
]
- for fpath, file_metric in summary.files.items():
- hunks = diff_files.get(fpath, [])
- unverified_lines = file_metric.unverified_decision_lines
+ for file_path, file_metric in summary.files.items():
+ hunks = diff_files.get(file_path, [])
+ unverified_decision_lines = file_metric.unverified_decision_lines
- lines.append(f"#### `{fpath}`")
+ lines.append(f"#### `{file_path}`")
lines.append("```diff")
for hunk in hunks:
lines.append(hunk.header)
- for l_type, text, l_num in hunk.lines:
- if l_type == "+":
- if l_num in file_metric.missed_lines:
- lines.append(f"- {text} // [MISSED]")
- elif l_num in unverified_lines:
- unverified_conds = ", ".join(unverified_lines[l_num])
- lines.append(f"! {text} // [PARTIAL MC/DC: {unverified_conds} unverified]")
- elif l_num in file_metric.covered_lines:
- lines.append(f"+ {text}")
+ for line_type, line_text, line_number in hunk.lines:
+ if line_type == "+":
+ if line_number in file_metric.missed_lines:
+ lines.append(f"- {line_text} // [MISSED]")
+ elif line_number in unverified_decision_lines:
+ unverified_conditions = ", ".join(unverified_decision_lines[line_number])
+ lines.append(f"! {line_text} // [PARTIAL MC/DC: {unverified_conditions} unverified]")
+ elif line_number in file_metric.covered_lines:
+ lines.append(f"+ {line_text}")
else:
- lines.append(f" {text}")
- elif l_type == " ":
- lines.append(f" {text}")
+ lines.append(f" {line_text}")
+ elif line_type == " ":
+ lines.append(f" {line_text}")
lines.append("```\n")
lines.append("</details>")
@@ -543,13 +554,13 @@ def format_annotated_diff(
def render_patch_report(
diff_files: Dict[str, List[DiffHunk]],
coverage_matrix: Dict[str, Dict[str, Any]],
- base_sha: Optional[str],
- head_sha: Optional[str],
- base_branch: Optional[str],
- head_branch: Optional[str],
- targets_str: Optional[str] = None,
- base_repo: str = DEFAULT_BASE_REPO,
- head_repo: str = DEFAULT_HEAD_REPO,
+ base_commit_sha: Optional[str],
+ head_commit_sha: Optional[str],
+ base_branch_name: Optional[str],
+ head_branch_name: Optional[str],
+ targeted_tests_string: Optional[str] = None,
+ base_repository: str = DEFAULT_BASE_REPOSITORY,
+ head_repository: str = DEFAULT_HEAD_REPOSITORY,
) -> None:
"""Composes and outputs the full Markdown report."""
summary = calculate_patch_statistics(diff_files, coverage_matrix)
@@ -560,15 +571,20 @@ def render_patch_report(
print("## LLVM-libc Patch Coverage Report\n")
if summary.total_lines == 0 or not summary.files:
- meta_str = format_metadata_section(
- base_sha, head_sha, base_branch, head_branch, targets_str, base_repo, head_repo
+ metadata_section_string = format_metadata_section(
+ base_commit_sha,
+ head_commit_sha,
+ base_branch_name,
+ head_branch_name,
+ targeted_tests_string,
+ base_repository,
+ head_repository,
)
- if meta_str:
- print(meta_str)
+ if metadata_section_string:
+ print(metadata_section_string)
print("\n---\n")
- print("> [!NOTE]")
- print("> ### Coverage Validated")
- print("> No `.cpp` source files in `libc/src/` were modified in this patch.")
+ print("### Coverage Summary")
+ print("No `.cpp` source files in `libc/src/` were modified in this patch.")
return
# 1. Status Banner
@@ -576,25 +592,27 @@ def render_patch_report(
print("")
# 2. Metadata Section
- meta_str = format_metadata_section(
- base_sha, head_sha, base_branch, head_branch, targets_str, base_repo, head_repo
+ metadata_section_string = format_metadata_section(
+ base_commit_sha,
+ head_commit_sha,
+ base_branch_name,
+ head_branch_name,
+ targeted_tests_string,
+ base_repository,
+ head_repository,
)
- if meta_str:
- print(meta_str)
+ if metadata_section_string:
+ print(metadata_section_string)
print("\n---\n")
# 3. Breakdown Table
- print(format_breakdown_table(summary, head_repo, head_sha))
+ print(format_breakdown_table(summary, head_repository, head_commit_sha))
print("")
# 4. Source Map Diff
print(format_annotated_diff(summary, diff_files))
-# -----------------------------------------------------------------------------
-# CLI Entry Point
-# -----------------------------------------------------------------------------
-
def main() -> None:
"""Parses command-line arguments and triggers report generation."""
parser = argparse.ArgumentParser(description="LLVM-libc Diff Coverage Analyzer")
@@ -610,32 +628,32 @@ def main() -> None:
parser.add_argument(
"base_repo",
nargs="?",
- default=DEFAULT_BASE_REPO,
- help=f"Base repository (default: {DEFAULT_BASE_REPO})",
+ default=DEFAULT_BASE_REPOSITORY,
+ help=f"Base repository (default: {DEFAULT_BASE_REPOSITORY})",
)
parser.add_argument(
"head_repo",
nargs="?",
- default=DEFAULT_HEAD_REPO,
- help=f"Head repository (default: {DEFAULT_HEAD_REPO})",
+ default=DEFAULT_HEAD_REPOSITORY,
+ help=f"Head repository (default: {DEFAULT_HEAD_REPOSITORY})",
)
- args = parser.parse_args()
+ arguments = parser.parse_args()
- diff_files = DiffParser.parse(args.diff_file)
- cov_data = CoverageJSONParser.load(args.json_file)
- coverage_matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
+ diff_files = DiffParser.parse(arguments.diff_file)
+ coverage_data = CoverageJSONParser.load(arguments.json_file)
+ coverage_matrix = CoverageJSONParser.extract_patch_matrix(coverage_data, diff_files)
render_patch_report(
diff_files,
coverage_matrix,
- args.base_sha,
- args.head_sha,
- args.base_branch,
- args.head_branch,
- args.targets,
- args.base_repo,
- args.head_repo,
+ arguments.base_sha,
+ arguments.head_sha,
+ arguments.base_branch,
+ arguments.head_branch,
+ arguments.targets,
+ arguments.base_repo,
+ arguments.head_repo,
)
diff --git a/libc/utils/coverage/test_codebase_coverage.py b/libc/utils/coverage/test_codebase_coverage.py
index de9820c0d05c5..972da7637c093 100644
--- a/libc/utils/coverage/test_codebase_coverage.py
+++ b/libc/utils/coverage/test_codebase_coverage.py
@@ -1,12 +1,12 @@
#!/usr/bin/env python3
#
-# ====- Unit tests for codebase coverage analyzer --------------*- python -*--==#
+# ===- Unit tests for codebase coverage analyzer -------------*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
-# ==-------------------------------------------------------------------------==#
+# ==------------------------------------------------------------------------==#
import io
import json
@@ -19,22 +19,22 @@
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from codebase_coverage import (
+ DirectoryCoverageMetrics,
FullCoverageSummary,
- SubsystemCoverageMetrics,
extract_full_coverage_statistics,
+ format_directory_breakdown_table,
format_global_summary_table,
format_overview_callout,
- format_subsystem_breakdown_table,
render_full_report,
resolve_dashboard_url,
)
-class TestSubsystemCoverageMetrics(unittest.TestCase):
+class TestDirectoryCoverageMetrics(unittest.TestCase):
"""Tests for mathematical calculation and edge-case handling in metric models."""
def test_metrics_percentages(self) -> None:
- metrics = SubsystemCoverageMetrics(
+ metrics = DirectoryCoverageMetrics(
name="math",
lines_cov=850,
lines_tot=1000,
@@ -52,7 +52,7 @@ def test_metrics_percentages(self) -> None:
self.assertEqual(metrics.missed_lines, 150)
def test_zero_division_guard(self) -> None:
- empty = SubsystemCoverageMetrics(name="empty")
+ empty = DirectoryCoverageMetrics(name="empty")
self.assertEqual(empty.line_pct, 0.0)
self.assertEqual(empty.func_pct, 0.0)
self.assertEqual(empty.mcdc_pct, 0.0)
@@ -83,14 +83,14 @@ def test_github_repository_fallback(self) -> None:
class TestDataExtraction(unittest.TestCase):
- """Tests for extracting and aggregating metrics across multi-directory subsystems."""
+ """Tests for extracting and aggregating metrics across multi-directory codebase."""
- def test_multi_subsystem_aggregation(self) -> None:
+ def test_multi_directory_aggregation(self) -> None:
cov_data = {
"data": [
{
"files": [
- # 1. ctype subsystem
+ # 1. ctype directory
{
"filename": "libc/src/ctype/isalpha.cpp",
"summary": {
@@ -102,7 +102,7 @@ def test_multi_subsystem_aggregation(self) -> None:
[10, 1, 10, 20, 2, 2, 2, 1, 1, [True, True]]
],
},
- # 2. math subsystem with nested directory
+ # 2. math directory with nested directory
{
"filename": "libc/src/math/generic/sin.cpp",
"summary": {
@@ -115,7 +115,7 @@ def test_multi_subsystem_aggregation(self) -> None:
[25, 1, 25, 30, 2, 2, 2, 1, 1, [True, True]],
],
},
- # 3. support subsystem
+ # 3. support directory
{
"filename": "libc/src/__support/OSUtil/linux/syscall.cpp",
"summary": {
@@ -143,14 +143,14 @@ def test_multi_subsystem_aggregation(self) -> None:
self.assertIsNotNone(summary)
assert summary is not None
- # Verify only 3 libc/src subsystems are tracked
- self.assertEqual(len(summary.subsystems), 3)
- self.assertIn("src/ctype", summary.subsystems)
- self.assertIn("src/math", summary.subsystems)
- self.assertIn("src/__support", summary.subsystems)
+ # Verify only 3 libc/src directories are tracked
+ self.assertEqual(len(summary.directories), 3)
+ self.assertIn("src/ctype", summary.directories)
+ self.assertIn("src/math", summary.directories)
+ self.assertIn("src/__support", summary.directories)
- # Verify subsystem-specific metrics
- math_m = summary.subsystems["src/math"]
+ # Verify directory-specific metrics
+ math_m = summary.directories["src/math"]
self.assertEqual(math_m.lines_cov, 40)
self.assertEqual(math_m.lines_tot, 50)
self.assertEqual(math_m.line_pct, 80.0)
@@ -180,14 +180,14 @@ class TestReportFormatting(unittest.TestCase):
def test_format_overview_callout(self) -> None:
# Standard line mode
- g_std = SubsystemCoverageMetrics(lines_cov=950, lines_tot=1000)
+ g_std = DirectoryCoverageMetrics(lines_cov=950, lines_tot=1000)
s_std = FullCoverageSummary(global_stats=g_std, dashboard_url="https://llvm.github.io/llvm-project/")
callout_std = format_overview_callout(s_std)
self.assertIn("95.00%", callout_std)
self.assertNotIn("MC/DC", callout_std)
# MC/DC mode
- g_mcdc = SubsystemCoverageMetrics(lines_cov=950, lines_tot=1000, mcdc_cov=90, mcdc_tot=100, decisions_tot=40)
+ g_mcdc = DirectoryCoverageMetrics(lines_cov=950, lines_tot=1000, mcdc_cov=90, mcdc_tot=100, decisions_tot=40)
s_mcdc = FullCoverageSummary(global_stats=g_mcdc, dashboard_url="https://llvm.github.io/llvm-project/mcdc/")
callout_mcdc = format_overview_callout(s_mcdc)
self.assertIn("95.00% Line", callout_mcdc)
@@ -195,7 +195,7 @@ def test_format_overview_callout(self) -> None:
self.assertIn("https://llvm.github.io/llvm-project/mcdc/", callout_mcdc)
def test_format_global_summary_table(self) -> None:
- g = SubsystemCoverageMetrics(
+ g = DirectoryCoverageMetrics(
lines_cov=950, lines_tot=1000, func_cov=98, func_tot=100,
mcdc_cov=90, mcdc_tot=100, decisions_full=36, decisions_tot=40
)
@@ -208,14 +208,14 @@ def test_format_global_summary_table(self) -> None:
self.assertIn("| **Executable Lines** | 950 | 1,000 | **95.00%** |", table)
self.assertIn("| **Functions** | 98 | 100 | **98.00%** |", table)
- def test_format_subsystem_breakdown_table_sorted(self) -> None:
- sub_metrics = {
- "src/string": SubsystemCoverageMetrics(name="src/string", lines_cov=20, lines_tot=20),
- "src/ctype": SubsystemCoverageMetrics(name="src/ctype", lines_cov=10, lines_tot=10),
- "src/math": SubsystemCoverageMetrics(name="src/math", lines_cov=40, lines_tot=50),
+ def test_format_directory_breakdown_table_sorted(self) -> None:
+ dir_metrics = {
+ "src/string": DirectoryCoverageMetrics(name="src/string", lines_cov=20, lines_tot=20),
+ "src/ctype": DirectoryCoverageMetrics(name="src/ctype", lines_cov=10, lines_tot=10),
+ "src/math": DirectoryCoverageMetrics(name="src/math", lines_cov=40, lines_tot=50),
}
- summary = FullCoverageSummary(subsystems=sub_metrics)
- table = format_subsystem_breakdown_table(summary)
+ summary = FullCoverageSummary(directories=dir_metrics)
+ table = format_directory_breakdown_table(summary)
# Must be alphabetically sorted: ctype, math, string
pos_ctype = table.find("`libc/src/ctype`")
diff --git a/libc/utils/coverage/test_diff_coverage.py b/libc/utils/coverage/test_diff_coverage.py
index da7ed98e7ef28..423533a2d363f 100644
--- a/libc/utils/coverage/test_diff_coverage.py
+++ b/libc/utils/coverage/test_diff_coverage.py
@@ -1,12 +1,12 @@
#!/usr/bin/env python3
#
-# ====- Unit tests for diff coverage analyzer ------------------*- python -*--==#
+# ===- Unit tests for diff coverage analyzer -----------------*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
-# ==-------------------------------------------------------------------------==#
+# ==------------------------------------------------------------------------==#
import io
import json
@@ -301,7 +301,7 @@ def test_line_coverage_calculation(self) -> None:
self.assertEqual(stats.total_covered_lines, 1) # line 11 (if x < 0)
self.assertEqual(stats.total_missed_lines, 1) # line 12 (return -x)
self.assertEqual(stats.total_lines, 2)
- self.assertEqual(stats.line_coverage_pct, 50.0)
+ self.assertEqual(stats.line_coverage_percentage, 50.0)
self.assertFalse(stats.has_mcdc)
def test_mcdc_decision_diagnostics(self) -> None:
@@ -336,115 +336,112 @@ def test_mcdc_decision_diagnostics(self) -> None:
stats = calculate_patch_statistics(diff_files, matrix)
self.assertTrue(stats.has_mcdc)
- self.assertEqual(stats.total_mcdc_cov, 2)
- self.assertEqual(stats.total_mcdc_tot, 3)
- self.assertEqual(stats.mcdc_coverage_pct, 66.66666666666666)
+ self.assertEqual(stats.total_mcdc_covered_conditions, 2)
+ self.assertEqual(stats.total_mcdc_total_conditions, 3)
+ self.assertEqual(stats.mcdc_coverage_percentage, 66.66666666666666)
self.assertEqual(stats.total_decisions_count, 1)
self.assertEqual(stats.fully_verified_decisions, 0)
- f_metrics = stats.files["libc/src/ctype/isspace.cpp"]
- self.assertIn("C3 unverified", f_metrics.condition_diagnostics[0])
- self.assertEqual(f_metrics.unverified_decision_lines[11], ["C3"])
+ file_metrics = stats.files["libc/src/ctype/isspace.cpp"]
+ self.assertIn("C3 unverified", file_metrics.condition_diagnostics[0])
+ self.assertEqual(file_metrics.unverified_decision_lines[11], ["C3"])
def test_format_status_banner(self) -> None:
# Full statement & full MC/DC
- s1 = PatchCoverageSummary(
+ summary_full = PatchCoverageSummary(
total_covered_lines=10,
total_missed_lines=0,
- total_mcdc_cov=4,
- total_mcdc_tot=4,
+ total_mcdc_covered_conditions=4,
+ total_mcdc_total_conditions=4,
total_decisions_count=2,
fully_verified_decisions=2,
)
- banner1 = format_status_banner(s1)
- self.assertIn("> [!TIP]", banner1)
- self.assertIn("100.00% Line", banner1)
- self.assertIn("100.00% MC/DC", banner1)
+ banner_full = format_status_banner(summary_full)
+ self.assertIn("100.00% Line", banner_full)
+ self.assertIn("100.00% MC/DC", banner_full)
# Full statement, partial MC/DC
- s2 = PatchCoverageSummary(
+ summary_partial = PatchCoverageSummary(
total_covered_lines=10,
total_missed_lines=0,
- total_mcdc_cov=3,
- total_mcdc_tot=4,
+ total_mcdc_covered_conditions=3,
+ total_mcdc_total_conditions=4,
total_decisions_count=2,
fully_verified_decisions=1,
)
- banner2 = format_status_banner(s2)
- self.assertIn("> [!NOTE]", banner2)
- self.assertIn("75.0% MC/DC", banner2)
+ banner_partial = format_status_banner(summary_partial)
+ self.assertIn("75.0% MC/DC", banner_partial)
- # Warning when lines missed
- s3 = PatchCoverageSummary(
+ # Lines missed
+ summary_warn = PatchCoverageSummary(
total_covered_lines=8,
total_missed_lines=2,
)
- banner3 = format_status_banner(s3)
- self.assertIn("> [!WARNING]", banner3)
- self.assertIn("80.00%", banner3)
- self.assertIn("unexecuted lines detected in patch", banner3)
+ banner_warn = format_status_banner(summary_warn)
+ self.assertIn("80.00%", banner_warn)
+ self.assertIn("unexecuted lines detected in patch", banner_warn)
def test_format_metadata_section(self) -> None:
- meta = format_metadata_section(
- base_sha="abcdef1234567890",
- head_sha="123456abcdef7890",
- base_branch="main",
- head_branch="my-pr",
- targets_str="libc.test.src.math.sin_test libc.test.src.math.cos_test",
- base_repo="llvm/llvm-project",
- head_repo="user/llvm-project",
+ metadata_string = format_metadata_section(
+ base_commit_sha="abcdef1234567890",
+ head_commit_sha="123456abcdef7890",
+ base_branch_name="main",
+ head_branch_name="my-pr",
+ targeted_tests_string="libc.test.src.math.sin_test libc.test.src.math.cos_test",
+ base_repository="llvm/llvm-project",
+ head_repository="user/llvm-project",
)
- self.assertIn("abcdef1", meta)
- self.assertIn("123456a", meta)
- self.assertIn("`libc.test.src.math.sin_test`, `libc.test.src.math.cos_test`", meta)
+ self.assertIn("abcdef1", metadata_string)
+ self.assertIn("123456a", metadata_string)
+ self.assertIn("`libc.test.src.math.sin_test`, `libc.test.src.math.cos_test`", metadata_string)
def test_format_breakdown_table_standard_and_mcdc(self) -> None:
- fm = FilePatchMetrics(
- fpath="libc/src/math/tan.cpp",
+ file_metrics = FilePatchMetrics(
+ file_path="libc/src/math/tan.cpp",
covered_lines={10, 11},
missed_lines={12},
- mcdc_cov=2,
- mcdc_tot=2,
- decisions_ver=1,
- decisions_tot=1,
+ mcdc_covered_conditions=2,
+ mcdc_total_conditions=2,
+ decisions_verified=1,
+ decisions_total=1,
condition_diagnostics=["`L10`: 2/2 verified"],
)
summary_mcdc = PatchCoverageSummary(
- files={"libc/src/math/tan.cpp": fm},
+ files={"libc/src/math/tan.cpp": file_metrics},
total_covered_lines=2,
total_missed_lines=1,
- total_mcdc_cov=2,
- total_mcdc_tot=2,
+ total_mcdc_covered_conditions=2,
+ total_mcdc_total_conditions=2,
total_decisions_count=1,
fully_verified_decisions=1,
)
- table_mcdc = format_breakdown_table(summary_mcdc, head_repo="llvm/llvm-project")
+ table_mcdc = format_breakdown_table(summary_mcdc, head_repository="llvm/llvm-project")
self.assertIn("MC/DC Conditions", table_mcdc)
self.assertIn("`L10`: 2/2 verified", table_mcdc)
summary_std = PatchCoverageSummary(
- files={"libc/src/math/tan.cpp": fm},
+ files={"libc/src/math/tan.cpp": file_metrics},
total_covered_lines=2,
total_missed_lines=1,
)
- table_std = format_breakdown_table(summary_std, head_repo="llvm/llvm-project")
+ table_std = format_breakdown_table(summary_std, head_repository="llvm/llvm-project")
self.assertIn("Unexecuted Line Spans", table_std)
self.assertIn("`L12`", table_std)
def test_render_patch_report_empty_diff(self) -> None:
- stdout_buf = io.StringIO()
- with redirect_stdout(stdout_buf):
+ stdout_buffer = io.StringIO()
+ with redirect_stdout(stdout_buffer):
render_patch_report(
diff_files={},
coverage_matrix={},
- base_sha="base",
- head_sha="head",
- base_branch="main",
- head_branch="patch",
+ base_commit_sha="base",
+ head_commit_sha="head",
+ base_branch_name="main",
+ head_branch_name="patch",
)
- output = stdout_buf.getvalue()
- self.assertIn("Coverage Validated", output)
- self.assertIn("No `.cpp` source files in `libc/src/` were modified", output)
+ output_text = stdout_buffer.getvalue()
+ self.assertIn("### Coverage Summary", output_text)
+ self.assertIn("No `.cpp` source files in `libc/src/` were modified", output_text)
if __name__ == "__main__":
>From 32c7d820550fb85bedf16154cd21e074fc435939 Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Thu, 27 Aug 2026 09:42:27 +0000
Subject: [PATCH 05/15] test(coverage): rewrite test suite for diff and
codebase coverage tools
---
libc/utils/coverage/test_codebase_coverage.py | 278 ++------
libc/utils/coverage/test_diff_coverage.py | 650 ++++++------------
2 files changed, 291 insertions(+), 637 deletions(-)
diff --git a/libc/utils/coverage/test_codebase_coverage.py b/libc/utils/coverage/test_codebase_coverage.py
index 972da7637c093..ce691385c8b28 100644
--- a/libc/utils/coverage/test_codebase_coverage.py
+++ b/libc/utils/coverage/test_codebase_coverage.py
@@ -1,236 +1,82 @@
-#!/usr/bin/env python3
-#
-# ===- Unit tests for codebase coverage analyzer -------------*- python -*--==#
+# ====- Unit tests for codebase_coverage.py ------------------*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
-# ==------------------------------------------------------------------------==#
+# ==-------------------------------------------------------------------------==#
-import io
-import json
-import os
-import sys
import unittest
-from contextlib import redirect_stdout
-from typing import Dict
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
+import json
from codebase_coverage import (
DirectoryCoverageMetrics,
FullCoverageSummary,
extract_full_coverage_statistics,
format_directory_breakdown_table,
- format_global_summary_table,
- format_overview_callout,
- render_full_report,
- resolve_dashboard_url,
+ render_full_report
)
-
-class TestDirectoryCoverageMetrics(unittest.TestCase):
- """Tests for mathematical calculation and edge-case handling in metric models."""
-
- def test_metrics_percentages(self) -> None:
- metrics = DirectoryCoverageMetrics(
- name="math",
- lines_cov=850,
- lines_tot=1000,
- func_cov=45,
- func_tot=50,
- mcdc_cov=90,
- mcdc_tot=100,
- decisions_tot=40,
- decisions_full=36,
- )
- self.assertEqual(metrics.line_pct, 85.0)
- self.assertEqual(metrics.func_pct, 90.0)
- self.assertEqual(metrics.mcdc_pct, 90.0)
- self.assertEqual(metrics.decisions_pct, 90.0)
- self.assertEqual(metrics.missed_lines, 150)
-
- def test_zero_division_guard(self) -> None:
- empty = DirectoryCoverageMetrics(name="empty")
- self.assertEqual(empty.line_pct, 0.0)
- self.assertEqual(empty.func_pct, 0.0)
- self.assertEqual(empty.mcdc_pct, 0.0)
- self.assertEqual(empty.decisions_pct, 0.0)
- self.assertEqual(empty.missed_lines, 0)
-
-
-class TestDashboardURLResolution(unittest.TestCase):
- """Tests for environment variable resolution and URL formatting."""
-
- def test_custom_environment_variable(self) -> None:
- os.environ["COVERAGE_DASHBOARD_URL"] = "https://custom-dashboard.internal/"
- url = resolve_dashboard_url(has_mcdc=False)
- self.assertEqual(url, "https://custom-dashboard.internal/")
-
- url_mcdc = resolve_dashboard_url(has_mcdc=True)
- self.assertEqual(url_mcdc, "https://custom-dashboard.internal/mcdc/")
-
- def test_github_repository_fallback(self) -> None:
- os.environ.pop("COVERAGE_DASHBOARD_URL", None)
- os.environ["GITHUB_REPOSITORY"] = "tapiwagonga/llvm-project"
-
- url = resolve_dashboard_url(has_mcdc=False)
- self.assertEqual(url, "https://tapiwagonga.github.io/llvm-project/")
-
- url_mcdc = resolve_dashboard_url(has_mcdc=True)
- self.assertEqual(url_mcdc, "https://tapiwagonga.github.io/llvm-project/mcdc/")
-
-
-class TestDataExtraction(unittest.TestCase):
- """Tests for extracting and aggregating metrics across multi-directory codebase."""
-
- def test_multi_directory_aggregation(self) -> None:
- cov_data = {
- "data": [
- {
- "files": [
- # 1. ctype directory
- {
- "filename": "libc/src/ctype/isalpha.cpp",
- "summary": {
- "lines": {"count": 10, "covered": 10},
- "functions": {"count": 1, "covered": 1},
- "mcdc": {"count": 2, "covered": 2},
- },
- "mcdc_records": [
- [10, 1, 10, 20, 2, 2, 2, 1, 1, [True, True]]
- ],
- },
- # 2. math directory with nested directory
- {
- "filename": "libc/src/math/generic/sin.cpp",
- "summary": {
- "lines": {"count": 50, "covered": 40},
- "functions": {"count": 2, "covered": 2},
- "mcdc": {"count": 6, "covered": 4},
- },
- "mcdc_records": [
- [15, 1, 15, 30, 2, 1, 2, 1, 1, [True, False]],
- [25, 1, 25, 30, 2, 2, 2, 1, 1, [True, True]],
- ],
- },
- # 3. support directory
- {
- "filename": "libc/src/__support/OSUtil/linux/syscall.cpp",
- "summary": {
- "lines": {"count": 100, "covered": 90},
- "functions": {"count": 5, "covered": 5},
- "mcdc": {"count": 0, "covered": 0},
- },
- "mcdc_records": [],
- },
- # 4. Ignored test and benchmark files
- {
- "filename": "libc/test/src/math/sin_test.cpp",
- "summary": {"lines": {"count": 200, "covered": 200}},
- },
- {
- "filename": "libc/utils/mathtools/ryu.py",
- "summary": {"lines": {"count": 80, "covered": 80}},
- },
- ]
- }
- ]
+class TestCodebaseDirectoryAggregation(unittest.TestCase):
+ def test_path_filtering_and_routing(self):
+ mock_json = {
+ "data": [{
+ "files": [
+ {
+ "filename": "/workspace/libc/src/math/sin.cpp",
+ "summary": {
+ "lines": {"count": 100, "covered": 50, "percent": 50.0},
+ "functions": {"count": 2, "covered": 1, "percent": 50.0}
+ }
+ },
+ {
+ "filename": "/workspace/libc/src/string/strcpy.cpp",
+ "summary": {
+ "lines": {"count": 200, "covered": 200, "percent": 100.0},
+ "functions": {"count": 4, "covered": 4, "percent": 100.0}
+ }
+ },
+ {
+ "filename": "/workspace/libc/test/src/math/sin_test.cpp",
+ "summary": {
+ "lines": {"count": 500, "covered": 500, "percent": 100.0},
+ "functions": {"count": 10, "covered": 10, "percent": 100.0}
+ }
+ }
+ ]
+ }]
}
-
- summary = extract_full_coverage_statistics(cov_data)
- self.assertIsNotNone(summary)
- assert summary is not None
-
- # Verify only 3 libc/src directories are tracked
- self.assertEqual(len(summary.directories), 3)
- self.assertIn("src/ctype", summary.directories)
+
+ summary = extract_full_coverage_statistics(mock_json)
+
+ # The test file should be ignored entirely
+ self.assertEqual(summary.global_stats.lines_tot, 300)
+ self.assertEqual(summary.global_stats.lines_cov, 250)
+ self.assertEqual(summary.global_stats.func_tot, 6)
+ self.assertEqual(summary.global_stats.func_cov, 5)
+
+ # Ensure proper bucket routing
self.assertIn("src/math", summary.directories)
- self.assertIn("src/__support", summary.directories)
-
- # Verify directory-specific metrics
- math_m = summary.directories["src/math"]
- self.assertEqual(math_m.lines_cov, 40)
- self.assertEqual(math_m.lines_tot, 50)
- self.assertEqual(math_m.line_pct, 80.0)
- self.assertEqual(math_m.mcdc_cov, 4)
- self.assertEqual(math_m.mcdc_tot, 6)
- self.assertEqual(math_m.decisions_tot, 2)
- self.assertEqual(math_m.decisions_full, 1)
-
- # Verify global aggregate sums
- self.assertEqual(summary.global_stats.lines_cov, 140) # 10 + 40 + 90
- self.assertEqual(summary.global_stats.lines_tot, 160) # 10 + 50 + 100
- self.assertEqual(summary.global_stats.func_cov, 8) # 1 + 2 + 5
- self.assertEqual(summary.global_stats.mcdc_cov, 6) # 2 + 4 + 0
- self.assertEqual(summary.global_stats.mcdc_tot, 8) # 2 + 6 + 0
- self.assertEqual(summary.global_stats.decisions_tot, 3)# 1 + 2 + 0
- self.assertEqual(summary.global_stats.decisions_full, 2)# 1 + 1 + 0
- self.assertTrue(summary.has_mcdc)
-
- def test_empty_or_malformed_json(self) -> None:
- self.assertIsNone(extract_full_coverage_statistics({}))
- self.assertIsNone(extract_full_coverage_statistics({"data": []}))
- self.assertIsNone(extract_full_coverage_statistics({"data": [{"files": []}]}))
-
-
-class TestReportFormatting(unittest.TestCase):
- """Tests for Markdown table generation, progress indicators, and banners."""
-
- def test_format_overview_callout(self) -> None:
- # Standard line mode
- g_std = DirectoryCoverageMetrics(lines_cov=950, lines_tot=1000)
- s_std = FullCoverageSummary(global_stats=g_std, dashboard_url="https://llvm.github.io/llvm-project/")
- callout_std = format_overview_callout(s_std)
- self.assertIn("95.00%", callout_std)
- self.assertNotIn("MC/DC", callout_std)
-
- # MC/DC mode
- g_mcdc = DirectoryCoverageMetrics(lines_cov=950, lines_tot=1000, mcdc_cov=90, mcdc_tot=100, decisions_tot=40)
- s_mcdc = FullCoverageSummary(global_stats=g_mcdc, dashboard_url="https://llvm.github.io/llvm-project/mcdc/")
- callout_mcdc = format_overview_callout(s_mcdc)
- self.assertIn("95.00% Line", callout_mcdc)
- self.assertIn("90.00% MC/DC", callout_mcdc)
- self.assertIn("https://llvm.github.io/llvm-project/mcdc/", callout_mcdc)
-
- def test_format_global_summary_table(self) -> None:
- g = DirectoryCoverageMetrics(
- lines_cov=950, lines_tot=1000, func_cov=98, func_tot=100,
- mcdc_cov=90, mcdc_tot=100, decisions_full=36, decisions_tot=40
+ self.assertEqual(summary.directories["src/math"].lines_tot, 100)
+ self.assertIn("src/string", summary.directories)
+ self.assertEqual(summary.directories["src/string"].lines_tot, 200)
+ self.assertNotIn("test/src/math", summary.directories)
+
+class TestCodebaseReportRendering(unittest.TestCase):
+ def test_format_directory_breakdown_table(self):
+ metrics_math = DirectoryCoverageMetrics(
+ name="src/math", lines_tot=100, lines_cov=50, func_tot=10, func_cov=5
)
- summary = FullCoverageSummary(global_stats=g)
- table = format_global_summary_table(summary)
-
- self.assertIn("### Overall", table)
- self.assertIn("| **MC/DC Condition Independence** | 90 | 100 | **90.00%** |", table)
- self.assertIn("| **Fully Verified Decisions** | 36 | 40 | **90.00%** |", table)
- self.assertIn("| **Executable Lines** | 950 | 1,000 | **95.00%** |", table)
- self.assertIn("| **Functions** | 98 | 100 | **98.00%** |", table)
-
- def test_format_directory_breakdown_table_sorted(self) -> None:
- dir_metrics = {
- "src/string": DirectoryCoverageMetrics(name="src/string", lines_cov=20, lines_tot=20),
- "src/ctype": DirectoryCoverageMetrics(name="src/ctype", lines_cov=10, lines_tot=10),
- "src/math": DirectoryCoverageMetrics(name="src/math", lines_cov=40, lines_tot=50),
- }
- summary = FullCoverageSummary(directories=dir_metrics)
+ metrics_string = DirectoryCoverageMetrics(
+ name="src/string", lines_tot=200, lines_cov=200, func_tot=20, func_cov=20
+ )
+ summary = FullCoverageSummary(
+ global_stats=metrics_math,
+ directories={"src/math": metrics_math, "src/string": metrics_string}
+ )
+
table = format_directory_breakdown_table(summary)
+ self.assertIn("`libc/src/math`", table)
+ self.assertIn("`libc/src/string`", table)
- # Must be alphabetically sorted: ctype, math, string
- pos_ctype = table.find("`libc/src/ctype`")
- pos_math = table.find("`libc/src/math`")
- pos_string = table.find("`libc/src/string`")
-
- self.assertTrue(pos_ctype < pos_math < pos_string)
-
- def test_render_full_report_empty_data(self) -> None:
- stdout_buf = io.StringIO()
- with redirect_stdout(stdout_buf):
- render_full_report({})
- output = stdout_buf.getvalue()
- self.assertIn("No Coverage Data Detected", output)
-
-
-if __name__ == "__main__":
+if __name__ == '__main__':
unittest.main()
diff --git a/libc/utils/coverage/test_diff_coverage.py b/libc/utils/coverage/test_diff_coverage.py
index 423533a2d363f..1aa6a14535c2e 100644
--- a/libc/utils/coverage/test_diff_coverage.py
+++ b/libc/utils/coverage/test_diff_coverage.py
@@ -1,448 +1,256 @@
-#!/usr/bin/env python3
-#
-# ===- Unit tests for diff coverage analyzer -----------------*- python -*--==#
+# ====- Unit tests for diff_coverage.py ----------------------*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
-# ==------------------------------------------------------------------------==#
+# ==-------------------------------------------------------------------------==#
-import io
-import json
-import os
-import sys
-import tempfile
import unittest
-from contextlib import redirect_stderr, redirect_stdout
-from typing import Dict, List, Set
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
+import json
from diff_coverage import (
- CoverageJSONParser,
- DiffHunk,
DiffParser,
- FilePatchMetrics,
- PatchCoverageSummary,
+ DiffHunk,
+ is_executable_line,
+ CoverageJSONParser,
calculate_patch_statistics,
- format_annotated_diff,
- format_breakdown_table,
- format_line_ranges,
- format_metadata_section,
format_status_banner,
- is_executable_line,
- render_patch_report,
+ format_breakdown_table,
+ PatchCoverageSummary,
+ FilePatchMetrics
)
+class TestExecutableLineHeuristics(unittest.TestCase):
+ def test_heuristics_matrix(self):
+ matrix = [
+ ("int x = 5;", True, "Basic assignment"),
+ ("return result;", True, "Return statement"),
+ ("if (x > 0) {", True, "Condition start"),
+ ("foo();", True, "Function call"),
+ (" // This is a comment", False, "Line comment"),
+ ("/* Block comment */", False, "Block comment line"),
+ ("#define LIBC_INLINE inline", False, "Preprocessor macro"),
+ ("namespace LIBC_NAMESPACE {", False, "Namespace declaration"),
+ ("};", False, "Struct/Class terminator"),
+ (" {", False, "Opening scope"),
+ (" }", False, "Closing scope"),
+ ("[[maybe_unused]] int x;", True, "Attribute with variable"),
+ ("", False, "Empty line"),
+ ]
+ for input_string, expected, description in matrix:
+ with self.subTest(msg=description, input_string=input_string):
+ self.assertEqual(is_executable_line(input_string), expected)
class TestDiffParser(unittest.TestCase):
- """Exhaustive unit tests for unified diff parsing."""
-
- def test_single_file_single_hunk(self) -> None:
- raw_diff = """diff --git a/libc/src/ctype/isalpha.cpp b/libc/src/ctype/isalpha.cpp
---- a/libc/src/ctype/isalpha.cpp
-+++ b/libc/src/ctype/isalpha.cpp
-@@ -10,3 +10,4 @@
- int isalpha(int c) {
-+ int x = c;
- return x;
- }
-"""
- files = DiffParser.parse(raw_diff)
- self.assertIn("libc/src/ctype/isalpha.cpp", files)
- hunks = files["libc/src/ctype/isalpha.cpp"]
+ def test_parse_diff_hunks_and_lines(self):
+ mock_diff = (
+ "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
+ "--- a/src/math/sin.cpp\n"
+ "+++ b/src/math/sin.cpp\n"
+ "@@ -10,3 +10,4 @@\n"
+ " context_line_1();\n"
+ "+added_line_1();\n"
+ "+added_line_2();\n"
+ " context_line_2();\n"
+ )
+ hunks_dict = DiffParser.parse(mock_diff)
+ self.assertIn("src/math/sin.cpp", hunks_dict)
+ hunks = hunks_dict["src/math/sin.cpp"]
self.assertEqual(len(hunks), 1)
- self.assertEqual(len(hunks[0].lines), 4)
- self.assertEqual(hunks[0].lines[0], (" ", "int isalpha(int c) {", 10))
- self.assertEqual(hunks[0].lines[1], ("+", " int x = c;", 11))
- self.assertEqual(hunks[0].lines[2], (" ", " return x;", 12))
- self.assertEqual(hunks[0].lines[3], (" ", "}", 13))
-
- def test_multi_file_multi_hunk(self) -> None:
- raw_diff = """diff --git a/libc/src/ctype/isalpha.cpp b/libc/src/ctype/isalpha.cpp
---- a/libc/src/ctype/isalpha.cpp
-+++ b/libc/src/ctype/isalpha.cpp
-@@ -5,2 +5,3 @@
-+// Header comment
- int isalpha(int c);
-@@ -20,2 +21,3 @@
-+ int z = 1;
- return z;
-diff --git a/libc/src/math/sin.cpp b/libc/src/math/sin.cpp
---- a/libc/src/math/sin.cpp
-+++ b/libc/src/math/sin.cpp
-@@ -1,3 +1,4 @@
-+// Math file
- double sin(double x) {
-+ return x;
- }
-"""
- files = DiffParser.parse(raw_diff)
- self.assertEqual(len(files), 2)
- self.assertIn("libc/src/ctype/isalpha.cpp", files)
- self.assertIn("libc/src/math/sin.cpp", files)
- self.assertEqual(len(files["libc/src/ctype/isalpha.cpp"]), 2)
- self.assertEqual(len(files["libc/src/math/sin.cpp"]), 1)
-
- def test_deleted_and_renamed_files(self) -> None:
- raw_diff = """diff --git a/libc/src/old.cpp b/libc/src/old.cpp
-deleted file mode 100644
---- a/libc/src/old.cpp
-+++ /dev/null
-@@ -1,3 +0,0 @@
--int old_func();
-"""
- files = DiffParser.parse(raw_diff)
- self.assertEqual(len(files), 0)
-
- def test_diff_from_temporary_file(self) -> None:
- raw_diff = """diff --git a/libc/src/string/strlen.cpp b/libc/src/string/strlen.cpp
---- a/libc/src/string/strlen.cpp
-+++ b/libc/src/string/strlen.cpp
-@@ -1,2 +1,3 @@
- size_t strlen(const char *s) {
-+ return 0;
- }
-"""
- with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as f:
- f.write(raw_diff)
- f_path = f.name
-
- try:
- files = DiffParser.parse(f_path)
- self.assertIn("libc/src/string/strlen.cpp", files)
- self.assertEqual(len(files["libc/src/string/strlen.cpp"][0].lines), 3)
- finally:
- os.remove(f_path)
-
- def test_diff_with_no_newline_warning(self) -> None:
- raw_diff = """diff --git a/libc/src/stdio/puts.cpp b/libc/src/stdio/puts.cpp
---- a/libc/src/stdio/puts.cpp
-+++ b/libc/src/stdio/puts.cpp
-@@ -1,2 +1,2 @@
--int puts(const char *s);
-+int puts(const char *str);
-\\ No newline at end of file
-"""
- files = DiffParser.parse(raw_diff)
- self.assertIn("libc/src/stdio/puts.cpp", files)
- lines = files["libc/src/stdio/puts.cpp"][0].lines
- self.assertEqual(len(lines), 1)
- self.assertEqual(lines[0], ("+", "int puts(const char *str);", 1))
-
-
-class TestExecutableLineFilter(unittest.TestCase):
- """Exhaustive tests for filtering executable lines vs comments and declarations."""
-
- def test_non_executable_comments_and_whitespace(self) -> None:
- test_cases = [
- "",
- " ",
- "\t\t",
- "// Single line comment",
- " // Indented comment",
- "/* Multi-line block start",
- " * Continuation line",
- " */ End of comment block",
- ]
- for line in test_cases:
- with self.subTest(line=line):
- self.assertFalse(is_executable_line(line))
-
- def test_non_executable_syntax_and_preprocessor(self) -> None:
- test_cases = [
- "{",
- "}",
- "};",
- "{};",
- ": m_val(0)",
- "#include <stddef.h>",
- "#define FOO 1",
- "#ifdef LIBC_ENABLE_COVERAGE",
- "#endif",
- "namespace __llvm_libc {",
- "extern \"C\" {",
- "using size_t = unsigned long;",
- "template <typename T>",
- "typedef int (*func_ptr)(void);",
- "__attribute__((noinline))",
- ]
- for line in test_cases:
- with self.subTest(line=line):
- self.assertFalse(is_executable_line(line))
-
- def test_non_executable_type_definitions(self) -> None:
- test_cases = [
- "struct Foo {",
- "struct Foo;",
- "class Bar {",
- "class Bar;",
- "enum Color {",
- "enum class Status : int {",
- ]
- for line in test_cases:
- with self.subTest(line=line):
- self.assertFalse(is_executable_line(line))
-
- def test_executable_statements(self) -> None:
- test_cases = [
- "int x = 5;",
- "return a + b;",
- "struct Foo f = init_foo();",
- "class Bar b(10);",
- "if (c >= 'a' && c <= 'z')",
- "for (int i = 0; i < 10; ++i) {",
- "while (*s++) {",
- "switch (op) {",
- "case 1:",
- "break;",
- "continue;",
- "goto cleanup;",
- "foo(); // inline comment",
- ]
- for line in test_cases:
- with self.subTest(line=line):
- self.assertTrue(is_executable_line(line))
-
-
-class TestFormatLineRanges(unittest.TestCase):
- """Tests for line number formatting into human-readable spans."""
-
- def test_formatting_variations(self) -> None:
- self.assertEqual(format_line_ranges(set()), "None")
- self.assertEqual(format_line_ranges({42}), "`L42`")
- self.assertEqual(format_line_ranges({10, 11, 12}), "`L10-L12`")
- self.assertEqual(
- format_line_ranges({5, 6, 7, 10, 15, 16, 20}),
- "`L5-L7`, `L10`, `L15-L16`, `L20`",
+
+ hunk = hunks[0]
+ self.assertEqual(hunk.header, "@@ -10,3 +10,4 @@")
+ added_lines = [line for line in hunk.lines if line[0] == "+"]
+ self.assertEqual(len(added_lines), 2)
+ self.assertEqual(added_lines[0], ("+", "added_line_1();", 11))
+ self.assertEqual(added_lines[1], ("+", "added_line_2();", 12))
+
+class TestSegmentIntersection(unittest.TestCase):
+ def test_calculate_patch_statistics_exact_match(self):
+ mock_diff = (
+ "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
+ "--- a/src/math/sin.cpp\n"
+ "+++ b/src/math/sin.cpp\n"
+ "@@ -10,3 +10,2 @@\n"
+ " context();\n"
+ "+return 0;\n"
)
-
-
-class TestCoverageJSONParser(unittest.TestCase):
- """Tests for parsing llvm-cov JSON export structures and mapping to diffs."""
-
- def test_invalid_json_handling(self) -> None:
- with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as f:
- f.write("{ invalid json")
- f_path = f.name
-
- try:
- with redirect_stderr(io.StringIO()):
- with self.assertRaises(SystemExit):
- CoverageJSONParser.load(f_path)
- finally:
- os.remove(f_path)
-
- def test_extract_patch_matrix_matching(self) -> None:
- cov_data = {
- "data": [
- {
- "files": [
- {
- "filename": "/runner/work/llvm-project/libc/src/math/cos.cpp",
- "segments": [
- [10, 1, 5, True, True, False],
- [12, 1, 0, True, True, False],
- [14, 1, 0, False, False, False],
- ],
- "mcdc_records": [
- [10, 5, 10, 25, 2, 2, 2, 1, 1, [True, True]]
- ],
- }
- ]
- }
- ]
+ mock_json = {
+ "data": [{
+ "files": [{
+ "filename": "/workspace/src/math/sin.cpp",
+ "segments": [
+ [10, 0, 1, 1, 1],
+ [11, 0, 1, 1, 1]
+ ],
+ "branches": []
+ }]
+ }]
}
- diff_files = {"libc/src/math/cos.cpp": []}
- matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
-
- self.assertIn("libc/src/math/cos.cpp", matrix)
- self.assertIn(10, matrix["libc/src/math/cos.cpp"]["covered"])
- self.assertIn(11, matrix["libc/src/math/cos.cpp"]["covered"])
- self.assertIn(12, matrix["libc/src/math/cos.cpp"]["missed"])
- self.assertEqual(len(matrix["libc/src/math/cos.cpp"]["mcdc_decisions"]), 1)
-
-
-class TestStatisticsAndReporting(unittest.TestCase):
- """Tests for patch coverage calculation, diagnostic generation, and Markdown rendering."""
-
- def test_line_coverage_calculation(self) -> None:
- diff_text = """diff --git a/libc/src/math/fabs.cpp b/libc/src/math/fabs.cpp
---- a/libc/src/math/fabs.cpp
-+++ b/libc/src/math/fabs.cpp
-@@ -10,3 +10,5 @@
- double fabs(double x) {
-+ if (x < 0)
-+ return -x;
- return x;
- }
-"""
- diff_files = DiffParser.parse(diff_text)
- cov_data = {
- "data": [
- {
- "files": [
- {
- "filename": "libc/src/math/fabs.cpp",
- "segments": [
- [10, 1, 10, True, True, False],
- [12, 1, 0, True, True, False],
- [14, 1, 0, False, False, False],
- ],
- }
- ]
- }
- ]
- }
- matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
- stats = calculate_patch_statistics(diff_files, matrix)
-
- self.assertEqual(stats.total_covered_lines, 1) # line 11 (if x < 0)
- self.assertEqual(stats.total_missed_lines, 1) # line 12 (return -x)
- self.assertEqual(stats.total_lines, 2)
- self.assertEqual(stats.line_coverage_percentage, 50.0)
- self.assertFalse(stats.has_mcdc)
-
- def test_mcdc_decision_diagnostics(self) -> None:
- diff_text = """diff --git a/libc/src/ctype/isspace.cpp b/libc/src/ctype/isspace.cpp
---- a/libc/src/ctype/isspace.cpp
-+++ b/libc/src/ctype/isspace.cpp
-@@ -10,2 +10,3 @@
- int isspace(int c) {
-+ if (c == ' ' || c == '\t' || c == '\n')
- return 1;
-"""
- diff_files = DiffParser.parse(diff_text)
- cov_data = {
- "data": [
- {
- "files": [
- {
- "filename": "libc/src/ctype/isspace.cpp",
- "segments": [
- [10, 1, 10, True, True, False],
- [13, 1, 0, False, False, False],
- ],
- "mcdc_records": [
- [11, 7, 11, 40, 3, 2, 3, 1, 1, [True, True, False]]
- ],
- }
- ]
- }
- ]
+
+ diff_files = DiffParser.parse(mock_diff)
+ coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
+ summary = calculate_patch_statistics(diff_files, coverage_matrix)
+
+ self.assertEqual(summary.total_lines, 1)
+ self.assertEqual(summary.total_covered_lines, 1)
+ self.assertEqual(summary.total_missed_lines, 0)
+
+ def test_calculate_patch_statistics_missed_line(self):
+ mock_diff = (
+ "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
+ "--- a/src/math/sin.cpp\n"
+ "+++ b/src/math/sin.cpp\n"
+ "@@ -10,3 +10,2 @@\n"
+ " context();\n"
+ "+return 0;\n"
+ )
+ mock_json = {
+ "data": [{
+ "files": [{
+ "filename": "/workspace/src/math/sin.cpp",
+ "segments": [
+ [10, 0, 1, 1, 1],
+ [11, 0, 0, 1, 1]
+ ],
+ "branches": []
+ }]
+ }]
}
- matrix = CoverageJSONParser.extract_patch_matrix(cov_data, diff_files)
- stats = calculate_patch_statistics(diff_files, matrix)
-
- self.assertTrue(stats.has_mcdc)
- self.assertEqual(stats.total_mcdc_covered_conditions, 2)
- self.assertEqual(stats.total_mcdc_total_conditions, 3)
- self.assertEqual(stats.mcdc_coverage_percentage, 66.66666666666666)
- self.assertEqual(stats.total_decisions_count, 1)
- self.assertEqual(stats.fully_verified_decisions, 0)
-
- file_metrics = stats.files["libc/src/ctype/isspace.cpp"]
- self.assertIn("C3 unverified", file_metrics.condition_diagnostics[0])
- self.assertEqual(file_metrics.unverified_decision_lines[11], ["C3"])
-
- def test_format_status_banner(self) -> None:
- # Full statement & full MC/DC
- summary_full = PatchCoverageSummary(
- total_covered_lines=10,
- total_missed_lines=0,
- total_mcdc_covered_conditions=4,
- total_mcdc_total_conditions=4,
- total_decisions_count=2,
- fully_verified_decisions=2,
+
+ diff_files = DiffParser.parse(mock_diff)
+ coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
+ summary = calculate_patch_statistics(diff_files, coverage_matrix)
+
+ self.assertEqual(summary.total_lines, 1)
+ self.assertEqual(summary.total_covered_lines, 0)
+ self.assertEqual(summary.total_missed_lines, 1)
+
+class TestPatchReportRendering(unittest.TestCase):
+ def test_format_status_banner(self):
+ summary = PatchCoverageSummary(
+ total_covered_lines=50,
+ total_missed_lines=50,
+ total_mcdc_total_conditions=10,
+ total_mcdc_covered_conditions=5,
+ files={}
)
- banner_full = format_status_banner(summary_full)
- self.assertIn("100.00% Line", banner_full)
- self.assertIn("100.00% MC/DC", banner_full)
-
- # Full statement, partial MC/DC
- summary_partial = PatchCoverageSummary(
- total_covered_lines=10,
- total_missed_lines=0,
- total_mcdc_covered_conditions=3,
- total_mcdc_total_conditions=4,
- total_decisions_count=2,
- fully_verified_decisions=1,
+ banner = format_status_banner(summary)
+ self.assertIn("### Patch Coverage:", banner)
+ self.assertIn("50.00% Line", banner)
+
+ def test_format_breakdown_table(self):
+ file_stat = FilePatchMetrics(
+ file_path="src/math/sin.cpp",
+ covered_lines={10, 11},
+ missed_lines=set(),
+ added_lines={10, 11}
)
- banner_partial = format_status_banner(summary_partial)
- self.assertIn("75.0% MC/DC", banner_partial)
-
- # Lines missed
- summary_warn = PatchCoverageSummary(
- total_covered_lines=8,
- total_missed_lines=2,
+ summary = PatchCoverageSummary(
+ total_covered_lines=2,
+ total_missed_lines=0,
+ files={"src/math/sin.cpp": file_stat}
)
- banner_warn = format_status_banner(summary_warn)
- self.assertIn("80.00%", banner_warn)
- self.assertIn("unexecuted lines detected in patch", banner_warn)
-
- def test_format_metadata_section(self) -> None:
- metadata_string = format_metadata_section(
- base_commit_sha="abcdef1234567890",
- head_commit_sha="123456abcdef7890",
- base_branch_name="main",
- head_branch_name="my-pr",
- targeted_tests_string="libc.test.src.math.sin_test libc.test.src.math.cos_test",
- base_repository="llvm/llvm-project",
- head_repository="user/llvm-project",
+ report = format_breakdown_table(summary)
+ self.assertIn("[`src/math/sin.cpp`]", report)
+ self.assertIn("**100.00%**", report)
+
+class TestDiffParserEdgeCases(unittest.TestCase):
+ def test_deleted_file(self):
+ mock_diff = (
+ "diff --git a/src/math/old.cpp b/src/math/old.cpp\n"
+ "deleted file mode 100644\n"
+ "--- a/src/math/old.cpp\n"
+ "+++ /dev/null\n"
+ "@@ -1,3 +0,0 @@\n"
+ "-deleted_line_1();\n"
+ "-deleted_line_2();\n"
)
- self.assertIn("abcdef1", metadata_string)
- self.assertIn("123456a", metadata_string)
- self.assertIn("`libc.test.src.math.sin_test`, `libc.test.src.math.cos_test`", metadata_string)
-
- def test_format_breakdown_table_standard_and_mcdc(self) -> None:
- file_metrics = FilePatchMetrics(
- file_path="libc/src/math/tan.cpp",
- covered_lines={10, 11},
- missed_lines={12},
- mcdc_covered_conditions=2,
- mcdc_total_conditions=2,
- decisions_verified=1,
- decisions_total=1,
- condition_diagnostics=["`L10`: 2/2 verified"],
+ hunks_dict = DiffParser.parse(mock_diff)
+ self.assertEqual(len(hunks_dict), 0)
+
+ def test_no_newline_at_eof(self):
+ mock_diff = (
+ "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
+ "--- a/src/math/sin.cpp\n"
+ "+++ b/src/math/sin.cpp\n"
+ "@@ -1,2 +1,3 @@\n"
+ " context();\n"
+ "+new_line();\n"
+ "\\ No newline at end of file\n"
)
- summary_mcdc = PatchCoverageSummary(
- files={"libc/src/math/tan.cpp": file_metrics},
- total_covered_lines=2,
- total_missed_lines=1,
- total_mcdc_covered_conditions=2,
- total_mcdc_total_conditions=2,
- total_decisions_count=1,
- fully_verified_decisions=1,
+ hunks_dict = DiffParser.parse(mock_diff)
+ hunks = hunks_dict["src/math/sin.cpp"]
+ added_lines = [line for line in hunks[0].lines if line[0] == "+"]
+ self.assertEqual(len(added_lines), 1)
+
+class TestPathResolution(unittest.TestCase):
+ def test_fuzzy_path_matching(self):
+ diff_files = {"libc/src/math/sin.cpp": []}
+ mock_json = {
+ "data": [{
+ "files": [{
+ "filename": "/home/runner/work/llvm-project/libc/src/math/sin.cpp",
+ "segments": [[10, 0, 1, 1, 1]],
+ "mcdc_records": []
+ }]
+ }]
+ }
+ coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
+ self.assertIn("libc/src/math/sin.cpp", coverage_matrix)
+
+class TestZeroStateBoundaries(unittest.TestCase):
+ def test_empty_json_data(self):
+ diff_files = {"src/math/sin.cpp": []}
+ mock_json = {"data": []}
+ coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
+ self.assertIn("src/math/sin.cpp", coverage_matrix)
+ self.assertEqual(len(coverage_matrix["src/math/sin.cpp"]["covered"]), 0)
+
+ def test_purely_cosmetic_patch(self):
+ mock_diff = (
+ "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
+ "--- a/src/math/sin.cpp\n"
+ "+++ b/src/math/sin.cpp\n"
+ "@@ -10,3 +10,2 @@\n"
+ " context();\n"
+ "+// Just a comment\n"
)
- table_mcdc = format_breakdown_table(summary_mcdc, head_repository="llvm/llvm-project")
- self.assertIn("MC/DC Conditions", table_mcdc)
- self.assertIn("`L10`: 2/2 verified", table_mcdc)
-
- summary_std = PatchCoverageSummary(
- files={"libc/src/math/tan.cpp": file_metrics},
- total_covered_lines=2,
- total_missed_lines=1,
+ diff_files = DiffParser.parse(mock_diff)
+ coverage_matrix = {"src/math/sin.cpp": {"covered": set(), "missed": set(), "mcdc_decisions": []}}
+ summary = calculate_patch_statistics(diff_files, coverage_matrix)
+ self.assertEqual(summary.total_lines, 0)
+ self.assertEqual(summary.line_coverage_percentage, 0.0)
+
+class TestMCDCIntersection(unittest.TestCase):
+ def test_mcdc_boolean_extraction(self):
+ mock_diff = (
+ "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
+ "--- a/src/math/sin.cpp\n"
+ "+++ b/src/math/sin.cpp\n"
+ "@@ -10,3 +10,2 @@\n"
+ " context();\n"
+ "+if (a && b) { return 0; }\n"
)
- table_std = format_breakdown_table(summary_std, head_repository="llvm/llvm-project")
- self.assertIn("Unexecuted Line Spans", table_std)
- self.assertIn("`L12`", table_std)
-
- def test_render_patch_report_empty_diff(self) -> None:
- stdout_buffer = io.StringIO()
- with redirect_stdout(stdout_buffer):
- render_patch_report(
- diff_files={},
- coverage_matrix={},
- base_commit_sha="base",
- head_commit_sha="head",
- base_branch_name="main",
- head_branch_name="patch",
- )
- output_text = stdout_buffer.getvalue()
- self.assertIn("### Coverage Summary", output_text)
- self.assertIn("No `.cpp` source files in `libc/src/` were modified", output_text)
-
-
-if __name__ == "__main__":
+ mock_json = {
+ "data": [{
+ "files": [{
+ "filename": "/workspace/src/math/sin.cpp",
+ "segments": [
+ [11, 0, 1, 1, 1]
+ ],
+ "mcdc_records": [
+ [11, 4, 11, 14, 0, 0, 0, 0, 0, [True, False]]
+ ]
+ }]
+ }]
+ }
+ diff_files = DiffParser.parse(mock_diff)
+ coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
+ summary = calculate_patch_statistics(diff_files, coverage_matrix)
+
+ self.assertEqual(summary.total_mcdc_total_conditions, 2)
+ self.assertEqual(summary.total_mcdc_covered_conditions, 1)
+
+if __name__ == '__main__':
unittest.main()
>From dd99bdce69210ffca09a54f019bdda09c9948d0b Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Wed, 2 Sep 2026 09:28:15 +0000
Subject: [PATCH 06/15] [libc][ci] Enable continuous profiling and fix
PYTHONPATH in workflows
---
.github/workflows/libc-full-coverage.yml | 4 ++--
.github/workflows/libc-full-mcdc.yml | 4 ++--
.github/workflows/libc-patch-coverage.yml | 4 ++--
.github/workflows/libc-patch-mcdc.yml | 4 ++--
4 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/libc-full-coverage.yml b/.github/workflows/libc-full-coverage.yml
index bded8dbe28d5d..fe8aece95488e 100644
--- a/.github/workflows/libc-full-coverage.yml
+++ b/.github/workflows/libc-full-coverage.yml
@@ -85,7 +85,7 @@ jobs:
echo "=== [STAGE 2/4] Running Full Unit Test Suite at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
START_TIME=$(date +%s)
- export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
+ export LLVM_PROFILE_FILE="%clibc_cov_%p.profraw"
ninja -k 0 -C build-cov libc-unit-tests || true
END_TIME=$(date +%s)
@@ -128,7 +128,7 @@ jobs:
touch coverage_html/.nojekyll
# 3. Run Codebase Coverage Analyzer
- python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
+ PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
- name: Deploy Coverage Dashboard to GitHub Pages
diff --git a/.github/workflows/libc-full-mcdc.yml b/.github/workflows/libc-full-mcdc.yml
index 8aec845f0f301..359ea5da59dde 100644
--- a/.github/workflows/libc-full-mcdc.yml
+++ b/.github/workflows/libc-full-mcdc.yml
@@ -86,7 +86,7 @@ jobs:
echo "=== [STAGE 2/4] Running Full Unit Test Suite at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
START_TIME=$(date +%s)
- export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
+ export LLVM_PROFILE_FILE="%clibc_cov_%p.profraw"
ninja -k 0 -C build-cov libc-unit-tests || true
END_TIME=$(date +%s)
@@ -131,7 +131,7 @@ jobs:
touch coverage_mcdc_html/.nojekyll
# 3. Run Codebase Coverage Analyzer
- python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
+ PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
- name: Deploy MC/DC Coverage Report to GitHub Pages
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
index db5257c8a4797..b63525095074c 100644
--- a/.github/workflows/libc-patch-coverage.yml
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -200,7 +200,7 @@ jobs:
echo "[LOG] Executing Ninja targets: $TARGETS"
if [ -n "$TARGETS" ]; then
- export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
+ export LLVM_PROFILE_FILE="%clibc_cov_%p.profraw"
ninja -C build-cov $TARGETS
echo "TARGETS=$TARGETS" >> $GITHUB_ENV
else
@@ -267,7 +267,7 @@ jobs:
git diff "$DIFF_BASE" HEAD -- libc/src/ > patch.diff
# 3. Run Diff Coverage Python Script using persisted commit metadata
- python3 libc/utils/coverage/diff_coverage.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
+ PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/diff_coverage.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
echo "[LOG] Coverage report successfully written to GITHUB_STEP_SUMMARY."
diff --git a/.github/workflows/libc-patch-mcdc.yml b/.github/workflows/libc-patch-mcdc.yml
index 22e91fcda8902..07c4c25586046 100644
--- a/.github/workflows/libc-patch-mcdc.yml
+++ b/.github/workflows/libc-patch-mcdc.yml
@@ -210,7 +210,7 @@ jobs:
echo "[LOG] Executing Ninja targets: $TARGETS"
if [ -n "$TARGETS" ]; then
- export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
+ export LLVM_PROFILE_FILE="%clibc_cov_%p.profraw"
ninja -C build-cov $TARGETS
echo "TARGETS=$TARGETS" >> $GITHUB_ENV
else
@@ -277,7 +277,7 @@ jobs:
git diff "$DIFF_BASE" HEAD -- libc/src/ > patch.diff
# 3. Run Diff Coverage Python Script using persisted commit metadata
- python3 libc/utils/coverage/diff_coverage.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
+ PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/diff_coverage.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
echo "[LOG] Coverage report successfully written to GITHUB_STEP_SUMMARY."
>From a30af49b0d1c3e17e4a7cc2f976260d19fadb6ea Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Wed, 2 Sep 2026 14:25:03 +0000
Subject: [PATCH 07/15] [libc][ci] Revert continuous profiling flag from
coverage workflows
---
.github/workflows/libc-full-coverage.yml | 2 +-
.github/workflows/libc-full-mcdc.yml | 2 +-
.github/workflows/libc-patch-coverage.yml | 2 +-
.github/workflows/libc-patch-mcdc.yml | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/libc-full-coverage.yml b/.github/workflows/libc-full-coverage.yml
index fe8aece95488e..ebaa9f2a39fb6 100644
--- a/.github/workflows/libc-full-coverage.yml
+++ b/.github/workflows/libc-full-coverage.yml
@@ -85,7 +85,7 @@ jobs:
echo "=== [STAGE 2/4] Running Full Unit Test Suite at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
START_TIME=$(date +%s)
- export LLVM_PROFILE_FILE="%clibc_cov_%p.profraw"
+ export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
ninja -k 0 -C build-cov libc-unit-tests || true
END_TIME=$(date +%s)
diff --git a/.github/workflows/libc-full-mcdc.yml b/.github/workflows/libc-full-mcdc.yml
index 359ea5da59dde..fdebc1c66f399 100644
--- a/.github/workflows/libc-full-mcdc.yml
+++ b/.github/workflows/libc-full-mcdc.yml
@@ -86,7 +86,7 @@ jobs:
echo "=== [STAGE 2/4] Running Full Unit Test Suite at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
START_TIME=$(date +%s)
- export LLVM_PROFILE_FILE="%clibc_cov_%p.profraw"
+ export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
ninja -k 0 -C build-cov libc-unit-tests || true
END_TIME=$(date +%s)
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
index b63525095074c..5cd2b723862e1 100644
--- a/.github/workflows/libc-patch-coverage.yml
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -200,7 +200,7 @@ jobs:
echo "[LOG] Executing Ninja targets: $TARGETS"
if [ -n "$TARGETS" ]; then
- export LLVM_PROFILE_FILE="%clibc_cov_%p.profraw"
+ export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
ninja -C build-cov $TARGETS
echo "TARGETS=$TARGETS" >> $GITHUB_ENV
else
diff --git a/.github/workflows/libc-patch-mcdc.yml b/.github/workflows/libc-patch-mcdc.yml
index 07c4c25586046..8e0b25d89e090 100644
--- a/.github/workflows/libc-patch-mcdc.yml
+++ b/.github/workflows/libc-patch-mcdc.yml
@@ -210,7 +210,7 @@ jobs:
echo "[LOG] Executing Ninja targets: $TARGETS"
if [ -n "$TARGETS" ]; then
- export LLVM_PROFILE_FILE="%clibc_cov_%p.profraw"
+ export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
ninja -C build-cov $TARGETS
echo "TARGETS=$TARGETS" >> $GITHUB_ENV
else
>From 80d89a220ddc1195c725d0a65c14671b665dad1a Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Thu, 3 Sep 2026 10:09:32 +0000
Subject: [PATCH 08/15] [libc][ci] Replace GitHub Pages deployment with
workflow artifact upload
Replace the third-party peaceiris/actions-gh-pages deployment action with
standard actions/upload-artifact at v4 in both full coverage workflows. This
aligns with LLVM CI security best practices by removing unnecessary
contents: write repository permissions and untrusted third-party actions.
Harden raw profile discovery and report generation with defensive guards.
Configure overwrite: true on artifact uploads to ensure robustness across
workflow re-runs. Update codebase_coverage.py and associated unit tests to
direct users to the workflow Artifacts section.
Assisted-by: Automated tooling, human reviewed.
---
.github/workflows/libc-full-coverage.yml | 36 ++++++-----
.github/workflows/libc-full-mcdc.yml | 37 ++++++-----
.github/workflows/libc-patch-coverage.yml | 2 +-
.github/workflows/libc-patch-mcdc.yml | 2 +-
libc/utils/coverage/codebase_coverage.py | 18 +++---
libc/utils/coverage/test_codebase_coverage.py | 61 ++++++++++++++++++-
6 files changed, 113 insertions(+), 43 deletions(-)
diff --git a/.github/workflows/libc-full-coverage.yml b/.github/workflows/libc-full-coverage.yml
index ebaa9f2a39fb6..c1122d88a49a1 100644
--- a/.github/workflows/libc-full-coverage.yml
+++ b/.github/workflows/libc-full-coverage.yml
@@ -1,7 +1,7 @@
name: Libc Full Codebase Coverage
permissions:
- contents: write # Required to publish live HTML dashboard to gh-pages branch
+ contents: read
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
@@ -94,11 +94,15 @@ jobs:
- name: Merge Profiles
run: |
echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- find . build-cov -name "libc_cov_*.profraw" > profraw_list.txt
+ find . -name "libc_cov_*.profraw" > profraw_list.txt
NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
echo "[LOG] Discovered $NUM_PROFS raw profile data files."
- llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
- echo "[LOG] Merged full codebase profile into libc_full.profdata."
+ if [ -s profraw_list.txt ]; then
+ llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
+ echo "[LOG] Merged full codebase profile into libc_full.profdata."
+ else
+ echo "[LOG] Warning: No profraw files found."
+ fi
- name: Generate Reports and Summary
env:
@@ -112,6 +116,13 @@ jobs:
echo "[LOG] Exporting coverage data across ${#EXECUTABLES[@]} test binaries."
+ if [ ! -f libc_full.profdata ] || [ ${#EXECUTABLES[@]} -eq 0 ]; then
+ echo "[LOG] Warning: Profile data or test executables missing. Skipping report generation."
+ echo "### LLVM-libc Coverage Report" >> $GITHUB_STEP_SUMMARY
+ echo "No coverage data was generated for this run." >> $GITHUB_STEP_SUMMARY
+ exit 0
+ fi
+
# 1. Generate JSON export for Full Coverage Analyzer
llvm-cov-23 export -format=text -instr-profile=libc_full.profdata "${EXECUTABLES[0]}" "${OBJECTS[@]}" > coverage.json
@@ -125,19 +136,14 @@ jobs:
--compilation-dir=. \
--path-equivalence="$GITHUB_WORKSPACE,." \
-ignore-filename-regex=".*(test|utils).*"
- touch coverage_html/.nojekyll
-
# 3. Run Codebase Coverage Analyzer
PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
- - name: Deploy Coverage Dashboard to GitHub Pages
- if: github.repository != 'llvm/llvm-project'
- continue-on-error: true
- uses: peaceiris/actions-gh-pages at v4
+ - name: Upload HTML Coverage Artifact
+ uses: actions/upload-artifact at v4
with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
- publish_dir: ./coverage_html
- keep_files: true
- enable_jekyll: false
- force_orphan: false
+ name: libc-coverage-html
+ path: coverage_html/
+ retention-days: 14
+ overwrite: true
diff --git a/.github/workflows/libc-full-mcdc.yml b/.github/workflows/libc-full-mcdc.yml
index fdebc1c66f399..a29f6e20fb661 100644
--- a/.github/workflows/libc-full-mcdc.yml
+++ b/.github/workflows/libc-full-mcdc.yml
@@ -1,7 +1,7 @@
name: Libc Full Codebase MC/DC Coverage
permissions:
- contents: write
+ contents: read
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
@@ -95,11 +95,15 @@ jobs:
- name: Merge Profiles
run: |
echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- find build-cov -name "libc_cov_*.profraw" > profraw_list.txt
+ find . -name "libc_cov_*.profraw" > profraw_list.txt
NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
echo "[LOG] Discovered $NUM_PROFS raw profile data files."
- llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
- echo "[LOG] Merged full codebase profile into libc_full.profdata."
+ if [ -s profraw_list.txt ]; then
+ llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
+ echo "[LOG] Merged full codebase profile into libc_full.profdata."
+ else
+ echo "[LOG] Warning: No profraw files found."
+ fi
- name: Generate Reports and Summary
env:
@@ -113,6 +117,13 @@ jobs:
echo "[LOG] Exporting coverage data across ${#EXECUTABLES[@]} test binaries."
+ if [ ! -f libc_full.profdata ] || [ ${#EXECUTABLES[@]} -eq 0 ]; then
+ echo "[LOG] Warning: Profile data or test executables missing. Skipping report generation."
+ echo "### LLVM-libc MC/DC Coverage Report" >> $GITHUB_STEP_SUMMARY
+ echo "No coverage data was generated for this run." >> $GITHUB_STEP_SUMMARY
+ exit 0
+ fi
+
# 1. Generate JSON export for Full Coverage Analyzer (contains mcdc_records)
llvm-cov-23 export -format=text -instr-profile=libc_full.profdata "${EXECUTABLES[0]}" "${OBJECTS[@]}" > coverage.json
@@ -128,22 +139,16 @@ jobs:
--compilation-dir=. \
--path-equivalence="$GITHUB_WORKSPACE,." \
-ignore-filename-regex=".*(test|utils).*"
- touch coverage_mcdc_html/.nojekyll
-
# 3. Run Codebase Coverage Analyzer
PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
- - name: Deploy MC/DC Coverage Report to GitHub Pages
- if: github.repository != 'llvm/llvm-project'
- continue-on-error: true
- uses: peaceiris/actions-gh-pages at v4
+ - name: Upload MC/DC Coverage Artifact
+ uses: actions/upload-artifact at v4
with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
- publish_dir: ./coverage_mcdc_html
- destination_dir: mcdc
- keep_files: true
- enable_jekyll: false
- force_orphan: false
+ name: libc-mcdc-coverage-html
+ path: coverage_mcdc_html/
+ retention-days: 14
+ overwrite: true
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
index 5cd2b723862e1..9c4c20a45a21d 100644
--- a/.github/workflows/libc-patch-coverage.yml
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -226,7 +226,7 @@ jobs:
if: env.TARGETS != ''
run: |
echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- find build-cov -name "libc_cov_*.profraw" > profraw_list.txt
+ find . -name "libc_cov_*.profraw" > profraw_list.txt
NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
echo "[LOG] Discovered $NUM_PROFS raw profile data files."
if [ -s profraw_list.txt ]; then
diff --git a/.github/workflows/libc-patch-mcdc.yml b/.github/workflows/libc-patch-mcdc.yml
index 8e0b25d89e090..d50e150347d4c 100644
--- a/.github/workflows/libc-patch-mcdc.yml
+++ b/.github/workflows/libc-patch-mcdc.yml
@@ -236,7 +236,7 @@ jobs:
if: env.TARGETS != ''
run: |
echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- find build-cov -name "libc_cov_*.profraw" > profraw_list.txt
+ find . -name "libc_cov_*.profraw" > profraw_list.txt
NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
echo "[LOG] Discovered $NUM_PROFS raw profile data files."
if [ -s profraw_list.txt ]; then
diff --git a/libc/utils/coverage/codebase_coverage.py b/libc/utils/coverage/codebase_coverage.py
index 7876c8e40106e..7b1663389d708 100644
--- a/libc/utils/coverage/codebase_coverage.py
+++ b/libc/utils/coverage/codebase_coverage.py
@@ -83,15 +83,10 @@ def has_mcdc(self) -> bool:
def resolve_dashboard_url(has_mcdc: bool) -> str:
- """Resolves the live dashboard URL based on repository environment variables."""
- pages_url = os.environ.get("COVERAGE_DASHBOARD_URL")
+ """Resolves the live dashboard URL if explicitly configured via environment variable."""
+ pages_url = os.environ.get("COVERAGE_DASHBOARD_URL", "").strip()
if not pages_url:
- repo = os.environ.get("GITHUB_REPOSITORY", DEFAULT_REPOSITORY)
- if "/" in repo:
- owner, repo_name = repo.split("/", 1)
- pages_url = f"https://{owner}.github.io/{repo_name}/"
- else:
- pages_url = f"https://{repo}.github.io/"
+ return ""
if has_mcdc and not pages_url.endswith("/mcdc/"):
return pages_url.rstrip("/") + "/mcdc/"
@@ -194,7 +189,12 @@ def format_overview_callout(summary: FullCoverageSummary) -> str:
)
lines.append("")
- lines.append(f"- **Coverage Dashboard:** [{summary.dashboard_url}]({summary.dashboard_url})")
+ if summary.dashboard_url:
+ lines.append(f"- **Coverage Dashboard:** [{summary.dashboard_url}]({summary.dashboard_url})")
+ else:
+ lines.append(
+ "- **HTML Coverage Report:** Available for download under the **Artifacts** section of this workflow run."
+ )
return "\n".join(lines)
diff --git a/libc/utils/coverage/test_codebase_coverage.py b/libc/utils/coverage/test_codebase_coverage.py
index ce691385c8b28..374fe46d77c66 100644
--- a/libc/utils/coverage/test_codebase_coverage.py
+++ b/libc/utils/coverage/test_codebase_coverage.py
@@ -13,7 +13,9 @@
FullCoverageSummary,
extract_full_coverage_statistics,
format_directory_breakdown_table,
- render_full_report
+ format_overview_callout,
+ render_full_report,
+ resolve_dashboard_url,
)
class TestCodebaseDirectoryAggregation(unittest.TestCase):
@@ -78,5 +80,62 @@ def test_format_directory_breakdown_table(self):
self.assertIn("`libc/src/math`", table)
self.assertIn("`libc/src/string`", table)
+ def test_format_overview_callout_without_dashboard_url(self):
+ metrics = DirectoryCoverageMetrics(name="global", lines_tot=100, lines_cov=75)
+ summary = FullCoverageSummary(global_stats=metrics, dashboard_url="")
+ callout = format_overview_callout(summary)
+ self.assertIn("Artifacts", callout)
+ self.assertIn("HTML Coverage Report", callout)
+ self.assertNotIn("Coverage Dashboard:", callout)
+
+ def test_format_overview_callout_with_dashboard_url(self):
+ metrics = DirectoryCoverageMetrics(name="global", lines_tot=100, lines_cov=75)
+ summary = FullCoverageSummary(
+ global_stats=metrics, dashboard_url="https://example.com/coverage/"
+ )
+ callout = format_overview_callout(summary)
+ self.assertIn("Coverage Dashboard:", callout)
+ self.assertIn("https://example.com/coverage/", callout)
+
+ def test_resolve_dashboard_url_default_is_empty(self):
+ import os
+ old_env = os.environ.get("COVERAGE_DASHBOARD_URL")
+ try:
+ if "COVERAGE_DASHBOARD_URL" in os.environ:
+ del os.environ["COVERAGE_DASHBOARD_URL"]
+ self.assertEqual(resolve_dashboard_url(has_mcdc=False), "")
+ self.assertEqual(resolve_dashboard_url(has_mcdc=True), "")
+ finally:
+ if old_env is not None:
+ os.environ["COVERAGE_DASHBOARD_URL"] = old_env
+
+ def test_render_full_report_end_to_end(self):
+ import io
+ from contextlib import redirect_stdout
+
+ mock_json = {
+ "data": [{
+ "files": [
+ {
+ "filename": "/workspace/libc/src/math/sin.cpp",
+ "summary": {
+ "lines": {"count": 100, "covered": 80, "percent": 80.0},
+ "functions": {"count": 2, "covered": 2, "percent": 100.0},
+ },
+ }
+ ]
+ }]
+ }
+ buf = io.StringIO()
+ with redirect_stdout(buf):
+ render_full_report(mock_json)
+ output = buf.getvalue()
+
+ self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
+ self.assertIn("HTML Coverage Report", output)
+ self.assertIn("Artifacts", output)
+ self.assertIn("### Overall", output)
+ self.assertIn("`libc/src/math`", output)
+
if __name__ == '__main__':
unittest.main()
>From 816a46948ae1d39c18e82fa500d7b22d1a43ea23 Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Thu, 3 Sep 2026 17:05:49 +0000
Subject: [PATCH 09/15] [libc][ci] Address reviewer feedback and bot check
failures
Pin workflow actions to 40-character commit hashes to comply with
Zizmor security policy.
Format codebase and diff coverage Python utilities and unit test suites
to conform to Black/darker standards (88-character limit, standard spacing).
Remove unnecessary container --privileged flag and redundant manual step
timing logic from coverage workflows.
Assisted-by: Automated tooling, human reviewed.
---
.github/workflows/libc-full-coverage.yml | 27 +--
.github/workflows/libc-full-mcdc.yml | 27 +--
.github/workflows/libc-patch-coverage.yml | 36 ++-
.github/workflows/libc-patch-mcdc.yml | 26 +--
libc/utils/coverage/codebase_coverage.py | 89 +++++--
libc/utils/coverage/diff_coverage.py | 219 +++++++++++++-----
libc/utils/coverage/test_codebase_coverage.py | 148 ++++++++----
libc/utils/coverage/test_diff_coverage.py | 140 ++++++-----
8 files changed, 448 insertions(+), 264 deletions(-)
diff --git a/.github/workflows/libc-full-coverage.yml b/.github/workflows/libc-full-coverage.yml
index c1122d88a49a1..4f8cc19563d62 100644
--- a/.github/workflows/libc-full-coverage.yml
+++ b/.github/workflows/libc-full-coverage.yml
@@ -35,19 +35,18 @@ jobs:
runs-on: ubuntu-24.04
container:
image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
- options: --privileged
defaults:
run:
shell: bash
steps:
- name: Checkout Code
- uses: actions/checkout at v4
+ uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
fetch-depth: 1
persist-credentials: false
- name: Setup Compiler Cache (sccache)
- uses: hendrikmuhs/ccache-action at v1.2.23
+ uses: hendrikmuhs/ccache-action at d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
with:
max-size: 1G
key: libc_coverage_unified_v2_x86_64
@@ -55,9 +54,6 @@ jobs:
- name: Configure CMake
run: |
- echo "=== [STAGE 1/4] Starting CMake Configuration at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- START_TIME=$(date +%s)
-
export CMAKE_FLAGS="
-G Ninja
-S runtimes
@@ -77,23 +73,13 @@ jobs:
"
cmake $CMAKE_FLAGS
- END_TIME=$(date +%s)
- echo "=== [STAGE 1/4] CMake Configuration Completed in $((END_TIME - START_TIME))s ==="
-
- name: Run Full Codebase Unit Tests
run: |
- echo "=== [STAGE 2/4] Running Full Unit Test Suite at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- START_TIME=$(date +%s)
-
export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
ninja -k 0 -C build-cov libc-unit-tests || true
- END_TIME=$(date +%s)
- echo "=== [STAGE 2/4] libc-unit-tests Completed in $((END_TIME - START_TIME))s ==="
-
- name: Merge Profiles
run: |
- echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
find . -name "libc_cov_*.profraw" > profraw_list.txt
NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
echo "[LOG] Discovered $NUM_PROFS raw profile data files."
@@ -109,7 +95,6 @@ jobs:
COMMIT_SHA: ${{ github.sha }}
BRANCH_REF: ${{ github.ref_name }}
run: |
- echo "=== [STAGE 4/4] Generating Full Codebase Coverage Reports at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
OBJECTS=("${EXECUTABLES[@]:1}")
OBJECTS=("${OBJECTS[@]/#/-object=}")
@@ -140,8 +125,14 @@ jobs:
PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
+ - name: Fallback Failure Summary
+ if: failure() && !hashFiles('coverage.json')
+ run: |
+ echo "### LLVM-libc Coverage Report" >> $GITHUB_STEP_SUMMARY
+ echo "Workflow run encountered an error before coverage reports could be generated. Inspect job logs for details." >> $GITHUB_STEP_SUMMARY
+
- name: Upload HTML Coverage Artifact
- uses: actions/upload-artifact at v4
+ uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: libc-coverage-html
path: coverage_html/
diff --git a/.github/workflows/libc-full-mcdc.yml b/.github/workflows/libc-full-mcdc.yml
index a29f6e20fb661..75acf682fd761 100644
--- a/.github/workflows/libc-full-mcdc.yml
+++ b/.github/workflows/libc-full-mcdc.yml
@@ -35,19 +35,18 @@ jobs:
runs-on: ubuntu-24.04
container:
image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
- options: --privileged
defaults:
run:
shell: bash
steps:
- name: Checkout Code
- uses: actions/checkout at v4
+ uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
fetch-depth: 1
persist-credentials: false
- name: Setup Compiler Cache (sccache)
- uses: hendrikmuhs/ccache-action at v1.2.23
+ uses: hendrikmuhs/ccache-action at d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
with:
max-size: 1G
key: libc_mcdc_coverage_unified_x86_64
@@ -55,9 +54,6 @@ jobs:
- name: Configure CMake with MC/DC
run: |
- echo "=== [STAGE 1/4] Starting CMake Configuration (MC/DC) at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- START_TIME=$(date +%s)
-
export CMAKE_FLAGS="
-G Ninja
-S runtimes
@@ -78,23 +74,13 @@ jobs:
"
cmake $CMAKE_FLAGS
- END_TIME=$(date +%s)
- echo "=== [STAGE 1/4] CMake Configuration Completed in $((END_TIME - START_TIME))s ==="
-
- name: Run Full Codebase Unit Tests
run: |
- echo "=== [STAGE 2/4] Running Full Unit Test Suite at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- START_TIME=$(date +%s)
-
export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
ninja -k 0 -C build-cov libc-unit-tests || true
- END_TIME=$(date +%s)
- echo "=== [STAGE 2/4] libc-unit-tests Completed in $((END_TIME - START_TIME))s ==="
-
- name: Merge Profiles
run: |
- echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
find . -name "libc_cov_*.profraw" > profraw_list.txt
NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
echo "[LOG] Discovered $NUM_PROFS raw profile data files."
@@ -110,7 +96,6 @@ jobs:
COMMIT_SHA: ${{ github.sha }}
BRANCH_REF: ${{ github.ref_name }}
run: |
- echo "=== [STAGE 4/4] Generating Full Codebase MC/DC Coverage Reports at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
OBJECTS=("${EXECUTABLES[@]:1}")
OBJECTS=("${OBJECTS[@]/#/-object=}")
@@ -143,8 +128,14 @@ jobs:
PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
+ - name: Fallback Failure Summary
+ if: failure() && !hashFiles('coverage.json')
+ run: |
+ echo "### LLVM-libc MC/DC Coverage Report" >> $GITHUB_STEP_SUMMARY
+ echo "Workflow run encountered an error before coverage reports could be generated. Inspect job logs for details." >> $GITHUB_STEP_SUMMARY
+
- name: Upload MC/DC Coverage Artifact
- uses: actions/upload-artifact at v4
+ uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: libc-mcdc-coverage-html
path: coverage_mcdc_html/
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
index 9c4c20a45a21d..e6dbf15c388cb 100644
--- a/.github/workflows/libc-patch-coverage.yml
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -49,21 +49,19 @@ jobs:
runs-on: ubuntu-24.04
container:
image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
- options: >-
- --privileged
defaults:
run:
shell: bash
steps:
- name: Checkout Code
- uses: actions/checkout at v4
+ uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
fetch-depth: 2
persist-credentials: false
- name: Setup Compiler Cache (sccache)
- uses: hendrikmuhs/ccache-action at v1.2.23
+ uses: hendrikmuhs/ccache-action at d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
with:
max-size: 1G
key: libc_coverage_unified_v2_x86_64
@@ -71,9 +69,6 @@ jobs:
- name: Configure CMake
run: |
- echo "=== [STAGE 1/4] Starting CMake Configuration at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- START_TIME=$(date +%s)
-
export CMAKE_FLAGS="
-G Ninja
-S runtimes
@@ -93,9 +88,6 @@ jobs:
"
cmake $CMAKE_FLAGS
- END_TIME=$(date +%s)
- echo "=== [STAGE 1/4] CMake Configuration Completed in $((END_TIME - START_TIME))s ==="
-
- name: Build and Run Targeted Tests
env:
EVENT_NAME: ${{ github.event_name }}
@@ -109,9 +101,6 @@ jobs:
BRANCH_REF: ${{ github.ref_name }}
REPO_NAME: ${{ github.repository }}
run: |
- echo "=== [STAGE 2/4] Starting Targeted Test Discovery at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- START_TIME=$(date +%s)
-
# Ensure upstream llvm-project remote is configured
git remote add upstream https://github.com/llvm/llvm-project.git 2>/dev/null || true
@@ -125,6 +114,16 @@ jobs:
HEAD_REPO="${PR_HEAD_REPO:-$REPO_NAME}"
git fetch upstream "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_REF" --depth=1 2>/dev/null || true
DIFF_BASE="$BASE_SHA"
+ elif [ "$BRANCH_REF" != "main" ]; then
+ # For feature branches, compare against main merge-base
+ git fetch origin main --depth=100 2>/dev/null || true
+ BASE_SHA=$(git merge-base origin/main HEAD 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || echo "HEAD~1")
+ HEAD_SHA="$COMMIT_SHA"
+ BASE_REF="main"
+ HEAD_REF="$BRANCH_REF"
+ BASE_REPO="llvm/llvm-project"
+ HEAD_REPO="$REPO_NAME"
+ DIFF_BASE="$BASE_SHA"
else
# For direct pushes, compare the pushed commit against its parent (HEAD~1)
BASE_SHA=$(git rev-parse HEAD~1 2>/dev/null || echo "HEAD~1")
@@ -219,13 +218,9 @@ jobs:
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
fi
- END_TIME=$(date +%s)
- echo "=== [STAGE 2/4] Targeted Tests Completed in $((END_TIME - START_TIME))s ==="
-
- name: Merge Profiles
if: env.TARGETS != ''
run: |
- echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
find . -name "libc_cov_*.profraw" > profraw_list.txt
NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
echo "[LOG] Discovered $NUM_PROFS raw profile data files."
@@ -239,11 +234,10 @@ jobs:
- name: Extract Executables and Generate Summary
if: env.TARGETS != ''
run: |
- echo "=== [STAGE 4/4] Generating Coverage Report at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
OBJECTS=("${EXECUTABLES[@]:1}")
OBJECTS=("${OBJECTS[@]/#/-object=}")
-
+
echo "[LOG] Discovered ${#EXECUTABLES[@]} instrumented binaries for llvm-cov export."
if [ ! -f libc_full.profdata ]; then
@@ -287,7 +281,7 @@ jobs:
- name: Post or Update Sticky PR Comment
if: always() && github.event_name == 'pull_request'
- uses: actions/github-script at v7
+ uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
@@ -314,7 +308,7 @@ jobs:
// Parse metrics from reportContent for history table entry
const headSha = context.payload.pull_request.head.sha.substring(0, 7);
const dateStr = new Date().toISOString().replace('T', ' ').substring(0, 16);
-
+
const summaryMatch = reportContent.match(/\|\s*\*\*Total \(Patch\)\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\|\s*([0-9]+)\s*\/\s*([0-9]+)\s*\|\s*\*\*([0-9]+)\*\*/);
const pct = summaryMatch ? summaryMatch[1] + '%' : '100.00%';
const totalLines = summaryMatch ? summaryMatch[3] : 'N/A';
diff --git a/.github/workflows/libc-patch-mcdc.yml b/.github/workflows/libc-patch-mcdc.yml
index d50e150347d4c..93098a49113e3 100644
--- a/.github/workflows/libc-patch-mcdc.yml
+++ b/.github/workflows/libc-patch-mcdc.yml
@@ -49,21 +49,19 @@ jobs:
runs-on: ubuntu-24.04
container:
image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
- options: >-
- --privileged
defaults:
run:
shell: bash
steps:
- name: Checkout Code
- uses: actions/checkout at v4
+ uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
fetch-depth: 2
persist-credentials: false
- name: Setup Compiler Cache (sccache)
- uses: hendrikmuhs/ccache-action at v1.2.23
+ uses: hendrikmuhs/ccache-action at d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
with:
max-size: 1G
key: libc_coverage_unified_v2_x86_64
@@ -71,9 +69,6 @@ jobs:
- name: Configure CMake
run: |
- echo "=== [STAGE 1/4] Starting CMake Configuration at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- START_TIME=$(date +%s)
-
export CMAKE_FLAGS="
-G Ninja
-S runtimes
@@ -94,9 +89,6 @@ jobs:
"
cmake $CMAKE_FLAGS
- END_TIME=$(date +%s)
- echo "=== [STAGE 1/4] CMake Configuration Completed in $((END_TIME - START_TIME))s ==="
-
- name: Build and Run Targeted Tests
env:
EVENT_NAME: ${{ github.event_name }}
@@ -110,9 +102,6 @@ jobs:
BRANCH_REF: ${{ github.ref_name }}
REPO_NAME: ${{ github.repository }}
run: |
- echo "=== [STAGE 2/4] Starting Targeted Test Discovery at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
- START_TIME=$(date +%s)
-
# Ensure upstream llvm-project remote is configured
git remote add upstream https://github.com/llvm/llvm-project.git 2>/dev/null || true
@@ -229,13 +218,9 @@ jobs:
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
fi
- END_TIME=$(date +%s)
- echo "=== [STAGE 2/4] Targeted Tests Completed in $((END_TIME - START_TIME))s ==="
-
- name: Merge Profiles
if: env.TARGETS != ''
run: |
- echo "=== [STAGE 3/4] Merging Raw Profiles at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
find . -name "libc_cov_*.profraw" > profraw_list.txt
NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
echo "[LOG] Discovered $NUM_PROFS raw profile data files."
@@ -249,11 +234,10 @@ jobs:
- name: Extract Executables and Generate Summary
if: env.TARGETS != ''
run: |
- echo "=== [STAGE 4/4] Generating Coverage Report at $(date -u '+%Y-%m-%d %H:%M:%S UTC') ==="
EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
OBJECTS=("${EXECUTABLES[@]:1}")
OBJECTS=("${OBJECTS[@]/#/-object=}")
-
+
echo "[LOG] Discovered ${#EXECUTABLES[@]} instrumented binaries for llvm-cov export."
if [ ! -f libc_full.profdata ]; then
@@ -297,7 +281,7 @@ jobs:
- name: Post or Update Sticky PR Comment
if: always() && github.event_name == 'pull_request'
- uses: actions/github-script at v7
+ uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
@@ -322,7 +306,7 @@ jobs:
const headSha = context.payload.pull_request.head.sha.substring(0, 7);
const dateStr = new Date().toISOString().replace('T', ' ').substring(0, 16);
-
+
const summaryMatch = reportContent.match(/\|\s*\*\*Total \(Patch\)\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\(([0-9]+)\/([0-9]+)\)\s*\|\s*\*\*([0-9.]+)%\*\*\s*\(([0-9]+)\/([0-9]+)\)\s*\|.*?\|\s*\*\*([0-9]+)\*\*/);
const linePct = summaryMatch ? summaryMatch[1] + '%' : '100.00%';
const mcdcPct = summaryMatch ? summaryMatch[4] + '%' : '100.00%';
diff --git a/libc/utils/coverage/codebase_coverage.py b/libc/utils/coverage/codebase_coverage.py
index 7b1663389d708..0c62bb11ebe57 100644
--- a/libc/utils/coverage/codebase_coverage.py
+++ b/libc/utils/coverage/codebase_coverage.py
@@ -9,11 +9,12 @@
# ==------------------------------------------------------------------------==#
"""
-Standalone file for generating whole-codebase statement, branch, and MC/DC coverage reports.
+Standalone file for generating whole-codebase statement, branch, and MC/DC
+coverage reports.
-This script parses full-codebase `llvm-cov export` JSON files, aggregates metrics
-across all top-level LLVM-libc directories (e.g. `src/ctype`, `src/math`, `src/string`),
-and outputs Markdown summary tables for CI step summaries.
+This script parses full-codebase `llvm-cov export` JSON files, aggregates
+metrics across all top-level LLVM-libc directories (e.g. `src/ctype`, `src/math`,
+`src/string`), and outputs Markdown summary tables for CI step summaries.
"""
from __future__ import annotations
@@ -30,7 +31,7 @@
@dataclass
class DirectoryCoverageMetrics:
- """Encapsulates coverage metrics and boolean decision counts for a directory or whole codebase."""
+ """Encapsulates coverage metrics and boolean decision counts for a directory."""
name: str = ""
lines_cov: int = 0
@@ -60,7 +61,9 @@ def mcdc_pct(self) -> float:
@property
def decisions_pct(self) -> float:
"""Percentage of fully verified boolean decisions."""
- return (self.decisions_full / self.decisions_tot * 100.0) if self.decisions_tot > 0 else 0.0
+ if self.decisions_tot == 0:
+ return 0.0
+ return self.decisions_full / self.decisions_tot * 100.0
@property
def missed_lines(self) -> int:
@@ -72,7 +75,9 @@ def missed_lines(self) -> int:
class FullCoverageSummary:
"""Encapsulates global and directory-level coverage statistics across LLVM-libc."""
- global_stats: DirectoryCoverageMetrics = field(default_factory=DirectoryCoverageMetrics)
+ global_stats: DirectoryCoverageMetrics = field(
+ default_factory=DirectoryCoverageMetrics
+ )
directories: Dict[str, DirectoryCoverageMetrics] = field(default_factory=dict)
dashboard_url: str = ""
@@ -83,7 +88,7 @@ def has_mcdc(self) -> bool:
def resolve_dashboard_url(has_mcdc: bool) -> str:
- """Resolves the live dashboard URL if explicitly configured via environment variable."""
+ """Resolves live dashboard URL if configured via environment variable."""
pages_url = os.environ.get("COVERAGE_DASHBOARD_URL", "").strip()
if not pages_url:
return ""
@@ -93,7 +98,9 @@ def resolve_dashboard_url(has_mcdc: bool) -> str:
return pages_url
-def extract_full_coverage_statistics(cov_data: dict) -> Optional[FullCoverageSummary]:
+def extract_full_coverage_statistics(
+ cov_data: dict,
+) -> Optional[FullCoverageSummary]:
"""Extracts global and per-directory metrics from llvm-cov export JSON data."""
if "data" not in cov_data or not cov_data["data"]:
return None
@@ -129,7 +136,9 @@ def extract_full_coverage_statistics(cov_data: dict) -> Optional[FullCoverageSum
mcdc_records = item.get("mcdc_records", [])
file_decisions_tot = len(mcdc_records)
file_decisions_full = sum(
- 1 for rec in mcdc_records if len(rec) >= 10 and isinstance(rec[9], list) and all(rec[9])
+ 1
+ for rec in mcdc_records
+ if len(rec) >= 10 and isinstance(rec[9], list) and all(rec[9])
)
global_metrics.lines_cov += line_cov
@@ -177,23 +186,31 @@ def format_overview_callout(summary: FullCoverageSummary) -> str:
if summary.has_mcdc:
lines.append(
- f"### Overall Codebase Coverage: **{g.line_pct:.2f}% Line** | **{g.mcdc_pct:.2f}% MC/DC**"
+ f"### Overall Codebase Coverage: **{g.line_pct:.2f}% Line**"
+ f" | **{g.mcdc_pct:.2f}% MC/DC**"
)
lines.append(
- f"Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines and **{g.mcdc_cov:,} / {g.mcdc_tot:,}** boolean conditions across **{g.decisions_tot:,}** decisions."
+ f"Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines "
+ f"and **{g.mcdc_cov:,} / {g.mcdc_tot:,}** boolean conditions "
+ f"across **{g.decisions_tot:,}** decisions."
)
else:
lines.append(f"### Overall Codebase Coverage: **{g.line_pct:.2f}%**")
lines.append(
- f"Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines across all LLVM-libc directories."
+ f"Tested **{g.lines_cov:,} / {g.lines_tot:,}** executable lines "
+ "across all LLVM-libc directories."
)
lines.append("")
if summary.dashboard_url:
- lines.append(f"- **Coverage Dashboard:** [{summary.dashboard_url}]({summary.dashboard_url})")
+ lines.append(
+ f"- **Coverage Dashboard:** "
+ f"[{summary.dashboard_url}]({summary.dashboard_url})"
+ )
else:
lines.append(
- "- **HTML Coverage Report:** Available for download under the **Artifacts** section of this workflow run."
+ "- **HTML Coverage Report:** Available for download under the "
+ "**Artifacts** section of this workflow run."
)
return "\n".join(lines)
@@ -209,17 +226,21 @@ def format_global_summary_table(summary: FullCoverageSummary) -> str:
if summary.has_mcdc:
lines.append(
- f"| **MC/DC Condition Independence** | {g.mcdc_cov:,} | {g.mcdc_tot:,} | **{g.mcdc_pct:.2f}%** |"
+ f"| **MC/DC Condition Independence** | {g.mcdc_cov:,} | "
+ f"{g.mcdc_tot:,} | **{g.mcdc_pct:.2f}%** |"
)
lines.append(
- f"| **Fully Verified Decisions** | {g.decisions_full:,} | {g.decisions_tot:,} | **{g.decisions_pct:.2f}%** |"
+ f"| **Fully Verified Decisions** | {g.decisions_full:,} | "
+ f"{g.decisions_tot:,} | **{g.decisions_pct:.2f}%** |"
)
lines.append(
- f"| **Executable Lines** | {g.lines_cov:,} | {g.lines_tot:,} | **{g.line_pct:.2f}%** |"
+ f"| **Executable Lines** | {g.lines_cov:,} | {g.lines_tot:,} | "
+ f"**{g.line_pct:.2f}%** |"
)
lines.append(
- f"| **Functions** | {g.func_cov:,} | {g.func_tot:,} | **{g.func_pct:.2f}%** |"
+ f"| **Functions** | {g.func_cov:,} | {g.func_tot:,} | "
+ f"**{g.func_pct:.2f}%** |"
)
return "\n".join(lines)
@@ -231,12 +252,14 @@ def format_directory_breakdown_table(summary: FullCoverageSummary) -> str:
if has_mcdc:
lines.append(
- "| Directory | MC/DC Conditions | Decisions (Verified / Total) | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
+ "| Directory | MC/DC Conditions | Decisions (Verified / Total) | "
+ "Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
)
lines.append("| :--- | :---: | :---: | :---: | :---: | :---: | :---: |")
else:
lines.append(
- "| Directory | Line Coverage | Function Coverage | Executable Lines | Missed Lines |"
+ "| Directory | Line Coverage | Function Coverage | "
+ "Executable Lines | Missed Lines |"
)
lines.append("| :--- | :---: | :---: | :---: | :---: |")
@@ -254,11 +277,15 @@ def format_directory_breakdown_table(summary: FullCoverageSummary) -> str:
else "N/A"
)
lines.append(
- f"| `libc/{dir_name}` | {mc_cell} | {dec_cell} | **{data.line_pct:.2f}%** | {data.func_pct:.2f}% | {data.lines_tot:,} | {data.missed_lines:,} |"
+ f"| `libc/{dir_name}` | {mc_cell} | {dec_cell} | "
+ f"**{data.line_pct:.2f}%** | {data.func_pct:.2f}% | "
+ f"{data.lines_tot:,} | {data.missed_lines:,} |"
)
else:
lines.append(
- f"| `libc/{dir_name}` | **{data.line_pct:.2f}%** | {data.func_pct:.2f}% | {data.lines_tot:,} | {data.missed_lines:,} |"
+ f"| `libc/{dir_name}` | **{data.line_pct:.2f}%** | "
+ f"{data.func_pct:.2f}% | {data.lines_tot:,} | "
+ f"{data.missed_lines:,} |"
)
return "\n".join(lines)
@@ -291,6 +318,18 @@ def main() -> None:
"""Parses command-line arguments and triggers report generation."""
parser = argparse.ArgumentParser(description="LLVM-libc Codebase Coverage Analyzer")
parser.add_argument("json_file", help="Path to llvm-cov export JSON file")
+ parser.add_argument(
+ "commit_sha",
+ nargs="?",
+ default="",
+ help="Commit SHA under evaluation",
+ )
+ parser.add_argument(
+ "branch_ref",
+ nargs="?",
+ default="",
+ help="Branch reference under evaluation",
+ )
args, _ = parser.parse_known_args()
@@ -298,7 +337,9 @@ def main() -> None:
with open(args.json_file, "r", encoding="utf-8") as f:
cov_data = json.load(f)
except Exception as err:
- sys.stderr.write(f"Error: Failed to parse coverage JSON from '{args.json_file}': {err}\n")
+ sys.stderr.write(
+ f"Error: Failed to parse coverage JSON from '{args.json_file}': {err}\n"
+ )
sys.exit(1)
render_full_report(cov_data)
diff --git a/libc/utils/coverage/diff_coverage.py b/libc/utils/coverage/diff_coverage.py
index 38f043e45fa44..e8c3587bf8b57 100644
--- a/libc/utils/coverage/diff_coverage.py
+++ b/libc/utils/coverage/diff_coverage.py
@@ -9,11 +9,13 @@
# ==------------------------------------------------------------------------==#
"""
-Standalone analyzer for evaluating diff-level statement, branch, and MC/DC coverage.
+Standalone analyzer for evaluating diff-level statement, branch, and MC/DC
+coverage.
This script parses unified git diffs alongside `llvm-cov export` JSON summaries,
-correlates added/modified lines with execution counts and boolean decision records,
-and outputs formatted Markdown reports for CI job summaries and PR comments.
+correlates added/modified lines with execution counts and boolean decision
+records, and outputs formatted Markdown reports for CI job summaries and PR
+comments.
"""
from __future__ import annotations
@@ -40,16 +42,20 @@
"typedef ",
)
+
@dataclass
class DiffHunk:
"""Represents a unified diff hunk with its header and line tokens."""
header: str
- lines: List[Tuple[str, str, int]] = field(default_factory=list) # (prefix, text, line_number)
+ lines: List[Tuple[str, str, int]] = field(
+ default_factory=list
+ ) # (prefix, text, line_number)
+
@dataclass
class FilePatchMetrics:
- """Encapsulates coverage metrics and decision records for a single modified file."""
+ """Encapsulates coverage metrics and decisions for a single modified file."""
file_path: str
covered_lines: Set[int] = field(default_factory=set)
@@ -85,6 +91,7 @@ def mcdc_coverage_percentage(self) -> float:
else 0.0
)
+
@dataclass
class PatchCoverageSummary:
"""Aggregated coverage statistics across all modified files in the patch."""
@@ -114,10 +121,12 @@ def line_coverage_percentage(self) -> float:
@property
def mcdc_coverage_percentage(self) -> float:
"""Aggregated patch MC/DC condition coverage percentage."""
+ if self.total_mcdc_total_conditions == 0:
+ return 0.0
return (
- (self.total_mcdc_covered_conditions / self.total_mcdc_total_conditions * 100.0)
- if self.total_mcdc_total_conditions > 0
- else 0.0
+ self.total_mcdc_covered_conditions
+ / self.total_mcdc_total_conditions
+ * 100.0
)
@property
@@ -131,14 +140,16 @@ class DiffParser:
@staticmethod
def parse(diff_source: str) -> Dict[str, List[DiffHunk]]:
- """Parses a diff file path or raw diff string into a mapping of file path to hunks."""
+ """Parses a diff file into a mapping of file path to hunks."""
files: Dict[str, List[DiffHunk]] = {}
current_file: Optional[str] = None
current_hunk: Optional[DiffHunk] = None
current_line_number: int = 0
if os.path.isfile(diff_source):
- with open(diff_source, "r", encoding="utf-8", errors="replace") as file_handle:
+ with open(
+ diff_source, "r", encoding="utf-8", errors="replace"
+ ) as file_handle:
lines = file_handle.readlines()
else:
lines = diff_source.splitlines(keepends=True)
@@ -184,7 +195,7 @@ def parse(diff_source: str) -> Dict[str, List[DiffHunk]]:
class CoverageJSONParser:
- """Parses and extracts statement segments and MC/DC records from llvm-cov JSON export."""
+ """Parses statement segments and MC/DC records from llvm-cov JSON export."""
@staticmethod
def load(json_path: str) -> dict:
@@ -202,9 +213,13 @@ def load(json_path: str) -> dict:
def extract_patch_matrix(
coverage_data: dict, diff_files: Dict[str, List[DiffHunk]]
) -> Dict[str, Dict[str, Any]]:
- """Maps statement coverage segments and MC/DC decision records to modified files."""
+ """Maps coverage segments and MC/DC decision records to modified files."""
coverage_matrix: Dict[str, Dict[str, Any]] = {
- file_path: {"covered": set(), "missed": set(), "mcdc_decisions": []}
+ file_path: {
+ "covered": set(),
+ "missed": set(),
+ "mcdc_decisions": [],
+ }
for file_path in diff_files.keys()
}
@@ -256,7 +271,9 @@ def extract_patch_matrix(
decision_start_line = record[0]
decision_end_line = record[2]
boolean_conditions = record[9]
- covered_conditions_count = sum(1 for condition in boolean_conditions if condition)
+ covered_conditions_count = sum(
+ 1 for condition in boolean_conditions if condition
+ )
coverage_matrix[relative_file_path]["mcdc_decisions"].append(
{
"line_start": decision_start_line,
@@ -283,8 +300,16 @@ def is_executable_line(line_text: str) -> bool:
return False
if any(stripped_line.startswith(prefix) for prefix in DECLARATION_PREFIXES):
return False
- if stripped_line.startswith("struct ") or stripped_line.startswith("class ") or stripped_line.startswith("enum "):
- if "{" in stripped_line or (stripped_line.endswith(";") and "=" not in stripped_line and "(" not in stripped_line):
+ if (
+ stripped_line.startswith("struct ")
+ or stripped_line.startswith("class ")
+ or stripped_line.startswith("enum ")
+ ):
+ if "{" in stripped_line or (
+ stripped_line.endswith(";")
+ and "=" not in stripped_line
+ and "(" not in stripped_line
+ ):
return False
return True
@@ -301,13 +326,17 @@ def format_line_ranges(line_numbers: Set[int]) -> str:
if current_number == end_line + 1:
end_line = current_number
else:
- formatted_ranges.append(
- f"`L{start_line}-L{end_line}`" if start_line != end_line else f"`L{start_line}`"
+ range_label = (
+ f"`L{start_line}-L{end_line}`"
+ if start_line != end_line
+ else f"`L{start_line}`"
)
+ formatted_ranges.append(range_label)
start_line = end_line = current_number
- formatted_ranges.append(
+ range_label = (
f"`L{start_line}-L{end_line}`" if start_line != end_line else f"`L{start_line}`"
)
+ formatted_ranges.append(range_label)
return ", ".join(formatted_ranges)
@@ -315,7 +344,7 @@ def calculate_patch_statistics(
diff_files: Dict[str, List[DiffHunk]],
coverage_matrix: Dict[str, Dict[str, Any]],
) -> PatchCoverageSummary:
- """Calculates granular line, branch, and MC/DC statistics for all modified patch files."""
+ """Calculates line, branch, and MC/DC statistics for modified patch files."""
summary = PatchCoverageSummary()
for file_path, file_data in coverage_matrix.items():
@@ -329,7 +358,9 @@ def calculate_patch_statistics(
continue
file_covered_lines = added_lines.intersection(file_data["covered"])
- file_missed_lines = (added_lines.intersection(file_data["missed"])) - file_covered_lines
+ file_missed_lines = (
+ added_lines.intersection(file_data["missed"])
+ ) - file_covered_lines
file_metrics = FilePatchMetrics(
file_path=file_path,
@@ -349,7 +380,10 @@ def calculate_patch_statistics(
for decision in file_data.get("mcdc_decisions", []):
decision_start_line = decision["line_start"]
decision_end_line = decision["line_end"]
- if any(decision_start_line <= line_number <= decision_end_line for line_number in added_lines):
+ if any(
+ decision_start_line <= line_number <= decision_end_line
+ for line_number in added_lines
+ ):
summary.total_decisions_count += 1
file_metrics.decisions_total += 1
file_metrics.mcdc_covered_conditions += decision["covered"]
@@ -361,21 +395,30 @@ def calculate_patch_statistics(
summary.fully_verified_decisions += 1
file_metrics.decisions_verified += 1
file_metrics.condition_diagnostics.append(
- f"`L{decision_start_line}`: {decision['covered']}/{decision['total']} verified"
+ f"`L{decision_start_line}`: "
+ f"{decision['covered']}/{decision['total']} verified"
)
else:
uncovered_indices = [
f"C{condition_index + 1}"
- for condition_index, is_covered in enumerate(decision["conditions"])
+ for condition_index, is_covered in enumerate(
+ decision["conditions"]
+ )
if not is_covered
]
unverified_conditions_string = ", ".join(uncovered_indices)
file_metrics.condition_diagnostics.append(
- f"`L{decision_start_line}`: {decision['covered']}/{decision['total']} verified ({unverified_conditions_string} unverified)"
+ f"`L{decision_start_line}`: "
+ f"{decision['covered']}/{decision['total']} verified "
+ f"({unverified_conditions_string} unverified)"
)
- for decision_line in range(decision_start_line, decision_end_line + 1):
+ for decision_line in range(
+ decision_start_line, decision_end_line + 1
+ ):
if decision_line in added_lines:
- file_metrics.unverified_decision_lines[decision_line] = uncovered_indices
+ file_metrics.unverified_decision_lines[
+ decision_line
+ ] = uncovered_indices
summary.files[file_path] = file_metrics
@@ -391,36 +434,62 @@ def format_status_banner(summary: PatchCoverageSummary) -> str:
f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}%**"
)
lines.append(
- f"All **{summary.total_lines}** newly added or modified executable lines are covered."
+ f"All **{summary.total_lines}** newly added or modified "
+ "executable lines are covered."
)
- elif summary.total_mcdc_covered_conditions == summary.total_mcdc_total_conditions:
+ elif (
+ summary.total_mcdc_covered_conditions == summary.total_mcdc_total_conditions
+ ):
lines.append(
- f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}% Line** | **100.00% MC/DC**"
+ f"### Patch Coverage: "
+ f"**{summary.line_coverage_percentage:.2f}% Line** | "
+ "**100.00% MC/DC**"
)
lines.append(
- f"All **{summary.total_lines}** executable lines and **{summary.total_mcdc_total_conditions}** boolean conditions across **{summary.total_decisions_count}** decisions are covered."
+ f"All **{summary.total_lines}** executable lines and "
+ f"**{summary.total_mcdc_total_conditions}** boolean conditions "
+ f"across **{summary.total_decisions_count}** decisions are covered."
)
else:
lines.append(
- f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}% Line** | **{summary.mcdc_coverage_percentage:.1f}% MC/DC**"
+ f"### Patch Coverage: "
+ f"**{summary.line_coverage_percentage:.2f}% Line** | "
+ f"**{summary.mcdc_coverage_percentage:.1f}% MC/DC**"
)
lines.append(
- f"Executed **{summary.total_covered_lines} / {summary.total_lines}** lines. **{summary.total_mcdc_covered_conditions} / {summary.total_mcdc_total_conditions}** boolean conditions achieved independence across **{summary.fully_verified_decisions} / {summary.total_decisions_count}** decisions."
+ f"Executed **{summary.total_covered_lines} / {summary.total_lines}** "
+ f"lines. **{summary.total_mcdc_covered_conditions} / "
+ f"{summary.total_mcdc_total_conditions}** boolean conditions "
+ f"achieved independence across **{summary.fully_verified_decisions} / "
+ f"{summary.total_decisions_count}** decisions."
)
else:
if not summary.has_mcdc:
lines.append(
- f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}%** ({summary.total_missed_lines} Missed Lines)"
+ f"### Patch Coverage: "
+ f"**{summary.line_coverage_percentage:.2f}%** "
+ f"({summary.total_missed_lines} Missed Lines)"
)
lines.append(
- f"Executed **{summary.total_covered_lines} / {summary.total_lines}** lines (**{summary.total_missed_lines}** unexecuted lines detected in patch)."
+ f"Executed **{summary.total_covered_lines} / {summary.total_lines}** "
+ f"lines (**{summary.total_missed_lines}** unexecuted lines "
+ "detected in patch)."
)
else:
lines.append(
- f"### Patch Coverage: **{summary.line_coverage_percentage:.2f}% Line** | **{summary.mcdc_coverage_percentage:.1f}% MC/DC** ({summary.total_missed_lines} Missed Lines)"
+ f"### Patch Coverage: "
+ f"**{summary.line_coverage_percentage:.2f}% Line** | "
+ f"**{summary.mcdc_coverage_percentage:.1f}% MC/DC** "
+ f"({summary.total_missed_lines} Missed Lines)"
)
lines.append(
- f"Executed **{summary.total_covered_lines} / {summary.total_lines}** lines. **{summary.total_mcdc_covered_conditions} / {summary.total_mcdc_total_conditions}** boolean conditions achieved independence across **{summary.fully_verified_decisions} / {summary.total_decisions_count}** decisions (**{summary.total_missed_lines}** unexecuted lines detected in patch)."
+ f"Executed **{summary.total_covered_lines} / {summary.total_lines}** "
+ f"lines. **{summary.total_mcdc_covered_conditions} / "
+ f"{summary.total_mcdc_total_conditions}** boolean conditions "
+ f"achieved independence across **{summary.fully_verified_decisions} / "
+ f"{summary.total_decisions_count}** decisions "
+ f"(**{summary.total_missed_lines}** unexecuted lines "
+ "detected in patch)."
)
return "\n".join(lines)
@@ -438,14 +507,18 @@ def format_metadata_section(
lines: List[str] = []
if base_commit_sha and head_commit_sha and base_branch_name and head_branch_name:
lines.append(
- f"- **Base Branch:** [`{base_branch_name}` ({base_commit_sha[:7]})](https://github.com/{base_repository}/commit/{base_commit_sha})"
+ f"- **Base Branch:** [`{base_branch_name}` ({base_commit_sha[:7]})]"
+ f"(https://github.com/{base_repository}/commit/{base_commit_sha})"
)
lines.append(
- f"- **Head Commit:** [`{head_branch_name}` ({head_commit_sha[:7]})](https://github.com/{head_repository}/commit/{head_commit_sha})"
+ f"- **Head Commit:** [`{head_branch_name}` ({head_commit_sha[:7]})]"
+ f"(https://github.com/{head_repository}/commit/{head_commit_sha})"
)
if targeted_tests_string:
formatted_targets = ", ".join(
- f"`{target.strip()}`" for target in targeted_tests_string.split() if target.strip()
+ f"`{target.strip()}`"
+ for target in targeted_tests_string.split()
+ if target.strip()
)
lines.append(f"- **Targeted Tests Executed:** {formatted_targets}")
return "\n".join(lines)
@@ -460,29 +533,37 @@ def format_breakdown_table(
lines: List[str] = ["### Coverage Breakdown"]
if summary.has_mcdc:
lines.append(
- "| Modified Source File | Line Coverage | MC/DC Conditions | Decisions (Verified / Total) | Missed Lines | Unverified Conditions |"
+ "| Modified Source File | Line Coverage | MC/DC Conditions | "
+ "Decisions (Verified / Total) | Missed Lines | Unverified Conditions |"
)
lines.append("| :--- | :---: | :---: | :---: | :---: | :--- |")
else:
lines.append(
- "| Modified Source File | Patch Coverage | Covered / Total | Missed Lines | Unexecuted Line Spans |"
+ "| Modified Source File | Patch Coverage | Covered / Total | "
+ "Missed Lines | Unexecuted Line Spans |"
)
lines.append("| :--- | :---: | :---: | :---: | :---: |")
for file_path, file_metric in summary.files.items():
- file_link = f"[`{file_path}`](https://github.com/{head_repository}/blob/{head_commit_sha or 'main'}/{file_path})"
+ commit_ref = head_commit_sha or "main"
+ file_link = (
+ f"[`{file_path}`]"
+ f"(https://github.com/{head_repository}/blob/{commit_ref}/{file_path})"
+ )
missed_lines = file_metric.missed_lines
covered_count = len(file_metric.covered_lines)
total_file_lines = file_metric.total_lines
if summary.has_mcdc:
+ mc_pct = file_metric.mcdc_coverage_percentage
+ mc_cov = file_metric.mcdc_covered_conditions
+ mc_tot = file_metric.mcdc_total_conditions
mcdc_cell = (
- f"**{file_metric.mcdc_coverage_percentage:.1f}%** ({file_metric.mcdc_covered_conditions}/{file_metric.mcdc_total_conditions})"
- if file_metric.mcdc_total_conditions > 0
- else "N/A"
+ f"**{mc_pct:.1f}%** ({mc_cov}/{mc_tot})" if mc_tot > 0 else "N/A"
)
- decision_cell = (
- f"**{file_metric.decisions_verified} / {file_metric.decisions_total}**"
+ dec_cell = (
+ f"**{file_metric.decisions_verified} / "
+ f"{file_metric.decisions_total}**"
if file_metric.decisions_total > 0
else "N/A"
)
@@ -492,23 +573,42 @@ def format_breakdown_table(
else "None"
)
lines.append(
- f"| {file_link} | **{file_metric.line_coverage_percentage:.2f}%** ({covered_count}/{total_file_lines}) | {mcdc_cell} | {decision_cell} | {len(missed_lines)} | {diagnostic_cell} |"
+ f"| {file_link} | "
+ f"**{file_metric.line_coverage_percentage:.2f}%** "
+ f"({covered_count}/{total_file_lines}) | "
+ f"{mcdc_cell} | {decision_cell} | "
+ f"{len(missed_lines)} | {diagnostic_cell} |"
)
else:
line_spans = format_line_ranges(missed_lines)
lines.append(
- f"| {file_link} | **{file_metric.line_coverage_percentage:.2f}%** | {covered_count} / {total_file_lines} | {len(missed_lines)} | {line_spans} |"
+ f"| {file_link} | "
+ f"**{file_metric.line_coverage_percentage:.2f}%** | "
+ f"{covered_count} / {total_file_lines} | "
+ f"{len(missed_lines)} | {line_spans} |"
)
# Summary Row
if summary.has_mcdc:
- total_decision_cell = f"**{summary.fully_verified_decisions} / {summary.total_decisions_count}**"
+ total_decision_cell = (
+ f"**{summary.fully_verified_decisions} / "
+ f"{summary.total_decisions_count}**"
+ )
lines.append(
- f"| **Total (Patch)** | **{summary.line_coverage_percentage:.2f}%** ({summary.total_covered_lines}/{summary.total_lines}) | **{summary.mcdc_coverage_percentage:.1f}%** ({summary.total_mcdc_covered_conditions}/{summary.total_mcdc_total_conditions}) | {total_decision_cell} | **{summary.total_missed_lines}** | - |"
+ f"| **Total (Patch)** | "
+ f"**{summary.line_coverage_percentage:.2f}%** "
+ f"({summary.total_covered_lines}/{summary.total_lines}) | "
+ f"**{summary.mcdc_coverage_percentage:.1f}%** "
+ f"({summary.total_mcdc_covered_conditions}/"
+ f"{summary.total_mcdc_total_conditions}) | "
+ f"{total_decision_cell} | **{summary.total_missed_lines}** | - |"
)
else:
lines.append(
- f"| **Total (Patch)** | **{summary.line_coverage_percentage:.2f}%** | {summary.total_covered_lines} / {summary.total_lines} | **{summary.total_missed_lines}** | - |"
+ f"| **Total (Patch)** | "
+ f"**{summary.line_coverage_percentage:.2f}%** | "
+ f"{summary.total_covered_lines} / {summary.total_lines} | "
+ f"**{summary.total_missed_lines}** | - |"
)
return "\n".join(lines)
@@ -537,8 +637,13 @@ def format_annotated_diff(
if line_number in file_metric.missed_lines:
lines.append(f"- {line_text} // [MISSED]")
elif line_number in unverified_decision_lines:
- unverified_conditions = ", ".join(unverified_decision_lines[line_number])
- lines.append(f"! {line_text} // [PARTIAL MC/DC: {unverified_conditions} unverified]")
+ unverified_conditions = ", ".join(
+ unverified_decision_lines[line_number]
+ )
+ lines.append(
+ f"! {line_text} // [PARTIAL MC/DC: "
+ f"{unverified_conditions} unverified]"
+ )
elif line_number in file_metric.covered_lines:
lines.append(f"+ {line_text}")
else:
@@ -623,7 +728,9 @@ def main() -> None:
parser.add_argument("base_branch", nargs="?", help="Base branch name")
parser.add_argument("head_branch", nargs="?", help="Head branch name")
parser.add_argument(
- "targets", nargs="?", help="Space-separated list of executed test targets"
+ "targets",
+ nargs="?",
+ help="Space-separated list of executed test targets",
)
parser.add_argument(
"base_repo",
diff --git a/libc/utils/coverage/test_codebase_coverage.py b/libc/utils/coverage/test_codebase_coverage.py
index 374fe46d77c66..571393975c09d 100644
--- a/libc/utils/coverage/test_codebase_coverage.py
+++ b/libc/utils/coverage/test_codebase_coverage.py
@@ -6,8 +6,12 @@
#
# ==-------------------------------------------------------------------------==#
-import unittest
+import io
import json
+import os
+import unittest
+from contextlib import redirect_stdout
+
from codebase_coverage import (
DirectoryCoverageMetrics,
FullCoverageSummary,
@@ -18,44 +22,71 @@
resolve_dashboard_url,
)
+
class TestCodebaseDirectoryAggregation(unittest.TestCase):
def test_path_filtering_and_routing(self):
mock_json = {
- "data": [{
- "files": [
- {
- "filename": "/workspace/libc/src/math/sin.cpp",
- "summary": {
- "lines": {"count": 100, "covered": 50, "percent": 50.0},
- "functions": {"count": 2, "covered": 1, "percent": 50.0}
- }
- },
- {
- "filename": "/workspace/libc/src/string/strcpy.cpp",
- "summary": {
- "lines": {"count": 200, "covered": 200, "percent": 100.0},
- "functions": {"count": 4, "covered": 4, "percent": 100.0}
- }
- },
- {
- "filename": "/workspace/libc/test/src/math/sin_test.cpp",
- "summary": {
- "lines": {"count": 500, "covered": 500, "percent": 100.0},
- "functions": {"count": 10, "covered": 10, "percent": 100.0}
- }
- }
- ]
- }]
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/src/math/sin.cpp",
+ "summary": {
+ "lines": {
+ "count": 100,
+ "covered": 50,
+ "percent": 50.0,
+ },
+ "functions": {
+ "count": 2,
+ "covered": 1,
+ "percent": 50.0,
+ },
+ },
+ },
+ {
+ "filename": ("/workspace/libc/src/string/strcpy.cpp"),
+ "summary": {
+ "lines": {
+ "count": 200,
+ "covered": 200,
+ "percent": 100.0,
+ },
+ "functions": {
+ "count": 4,
+ "covered": 4,
+ "percent": 100.0,
+ },
+ },
+ },
+ {
+ "filename": ("/workspace/libc/test/src/math/sin_test.cpp"),
+ "summary": {
+ "lines": {
+ "count": 500,
+ "covered": 500,
+ "percent": 100.0,
+ },
+ "functions": {
+ "count": 10,
+ "covered": 10,
+ "percent": 100.0,
+ },
+ },
+ },
+ ]
+ }
+ ]
}
-
+
summary = extract_full_coverage_statistics(mock_json)
-
+
# The test file should be ignored entirely
self.assertEqual(summary.global_stats.lines_tot, 300)
self.assertEqual(summary.global_stats.lines_cov, 250)
self.assertEqual(summary.global_stats.func_tot, 6)
self.assertEqual(summary.global_stats.func_cov, 5)
-
+
# Ensure proper bucket routing
self.assertIn("src/math", summary.directories)
self.assertEqual(summary.directories["src/math"].lines_tot, 100)
@@ -63,19 +94,31 @@ def test_path_filtering_and_routing(self):
self.assertEqual(summary.directories["src/string"].lines_tot, 200)
self.assertNotIn("test/src/math", summary.directories)
+
class TestCodebaseReportRendering(unittest.TestCase):
def test_format_directory_breakdown_table(self):
metrics_math = DirectoryCoverageMetrics(
- name="src/math", lines_tot=100, lines_cov=50, func_tot=10, func_cov=5
+ name="src/math",
+ lines_tot=100,
+ lines_cov=50,
+ func_tot=10,
+ func_cov=5,
)
metrics_string = DirectoryCoverageMetrics(
- name="src/string", lines_tot=200, lines_cov=200, func_tot=20, func_cov=20
+ name="src/string",
+ lines_tot=200,
+ lines_cov=200,
+ func_tot=20,
+ func_cov=20,
)
summary = FullCoverageSummary(
global_stats=metrics_math,
- directories={"src/math": metrics_math, "src/string": metrics_string}
+ directories={
+ "src/math": metrics_math,
+ "src/string": metrics_string,
+ },
)
-
+
table = format_directory_breakdown_table(summary)
self.assertIn("`libc/src/math`", table)
self.assertIn("`libc/src/string`", table)
@@ -98,7 +141,6 @@ def test_format_overview_callout_with_dashboard_url(self):
self.assertIn("https://example.com/coverage/", callout)
def test_resolve_dashboard_url_default_is_empty(self):
- import os
old_env = os.environ.get("COVERAGE_DASHBOARD_URL")
try:
if "COVERAGE_DASHBOARD_URL" in os.environ:
@@ -110,21 +152,28 @@ def test_resolve_dashboard_url_default_is_empty(self):
os.environ["COVERAGE_DASHBOARD_URL"] = old_env
def test_render_full_report_end_to_end(self):
- import io
- from contextlib import redirect_stdout
-
mock_json = {
- "data": [{
- "files": [
- {
- "filename": "/workspace/libc/src/math/sin.cpp",
- "summary": {
- "lines": {"count": 100, "covered": 80, "percent": 80.0},
- "functions": {"count": 2, "covered": 2, "percent": 100.0},
- },
- }
- ]
- }]
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/src/math/sin.cpp",
+ "summary": {
+ "lines": {
+ "count": 100,
+ "covered": 80,
+ "percent": 80.0,
+ },
+ "functions": {
+ "count": 2,
+ "covered": 2,
+ "percent": 100.0,
+ },
+ },
+ }
+ ]
+ }
+ ]
}
buf = io.StringIO()
with redirect_stdout(buf):
@@ -137,5 +186,6 @@ def test_render_full_report_end_to_end(self):
self.assertIn("### Overall", output)
self.assertIn("`libc/src/math`", output)
-if __name__ == '__main__':
+
+if __name__ == "__main__":
unittest.main()
diff --git a/libc/utils/coverage/test_diff_coverage.py b/libc/utils/coverage/test_diff_coverage.py
index 1aa6a14535c2e..49683c3d6f5b7 100644
--- a/libc/utils/coverage/test_diff_coverage.py
+++ b/libc/utils/coverage/test_diff_coverage.py
@@ -6,20 +6,22 @@
#
# ==-------------------------------------------------------------------------==#
-import unittest
import json
+import unittest
+
from diff_coverage import (
- DiffParser,
- DiffHunk,
- is_executable_line,
CoverageJSONParser,
+ DiffHunk,
+ DiffParser,
+ FilePatchMetrics,
+ PatchCoverageSummary,
calculate_patch_statistics,
- format_status_banner,
format_breakdown_table,
- PatchCoverageSummary,
- FilePatchMetrics
+ format_status_banner,
+ is_executable_line,
)
+
class TestExecutableLineHeuristics(unittest.TestCase):
def test_heuristics_matrix(self):
matrix = [
@@ -41,6 +43,7 @@ def test_heuristics_matrix(self):
with self.subTest(msg=description, input_string=input_string):
self.assertEqual(is_executable_line(input_string), expected)
+
class TestDiffParser(unittest.TestCase):
def test_parse_diff_hunks_and_lines(self):
mock_diff = (
@@ -57,7 +60,7 @@ def test_parse_diff_hunks_and_lines(self):
self.assertIn("src/math/sin.cpp", hunks_dict)
hunks = hunks_dict["src/math/sin.cpp"]
self.assertEqual(len(hunks), 1)
-
+
hunk = hunks[0]
self.assertEqual(hunk.header, "@@ -10,3 +10,4 @@")
added_lines = [line for line in hunk.lines if line[0] == "+"]
@@ -65,6 +68,7 @@ def test_parse_diff_hunks_and_lines(self):
self.assertEqual(added_lines[0], ("+", "added_line_1();", 11))
self.assertEqual(added_lines[1], ("+", "added_line_2();", 12))
+
class TestSegmentIntersection(unittest.TestCase):
def test_calculate_patch_statistics_exact_match(self):
mock_diff = (
@@ -76,26 +80,27 @@ def test_calculate_patch_statistics_exact_match(self):
"+return 0;\n"
)
mock_json = {
- "data": [{
- "files": [{
- "filename": "/workspace/src/math/sin.cpp",
- "segments": [
- [10, 0, 1, 1, 1],
- [11, 0, 1, 1, 1]
- ],
- "branches": []
- }]
- }]
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/src/math/sin.cpp",
+ "segments": [[10, 0, 1, 1, 1], [11, 0, 1, 1, 1]],
+ "branches": [],
+ }
+ ]
+ }
+ ]
}
-
+
diff_files = DiffParser.parse(mock_diff)
coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
summary = calculate_patch_statistics(diff_files, coverage_matrix)
-
+
self.assertEqual(summary.total_lines, 1)
self.assertEqual(summary.total_covered_lines, 1)
self.assertEqual(summary.total_missed_lines, 0)
-
+
def test_calculate_patch_statistics_missed_line(self):
mock_diff = (
"diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
@@ -106,26 +111,28 @@ def test_calculate_patch_statistics_missed_line(self):
"+return 0;\n"
)
mock_json = {
- "data": [{
- "files": [{
- "filename": "/workspace/src/math/sin.cpp",
- "segments": [
- [10, 0, 1, 1, 1],
- [11, 0, 0, 1, 1]
- ],
- "branches": []
- }]
- }]
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/src/math/sin.cpp",
+ "segments": [[10, 0, 1, 1, 1], [11, 0, 0, 1, 1]],
+ "branches": [],
+ }
+ ]
+ }
+ ]
}
-
+
diff_files = DiffParser.parse(mock_diff)
coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
summary = calculate_patch_statistics(diff_files, coverage_matrix)
-
+
self.assertEqual(summary.total_lines, 1)
self.assertEqual(summary.total_covered_lines, 0)
self.assertEqual(summary.total_missed_lines, 1)
+
class TestPatchReportRendering(unittest.TestCase):
def test_format_status_banner(self):
summary = PatchCoverageSummary(
@@ -133,28 +140,29 @@ def test_format_status_banner(self):
total_missed_lines=50,
total_mcdc_total_conditions=10,
total_mcdc_covered_conditions=5,
- files={}
+ files={},
)
banner = format_status_banner(summary)
self.assertIn("### Patch Coverage:", banner)
self.assertIn("50.00% Line", banner)
-
+
def test_format_breakdown_table(self):
file_stat = FilePatchMetrics(
file_path="src/math/sin.cpp",
covered_lines={10, 11},
missed_lines=set(),
- added_lines={10, 11}
+ added_lines={10, 11},
)
summary = PatchCoverageSummary(
total_covered_lines=2,
total_missed_lines=0,
- files={"src/math/sin.cpp": file_stat}
+ files={"src/math/sin.cpp": file_stat},
)
report = format_breakdown_table(summary)
self.assertIn("[`src/math/sin.cpp`]", report)
self.assertIn("**100.00%**", report)
+
class TestDiffParserEdgeCases(unittest.TestCase):
def test_deleted_file(self):
mock_diff = (
@@ -184,21 +192,29 @@ def test_no_newline_at_eof(self):
added_lines = [line for line in hunks[0].lines if line[0] == "+"]
self.assertEqual(len(added_lines), 1)
+
class TestPathResolution(unittest.TestCase):
def test_fuzzy_path_matching(self):
diff_files = {"libc/src/math/sin.cpp": []}
mock_json = {
- "data": [{
- "files": [{
- "filename": "/home/runner/work/llvm-project/libc/src/math/sin.cpp",
- "segments": [[10, 0, 1, 1, 1]],
- "mcdc_records": []
- }]
- }]
+ "data": [
+ {
+ "files": [
+ {
+ "filename": (
+ "/home/runner/work/llvm-project/libc/src/math/sin.cpp"
+ ),
+ "segments": [[10, 0, 1, 1, 1]],
+ "mcdc_records": [],
+ }
+ ]
+ }
+ ]
}
coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
self.assertIn("libc/src/math/sin.cpp", coverage_matrix)
+
class TestZeroStateBoundaries(unittest.TestCase):
def test_empty_json_data(self):
diff_files = {"src/math/sin.cpp": []}
@@ -217,11 +233,18 @@ def test_purely_cosmetic_patch(self):
"+// Just a comment\n"
)
diff_files = DiffParser.parse(mock_diff)
- coverage_matrix = {"src/math/sin.cpp": {"covered": set(), "missed": set(), "mcdc_decisions": []}}
+ coverage_matrix = {
+ "src/math/sin.cpp": {
+ "covered": set(),
+ "missed": set(),
+ "mcdc_decisions": [],
+ }
+ }
summary = calculate_patch_statistics(diff_files, coverage_matrix)
self.assertEqual(summary.total_lines, 0)
self.assertEqual(summary.line_coverage_percentage, 0.0)
+
class TestMCDCIntersection(unittest.TestCase):
def test_mcdc_boolean_extraction(self):
mock_diff = (
@@ -233,24 +256,27 @@ def test_mcdc_boolean_extraction(self):
"+if (a && b) { return 0; }\n"
)
mock_json = {
- "data": [{
- "files": [{
- "filename": "/workspace/src/math/sin.cpp",
- "segments": [
- [11, 0, 1, 1, 1]
- ],
- "mcdc_records": [
- [11, 4, 11, 14, 0, 0, 0, 0, 0, [True, False]]
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/src/math/sin.cpp",
+ "segments": [[11, 0, 1, 1, 1]],
+ "mcdc_records": [
+ [11, 4, 11, 14, 0, 0, 0, 0, 0, [True, False]]
+ ],
+ }
]
- }]
- }]
+ }
+ ]
}
diff_files = DiffParser.parse(mock_diff)
coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
summary = calculate_patch_statistics(diff_files, coverage_matrix)
-
+
self.assertEqual(summary.total_mcdc_total_conditions, 2)
self.assertEqual(summary.total_mcdc_covered_conditions, 1)
-if __name__ == '__main__':
+
+if __name__ == "__main__":
unittest.main()
>From 274f8a1c7a50d70fb1a43e32699ea3fd6b28ce7d Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Thu, 3 Sep 2026 18:00:49 +0000
Subject: [PATCH 10/15] [libc][ci] Restrict workflow push and pull request
triggers to main
Remove development and testing branch filters from coverage workflows so
they only trigger on upstream main and pull requests targeting main.
Assisted-by: Automated tooling, human reviewed.
---
.github/workflows/libc-full-coverage.yml | 4 +---
.github/workflows/libc-full-mcdc.yml | 4 +---
.github/workflows/libc-patch-coverage.yml | 4 ++--
.github/workflows/libc-patch-mcdc.yml | 4 ++--
4 files changed, 6 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/libc-full-coverage.yml b/.github/workflows/libc-full-coverage.yml
index 4f8cc19563d62..aee5736372305 100644
--- a/.github/workflows/libc-full-coverage.yml
+++ b/.github/workflows/libc-full-coverage.yml
@@ -14,12 +14,10 @@ on:
# Allow manual on-demand execution from GitHub Actions UI
workflow_dispatch:
- # Trigger on pushes to main / libc-coverage-ci-bots / post-commit-bot
+ # Trigger on pushes to main
push:
branches:
- main
- - libc-coverage-ci-bots
- - post-commit-bot
paths:
- 'libc/**'
- '.github/workflows/libc-full-coverage.yml'
diff --git a/.github/workflows/libc-full-mcdc.yml b/.github/workflows/libc-full-mcdc.yml
index 75acf682fd761..fd2cd5f45d410 100644
--- a/.github/workflows/libc-full-mcdc.yml
+++ b/.github/workflows/libc-full-mcdc.yml
@@ -14,12 +14,10 @@ on:
# Allow manual on-demand execution from GitHub Actions UI ("Run workflow" button)
workflow_dispatch:
- # Trigger on pushes to main / libc-mcdc-coverage / post-commit-bot
+ # Trigger on pushes to main
push:
branches:
- main
- - libc-mcdc-coverage
- - post-commit-bot
paths:
- 'libc/**'
- '.github/workflows/libc-full-mcdc.yml'
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
index e6dbf15c388cb..b21df6efcd864 100644
--- a/.github/workflows/libc-patch-coverage.yml
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -11,8 +11,6 @@ on:
workflow_dispatch:
push:
branches:
- - pre-commit-bot
- - libc-coverage-ci-bots
- main
paths:
- 'libc/src/**'
@@ -26,6 +24,8 @@ on:
- '!libc/utils/**'
- '!**.md'
pull_request:
+ branches:
+ - main
paths:
- 'libc/src/**'
- 'libc/include/**'
diff --git a/.github/workflows/libc-patch-mcdc.yml b/.github/workflows/libc-patch-mcdc.yml
index 93098a49113e3..55355a839aad9 100644
--- a/.github/workflows/libc-patch-mcdc.yml
+++ b/.github/workflows/libc-patch-mcdc.yml
@@ -11,8 +11,6 @@ on:
workflow_dispatch:
push:
branches:
- - 'test-*'
- - 'libc-mcdc-*'
- main
paths:
- 'libc/src/**'
@@ -26,6 +24,8 @@ on:
- '!libc/utils/**'
- '!**.md'
pull_request:
+ branches:
+ - main
paths:
- 'libc/src/**'
- 'libc/include/**'
>From 9183f1c7c874d0d286023bc4686d4482411f234f Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Fri, 4 Sep 2026 17:48:17 +0000
Subject: [PATCH 11/15] [libc][ci] Streamline coverage test suites, harden
reporting, and simplify CI summaries
Streamline test_diff_coverage.py by 41% (reducing 360+ lines of fixture
boilerplate) while preserving 100% statement and branch coverage.
Harden diff and codebase coverage utilities against C++ syntax edge cases,
including closing braces with namespace comments, C++ access specifiers,
and compile-time declarations (static_assert, friend). Filter non-source files
from patch diff evaluations and normalize repository URLs for GitHub links.
Remove the redundant sticky PR comment step from patch workflows in favor of
standard GITHUB_STEP_SUMMARY and artifact uploads, restricting workflow
permissions strictly to contents: read to avoid fork token permission failures.
Add downloadable artifact packaging with if-no-files-found guards, merge-base
diff resolution, and executable verification guards across CI.
Assisted-by: Automated tooling, human reviewed.
---
.github/workflows/libc-full-coverage.yml | 16 +-
.github/workflows/libc-full-mcdc.yml | 16 +-
.github/workflows/libc-patch-coverage.yml | 102 +--
.github/workflows/libc-patch-mcdc.yml | 100 +--
libc/utils/coverage/codebase_coverage.py | 46 +-
libc/utils/coverage/diff_coverage.py | 62 +-
libc/utils/coverage/test_codebase_coverage.py | 718 ++++++++++++++---
libc/utils/coverage/test_diff_coverage.py | 736 +++++++++++++-----
8 files changed, 1287 insertions(+), 509 deletions(-)
diff --git a/.github/workflows/libc-full-coverage.yml b/.github/workflows/libc-full-coverage.yml
index aee5736372305..7604907b75d69 100644
--- a/.github/workflows/libc-full-coverage.yml
+++ b/.github/workflows/libc-full-coverage.yml
@@ -120,7 +120,8 @@ jobs:
--path-equivalence="$GITHUB_WORKSPACE,." \
-ignore-filename-regex=".*(test|utils).*"
# 3. Run Codebase Coverage Analyzer
- PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
+ PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" > coverage_summary.md
+ cat coverage_summary.md >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
- name: Fallback Failure Summary
@@ -129,7 +130,20 @@ jobs:
echo "### LLVM-libc Coverage Report" >> $GITHUB_STEP_SUMMARY
echo "Workflow run encountered an error before coverage reports could be generated. Inspect job logs for details." >> $GITHUB_STEP_SUMMARY
+ - name: Upload Coverage Summary Artifact
+ if: always() && hashFiles('coverage_summary.md')
+ uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: libc-coverage-summary
+ path: |
+ coverage_summary.md
+ coverage.json
+ if-no-files-found: ignore
+ retention-days: 14
+ overwrite: true
+
- name: Upload HTML Coverage Artifact
+ if: always() && hashFiles('coverage_html/**')
uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: libc-coverage-html
diff --git a/.github/workflows/libc-full-mcdc.yml b/.github/workflows/libc-full-mcdc.yml
index fd2cd5f45d410..9aa6d96943891 100644
--- a/.github/workflows/libc-full-mcdc.yml
+++ b/.github/workflows/libc-full-mcdc.yml
@@ -123,7 +123,8 @@ jobs:
--path-equivalence="$GITHUB_WORKSPACE,." \
-ignore-filename-regex=".*(test|utils).*"
# 3. Run Codebase Coverage Analyzer
- PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" >> $GITHUB_STEP_SUMMARY
+ PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" > coverage_summary.md
+ cat coverage_summary.md >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
- name: Fallback Failure Summary
@@ -132,7 +133,20 @@ jobs:
echo "### LLVM-libc MC/DC Coverage Report" >> $GITHUB_STEP_SUMMARY
echo "Workflow run encountered an error before coverage reports could be generated. Inspect job logs for details." >> $GITHUB_STEP_SUMMARY
+ - name: Upload MC/DC Coverage Summary Artifact
+ if: always() && hashFiles('coverage_summary.md')
+ uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: libc-mcdc-coverage-summary
+ path: |
+ coverage_summary.md
+ coverage.json
+ if-no-files-found: ignore
+ retention-days: 14
+ overwrite: true
+
- name: Upload MC/DC Coverage Artifact
+ if: always() && hashFiles('coverage_mcdc_html/**')
uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: libc-mcdc-coverage-html
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
index b21df6efcd864..c5637301575ea 100644
--- a/.github/workflows/libc-patch-coverage.yml
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -2,7 +2,6 @@ name: Libc Patch Code Coverage
permissions:
contents: read
- pull-requests: write # Required to post/update patch coverage comments
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
@@ -57,7 +56,7 @@ jobs:
uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- fetch-depth: 2
+ fetch-depth: 100
persist-credentials: false
- name: Setup Compiler Cache (sccache)
@@ -112,8 +111,13 @@ jobs:
HEAD_REF="$PR_HEAD_REF"
BASE_REPO="${PR_BASE_REPO:-llvm/llvm-project}"
HEAD_REPO="${PR_HEAD_REPO:-$REPO_NAME}"
- git fetch upstream "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_REF" --depth=1 2>/dev/null || true
- DIFF_BASE="$BASE_SHA"
+ # Fetch base branch history to resolve merge base
+ git fetch upstream "$BASE_REF" --depth=100 2>/dev/null || git fetch origin "$BASE_REF" --depth=100 2>/dev/null || git fetch upstream "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 2>/dev/null || true
+ DIFF_BASE=$(git merge-base "$BASE_SHA" HEAD 2>/dev/null || true)
+ if [ -z "$DIFF_BASE" ]; then
+ git fetch --no-tags --deepen=200 2>/dev/null || true
+ DIFF_BASE=$(git merge-base "$BASE_SHA" HEAD 2>/dev/null || echo "$BASE_SHA")
+ fi
elif [ "$BRANCH_REF" != "main" ]; then
# For feature branches, compare against main merge-base
git fetch origin main --depth=100 2>/dev/null || true
@@ -240,8 +244,8 @@ jobs:
echo "[LOG] Discovered ${#EXECUTABLES[@]} instrumented binaries for llvm-cov export."
- if [ ! -f libc_full.profdata ]; then
- echo "[LOG] Notice: libc_full.profdata not found. Generating non-coverage summary."
+ if [ ! -f libc_full.profdata ] || [ ${#EXECUTABLES[@]} -eq 0 ]; then
+ echo "[LOG] Notice: Profile data or instrumented executables not found. Generating non-coverage summary."
echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
echo "" >> coverage_report.md
echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
@@ -279,78 +283,16 @@ jobs:
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
- - name: Post or Update Sticky PR Comment
- if: always() && github.event_name == 'pull_request'
- uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+
+ - name: Upload Patch Coverage Artifacts
+ if: always() && hashFiles('coverage_report.md')
+ uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- github-token: ${{ secrets.GITHUB_TOKEN }}
- script: |
- const fs = require('fs');
- if (!fs.existsSync('coverage_report.md')) return;
-
- const reportContent = fs.readFileSync('coverage_report.md', 'utf8');
- const identifier = '<!-- LLVM-LIBC-PRE-COMMIT-COVERAGE-COMMENT -->';
- const issue_number = context.payload.pull_request.number;
- const owner = context.repo.owner;
- const repo = context.repo.repo;
-
- try {
- // Fetch existing PR comments
- const comments = await github.rest.issues.listComments({
- owner,
- repo,
- issue_number,
- per_page: 100
- });
-
- const botComment = comments.data.find(c => c.body && c.body.includes(identifier));
-
- // Parse metrics from reportContent for history table entry
- const headSha = context.payload.pull_request.head.sha.substring(0, 7);
- const dateStr = new Date().toISOString().replace('T', ' ').substring(0, 16);
-
- const summaryMatch = reportContent.match(/\|\s*\*\*Total \(Patch\)\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\|\s*([0-9]+)\s*\/\s*([0-9]+)\s*\|\s*\*\*([0-9]+)\*\*/);
- const pct = summaryMatch ? summaryMatch[1] + '%' : '100.00%';
- const totalLines = summaryMatch ? summaryMatch[3] : 'N/A';
- const missedLines = summaryMatch ? summaryMatch[4] : '0';
-
- const newHistoryRow = `| [\`${headSha}\`](https://github.com/${owner}/${repo}/commit/${context.payload.pull_request.head.sha}) | ${dateStr} | ${totalLines} | **${pct}** | ${missedLines} |`;
-
- let existingHistory = '';
- if (botComment && botComment.body.includes('<!-- HISTORY_START -->')) {
- const historyStart = botComment.body.indexOf('<!-- HISTORY_START -->') + '<!-- HISTORY_START -->'.length;
- const historyEnd = botComment.body.indexOf('<!-- HISTORY_END -->');
- if (historyStart !== -1 && historyEnd !== -1) {
- existingHistory = botComment.body.substring(historyStart, historyEnd).trim();
- }
- }
-
- let historyTable = existingHistory;
- if (!historyTable) {
- historyTable = `| Commit | Date (UTC) | Executable Lines | Patch Coverage | Missed Lines |\n| :---: | :---: | :---: | :---: | :---: |\n${newHistoryRow}`;
- } else {
- if (!historyTable.includes(headSha)) {
- historyTable += `\n${newHistoryRow}`;
- }
- }
-
- const fullBody = `${identifier}\n${reportContent}\n\n<details>\n<summary><b>View Coverage History for this PR</b></summary>\n\n<!-- HISTORY_START -->\n${historyTable}\n<!-- HISTORY_END -->\n\n</details>`;
-
- if (botComment) {
- await github.rest.issues.updateComment({
- owner,
- repo,
- comment_id: botComment.id,
- body: fullBody
- });
- } else {
- await github.rest.issues.createComment({
- owner,
- repo,
- issue_number,
- body: fullBody
- });
- }
- } catch (err) {
- core.warning(`Could not post/update sticky PR comment: ${err.message}`);
- }
+ name: libc-patch-coverage-report
+ path: |
+ coverage_report.md
+ patch.diff
+ coverage.json
+ if-no-files-found: ignore
+ retention-days: 14
+ overwrite: true
diff --git a/.github/workflows/libc-patch-mcdc.yml b/.github/workflows/libc-patch-mcdc.yml
index 55355a839aad9..c4e6c238e4b8d 100644
--- a/.github/workflows/libc-patch-mcdc.yml
+++ b/.github/workflows/libc-patch-mcdc.yml
@@ -2,7 +2,6 @@ name: Libc Patch MC/DC Coverage
permissions:
contents: read
- pull-requests: write # Required to post/update patch coverage comments
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
@@ -57,7 +56,7 @@ jobs:
uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- fetch-depth: 2
+ fetch-depth: 100
persist-credentials: false
- name: Setup Compiler Cache (sccache)
@@ -113,8 +112,13 @@ jobs:
HEAD_REF="$PR_HEAD_REF"
BASE_REPO="${PR_BASE_REPO:-llvm/llvm-project}"
HEAD_REPO="${PR_HEAD_REPO:-$REPO_NAME}"
- git fetch upstream "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_REF" --depth=1 2>/dev/null || true
- DIFF_BASE="$BASE_SHA"
+ # Fetch base branch history to resolve merge base
+ git fetch upstream "$BASE_REF" --depth=100 2>/dev/null || git fetch origin "$BASE_REF" --depth=100 2>/dev/null || git fetch upstream "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 2>/dev/null || true
+ DIFF_BASE=$(git merge-base "$BASE_SHA" HEAD 2>/dev/null || true)
+ if [ -z "$DIFF_BASE" ]; then
+ git fetch --no-tags --deepen=200 2>/dev/null || true
+ DIFF_BASE=$(git merge-base "$BASE_SHA" HEAD 2>/dev/null || echo "$BASE_SHA")
+ fi
elif [ "$BRANCH_REF" != "main" ]; then
# For feature branches, compare against main merge-base
git fetch origin main --depth=100 2>/dev/null || true
@@ -240,8 +244,8 @@ jobs:
echo "[LOG] Discovered ${#EXECUTABLES[@]} instrumented binaries for llvm-cov export."
- if [ ! -f libc_full.profdata ]; then
- echo "[LOG] Notice: libc_full.profdata not found. Generating non-coverage summary."
+ if [ ! -f libc_full.profdata ] || [ ${#EXECUTABLES[@]} -eq 0 ]; then
+ echo "[LOG] Notice: Profile data or instrumented executables not found. Generating non-coverage summary."
echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
echo "" >> coverage_report.md
echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
@@ -279,76 +283,16 @@ jobs:
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
- - name: Post or Update Sticky PR Comment
- if: always() && github.event_name == 'pull_request'
- uses: actions/github-script at 3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+
+ - name: Upload Patch MC/DC Coverage Artifacts
+ if: always() && hashFiles('coverage_report.md')
+ uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- github-token: ${{ secrets.GITHUB_TOKEN }}
- script: |
- const fs = require('fs');
- if (!fs.existsSync('coverage_report.md')) return;
-
- const reportContent = fs.readFileSync('coverage_report.md', 'utf8');
- const identifier = '<!-- LLVM-LIBC-PRE-COMMIT-MCDC-COMMENT -->';
- const issue_number = context.payload.pull_request.number;
- const owner = context.repo.owner;
- const repo = context.repo.repo;
-
- try {
- const comments = await github.rest.issues.listComments({
- owner,
- repo,
- issue_number,
- per_page: 100
- });
-
- const botComment = comments.data.find(c => c.body && c.body.includes(identifier));
-
- const headSha = context.payload.pull_request.head.sha.substring(0, 7);
- const dateStr = new Date().toISOString().replace('T', ' ').substring(0, 16);
-
- const summaryMatch = reportContent.match(/\|\s*\*\*Total \(Patch\)\*\*\s*\|\s*\*\*([0-9.]+)%\*\*\s*\(([0-9]+)\/([0-9]+)\)\s*\|\s*\*\*([0-9.]+)%\*\*\s*\(([0-9]+)\/([0-9]+)\)\s*\|.*?\|\s*\*\*([0-9]+)\*\*/);
- const linePct = summaryMatch ? summaryMatch[1] + '%' : '100.00%';
- const mcdcPct = summaryMatch ? summaryMatch[4] + '%' : '100.00%';
- const missedLines = summaryMatch ? summaryMatch[7] : '0';
-
- const newHistoryRow = `| [\`${headSha}\`](https://github.com/${owner}/${repo}/commit/${context.payload.pull_request.head.sha}) | ${dateStr} | **${mcdcPct}** | **${linePct}** | ${missedLines} |`;
-
- let existingHistory = '';
- if (botComment && botComment.body.includes('<!-- HISTORY_START -->')) {
- const historyStart = botComment.body.indexOf('<!-- HISTORY_START -->') + '<!-- HISTORY_START -->'.length;
- const historyEnd = botComment.body.indexOf('<!-- HISTORY_END -->');
- if (historyStart !== -1 && historyEnd !== -1) {
- existingHistory = botComment.body.substring(historyStart, historyEnd).trim();
- }
- }
-
- let historyTable = existingHistory;
- if (!historyTable) {
- historyTable = `| Commit | Date (UTC) | MC/DC Coverage | Line Coverage | Missed Lines |\n| :---: | :---: | :---: | :---: | :---: |\n${newHistoryRow}`;
- } else {
- if (!historyTable.includes(headSha)) {
- historyTable += `\n${newHistoryRow}`;
- }
- }
-
- const fullBody = `${identifier}\n${reportContent}\n\n<details>\n<summary><b>View Coverage History for this PR</b></summary>\n\n<!-- HISTORY_START -->\n${historyTable}\n<!-- HISTORY_END -->\n\n</details>`;
-
- if (botComment) {
- await github.rest.issues.updateComment({
- owner,
- repo,
- comment_id: botComment.id,
- body: fullBody
- });
- } else {
- await github.rest.issues.createComment({
- owner,
- repo,
- issue_number,
- body: fullBody
- });
- }
- } catch (err) {
- core.warning(`Could not post/update sticky PR comment: ${err.message}`);
- }
+ name: libc-patch-mcdc-coverage-report
+ path: |
+ coverage_report.md
+ patch.diff
+ coverage.json
+ if-no-files-found: ignore
+ retention-days: 14
+ overwrite: true
diff --git a/libc/utils/coverage/codebase_coverage.py b/libc/utils/coverage/codebase_coverage.py
index 0c62bb11ebe57..03b35e84ace70 100644
--- a/libc/utils/coverage/codebase_coverage.py
+++ b/libc/utils/coverage/codebase_coverage.py
@@ -21,7 +21,6 @@
import argparse
import json
-import os
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
@@ -79,7 +78,6 @@ class FullCoverageSummary:
default_factory=DirectoryCoverageMetrics
)
directories: Dict[str, DirectoryCoverageMetrics] = field(default_factory=dict)
- dashboard_url: str = ""
@property
def has_mcdc(self) -> bool:
@@ -87,17 +85,6 @@ def has_mcdc(self) -> bool:
return self.global_stats.mcdc_tot > 0
-def resolve_dashboard_url(has_mcdc: bool) -> str:
- """Resolves live dashboard URL if configured via environment variable."""
- pages_url = os.environ.get("COVERAGE_DASHBOARD_URL", "").strip()
- if not pages_url:
- return ""
-
- if has_mcdc and not pages_url.endswith("/mcdc/"):
- return pages_url.rstrip("/") + "/mcdc/"
- return pages_url
-
-
def extract_full_coverage_statistics(
cov_data: dict,
) -> Optional[FullCoverageSummary]:
@@ -114,8 +101,6 @@ def extract_full_coverage_statistics(
continue
idx = file_path.find("src/")
- if idx == -1:
- continue
rel_path = file_path[idx:]
summary = item.get("summary", {})
@@ -134,12 +119,13 @@ def extract_full_coverage_statistics(
continue
mcdc_records = item.get("mcdc_records", [])
- file_decisions_tot = len(mcdc_records)
- file_decisions_full = sum(
- 1
+ valid_mcdc_records = [
+ rec
for rec in mcdc_records
- if len(rec) >= 10 and isinstance(rec[9], list) and all(rec[9])
- )
+ if len(rec) >= 10 and isinstance(rec[9], list) and len(rec[9]) > 0
+ ]
+ file_decisions_tot = len(valid_mcdc_records)
+ file_decisions_full = sum(1 for rec in valid_mcdc_records if all(rec[9]))
global_metrics.lines_cov += line_cov
global_metrics.lines_tot += line_tot
@@ -169,18 +155,14 @@ def extract_full_coverage_statistics(
if global_metrics.lines_tot == 0:
return None
- has_mcdc = global_metrics.mcdc_tot > 0
- dashboard_url = resolve_dashboard_url(has_mcdc)
-
return FullCoverageSummary(
global_stats=global_metrics,
directories=directories,
- dashboard_url=dashboard_url,
)
def format_overview_callout(summary: FullCoverageSummary) -> str:
- """Generates the executive summary banner with dashboard link."""
+ """Generates the executive summary banner."""
g = summary.global_stats
lines: List[str] = []
@@ -202,16 +184,10 @@ def format_overview_callout(summary: FullCoverageSummary) -> str:
)
lines.append("")
- if summary.dashboard_url:
- lines.append(
- f"- **Coverage Dashboard:** "
- f"[{summary.dashboard_url}]({summary.dashboard_url})"
- )
- else:
- lines.append(
- "- **HTML Coverage Report:** Available for download under the "
- "**Artifacts** section of this workflow run."
- )
+ lines.append(
+ "- **HTML Coverage Report:** Available for download under the "
+ "**Artifacts** section of this workflow run."
+ )
return "\n".join(lines)
diff --git a/libc/utils/coverage/diff_coverage.py b/libc/utils/coverage/diff_coverage.py
index e8c3587bf8b57..8bfbe78e1cda5 100644
--- a/libc/utils/coverage/diff_coverage.py
+++ b/libc/utils/coverage/diff_coverage.py
@@ -40,7 +40,10 @@
"__attribute__",
"template",
"typedef ",
+ "friend ",
+ "static_assert",
)
+ACCESS_SPECIFIERS = ("public:", "private:", "protected:")
@dataclass
@@ -267,7 +270,11 @@ def extract_patch_matrix(
# 2. Process MC/DC decision records
mcdc_records = item.get("mcdc_records", [])
for record in mcdc_records:
- if len(record) >= 10 and isinstance(record[9], list):
+ if (
+ len(record) >= 10
+ and isinstance(record[9], list)
+ and len(record[9]) > 0
+ ):
decision_start_line = record[0]
decision_end_line = record[2]
boolean_conditions = record[9]
@@ -292,23 +299,32 @@ def is_executable_line(line_text: str) -> bool:
stripped_line = line_text.strip()
if not stripped_line:
return False
- if any(stripped_line.startswith(prefix) for prefix in COMMENT_PREFIXES):
+ # Strip block comments on the same line and trailing line comments
+ clean_line = re.sub(r"/\*.*?\*/", "", stripped_line)
+ clean_line = re.sub(r"//.*$", "", clean_line).strip()
+ if not clean_line:
return False
- if stripped_line in STRUCTURAL_TOKENS or stripped_line.startswith(":"):
+ if (
+ clean_line.startswith("*")
+ or clean_line.startswith("/*")
+ or clean_line.startswith("*/")
+ ):
+ return False
+ if clean_line in STRUCTURAL_TOKENS or clean_line.startswith(":"):
return False
- if stripped_line.startswith("#"):
+ if clean_line in ACCESS_SPECIFIERS:
return False
- if any(stripped_line.startswith(prefix) for prefix in DECLARATION_PREFIXES):
+ if clean_line.startswith("#"):
+ return False
+ if any(clean_line.startswith(prefix) for prefix in DECLARATION_PREFIXES):
return False
if (
- stripped_line.startswith("struct ")
- or stripped_line.startswith("class ")
- or stripped_line.startswith("enum ")
+ clean_line.startswith("struct ")
+ or clean_line.startswith("class ")
+ or clean_line.startswith("enum ")
):
- if "{" in stripped_line or (
- stripped_line.endswith(";")
- and "=" not in stripped_line
- and "(" not in stripped_line
+ if ("{" in clean_line and "=" not in clean_line) or (
+ clean_line.endswith(";") and "=" not in clean_line and "(" not in clean_line
):
return False
return True
@@ -348,6 +364,15 @@ def calculate_patch_statistics(
summary = PatchCoverageSummary()
for file_path, file_data in coverage_matrix.items():
+ if (
+ not any(file_path.endswith(ext) for ext in (".cpp", ".c", ".h", ".inc"))
+ or "/test/" in file_path
+ or file_path.startswith("test/")
+ or "/utils/" in file_path
+ or file_path.startswith("utils/")
+ ):
+ continue
+
added_lines: Set[int] = set()
for hunk in diff_files.get(file_path, []):
for line_type, text, line_number in hunk.lines:
@@ -546,9 +571,12 @@ def format_breakdown_table(
for file_path, file_metric in summary.files.items():
commit_ref = head_commit_sha or "main"
+ repo_file_path = (
+ file_path if file_path.startswith("libc/") else f"libc/{file_path}"
+ )
file_link = (
f"[`{file_path}`]"
- f"(https://github.com/{head_repository}/blob/{commit_ref}/{file_path})"
+ f"(https://github.com/{head_repository}/blob/{commit_ref}/{repo_file_path})"
)
missed_lines = file_metric.missed_lines
covered_count = len(file_metric.covered_lines)
@@ -576,7 +604,7 @@ def format_breakdown_table(
f"| {file_link} | "
f"**{file_metric.line_coverage_percentage:.2f}%** "
f"({covered_count}/{total_file_lines}) | "
- f"{mcdc_cell} | {decision_cell} | "
+ f"{mcdc_cell} | {dec_cell} | "
f"{len(missed_lines)} | {diagnostic_cell} |"
)
else:
@@ -689,7 +717,7 @@ def render_patch_report(
print(metadata_section_string)
print("\n---\n")
print("### Coverage Summary")
- print("No `.cpp` source files in `libc/src/` were modified in this patch.")
+ print("No executable lines were added or modified in this patch.")
return
# 1. Status Banner
@@ -747,6 +775,10 @@ def main() -> None:
arguments = parser.parse_args()
+ if not os.path.isfile(arguments.diff_file):
+ sys.stderr.write(f"Error: Diff file not found: '{arguments.diff_file}'\n")
+ sys.exit(1)
+
diff_files = DiffParser.parse(arguments.diff_file)
coverage_data = CoverageJSONParser.load(arguments.json_file)
coverage_matrix = CoverageJSONParser.extract_patch_matrix(coverage_data, diff_files)
diff --git a/libc/utils/coverage/test_codebase_coverage.py b/libc/utils/coverage/test_codebase_coverage.py
index 571393975c09d..9d9d19ee3fad0 100644
--- a/libc/utils/coverage/test_codebase_coverage.py
+++ b/libc/utils/coverage/test_codebase_coverage.py
@@ -6,72 +6,204 @@
#
# ==-------------------------------------------------------------------------==#
+"""Unit tests for codebase_coverage.py."""
+
import io
import json
import os
+import sys
+import tempfile
import unittest
from contextlib import redirect_stdout
+from unittest.mock import patch
+
+# Ensure libc/utils/coverage is in sys.path when running from any working directory
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from codebase_coverage import (
DirectoryCoverageMetrics,
FullCoverageSummary,
extract_full_coverage_statistics,
format_directory_breakdown_table,
+ format_global_summary_table,
format_overview_callout,
+ main,
render_full_report,
- resolve_dashboard_url,
)
-class TestCodebaseDirectoryAggregation(unittest.TestCase):
- def test_path_filtering_and_routing(self):
- mock_json = {
+class TestDirectoryCoverageMetrics(unittest.TestCase):
+ """Tests DirectoryCoverageMetrics mathematical operations and property safeguards."""
+
+ def test_zero_totals_return_zero_percentages(self):
+ """Zero totals must safely evaluate to 0.0 without ZeroDivisionError."""
+ metrics = DirectoryCoverageMetrics(name="empty")
+ self.assertEqual(metrics.line_pct, 0.0)
+ self.assertEqual(metrics.func_pct, 0.0)
+ self.assertEqual(metrics.mcdc_pct, 0.0)
+ self.assertEqual(metrics.decisions_pct, 0.0)
+ self.assertEqual(metrics.missed_lines, 0)
+
+ def test_percentage_calculations(self):
+ """Percentages must correctly compute ratios across lines, functions, and MC/DC."""
+ metrics = DirectoryCoverageMetrics(
+ name="src/math",
+ lines_cov=75,
+ lines_tot=100,
+ func_cov=3,
+ func_tot=4,
+ mcdc_cov=7,
+ mcdc_tot=10,
+ decisions_tot=5,
+ decisions_full=4,
+ )
+ self.assertAlmostEqual(metrics.line_pct, 75.0, places=2)
+ self.assertAlmostEqual(metrics.func_pct, 75.0, places=2)
+ self.assertAlmostEqual(metrics.mcdc_pct, 70.0, places=2)
+ self.assertAlmostEqual(metrics.decisions_pct, 80.0, places=2)
+ self.assertEqual(metrics.missed_lines, 25)
+
+ def test_missed_lines_clamping(self):
+ """Missed lines must clamp to 0 if covered lines exceed total lines."""
+ metrics = DirectoryCoverageMetrics(
+ name="src/clamped", lines_cov=120, lines_tot=100
+ )
+ self.assertEqual(metrics.missed_lines, 0)
+
+ def test_all_zero_metrics(self):
+ """Default initialized metrics object must have zero values and empty name."""
+ metrics = DirectoryCoverageMetrics()
+ self.assertEqual(metrics.name, "")
+ self.assertEqual(metrics.lines_cov, 0)
+ self.assertEqual(metrics.lines_tot, 0)
+ self.assertEqual(metrics.decisions_pct, 0.0)
+
+
+class TestFullCoverageSummary(unittest.TestCase):
+ """Tests FullCoverageSummary attributes and condition indicators."""
+
+ def test_has_mcdc_property(self):
+ """has_mcdc must reflect whether global MC/DC conditions exist."""
+ without_mcdc = FullCoverageSummary(
+ global_stats=DirectoryCoverageMetrics(mcdc_tot=0)
+ )
+ self.assertFalse(without_mcdc.has_mcdc)
+
+ with_mcdc = FullCoverageSummary(
+ global_stats=DirectoryCoverageMetrics(mcdc_tot=12)
+ )
+ self.assertTrue(with_mcdc.has_mcdc)
+
+ def test_default_initialization(self):
+ """Default FullCoverageSummary must initialize empty directories dictionary."""
+ summary = FullCoverageSummary()
+ self.assertFalse(summary.has_mcdc)
+ self.assertEqual(len(summary.directories), 0)
+
+
+class TestExtractFullCoverageStatistics(unittest.TestCase):
+ """Tests JSON extraction, file path filtering, directory grouping, and MC/DC aggregation."""
+
+ def test_empty_or_invalid_payloads_return_none(self):
+ """Missing or malformed payload structures must return None."""
+ test_cases = [
+ ({}, "Empty dictionary"),
+ ({"data": []}, "Empty data array"),
+ ({"data": [{}]}, "Data array without files key"),
+ ({"data": [{"files": []}]}, "Empty files array"),
+ ]
+ for payload, description in test_cases:
+ with self.subTest(msg=description):
+ self.assertIsNone(extract_full_coverage_statistics(payload))
+
+ def test_all_files_zero_lines_returns_none(self):
+ """Payload containing only files with zero total lines must return None."""
+ payload = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/src/empty.cpp",
+ "summary": {
+ "lines": {"count": 0, "covered": 0},
+ "functions": {"count": 0, "covered": 0},
+ },
+ }
+ ]
+ }
+ ]
+ }
+ self.assertIsNone(extract_full_coverage_statistics(payload))
+
+ def test_all_files_excluded_returns_none(self):
+ """Payload containing only test or utility files must return None."""
+ payload = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/test/src/math/sin_test.cpp",
+ "summary": {
+ "lines": {"count": 100, "covered": 100},
+ "functions": {"count": 1, "covered": 1},
+ },
+ },
+ {
+ "filename": "/workspace/libc/utils/MPFRWrapper/MPFRUtils.cpp",
+ "summary": {
+ "lines": {"count": 200, "covered": 200},
+ "functions": {"count": 2, "covered": 2},
+ },
+ },
+ ]
+ }
+ ]
+ }
+ self.assertIsNone(extract_full_coverage_statistics(payload))
+
+ def test_file_path_filtering(self):
+ """Test and utility directories must be excluded from codebase coverage."""
+ payload = {
"data": [
{
"files": [
{
"filename": "/workspace/libc/src/math/sin.cpp",
"summary": {
- "lines": {
- "count": 100,
- "covered": 50,
- "percent": 50.0,
- },
- "functions": {
- "count": 2,
- "covered": 1,
- "percent": 50.0,
- },
+ "lines": {"count": 100, "covered": 80},
+ "functions": {"count": 2, "covered": 2},
},
},
{
- "filename": ("/workspace/libc/src/string/strcpy.cpp"),
+ # Test file: must be excluded
+ "filename": "/workspace/libc/test/src/math/sin_test.cpp",
"summary": {
- "lines": {
- "count": 200,
- "covered": 200,
- "percent": 100.0,
- },
- "functions": {
- "count": 4,
- "covered": 4,
- "percent": 100.0,
- },
+ "lines": {"count": 500, "covered": 500},
+ "functions": {"count": 5, "covered": 5},
},
},
{
- "filename": ("/workspace/libc/test/src/math/sin_test.cpp"),
+ # Utility file: must be excluded
+ "filename": "/workspace/libc/utils/MPFRWrapper/MPFRUtils.cpp",
"summary": {
- "lines": {
- "count": 500,
- "covered": 500,
- "percent": 100.0,
- },
- "functions": {
- "count": 10,
- "covered": 10,
- "percent": 100.0,
- },
+ "lines": {"count": 300, "covered": 300},
+ "functions": {"count": 4, "covered": 4},
+ },
+ },
+ {
+ # Non-src file: must be excluded
+ "filename": "/workspace/libc/include/llvm-libc-types/size_t.h",
+ "summary": {
+ "lines": {"count": 50, "covered": 50},
+ "functions": {"count": 1, "covered": 1},
+ },
+ },
+ {
+ # File with zero total lines: must be excluded
+ "filename": "/workspace/libc/src/empty.cpp",
+ "summary": {
+ "lines": {"count": 0, "covered": 0},
+ "functions": {"count": 0, "covered": 0},
},
},
]
@@ -79,96 +211,332 @@ def test_path_filtering_and_routing(self):
]
}
- summary = extract_full_coverage_statistics(mock_json)
+ summary = extract_full_coverage_statistics(payload)
+ self.assertIsNotNone(summary)
+ self.assertEqual(summary.global_stats.lines_tot, 100)
+ self.assertEqual(summary.global_stats.lines_cov, 80)
+ self.assertEqual(summary.global_stats.func_tot, 2)
+ self.assertEqual(summary.global_stats.func_cov, 2)
+ self.assertIn("src/math", summary.directories)
+ self.assertEqual(len(summary.directories), 1)
+
+ def test_directory_bucketing_and_nested_paths(self):
+ """Files within identical top-level directories or deep subpaths must aggregate properly."""
+ payload = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/src/math/sin.cpp",
+ "summary": {
+ "lines": {"count": 100, "covered": 60},
+ "functions": {"count": 2, "covered": 1},
+ },
+ },
+ {
+ "filename": "/workspace/libc/src/math/cos.cpp",
+ "summary": {
+ "lines": {"count": 80, "covered": 80},
+ "functions": {"count": 2, "covered": 2},
+ },
+ },
+ {
+ # Nested subpath under src/string/
+ "filename": "/workspace/libc/src/string/memory_utils/op_builtin.cpp",
+ "summary": {
+ "lines": {"count": 120, "covered": 100},
+ "functions": {"count": 4, "covered": 3},
+ },
+ },
+ ]
+ }
+ ]
+ }
- # The test file should be ignored entirely
+ summary = extract_full_coverage_statistics(payload)
+ self.assertIsNotNone(summary)
self.assertEqual(summary.global_stats.lines_tot, 300)
- self.assertEqual(summary.global_stats.lines_cov, 250)
- self.assertEqual(summary.global_stats.func_tot, 6)
- self.assertEqual(summary.global_stats.func_cov, 5)
+ self.assertEqual(summary.global_stats.lines_cov, 240)
- # Ensure proper bucket routing
self.assertIn("src/math", summary.directories)
- self.assertEqual(summary.directories["src/math"].lines_tot, 100)
+ math_dir = summary.directories["src/math"]
+ self.assertEqual(math_dir.lines_tot, 180)
+ self.assertEqual(math_dir.lines_cov, 140)
+ self.assertEqual(math_dir.func_tot, 4)
+ self.assertEqual(math_dir.func_cov, 3)
+
self.assertIn("src/string", summary.directories)
- self.assertEqual(summary.directories["src/string"].lines_tot, 200)
- self.assertNotIn("test/src/math", summary.directories)
+ str_dir = summary.directories["src/string"]
+ self.assertEqual(str_dir.lines_tot, 120)
+ self.assertEqual(str_dir.lines_cov, 100)
+ def test_mcdc_records_aggregation_and_decision_tracking(self):
+ """MC/DC records must be parsed for total conditions and full decision verification."""
+ payload = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/src/math/fma.cpp",
+ "summary": {
+ "lines": {"count": 50, "covered": 50},
+ "functions": {"count": 1, "covered": 1},
+ "mcdc": {"count": 4, "covered": 3},
+ },
+ "mcdc_records": [
+ # Fully verified decision: [True, True]
+ [10, 5, 10, 20, 0, 0, 0, 0, 0, [True, True]],
+ # Partially verified decision: [True, False]
+ [25, 5, 25, 25, 0, 0, 0, 0, 0, [True, False]],
+ # Malformed record: ignored
+ [30, 5, 30, 20],
+ ],
+ }
+ ]
+ }
+ ]
+ }
-class TestCodebaseReportRendering(unittest.TestCase):
- def test_format_directory_breakdown_table(self):
- metrics_math = DirectoryCoverageMetrics(
- name="src/math",
- lines_tot=100,
- lines_cov=50,
- func_tot=10,
- func_cov=5,
+ summary = extract_full_coverage_statistics(payload)
+ self.assertIsNotNone(summary)
+ self.assertTrue(summary.has_mcdc)
+ self.assertEqual(summary.global_stats.mcdc_tot, 4)
+ self.assertEqual(summary.global_stats.mcdc_cov, 3)
+ self.assertEqual(summary.global_stats.decisions_tot, 2)
+ self.assertEqual(summary.global_stats.decisions_full, 1)
+
+ def test_malformed_and_empty_mcdc_records_ignored(self):
+ """Malformed, non-list, or empty condition vectors must not count as valid decisions."""
+ payload = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/src/math/exp.cpp",
+ "summary": {
+ "lines": {"count": 50, "covered": 50},
+ "functions": {"count": 1, "covered": 1},
+ "mcdc": {"count": 2, "covered": 2},
+ },
+ "mcdc_records": [
+ # Valid fully verified decision
+ [10, 5, 10, 20, 0, 0, 0, 0, 0, [True]],
+ # Empty list: not a valid decision
+ [20, 5, 20, 20, 0, 0, 0, 0, 0, []],
+ # Non-list 10th element: not a valid decision
+ [30, 5, 30, 20, 0, 0, 0, 0, 0, None],
+ # Record length < 10: not a valid decision
+ [40, 5, 40, 20],
+ ],
+ }
+ ]
+ }
+ ]
+ }
+
+ summary = extract_full_coverage_statistics(payload)
+ self.assertIsNotNone(summary)
+ self.assertEqual(summary.global_stats.decisions_tot, 1)
+ self.assertEqual(summary.global_stats.decisions_full, 1)
+
+
+class TestCodebaseReportFormatting(unittest.TestCase):
+ """Tests Markdown rendering of callouts, summary tables, and directory breakdowns."""
+
+ def test_format_overview_callout_without_mcdc(self):
+ """Callout without MC/DC must render line coverage and artifacts instructions."""
+ metrics = DirectoryCoverageMetrics(name="global", lines_tot=1000, lines_cov=850)
+ summary = FullCoverageSummary(global_stats=metrics)
+ callout = format_overview_callout(summary)
+
+ self.assertIn("### Overall Codebase Coverage: **85.00%**", callout)
+ self.assertIn("Tested **850 / 1,000** executable lines", callout)
+ self.assertIn("Artifacts", callout)
+ self.assertIn("HTML Coverage Report", callout)
+ self.assertNotIn("MC/DC", callout)
+
+ def test_format_overview_callout_with_mcdc(self):
+ """Callout with MC/DC must render both line and condition coverage metrics."""
+ metrics = DirectoryCoverageMetrics(
+ name="global",
+ lines_tot=2000,
+ lines_cov=1800,
+ mcdc_tot=50,
+ mcdc_cov=45,
+ decisions_tot=20,
+ decisions_full=18,
+ )
+ summary = FullCoverageSummary(global_stats=metrics)
+ callout = format_overview_callout(summary)
+
+ self.assertIn(
+ "### Overall Codebase Coverage: **90.00% Line** | **90.00% MC/DC**",
+ callout,
+ )
+ self.assertIn("Tested **1,800 / 2,000** executable lines", callout)
+ self.assertIn("and **45 / 50** boolean conditions", callout)
+ self.assertIn("across **20** decisions.", callout)
+ self.assertIn("Artifacts", callout)
+
+ def test_format_global_summary_table_without_mcdc(self):
+ """Global table without MC/DC must display lines and functions only."""
+ metrics = DirectoryCoverageMetrics(
+ name="global",
+ lines_tot=500,
+ lines_cov=400,
+ func_tot=50,
+ func_cov=40,
+ )
+ summary = FullCoverageSummary(global_stats=metrics)
+ table = format_global_summary_table(summary)
+
+ self.assertIn("| **Executable Lines** | 400 | 500 | **80.00%** |", table)
+ self.assertIn("| **Functions** | 40 | 50 | **80.00%** |", table)
+ self.assertNotIn("MC/DC", table)
+
+ def test_format_global_summary_table_with_mcdc(self):
+ """Global table with MC/DC must include condition independence and decision verification."""
+ metrics = DirectoryCoverageMetrics(
+ name="global",
+ lines_tot=1000,
+ lines_cov=900,
+ func_tot=100,
+ func_cov=95,
+ mcdc_tot=80,
+ mcdc_cov=60,
+ decisions_tot=30,
+ decisions_full=25,
+ )
+ summary = FullCoverageSummary(global_stats=metrics)
+ table = format_global_summary_table(summary)
+
+ self.assertIn(
+ "| **MC/DC Condition Independence** | 60 | 80 | **75.00%** |", table
+ )
+ self.assertIn("| **Fully Verified Decisions** | 25 | 30 | **83.33%** |", table)
+
+ def test_format_directory_breakdown_table_alphabetical_sorting(self):
+ """Directory breakdown table must sort directory names alphabetically."""
+ dir_string = DirectoryCoverageMetrics(
+ name="src/string", lines_tot=100, lines_cov=100
+ )
+ dir_math = DirectoryCoverageMetrics(
+ name="src/math", lines_tot=100, lines_cov=80
)
- metrics_string = DirectoryCoverageMetrics(
- name="src/string",
- lines_tot=200,
- lines_cov=200,
- func_tot=20,
- func_cov=20,
+ dir_ctype = DirectoryCoverageMetrics(
+ name="src/ctype", lines_tot=100, lines_cov=90
)
+
summary = FullCoverageSummary(
- global_stats=metrics_math,
+ global_stats=DirectoryCoverageMetrics(lines_tot=300, lines_cov=270),
directories={
- "src/math": metrics_math,
- "src/string": metrics_string,
+ "src/string": dir_string,
+ "src/math": dir_math,
+ "src/ctype": dir_ctype,
},
)
+ table = format_directory_breakdown_table(summary)
+
+ # Check alphabetical order in markdown output
+ idx_ctype = table.find("`libc/src/ctype`")
+ idx_math = table.find("`libc/src/math`")
+ idx_string = table.find("`libc/src/string`")
+ self.assertTrue(0 <= idx_ctype < idx_math < idx_string)
+
+ def test_format_directory_breakdown_table_without_mcdc(self):
+ """Directory table without MC/DC must show line and function coverage columns."""
+ dir_math = DirectoryCoverageMetrics(
+ name="src/math", lines_tot=100, lines_cov=80, func_tot=2, func_cov=2
+ )
+ summary = FullCoverageSummary(
+ global_stats=DirectoryCoverageMetrics(lines_tot=100, lines_cov=80),
+ directories={"src/math": dir_math},
+ )
table = format_directory_breakdown_table(summary)
- self.assertIn("`libc/src/math`", table)
- self.assertIn("`libc/src/string`", table)
+ self.assertNotIn("MC/DC Conditions", table)
+ self.assertIn("`libc/src/math` | **80.00%** | 100.00% | 100 | 20 |", table)
- def test_format_overview_callout_without_dashboard_url(self):
- metrics = DirectoryCoverageMetrics(name="global", lines_tot=100, lines_cov=75)
- summary = FullCoverageSummary(global_stats=metrics, dashboard_url="")
- callout = format_overview_callout(summary)
- self.assertIn("Artifacts", callout)
- self.assertIn("HTML Coverage Report", callout)
- self.assertNotIn("Coverage Dashboard:", callout)
+ def test_format_directory_breakdown_table_with_mcdc(self):
+ """Directory table with MC/DC must show MC/DC condition columns."""
+ dir_math = DirectoryCoverageMetrics(
+ name="src/math",
+ lines_tot=100,
+ lines_cov=80,
+ mcdc_tot=10,
+ mcdc_cov=8,
+ decisions_tot=4,
+ decisions_full=3,
+ )
+ summary = FullCoverageSummary(
+ global_stats=DirectoryCoverageMetrics(
+ lines_tot=100, lines_cov=80, mcdc_tot=10, mcdc_cov=8
+ ),
+ directories={"src/math": dir_math},
+ )
+ table = format_directory_breakdown_table(summary)
+
+ self.assertIn("MC/DC Conditions", table)
+ self.assertIn("**80.0%** (8/10)", table)
- def test_format_overview_callout_with_dashboard_url(self):
- metrics = DirectoryCoverageMetrics(name="global", lines_tot=100, lines_cov=75)
+ def test_format_directory_breakdown_table_mixed_mcdc(self):
+ """Directories without MC/DC records in an MC/DC run must display N/A."""
+ dir_math = DirectoryCoverageMetrics(
+ name="src/math",
+ lines_tot=100,
+ lines_cov=80,
+ mcdc_tot=10,
+ mcdc_cov=8,
+ decisions_tot=4,
+ decisions_full=3,
+ )
+ dir_ctype = DirectoryCoverageMetrics(
+ name="src/ctype",
+ lines_tot=50,
+ lines_cov=50,
+ mcdc_tot=0,
+ mcdc_cov=0,
+ decisions_tot=0,
+ decisions_full=0,
+ )
summary = FullCoverageSummary(
- global_stats=metrics, dashboard_url="https://example.com/coverage/"
+ global_stats=DirectoryCoverageMetrics(
+ lines_tot=150, lines_cov=130, mcdc_tot=10, mcdc_cov=8
+ ),
+ directories={"src/math": dir_math, "src/ctype": dir_ctype},
)
- callout = format_overview_callout(summary)
- self.assertIn("Coverage Dashboard:", callout)
- self.assertIn("https://example.com/coverage/", callout)
+ table = format_directory_breakdown_table(summary)
+ self.assertIn("`libc/src/math` | **80.0%** (8/10) | 3 / 4", table)
+ self.assertIn("`libc/src/ctype` | N/A | N/A", table)
- def test_resolve_dashboard_url_default_is_empty(self):
- old_env = os.environ.get("COVERAGE_DASHBOARD_URL")
- try:
- if "COVERAGE_DASHBOARD_URL" in os.environ:
- del os.environ["COVERAGE_DASHBOARD_URL"]
- self.assertEqual(resolve_dashboard_url(has_mcdc=False), "")
- self.assertEqual(resolve_dashboard_url(has_mcdc=True), "")
- finally:
- if old_env is not None:
- os.environ["COVERAGE_DASHBOARD_URL"] = old_env
- def test_render_full_report_end_to_end(self):
- mock_json = {
+class TestRenderFullReportEndToEnd(unittest.TestCase):
+ """Tests full Markdown report composition from JSON payload to stdout."""
+
+ def test_render_empty_payload_fallback(self):
+ """Empty payload must output fallback message without crashing."""
+ buf = io.StringIO()
+ with redirect_stdout(buf):
+ render_full_report({})
+ output = buf.getvalue()
+ self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
+ self.assertIn("### No Coverage Data Detected", output)
+ self.assertIn(
+ "The test execution completed but no coverage profiles were exported.",
+ output,
+ )
+
+ def test_render_complete_report_without_mcdc(self):
+ """Valid report without MC/DC must render line coverage and summary tables."""
+ payload = {
"data": [
{
"files": [
{
"filename": "/workspace/libc/src/math/sin.cpp",
"summary": {
- "lines": {
- "count": 100,
- "covered": 80,
- "percent": 80.0,
- },
- "functions": {
- "count": 2,
- "covered": 2,
- "percent": 100.0,
- },
+ "lines": {"count": 100, "covered": 90},
+ "functions": {"count": 2, "covered": 2},
},
}
]
@@ -177,15 +545,161 @@ def test_render_full_report_end_to_end(self):
}
buf = io.StringIO()
with redirect_stdout(buf):
- render_full_report(mock_json)
+ render_full_report(payload)
output = buf.getvalue()
self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
- self.assertIn("HTML Coverage Report", output)
- self.assertIn("Artifacts", output)
+ self.assertIn("### Overall Codebase Coverage: **90.00%**", output)
+ self.assertIn("`libc/src/math`", output)
+ self.assertNotIn("MC/DC", output)
+
+ def test_render_complete_report_with_mcdc(self):
+ """Valid report with MC/DC must render full callouts, global table, and directory table."""
+ payload = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/src/math/sin.cpp",
+ "summary": {
+ "lines": {"count": 100, "covered": 90},
+ "functions": {"count": 2, "covered": 2},
+ "mcdc": {"count": 6, "covered": 6},
+ },
+ "mcdc_records": [
+ [10, 5, 10, 20, 0, 0, 0, 0, 0, [True, True]]
+ ],
+ }
+ ]
+ }
+ ]
+ }
+
+ buf = io.StringIO()
+ with redirect_stdout(buf):
+ render_full_report(payload)
+ output = buf.getvalue()
+
+ self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
+ self.assertIn("### Overall Codebase Coverage:", output)
self.assertIn("### Overall", output)
+ self.assertIn("### Coverage Breakdown", output)
self.assertIn("`libc/src/math`", output)
+class TestCommandLineInterface(unittest.TestCase):
+ """Tests CLI invocation, arguments parsing, and file handling."""
+
+ def test_cli_execution_with_file(self):
+ """CLI must read JSON coverage file from disk and write report to stdout."""
+ payload = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/src/math/sin.cpp",
+ "summary": {
+ "lines": {"count": 50, "covered": 40},
+ "functions": {"count": 1, "covered": 1},
+ },
+ }
+ ]
+ }
+ ]
+ }
+
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".json", delete=False
+ ) as tmp_file:
+ json.dump(payload, tmp_file)
+ tmp_path = tmp_file.name
+
+ try:
+ buf = io.StringIO()
+ with patch.object(
+ sys,
+ "argv",
+ ["codebase_coverage.py", tmp_path, "aabbccdd1122", "main"],
+ ):
+ with redirect_stdout(buf):
+ main()
+ output = buf.getvalue()
+ self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
+ self.assertIn("`libc/src/math`", output)
+ finally:
+ if os.path.exists(tmp_path):
+ os.remove(tmp_path)
+
+ def test_cli_execution_with_minimal_arguments(self):
+ """CLI must execute successfully when commit SHA and branch ref are omitted."""
+ payload = {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": "/workspace/libc/src/ctype/isalnum.cpp",
+ "summary": {
+ "lines": {"count": 20, "covered": 20},
+ "functions": {"count": 1, "covered": 1},
+ },
+ }
+ ]
+ }
+ ]
+ }
+
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".json", delete=False
+ ) as tmp_file:
+ json.dump(payload, tmp_file)
+ tmp_path = tmp_file.name
+
+ try:
+ buf = io.StringIO()
+ with patch.object(sys, "argv", ["codebase_coverage.py", tmp_path]):
+ with redirect_stdout(buf):
+ main()
+ output = buf.getvalue()
+ self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
+ self.assertIn("`libc/src/ctype`", output)
+ finally:
+ if os.path.exists(tmp_path):
+ os.remove(tmp_path)
+
+ def test_cli_nonexistent_file_exits_with_error(self):
+ """CLI must exit with code 1 when targeted file does not exist."""
+ stderr_buf = io.StringIO()
+ with patch.object(
+ sys,
+ "argv",
+ ["codebase_coverage.py", "/nonexistent/path/coverage.json"],
+ ):
+ with patch("sys.stderr", stderr_buf):
+ with self.assertRaises(SystemExit) as cm:
+ main()
+ self.assertEqual(cm.exception.code, 1)
+ self.assertIn("Error: Failed to parse coverage JSON", stderr_buf.getvalue())
+
+ def test_cli_invalid_json_exits_with_error(self):
+ """CLI must exit with code 1 when targeted file contains invalid JSON syntax."""
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".json", delete=False
+ ) as tmp_file:
+ tmp_file.write("INVALID JSON CONTENT")
+ tmp_path = tmp_file.name
+
+ stderr_buf = io.StringIO()
+ try:
+ with patch.object(sys, "argv", ["codebase_coverage.py", tmp_path]):
+ with patch("sys.stderr", stderr_buf):
+ with self.assertRaises(SystemExit) as cm:
+ main()
+ self.assertEqual(cm.exception.code, 1)
+ self.assertIn("Error: Failed to parse coverage JSON", stderr_buf.getvalue())
+ finally:
+ if os.path.exists(tmp_path):
+ os.remove(tmp_path)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/libc/utils/coverage/test_diff_coverage.py b/libc/utils/coverage/test_diff_coverage.py
index 49683c3d6f5b7..e369a6a5f7004 100644
--- a/libc/utils/coverage/test_diff_coverage.py
+++ b/libc/utils/coverage/test_diff_coverage.py
@@ -6,276 +6,618 @@
#
# ==-------------------------------------------------------------------------==#
+"""Unit tests for diff_coverage.py."""
+
+import io
import json
+import os
+import sys
+import tempfile
import unittest
+from contextlib import redirect_stdout
+from unittest.mock import patch
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from diff_coverage import (
+ DEFAULT_BASE_REPOSITORY,
+ DEFAULT_HEAD_REPOSITORY,
CoverageJSONParser,
DiffHunk,
DiffParser,
FilePatchMetrics,
PatchCoverageSummary,
calculate_patch_statistics,
+ format_annotated_diff,
format_breakdown_table,
+ format_line_ranges,
+ format_metadata_section,
format_status_banner,
is_executable_line,
+ main,
+ render_patch_report,
)
-class TestExecutableLineHeuristics(unittest.TestCase):
- def test_heuristics_matrix(self):
- matrix = [
- ("int x = 5;", True, "Basic assignment"),
+def _make_cov_json(filename, segments, mcdc_records=None):
+ """Constructs a minimal llvm-cov JSON export payload."""
+ return {
+ "data": [
+ {
+ "files": [
+ {
+ "filename": filename,
+ "segments": segments,
+ "mcdc_records": mcdc_records or [],
+ }
+ ]
+ }
+ ]
+ }
+
+
+class TestCoverageDataStructures(unittest.TestCase):
+ """Tests FilePatchMetrics and PatchCoverageSummary ratio logic and zero-division guards."""
+
+ def test_file_patch_metrics_calculations(self):
+ """FilePatchMetrics must correctly compute line and MC/DC ratios or 0.0 on zero totals."""
+ empty = FilePatchMetrics(file_path="src/math/sin.cpp")
+ self.assertEqual(empty.total_lines, 0)
+ self.assertEqual(empty.line_coverage_percentage, 0.0)
+ self.assertEqual(empty.mcdc_coverage_percentage, 0.0)
+
+ metrics = FilePatchMetrics(
+ file_path="src/math/sin.cpp",
+ covered_lines={10, 11, 12},
+ missed_lines={13},
+ mcdc_covered_conditions=3,
+ mcdc_total_conditions=4,
+ )
+ self.assertEqual(metrics.total_lines, 4)
+ self.assertAlmostEqual(metrics.line_coverage_percentage, 75.0, places=2)
+ self.assertAlmostEqual(metrics.mcdc_coverage_percentage, 75.0, places=2)
+
+ def test_patch_coverage_summary_aggregation(self):
+ """PatchCoverageSummary must correctly aggregate metrics and condition indicators."""
+ empty = PatchCoverageSummary()
+ self.assertEqual(empty.total_lines, 0)
+ self.assertEqual(empty.line_coverage_percentage, 0.0)
+ self.assertEqual(empty.mcdc_coverage_percentage, 0.0)
+ self.assertFalse(empty.has_mcdc)
+
+ summary = PatchCoverageSummary(
+ total_covered_lines=6,
+ total_missed_lines=2,
+ total_mcdc_covered_conditions=1,
+ total_mcdc_total_conditions=2,
+ )
+ self.assertEqual(summary.total_lines, 8)
+ self.assertAlmostEqual(summary.line_coverage_percentage, 75.0, places=2)
+ self.assertAlmostEqual(summary.mcdc_coverage_percentage, 50.0, places=2)
+ self.assertTrue(summary.has_mcdc)
+
+
+class TestExecutableLineFiltering(unittest.TestCase):
+ """Tests statement heuristics distinguishing executable C++ statements from non-code."""
+
+ def test_statement_heuristics(self):
+ """Verifies statement classification across distinct C/C++ syntactic forms."""
+ test_cases = [
+ ("int x = 42;", True, "Variable assignment"),
+ ("x += y;", True, "Compound arithmetic assignment"),
("return result;", True, "Return statement"),
- ("if (x > 0) {", True, "Condition start"),
- ("foo();", True, "Function call"),
- (" // This is a comment", False, "Line comment"),
- ("/* Block comment */", False, "Block comment line"),
- ("#define LIBC_INLINE inline", False, "Preprocessor macro"),
- ("namespace LIBC_NAMESPACE {", False, "Namespace declaration"),
- ("};", False, "Struct/Class terminator"),
- (" {", False, "Opening scope"),
- (" }", False, "Closing scope"),
- ("[[maybe_unused]] int x;", True, "Attribute with variable"),
+ ("if (x > 0) {", True, "Branch condition header"),
+ ("for (size_t i = 0; i < count; ++i) {", True, "For loop header"),
+ ("do_work(a, b);", True, "Function call"),
+ ("struct Point p = {1, 2};", True, "Struct variable assignment"),
+ ("int x = 42; // assignment", True, "Statement with line comment"),
+ ("int x = 42; /* inline comment */", True, "Statement with block comment"),
+ ("// Single line comment", False, "Line comment"),
+ ("/* Block comment start", False, "Block comment start"),
+ (" * Continuation line", False, "Block comment middle"),
+ (" */", False, "Block comment end"),
+ (
+ "} // namespace LIBC_NAMESPACE_DECL",
+ False,
+ "Brace with namespace comment",
+ ),
+ ("}; // struct Point", False, "Scope end with comment"),
+ ("{ // begin loop", False, "Opening brace with comment"),
+ ("} /* namespace */", False, "Brace with block comment"),
+ ("public:", False, "Access specifier"),
+ ("private: // methods", False, "Access specifier with comment"),
+ (
+ 'static_assert(sizeof(long) == 8, "msg");',
+ False,
+ "Compile-time assertion",
+ ),
+ ("friend class Peer;", False, "Friend declaration"),
+ ("{", False, "Opening brace"),
+ ("}", False, "Closing brace"),
+ ("};", False, "Scope terminator"),
+ (": value_(0) {", False, "Constructor initializer header"),
+ ("#include <stddef.h>", False, "Preprocessor include"),
+ ("namespace LIBC_NAMESPACE {", False, "Namespace definition"),
+ ("using size_t = unsigned long;", False, "Type alias"),
+ ("struct ListNode;", False, "Forward struct declaration"),
+ ("enum class Status : uint8_t {", False, "Enum definition header"),
("", False, "Empty line"),
+ (" ", False, "Whitespace line"),
]
- for input_string, expected, description in matrix:
- with self.subTest(msg=description, input_string=input_string):
- self.assertEqual(is_executable_line(input_string), expected)
+ for line, expected, desc in test_cases:
+ with self.subTest(msg=desc, line=line):
+ self.assertEqual(is_executable_line(line), expected)
+
+
+class TestFormatLineRanges(unittest.TestCase):
+ """Tests line number set formatting into concise span representations."""
+
+ def test_formatting_spans(self):
+ """Line number sets must format as empty, single, contiguous, or disjoint spans."""
+ self.assertEqual(format_line_ranges(set()), "None")
+ self.assertEqual(format_line_ranges({42}), "`L42`")
+ self.assertEqual(format_line_ranges({10, 11, 12}), "`L10-L12`")
+ disjoint = {1, 2, 5, 8, 9, 100}
+ self.assertEqual(format_line_ranges(disjoint), "`L1-L2`, `L5`, `L8-L9`, `L100`")
class TestDiffParser(unittest.TestCase):
- def test_parse_diff_hunks_and_lines(self):
- mock_diff = (
+ """Tests Unified Diff parsing across single/multiple hunks, creations, and deletions."""
+
+ def test_parse_diff_hunks(self):
+ """DiffParser must extract added and context lines across hunks while ignoring deletions."""
+ diff_text = (
"diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
"--- a/src/math/sin.cpp\n"
"+++ b/src/math/sin.cpp\n"
"@@ -10,3 +10,4 @@\n"
- " context_line_1();\n"
- "+added_line_1();\n"
- "+added_line_2();\n"
- " context_line_2();\n"
+ " ctx1();\n"
+ "-deleted();\n"
+ "+added1();\n"
+ "+added2();\n"
+ "@@ -50,1 +51,2 @@\n"
+ " ctx2();\n"
+ "+added3();\n"
)
- hunks_dict = DiffParser.parse(mock_diff)
- self.assertIn("src/math/sin.cpp", hunks_dict)
- hunks = hunks_dict["src/math/sin.cpp"]
- self.assertEqual(len(hunks), 1)
-
- hunk = hunks[0]
- self.assertEqual(hunk.header, "@@ -10,3 +10,4 @@")
- added_lines = [line for line in hunk.lines if line[0] == "+"]
- self.assertEqual(len(added_lines), 2)
- self.assertEqual(added_lines[0], ("+", "added_line_1();", 11))
- self.assertEqual(added_lines[1], ("+", "added_line_2();", 12))
-
-
-class TestSegmentIntersection(unittest.TestCase):
- def test_calculate_patch_statistics_exact_match(self):
- mock_diff = (
- "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
- "--- a/src/math/sin.cpp\n"
- "+++ b/src/math/sin.cpp\n"
- "@@ -10,3 +10,2 @@\n"
- " context();\n"
- "+return 0;\n"
+ parsed = DiffParser.parse(diff_text)
+ self.assertIn("src/math/sin.cpp", parsed)
+ hunks = parsed["src/math/sin.cpp"]
+ self.assertEqual(len(hunks), 2)
+ added_hunk1 = [line for line in hunks[0].lines if line[0] == "+"]
+ added_hunk2 = [line for line in hunks[1].lines if line[0] == "+"]
+ self.assertEqual(added_hunk1, [("+", "added1();", 11), ("+", "added2();", 12)])
+ self.assertEqual(added_hunk2, [("+", "added3();", 52)])
+
+ def test_parse_special_files(self):
+ """Newly created files must start at line 1, deleted files and headers must be skipped."""
+ diff_text = (
+ "diff --git a/src/math/new.cpp b/src/math/new.cpp\n"
+ "new file mode 100644\n"
+ "--- /dev/null\n"
+ "+++ b/src/math/new.cpp\n"
+ "index 0000..1111\n"
+ "@@ -0,0 +1,1 @@\n"
+ "+int new_func();\n"
+ "diff --git a/src/math/old.cpp b/src/math/old.cpp\n"
+ "--- a/src/math/old.cpp\n"
+ "+++ /dev/null\n"
+ "@@ -1,1 +0,0 @@\n"
+ "-deleted();\n"
+ )
+ parsed = DiffParser.parse(diff_text)
+ self.assertIn("src/math/new.cpp", parsed)
+ self.assertNotIn("src/math/old.cpp", parsed)
+ self.assertEqual(
+ parsed["src/math/new.cpp"][0].lines[0], ("+", "int new_func();", 1)
)
- mock_json = {
+ self.assertEqual(DiffParser.parse(""), {})
+
+ def test_parse_from_disk_file(self):
+ """DiffParser must successfully read and parse diff files from disk."""
+ diff_text = "diff --git a/a b/b\n+++ b/src/math/f.cpp\n@@ -1,1 +1,2 @@\n ctx();\n+line();\n"
+ with tempfile.NamedTemporaryFile(mode="w", suffix=".diff", delete=False) as tmp:
+ tmp.write(diff_text)
+ tmp_path = tmp.name
+ try:
+ parsed = DiffParser.parse(tmp_path)
+ self.assertIn("src/math/f.cpp", parsed)
+ finally:
+ if os.path.exists(tmp_path):
+ os.remove(tmp_path)
+
+
+class TestCoverageJSONParser(unittest.TestCase):
+ """Tests JSON parsing, segment expansion, MC/DC extraction, and loader safeguards."""
+
+ def test_segment_expansion_and_path_normalization(self):
+ """Segments spanning multiple lines must mark each line covered, uncounted must skip."""
+ diff_files = {"libc/src/math/sin.cpp": [], "src/math/cos.cpp": []}
+ json_data = {
"data": [
{
"files": [
{
- "filename": "/workspace/src/math/sin.cpp",
- "segments": [[10, 0, 1, 1, 1], [11, 0, 1, 1, 1]],
- "branches": [],
- }
+ "filename": "/runner/work/llvm-project/libc/src/math/sin.cpp",
+ "segments": [
+ [10, 0, 5, 1, 1],
+ [13, 0, 0, 1, 1],
+ [20, 0, 0, 0, 1],
+ ],
+ "mcdc_records": [],
+ },
+ {
+ # Tests target_path.endswith("/" + file_name)
+ "filename": "cos.cpp",
+ "segments": [[5, 0, 1, 1, 1]],
+ "mcdc_records": [],
+ },
+ {
+ # Unmatched file: tests continue branch
+ "filename": "/runner/work/llvm-project/libc/src/math/other.cpp",
+ "segments": [[1, 0, 1, 1, 1]],
+ "mcdc_records": [],
+ },
]
}
]
}
+ matrix = CoverageJSONParser.extract_patch_matrix(json_data, diff_files)
+ covered = matrix["libc/src/math/sin.cpp"]["covered"]
+ missed = matrix["libc/src/math/sin.cpp"]["missed"]
+ self.assertEqual(covered, {10, 11, 12})
+ self.assertIn(13, missed)
+ self.assertIn(19, missed)
+ self.assertNotIn(20, missed)
- diff_files = DiffParser.parse(mock_diff)
- coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
- summary = calculate_patch_statistics(diff_files, coverage_matrix)
+ def test_mcdc_records_extraction(self):
+ """Valid MC/DC records must be extracted; truncated or empty records must be ignored."""
+ diff_files = {"src/math/sin.cpp": []}
+ json_data = _make_cov_json(
+ filename="/workspace/src/math/sin.cpp",
+ segments=[[10, 0, 1, 1, 1]],
+ mcdc_records=[
+ [10, 4, 10, 14, 0, 0, 0, 0, 0, [True, False]], # Valid
+ [11, 4, 11, 14], # Truncated (< 10)
+ [12, 4, 12, 14, 0, 0, 0, 0, 0, []], # Empty condition vector
+ ],
+ )
+ matrix = CoverageJSONParser.extract_patch_matrix(json_data, diff_files)
+ decisions = matrix["src/math/sin.cpp"]["mcdc_decisions"]
+ self.assertEqual(len(decisions), 1)
+ self.assertEqual(decisions[0]["line_start"], 10)
+ self.assertEqual(decisions[0]["covered"], 1)
+ self.assertEqual(decisions[0]["total"], 2)
+
+ def test_load_and_empty_payload_safeguards(self):
+ """CoverageJSONParser must load valid JSON, fallback on empty data, and exit on error."""
+ diff_files = {"src/math/sin.cpp": []}
+ empty_matrix = CoverageJSONParser.extract_patch_matrix({}, diff_files)
+ self.assertEqual(len(empty_matrix["src/math/sin.cpp"]["covered"]), 0)
+
+ with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp:
+ json.dump({"key": "val"}, tmp)
+ tmp_path = tmp.name
+ try:
+ self.assertEqual(CoverageJSONParser.load(tmp_path), {"key": "val"})
+ finally:
+ if os.path.exists(tmp_path):
+ os.remove(tmp_path)
- self.assertEqual(summary.total_lines, 1)
- self.assertEqual(summary.total_covered_lines, 1)
- self.assertEqual(summary.total_missed_lines, 0)
+ stderr_buf = io.StringIO()
+ with patch("sys.stderr", stderr_buf):
+ with self.assertRaises(SystemExit):
+ CoverageJSONParser.load("/nonexistent/cov.json")
+ self.assertIn("Error: Failed to parse coverage JSON", stderr_buf.getvalue())
- def test_calculate_patch_statistics_missed_line(self):
- mock_diff = (
+
+class TestCalculatePatchStatistics(unittest.TestCase):
+ """Tests correlating patch lines against coverage segments and MC/DC truth tables."""
+
+ def test_calculate_patch_statistics(self):
+ """Covered lines take precedence, uninstrumented files miss, and MC/DC diagnoses unverified."""
+ diff_text = (
"diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
"--- a/src/math/sin.cpp\n"
"+++ b/src/math/sin.cpp\n"
- "@@ -10,3 +10,2 @@\n"
- " context();\n"
- "+return 0;\n"
+ "@@ -10,2 +10,3 @@\n"
+ " ctx();\n"
+ "+int covered_and_missed = 1;\n"
+ "+if (a && b) return 1;\n"
+ "diff --git a/src/math/untested.cpp b/src/math/untested.cpp\n"
+ "--- a/src/math/untested.cpp\n"
+ "+++ b/src/math/untested.cpp\n"
+ "@@ -1,1 +1,2 @@\n"
+ " ctx();\n"
+ "+int untested = 1;\n"
)
- mock_json = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/src/math/sin.cpp",
- "segments": [[10, 0, 1, 1, 1], [11, 0, 0, 1, 1]],
- "branches": [],
- }
- ]
- }
- ]
+ diff_files = DiffParser.parse(diff_text)
+ coverage_matrix = {
+ "src/math/sin.cpp": {
+ "covered": {11, 12},
+ "missed": {11}, # Covered takes precedence
+ "mcdc_decisions": [
+ {
+ "line_start": 12,
+ "line_end": 12,
+ "conditions": [True, False],
+ "covered": 1,
+ "total": 2,
+ }
+ ],
+ },
+ "src/math/untested.cpp": {
+ "covered": set(),
+ "missed": set(),
+ "mcdc_decisions": [],
+ },
}
+ summary = calculate_patch_statistics(diff_files, coverage_matrix)
+ self.assertEqual(summary.total_lines, 3)
+ self.assertEqual(summary.total_covered_lines, 2)
+ self.assertEqual(summary.total_missed_lines, 1)
+ self.assertEqual(summary.total_mcdc_covered_conditions, 1)
+ self.assertEqual(summary.total_mcdc_total_conditions, 2)
+
+ sin_metric = summary.files["src/math/sin.cpp"]
+ self.assertIn(
+ "1/2 verified (C2 unverified)", sin_metric.condition_diagnostics[0]
+ )
+ self.assertEqual(sin_metric.unverified_decision_lines[12], ["C2"])
- diff_files = DiffParser.parse(mock_diff)
- coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
+ def test_non_source_and_comment_files_skipped(self):
+ """Test files, documentation, and files with only comment additions must be skipped."""
+ diff_text = (
+ "diff --git a/libc/test/src/math/sin_test.cpp b/libc/test/src/math/sin_test.cpp\n"
+ "+++ b/libc/test/src/math/sin_test.cpp\n"
+ "@@ -1,1 +1,2 @@\n"
+ "+TEST(Foo, Bar) {}\n"
+ "diff --git a/src/math/comment_only.cpp b/src/math/comment_only.cpp\n"
+ "+++ b/src/math/comment_only.cpp\n"
+ "@@ -1,1 +1,2 @@\n"
+ "+// comment only\n"
+ )
+ diff_files = DiffParser.parse(diff_text)
+ coverage_matrix = {
+ "libc/test/src/math/sin_test.cpp": {
+ "covered": set(),
+ "missed": set(),
+ "mcdc_decisions": [],
+ },
+ "src/math/comment_only.cpp": {
+ "covered": set(),
+ "missed": set(),
+ "mcdc_decisions": [],
+ },
+ }
summary = calculate_patch_statistics(diff_files, coverage_matrix)
+ self.assertEqual(summary.total_lines, 0)
+ self.assertEqual(len(summary.files), 0)
- self.assertEqual(summary.total_lines, 1)
- self.assertEqual(summary.total_covered_lines, 0)
- self.assertEqual(summary.total_missed_lines, 1)
+class TestPatchReportFormatting(unittest.TestCase):
+ """Tests Markdown formatting across status banners, metadata, tables, and annotated diffs."""
-class TestPatchReportRendering(unittest.TestCase):
- def test_format_status_banner(self):
- summary = PatchCoverageSummary(
- total_covered_lines=50,
- total_missed_lines=50,
- total_mcdc_total_conditions=10,
- total_mcdc_covered_conditions=5,
- files={},
+ def test_format_status_banner_variants(self):
+ """Verifies phrasing across all 5 status banner operational conditions."""
+ cases = [
+ (
+ PatchCoverageSummary(total_covered_lines=5, total_missed_lines=0),
+ "All **5** newly added",
+ ),
+ (
+ PatchCoverageSummary(
+ total_covered_lines=5,
+ total_missed_lines=0,
+ total_mcdc_covered_conditions=2,
+ total_mcdc_total_conditions=2,
+ total_decisions_count=1,
+ fully_verified_decisions=1,
+ ),
+ "All **5** executable lines and **2** boolean conditions",
+ ),
+ (
+ PatchCoverageSummary(
+ total_covered_lines=5,
+ total_missed_lines=0,
+ total_mcdc_covered_conditions=1,
+ total_mcdc_total_conditions=2,
+ total_decisions_count=1,
+ fully_verified_decisions=0,
+ ),
+ "Executed **5 / 5** lines. **1 / 2** boolean conditions",
+ ),
+ (
+ PatchCoverageSummary(total_covered_lines=4, total_missed_lines=1),
+ "Executed **4 / 5** lines (**1** unexecuted",
+ ),
+ (
+ PatchCoverageSummary(
+ total_covered_lines=4,
+ total_missed_lines=1,
+ total_mcdc_covered_conditions=1,
+ total_mcdc_total_conditions=2,
+ total_decisions_count=1,
+ fully_verified_decisions=0,
+ ),
+ "(**1** unexecuted lines detected in patch).",
+ ),
+ ]
+ for summary, expected in cases:
+ with self.subTest(expected=expected):
+ self.assertIn(expected, format_status_banner(summary))
+
+ def test_format_metadata_section(self):
+ """Metadata section must format commits, tests, or return empty on missing arguments."""
+ metadata = format_metadata_section(
+ "1111111", "2222222", "main", "patch", "test_target"
)
- banner = format_status_banner(summary)
- self.assertIn("### Patch Coverage:", banner)
- self.assertIn("50.00% Line", banner)
+ self.assertIn("Base Branch", metadata)
+ self.assertIn("`test_target`", metadata)
+ self.assertEqual(format_metadata_section(None, None, None, None), "")
def test_format_breakdown_table(self):
- file_stat = FilePatchMetrics(
+ """Breakdown tables must render line/MCDC stats and normalize paths to libc/ on GitHub."""
+ file_mcdc = FilePatchMetrics(
file_path="src/math/sin.cpp",
- covered_lines={10, 11},
+ covered_lines={10},
missed_lines=set(),
- added_lines={10, 11},
+ added_lines={10},
+ mcdc_covered_conditions=2,
+ mcdc_total_conditions=2,
+ decisions_verified=1,
+ decisions_total=1,
+ condition_diagnostics=["`L10`: 2/2 verified"],
+ )
+ file_no_mcdc = FilePatchMetrics(
+ file_path="src/string/strlen.cpp",
+ covered_lines={20},
+ missed_lines={21},
+ added_lines={20, 21},
)
summary = PatchCoverageSummary(
total_covered_lines=2,
- total_missed_lines=0,
- files={"src/math/sin.cpp": file_stat},
+ total_missed_lines=1,
+ total_mcdc_covered_conditions=2,
+ total_mcdc_total_conditions=2,
+ fully_verified_decisions=1,
+ total_decisions_count=1,
+ files={
+ "src/math/sin.cpp": file_mcdc,
+ "src/string/strlen.cpp": file_no_mcdc,
+ },
)
- report = format_breakdown_table(summary)
- self.assertIn("[`src/math/sin.cpp`]", report)
- self.assertIn("**100.00%**", report)
+ table = format_breakdown_table(summary, head_commit_sha="abcd123")
+ self.assertIn("blob/abcd123/libc/src/math/sin.cpp", table)
+ self.assertIn("MC/DC Conditions", table)
+ self.assertIn("N/A | N/A", table) # strlen has no MC/DC
-
-class TestDiffParserEdgeCases(unittest.TestCase):
- def test_deleted_file(self):
- mock_diff = (
- "diff --git a/src/math/old.cpp b/src/math/old.cpp\n"
- "deleted file mode 100644\n"
- "--- a/src/math/old.cpp\n"
- "+++ /dev/null\n"
- "@@ -1,3 +0,0 @@\n"
- "-deleted_line_1();\n"
- "-deleted_line_2();\n"
- )
- hunks_dict = DiffParser.parse(mock_diff)
- self.assertEqual(len(hunks_dict), 0)
-
- def test_no_newline_at_eof(self):
- mock_diff = (
+ def test_format_annotated_diff(self):
+ """Annotated diff must output covered, missed, partial MC/DC, non-executable, and context lines."""
+ diff_text = (
"diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
- "--- a/src/math/sin.cpp\n"
"+++ b/src/math/sin.cpp\n"
- "@@ -1,2 +1,3 @@\n"
- " context();\n"
- "+new_line();\n"
- "\\ No newline at end of file\n"
+ "@@ -10,4 +10,5 @@\n"
+ " ctx();\n"
+ "+covered();\n"
+ "+missed();\n"
+ "+if (a && b) {}\n"
+ "+{\n"
)
- hunks_dict = DiffParser.parse(mock_diff)
- hunks = hunks_dict["src/math/sin.cpp"]
- added_lines = [line for line in hunks[0].lines if line[0] == "+"]
- self.assertEqual(len(added_lines), 1)
-
-
-class TestPathResolution(unittest.TestCase):
- def test_fuzzy_path_matching(self):
- diff_files = {"libc/src/math/sin.cpp": []}
- mock_json = {
- "data": [
- {
- "files": [
- {
- "filename": (
- "/home/runner/work/llvm-project/libc/src/math/sin.cpp"
- ),
- "segments": [[10, 0, 1, 1, 1]],
- "mcdc_records": [],
- }
- ]
- }
- ]
- }
- coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
- self.assertIn("libc/src/math/sin.cpp", coverage_matrix)
+ diff_files = DiffParser.parse(diff_text)
+ file_metrics = FilePatchMetrics(
+ file_path="src/math/sin.cpp",
+ covered_lines={11, 13},
+ missed_lines={12},
+ unverified_decision_lines={13: ["C2"]},
+ )
+ summary = PatchCoverageSummary(files={"src/math/sin.cpp": file_metrics})
+ annotated = format_annotated_diff(summary, diff_files)
+ self.assertIn(" ctx();", annotated)
+ self.assertIn("+ covered();", annotated)
+ self.assertIn("- missed(); // [MISSED]", annotated)
+ self.assertIn("! if (a && b) {} // [PARTIAL MC/DC: C2 unverified]", annotated)
+ self.assertIn(" {", annotated)
-class TestZeroStateBoundaries(unittest.TestCase):
- def test_empty_json_data(self):
- diff_files = {"src/math/sin.cpp": []}
- mock_json = {"data": []}
- coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
- self.assertIn("src/math/sin.cpp", coverage_matrix)
- self.assertEqual(len(coverage_matrix["src/math/sin.cpp"]["covered"]), 0)
+class TestRenderPatchReportEndToEnd(unittest.TestCase):
+ """Tests full Markdown report composition from inputs to stdout."""
- def test_purely_cosmetic_patch(self):
- mock_diff = (
- "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
- "--- a/src/math/sin.cpp\n"
- "+++ b/src/math/sin.cpp\n"
- "@@ -10,3 +10,2 @@\n"
- " context();\n"
- "+// Just a comment\n"
+ def test_render_empty_diff(self):
+ """Empty diff must render coverage notice without failing."""
+ buf = io.StringIO()
+ with redirect_stdout(buf):
+ render_patch_report({}, {}, "1111", "2222", "main", "feature")
+ self.assertIn(
+ "No executable lines were added or modified in this patch.", buf.getvalue()
)
- diff_files = DiffParser.parse(mock_diff)
+
+ def test_render_full_report_with_mcdc(self):
+ """Patch report with MC/DC must display the MC/DC report title and full tables."""
+ diff_text = "diff --git a/src/math/f.cpp b/src/math/f.cpp\n+++ b/src/math/f.cpp\n@@ -1,1 +1,2 @@\n ctx();\n+return a && b;\n"
+ diff_files = DiffParser.parse(diff_text)
coverage_matrix = {
- "src/math/sin.cpp": {
- "covered": set(),
+ "src/math/f.cpp": {
+ "covered": {2},
"missed": set(),
- "mcdc_decisions": [],
+ "mcdc_decisions": [
+ {
+ "line_start": 2,
+ "line_end": 2,
+ "conditions": [True, True],
+ "covered": 2,
+ "total": 2,
+ }
+ ],
}
}
- summary = calculate_patch_statistics(diff_files, coverage_matrix)
- self.assertEqual(summary.total_lines, 0)
- self.assertEqual(summary.line_coverage_percentage, 0.0)
+ buf = io.StringIO()
+ with redirect_stdout(buf):
+ render_patch_report(
+ diff_files, coverage_matrix, "1111", "2222", "main", "feature"
+ )
+ output = buf.getvalue()
+ self.assertIn("## LLVM-libc MC/DC Patch Coverage Report", output)
+ self.assertIn(
+ "### Patch Coverage: **100.00% Line** | **100.00% MC/DC**", output
+ )
+ self.assertIn("View Annotated Patch Diff", output)
-class TestMCDCIntersection(unittest.TestCase):
- def test_mcdc_boolean_extraction(self):
- mock_diff = (
- "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
- "--- a/src/math/sin.cpp\n"
- "+++ b/src/math/sin.cpp\n"
- "@@ -10,3 +10,2 @@\n"
- " context();\n"
- "+if (a && b) { return 0; }\n"
- )
- mock_json = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/src/math/sin.cpp",
- "segments": [[11, 0, 1, 1, 1]],
- "mcdc_records": [
- [11, 4, 11, 14, 0, 0, 0, 0, 0, [True, False]]
- ],
- }
- ]
- }
- ]
- }
- diff_files = DiffParser.parse(mock_diff)
- coverage_matrix = CoverageJSONParser.extract_patch_matrix(mock_json, diff_files)
- summary = calculate_patch_statistics(diff_files, coverage_matrix)
+class TestCommandLineInterfaceDiff(unittest.TestCase):
+ """Tests CLI execution and file validation safeguards."""
- self.assertEqual(summary.total_mcdc_total_conditions, 2)
- self.assertEqual(summary.total_mcdc_covered_conditions, 1)
+ def test_cli_execution(self):
+ """CLI must read diff and JSON files from disk and print the report to stdout."""
+ diff_text = "diff --git a/src/math/s.cpp b/src/math/s.cpp\n+++ b/src/math/s.cpp\n@@ -1,1 +1,2 @@\n ctx();\n+int x = 1;\n"
+ json_data = _make_cov_json("/workspace/src/math/s.cpp", [[2, 0, 1, 1, 1]])
+
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".diff", delete=False
+ ) as f_diff:
+ f_diff.write(diff_text)
+ path_diff = f_diff.name
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".json", delete=False
+ ) as f_json:
+ json.dump(json_data, f_json)
+ path_json = f_json.name
+
+ try:
+ buf = io.StringIO()
+ with patch.object(
+ sys,
+ "argv",
+ [
+ "diff_coverage.py",
+ path_diff,
+ path_json,
+ "111",
+ "222",
+ "m",
+ "f",
+ "target",
+ ],
+ ):
+ with redirect_stdout(buf):
+ main()
+ self.assertIn("## LLVM-libc Patch Coverage Report", buf.getvalue())
+ self.assertIn("[`src/math/s.cpp`]", buf.getvalue())
+ finally:
+ if os.path.exists(path_diff):
+ os.remove(path_diff)
+ if os.path.exists(path_json):
+ os.remove(path_json)
+
+ def test_cli_missing_files_exit(self):
+ """CLI must exit with code 1 when diff file or JSON file is missing."""
+ stderr_buf = io.StringIO()
+ with patch.object(
+ sys, "argv", ["diff_coverage.py", "/missing.diff", "/missing.json"]
+ ):
+ with patch("sys.stderr", stderr_buf):
+ with self.assertRaises(SystemExit):
+ main()
+ self.assertIn("Error: Diff file not found", stderr_buf.getvalue())
if __name__ == "__main__":
>From 6dc129e0bf2b94b4d112e4a0681857690d847247 Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Mon, 7 Sep 2026 09:40:46 +0000
Subject: [PATCH 12/15] [libc][ci] Consolidate coverage workflows and
standardize test suites
Merge full and patch coverage workflows into matrix jobs across line and
MC/DC coverage, eliminating duplicate workflow definitions. Rename unit
test files to patch_coverage_test.py and full_coverage_test.py, and remove
synthetic commit SHAs, dummy branch names, and mock repository fixtures
from test cases. Remove redundant Node.js environment settings and ensure
strict compliance with darker, Black, and git-clang-format.
Assisted-by: Automated tooling, human reviewed.
---
.github/workflows/libc-full-coverage.yml | 42 +-
.github/workflows/libc-full-mcdc.yml | 157 ------
.github/workflows/libc-patch-coverage.yml | 39 +-
.github/workflows/libc-patch-mcdc.yml | 298 ----------
...base_coverage.py => full_coverage_test.py} | 487 ++++++++---------
...iff_coverage.py => patch_coverage_test.py} | 514 ++++++++++--------
6 files changed, 577 insertions(+), 960 deletions(-)
delete mode 100644 .github/workflows/libc-full-mcdc.yml
delete mode 100644 .github/workflows/libc-patch-mcdc.yml
rename libc/utils/coverage/{test_codebase_coverage.py => full_coverage_test.py} (62%)
rename libc/utils/coverage/{test_diff_coverage.py => patch_coverage_test.py} (56%)
diff --git a/.github/workflows/libc-full-coverage.yml b/.github/workflows/libc-full-coverage.yml
index 7604907b75d69..11a97471b53b0 100644
--- a/.github/workflows/libc-full-coverage.yml
+++ b/.github/workflows/libc-full-coverage.yml
@@ -3,9 +3,6 @@ name: Libc Full Codebase Coverage
permissions:
contents: read
-env:
- FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
-
on:
# Daily overnight run at 02:00 UTC
schedule:
@@ -29,13 +26,31 @@ concurrency:
jobs:
full-coverage:
timeout-minutes: 60
- name: libc-full-coverage
+ name: libc-full-coverage (${{ matrix.name }})
runs-on: ubuntu-24.04
container:
image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
defaults:
run:
shell: bash
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: Line Coverage
+ enable_mcdc: "OFF"
+ llvm_cov_show_flags: ""
+ sccache_key: libc_coverage_unified_v2_x86_64
+ summary_artifact: libc-coverage-summary
+ html_artifact: libc-coverage-html
+ report_title: "LLVM-libc Coverage Report"
+ - name: MC/DC Coverage
+ enable_mcdc: "ON"
+ llvm_cov_show_flags: "--show-mcdc --show-mcdc-summary"
+ sccache_key: libc_mcdc_coverage_unified_x86_64
+ summary_artifact: libc-mcdc-coverage-summary
+ html_artifact: libc-mcdc-coverage-html
+ report_title: "LLVM-libc MC/DC Coverage Report"
steps:
- name: Checkout Code
uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -47,7 +62,7 @@ jobs:
uses: hendrikmuhs/ccache-action at d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
with:
max-size: 1G
- key: libc_coverage_unified_v2_x86_64
+ key: ${{ matrix.sccache_key }}
variant: sccache
- name: Configure CMake
@@ -65,6 +80,7 @@ jobs:
-DLLVM_ENABLE_RUNTIMES=libc
-DLLVM_LIBC_FULL_BUILD=ON
-DLIBC_ENABLE_COVERAGE=ON
+ -DLIBC_ENABLE_MCDC=${{ matrix.enable_mcdc }}
-DLIBC_TEST_UNIT_TEST_ONLY=ON
-DLIBC_TEST_SKIP_DEATH_TESTS=ON
-DLIBC_TEST_SKIP_SHARED_TESTS=ON
@@ -101,7 +117,7 @@ jobs:
if [ ! -f libc_full.profdata ] || [ ${#EXECUTABLES[@]} -eq 0 ]; then
echo "[LOG] Warning: Profile data or test executables missing. Skipping report generation."
- echo "### LLVM-libc Coverage Report" >> $GITHUB_STEP_SUMMARY
+ echo "### ${{ matrix.report_title }}" >> $GITHUB_STEP_SUMMARY
echo "No coverage data was generated for this run." >> $GITHUB_STEP_SUMMARY
exit 0
fi
@@ -116,10 +132,12 @@ jobs:
"${EXECUTABLES[0]}" "${OBJECTS[@]}" \
--show-directory-coverage \
--show-branches=count \
+ ${{ matrix.llvm_cov_show_flags }} \
--compilation-dir=. \
--path-equivalence="$GITHUB_WORKSPACE,." \
-ignore-filename-regex=".*(test|utils).*"
- # 3. Run Codebase Coverage Analyzer
+
+ # 3. Run Full Codebase Coverage Analyzer
PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" > coverage_summary.md
cat coverage_summary.md >> $GITHUB_STEP_SUMMARY
echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
@@ -127,14 +145,14 @@ jobs:
- name: Fallback Failure Summary
if: failure() && !hashFiles('coverage.json')
run: |
- echo "### LLVM-libc Coverage Report" >> $GITHUB_STEP_SUMMARY
+ echo "### ${{ matrix.report_title }}" >> $GITHUB_STEP_SUMMARY
echo "Workflow run encountered an error before coverage reports could be generated. Inspect job logs for details." >> $GITHUB_STEP_SUMMARY
- name: Upload Coverage Summary Artifact
- if: always() && hashFiles('coverage_summary.md')
+ if: ${{ !cancelled() && hashFiles('coverage_summary.md') }}
uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: libc-coverage-summary
+ name: ${{ matrix.summary_artifact }}
path: |
coverage_summary.md
coverage.json
@@ -143,10 +161,10 @@ jobs:
overwrite: true
- name: Upload HTML Coverage Artifact
- if: always() && hashFiles('coverage_html/**')
+ if: ${{ !cancelled() && hashFiles('coverage_html/**') }}
uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: libc-coverage-html
+ name: ${{ matrix.html_artifact }}
path: coverage_html/
retention-days: 14
overwrite: true
diff --git a/.github/workflows/libc-full-mcdc.yml b/.github/workflows/libc-full-mcdc.yml
deleted file mode 100644
index 9aa6d96943891..0000000000000
--- a/.github/workflows/libc-full-mcdc.yml
+++ /dev/null
@@ -1,157 +0,0 @@
-name: Libc Full Codebase MC/DC Coverage
-
-permissions:
- contents: read
-
-env:
- FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
-
-on:
- # Daily overnight run at 02:30 UTC (staggered by 30 mins from baseline sweep)
- schedule:
- - cron: '30 2 * * *'
-
- # Allow manual on-demand execution from GitHub Actions UI ("Run workflow" button)
- workflow_dispatch:
-
- # Trigger on pushes to main
- push:
- branches:
- - main
- paths:
- - 'libc/**'
- - '.github/workflows/libc-full-mcdc.yml'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
- full-mcdc-coverage:
- timeout-minutes: 60
- name: libc-full-mcdc-coverage
- runs-on: ubuntu-24.04
- container:
- image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
- defaults:
- run:
- shell: bash
- steps:
- - name: Checkout Code
- uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- with:
- fetch-depth: 1
- persist-credentials: false
-
- - name: Setup Compiler Cache (sccache)
- uses: hendrikmuhs/ccache-action at d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
- with:
- max-size: 1G
- key: libc_mcdc_coverage_unified_x86_64
- variant: sccache
-
- - name: Configure CMake with MC/DC
- run: |
- export CMAKE_FLAGS="
- -G Ninja
- -S runtimes
- -B build-cov
- -DCMAKE_C_COMPILER=clang-23
- -DCMAKE_CXX_COMPILER=clang++-23
- -DCMAKE_BUILD_TYPE=Debug
- -DCMAKE_C_COMPILER_LAUNCHER=sccache
- -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
- -DLLVM_USE_LINKER=lld-23
- -DLLVM_ENABLE_RUNTIMES=libc
- -DLLVM_LIBC_FULL_BUILD=ON
- -DLIBC_ENABLE_COVERAGE=ON
- -DLIBC_ENABLE_MCDC=ON
- -DLIBC_TEST_UNIT_TEST_ONLY=ON
- -DLIBC_TEST_SKIP_DEATH_TESTS=ON
- -DLIBC_TEST_SKIP_SHARED_TESTS=ON
- "
- cmake $CMAKE_FLAGS
-
- - name: Run Full Codebase Unit Tests
- run: |
- export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
- ninja -k 0 -C build-cov libc-unit-tests || true
-
- - name: Merge Profiles
- run: |
- find . -name "libc_cov_*.profraw" > profraw_list.txt
- NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
- echo "[LOG] Discovered $NUM_PROFS raw profile data files."
- if [ -s profraw_list.txt ]; then
- llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
- echo "[LOG] Merged full codebase profile into libc_full.profdata."
- else
- echo "[LOG] Warning: No profraw files found."
- fi
-
- - name: Generate Reports and Summary
- env:
- COMMIT_SHA: ${{ github.sha }}
- BRANCH_REF: ${{ github.ref_name }}
- run: |
- EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
- OBJECTS=("${EXECUTABLES[@]:1}")
- OBJECTS=("${OBJECTS[@]/#/-object=}")
-
- echo "[LOG] Exporting coverage data across ${#EXECUTABLES[@]} test binaries."
-
- if [ ! -f libc_full.profdata ] || [ ${#EXECUTABLES[@]} -eq 0 ]; then
- echo "[LOG] Warning: Profile data or test executables missing. Skipping report generation."
- echo "### LLVM-libc MC/DC Coverage Report" >> $GITHUB_STEP_SUMMARY
- echo "No coverage data was generated for this run." >> $GITHUB_STEP_SUMMARY
- exit 0
- fi
-
- # 1. Generate JSON export for Full Coverage Analyzer (contains mcdc_records)
- llvm-cov-23 export -format=text -instr-profile=libc_full.profdata "${EXECUTABLES[0]}" "${OBJECTS[@]}" > coverage.json
-
- # 2. Generate HTML Coverage Report with directory hierarchy, branch tracking, and MC/DC truth tables
- llvm-cov-23 show -format=html \
- -output-dir=coverage_mcdc_html \
- -instr-profile=libc_full.profdata \
- "${EXECUTABLES[0]}" "${OBJECTS[@]}" \
- --show-directory-coverage \
- --show-branches=count \
- --show-mcdc \
- --show-mcdc-summary \
- --compilation-dir=. \
- --path-equivalence="$GITHUB_WORKSPACE,." \
- -ignore-filename-regex=".*(test|utils).*"
- # 3. Run Codebase Coverage Analyzer
- PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/codebase_coverage.py coverage.json "$COMMIT_SHA" "$BRANCH_REF" > coverage_summary.md
- cat coverage_summary.md >> $GITHUB_STEP_SUMMARY
- echo "[LOG] Post-commit summary report written to GITHUB_STEP_SUMMARY."
-
- - name: Fallback Failure Summary
- if: failure() && !hashFiles('coverage.json')
- run: |
- echo "### LLVM-libc MC/DC Coverage Report" >> $GITHUB_STEP_SUMMARY
- echo "Workflow run encountered an error before coverage reports could be generated. Inspect job logs for details." >> $GITHUB_STEP_SUMMARY
-
- - name: Upload MC/DC Coverage Summary Artifact
- if: always() && hashFiles('coverage_summary.md')
- uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: libc-mcdc-coverage-summary
- path: |
- coverage_summary.md
- coverage.json
- if-no-files-found: ignore
- retention-days: 14
- overwrite: true
-
- - name: Upload MC/DC Coverage Artifact
- if: always() && hashFiles('coverage_mcdc_html/**')
- uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: libc-mcdc-coverage-html
- path: coverage_mcdc_html/
- retention-days: 14
- overwrite: true
-
-
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
index c5637301575ea..6ed0a368bb281 100644
--- a/.github/workflows/libc-patch-coverage.yml
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -1,11 +1,8 @@
-name: Libc Patch Code Coverage
+name: Libc Patch Coverage
permissions:
contents: read
-env:
- FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
-
on:
workflow_dispatch:
push:
@@ -44,13 +41,27 @@ concurrency:
jobs:
pre-commit-coverage:
timeout-minutes: 60
- name: libc-pre-commit-coverage
+ name: libc-patch-coverage (${{ matrix.name }})
runs-on: ubuntu-24.04
container:
image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
defaults:
run:
shell: bash
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: Line Coverage
+ enable_mcdc: "OFF"
+ sccache_key: libc_coverage_unified_v2_x86_64
+ artifact_name: libc-patch-coverage-report
+ report_title: "LLVM-libc Patch Coverage Report"
+ - name: MC/DC Coverage
+ enable_mcdc: "ON"
+ sccache_key: libc_mcdc_coverage_unified_x86_64
+ artifact_name: libc-patch-mcdc-coverage-report
+ report_title: "LLVM-libc Patch MC/DC Coverage Report"
steps:
- name: Checkout Code
uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -63,7 +74,7 @@ jobs:
uses: hendrikmuhs/ccache-action at d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
with:
max-size: 1G
- key: libc_coverage_unified_v2_x86_64
+ key: ${{ matrix.sccache_key }}
variant: sccache
- name: Configure CMake
@@ -81,6 +92,7 @@ jobs:
-DLLVM_ENABLE_RUNTIMES=libc
-DLLVM_LIBC_FULL_BUILD=ON
-DLIBC_ENABLE_COVERAGE=ON
+ -DLIBC_ENABLE_MCDC=${{ matrix.enable_mcdc }}
-DLIBC_TEST_UNIT_TEST_ONLY=ON
-DLIBC_TEST_SKIP_DEATH_TESTS=ON
-DLIBC_TEST_SKIP_SHARED_TESTS=ON
@@ -159,7 +171,7 @@ jobs:
echo "[LOG] No source files modified in libc/src/. Exiting successfully."
echo "TARGETS=" >> $GITHUB_ENV
- echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "## ${{ matrix.report_title }}" > coverage_report.md
echo "" >> coverage_report.md
echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
@@ -210,7 +222,7 @@ jobs:
echo "[LOG] No standalone unit test targets to run."
echo "TARGETS=" >> $GITHUB_ENV
- echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "## ${{ matrix.report_title }}" > coverage_report.md
echo "" >> coverage_report.md
echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
@@ -246,7 +258,7 @@ jobs:
if [ ! -f libc_full.profdata ] || [ ${#EXECUTABLES[@]} -eq 0 ]; then
echo "[LOG] Notice: Profile data or instrumented executables not found. Generating non-coverage summary."
- echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "## ${{ matrix.report_title }}" > coverage_report.md
echo "" >> coverage_report.md
echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
@@ -264,7 +276,7 @@ jobs:
# 2. Generate git diff for modified libc source files
git diff "$DIFF_BASE" HEAD -- libc/src/ > patch.diff
- # 3. Run Diff Coverage Python Script using persisted commit metadata
+ # 3. Run Patch Coverage Python Script using persisted commit metadata
PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/diff_coverage.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
echo "[LOG] Coverage report successfully written to GITHUB_STEP_SUMMARY."
@@ -272,7 +284,7 @@ jobs:
- name: Fallback Failure Summary
if: failure() && !hashFiles('coverage_report.md')
run: |
- echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
+ echo "## ${{ matrix.report_title }}" > coverage_report.md
echo "" >> coverage_report.md
echo "- **Base Branch:** [\`${BASE_REF:-main}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO:-llvm/llvm-project}/commit/${BASE_SHA:-main})" >> coverage_report.md
echo "- **Head Commit:** [\`${HEAD_REF:-HEAD}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO:-llvm/llvm-project}/commit/${HEAD_SHA:-HEAD})" >> coverage_report.md
@@ -283,12 +295,11 @@ jobs:
cat coverage_report.md >> $GITHUB_STEP_SUMMARY
-
- name: Upload Patch Coverage Artifacts
- if: always() && hashFiles('coverage_report.md')
+ if: ${{ !cancelled() && hashFiles('coverage_report.md') }}
uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: libc-patch-coverage-report
+ name: ${{ matrix.artifact_name }}
path: |
coverage_report.md
patch.diff
diff --git a/.github/workflows/libc-patch-mcdc.yml b/.github/workflows/libc-patch-mcdc.yml
deleted file mode 100644
index c4e6c238e4b8d..0000000000000
--- a/.github/workflows/libc-patch-mcdc.yml
+++ /dev/null
@@ -1,298 +0,0 @@
-name: Libc Patch MC/DC Coverage
-
-permissions:
- contents: read
-
-env:
- FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
-
-on:
- workflow_dispatch:
- push:
- branches:
- - main
- paths:
- - 'libc/src/**'
- - 'libc/include/**'
- - 'libc/test/**'
- - 'libc/CMakeLists.txt'
- - '.github/workflows/libc-patch-mcdc.yml'
- - '!libc/docs/**'
- - '!libc/benchmarks/**'
- - '!libc/fuzzing/**'
- - '!libc/utils/**'
- - '!**.md'
- pull_request:
- branches:
- - main
- paths:
- - 'libc/src/**'
- - 'libc/include/**'
- - 'libc/test/**'
- - 'libc/CMakeLists.txt'
- - '.github/workflows/libc-patch-mcdc.yml'
- - '!libc/docs/**'
- - '!libc/benchmarks/**'
- - '!libc/fuzzing/**'
- - '!libc/utils/**'
- - '!**.md'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
- cancel-in-progress: true
-
-jobs:
- pre-commit-mcdc-coverage:
- timeout-minutes: 60
- name: libc-pre-commit-mcdc-coverage
- runs-on: ubuntu-24.04
- container:
- image: ghcr.io/llvm/libc-ubuntu-24.04:latest at sha256:8fee4c9ce5a1fd095686593cd36e032c48a31b6ae575378c82accd1d86a08d59
- defaults:
- run:
- shell: bash
- steps:
- - name: Checkout Code
- uses: actions/checkout at df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- with:
- ref: ${{ github.event.pull_request.head.sha || github.sha }}
- fetch-depth: 100
- persist-credentials: false
-
- - name: Setup Compiler Cache (sccache)
- uses: hendrikmuhs/ccache-action at d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23
- with:
- max-size: 1G
- key: libc_coverage_unified_v2_x86_64
- variant: sccache
-
- - name: Configure CMake
- run: |
- export CMAKE_FLAGS="
- -G Ninja
- -S runtimes
- -B build-cov
- -DCMAKE_C_COMPILER=clang-23
- -DCMAKE_CXX_COMPILER=clang++-23
- -DCMAKE_BUILD_TYPE=Debug
- -DCMAKE_C_COMPILER_LAUNCHER=sccache
- -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
- -DLLVM_USE_LINKER=lld-23
- -DLLVM_ENABLE_RUNTIMES=libc
- -DLLVM_LIBC_FULL_BUILD=ON
- -DLIBC_ENABLE_COVERAGE=ON
- -DLIBC_ENABLE_MCDC=ON
- -DLIBC_TEST_UNIT_TEST_ONLY=ON
- -DLIBC_TEST_SKIP_DEATH_TESTS=ON
- -DLIBC_TEST_SKIP_SHARED_TESTS=ON
- "
- cmake $CMAKE_FLAGS
-
- - name: Build and Run Targeted Tests
- env:
- EVENT_NAME: ${{ github.event_name }}
- PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
- PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
- PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
- PR_BASE_REPO: ${{ github.event.pull_request.base.repo.full_name }}
- PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
- COMMIT_SHA: ${{ github.sha }}
- BRANCH_REF: ${{ github.ref_name }}
- REPO_NAME: ${{ github.repository }}
- run: |
- # Ensure upstream llvm-project remote is configured
- git remote add upstream https://github.com/llvm/llvm-project.git 2>/dev/null || true
-
- # 1. Deterministically resolve Base and Head references against upstream llvm-project
- if [ "$EVENT_NAME" == "pull_request" ]; then
- BASE_SHA="$PR_BASE_SHA"
- HEAD_SHA="$PR_HEAD_SHA"
- BASE_REF="$PR_BASE_REF"
- HEAD_REF="$PR_HEAD_REF"
- BASE_REPO="${PR_BASE_REPO:-llvm/llvm-project}"
- HEAD_REPO="${PR_HEAD_REPO:-$REPO_NAME}"
- # Fetch base branch history to resolve merge base
- git fetch upstream "$BASE_REF" --depth=100 2>/dev/null || git fetch origin "$BASE_REF" --depth=100 2>/dev/null || git fetch upstream "$BASE_SHA" --depth=1 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 2>/dev/null || true
- DIFF_BASE=$(git merge-base "$BASE_SHA" HEAD 2>/dev/null || true)
- if [ -z "$DIFF_BASE" ]; then
- git fetch --no-tags --deepen=200 2>/dev/null || true
- DIFF_BASE=$(git merge-base "$BASE_SHA" HEAD 2>/dev/null || echo "$BASE_SHA")
- fi
- elif [ "$BRANCH_REF" != "main" ]; then
- # For feature branches, compare against main merge-base
- git fetch origin main --depth=100 2>/dev/null || true
- BASE_SHA=$(git merge-base origin/main HEAD 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || echo "HEAD~1")
- HEAD_SHA="$COMMIT_SHA"
- BASE_REF="main"
- HEAD_REF="$BRANCH_REF"
- BASE_REPO="llvm/llvm-project"
- HEAD_REPO="$REPO_NAME"
- DIFF_BASE="$BASE_SHA"
- else
- BASE_SHA=$(git rev-parse HEAD~1 2>/dev/null || echo "HEAD~1")
- HEAD_SHA="$COMMIT_SHA"
- BASE_REF="main"
- HEAD_REF="$BRANCH_REF"
- BASE_REPO="llvm/llvm-project"
- HEAD_REPO="$REPO_NAME"
- DIFF_BASE="HEAD~1"
- fi
-
- # Persist resolved commit references for all downstream steps
- echo "DIFF_BASE=$DIFF_BASE" >> $GITHUB_ENV
- echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV
- echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_ENV
- echo "BASE_REF=$BASE_REF" >> $GITHUB_ENV
- echo "HEAD_REF=$HEAD_REF" >> $GITHUB_ENV
- echo "BASE_REPO=$BASE_REPO" >> $GITHUB_ENV
- echo "HEAD_REPO=$HEAD_REPO" >> $GITHUB_ENV
-
- echo "[LOG] Resolved Base: ${BASE_REF} (${BASE_SHA:0:7}) in ${BASE_REPO}"
- echo "[LOG] Resolved Head: ${HEAD_REF} (${HEAD_SHA:0:7}) in ${HEAD_REPO}"
-
- MODIFIED_FILES=$(git diff --name-only "$DIFF_BASE" HEAD -- libc/src/ || true)
- echo "[LOG] Modified files in libc/src/:"
- echo "$MODIFIED_FILES"
-
- if [ -z "$MODIFIED_FILES" ]; then
- echo "[LOG] No source files modified in libc/src/. Exiting successfully."
- echo "TARGETS=" >> $GITHUB_ENV
-
- echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
- echo "" >> coverage_report.md
- echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
- echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
- echo "---" >> coverage_report.md
- echo "" >> coverage_report.md
- echo "**Coverage Validated**" >> coverage_report.md
- echo "No \`.cpp\` source files in \`libc/src/\` were modified in this patch." >> coverage_report.md
-
- cat coverage_report.md >> $GITHUB_STEP_SUMMARY
- exit 0
- fi
-
- # Get all valid ninja targets
- ninja -C build-cov -t targets > all_targets.txt
-
- TARGETS=""
-
- for FILE in $MODIFIED_FILES; do
- # Skip core support files from direct target mapping (tested transitively)
- if [[ "$FILE" == *"__support/"* ]]; then
- echo "[LOG] -> Support file ($FILE) will be tested transitively."
- continue
- fi
-
- if [[ "$FILE" =~ ^libc/src/([^/]+)/(.*/)?([^/]+)\.cpp$ ]]; then
- DIR="${BASH_REMATCH[1]}"
- FUNC="${BASH_REMATCH[3]}"
- ALL_MATCHES=$(grep -oE "^libc\.test\.src\.${DIR}\.([a-zA-Z0-9_]+\.)*${FUNC}_test\.__unit__" all_targets.txt || true)
- FOUND_TARGET=$(echo "$ALL_MATCHES" | head -n 1 | tr -d '\r\n ' || true)
- if [ -n "$FOUND_TARGET" ]; then
- echo "[LOG] -> Matched $FILE -> $FOUND_TARGET"
- TARGETS="$TARGETS $FOUND_TARGET"
- else
- echo "[LOG] -> Notice: No direct unit test target found for $FILE."
- fi
- fi
- done
-
- # Remove duplicate targets
- TARGETS=$(echo "$TARGETS" | xargs -n1 | sort -u | xargs || true)
- echo "[LOG] Executing Ninja targets: $TARGETS"
-
- if [ -n "$TARGETS" ]; then
- export LLVM_PROFILE_FILE="libc_cov_%p.profraw"
- ninja -C build-cov $TARGETS
- echo "TARGETS=$TARGETS" >> $GITHUB_ENV
- else
- echo "[LOG] No standalone unit test targets to run."
- echo "TARGETS=" >> $GITHUB_ENV
-
- echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
- echo "" >> coverage_report.md
- echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
- echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
- echo "---" >> coverage_report.md
- echo "" >> coverage_report.md
- echo "**Coverage Validated**" >> coverage_report.md
- echo "Modified files do not have standalone unit test targets." >> coverage_report.md
-
- cat coverage_report.md >> $GITHUB_STEP_SUMMARY
- fi
-
- - name: Merge Profiles
- if: env.TARGETS != ''
- run: |
- find . -name "libc_cov_*.profraw" > profraw_list.txt
- NUM_PROFS=$(wc -l < profraw_list.txt || echo 0)
- echo "[LOG] Discovered $NUM_PROFS raw profile data files."
- if [ -s profraw_list.txt ]; then
- llvm-profdata-23 merge -sparse --input-files=profraw_list.txt -o libc_full.profdata
- echo "[LOG] Successfully merged profile data into libc_full.profdata."
- else
- echo "[LOG] Warning: No profraw files found."
- fi
-
- - name: Extract Executables and Generate Summary
- if: env.TARGETS != ''
- run: |
- EXECUTABLES=($(find build-cov -type f -executable -name "*__build__"))
- OBJECTS=("${EXECUTABLES[@]:1}")
- OBJECTS=("${OBJECTS[@]/#/-object=}")
-
- echo "[LOG] Discovered ${#EXECUTABLES[@]} instrumented binaries for llvm-cov export."
-
- if [ ! -f libc_full.profdata ] || [ ${#EXECUTABLES[@]} -eq 0 ]; then
- echo "[LOG] Notice: Profile data or instrumented executables not found. Generating non-coverage summary."
- echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
- echo "" >> coverage_report.md
- echo "- **Base Branch:** [\`${BASE_REF}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO}/commit/${BASE_SHA})" >> coverage_report.md
- echo "- **Head Commit:** [\`${HEAD_REF}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO}/commit/${HEAD_SHA})" >> coverage_report.md
- echo "---" >> coverage_report.md
- echo "" >> coverage_report.md
- echo "**Coverage Validated**" >> coverage_report.md
- echo "No executable statement profile data was collected for the modified lines in this patch." >> coverage_report.md
- cat coverage_report.md >> $GITHUB_STEP_SUMMARY
- exit 0
- fi
-
- # 1. Generate JSON report for Python script
- llvm-cov-23 export -format=text -instr-profile=libc_full.profdata "${EXECUTABLES[0]}" "${OBJECTS[@]}" > coverage.json
-
- # 2. Generate git diff for modified libc source files
- git diff "$DIFF_BASE" HEAD -- libc/src/ > patch.diff
-
- # 3. Run Diff Coverage Python Script using persisted commit metadata
- PYTHONPATH="libc/utils/coverage" python3 libc/utils/coverage/diff_coverage.py patch.diff coverage.json "$BASE_SHA" "$HEAD_SHA" "$BASE_REF" "$HEAD_REF" "$TARGETS" "$BASE_REPO" "$HEAD_REPO" > coverage_report.md
- cat coverage_report.md >> $GITHUB_STEP_SUMMARY
- echo "[LOG] Coverage report successfully written to GITHUB_STEP_SUMMARY."
-
- - name: Fallback Failure Summary
- if: failure() && !hashFiles('coverage_report.md')
- run: |
- echo "## LLVM-libc Patch Coverage Report" > coverage_report.md
- echo "" >> coverage_report.md
- echo "- **Base Branch:** [\`${BASE_REF:-main}\` (${BASE_SHA:0:7})](https://github.com/${BASE_REPO:-llvm/llvm-project}/commit/${BASE_SHA:-main})" >> coverage_report.md
- echo "- **Head Commit:** [\`${HEAD_REF:-HEAD}\` (${HEAD_SHA:0:7})](https://github.com/${HEAD_REPO:-llvm/llvm-project}/commit/${HEAD_SHA:-HEAD})" >> coverage_report.md
- echo "---" >> coverage_report.md
- echo "" >> coverage_report.md
- echo "**Coverage Validated**" >> coverage_report.md
- echo "The targeted build or test execution encountered an error before coverage data could be finalized. Inspect the job logs for details." >> coverage_report.md
-
- cat coverage_report.md >> $GITHUB_STEP_SUMMARY
-
-
- - name: Upload Patch MC/DC Coverage Artifacts
- if: always() && hashFiles('coverage_report.md')
- uses: actions/upload-artifact at 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: libc-patch-mcdc-coverage-report
- path: |
- coverage_report.md
- patch.diff
- coverage.json
- if-no-files-found: ignore
- retention-days: 14
- overwrite: true
diff --git a/libc/utils/coverage/test_codebase_coverage.py b/libc/utils/coverage/full_coverage_test.py
similarity index 62%
rename from libc/utils/coverage/test_codebase_coverage.py
rename to libc/utils/coverage/full_coverage_test.py
index 9d9d19ee3fad0..080ca3905fbbc 100644
--- a/libc/utils/coverage/test_codebase_coverage.py
+++ b/libc/utils/coverage/full_coverage_test.py
@@ -1,10 +1,13 @@
-# ====- Unit tests for codebase_coverage.py ------------------*- python -*--==#
+# ===- Unit tests for full codebase coverage ----------------*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
-# ==-------------------------------------------------------------------------==#
+# ==------------------------------------------------------------------------==#
+
+# To run these tests:
+# python3 -m unittest full_coverage_test.py
"""Unit tests for codebase_coverage.py."""
@@ -15,6 +18,7 @@
import tempfile
import unittest
from contextlib import redirect_stdout
+from typing import List, Optional
from unittest.mock import patch
# Ensure libc/utils/coverage is in sys.path when running from any working directory
@@ -32,6 +36,36 @@
)
+def _make_file_entry(
+ filename: str,
+ lines_total: int,
+ lines_covered: int,
+ func_total: int = 1,
+ func_covered: int = 1,
+ mcdc_total: int = 0,
+ mcdc_covered: int = 0,
+ mcdc_records: Optional[list] = None,
+) -> dict:
+ """Constructs a single file coverage record for llvm-cov JSON export."""
+ entry = {
+ "filename": filename,
+ "summary": {
+ "lines": {"count": lines_total, "covered": lines_covered},
+ "functions": {"count": func_total, "covered": func_covered},
+ },
+ }
+ if mcdc_total > 0 or mcdc_records:
+ entry["summary"]["mcdc"] = {"count": mcdc_total, "covered": mcdc_covered}
+ if mcdc_records is not None:
+ entry["mcdc_records"] = mcdc_records
+ return entry
+
+
+def _make_codebase_payload(files: List[dict]) -> dict:
+ """Wraps file coverage entries in the top-level llvm-cov JSON export structure."""
+ return {"data": [{"files": files}]}
+
+
class TestDirectoryCoverageMetrics(unittest.TestCase):
"""Tests DirectoryCoverageMetrics mathematical operations and property safeguards."""
@@ -118,99 +152,78 @@ def test_empty_or_invalid_payloads_return_none(self):
def test_all_files_zero_lines_returns_none(self):
"""Payload containing only files with zero total lines must return None."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/src/empty.cpp",
- "summary": {
- "lines": {"count": 0, "covered": 0},
- "functions": {"count": 0, "covered": 0},
- },
- }
- ]
- }
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/src/empty.cpp",
+ lines_total=0,
+ lines_covered=0,
+ func_total=0,
+ func_covered=0,
+ )
]
- }
+ )
self.assertIsNone(extract_full_coverage_statistics(payload))
def test_all_files_excluded_returns_none(self):
"""Payload containing only test or utility files must return None."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/test/src/math/sin_test.cpp",
- "summary": {
- "lines": {"count": 100, "covered": 100},
- "functions": {"count": 1, "covered": 1},
- },
- },
- {
- "filename": "/workspace/libc/utils/MPFRWrapper/MPFRUtils.cpp",
- "summary": {
- "lines": {"count": 200, "covered": 200},
- "functions": {"count": 2, "covered": 2},
- },
- },
- ]
- }
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/test/src/math/sin_test.cpp",
+ lines_total=100,
+ lines_covered=100,
+ ),
+ _make_file_entry(
+ "/workspace/libc/utils/MPFRWrapper/MPFRUtils.cpp",
+ lines_total=200,
+ lines_covered=200,
+ func_total=2,
+ func_covered=2,
+ ),
]
- }
+ )
self.assertIsNone(extract_full_coverage_statistics(payload))
def test_file_path_filtering(self):
"""Test and utility directories must be excluded from codebase coverage."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/src/math/sin.cpp",
- "summary": {
- "lines": {"count": 100, "covered": 80},
- "functions": {"count": 2, "covered": 2},
- },
- },
- {
- # Test file: must be excluded
- "filename": "/workspace/libc/test/src/math/sin_test.cpp",
- "summary": {
- "lines": {"count": 500, "covered": 500},
- "functions": {"count": 5, "covered": 5},
- },
- },
- {
- # Utility file: must be excluded
- "filename": "/workspace/libc/utils/MPFRWrapper/MPFRUtils.cpp",
- "summary": {
- "lines": {"count": 300, "covered": 300},
- "functions": {"count": 4, "covered": 4},
- },
- },
- {
- # Non-src file: must be excluded
- "filename": "/workspace/libc/include/llvm-libc-types/size_t.h",
- "summary": {
- "lines": {"count": 50, "covered": 50},
- "functions": {"count": 1, "covered": 1},
- },
- },
- {
- # File with zero total lines: must be excluded
- "filename": "/workspace/libc/src/empty.cpp",
- "summary": {
- "lines": {"count": 0, "covered": 0},
- "functions": {"count": 0, "covered": 0},
- },
- },
- ]
- }
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/src/math/sin.cpp",
+ lines_total=100,
+ lines_covered=80,
+ func_total=2,
+ func_covered=2,
+ ),
+ _make_file_entry(
+ "/workspace/libc/test/src/math/sin_test.cpp",
+ lines_total=500,
+ lines_covered=500,
+ func_total=5,
+ func_covered=5,
+ ),
+ _make_file_entry(
+ "/workspace/libc/utils/MPFRWrapper/MPFRUtils.cpp",
+ lines_total=300,
+ lines_covered=300,
+ func_total=4,
+ func_covered=4,
+ ),
+ _make_file_entry(
+ "/workspace/libc/include/llvm-libc-types/size_t.h",
+ lines_total=50,
+ lines_covered=50,
+ ),
+ _make_file_entry(
+ "/workspace/libc/src/empty.cpp",
+ lines_total=0,
+ lines_covered=0,
+ func_total=0,
+ func_covered=0,
+ ),
]
- }
-
+ )
summary = extract_full_coverage_statistics(payload)
self.assertIsNotNone(summary)
self.assertEqual(summary.global_stats.lines_tot, 100)
@@ -222,37 +235,31 @@ def test_file_path_filtering(self):
def test_directory_bucketing_and_nested_paths(self):
"""Files within identical top-level directories or deep subpaths must aggregate properly."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/src/math/sin.cpp",
- "summary": {
- "lines": {"count": 100, "covered": 60},
- "functions": {"count": 2, "covered": 1},
- },
- },
- {
- "filename": "/workspace/libc/src/math/cos.cpp",
- "summary": {
- "lines": {"count": 80, "covered": 80},
- "functions": {"count": 2, "covered": 2},
- },
- },
- {
- # Nested subpath under src/string/
- "filename": "/workspace/libc/src/string/memory_utils/op_builtin.cpp",
- "summary": {
- "lines": {"count": 120, "covered": 100},
- "functions": {"count": 4, "covered": 3},
- },
- },
- ]
- }
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/src/math/sin.cpp",
+ lines_total=100,
+ lines_covered=60,
+ func_total=2,
+ func_covered=1,
+ ),
+ _make_file_entry(
+ "/workspace/libc/src/math/cos.cpp",
+ lines_total=80,
+ lines_covered=80,
+ func_total=2,
+ func_covered=2,
+ ),
+ _make_file_entry(
+ "/workspace/libc/src/string/memory_utils/op_builtin.cpp",
+ lines_total=120,
+ lines_covered=100,
+ func_total=4,
+ func_covered=3,
+ ),
]
- }
-
+ )
summary = extract_full_coverage_statistics(payload)
self.assertIsNotNone(summary)
self.assertEqual(summary.global_stats.lines_tot, 300)
@@ -272,31 +279,24 @@ def test_directory_bucketing_and_nested_paths(self):
def test_mcdc_records_aggregation_and_decision_tracking(self):
"""MC/DC records must be parsed for total conditions and full decision verification."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/src/math/fma.cpp",
- "summary": {
- "lines": {"count": 50, "covered": 50},
- "functions": {"count": 1, "covered": 1},
- "mcdc": {"count": 4, "covered": 3},
- },
- "mcdc_records": [
- # Fully verified decision: [True, True]
- [10, 5, 10, 20, 0, 0, 0, 0, 0, [True, True]],
- # Partially verified decision: [True, False]
- [25, 5, 25, 25, 0, 0, 0, 0, 0, [True, False]],
- # Malformed record: ignored
- [30, 5, 30, 20],
- ],
- }
- ]
- }
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/src/math/fma.cpp",
+ lines_total=50,
+ lines_covered=50,
+ func_total=1,
+ func_covered=1,
+ mcdc_total=4,
+ mcdc_covered=3,
+ mcdc_records=[
+ [10, 5, 10, 20, 0, 0, 0, 0, 0, [True, True]],
+ [25, 5, 25, 25, 0, 0, 0, 0, 0, [True, False]],
+ [30, 5, 30, 20], # Malformed: ignored
+ ],
+ )
]
- }
-
+ )
summary = extract_full_coverage_statistics(payload)
self.assertIsNotNone(summary)
self.assertTrue(summary.has_mcdc)
@@ -307,33 +307,25 @@ def test_mcdc_records_aggregation_and_decision_tracking(self):
def test_malformed_and_empty_mcdc_records_ignored(self):
"""Malformed, non-list, or empty condition vectors must not count as valid decisions."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/src/math/exp.cpp",
- "summary": {
- "lines": {"count": 50, "covered": 50},
- "functions": {"count": 1, "covered": 1},
- "mcdc": {"count": 2, "covered": 2},
- },
- "mcdc_records": [
- # Valid fully verified decision
- [10, 5, 10, 20, 0, 0, 0, 0, 0, [True]],
- # Empty list: not a valid decision
- [20, 5, 20, 20, 0, 0, 0, 0, 0, []],
- # Non-list 10th element: not a valid decision
- [30, 5, 30, 20, 0, 0, 0, 0, 0, None],
- # Record length < 10: not a valid decision
- [40, 5, 40, 20],
- ],
- }
- ]
- }
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/src/math/exp.cpp",
+ lines_total=50,
+ lines_covered=50,
+ func_total=1,
+ func_covered=1,
+ mcdc_total=2,
+ mcdc_covered=2,
+ mcdc_records=[
+ [10, 5, 10, 20, 0, 0, 0, 0, 0, [True]],
+ [20, 5, 20, 20, 0, 0, 0, 0, 0, []],
+ [30, 5, 30, 20, 0, 0, 0, 0, 0, None],
+ [40, 5, 40, 20],
+ ],
+ )
]
- }
-
+ )
summary = extract_full_coverage_statistics(payload)
self.assertIsNotNone(summary)
self.assertEqual(summary.global_stats.decisions_tot, 1)
@@ -437,7 +429,6 @@ def test_format_directory_breakdown_table_alphabetical_sorting(self):
)
table = format_directory_breakdown_table(summary)
- # Check alphabetical order in markdown output
idx_ctype = table.find("`libc/src/ctype`")
idx_math = table.find("`libc/src/math`")
idx_string = table.find("`libc/src/string`")
@@ -528,21 +519,17 @@ def test_render_empty_payload_fallback(self):
def test_render_complete_report_without_mcdc(self):
"""Valid report without MC/DC must render line coverage and summary tables."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/src/math/sin.cpp",
- "summary": {
- "lines": {"count": 100, "covered": 90},
- "functions": {"count": 2, "covered": 2},
- },
- }
- ]
- }
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/src/math/sin.cpp",
+ lines_total=100,
+ lines_covered=90,
+ func_total=2,
+ func_covered=2,
+ )
]
- }
+ )
buf = io.StringIO()
with redirect_stdout(buf):
render_full_report(payload)
@@ -555,26 +542,20 @@ def test_render_complete_report_without_mcdc(self):
def test_render_complete_report_with_mcdc(self):
"""Valid report with MC/DC must render full callouts, global table, and directory table."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/src/math/sin.cpp",
- "summary": {
- "lines": {"count": 100, "covered": 90},
- "functions": {"count": 2, "covered": 2},
- "mcdc": {"count": 6, "covered": 6},
- },
- "mcdc_records": [
- [10, 5, 10, 20, 0, 0, 0, 0, 0, [True, True]]
- ],
- }
- ]
- }
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/src/math/sin.cpp",
+ lines_total=100,
+ lines_covered=90,
+ func_total=2,
+ func_covered=2,
+ mcdc_total=6,
+ mcdc_covered=6,
+ mcdc_records=[[10, 5, 10, 20, 0, 0, 0, 0, 0, [True, True]]],
+ )
]
- }
-
+ )
buf = io.StringIO()
with redirect_stdout(buf):
render_full_report(payload)
@@ -590,81 +571,60 @@ def test_render_complete_report_with_mcdc(self):
class TestCommandLineInterface(unittest.TestCase):
"""Tests CLI invocation, arguments parsing, and file handling."""
- def test_cli_execution_with_file(self):
+ def test_cli_execution(self):
"""CLI must read JSON coverage file from disk and write report to stdout."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/src/math/sin.cpp",
- "summary": {
- "lines": {"count": 50, "covered": 40},
- "functions": {"count": 1, "covered": 1},
- },
- }
- ]
- }
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/src/math/sin.cpp",
+ lines_total=50,
+ lines_covered=40,
+ )
]
- }
-
- with tempfile.NamedTemporaryFile(
- mode="w", suffix=".json", delete=False
- ) as tmp_file:
- json.dump(payload, tmp_file)
- tmp_path = tmp_file.name
+ )
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_path = os.path.join(tmp_dir, "cov.json")
+ with open(tmp_path, "w") as tmp_file:
+ json.dump(payload, tmp_file)
- try:
buf = io.StringIO()
- with patch.object(
- sys,
- "argv",
- ["codebase_coverage.py", tmp_path, "aabbccdd1122", "main"],
- ):
+ with patch.object(sys, "argv", ["codebase_coverage.py", tmp_path]):
with redirect_stdout(buf):
main()
output = buf.getvalue()
self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
self.assertIn("`libc/src/math`", output)
- finally:
- if os.path.exists(tmp_path):
- os.remove(tmp_path)
-
- def test_cli_execution_with_minimal_arguments(self):
- """CLI must execute successfully when commit SHA and branch ref are omitted."""
- payload = {
- "data": [
- {
- "files": [
- {
- "filename": "/workspace/libc/src/ctype/isalnum.cpp",
- "summary": {
- "lines": {"count": 20, "covered": 20},
- "functions": {"count": 1, "covered": 1},
- },
- }
- ]
- }
- ]
- }
- with tempfile.NamedTemporaryFile(
- mode="w", suffix=".json", delete=False
- ) as tmp_file:
- json.dump(payload, tmp_file)
- tmp_path = tmp_file.name
+ def test_cli_execution_with_mcdc(self):
+ """CLI must render MC/DC breakdown tables when payload includes MC/DC records."""
+ payload = _make_codebase_payload(
+ [
+ _make_file_entry(
+ "/workspace/libc/src/math/sin.cpp",
+ lines_total=60,
+ lines_covered=50,
+ func_total=2,
+ func_covered=2,
+ mcdc_total=4,
+ mcdc_covered=4,
+ mcdc_records=[[10, 5, 10, 20, 0, 0, 0, 0, 0, [True, True]]],
+ )
+ ]
+ )
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_path = os.path.join(tmp_dir, "cov.json")
+ with open(tmp_path, "w") as tmp_file:
+ json.dump(payload, tmp_file)
- try:
buf = io.StringIO()
with patch.object(sys, "argv", ["codebase_coverage.py", tmp_path]):
with redirect_stdout(buf):
main()
output = buf.getvalue()
self.assertIn("## LLVM-libc Full Codebase Coverage Report", output)
- self.assertIn("`libc/src/ctype`", output)
- finally:
- if os.path.exists(tmp_path):
- os.remove(tmp_path)
+ self.assertIn("`libc/src/math`", output)
+ self.assertIn("MC/DC Conditions", output)
+ self.assertIn("Decisions (Verified / Total)", output)
def test_cli_nonexistent_file_exits_with_error(self):
"""CLI must exit with code 1 when targeted file does not exist."""
@@ -682,23 +642,18 @@ def test_cli_nonexistent_file_exits_with_error(self):
def test_cli_invalid_json_exits_with_error(self):
"""CLI must exit with code 1 when targeted file contains invalid JSON syntax."""
- with tempfile.NamedTemporaryFile(
- mode="w", suffix=".json", delete=False
- ) as tmp_file:
- tmp_file.write("INVALID JSON CONTENT")
- tmp_path = tmp_file.name
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_path = os.path.join(tmp_dir, "cov.json")
+ with open(tmp_path, "w") as tmp_file:
+ tmp_file.write("INVALID JSON CONTENT")
- stderr_buf = io.StringIO()
- try:
+ stderr_buf = io.StringIO()
with patch.object(sys, "argv", ["codebase_coverage.py", tmp_path]):
with patch("sys.stderr", stderr_buf):
with self.assertRaises(SystemExit) as cm:
main()
self.assertEqual(cm.exception.code, 1)
self.assertIn("Error: Failed to parse coverage JSON", stderr_buf.getvalue())
- finally:
- if os.path.exists(tmp_path):
- os.remove(tmp_path)
if __name__ == "__main__":
diff --git a/libc/utils/coverage/test_diff_coverage.py b/libc/utils/coverage/patch_coverage_test.py
similarity index 56%
rename from libc/utils/coverage/test_diff_coverage.py
rename to libc/utils/coverage/patch_coverage_test.py
index e369a6a5f7004..0246176b9c78c 100644
--- a/libc/utils/coverage/test_diff_coverage.py
+++ b/libc/utils/coverage/patch_coverage_test.py
@@ -1,29 +1,31 @@
-# ====- Unit tests for diff_coverage.py ----------------------*- python -*--==#
+# ===- Unit tests for patch coverage ------------------------*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
-# ==-------------------------------------------------------------------------==#
+# ==------------------------------------------------------------------------==#
-"""Unit tests for diff_coverage.py."""
+# To run these tests:
+# python3 -m unittest patch_coverage_test.py
+
+"""Unit tests for patch coverage analyzer."""
import io
import json
import os
import sys
import tempfile
+import textwrap
import unittest
from contextlib import redirect_stdout
+from typing import List, Optional
from unittest.mock import patch
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from diff_coverage import (
- DEFAULT_BASE_REPOSITORY,
- DEFAULT_HEAD_REPOSITORY,
CoverageJSONParser,
- DiffHunk,
DiffParser,
FilePatchMetrics,
PatchCoverageSummary,
@@ -39,23 +41,51 @@
)
-def _make_cov_json(filename, segments, mcdc_records=None):
- """Constructs a minimal llvm-cov JSON export payload."""
+def _make_unified_diff(
+ filepath: str,
+ added_lines: List[str],
+ start_line: int = 1,
+ context_lines: Optional[List[str]] = None,
+) -> str:
+ """Constructs a valid Git unified diff string for testing patch correlation."""
+ ctx = context_lines or ["ctx();"]
+ old_count = len(ctx)
+ new_count = old_count + len(added_lines)
+ header = f"@@ -{start_line},{old_count} +{start_line},{new_count} @@"
+ lines = [
+ f"diff --git a/{filepath} b/{filepath}",
+ f"--- a/{filepath}",
+ f"+++ b/{filepath}",
+ header,
+ ]
+ for c in ctx:
+ lines.append(f" {c}")
+ for a in added_lines:
+ lines.append(f"+{a}")
+ lines.append("")
+ return "\n".join(lines)
+
+
+def _make_cov_file(
+ filename: str, segments: list, mcdc_records: Optional[list] = None
+) -> dict:
+ """Constructs a single file coverage record for llvm-cov JSON export."""
return {
- "data": [
- {
- "files": [
- {
- "filename": filename,
- "segments": segments,
- "mcdc_records": mcdc_records or [],
- }
- ]
- }
- ]
+ "filename": filename,
+ "segments": segments,
+ "mcdc_records": mcdc_records or [],
}
+def _make_cov_json(filename_or_files, segments=None, mcdc_records=None) -> dict:
+ """Constructs a minimal llvm-cov JSON export payload."""
+ if isinstance(filename_or_files, list):
+ files = filename_or_files
+ else:
+ files = [_make_cov_file(filename_or_files, segments or [], mcdc_records)]
+ return {"data": [{"files": files}]}
+
+
class TestCoverageDataStructures(unittest.TestCase):
"""Tests FilePatchMetrics and PatchCoverageSummary ratio logic and zero-division guards."""
@@ -100,53 +130,75 @@ def test_patch_coverage_summary_aggregation(self):
class TestExecutableLineFiltering(unittest.TestCase):
"""Tests statement heuristics distinguishing executable C++ statements from non-code."""
- def test_statement_heuristics(self):
- """Verifies statement classification across distinct C/C++ syntactic forms."""
- test_cases = [
- ("int x = 42;", True, "Variable assignment"),
- ("x += y;", True, "Compound arithmetic assignment"),
- ("return result;", True, "Return statement"),
- ("if (x > 0) {", True, "Branch condition header"),
- ("for (size_t i = 0; i < count; ++i) {", True, "For loop header"),
- ("do_work(a, b);", True, "Function call"),
- ("struct Point p = {1, 2};", True, "Struct variable assignment"),
- ("int x = 42; // assignment", True, "Statement with line comment"),
- ("int x = 42; /* inline comment */", True, "Statement with block comment"),
- ("// Single line comment", False, "Line comment"),
- ("/* Block comment start", False, "Block comment start"),
- (" * Continuation line", False, "Block comment middle"),
- (" */", False, "Block comment end"),
+ def test_executable_statements(self):
+ """Statements with assignments, function calls, returns, and control flow are executable."""
+ cases = [
+ ("int x = 42;", "Variable assignment"),
+ ("x += y;", "Compound arithmetic assignment"),
+ ("return result;", "Return statement"),
+ ("if (x > 0) {", "Branch condition header"),
+ ("for (size_t i = 0; i < count; ++i) {", "For loop header"),
+ ("do_work(a, b);", "Function call"),
+ ("struct Point p = {1, 2};", "Struct variable initialization"),
+ ("int x = 42; // assignment", "Statement with trailing line comment"),
+ ("int x = 42; /* inline comment */", "Statement with inline block comment"),
+ ]
+ for line, desc in cases:
+ with self.subTest(msg=desc, line=line):
+ self.assertTrue(is_executable_line(line))
+
+ def test_structural_syntax_and_braces(self):
+ """Standalone braces, access specifiers, and constructor colons are non-executable."""
+ cases = [
+ ("{", "Opening brace"),
+ ("}", "Closing brace"),
+ ("};", "Scope terminator"),
+ ("public:", "Public access specifier"),
+ ("private: // methods", "Private access specifier with comment"),
+ (": value_(0) {", "Constructor initializer header"),
+ ]
+ for line, desc in cases:
+ with self.subTest(msg=desc, line=line):
+ self.assertFalse(is_executable_line(line))
+
+ def test_comments_and_whitespace(self):
+ """Line comments, block comments, blank lines, and commented braces are non-executable."""
+ cases = [
+ ("// Single line comment", "Single-line comment"),
+ ("/* Block comment start", "Block comment start"),
+ (" * Continuation line", "Block comment continuation"),
+ (" */", "Block comment end"),
(
"} // namespace LIBC_NAMESPACE_DECL",
- False,
- "Brace with namespace comment",
+ "Closing brace with namespace comment",
),
- ("}; // struct Point", False, "Scope end with comment"),
- ("{ // begin loop", False, "Opening brace with comment"),
- ("} /* namespace */", False, "Brace with block comment"),
- ("public:", False, "Access specifier"),
- ("private: // methods", False, "Access specifier with comment"),
+ ("}; // struct Point", "Scope end with comment"),
+ ("{ // begin loop", "Opening brace with comment"),
+ ("} /* namespace */", "Closing brace with block comment"),
+ ("", "Empty line"),
+ (" ", "Whitespace indentation only"),
+ ]
+ for line, desc in cases:
+ with self.subTest(msg=desc, line=line):
+ self.assertFalse(is_executable_line(line))
+
+ def test_declarations_and_preprocessor(self):
+ """Includes, namespaces, type aliases, forward declarations, and static asserts are non-executable."""
+ cases = [
+ ("#include <stddef.h>", "Preprocessor include"),
+ ("namespace LIBC_NAMESPACE {", "Namespace definition"),
+ ("using size_t = unsigned long;", "Type alias"),
+ ("struct ListNode;", "Forward struct declaration"),
+ ("enum class Status : uint8_t {", "Enum definition header"),
(
'static_assert(sizeof(long) == 8, "msg");',
- False,
- "Compile-time assertion",
+ "Compile-time static assertion",
),
- ("friend class Peer;", False, "Friend declaration"),
- ("{", False, "Opening brace"),
- ("}", False, "Closing brace"),
- ("};", False, "Scope terminator"),
- (": value_(0) {", False, "Constructor initializer header"),
- ("#include <stddef.h>", False, "Preprocessor include"),
- ("namespace LIBC_NAMESPACE {", False, "Namespace definition"),
- ("using size_t = unsigned long;", False, "Type alias"),
- ("struct ListNode;", False, "Forward struct declaration"),
- ("enum class Status : uint8_t {", False, "Enum definition header"),
- ("", False, "Empty line"),
- (" ", False, "Whitespace line"),
+ ("friend class Peer;", "Friend class declaration"),
]
- for line, expected, desc in test_cases:
+ for line, desc in cases:
with self.subTest(msg=desc, line=line):
- self.assertEqual(is_executable_line(line), expected)
+ self.assertFalse(is_executable_line(line))
class TestFormatLineRanges(unittest.TestCase):
@@ -166,18 +218,20 @@ class TestDiffParser(unittest.TestCase):
def test_parse_diff_hunks(self):
"""DiffParser must extract added and context lines across hunks while ignoring deletions."""
- diff_text = (
- "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
- "--- a/src/math/sin.cpp\n"
- "+++ b/src/math/sin.cpp\n"
- "@@ -10,3 +10,4 @@\n"
- " ctx1();\n"
- "-deleted();\n"
- "+added1();\n"
- "+added2();\n"
- "@@ -50,1 +51,2 @@\n"
- " ctx2();\n"
- "+added3();\n"
+ diff_text = textwrap.dedent(
+ """\
+ diff --git a/src/math/sin.cpp b/src/math/sin.cpp
+ --- a/src/math/sin.cpp
+ +++ b/src/math/sin.cpp
+ @@ -10,3 +10,4 @@
+ ctx1();
+ -deleted();
+ +added1();
+ +added2();
+ @@ -50,1 +51,2 @@
+ ctx2();
+ +added3();
+ """
)
parsed = DiffParser.parse(diff_text)
self.assertIn("src/math/sin.cpp", parsed)
@@ -190,19 +244,21 @@ def test_parse_diff_hunks(self):
def test_parse_special_files(self):
"""Newly created files must start at line 1, deleted files and headers must be skipped."""
- diff_text = (
- "diff --git a/src/math/new.cpp b/src/math/new.cpp\n"
- "new file mode 100644\n"
- "--- /dev/null\n"
- "+++ b/src/math/new.cpp\n"
- "index 0000..1111\n"
- "@@ -0,0 +1,1 @@\n"
- "+int new_func();\n"
- "diff --git a/src/math/old.cpp b/src/math/old.cpp\n"
- "--- a/src/math/old.cpp\n"
- "+++ /dev/null\n"
- "@@ -1,1 +0,0 @@\n"
- "-deleted();\n"
+ diff_text = textwrap.dedent(
+ """\
+ diff --git a/src/math/new.cpp b/src/math/new.cpp
+ new file mode 100644
+ --- /dev/null
+ +++ b/src/math/new.cpp
+ index 0000..1111
+ @@ -0,0 +1,1 @@
+ +int new_func();
+ diff --git a/src/math/old.cpp b/src/math/old.cpp
+ --- a/src/math/old.cpp
+ +++ /dev/null
+ @@ -1,1 +0,0 @@
+ -deleted();
+ """
)
parsed = DiffParser.parse(diff_text)
self.assertIn("src/math/new.cpp", parsed)
@@ -214,16 +270,13 @@ def test_parse_special_files(self):
def test_parse_from_disk_file(self):
"""DiffParser must successfully read and parse diff files from disk."""
- diff_text = "diff --git a/a b/b\n+++ b/src/math/f.cpp\n@@ -1,1 +1,2 @@\n ctx();\n+line();\n"
- with tempfile.NamedTemporaryFile(mode="w", suffix=".diff", delete=False) as tmp:
- tmp.write(diff_text)
- tmp_path = tmp.name
- try:
+ diff_text = _make_unified_diff("src/math/f.cpp", ["line();"])
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_path = os.path.join(tmp_dir, "test.diff")
+ with open(tmp_path, "w") as f:
+ f.write(diff_text)
parsed = DiffParser.parse(tmp_path)
self.assertIn("src/math/f.cpp", parsed)
- finally:
- if os.path.exists(tmp_path):
- os.remove(tmp_path)
class TestCoverageJSONParser(unittest.TestCase):
@@ -232,51 +285,35 @@ class TestCoverageJSONParser(unittest.TestCase):
def test_segment_expansion_and_path_normalization(self):
"""Segments spanning multiple lines must mark each line covered, uncounted must skip."""
diff_files = {"libc/src/math/sin.cpp": [], "src/math/cos.cpp": []}
- json_data = {
- "data": [
- {
- "files": [
- {
- "filename": "/runner/work/llvm-project/libc/src/math/sin.cpp",
- "segments": [
- [10, 0, 5, 1, 1],
- [13, 0, 0, 1, 1],
- [20, 0, 0, 0, 1],
- ],
- "mcdc_records": [],
- },
- {
- # Tests target_path.endswith("/" + file_name)
- "filename": "cos.cpp",
- "segments": [[5, 0, 1, 1, 1]],
- "mcdc_records": [],
- },
- {
- # Unmatched file: tests continue branch
- "filename": "/runner/work/llvm-project/libc/src/math/other.cpp",
- "segments": [[1, 0, 1, 1, 1]],
- "mcdc_records": [],
- },
- ]
- }
+ json_data = _make_cov_json(
+ [
+ _make_cov_file(
+ "/runner/work/llvm-project/libc/src/math/sin.cpp",
+ segments=[
+ [10, 0, 5, 1, 1],
+ [13, 0, 0, 1, 1],
+ [20, 0, 0, 0, 1],
+ ],
+ ),
+ _make_cov_file("cos.cpp", segments=[[5, 0, 1, 1, 1]]),
+ _make_cov_file(
+ "/runner/work/llvm-project/libc/src/math/other.cpp",
+ segments=[[1, 0, 1, 1, 1]],
+ ),
]
- }
+ )
matrix = CoverageJSONParser.extract_patch_matrix(json_data, diff_files)
- covered = matrix["libc/src/math/sin.cpp"]["covered"]
- missed = matrix["libc/src/math/sin.cpp"]["missed"]
- self.assertEqual(covered, {10, 11, 12})
- self.assertIn(13, missed)
- self.assertIn(19, missed)
- self.assertNotIn(20, missed)
+ self.assertEqual(matrix["libc/src/math/sin.cpp"]["covered"], {10, 11, 12})
+ self.assertEqual(matrix["src/math/cos.cpp"]["covered"], {5})
def test_mcdc_records_extraction(self):
"""Valid MC/DC records must be extracted; truncated or empty records must be ignored."""
diff_files = {"src/math/sin.cpp": []}
json_data = _make_cov_json(
- filename="/workspace/src/math/sin.cpp",
- segments=[[10, 0, 1, 1, 1]],
+ "/runner/work/llvm-project/libc/src/math/sin.cpp",
+ segments=[],
mcdc_records=[
- [10, 4, 10, 14, 0, 0, 0, 0, 0, [True, False]], # Valid
+ [10, 4, 10, 14, 0, 0, 0, 0, 0, [True, False]],
[11, 4, 11, 14], # Truncated (< 10)
[12, 4, 12, 14, 0, 0, 0, 0, 0, []], # Empty condition vector
],
@@ -294,14 +331,11 @@ def test_load_and_empty_payload_safeguards(self):
empty_matrix = CoverageJSONParser.extract_patch_matrix({}, diff_files)
self.assertEqual(len(empty_matrix["src/math/sin.cpp"]["covered"]), 0)
- with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp:
- json.dump({"key": "val"}, tmp)
- tmp_path = tmp.name
- try:
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_path = os.path.join(tmp_dir, "cov.json")
+ with open(tmp_path, "w") as tmp:
+ json.dump({"key": "val"}, tmp)
self.assertEqual(CoverageJSONParser.load(tmp_path), {"key": "val"})
- finally:
- if os.path.exists(tmp_path):
- os.remove(tmp_path)
stderr_buf = io.StringIO()
with patch("sys.stderr", stderr_buf):
@@ -315,22 +349,17 @@ class TestCalculatePatchStatistics(unittest.TestCase):
def test_calculate_patch_statistics(self):
"""Covered lines take precedence, uninstrumented files miss, and MC/DC diagnoses unverified."""
- diff_text = (
- "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
- "--- a/src/math/sin.cpp\n"
- "+++ b/src/math/sin.cpp\n"
- "@@ -10,2 +10,3 @@\n"
- " ctx();\n"
- "+int covered_and_missed = 1;\n"
- "+if (a && b) return 1;\n"
- "diff --git a/src/math/untested.cpp b/src/math/untested.cpp\n"
- "--- a/src/math/untested.cpp\n"
- "+++ b/src/math/untested.cpp\n"
- "@@ -1,1 +1,2 @@\n"
- " ctx();\n"
- "+int untested = 1;\n"
+ diff_sin = _make_unified_diff(
+ "src/math/sin.cpp",
+ ["int covered_and_missed = 1;", "if (a && b) return 1;"],
+ start_line=10,
)
- diff_files = DiffParser.parse(diff_text)
+ diff_untested = _make_unified_diff(
+ "src/math/untested.cpp",
+ ["int untested = 1;"],
+ start_line=1,
+ )
+ diff_files = DiffParser.parse(f"{diff_sin}\n{diff_untested}")
coverage_matrix = {
"src/math/sin.cpp": {
"covered": {11, 12},
@@ -366,17 +395,13 @@ def test_calculate_patch_statistics(self):
def test_non_source_and_comment_files_skipped(self):
"""Test files, documentation, and files with only comment additions must be skipped."""
- diff_text = (
- "diff --git a/libc/test/src/math/sin_test.cpp b/libc/test/src/math/sin_test.cpp\n"
- "+++ b/libc/test/src/math/sin_test.cpp\n"
- "@@ -1,1 +1,2 @@\n"
- "+TEST(Foo, Bar) {}\n"
- "diff --git a/src/math/comment_only.cpp b/src/math/comment_only.cpp\n"
- "+++ b/src/math/comment_only.cpp\n"
- "@@ -1,1 +1,2 @@\n"
- "+// comment only\n"
+ diff_test = _make_unified_diff(
+ "libc/test/src/math/sin_test.cpp", ["TEST(Foo, Bar) {}"]
)
- diff_files = DiffParser.parse(diff_text)
+ diff_comment = _make_unified_diff(
+ "src/math/comment_only.cpp", ["// comment only"]
+ )
+ diff_files = DiffParser.parse(f"{diff_test}\n{diff_comment}")
coverage_matrix = {
"libc/test/src/math/sin_test.cpp": {
"covered": set(),
@@ -447,13 +472,13 @@ def test_format_status_banner_variants(self):
self.assertIn(expected, format_status_banner(summary))
def test_format_metadata_section(self):
- """Metadata section must format commits, tests, or return empty on missing arguments."""
+ """Metadata section must format targeted tests or return empty on missing arguments."""
+ self.assertEqual(format_metadata_section(None, None, None, None), "")
metadata = format_metadata_section(
- "1111111", "2222222", "main", "patch", "test_target"
+ None, None, None, None, targeted_tests_string="libc-math-unit-tests"
)
- self.assertIn("Base Branch", metadata)
- self.assertIn("`test_target`", metadata)
- self.assertEqual(format_metadata_section(None, None, None, None), "")
+ self.assertIn("`libc-math-unit-tests`", metadata)
+ self.assertIn("Targeted Tests Executed", metadata)
def test_format_breakdown_table(self):
"""Breakdown tables must render line/MCDC stats and normalize paths to libc/ on GitHub."""
@@ -486,22 +511,17 @@ def test_format_breakdown_table(self):
"src/string/strlen.cpp": file_no_mcdc,
},
)
- table = format_breakdown_table(summary, head_commit_sha="abcd123")
- self.assertIn("blob/abcd123/libc/src/math/sin.cpp", table)
+ table = format_breakdown_table(summary)
+ self.assertIn("blob/main/libc/src/math/sin.cpp", table)
self.assertIn("MC/DC Conditions", table)
self.assertIn("N/A | N/A", table) # strlen has no MC/DC
def test_format_annotated_diff(self):
"""Annotated diff must output covered, missed, partial MC/DC, non-executable, and context lines."""
- diff_text = (
- "diff --git a/src/math/sin.cpp b/src/math/sin.cpp\n"
- "+++ b/src/math/sin.cpp\n"
- "@@ -10,4 +10,5 @@\n"
- " ctx();\n"
- "+covered();\n"
- "+missed();\n"
- "+if (a && b) {}\n"
- "+{\n"
+ diff_text = _make_unified_diff(
+ "src/math/sin.cpp",
+ ["covered();", "missed();", "if (a && b) {}", "{\n"],
+ start_line=10,
)
diff_files = DiffParser.parse(diff_text)
file_metrics = FilePatchMetrics(
@@ -526,14 +546,22 @@ def test_render_empty_diff(self):
"""Empty diff must render coverage notice without failing."""
buf = io.StringIO()
with redirect_stdout(buf):
- render_patch_report({}, {}, "1111", "2222", "main", "feature")
+ render_patch_report(
+ {},
+ {},
+ None,
+ None,
+ None,
+ None,
+ )
self.assertIn(
- "No executable lines were added or modified in this patch.", buf.getvalue()
+ "No executable lines were added or modified in this patch.",
+ buf.getvalue(),
)
def test_render_full_report_with_mcdc(self):
"""Patch report with MC/DC must display the MC/DC report title and full tables."""
- diff_text = "diff --git a/src/math/f.cpp b/src/math/f.cpp\n+++ b/src/math/f.cpp\n@@ -1,1 +1,2 @@\n ctx();\n+return a && b;\n"
+ diff_text = _make_unified_diff("src/math/f.cpp", ["return a && b;"])
diff_files = DiffParser.parse(diff_text)
coverage_matrix = {
"src/math/f.cpp": {
@@ -553,7 +581,12 @@ def test_render_full_report_with_mcdc(self):
buf = io.StringIO()
with redirect_stdout(buf):
render_patch_report(
- diff_files, coverage_matrix, "1111", "2222", "main", "feature"
+ diff_files,
+ coverage_matrix,
+ None,
+ None,
+ None,
+ None,
)
output = buf.getvalue()
self.assertIn("## LLVM-libc MC/DC Patch Coverage Report", output)
@@ -562,51 +595,88 @@ def test_render_full_report_with_mcdc(self):
)
self.assertIn("View Annotated Patch Diff", output)
+ def test_render_full_report_line_coverage_only(self):
+ """Line-coverage-only reports must omit MC/DC headers and condition columns."""
+ diff_text = _make_unified_diff("src/math/sin.cpp", ["int x = 1;"])
+ diff_files = DiffParser.parse(diff_text)
+ coverage_matrix = {
+ "src/math/sin.cpp": {
+ "covered": {2},
+ "missed": set(),
+ "mcdc_decisions": [],
+ }
+ }
+ buf = io.StringIO()
+ with redirect_stdout(buf):
+ render_patch_report(
+ diff_files,
+ coverage_matrix,
+ None,
+ None,
+ None,
+ None,
+ "libc-math-unit-tests",
+ )
+ output = buf.getvalue()
+ self.assertIn("## LLVM-libc Patch Coverage Report", output)
+ self.assertNotIn("MC/DC", output)
+ self.assertIn("### Patch Coverage: **100.00%**", output)
+ self.assertIn("`libc-math-unit-tests`", output)
+
-class TestCommandLineInterfaceDiff(unittest.TestCase):
+class TestCommandLineInterfacePatch(unittest.TestCase):
"""Tests CLI execution and file validation safeguards."""
- def test_cli_execution(self):
- """CLI must read diff and JSON files from disk and print the report to stdout."""
- diff_text = "diff --git a/src/math/s.cpp b/src/math/s.cpp\n+++ b/src/math/s.cpp\n@@ -1,1 +1,2 @@\n ctx();\n+int x = 1;\n"
+ def test_cli_minimal_arguments(self):
+ """CLI must execute successfully when only required diff and JSON files are provided."""
+ diff_text = _make_unified_diff("src/math/s.cpp", ["int x = 1;"])
json_data = _make_cov_json("/workspace/src/math/s.cpp", [[2, 0, 1, 1, 1]])
- with tempfile.NamedTemporaryFile(
- mode="w", suffix=".diff", delete=False
- ) as f_diff:
- f_diff.write(diff_text)
- path_diff = f_diff.name
- with tempfile.NamedTemporaryFile(
- mode="w", suffix=".json", delete=False
- ) as f_json:
- json.dump(json_data, f_json)
- path_json = f_json.name
-
- try:
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ path_diff = os.path.join(tmp_dir, "patch.diff")
+ path_json = os.path.join(tmp_dir, "cov.json")
+ with open(path_diff, "w") as f_diff:
+ f_diff.write(diff_text)
+ with open(path_json, "w") as f_json:
+ json.dump(json_data, f_json)
+
+ buf = io.StringIO()
+ with patch.object(sys, "argv", ["diff_coverage.py", path_diff, path_json]):
+ with redirect_stdout(buf):
+ main()
+ self.assertIn("## LLVM-libc Patch Coverage Report", buf.getvalue())
+ self.assertIn("[`src/math/s.cpp`]", buf.getvalue())
+
+ def test_cli_execution_with_mcdc(self):
+ """CLI must process MC/DC records and format tables when executed with MC/DC data."""
+ diff_text = _make_unified_diff("src/math/sin.cpp", ["if (a && b) return 0;"])
+ json_data = _make_cov_json(
+ "/workspace/libc/src/math/sin.cpp",
+ [[2, 0, 5, 1, 1]],
+ mcdc_records=[[2, 5, 2, 20, 0, 0, 0, 0, 0, [True, True]]],
+ )
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ path_diff = os.path.join(tmp_dir, "patch.diff")
+ path_json = os.path.join(tmp_dir, "cov.json")
+ with open(path_diff, "w") as f_diff:
+ f_diff.write(diff_text)
+ with open(path_json, "w") as f_json:
+ json.dump(json_data, f_json)
+
buf = io.StringIO()
with patch.object(
sys,
"argv",
- [
- "diff_coverage.py",
- path_diff,
- path_json,
- "111",
- "222",
- "m",
- "f",
- "target",
- ],
+ ["diff_coverage.py", path_diff, path_json],
):
with redirect_stdout(buf):
main()
- self.assertIn("## LLVM-libc Patch Coverage Report", buf.getvalue())
- self.assertIn("[`src/math/s.cpp`]", buf.getvalue())
- finally:
- if os.path.exists(path_diff):
- os.remove(path_diff)
- if os.path.exists(path_json):
- os.remove(path_json)
+ output = buf.getvalue()
+ self.assertIn("## LLVM-libc MC/DC Patch Coverage Report", output)
+ self.assertIn("blob/main/libc/src/math/sin.cpp", output)
+ self.assertIn("MC/DC Conditions", output)
+ self.assertIn("Decisions (Verified / Total)", output)
def test_cli_missing_files_exit(self):
"""CLI must exit with code 1 when diff file or JSON file is missing."""
@@ -619,6 +689,24 @@ def test_cli_missing_files_exit(self):
main()
self.assertIn("Error: Diff file not found", stderr_buf.getvalue())
+ def test_cli_invalid_json_exits_with_error(self):
+ """CLI must exit with code 1 when diff exists but coverage JSON is malformed."""
+ diff_text = "diff --git a/a b/b\n"
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ path_diff = os.path.join(tmp_dir, "patch.diff")
+ path_json = os.path.join(tmp_dir, "cov.json")
+ with open(path_diff, "w") as f_diff:
+ f_diff.write(diff_text)
+ with open(path_json, "w") as f_json:
+ f_json.write("MALFORMED JSON")
+
+ stderr_buf = io.StringIO()
+ with patch.object(sys, "argv", ["diff_coverage.py", path_diff, path_json]):
+ with patch("sys.stderr", stderr_buf):
+ with self.assertRaises(SystemExit):
+ main()
+ self.assertIn("Error: Failed to parse coverage JSON", stderr_buf.getvalue())
+
if __name__ == "__main__":
unittest.main()
>From b8c461770ab7ff59da4a20cf713de3640c785c67 Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Tue, 8 Sep 2026 12:44:50 +0000
Subject: [PATCH 13/15] [libc][test] Test patch coverage bot on isascii
Assisted-by: Automated tooling, human reviewed.
---
libc/src/ctype/isascii.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/libc/src/ctype/isascii.cpp b/libc/src/ctype/isascii.cpp
index ef3788d136e9d..2102c90d5a0ab 100644
--- a/libc/src/ctype/isascii.cpp
+++ b/libc/src/ctype/isascii.cpp
@@ -14,7 +14,9 @@
namespace LIBC_NAMESPACE_DECL {
LLVM_LIBC_FUNCTION(int, isascii, (int c)) {
- return static_cast<int>((c & (~0x7f)) == 0);
+ if (c < 0 || c > 127)
+ return 0;
+ return 1;
}
} // namespace LIBC_NAMESPACE_DECL
>From b8e2627f408d4725e89a07985b7a00c0d4ebe179 Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Tue, 8 Sep 2026 12:48:56 +0000
Subject: [PATCH 14/15] [libc][ci] Add libc-coverage-bots to workflow triggers
for testing
Temporarily enable patch coverage workflow runs on libc-coverage-bots for testing purposes.
Assisted-by: Automated tooling, human reviewed.
---
.github/workflows/libc-patch-coverage.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/libc-patch-coverage.yml b/.github/workflows/libc-patch-coverage.yml
index 6ed0a368bb281..4551f2afa8204 100644
--- a/.github/workflows/libc-patch-coverage.yml
+++ b/.github/workflows/libc-patch-coverage.yml
@@ -8,6 +8,7 @@ on:
push:
branches:
- main
+ - libc-coverage-bots
paths:
- 'libc/src/**'
- 'libc/include/**'
@@ -22,6 +23,7 @@ on:
pull_request:
branches:
- main
+ - libc-coverage-bots
paths:
- 'libc/src/**'
- 'libc/include/**'
>From 3ede87c564e04d79aaddb4c7dba9c5e04d319ee6 Mon Sep 17 00:00:00 2001
From: Tapiwa Gonga <tapiwagonga at google.com>
Date: Tue, 8 Sep 2026 13:09:12 +0000
Subject: [PATCH 15/15] [libc][test] Update isascii logic to test patch
coverage summary
Assisted-by: Automated tooling, human reviewed.
---
libc/src/ctype/isascii.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/libc/src/ctype/isascii.cpp b/libc/src/ctype/isascii.cpp
index 2102c90d5a0ab..71d8906f0ade6 100644
--- a/libc/src/ctype/isascii.cpp
+++ b/libc/src/ctype/isascii.cpp
@@ -14,9 +14,9 @@
namespace LIBC_NAMESPACE_DECL {
LLVM_LIBC_FUNCTION(int, isascii, (int c)) {
- if (c < 0 || c > 127)
- return 0;
- return 1;
+ if (c >= 0 && c <= 127)
+ return 1;
+ return 0;
}
} // namespace LIBC_NAMESPACE_DECL
More information about the libc-commits
mailing list