[PATCH] D22544: [utils] Add script to check for code coverage regressions

[Vedant Kumar via llvm-commits]

vsk created this revision.
vsk added reviewers: cmatthews, silvas, MaggieYi.
vsk added a subscriber: llvm-commits.

This script is meant to be run on code coverage bots in conjunction with `utils/prepare-code-coverage-artifact.py` to flag coverage regressions in a window of commits.

https://reviews.llvm.org/D22544

Files:
  utils/check-coverage-regressions.py

Index: utils/check-coverage-regressions.py
===================================================================
--- /dev/null
+++ utils/check-coverage-regressions.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+
+'''Compare two coverage summaries for regressions.
+
+You can create a coverage summary by using the `llvm-cov report` command.
+Alternatively, you can use `utils/prepare-code-coverage-artifact.py` which
+creates summaries as well as file-based html reports.
+'''
+
+import argparse
+import collections
+import re
+
+FileCoverage = collections.namedtuple('FileCoverage',
+        ['Regions', 'Missed', 'Coverage', 'Functions', 'Executed'])
+
+CoverageEntry = re.compile('^(.*) +(\d+) +(\d+) +([\d.]+)% +(\d+) +([\d.]+)%$')
+
+kThresh = 0.95
+
+def parse_summary(path):
+    '''Parse the summary at @path. Return a dictionary mapping filenames to
+       FileCoverage instances.'''
+
+    with open(path, 'r') as f:
+        lines = f.readlines()
+
+    # Drop the header and the cell dividers. Include "TOTAL" in this list.
+    file_coverages = lines[2:-2] + [lines[-1]]
+
+    summary = {}
+
+    for line in file_coverages:
+        m = re.match(CoverageEntry, line)
+        if not m:
+            print "Could not read coverage summary:", line
+            exit(1)
+
+        groups = m.groups()
+        filename = groups[0].strip()
+        regions = int(groups[1])
+        missed = int(groups[2])
+        coverage = float(groups[3])
+        functions = int(groups[4])
+        executed = float(groups[5])
+        fc = FileCoverage(regions, missed, coverage, functions, executed)
+        summary[filename] = fc
+
+    return summary
+
+def find_coverage_regressions(old_coverage, new_coverage):
+    '''Given two coverage summaries, generate coverage regressions of the form:
+       (filename, old FileCoverage, new FileCoverage).'''
+
+    for filename in old_coverage.keys():
+        if filename not in new_coverage:
+            continue
+
+        old_fc = old_coverage[filename]
+        new_fc = new_coverage[filename]
+        if new_fc.Coverage < kThresh * old_fc.Coverage or \
+                new_fc.Executed < kThresh * old_fc.Executed:
+            yield (filename, old_fc, new_fc)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('old_summary', help='Path to the old coverage summary')
+    parser.add_argument('new_summary', help='Path to the new coverage summary')
+    args = parser.parse_args()
+
+    old_coverage = parse_summary(args.old_summary)
+    new_coverage = parse_summary(args.new_summary)
+
+    num_regressions = 0
+    for filename, old_fc, new_fc in \
+            find_coverage_regressions(old_coverage, new_coverage):
+        print "Coverage regression in:", filename
+        print "    Old coverage:", old_fc
+        print "    New coverage:", new_fc
+
+    if num_regressions > 0:
+        exit(1)
+    exit(0)


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D22544.64578.patch
Type: text/x-patch
Size: 2937 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20160719/142514cd/attachment.bin>