[llvm-commits] [zorg] r147165 - in /zorg/trunk/lnt/lnt: server/reporting/analysis.py server/reporting/runs.py util/NTEmailReport.py
Daniel Dunbar
daniel at zuster.org
Thu Dec 22 11:24:32 PST 2011
Author: ddunbar
Date: Thu Dec 22 13:24:32 2011
New Revision: 147165
URL: http://llvm.org/viewvc/llvm-project?rev=147165&view=rev
Log:
[lnt/v0.4] lnt.server.reporting: Continue defining V4 report implementation.
- Generates the summary table counts.
- Returns the changes in a format that can be reported by the lnt submit client.
Added:
zorg/trunk/lnt/lnt/server/reporting/analysis.py
Modified:
zorg/trunk/lnt/lnt/server/reporting/runs.py
zorg/trunk/lnt/lnt/util/NTEmailReport.py
Added: zorg/trunk/lnt/lnt/server/reporting/analysis.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/server/reporting/analysis.py?rev=147165&view=auto
==============================================================================
--- zorg/trunk/lnt/lnt/server/reporting/analysis.py (added)
+++ zorg/trunk/lnt/lnt/server/reporting/analysis.py Thu Dec 22 13:24:32 2011
@@ -0,0 +1,132 @@
+"""
+Utilities for helping with the analysis of data, for reporting purposes.
+"""
+
+from lnt.server.ui import util
+from lnt.db.runinfo import ComparisonResult
+from lnt.testing import PASS, FAIL, XFAIL
+
+class RunInfo(object):
+ def __init__(self, testsuite):
+ self.testsuite = testsuite
+
+ self.sample_map = util.multidict()
+ self.loaded_run_ids = set()
+
+ def get_run_comparison_result(self, run, compare_to, test_id, field,
+ comparison_window=[]):
+ # Get the field which indicates the requested field's status.
+ status_field = field.status_field
+
+ # Load the sample data for the current and previous runs and the
+ # comparison window.
+ if compare_to is None:
+ compare_id = None
+ else:
+ compare_id = compare_to.id
+ runs_to_load = set([r.id for r in comparison_window])
+ runs_to_load.add(run.id)
+ if compare_id is not None:
+ runs_to_load.add(compare_id)
+ self._load_samples_for_runs(runs_to_load)
+
+ # Lookup the current and previous samples.
+ run_samples = self.sample_map.get((run.id, test_id), ())
+ prev_samples = self.sample_map.get((compare_id, test_id), ())
+
+ # Determine whether this (test,pset) passed or failed in the current and
+ # previous runs.
+ #
+ # FIXME: Support XFAILs and non-determinism (mixed fail and pass)
+ # better.
+ run_failed = prev_failed = False
+ if status_field:
+ for sample in run_samples:
+ run_failed |= sample.get_field(status_field) == FAIL
+ for sample in prev_samples:
+ prev_failed |= sample.get_field(status_field) == FAIL
+
+ # Get the current and previous values.
+ run_values = [s.get_field(field) for s in run_samples]
+ prev_values = [s.get_field(field) for s in prev_samples]
+ if run_values:
+ run_value = min(run_values)
+ else:
+ run_value = None
+ if prev_values:
+ prev_value = min(prev_values)
+ else:
+ prev_value = None
+
+ # If we have multiple values for this run, use that to estimate the
+ # distribution.
+ if run_values and len(run_values) > 1:
+ stddev = stats.standard_deviation(run_values)
+ MAD = stats.median_absolute_deviation(run_values)
+ stddev_mean = stats.mean(run_values)
+ stddev_is_estimated = False
+ else:
+ stddev = None
+ MAD = None
+ stddev_mean = None
+ stddev_is_estimated = False
+
+ # If we are missing current or comparison values we are done.
+ if run_value is None or prev_value is None:
+ return ComparisonResult(
+ run_value, prev_value, delta=None,
+ pct_delta = None, stddev = stddev, MAD = MAD,
+ cur_failed = run_failed, prev_failed = prev_failed,
+ samples = run_values)
+
+ # Compute the comparison status for the test value.
+ delta = run_value - prev_value
+ if prev_value != 0:
+ pct_delta = delta / prev_value
+ else:
+ pct_delta = 0.0
+
+ # If we don't have an estimate for the distribution, attempt to "guess"
+ # it using the comparison window.
+ #
+ # FIXME: We can substantially improve the algorithm for guessing the
+ # noise level from a list of values. Probably better to just find a way
+ # to kill this code though.
+ if stddev is None:
+ # Get all previous values in the comparison window, for passing
+ # runs.
+ #
+ # FIXME: This is using the wrong status kind. :/
+ prev_samples = [v for run_id in comparison_window
+ for v in self.sample_map.get((run_id, test_id), ())]
+ if prev_samples:
+ # Filter out failing samples.
+ if status_field:
+ prev_samples = [s for s in prev_samples
+ if s.get_field(status_field) == PASS]
+ prev_values = [s.get_field(field)
+ for s in prev_samples]
+ stddev = stats.standard_deviation(prev_values)
+ MAD = stats.median_absolute_deviation(prev_values)
+ stddev_mean = stats.mean(prev_values)
+ stddev_is_estimated = True
+
+ return ComparisonResult(run_value, prev_value, delta,
+ pct_delta, stddev, MAD,
+ run_failed, prev_failed, run_values,
+ stddev_mean, stddev_is_estimated)
+
+ def _load_samples_for_runs(self, run_ids):
+ # Find the set of new runs to load.
+ to_load = set(run_ids) - self.loaded_run_ids
+ if not to_load:
+ return
+
+ # Batch load all of the samples for the needed runs.
+ q = self.testsuite.query(self.testsuite.Sample)
+ q = q.filter(self.testsuite.Sample.run_id.in_(to_load))
+ for sample in q:
+ self.sample_map[(sample.run_id, sample.test_id)] = sample
+
+ self.loaded_run_ids |= to_load
+
Modified: zorg/trunk/lnt/lnt/server/reporting/runs.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/server/reporting/runs.py?rev=147165&r1=147164&r2=147165&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/server/reporting/runs.py (original)
+++ zorg/trunk/lnt/lnt/server/reporting/runs.py Thu Dec 22 13:24:32 2011
@@ -5,8 +5,11 @@
import StringIO
import os
+import lnt.server.reporting.analysis
+from lnt.db import runinfo
+
def generate_run_report(run, baseurl, only_html_body = False,
- num_comparison_runs = 10):
+ num_comparison_runs = 10, result = None):
"""
generate_run_report(...) -> (str: subject, str: text_report,
str: html_report)
@@ -17,10 +20,10 @@
assert num_comparison_runs > 0
-
ts = run.testsuite
machine = run.machine
machine_parameters = machine.parameters
+ sri = lnt.server.reporting.analysis.RunInfo(ts)
# Gather the runs to use for statistical data.
comparison_window = list(ts.get_previous_runs_on_machine(
@@ -32,6 +35,70 @@
else:
compare_to = None
+ # Get the test names.
+ test_names = ts.query(ts.Test.name, ts.Test.id).order_by(ts.Test.name).all()
+
+ # Gather the changes to report, organized by field and then collated by
+ # change type.
+ primary_fields = list(ts.Sample.get_primary_fields())
+ num_total_tests = len(primary_fields) * len(test_names)
+ test_results = []
+ for field in primary_fields:
+ new_failures = []
+ new_passes = []
+ perf_regressions = []
+ perf_improvements = []
+ removed_tests = []
+ added_tests = []
+ existing_failures = []
+ unchanged_tests = []
+ for name,test_id in test_names:
+ cr = sri.get_run_comparison_result(run, compare_to, test_id, field,
+ comparison_window)
+ test_status = cr.get_test_status()
+ perf_status = cr.get_value_status()
+ if test_status == runinfo.REGRESSED:
+ bucket = new_failures
+ elif test_status == runinfo.IMPROVED:
+ bucket = new_passes
+ elif cr.current is None and cr.previous is not None:
+ bucket = removed_tests
+ elif cr.current is not None and cr.previous is None:
+ bucket = added_tests
+ elif test_status == runinfo.UNCHANGED_FAIL:
+ bucket = existing_failures
+ elif perf_status == runinfo.REGRESSED:
+ bucket = perf_regressions
+ elif perf_status == runinfo.IMPROVED:
+ bucket = perf_improvements
+ else:
+ bucket = unchanged_tests
+
+ bucket.append((name, cr))
+
+ test_results.append(
+ (field, (('New Failures', new_failures, False),
+ ('New Passes', new_passes, False),
+ ('Performance Regressions', perf_regressions, True),
+ ('Performance Improvements', perf_improvements, True),
+ ('Removed Tests', removed_tests, False),
+ ('Added Tests', added_tests, False),
+ ('Existing Failures', existing_failures, False),
+ ('Unchanged Tests', unchanged_tests, False))))
+
+ # Collect the simplified results, if desired, for sending back to clients.
+ if result is not None:
+ pset_results = []
+ result['test_results'] = [{ 'pset' : (), 'results' : pset_results}]
+ for field,field_results in test_results:
+ for _,bucket,_ in field_results:
+ for name,cr in bucket:
+ # FIXME: Include additional information about performance
+ # changes.
+ pset_results.append(("%s.%s" % (name, field.name),
+ cr.get_test_status(),
+ cr.get_value_status()))
+
# Begin report generation...
subject = """%s test results: %s""" % (
machine.name, run.start_time.strftime('%Y-%m-%d %H:%M:%S %Z PST'))
@@ -107,6 +174,41 @@
print >>html_report, """(%s:%d)</b></p>""" % (compare_to.machine.name,
compare_to.machine.id)
+ # Generate the summary of the changes.
+ total_changes = sum(len(bucket)
+ for _,field_results in test_results
+ for name,bucket,_ in field_results
+ if name != 'Unchanged Tests')
+
+ print >>report, """==============="""
+ print >>report, """Tests Summary"""
+ print >>report, """==============="""
+ print >>report
+ print >>html_report, """
+<hr>
+<h3>Tests Summary</h3>
+<table>
+<thead><tr><th>Status Group</th><th align="right">#</th></tr></thead>
+"""
+ # For now, we aggregate across all bucket types for reports.
+ for i,(name,_,_) in enumerate(test_results[0][1]):
+ num_items = sum(len(field_results[i][1])
+ for _,field_results in test_results)
+ if num_items:
+ print >>report, '%s: %d' % (name, num_items)
+ print >>html_report, """
+<tr><td>%s</td><td align="right">%d</td></tr>""" % (
+ name, num_items)
+ print >>report, """Total Tests: %d""" % num_total_tests
+ print >>report
+ print >>html_report, """
+<tfoot>
+ <tr><td><b>Total Tests</b></td><td align="right"><b>%d</b></td></tr>
+</tfoot>
+</table>
+""" % num_total_tests
+
+ # Finish up the HTML report (wrapping the body, if necessary).
html_report = html_report.getvalue()
if not only_html_body:
# We embed the additional resources, so that the message is self
@@ -116,7 +218,7 @@
style_css = open(os.path.join(static_path,
"style.css")).read()
- html_report = """
+ html_report = """\
<html>
<head>
<style type="text/css">
Modified: zorg/trunk/lnt/lnt/util/NTEmailReport.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/util/NTEmailReport.py?rev=147165&r1=147164&r2=147165&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/util/NTEmailReport.py (original)
+++ zorg/trunk/lnt/lnt/util/NTEmailReport.py Thu Dec 22 13:24:32 2011
@@ -470,7 +470,8 @@
# We haven't implemented V4DB support yet in reports.
if isinstance(db, lnt.server.db.v4db.V4DB):
return lnt.server.reporting.runs.generate_run_report(
- run, baseurl=baseurl, only_html_body=only_html_body)
+ run, baseurl=baseurl, only_html_body=only_html_body,
+ result=result)
# Use a simple report unless the tag indicates this is an old style nightly
# test run.
More information about the llvm-commits
mailing list