[LNT] r235586 - [LNT] Add the ability for test samples to be interpreted as "bigger is better"
James Molloy
james.molloy at arm.com
Thu Apr 23 05:18:58 PDT 2015
Author: jamesm
Date: Thu Apr 23 07:18:58 2015
New Revision: 235586
URL: http://llvm.org/viewvc/llvm-project?rev=235586&view=rev
Log:
[LNT] Add the ability for test samples to be interpreted as "bigger is better"
The default assumption in LNT currently is that smaller sample values are
better. This adds a new column to the SampleFields table (and a corresponding
migration), and does the small amount of plumbing to feed it through.
Added:
lnt/trunk/lnt/server/db/migrations/upgrade_3_to_4.py
Modified:
lnt/trunk/lnt/server/db/testsuite.py
lnt/trunk/lnt/server/reporting/analysis.py
lnt/trunk/lnt/server/ui/templates/reporting/runs.html
lnt/trunk/lnt/server/ui/templates/v4_run.html
Added: lnt/trunk/lnt/server/db/migrations/upgrade_3_to_4.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_3_to_4.py?rev=235586&view=auto
==============================================================================
--- lnt/trunk/lnt/server/db/migrations/upgrade_3_to_4.py (added)
+++ lnt/trunk/lnt/server/db/migrations/upgrade_3_to_4.py Thu Apr 23 07:18:58 2015
@@ -0,0 +1,22 @@
+# Version 4 of the database adds the bigger_is_better column to StatusField.
+
+import os
+import sys
+
+import sqlalchemy
+
+###
+# Upgrade TestSuite
+
+def upgrade(engine):
+ # Create a session.
+ session = sqlalchemy.orm.sessionmaker(engine)()
+
+ # Add our new column. SQLAlchemy doesn't really support adding a new column to an
+ # existing table, so instead of requiring SQLAlchemy-Migrate, just execute the raw SQL.
+ session.connection().execute("""
+ALTER TABLE TestSuiteSampleFields
+ADD COLUMN bigger_is_better INTEGER DEFAULT 0
+""")
+
+ session.commit()
Modified: lnt/trunk/lnt/server/db/testsuite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/testsuite.py?rev=235586&r1=235585&r2=235586&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/testsuite.py (original)
+++ lnt/trunk/lnt/server/db/testsuite.py Thu Apr 23 07:18:58 2015
@@ -194,11 +194,17 @@ class SampleField(FieldMixin, Base):
'TestSuiteSampleFields.ID'))
status_field = relation('SampleField', remote_side=id)
- def __init__(self, name, type, info_key, status_field = None):
+ # Most real type samples assume lower values are better than higher values.
+ # This assumption can be inverted by setting this column to nonzero.
+ bigger_is_better = Column("bigger_is_better", Integer)
+
+ def __init__(self, name, type, info_key, status_field = None,
+ bigger_is_better = 0):
self.name = name
self.type = type
self.info_key = info_key
self.status_field = status_field
+ self.bigger_is_better = bigger_is_better
# Index of this column.
self.index = None
Modified: lnt/trunk/lnt/server/reporting/analysis.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/analysis.py?rev=235586&r1=235585&r2=235586&view=diff
==============================================================================
--- lnt/trunk/lnt/server/reporting/analysis.py (original)
+++ lnt/trunk/lnt/server/reporting/analysis.py Thu Apr 23 07:18:58 2015
@@ -30,7 +30,7 @@ def calc_geomean(run_values):
class ComparisonResult:
def __init__(self, cur_value, prev_value, delta, pct_delta, stddev, MAD,
cur_failed, prev_failed, samples, prev_samples, stddev_mean = None,
- confidence_lv = .05):
+ confidence_lv = .05, bigger_is_better = False):
self.current = cur_value
self.previous = prev_value
self.delta = delta
@@ -43,6 +43,7 @@ class ComparisonResult:
self.prev_samples = prev_samples
self.stddev_mean = stddev_mean
self.confidence_lv = confidence_lv
+ self.bigger_is_better = bigger_is_better
def get_samples(self):
return self.samples
@@ -122,9 +123,9 @@ class ComparisonResult:
# If the delta is significant, return
if is_significant:
if self.delta < 0:
- return IMPROVED
+ return REGRESSED if self.bigger_is_better else IMPROVED
else:
- return REGRESSED
+ return IMPROVED if self.bigger_is_better else REGRESSED
else:
return UNCHANGED_PASS
@@ -133,9 +134,9 @@ class ComparisonResult:
# accurately.
if not ignore_small or abs(self.pct_delta) >= .002:
if self.pct_delta < 0:
- return IMPROVED
+ return REGRESSED if self.bigger_is_better else IMPROVED
else:
- return REGRESSED
+ return IMPROVED if self.bigger_is_better else REGRESSED
else:
return UNCHANGED_PASS
@@ -241,7 +242,8 @@ class RunInfo(object):
pct_delta = None, stddev = stddev, MAD = MAD,
cur_failed = run_failed, prev_failed = prev_failed,
samples = run_values, prev_samples = prev_values,
- confidence_lv = self.confidence_lv)
+ confidence_lv = self.confidence_lv,
+ bigger_is_better = field.bigger_is_better)
# Compute the comparison status for the test value.
delta = run_value - prev_value
@@ -253,7 +255,8 @@ class RunInfo(object):
return ComparisonResult(run_value, prev_value, delta,
pct_delta, stddev, MAD,
run_failed, prev_failed, run_values,
- prev_values, stddev_mean, self.confidence_lv)
+ prev_values, stddev_mean, self.confidence_lv,
+ bigger_is_better = field.bigger_is_better)
def get_geomean_comparison_result(self, run, compare_to, field, tests,
@@ -281,7 +284,8 @@ class RunInfo(object):
prev_failed=prev_values and not prev_geomean,
samples=[run_geomean] if run_geomean else [],
prev_samples=[prev_geomean] if prev_geomean else [],
- confidence_lv=0)
+ confidence_lv=0,
+ bigger_is_better = field.bigger_is_better)
def _load_samples_for_runs(self, run_ids):
# Find the set of new runs to load.
Modified: lnt/trunk/lnt/server/ui/templates/reporting/runs.html
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/reporting/runs.html?rev=235586&r1=235585&r2=235586&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/templates/reporting/runs.html (original)
+++ lnt/trunk/lnt/server/ui/templates/reporting/runs.html Thu Apr 23 07:18:58 2015
@@ -36,13 +36,13 @@
<td style="{{ styles['td'] }}">
<a href="{{ run_url }}/graph?{{ [('test.%d' % test_id, field_index)] | urlencode }}">{{ name }}</a>
</td>
- {{ cr.pct_delta|aspctcell(style=styles['td'])|safe }}
+ {{ cr.pct_delta|aspctcell(style=styles['td'],reverse=cr.bigger_is_better)|safe }}
<td style="{{ styles['td'] }}">{{ "%.4f" | format(cr.previous) }}</td>
<td style="{{ styles['td'] }}">{{ "%.4f" | format(cr.current) }}</td>
<td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(cr.stddev, '-') }}</td>
{% if secondary_info %}
{% set a_cr = secondary_info[(name, field)] %}
- {{ a_cr.pct_delta|aspctcell(style=styles['td'])|safe }}
+ {{ a_cr.pct_delta|aspctcell(style=styles['td'],reverse=cr.bigger_is_better)|safe }}
<td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(a_cr.stddev, '-') }}</td>
{% endif %}
</tr>
Modified: lnt/trunk/lnt/server/ui/templates/v4_run.html
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/v4_run.html?rev=235586&r1=235585&r2=235586&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/templates/v4_run.html (original)
+++ lnt/trunk/lnt/server/ui/templates/v4_run.html Thu Apr 23 07:18:58 2015
@@ -53,7 +53,7 @@
{% if (options.show_all or
value_status == analysis.REGRESSED or
value_status == analysis.IMPROVED) %}
- {{ cr.pct_delta|aspctcell|safe }}
+ {{ cr.pct_delta|aspctcell(reverse=cr.bigger_is_better)|safe }}
{% else %}
<td>-</td>
{% endif %}
More information about the llvm-commits
mailing list