[LNT] r255953 - Initial regression correlation
Chris Matthews via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 17 17:17:09 PST 2015
Author: cmatthews
Date: Thu Dec 17 19:17:08 2015
New Revision: 255953
URL: http://llvm.org/viewvc/llvm-project?rev=255953&view=rev
Log:
Initial regression correlation
Added:
lnt/trunk/tests/server/ui/change_processing.py
Modified:
lnt/trunk/lnt/server/db/fieldchange.py
Modified: lnt/trunk/lnt/server/db/fieldchange.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/fieldchange.py?rev=255953&r1=255952&r2=255953&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/fieldchange.py (original)
+++ lnt/trunk/lnt/server/db/fieldchange.py Thu Dec 17 19:17:08 2015
@@ -1,7 +1,9 @@
+import re
import sqlalchemy.sql
import lnt.server.reporting.analysis
from lnt.testing.util.commands import warning
from lnt.testing.util.commands import note
+from lnt.server.ui.regression_views import new_regression
# How many runs backwards to use in the previous run set.
# More runs are slower (more DB access), but may provide
# more accurate results.
@@ -19,6 +21,7 @@ def regenerate_fieldchanges_for_run(ts,
filter(ts.Run.order_id == run.order_id). \
filter(ts.Run.machine_id == run.machine_id). \
all()
+ regressions = ts.query(ts.Regression).all()[::-1]
previous_runs = ts.get_previous_runs_on_machine(run, FIELD_CHANGE_LOOKBACK)
next_runs = ts.get_next_runs_on_machine(run, FIELD_CHANGE_LOOKBACK)
@@ -78,11 +81,69 @@ def regenerate_fieldchanges_for_run(ts,
field=field)
f.test_id = test_id
ts.add(f)
+ ts.commit()
+ found, new_reg = identify_related_changes(ts, regressions, f)
+ if found:
+ regressions.append(new_reg)
note("Found field change: {}".format(run.machine))
-
+
# Always update FCs with new values.
if f:
f.old_value = result.previous
f.new_value = result.current
f.run = run
ts.commit()
+
+
+def is_overlaping(fc1, fc2):
+ """"Returns true if these two orders intersect. """
+ r1_min = fc1.start_order
+ r1_max = fc1.end_order
+ r2_min = fc2.start_order
+ r2_max = fc2.end_order
+ return (r1_min == r2_min and r1_max == r2_max) or \
+ (r1_min < r2_max and r2_min < r1_max)
+
+
+def rebuild_title(regression, new_size):
+ """Update the title of a regresson."""
+ if re.match("Regression of \d+ benchmarks.*", regression.title):
+ new_title = "Regression of {} benchmarks".format(new_size)
+ regression.title = new_title
+ print new_title
+ return regression
+
+
+def identify_related_changes(ts, regressions, fc):
+ """Can we find a home for this change in some existing regression? """
+ for regression in regressions:
+ regression_indicators = ts.query(ts.RegressionIndicator) \
+ .filter(ts.RegressionIndicator.regression_id == regression.id) \
+ .all()
+ for change in regression_indicators:
+ regression_change = change.field_change
+ if is_overlaping(regression_change, fc):
+ confidence = 0
+ relation = ["Revision"]
+ if regression_change.machine == fc.machine:
+ confidence += 1
+ relation.append("Machine")
+ if regression_change.test == fc.test:
+ confidence += 1
+ relation.append("Test")
+ if regression_change.field == fc.field:
+ confidence += 1
+ relation.append("Field")
+
+ if confidence >= 2:
+ # Matching
+ note("Found a match:" + str(regression) + " On " +
+ ', '.join(relation))
+ ri = ts.RegressionIndicator(regression, fc)
+ ts.add(ri)
+ # Update the default title if needed.
+ rebuild_title(regression, len(regression_indicators) + 1)
+ return (True, regression)
+ note("Could not find a partner, creating new Regression for change")
+ new_reg = new_regression(ts, [fc.id])
+ return (False, new_reg)
Added: lnt/trunk/tests/server/ui/change_processing.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/ui/change_processing.py?rev=255953&view=auto
==============================================================================
--- lnt/trunk/tests/server/ui/change_processing.py (added)
+++ lnt/trunk/tests/server/ui/change_processing.py Thu Dec 17 19:17:08 2015
@@ -0,0 +1,148 @@
+# Check that the LNT REST JSON API is working.
+# create temporary instance
+# RUN: rm -rf %t.instance
+# RUN: python %{shared_inputs}/create_temp_instance.py \
+# RUN: %s %{shared_inputs}/SmallInstance \
+# RUN: %t.instance %S/Inputs/V4Pages_extra_records.sql
+#
+# RUN: python %s %t.instance
+
+import unittest
+import logging
+import sys
+import datetime
+
+from lnt.server.config import Config
+from lnt.server.db import v4db
+from lnt.server.db.fieldchange import is_overlaping, identify_related_changes
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+class ChangeProcessingTests(unittest.TestCase):
+ """Test the REST api."""
+
+ def setUp(self):
+ """Bind to the LNT test instance."""
+ db = v4db.V4DB("sqlite:///:memory:", Config.dummyInstance(), echo=False)
+
+ # Get the test suite wrapper.
+ self.ts_db = db.testsuite['nts']
+
+ def test_fc(self):
+ pass
+
+ def _mkorder(self, ts, rev):
+ order = ts.Order()
+ order.llvm_project_revision = rev
+ ts.add(order)
+ return order
+
+ def test_change_grouping_criteria(self):
+ ts_db = self.ts_db
+ order1234 = self._mkorder(ts_db, "1234")
+ order1235 = self._mkorder(ts_db, "1235")
+ order1236 = self._mkorder(ts_db, "1236")
+ order1237 = self._mkorder(ts_db, "1237")
+ order1238 = self._mkorder(ts_db, "1238")
+
+ start_time = end_time = datetime.datetime.utcnow()
+ machine = ts_db.Machine("test-machine")
+ test = ts_db.Test("test-a")
+ machine2 = ts_db.Machine("test-machine2")
+ test2 = ts_db.Test("test-b")
+
+ run = ts_db.Run(machine, order1235, start_time,
+ end_time)
+ sample = ts_db.Sample(run, test, compile_time=1.0,
+ score=4.2)
+ a_field = list(sample.get_primary_fields())[0]
+ a_field2 = list(sample.get_primary_fields())[1]
+
+ field_change = ts_db.FieldChange(order1234,
+ order1235,
+ machine,
+ test,
+ a_field)
+ ts_db.add(field_change)
+
+ field_change2 = ts_db.FieldChange(order1235, order1236, machine,
+ test,
+ a_field)
+ ts_db.add(field_change2)
+
+ field_change3 = ts_db.FieldChange(order1237, order1238, machine,
+ test,
+ a_field)
+ ts_db.add(field_change3)
+
+ regression = ts_db.Regression("Some regression title", "PR1234")
+ ts_db.add(regression)
+
+ regression_indicator1 = ts_db.RegressionIndicator(regression,
+ field_change)
+ regression_indicator2 = ts_db.RegressionIndicator(regression,
+ field_change2)
+
+ ts_db.add(regression_indicator1)
+ ts_db.add(regression_indicator2)
+
+ # All the regressions we detected.
+ regressions = [regression]
+
+ # Check simple overlap checks work.
+ self.assertTrue(is_overlaping(field_change, field_change2),
+ "Should be overlapping")
+ self.assertFalse(is_overlaping(field_change, field_change3),
+ "Should not be overlapping")
+
+ # Check non-overlapping changes are always False.
+ ret, reg = identify_related_changes(ts_db, regressions, field_change3)
+ self.assertFalse(ret, "Ranges don't overlap, should not match")
+ regressions.append(reg)
+ # Check a regression matches if all fields match.
+ ret, _ = identify_related_changes(ts_db, regressions, field_change)
+ self.assertTrue(ret, "Should Match.")
+
+ field_change7 = ts_db.FieldChange(order1234,
+ order1235,
+ machine2,
+ test2,
+ a_field)
+ ts_db.add(field_change7)
+ ret, reg = identify_related_changes(ts_db, regressions, field_change7)
+ self.assertNotEquals(regression, reg)
+ self.assertFalse(ret, "Should not match with differnt machine and tests.")
+ regressions.append(reg)
+ field_change4 = ts_db.FieldChange(order1234,
+ order1235,
+ machine2,
+ test,
+ a_field)
+
+ # Check a regression matches if all fields match.
+ ret, _ = identify_related_changes(ts_db, regressions, field_change4)
+ self.assertTrue(ret, "Should Match with differnt machine.")
+
+ field_change5 = ts_db.FieldChange(order1234,
+ order1235,
+ machine,
+ test2,
+ a_field)
+
+ # Check a regression matches if all fields match.
+ ret, _ = identify_related_changes(ts_db, regressions, field_change5)
+ self.assertTrue(ret, "Should Match with differnt tests.")
+ field_change6 = ts_db.FieldChange(order1234,
+ order1235,
+ machine,
+ test,
+ a_field2)
+
+ # Check a regression matches if all fields match.
+ ret, _ = identify_related_changes(ts_db, regressions, field_change6)
+ self.assertTrue(ret, "Should Match with differnt fields.")
+
+
+if __name__ == '__main__':
+ unittest.main(argv=[sys.argv[0], ])
More information about the llvm-commits
mailing list