[LNT] r255957 - Regression evolution rule: detect fixed regressions and move them
Chris Matthews via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 17 17:32:48 PST 2015
Author: cmatthews
Date: Thu Dec 17 19:32:48 2015
New Revision: 255957
URL: http://llvm.org/viewvc/llvm-project?rev=255957&view=rev
Log:
Regression evolution rule: detect fixed regressions and move them
Evolve regressions. When regressions that no longer exist are detected in the system, move them to a new state (ignored or verified depending on the original state).
Added:
lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py
Modified:
lnt/trunk/lnt/server/db/fieldchange.py
lnt/trunk/lnt/server/db/rules.py
lnt/trunk/lnt/server/db/v4db.py
lnt/trunk/lnt/server/ui/regression_views.py
lnt/trunk/tests/server/ui/change_processing.py
Modified: lnt/trunk/lnt/server/db/fieldchange.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/fieldchange.py?rev=255957&r1=255956&r2=255957&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/fieldchange.py (original)
+++ lnt/trunk/lnt/server/db/fieldchange.py Thu Dec 17 19:32:48 2015
@@ -4,6 +4,8 @@ import lnt.server.reporting.analysis
from lnt.testing.util.commands import warning
from lnt.testing.util.commands import note
from lnt.server.ui.regression_views import new_regression, RegressionState
+from lnt.server.ui.regression_views import get_ris
+from lnt.server.db import rules
# How many runs backwards to use in the previous run set.
# More runs are slower (more DB access), but may provide
# more accurate results.
@@ -92,6 +94,8 @@ def regenerate_fieldchanges_for_run(ts,
f.new_value = result.current
f.run = run
ts.commit()
+ rules.post_submission_hooks(ts, regressions)
+
def is_overlaping(fc1, fc2):
@@ -130,12 +134,11 @@ def rebuild_title(ts, regression):
return regression
+
def identify_related_changes(ts, regressions, fc):
"""Can we find a home for this change in some existing regression? """
for regression in regressions:
- regression_indicators = ts.query(ts.RegressionIndicator) \
- .filter(ts.RegressionIndicator.regression_id == regression.id) \
- .all()
+ regression_indicators = get_ris(ts, regression)
for change in regression_indicators:
regression_change = change.field_change
if is_overlaping(regression_change, fc):
Modified: lnt/trunk/lnt/server/db/rules.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/rules.py?rev=255957&r1=255956&r2=255957&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/rules.py (original)
+++ lnt/trunk/lnt/server/db/rules.py Thu Dec 17 19:32:48 2015
@@ -65,5 +65,9 @@ def register_hooks():
HOOKS[hook_name].append(globals[hook_name])
return HOOKS
+def post_submission_hooks(ts, run_id):
+ """Run all the post submission hooks on the submitted run."""
+ for func in HOOKS['post_submission_hook']:
+ func(ts, run)
logger = logging.getLogger(__name__)
Added: lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py?rev=255957&view=auto
==============================================================================
--- lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py (added)
+++ lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py Thu Dec 17 19:32:48 2015
@@ -0,0 +1,65 @@
+"""Check if a regression is fixed, and move to differnt sate.
+Detcted + fixed -> Ignored
+Staged or Active + fixed -> Verify
+"""
+from lnt.server.ui.regression_views import RegressionState
+from lnt.server.ui.regression_views import get_cr_for_field_change, get_ris
+from lnt.testing.util.commands import note
+
+def _fixed_rind(ts, rind):
+ """Is this regression indicator fixed?"""
+ fc = rind.field_change
+ if fc is None:
+ return False
+ current_cr, _ = get_cr_for_field_change(ts, fc, current=True)
+ if current_cr.pct_delta < 0.01:
+ return True
+ else:
+ return False
+
+def is_fixed(ts, regression):
+ """Comparing the current value to the regression, is this regression now
+ fixed?
+ """
+ r_inds = get_ris(ts, regression)
+ changes = [r.field_change for r in r_inds]
+ fixes = [_fixed_rind(ts, x) for x in changes]
+ return all(fixes)
+
+
+
+def regression_evolution(ts, regressions):
+ """Analyse regressions. If they have changes, process them.
+ Look at each regression in state detect. Move to ignore if it is fixed.
+ Look at each regression in state stage. Move to verify if fixed.
+ Look at regressions in detect, do they match our policy? If no, move to NTBF.
+
+ """
+ detects = [r for r in regressions if r.state == RegressionState.DETECTED]
+
+ for regression in detects:
+ if is_fixed(ts, regression):
+ note("Detected fixed regression" + str(regression))
+ regression.state = RegressionState.IGNORED
+ regression.title = regression.title + " [Detected Fixed]"
+ ts.commit()
+
+ staged = [r for r in regressions if r.state == RegressionState.STAGED]
+
+ for regression in staged:
+ if is_fixed(ts, regression):
+ note("Staged fixed regression" + str(regression))
+ regression.state = RegressionState.DETECTED_FIXED
+ regression.title = regression.title + " [Detected Fixed]"
+ ts.commit()
+
+ active = [r for r in regressions if r.state == RegressionState.ACTIVE]
+
+ for regression in active:
+ if is_fixed(ts, regression):
+ note("Active fixed regression" + str(regression))
+ regression.state = RegressionState.DETECTED_FIXED
+ regression.title = regression.title + " [Detected Fixed]"
+ ts.commit()
+
+post_submission_hook = regression_evolution
Modified: lnt/trunk/lnt/server/db/v4db.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/v4db.py?rev=255957&r1=255956&r2=255957&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/v4db.py (original)
+++ lnt/trunk/lnt/server/db/v4db.py Thu Dec 17 19:32:48 2015
@@ -131,6 +131,13 @@ class V4DB(object):
def close(self):
if self.session is not None:
self.session.close()
+
+ def close_engine(self):
+ """Rip down everything about this path, so we can make it
+ new again. This is used for tests that need to make a fresh
+ in memory database."""
+ self._engine.pop(self.path)
+ V4DB._db_updated.remove(self.path)
def settings(self):
"""All the setting needed to recreate this instnace elsewhere."""
Modified: lnt/trunk/lnt/server/ui/regression_views.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/regression_views.py?rev=255957&r1=255956&r2=255957&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/regression_views.py (original)
+++ lnt/trunk/lnt/server/ui/regression_views.py Thu Dec 17 19:32:48 2015
@@ -310,6 +310,12 @@ class EditRegressionForm(Form):
state = SelectField(u'State', choices=choices)
+def get_ris(ts, regression):
+ return ts.query(ts.RegressionIndicator) \
+ .filter(ts.RegressionIndicator.regression_id == regression.id) \
+ .all()
+
+
@v4_route("/regressions/<int:id>", methods=["GET", "POST"])
def v4_regression_detail(id):
ts = request.get_testsuite()
Modified: lnt/trunk/tests/server/ui/change_processing.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/ui/change_processing.py?rev=255957&r1=255956&r2=255957&view=diff
==============================================================================
--- lnt/trunk/tests/server/ui/change_processing.py (original)
+++ lnt/trunk/tests/server/ui/change_processing.py Thu Dec 17 19:32:48 2015
@@ -15,134 +15,159 @@ import datetime
from lnt.server.config import Config
from lnt.server.db import v4db
from lnt.server.db.fieldchange import is_overlaping, identify_related_changes
+from lnt.server.db.fieldchange import rebuild_title, RegressionState
+from lnt.server.db.rules import rule_update_fixed_regressions
logging.basicConfig(level=logging.DEBUG)
class ChangeProcessingTests(unittest.TestCase):
- """Test the REST api."""
+ """Test fieldchange and regression building."""
def setUp(self):
"""Bind to the LNT test instance."""
- db = v4db.V4DB("sqlite:///:memory:", Config.dummyInstance(), echo=False)
+
+ self.db = v4db.V4DB("sqlite:///:memory:", Config.dummyInstance(), echo=False)
# Get the test suite wrapper.
- self.ts_db = db.testsuite['nts']
-
- def test_fc(self):
- pass
-
- def _mkorder(self, ts, rev):
- order = ts.Order()
- order.llvm_project_revision = rev
- ts.add(order)
- return order
-
- def test_change_grouping_criteria(self):
- ts_db = self.ts_db
- order1234 = self._mkorder(ts_db, "1234")
- order1235 = self._mkorder(ts_db, "1235")
- order1236 = self._mkorder(ts_db, "1236")
- order1237 = self._mkorder(ts_db, "1237")
- order1238 = self._mkorder(ts_db, "1238")
+ ts_db = self.ts_db = self.db.testsuite['nts']
+
+ order1234 = self.order1234 = self._mkorder(ts_db, "1234")
+ order1235 = self.order1235 = self._mkorder(ts_db, "1235")
+ order1236 = self.order1236 = self._mkorder(ts_db, "1236")
+ order1237 = self.order1237 = self._mkorder(ts_db, "1237")
+ order1238 = self.order1238 = self._mkorder(ts_db, "1238")
start_time = end_time = datetime.datetime.utcnow()
- machine = ts_db.Machine("test-machine")
- test = ts_db.Test("test-a")
- machine2 = ts_db.Machine("test-machine2")
- test2 = ts_db.Test("test-b")
+ machine = self.machine = ts_db.Machine("test-machine")
+ test = self.test = ts_db.Test("test-a")
+ machine2 = self.machine2 = ts_db.Machine("test-machine2")
+ test2 = self.test2 = ts_db.Test("test-b")
- run = ts_db.Run(machine, order1235, start_time,
+ run = self.run = ts_db.Run(machine, order1235, start_time,
end_time)
sample = ts_db.Sample(run, test, compile_time=1.0,
score=4.2)
- a_field = list(sample.get_primary_fields())[0]
- a_field2 = list(sample.get_primary_fields())[1]
+ a_field = self.a_field = list(sample.get_primary_fields())[0]
+ a_field2 = self.a_field2 = list(sample.get_primary_fields())[1]
- field_change = ts_db.FieldChange(order1234,
- order1235,
+ field_change = self.field_change = ts_db.FieldChange(order1234,
+ order1236,
machine,
test,
a_field)
ts_db.add(field_change)
- field_change2 = ts_db.FieldChange(order1235, order1236, machine,
+ field_change2 = self.field_change2 = ts_db.FieldChange(order1235, order1236, machine,
test,
a_field)
ts_db.add(field_change2)
- field_change3 = ts_db.FieldChange(order1237, order1238, machine,
+ field_change3 = self.field_change3 = ts_db.FieldChange(order1237, order1238, machine,
test,
a_field)
ts_db.add(field_change3)
- regression = ts_db.Regression("Some regression title", "PR1234")
- ts_db.add(regression)
+ regression = self.regression = ts_db.Regression("Regression of 1 benchmarks:", "PR1234",
+ RegressionState.DETECTED)
+ ts_db.add(self.regression)
- regression_indicator1 = ts_db.RegressionIndicator(regression,
+ self.regression_indicator1 = ts_db.RegressionIndicator(regression,
field_change)
- regression_indicator2 = ts_db.RegressionIndicator(regression,
+ self.regression_indicator2 = ts_db.RegressionIndicator(regression,
field_change2)
- ts_db.add(regression_indicator1)
- ts_db.add(regression_indicator2)
+ ts_db.add(self.regression_indicator1)
+ ts_db.add(self.regression_indicator2)
# All the regressions we detected.
- regressions = [regression]
+ self.regressions = [regression]
+
+ def tearDown(self):
+ self.db.close_engine()
+
+ def test_fc(self):
+ pass
+
+ def _mkorder(self, ts, rev):
+ order = ts.Order()
+ order.llvm_project_revision = rev
+ ts.add(order)
+ return order
+
+ def test_rebuild_title(self):
+ ts = self.ts_db
+
+ def test_change_grouping_criteria(self):
+ ts_db = self.ts_db
+
# Check simple overlap checks work.
- self.assertTrue(is_overlaping(field_change, field_change2),
+ self.assertTrue(is_overlaping(self.field_change, self.field_change2),
"Should be overlapping")
- self.assertFalse(is_overlaping(field_change, field_change3),
+ self.assertFalse(is_overlaping(self.field_change, self.field_change3),
"Should not be overlapping")
# Check non-overlapping changes are always False.
- ret, reg = identify_related_changes(ts_db, regressions, field_change3)
+ ret, reg = identify_related_changes(ts_db, self.regressions,
+ self.field_change3)
+
self.assertFalse(ret, "Ranges don't overlap, should not match")
- regressions.append(reg)
+ self.regressions.append(reg)
# Check a regression matches if all fields match.
- ret, _ = identify_related_changes(ts_db, regressions, field_change)
+ ret, _ = identify_related_changes(ts_db, self.regressions,
+ self.field_change)
self.assertTrue(ret, "Should Match.")
- field_change7 = ts_db.FieldChange(order1234,
- order1235,
- machine2,
- test2,
- a_field)
+ field_change7 = ts_db.FieldChange(self.order1234,
+ self.order1235,
+ self.machine2,
+ self.test2,
+ self.a_field)
ts_db.add(field_change7)
- ret, reg = identify_related_changes(ts_db, regressions, field_change7)
- self.assertNotEquals(regression, reg)
+ ret, reg = identify_related_changes(ts_db, self.regressions, field_change7)
+ self.assertNotEquals(self.regression, reg)
self.assertFalse(ret, "Should not match with differnt machine and tests.")
- regressions.append(reg)
- field_change4 = ts_db.FieldChange(order1234,
- order1235,
- machine2,
- test,
- a_field)
+ self.regressions.append(reg)
+ field_change4 = ts_db.FieldChange(self.order1234,
+ self.order1235,
+ self.machine2,
+ self.test,
+ self.a_field)
# Check a regression matches if all fields match.
- ret, _ = identify_related_changes(ts_db, regressions, field_change4)
+ ret, _ = identify_related_changes(ts_db, self.regressions, field_change4)
self.assertTrue(ret, "Should Match with differnt machine.")
- field_change5 = ts_db.FieldChange(order1234,
- order1235,
- machine,
- test2,
- a_field)
+ field_change5 = ts_db.FieldChange(self.order1234,
+ self.order1235,
+ self.machine,
+ self.test2,
+ self.a_field)
# Check a regression matches if all fields match.
- ret, _ = identify_related_changes(ts_db, regressions, field_change5)
+ ret, _ = identify_related_changes(ts_db, self.regressions, field_change5)
self.assertTrue(ret, "Should Match with differnt tests.")
- field_change6 = ts_db.FieldChange(order1234,
- order1235,
- machine,
- test,
- a_field2)
+ field_change6 = ts_db.FieldChange(self.order1234,
+ self.order1235,
+ self.machine,
+ self.test,
+ self.a_field2)
# Check a regression matches if all fields match.
- ret, _ = identify_related_changes(ts_db, regressions, field_change6)
+ ret, _ = identify_related_changes(ts_db, self.regressions, field_change6)
self.assertTrue(ret, "Should Match with differnt fields.")
+ ts_db.commit()
+
+ r2 = rebuild_title(ts_db, self.regression)
+ EXPECTED_TITLE = "Regression of 6 benchmarks: test-a, test-b"
+ self.assertEquals(r2.title, EXPECTED_TITLE)
+
+ def test_regression_evolution(self):
+ ts_db = self.ts_db
+ rule_update_fixed_regressions.regression_evolution(ts_db, self.regressions)
+
if __name__ == '__main__':
unittest.main(argv=[sys.argv[0], ])
More information about the llvm-commits
mailing list