[LNT] r284562 - Make change grouping control regression lists
Chris Matthews via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 18 17:40:21 PDT 2016
Author: cmatthews
Date: Tue Oct 18 19:40:21 2016
New Revision: 284562
URL: http://llvm.org/viewvc/llvm-project?rev=284562&view=rev
Log:
Make change grouping control regression lists
We have had problems keeping consistent regression lists during
processing. Now query them every time, but use a faster query.
Modified:
lnt/trunk/lnt/server/db/fieldchange.py
lnt/trunk/lnt/server/db/regression.py
lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py
lnt/trunk/tests/server/ui/change_processing.py
Modified: lnt/trunk/lnt/server/db/fieldchange.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/fieldchange.py?rev=284562&r1=284561&r2=284562&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/fieldchange.py (original)
+++ lnt/trunk/lnt/server/db/fieldchange.py Tue Oct 18 19:40:21 2016
@@ -7,7 +7,7 @@ from lnt.testing.util.commands import no
from lnt.server.db.regression import new_regression, RegressionState
from lnt.server.db.regression import get_ris
from lnt.server.db.regression import rebuild_title
-
+from sqlalchemy import or_
from lnt.server.db import rules_manager as rules
# How many runs backwards to use in the previous run set.
# More runs are slower (more DB access), but may provide
@@ -39,6 +39,7 @@ def delete_fieldchange(ts, change):
# We might have just created a regression with no changes.
# If so, delete it as well.
+ deleted_ids = []
for r in regression_ids:
remaining = ts.query(ts.RegressionIndicator). \
filter(ts.RegressionIndicator.regression_id == r). \
@@ -47,7 +48,9 @@ def delete_fieldchange(ts, change):
r = ts.query(ts.Regression).get(r)
note("Deleting regression because it has not changes:" + repr(r))
ts.delete(r)
+ deleted_ids.append(r)
ts.commit()
+ return deleted_ids
@timed
@@ -62,7 +65,7 @@ def regenerate_fieldchanges_for_run(ts,
filter(ts.Run.order_id == run.order_id). \
filter(ts.Run.machine_id == run.machine_id). \
all()
- regressions = ts.query(ts.Regression).all()[::-1]
+
previous_runs = ts.get_previous_runs_on_machine(run, FIELD_CHANGE_LOOKBACK)
next_runs = ts.get_next_runs_on_machine(run, FIELD_CHANGE_LOOKBACK)
@@ -111,7 +114,7 @@ def regenerate_fieldchanges_for_run(ts,
if not result.is_result_performance_change() and f:
# With more data, its not a regression. Kill it!
note("Removing field change: {}".format(f.id))
- delete_fieldchange(ts, f)
+ deleted = delete_fieldchange(ts, f)
continue
if result.is_result_performance_change() and not f:
@@ -126,15 +129,13 @@ def regenerate_fieldchanges_for_run(ts,
ts.add(f)
ts.commit()
try:
- found, new_reg = identify_related_changes(ts, regressions, f)
+ found, new_reg = identify_related_changes(ts, f)
except ObjectDeletedError:
# This can happen from time to time.
# So, lets retry once.
- regressions = ts.query(ts.Regression).all()[::-1]
- found, new_reg = identify_related_changes(ts, regressions, f)
+ found, new_reg = identify_related_changes(ts, f)
if found:
- regressions.append(new_reg)
note("Found field change: {}".format(run.machine))
# Always update FCs with new values.
@@ -143,6 +144,8 @@ def regenerate_fieldchanges_for_run(ts,
f.new_value = result.current
f.run = run
ts.commit()
+
+ regressions = ts.query(ts.Regression).all()[::-1]
rules.post_submission_hooks(ts, regressions)
@@ -172,7 +175,7 @@ def percent_similar(a, b):
@timed
-def identify_related_changes(ts, regressions, fc):
+def identify_related_changes(ts, fc):
"""Can we find a home for this change in some existing regression? If a
match is found add a regression indicator adding this change to that
regression, otherwise create a new regression for this change.
@@ -181,8 +184,15 @@ def identify_related_changes(ts, regress
ranges. Then looks for changes that are similar.
"""
- for regression in regressions:
- regression_indicators = get_ris(ts, regression)
+ regressions = ts.query(ts.Regression.id) \
+ .filter(or_(ts.Regression.state == RegressionState.DETECTED,
+ ts.Regression.state == RegressionState.DETECTED_FIXED)) \
+ .all()
+
+ for regression_packed in regressions:
+ regression_id = regression_packed[0]
+ regression_indicators = get_ris(ts, regression_id)
+ print "RIs:", regression_indicators
for change in regression_indicators:
regression_change = change.field_change
if is_overlaping(regression_change, fc):
@@ -198,12 +208,14 @@ def identify_related_changes(ts, regress
if confidence >= 2.0:
# Matching
MSG = "Found a match: {} with score {}."
+ regression = ts.query(ts.Regression).get(regression_id)
note(MSG.format(str(regression),
confidence))
ri = ts.RegressionIndicator(regression, fc)
ts.add(ri)
# Update the default title if needed.
rebuild_title(ts, regression)
+ ts.commit()
return True, regression
note("Could not find a partner, creating new Regression for change")
new_reg = new_regression(ts, [fc.id])
Modified: lnt/trunk/lnt/server/db/regression.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/regression.py?rev=284562&r1=284561&r2=284562&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/regression.py (original)
+++ lnt/trunk/lnt/server/db/regression.py Tue Oct 18 19:40:21 2016
@@ -80,9 +80,9 @@ def get_all_orders_for_machine(ts, machi
.all()
-def get_ris(ts, regression):
+def get_ris(ts, regression_id):
return ts.query(ts.RegressionIndicator) \
- .filter(ts.RegressionIndicator.regression_id == regression.id) \
+ .filter(ts.RegressionIndicator.regression_id == regression_id) \
.all()
Modified: lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py?rev=284562&r1=284561&r2=284562&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py (original)
+++ lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py Tue Oct 18 19:40:21 2016
@@ -21,7 +21,7 @@ def is_fixed(ts, regression):
"""Comparing the current value to the regression, is this regression now
fixed?
"""
- r_inds = get_ris(ts, regression)
+ r_inds = get_ris(ts, regression.id)
fixes = [_fixed_rind(ts, x) for x in r_inds]
return all(fixes)
Modified: lnt/trunk/tests/server/ui/change_processing.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/ui/change_processing.py?rev=284562&r1=284561&r2=284562&view=diff
==============================================================================
--- lnt/trunk/tests/server/ui/change_processing.py (original)
+++ lnt/trunk/tests/server/ui/change_processing.py Tue Oct 18 19:40:21 2016
@@ -130,14 +130,12 @@ class ChangeProcessingTests(unittest.Tes
"Should not be overlapping")
# Check non-overlapping changes are always False.
- ret, reg = identify_related_changes(ts_db, self.regressions,
- self.field_change3)
-
+ ret, reg = identify_related_changes(ts_db, self.field_change3)
+
self.assertFalse(ret, "Ranges don't overlap, should not match")
self.regressions.append(reg)
# Check a regression matches if all fields match.
- ret, _ = identify_related_changes(ts_db, self.regressions,
- self.field_change)
+ ret, _ = identify_related_changes(ts_db, self.field_change)
self.assertTrue(ret, "Should Match.")
field_change7 = ts_db.FieldChange(self.order1234,
@@ -146,7 +144,7 @@ class ChangeProcessingTests(unittest.Tes
self.test2,
self.a_field)
ts_db.add(field_change7)
- ret, reg = identify_related_changes(ts_db, self.regressions, field_change7)
+ ret, reg = identify_related_changes(ts_db, field_change7)
self.assertNotEquals(self.regression, reg)
self.assertFalse(ret, "No match with different machine and tests.")
self.regressions.append(reg)
@@ -157,7 +155,7 @@ class ChangeProcessingTests(unittest.Tes
self.a_field)
# Check a regression matches if all fields match.
- ret, _ = identify_related_changes(ts_db, self.regressions, field_change4)
+ ret, _ = identify_related_changes(ts_db, field_change4)
self.assertTrue(ret, "Should Match with different machine.")
field_change5 = ts_db.FieldChange(self.order1234,
@@ -167,7 +165,7 @@ class ChangeProcessingTests(unittest.Tes
self.a_field)
# Check a regression matches if all fields match.
- ret, _ = identify_related_changes(ts_db, self.regressions, field_change5)
+ ret, _ = identify_related_changes(ts_db, field_change5)
self.assertTrue(ret, "Should Match with different tests.")
field_change6 = ts_db.FieldChange(self.order1234,
self.order1235,
@@ -176,7 +174,7 @@ class ChangeProcessingTests(unittest.Tes
self.a_field2)
# Check a regression matches if all fields match.
- ret, _ = identify_related_changes(ts_db, self.regressions, field_change6)
+ ret, _ = identify_related_changes(ts_db, field_change6)
self.assertTrue(ret, "Should Match with different fields.")
ts_db.commit()
More information about the llvm-commits
mailing list