[LNT] r267787 - Server side blacklist for changes
Chris Matthews via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 27 13:47:41 PDT 2016
Author: cmatthews
Date: Wed Apr 27 15:47:41 2016
New Revision: 267787
URL: http://llvm.org/viewvc/llvm-project?rev=267787&view=rev
Log:
Server side blacklist for changes
There are some changes we don't care about. For instance, changes in
Unittests and the known to be evil benchmark ks. This implementes a
regex based blacklist for LNT change detection. A blacklist path can be
added to your config file which will point to a file with regex per
line, of things to ignore. When we do change detection, any changes
that match those regexes will be dropped. A default blacklist is not
provided, so nothing is filtered unless you set it up.
Added:
lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py
lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/blacklist
lnt/trunk/tests/server/db/blacklist
lnt/trunk/tests/server/db/blacklist.py
Modified:
lnt/trunk/lnt/server/config.py
lnt/trunk/lnt/server/db/fieldchange.py
lnt/trunk/lnt/server/db/rules_manager.py
lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/lnt.cfg
Modified: lnt/trunk/lnt/server/config.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/config.py?rev=267787&r1=267786&r2=267787&view=diff
==============================================================================
--- lnt/trunk/lnt/server/config.py (original)
+++ lnt/trunk/lnt/server/config.py Wed Apr 27 15:47:41 2016
@@ -106,16 +106,21 @@ class Config:
# FIXME: Remove this default.
tempDir = data.get('tmp_dir', 'viewer/resources/graphs')
-
+ blacklist = data.get('blacklist', None)
+ if blacklist and baseDir:
+ blacklist = os.path.join(baseDir, blacklist)
+ else:
+ blacklist = None
secretKey = data.get('secret_key', None)
return Config(data.get('name', 'LNT'), data['zorgURL'],
dbDir, os.path.join(baseDir, tempDir),
os.path.join(baseDir, profileDir), secretKey,
- dict([(k,DBInfo.fromData(dbDirPath, v,
- default_email_config,
- 0))
- for k,v in data['databases'].items()]))
+ dict([(k, DBInfo.fromData(dbDirPath, v,
+ default_email_config,
+ 0))
+ for k, v in data['databases'].items()]),
+ blacklist)
@staticmethod
def dummyInstance():
@@ -125,16 +130,18 @@ class Config:
profileDirPath = os.path.join(baseDir, 'profiles')
tempDir = os.path.join(baseDir, 'tmp')
secretKey = None
- dbInfo = {'dummy': DBInfo.dummyInstance()}
+ dbInfo = {'dummy': DBInfo.dummyInstance(),}
+ blacklist = None
- return Config('LNT', 'http://localhost:8000', dbDir, tempDir, profileDirPath, secretKey, dbInfo)
+ return Config('LNT', 'http://localhost:8000', dbDir, tempDir, profileDirPath, secretKey, dbInfo, blacklist)
- def __init__(self, name, zorgURL, dbDir, tempDir, profileDir, secretKey, databases):
+ def __init__(self, name, zorgURL, dbDir, tempDir, profileDir, secretKey, databases, blacklist):
self.name = name
self.zorgURL = zorgURL
self.dbDir = dbDir
self.tempDir = tempDir
self.secretKey = secretKey
+ self.blacklist = blacklist
self.profileDir = profileDir
while self.zorgURL.endswith('/'):
self.zorgURL = zorgURL[:-1]
Modified: lnt/trunk/lnt/server/db/fieldchange.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/fieldchange.py?rev=267787&r1=267786&r2=267787&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/fieldchange.py (original)
+++ lnt/trunk/lnt/server/db/fieldchange.py Wed Apr 27 15:47:41 2016
@@ -121,19 +121,21 @@ def regenerate_fieldchanges_for_run(ts,
machine=run.machine,
test=test,
field=field)
- ts.add(f)
- ts.commit()
- try:
- found, new_reg = identify_related_changes(ts, regressions, f)
- except ObjectDeletedError:
- # This can happen from time to time.
- # So, lets retry once.
- regressions = ts.query(ts.Regression).all()[::-1]
- found, new_reg = identify_related_changes(ts, regressions, f)
-
- if found:
- regressions.append(new_reg)
- note("Found field change: {}".format(run.machine))
+ # Check the rules to see if this change matters.
+ if rules.is_useful_change(ts, f):
+ ts.add(f)
+ ts.commit()
+ try:
+ found, new_reg = identify_related_changes(ts, regressions, f)
+ except ObjectDeletedError:
+ # This can happen from time to time.
+ # So, lets retry once.
+ regressions = ts.query(ts.Regression).all()[::-1]
+ found, new_reg = identify_related_changes(ts, regressions, f)
+
+ if found:
+ regressions.append(new_reg)
+ note("Found field change: {}".format(run.machine))
# Always update FCs with new values.
if f:
Added: lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py?rev=267787&view=auto
==============================================================================
--- lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py (added)
+++ lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py Wed Apr 27 15:47:41 2016
@@ -0,0 +1,49 @@
+"""Given a set of filed changes - figure out if we really care.
+
+This can be used to implement server side black lists.
+
+"""
+import re
+import os
+import sys
+from lnt.testing.util.commands import note, warning
+from flask import current_app
+
+ignored = None
+
+
+# Try and find the blacklist.
+def populate_blacklist():
+ global ignored
+ ignored = []
+ try:
+ path = current_app.old_config.blacklist
+ except RuntimeError:
+ path = os.path.join(os.path.dirname(sys.argv[0]), "blacklist")
+
+ if path and os.path.isfile(path):
+ note("Loading blacklist file: {}".format(path))
+ with open(path, 'r') as f:
+ for l in f.readlines():
+ ignored.append(re.compile(l.strip()))
+ else:
+ warning("Ignoring blacklist file: {}".format(path))
+
+
+def filter_by_benchmark_name(ts, field_change):
+ """Is this a fieldchanges we care about?
+ """
+ if ignored is None:
+ populate_blacklist()
+ benchmark_name = field_change.test.name
+ ts_name = ts.name
+ full_name = ts_name + "." + benchmark_name
+
+ for regex in ignored:
+ if regex.match(full_name):
+ note("Dropping field change {} because it matches {}".format(full_name,
+ regex.pattern))
+ return False
+ return True
+
+is_useful_change = filter_by_benchmark_name
Modified: lnt/trunk/lnt/server/db/rules_manager.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/rules_manager.py?rev=267787&r1=267786&r2=267787&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/rules_manager.py (original)
+++ lnt/trunk/lnt/server/db/rules_manager.py Wed Apr 27 15:47:41 2016
@@ -46,8 +46,8 @@ def load_rules():
# Places our rules can hook to.
HOOKS = {'post_test_hook':[],
'post_submission_hook':[],
- 'post_regression_create_hook':[]}
-
+ 'post_regression_create_hook':[],
+ 'is_useful_change': []}
DESCRIPTIONS = {}
def register_hooks():
@@ -67,3 +67,15 @@ def post_submission_hooks(ts, run_id):
"""Run all the post submission hooks on the submitted run."""
for func in HOOKS['post_submission_hook']:
func(ts, run_id)
+
+def is_useful_change(ts, field_change):
+ """Run all the change filters. If any are false, drop this change."""
+ all_filters = []
+ for func in HOOKS['is_useful_change']:
+ decision = func(ts, field_change)
+ all_filters.append(decision)
+ if len(all_filters) == 0:
+ return True
+ else:
+ # If any filter ignores, we ignore.
+ return all(all_filters)
Added: lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/blacklist
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/blacklist?rev=267787&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/blacklist (added)
+++ lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/blacklist Wed Apr 27 15:47:41 2016
@@ -0,0 +1,2 @@
+nts\..*\/UnitTests\/.*
+nts.MultiSource/Benchmarks/Ptrdist/ks/ks
Modified: lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/lnt.cfg
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/lnt.cfg?rev=267787&r1=267786&r2=267787&view=diff
==============================================================================
--- lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/lnt.cfg (original)
+++ lnt/trunk/tests/server/db/Inputs/lnt_v0.4.0_basic_instance/lnt.cfg Wed Apr 27 15:47:41 2016
@@ -48,3 +48,5 @@ nt_emailer = {
# Enable automatic restart using the wsgi_restart module; this should be off in
# a production environment.
wsgi_restart = False
+
+blacklist = 'blacklist'
Added: lnt/trunk/tests/server/db/blacklist
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/blacklist?rev=267787&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/blacklist (added)
+++ lnt/trunk/tests/server/db/blacklist Wed Apr 27 15:47:41 2016
@@ -0,0 +1,2 @@
+nts\..*\/UnitTests\/.*
+nts.MultiSource/Benchmarks/Ptrdist/ks/ks
Added: lnt/trunk/tests/server/db/blacklist.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/blacklist.py?rev=267787&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/blacklist.py (added)
+++ lnt/trunk/tests/server/db/blacklist.py Wed Apr 27 15:47:41 2016
@@ -0,0 +1,87 @@
+# Check the blacklist.
+# RUN: python %s
+"""Test the blacklist module"""
+import unittest
+import logging
+import sys
+import datetime
+from lnt.server.config import Config
+from lnt.server.db import v4db
+
+import lnt.server.db.rules.rule_blacklist_benchmarks_by_name as blacklist
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+class BlacklistProcessingTest(unittest.TestCase):
+ """Test the Rules facility."""
+
+ def _mkorder(self, ts, rev):
+ order = ts.Order()
+ order.llvm_project_revision = rev
+ ts.add(order)
+ return order
+
+ def setUp(self):
+ self.db = v4db.V4DB("sqlite:///:memory:", Config.dummyInstance(), echo=False)
+
+ # Get the test suite wrapper.
+ ts_db = self.ts_db = self.db.testsuite['nts']
+
+ order1234 = self.order1234 = self._mkorder(ts_db, "1234")
+ order1235 = self.order1235 = self._mkorder(ts_db, "1235")
+ order1236 = self.order1236 = self._mkorder(ts_db, "1236")
+ order1237 = self.order1237 = self._mkorder(ts_db, "1237")
+ order1238 = self.order1238 = self._mkorder(ts_db, "1238")
+
+ start_time = end_time = datetime.datetime.utcnow()
+ machine = self.machine = ts_db.Machine("test-machine")
+ ts_db.add(machine)
+
+ test = self.test = ts_db.Test("Foo")
+ test2 = self.test2 = ts_db.Test("SingleSource/Foo/Bar/baz")
+ test3 = self.test3 = ts_db.Test("SingleSource/UnitTests/Bar/baz")
+ test4 = self.test4 = ts_db.Test("MultiSource/Benchmarks/Ptrdist/ks/ks")
+
+ a_field = None
+
+ self.field_change1 = ts_db.FieldChange(order1234,
+ order1236,
+ None,
+ test,
+ a_field)
+ self.field_change2 = ts_db.FieldChange(order1234,
+ order1236,
+ None,
+ test2,
+ a_field)
+ self.field_change3 = ts_db.FieldChange(order1234,
+ order1236,
+ None,
+ test3,
+ a_field)
+ self.field_change4 = ts_db.FieldChange(order1234,
+ order1236,
+ None,
+ test4,
+ a_field)
+
+ def test_blacklist(self):
+ """Check we filter by benchmark name correctly."""
+ ts = self.ts_db
+ fc1 = self.field_change1
+ fc2 = self.field_change2
+ fc3 = self.field_change3
+ fc4 = self.field_change4
+
+ valid = blacklist.filter_by_benchmark_name(ts, fc1)
+ self.assertTrue(valid, "Expect this to not be filtered.")
+ valid = blacklist.filter_by_benchmark_name(ts ,fc2)
+ self.assertTrue(valid, "Expect this to not be filtered.")
+ bad = blacklist.filter_by_benchmark_name(ts, fc3)
+ self.assertFalse(bad, "Expect this to be filtered by regex.")
+ bad = blacklist.filter_by_benchmark_name(ts, fc4)
+ self.assertFalse(bad, "Expect this to be filtered by blacklist.")
+
+if __name__ == '__main__':
+ unittest.main(argv=[sys.argv[0], ])
More information about the llvm-commits
mailing list