[LNT] r318198 - Don't run regression analysis in the background
Chris Matthews via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 14 13:12:20 PST 2017
Author: cmatthews
Date: Tue Nov 14 13:12:20 2017
New Revision: 318198
URL: http://llvm.org/viewvc/llvm-project?rev=318198&view=rev
Log:
Don't run regression analysis in the background
This remove the background workers. We have had a lot of problems with
system stability because of them. We will do the work in request
instead.
This patch will be followed up with some performance fixes to make
running in request more reasonable.
Removed:
lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py
lnt/trunk/lnt/util/async_ops.py
lnt/trunk/tests/server/db/blacklist
lnt/trunk/tests/server/db/blacklist.py
Modified:
lnt/trunk/lnt/server/db/fieldchange.py
lnt/trunk/lnt/server/ui/views.py
lnt/trunk/lnt/util/ImportData.py
Modified: lnt/trunk/lnt/server/db/fieldchange.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/fieldchange.py?rev=318198&r1=318197&r2=318198&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/fieldchange.py (original)
+++ lnt/trunk/lnt/server/db/fieldchange.py Tue Nov 14 13:12:20 2017
@@ -12,10 +12,14 @@ from lnt.server.db import rules_manager
# How many runs backwards to use in the previous run set.
# More runs are slower (more DB access), but may provide
# more accurate results.
+
FIELD_CHANGE_LOOKBACK = 10
def post_submit_tasks(session, ts, run_id):
+ """Run the field change related post submission tasks.
+
+ """
regenerate_fieldchanges_for_run(session, ts, run_id)
@@ -130,22 +134,19 @@ def regenerate_fieldchanges_for_run(sess
machine=run.machine,
test=test,
field_id=field.id)
- f.field = field
- # Check the rules to see if this change matters.
- if rules.is_useful_change(session, ts, f):
- session.add(f)
- try:
- found, new_reg = identify_related_changes(session, ts,
- f)
- except ObjectDeletedError:
- # This can happen from time to time.
- # So, lets retry once.
- found, new_reg = identify_related_changes(session, ts,
- f)
-
- if found:
- logger.info("Found field change: {}".format(
- run.machine))
+ session.add(f)
+ try:
+ found, new_reg = identify_related_changes(session, ts,
+ f)
+ except ObjectDeletedError:
+ # This can happen from time to time.
+ # So, lets retry once.
+ found, new_reg = identify_related_changes(session, ts,
+ f)
+
+ if found:
+ logger.info("Found field change: {}".format(
+ run.machine))
# Always update FCs with new values.
if f:
Removed: lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py?rev=318197&view=auto
==============================================================================
--- lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py (original)
+++ lnt/trunk/lnt/server/db/rules/rule_blacklist_benchmarks_by_name.py (removed)
@@ -1,53 +0,0 @@
-"""Given a set of filed changes - figure out if we really care.
-
-This can be used to implement server side black lists.
-
-"""
-import re
-import os
-import sys
-from lnt.util import logger
-from flask import current_app
-
-ignored = None
-
-
-# Try and find the blacklist.
-def _populate_blacklist():
- global ignored
- ignored = []
- try:
- path = current_app.old_config.blacklist
- except RuntimeError:
- path = os.path.join(os.path.dirname(sys.argv[0]), "blacklist")
-
- if path and os.path.isfile(path):
- logger.info("Loading blacklist file: {}".format(path))
- with open(path, 'r') as f:
- for l in f.readlines():
- ignored.append(re.compile(l.strip()))
- else:
- logger.warning("Ignoring blacklist file: {}".format(path))
-
-
-def filter_by_benchmark_name(session, ts, field_change):
- """Is this a fieldchanges we care about?
- """
- if ignored is None:
- _populate_blacklist()
- benchmark_name = field_change.test.name
- ts_name = ts.name
- full_name = '.'.join([ts_name,
- field_change.machine.name,
- benchmark_name,
- field_change.field.name])
- logger.info(full_name)
- for regex in ignored:
- if regex.match(full_name):
- logger.info("Dropping field change {} because it matches {}"
- .format(full_name, regex.pattern))
- return False
- return True
-
-
-is_useful_change = filter_by_benchmark_name
Modified: lnt/trunk/lnt/server/ui/views.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/views.py?rev=318198&r1=318197&r2=318198&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/views.py (original)
+++ lnt/trunk/lnt/server/ui/views.py Tue Nov 14 13:12:20 2017
@@ -38,7 +38,7 @@ from lnt.server.ui.globals import db_url
from lnt.server.ui.util import FLASH_DANGER, FLASH_SUCCESS, FLASH_INFO
from lnt.server.ui.util import PrecomputedCR
from lnt.server.ui.util import mean
-from lnt.util import async_ops
+
from lnt.util import logger
from lnt.util import multidict
from lnt.server.ui.util import baseline_key, convert_revision
@@ -1538,7 +1538,6 @@ def rules():
@frontend.route('/log')
def log():
- async_ops.check_workers(True)
with open(current_app.config['log_file_name'], 'r') as f:
log_lines = f.readlines()
r'2017-07-21 15:02:15,143 ERROR:'
Modified: lnt/trunk/lnt/util/ImportData.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/util/ImportData.py?rev=318198&r1=318197&r2=318198&view=diff
==============================================================================
--- lnt/trunk/lnt/util/ImportData.py (original)
+++ lnt/trunk/lnt/util/ImportData.py Tue Nov 14 13:12:20 2017
@@ -1,5 +1,5 @@
from lnt.util import NTEmailReport
-from lnt.util import async_ops
+
from lnt.util import logger
import collections
import datetime
@@ -7,10 +7,12 @@ import lnt.formats
import lnt.server.reporting.analysis
import lnt.testing
import os
-import re
+
import tempfile
import time
+from lnt.server.db import fieldchange
+
def import_and_report(config, db_name, db, session, file, format, ts_name,
show_sample_count=False, disable_email=False,
@@ -144,11 +146,8 @@ def import_and_report(config, db_name, d
result['committed'] = True
result['run_id'] = run.id
session.commit()
- if db_config:
- # If we are not in a dummy instance, also run background jobs.
- # We have to have a commit before we run, so subprocesses can
- # see the submitted data.
- async_ops.async_fieldchange_calc(db_name, ts, run, config)
+
+ fieldchange.post_submit_tasks(session, ts, run.id)
# Add a handy relative link to the submitted run.
result['result_url'] = "db_{}/v4/{}/{}".format(db_name, ts_name, run.id)
Removed: lnt/trunk/lnt/util/async_ops.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/util/async_ops.py?rev=318197&view=auto
==============================================================================
--- lnt/trunk/lnt/util/async_ops.py (original)
+++ lnt/trunk/lnt/util/async_ops.py (removed)
@@ -1,148 +0,0 @@
-"""Asynchrounus operations for LNT.
-
-For big tasks it is nice to be able to run in the backgorund. This module
-contains wrappers to run particular LNT tasks in subprocesess.
-
-Because multiprocessing cannot directly use the LNT test-suite objects in
-subprocesses (because they are not serializable because they don't have a fix
-package in the system, but are generated on program load) we recreate the test
-suite that we need inside each subprocess before we execute the work job.
-"""
-import atexit
-import os
-import time
-from flask import current_app, g
-import sys
-import lnt.server.db.fieldchange as fieldchange
-import lnt.server.db.v4db
-import traceback
-import signal
-from time import sleep
-import contextlib
-import multiprocessing
-from multiprocessing import Pool, TimeoutError, Manager, Process
-from threading import Lock
-from lnt.util import logger
-NUM_WORKERS = 4 # The number of subprocesses to spawn per LNT process.
-WORKERS = None # The worker pool.
-WORKERS_LOCK = Lock()
-
-JOBS = []
-
-
-def launch_workers():
- """Make sure we have a worker pool ready to queue."""
- global WORKERS
- global WORKERS_LOCK
- WORKERS_LOCK.acquire()
- try:
- if not WORKERS:
- logger.info("Starting workers")
- WORKERS = True
- finally:
- WORKERS_LOCK.release()
-
-
-def sig_handler(signo, frame):
- cleanup()
- sys.exit(0)
-
-
-def cleanup():
- logger.info("Running process cleanup.")
- for p in JOBS:
- logger.info("Waiting for %s %s" % (p.name, p.pid))
- if p.is_alive:
- p.join()
-
-
-atexit.register(cleanup)
-signal.signal(signal.SIGTERM, sig_handler)
-
-
-def async_fieldchange_calc(db_name, ts, run, db_config):
- """Run regenerate field changes in the background."""
- func_args = {'run_id': run.id}
- # Make sure this run is in the database!
- async_run_job(fieldchange.post_submit_tasks,
- db_name, ts,
- func_args, db_config)
-
-
-def check_workers(is_logged):
- global JOBS
- alive_jobs_list = []
- for j in JOBS:
- if j.is_alive():
- alive_jobs_list.append(j)
- else:
- j.join()
- JOBS = alive_jobs_list
- still_running = len(JOBS)
- msg = "{} Job(s) in the queue.".format(still_running)
- if is_logged:
- if still_running > 5:
- # This could be run outside of the application context, so use
- # full logger name.
- logger.warning(msg)
- elif still_running > 0:
- logger.info(msg)
- else:
- logger.info("Job queue empty.")
- return len(JOBS)
-
-
-def async_run_job(job, db_name, ts, func_args, db_config):
- """Send a job to the async wrapper in the subprocess."""
- # If the run is not in the database, we can't do anything more.
- logger.info("Queuing background job to process fieldchanges " +
- str(os.getpid()))
- launch_workers()
- check_workers(True)
-
- args = {'tsname': ts.name,
- 'db': db_name,
- 'db_info': db_config}
- job = Process(target=async_wrapper,
- args=[job, args, func_args])
-
- # Set this to make sure when parent dies, children are killed.
- job.daemon = True
-
- job.start()
- JOBS.append(job)
-
-
-def async_wrapper(job, ts_args, func_args):
- """Setup test-suite in this subprocess and run something.
-
- Because of multipocessing, capture excptions and log messages,
- and return them.
- """
- try:
- start_time = time.time()
-
- sleep(3)
- logger.info("Running async wrapper: {} ".format(job.__name__) +
- str(os.getpid()))
- config = ts_args['db_info']
- db = config.get_database(ts_args['db'])
- with contextlib.closing(db):
- session = db.make_session()
- with contextlib.closing(session):
- ts = db.testsuite[ts_args['tsname']]
- nothing = job(session, ts, **func_args)
- assert nothing is None
- end_time = time.time()
- delta = end_time-start_time
- msg = "Finished: {name} in {time:.2f}s ".format(name=job.__name__,
- time=delta)
- if delta < 100:
- logger.info(msg)
- else:
- logger.warning(msg)
- except Exception:
- # Put all exception text into an exception and raise that for our
- # parent process.
- logger.error("Subprocess failed with:" +
- "".join(traceback.format_exception(*sys.exc_info())))
Removed: lnt/trunk/tests/server/db/blacklist
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/blacklist?rev=318197&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/blacklist (original)
+++ lnt/trunk/tests/server/db/blacklist (removed)
@@ -1,2 +0,0 @@
-nts\.test-machine\..*\/UnitTests\/.*
-nts\.test-machine\.MultiSource\/Benchmarks\/Ptrdist\/ks\/ks.*
Removed: lnt/trunk/tests/server/db/blacklist.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/blacklist.py?rev=318197&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/blacklist.py (original)
+++ lnt/trunk/tests/server/db/blacklist.py (removed)
@@ -1,116 +0,0 @@
-# Check the blacklist.
-# RUN: rm -rf %t.instance
-# RUN: python %{shared_inputs}/create_temp_instance.py \
-# RUN: %s %{shared_inputs}/SmallInstance %t.instance \
-# RUN: %S/Inputs/V4Pages_extra_records.sql
-#
-# RUN: python %s %t.instance
-
-"""Test the blacklist module"""
-import unittest
-import os
-import logging
-import sys
-
-import lnt
-from lnt.server.config import Config
-from lnt.server.db import v4db
-
-import lnt.server.db.rules.rule_blacklist_benchmarks_by_name as blacklist
-import lnt.server.ui.app
-
-here = os.path.dirname(__file__)
-
-logging.basicConfig(level=logging.DEBUG)
-
-
-class BlacklistProcessingTest(unittest.TestCase):
- """Test the Rules facility."""
-
- def _mkorder(self, session, ts, rev):
- order = ts.Order()
- order.llvm_project_revision = rev
- session.add(order)
- return order
-
- def setUp(self):
- _, instance_path = sys.argv
-
- # Create the application instance.
- app = lnt.server.ui.app.App.create_standalone(instance_path)
- app.old_config.blacklist = here + "/blacklist"
- app.app_context().push()
- # Get the test suite wrapper.
- with app.test_request_context('/db_default/nts/foo') as r:
- app.preprocess_request()
- r.g.db_name = "default"
- r.g.testsuite_name = "nts"
- r.request.db = app.instance.get_database(r.g.db_name)
- r.request.session = r.request.db.make_session()
- self.ts = r.request.get_testsuite()
- self.ts_db = self.ts
- self.session = r.request.session
- session = self.session
- ts_db = self.ts_db
- order1234 = self.order1234 = self._mkorder(session, ts_db, "1234")
- order1236 = self.order1236 = self._mkorder(session, ts_db, "1236")
-
- machine = self.machine = ts_db.Machine("test-machine")
- session.add(machine)
-
- a_field = ts_db.Sample.fields[0]
-
- session.commit()
-
- test = self.test = ts_db.Test("Foo")
- test2 = self.test2 = ts_db.Test("SingleSource/Foo/Bar/baz")
- test3 = self.test3 = ts_db.Test("SingleSource/UnitTests/Bar/baz")
- test4 = self.test4 = ts_db.Test("MultiSource/Benchmarks/Ptrdist/ks/ks")
-
- self.field_change1 = ts_db.FieldChange(order1234,
- order1236,
- machine,
- test,
- a_field.id)
- self.field_change2 = ts_db.FieldChange(order1234,
- order1236,
- machine,
- test2,
- a_field.id)
- self.field_change3 = ts_db.FieldChange(order1234,
- order1236,
- machine,
- test3,
- a_field.id)
- self.field_change4 = ts_db.FieldChange(order1234,
- order1236,
- machine,
- test4,
- a_field.id)
- session.add(self.field_change1)
- session.add(self.field_change2)
- session.add(self.field_change3)
- session.add(self.field_change4)
-
- session.commit()
-
- def test_blacklist(self):
- """Check we filter by benchmark name correctly."""
- session = self.session
- ts = self.ts_db
- fc1 = self.field_change1
- fc2 = self.field_change2
- fc3 = self.field_change3
- fc4 = self.field_change4
-
- valid = blacklist.filter_by_benchmark_name(session, ts, fc1)
- self.assertTrue(valid, "Expect this to not be filtered.")
- valid = blacklist.filter_by_benchmark_name(session, ts, fc2)
- self.assertTrue(valid, "Expect this to not be filtered.")
- bad = blacklist.filter_by_benchmark_name(session, ts, fc3)
- self.assertFalse(bad, "Expect this to be filtered by regex.")
- bad = blacklist.filter_by_benchmark_name(session, ts, fc4)
- self.assertFalse(bad, "Expect this to be filtered by blacklist.")
-
-if __name__ == '__main__':
- unittest.main(argv=[sys.argv[0], ])
More information about the llvm-commits
mailing list