[LNT] r255958 - Fix LNT's logging, add /log endpoint to view logs from web
James Molloy via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 18 02:54:42 PST 2015
Massive +1 from me for this! Nice work!
On Fri, 18 Dec 2015 at 01:41 Chris Matthews via llvm-commits <
llvm-commits at lists.llvm.org> wrote:
> Author: cmatthews
> Date: Thu Dec 17 19:37:53 2015
> New Revision: 255958
>
> URL: http://llvm.org/viewvc/llvm-project?rev=255958&view=rev
> Log:
> Fix LNT's logging, add /log endpoint to view logs from web
>
> LNT's logging support has always been horrible. This makes debugging
> server problems hard. This patch tries to get all the parts of logging in
> LNT working together better. First replace the stderr loggers in commands
> with python logging package. Replace all uses of the python logging
> package with a single logger, so all messages from LNT will be in one log.
> Add a in memory logging buffer, and allow the user to see that through the
> /log page. Make sure the rotating file logging is used as well.
>
> Later patches will add support for subprocesses.
>
> Added:
> lnt/trunk/lnt/server/ui/templates/log.html
> Modified:
> lnt/trunk/lnt/lnttool/main.py
> lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py
> lnt/trunk/lnt/server/ui/app.py
> lnt/trunk/lnt/server/ui/views.py
> lnt/trunk/lnt/testing/util/commands.py
>
> Modified: lnt/trunk/lnt/lnttool/main.py
> URL:
> http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/lnttool/main.py?rev=255958&r1=255957&r2=255958&view=diff
>
> ==============================================================================
> --- lnt/trunk/lnt/lnttool/main.py (original)
> +++ lnt/trunk/lnt/lnttool/main.py Thu Dec 17 19:37:53 2015
> @@ -65,10 +65,12 @@ view the results.\
> input_path, = args
>
> # Setup the base LNT logger.
> - logger = logging.getLogger("lnt")
> + # Root logger in debug.
> + logger = logging.getLogger("LNT")
> if opts.debugger:
> logger.setLevel(logging.DEBUG)
> - handler = logging.StreamHandler(sys.stderr)
> + handler = logging.StreamHandler()
> + handler.setLevel(logging.DEBUG)
> handler.setFormatter(logging.Formatter(
> '%(asctime)s %(levelname)s: %(message)s',
> datefmt='%Y-%m-%d %H:%M:%S'))
> @@ -79,7 +81,7 @@ view the results.\
> sa_logger = logging.getLogger("sqlalchemy")
> if opts.debugger:
> sa_logger.setLevel(logging.DEBUG)
> - sa_logger.setLevel(logging.DEBUG)
> + sa_logger.setLevel(logging.INFO)
> sa_logger.addHandler(handler)
>
> import lnt.server.ui.app
> @@ -158,6 +160,14 @@ def action_runtest(name, args):
> warning("--{} should be passed directly to the"
> " test suite.".format(key))
>
> + logger = logging.getLogger("LNT")
> + logger.setLevel(logging.DEBUG)
> + handler = logging.StreamHandler()
> + handler.setLevel(logging.DEBUG)
> + handler.setFormatter(logging.Formatter(
> + '%(asctime)s %(levelname)s: %(message)s',
> + datefmt='%Y-%m-%d %H:%M:%S'))
> + logger.addHandler(handler)
> import lnt.tests
> try:
> test_instance = lnt.tests.get_test_instance(test_name)
>
> Modified: lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py
> URL:
> http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py?rev=255958&r1=255957&r2=255958&view=diff
>
> ==============================================================================
> --- lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py
> (original)
> +++ lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py Thu Dec
> 17 19:37:53 2015
> @@ -35,6 +35,7 @@ def regression_evolution(ts, regressions
> Look at regressions in detect, do they match our policy? If no, move
> to NTBF.
>
> """
> + note("Running regression evolution")
> detects = [r for r in regressions if r.state ==
> RegressionState.DETECTED]
>
> for regression in detects:
>
> Modified: lnt/trunk/lnt/server/ui/app.py
> URL:
> http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/app.py?rev=255958&r1=255957&r2=255958&view=diff
>
> ==============================================================================
> --- lnt/trunk/lnt/server/ui/app.py (original)
> +++ lnt/trunk/lnt/server/ui/app.py Thu Dec 17 19:37:53 2015
> @@ -5,6 +5,7 @@ import logging.handlers
> from logging import Formatter
> import os
> import time
> +import StringIO
>
> import flask
> from flask import current_app
> @@ -64,7 +65,6 @@ class Request(flask.Request):
> #
> # FIXME: Conditionalize on an is_production variable.
> if echo:
> - import logging, StringIO
> g.db_log = StringIO.StringIO()
> logger = logging.getLogger("sqlalchemy")
> logger.addHandler(logging.StreamHandler(g.db_log))
> @@ -137,7 +137,10 @@ class App(flask.Flask):
> # Inject a fix for missing slashes on the root URL (see Flask
> issue
> # #169).
> self.wsgi_app = RootSlashPatchMiddleware(self.wsgi_app)
> -
> +
> + self.lnt_logger = logging.getLogger('LNT')
> + self.lnt_logger.setLevel(logging.DEBUG)
> +
> def load_config(self, instance):
> self.instance = instance
> self.old_config = self.instance.config
> @@ -154,9 +157,21 @@ class App(flask.Flask):
> to stderr, so just log to a file as well.
>
> """
> + # Print to screen.
> + ch = logging.StreamHandler()
> + ch.setLevel(logging.DEBUG)
> + self.lnt_logger.addHandler(ch)
> +
> + # Log to mem for the /log view.
> + h = logging.handlers.MemoryHandler(1024 * 1024)
> + h.setLevel(logging.DEBUG)
> + self.lnt_logger.addHandler(h)
> + self.logger.addHandler(h)
> + self.old_config.mem_logger = h
> +
> if not self.debug:
> LOG_FILENAME = "lnt.log"
> - try:
> + try:
> rotating = logging.handlers.RotatingFileHandler(
> LOG_FILENAME, maxBytes=1048576, backupCount=5)
> rotating.setFormatter(Formatter(
> @@ -164,7 +179,7 @@ class App(flask.Flask):
> '[in %(pathname)s:%(lineno)d]'
> ))
> rotating.setLevel(logging.DEBUG)
> - self.logger.addHandler(rotating)
> + self.lnt_logger.addHandler(rotating)
> except (OSError, IOError) as e:
> print >> sys.stderr, "Error making log file",
> LOG_FILENAME, str(e)
> print >> sys.stderr, "Will not log to file."
>
> Added: lnt/trunk/lnt/server/ui/templates/log.html
> URL:
> http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/log.html?rev=255958&view=auto
>
> ==============================================================================
> --- lnt/trunk/lnt/server/ui/templates/log.html (added)
> +++ lnt/trunk/lnt/server/ui/templates/log.html Thu Dec 17 19:37:53 2015
> @@ -0,0 +1,35 @@
> +{% import "utils.html" as utils %}
> +
> +{% extends "layout.html" %}
> +{% set components = [] %}
> +
> +{% block title %}Log{%endblock%}
> +
> +{% block body %}
> +
> +<table class="table table-striped table-hover table-condensed">
> + <thead>
> + <tr>
> + <th>Kind</th>
> + <th>Location</th>
> + <th>Time</th>
> + <th>Message<th>
> + </tr>
> + </thead>
> + <tbody class="searchable">
> +{% for item in old_config.mem_logger.buffer %}
> + {% if item.levelname|string() == 'WARNING' %}
> + <tr class="warning">
> + {% elif item.levelname|string() == 'ERROR' %}
> + <tr class="error">
> + {% else %}
> + <tr class="info">
> + {% endif %}
> + <td>{{ item.levelname}}</td>
> + <td>{{item.filename}}:{{item.lineno}}</td>
> + <td>{{ item.asctime}}</td>
> + <td>{{ item.msg }}</td>
> + </tr>
> +{% endfor %}
> +
> +{% endblock %}
>
> Modified: lnt/trunk/lnt/server/ui/views.py
> URL:
> http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/views.py?rev=255958&r1=255957&r2=255958&view=diff
>
> ==============================================================================
> --- lnt/trunk/lnt/server/ui/views.py (original)
> +++ lnt/trunk/lnt/server/ui/views.py Thu Dec 17 19:37:53 2015
> @@ -1169,3 +1169,7 @@ def rules():
> lnt.server.db.rules.register_hooks()
> discovered_rules = lnt.server.db.rules.DESCRIPTIONS
> return render_template("rules.html",rules=discovered_rules)
> +
> + at frontend.route('/log')
> +def log():
> + return render_template("log.html")
>
> Modified: lnt/trunk/lnt/testing/util/commands.py
> URL:
> http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/testing/util/commands.py?rev=255958&r1=255957&r2=255958&view=diff
>
> ==============================================================================
> --- lnt/trunk/lnt/testing/util/commands.py (original)
> +++ lnt/trunk/lnt/testing/util/commands.py Thu Dec 17 19:37:53 2015
> @@ -6,23 +6,18 @@ import errno
> import inspect
> import os
> import sys
> -
> +import logging
> # FIXME: Find a better place for this code.
>
> -def _write_message(kind, message):
> - # Get the file/line where this message was generated.
> - f = inspect.currentframe()
> - # Step out of _write_message, and then out of wrapper.
> - f = f.f_back.f_back
> - file,line,_,_,_ = inspect.getframeinfo(f)
> - location = '%s:%d' % (os.path.basename(file), line)
> +note = lambda message: logging.getLogger('LNT').info(message)
> +warning = lambda message: logging.getLogger('LNT').warning(message)
> +error = lambda message: logging.getLogger('LNT').error(message)
> +
>
> - print >>sys.stderr, '%s: %s: %s' % (location, kind, message)
> +def fatal(message):
> + logging.getLogger('LNT').critical(message)
> + sys.exit(1)
>
> -note = lambda message: _write_message('note', message)
> -warning = lambda message: _write_message('warning', message)
> -error = lambda message: _write_message('error', message)
> -fatal = lambda message: (_write_message('fatal error', message),
> sys.exit(1))
>
> def rm_f(path):
> try:
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20151218/aafc0478/attachment.html>
More information about the llvm-commits
mailing list