<div dir="ltr">Massive +1 from me for this! Nice work!</div><br><div class="gmail_quote"><div dir="ltr">On Fri, 18 Dec 2015 at 01:41 Chris Matthews via llvm-commits <<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: cmatthews<br>
Date: Thu Dec 17 19:37:53 2015<br>
New Revision: 255958<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=255958&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=255958&view=rev</a><br>
Log:<br>
Fix LNT's logging, add /log endpoint to view logs from web<br>
<br>
LNT's logging support has always been horrible. This makes debugging server problems hard. This patch tries to get all the parts of logging in LNT working together better. First replace the stderr loggers in commands with python logging package. Replace all uses of the python logging package with a single logger, so all messages from LNT will be in one log. Add a in memory logging buffer, and allow the user to see that through the /log page. Make sure the rotating file logging is used as well.<br>
<br>
Later patches will add support for subprocesses.<br>
<br>
Added:<br>
lnt/trunk/lnt/server/ui/templates/log.html<br>
Modified:<br>
lnt/trunk/lnt/lnttool/main.py<br>
lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py<br>
lnt/trunk/lnt/server/ui/app.py<br>
lnt/trunk/lnt/server/ui/views.py<br>
lnt/trunk/lnt/testing/util/commands.py<br>
<br>
Modified: lnt/trunk/lnt/lnttool/main.py<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/lnttool/main.py?rev=255958&r1=255957&r2=255958&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/lnttool/main.py?rev=255958&r1=255957&r2=255958&view=diff</a><br>
==============================================================================<br>
--- lnt/trunk/lnt/lnttool/main.py (original)<br>
+++ lnt/trunk/lnt/lnttool/main.py Thu Dec 17 19:37:53 2015<br>
@@ -65,10 +65,12 @@ view the results.\<br>
input_path, = args<br>
<br>
# Setup the base LNT logger.<br>
- logger = logging.getLogger("lnt")<br>
+ # Root logger in debug.<br>
+ logger = logging.getLogger("LNT")<br>
if opts.debugger:<br>
logger.setLevel(logging.DEBUG)<br>
- handler = logging.StreamHandler(sys.stderr)<br>
+ handler = logging.StreamHandler()<br>
+ handler.setLevel(logging.DEBUG)<br>
handler.setFormatter(logging.Formatter(<br>
'%(asctime)s %(levelname)s: %(message)s',<br>
datefmt='%Y-%m-%d %H:%M:%S'))<br>
@@ -79,7 +81,7 @@ view the results.\<br>
sa_logger = logging.getLogger("sqlalchemy")<br>
if opts.debugger:<br>
sa_logger.setLevel(logging.DEBUG)<br>
- sa_logger.setLevel(logging.DEBUG)<br>
+ sa_logger.setLevel(logging.INFO)<br>
sa_logger.addHandler(handler)<br>
<br>
import lnt.server.ui.app<br>
@@ -158,6 +160,14 @@ def action_runtest(name, args):<br>
warning("--{} should be passed directly to the"<br>
" test suite.".format(key))<br>
<br>
+ logger = logging.getLogger("LNT")<br>
+ logger.setLevel(logging.DEBUG)<br>
+ handler = logging.StreamHandler()<br>
+ handler.setLevel(logging.DEBUG)<br>
+ handler.setFormatter(logging.Formatter(<br>
+ '%(asctime)s %(levelname)s: %(message)s',<br>
+ datefmt='%Y-%m-%d %H:%M:%S'))<br>
+ logger.addHandler(handler)<br>
import lnt.tests<br>
try:<br>
test_instance = lnt.tests.get_test_instance(test_name)<br>
<br>
Modified: lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py?rev=255958&r1=255957&r2=255958&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py?rev=255958&r1=255957&r2=255958&view=diff</a><br>
==============================================================================<br>
--- lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py (original)<br>
+++ lnt/trunk/lnt/server/db/rules/rule_update_fixed_regressions.py Thu Dec 17 19:37:53 2015<br>
@@ -35,6 +35,7 @@ def regression_evolution(ts, regressions<br>
Look at regressions in detect, do they match our policy? If no, move to NTBF.<br>
<br>
"""<br>
+ note("Running regression evolution")<br>
detects = [r for r in regressions if r.state == RegressionState.DETECTED]<br>
<br>
for regression in detects:<br>
<br>
Modified: lnt/trunk/lnt/server/ui/app.py<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/app.py?rev=255958&r1=255957&r2=255958&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/app.py?rev=255958&r1=255957&r2=255958&view=diff</a><br>
==============================================================================<br>
--- lnt/trunk/lnt/server/ui/app.py (original)<br>
+++ lnt/trunk/lnt/server/ui/app.py Thu Dec 17 19:37:53 2015<br>
@@ -5,6 +5,7 @@ import logging.handlers<br>
from logging import Formatter<br>
import os<br>
import time<br>
+import StringIO<br>
<br>
import flask<br>
from flask import current_app<br>
@@ -64,7 +65,6 @@ class Request(flask.Request):<br>
#<br>
# FIXME: Conditionalize on an is_production variable.<br>
if echo:<br>
- import logging, StringIO<br>
g.db_log = StringIO.StringIO()<br>
logger = logging.getLogger("sqlalchemy")<br>
logger.addHandler(logging.StreamHandler(g.db_log))<br>
@@ -137,7 +137,10 @@ class App(flask.Flask):<br>
# Inject a fix for missing slashes on the root URL (see Flask issue<br>
# #169).<br>
self.wsgi_app = RootSlashPatchMiddleware(self.wsgi_app)<br>
-<br>
+<br>
+ self.lnt_logger = logging.getLogger('LNT')<br>
+ self.lnt_logger.setLevel(logging.DEBUG)<br>
+<br>
def load_config(self, instance):<br>
self.instance = instance<br>
self.old_config = self.instance.config<br>
@@ -154,9 +157,21 @@ class App(flask.Flask):<br>
to stderr, so just log to a file as well.<br>
<br>
"""<br>
+ # Print to screen.<br>
+ ch = logging.StreamHandler()<br>
+ ch.setLevel(logging.DEBUG)<br>
+ self.lnt_logger.addHandler(ch)<br>
+<br>
+ # Log to mem for the /log view.<br>
+ h = logging.handlers.MemoryHandler(1024 * 1024)<br>
+ h.setLevel(logging.DEBUG)<br>
+ self.lnt_logger.addHandler(h)<br>
+ self.logger.addHandler(h)<br>
+ self.old_config.mem_logger = h<br>
+<br>
if not self.debug:<br>
LOG_FILENAME = "lnt.log"<br>
- try:<br>
+ try:<br>
rotating = logging.handlers.RotatingFileHandler(<br>
LOG_FILENAME, maxBytes=1048576, backupCount=5)<br>
rotating.setFormatter(Formatter(<br>
@@ -164,7 +179,7 @@ class App(flask.Flask):<br>
'[in %(pathname)s:%(lineno)d]'<br>
))<br>
rotating.setLevel(logging.DEBUG)<br>
- self.logger.addHandler(rotating)<br>
+ self.lnt_logger.addHandler(rotating)<br>
except (OSError, IOError) as e:<br>
print >> sys.stderr, "Error making log file", LOG_FILENAME, str(e)<br>
print >> sys.stderr, "Will not log to file."<br>
<br>
Added: lnt/trunk/lnt/server/ui/templates/log.html<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/log.html?rev=255958&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/log.html?rev=255958&view=auto</a><br>
==============================================================================<br>
--- lnt/trunk/lnt/server/ui/templates/log.html (added)<br>
+++ lnt/trunk/lnt/server/ui/templates/log.html Thu Dec 17 19:37:53 2015<br>
@@ -0,0 +1,35 @@<br>
+{% import "utils.html" as utils %}<br>
+<br>
+{% extends "layout.html" %}<br>
+{% set components = [] %}<br>
+<br>
+{% block title %}Log{%endblock%}<br>
+<br>
+{% block body %}<br>
+<br>
+<table class="table table-striped table-hover table-condensed"><br>
+ <thead><br>
+ <tr><br>
+ <th>Kind</th><br>
+ <th>Location</th><br>
+ <th>Time</th><br>
+ <th>Message<th><br>
+ </tr><br>
+ </thead><br>
+ <tbody class="searchable"><br>
+{% for item in old_config.mem_logger.buffer %}<br>
+ {% if item.levelname|string() == 'WARNING' %}<br>
+ <tr class="warning"><br>
+ {% elif item.levelname|string() == 'ERROR' %}<br>
+ <tr class="error"><br>
+ {% else %}<br>
+ <tr class="info"><br>
+ {% endif %}<br>
+ <td>{{ item.levelname}}</td><br>
+ <td>{{item.filename}}:{{item.lineno}}</td><br>
+ <td>{{ item.asctime}}</td><br>
+ <td>{{ item.msg }}</td><br>
+ </tr><br>
+{% endfor %}<br>
+<br>
+{% endblock %}<br>
<br>
Modified: lnt/trunk/lnt/server/ui/views.py<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/views.py?rev=255958&r1=255957&r2=255958&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/views.py?rev=255958&r1=255957&r2=255958&view=diff</a><br>
==============================================================================<br>
--- lnt/trunk/lnt/server/ui/views.py (original)<br>
+++ lnt/trunk/lnt/server/ui/views.py Thu Dec 17 19:37:53 2015<br>
@@ -1169,3 +1169,7 @@ def rules():<br>
lnt.server.db.rules.register_hooks()<br>
discovered_rules = lnt.server.db.rules.DESCRIPTIONS<br>
return render_template("rules.html",rules=discovered_rules)<br>
+<br>
+@frontend.route('/log')<br>
+def log():<br>
+ return render_template("log.html")<br>
<br>
Modified: lnt/trunk/lnt/testing/util/commands.py<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/testing/util/commands.py?rev=255958&r1=255957&r2=255958&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/testing/util/commands.py?rev=255958&r1=255957&r2=255958&view=diff</a><br>
==============================================================================<br>
--- lnt/trunk/lnt/testing/util/commands.py (original)<br>
+++ lnt/trunk/lnt/testing/util/commands.py Thu Dec 17 19:37:53 2015<br>
@@ -6,23 +6,18 @@ import errno<br>
import inspect<br>
import os<br>
import sys<br>
-<br>
+import logging<br>
# FIXME: Find a better place for this code.<br>
<br>
-def _write_message(kind, message):<br>
- # Get the file/line where this message was generated.<br>
- f = inspect.currentframe()<br>
- # Step out of _write_message, and then out of wrapper.<br>
- f = f.f_back.f_back<br>
- file,line,_,_,_ = inspect.getframeinfo(f)<br>
- location = '%s:%d' % (os.path.basename(file), line)<br>
+note = lambda message: logging.getLogger('LNT').info(message)<br>
+warning = lambda message: logging.getLogger('LNT').warning(message)<br>
+error = lambda message: logging.getLogger('LNT').error(message)<br>
+<br>
<br>
- print >>sys.stderr, '%s: %s: %s' % (location, kind, message)<br>
+def fatal(message):<br>
+ logging.getLogger('LNT').critical(message)<br>
+ sys.exit(1)<br>
<br>
-note = lambda message: _write_message('note', message)<br>
-warning = lambda message: _write_message('warning', message)<br>
-error = lambda message: _write_message('error', message)<br>
-fatal = lambda message: (_write_message('fatal error', message), sys.exit(1))<br>
<br>
def rm_f(path):<br>
try:<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@lists.llvm.org" target="_blank">llvm-commits@lists.llvm.org</a><br>
<a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits</a><br>
</blockquote></div>