[llvm-commits] [zorg] r99274 - in /zorg/trunk/lnt: examples/ examples/functions.py lnt/lnttool/__init__.py lnt/testing/ lnt/testing/__init__.py lnt/viewer/Config.py lnt/viewer/PerfDB.py lnt/viewer/root.ptl lnt/viewer/simple.ptl
Daniel Dunbar
daniel at zuster.org
Tue Mar 23 02:59:31 PDT 2010
Author: ddunbar
Date: Tue Mar 23 04:59:31 2010
New Revision: 99274
URL: http://llvm.org/viewvc/llvm-project?rev=99274&view=rev
Log:
LNT: Add a new 'simple' UI, which is an NT like viewer for generic data, and works with the full range of the database features (in particular, multiple samples per test, and test parameters).
Added:
zorg/trunk/lnt/examples/
zorg/trunk/lnt/examples/functions.py
zorg/trunk/lnt/lnt/testing/
zorg/trunk/lnt/lnt/testing/__init__.py
zorg/trunk/lnt/lnt/viewer/simple.ptl
Modified:
zorg/trunk/lnt/lnt/lnttool/__init__.py
zorg/trunk/lnt/lnt/viewer/Config.py
zorg/trunk/lnt/lnt/viewer/PerfDB.py
zorg/trunk/lnt/lnt/viewer/root.ptl
Added: zorg/trunk/lnt/examples/functions.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/examples/functions.py?rev=99274&view=auto
==============================================================================
--- zorg/trunk/lnt/examples/functions.py (added)
+++ zorg/trunk/lnt/examples/functions.py Tue Mar 23 04:59:31 2010
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+"""
+Simple example of a test generator which just produces data on some mathematical
+functions, keyed off of the current time.
+"""
+
+import time
+import math, random
+
+from lnt.testing import *
+
+def main():
+ offset = math.pi/5
+ delay = 120.
+
+ machine = Machine('Mr. Sin Wave', info = { 'delay' : delay })
+
+ start = time.time()
+
+ run = Run(start, start, info = { 't' : start,
+ 'tag' : 'simple' })
+ tests = [TestSamples('simple.%s' % name,
+ [fn(start*2*math.pi / delay + j * offset)],
+ info = { 'offset' : j })
+ for j in range(5)
+ for name,fn in (('sin',math.sin),
+ ('cos',math.cos),
+ ('random',lambda x: random.random()))]
+
+ report = Report(machine, run, tests)
+
+ print report.render()
+
+if __name__ == '__main__':
+ main()
Modified: zorg/trunk/lnt/lnt/lnttool/__init__.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/lnttool/__init__.py?rev=99274&r1=99273&r2=99274&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/lnttool/__init__.py (original)
+++ zorg/trunk/lnt/lnt/lnttool/__init__.py Tue Mar 23 04:59:31 2010
@@ -3,6 +3,8 @@
import os
import sys
+import StringIO
+
def action_runserver(name, args):
"""start a new development server."""
@@ -48,6 +50,32 @@
from convert import action_convert
from import_data import action_import
+def action_checkformat(name, args):
+ """check the format of an LNT test report file."""
+
+ from optparse import OptionParser, OptionGroup
+ parser = OptionParser("%%prog %s [options] files" % name)
+
+ (opts, args) = parser.parse_args(args)
+ if len(args) > 1:
+ parser.error("incorrect number of argments")
+
+ if len(args) == 0:
+ input = '-'
+ else:
+ input, = args
+
+ if input == '-':
+ input = StringIO.StringIO(sys.stdin.read())
+
+ from lnt import formats
+ from lnt.viewer import PerfDB
+
+ db = PerfDB.PerfDB('sqlite:///:memory:')
+
+ data = formats.read_any(input, '<auto>')
+ PerfDB.importDataFromDict(db, data)
+
def action_submit(name, args):
"""submit a test report to the server."""
@@ -83,7 +111,7 @@
if len(sys.argv) < 2 or sys.argv[1] not in commands:
if len(sys.argv) >= 2:
- print >>sys.sterr,"error: invalid command %r\n" % sys.argv[1]
+ print >>sys.stderr,"error: invalid command %r\n" % sys.argv[1]
usage()
Added: zorg/trunk/lnt/lnt/testing/__init__.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/testing/__init__.py?rev=99274&view=auto
==============================================================================
--- zorg/trunk/lnt/lnt/testing/__init__.py (added)
+++ zorg/trunk/lnt/lnt/testing/__init__.py Tue Mar 23 04:59:31 2010
@@ -0,0 +1,72 @@
+"""
+Utilities for working with the LNT test format.
+"""
+
+import time
+import datetime
+import json
+
+def normalize_time(t):
+ if isinstance(t,float):
+ t = datetime.datetime.utcfromtimestamp(t)
+ elif not isinstance(t, datatime.datetime):
+ t = time.strptime(start_time, '%Y-%m-%d %H:%M:%S')
+ return t.strftime('%Y-%m-%d %H:%M:%S')
+
+class Machine:
+ def __init__(self, name, info={}):
+ self.name = str(name)
+ self.info = dict((str(key),str(value))
+ for key,value in info.items())
+
+ def render(self):
+ return { 'Name' : self.name,
+ 'Info' : self.info }
+
+class Run:
+ def __init__(self, start_time, end_time, info={}):
+ if start_time is None:
+ start_time = datetime.datetime.now()
+ if end_time is None:
+ end_time = datetime.datetime.now()
+
+ self.start_time = normalize_time(start_time)
+ self.end_time = normalize_time(end_time)
+ self.info = dict((str(key),str(value))
+ for key,value in info.items())
+
+ def render(self):
+ return { 'Start Time' : self.start_time,
+ 'End Time' : self.end_time,
+ 'Info' : self.info }
+
+class TestSamples:
+ def __init__(self, name, data, info={}):
+ self.name = str(name)
+ self.info = dict((str(key),str(value))
+ for key,value in info.items())
+ self.data = map(float, data)
+
+ def render(self):
+ return { 'Name' : self.name,
+ 'Info' : self.info,
+ 'Data' : self.data }
+
+class Report:
+ def __init__(self, machine, run, tests):
+ self.machine = machine
+ self.run = run
+ self.tests = list(tests)
+
+ assert isinstance(self.machine, Machine)
+ assert isinstance(self.run, Run)
+ for t in self.tests:
+ assert isinstance(t, TestSamples)
+
+ def render(self):
+ return json.dumps({ 'Machine' : self.machine.render(),
+ 'Run' : self.run.render(),
+ 'Tests' : [t.render() for t in self.tests] },
+ sort_keys=True, indent=4)
+
+__all__ = ['Machine', 'Run', 'TestSamples', 'Report']
Modified: zorg/trunk/lnt/lnt/viewer/Config.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/viewer/Config.py?rev=99274&r1=99273&r2=99274&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/viewer/Config.py (original)
+++ zorg/trunk/lnt/lnt/viewer/Config.py Tue Mar 23 04:59:31 2010
@@ -12,12 +12,14 @@
dbPath = os.path.join(baseDir, dbPath)
return DBInfo(dbPath,
bool(dict.get('showNightlytest')),
- bool(dict.get('showGeneral')))
+ bool(dict.get('showGeneral')),
+ bool(dict.get('showSimple')))
- def __init__(self, path, showNightlytest, showGeneral):
+ def __init__(self, path, showNightlytest, showGeneral, showSimple):
self.path = path
- self.showNightlytest = showNightlytest
self.showGeneral = showGeneral
+ self.showNightlytest = showNightlytest
+ self.showSimple = showSimple
class Config:
@staticmethod
Modified: zorg/trunk/lnt/lnt/viewer/PerfDB.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/viewer/PerfDB.py?rev=99274&r1=99273&r2=99274&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/viewer/PerfDB.py (original)
+++ zorg/trunk/lnt/lnt/viewer/PerfDB.py Tue Mar 23 04:59:31 2010
@@ -321,11 +321,12 @@
test_info = {}
for id,k,v in db.session.query(TestInfo.test_id, TestInfo.key,
TestInfo.value):
- test_info[id] = (str(k),str(v))
+ info = test_info[id] = test_info.get(id,{})
+ info[str(k)] = str(v)
testMap = {}
for test_id,test_name in db.session.query(Test.id, Test.name):
- info = test_info.get(test_id,[])
+ info = test_info.get(test_id,{}).items()
info.sort()
testMap[(str(test_name),tuple(info))] = test_id
@@ -338,7 +339,7 @@
info.sort()
test_id = testMap.get((name,tuple(info)))
if test_id is None:
- test,created = db.getOrCreateTest(testData['Name'],testData['Info'])
+ test,created = db.getOrCreateTest(testData['Name'],info)
assert created
late_ids.append((i,test))
test_ids.append(test_id)
Modified: zorg/trunk/lnt/lnt/viewer/root.ptl
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/viewer/root.ptl?rev=99274&r1=99273&r2=99274&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/viewer/root.ptl (original)
+++ zorg/trunk/lnt/lnt/viewer/root.ptl Tue Mar 23 04:59:31 2010
@@ -21,7 +21,7 @@
class RootDirectory(Resolving, Directory):
_q_exports = ["", "resources", "js", "machines", "runs", "tests",
- "browse", "submitRun", "nightlytest", "zview",
+ "browse", "submitRun", "nightlytest", "simple", "zview",
# Redirections.
"select_db",
@@ -172,6 +172,11 @@
# Available UIs.
+ if self.dbInfo.showSimple:
+ """
+ <a href="simple/">Simple Test Viewer</a>
+ """
+
if self.dbInfo.showNightlytest:
"""
<h3>Nightly Test Results</h3>
@@ -359,6 +364,9 @@
if component == 'nightlytest':
import nightlytest
return nightlytest.NightlyTestDirectory(self)
+ if component == 'simple':
+ import simple
+ return simple.RootDirectory(self)
if component == 'zview':
from zview import zviewui
return zviewui.ZViewUI(self)
Added: zorg/trunk/lnt/lnt/viewer/simple.ptl
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/viewer/simple.ptl?rev=99274&view=auto
==============================================================================
--- zorg/trunk/lnt/lnt/viewer/simple.ptl (added)
+++ zorg/trunk/lnt/lnt/viewer/simple.ptl Tue Mar 23 04:59:31 2010
@@ -0,0 +1,471 @@
+# -*- python -*-
+
+"""
+Nightly Test UI instance for actual nightly test data.
+"""
+
+# FIXME: The NTStyleBrowser abstraction is no longer useful. We should kill it.
+
+import sys
+import time
+
+import quixote
+from quixote.directory import Directory
+from quixote.errors import TraversalError
+
+import Util, NTStyleBrowser
+from Util import safediv
+from NTUtil import *
+
+from PerfDB import Machine, Run, Test
+
+class SimpleRunUI(Directory):
+ _q_exports = ["", "graph"]
+
+ def __init__(self, root, idstr):
+ self.root = root
+ try:
+ self.id = int(idstr)
+ except ValueError, exc:
+ raise TraversalError(str(exc))
+
+ def getInfo(self, db):
+ request = quixote.get_request()
+
+ compareToID = request.form.get('compare', '')
+ compareTo = None
+ if compareToID:
+ try:
+ compareTo = db.getRun(int(compareToID))
+ except:
+ pass
+
+ run = db.getRun(self.id)
+
+ # Find previous runs, ordered by time.
+ runs = db.runs(run.machine).order_by(Run.start_time.desc()).all()
+ runs = [r for r in runs
+ if 'tag' in r.info and r.info['tag'].value == 'simple']
+
+ # Find previous run to compare to.
+ if compareTo is None:
+ for r in runs:
+ # FIXME: Compare revisions, not times.
+ if r != run and r.start_time < run.start_time:
+ compareTo = r
+ break
+
+ return run, runs, compareTo
+
+ def show_run_page [html] (self, db, run, runs, compareTo, contents_fn):
+ machine = run.machine
+
+ """
+ <center>
+ <table>
+ <tr>
+ <td align=right>Machine:</td>
+ <td>%s:%d</td>
+ </tr>
+ <tr>
+ <td align=right>Run:</td>
+ <td>%s</td>
+ </tr>
+ """ % (machine.name, machine.number, run.start_time)
+ if compareTo:
+ """
+ <tr>
+ <td align=right>Compare To:</td>
+ <td>%s</td>
+ </tr>
+ """ % (compareTo.start_time,)
+ """
+ </table>
+ </center>
+ <p>
+ """
+
+ """
+ <table width="100%%" border=1>
+ <tr>
+ <td valign="top" width="200">
+ <a href="..">Homepage</a>
+ <h4>Machine:</h4>
+ <a href="../machines/%d/">%s:%d</a>
+ <h4>Runs:</h4>
+ <ul>
+ """ % (machine.id, machine.name, machine.number)
+
+ # Show a small number of neighboring runs.
+ runIndex = runs.index(run)
+ for r in runs[max(0,runIndex-3):runIndex+6]:
+ if r == run:
+ """ <li> <h3><a href="../%d/">%s</a></h3> """ % (r.id,
+ r.start_time)
+ else:
+ """ <li> <a href="../%d/">%s</a> """ % (r.id, r.start_time)
+ """
+ </ul>
+ </td>
+ <td valign="top">
+ <table border=1>
+ <tr>
+ <td> <b>Nickname</b> </td>
+ <td> %s </td>
+ </tr>
+ <tr>
+ <td> <b>Machine ID</b> </td>
+ <td> %d </td>
+ </tr>
+ </table>
+ <p>
+ <table border=1>
+ """ % (machine.name, machine.id)
+ for mi in machine.info.values():
+ """
+ <tr>
+ <td> <b>%s</b> </td>
+ <td>%s</td>
+ </tr>
+ """ % (mi.key, mi.value)
+ """
+ </table>
+ <p>
+ <table border=1>
+ """
+ for ri in run.info.values():
+ """
+ <tr>
+ <td> <b>%s</b> </td>
+ <td>%s</td>
+ </tr>
+ """ % (ri.key, ri.value)
+ """
+ </table>
+ """
+
+ contents_fn(db, run, runs, compareTo)
+
+ """
+ </td>
+ </tr>
+ </table>
+ """
+
+ self.root.getFooter()
+
+ def _q_index [html] (self):
+ # Get a DB connection.
+ db = self.root.getDB()
+
+ run,runs,compareTo = self.getInfo(db)
+ machine = run.machine
+
+ self.root.getHeader('Run Results', "../..",
+ components=(('simple','simple'),
+ ('machine',
+ 'simple/machines/%d' % machine.id)),
+ addPopupJS=True, addFormCSS=True)
+
+ self.show_run_page(db, run, runs, compareTo, self._q_index_body)
+
+ def graph [html] (self):
+ request = quixote.get_request()
+
+ # Get a DB connection.
+ db = self.root.getDB()
+
+ run,runs,compareTo = self.getInfo(db)
+ machine = run.machine
+
+ # Load the metadata.
+
+ test_names,test_map,parameter_keys,parameter_sets = self.\
+ get_simple_metadata(db)
+
+ # Load the form data.
+ graph_tests = []
+ graph_psets = []
+ for name,value in request.form.items():
+ if name.startswith(str('test.')):
+ graph_tests.append(name[5:])
+ elif name.startswith(str('pset.')):
+ graph_psets.append(parameter_sets[int(name[5:])])
+
+ # Get the test ids we want data for.
+ test_ids = [test_map[(name,pset)].id
+ for name in graph_tests
+ for pset in graph_psets]
+
+ # Load all the samples for those tests and this machine.
+ q = db.session.query(Sample.run_id,Sample.test_id,
+ Sample.value).join(Run)
+ q = q.filter(Run.machine_id == machine.id)
+ q = q.filter(Sample.test_id.in_(test_ids))
+ samples = list(q)
+
+ # Aggregate by test id and then run id.
+ #
+ # FIXME: Pretty expensive.
+ samples_by_test_id = {}
+ for run_id,test_id,value in samples:
+ d = samples_by_test_id.get(test_id)
+ if d is None:
+ d = samples_by_test_id[test_id] = Util.multidict()
+ d[run_id] = value
+
+ # Build the graph data
+ run_id_map = dict([(r.id,r) for r in runs])
+ pset_id_map = dict([(pset,i) for i,pset in enumerate(parameter_sets)])
+ legend = []
+ plots = ""
+ num_plots = len(graph_tests) * len(graph_psets)
+ num_points = 0
+ index = 0
+ for name in graph_tests:
+ for pset in graph_psets:
+ test_id = test_map[(name,pset)].id
+
+ # Get the plot for this test.
+ #
+ # FIXME: Support order by something other than time.
+ data = []
+ for run_id,values in samples_by_test_id.get(test_id,{}).items():
+ r = run_id_map.get(run_id)
+ if not r:
+ continue
+ timeval = time.mktime(r.start_time.timetuple())
+ data.append((timeval, min(values)))
+ data.sort()
+ num_points += len(data)
+
+ col = list(Util.makeDarkColor(float(index) / num_plots))
+ pts = ','.join(['[%f,%f]' % (t,v) for t,v in data])
+ style = "new Graph2D_LinePlotStyle(1, %r)" % col
+ plots += " graph.addPlot([%s], %s);\n" % (pts,style)
+
+ legend.append(("%s : P%d" % (name, pset_id_map[pset]), col))
+ index += 1
+
+ def graph_body [html] (db, run, runs, compare_to):
+ """
+ <h3>Graph</h3>
+ <table>
+ <tr>
+ <td rowspan=2 valign="top">
+ <canvas id="graph" width="600" height="400"></canvas>
+ </td>
+ <td valign="top">
+ <table cellspacing=4 border=1>
+ <tr><th colspan=2>Test</th></tr>
+ """
+ for name,col in legend:
+ """
+ <tr><td bgcolor="%02x%02x%02x"> </td>
+ <td>%s</td></tr>""" % (255*col[0], 255*col[1], 255*col[2], name)
+ """
+ </table>
+ </td></tr>
+ <tr><td align="right" valign="bottom">
+ <font size="-2">
+ Shift-Left Mouse: Pan<br>
+ Alt/Meta-Left Mouse: Zoom<br>
+ Wheel: Zoom (<i>Shift Slows</i>)<br>
+ </font>
+ </td></tr>
+ </table>
+ <p>
+ <b>Plots</b>: %d<br>
+ <b>Num Points<b>: %d<br>
+ """ % (num_plots, num_points)
+
+ graph_init = """\
+ function init() {
+ graph = new Graph2D("graph");
+ graph.clearColor = [1, 1, 1];
+ %s
+ graph.xAxis.format = graph.xAxis.formats.day;
+ graph.draw();
+ }
+ """ % (plots,)
+ self.root.getHeader('Run Results', "..",
+ components=(('simple','simple'),
+ ('machine',
+ 'simple/machines/%d' % machine.id),
+ ('run', 'simple/%d' % run.id)),
+ addPopupJS=True, addGraphJS=True,
+ addJSScript=graph_init,
+ onload="init()")
+
+ self.show_run_page(db, run, runs, compareTo, graph_body)
+
+ def get_simple_metadata(self, db):
+ """Compute the metadata about tests, parameter sets, etc."""
+
+ # FIXME: We can cache this in a number of ways.
+
+ # Find all test names.
+ q = db.session.query(Test)
+ q = q.filter(Test.name.startswith(str('simple.')))
+ tests = list(q)
+
+ # Collect all the test data.
+ test_names = set()
+ parameter_sets = set()
+ test_map = {}
+ for t in tests:
+ name = t.name.split(str('.'),1)[1]
+ test_names.add(name)
+
+ items = [(k,v.value) for k,v in t.info.items()]
+ items.sort()
+ key = tuple(items)
+
+ parameter_sets.add(key)
+ test_map[(name, key)] = t
+
+ # Order the test names.
+ test_names = list(test_names)
+ test_names.sort()
+
+ # Collect the set of all parameter keys.
+ parameter_keys = list(set([k for pset in parameter_sets
+ for k,v in pset]))
+ parameter_keys.sort()
+
+ # Order the parameter sets and convert to dictionaries.
+ parameter_sets = list(parameter_sets)
+ parameter_sets.sort()
+
+ return test_names,test_map,parameter_keys,parameter_sets
+
+ def _q_index_body [html] (self, db, run, runs, compare_to):
+ # Find the tests. The simple UI maps all tests that start with
+ # 'simple.'.
+ #
+ # One sensible addition would be to allow 'simple.foo.success' as a test
+ # to indicate the success or failure of the test. We would assume that
+ # the test succeeded if its .success test was missing, which leads to a
+ # nice compact format (failures are expected to be rare).
+
+ if compare_to:
+ prev_id = compare_to.id
+ interesting_runs = (run.id, prev_id)
+ else:
+ prev_id = None
+ interesting_runs = (run.id,)
+
+ # Load the metadata.
+
+ test_names,test_map,parameter_keys,parameter_sets = self.\
+ get_simple_metadata(db)
+
+ # Load the run sample data.
+
+ q = db.session.query(Sample.value, Sample.run_id, Sample.test_id)
+ q = q.filter(Sample.run_id.in_(interesting_runs))
+
+ sample_map = Util.multidict()
+ for value,run_id,test_id in q:
+ key = (run_id,test_id)
+ sample_map[key] = value
+
+ # Render the page.
+
+ def get_cell_value [html] (test, name, pset):
+ run_values = sample_map.get((run.id,test.id))
+ prev_values = sample_map.get((prev_id,test.id))
+
+ # FIXME: Check success
+ failed = not run_values
+
+ run_cell_value = "-"
+ if run_values:
+ run_cell_value = "%.2f" % min(run_values)
+
+ if failed:
+ """
+ <td bgcolor="#FF0000">%s</td""" % run_cell_value
+ else:
+ """
+ <td>%s</td""" % run_cell_value
+
+ if prev_values and run_values:
+ prev_value = min(prev_values)
+ pct = safediv(min(run_values), prev_value,
+ '<center><font size=-2>nan</font></center>')
+ Util.PctCell(pct, delta=True).render()
+ else:
+ """<td>-</td>"""
+
+
+ """
+ <h3>Parameter Sets</h3>
+ <table border=1>
+ <tr>
+ <th rowspan=2>Name</th>
+ <th colspan=%d>Parameters</th>
+ </tr><tr>""" % len(parameter_sets)
+ for key in parameter_keys:
+ """
+ <th>%s</th>""" % key
+ """
+ </tr>"""
+ for (i,pset) in enumerate(parameter_sets):
+ """
+ <tr>
+ <td>P%s</td>""" % (i,)
+ pmap = dict(pset)
+ for key in parameter_keys:
+ item = pmap.get(key)
+ if item is None:
+ item = "-"
+ """
+ <td>%s</td>""" % item
+ """
+ </tr>"""
+ """
+ </table>"""
+
+ """
+ <h3>Tests</h3>"""
+
+ """
+ <form method="GET" action="graph">
+ <table border=1>
+ <tr>
+ <th></th><th>Name</th>"""
+ for i in range(len(parameter_sets)):
+ """
+ <th><input type="checkbox" name="pset.%d">P%d</th>
+ <th>%%</th>""" % (i, i)
+ """
+ </tr>"""
+ for name in test_names:
+ """
+ <tr>
+ <td><input type="checkbox" name="test.%s"></td>
+ <td>%s</td>""" % (name, name)
+ for pset in parameter_sets:
+ test = test_map.get((name,pset))
+ if test is None:
+ """
+ <td></td><td></td>"""
+ continue
+
+ get_cell_value(test, name, pset)
+ """
+ </tr>"""
+ """
+ </table>
+ <input type="submit" value="Graph">
+ </form>"""
+
+class RootDirectory(NTStyleBrowser.RecentMachineDirectory):
+ _q_exports = [""]
+
+ def getTags(self):
+ return ('simple',)
+
+ def getTestRunUI(self, component):
+ return SimpleRunUI(self.root, component)
More information about the llvm-commits
mailing list