[LNT] r273216 - LNT Matrix view
Chris Matthews via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 20 16:04:30 PDT 2016
Author: cmatthews
Date: Mon Jun 20 18:04:29 2016
New Revision: 273216
URL: http://llvm.org/viewvc/llvm-project?rev=273216&view=rev
Log:
LNT Matrix view
This adds a new view, similar to graph view which shows result data in a order aligned table. When monitoring a small number of important benchmarks, this view will be useful.
Added:
lnt/trunk/lnt/server/ui/templates/v4_matrix.html
lnt/trunk/tests/server/ui/test_matrix_page.py
Modified:
lnt/trunk/lnt/server/ui/templates/v4_run.html
lnt/trunk/lnt/server/ui/util.py
lnt/trunk/lnt/server/ui/views.py
Added: lnt/trunk/lnt/server/ui/templates/v4_matrix.html
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/v4_matrix.html?rev=273216&view=auto
==============================================================================
--- lnt/trunk/lnt/server/ui/templates/v4_matrix.html (added)
+++ lnt/trunk/lnt/server/ui/templates/v4_matrix.html Mon Jun 20 18:04:29 2016
@@ -0,0 +1,81 @@
+{% set nosidebar = True %}
+{% import "utils.html" as utils %}
+{% import "local.html" as local %}
+{% set ts = request.get_testsuite() %}
+
+{% extends "layout.html" %}{
+{% set components = [(testsuite_name, v4_url_for("v4_recent_activity"))] %}
+{% block head %}
+{% endblock %}
+
+
+
+{% macro get_cell_value(cr) %}
+ {% set test_status = cr.get_test_status() %}
+ {% set value_status = cr.get_value_status(ignore_small=not options.show_small_diff) %}
+ {% set run_cell_value = "-" if cr.current is none else "%.4f" % cr.current %}
+
+ {% if value_status != analysis.UNCHANGED_PASS and value_status != analysis.UNCHANGED_FAIL %}
+ {{ cr.pct_delta|aspctcell(reverse=cr.bigger_is_better, data=run_cell_value)|safe }}
+ {% else %}
+ <td>{{ run_cell_value }}</td>
+ {% endif%}
+
+
+{% endmacro %}
+
+
+{% block title %}Matrix View{% endblock %}
+
+{% block sidebar %}
+ <ul class="nav nav-list bs-docs-sidenav">
+ </ul>
+ {% endblock %}
+
+
+{% block body %}
+<h2>Matrix View</h2>
+<form method="POST" action="">
+{{ form.hidden_tag() }}
+{{ form.limit(class="auto_submit_item") }}
+<h3>Baseline: {{baseline_rev}}</h3>
+<div id="matrix_div">
+ {% if machine_name_common %}
+ <p>Machine: <a href="{{ v4_url_for('v4_machine', id=machine_id_common) }}">{{ machine_name_common }}</a></p>
+ {% endif %}
+ <table class="table table-hover" style="width:auto">
+ <tr>
+ <th>Order</th>
+ {% for r in associated_runs: %}
+ <th>
+ {% if not machine_name_common %}
+ <a href="{{ v4_url_for('v4_machine', id=r.machine.id) }}">{{ r.machine.name }}</a>
+ {% endif %}
+ {{ r.test.name | shortname }}</th>
+ {% endfor %}
+ <th>Geomean</th>
+ </tr>
+ {% for order in orders: %}
+ {% set baseline_class = "info" if order == baseline_rev else "" %}
+ <tr class="{{baseline_class}}">
+ <td><a href="{{v4_url_for('v4_order', id=order_to_id[order])}}">{{order}}</a></td>
+ {% for req in associated_runs: %}
+ {{ get_cell_value(req.change.get(order)) }}
+ {% endfor %}
+ {{ get_cell_value(geomeans[order]) }}
+ </tr>
+ {% endfor %}
+
+</table>
+
+</div>
+
+<script type="text/javascript">
+ $(function() {
+ $(".auto_submit_item").change(function() {
+ $("form").submit();
+ });
+ });
+</script>
+
+ {% endblock %}
Modified: lnt/trunk/lnt/server/ui/templates/v4_run.html
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/v4_run.html?rev=273216&r1=273215&r2=273216&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/templates/v4_run.html (original)
+++ lnt/trunk/lnt/server/ui/templates/v4_run.html Mon Jun 20 18:04:29 2016
@@ -317,7 +317,7 @@ $('.profile-prev-only').tooltip();
{{ utils.render_popup_end() }}
{% set graph_base=v4_url_for('v4_graph', highlight_run=run.id) %}
- <form method="GET" action="{{ graph_base }}">
+ <form id="graph_selection_form" method="GET" action="{{ graph_base }}">
{# Report one table for each primary field. #}
{% for field in metric_fields %}
@@ -377,7 +377,7 @@ $('.profile-prev-only').tooltip();
{{ utils.render_popup_end() }}
</section>
{% endfor %}
- <p><input type="submit" value="Graph">
+ <p><input type="submit" value="Graph"> <input id="matrix_button" type="submit" value="Matrix">
</form>
</section>
@@ -414,5 +414,16 @@ $('.profile-prev-only').tooltip();
</tbody>
</table>
{% endif %}
+<script type="text/javascript" language="Javascript">
+$("#matrix_button").on("click", function(e){
+ e.preventDefault();
+ {% set matrix_base=v4_url_for('v4_matrix') %}
+ $('#graph_selection_form').attr('action', "{{ matrix_base }}").submit();
+});
+
+</script>
+
+
+</script>
{% endblock %}
Modified: lnt/trunk/lnt/server/ui/util.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/util.py?rev=273216&r1=273215&r2=273216&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/util.py (original)
+++ lnt/trunk/lnt/server/ui/util.py Mon Jun 20 18:04:29 2016
@@ -199,6 +199,8 @@ def lerp(a,b,t):
t_ = 1. - t
return tuple([av*t_ + bv*t for av,bv in zip(a,b)])
+
+
class PctCell:
# Color levels
kNeutralColor = (1,1,1)
@@ -208,12 +210,13 @@ class PctCell:
kNANColor = (.86,.86,.86)
kInvalidColor = (0,0,1)
- def __init__(self, value, reverse=False, precision=2, delta=False):
+ def __init__(self, value, reverse=False, precision=2, delta=False, data=None):
if delta and isinstance(value, float):
value -= 1
self.value = value
self.reverse = reverse
self.precision = precision
+ self.data = data
def getColor(self):
v = self.value
@@ -263,7 +266,10 @@ class PctCell:
for key, value in attributes.items():
attrs.append('%s="%s"' % (key, value))
attr_string = ' '.join(attrs)
- return '<td %s>%s</td>' % (attr_string, self.getValue())
+ if self.data:
+ return '<td %s>%s (%s)</td>' % (attr_string, self.data, self.getValue())
+ else:
+ return '<td %s>%s</td>' % (attr_string, self.getValue())
def sorted(l, *args, **kwargs):
Modified: lnt/trunk/lnt/server/ui/views.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/views.py?rev=273216&r1=273215&r2=273216&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/views.py (original)
+++ lnt/trunk/lnt/server/ui/views.py Mon Jun 20 18:04:29 2016
@@ -20,21 +20,26 @@ from lnt.testing.util.commands import wa
import sqlalchemy.sql
from sqlalchemy.orm.exc import NoResultFound
+from flask_wtf import Form
+from wtforms import SelectField
+
import lnt.util
import lnt.util.ImportData
import lnt.util.stats
from lnt.server.ui.globals import db_url_for, v4_url_for
import lnt.server.reporting.analysis
+from lnt.server.reporting.analysis import ComparisonResult, calc_geomean
import lnt.server.reporting.runs
from lnt.server.ui.decorators import frontend, db_route, v4_route
import lnt.server.ui.util
+from lnt.server.ui.util import mean
import lnt.server.reporting.dailyreport
import lnt.server.reporting.summaryreport
import lnt.server.db.rules_manager
import lnt.server.db.search
-from collections import namedtuple
+from lnt.server.ui.regression_views import PrecomputedCR
+from collections import namedtuple, defaultdict
from lnt.util import async_ops
-
integral_rex = re.compile(r"[\d]+")
###
@@ -1262,3 +1267,214 @@ def v4_search():
[('%s #%s' % (r.machine.name, r.order.llvm_project_revision),
r.id)
for r in results])
+
+
+class MatrixDataRequest(object):
+ def __init__(self, machine, test, field):
+ self.machine = machine
+ self.test = test
+ self.field = field
+ def __repr__(self):
+ return "{}:{}({} samples)" \
+ .format(self.machine.name,
+ self.test.name,
+ len(self.samples) if self.samples else "No")
+
+
+# How much data to render in the Matrix view.
+MATRIX_LIMITS = [('12', 'Small'),
+ ('50', 'Medium'),
+ ('250', 'Large'),
+ ('-1', 'All')]
+
+class MatrixOptions(Form):
+ limit = SelectField('Size', choices=MATRIX_LIMITS)
+
+
+ at v4_route("/matrix", methods=['GET', 'POST'])
+def v4_matrix():
+ """A table view for Run sample data, because *some* people really
+ like to be able to see results textually.
+ request.args.limit limits the number of samples.
+ for each dataset to add, there will be a "plot.n=.m.b.f" where m is machine
+ ID, b is benchmark ID and f os field kind offset. "n" is used to unique
+ the paramters, and is ignored.
+
+ """
+ ts = request.get_testsuite()
+ # Load the matrix request parameters.
+ form = MatrixOptions(request.form)
+ if request.method == 'POST':
+ post_limit = form.limit.data
+ else:
+ post_limit = MATRIX_LIMITS[0][0]
+ data_parameters = []
+ for name, value in request.args.items():
+ # plot.<unused>=<machine id>.<test id>.<field index>
+ if not name.startswith(str('plot.')):
+ continue
+
+ # Ignore the extra part of the key, it is unused.
+ machine_id_str, test_id_str, field_index_str = value.split('.')
+ try:
+ machine_id = int(machine_id_str)
+ test_id = int(test_id_str)
+ field_index = int(field_index_str)
+ except:
+ err_msg = "data {} was malformed. {} must be int.int.int"
+ return abort(400, err_msg.format(name, value))
+
+ if not (0 <= field_index < len(ts.sample_fields)):
+ return abort(404, "Invalid field index: {}".format(field_index))
+
+ try:
+ machine = \
+ ts.query(ts.Machine).filter(ts.Machine.id == machine_id).one()
+ except NoResultFound:
+ return abort(404, "Invalid machine ID: {}".format(machine_id))
+ try:
+ test = ts.query(ts.Test).filter(ts.Test.id == test_id).one()
+ except NoResultFound:
+ return abort(404, "Invalid test ID: {}".format(test_id))
+ try:
+ field = ts.sample_fields[field_index]
+ except NoResultFound:
+ return abort(404, "Invalid field_index: {}".format(field_index))
+
+ valid_request = MatrixDataRequest(machine, test, field)
+ data_parameters.append(valid_request)
+
+ if not data_parameters:
+ abort(404, "Request requires some data arguments.")
+
+ # Feature: if all of the results are from the same machine, hide the name to
+ # make the headers more compact.
+ dedup = True
+ for r in data_parameters:
+ if r.machine.id != data_parameters[0].machine.id:
+ dedup = False
+ if dedup:
+ machine_name_common = data_parameters[0].machine.name
+ machine_id_common = data_parameters[0].machine.id
+ else:
+ machine_name_common = machine_id_common = None
+
+ # It is nice for the columns to be sorted by name.
+ data_parameters.sort(key=lambda x: x.test.name),
+
+ # Now lets get the data.
+ all_orders = set()
+ order_to_id = {}
+ for req in data_parameters:
+ q = ts.query(req.field.column, ts.Order.llvm_project_revision, ts.Order.id) \
+ .join(ts.Run) \
+ .join(ts.Order) \
+ .filter(ts.Run.machine_id == req.machine.id) \
+ .filter(ts.Sample.test == req.test) \
+ .filter(req.field.column != None) \
+ .order_by(ts.Order.llvm_project_revision.desc())
+
+ limit = request.args.get('limit', post_limit)
+ if limit or post_limit:
+ limit = int(limit)
+ if limit != -1:
+ q = q.limit(limit)
+
+ req.samples = defaultdict(list)
+
+ for s in q.all():
+ req.samples[s[1]].append(s[0])
+ all_orders.add(s[1])
+ order_to_id[s[1]] = s[2]
+ req.derive_stat = {}
+ for order, samples in req.samples.items():
+ req.derive_stat[order] = mean(samples)
+ if not all_orders:
+ abort(404, "No data found.")
+ # Now grab the baseline data.
+ baseline_rev = machine.DEFAULT_BASELINE_REVISION
+ if baseline_rev:
+ all_orders.add(baseline_rev)
+ else:
+ baseline_rev = next(iter(all_orders))
+
+ for req in data_parameters:
+ q_baseline = ts.query(req.field.column, ts.Order.llvm_project_revision, ts.Order.id) \
+ .join(ts.Run) \
+ .join(ts.Order) \
+ .filter(ts.Run.machine_id == req.machine.id) \
+ .filter(ts.Sample.test == req.test) \
+ .filter(req.field.column != None) \
+ .filter(ts.Order.llvm_project_revision == baseline_rev)
+ for s in q_baseline.all():
+ req.samples[s[1]].append(s[0])
+ all_orders.add(s[1])
+ order_to_id[s[1]] = s[2]
+
+ all_orders = list(all_orders)
+ all_orders.sort()
+ # Now calculate Changes between each run.
+
+ for req in data_parameters:
+ req.change = {}
+ for order in all_orders:
+ cur_samples = req.samples[order]
+ prev_samples = req.samples.get(baseline_rev, None)
+ cr = ComparisonResult(mean,
+ False, False,
+ cur_samples,
+ prev_samples,
+ None, None,
+ confidence_lv=0.05,
+ bigger_is_better=False)
+ req.change[order] = cr
+
+ # Calculate Geomean for each order.
+ order_to_geomean = {}
+ curr_geomean = None
+ for order in all_orders:
+ curr_samples = []
+ prev_samples = []
+ for req in data_parameters:
+ curr_samples.extend(req.samples[order])
+ prev_samples.extend(req.samples[baseline_rev])
+ prev_geomean = calc_geomean(prev_samples)
+ curr_geomean = calc_geomean(curr_samples)
+ if prev_geomean:
+ cr = ComparisonResult(mean,
+ False, False,
+ [curr_geomean],
+ [prev_geomean],
+ None, None,
+ confidence_lv=0.05,
+ bigger_is_better=False)
+ order_to_geomean[order] = cr
+ else:
+ # There will be no change here, but display current val.
+ if curr_geomean:
+ order_to_geomean[order] = PrecomputedCR(curr_geomean,
+ curr_geomean,
+ False)
+
+ class FakeOptions(object):
+ show_small_diff = False
+ show_previous = False
+ show_all = True
+ show_delta = False
+ show_stddev = False
+ show_mad = False
+ show_all_samples = False
+ show_sample_counts = False
+
+ return render_template("v4_matrix.html",
+ testsuite_name=g.testsuite_name,
+ associated_runs=data_parameters,
+ orders=all_orders,
+ options=FakeOptions(),
+ analysis=lnt.server.reporting.analysis,
+ geomeans=order_to_geomean,
+ order_to_id=order_to_id,
+ form=form,
+ baseline_rev=baseline_rev,
+ machine_name_common=machine_name_common,
+ machine_id_common=machine_id_common)
Added: lnt/trunk/tests/server/ui/test_matrix_page.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/ui/test_matrix_page.py?rev=273216&view=auto
==============================================================================
--- lnt/trunk/tests/server/ui/test_matrix_page.py (added)
+++ lnt/trunk/tests/server/ui/test_matrix_page.py Mon Jun 20 18:04:29 2016
@@ -0,0 +1,79 @@
+# Check that the LNT REST JSON API is working.
+# create temporary instance
+# RUN: rm -rf %t.instance
+# RUN: python %{shared_inputs}/create_temp_instance.py \
+# RUN: %s %{shared_inputs}/SmallInstance \
+# RUN: %t.instance %S/Inputs/V4Pages_extra_records.sql
+#
+# RUN: python %s %t.instance
+
+import unittest
+import logging
+import sys
+
+import lnt.server.db.migrate
+import lnt.server.ui.app
+
+from V4Pages import check_json, check_code
+logging.basicConfig(level=logging.DEBUG)
+
+import json
+
+logging.basicConfig(level=logging.DEBUG)
+
+HTTP_BAD_REQUEST = 400
+HTTP_NOT_FOUND = 404
+HTTP_OK = 200
+
+
+class MatrixViewTester(unittest.TestCase):
+ """Test the Matrix view."""
+
+ def setUp(self):
+ """Bind to the LNT test instance."""
+ _, instance_path = sys.argv
+ app = lnt.server.ui.app.App.create_standalone(instance_path)
+ app.testing = True
+ self.client = app.test_client()
+
+ def test_config_errors(self):
+ """Does passing bad arugments to matrix view error correctly.
+ """
+ client = self.client
+ reply = check_code(client, '/v4/nts/matrix',
+ expected_code=HTTP_NOT_FOUND)
+ self.assertIn("Request requires some data arguments.", reply.data)
+
+ reply = check_code(client, '/v4/nts/matrix?plot.0=1.1.1',
+ expected_code=HTTP_NOT_FOUND)
+ self.assertIn("No data found.", reply.data)
+
+ reply = check_code(client, '/v4/nts/matrix?plot.0=a.2.0',
+ expected_code=HTTP_BAD_REQUEST)
+ self.assertIn("malformed", reply.data)
+
+ reply = check_code(client, '/v4/nts/matrix?plot.0=999.0.0',
+ expected_code=HTTP_NOT_FOUND)
+ self.assertIn("Invalid machine", reply.data)
+ reply = check_code(client, '/v4/nts/matrix?plot.0=1.999.0',
+ expected_code=HTTP_NOT_FOUND)
+ self.assertIn("Invalid test", reply.data)
+ reply = check_code(client, '/v4/nts/matrix?plot.0=1.1.999',
+ expected_code=HTTP_NOT_FOUND)
+ self.assertIn("Invalid field", reply.data)
+
+ def test_matrix_view(self):
+ """Does the page load with the data as expected.
+ """
+ client = self.client
+ reply = check_code(client, '/v4/nts/matrix?plot.0=2.6.3')
+ # Make sure the data is in the page.
+ self.assertIn("test6", reply.data)
+ self.assertIn("1.0000", reply.data)
+ self.assertIn("1.2000", reply.data)
+
+ reply = check_code(client, '/v4/nts/matrix?plot.0=2.6.3&limit=1')
+
+
+if __name__ == '__main__':
+ unittest.main(argv=[sys.argv[0], ])
More information about the llvm-commits
mailing list