[LNT] r240568 - Adapt daily report to handle runs missing on some days.
Kristof Beyls
kristof.beyls at arm.com
Wed Jun 24 12:02:33 PDT 2015
Author: kbeyls
Date: Wed Jun 24 14:02:33 2015
New Revision: 240568
URL: http://llvm.org/viewvc/llvm-project?rev=240568&view=rev
Log:
Adapt daily report to handle runs missing on some days.
When a machine on a particular day doesn't produce a result, the daily
report page won't report regressions/improvements for the day after.
This patch fixes that, so that when a machine doesn't produce results
for a particular day, for what-ever reason, regressions and improvements
are still analyzed and reported.
Reasons for a machine not producing results every day could be the
machine having a hick-up for a day; or simply because that machine
is scheduled to not run a large benchmark every day, but e.g. only
every week.
Modified:
lnt/trunk/lnt/server/reporting/analysis.py
lnt/trunk/lnt/server/reporting/dailyreport.py
lnt/trunk/lnt/server/ui/templates/reporting/daily_report.html
lnt/trunk/tests/server/ui/Inputs/V4Pages_extra_records.sql
lnt/trunk/tests/server/ui/V4Pages.py
Modified: lnt/trunk/lnt/server/reporting/analysis.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/analysis.py?rev=240568&r1=240567&r2=240568&view=diff
==============================================================================
--- lnt/trunk/lnt/server/reporting/analysis.py (original)
+++ lnt/trunk/lnt/server/reporting/analysis.py Wed Jun 24 14:02:33 2015
@@ -254,22 +254,22 @@ class RunInfo(object):
compare_to = []
return self.get_comparison_result([run], compare_to, test_id, field)
+ def get_samples(self, runs, test_id, field):
+ all_samples = []
+ for run in runs:
+ samples = self.sample_map.get((run.id, test_id))
+ if samples is not None:
+ all_samples.extend(samples)
+ return all_samples
+
def get_comparison_result(self, runs, compare_runs, test_id, field):
# Get the field which indicates the requested field's status.
status_field = field.status_field
# Load the sample data for the current and previous runs and the
# comparison window.
- run_samples = []
- prev_samples = []
- for run in runs:
- samples = self.sample_map.get((run.id, test_id))
- if samples is not None:
- run_samples.extend(samples)
- for run in compare_runs:
- samples = self.sample_map.get((run.id, test_id))
- if samples is not None:
- prev_samples.extend(samples)
+ run_samples = self.get_samples(runs, test_id, field)
+ prev_samples = self.get_samples(compare_runs, test_id, field)
# Determine whether this (test,pset) passed or failed in the current and
# previous runs.
Modified: lnt/trunk/lnt/server/reporting/dailyreport.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/dailyreport.py?rev=240568&r1=240567&r2=240568&view=diff
==============================================================================
--- lnt/trunk/lnt/server/reporting/dailyreport.py (original)
+++ lnt/trunk/lnt/server/reporting/dailyreport.py Wed Jun 24 14:02:33 2015
@@ -275,11 +275,26 @@ class DailyReport(object):
visible_results = []
for machine in self.reporting_machines:
# Get the most recent comparison result.
- day_runs = machine_runs.get((machine.id, 0), ())
- prev_runs = self.machine_past_runs.get((machine.id, 1), ())
+ # Record which days have samples, so that we'll compare
+ # also consecutive runs that are further than a day
+ # apart if no runs happened in between.
+ day_has_samples = []
+ for i in range(0, self.num_prior_days_to_include):
+ runs = self.machine_past_runs.get((machine.id, i), ())
+ samples = sri.get_samples(runs, test.id, field)
+ day_has_samples.append(len(samples) > 0)
- prev_day_run = machine_runs.get((machine.id, 1), ())
+ def find_most_recent_run_with_samples(day_nr):
+ for i in range(day_nr+1,
+ self.num_prior_days_to_include):
+ if day_has_samples[i]:
+ return i
+ return day_nr+1
+ prev_day_index = find_most_recent_run_with_samples(0)
+ day_runs = machine_runs.get((machine.id, 0), ())
+ prev_runs = self.machine_past_runs.get(
+ (machine.id, prev_day_index), ())
cr = sri.get_comparison_result(
day_runs, prev_runs, test.id, field)
@@ -291,10 +306,14 @@ class DailyReport(object):
day_results = DayResults()
day_results.append(DayResult(cr))
for i in range(1, self.num_prior_days_to_include):
- day_runs = prev_day_run
- prev_day_run = machine_runs.get((machine.id, i+1), ())
+ day_runs = machine_runs.get((machine.id, i), ())
+ if len(day_runs) == 0:
+ day_results.append(None)
+ continue
+
+ prev_day_index = find_most_recent_run_with_samples(i)
prev_runs = self.machine_past_runs.get(
- (machine.id, i+1), ())
+ (machine.id, prev_day_index), ())
cr = sri.get_comparison_result(day_runs, prev_runs,
test.id, field)
day_results.append(DayResult(cr))
Modified: lnt/trunk/lnt/server/ui/templates/reporting/daily_report.html
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/reporting/daily_report.html?rev=240568&r1=240567&r2=240568&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/templates/reporting/daily_report.html (original)
+++ lnt/trunk/lnt/server/ui/templates/reporting/daily_report.html Wed Jun 24 14:02:33 2015
@@ -178,11 +178,13 @@
{% endfor %}
<polyline points="
{% for dr in day_results %}
- {% set cr = dr.cr %}
- {% set day_nr = loop.index %}
- {% if cr.current is not none %}
- {{ spark_x_coord(day_nr) }}
- {{ spark_y_coord(day_nr, cr.current) }}
+ {% if dr is not none %}
+ {% set cr = dr.cr %}
+ {% set day_nr = loop.index %}
+ {% if cr.current is not none %}
+ {{ spark_x_coord(day_nr) }}
+ {{ spark_y_coord(day_nr, cr.current) }}
+ {% endif %}
{% endif %}
{% endfor %}
" fill="none" stroke="red" stroke-width="1"/>
@@ -206,11 +208,20 @@
<td style="{{ styles.td }}"> </td>
<td class="machine-name" style="{{ styles.td }}"><a href="{{
ts_url}}/graph?plot.0={{machine.id}}.{{test.id}}.{{
- field.index}}&highlight_run={{key_run.id}}">{{
+ field.index}}&highlight_run={{key_run.id}}">{{
machine.name}}</a></td>
- {{ get_initial_cell_value(day_results[-1]) }}
-{% for day_result in day_results[:-1]|reverse %}
+{% set first_result_shown = false %}
+{% for day_result in day_results|reverse %}
+{% if day_result is none %}
+ <td style="{{ styles.td }}">-</td>
+{% else %}
+{% if first_result_shown %}
{{ get_cell_value(day_result) }}
+{% else %}
+{% set first_result_shown = true %}
+ {{ get_initial_cell_value(day_result) }}
+{% endif %}
+{% endif %}
{% endfor %}
<td>{{ spark_plot(day_results) }}</td>
</tr>
Modified: lnt/trunk/tests/server/ui/Inputs/V4Pages_extra_records.sql
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/ui/Inputs/V4Pages_extra_records.sql?rev=240568&r1=240567&r2=240568&view=diff
==============================================================================
--- lnt/trunk/tests/server/ui/Inputs/V4Pages_extra_records.sql (original)
+++ lnt/trunk/tests/server/ui/Inputs/V4Pages_extra_records.sql Wed Jun 24 14:02:33 2015
@@ -11,4 +11,16 @@ INSERT INTO "NT_Machine" VALUES(3,'machi
INSERT INTO "NT_Order" VALUES(6,5,NULL,'152291');
INSERT INTO "NT_Run" VALUES(4,3,6,'run4.json','2012-04-11 16:28:24.000000','2012-04-11 16:28:59.000000',NULL,'[]');
INSERT INTO "NT_Sample" VALUES(4,4,1,NULL,NULL,0.001,0.0001,NULL);
+-- check that a regression on consecutive runs more than 1 day apart can be detected:
+INSERT INTO "NT_Test" VALUES(88,'test1');
+INSERT INTO "NT_Test" VALUES(89,'test2');
+INSERT INTO "NT_Order" VALUES(7,NULL,8,'152292');
+INSERT INTO "NT_Run" VALUES(5,2,7,'run5.json','2012-05-01 16:28:23.000000','2012-05-01 16:28:58.000000',NULL,'[]');
+INSERT INTO "NT_Sample" VALUES(5,5,88,0,0,0.001,1.0,NULL); -- passing result
+INSERT INTO "NT_Sample" VALUES(6,5,89,0,1,0.001,1.0,NULL); -- failing result
+INSERT INTO "NT_Order" VALUES(8,7,NULL,'152293');
+INSERT INTO "NT_Run" VALUES(6,2,8,'run6.json','2012-05-03 16:28:24.000000','2012-05-03 16:28:59.000000',NULL,'[]');
+INSERT INTO "NT_Sample" VALUES(7,6,88,0,0,0.001,10.0,NULL); -- passing result 10x slower
+INSERT INTO "NT_Sample" VALUES(8,5,89,0,0,0.001,1.0,NULL); -- passing result
+
COMMIT;
Modified: lnt/trunk/tests/server/ui/V4Pages.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/ui/V4Pages.py?rev=240568&r1=240567&r2=240568&view=diff
==============================================================================
--- lnt/trunk/tests/server/ui/V4Pages.py (original)
+++ lnt/trunk/tests/server/ui/V4Pages.py Wed Jun 24 14:02:33 2015
@@ -1,5 +1,4 @@
-# Perform basic sanity checking of the V4 UI pages. Currently this really only
-# checks that we don't crash on any of them.
+# Perform basic sanity checking of the V4 UI pages.
#
# create temporary instance
# Cleanup temporary directory in case one remained from a previous run - also see PR9904.
@@ -52,6 +51,21 @@ def get_xml_tree(html_string):
raise
return tree
+
+def find_table_with_heading(tree, table_heading):
+ table_parent_elements = tree.findall(".//table/..")
+ found_header = False
+ for parent in table_parent_elements:
+ for child in parent.findall('*'):
+ if found_header:
+ if child.tag == "table":
+ return child
+ elif (child.tag.startswith('h') and
+ child.text == table_heading):
+ found_header = True
+ return None
+
+
def check_nr_machines_reported(client, url, expected_nr_machines):
resp = check_code(client, url)
html = resp.data
@@ -60,24 +74,36 @@ def check_nr_machines_reported(client, u
# do this by looking for the title containing "Reported Machine Order"
# and assuming that the first <table> at the same level after it is the
# one we're looking for.
- reported_machine_order_table = None
- table_parent_elements = tree.findall(".//table/..")
- found_header = False
- for parent in table_parent_elements:
- for child in parent.findall('*'):
- if found_header:
- if child.tag == "table":
- reported_machine_order_table = child
- found_header = False
- elif child.tag.startswith('h') and \
- child.text == 'Reported Machine Order':
- found_header = True
+ reported_machine_order_table = \
+ find_table_with_heading(tree, 'Reported Machine Order')
if reported_machine_order_table is None:
nr_machines = 0
else:
nr_machines = len(reported_machine_order_table.findall("./tr"))
assert expected_nr_machines == nr_machines
+
+def convert_html_to_text(element):
+ return ("".join(element.itertext()))
+
+
+def check_body_result_table(client, url, fieldname,
+ expected_table_body_content):
+ resp = check_code(client, url)
+ html = resp.data
+ tree = get_xml_tree(html)
+ table_header = "Result Table (%s)" % fieldname
+ table = find_table_with_heading(tree, table_header)
+ assert table is not None, \
+ "Couldn't find table with header '%s'" % table_header
+ body_content = [[convert_html_to_text(cell).strip()
+ for cell in row.findall("./td")]
+ for row in table.findall("./tr")]
+ assert expected_table_body_content == body_content, \
+ "Expected table content %s, found %s" % \
+ (expected_table_body_content, body_content)
+
+
def main():
_,instance_path = sys.argv
@@ -154,6 +180,15 @@ def main():
# Don't crash on an invalid regular expression:
check_nr_machines_reported(client, '/v4/nts/daily_report/2012/4/12?filter-machine-regex=?', 3)
+ # check that a regression seen between 2 consecutive runs that are
+ # more than a day apart gets reported
+ check_body_result_table(client, '/v4/nts/daily_report/2012/5/04',
+ "execution_time",
+ [["test1", ""],
+ ["", "machine2", "1.0000", "-", "900.00%", ""],
+ ["test2", ""],
+ ["", "machine2", "FAIL", "-", "PASS", ""]])
+
# Now check the compile report
# Get the V4 overview page.
check_code(client, '/v4/compile/')
More information about the llvm-commits
mailing list