[LNT] r312061 - tests: Integrate html-tidy checks into ui page tests
Matthias Braun via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 29 15:59:29 PDT 2017
Author: matze
Date: Tue Aug 29 15:59:28 2017
New Revision: 312061
URL: http://llvm.org/viewvc/llvm-project?rev=312061&view=rev
Log:
tests: Integrate html-tidy checks into ui page tests
This adds an option to check generated html pages for errors with tidy-html5
Modified:
lnt/trunk/tests/lit.cfg
lnt/trunk/tests/server/ui/V4Pages.py
lnt/trunk/tests/server/ui/test_matrix_page.py
lnt/trunk/tests/server/ui/test_system_info.py
Modified: lnt/trunk/tests/lit.cfg
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/lit.cfg?rev=312061&r1=312060&r2=312061&view=diff
==============================================================================
--- lnt/trunk/tests/lit.cfg (original)
+++ lnt/trunk/tests/lit.cfg Tue Aug 29 15:59:28 2017
@@ -56,6 +56,12 @@ if lit_config.params.get('postgres', Non
if lit_config.params.get('mysql', None):
config.available_features.add('mysql')
+# Enable tidylib testing. This requires pytidylib and tidy-html5.
+if lit_config.params.get('tidylib', None):
+ config.substitutions.append(('%{tidylib}', '--use-tidylib'))
+else:
+ config.substitutions.append(('%{tidylib}', ''))
+
config.available_features.add(platform.system())
# Enable coverage.py reporting, assuming the coverage module has been installed
Modified: lnt/trunk/tests/server/ui/V4Pages.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/ui/V4Pages.py?rev=312061&r1=312060&r2=312061&view=diff
==============================================================================
--- lnt/trunk/tests/server/ui/V4Pages.py (original)
+++ lnt/trunk/tests/server/ui/V4Pages.py Tue Aug 29 15:59:28 2017
@@ -8,7 +8,7 @@
# RUN: %s %{shared_inputs}/SmallInstance %t.instance \
# RUN: %S/Inputs/V4Pages_extra_records.sql
#
-# RUN: python %s %t.instance
+# RUN: python %s %t.instance %{tidylib}
import logging
import re
@@ -19,6 +19,38 @@ from flask import session
import lnt.server.db.migrate
import lnt.server.ui.app
import json
+import HTMLParser
+
+# We can validate html if pytidylib is available and tidy-html5 is installed.
+# The user can indicate this by passing --use-tidylib to the script (triggered
+# by `lit -Dtidylib=1`)
+if '--use-tidylib' in sys.argv:
+ import tidylib
+ def validate_html(text):
+ document, errors = tidylib.tidy_document(text)
+ had_error = False
+ ignore = [
+ "Warning: trimming empty",
+ "Warning: inserting implicit",
+ ]
+ for e in errors.splitlines():
+ ignore_line = False
+ for i in ignore:
+ if i in e:
+ ignore_line = True
+ break
+ if ignore_line:
+ continue
+ sys.stderr.write(e + '\n')
+ had_error = True
+ if had_error:
+ with open('/tmp/lntpage.html', 'w') as out:
+ out.write(text)
+ sys.stderr.write("Note: html saved in /tmp/lntpage.html\n")
+ assert not had_error
+else:
+ def validate_html(text):
+ pass
logging.basicConfig(level=logging.DEBUG)
@@ -37,6 +69,12 @@ def check_code(client, url, expected_cod
return resp
+def check_html(client, url, expected_code=HTTP_OK, data_to_send=None):
+ resp = check_code(client, url, expected_code, data_to_send)
+ validate_html(resp.data)
+ return resp
+
+
def check_json(client, url, expected_code=HTTP_OK, data_to_send=None):
"""Call a flask url, make sure the return code is good,
and grab reply data from the json payload."""
@@ -227,7 +265,7 @@ def extract_background_colors(sparkline_
def main():
- _, instance_path = sys.argv
+ instance_path = sys.argv[1]
# Create the application instance.
app = lnt.server.ui.app.App.create_standalone(instance_path)
@@ -240,23 +278,23 @@ def main():
client = app.test_client()
# Fetch the index page.
- check_code(client, '/')
+ check_html(client, '/')
# Get the V4 overview page.
- check_code(client, '/v4/nts/')
+ check_html(client, '/v4/nts/')
# Get a machine overview page.
- check_code(client, '/v4/nts/machine/1')
+ check_html(client, '/v4/nts/machine/1')
# Check invalid machine gives error.
check_code(client, '/v4/nts/machine/9999', expected_code=HTTP_NOT_FOUND)
# Get a machine overview page in JSON format.
- check_code(client, '/v4/nts/machine/1?json=true')
+ check_json(client, '/v4/nts/machine/1?json=true')
# Get the order summary page.
- check_code(client, '/v4/nts/all_orders')
+ check_html(client, '/v4/nts/all_orders')
# Get an order page.
- check_code(client, '/v4/nts/order/3')
+ check_html(client, '/v4/nts/order/3')
# Check invalid order gives error.
check_code(client, '/v4/nts/order/9999', expected_code=HTTP_NOT_FOUND)
@@ -296,9 +334,9 @@ def main():
session.get('baseline-default-nts') == 1
# Get a run result page (and associated views).
- check_code(client, '/v4/nts/1')
- check_code(client, '/v4/nts/1?json=true')
- check_code(client, '/v4/nts/1/report')
+ check_html(client, '/v4/nts/1')
+ check_json(client, '/v4/nts/1?json=true')
+ check_html(client, '/v4/nts/1/report')
check_code(client, '/v4/nts/1/text_report')
# Check invalid run numbers give errors.
check_code(client, '/v4/nts/9999',
@@ -325,32 +363,32 @@ def main():
'0:00:35', 'some-builder #987'])
# Get the new graph page.
- check_code(client, '/v4/nts/graph?plot.0=1.3.2')
+ check_html(client, '/v4/nts/graph?plot.0=1.3.2')
# Don't crash when requesting non-existing data
- check_code(client, '/v4/nts/graph?plot.9999=1.3.2')
+ check_html(client, '/v4/nts/graph?plot.9999=1.3.2')
check_code(client, '/v4/nts/graph?plot.0=9999.3.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?plot.0=1.3.9999',
expected_code=HTTP_NOT_FOUND)
- check_code(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
+ check_json(client, '/v4/nts/graph?plot.9999=1.3.2&json=True')
# Get the mean graph page.
- check_code(client, '/v4/nts/graph?mean=1.2')
+ check_html(client, '/v4/nts/graph?mean=1.2')
# Don't crash when requesting non-existing data
check_code(client, '/v4/nts/graph?mean=9999.2',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/graph?mean=1.9999',
expected_code=HTTP_NOT_FOUND)
# Check baselines work.
- check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')
+ check_html(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')
# Check some variations of the daily report work.
- check_code(client, '/v4/nts/daily_report/2012/4/12')
- check_code(client, '/v4/nts/daily_report/2012/4/11')
- check_code(client, '/v4/nts/daily_report/2012/4/13')
- check_code(client, '/v4/nts/daily_report/2012/4/10')
- check_code(client, '/v4/nts/daily_report/2012/4/14')
+ check_html(client, '/v4/nts/daily_report/2012/4/12')
+ check_html(client, '/v4/nts/daily_report/2012/4/11')
+ check_html(client, '/v4/nts/daily_report/2012/4/13')
+ check_html(client, '/v4/nts/daily_report/2012/4/10')
+ check_html(client, '/v4/nts/daily_report/2012/4/14')
check_redirect(client, '/v4/nts/daily_report',
'/v4/nts/daily_report/\d+/\d+/\d+$')
check_redirect(client, '/v4/nts/daily_report?num_days=7',
@@ -360,7 +398,7 @@ def main():
check_redirect(client, '/v4/nts/daily_report?day=15',
'/v4/nts/daily_report/\d+/\d+/\d+$')
# Don't crash when requesting non-existing data
- check_code(client, '/v4/nts/daily_report/1999/4/12')
+ check_html(client, '/v4/nts/daily_report/1999/4/12')
check_code(client, '/v4/nts/daily_report/-1/4/12',
expected_code=HTTP_NOT_FOUND)
check_code(client, '/v4/nts/daily_report/2012/13/12',
@@ -464,19 +502,19 @@ def main():
check_redirect(client, '/db_default/submitRun',
'/db_default/v4/nts/submitRun')
- check_code(client, '/db_default/v4/nts/submitRun')
+ check_html(client, '/db_default/v4/nts/submitRun')
- check_code(client, '/v4/nts/global_status')
+ check_html(client, '/v4/nts/global_status')
- check_code(client, '/v4/nts/recent_activity')
+ check_html(client, '/v4/nts/recent_activity')
# Now check the compile report
# Get the V4 overview page.
- check_code(client, '/v4/compile/')
+ check_html(client, '/v4/compile/')
# Get a machine overview page.
- check_code(client, '/v4/compile/machine/1')
- check_code(client, '/v4/compile/machine/2')
+ check_html(client, '/v4/compile/machine/1')
+ check_html(client, '/v4/compile/machine/2')
check_code(client, '/v4/compile/machine/2/latest', expected_code=HTTP_REDIRECT)
# Don't crash when requesting non-existing data
check_code(client, '/v4/compile/machine/9999',
@@ -491,59 +529,61 @@ def main():
assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9"
# Get the order summary page.
- check_code(client, '/v4/compile/all_orders')
+ check_html(client, '/v4/compile/all_orders')
# Get an order page.
- check_code(client, '/v4/compile/order/3')
+ check_html(client, '/v4/compile/order/3')
# Get a run result page (and associated views).
- check_code(client, '/v4/compile/1')
- check_code(client, '/v4/compile/2')
- check_code(client, '/v4/compile/3')
- check_code(client, '/v4/compile/4')
+ check_html(client, '/v4/compile/1')
+ check_html(client, '/v4/compile/2')
+ check_html(client, '/v4/compile/3')
+ check_html(client, '/v4/compile/4')
check_code(client, '/v4/compile/9999', expected_code=HTTP_NOT_FOUND)
- check_code(client, '/v4/compile/1/report')
+ check_html(client, '/v4/compile/1/report')
check_code(client, '/v4/compile/1/text_report')
# Get the new graph page.
- check_code(client, '/v4/compile/graph?plot.3=2.3.9')
+ check_html(client, '/v4/compile/graph?plot.3=2.3.9')
# Get the mean graph page.
- check_code(client, 'v4/compile/graph?mean=2.9')
+ check_html(client, 'v4/compile/graph?mean=2.9')
# Check some variations of the daily report work.
- check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
- check_code(client, '/v4/compile/daily_report/2014/6/4')
+ check_html(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
+ check_html(client, '/v4/compile/daily_report/2014/6/4')
check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
- check_code(client, '/v4/nts/regressions/')
- check_code(client, '/v4/nts/regressions/?machine_filter=machine2')
- check_code(client, '/v4/nts/regressions/?machine_filter=machine0')
+ check_html(client, '/v4/nts/regressions/')
+ check_html(client, '/v4/nts/regressions/?machine_filter=machine2')
+ check_html(client, '/v4/nts/regressions/?machine_filter=machine0')
- check_code(client, '/v4/nts/regressions/1')
+ check_html(client, '/v4/nts/regressions/1')
check_json(client, '/v4/nts/regressions/1?json=True')
# Make sure the new option does not break anything
- check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update')
+ check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update')
check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update')
- check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
+ check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
app.testing = False
- error_page = check_code(client, '/explode', expected_code=500)
+ error_page = check_html(client, '/explode', expected_code=500)
assert "integer division or modulo by zero" in error_page.data
- error_page = check_code(client, '/gone', expected_code=404)
+ error_page = check_html(client, '/gone', expected_code=404)
assert "test" in error_page.data
- check_code(client, '/db_default/summary_report')
+ check_html(client, '/db_default/summary_report')
- check_code(client, '/rules')
- check_code(client, '/log')
- check_code(client, '/__health')
- check_code(client, '/ping')
+ check_html(client, '/rules')
+ check_html(client, '/log')
+ resp = check_code(client, '/__health')
+ assert resp.data == "Ok"
+ resp = check_code(client, '/ping')
+ assert resp.data == "pong"
if __name__ == '__main__':
main()
Modified: lnt/trunk/tests/server/ui/test_matrix_page.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/ui/test_matrix_page.py?rev=312061&r1=312060&r2=312061&view=diff
==============================================================================
--- lnt/trunk/tests/server/ui/test_matrix_page.py (original)
+++ lnt/trunk/tests/server/ui/test_matrix_page.py Tue Aug 29 15:59:28 2017
@@ -5,7 +5,7 @@
# RUN: %s %{shared_inputs}/SmallInstance \
# RUN: %t.instance %S/Inputs/V4Pages_extra_records.sql
#
-# RUN: python %s %t.instance
+# RUN: python %s %t.instance %{tidylib}
import unittest
import logging
@@ -14,7 +14,7 @@ import sys
import lnt.server.db.migrate
import lnt.server.ui.app
-from V4Pages import check_json, check_code
+from V4Pages import check_json, check_code, check_html
from V4Pages import HTTP_REDIRECT, HTTP_OK, HTTP_BAD_REQUEST, HTTP_NOT_FOUND
logging.basicConfig(level=logging.DEBUG)
@@ -23,7 +23,7 @@ class MatrixViewTester(unittest.TestCase
def setUp(self):
"""Bind to the LNT test instance."""
- _, instance_path = sys.argv
+ instance_path = sys.argv[1]
app = lnt.server.ui.app.App.create_standalone(instance_path)
app.testing = True
app.config['WTF_CSRF_ENABLED'] = False
@@ -59,7 +59,7 @@ class MatrixViewTester(unittest.TestCase
"""Does the page load with the data as expected.
"""
client = self.client
- reply = check_code(client, '/v4/nts/matrix?plot.0=2.6.3')
+ reply = check_html(client, '/v4/nts/matrix?plot.0=2.6.3')
# Set a baseline and run again.
form_data = dict(name="foo_baseline",
description="foo_description",
@@ -69,13 +69,13 @@ class MatrixViewTester(unittest.TestCase
check_code(client, '/v4/nts/set_baseline/1',
expected_code=HTTP_REDIRECT)
- reply = check_code(client, '/v4/nts/matrix?plot.0=2.6.3')
+ reply = check_html(client, '/v4/nts/matrix?plot.0=2.6.3')
# Make sure the data is in the page.
self.assertIn("test6", reply.data)
self.assertIn("1.0000", reply.data)
self.assertIn("1.2000", reply.data)
- reply = check_code(client, '/v4/nts/matrix?plot.0=2.6.3&limit=1')
+ reply = check_html(client, '/v4/nts/matrix?plot.0=2.6.3&limit=1')
if __name__ == '__main__':
Modified: lnt/trunk/tests/server/ui/test_system_info.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/ui/test_system_info.py?rev=312061&r1=312060&r2=312061&view=diff
==============================================================================
--- lnt/trunk/tests/server/ui/test_system_info.py (original)
+++ lnt/trunk/tests/server/ui/test_system_info.py Tue Aug 29 15:59:28 2017
@@ -4,7 +4,7 @@
# RUN: %s %{shared_inputs}/SmallInstance \
# RUN: %t.instance %S/Inputs/V4Pages_extra_records.sql
#
-# RUN: python %s %t.instance
+# RUN: python %s %t.instance %{tidylib}
import unittest
import logging
@@ -13,7 +13,7 @@ import sys
import lnt.server.db.migrate
import lnt.server.ui.app
-from V4Pages import check_code
+from V4Pages import check_html
from V4Pages import HTTP_OK
logging.basicConfig(level=logging.DEBUG)
@@ -23,7 +23,7 @@ class SystemInfoTester(unittest.TestCase
def setUp(self):
"""Bind to the LNT test instance."""
- _, instance_path = sys.argv
+ instance_path = sys.argv[1]
app = lnt.server.ui.app.App.create_standalone(instance_path)
app.testing = True
app.config['WTF_CSRF_ENABLED'] = False
@@ -32,7 +32,7 @@ class SystemInfoTester(unittest.TestCase
def test_profile_view(self):
"""Does the page load without crashing the server?
"""
- check_code(self.client, '/profile/admin', expected_code=HTTP_OK)
+ check_html(self.client, '/profile/admin', expected_code=HTTP_OK)
if __name__ == '__main__':
More information about the llvm-commits
mailing list