[LNT] r264720 - [db] Add a search API
James Molloy via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 29 04:08:02 PDT 2016
Author: jamesm
Date: Tue Mar 29 06:08:02 2016
New Revision: 264720
URL: http://llvm.org/viewvc/llvm-project?rev=264720&view=rev
Log:
[db] Add a search API
This is used for finding runs quickly via a javascript dropdown in the UI. Currently the profiling infrastructure has its own search facility - this promotes it to a first class citizen and adds proper tests for it.
This currently has no users.
Added:
lnt/trunk/lnt/server/db/search.py
lnt/trunk/tests/server/db/Inputs/report.json.in
lnt/trunk/tests/server/db/search.py
Added: lnt/trunk/lnt/server/db/search.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/search.py?rev=264720&view=auto
==============================================================================
--- lnt/trunk/lnt/server/db/search.py (added)
+++ lnt/trunk/lnt/server/db/search.py Tue Mar 29 06:08:02 2016
@@ -0,0 +1,69 @@
+import re
+
+def _naive_search_for_run(ts, query, num_results, default_machine, default_order):
+ """
+ This 'naive' search doesn't rely on any indexes so can be used without
+ full-text search enabled. This does make it less clever however.
+
+ It is able to match queries for machine names and order numbers (specifically
+ llvm_project_revision numbers). The revision numbers may be partial and may be
+ preceded by '#' or 'r'. Any other non-integer tokens are considered to be partial
+ matches for a machine name; any machine that contains ALL of the tokens will be
+ searched.
+ """
+
+ order_re = re.compile(r'[r#]?(\d+)')
+ machine_queries = []
+ order_queries = []
+
+ # First, tokenize the query string.
+ for q in query.split(' '):
+ if not q:
+ # Prune zero-length tokens
+ continue
+ m = order_re.match(q)
+ if m:
+ order_queries.append(int(m.group(1)))
+ else:
+ machine_queries.append(q)
+
+ # If we didn't have any machines queries but we do have a default, query that.
+ if not machine_queries and default_machine:
+ machine_queries = [default_machine]
+ elif not machine_queries:
+ # No machines to query: no matches. We can't query all machines, we'd end up
+ # doing a full table scan and that is not scalable.
+ return []
+
+ if default_order:
+ order_queries.append(default_order)
+
+ machines = []
+ for m in ts.query(ts.Machine).all():
+ if all(q in m.name for q in machine_queries):
+ machines.append(m.id)
+ if not machines:
+ return []
+
+ llvm_project_revision_idx = [i
+ for i, f in enumerate(ts.Order.fields)
+ if f.name == 'llvm_project_revision'][0]
+ llvm_project_revision_col = ts.Order.fields[llvm_project_revision_idx].column
+
+ q = ts.query(ts.Run) \
+ .filter(ts.Run.machine_id.in_(machines)) \
+ .filter(ts.Run.order_id == ts.Order.id)
+ if order_queries:
+ oq = '%' + str(order_queries[0]) + '%'
+ q = q.filter(llvm_project_revision_col.like(oq))
+
+ return q.order_by(llvm_project_revision_col.desc()).limit(num_results).all()
+
+
+def search(ts, query,
+ num_results=8, default_machine=None, default_order=None):
+ """
+ """
+
+ return _naive_search_for_run(ts, query,
+ num_results, default_machine, default_order)
Added: lnt/trunk/tests/server/db/Inputs/report.json.in
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/Inputs/report.json.in?rev=264720&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/Inputs/report.json.in (added)
+++ lnt/trunk/tests/server/db/Inputs/report.json.in Tue Mar 29 06:08:02 2016
@@ -0,0 +1,61 @@
+{
+ "Machine": {
+ "Info": {},
+ "Name": "@@MACHINE@@"
+ },
+ "Run": {
+ "End Time": "2016-03-14 14:20:29",
+ "Info": {
+ "__report_version__": "1",
+ "cc1_exec_hash": "c431213c8e77728ee825579745d700e79c3521f2",
+ "cc_alt_src_branch": "llvm/trunk",
+ "cc_alt_src_revision": "154329",
+ "cc_as_version": "unrecognized argument vector: ('-c', '-Wa,-v', '-o', '/dev/null', '-x', 'assembler', '/dev/null')",
+ "cc_build": "DEV",
+ "cc_dumpmachine": "x86_64-apple-darwin11.0.0",
+ "cc_exec_hash": "c431213c8e77728ee825579745d700e79c3521f2",
+ "cc_ld_version": "unrecognized argument vector: ('-Wl,-v', '-o', '/dev/null', '/tmp/tmpzWa737.c')",
+ "cc_name": "clang",
+ "cc_src_branch": "trunk",
+ "cc_src_revision": "154331",
+ "cc_target": "x86_64-apple-darwin11.0.0",
+ "cc_target_assembly": "; ModuleID = '/dev/null'\ntarget datalayout = \"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64\"\ntarget triple = \"x86_64-apple-darwin11.0.0\"",
+ "cc_version": "clang version 3.1 (trunk 154331) (llvm/trunk 154329)\nTarget: x86_64-apple-darwin11.3.0\nThread model: posix\nInstalledDir: /home/foo/bin\n\n \"/Users/jammol01/Code/lnt/tests/SharedInputs/FakeCompilers/clang-r154331\" \"-cc1\" \"-E\" ... more boring stuff here ...",
+ "cc_version_number": "3.1",
+ "inferred_run_order": "@@ORDER@@",
+ "run_order": "@@ORDER@@",
+ "tag": "nts"
+ },
+ "Start Time": "2016-03-14 14:20:28"
+ },
+ "Tests": [
+ {
+ "Data": [
+ 1.4
+ ],
+ "Info": {},
+ "Name": "nts.foo.exec"
+ },
+ {
+ "Data": [
+ 1.3
+ ],
+ "Info": {},
+ "Name": "nts.foo.compile"
+ },
+ {
+ "Data": [
+ 1.5
+ ],
+ "Info": {},
+ "Name": "nts.foo.score"
+ },
+ {
+ "Data": [
+ "xyz"
+ ],
+ "Info": {},
+ "Name": "nts.foo.hash"
+ }
+ ]
+}
Added: lnt/trunk/tests/server/db/search.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/search.py?rev=264720&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/search.py (added)
+++ lnt/trunk/tests/server/db/search.py Tue Mar 29 06:08:02 2016
@@ -0,0 +1,102 @@
+# RUN: python %s
+
+import unittest, tempfile, shutil, logging, sys, os, contextlib
+import lnt.util.ImportData
+import lnt.server.instance
+from lnt.server.db.search import search
+
+#logging.basicConfig(level=logging.DEBUG)
+
+class SearchTest(unittest.TestCase):
+ def setUp(self):
+
+ master_path = 'Inputs/lnt_v0.4.0_filled_instance'
+ slave_path = os.path.join(tempfile.mkdtemp(), 'lnt')
+ shutil.copytree(master_path, slave_path)
+ instance = lnt.server.instance.Instance.frompath(slave_path)
+ config = instance.config
+
+ imported_runs = [('machine1', '5624'),
+ ('machine1', '5625'),
+ ('machine2', '6512'),
+ ('machine2', '7623'),
+ ('supermachine', '1324'),
+ ('supermachine', '7623')]
+
+ # Get the database.
+ self.db = config.get_database('default', echo=False)
+ # Load the database.
+ success = True
+ for r in imported_runs:
+ with tempfile.NamedTemporaryFile() as f:
+ data = open('Inputs/report.json.in') \
+ .read() \
+ .replace('@@MACHINE@@', r[0]) \
+ .replace('@@ORDER@@', r[1])
+ open(f.name, 'w').write(data)
+
+ result = lnt.util.ImportData.import_and_report(
+ None, 'default', self.db, f.name,
+ '<auto>', True, False,
+ True, True)
+
+ success &= result.get('success', False)
+
+ assert success
+
+ def _mangleResults(self, rs):
+ return [(r.machine.name, str(r.order.llvm_project_revision))
+ for r in rs]
+
+ def test_specific(self):
+ ts = self.db.testsuite.get('nts')
+
+ results = self._mangleResults(search(ts, 'machine1 #5625'))
+ self.assertEqual(results, [
+ ('machine1', '5625')
+ ])
+
+ results = self._mangleResults(search(ts, 'machine1 #5624'))
+ self.assertEqual(results, [
+ ('machine1', '5624')
+ ])
+
+ def test_multiple_orders(self):
+ ts = self.db.testsuite.get('nts')
+
+ results = self._mangleResults(search(ts, 'machine1 #56'))
+ self.assertEqual(results, [
+ ('machine1', '5625'), ('machine1', '5624')
+ ])
+
+ def test_nohash(self):
+ ts = self.db.testsuite.get('nts')
+
+ results = self._mangleResults(search(ts, 'machine1 r56'))
+ self.assertEqual(results, [
+ ('machine1', '5625'), ('machine1', '5624')
+ ])
+
+ results = self._mangleResults(search(ts, 'machine1 56'))
+ self.assertEqual(results, [
+ ('machine1', '5625'), ('machine1', '5624')
+ ])
+
+ def test_default_order(self):
+ ts = self.db.testsuite.get('nts')
+
+ results = self._mangleResults(search(ts, 'machi ne2'))
+ self.assertEqual(results, [
+ ('machine2', '7623'), ('machine2', '6512')
+ ])
+
+ def test_default_machine(self):
+ ts = self.db.testsuite.get('nts')
+
+ results = self._mangleResults(search(ts, '65', default_machine='machine2'))
+ self.assertEqual(results, [
+ ('machine2', '6512')
+ ])
+
+if __name__ == '__main__':
+ unittest.main(argv=[sys.argv[0], ])
More information about the llvm-commits
mailing list