[LNT] r318219 - Optimize FieldChange lookup

[Chris Matthews via llvm-commits]

Author: cmatthews
Date: Tue Nov 14 15:30:17 2017
New Revision: 318219

URL: http://llvm.org/viewvc/llvm-project?rev=318219&view=rev
Log:
Optimize FieldChange lookup

One thing that makes change ingestion slow is the field change
deduplication query, which we make many times.  This should hopefully
get us off of the full table search, and make ingestion times a little
shorter.

Added:
    lnt/trunk/lnt/server/db/migrations/upgrade_15_to_16.py
Modified:
    lnt/trunk/lnt/server/db/testsuitedb.py

Added: lnt/trunk/lnt/server/db/migrations/upgrade_15_to_16.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_15_to_16.py?rev=318219&view=auto
==============================================================================
--- lnt/trunk/lnt/server/db/migrations/upgrade_15_to_16.py (added)
+++ lnt/trunk/lnt/server/db/migrations/upgrade_15_to_16.py Tue Nov 14 15:30:17 2017
@@ -0,0 +1,31 @@
+"""This upgrade adds a index on the fieldchange start order, to optimize the deduplication query that is
+pretty slow.
+"""
+
+import sqlalchemy
+from sqlalchemy import Index, select, all_
+from lnt.server.db.migrations.util import introspect_table
+from logging import getLogger
+
+log = getLogger(__name__)
+
+
+def _mk_index_on(engine, ts_name):
+    fc_table = introspect_table(engine, "{}_FieldChangeV2".format(ts_name))
+
+    fast_fc_lookup = Index('idx_fast_fieldchange_lookup', fc_table.c.StartOrderID)
+    try:
+        fast_fc_lookup.create(engine)
+    except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.ProgrammingError) as e:
+        log.warning("Skipping index creation on {}, because of {}".format(fc_table.name, e.message))
+
+
+def upgrade(engine):
+    """Add an index to FieldChangeV2 for each fo the test-suites.
+    """
+    with engine.begin() as trans:
+        test_suite = introspect_table(engine, 'TestSuite')
+
+        db_keys = trans.execute(select([test_suite]))
+        for suite in db_keys:
+            _mk_index_on(trans, suite[2])

Modified: lnt/trunk/lnt/server/db/testsuitedb.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/testsuitedb.py?rev=318219&r1=318218&r2=318219&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/testsuitedb.py (original)
+++ lnt/trunk/lnt/server/db/testsuitedb.py Tue Nov 14 15:30:17 2017
@@ -589,7 +589,7 @@ class TestSuiteDB(object):
             old_value = Column("OldValue", Float)
             new_value = Column("NewValue", Float)
             start_order_id = Column("StartOrderID", Integer,
-                                    ForeignKey(Order.id))
+                                    ForeignKey(Order.id), index=True)
             end_order_id = Column("EndOrderID", Integer, ForeignKey(Order.id))
             test_id = Column("TestID", Integer, ForeignKey(Test.id))
             machine_id = Column("MachineID", Integer, ForeignKey(Machine.id))