[llvm-commits] [zorg] r147994 - in /zorg/trunk/lnt: lnt/server/db/testsuitedb.py lnt/server/ui/templates/v4_order.html lnt/server/ui/templates/v4_overview.html lnt/server/ui/views.py tests/server/db/CreateV4TestSuiteInstance.py tests/server/db/ImportV4TestSuiteInstance.py
Daniel Dunbar
daniel at zuster.org
Wed Jan 11 17:27:26 PST 2012
Author: ddunbar
Date: Wed Jan 11 19:27:26 2012
New Revision: 147994
URL: http://llvm.org/viewvc/llvm-project?rev=147994&view=rev
Log:
[lnt/v0.4] lnt.server.db: Change Order objects to form a doubly linked list instead of caching their ordinal in the total lexicographic ordering.
- This eliminates one potentially large scalability problem when we couldn't guarantee runs were normally inserted in increasing order.
- <rdar://problem/10678610> [lnt/v0.4] run order ordinal computation is not scalable
Modified:
zorg/trunk/lnt/lnt/server/db/testsuitedb.py
zorg/trunk/lnt/lnt/server/ui/templates/v4_order.html
zorg/trunk/lnt/lnt/server/ui/templates/v4_overview.html
zorg/trunk/lnt/lnt/server/ui/views.py
zorg/trunk/lnt/tests/server/db/CreateV4TestSuiteInstance.py
zorg/trunk/lnt/tests/server/db/ImportV4TestSuiteInstance.py
Modified: zorg/trunk/lnt/lnt/server/db/testsuitedb.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/server/db/testsuitedb.py?rev=147994&r1=147993&r2=147994&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/server/db/testsuitedb.py (original)
+++ zorg/trunk/lnt/lnt/server/db/testsuitedb.py Wed Jan 11 19:27:26 2012
@@ -106,17 +106,16 @@
id = Column("ID", Integer, primary_key=True)
- # The ordinal defines the placement of the Order record within the
- # total ordering. The ordinals are always [0, count(Order)) and are
- # maintained by the insertion code.
- #
- # FIXME: There are many other ways we could deal with this
- # information. Obviously we could always manage it outside the
- # database, but we could also store things like previous and next
- # links in the orders (which would be easier to update, but harder
- # to query, but also supports more complicated ordering schemes).
- ordinal = Column("Ordinal", Integer, nullable=False, unique=True,
- index=True)
+ # Define two common columns which are used to store the previous and
+ # next links for the total ordering amongst run orders.
+ next_order_id = Column("NextOrder", Integer, ForeignKey(
+ "%s.ID" % __tablename__))
+ previous_order_id = Column("PreviousOrder", Integer, ForeignKey(
+ "%s.ID" % __tablename__))
+
+ # FIXME: <sheepish> I would really like to have next_order and
+ # previous_order relation's here, but can't figure out how to
+ # declare them </sheepsih>.
# Dynamically create fields for all of the test suite defined order
# fields.
@@ -129,8 +128,10 @@
class_dict[item.name] = item.column = Column(
item.name, String(256))
- def __init__(self, ordinal, **kwargs):
- self.ordinal = ordinal
+ def __init__(self, previous_order_id = None, next_order_id = None,
+ **kwargs):
+ self.previous_order_id = previous_order_id
+ self.next_order_id = next_order_id
# Initialize fields (defaulting to None, for now).
for item in self.fields:
@@ -140,8 +141,9 @@
fields = dict((item.name, self.get_field(item))
for item in self.fields)
- return '%s_%s(%r, **%r)' % (
- db_key_name, self.__class__.__name__, self.ordinal, fields)
+ return '%s_%s(%r, %r, **%r)' % (
+ db_key_name, self.__class__.__name__,
+ self.previous_order_id, self.next_order_id, fields)
def __cmp__(self, b):
# SA occassionally uses comparison to check model instances
@@ -384,7 +386,7 @@
"""
query = self.query(self.Order)
- order = self.Order(ordinal = None)
+ order = self.Order()
# First, extract all of the specified order fields.
for item in self.order_fields:
@@ -403,44 +405,32 @@
try:
return query.one(),False
except sqlalchemy.orm.exc.NoResultFound:
- # If not, then we need to assign an ordinal to this run.
- #
- # For now, we do this in the simple, slow, and stupid fashion in
- # which we just recompute the total ordering and reassign the
- # ordinals.
- #
- # FIXME: Optimize this for the common case, in which the new ordinal
- # will almost always be very close to the top value, and will
- # require shifting only a few (or no) other order ordinals.
+ # If not, then we need to insert this order into the total ordering
+ # linked list.
+
+ # Add the new order and flush, to assign an ID.
+ self.add(order)
+ self.v4db.session.flush()
# Load all the orders.
orders = list(self.query(self.Order))
- orders.append(order)
# Sort the objects to form the total ordering.
orders.sort()
- # Assign ordinals.
- #
- # We iterate in reverse order to guarantee that we do not create
- # unique conflicts.
- for i in range(len(orders)-1,-1,-1):
- # FIXME: Figure out whether or not SA checks modified status on
- # write or on value change.
- o = orders[i]
- if o.ordinal != i:
- o.ordinal = i
- # We have to flush now in order to assure that SA will not
- # present non-unique values to the database.
- #
- # FIXME: This is really horrible from a performance point of
- # view. If we can't figure out how to get SA to do a batch
- # update then we should write the SQL query to do the update
- # directly.
- self.v4db.session.flush()
+ # Find the order we just added.
+ index = orders.index(order)
- # Finally, add the new order.
- self.add(order)
+ # Insert this order into the linked list which forms the total
+ # ordering.
+ if index > 0:
+ previous_order = orders[index - 1]
+ previous_order.next_order_id = order.id
+ order.previous_order_id = previous_order.id
+ if index + 1 < len(orders):
+ next_order = orders[index + 1]
+ next_order.previous_order_id = order.id
+ order.next_order_id = next_order.id
return order,True
@@ -627,18 +617,30 @@
The direction must be -1 or 1 and specified whether or not the
preceeding or following runs should be returned.
"""
+ assert N > 0, "invalid count"
assert direction in (-1, 1), "invalid direction"
- ordinal = run.order.ordinal + direction
- # FIXME: This probably isn't a great way to limit our search, at least
- # for SQLite which can't answer this quickly.
- last_ordinal = self.query(self.Order).count()
- while 0 <= ordinal <= last_ordinal and N > 0:
- # Find all the runs on this machine for the current ordinal.
+ order = run.order
+ while N:
+ # Update the order in the direction we are searching.
+ if direction == -1:
+ next_id = order.previous_order_id
+ else:
+ next_id = order.next_order_id
+
+ # If we have reached the end of the order chain, we are done.
+ if next_id is None:
+ break
+
+ # Otherwise fetch the run.
+ order = self.query(self.Order).\
+ filter(self.Order.id == next_id).first()
+ assert order is not None
+
+ # Find all the runs on this machine for the current order.
found_any = False
for item in self.query(self.Run).\
- join(self.Order).\
- filter(self.Order.ordinal == ordinal).\
+ filter(self.Run.order == order).\
filter(self.Run.machine == run.machine):
yield item
found_any = True
@@ -648,9 +650,6 @@
if found_any:
N -= 1
- # Update the ordinal we are searching for runs for.
- ordinal += direction
-
def get_previous_runs_on_machine(self, run, N):
return self.get_adjacent_runs_on_machine(run, N, direction = -1)
Modified: zorg/trunk/lnt/lnt/server/ui/templates/v4_order.html
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/server/ui/templates/v4_order.html?rev=147994&r1=147993&r2=147994&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/server/ui/templates/v4_order.html (original)
+++ zorg/trunk/lnt/lnt/server/ui/templates/v4_order.html Wed Jan 11 19:27:26 2012
@@ -21,8 +21,12 @@
<td>{{order.id}}</td>
</tr>
<tr>
- <td>Ordinal</td>
- <td>{{order.ordinal}}</td>
+ <td>Previous Order ID</td>
+ <td>{{order.previous_order_id}}</td>
+ </tr>
+ <tr>
+ <td>Next ID</td>
+ <td>{{order.next_order_id}}</td>
</tr>
{% for field in order.fields %}
<tr>
@@ -33,10 +37,12 @@
</table>
{# Provide links to the previous and next orders. #}
-{% if order.ordinal != 0 %}
-<a href="{{v4_url_for('v4_order', ordinal=order.ordinal - 1)}}">Previous</a>
+{% if order.previous_order_id is not null %}
+<a href="{{v4_url_for('v4_order', id=order.previous_order_id)}}">Previous</a>
+{% endif %}
+{% if order.next_order_id is not null %}
+<a href="{{v4_url_for('v4_order', id=order.next_order_id)}}">Next</a>
{% endif %}
-<a href="{{v4_url_for('v4_order', ordinal=order.ordinal + 1)}}">Next</a>
{# List all submissions which reported for this order. #}
<h3>Submissions</h3>
Modified: zorg/trunk/lnt/lnt/server/ui/templates/v4_overview.html
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/server/ui/templates/v4_overview.html?rev=147994&r1=147993&r2=147994&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/server/ui/templates/v4_overview.html (original)
+++ zorg/trunk/lnt/lnt/server/ui/templates/v4_overview.html Wed Jan 11 19:27:26 2012
@@ -53,7 +53,7 @@
{% for r,run_order in active_submissions %}
{% set m = r.machine %}
<tr>
- <td><a href="{{v4_url_for('v4_order', ordinal=r.order.ordinal)}}">{{
+ <td><a href="{{v4_url_for('v4_order', id=r.order.id)}}">{{
run_order}}</a></td></td>
<td>{{r.start_time}}</td>
<td>{{r.end_time}}</td>
Modified: zorg/trunk/lnt/lnt/server/ui/views.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/server/ui/views.py?rev=147994&r1=147993&r2=147994&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/server/ui/views.py (original)
+++ zorg/trunk/lnt/lnt/server/ui/views.py Wed Jan 11 19:27:26 2012
@@ -725,13 +725,13 @@
options=options, neighboring_runs=neighboring_runs,
text_report=text_report, html_report=html_report)
- at v4_route("/order/<int:ordinal>")
-def v4_order(ordinal):
+ at v4_route("/order/<int:id>")
+def v4_order(id):
# Get the testsuite.
ts = request.get_testsuite()
# Get the order.
- order = ts.query(ts.Order).filter_by(ordinal = ordinal).first()
+ order = ts.query(ts.Order).filter(ts.Order.id == id).first()
if order is None:
abort(404)
@@ -811,7 +811,9 @@
#
# FIXME: Don't join to Order here, aggregate this across all the tests
# we want to load. Actually, we should just make this a single query.
- q = ts.query(field.column, ts.Order.ordinal).\
+ #
+ # FIXME: Don't hard code field name.
+ q = ts.query(field.column, ts.Order.llvm_project_revision).\
join(ts.Run).join(ts.Order).\
filter(ts.Run.machine == run.machine).\
filter(ts.Sample.test == test)
@@ -821,8 +823,8 @@
if field.status_field:
q = q.filter(field.status_field.column == PASS)
- # Aggregate by run order id.
- data = util.multidict((r,v)
+ # Aggregate by revision.
+ data = util.multidict((int(r),v)
for v,r in q).items()
data.sort()
Modified: zorg/trunk/lnt/tests/server/db/CreateV4TestSuiteInstance.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/tests/server/db/CreateV4TestSuiteInstance.py?rev=147994&r1=147993&r2=147994&view=diff
==============================================================================
--- zorg/trunk/lnt/tests/server/db/CreateV4TestSuiteInstance.py (original)
+++ zorg/trunk/lnt/tests/server/db/CreateV4TestSuiteInstance.py Wed Jan 11 19:27:26 2012
@@ -43,7 +43,7 @@
machine = ts_db.Machine("test-machine")
machine.uname = "test-uname"
-order = ts_db.Order(ordinal = 0)
+order = ts_db.Order()
order.llvm_revision = "test-revision"
run = ts_db.Run(machine, order, start_time, end_time)
run.arch = "test-arch"
@@ -85,7 +85,8 @@
assert machine.name == "test-machine"
assert machine.uname == "test-uname"
-assert order.ordinal == 0
+assert order.next_order_id is None
+assert order.previous_order_id is None
assert order.llvm_revision == "test-revision"
assert run.machine is machine
Modified: zorg/trunk/lnt/tests/server/db/ImportV4TestSuiteInstance.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/tests/server/db/ImportV4TestSuiteInstance.py?rev=147994&r1=147993&r2=147994&view=diff
==============================================================================
--- zorg/trunk/lnt/tests/server/db/ImportV4TestSuiteInstance.py (original)
+++ zorg/trunk/lnt/tests/server/db/ImportV4TestSuiteInstance.py Wed Jan 11 19:27:26 2012
@@ -73,12 +73,16 @@
assert tests[0].name == 'sampletest'
# Validate the orders.
-orders = list(ts.query(ts.Order).order_by(ts.Order.ordinal))
+orders = list(ts.query(ts.Order).order_by(ts.Order.llvm_project_revision))
assert len(orders) == 2
order_a,order_b = orders
-assert order_a.ordinal == 0
+print order_a
+print order_b
+assert order_a.previous_order_id is None
+assert order_a.next_order_id is order_b.id
assert order_a.llvm_project_revision == u'% 7d' % 1
-assert order_b.ordinal == 1
+assert order_b.previous_order_id is order_a.id
+assert order_b.next_order_id is None
assert order_b.llvm_project_revision == u'% 7d' % 2
# Validate the runs.
More information about the llvm-commits
mailing list