[LNT] r307223 - Schema migration logic for custom schemas
Matthias Braun via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 5 16:34:39 PDT 2017
Author: matze
Date: Wed Jul 5 16:34:39 2017
New Revision: 307223
URL: http://llvm.org/viewvc/llvm-project?rev=307223&view=rev
Log:
Schema migration logic for custom schemas
For yaml based testsuite schemas: This saves the last schema used in the
database. When the schema in the database differs from the configured
schema a migration strategy is tried: New metrics, run_fields and
machine_field columns are added as needed. If there are any other
differences in the schema the schema is rejected and server startup
aborted.
This also changes the default value for run fields and machine fields
from '' to null/None to better match the effects when adding a new
column. The rest of the UI seems unaffected by this.
Added:
lnt/trunk/lnt/server/db/migrations/upgrade_12_to_13.py
lnt/trunk/tests/server/db/Inputs/customschema-report2.json
lnt/trunk/tests/server/db/Inputs/schema-example-migratable.yaml
lnt/trunk/tests/server/db/Inputs/schema-example-nomigration0.yaml
lnt/trunk/tests/server/db/Inputs/schema-example-nomigration1.yaml
lnt/trunk/tests/server/db/Inputs/schema-example-nomigration2.yaml
Modified:
lnt/trunk/lnt/server/db/testsuite.py
lnt/trunk/lnt/server/db/testsuitedb.py
lnt/trunk/lnt/server/db/util.py
lnt/trunk/lnt/server/db/v4db.py
lnt/trunk/tests/server/db/yamlschema.shtest
Added: lnt/trunk/lnt/server/db/migrations/upgrade_12_to_13.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_12_to_13.py?rev=307223&view=auto
==============================================================================
--- lnt/trunk/lnt/server/db/migrations/upgrade_12_to_13.py (added)
+++ lnt/trunk/lnt/server/db/migrations/upgrade_12_to_13.py Wed Jul 5 16:34:39 2017
@@ -0,0 +1,12 @@
+# Adds new table to store jsonschema previously used to construct testsuite.
+import sqlalchemy
+from sqlalchemy import Column, String, Binary
+
+Base = sqlalchemy.ext.declarative.declarative_base()
+class TestSuiteJSONSchema(Base):
+ __tablename__ = "TestSuiteJSONSchemas"
+ testsuite_name = Column("TestSuiteName", String(256), primary_key=True)
+ jsonschema = Column("JSONSchema", Binary)
+
+def upgrade(engine):
+ Base.metadata.create_all(engine)
Modified: lnt/trunk/lnt/server/db/testsuite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/testsuite.py?rev=307223&r1=307222&r2=307223&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/testsuite.py (original)
+++ lnt/trunk/lnt/server/db/testsuite.py Wed Jul 5 16:34:39 2017
@@ -2,7 +2,12 @@
Database models for the TestSuites abstraction.
"""
+import json
import lnt
+import logging
+import sys
+import testsuitedb
+import util
import sqlalchemy
import sqlalchemy.ext.declarative
@@ -56,6 +61,106 @@ class StatusKind(Base):
return '%s%r' % (self.__class__.__name__, (self.name,))
+class _MigrationError(Exception):
+ def __init__(self, message):
+ full_message = \
+ "Cannot automatically migrate database: %s" % message
+ super(_MigrationError, self).__init__(full_message)
+
+
+class TestSuiteJSONSchema(Base):
+ """
+ Saves the json schema used when creating a testsuite. Only used for suites
+ created with a json schema description.
+ """
+ __tablename__ = 'TestSuiteJSONSchemas'
+ testsuite_name = Column("TestSuiteName", String(256), primary_key=True)
+ jsonschema = Column("JSONSchema", Binary)
+
+ def __init__(self, testsuite_name, data):
+ self.testsuite_name = testsuite_name
+ self.jsonschema = json.dumps(data, encoding='utf-8', sort_keys=True)
+
+ def upgrade_to(self, engine, new_schema, dry_run=False):
+ new = json.loads(new_schema.jsonschema)
+ old = json.loads(self.jsonschema)
+ ts_name = new['name']
+ if old['name'] != ts_name:
+ raise _MigrationError("Schema names differ?!?")
+
+ old_metrics = {}
+ for metric_desc in old.get('metrics', []):
+ old_metrics[metric_desc['name']] = metric_desc
+
+ for metric_desc in new.get('metrics', []):
+ name = metric_desc['name']
+ old_metric = old_metrics.pop(name, None)
+ type = metric_desc['type']
+ if old_metric is not None:
+ if old_metric['type'] != type:
+ raise _MigrationError("Type mismatch in metric '%s'" % name)
+ elif not dry_run:
+ # Add missing columns
+ column = testsuitedb.make_sample_column(name, type)
+ util.add_sqlalchemy_column(engine, '%s_Sample' % ts_name,
+ column)
+
+ if len(old_metrics) != 0:
+ raise _MigrationError("Metrics removed: %s" %
+ ", ".join(old_metrics.keys()))
+
+ old_run_fields = {}
+ old_order_fields = {}
+ for field_desc in old.get('run_fields', []):
+ if field_desc.get('order', False):
+ old_order_fields[field_desc['name']] = field_desc
+ continue
+ old_run_fields[field_desc['name']] = field_desc
+
+ for field_desc in new.get('run_fields', []):
+ name = field_desc['name']
+ if field_desc.get('order', False):
+ old_order_field = old_order_fields.pop(name, None)
+ if old_order_field is None:
+ raise _MigrationError("Cannot add order field '%s'" %
+ name)
+ continue
+
+ old_field = old_run_fields.pop(name, None)
+ # Add missing columns
+ if old_field is None and not dry_run:
+ column = testsuitedb.make_run_column(name)
+ util.add_sqlalchemy_column(engine, '%s_Run' % ts_name, column)
+
+ if len(old_run_fields) > 0:
+ raise _MigrationError("Run fields removed: %s" %
+ ", ".join(old_run_fields.keys()))
+ if len(old_order_fields) > 0:
+ raise _MigrationError("Order fields removed: %s" %
+ ", ".join(old_order_fields.keys()))
+
+
+ old_machine_fields = {}
+ for field_desc in old.get('machine_fields', []):
+ name = field_desc['name']
+ old_machine_fields[name] = field_desc
+
+ for field_desc in new.get('machine_fields', []):
+ name = field_desc['name']
+ old_field = old_machine_fields.pop(name, None)
+ # Add missing columns
+ if old_field is None and not dry_run:
+ column = testsuitedb.make_machine_column(name)
+ util.add_sqlalchemy_column(engine, '%s_Machine' % ts_name,
+ column)
+
+ if len(old_machine_fields) > 0:
+ raise _MigrationError("Machine fields removed: %s" %
+ ", ".join(old_machine_fields.keys()))
+ # The rest should just be metadata that we can upgrade
+ return True
+
+
class TestSuite(Base):
__tablename__ = 'TestSuite'
@@ -140,8 +245,32 @@ class TestSuite(Base):
bigger_is_better=bigger_is_better_int)
sample_fields.append(field)
ts.sample_fields = sample_fields
+ ts.jsonschema = data
return ts
+ def check_json_schema_changes(self, v4db):
+ name = self.name
+ schema = TestSuiteJSONSchema(name, self.jsonschema)
+ prev_schema = v4db.query(TestSuiteJSONSchema) \
+ .filter(TestSuiteJSONSchema.testsuite_name == name) \
+ .first()
+ if prev_schema is not None:
+ if prev_schema.jsonschema != schema.jsonschema:
+ logging.info("Previous Schema:")
+ logging.info(json.dumps(json.loads(prev_schema.jsonschema),
+ indent=2))
+ # New schema? Save it in the database and we are good.
+ engine = v4db.engine
+ prev_schema.upgrade_to(engine, schema, dry_run=True)
+ prev_schema.upgrade_to(engine, schema)
+
+ prev_schema.jsonschema = schema.jsonschema
+ v4db.add(prev_schema)
+ v4db.commit()
+ else:
+ v4db.add(schema)
+ v4db.commit()
+
class FieldMixin(object):
@property
Modified: lnt/trunk/lnt/server/db/testsuitedb.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/testsuitedb.py?rev=307223&r1=307222&r2=307223&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/testsuitedb.py (original)
+++ lnt/trunk/lnt/server/db/testsuitedb.py Wed Jul 5 16:34:39 2017
@@ -27,6 +27,32 @@ def strip(obj):
return new_dict
+_sample_type_to_sql = {
+ 'Real': Float,
+ 'Integer': Integer,
+ 'Hash': String,
+ 'Status': Integer
+}
+
+
+def make_sample_column(name, type):
+ sqltype = _sample_type_to_sql.get(type)
+ if sqltype is None:
+ raise ValueError("test suite defines unknown sample type %r" % type)
+ options = []
+ if type == 'Status':
+ options.append(ForeignKey(testsuite.StatusKind.id))
+ return Column(name, sqltype, *options)
+
+
+def make_run_column(name):
+ return Column(name, String(256))
+
+
+def make_machine_column(name):
+ return Column(name, String(256))
+
+
class TestSuiteDB(object):
"""
Wrapper object for an individual test suites database tables.
@@ -90,12 +116,12 @@ class TestSuiteDB(object):
# machine fields.
class_dict = locals()
for item in fields:
- if item.name in class_dict:
+ iname = item.name
+ if iname in class_dict:
raise ValueError("test suite defines reserved key %r" % (
- name))
+ iname))
- class_dict[item.name] = item.column = Column(
- item.name, String(256))
+ class_dict[iname] = item.column = make_machine_column(iname)
def __init__(self, name_value):
self.name = name_value
@@ -289,12 +315,12 @@ class TestSuiteDB(object):
# but need a bit for that in the test suite definition.
class_dict = locals()
for item in fields:
- if item.name in class_dict:
+ iname = item.name
+ if iname in class_dict:
raise ValueError("test suite defines reserved key %r" %
- (name,))
+ (iname,))
- class_dict[item.name] = item.column = Column(
- item.name, String(256))
+ class_dict[iname] = item.column = make_run_column(iname)
def __init__(self, machine, order, start_time, end_time):
self.machine = machine
@@ -445,24 +471,13 @@ class TestSuiteDB(object):
# the new UI is up.
class_dict = locals()
for item in self.sample_fields:
- if item.name in class_dict:
+ iname = item.name
+ if iname in class_dict:
raise ValueError("test suite defines reserved key %r" %
- (name,))
-
- if item.type.name == 'Real':
- item.column = Column(item.name, Float)
- elif item.type.name == 'Integer':
- item.column = Column(item.name, Integer)
- elif item.type.name == 'Status':
- item.column = Column(item.name, Integer, ForeignKey(
- testsuite.StatusKind.id))
- elif item.type.name == 'Hash':
- item.column = Column(item.name, String)
- else:
- raise ValueError("Unknown sample type %r" %
- (item.type.name,))
+ (iname,))
- class_dict[item.name] = item.column
+ item.column = make_sample_column(iname, item.type.name)
+ class_dict[iname] = item.column
def __init__(self, run, test, **kwargs):
self.run = run
@@ -701,11 +716,9 @@ class TestSuiteDB(object):
# Convert the machine data into a machine record. We construct the
# query to look for any existing machine at the same time as we build
# up the record to possibly add.
- #
- # FIXME: This feels inelegant, can't SA help us out here?
- query = self.query(self.Machine).\
- filter(self.Machine.name == machine_data['name'])
- machine = self.Machine(machine_data['name'])
+ name = machine_data['name']
+ query = self.query(self.Machine).filter(self.Machine.name == name)
+ machine = self.Machine(name)
machine_parameters = machine_data.copy()
machine_parameters.pop('name')
# Ignore incoming ids; we will create our own.
@@ -715,12 +728,7 @@ class TestSuiteDB(object):
# First, extract all of the specified machine fields.
for item in self.machine_fields:
- # For now, insert empty values for any missing fields. We don't
- # want to insert NULLs, so we should probably allow the test
- # suite to define defaults.
- default_value = ''
-
- value = machine_parameters.pop(item.name, default_value)
+ value = machine_parameters.pop(item.name, None)
query = query.filter(item.column == value)
machine.set_field(item, value)
@@ -732,12 +740,11 @@ class TestSuiteDB(object):
machine.parameters_data)
# Execute the query to see if we already have this machine.
- try:
- return query.one(), False
- except sqlalchemy.orm.exc.NoResultFound:
- # If not, add the machine.
+ existing_machine = query.first()
+ if existing_machine is not None:
+ return existing_machine, False
+ else:
self.add(machine)
-
return machine, True
def _getOrCreateOrder(self, run_parameters):
@@ -840,12 +847,7 @@ class TestSuiteDB(object):
# First, extract all of the specified run fields.
for item in self.run_fields:
- # For now, insert empty values for any missing fields. We don't
- # want to insert NULLs, so we should probably allow the test
- # suite to define defaults.
- default_value = ''
-
- value = run_parameters.pop(item.name, default_value)
+ value = run_parameters.pop(item.name, None)
query = query.filter(item.column == value)
run.set_field(item, value)
Modified: lnt/trunk/lnt/server/db/util.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/util.py?rev=307223&r1=307222&r2=307223&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/util.py (original)
+++ lnt/trunk/lnt/server/db/util.py Wed Jul 5 16:34:39 2017
@@ -1,7 +1,24 @@
-
+import sqlalchemy
+import sqlalchemy.ext.compiler
import re
PATH_DATABASE_TYPE_RE = re.compile('\w+\:\/\/')
def path_has_no_database_type(path):
return PATH_DATABASE_TYPE_RE.match(path) is None
+
+
+def _alter_table_statement(dialect, table_name, column):
+ """Given an SQLAlchemy Column object, create an `ALTER TABLE` statement
+ that adds the column to the existing table."""
+ # Code inspired by sqlalchemy.schema.CreateColumn documentation.
+ compiler = dialect.ddl_compiler(dialect, None)
+ text = "ALTER TABLE \"%s\" ADD COLUMN " % table_name
+ text += compiler.get_column_specification(column)
+ return text
+
+
+def add_sqlalchemy_column(engine, table_name, column):
+ statement = _alter_table_statement(engine.dialect, table_name, column)
+ with engine.begin() as trans:
+ trans.execute(statement)
Modified: lnt/trunk/lnt/server/db/v4db.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/v4db.py?rev=307223&r1=307222&r2=307223&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/v4db.py (original)
+++ lnt/trunk/lnt/server/db/v4db.py Wed Jul 5 16:34:39 2017
@@ -54,6 +54,7 @@ class V4DB(object):
create_tables = False
ts = self._extra_suites.get(name)
if ts:
+ ts.check_json_schema_changes(self.v4db)
create_tables = True
else:
# Get the test suite object.
Added: lnt/trunk/tests/server/db/Inputs/customschema-report2.json
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/Inputs/customschema-report2.json?rev=307223&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/Inputs/customschema-report2.json (added)
+++ lnt/trunk/tests/server/db/Inputs/customschema-report2.json Wed Jul 5 16:34:39 2017
@@ -0,0 +1,20 @@
+{
+ "format_version": "2",
+ "machine": {
+ "name": "sizetester0",
+ "hostname": "mymachine.local"
+ },
+ "run": {
+ "llvm_project_revision": "642042",
+ "end_time": "2017-04-18 23:31:18",
+ "start_time": "2017-04-18 23:01:34"
+ },
+ "tests": [
+ {
+ "name": "/obj/Debug/src/lib/embUnit~armv7_cortex-r5_Oz_floatabisoft/embUnit/AssertImpl_c~armv7_cortex-r5_Oz_floatabisoft_o",
+ "text_size": 402,
+ "data_size": 568,
+ "newfield": 42.42
+ }
+ ]
+}
Added: lnt/trunk/tests/server/db/Inputs/schema-example-migratable.yaml
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/Inputs/schema-example-migratable.yaml?rev=307223&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/Inputs/schema-example-migratable.yaml (added)
+++ lnt/trunk/tests/server/db/Inputs/schema-example-migratable.yaml Wed Jul 5 16:34:39 2017
@@ -0,0 +1,28 @@
+# This adds a new metric "newfield", as well as a new run and machine field
+# over the default example schema. Migration should succeed as this only
+# requires the create of new columns.
+format_version: "2"
+name: size
+metrics:
+ - name: text_size
+ bigger_is_better: false
+ type: Integer
+ - name: data_size
+ bigger_is_better: false
+ type: Integer
+ - name: score
+ bigger_is_better: true
+ type: Real
+ - name: hash
+ type: Hash
+ - name: newfield
+ bigger_is_better: true
+ type: Real
+run_fields:
+ - name: llvm_project_revision
+ order: true
+ - name: new_run_field
+machine_fields:
+ - name: new_machine_field
+ - name: hardware
+ - name: os
Added: lnt/trunk/tests/server/db/Inputs/schema-example-nomigration0.yaml
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/Inputs/schema-example-nomigration0.yaml?rev=307223&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/Inputs/schema-example-nomigration0.yaml (added)
+++ lnt/trunk/tests/server/db/Inputs/schema-example-nomigration0.yaml Wed Jul 5 16:34:39 2017
@@ -0,0 +1,22 @@
+# We cannot automatically migrate this as the metric "data_size" metric
+# disappears.
+format_version: "2"
+name: size
+metrics:
+ - name: text_size
+ bigger_is_better: false
+ type: Integer
+ - name: hash
+ type: Hash
+ - name: score
+ bigger_is_better: true
+ type: Real
+ - name: newfield
+ bigger_is_better: true
+ type: Real
+run_fields:
+ - name: llvm_project_revision
+ order: true
+machine_fields:
+ - name: hardware
+ - name: os
Added: lnt/trunk/tests/server/db/Inputs/schema-example-nomigration1.yaml
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/Inputs/schema-example-nomigration1.yaml?rev=307223&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/Inputs/schema-example-nomigration1.yaml (added)
+++ lnt/trunk/tests/server/db/Inputs/schema-example-nomigration1.yaml Wed Jul 5 16:34:39 2017
@@ -0,0 +1,24 @@
+# We cannot automatically migrate this it removes the existing machine field
+# "os"
+format_version: "2"
+name: size
+metrics:
+ - name: text_size
+ bigger_is_better: false
+ type: Integer
+ - name: data_size
+ bigger_is_better: false
+ type: Integer
+ - name: score
+ bigger_is_better: true
+ type: Real
+ - name: hash
+ type: Hash
+ - name: newfield
+ bigger_is_better: true
+ type: Real
+run_fields:
+ - name: llvm_project_revision
+ order: true
+machine_fields:
+ - name: hardware
Added: lnt/trunk/tests/server/db/Inputs/schema-example-nomigration2.yaml
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/Inputs/schema-example-nomigration2.yaml?rev=307223&view=auto
==============================================================================
--- lnt/trunk/tests/server/db/Inputs/schema-example-nomigration2.yaml (added)
+++ lnt/trunk/tests/server/db/Inputs/schema-example-nomigration2.yaml Wed Jul 5 16:34:39 2017
@@ -0,0 +1,22 @@
+# We cannot automatically migrate this it removes the existing machine field
+# "os"
+format_version: "2"
+name: size
+metrics:
+ - name: text_size
+ bigger_is_better: false
+ type: Integer
+ - name: data_size
+ bigger_is_better: false
+ type: Real
+ - name: score
+ bigger_is_better: true
+ type: Real
+ - name: hash
+ type: Hash
+run_fields:
+ - name: llvm_project_revision
+ order: true
+machine_fields:
+ - name: hardware
+ - name: os
Modified: lnt/trunk/tests/server/db/yamlschema.shtest
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/yamlschema.shtest?rev=307223&r1=307222&r2=307223&view=diff
==============================================================================
--- lnt/trunk/tests/server/db/yamlschema.shtest (original)
+++ lnt/trunk/tests/server/db/yamlschema.shtest Wed Jul 5 16:34:39 2017
@@ -5,11 +5,53 @@
# CHECK: Import succeeded.
# CHECK: Imported Data
-# CHECK: -------------
-# CHECK: Added Machines: 1
-# CHECK: Added Runs : 1
-# CHECK: Added Tests : 1
+# CHECK-NEXT: -------------
+# CHECK-NEXT: Added Machines: 1
+# CHECK-NEXT: Added Runs : 1
+# CHECK-NEXT: Added Tests : 1
#
# CHECK: Results
# CHECK: ----------------
# CHECK: PASS : 3
+
+
+# Migration Tests
+# ===============
+#
+# Inserting with an extra field shouldn't work just yet
+# RUN: not lnt import "%t.install" -s size "%S/Inputs/customschema-report2.json" --commit=1 2>&1 | FileCheck %s --check-prefix=NOTUPGRADED
+# NOTUPGRADED: Metric u'newfield' unknown in suite
+
+# Upgrading to a schema with metrics/fields removed should fail
+# RUN: rm -f "%t.install/schemas/size.yaml"
+# RUN: ln -sf "%S/Inputs/schema-example-nomigration0.yaml" "%t.install/schemas/size.yaml"
+# RUN: not lnt import "%t.install" -s size "%S/Inputs/customschema-report.json" --commit=1 2>&1 | FileCheck %s --check-prefix=NOMIGRATION0
+# NOMIGRATION0: Cannot automatically migrate database: Metrics removed: data_size
+#
+# RUN: rm -f "%t.install/schemas/size.yaml"
+# RUN: ln -sf "%S/Inputs/schema-example-nomigration1.yaml" "%t.install/schemas/size.yaml"
+# RUN: not lnt import "%t.install" -s size "%S/Inputs/customschema-report.json" --commit=1 2>&1 | FileCheck %s --check-prefix=NOMIGRATION1
+# NOMIGRATION1: Cannot automatically migrate database: Machine fields removed: os
+#
+# RUN: rm -f "%t.install/schemas/size.yaml"
+# RUN: ln -sf "%S/Inputs/schema-example-nomigration2.yaml" "%t.install/schemas/size.yaml"
+# RUN: not lnt import "%t.install" -s size "%S/Inputs/customschema-report.json" --commit=1 2>&1 | FileCheck %s --check-prefix=NOMIGRATION2
+# NOMIGRATION2: Cannot automatically migrate database: Type mismatch in metric 'data_size'
+
+
+# This upgrade should finally work
+# RUN: rm -f "%t.install/schemas/size.yaml"
+# RUN: ln -sf "%S/Inputs/schema-example-migratable.yaml" "%t.install/schemas/size.yaml"
+# RUN: lnt import "%t.install" "%S/Inputs/customschema-report2.json" -s size --commit=1 --show-sql 2>&1 | FileCheck %s --check-prefix=MIGRATION
+#
+# MIGRATION: ALTER TABLE "size_Sample" ADD COLUMN newfield FLOAT
+# MIGRATION: ALTER TABLE "size_Run" ADD COLUMN new_run_field VARCHAR(256)
+# MIGRATION: ALTER TABLE "size_Machine" ADD COLUMN new_machine_field VARCHAR(256)
+#
+# MIGRATION: Import succeeded.
+# MIGRATION: Imported Data
+# MIGRATION-NEXT: -------------
+# MIGRATION-NEXT: Added Runs : 1
+#
+# MIGRATION: Results
+# MIGRATION: PASS : 4
More information about the llvm-commits
mailing list