[LNT] r307562 - Use yaml schema mechanism to describe compile suite
Matthias Braun via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 10 12:56:51 PDT 2017
Uhoh, somehow I didn't expect references to the "metatables".
Apart from the migration failing, I probably have to extend the code to actually add the entries to the metatables even if we load from the .yaml file.
And of course I now wonder, why I split the migration into two transactions instead of one... I assume for recovery we have to manually issue a whole bunch of
ALTER TABLE "compile_XXX" RENAME TO "Compile_XXX" basically for everything here:
renames = {
'Compile_Machine': 'compile_Machine',
'Compile_Order': 'compile_Order',
'Compile_Run': 'compile_Run',
'Compile_Sample': 'compile_Sample',
'Compile_Profile': 'compile_Profile',
'Compile_FieldChange': 'compile_FieldChange',
'Compile_FieldChangeV2': 'compile_FieldChangeV2',
'Compile_Regression': 'compile_Regression',
'Compile_RegressionIndicator': 'compile_RegressionIndicator',
'Compile_ChangeIgnore': 'compile_ChangeIgnore',
'Compile_BaseLine': 'compile_Baseline',
}
because that part of the migration probably went through already.
- Matthias
> On Jul 10, 2017, at 12:11 PM, Chris Matthews via llvm-commits <llvm-commits at lists.llvm.org> wrote:
>
> Hi Matthias,
>
> I revert this. It seems to explode quite badly during the upgrade on our server:
>
> (IntegrityError) update or delete on table "TestSuiteSampleFields" violates foreign key constraint "Compile_FieldChangeV2_FieldID_fkey" on table "Compile_FieldChangeV2" DETAIL: Key (ID)=(10) is still referenced from table "Compile_FieldChangeV2". '\nDELETE FROM "TestSuiteSampleFields"\n WHERE "TestSuiteID" IN\n (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')\n' {}
>
> IntegrityError: (IntegrityError) update or delete on table "TestSuiteSampleFields" violates foreign key constraint "Compile_FieldChangeV2_FieldID_fkey" on table "Compile_FieldChangeV2"
> DETAIL: Key (ID)=(10) is still referenced from table "Compile_FieldChangeV2".
> '\nDELETE FROM "TestSuiteSampleFields"\n WHERE "TestSuiteID" IN\n (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')\n' {}
> File "flask/app.py", line 1817, in wsgi_app
> response = self.full_dispatch_request()
> File "flask/app.py", line 1477, in full_dispatch_request
> rv = self.handle_user_exception(e)
> File "flask_restful/__init__.py", line 270, in error_router
> return original_handler(e)
> File "flask/app.py", line 1381, in handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "flask/app.py", line 1475, in full_dispatch_request
> rv = self.dispatch_request()
> File "flask/app.py", line 1461, in dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "lnt/server/ui/decorators.py", line 32, in wrap
> result = f(**args)
> File "lnt/server/ui/views.py", line 187, in submit_run
> return _do_submit()
> File "lnt/server/ui/views.py", line 161, in _do_submit
> db = request.get_db()
> File "lnt/server/ui/app.py", line 101, in get_db
> self.db = current_app.old_config.get_database(g.db_name, echo=echo)
> File "lnt/server/config.py", line 183, in get_database
> echo)
> File "lnt/server/db/v4db.py", line 122, in __init__
> lnt.server.db.migrate.update(self.engine)
> File "lnt/server/db/migrate.py", line 220, in update
> available_migrations, '__core__')
> File "lnt/server/db/migrate.py", line 172, in update_schema
> upgrade_method(engine)
> File "/opt/llvm/lnt/lnt/server/db/migrations/upgrade_13_to_14.py", line 71, in upgrade
> ''')
> File "sqlalchemy/engine/base.py", line 712, in execute
> return self._execute_text(object, multiparams, params)
> File "sqlalchemy/engine/base.py", line 861, in _execute_text
> statement, parameters
> File "sqlalchemy/engine/base.py", line 947, in _execute_context
> context)
> File "sqlalchemy/engine/base.py", line 1108, in _handle_dbapi_exception
> exc_info
> File "sqlalchemy/util/compat.py", line 185, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb)
> File "sqlalchemy/engine/base.py", line 940, in _execute_context
> context)
> File "sqlalchemy/engine/default.py", line 435, in do_execute
> cursor.execute(statement, parameters)
>
>> On Jul 10, 2017, at 11:10 AM, Matthias Braun via llvm-commits <llvm-commits at lists.llvm.org> wrote:
>>
>> Author: matze
>> Date: Mon Jul 10 11:10:32 2017
>> New Revision: 307562
>>
>> URL: http://llvm.org/viewvc/llvm-project?rev=307562&view=rev
>> Log:
>> Use yaml schema mechanism to describe compile suite
>>
>> This converts the existing compile suite schema which was previously
>> configured via the TestSuiteXXX tables in lnt/server/db/migrations to
>> use the new yaml schema mechanism instead.
>>
>> - Adds schemas/compile.yaml with the test suite definition.
>> - Adds a migration that removes the compile definitions in the
>> TestSuiteXXX meta tables. It also drops the compile tables if they are
>> empty (and otherwise leaves them untouched).
>> - This means by default the compile suite will not be visible anymore.
>> Users using the 'compile' suite need to copy or symlink
>> schemas/compile.yaml into $LNTINSTANCEPATH/schemas/compile.yaml
>>
>> Added:
>> lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py
>> lnt/trunk/schemas/
>> lnt/trunk/schemas/README.md
>> lnt/trunk/schemas/compile.yaml
>> Modified:
>> lnt/trunk/tests/SharedInputs/create_temp_instance.py
>> lnt/trunk/tests/server/db/CreateV4TestSuite.py
>>
>> Added: lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py
>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py?rev=307562&view=auto
>> ==============================================================================
>> --- lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py (added)
>> +++ lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py Mon Jul 10 11:10:32 2017
>> @@ -0,0 +1,74 @@
>> +# The "compile" suite is handled by the compile.yaml schema file now.
>> +# This means we drop the entry in the TestSuite meta tables.
>> +# The existing tables are either dropped if they are empty. We have to rename
>> +# them if they are not empty, as previously the test-suite name was different
>> +# from the prefix used in the tables. In yaml schemas the name and prefix is
>> +# always the same so we have to rename from `Compile_XXXX` to `compile_XXX`.
>> +import sqlalchemy
>> +
>> +
>> +def _drop_table_if_empty(trans, name):
>> + num = trans.execute('SELECT COUNT(*) FROM "%s"' % name).first()
>> + if num[0] == 0:
>> + trans.execute('DROP TABLE "%s"' % name)
>> + return True
>> + return False
>> +
>> +
>> +def upgrade(engine):
>> + # The following is expected to fail if the user never had an old
>> + # version of the database that create the Compile_XXX tables.
>> + try:
>> + renames = {
>> + 'Compile_Machine': 'compile_Machine',
>> + 'Compile_Order': 'compile_Order',
>> + 'Compile_Run': 'compile_Run',
>> + 'Compile_Sample': 'compile_Sample',
>> + 'Compile_Profile': 'compile_Profile',
>> + 'Compile_FieldChange': 'compile_FieldChange',
>> + 'Compile_FieldChangeV2': 'compile_FieldChangeV2',
>> + 'Compile_Regression': 'compile_Regression',
>> + 'Compile_RegressionIndicator': 'compile_RegressionIndicator',
>> + 'Compile_ChangeIgnore': 'compile_ChangeIgnore',
>> + 'Compile_BaseLine': 'compile_Baseline',
>> + }
>> + with engine.begin() as trans:
>> + for old_name, new_name in renames.items():
>> + if _drop_table_if_empty(trans, old_name):
>> + continue;
>> + env = {'old_name': old_name, 'new_name': new_name}
>> + trans.execute('''
>> +ALTER TABLE "%(old_name)s" RENAME TO "%(new_name)s_x"
>> +''' % env)
>> + trans.execute('''
>> +ALTER TABLE "%(new_name)s_x" RENAME TO \"%(new_name)s\"
>> +''' % env)
>> + except Exception as e:
>> + import traceback
>> + traceback.print_exc()
>> +
>> + # Drop Compile suite information from meta tables
>> + with engine.begin() as trans:
>> + trans.execute('''
>> +DELETE FROM "TestSuiteOrderFields"
>> + WHERE "TestSuiteID" IN
>> + (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')
>> +''')
>> + trans.execute('''
>> +DELETE FROM "TestSuiteMachineFields"
>> + WHERE "TestSuiteID" IN
>> + (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')
>> +''')
>> + trans.execute('''
>> +DELETE FROM "TestSuiteRunFields"
>> + WHERE "TestSuiteID" IN
>> + (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')
>> +''')
>> + trans.execute('''
>> +DELETE FROM "TestSuiteSampleFields"
>> + WHERE "TestSuiteID" IN
>> + (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')
>> +''')
>> + trans.execute('''
>> +DELETE FROM "TestSuite" WHERE "Name"='compile'
>> +''')
>>
>> Added: lnt/trunk/schemas/README.md
>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/schemas/README.md?rev=307562&view=auto
>> ==============================================================================
>> --- lnt/trunk/schemas/README.md (added)
>> +++ lnt/trunk/schemas/README.md Mon Jul 10 11:10:32 2017
>> @@ -0,0 +1,2 @@
>> +This directory contains schema files describing test suites. You can copy them
>> +or create symlinks to the schemas directory of an lnt instance to enable them.
>>
>> Added: lnt/trunk/schemas/compile.yaml
>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/schemas/compile.yaml?rev=307562&view=auto
>> ==============================================================================
>> --- lnt/trunk/schemas/compile.yaml (added)
>> +++ lnt/trunk/schemas/compile.yaml Mon Jul 10 11:10:32 2017
>> @@ -0,0 +1,30 @@
>> +# This schema is used by the `lnt runtest compile` suite.
>> +format_version: '2'
>> +name: compile
>> +metrics:
>> + - name: user_time
>> + type: Real
>> + - name: user_status
>> + type: Status
>> + - name: sys_time
>> + type: Real
>> + - name: sys_status
>> + type: Status
>> + - name: wall_time
>> + type: Real
>> + - name: wall_status
>> + type: Status
>> + - name: size_bytes
>> + type: Real # Should be Integer but we don't want to invalidate old data
>> + - name: size_status
>> + type: Status
>> + - name: mem_bytes
>> + type: Real
>> + - name: mem_status
>> + type: Status
>> +run_fields:
>> + - name: llvm_project_revision
>> + order: true
>> +machine_fields:
>> + - name: hardware
>> + - name: os_version
>>
>> Modified: lnt/trunk/tests/SharedInputs/create_temp_instance.py
>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/SharedInputs/create_temp_instance.py?rev=307562&r1=307561&r2=307562&view=diff
>> ==============================================================================
>> --- lnt/trunk/tests/SharedInputs/create_temp_instance.py (original)
>> +++ lnt/trunk/tests/SharedInputs/create_temp_instance.py Mon Jul 10 11:10:32 2017
>> @@ -128,5 +128,9 @@ def main():
>> dest_dir)
>> if extra_sql:
>> run_sql_file(lnt_db, extra_sql, dest_dir)
>> + os.mkdir(os.path.join(dest_dir, 'schemas'))
>> + filedir = os.path.dirname(__file__)
>> + os.symlink(os.path.join(filedir, '..', '..', 'schemas', 'compile.yaml'),
>> + os.path.join(dest_dir, 'schemas', 'compile.yaml'))
>>
>> main()
>>
>> Modified: lnt/trunk/tests/server/db/CreateV4TestSuite.py
>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/CreateV4TestSuite.py?rev=307562&r1=307561&r2=307562&view=diff
>> ==============================================================================
>> --- lnt/trunk/tests/server/db/CreateV4TestSuite.py (original)
>> +++ lnt/trunk/tests/server/db/CreateV4TestSuite.py Mon Jul 10 11:10:32 2017
>> @@ -13,9 +13,9 @@ from lnt.server.db import v4db
>> # Create an in memory database.
>> db = v4db.V4DB("sqlite:///:memory:", Config.dummy_instance(), echo=True)
>>
>> -# We expect exactly two test suites, one for NTS and one for Compile.
>> +# We expect exactly the NTS test suite.
>> test_suites = list(db.query(testsuite.TestSuite))
>> -assert len(test_suites) == 2
>> +assert len(test_suites) == 1
>>
>> # Check the NTS test suite.
>> ts = db.query(testsuite.TestSuite).filter_by(name="nts").first()
>>
>>
>> _______________________________________________
>> llvm-commits mailing list
>> llvm-commits at lists.llvm.org
>> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list