[LNT] r307562 - Use yaml schema mechanism to describe compile suite
Chris Matthews via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 10 12:11:38 PDT 2017
Hi Matthias,
I revert this. It seems to explode quite badly during the upgrade on our server:
(IntegrityError) update or delete on table "TestSuiteSampleFields" violates foreign key constraint "Compile_FieldChangeV2_FieldID_fkey" on table "Compile_FieldChangeV2" DETAIL: Key (ID)=(10) is still referenced from table "Compile_FieldChangeV2". '\nDELETE FROM "TestSuiteSampleFields"\n WHERE "TestSuiteID" IN\n (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')\n' {}
IntegrityError: (IntegrityError) update or delete on table "TestSuiteSampleFields" violates foreign key constraint "Compile_FieldChangeV2_FieldID_fkey" on table "Compile_FieldChangeV2"
DETAIL: Key (ID)=(10) is still referenced from table "Compile_FieldChangeV2".
'\nDELETE FROM "TestSuiteSampleFields"\n WHERE "TestSuiteID" IN\n (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')\n' {}
File "flask/app.py", line 1817, in wsgi_app
response = self.full_dispatch_request()
File "flask/app.py", line 1477, in full_dispatch_request
rv = self.handle_user_exception(e)
File "flask_restful/__init__.py", line 270, in error_router
return original_handler(e)
File "flask/app.py", line 1381, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "flask/app.py", line 1475, in full_dispatch_request
rv = self.dispatch_request()
File "flask/app.py", line 1461, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "lnt/server/ui/decorators.py", line 32, in wrap
result = f(**args)
File "lnt/server/ui/views.py", line 187, in submit_run
return _do_submit()
File "lnt/server/ui/views.py", line 161, in _do_submit
db = request.get_db()
File "lnt/server/ui/app.py", line 101, in get_db
self.db = current_app.old_config.get_database(g.db_name, echo=echo)
File "lnt/server/config.py", line 183, in get_database
echo)
File "lnt/server/db/v4db.py", line 122, in __init__
lnt.server.db.migrate.update(self.engine)
File "lnt/server/db/migrate.py", line 220, in update
available_migrations, '__core__')
File "lnt/server/db/migrate.py", line 172, in update_schema
upgrade_method(engine)
File "/opt/llvm/lnt/lnt/server/db/migrations/upgrade_13_to_14.py", line 71, in upgrade
''')
File "sqlalchemy/engine/base.py", line 712, in execute
return self._execute_text(object, multiparams, params)
File "sqlalchemy/engine/base.py", line 861, in _execute_text
statement, parameters
File "sqlalchemy/engine/base.py", line 947, in _execute_context
context)
File "sqlalchemy/engine/base.py", line 1108, in _handle_dbapi_exception
exc_info
File "sqlalchemy/util/compat.py", line 185, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "sqlalchemy/engine/base.py", line 940, in _execute_context
context)
File "sqlalchemy/engine/default.py", line 435, in do_execute
cursor.execute(statement, parameters)
> On Jul 10, 2017, at 11:10 AM, Matthias Braun via llvm-commits <llvm-commits at lists.llvm.org> wrote:
>
> Author: matze
> Date: Mon Jul 10 11:10:32 2017
> New Revision: 307562
>
> URL: http://llvm.org/viewvc/llvm-project?rev=307562&view=rev
> Log:
> Use yaml schema mechanism to describe compile suite
>
> This converts the existing compile suite schema which was previously
> configured via the TestSuiteXXX tables in lnt/server/db/migrations to
> use the new yaml schema mechanism instead.
>
> - Adds schemas/compile.yaml with the test suite definition.
> - Adds a migration that removes the compile definitions in the
> TestSuiteXXX meta tables. It also drops the compile tables if they are
> empty (and otherwise leaves them untouched).
> - This means by default the compile suite will not be visible anymore.
> Users using the 'compile' suite need to copy or symlink
> schemas/compile.yaml into $LNTINSTANCEPATH/schemas/compile.yaml
>
> Added:
> lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py
> lnt/trunk/schemas/
> lnt/trunk/schemas/README.md
> lnt/trunk/schemas/compile.yaml
> Modified:
> lnt/trunk/tests/SharedInputs/create_temp_instance.py
> lnt/trunk/tests/server/db/CreateV4TestSuite.py
>
> Added: lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py?rev=307562&view=auto
> ==============================================================================
> --- lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py (added)
> +++ lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py Mon Jul 10 11:10:32 2017
> @@ -0,0 +1,74 @@
> +# The "compile" suite is handled by the compile.yaml schema file now.
> +# This means we drop the entry in the TestSuite meta tables.
> +# The existing tables are either dropped if they are empty. We have to rename
> +# them if they are not empty, as previously the test-suite name was different
> +# from the prefix used in the tables. In yaml schemas the name and prefix is
> +# always the same so we have to rename from `Compile_XXXX` to `compile_XXX`.
> +import sqlalchemy
> +
> +
> +def _drop_table_if_empty(trans, name):
> + num = trans.execute('SELECT COUNT(*) FROM "%s"' % name).first()
> + if num[0] == 0:
> + trans.execute('DROP TABLE "%s"' % name)
> + return True
> + return False
> +
> +
> +def upgrade(engine):
> + # The following is expected to fail if the user never had an old
> + # version of the database that create the Compile_XXX tables.
> + try:
> + renames = {
> + 'Compile_Machine': 'compile_Machine',
> + 'Compile_Order': 'compile_Order',
> + 'Compile_Run': 'compile_Run',
> + 'Compile_Sample': 'compile_Sample',
> + 'Compile_Profile': 'compile_Profile',
> + 'Compile_FieldChange': 'compile_FieldChange',
> + 'Compile_FieldChangeV2': 'compile_FieldChangeV2',
> + 'Compile_Regression': 'compile_Regression',
> + 'Compile_RegressionIndicator': 'compile_RegressionIndicator',
> + 'Compile_ChangeIgnore': 'compile_ChangeIgnore',
> + 'Compile_BaseLine': 'compile_Baseline',
> + }
> + with engine.begin() as trans:
> + for old_name, new_name in renames.items():
> + if _drop_table_if_empty(trans, old_name):
> + continue;
> + env = {'old_name': old_name, 'new_name': new_name}
> + trans.execute('''
> +ALTER TABLE "%(old_name)s" RENAME TO "%(new_name)s_x"
> +''' % env)
> + trans.execute('''
> +ALTER TABLE "%(new_name)s_x" RENAME TO \"%(new_name)s\"
> +''' % env)
> + except Exception as e:
> + import traceback
> + traceback.print_exc()
> +
> + # Drop Compile suite information from meta tables
> + with engine.begin() as trans:
> + trans.execute('''
> +DELETE FROM "TestSuiteOrderFields"
> + WHERE "TestSuiteID" IN
> + (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')
> +''')
> + trans.execute('''
> +DELETE FROM "TestSuiteMachineFields"
> + WHERE "TestSuiteID" IN
> + (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')
> +''')
> + trans.execute('''
> +DELETE FROM "TestSuiteRunFields"
> + WHERE "TestSuiteID" IN
> + (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')
> +''')
> + trans.execute('''
> +DELETE FROM "TestSuiteSampleFields"
> + WHERE "TestSuiteID" IN
> + (SELECT "ID" FROM "TestSuite" WHERE "Name"=\'compile\')
> +''')
> + trans.execute('''
> +DELETE FROM "TestSuite" WHERE "Name"='compile'
> +''')
>
> Added: lnt/trunk/schemas/README.md
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/schemas/README.md?rev=307562&view=auto
> ==============================================================================
> --- lnt/trunk/schemas/README.md (added)
> +++ lnt/trunk/schemas/README.md Mon Jul 10 11:10:32 2017
> @@ -0,0 +1,2 @@
> +This directory contains schema files describing test suites. You can copy them
> +or create symlinks to the schemas directory of an lnt instance to enable them.
>
> Added: lnt/trunk/schemas/compile.yaml
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/schemas/compile.yaml?rev=307562&view=auto
> ==============================================================================
> --- lnt/trunk/schemas/compile.yaml (added)
> +++ lnt/trunk/schemas/compile.yaml Mon Jul 10 11:10:32 2017
> @@ -0,0 +1,30 @@
> +# This schema is used by the `lnt runtest compile` suite.
> +format_version: '2'
> +name: compile
> +metrics:
> + - name: user_time
> + type: Real
> + - name: user_status
> + type: Status
> + - name: sys_time
> + type: Real
> + - name: sys_status
> + type: Status
> + - name: wall_time
> + type: Real
> + - name: wall_status
> + type: Status
> + - name: size_bytes
> + type: Real # Should be Integer but we don't want to invalidate old data
> + - name: size_status
> + type: Status
> + - name: mem_bytes
> + type: Real
> + - name: mem_status
> + type: Status
> +run_fields:
> + - name: llvm_project_revision
> + order: true
> +machine_fields:
> + - name: hardware
> + - name: os_version
>
> Modified: lnt/trunk/tests/SharedInputs/create_temp_instance.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/SharedInputs/create_temp_instance.py?rev=307562&r1=307561&r2=307562&view=diff
> ==============================================================================
> --- lnt/trunk/tests/SharedInputs/create_temp_instance.py (original)
> +++ lnt/trunk/tests/SharedInputs/create_temp_instance.py Mon Jul 10 11:10:32 2017
> @@ -128,5 +128,9 @@ def main():
> dest_dir)
> if extra_sql:
> run_sql_file(lnt_db, extra_sql, dest_dir)
> + os.mkdir(os.path.join(dest_dir, 'schemas'))
> + filedir = os.path.dirname(__file__)
> + os.symlink(os.path.join(filedir, '..', '..', 'schemas', 'compile.yaml'),
> + os.path.join(dest_dir, 'schemas', 'compile.yaml'))
>
> main()
>
> Modified: lnt/trunk/tests/server/db/CreateV4TestSuite.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/db/CreateV4TestSuite.py?rev=307562&r1=307561&r2=307562&view=diff
> ==============================================================================
> --- lnt/trunk/tests/server/db/CreateV4TestSuite.py (original)
> +++ lnt/trunk/tests/server/db/CreateV4TestSuite.py Mon Jul 10 11:10:32 2017
> @@ -13,9 +13,9 @@ from lnt.server.db import v4db
> # Create an in memory database.
> db = v4db.V4DB("sqlite:///:memory:", Config.dummy_instance(), echo=True)
>
> -# We expect exactly two test suites, one for NTS and one for Compile.
> +# We expect exactly the NTS test suite.
> test_suites = list(db.query(testsuite.TestSuite))
> -assert len(test_suites) == 2
> +assert len(test_suites) == 1
>
> # Check the NTS test suite.
> ts = db.query(testsuite.TestSuite).filter_by(name="nts").first()
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list