[LNT] r310665 - MySQL support for LNT
Chris Matthews via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 10 14:38:39 PDT 2017
Author: cmatthews
Date: Thu Aug 10 14:38:39 2017
New Revision: 310665
URL: http://llvm.org/viewvc/llvm-project?rev=310665&view=rev
Log:
MySQL support for LNT
This is an attempt to merge in LNT's MySQL support from an experimental
branch. The lnt.llvm.org server runs on MySQL.
Add a test for MySQL support which brings up a simple instance and runs
migrations, and a mysql wrapper tester, like the postgres one, for
creating a dedicated DB instance during testing. Add a mysql lit param
to enable the test.
Change in LNT to support MySQL:
- turn off autoincrement on some values which we want to manually
assign IDs to.
- turn off an unused index whose type MySQL did not support.
- Update some error inspections to notice mysql error messages.
- Change how test suite are delete to fix a circular foreign key issue
in mysql.
Added:
lnt/trunk/tests/SharedInputs/mysql_wrapper.sh (with props)
lnt/trunk/tests/lnttool/MySQLDB.shtest
Modified:
lnt/trunk/docs/developer_guide.rst
lnt/trunk/lnt/server/db/migrate.py
lnt/trunk/lnt/server/db/migrations/upgrade_0_to_1.py
lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py
lnt/trunk/tests/lit.cfg
Modified: lnt/trunk/docs/developer_guide.rst
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/docs/developer_guide.rst?rev=310665&r1=310664&r2=310665&view=diff
==============================================================================
--- lnt/trunk/docs/developer_guide.rst (original)
+++ lnt/trunk/docs/developer_guide.rst Thu Aug 10 14:38:39 2017
@@ -52,3 +52,12 @@ database, use a command like the followi
PATH=$LLVMBUILD/bin:$LNTINSTALL/bin:$PATH llvm-lit -sv -Dpostgres=1 ../lnt/tests
You'll need to use at least postgres version 9.2 to run the regression tests.
+
+LNT MySQL Support
+=================
+
+The lnt.llvm.org server runs on MySQL. Testing has been more limited than sqlite and Postgres. To run the regression
+tests there needs to be MySQL installed in the local path. The LNT tests will create their own instance of a MySQL
+server and database to test in::
+
+ llvm-lit -sv -Dmysql=1 ../lnt/tests
Modified: lnt/trunk/lnt/server/db/migrate.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrate.py?rev=310665&r1=310664&r2=310665&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/migrate.py (original)
+++ lnt/trunk/lnt/server/db/migrate.py Thu Aug 10 14:38:39 2017
@@ -14,6 +14,7 @@ import sqlalchemy
import sqlalchemy.ext.declarative
import sqlalchemy.orm
from sqlalchemy import Column, String, Integer
+from sqlalchemy.exc import ProgrammingError, OperationalError
from lnt.util import logger
import lnt.server.db.util
Modified: lnt/trunk/lnt/server/db/migrations/upgrade_0_to_1.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_0_to_1.py?rev=310665&r1=310664&r2=310665&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/migrations/upgrade_0_to_1.py (original)
+++ lnt/trunk/lnt/server/db/migrations/upgrade_0_to_1.py Thu Aug 10 14:38:39 2017
@@ -197,7 +197,7 @@ def get_base_for_testsuite(test_suite):
id = Column("ID", Integer, primary_key=True)
name = Column("Name", String(256), index=True)
- parameters_data = Column("Parameters", Binary)
+ parameters_data = Column("Parameters", Binary, index=False, unique=False)
class_dict = locals()
for item in test_suite.machine_fields:
@@ -221,8 +221,8 @@ def get_base_for_testsuite(test_suite):
class_dict = locals()
for item in test_suite.order_fields:
if item.name in class_dict:
- raise ValueError("test suite defines reserved key %r" %
- (name,))
+ raise ValueError,"test suite defines reserved key %r" % (
+ item.name,)
class_dict[item.name] = item.column = Column(
item.name, String(256))
@@ -240,7 +240,7 @@ def get_base_for_testsuite(test_suite):
end_time = Column("EndTime", DateTime)
simple_run_id = Column("SimpleRunID", Integer)
- parameters_data = Column("Parameters", Binary)
+ parameters_data = Column("Parameters", Binary, index=False, unique=False)
machine = sqlalchemy.orm.relation(Machine)
order = sqlalchemy.orm.relation(Order)
@@ -248,8 +248,8 @@ def get_base_for_testsuite(test_suite):
class_dict = locals()
for item in test_suite.run_fields:
if item.name in class_dict:
- raise ValueError("test suite defines reserved key %r" %
- (name,))
+ raise ValueError,"test suite defines reserved key %r" % (
+ item.name,)
class_dict[item.name] = item.column = Column(
item.name, String(256))
Modified: lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py?rev=310665&r1=310664&r2=310665&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py (original)
+++ lnt/trunk/lnt/server/db/migrations/upgrade_13_to_14.py Thu Aug 10 14:38:39 2017
@@ -6,7 +6,7 @@
# always the same so we have to rename from `Compile_XXXX` to `compile_XXX`.
import collections
-from sqlalchemy import delete, select, update, func
+from sqlalchemy import delete, select, update, func, and_
from lnt.server.db.migrations.util import introspect_table, rename_table
@@ -29,7 +29,8 @@ def _drop_suite(trans, name, engine):
drop_fields(engine, test_suite_id, 'TestSuiteOrderFields', trans)
drop_fields(engine, test_suite_id, 'TestSuiteMachineFields', trans)
drop_fields(engine, test_suite_id, 'TestSuiteRunFields', trans)
- drop_fields(engine, test_suite_id, 'TestSuiteSampleFields', trans)
+
+ drop_samples_fields(engine, test_suite_id, trans)
trans.execute(delete(test_suite).where(test_suite.c.Name == name))
@@ -44,6 +45,23 @@ def drop_fields(engine, test_suite_id, n
return fields_table
+def drop_samples_fields(engine, test_suite_id, trans):
+ """In the TestSuiteSampleFields, drop entries related to the test_suite_id.
+
+ This extra function is needed because in MySQL it can't sort out the forign
+ keys in the same table.
+ """
+ samples_table = introspect_table(engine, 'TestSuiteSampleFields')
+ order_files = delete(samples_table,
+ and_(samples_table.c.TestSuiteID == test_suite_id,
+ samples_table.c.status_field.isnot(None)))
+ trans.execute(order_files)
+ order_files = delete(samples_table,
+ samples_table.c.TestSuiteID == test_suite_id)
+ trans.execute(order_files)
+ return samples_table
+
+
TableRename = collections.namedtuple('TableRename', 'old_name new_name')
Added: lnt/trunk/tests/SharedInputs/mysql_wrapper.sh
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/SharedInputs/mysql_wrapper.sh?rev=310665&view=auto
==============================================================================
--- lnt/trunk/tests/SharedInputs/mysql_wrapper.sh (added)
+++ lnt/trunk/tests/SharedInputs/mysql_wrapper.sh Thu Aug 10 14:38:39 2017
@@ -0,0 +1,88 @@
+#!/bin/sh
+# This works like postgres_wrapper.sh but for mysql.
+#
+# This will bind to localhost on the specified port. There is a database
+# called testdb; user 'root' can login without a password.
+#
+# Example:
+# ./mysql_wrapper /tmp/myinstance 9101 mysql -u root -P 9101 testdb
+#
+# Inspired by https://github.com/tk0miya/testing.mysqld
+set -eu
+
+DBDIR="$1"
+shift
+if [ -d "${DBDIR}" ]; then
+ echo 1>&2 "${DBDIR} already exists"
+ exit 1
+fi
+PORT="$1"
+shift
+
+mkdir -p "${DBDIR}"
+mkdir -p "${DBDIR}/etc"
+mkdir -p "${DBDIR}/data"
+cat > "${DBDIR}/etc/my.cnf" << __EOF__
+[mysqld]
+bind-address = 127.0.0.1
+port = ${PORT}
+user = $(whoami)
+__EOF__
+
+MYSQL_INSTALL_DB_FLAGS+=" --defaults-file=\"${DBDIR}/etc/my.cnf\""
+MYSQL_INSTALL_DB_FLAGS+=" --initialize-insecure"
+MYSQL_INSTALL_DB_FLAGS+=" --user=$(whoami)"
+MYSQL_INSTALL_DB_FLAGS+=" --datadir=\"${DBDIR}/data\""
+MYSQL_INSTALL_DB_FLAGS+=" >& \"${DBDIR}/install_db.log\""
+#echo "$ mysql_install_db ${MYSQL_INSTALL_DB_FLAGS}"
+#eval mysql_install_db ${MYSQL_INSTALL_DB_FLAGS}
+echo "$ mysqld ${MYSQL_INSTALL_DB_FLAGS}"
+eval mysqld ${MYSQL_INSTALL_DB_FLAGS}
+
+
+MYSQLD_FLAGS+=" --defaults-file=\"${DBDIR}/etc/my.cnf\""
+MYSQLD_FLAGS+=" --datadir=\"${DBDIR}/data\""
+MYSQLD_FLAGS+=" --pid-file=\"${DBDIR}/mysqld.pid\""
+MYSQLD_FLAGS+=" --log-error=\"${DBDIR}/mysqld.error.log\""
+MYSQLD_FLAGS+="&"
+echo "$ mysqld ${MYSQLD_FLAGS}"
+eval mysqld ${MYSQLD_FLAGS}
+
+MYSQLADMIN_FLAGS+=" --defaults-file=\"${DBDIR}/etc/my.cnf\""
+MYSQLADMIN_FLAGS+=" -h 127.0.0.1"
+MYSQLADMIN_FLAGS+=" -P ${PORT}"
+MYSQLADMIN_FLAGS+=" -u root"
+
+# Poll server for 10 seconds to see when it is up.
+set +e
+for i in {1..100}
+do
+ sleep 0.1
+ echo "$ mysqladmin ${MYSQLADMIN_FLAGS} status"
+ eval mysqladmin ${MYSQLADMIN_FLAGS} status
+ if [ $? -e 0 ]; then
+ break
+ fi
+done
+set -e
+
+set +e
+# This may not be there if the test has not been run before.
+echo "$ mysqladmin ${MYSQLADMIN_FLAGS} drop testdb"
+eval mysqladmin ${MYSQLADMIN_FLAGS} drop --force testdb
+set -e
+
+echo "$ mysqladmin ${MYSQLADMIN_FLAGS} create testdb"
+eval mysqladmin ${MYSQLADMIN_FLAGS} create testdb
+
+
+# Execute command
+eval "$@"
+RC=$?
+
+# Kill server
+MYSQLD_PID="$(cat "${DBDIR}/mysqld.pid")"
+kill -15 ${MYSQLD_PID}
+[ $? -ne 0 ] && (echo 1>&1 "Could not kill mysql server"; exit 1)
+wait "${MYSQLD_PID}"
+exit $RC
Propchange: lnt/trunk/tests/SharedInputs/mysql_wrapper.sh
------------------------------------------------------------------------------
svn:executable = *
Modified: lnt/trunk/tests/lit.cfg
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/lit.cfg?rev=310665&r1=310664&r2=310665&view=diff
==============================================================================
--- lnt/trunk/tests/lit.cfg (original)
+++ lnt/trunk/tests/lit.cfg Thu Aug 10 14:38:39 2017
@@ -51,6 +51,11 @@ if lit_config.params.get('long', None):
if lit_config.params.get('postgres', None):
config.available_features.add('postgres')
+# Enable MySQL testing. This requires mysqladmin binary in PATH, and a server to be running
+# on localhost with user root/no password.
+if lit_config.params.get('mysql', None):
+ config.available_features.add('mysql')
+
config.available_features.add(platform.system())
# Enable coverage.py reporting, assuming the coverage module has been installed
Added: lnt/trunk/tests/lnttool/MySQLDB.shtest
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/lnttool/MySQLDB.shtest?rev=310665&view=auto
==============================================================================
--- lnt/trunk/tests/lnttool/MySQLDB.shtest (added)
+++ lnt/trunk/tests/lnttool/MySQLDB.shtest Thu Aug 10 14:38:39 2017
@@ -0,0 +1,12 @@
+# REQUIRES: mysql
+# RUN: rm -rf "%t.install"
+# RUN: %{shared_inputs}/mysql_wrapper.sh "%t.install" 9109 /bin/sh %s "%t.install" "%{shared_inputs}"
+set -eux
+
+TESTDIR="$1"
+SHARED_INPUTS="$2"
+
+lnt create "${TESTDIR}/instance" --db-dir mysql://root@localhost:9109 --default-db testdb --show-sql
+
+# Import a test set.
+#lnt import "${TESTDIR}/instance" "${SHARED_INPUTS}/sample-a-small.plist" --commit --show-sample-count
More information about the llvm-commits
mailing list