[LNT] r263877 - [profile] Add the Profile DB table
Mehdi Amini via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 29 21:57:19 PDT 2016
Hi James,
This commit broke `lnt view-comparison` I think.
--
Mehdi
> On Mar 19, 2016, at 7:46 AM, James Molloy via llvm-commits <llvm-commits at lists.llvm.org> wrote:
>
> Author: jamesm
> Date: Sat Mar 19 09:46:20 2016
> New Revision: 263877
>
> URL: http://llvm.org/viewvc/llvm-project?rev=263877&view=rev
> Log:
> [profile] Add the Profile DB table
>
> This maps a Sample to a profile on disk. Also add the config file plumbing.
>
> Added:
> lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py
> Modified:
> lnt/trunk/lnt/lnttool/create.py
> lnt/trunk/lnt/server/config.py
> lnt/trunk/lnt/server/db/testsuitedb.py
> lnt/trunk/lnt/testing/profile/profile.py
>
> Modified: lnt/trunk/lnt/lnttool/create.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/lnttool/create.py?rev=263877&r1=263876&r2=263877&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/lnttool/create.py (original)
> +++ lnt/trunk/lnt/lnttool/create.py Sat Mar 19 09:46:20 2016
> @@ -32,6 +32,9 @@ tmp_dir = %(tmp_dir)r
> # paths are resolved relative to the config path + this path.
> db_dir = %(db_dir)r
>
> +# Profile directory, where profiles are kept.
> +profile_dir = %(profile_dir)r
> +
> # Secret key for this server instance.
> secret_key = %(secret_key)r
>
> @@ -98,6 +101,8 @@ def action_create(name, args):
> help="name of the temp file directory [%default]")
> parser.add_option("", "--db-dir", dest="db_dir", default="data",
> help="name of the directory to hold databases")
> + parser.add_option("", "--profile-dir", dest="profile_dir", default="data/profiles",
> + help="name of the directory to hold databases")
> parser.add_option("", "--default-db", dest="default_db", default="lnt.db",
> help="name for the default db [%default]", metavar="NAME")
> parser.add_option("", "--secret-key", dest="secret_key", default=None,
> @@ -139,6 +144,7 @@ def action_create(name, args):
> wsgi = opts.wsgi
> tmp_dir = opts.tmp_dir
> db_dir = opts.db_dir
> + profile_dir = opts.profile_dir
> default_db = opts.default_db
> hostname = opts.hostname
> hostsuffix = opts.hostsuffix
>
> Modified: lnt/trunk/lnt/server/config.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/config.py?rev=263877&r1=263876&r2=263877&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/config.py (original)
> +++ lnt/trunk/lnt/server/config.py Sat Mar 19 09:46:20 2016
> @@ -99,7 +99,8 @@ class Config:
> default_email_config = EmailConfig(False, '', '', [])
>
> dbDir = data.get('db_dir', '.')
> -
> + profileDir = data.get('profile_dir', 'data/profiles')
> +
> # If the path does not contain database type, assume relative path.
> dbDirPath = dbDir if "://" in dbDir else os.path.join(baseDir, dbDir)
>
> @@ -109,7 +110,8 @@ class Config:
> secretKey = data.get('secret_key', None)
>
> return Config(data.get('name', 'LNT'), data['zorgURL'],
> - dbDir, os.path.join(baseDir, tempDir), secretKey,
> + dbDir, os.path.join(baseDir, tempDir),
> + os.path.join(baseDir, profileDir), secretKey,
> dict([(k,DBInfo.fromData(dbDirPath, v,
> default_email_config,
> 0))
> @@ -120,18 +122,20 @@ class Config:
> baseDir = tempfile.mkdtemp()
> dbDir = '.'
> dbDirPath = os.path.join(baseDir, dbDir)
> + profileDirPath = os.path.join(baseDir, 'profiles')
> tempDir = os.path.join(baseDir, 'tmp')
> secretKey = None
> dbInfo = {'dummy': DBInfo.dummyInstance()}
>
> - return Config('LNT', 'http://localhost:8000', dbDir, tempDir, secretKey, dbInfo)
> + return Config('LNT', 'http://localhost:8000', dbDir, tempDir, profileDirPath, secretKey, dbInfo)
>
> - def __init__(self, name, zorgURL, dbDir, tempDir, secretKey, databases):
> + def __init__(self, name, zorgURL, dbDir, tempDir, profileDir, secretKey, databases):
> self.name = name
> self.zorgURL = zorgURL
> self.dbDir = dbDir
> self.tempDir = tempDir
> self.secretKey = secretKey
> + self.profileDir = profileDir
> while self.zorgURL.endswith('/'):
> self.zorgURL = zorgURL[:-1]
> self.databases = databases
>
> Added: lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py?rev=263877&view=auto
> ==============================================================================
> --- lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py (added)
> +++ lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py Sat Mar 19 09:46:20 2016
> @@ -0,0 +1,73 @@
> +# Version 9 of the database updates Sample to add the profile field, and
> +# adds Profiles.
> +
> +import os
> +import sys
> +
> +import sqlalchemy
> +from sqlalchemy import *
> +from sqlalchemy.schema import Index
> +from sqlalchemy.orm import relation
> +
> +# Import the original schema from upgrade_0_to_1 since upgrade_1_to_2 does not
> +# change the actual schema, but rather adds functionality vis-a-vis orders.
> +import lnt.server.db.migrations.upgrade_0_to_1 as upgrade_0_to_1
> +import lnt.server.db.migrations.upgrade_2_to_3 as upgrade_2_to_3
> +
> +
> +###
> +# Upgrade TestSuite
> +def add_profiles(test_suite):
> + """Given a test suite with a database connection and a test-suite
> + name, make the profile sqlalchemy database objects for that test-suite.
> + """
> + # Grab the Base for the previous schema so that we have all
> + # the definitions we need.
> + Base = upgrade_2_to_3.get_base(test_suite)
> + # Grab our db_key_name for our test suite so we can properly
> + # prefix our fields/table names.
> + db_key_name = test_suite.db_key_name
> +
> + class Profile(Base):
> + __tablename__ = db_key_name + '_Profile'
> +
> + id = Column("ID", Integer, primary_key=True)
> + created_time = Column("CreatedTime", DateTime)
> + accessed_time = Column("AccessedTime", DateTime)
> + filename = Column("Filename", String(256))
> + counters = Column("Counters", String(512))
> +
> + return Base
> +
> +def upgrade_testsuite(engine, session, name):
> + # Grab Test Suite.
> + test_suite = session.query(upgrade_0_to_1.TestSuite).\
> + filter_by(name=name).first()
> + assert(test_suite is not None)
> + db_key_name = test_suite.db_key_name
> +
> + # Add FieldChange to the test suite.
> + Base = add_profiles(test_suite)
> +
> + session.connection().execute("""
> +ALTER TABLE "%s_Sample"
> +ADD COLUMN "ProfileID" INTEGER
> + """ % (db_key_name,))
> +
> + # Create tables. We commit now since databases like Postgres run
> + # into deadlocking issues due to previous queries that we have run
> + # during the upgrade process. The commit closes all of the
> + # relevant transactions allowing us to then perform our upgrade.
> + session.commit()
> + Base.metadata.create_all(engine)
> + # Commit changes (also closing all relevant transactions with
> + # respect to Postgres like databases).
> + session.commit()
> +
> +def upgrade(engine):
> + # Create a session.
> + session = sqlalchemy.orm.sessionmaker(engine)()
> +
> + # Create our FieldChangeField table and commit.
> + upgrade_testsuite(engine, session, 'nts')
> + upgrade_testsuite(engine, session, 'compile')
>
> Modified: lnt/trunk/lnt/server/db/testsuitedb.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/testsuitedb.py?rev=263877&r1=263876&r2=263877&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/db/testsuitedb.py (original)
> +++ lnt/trunk/lnt/server/db/testsuitedb.py Sat Mar 19 09:46:20 2016
> @@ -7,12 +7,13 @@ suite metadata, so we only create the cl
>
> import datetime
> import json
> +import os
>
> import sqlalchemy
> from sqlalchemy import *
>
> import testsuite
> -
> +import lnt.testing.profile.profile as profile
>
> def strip(obj):
> """Give back a dict without sqlalchemy stuff."""
> @@ -327,6 +328,38 @@ class TestSuiteDB(object):
> def __json__(self):
> return strip(self.__dict__)
>
> + class Profile(self.base):
> + __tablename__ = db_key_name + '_Profile'
> +
> + id = Column("ID", Integer, primary_key=True)
> + created_time = Column("CreatedTime", DateTime)
> + accessed_time = Column("AccessedTime", DateTime)
> + filename = Column("Filename", String(256))
> + counters = Column("Counters", String(512))
> +
> + def __init__(self, encoded, config, testid):
> + self.created_time = datetime.datetime.now()
> + self.accessed_time = datetime.datetime.now()
> +
> + p = profile.Profile.fromRendered(encoded)
> + if config is not None:
> + self.filename = p.save(profileDir=config.config.profileDir,
> + prefix='t-%s-s-' % testid)
> +
> + s = ','.join('%s=%s' % (k,v)
> + for k,v in p.getTopLevelCounters().items())
> + self.counters = s[:512]
> +
> + def getTopLevelCounters(self):
> + d = dict()
> + for i in self.counters.split('='):
> + k, v = i.split(',')
> + d[k] = v
> + return d
> +
> + def load(self, profileDir):
> + return profile.Profile.fromFile(os.path.join(self.filename))
> +
> class Sample(self.base, ParameterizedMixin):
> __tablename__ = db_key_name + '_Sample'
>
> @@ -336,9 +369,11 @@ class TestSuiteDB(object):
> # (Run(ID),Test(ID)) index we create below.
> run_id = Column("RunID", Integer, ForeignKey(Run.id))
> test_id = Column("TestID", Integer, ForeignKey(Test.id), index=True)
> -
> + profile_id = Column("ProfileID", Integer, ForeignKey(Profile.id))
> +
> run = sqlalchemy.orm.relation(Run)
> test = sqlalchemy.orm.relation(Test)
> + profile = sqlalchemy.orm.relation(Profile)
>
> @staticmethod
> def get_primary_fields():
> @@ -556,6 +591,7 @@ class TestSuiteDB(object):
> self.Machine = Machine
> self.Run = Run
> self.Test = Test
> + self.Profile = Profile
> self.Sample = Sample
> self.Order = Order
> self.FieldChange = FieldChange
>
> Modified: lnt/trunk/lnt/testing/profile/profile.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/testing/profile/profile.py?rev=263877&r1=263876&r2=263877&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/testing/profile/profile.py (original)
> +++ lnt/trunk/lnt/testing/profile/profile.py Sat Mar 19 09:46:20 2016
> @@ -1,4 +1,5 @@
> import os, tempfile, base64
> +import lnt.testing.profile
>
> class Profile(object):
> """Profile objects hold a performance profile.
> @@ -76,7 +77,7 @@ class Profile(object):
> self.impl.serialize(tf.name)
>
> # FIXME: make the returned filepath relative to baseDir?
> - return tf.name
> + return os.path.relpath(tf.name, profileDir)
>
> def render(self):
> """
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list