[LNT] r263877 - [profile] Add the Profile DB table
James Molloy via llvm-commits
llvm-commits at lists.llvm.org
Thu May 5 03:47:41 PDT 2016
Hi Mehdi,
In an attempt to organise the work on LNT a bit better, I’ve raised PR27655 for this.
Cheers,
James
> On 30 Apr 2016, at 18:24, James Molloy via llvm-commits <llvm-commits at lists.llvm.org> wrote:
>
> Hi Medhi,
>
> Thanks for the report, I'll take a look when I return to work on Wednesday.
>
> James
>
>> On 30 Apr 2016, at 06:57, Mehdi Amini <mehdi.amini at apple.com> wrote:
>>
>> Hi James,
>>
>> This commit broke `lnt view-comparison` I think.
>>
>> --
>> Mehdi
>>
>>> On Mar 19, 2016, at 7:46 AM, James Molloy via llvm-commits <llvm-commits at lists.llvm.org> wrote:
>>>
>>> Author: jamesm
>>> Date: Sat Mar 19 09:46:20 2016
>>> New Revision: 263877
>>>
>>> URL: http://llvm.org/viewvc/llvm-project?rev=263877&view=rev
>>> Log:
>>> [profile] Add the Profile DB table
>>>
>>> This maps a Sample to a profile on disk. Also add the config file plumbing.
>>>
>>> Added:
>>> lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py
>>> Modified:
>>> lnt/trunk/lnt/lnttool/create.py
>>> lnt/trunk/lnt/server/config.py
>>> lnt/trunk/lnt/server/db/testsuitedb.py
>>> lnt/trunk/lnt/testing/profile/profile.py
>>>
>>> Modified: lnt/trunk/lnt/lnttool/create.py
>>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/lnttool/create.py?rev=263877&r1=263876&r2=263877&view=diff
>>> ==============================================================================
>>> --- lnt/trunk/lnt/lnttool/create.py (original)
>>> +++ lnt/trunk/lnt/lnttool/create.py Sat Mar 19 09:46:20 2016
>>> @@ -32,6 +32,9 @@ tmp_dir = %(tmp_dir)r
>>> # paths are resolved relative to the config path + this path.
>>> db_dir = %(db_dir)r
>>>
>>> +# Profile directory, where profiles are kept.
>>> +profile_dir = %(profile_dir)r
>>> +
>>> # Secret key for this server instance.
>>> secret_key = %(secret_key)r
>>>
>>> @@ -98,6 +101,8 @@ def action_create(name, args):
>>> help="name of the temp file directory [%default]")
>>> parser.add_option("", "--db-dir", dest="db_dir", default="data",
>>> help="name of the directory to hold databases")
>>> + parser.add_option("", "--profile-dir", dest="profile_dir", default="data/profiles",
>>> + help="name of the directory to hold databases")
>>> parser.add_option("", "--default-db", dest="default_db", default="lnt.db",
>>> help="name for the default db [%default]", metavar="NAME")
>>> parser.add_option("", "--secret-key", dest="secret_key", default=None,
>>> @@ -139,6 +144,7 @@ def action_create(name, args):
>>> wsgi = opts.wsgi
>>> tmp_dir = opts.tmp_dir
>>> db_dir = opts.db_dir
>>> + profile_dir = opts.profile_dir
>>> default_db = opts.default_db
>>> hostname = opts.hostname
>>> hostsuffix = opts.hostsuffix
>>>
>>> Modified: lnt/trunk/lnt/server/config.py
>>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/config.py?rev=263877&r1=263876&r2=263877&view=diff
>>> ==============================================================================
>>> --- lnt/trunk/lnt/server/config.py (original)
>>> +++ lnt/trunk/lnt/server/config.py Sat Mar 19 09:46:20 2016
>>> @@ -99,7 +99,8 @@ class Config:
>>> default_email_config = EmailConfig(False, '', '', [])
>>>
>>> dbDir = data.get('db_dir', '.')
>>> -
>>> + profileDir = data.get('profile_dir', 'data/profiles')
>>> +
>>> # If the path does not contain database type, assume relative path.
>>> dbDirPath = dbDir if "://" in dbDir else os.path.join(baseDir, dbDir)
>>>
>>> @@ -109,7 +110,8 @@ class Config:
>>> secretKey = data.get('secret_key', None)
>>>
>>> return Config(data.get('name', 'LNT'), data['zorgURL'],
>>> - dbDir, os.path.join(baseDir, tempDir), secretKey,
>>> + dbDir, os.path.join(baseDir, tempDir),
>>> + os.path.join(baseDir, profileDir), secretKey,
>>> dict([(k,DBInfo.fromData(dbDirPath, v,
>>> default_email_config,
>>> 0))
>>> @@ -120,18 +122,20 @@ class Config:
>>> baseDir = tempfile.mkdtemp()
>>> dbDir = '.'
>>> dbDirPath = os.path.join(baseDir, dbDir)
>>> + profileDirPath = os.path.join(baseDir, 'profiles')
>>> tempDir = os.path.join(baseDir, 'tmp')
>>> secretKey = None
>>> dbInfo = {'dummy': DBInfo.dummyInstance()}
>>>
>>> - return Config('LNT', 'http://localhost:8000', dbDir, tempDir, secretKey, dbInfo)
>>> + return Config('LNT', 'http://localhost:8000', dbDir, tempDir, profileDirPath, secretKey, dbInfo)
>>>
>>> - def __init__(self, name, zorgURL, dbDir, tempDir, secretKey, databases):
>>> + def __init__(self, name, zorgURL, dbDir, tempDir, profileDir, secretKey, databases):
>>> self.name = name
>>> self.zorgURL = zorgURL
>>> self.dbDir = dbDir
>>> self.tempDir = tempDir
>>> self.secretKey = secretKey
>>> + self.profileDir = profileDir
>>> while self.zorgURL.endswith('/'):
>>> self.zorgURL = zorgURL[:-1]
>>> self.databases = databases
>>>
>>> Added: lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py
>>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py?rev=263877&view=auto
>>> ==============================================================================
>>> --- lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py (added)
>>> +++ lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py Sat Mar 19 09:46:20 2016
>>> @@ -0,0 +1,73 @@
>>> +# Version 9 of the database updates Sample to add the profile field, and
>>> +# adds Profiles.
>>> +
>>> +import os
>>> +import sys
>>> +
>>> +import sqlalchemy
>>> +from sqlalchemy import *
>>> +from sqlalchemy.schema import Index
>>> +from sqlalchemy.orm import relation
>>> +
>>> +# Import the original schema from upgrade_0_to_1 since upgrade_1_to_2 does not
>>> +# change the actual schema, but rather adds functionality vis-a-vis orders.
>>> +import lnt.server.db.migrations.upgrade_0_to_1 as upgrade_0_to_1
>>> +import lnt.server.db.migrations.upgrade_2_to_3 as upgrade_2_to_3
>>> +
>>> +
>>> +###
>>> +# Upgrade TestSuite
>>> +def add_profiles(test_suite):
>>> + """Given a test suite with a database connection and a test-suite
>>> + name, make the profile sqlalchemy database objects for that test-suite.
>>> + """
>>> + # Grab the Base for the previous schema so that we have all
>>> + # the definitions we need.
>>> + Base = upgrade_2_to_3.get_base(test_suite)
>>> + # Grab our db_key_name for our test suite so we can properly
>>> + # prefix our fields/table names.
>>> + db_key_name = test_suite.db_key_name
>>> +
>>> + class Profile(Base):
>>> + __tablename__ = db_key_name + '_Profile'
>>> +
>>> + id = Column("ID", Integer, primary_key=True)
>>> + created_time = Column("CreatedTime", DateTime)
>>> + accessed_time = Column("AccessedTime", DateTime)
>>> + filename = Column("Filename", String(256))
>>> + counters = Column("Counters", String(512))
>>> +
>>> + return Base
>>> +
>>> +def upgrade_testsuite(engine, session, name):
>>> + # Grab Test Suite.
>>> + test_suite = session.query(upgrade_0_to_1.TestSuite).\
>>> + filter_by(name=name).first()
>>> + assert(test_suite is not None)
>>> + db_key_name = test_suite.db_key_name
>>> +
>>> + # Add FieldChange to the test suite.
>>> + Base = add_profiles(test_suite)
>>> +
>>> + session.connection().execute("""
>>> +ALTER TABLE "%s_Sample"
>>> +ADD COLUMN "ProfileID" INTEGER
>>> + """ % (db_key_name,))
>>> +
>>> + # Create tables. We commit now since databases like Postgres run
>>> + # into deadlocking issues due to previous queries that we have run
>>> + # during the upgrade process. The commit closes all of the
>>> + # relevant transactions allowing us to then perform our upgrade.
>>> + session.commit()
>>> + Base.metadata.create_all(engine)
>>> + # Commit changes (also closing all relevant transactions with
>>> + # respect to Postgres like databases).
>>> + session.commit()
>>> +
>>> +def upgrade(engine):
>>> + # Create a session.
>>> + session = sqlalchemy.orm.sessionmaker(engine)()
>>> +
>>> + # Create our FieldChangeField table and commit.
>>> + upgrade_testsuite(engine, session, 'nts')
>>> + upgrade_testsuite(engine, session, 'compile')
>>>
>>> Modified: lnt/trunk/lnt/server/db/testsuitedb.py
>>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/testsuitedb.py?rev=263877&r1=263876&r2=263877&view=diff
>>> ==============================================================================
>>> --- lnt/trunk/lnt/server/db/testsuitedb.py (original)
>>> +++ lnt/trunk/lnt/server/db/testsuitedb.py Sat Mar 19 09:46:20 2016
>>> @@ -7,12 +7,13 @@ suite metadata, so we only create the cl
>>>
>>> import datetime
>>> import json
>>> +import os
>>>
>>> import sqlalchemy
>>> from sqlalchemy import *
>>>
>>> import testsuite
>>> -
>>> +import lnt.testing.profile.profile as profile
>>>
>>> def strip(obj):
>>> """Give back a dict without sqlalchemy stuff."""
>>> @@ -327,6 +328,38 @@ class TestSuiteDB(object):
>>> def __json__(self):
>>> return strip(self.__dict__)
>>>
>>> + class Profile(self.base):
>>> + __tablename__ = db_key_name + '_Profile'
>>> +
>>> + id = Column("ID", Integer, primary_key=True)
>>> + created_time = Column("CreatedTime", DateTime)
>>> + accessed_time = Column("AccessedTime", DateTime)
>>> + filename = Column("Filename", String(256))
>>> + counters = Column("Counters", String(512))
>>> +
>>> + def __init__(self, encoded, config, testid):
>>> + self.created_time = datetime.datetime.now()
>>> + self.accessed_time = datetime.datetime.now()
>>> +
>>> + p = profile.Profile.fromRendered(encoded)
>>> + if config is not None:
>>> + self.filename = p.save(profileDir=config.config.profileDir,
>>> + prefix='t-%s-s-' % testid)
>>> +
>>> + s = ','.join('%s=%s' % (k,v)
>>> + for k,v in p.getTopLevelCounters().items())
>>> + self.counters = s[:512]
>>> +
>>> + def getTopLevelCounters(self):
>>> + d = dict()
>>> + for i in self.counters.split('='):
>>> + k, v = i.split(',')
>>> + d[k] = v
>>> + return d
>>> +
>>> + def load(self, profileDir):
>>> + return profile.Profile.fromFile(os.path.join(self.filename))
>>> +
>>> class Sample(self.base, ParameterizedMixin):
>>> __tablename__ = db_key_name + '_Sample'
>>>
>>> @@ -336,9 +369,11 @@ class TestSuiteDB(object):
>>> # (Run(ID),Test(ID)) index we create below.
>>> run_id = Column("RunID", Integer, ForeignKey(Run.id))
>>> test_id = Column("TestID", Integer, ForeignKey(Test.id), index=True)
>>> -
>>> + profile_id = Column("ProfileID", Integer, ForeignKey(Profile.id))
>>> +
>>> run = sqlalchemy.orm.relation(Run)
>>> test = sqlalchemy.orm.relation(Test)
>>> + profile = sqlalchemy.orm.relation(Profile)
>>>
>>> @staticmethod
>>> def get_primary_fields():
>>> @@ -556,6 +591,7 @@ class TestSuiteDB(object):
>>> self.Machine = Machine
>>> self.Run = Run
>>> self.Test = Test
>>> + self.Profile = Profile
>>> self.Sample = Sample
>>> self.Order = Order
>>> self.FieldChange = FieldChange
>>>
>>> Modified: lnt/trunk/lnt/testing/profile/profile.py
>>> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/testing/profile/profile.py?rev=263877&r1=263876&r2=263877&view=diff
>>> ==============================================================================
>>> --- lnt/trunk/lnt/testing/profile/profile.py (original)
>>> +++ lnt/trunk/lnt/testing/profile/profile.py Sat Mar 19 09:46:20 2016
>>> @@ -1,4 +1,5 @@
>>> import os, tempfile, base64
>>> +import lnt.testing.profile
>>>
>>> class Profile(object):
>>> """Profile objects hold a performance profile.
>>> @@ -76,7 +77,7 @@ class Profile(object):
>>> self.impl.serialize(tf.name)
>>>
>>> # FIXME: make the returned filepath relative to baseDir?
>>> - return tf.name
>>> + return os.path.relpath(tf.name, profileDir)
>>>
>>> def render(self):
>>> """
>>>
>>>
>>> _______________________________________________
>>> llvm-commits mailing list
>>> llvm-commits at lists.llvm.org
>>> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
>>
> IMPORTANT NOTICE: The contents of this email and any attachments are confidential and may also be privileged. If you are not the intended recipient, please notify the sender immediately and do not disclose the contents to any other person, use it for any purpose, or store or copy the information in any medium. Thank you.
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list