[LNT] r263877 - [profile] Add the Profile DB table
James Molloy via llvm-commits
llvm-commits at lists.llvm.org
Sat Mar 19 07:46:21 PDT 2016
Author: jamesm
Date: Sat Mar 19 09:46:20 2016
New Revision: 263877
URL: http://llvm.org/viewvc/llvm-project?rev=263877&view=rev
Log:
[profile] Add the Profile DB table
This maps a Sample to a profile on disk. Also add the config file plumbing.
Added:
lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py
Modified:
lnt/trunk/lnt/lnttool/create.py
lnt/trunk/lnt/server/config.py
lnt/trunk/lnt/server/db/testsuitedb.py
lnt/trunk/lnt/testing/profile/profile.py
Modified: lnt/trunk/lnt/lnttool/create.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/lnttool/create.py?rev=263877&r1=263876&r2=263877&view=diff
==============================================================================
--- lnt/trunk/lnt/lnttool/create.py (original)
+++ lnt/trunk/lnt/lnttool/create.py Sat Mar 19 09:46:20 2016
@@ -32,6 +32,9 @@ tmp_dir = %(tmp_dir)r
# paths are resolved relative to the config path + this path.
db_dir = %(db_dir)r
+# Profile directory, where profiles are kept.
+profile_dir = %(profile_dir)r
+
# Secret key for this server instance.
secret_key = %(secret_key)r
@@ -98,6 +101,8 @@ def action_create(name, args):
help="name of the temp file directory [%default]")
parser.add_option("", "--db-dir", dest="db_dir", default="data",
help="name of the directory to hold databases")
+ parser.add_option("", "--profile-dir", dest="profile_dir", default="data/profiles",
+ help="name of the directory to hold databases")
parser.add_option("", "--default-db", dest="default_db", default="lnt.db",
help="name for the default db [%default]", metavar="NAME")
parser.add_option("", "--secret-key", dest="secret_key", default=None,
@@ -139,6 +144,7 @@ def action_create(name, args):
wsgi = opts.wsgi
tmp_dir = opts.tmp_dir
db_dir = opts.db_dir
+ profile_dir = opts.profile_dir
default_db = opts.default_db
hostname = opts.hostname
hostsuffix = opts.hostsuffix
Modified: lnt/trunk/lnt/server/config.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/config.py?rev=263877&r1=263876&r2=263877&view=diff
==============================================================================
--- lnt/trunk/lnt/server/config.py (original)
+++ lnt/trunk/lnt/server/config.py Sat Mar 19 09:46:20 2016
@@ -99,7 +99,8 @@ class Config:
default_email_config = EmailConfig(False, '', '', [])
dbDir = data.get('db_dir', '.')
-
+ profileDir = data.get('profile_dir', 'data/profiles')
+
# If the path does not contain database type, assume relative path.
dbDirPath = dbDir if "://" in dbDir else os.path.join(baseDir, dbDir)
@@ -109,7 +110,8 @@ class Config:
secretKey = data.get('secret_key', None)
return Config(data.get('name', 'LNT'), data['zorgURL'],
- dbDir, os.path.join(baseDir, tempDir), secretKey,
+ dbDir, os.path.join(baseDir, tempDir),
+ os.path.join(baseDir, profileDir), secretKey,
dict([(k,DBInfo.fromData(dbDirPath, v,
default_email_config,
0))
@@ -120,18 +122,20 @@ class Config:
baseDir = tempfile.mkdtemp()
dbDir = '.'
dbDirPath = os.path.join(baseDir, dbDir)
+ profileDirPath = os.path.join(baseDir, 'profiles')
tempDir = os.path.join(baseDir, 'tmp')
secretKey = None
dbInfo = {'dummy': DBInfo.dummyInstance()}
- return Config('LNT', 'http://localhost:8000', dbDir, tempDir, secretKey, dbInfo)
+ return Config('LNT', 'http://localhost:8000', dbDir, tempDir, profileDirPath, secretKey, dbInfo)
- def __init__(self, name, zorgURL, dbDir, tempDir, secretKey, databases):
+ def __init__(self, name, zorgURL, dbDir, tempDir, profileDir, secretKey, databases):
self.name = name
self.zorgURL = zorgURL
self.dbDir = dbDir
self.tempDir = tempDir
self.secretKey = secretKey
+ self.profileDir = profileDir
while self.zorgURL.endswith('/'):
self.zorgURL = zorgURL[:-1]
self.databases = databases
Added: lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py?rev=263877&view=auto
==============================================================================
--- lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py (added)
+++ lnt/trunk/lnt/server/db/migrations/upgrade_8_to_9.py Sat Mar 19 09:46:20 2016
@@ -0,0 +1,73 @@
+# Version 9 of the database updates Sample to add the profile field, and
+# adds Profiles.
+
+import os
+import sys
+
+import sqlalchemy
+from sqlalchemy import *
+from sqlalchemy.schema import Index
+from sqlalchemy.orm import relation
+
+# Import the original schema from upgrade_0_to_1 since upgrade_1_to_2 does not
+# change the actual schema, but rather adds functionality vis-a-vis orders.
+import lnt.server.db.migrations.upgrade_0_to_1 as upgrade_0_to_1
+import lnt.server.db.migrations.upgrade_2_to_3 as upgrade_2_to_3
+
+
+###
+# Upgrade TestSuite
+def add_profiles(test_suite):
+ """Given a test suite with a database connection and a test-suite
+ name, make the profile sqlalchemy database objects for that test-suite.
+ """
+ # Grab the Base for the previous schema so that we have all
+ # the definitions we need.
+ Base = upgrade_2_to_3.get_base(test_suite)
+ # Grab our db_key_name for our test suite so we can properly
+ # prefix our fields/table names.
+ db_key_name = test_suite.db_key_name
+
+ class Profile(Base):
+ __tablename__ = db_key_name + '_Profile'
+
+ id = Column("ID", Integer, primary_key=True)
+ created_time = Column("CreatedTime", DateTime)
+ accessed_time = Column("AccessedTime", DateTime)
+ filename = Column("Filename", String(256))
+ counters = Column("Counters", String(512))
+
+ return Base
+
+def upgrade_testsuite(engine, session, name):
+ # Grab Test Suite.
+ test_suite = session.query(upgrade_0_to_1.TestSuite).\
+ filter_by(name=name).first()
+ assert(test_suite is not None)
+ db_key_name = test_suite.db_key_name
+
+ # Add FieldChange to the test suite.
+ Base = add_profiles(test_suite)
+
+ session.connection().execute("""
+ALTER TABLE "%s_Sample"
+ADD COLUMN "ProfileID" INTEGER
+ """ % (db_key_name,))
+
+ # Create tables. We commit now since databases like Postgres run
+ # into deadlocking issues due to previous queries that we have run
+ # during the upgrade process. The commit closes all of the
+ # relevant transactions allowing us to then perform our upgrade.
+ session.commit()
+ Base.metadata.create_all(engine)
+ # Commit changes (also closing all relevant transactions with
+ # respect to Postgres like databases).
+ session.commit()
+
+def upgrade(engine):
+ # Create a session.
+ session = sqlalchemy.orm.sessionmaker(engine)()
+
+ # Create our FieldChangeField table and commit.
+ upgrade_testsuite(engine, session, 'nts')
+ upgrade_testsuite(engine, session, 'compile')
Modified: lnt/trunk/lnt/server/db/testsuitedb.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/db/testsuitedb.py?rev=263877&r1=263876&r2=263877&view=diff
==============================================================================
--- lnt/trunk/lnt/server/db/testsuitedb.py (original)
+++ lnt/trunk/lnt/server/db/testsuitedb.py Sat Mar 19 09:46:20 2016
@@ -7,12 +7,13 @@ suite metadata, so we only create the cl
import datetime
import json
+import os
import sqlalchemy
from sqlalchemy import *
import testsuite
-
+import lnt.testing.profile.profile as profile
def strip(obj):
"""Give back a dict without sqlalchemy stuff."""
@@ -327,6 +328,38 @@ class TestSuiteDB(object):
def __json__(self):
return strip(self.__dict__)
+ class Profile(self.base):
+ __tablename__ = db_key_name + '_Profile'
+
+ id = Column("ID", Integer, primary_key=True)
+ created_time = Column("CreatedTime", DateTime)
+ accessed_time = Column("AccessedTime", DateTime)
+ filename = Column("Filename", String(256))
+ counters = Column("Counters", String(512))
+
+ def __init__(self, encoded, config, testid):
+ self.created_time = datetime.datetime.now()
+ self.accessed_time = datetime.datetime.now()
+
+ p = profile.Profile.fromRendered(encoded)
+ if config is not None:
+ self.filename = p.save(profileDir=config.config.profileDir,
+ prefix='t-%s-s-' % testid)
+
+ s = ','.join('%s=%s' % (k,v)
+ for k,v in p.getTopLevelCounters().items())
+ self.counters = s[:512]
+
+ def getTopLevelCounters(self):
+ d = dict()
+ for i in self.counters.split('='):
+ k, v = i.split(',')
+ d[k] = v
+ return d
+
+ def load(self, profileDir):
+ return profile.Profile.fromFile(os.path.join(self.filename))
+
class Sample(self.base, ParameterizedMixin):
__tablename__ = db_key_name + '_Sample'
@@ -336,9 +369,11 @@ class TestSuiteDB(object):
# (Run(ID),Test(ID)) index we create below.
run_id = Column("RunID", Integer, ForeignKey(Run.id))
test_id = Column("TestID", Integer, ForeignKey(Test.id), index=True)
-
+ profile_id = Column("ProfileID", Integer, ForeignKey(Profile.id))
+
run = sqlalchemy.orm.relation(Run)
test = sqlalchemy.orm.relation(Test)
+ profile = sqlalchemy.orm.relation(Profile)
@staticmethod
def get_primary_fields():
@@ -556,6 +591,7 @@ class TestSuiteDB(object):
self.Machine = Machine
self.Run = Run
self.Test = Test
+ self.Profile = Profile
self.Sample = Sample
self.Order = Order
self.FieldChange = FieldChange
Modified: lnt/trunk/lnt/testing/profile/profile.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/testing/profile/profile.py?rev=263877&r1=263876&r2=263877&view=diff
==============================================================================
--- lnt/trunk/lnt/testing/profile/profile.py (original)
+++ lnt/trunk/lnt/testing/profile/profile.py Sat Mar 19 09:46:20 2016
@@ -1,4 +1,5 @@
import os, tempfile, base64
+import lnt.testing.profile
class Profile(object):
"""Profile objects hold a performance profile.
@@ -76,7 +77,7 @@ class Profile(object):
self.impl.serialize(tf.name)
# FIXME: make the returned filepath relative to baseDir?
- return tf.name
+ return os.path.relpath(tf.name, profileDir)
def render(self):
"""
More information about the llvm-commits
mailing list