[llvm-commits] [zorg] r99158 - in /zorg/trunk/lnt: README.txt lnt/import/ImportData lnt/import/NTEmailReport.py lnt/import/ServerUtil.py lnt/viewer/root.ptl tests/Web/WebSubmit.py
Daniel Dunbar
daniel at zuster.org
Sun Mar 21 17:55:38 PDT 2010
Author: ddunbar
Date: Sun Mar 21 19:55:38 2010
New Revision: 99158
URL: http://llvm.org/viewvc/llvm-project?rev=99158&view=rev
Log:
LNT: Add support for submitting test results directly (instead of in a file), and eliminate urllib2_file dependency.
Also, tweak email reports to include a bit more information about things like submitting runs with commit=0, or submitting runs that end up being duplicates.
Added:
zorg/trunk/lnt/tests/Web/WebSubmit.py
Modified:
zorg/trunk/lnt/README.txt
zorg/trunk/lnt/lnt/import/ImportData
zorg/trunk/lnt/lnt/import/NTEmailReport.py
zorg/trunk/lnt/lnt/import/ServerUtil.py
zorg/trunk/lnt/lnt/viewer/root.ptl
Modified: zorg/trunk/lnt/README.txt
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/README.txt?rev=99158&r1=99157&r2=99158&view=diff
==============================================================================
--- zorg/trunk/lnt/README.txt (original)
+++ zorg/trunk/lnt/README.txt Sun Mar 21 19:55:38 2010
@@ -28,8 +28,7 @@
Installation Instructions
-------------------------
-External Dependencies: SQLAlchemy, Quixote, mod_wsgi, SQLite,
- MySQL (optional), urllib2_file
+External Dependencies: SQLAlchemy, Quixote, mod_wsgi, SQLite, MySQL (optional)
Internal Dependencies: MooTools
Modified: zorg/trunk/lnt/lnt/import/ImportData
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/import/ImportData?rev=99158&r1=99157&r2=99158&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/import/ImportData (original)
+++ zorg/trunk/lnt/lnt/import/ImportData Sun Mar 21 19:55:38 2010
@@ -41,6 +41,8 @@
default=False)
parser.add_option("", "--show-sample-count", dest="showSampleCount",
action="store_true", default=False)
+ parser.add_option("", "--always-email", dest="alwaysEmail",
+ action="store_true", default=False)
opts,args = parser.parse_args()
if len(args) < 2:
@@ -52,9 +54,13 @@
db = PerfDB.PerfDB(dbpath, echo=opts.showSQL)
importFiles(db, args[1:])
if opts.commit:
+ print 'COMMITTING RESULT:',
db.commit()
+ print 'DONE'
else:
+ print 'DISCARDING RESULT:',
db.rollback()
+ print 'DONE'
print 'TOTAL IMPORT TIME: %.2fs' % (time.time() - startTime,)
def importFiles(db, files):
@@ -99,15 +105,17 @@
print " END : %s" % (run.end_time, )
for ri in run.info.values():
print " INFO : %r = %r" % (ri.key, ri.value)
- continue
- else:
- if opts.emailOnImport:
- import NTEmailReport
- NTEmailReport.emailReport(db, run,
- opts.emailReportURL,
- opts.emailReportHost,
- opts.emailReportFrom,
- opts.emailReportTo)
+
+ if opts.emailOnImport and (success or opts.alwaysEmail):
+ import NTEmailReport
+ print "\nMAILING RESULTS TO: %r\n" % opts.emailReportTo
+ NTEmailReport.emailReport(db, run,
+ opts.emailReportURL,
+ opts.emailReportHost,
+ opts.emailReportFrom,
+ opts.emailReportTo,
+ success,
+ opts.commit)
print "ADDED: %d machines" % (db.getNumMachines() - numMachines,)
print "ADDED: %d runs" % (db.getNumRuns() - numRuns,)
Modified: zorg/trunk/lnt/lnt/import/NTEmailReport.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/import/NTEmailReport.py?rev=99158&r1=99157&r2=99158&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/import/NTEmailReport.py (original)
+++ zorg/trunk/lnt/lnt/import/NTEmailReport.py Sun Mar 21 19:55:38 2010
@@ -32,10 +32,11 @@
emailReport(db, run, baseurl, host, from_, to)
-def emailReport(db, run, baseurl, host, from_, to):
+def emailReport(db, run, baseurl, host, from_, to, was_added=True,
+ will_commit=True):
import email.mime.text
- subject, report = getReport(db, run, baseurl)
+ subject, report = getReport(db, run, baseurl, was_added, will_commit)
msg = email.mime.text.MIMEText(report)
msg['Subject'] = subject
@@ -61,7 +62,7 @@
best = r
return best
-def getReport(db, run, baseurl):
+def getReport(db, run, baseurl, was_added, will_commit):
report = StringIO.StringIO()
machine = run.machine
@@ -159,15 +160,29 @@
changes[name] = (testname, curValue, prevValue, pctDelta)
+ if will_commit:
+ if not was_added:
+ print >>report, ("*** NOTE ***: This was a duplicate submission, "
+ "and did not modify the database.\n")
+ else:
+ if was_added:
+ print >>report, ("*** NOTE ***: This is a test submission, "
+ "it will not be committed to the database.\n")
+ else:
+ print >>report, ("*** NOTE ***: This is a test submission, "
+ "and was a duplicate of an existing run.\n")
+
if baseurl[-1] == '/':
baseurl = baseurl[:-1]
print >>report, """%s/%d/""" % (baseurl, run.id)
print >>report, """Name: %s""" % (machine.info['name'].value,)
print >>report, """Nickname: %s:%d""" % (machine.name, machine.number)
print >>report
- print >>report, """Run: %d, Start Time: %s, End Time: %s""" % (run.id, run.start_time, run.end_time)
+ print >>report, """Run: %d, Start Time: %s, End Time: %s""" % (
+ run.id, run.start_time, run.end_time)
if compareTo:
- print >>report, """Comparing To: %d, Start Time: %s, End Time: %s""" % (compareTo.id, compareTo.start_time, compareTo.end_time)
+ print >>report, """Comparing To: %d, Start Time: %s, End Time: %s""" % (
+ compareTo.id, compareTo.start_time, compareTo.end_time)
if compareCrossesMachine:
print >>report, """*** WARNING ***:""",
print >>report, """comparison is against a different machine""",
@@ -210,7 +225,8 @@
for name,changelist in changes.items():
print >>report, """%s:""" % name
for name,curValue,prevValue,delta in Util.sorted(changelist):
- print >>report, """ %s: %.2f%% (%.4f => %.4f)""" % (name, delta*100, prevValue, curValue)
+ print >>report, """ %s: %.2f%% (%.4f => %.4f)""" % (
+ name, delta*100, prevValue, curValue)
# FIXME: Where is the old mailer getting the arch from?
subject = """%s nightly tester results""" % machine.name
Modified: zorg/trunk/lnt/lnt/import/ServerUtil.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/import/ServerUtil.py?rev=99158&r1=99157&r2=99158&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/import/ServerUtil.py (original)
+++ zorg/trunk/lnt/lnt/import/ServerUtil.py Sun Mar 21 19:55:38 2010
@@ -5,14 +5,16 @@
import plistlib
import urllib
import urllib2
-import urllib2_file
def submitFiles(url, files, commit):
for file in files:
- data = { 'file' : open(file),
- 'commit' : ("0","1")[not not commit] }
+ f = open(file, 'rb')
+ values = { 'input_data' : f.read(),
+ 'commit' : ("0","1")[not not commit] }
+ f.close()
- response = urllib2.urlopen(url, data)
+ data = urllib.urlencode(values)
+ response = urllib2.urlopen(urllib2.Request(url, data))
the_page = response.read()
print the_page
Modified: zorg/trunk/lnt/lnt/viewer/root.ptl
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/lnt/viewer/root.ptl?rev=99158&r1=99157&r2=99158&view=diff
==============================================================================
--- zorg/trunk/lnt/lnt/viewer/root.ptl (original)
+++ zorg/trunk/lnt/lnt/viewer/root.ptl Sun Mar 21 19:55:38 2010
@@ -11,6 +11,7 @@
import quixote
import quixote.form
import quixote.form.css
+import quixote.errors
from quixote import get_response
from quixote.directory import Directory, Resolving
from quixote.util import StaticDirectory
@@ -271,7 +272,9 @@
def submitRun(self):
form = quixote.form.Form(enctype="multipart/form-data")
form.add(quixote.form.FileWidget, "file",
- title="Input File (plist)", required=True)
+ title="Input File (plist)")
+ form.add(quixote.form.TextWidget, "input_data",
+ title="Input Data (plist)")
form.add(quixote.form.SingleSelectWidget, "commit",
title="Commit", value="0",
options=["0", "1"], required=True)
@@ -285,16 +288,29 @@
def process():
import plistlib
import tempfile
- fileWidget = form.get_widget('file')
- value = fileWidget.parse()
- data = value.fp.read()
- value.fp.close()
+ # Get the input data.
+ input_file = form.get_widget('file')
+ file_value = input_file.parse()
+
+ input_data = form.get_widget('input_data')
+ data_value = input_data.parse()
+
+ if ((file_value is None and data_value is None) or
+ (file_value is not None and data_value is not None)):
+ raise quixote.errors.QueryError(
+ "Must supply either an input file or input text data")
+
+ if file_value is not None:
+ data_value = file_value.fp.read()
+ file_value.fp.close()
+
+ # Stash a copy of the submission.
prefix = time.strftime("data-%Y-%m-%d_%H-%M-%S")
fd,path = tempfile.mkstemp(prefix=prefix,
suffix='.plist',
dir=self.config.tempDir)
- os.write(fd, data)
+ os.write(fd, data_value)
os.close(fd)
# Find the email address for this machine's results.
@@ -333,13 +349,13 @@
"--email-host=%s" % self.config.ntEmailHost,
"--email-from=%s" % self.config.ntEmailFrom,
"--email-to=%s" % toAddress,
+ "--always-email",
self.dbInfo.path,
path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout,stderr = p.communicate(None)
res = p.wait()
- stdout += "\nMAILING RESULTS TO: %r\n" % toAddress
return res,stdout,stderr
def result [plain] ():
Added: zorg/trunk/lnt/tests/Web/WebSubmit.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/lnt/tests/Web/WebSubmit.py?rev=99158&view=auto
==============================================================================
--- zorg/trunk/lnt/tests/Web/WebSubmit.py (added)
+++ zorg/trunk/lnt/tests/Web/WebSubmit.py Sun Mar 21 19:55:38 2010
@@ -0,0 +1,19 @@
+# RUN: %src_root/lnt/import/SubmitData %base_url/submitRun \
+# RUN: ../DB/Inputs/sample-a-small.plist > %t.log
+# RUN: FileCheck %s < %t.log
+
+# CHECK: STATUS: 0
+
+# CHECK: OUTPUT:
+# CHECK: IMPORT: {{.*}}/lnt_tmp/{{.*}}.plist
+# CHECK: LOAD TIME: 0.03s
+# CHECK: IMPORT TIME: 0.03s
+# CHECK: MACHINE: 107
+# CHECK: START : {{.*}}
+# CHECK: END : {{.*}}
+# CHECK: INFO : u'tag' = u'nightlytest'
+# CHECK: MAILING RESULTS TO: {{.*}}
+# CHECK: ADDED: {{[0-9]*}} machines
+# CHECK: ADDED: {{[0-9]*}} runs
+# CHECK: ADDED: {{[0-9]*}} tests
+# CHECK: DISCARDING RESULT: DONE
More information about the llvm-commits
mailing list