[llvm-commits] [zorg] r143473 - in /zorg/trunk: buildbot/osuosl/master/templates/ buildbot/osuosl/master/templates/root.html zorg/buildbot/changes/ zorg/buildbot/changes/__init__.py zorg/buildbot/changes/llvmpoller.py
Galina Kistanova
gkistanova at gmail.com
Tue Nov 1 11:41:13 PDT 2011
Author: gkistanova
Date: Tue Nov 1 13:41:13 2011
New Revision: 143473
URL: http://llvm.org/viewvc/llvm-project?rev=143473&view=rev
Log:
Added template for buildmaster home page and llvmpoller.py
Added:
zorg/trunk/buildbot/osuosl/master/templates/
zorg/trunk/buildbot/osuosl/master/templates/root.html
zorg/trunk/zorg/buildbot/changes/
zorg/trunk/zorg/buildbot/changes/__init__.py
zorg/trunk/zorg/buildbot/changes/llvmpoller.py
Added: zorg/trunk/buildbot/osuosl/master/templates/root.html
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/templates/root.html?rev=143473&view=auto
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/templates/root.html (added)
+++ zorg/trunk/buildbot/osuosl/master/templates/root.html Tue Nov 1 13:41:13 2011
@@ -0,0 +1,63 @@
+{% extends 'layout.html' %}
+{% import 'forms.html' as forms %}
+
+{% block content %}
+
+<h1>Welcome to the Buildbot
+{%- if title -%}
+ for the
+ {%- if title_url -%}
+ <a href="{{ title_url }}">{{ title }}</a>
+ {%- else -%}
+ {{ title }}
+ {%- endif -%}
+ project
+{%- endif -%}
+!
+</h1>
+
+<div class="column">
+
+<ul>
+ {% set item_class=cycler('alt', '') %}
+
+ <li class="{{ item_class.next() }}">The <a href="waterfall">Waterfall Display</a> will give you a
+ time-oriented summary of recent buildbot activity. <a href="waterfall/help">Waterfall Help.</a></li>
+
+ <li class="{{ item_class.next() }}">The <a href="grid">Grid Display</a> will give you a
+ developer-oriented summary of recent buildbot activity.</li>
+
+ <li class="{{ item_class.next() }}">The <a href="tgrid">Transposed Grid Display</a> presents
+ the same information as the grid, but lists the revisions down the side.</li>
+
+ <li class="{{ item_class.next() }}">The <a href="console">Console</a> presents
+ a user-oriented status page.</li>
+
+ <li class="{{ item_class.next() }}">The <a href="builders">Builders</a> and their most recent builds are
+ here.</li>
+
+ <li class="{{ item_class.next() }}"><a href="one_line_per_build">Recent Builds</a> are summarized here, one
+ per line.</li>
+
+ <li class="{{ item_class.next() }}"><a href="buildslaves">Buildslave</a> information</li>
+ <li class="{{ item_class.next() }}"><a href="changes">Changesource</a> information.</li>
+
+ <li class="{{ item_class.next() }}">How to <a href="http://llvm.org/docs/HowToAddABuilder.html">Add Your Build Configuration</a> to LLVM buildbot infrastructure.</li>
+
+ <li class="{{ item_class.next() }}"><a href="about">About</a> this Buildbot</li>
+</ul>
+
+{%- if authz.advertiseAction('cleanShutdown') -%}
+{%- if shutting_down -%}
+Master is shutting down<br/>
+{{ forms.cancel_clean_shutdown(cancel_shutdown_url, authz) }}
+{%- else -%}
+{{ forms.clean_shutdown(shutdown_url, authz) }}
+{%- endif -%}
+{%- endif -%}
+
+<p><i>Admin: Galina Kistanova gkistanova at gmail.com</i></p>
+
+</div>
+
+{% endblock %}
Added: zorg/trunk/zorg/buildbot/changes/__init__.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/changes/__init__.py?rev=143473&view=auto
==============================================================================
(empty)
Added: zorg/trunk/zorg/buildbot/changes/llvmpoller.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/zorg/buildbot/changes/llvmpoller.py?rev=143473&view=auto
==============================================================================
--- zorg/trunk/zorg/buildbot/changes/llvmpoller.py (added)
+++ zorg/trunk/zorg/buildbot/changes/llvmpoller.py Tue Nov 1 13:41:13 2011
@@ -0,0 +1,296 @@
+# LLVM buildbot needs to watch multiple projects within a single repository.
+
+# Based on the buildbot.changes.svnpoller.SVNPoller source code.
+
+from twisted.python import log
+from twisted.internet import defer, utils
+
+from buildbot import util
+from buildbot.changes import base
+
+import xml.dom.minidom
+import os, urllib, collections
+
+class LLVMPoller(base.PollingChangeSource, util.ComparableMixin):
+ """
+ Poll LLVM repository for changes and submit them to the change master.
+ Following Multiple Projects.
+ """
+
+ _svnurl="http://llvm.org/svn/llvm-project"
+ _revlinktmpl="http://llvm.org/viewvc/llvm-project/?view=rev&revision=%s"
+
+ compare_attrs = ["svnurl", "split_svn_path",
+ "svnuser", "svnpasswd",
+ "pollInterval", "histmax",
+ "svnbin", "category", "cachepath",
+ "projects"]
+
+ parent = None # filled in when we're added
+ last_change = None
+ loop = None
+
+ def __init__(self, svnurl=_svnurl, svnuser=None, svnpasswd=None,
+ pollInterval=2*60, histmax=10,
+ svnbin='svn', revlinktmpl=_revlinktmpl, category=None,
+ projects=None, cachepath=None):
+
+ # projects is a list of projects to watch or None to watch all.
+ if projects:
+ if isinstance(projects, str):
+ projects = [projects]
+ assert isinstance(projects, list)
+ assert len(projects) > 0
+ self.projects = projects
+
+ if svnurl.endswith("/"):
+ svnurl = svnurl[:-1] # strip the trailing slash.
+ self.svnurl = svnurl
+ self._prefix = svnurl # svnurl is the LLVM repository root.
+
+ self.svnuser = svnuser
+ self.svnpasswd = svnpasswd
+
+ self.revlinktmpl = revlinktmpl
+
+ self.environ = os.environ.copy() # include environment variables
+ # required for ssh-agent auth.
+
+ self.svnbin = svnbin
+ self.pollInterval = pollInterval
+ self.histmax = histmax
+ self.category = category
+
+ self.cachepath = cachepath
+ if self.cachepath and os.path.exists(self.cachepath):
+ try:
+ f = open(self.cachepath, "r")
+ self.last_change = int(f.read().strip())
+ log.msg("LLVMPoller(%s): Setting last_change to %s" % (self.svnurl, self.last_change))
+ f.close()
+ # try writing it, too
+ f = open(self.cachepath, "w")
+ f.write(str(self.last_change))
+ f.close()
+ except:
+ self.cachepath = None
+ log.msg(("LLVMPoller(%s): Cache file corrupt or unwriteable; " +
+ "skipping and not using") % self.svnurl)
+ log.err()
+
+ def describe(self):
+ return "LLVMPoller: watching %s" % self.svnurl
+
+ def poll(self):
+ # Return value is only used for unit testing.
+ if self.projects:
+ log.msg("LLVMPoller(%s): Polling %s projects" % (self.svnurl, self.projects))
+ else:
+ log.msg("LLVMPoller(%s): Polling all projects" % self.svnurl)
+
+ d = defer.succeed(None)
+
+ d.addCallback(self.get_logs)
+ d.addCallback(self.parse_logs)
+ d.addCallback(self.get_new_logentries)
+ d.addCallback(self.create_changes)
+ d.addCallback(self.submit_changes)
+ d.addCallback(self.finished_ok)
+ d.addErrback(log.err, 'LLVMPoller: Error in while polling') # eat errors
+
+ return d
+
+ def getProcessOutput(self, args):
+ # This exists so we can override it during the unit tests.
+ d = utils.getProcessOutput(self.svnbin, args, self.environ)
+ return d
+
+ def get_logs(self, _):
+ args = []
+ args.extend(["log", "--xml", "--verbose", "--non-interactive"])
+ if self.svnuser:
+ args.extend(["--username=%s" % self.svnuser])
+ if self.svnpasswd:
+ args.extend(["--password=%s" % self.svnpasswd])
+ args.extend(["--limit=%d" % (self.histmax), self.svnurl])
+ d = self.getProcessOutput(args)
+ return d
+
+ def parse_logs(self, output):
+ # Parse the XML output, return a list of <logentry> nodes.
+ try:
+ doc = xml.dom.minidom.parseString(output)
+ except xml.parsers.expat.ExpatError:
+ log.msg("LLVMPoller(%s): LLVMPoller.parse_logs: ExpatError in '%s'" % (self.svnurl, output))
+ raise
+ logentries = doc.getElementsByTagName("logentry")
+ return logentries
+
+ def get_new_logentries(self, logentries):
+ last_change = old_last_change = self.last_change
+
+ # Given a list of logentries, calculate new_last_change, and
+ # new_logentries, where new_logentries contains only the ones after
+ # last_change.
+
+ new_last_change = None
+ new_logentries = []
+ if logentries:
+ new_last_change = int(logentries[0].getAttribute("revision"))
+
+ if last_change is None:
+ # If this is the first time we've been run, ignore any changes
+ # that occurred before now. This prevents a build at every
+ # startup.
+ log.msg('LLVMPoller(%s): Starting at change %s' % (self.svnurl, new_last_change))
+ elif last_change == new_last_change:
+ # An unmodified repository will hit this case.
+ log.msg('LLVMPoller(%s): No changes' % self.svnurl)
+ assert len(new_logentries) == 0
+ return [] # No new logentries.
+ else:
+ for el in logentries:
+ if last_change == int(el.getAttribute("revision")):
+ break
+ new_logentries.append(el)
+ new_logentries.reverse() # Return the oldest first.
+
+ self.last_change = new_last_change
+ log.msg('LLVMPoller(%s): Last change set from %s to %s' %
+ (self.svnurl, old_last_change, new_last_change))
+ return new_logentries
+
+ def _get_text(self, element, tag_name):
+ try:
+ child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
+ text = "".join([t.data for t in child_nodes])
+ except:
+ text = "<unknown>"
+ return text
+
+ def _transform_path(self, path):
+ """
+ Parses the given path, and returns a three-entry tuple
+ (PROJECT, BRANCH, FILEPATH) if PROJECT is watched one,
+ or None otherwise.
+ """
+
+ relative_path = path
+ if relative_path.startswith(self._prefix):
+ relative_path = path[len(self._prefix):]
+ if relative_path.startswith("/"):
+ relative_path = relative_path[1:]
+
+ # turn llvm/trunk/lib/CodeGen/Analysis.cpp into
+ # ("llvm", "trunk", "lib/CodeGen/Analysis.cpp")
+ # llvm/branches/release_30/lib/CodeGen/Analysis.cpp into
+ # ("llvm", "branches/release_30", "lib/CodeGen/Analysis.cpp")
+ # and llvm/tags/RELEASE_30/rc1/lib/CodeGen/Analysis.cpp into
+ # ("llvm", "tags/RELEASE_30/rc1", "lib/CodeGen/Analysis.cpp")
+
+ pieces = relative_path.split('/')
+ project = pieces.pop(0)
+
+ if self.projects:
+ if project not in self.projects:
+ # Handle only projects we are watching.
+ return None
+
+ if pieces[0] == "trunk":
+ return (project, pieces[0], '/'.join(pieces[1:]))
+ elif pieces[0] == "branches":
+ return (project, '/'.join(pieces[0:2]), '/'.join(pieces[2:]))
+ elif pieces[0] == "tags":
+ return (project, '/'.join(pieces[0:3]), '/'.join(pieces[3:]))
+ else:
+ # Something we do not expect.
+ log.msg("LLVMPoller(%s) cannot parse the path (%s). Ignored." % (self.svnuurl, path))
+ return None
+
+ def create_changes(self, new_logentries):
+ changes = []
+
+ for el in new_logentries:
+ revision = str(el.getAttribute("revision"))
+
+ revlink = ''
+
+ if self.revlinktmpl:
+ if revision:
+ revlink = self.revlinktmpl % urllib.quote_plus(revision)
+
+ log.msg("LLVMPoller(%s): Adding change revision %s" % (self.svnurl, revision))
+ author = self._get_text(el, "author")
+ comments = self._get_text(el, "msg")
+ # there is a "date" field, but it provides localtime in the
+ # repository's timezone, whereas we care about buildmaster's
+ # localtime (since this will get used to position the boxes on
+ # the Waterfall display, etc). So ignore the date field, and
+ # addChange will fill in with the current time
+ branches = {}
+ try:
+ pathlist = el.getElementsByTagName("paths")[0]
+ except IndexError: # weird, we got an empty revision
+ log.msg("LLVMPoller(%s): Ignoring commit with no paths." % self.svnurl)
+ continue
+
+ for p in pathlist.getElementsByTagName("path"):
+ action = p.getAttribute("action")
+ path = "".join([t.data for t in p.childNodes])
+ # the rest of buildbot is certaily not yet ready to handle
+ # unicode filenames, because they get put in RemoteCommands
+ # which get sent via PB to the buildslave, and PB doesn't
+ # handle unicode.
+ path = path.encode("ascii")
+ if path.startswith("/"):
+ path = path[1:]
+ where = self._transform_path(path)
+
+ # if 'where' is None, the file was outside any project that
+ # we care about and we should ignore it.
+ if where:
+ assert len(where) == 3
+ project, branch, filename = where
+ if not branch in branches:
+ branches[branch] = {'files': []}
+ branches[branch]['files'].append(filename)
+
+ if not branches[branch].has_key('action'):
+ branches[branch]['action'] = action
+
+ for branch in branches.keys():
+ action = branches[branch]['action']
+ files = branches[branch]['files']
+ number_of_files_changed = len(files)
+
+ if action == u'D' and number_of_files_changed == 1 and files[0] == '':
+ log.msg("LLVMPoller(%s): Ignoring deletion of branch '%s'" % (self.svnurl, branch))
+ else:
+ chdict = dict(author=author,
+ files=files,
+ comments=comments,
+ revision=revision,
+ branch=branch,
+ revlink=revlink,
+ category=self.category,
+ repository=self.svnurl,
+ project=project)
+ changes.append(chdict)
+
+ return changes
+
+ @defer.deferredGenerator
+ def submit_changes(self, changes):
+ for chdict in changes:
+ wfd = defer.waitForDeferred(self.master.addChange(**chdict))
+ yield wfd
+ wfd.getResult()
+
+ def finished_ok(self, res):
+ if self.cachepath:
+ f = open(self.cachepath, "w")
+ f.write(str(self.last_change))
+ f.close()
+
+ log.msg("LLVMPoller: Finished polling with res %s" % res)
+ return res
More information about the llvm-commits
mailing list