[Buildbot-commits] buildbot/buildbot/changes svnpoller.py,1.1,1.2
Brian Warner
warner at users.sourceforge.net
Mon Oct 2 00:13:20 UTC 2006
Update of /cvsroot/buildbot/buildbot/buildbot/changes
In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17137/buildbot/changes
Modified Files:
svnpoller.py
Log Message:
[project @ svnpoller: lots of work, make branches behave properly, write lots of tests, docs]
Original author: warner at lothar.com
Date: 2006-10-01 23:23:25
Index: svnpoller.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/svnpoller.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- svnpoller.py 2 Oct 2006 00:13:11 -0000 1.1
+++ svnpoller.py 2 Oct 2006 00:13:18 -0000 1.2
@@ -2,12 +2,12 @@
# Based on the work of Dave Peticolas for the P4poll
# Changed to svn (using xml.dom.minidom) by Niklaus Giger
+# Hacked beyond recognition by Brian Warner
import time
-from twisted.python import log, failure
-from twisted.internet import defer, reactor
-from twisted.internet.utils import getProcessOutput
+from twisted.python import log
+from twisted.internet import defer, reactor, utils
from twisted.internet.task import LoopingCall
from buildbot import util
@@ -31,21 +31,25 @@
return (None, path)
def split_file_branches(path):
+ # turn trunk/subdir/file.c into (None, "subdir/file.c")
+ # and branches/1.5.x/subdir/file.c into ("branches/1.5.x", "subdir/file.c")
pieces = path.split('/')
if pieces[0] == 'trunk':
return (None, '/'.join(pieces[1:]))
elif pieces[0] == 'branches':
- return (pieces[1], '/'.join(pieces[2:]))
+ return ('/'.join(pieces[0:2]), '/'.join(pieces[2:]))
else:
return None
class SvnSource(base.ChangeSource, util.ComparableMixin):
- """This source will poll a perforce repository for changes and submit
+ """This source will poll a Subversion repository for changes and submit
them to the change master."""
- compare_attrs = ["svnuser", "svnpasswd", "svnurl",
- "pollinterval", "histmax"]
+ compare_attrs = ["svnurl", "split_file_function",
+ "svnuser", "svnpasswd",
+ "pollinterval", "histmax",
+ "svnbin"]
parent = None # filled in when we're added
last_change = None
@@ -54,8 +58,8 @@
def __init__(self, svnurl, split_file=None,
svnuser=None, svnpasswd=None,
- svnbin='svn',
- pollinterval=10*60, histmax=100):
+ pollinterval=10*60, histmax=100,
+ svnbin='svn'):
"""
@type svnurl: string
@param svnurl: the SVN URL that describes the repository and
@@ -100,7 +104,8 @@
if pieces[0] == 'trunk':
return (None, '/'.join(pieces[1:]))
elif pieces[0] == 'branches':
- return (pieces[1], '/'.join(pieces[2:]))
+ return ('/'.join(pieces[0:2]),
+ '/'.join(pieces[2:]))
else:
return None
@@ -115,7 +120,8 @@
pieces.pop(0) # remove 'trunk'
elif pieces[0] == 'branches':
pieces.pop(0) # remove 'branches'
- branch = pieces.pop(0) # grab branch name
+ # grab branch name
+ branch = 'branches/' + pieces.pop(0)
else:
return None # something weird
projectname = pieces.pop(0)
@@ -141,11 +147,6 @@
@type svnpasswd: string
@param svnpasswd: If set, the --password option will be added.
- @type svnbin: string
- @param svnbin: path to svn binary, defaults to just 'svn'. Use
- this if your subversion command lives in an
- unusual location.
-
@type pollinterval: int
@param pollinterval: interval in seconds between polls. The default
is 600 seconds (10 minutes). Smaller values
@@ -159,6 +160,11 @@
system load, but if more than histmax changes
are recorded between polls, the extra ones will
be silently lost.
+
+ @type svnbin: string
+ @param svnbin: path to svn binary, defaults to just 'svn'. Use
+ this if your subversion command lives in an
+ unusual location.
"""
if svnurl.endswith("/"):
@@ -171,7 +177,8 @@
self.svnbin = svnbin
self.pollinterval = pollinterval
self.histmax = histmax
- self._root = None
+ self._prefix = None
+ self.overrun_counter = 0
self.loop = LoopingCall(self.checksvn)
def split_file(self, path):
@@ -181,6 +188,7 @@
return f(path)
def startService(self):
+ log.msg("SvnSource(%s) starting" % self.svnurl)
base.ChangeSource.startService(self)
# Don't start the loop just yet because the reactor isn't running.
# Give it a chance to go and install our SIGCHLD handler before
@@ -188,11 +196,12 @@
reactor.callLater(0, self.loop.start, self.pollinterval)
def stopService(self):
+ log.msg("SvnSource(%s) shutting down" % self.svnurl)
self.loop.stop()
return base.ChangeSource.stopService(self)
def describe(self):
- return "svnsource %s branch %s" % ( self.svnurl, self.branch)
+ return "SvnSource watching %s" % self.svnurl
def checksvn(self):
# Our return value is only used for unit testing.
@@ -226,43 +235,53 @@
# whew.
+ if self.working:
+ log.msg("SvnSource(%s) overrun: timer fired but the previous "
+ "poll had not yet finished.")
+ self.overrun_counter += 1
+ return defer.succeed(None)
+ self.working = True
+
+ log.msg("SvnSource polling")
if not self._prefix:
- # this sets self._prefix when it finishes
- d = self._determine_prefix()
+ # this sets self._prefix when it finishes. It fires with
+ # self._prefix as well, because that makes the unit tests easier
+ # to write.
+ d = self.get_root()
+ d.addCallback(self.determine_prefix)
else:
- d = defer.succeed(None)
+ d = defer.succeed(self._prefix)
- if self.working:
- dbgMsg("Skipping checksvn because last one has not finished")
- return d
+ d.addCallback(self.get_logs)
+ d.addCallback(self.parse_logs)
+ d.addCallback(self.get_new_logentries)
+ d.addCallback(self.create_changes)
+ d.addCallback(self.submit_changes)
+ d.addBoth(self.finished)
+ return d
- self.working = True
- d.addCallback(lambda res: self._get_logs())
- d.addCallback(self._parse_logs)
- d.addCallback(self._get_new_logentries)
- d.addCallback(self._create_changes)
- d.addCallback(self._submit_changes)
- d.addBoth(self._finished)
+ def getProcessOutput(self, args):
+ # this exists so we can override it during the unit tests
+ d = utils.getProcessOutput(self.svnbin, args, {})
return d
- def _determine_prefix(self):
+ def get_root(self):
args = ["info", "--xml", "--non-interactive", self.svnurl]
if self.svnuser:
args.extend(["--username=%s" % self.svnuser])
if self.svnpasswd:
args.extend(["--password=%s" % self.svnpasswd])
- d = getProcessOutput(self.svnbin, args, {})
- d.addCallback(self._determine_prefix_2)
+ d = self.getProcessOutput(args)
return d
- def _determine_prefix_2(self, output):
- try:
- doc = xml.dom.minidom.parseString(output)
- except xml.parsers.expat.ExpatError:
- dbgMsg("_process_changes: ExpatError in %s" % output)
+ def determine_prefix(self, output):
+ try:
+ doc = xml.dom.minidom.parseString(output)
+ except xml.parsers.expat.ExpatError:
+ dbgMsg("_process_changes: ExpatError in %s" % output)
log.msg("SvnSource._determine_prefix_2: ExpatError in '%s'"
% output)
- raise
+ raise
rootnodes = doc.getElementsByTagName("root")
if not rootnodes:
# this happens if the URL we gave was already the root. In this
@@ -282,7 +301,7 @@
(self.svnurl, root, self._prefix))
return self._prefix
- def _get_logs(self):
+ def get_logs(self, ignored_prefix=None):
args = []
args.extend(["log", "--xml", "--verbose", "--non-interactive"])
if self.svnuser:
@@ -290,22 +309,21 @@
if self.svnpasswd:
args.extend(["--password=%s" % self.svnpasswd])
args.extend(["--limit=%d" % (self.histmax), self.svnurl])
- env = {}
- res = getProcessOutput(self.svnbin, args, env)
- log.msg("_get_changes args %s end %s returns %s" % (args, env, res))
- return res
+ d = self.getProcessOutput(args)
+ return d
- def _parse_logs(self, output):
+ def parse_logs(self, output):
# parse the XML output, return a list of <logentry> nodes
- try:
- doc = xml.dom.minidom.parseString(output)
- except xml.parsers.expat.ExpatError:
- dbgMsg("_process_changes: ExpatError in %s" % output)
+ try:
+ doc = xml.dom.minidom.parseString(output)
+ except xml.parsers.expat.ExpatError:
+ dbgMsg("_process_changes: ExpatError in %s" % output)
log.msg("SvnSource._parse_changes: ExpatError in '%s'" % output)
raise
logentries = doc.getElementsByTagName("logentry")
return logentries
+
def _filter_new_logentries(self, logentries, last_change):
# given a list of logentries, return a tuple of (new_last_change,
# new_logentries), where new_logentries contains only the ones after
@@ -314,30 +332,40 @@
# no entries, so last_change must stay at None
return (None, [])
- mostRecent = int(logentries[0].getAttribute("revision"))
+ mostRecent = int(logentries[0].getAttribute("revision"))
if last_change is None:
# if this is the first time we've been run, ignore any changes
# that occurred before now. This prevents a build at every
# startup.
- log.msg('svnPoller: starting at change %s' % mostRecent)
- self.last_change = mostRecent
- return (mostRecent, [])
+ log.msg('svnPoller: starting at change %s' % mostRecent)
+ return (mostRecent, [])
- if last_change == mostRecent:
+ if last_change == mostRecent:
# an unmodified repository will hit this case
- log.msg('svnPoller: _process_changes last %s mostRecent %s' % (
- last_change, mostRecent))
+ log.msg('svnPoller: _process_changes last %s mostRecent %s' % (
+ last_change, mostRecent))
return (mostRecent, [])
new_logentries = []
- for el in logentries:
+ for el in logentries:
if last_change == int(el.getAttribute("revision")):
- break
+ break
new_logentries.append(el)
new_logentries.reverse() # return oldest first
return (mostRecent, new_logentries)
+ def get_new_logentries(self, logentries):
+ last_change = self.last_change
+ (new_last_change,
+ new_logentries) = self._filter_new_logentries(logentries,
+ self.last_change)
+ self.last_change = new_last_change
+ log.msg('svnPoller: _process_changes %s .. %s' %
+ (last_change, new_last_change))
+ return new_logentries
+
+
def _get_text(self, element, tag_name):
child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
text = "".join([t.data for t in child_nodes])
@@ -350,41 +378,39 @@
relative_path = path[len(self._prefix):]
if relative_path.startswith("/"):
relative_path = relative_path[1:]
- branch, final_path = self.split_file(relative_path)
- return branch, final_path
-
- def _get_new_logentries(self, logentries):
- last_change = self.last_change
- (new_last_change,
- new_logentries) = self._filter_new_logentries(logentries,
- self.last_change)
- self.last_change = new_last_change
- log.msg('svnPoller: _process_changes %s .. %s' %
- (last_change, new_last_change))
- return new_logentries
+ where = self.split_file(relative_path)
+ # 'where' is either None or (branch, final_path)
+ return where
- def _create_changes(self, new_logentries):
+ def create_changes(self, new_logentries):
changes = []
for el in new_logentries:
- branch_files = [] # get oldest change first
- revision = int(el.getAttribute("revision"))
- dbgMsg("Adding change revision %s" % (revision,))
- author = self._get_text(el, "author")
- comments = self._get_text(el, "msg")
- when = self._get_text(el, "date")
- when = time.mktime(time.strptime("%.19s" % when,
+ branch_files = [] # get oldest change first
+ # TODO: revisit this, I think I've settled on Change.revision
+ # being a string everywhere, and leaving the interpretation
+ # of that string up to b.s.source.SVN methods
+ revision = int(el.getAttribute("revision"))
+ dbgMsg("Adding change revision %s" % (revision,))
+ author = self._get_text(el, "author")
+ comments = self._get_text(el, "msg")
+ when = self._get_text(el, "date")
+ when = time.mktime(time.strptime("%.19s" % when,
"%Y-%m-%dT%H:%M:%S"))
branches = {}
- pathlist = el.getElementsByTagName("paths")[0]
- for p in pathlist.getElementsByTagName("path"):
- path = "".join([t.data for t in p.childNodes])
+ pathlist = el.getElementsByTagName("paths")[0]
+ for p in pathlist.getElementsByTagName("path"):
+ path = "".join([t.data for t in p.childNodes])
if path.startswith("/"):
path = path[1:]
- branch, filename = self._transform_path(path)
- if not branch in branches:
- branches[branch] = []
- branches[branch].append(filename)
+ where = self._transform_path(path)
+ # if 'where' is None, the file was outside any project that
+ # we care about and we should ignore it
+ if where:
+ branch, filename = where
+ if not branch in branches:
+ branches[branch] = []
+ branches[branch].append(filename)
for branch in branches:
c = Change(who=author,
@@ -397,17 +423,13 @@
return changes
- def _submit_changes(self, changes):
+ def submit_changes(self, changes):
for c in changes:
- self.parent.addChange(c)
+ self.parent.addChange(c)
- def _finished(self, res):
+ def finished(self, res):
+ log.msg("SvnSource finished polling")
dbgMsg('_finished : %s' % res)
assert self.working
self.working = False
-
- # Again, the return value is only for unit testing.
- # If there's a failure, log it so it isn't lost.
- if isinstance(res, failure.Failure):
- dbgMsg('checksvn failed: %s' % res)
return res
More information about the Commits
mailing list