[Buildbot-devel] Buildbot-devel Digest, Vol 6, Issue 5
Minesh Patel
mpatel at 2wire.com
Thu Oct 5 00:02:02 UTC 2006
Sorry if you got this already but I couldn't observe the patches so I'll
put them inline..
Is there a proper way to submit these patches?
Thanks,
Minesh
svnpoller.py
==========# -*- test-case-name: buildbot.test.test_svnpoller -*-
# Many thanks to Dave Peticolas for contributing this module
import re
import time
from twisted.python import log, failure
from twisted.internet import defer, reactor
from twisted.internet.utils import getProcessOutput
from twisted.internet.task import LoopingCall
from buildbot import util
from buildbot.changes import base, changes
from buildbot.changes.p4poller import get_simple_split
import xml.dom.minidom
class SVNSource(base.ChangeSource, util.ComparableMixin):
"""This source will poll a subversion repository for changes and submit
them to the change master."""
compare_attrs = ["svnurl", "svnuser", "svnpasswd", "svnbase",
"svnbin", "pollinterval", "histmax"]
parent = None # filled in when we're added
last_change = None
loop = None
volatile = ['loop']
working = False
def __init__(self, svnurl=None, svnuser=None, svnpasswd=None,
svnbase='/', svnbin='svn',
split_file=lambda svnpath: (None, svnpath),
pollinterval=60 * 10, histmax=10):
"""
@type svnurl: string
@param svnurl: svn url definition (http(s)://...)
@type svnuser: string
@param svnuser: svn user
@type svnpasswd: string
@param svnpasswd: svn passwd
@type svnbase: string
@param svnbase: svn path specification to limit a poll to
a certain path after svnurl '...' (i.e., /foo/)
@type svnbin: string
@param svnbin: path to svn binary, defaults to just 'svn'
@type split_file: func
$param split_file: splits a filename into branch and filename.
@type pollinterval: int
@param pollinterval: interval in seconds between polls
@type histmax: int
@param histmax: maximum number of changes to look back through
"""
if svnurl.endswith("/"):
svnurl = svnurl[:-1] # strip the trailing slash
self.svnurl = (svnurl + svnbase)
self.svnuser = svnuser
self.svnpasswd = svnpasswd
self.svnbase = svnbase
self.svnbin = svnbin
self.split_file = split_file
self.pollinterval = pollinterval
self.histmax = str(histmax)
def startService(self):
self.loop = LoopingCall(self.checksvn)
base.ChangeSource.startService(self)
# Don't start the loop just yet because the reactor isn't running.
# Give it a chance to go and install our SIGCHLD handler before
# spawning processes.
reactor.callLater(0, self.loop.start, self.pollinterval)
def stopService(self):
self.loop.stop()
return base.ChangeSource.stopService(self)
def describe(self):
return "svnsource url:%s base:%s" % (self.svnurl, self.svnbase)
def checksvn(self):
# Our return value is only used for unit testing.
if self.working:
log.msg("Skipping checksvn because last one has not finished")
return defer.succeed(None)
else:
self.working = True
d = self._get_changes()
d.addCallback(self._process_changes)
d.addBoth(self._finished)
return d
def _finished(self, res):
assert self.working
self.working = False
# Again, the return value is only for unit testing.
# If there's a failure, log it so it isn't lost.
if isinstance(res, failure.Failure):
log.msg('SVN poll failed: %s' % res)
return res
def _get_changes(self):
args = []
if self.svnuser:
args.extend(['--username', self.svnuser])
if self.svnpasswd:
args.extend(['--password', self.svnpasswd])
if self.histmax:
args.extend(['--limit', self.histmax])
args.extend(['--verbose', '--xml', '--non-interactive'])
args.extend(['log', self.svnurl])
env = {}
return getProcessOutput(self.svnbin, args, env)
def _get_text(self, element, tag_name):
child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
text = "".join([t.data for t in child_nodes])
return text
def _process_changes(self, result):
branch_files = {}
branch_rev = {}
branch_author = {}
branch_comments = {}
try:
doc = xml.dom.minidom.parseString(result)
except xml.parsers.expat.ExpatError:
log.msg("_process_changes: ExpatError in %s" % result)
log.msg("SvnSource._determine_prefix_2: ExpatError in '%s'"
% output)
raise
logentries = doc.getElementsByTagName("logentry")
mostRecent = int(logentries[0].getAttribute("revision"))
for entry in logentries:
rev = entry.getAttribute("revision")
rev = rev.encode("ascii")
author = self._get_text(entry, "author")
author = author.encode("ascii")
comments = self._get_text(entry, "msg")
comments = comments.encode("ascii")
pathlist = entry.getElementsByTagName("paths")[0]
if self.last_change == None:
log.msg('SVNPoller: starting at change %s' % rev)
self.last_change = rev
return []
if self.last_change == rev:
break
for p in pathlist.getElementsByTagName("path"):
path = "".join([t.data for t in p.childNodes])
path = path.encode("ascii")
if path.startswith(self.svnbase):
branch, file = self.split_file(path[len(self.svnbase):])
if (branch == None and file == None): continue
if branch_files.has_key(branch):
branch_files[branch].append(path)
else:
branch_files[branch] = [path]
branch_author[branch] = author
branch_rev[branch] = rev
branch_comments[branch] = comments
for branch in branch_files:
c = changes.Change(who=branch_author[branch],
files=branch_files[branch],
comments=branch_comments[branch],
revision=branch_rev[branch],
branch=(self.svnbase + branch))
self.parent.addChange(c)
if branch_rev.values():
self.last_change = branch_rev.values()[0]
test_svnpoller.py
=============
import time
from twisted.internet import defer
from twisted.trial import unittest
from buildbot.twcompat import maybeWait
from buildbot.changes.changes import Change
from buildbot.changes.svnpoller import SVNSource
from buildbot.changes.p4poller import get_simple_split
first_svnlist = \
"""<?xml version="1.0" encoding="utf-8"?>
<log>
<logentry
revision="1">
<author>mpatel</author>
<date>2006-09-26T20:32:41.670553Z</date>
<paths>
<path
action="A">/builds/5.20.0.10/buildbot.c</path>
</paths>
<msg>test</msg>
</logentry>
</log>
"""
second_svnlist = \
"""<?xml version="1.0" encoding="utf-8"?>
<log>
<logentry
revision="21">
<author>foo</author>
<date>2006-09-26T20:32:41.670553Z</date>
<paths>
<path
action="A">/builds/5.20.0.32/foo.c</path>
<path
action="A">/builds/5.20.0.33/test.c</path>
</paths>
<msg>test</msg>
</logentry>
</log>
"""
third_svnlist = \
"""<?xml version="1.0" encoding="utf-8"?>
<log>
<logentry
revision="43">
<author>user</author>
<date>2006-09-26T20:32:41.670553Z</date>
<paths>
<path
action="A">/builds/5.23.3.46/foo.c</path>
<path
action="A">/builds/5.24.3.46/foo.c</path>
<path
action="A">/builds/5.25.3.46/foo.c</path>
</paths>
<msg>test</msg>
</logentry>
</log>
"""
fourth_svnlist = \
"""<?xml version="1.0" encoding="utf-8"?>
<log>
<logentry
revision="12">
<author>mpatel</author>
<date>2006-09-26T20:32:41.670553Z</date>
<paths>
<path
action="A">/builds/5.20.9.10/buildbot.c</path>
</paths>
<msg>test</msg>
</logentry>
<logentry
revision="14">
<author>slamb</author>
<date>2006-09-26T20:32:41.670553Z</date>
<paths>
<path
action="A">/builds/5.20.0.10/buildbot.c</path>
<path
action="A">/builds/6.20.0.10/buildbot.c</path>
</paths>
<msg>test</msg>
</logentry>
</log>
"""
class MockSVNSource(SVNSource):
"""Test SVNSource which doesn't actually invoke svn."""
invocation = 0
def __init__(self, svnadd, *args, **kwargs):
SVNSource.__init__(self, *args, **kwargs)
self.svnadd = svnadd
def _get_changes(self):
assert self.working
result = self.svnadd[self.invocation]
self.invocation += 1
return defer.succeed(result)
def _get_describe(self, dummy, num):
assert self.working
return defer.succeed(self.p4change[num])
class TestSVNPoller(unittest.TestCase):
def setUp(self):
self.changes = []
self.addChange = self.changes.append
def testCheck(self):
"""Base successful checks"""
self.t = MockSVNSource(svnadd=[first_svnlist, second_svnlist],
svnurl='http://path/to/repo', svnuser=None,
svnbase='/builds/',
split_file=lambda x: x.split('/', 1))
self.t.parent = self
# The first time, it just learns the change to start at.
self.assert_(self.t.last_change is None)
self.assert_(not self.t.working)
return maybeWait(self.t.checksvn().addCallback(self._testCheck2))
def _testCheck2(self, res):
self.assertEquals(self.changes, [])
self.assertEquals(self.t.last_change, '1')
# Subsequent times, it returns Change objects for new changes.
return self.t.checksvn().addCallback(self._testCheck3)
def _testCheck3(self, res):
self.assertEquals(len(self.changes), 2)
self.assertEquals(self.t.last_change, '21')
self.assert_(not self.t.working)
# They're supposed to go down in revision(head rev first)
self.assertEquals(self.changes[0].asText(),
Change(who='foo',
files=['/builds/5.20.0.33/test.c'],
comments='test',
revision='21',
#when=self.makeTime("2006-09-26 12:01:49"),
branch='5.20.0.33').asText())
def testCheckNoBase(self):
"""Empty base successful checks"""
self.t = MockSVNSource(svnadd=[first_svnlist, second_svnlist],
svnurl='http://path/to/repo', svnuser=None,
svnbase='/',
split_file=lambda x: x.split('/', 1))
self.t.parent = self
# The first time, it just learns the change to start at.
self.assert_(self.t.last_change is None)
self.assert_(not self.t.working)
return
maybeWait(self.t.checksvn().addCallback(self._testCheckNoBase1))
def _testCheckNoBase1(self, res):
self.assertEquals(self.changes, [])
self.assertEquals(self.t.last_change, '1')
# Subsequent times, it returns Change objects for new changes.
return self.t.checksvn().addCallback(self._testCheckNoBase2)
def _testCheckNoBase2(self, res):
self.assertEquals(len(self.changes), 1)
self.assertEquals(self.t.last_change, '21')
self.assert_(not self.t.working)
# They're supposed to go down in revision(head rev first)
self.assertEquals(self.changes[0].asText(),
Change(who='foo',
files=['/builds/5.20.0.32/foo.c','/builds/5.20.0.33/test.c'],
comments='test',
revision='21',
#when=self.makeTime("2006-09-26 12:01:49"),
branch='5.20.0.33').asText())
def makeTime(self, timestring):
datefmt = '%Y-%m-%d %H:%M:%S'
when = time.mktime(time.strptime(timestring, datefmt))
return when
def testAlreadyWorking(self):
"""don't launch a new poll while old is still going"""
self.t = SVNSource(svnurl='http://path/to/repo/')
self.t.working = True
self.assert_(self.t.last_change is None)
d = self.t.checksvn()
d.addCallback(self._testAlreadyWorking2)
def _testAlreadyWorking2(self, res):
self.assert_(self.t.last_change is None)
def testNumofUsers(self):
"""Test the number of users """
self.t = MockSVNSource(svnadd=[fourth_svnlist],
svnurl='http://path/to/repo', svnuser=None,
svnbase='/builds/',
split_file=lambda x: x.split('/', 1))
self.t.parent = self
self.t.last_change = 1
d = self.t.checksvn()
d.addCallback(self._testNumofUsers)
def _testNumofUsers(self, res):
users = []
for c in self.changes:
if c.who not in users:
users.append(c.who)
self.assertEquals(len(users), 2)
def testSplitFile(self):
"""Make sure split file works on branch only changes"""
self.t = MockSVNSource(svnadd=[third_svnlist],
svnurl='http://path/to/repo', svnuser=None,
svnbase='/builds/',
split_file=get_simple_split)
self.t.parent = self
self.t.last_change = 43
d = self.t.checksvn()
d.addCallback(self._testSplitFile)
def _testSplitFile(self, res):
self.assertEquals(len(self.changes), 3)
self.assertEquals(self.t.last_change, '43')
buildbot-devel-request at lists.sourceforge.net wrote:
> Send Buildbot-devel mailing list submissions to
> buildbot-devel at lists.sourceforge.net
>
> To subscribe or unsubscribe via the World Wide Web, visit
> https://lists.sourceforge.net/lists/listinfo/buildbot-devel
> or, via email, send a message with subject or body 'help' to
> buildbot-devel-request at lists.sourceforge.net
>
> You can reach the person managing the list at
> buildbot-devel-owner at lists.sourceforge.net
>
> When replying, please edit your Subject line so it is more specific
> than "Re: Contents of Buildbot-devel digest..."
>
>
> Today's Topics:
>
> 1. Re: SvnSource poller (Niklaus Giger)
> 2. Re: SVNPoller (Minesh Patel)
>
>
> ----------------------------------------------------------------------
>
> Message: 1
> Date: Wed, 4 Oct 2006 21:22:08 +0200
> From: Niklaus Giger <niklaus.giger at member.fsf.org>
> Subject: Re: [Buildbot-devel] SvnSource poller
> To: Brian Warner <warner-buildbot at lothar.com>
> Cc: buildbot-devel at lists.sourceforge.net
> Message-ID: <200610042122.09165.niklaus.giger at member.fsf.org>
> Content-Type: text/plain; charset="iso-8859-1"
>
> Am Montag, 2. Oktober 2006 02:25 schrieb Brian Warner:
>
>> I spent the weekend working on the SvnSource code developed by Niklaus and
>> others, and have finally checked it in. Thank you guys so much for writing
>> this! I rearranged a lot of stuff.. I hope it still works in a useful way.
>>
>> Could the folks who were involved with this take a look at the latest
>> CVS/Darcs and review my changes?
>>
> After being away a few days and having guests, I took a look at your changes.
> Thank you a lot for reworking my proposed patches. I am learning more Python
> and a buildbot internals.
>
>
>> I changed the API so that for situations where you're only following trunk
>> you only have to provide one parameter, the svn URL that points at the top
>> of the tree of interest. If you want to follow multiple branches at once,
>> you have to provide a "split_file" function that knows about your branch
>> naming policy. There are a lot of docs and examples in the User's Manual
>> that I hope will explain how to use split_file().
>>
> I had no problem with these changes and they make sense to me.
>
> <..>
>
>> Rather than setting the Change's timestamp to the value provided by the
>> SVN server (which seemed to be in the server's timezone when I tried it), I
>> just had the ChangeMaster assign the current time to each Change as it is
>> submitted. Before doing this, I found that the Changes appeared several
>> hours in the future, messing up the Waterfall display.
>>
> Good idea.
>
>> I took the unicode objects returned by minidom for filenames and forced
>> them back into ascii, since sometimes they get included in commands and
>> that would cause some problems. I left "comments" and "author" as unicode,
>> but fixed up the HTML-rendering code that wasn't able to deal with it. I
>> suspect that some other status targets may get screwed up by this, but I'd
>> rather discover those and fix them than forbid unicode everywhere.
>> Eventually I'd like everything to be unicode-clean but I suspect it will
>> take a while.
>>
> Adding comments with german Umlauts and french accents worked in my
> environment (using konqueror as a web browser). But I am not sure whether the
> generated HTML code is correct.
>
>
>> If you could, please see if you can make the new code do the things that
>> the old code did. Hopefully I didn't break any of the functionality when I
>> dived in and started rearranging everything.
>>
>
> I ran a few simple tests here and things behaved the way I expected.
>
> I just noted the following small problem:
> Specifiying (note the missing slash at the end of the URL)
> c['buildbotURL'] = "http://localhost:8081"
> generated in the build displays section "Steps and Logfiles:" (e.g.
> http://localhost:8081/svn-better/builds/2) incorrect references like
> http://localhost:8081svn-better/builds/2/step-svn)
>
> I do not now whether it is new or not.
>
> Thanks for your great work!
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://buildbot.net/pipermail/devel/attachments/20061004/2920356a/attachment.html>
More information about the devel
mailing list