[Buildbot-devel] Buildbot-devel Digest, Vol 6, Issue 5
Ben Hearsum
ben.hearsum at senecac.on.ca
Thu Oct 5 01:27:20 UTC 2006
Try the 'patches' section of the sourceforge page.
Minesh Patel wrote:
> Sorry if you got this already but I couldn't observe the patches so
> I'll put them inline..
> Is there a proper way to submit these patches?
>
> Thanks,
> Minesh
>
> svnpoller.py
> ==========# -*- test-case-name: buildbot.test.test_svnpoller -*-
>
> # Many thanks to Dave Peticolas for contributing this module
>
> import re
> import time
>
> from twisted.python import log, failure
> from twisted.internet import defer, reactor
> from twisted.internet.utils import getProcessOutput
> from twisted.internet.task import LoopingCall
>
> from buildbot import util
> from buildbot.changes import base, changes
> from buildbot.changes.p4poller import get_simple_split
>
> import xml.dom.minidom
>
> class SVNSource(base.ChangeSource, util.ComparableMixin):
> """This source will poll a subversion repository for changes and submit
> them to the change master."""
>
> compare_attrs = ["svnurl", "svnuser", "svnpasswd", "svnbase",
> "svnbin", "pollinterval", "histmax"]
>
> parent = None # filled in when we're added
> last_change = None
> loop = None
> volatile = ['loop']
> working = False
>
> def __init__(self, svnurl=None, svnuser=None, svnpasswd=None,
> svnbase='/', svnbin='svn',
> split_file=lambda svnpath: (None, svnpath),
> pollinterval=60 * 10, histmax=10):
> """
> @type svnurl: string
> @param svnurl: svn url definition (http(s)://...)
> @type svnuser: string
> @param svnuser: svn user
> @type svnpasswd: string
> @param svnpasswd: svn passwd
> @type svnbase: string
> @param svnbase: svn path specification to limit a poll to
> a certain path after svnurl '...' (i.e., /foo/)
> @type svnbin: string
> @param svnbin: path to svn binary, defaults to just 'svn'
> @type split_file: func
> $param split_file: splits a filename into branch and filename.
> @type pollinterval: int
> @param pollinterval: interval in seconds between polls
> @type histmax: int
> @param histmax: maximum number of changes to look back through
> """
>
> if svnurl.endswith("/"):
> svnurl = svnurl[:-1] # strip the trailing slash
> self.svnurl = (svnurl + svnbase)
> self.svnuser = svnuser
> self.svnpasswd = svnpasswd
> self.svnbase = svnbase
> self.svnbin = svnbin
> self.split_file = split_file
> self.pollinterval = pollinterval
> self.histmax = str(histmax)
>
> def startService(self):
> self.loop = LoopingCall(self.checksvn)
> base.ChangeSource.startService(self)
>
> # Don't start the loop just yet because the reactor isn't running.
> # Give it a chance to go and install our SIGCHLD handler before
> # spawning processes.
> reactor.callLater(0, self.loop.start, self.pollinterval)
>
> def stopService(self):
> self.loop.stop()
> return base.ChangeSource.stopService(self)
>
> def describe(self):
> return "svnsource url:%s base:%s" % (self.svnurl, self.svnbase)
>
> def checksvn(self):
> # Our return value is only used for unit testing.
> if self.working:
> log.msg("Skipping checksvn because last one has not finished")
> return defer.succeed(None)
> else:
> self.working = True
> d = self._get_changes()
> d.addCallback(self._process_changes)
> d.addBoth(self._finished)
> return d
>
> def _finished(self, res):
> assert self.working
> self.working = False
>
> # Again, the return value is only for unit testing.
> # If there's a failure, log it so it isn't lost.
> if isinstance(res, failure.Failure):
> log.msg('SVN poll failed: %s' % res)
> return res
>
> def _get_changes(self):
> args = []
> if self.svnuser:
> args.extend(['--username', self.svnuser])
> if self.svnpasswd:
> args.extend(['--password', self.svnpasswd])
> if self.histmax:
> args.extend(['--limit', self.histmax])
> args.extend(['--verbose', '--xml', '--non-interactive'])
> args.extend(['log', self.svnurl])
> env = {}
> return getProcessOutput(self.svnbin, args, env)
>
> def _get_text(self, element, tag_name):
> child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
> text = "".join([t.data for t in child_nodes])
> return text
>
> def _process_changes(self, result):
> branch_files = {}
> branch_rev = {}
> branch_author = {}
> branch_comments = {}
>
> try:
> doc = xml.dom.minidom.parseString(result)
> except xml.parsers.expat.ExpatError:
> log.msg("_process_changes: ExpatError in %s" % result)
> log.msg("SvnSource._determine_prefix_2: ExpatError in '%s'"
> % output)
> raise
> logentries = doc.getElementsByTagName("logentry")
> mostRecent = int(logentries[0].getAttribute("revision"))
>
> for entry in logentries:
> rev = entry.getAttribute("revision")
> rev = rev.encode("ascii")
> author = self._get_text(entry, "author")
> author = author.encode("ascii")
> comments = self._get_text(entry, "msg")
> comments = comments.encode("ascii")
> pathlist = entry.getElementsByTagName("paths")[0]
> if self.last_change == None:
> log.msg('SVNPoller: starting at change %s' % rev)
> self.last_change = rev
> return []
> if self.last_change == rev:
> break
>
> for p in pathlist.getElementsByTagName("path"):
> path = "".join([t.data for t in p.childNodes])
> path = path.encode("ascii")
> if path.startswith(self.svnbase):
> branch, file = self.split_file(path[len(self.svnbase):])
> if (branch == None and file == None): continue
> if branch_files.has_key(branch):
> branch_files[branch].append(path)
> else:
> branch_files[branch] = [path]
> branch_author[branch] = author
> branch_rev[branch] = rev
> branch_comments[branch] = comments
>
> for branch in branch_files:
> c = changes.Change(who=branch_author[branch],
> files=branch_files[branch],
> comments=branch_comments[branch],
> revision=branch_rev[branch],
> branch=(self.svnbase + branch))
> self.parent.addChange(c)
>
> if branch_rev.values():
> self.last_change = branch_rev.values()[0]
>
> test_svnpoller.py
> =============
> import time
>
> from twisted.internet import defer
> from twisted.trial import unittest
>
> from buildbot.twcompat import maybeWait
> from buildbot.changes.changes import Change
> from buildbot.changes.svnpoller import SVNSource
> from buildbot.changes.p4poller import get_simple_split
>
> first_svnlist = \
> """<?xml version="1.0" encoding="utf-8"?>
> <log>
> <logentry
> revision="1">
> <author>mpatel</author>
> <date>2006-09-26T20:32:41.670553Z</date>
> <paths>
> <path
> action="A">/builds/5.20.0.10/buildbot.c</path>
> </paths>
> <msg>test</msg>
> </logentry>
> </log>
> """
> second_svnlist = \
> """<?xml version="1.0" encoding="utf-8"?>
> <log>
> <logentry
> revision="21">
> <author>foo</author>
> <date>2006-09-26T20:32:41.670553Z</date>
> <paths>
> <path
> action="A">/builds/5.20.0.32/foo.c</path>
> <path
> action="A">/builds/5.20.0.33/test.c</path>
> </paths>
> <msg>test</msg>
> </logentry>
> </log>
> """
> third_svnlist = \
> """<?xml version="1.0" encoding="utf-8"?>
> <log>
> <logentry
> revision="43">
> <author>user</author>
> <date>2006-09-26T20:32:41.670553Z</date>
> <paths>
> <path
> action="A">/builds/5.23.3.46/foo.c</path>
> <path
> action="A">/builds/5.24.3.46/foo.c</path>
> <path
> action="A">/builds/5.25.3.46/foo.c</path>
> </paths>
> <msg>test</msg>
> </logentry>
> </log>
> """
> fourth_svnlist = \
> """<?xml version="1.0" encoding="utf-8"?>
> <log>
> <logentry
> revision="12">
> <author>mpatel</author>
> <date>2006-09-26T20:32:41.670553Z</date>
> <paths>
> <path
> action="A">/builds/5.20.9.10/buildbot.c</path>
> </paths>
> <msg>test</msg>
> </logentry>
> <logentry
> revision="14">
> <author>slamb</author>
> <date>2006-09-26T20:32:41.670553Z</date>
> <paths>
> <path
> action="A">/builds/5.20.0.10/buildbot.c</path>
> <path
> action="A">/builds/6.20.0.10/buildbot.c</path>
> </paths>
> <msg>test</msg>
> </logentry>
> </log>
> """
>
> class MockSVNSource(SVNSource):
> """Test SVNSource which doesn't actually invoke svn."""
> invocation = 0
>
> def __init__(self, svnadd, *args, **kwargs):
> SVNSource.__init__(self, *args, **kwargs)
> self.svnadd = svnadd
>
> def _get_changes(self):
> assert self.working
> result = self.svnadd[self.invocation]
> self.invocation += 1
> return defer.succeed(result)
>
> def _get_describe(self, dummy, num):
> assert self.working
> return defer.succeed(self.p4change[num])
>
> class TestSVNPoller(unittest.TestCase):
> def setUp(self):
> self.changes = []
> self.addChange = self.changes.append
>
> def testCheck(self):
> """Base successful checks"""
> self.t = MockSVNSource(svnadd=[first_svnlist, second_svnlist],
> svnurl='http://path/to/repo', svnuser=None,
> svnbase='/builds/',
> split_file=lambda x: x.split('/', 1))
> self.t.parent = self
>
> # The first time, it just learns the change to start at.
> self.assert_(self.t.last_change is None)
> self.assert_(not self.t.working)
> return maybeWait(self.t.checksvn().addCallback(self._testCheck2))
>
> def _testCheck2(self, res):
> self.assertEquals(self.changes, [])
> self.assertEquals(self.t.last_change, '1')
>
> # Subsequent times, it returns Change objects for new changes.
> return self.t.checksvn().addCallback(self._testCheck3)
>
> def _testCheck3(self, res):
> self.assertEquals(len(self.changes), 2)
> self.assertEquals(self.t.last_change, '21')
> self.assert_(not self.t.working)
>
> # They're supposed to go down in revision(head rev first)
> self.assertEquals(self.changes[0].asText(),
> Change(who='foo',
> files=['/builds/5.20.0.33/test.c'],
> comments='test',
> revision='21',
> #when=self.makeTime("2006-09-26 12:01:49"),
> branch='5.20.0.33').asText())
>
> def testCheckNoBase(self):
> """Empty base successful checks"""
> self.t = MockSVNSource(svnadd=[first_svnlist, second_svnlist],
> svnurl='http://path/to/repo', svnuser=None,
> svnbase='/',
> split_file=lambda x: x.split('/', 1))
> self.t.parent = self
>
> # The first time, it just learns the change to start at.
> self.assert_(self.t.last_change is None)
> self.assert_(not self.t.working)
> return
> maybeWait(self.t.checksvn().addCallback(self._testCheckNoBase1))
>
> def _testCheckNoBase1(self, res):
> self.assertEquals(self.changes, [])
> self.assertEquals(self.t.last_change, '1')
>
> # Subsequent times, it returns Change objects for new changes.
> return self.t.checksvn().addCallback(self._testCheckNoBase2)
>
> def _testCheckNoBase2(self, res):
> self.assertEquals(len(self.changes), 1)
> self.assertEquals(self.t.last_change, '21')
> self.assert_(not self.t.working)
>
> # They're supposed to go down in revision(head rev first)
> self.assertEquals(self.changes[0].asText(),
> Change(who='foo',
>
> files=['/builds/5.20.0.32/foo.c','/builds/5.20.0.33/test.c'],
> comments='test',
> revision='21',
> #when=self.makeTime("2006-09-26 12:01:49"),
> branch='5.20.0.33').asText())
>
> def makeTime(self, timestring):
> datefmt = '%Y-%m-%d %H:%M:%S'
> when = time.mktime(time.strptime(timestring, datefmt))
> return when
>
> def testAlreadyWorking(self):
> """don't launch a new poll while old is still going"""
> self.t = SVNSource(svnurl='http://path/to/repo/')
> self.t.working = True
> self.assert_(self.t.last_change is None)
> d = self.t.checksvn()
> d.addCallback(self._testAlreadyWorking2)
>
> def _testAlreadyWorking2(self, res):
> self.assert_(self.t.last_change is None)
>
> def testNumofUsers(self):
> """Test the number of users """
> self.t = MockSVNSource(svnadd=[fourth_svnlist],
> svnurl='http://path/to/repo', svnuser=None,
> svnbase='/builds/',
> split_file=lambda x: x.split('/', 1))
> self.t.parent = self
> self.t.last_change = 1
> d = self.t.checksvn()
> d.addCallback(self._testNumofUsers)
>
> def _testNumofUsers(self, res):
> users = []
> for c in self.changes:
> if c.who not in users:
> users.append(c.who)
> self.assertEquals(len(users), 2)
>
> def testSplitFile(self):
> """Make sure split file works on branch only changes"""
> self.t = MockSVNSource(svnadd=[third_svnlist],
> svnurl='http://path/to/repo', svnuser=None,
> svnbase='/builds/',
> split_file=get_simple_split)
> self.t.parent = self
> self.t.last_change = 43
> d = self.t.checksvn()
> d.addCallback(self._testSplitFile)
>
> def _testSplitFile(self, res):
> self.assertEquals(len(self.changes), 3)
> self.assertEquals(self.t.last_change, '43')
>
>
>
> buildbot-devel-request at lists.sourceforge.net
> <mailto:buildbot-devel-request at lists.sourceforge.net> wrote:
>> Send Buildbot-devel mailing list submissions to
>> buildbot-devel at lists.sourceforge.net <mailto:buildbot-devel at lists.sourceforge.net>
>>
>> To subscribe or unsubscribe via the World Wide Web, visit
>> https://lists.sourceforge.net/lists/listinfo/buildbot-devel
>> or, via email, send a message with subject or body 'help' to
>> buildbot-devel-request at lists.sourceforge.net <mailto:buildbot-devel-request at lists.sourceforge.net>
>>
>> You can reach the person managing the list at
>> buildbot-devel-owner at lists.sourceforge.net <mailto:buildbot-devel-owner at lists.sourceforge.net>
>>
>> When replying, please edit your Subject line so it is more specific
>> than "Re: Contents of Buildbot-devel digest..."
>>
>>
>> Today's Topics:
>>
>> 1. Re: SvnSource poller (Niklaus Giger)
>> 2. Re: SVNPoller (Minesh Patel)
>>
>>
>> ----------------------------------------------------------------------
>>
>> Message: 1
>> Date: Wed, 4 Oct 2006 21:22:08 +0200
>> From: Niklaus Giger <niklaus.giger at member.fsf.org> <mailto:niklaus.giger at member.fsf.org>
>> Subject: Re: [Buildbot-devel] SvnSource poller
>> To: Brian Warner <warner-buildbot at lothar.com> <mailto:warner-buildbot at lothar.com>
>> Cc: buildbot-devel at lists.sourceforge.net <mailto:buildbot-devel at lists.sourceforge.net>
>> Message-ID: <200610042122.09165.niklaus.giger at member.fsf.org> <mailto:200610042122.09165.niklaus.giger at member.fsf.org>
>> Content-Type: text/plain; charset="iso-8859-1"
>>
>> Am Montag, 2. Oktober 2006 02:25 schrieb Brian Warner:
>>
>>> I spent the weekend working on the SvnSource code developed by Niklaus and
>>> others, and have finally checked it in. Thank you guys so much for writing
>>> this! I rearranged a lot of stuff.. I hope it still works in a useful way.
>>>
>>> Could the folks who were involved with this take a look at the latest
>>> CVS/Darcs and review my changes?
>>>
>> After being away a few days and having guests, I took a look at your changes.
>> Thank you a lot for reworking my proposed patches. I am learning more Python
>> and a buildbot internals.
>>
>>
>>> I changed the API so that for situations where you're only following trunk
>>> you only have to provide one parameter, the svn URL that points at the top
>>> of the tree of interest. If you want to follow multiple branches at once,
>>> you have to provide a "split_file" function that knows about your branch
>>> naming policy. There are a lot of docs and examples in the User's Manual
>>> that I hope will explain how to use split_file().
>>>
>> I had no problem with these changes and they make sense to me.
>>
>> <..>
>>
>>> Rather than setting the Change's timestamp to the value provided by the
>>> SVN server (which seemed to be in the server's timezone when I tried it), I
>>> just had the ChangeMaster assign the current time to each Change as it is
>>> submitted. Before doing this, I found that the Changes appeared several
>>> hours in the future, messing up the Waterfall display.
>>>
>> Good idea.
>>
>>> I took the unicode objects returned by minidom for filenames and forced
>>> them back into ascii, since sometimes they get included in commands and
>>> that would cause some problems. I left "comments" and "author" as unicode,
>>> but fixed up the HTML-rendering code that wasn't able to deal with it. I
>>> suspect that some other status targets may get screwed up by this, but I'd
>>> rather discover those and fix them than forbid unicode everywhere.
>>> Eventually I'd like everything to be unicode-clean but I suspect it will
>>> take a while.
>>>
>> Adding comments with german Umlauts and french accents worked in my
>> environment (using konqueror as a web browser). But I am not sure whether the
>> generated HTML code is correct.
>>
>>
>>> If you could, please see if you can make the new code do the things that
>>> the old code did. Hopefully I didn't break any of the functionality when I
>>> dived in and started rearranging everything.
>>>
>> I ran a few simple tests here and things behaved the way I expected.
>>
>> I just noted the following small problem:
>> Specifiying (note the missing slash at the end of the URL)
>> c['buildbotURL'] = "http://localhost:8081"
>> generated in the build displays section "Steps and Logfiles:" (e.g.
>> http://localhost:8081/svn-better/builds/2) incorrect references like
>> http://localhost:8081svn-better/builds/2/step-svn)
>>
>> I do not now whether it is new or not.
>>
>> Thanks for your great work!
>>
>>
>
>
> ------------------------------------------------------------------------
>
> -------------------------------------------------------------------------
> Take Surveys. Earn Cash. Influence the Future of IT
> Join SourceForge.net's Techsay panel and you'll get the chance to share your
> opinions on IT & business topics through brief surveys -- and earn cash
> http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
>
>
> ------------------------------------------------------------------------
>
> _______________________________________________
> Buildbot-devel mailing list
> Buildbot-devel at lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/buildbot-devel
More information about the devel
mailing list