From warner at users.sourceforge.net Sun Aug 6 02:49:36 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 02:49:36 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.56,1.57 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8128/buildbot/slave Modified Files: commands.py Log Message: [project @ update commands.py version number] Original author: warner at lothar.com Date: 2006-08-06 02:48:16 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.56 retrieving revision 1.57 diff -u -d -r1.56 -r1.57 --- commands.py 28 Jun 2006 20:05:46 -0000 1.56 +++ commands.py 6 Aug 2006 02:49:34 -0000 1.57 @@ -11,7 +11,10 @@ from buildbot.slave.interfaces import ISlaveCommand from buildbot.slave.registry import registerSlaveCommand -cvs_ver = '$Revision$'[1+len("Revision: "):-2] +# this used to be a CVS $-style "Revision" auto-updated keyword, but since I +# moved to Darcs as the primary repository, this is updated manually each +# time this file is changed. The last cvs_ver that was here was 1.51 . +command_version = "2.1" # version history: # >=1.17: commands are interruptable @@ -20,6 +23,14 @@ # >=1.39: Source classes correctly handle changes in branch (except Git) # Darcs accepts 'revision' (now all do but Git) (well, and P4Sync) # Arch/Baz should accept 'build-config' +# >=1.51: (release 0.7.3) +# >= 2.1: SlaveShellCommand now accepts 'initial_stdin', 'keep_stdin_open', +# and 'logfiles'. It now sends 'log' messages in addition to +# stdout/stdin/header/rc. It acquired writeStdin/closeStdin methods, +# but these are not remotely callable yet. +# (not externally visible: ShellCommandPP has writeStdin/closeStdin. +# ShellCommand accepts new arguments (logfiles=, initialStdin=, +# keepStdinOpen=) and no longer accepts stdin=) class CommandInterrupted(Exception): pass @@ -699,7 +710,7 @@ def closeStdin(self): self.command.closeStdin() -registerSlaveCommand("shell", SlaveShellCommand, cvs_ver) +registerSlaveCommand("shell", SlaveShellCommand, command_version) class DummyCommand(Command): @@ -736,7 +747,7 @@ self.sendStatus({'rc': 0}) self.d.callback(0) -registerSlaveCommand("dummy", DummyCommand, cvs_ver) +registerSlaveCommand("dummy", DummyCommand, command_version) class SourceBase(Command): @@ -1101,7 +1112,7 @@ # reasonably-well-synchronized with the repository. return time.strftime("%Y-%m-%d %H:%M:%S +0000", time.gmtime()) -registerSlaveCommand("cvs", CVS, cvs_ver) +registerSlaveCommand("cvs", CVS, command_version) class SVN(SourceBase): """Subversion-specific VC operation. In addition to the arguments @@ -1185,7 +1196,7 @@ return d -registerSlaveCommand("svn", SVN, cvs_ver) +registerSlaveCommand("svn", SVN, command_version) class Darcs(SourceBase): """Darcs-specific VC operation. In addition to the arguments @@ -1264,7 +1275,7 @@ d.addCallback(lambda res: c.stdout) return d -registerSlaveCommand("darcs", Darcs, cvs_ver) +registerSlaveCommand("darcs", Darcs, command_version) class Git(SourceBase): """Git specific VC operation. In addition to the arguments @@ -1304,7 +1315,7 @@ self.command = c return c.start() -registerSlaveCommand("git", Git, cvs_ver) +registerSlaveCommand("git", Git, command_version) class Arch(SourceBase): """Arch-specific (tla-specific) VC operation. In addition to the @@ -1438,7 +1449,7 @@ d.addCallback(_parse) return d -registerSlaveCommand("arch", Arch, cvs_ver) +registerSlaveCommand("arch", Arch, command_version) class Bazaar(Arch): """Bazaar (/usr/bin/baz) is an alternative client for Arch repositories. @@ -1499,7 +1510,7 @@ d.addCallback(_parse) return d -registerSlaveCommand("bazaar", Bazaar, cvs_ver) +registerSlaveCommand("bazaar", Bazaar, command_version) class Mercurial(SourceBase): @@ -1579,7 +1590,7 @@ d.addCallback(_parse) return d -registerSlaveCommand("hg", Mercurial, cvs_ver) +registerSlaveCommand("hg", Mercurial, command_version) class P4(SourceBase): @@ -1701,7 +1712,7 @@ d.addCallback(lambda _: self._doP4Sync(force=True)) return d -registerSlaveCommand("p4", P4, cvs_ver) +registerSlaveCommand("p4", P4, command_version) class P4Sync(SourceBase): @@ -1756,4 +1767,4 @@ def doVCFull(self): return self._doVC(force=True) -registerSlaveCommand("p4sync", P4Sync, cvs_ver) +registerSlaveCommand("p4sync", P4Sync, command_version) From warner at users.sourceforge.net Sun Aug 6 02:49:36 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 02:49:36 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.691,1.692 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8128 Modified Files: ChangeLog Log Message: [project @ update commands.py version number] Original author: warner at lothar.com Date: 2006-08-06 02:48:16 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.691 retrieving revision 1.692 diff -u -d -r1.691 -r1.692 --- ChangeLog 31 Jul 2006 08:31:33 -0000 1.691 +++ ChangeLog 6 Aug 2006 02:49:33 -0000 1.692 @@ -1,3 +1,12 @@ +2006-08-05 Brian Warner + + * buildbot/slave/commands.py (command_version): replace the CVS + auto-updated cvs_ver keyword with a manually-updated variable, + since CVS is no longer the master repository. Add a description of + the remote API change starting in this version (2.1), specifically + the fact that SlaveShellCommand now accepts 'initial_stdin', + 'keep_stdin_open', and 'logfiles'. + 2006-07-31 Brian Warner * docs/buildbot.texinfo (System Architecture): Finally add lots of From warner at users.sourceforge.net Sun Aug 6 07:05:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 07:05:51 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.692,1.693 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25502 Modified Files: ChangeLog Log Message: [project @ test_steps.py: update to match s/cvs_ver/command_version/ change] Original author: warner at lothar.com Date: 2006-08-06 07:04:57 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.692 retrieving revision 1.693 diff -u -d -r1.692 -r1.693 --- ChangeLog 6 Aug 2006 02:49:33 -0000 1.692 +++ ChangeLog 6 Aug 2006 07:05:48 -0000 1.693 @@ -1,3 +1,8 @@ +2006-08-06 Brian Warner + + * buildbot/test/test_steps.py (Version.checkCompare): oops, update + to match the s/cvs_ver/command_version/ change + 2006-08-05 Brian Warner * buildbot/slave/commands.py (command_version): replace the CVS From warner at users.sourceforge.net Sun Aug 6 07:05:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 07:05:51 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_steps.py,1.22,1.23 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25502/buildbot/test Modified Files: test_steps.py Log Message: [project @ test_steps.py: update to match s/cvs_ver/command_version/ change] Original author: warner at lothar.com Date: 2006-08-06 07:04:57 Index: test_steps.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_steps.py,v retrieving revision 1.22 retrieving revision 1.23 diff -u -d -r1.22 -r1.23 --- test_steps.py 20 Jun 2006 16:08:47 -0000 1.22 +++ test_steps.py 6 Aug 2006 07:05:49 -0000 1.23 @@ -217,9 +217,10 @@ def checkCompare(self, s): + cver = commands.command_version v = s.slaveVersion("svn", None) # this insures that we are getting the version correctly - self.failUnlessEqual(s.slaveVersion("svn", None), commands.cvs_ver) + self.failUnlessEqual(s.slaveVersion("svn", None), cver) # and that non-existent commands do not provide a version self.failUnlessEqual(s.slaveVersion("NOSUCHCOMMAND"), None) # TODO: verify that a <=0.5.0 buildslave (which does not implement @@ -228,10 +229,9 @@ #self.failUnlessEqual(s.slaveVersion("NOSUCHCOMMAND", "old"), "old") # now check the comparison functions - self.failIf(s.slaveVersionIsOlderThan("svn", commands.cvs_ver)) + self.failIf(s.slaveVersionIsOlderThan("svn", cver)) self.failIf(s.slaveVersionIsOlderThan("svn", "1.1")) - self.failUnless(s.slaveVersionIsOlderThan("svn", - commands.cvs_ver + ".1")) + self.failUnless(s.slaveVersionIsOlderThan("svn", cver + ".1")) def testCompare(self): self.master._checker = self.checkCompare From warner at users.sourceforge.net Sun Aug 6 20:05:23 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 20:05:23 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.693,1.694 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17459 Modified Files: ChangeLog Log Message: [project @ add BuildStep.getSlaveName, make useLog more flexible] Original author: warner at lothar.com Date: 2006-08-06 19:40:26 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.693 retrieving revision 1.694 diff -u -d -r1.693 -r1.694 --- ChangeLog 6 Aug 2006 07:05:48 -0000 1.693 +++ ChangeLog 6 Aug 2006 20:05:21 -0000 1.694 @@ -1,5 +1,12 @@ 2006-08-06 Brian Warner + * buildbot/process/step.py (LoggedRemoteCommand.useLog): allow + callers to provide the slave-side logfile name, rather than + forcing it to come from the local name of the LogFile. + (BuildStep.getSlaveName): new method + * buildbot/process/base.py (Build.getSlaveName): new method, so + steps can find out which buildslave they're running on + * buildbot/test/test_steps.py (Version.checkCompare): oops, update to match the s/cvs_ver/command_version/ change From warner at users.sourceforge.net Sun Aug 6 20:05:23 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 20:05:23 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process base.py, 1.67, 1.68 step.py, 1.95, 1.96 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17459/buildbot/process Modified Files: base.py step.py Log Message: [project @ add BuildStep.getSlaveName, make useLog more flexible] Original author: warner at lothar.com Date: 2006-08-06 19:40:26 Index: base.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/base.py,v retrieving revision 1.67 retrieving revision 1.68 diff -u -d -r1.67 -r1.68 --- base.py 16 Jun 2006 05:28:13 -0000 1.67 +++ base.py 6 Aug 2006 20:05:21 -0000 1.68 @@ -263,6 +263,8 @@ def getSlaveCommandVersion(self, command, oldversion=None): return self.slavebuilder.getSlaveCommandVersion(command, oldversion) + def getSlaveName(self): + return self.slavebuilder.slave.slavename def setupStatus(self, build_status): self.build_status = build_status Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.95 retrieving revision 1.96 diff -u -d -r1.95 -r1.96 --- step.py 20 Jun 2006 08:09:15 -0000 1.95 +++ step.py 6 Aug 2006 20:05:21 -0000 1.96 @@ -266,12 +266,36 @@ def __repr__(self): return "" % (self.remote_command, id(self)) - def useLog(self, loog, closeWhenFinished=False): + def useLog(self, loog, closeWhenFinished=False, logfileName=None): + """Start routing messages from a remote logfile to a local LogFile + + I take a local ILogFile instance in 'loog', and arrange to route + remote log messages for the logfile named 'logfileName' into it. By + default this logfileName comes from the ILogFile itself (using the + name by which the ILogFile will be displayed), but the 'logfileName' + argument can be used to override this. For example, if + logfileName='stdio', this logfile will collect text from the stdout + and stderr of the command. + + @param loog: an instance which implements ILogFile + @param closeWhenFinished: a boolean, set to False if the logfile + will be shared between multiple + RemoteCommands. If True, the logfile will + be closed when this ShellCommand is done + with it. + @param logfileName: a string, which indicates which remote log file + should be routed into this ILogFile. This should + match one of the keys of the logfiles= argument + to ShellCommand. + + """ + assert providedBy(loog, interfaces.ILogFile) - name = loog.getName() - assert name not in self.logs - self.logs[name] = loog - self._closeWhenFinished[name] = closeWhenFinished + if not logfileName: + logfileName = loog.getName() + assert logfileName not in self.logs + self.logs[logfileName] = loog + self._closeWhenFinished[logfileName] = closeWhenFinished def start(self): log.msg("LoggedRemoteCommand.start") @@ -797,11 +821,19 @@ return True return False + def getSlaveName(self): + return self.build.getSlaveName() + def addLog(self, name): loog = self.step_status.addLog(name) self._connectPendingLogObservers() return loog + # TODO: add a getLog() ? At the moment all logs have to be retrieved from + # the RemoteCommand that created them, but for status summarizers it + # would be more convenient to get them from the BuildStep / BSStatus, + # especially if there are multiple RemoteCommands involved. + def addCompleteLog(self, name, text): log.msg("addCompleteLog(%s)" % name) loog = self.step_status.addLog(name) From warner at users.sourceforge.net Sun Aug 6 20:05:33 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 20:05:33 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.96,1.97 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17495/buildbot/process Modified Files: step.py Log Message: [project @ refactor command-completion handling, warn about old slaves and logfiles=] Original author: warner at lothar.com Date: 2006-08-06 20:02:38 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.96 retrieving revision 1.97 diff -u -d -r1.96 -r1.97 --- step.py 6 Aug 2006 20:05:21 -0000 1.96 +++ step.py 6 Aug 2006 20:05:31 -0000 1.97 @@ -910,8 +910,15 @@ cmd.useLog(stdio_log, True) for em in errorMessages: stdio_log.addHeader(em) - d = self.runCommand(cmd) - d.addCallbacks(self._commandComplete, self.checkDisconnect) + d = self.runCommand(cmd) # might raise ConnectionLost + d.addCallback(lambda res: self.commandComplete(cmd)) + d.addCallback(lambda res: self.createSummary(cmd.logs['stdio'])) + d.addCallback(lambda res: self.evaluateCommand(cmd)) # returns results + def _gotResults(results): + self.setStatus(cmd, results) + return results + d.addCallback(_gotResults) # returns results + d.addCallbacks(self.finished, self.checkDisconnect) d.addErrback(self.failed) def interrupt(self, reason): @@ -930,13 +937,6 @@ self.step_status.setText2(["failed", "slave", "lost"]) return self.finished(FAILURE) - def _commandComplete(self, cmd): - self.commandComplete(cmd) - self.createSummary(cmd.logs['stdio']) - results = self.evaluateCommand(cmd) - self.setStatus(cmd, results) - return self.finished(results) - # to refine the status output, override one or more of the following # methods. Change as little as possible: start with the first ones on # this list and only proceed further if you have to @@ -1203,12 +1203,29 @@ # dictionary, so we shouldn't be affecting anyone but ourselves. def setupLogfiles(self, cmd, logfiles): - if logfiles: + if not logfiles: + return + if self.slaveVersionIsOlderThan("shell", "2.1"): + # this buildslave is too old and will ignore the 'logfiles' + # argument. You'll either have to pull the logfiles manually + # (say, by using 'cat' in a separate RemoteShellCommand) or + # upgrade the buildslave. + msg1 = ("Warning: buildslave %s is too old " + "to understand logfiles=, ignoring it." + % self.getSlaveName()) + msg2 = "You will have to pull this logfile (%s) manually." + log.msg(msg1) for logname,remotefilename in logfiles.items(): - # tell the BuildStepStatus to add a LogFile newlog = self.addLog(logname) - # and tell the LoggedRemoteCommand to feed it - cmd.useLog(newlog, True) + newlog.addHeader(msg1 + "\n") + newlog.addHeader(msg2 % remotefilename + "\n") + newlog.finish() + return + for logname,remotefilename in logfiles.items(): + # tell the BuildStepStatus to add a LogFile + newlog = self.addLog(logname) + # and tell the LoggedRemoteCommand to feed it + cmd.useLog(newlog, True) def start(self): command = self._interpolateProperties(self.command) From warner at users.sourceforge.net Sun Aug 6 20:05:33 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 20:05:33 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.694,1.695 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17495 Modified Files: ChangeLog Log Message: [project @ refactor command-completion handling, warn about old slaves and logfiles=] Original author: warner at lothar.com Date: 2006-08-06 20:02:38 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.694 retrieving revision 1.695 diff -u -d -r1.694 -r1.695 --- ChangeLog 6 Aug 2006 20:05:21 -0000 1.694 +++ ChangeLog 6 Aug 2006 20:05:31 -0000 1.695 @@ -1,9 +1,18 @@ 2006-08-06 Brian Warner + * buildbot/process/step.py (LoggingBuildStep.startCommand): + refactor command-completion handling, to allow methods like + commandComplete/createSummary/evaluateCommand to return Deferreds. + (LoggingBuildStep._commandComplete): delete the refactored method + (ShellCommand.setupLogfiles): if the buildslave is too old to + understand logfiles=, put a warning message both into twistd.log + and into the otherwise empty user-visible LogFiles. + * buildbot/process/step.py (LoggedRemoteCommand.useLog): allow callers to provide the slave-side logfile name, rather than forcing it to come from the local name of the LogFile. (BuildStep.getSlaveName): new method + * buildbot/process/base.py (Build.getSlaveName): new method, so steps can find out which buildslave they're running on From warner at users.sourceforge.net Sun Aug 6 20:05:40 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 20:05:40 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step_twisted.py, 1.82, 1.83 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17513/buildbot/process Modified Files: step_twisted.py Log Message: [project @ Trial: fall back to 'cat' if the slave is too old to use logfiles=] Original author: warner at lothar.com Date: 2006-08-06 20:03:51 Index: step_twisted.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step_twisted.py,v retrieving revision 1.82 retrieving revision 1.83 diff -u -d -r1.82 -r1.83 --- step_twisted.py 20 Jun 2006 08:09:21 -0000 1.82 +++ step_twisted.py 6 Aug 2006 20:05:38 -0000 1.83 @@ -444,20 +444,47 @@ else: self.command.extend(self.tests) log.msg("Trial.start: command is", self.command) + + # if our slave is too old to understand logfiles=, fetch them + # manually. This is a fallback for the Twisted buildbot and some old + # buildslaves. + self._needToPullTestDotLog = False + if self.slaveVersionIsOlderThan("shell", "2.1"): + log.msg("Trial: buildslave %s is too old to accept logfiles=" % + self.getSlaveName()) + log.msg(" falling back to 'cat _trial_temp/test.log' instead") + self.logfiles = {} + self._needToPullTestDotLog = True + ShellCommand.start(self) + def commandComplete(self, cmd): + if not self._needToPullTestDotLog: + return self._gotTestDotLog(cmd) + + # if the buildslave was too old, pull test.log now + catcmd = ["cat", "_trial_temp/test.log"] + c2 = step.RemoteShellCommand(command=catcmd, workdir=self.workdir) + loog = self.addLog("test.log") + c2.useLog(loog, True, logfileName="stdio") + self.cmd = c2 # to allow interrupts + d = c2.run(self, self.remote) + d.addCallback(lambda res: self._gotTestDotLog(cmd)) + return d + def rtext(self, fmt='%s'): if self.reactor: rtext = fmt % self.reactor return rtext.replace("reactor", "") return "" - - def commandComplete(self, cmd): + def _gotTestDotLog(self, cmd): # figure out all status, then let the various hook functions return # different pieces of it + # 'cmd' is the original trial command, so cmd.logs['stdio'] is the + # trial output. We don't have access to test.log from here. output = cmd.logs['stdio'].getText() counts = countFailedTests(output) From warner at users.sourceforge.net Sun Aug 6 20:05:40 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 20:05:40 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.695,1.696 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17513 Modified Files: ChangeLog Log Message: [project @ Trial: fall back to 'cat' if the slave is too old to use logfiles=] Original author: warner at lothar.com Date: 2006-08-06 20:03:51 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.695 retrieving revision 1.696 diff -u -d -r1.695 -r1.696 --- ChangeLog 6 Aug 2006 20:05:31 -0000 1.695 +++ ChangeLog 6 Aug 2006 20:05:38 -0000 1.696 @@ -1,5 +1,12 @@ 2006-08-06 Brian Warner + * buildbot/process/step_twisted.py (Trial.start): if the + buildslave is too old to understand logfiles=, fall back to + running 'cat _trial_temp/test.log' like before. + (Trial.commandComplete): same. this takes advantage of the + LoggingBuildStep refactoring to stall commandComplete long enough + to run a second RemoteShellCommand. + * buildbot/process/step.py (LoggingBuildStep.startCommand): refactor command-completion handling, to allow methods like commandComplete/createSummary/evaluateCommand to return Deferreds. From warner at users.sourceforge.net Sun Aug 6 21:14:46 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 21:14:46 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.60,1.61 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12505/docs Modified Files: buildbot.texinfo Log Message: [project @ add Manhole .ssh/config suggestion, update NEWS file] Original author: warner at lothar.com Date: 2006-08-06 21:14:00 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.60 retrieving revision 1.61 diff -u -d -r1.60 -r1.61 --- buildbot.texinfo 31 Jul 2006 08:31:33 -0000 1.60 +++ buildbot.texinfo 6 Aug 2006 21:14:44 -0000 1.61 @@ -2346,6 +2346,23 @@ Note that using any Manhole requires that the TwistedConch package be installed, and that you be using Twisted version 2.0 or later. +The buildmaster's SSH server will use a different host key than the +normal sshd running on a typical unix host. This will cause the ssh +client to complain about a ``host key mismatch'', because it does not +realize there are two separate servers running on the same host. To +avoid this, use a clause like the following in your @file{.ssh/config} +file: + + at example +Host remotehost-buildbot + HostName remotehost + HostKeyAlias remotehost-buildbot + Port 9999 + # use 'user' if you use PasswordManhole and your name is not 'admin'. + # if you use AuthorizedKeysManhole, this probably doesn't matter. + User admin + at end example + @node Getting Source Code Changes, Build Process, Configuration, Top @chapter Getting Source Code Changes From warner at users.sourceforge.net Sun Aug 6 21:14:46 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 21:14:46 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.696,1.697 NEWS,1.55,1.56 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12505 Modified Files: ChangeLog NEWS Log Message: [project @ add Manhole .ssh/config suggestion, update NEWS file] Original author: warner at lothar.com Date: 2006-08-06 21:14:00 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.696 retrieving revision 1.697 diff -u -d -r1.696 -r1.697 --- ChangeLog 6 Aug 2006 20:05:38 -0000 1.696 +++ ChangeLog 6 Aug 2006 21:14:44 -0000 1.697 @@ -1,5 +1,10 @@ 2006-08-06 Brian Warner + * NEWS: summarize recent changes + + * docs/buildbot.texinfo (Debug options): suggest an .ssh/options + clause to avoid the "host key mismatch" warning + * buildbot/process/step_twisted.py (Trial.start): if the buildslave is too old to understand logfiles=, fall back to running 'cat _trial_temp/test.log' like before. Index: NEWS =================================================================== RCS file: /cvsroot/buildbot/buildbot/NEWS,v retrieving revision 1.55 retrieving revision 1.56 diff -u -d -r1.55 -r1.56 --- NEWS 23 May 2006 16:49:07 -0000 1.55 +++ NEWS 6 Aug 2006 21:14:44 -0000 1.56 @@ -1,5 +1,99 @@ User visible changes in Buildbot. +* Release ?.?.? (?) + +** new features + +*** full Perforce support + +SF#1473939: large patch from Scott Lamb, with docs and unit tests! This +includes both the step.P4 source-checkout BuildStep, and the changes.p4poller +ChangeSource you'll want to feed it. P4 is now supported just as well as all +the other VC systems. Thanks Scott! + +*** SSH-based Manhole + +The 'manhole' feature allows buildbot developers to get access to a python +read/eval/print loop (REPL) inside the buildmaster through a network +connection. Previously, this ran over unencrypted telnet, using a simple +username/password for access control. The new release defaults to encrypted +SSH access, using either username/password or an authorized_keys file (just +like sshd). There also exists an unencrypted telnet form, but its use is +discouraged. The syntax for setting up a manhole has changed, so master.cfg +files that use them must be updated. The "Debug options" section in the +user's manual provides a complete description. + +*** LogObservers + +BuildSteps can now attach LogObservers to various logfiles, allowing them to +get real-time log output. They can use this to watch for progress-indicating +events (like counting the number of files compiled, or the number of tests +which have run), and update both ETA/progress-tracking and step text. This +allows for more accurate ETA information, and more information passed to the +user about how much of the process has completed. + +The 'Trial' buildstep has been updated to use this for progress tracking, by +counting how many test cases have run. + +*** Multiple Logfiles + +BuildSteps can watch multiple log files in realtime, not just stdout/stderr. +This works in a similar fashion to 'tail -f': the file is polled once per +second, and any new data is sent to the buildmaster. + +This requires a buildslave running 0.7.4 or later, and a warning message is +produced if used against an old buildslave. Use "logfiles={'name': +'filename'}" to take advantage of this feature from master.cfg, and see the +"ShellCommand" section of the user's manual for full documentation. + +The 'Trial' buildstep has been updated to use this, to display +_trial_temp/test.log in realtime. + +** new documentation + +What classes are useful in your master.cfg file? A table of them has been +added to the user's manual, in a section called "Index of Useful Classes". + +A number of pretty diagrams have been added to the "System Architecture" +portion of the manual, explaining how all the buildbot pieces fit together. + +** bugfixes + +SF#1398174: ignore SVN property changes better, fixed by Olivier Bonnet + +SF#1452801: don't double-escape the build URL, fixed by Olivier Bonnet + +SF#1401121: add support for running py2exe on windows, by Mark Hammond + +reloading unchanged config files with WithProperties shouldn't change anything. + +All svn commands now include --non-interactive so they won't ask for +passwords. Instead the command fails if it cannot be performed without user +input. + +Deprecation warnings with newer versions of Twisted have been hushed. + +** compatibility + +I haven't actually removed support for Twisted-1.3.0 yet, but I'd like to. + +The step_twisted default value for --reporter matches modern Twisteds, +though, and won't work under 1.3.0. + +ShellCommand.flunkOnFailure now defaults to True, so any shell command which +fails counts as a build failure. Set this to False if you don't want this +behavior. + +** minor features + +contrib/darcs_buildbot.py contains a new script suitable for use in a darcs +commit-hook. + +Hovering a cursor over the yellow "Build #123" box in the Waterfall display +will pop up an HTML tooltip to show the reason for the build. Thanks to Zandr +Milewski for the suggestion. + + * Release 0.7.3 (23 May 2006) ** compatibility From warner at users.sourceforge.net Sun Aug 6 22:37:22 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 22:37:22 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test emitlogs.py, 1.2, 1.3 test_shell.py, 1.2, 1.3 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12486/buildbot/test Modified Files: emitlogs.py test_shell.py Log Message: [project @ LogFileWatcher: survive logfiles which aren't around at startup correctly] Original author: warner at lothar.com Date: 2006-08-06 22:34:59 Index: emitlogs.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/emitlogs.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- emitlogs.py 20 Jun 2006 08:08:58 -0000 1.2 +++ emitlogs.py 6 Aug 2006 22:37:20 -0000 1.3 @@ -1,9 +1,27 @@ #! /usr/bin/python -import sys, time +import sys, time, os.path, StringIO -log2 = open("log2.out", "wt") -log3 = open("log3.out", "wt") +mode = 0 +if len(sys.argv) > 1: + mode = int(sys.argv[1]) + +if mode == 0: + log2 = open("log2.out", "wt") + log3 = open("log3.out", "wt") +elif mode == 1: + # delete the logfiles first, and wait a moment to exercise a failure path + if os.path.exists("log2.out"): + os.unlink("log2.out") + if os.path.exists("log3.out"): + os.unlink("log3.out") + time.sleep(2) + log2 = open("log2.out", "wt") + log3 = open("log3.out", "wt") +elif mode == 2: + # don't create the logfiles at all + log2 = StringIO.StringIO() + log3 = StringIO.StringIO() def write(i): log2.write("this is log2 %d\n" % i) Index: test_shell.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_shell.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- test_shell.py 20 Jun 2006 08:08:58 -0000 1.2 +++ test_shell.py 6 Aug 2006 22:37:20 -0000 1.3 @@ -35,27 +35,50 @@ lines.append("this is %s %d\n" % (filename, i)) return "".join(lines) - def testLogFiles(self): + def testLogFiles_0(self): + return self._testLogFiles(0) + + def testLogFiles_1(self): + return self._testLogFiles(1) + + def testLogFiles_2(self): + return self._testLogFiles(2) + + def testLogFiles_3(self): + return self._testLogFiles(3) + + def _testLogFiles(self, mode): basedir = "test_shell.testLogFiles" self.setUpBuilder(basedir) # emitlogs.py writes two lines to stdout and two logfiles, one second # apart. Then it waits for us to write something to stdin, then it # writes one more line. - # we write something to the log file first, to exercise the logic - # that distinguishes between the old file and the one as modified by - # the ShellCommand. We set the timestamp back 5 seconds so that - # timestamps can be used to distinguish old from new. - log2file = os.path.join(basedir, "log2.out") - f = open(log2file, "w") - f.write("dummy text\n") - f.close() - earlier = time.time() - 5 - os.utime(log2file, (earlier, earlier)) + if mode != 3: + # we write something to the log file first, to exercise the logic + # that distinguishes between the old file and the one as modified + # by the ShellCommand. We set the timestamp back 5 seconds so + # that timestamps can be used to distinguish old from new. + log2file = os.path.join(basedir, "log2.out") + f = open(log2file, "w") + f.write("dummy text\n") + f.close() + earlier = time.time() - 5 + os.utime(log2file, (earlier, earlier)) + if mode == 3: + # mode=3 doesn't create the old logfiles in the first place, but + # then behaves like mode=1 (where the command pauses before + # creating them). + mode = 1 + + # mode=1 will cause emitlogs.py to delete the old logfiles first, and + # then wait two seconds before creating the new files. mode=0 does + # not do this. args = { 'command': [sys.executable, - util.sibpath(__file__, "emitlogs.py")], + util.sibpath(__file__, "emitlogs.py"), + "%s" % mode], 'workdir': ".", 'logfiles': {"log2": "log2.out", "log3": "log3.out"}, @@ -74,10 +97,14 @@ d.addCallback(self.collectUpdates) def _check(logs): self.failUnlessEqual(logs['stdout'], self._generateText("stdout")) - self.failUnlessEqual(logs[('log','log2')], - self._generateText("log2")) - self.failUnlessEqual(logs[('log','log3')], - self._generateText("log3")) + if mode == 2: + self.failIf(('log','log2') in logs) + self.failIf(('log','log3') in logs) + else: + self.failUnlessEqual(logs[('log','log2')], + self._generateText("log2")) + self.failUnlessEqual(logs[('log','log3')], + self._generateText("log3")) d.addCallback(_check) d.addBoth(self._maybePrintError) return maybeWait(d) From warner at users.sourceforge.net Sun Aug 6 22:37:22 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 22:37:22 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.57,1.58 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12486/buildbot/slave Modified Files: commands.py Log Message: [project @ LogFileWatcher: survive logfiles which aren't around at startup correctly] Original author: warner at lothar.com Date: 2006-08-06 22:34:59 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.57 retrieving revision 1.58 diff -u -d -r1.57 -r1.58 --- commands.py 6 Aug 2006 02:49:34 -0000 1.57 +++ commands.py 6 Aug 2006 22:37:20 -0000 1.58 @@ -162,11 +162,16 @@ self.poller = task.LoopingCall(self.poll) def start(self): - self.poller.start(self.POLL_INTERVAL) + self.poller.start(self.POLL_INTERVAL).addErrback(self._cleanupPoll) + + def _cleanupPoll(self, err): + log.err(err, msg="Polling error") + self.poller = None def stop(self): self.poll() - self.poller.stop() + if self.poller is not None: + self.poller.stop() def statFile(self): if os.path.exists(self.logfile): @@ -179,6 +184,12 @@ s = self.statFile() if s == self.old_logfile_stats: return # not started yet + if not s: + # the file was there, but now it's deleted. Forget about the + # initial state, clearly the process has deleted the logfile + # in preparation for creating a new one. + self.old_logfile_stats = None + return # no file to work with self.f = open(self.logfile, "rb") self.started = True while True: From warner at users.sourceforge.net Sun Aug 6 22:37:22 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 22:37:22 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.697,1.698 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12486 Modified Files: ChangeLog Log Message: [project @ LogFileWatcher: survive logfiles which aren't around at startup correctly] Original author: warner at lothar.com Date: 2006-08-06 22:34:59 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.697 retrieving revision 1.698 diff -u -d -r1.697 -r1.698 --- ChangeLog 6 Aug 2006 21:14:44 -0000 1.697 +++ ChangeLog 6 Aug 2006 22:37:20 -0000 1.698 @@ -1,5 +1,14 @@ 2006-08-06 Brian Warner + * buildbot/slave/commands.py (LogFileWatcher): handle logfiles + which are deleted (or not yet created) correctly. Also add + failsafe code to not explode if the file-watching poller doesn't + get started. Thanks to JP Calderone for the catch and the poller + patch. + * buildbot/test/test_shell.py (SlaveSide._testLogFiles): add test + for that case + * buildbot/test/emitlogs.py: same + * NEWS: summarize recent changes * docs/buildbot.texinfo (Debug options): suggest an .ssh/options From warner at users.sourceforge.net Sun Aug 6 23:29:55 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 23:29:55 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_vc.py,1.64,1.65 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv32463/buildbot/test Modified Files: test_vc.py Log Message: [project @ test.test_vc.BaseHelper.dovc: let command be a list] Original author: Kevin Turner Date: 2006-08-01 22:52:01 Index: test_vc.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_vc.py,v retrieving revision 1.64 retrieving revision 1.65 diff -u -d -r1.64 -r1.65 --- test_vc.py 20 Jun 2006 08:08:58 -0000 1.64 +++ test_vc.py 6 Aug 2006 23:29:53 -0000 1.65 @@ -200,6 +200,9 @@ } ''' +def wq(s): + return s.split() + class VCS_Helper: # this is a helper class which keeps track of whether each VC system is # available, and whether the repository for each has been created. There @@ -386,7 +389,11 @@ def dovc(self, basedir, command, failureIsOk=False, stdin=None): """Like do(), but the VC binary will be prepended to COMMAND.""" - command = self.vcexe + " " + command + if isinstance(command, (str, unicode)): + command = self.vcexe + " " + command + else: + # command is a list + command = [self.vcexe] + command return self.do(basedir, command, failureIsOk, stdin) class VCBase(SignalMixin): @@ -1538,31 +1545,31 @@ tmp = os.path.join(self.repbase, "darcstmp") os.makedirs(self.rep_trunk) - w = self.dovc(self.rep_trunk, "initialize") + w = self.dovc(self.rep_trunk, ["initialize"]) yield w; w.getResult() os.makedirs(self.rep_branch) - w = self.dovc(self.rep_branch, "initialize") + w = self.dovc(self.rep_branch, ["initialize"]) yield w; w.getResult() self.populate(tmp) - w = self.dovc(tmp, "initialize") + w = self.dovc(tmp, wq("initialize")) yield w; w.getResult() - w = self.dovc(tmp, "add -r .") + w = self.dovc(tmp, wq("add -r .")) yield w; w.getResult() - w = self.dovc(tmp, "record -a -m initial_import --skip-long-comment -A test at buildbot.sf.net") + w = self.dovc(tmp, wq("record -a -m initial_import --skip-long-comment -A test at buildbot.sf.net")) yield w; w.getResult() - w = self.dovc(tmp, "push -a %s" % self.rep_trunk) + w = self.dovc(tmp, ["push", "-a", self.rep_trunk]) yield w; w.getResult() - w = self.dovc(tmp, "changes --context") + w = self.dovc(tmp, wq("changes --context")) yield w; out = w.getResult() self.addTrunkRev(out) self.populate_branch(tmp) - w = self.dovc(tmp, "record -a --ignore-times -m commit_on_branch --skip-long-comment -A test at buildbot.sf.net") + w = self.dovc(tmp, wq("record -a --ignore-times -m commit_on_branch --skip-long-comment -A test at buildbot.sf.net")) yield w; w.getResult() - w = self.dovc(tmp, "push -a %s" % self.rep_branch) + w = self.dovc(tmp, ["push", "-a", self.rep_branch]) yield w; w.getResult() - w = self.dovc(tmp, "changes --context") + w = self.dovc(tmp, wq("changes --context")) yield w; out = w.getResult() self.addBranchRev(out) rmdirRecursive(tmp) @@ -1571,19 +1578,19 @@ def vc_revise(self): tmp = os.path.join(self.repbase, "darcstmp") os.makedirs(tmp) - w = self.dovc(tmp, "initialize") + w = self.dovc(tmp, wq("initialize")) yield w; w.getResult() - w = self.dovc(tmp, "pull -a %s" % self.rep_trunk) + w = self.dovc(tmp, ["pull", "-a", self.rep_trunk]) yield w; w.getResult() self.version += 1 version_c = VERSION_C % self.version open(os.path.join(tmp, "version.c"), "w").write(version_c) - w = self.dovc(tmp, "record -a --ignore-times -m revised_to_%d --skip-long-comment -A test at buildbot.sf.net" % self.version) + w = self.dovc(tmp, wq("record -a --ignore-times -m revised_to_%d --skip-long-comment -A test at buildbot.sf.net" % self.version)) yield w; w.getResult() - w = self.dovc(tmp, "push -a %s" % self.rep_trunk) + w = self.dovc(tmp, ["push", "-a", self.rep_trunk]) yield w; w.getResult() - w = self.dovc(tmp, "changes --context") + w = self.dovc(tmp, wq("changes --context")) yield w; out = w.getResult() self.addTrunkRev(out) rmdirRecursive(tmp) @@ -1594,13 +1601,13 @@ if os.path.exists(workdir): rmdirRecursive(workdir) os.makedirs(workdir) - w = self.dovc(workdir, "initialize") + w = self.dovc(workdir, wq("initialize")) yield w; w.getResult() if not branch: rep = self.rep_trunk else: rep = os.path.join(self.darcs_base, branch) - w = self.dovc(workdir, "pull -a %s" % rep) + w = self.dovc(workdir, ["pull", "-a", rep]) yield w; w.getResult() open(os.path.join(workdir, "subdir", "subdir.c"), "w").write(TRY_C) vc_try_checkout = deferredGenerator(vc_try_checkout) From warner at users.sourceforge.net Sun Aug 6 23:30:02 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 23:30:02 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.698,1.699 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv32482 Modified Files: ChangeLog Log Message: [project @ add docs to kevin's test_vc patch, rename helper function] Original author: warner at lothar.com Date: 2006-08-06 23:02:15 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.698 retrieving revision 1.699 diff -u -d -r1.698 -r1.699 --- ChangeLog 6 Aug 2006 22:37:20 -0000 1.698 +++ ChangeLog 6 Aug 2006 23:30:00 -0000 1.699 @@ -1,5 +1,13 @@ 2006-08-06 Brian Warner + * buildbot/test/test_vc.py (BaseHelper.dovc): patch from Kevin + Turner to prefer lists over strings when creating/running VC + commands during unit tests. This is clearly necessary to survive + vcexe containing spaces, like "C:\Program Files\darcs.exe". I + renamed the wq() function to qw() though, since that's how it's + spelled in perl. Eventually I'd prefer all commands to be + specified with lists. + * buildbot/slave/commands.py (LogFileWatcher): handle logfiles which are deleted (or not yet created) correctly. Also add failsafe code to not explode if the file-watching poller doesn't From warner at users.sourceforge.net Sun Aug 6 23:30:02 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 23:30:02 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_vc.py,1.65,1.66 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv32482/buildbot/test Modified Files: test_vc.py Log Message: [project @ add docs to kevin's test_vc patch, rename helper function] Original author: warner at lothar.com Date: 2006-08-06 23:02:15 Index: test_vc.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_vc.py,v retrieving revision 1.65 retrieving revision 1.66 diff -u -d -r1.65 -r1.66 --- test_vc.py 6 Aug 2006 23:29:53 -0000 1.65 +++ test_vc.py 6 Aug 2006 23:30:00 -0000 1.66 @@ -200,7 +200,7 @@ } ''' -def wq(s): +def qw(s): return s.split() class VCS_Helper: @@ -1552,24 +1552,24 @@ yield w; w.getResult() self.populate(tmp) - w = self.dovc(tmp, wq("initialize")) + w = self.dovc(tmp, qw("initialize")) yield w; w.getResult() - w = self.dovc(tmp, wq("add -r .")) + w = self.dovc(tmp, qw("add -r .")) yield w; w.getResult() - w = self.dovc(tmp, wq("record -a -m initial_import --skip-long-comment -A test at buildbot.sf.net")) + w = self.dovc(tmp, qw("record -a -m initial_import --skip-long-comment -A test at buildbot.sf.net")) yield w; w.getResult() w = self.dovc(tmp, ["push", "-a", self.rep_trunk]) yield w; w.getResult() - w = self.dovc(tmp, wq("changes --context")) + w = self.dovc(tmp, qw("changes --context")) yield w; out = w.getResult() self.addTrunkRev(out) self.populate_branch(tmp) - w = self.dovc(tmp, wq("record -a --ignore-times -m commit_on_branch --skip-long-comment -A test at buildbot.sf.net")) + w = self.dovc(tmp, qw("record -a --ignore-times -m commit_on_branch --skip-long-comment -A test at buildbot.sf.net")) yield w; w.getResult() w = self.dovc(tmp, ["push", "-a", self.rep_branch]) yield w; w.getResult() - w = self.dovc(tmp, wq("changes --context")) + w = self.dovc(tmp, qw("changes --context")) yield w; out = w.getResult() self.addBranchRev(out) rmdirRecursive(tmp) @@ -1578,7 +1578,7 @@ def vc_revise(self): tmp = os.path.join(self.repbase, "darcstmp") os.makedirs(tmp) - w = self.dovc(tmp, wq("initialize")) + w = self.dovc(tmp, qw("initialize")) yield w; w.getResult() w = self.dovc(tmp, ["pull", "-a", self.rep_trunk]) yield w; w.getResult() @@ -1586,11 +1586,11 @@ self.version += 1 version_c = VERSION_C % self.version open(os.path.join(tmp, "version.c"), "w").write(version_c) - w = self.dovc(tmp, wq("record -a --ignore-times -m revised_to_%d --skip-long-comment -A test at buildbot.sf.net" % self.version)) + w = self.dovc(tmp, qw("record -a --ignore-times -m revised_to_%d --skip-long-comment -A test at buildbot.sf.net" % self.version)) yield w; w.getResult() w = self.dovc(tmp, ["push", "-a", self.rep_trunk]) yield w; w.getResult() - w = self.dovc(tmp, wq("changes --context")) + w = self.dovc(tmp, qw("changes --context")) yield w; out = w.getResult() self.addTrunkRev(out) rmdirRecursive(tmp) @@ -1601,7 +1601,7 @@ if os.path.exists(workdir): rmdirRecursive(workdir) os.makedirs(workdir) - w = self.dovc(workdir, wq("initialize")) + w = self.dovc(workdir, qw("initialize")) yield w; w.getResult() if not branch: rep = self.rep_trunk From warner at users.sourceforge.net Sun Aug 6 23:30:20 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 23:30:20 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.58,1.59 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv382/buildbot/slave Modified Files: commands.py Log Message: [project @ slave.commands.ShellCommand: handle os.pathsep in PYTHONPATH] making PYTHONPATH even more of a magic special-case in the environs. Original author: Kevin Turner Date: 2006-08-02 02:58:45 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.58 retrieving revision 1.59 diff -u -d -r1.58 -r1.59 --- commands.py 6 Aug 2006 22:37:20 -0000 1.58 +++ commands.py 6 Aug 2006 23:30:18 -0000 1.59 @@ -231,15 +231,24 @@ self.workdir = workdir self.environ = os.environ.copy() if environ: - if (self.environ.has_key('PYTHONPATH') - and environ.has_key('PYTHONPATH')): - # special case, prepend the builder's items to the existing - # ones. This will break if you send over empty strings, so - # don't do that. - environ['PYTHONPATH'] = (environ['PYTHONPATH'] - + os.pathsep - + self.environ['PYTHONPATH']) - # this will proceed to replace the old one + if environ.has_key('PYTHONPATH'): + ppath = environ['PYTHONPATH'] + # Need to do os.pathsep translation. We could either do that + # by replacing all incoming ':'s with os.pathsep, or by + # accepting lists. I like lists better. + if not isinstance(ppath, str): + # If it's not a string, treat it as a sequence to be + # turned in to a string. + ppath = os.pathsep.join(ppath) + + if self.environ.has_key('PYTHONPATH'): + # special case, prepend the builder's items to the + # existing ones. This will break if you send over empty + # strings, so don't do that. + ppath = ppath + os.pathsep + self.environ['PYTHONPATH'] + + environ['PYTHONPATH'] = ppath + self.environ.update(environ) self.initialStdin = initialStdin self.keepStdinOpen = keepStdinOpen From warner at users.sourceforge.net Sun Aug 6 23:30:30 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 23:30:30 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.59,1.60 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv425/buildbot/slave Modified Files: commands.py Log Message: [project @ added docs to Kevin's PYTHONPATH-as-list patch] Original author: warner at lothar.com Date: 2006-08-06 23:29:10 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.59 retrieving revision 1.60 diff -u -d -r1.59 -r1.60 --- commands.py 6 Aug 2006 23:30:18 -0000 1.59 +++ commands.py 6 Aug 2006 23:30:28 -0000 1.60 @@ -676,7 +676,9 @@ - ['workdir'] (required): subdirectory in which the command will be run, relative to the builder dir - ['env']: a dict of environment variables to augment/replace - os.environ + os.environ . PYTHONPATH is treated specially, and + should be a list of path components to be prepended to + any existing PYTHONPATH environment variable. - ['initial_stdin']: a string which will be written to the command's stdin as soon as it starts - ['keep_stdin_open']: unless True, the command's stdin will be From warner at users.sourceforge.net Sun Aug 6 23:30:30 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 23:30:30 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.699,1.700 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv425 Modified Files: ChangeLog Log Message: [project @ added docs to Kevin's PYTHONPATH-as-list patch] Original author: warner at lothar.com Date: 2006-08-06 23:29:10 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.699 retrieving revision 1.700 diff -u -d -r1.699 -r1.700 --- ChangeLog 6 Aug 2006 23:30:00 -0000 1.699 +++ ChangeLog 6 Aug 2006 23:30:28 -0000 1.700 @@ -1,5 +1,16 @@ 2006-08-06 Brian Warner + * buildbot/slave/commands.py (ShellCommand.__init__): patch from + Kevin Turner to prefer the environ= argument be a list rather than + a string. If it is a list, it will be joined with a platform-local + os.pathsep delimiter, and then prepended to any existing + $PYTHONPATH value. This works better in cross-platform (i.e. + windows buildslaves) environments when you need to push multiple + directories onto the front of the path. + (SlaveShellCommand): documented the new magic + * docs/buildbot.texinfo (ShellCommand): documented the new magic + in a user-visible form + * buildbot/test/test_vc.py (BaseHelper.dovc): patch from Kevin Turner to prefer lists over strings when creating/running VC commands during unit tests. This is clearly necessary to survive From warner at users.sourceforge.net Sun Aug 6 23:30:31 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 06 Aug 2006 23:30:31 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.61,1.62 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv425/docs Modified Files: buildbot.texinfo Log Message: [project @ added docs to Kevin's PYTHONPATH-as-list patch] Original author: warner at lothar.com Date: 2006-08-06 23:29:10 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.61 retrieving revision 1.62 diff -u -d -r1.61 -r1.62 --- buildbot.texinfo 6 Aug 2006 21:14:44 -0000 1.61 +++ buildbot.texinfo 6 Aug 2006 23:30:28 -0000 1.62 @@ -3334,7 +3334,28 @@ @item env a dictionary of environment strings which will be added to the child -command's environment. +command's environment. For example, to run tests with a different i18n +language setting, you might use + + at example +s(ShellCommand, command=["make", "test"], + env=@{'LANG': 'fr_FR'@}) + at end example + +These variable settings will override any existing ones in the +buildslave's environment. The exception is PYTHONPATH, which is merged +with (actually prepended to) any existing $PYTHONPATH setting. The +value is treated as a list of directories to prepend, and a single +string is treated like a one-item list. For example, to prepend both + at file{/usr/local/lib/python2.3} and @file{/home/buildbot/lib/python} +to any existing $PYTHONPATH setting, you would do something like the +following: + + at example +s(ShellCommand, command=["make", "test"], + env=@{'PYTHONPATH': ["/usr/local/lib/python2.3", + "/home/buildbot/lib/python"] @}) + at end example @item want_stdout if False, stdout from the child process is discarded rather than being From warner at users.sourceforge.net Mon Aug 7 00:14:30 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 07 Aug 2006 00:14:30 +0000 Subject: [Buildbot-commits] site ChangeLog, 1.43, 1.44 index.html, 1.70, 1.71 manual-CVS.html, 1.5, 1.6 Message-ID: Update of /cvsroot/buildbot/site In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17209 Modified Files: ChangeLog index.html manual-CVS.html Log Message: updates Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/site/ChangeLog,v retrieving revision 1.43 retrieving revision 1.44 diff -u -d -r1.43 -r1.44 --- ChangeLog 31 Jul 2006 08:36:52 -0000 1.43 +++ ChangeLog 7 Aug 2006 00:14:27 -0000 1.44 @@ -1,3 +1,8 @@ +2006-08-06 Brian Warner + + * index.html: added Cheesecake, ReactOS + * manual-CVS.html: updated + 2006-07-31 Brian Warner * manual-CVS.html: updated, added images Index: index.html =================================================================== RCS file: /cvsroot/buildbot/site/index.html,v retrieving revision 1.70 retrieving revision 1.71 diff -u -d -r1.70 -r1.71 --- index.html 12 Jul 2006 21:36:32 -0000 1.70 +++ index.html 7 Aug 2006 00:14:27 -0000 1.71 @@ -520,7 +520,36 @@ composition of the highest energy cosmic rays, and is operated by a collaboration of about 300 physicists (running code on quite a few different platforms)". - + + + + Cheesecake + + Buildbot + + + home page + + Michal Kwiatkowski reports that the pycheesecake project is using + a buildbot to run all their functional and unit tests. This project + ranks many python packages according to such quality metrics like + pylint scores, percentage of documented methods, and + downloadability. + + + + ReactOS + + Buildbot + + + home page + + Aleksey Bragin has set up a buildbot for the ReactOS project, to + automate the creation of installation CD images. This project is + dedicated to creating a GPLed windows-compatible operating system + from the ground up. + @@ -574,5 +603,5 @@ href="http://creativecommons.org/licenses/by-sa/2.5/">Creative Commons Attribution Share-Alike license.

-Last modified: Wed Jul 12 14:32:41 PDT 2006 +Last modified: Sun Aug 6 17:11:15 PDT 2006 Index: manual-CVS.html =================================================================== RCS file: /cvsroot/buildbot/site/manual-CVS.html,v retrieving revision 1.5 retrieving revision 1.6 diff -u -d -r1.5 -r1.6 --- manual-CVS.html 31 Jul 2006 08:36:52 -0000 1.5 +++ manual-CVS.html 7 Aug 2006 00:14:27 -0000 1.6 @@ -2765,7 +2765,22 @@

Note that using any Manhole requires that the TwistedConch package be installed, and that you be using Twisted version 2.0 or later. -

+

The buildmaster's SSH server will use a different host key than the +normal sshd running on a typical unix host. This will cause the ssh +client to complain about a “host key mismatch”, because it does not +realize there are two separate servers running on the same host. To +avoid this, use a clause like the following in your .ssh/config +file: + +

     Host remotehost-buildbot
+      HostName remotehost
+      HostKeyAlias remotehost-buildbot
+      Port 9999
+      # use 'user' if you use PasswordManhole and your name is not 'admin'.
+      # if you use AuthorizedKeysManhole, this probably doesn't matter.
+      User admin
+
+

Next: , @@ -3789,8 +3804,25 @@ specifies the command to be run
env
a dictionary of environment strings which will be added to the child -command's environment. +command's environment. For example, to run tests with a different i18n +language setting, you might use + + 
          s(ShellCommand, command=["make", "test"],
+            env={'LANG': 'fr_FR'})
+
+

These variable settings will override any existing ones in the +buildslave's environment. The exception is PYTHONPATH, which is merged +with (actually prepended to) any existing $PYTHONPATH setting. The +value is treated as a list of directories to prepend, and a single +string is treated like a one-item list. For example, to prepend both +/usr/local/lib/python2.3 and /home/buildbot/lib/python +to any existing $PYTHONPATH setting, you would do something like the +following: + 

          s(ShellCommand, command=["make", "test"],
+           env={'PYTHONPATH': ["/usr/local/lib/python2.3",
+                               "/home/buildbot/lib/python"] })
+

want_stdout
if False, stdout from the child process is discarded rather than being sent to the buildmaster for inclusion in the step's LogFile. From warner at users.sourceforge.net Mon Aug 7 00:25:33 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 07 Aug 2006 00:25:33 +0000 Subject: [Buildbot-commits] site ChangeLog,1.44,1.45 index.html,1.71,1.72 Message-ID: Update of /cvsroot/buildbot/site In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21456 Modified Files: ChangeLog index.html Log Message: updated Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/site/ChangeLog,v retrieving revision 1.44 retrieving revision 1.45 diff -u -d -r1.44 -r1.45 --- ChangeLog 7 Aug 2006 00:14:27 -0000 1.44 +++ ChangeLog 7 Aug 2006 00:25:31 -0000 1.45 @@ -1,6 +1,8 @@ 2006-08-06 Brian Warner * index.html: added Cheesecake, ReactOS + add a link to manual-CVS.html + * manual-CVS.html: updated 2006-07-31 Brian Warner Index: index.html =================================================================== RCS file: /cvsroot/buildbot/site/index.html,v retrieving revision 1.71 retrieving revision 1.72 diff -u -d -r1.71 -r1.72 --- index.html 7 Aug 2006 00:14:27 -0000 1.71 +++ index.html 7 Aug 2006 00:25:31 -0000 1.72 @@ -55,7 +55,9 @@ or read-only checkout. There are also Arch and Darcs repositories which track the main CVS tree and provide lower-latency access than anonymous CVS, details - are here. + are here. There is also a copy of the CVS + user's manual available, which may give you + an idea of what sorts of features are coming up in the next release.
  • The README file contains an overview, while the User's Manual contains complete documentation. @@ -603,5 +605,5 @@ href="http://creativecommons.org/licenses/by-sa/2.5/">Creative Commons Attribution Share-Alike license.

    -Last modified: Sun Aug 6 17:11:15 PDT 2006 +Last modified: Sun Aug 6 17:24:52 PDT 2006 From warner at users.sourceforge.net Fri Aug 11 07:30:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:30:51 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.700,1.701 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26010 Modified Files: ChangeLog Log Message: [project @ reimplement 'useful classes' index with real texinfo indices] Original author: warner at lothar.com Date: 2006-08-11 03:59:59 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.700 retrieving revision 1.701 diff -u -d -r1.700 -r1.701 --- ChangeLog 6 Aug 2006 23:30:28 -0000 1.700 +++ ChangeLog 11 Aug 2006 07:30:49 -0000 1.701 @@ -1,3 +1,11 @@ +2006-08-10 Brian Warner + + * docs/buildbot.texinfo: reimplement the "useful classes" index + with actual texinfo indices. The .info rendering is a bit + weird-looking but it works well, and the HTML rendering is quite + nice. This also puts the index targets in the regular flow of the + text, which is easier to maintain. + 2006-08-06 Brian Warner * buildbot/slave/commands.py (ShellCommand.__init__): patch from From warner at users.sourceforge.net Fri Aug 11 07:30:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:30:51 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.62,1.63 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26010/docs Modified Files: buildbot.texinfo Log Message: [project @ reimplement 'useful classes' index with real texinfo indices] Original author: warner at lothar.com Date: 2006-08-11 03:59:59 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.62 retrieving revision 1.63 diff -u -d -r1.62 -r1.63 --- buildbot.texinfo 6 Aug 2006 23:30:28 -0000 1.62 +++ buildbot.texinfo 11 Aug 2006 07:30:49 -0000 1.63 @@ -2,8 +2,20 @@ @c %**start of header @setfilename buildbot.info @settitle BuildBot Manual 0.7.3+ + at defcodeindex cs + at defcodeindex sl + at defcodeindex bf + at defcodeindex bs + at defcodeindex st @c %**end of header + at c these indices are for classes useful in a master.cfg config file + at c @csindex : Change Sources + at c @slindex : Schedulers and Locks + at c @bfindex : Build Factories + at c @bsindex : Build Steps + at c @stindex : Status Targets + @copying This is the BuildBot manual. @@ -1996,6 +2008,10 @@ @end menu @node Scheduler Types, Build Dependencies, Listing Change Sources and Schedulers, Listing Change Sources and Schedulers + at slindex buildbot.scheduler.Scheduler + at slindex buildbot.scheduler.AnyBranchScheduler + at slindex buildbot.scheduler.Periodic + at slindex buildbot.scheduler.Nightly @subsection Scheduler Types Here is a brief catalog of the available Scheduler types. All these @@ -2044,10 +2060,10 @@ @end table @node Build Dependencies, , Scheduler Types, Listing Change Sources and Schedulers - at subsection Build Dependencies - @cindex Dependent @cindex Dependencies + at slindex buildbot.scheduler.Dependent + at subsection Build Dependencies It is common to wind up with one kind of build which should only be performed if the same source code was successfully handled by some @@ -2523,6 +2539,7 @@ @node CVSToys - PBService, CVSToys - mail notification, Choosing ChangeSources, Change Sources + at csindex buildbot.changes.freshcvs.FreshCVSSource @subsection CVSToys - PBService The @uref{http://purl.net/net/CVSToys, CVSToys} package provides a @@ -2595,6 +2612,7 @@ @node CVSToys - mail notification, Other mail notification ChangeSources, CVSToys - PBService, Change Sources + at csindex buildbot.changes.mail.FCMaildirSource @subsection CVSToys - mail notification CVSToys also provides a @code{MailNotification} action which will send @@ -2629,6 +2647,8 @@ directory name of the maildir root, and the prefix to strip. @node Other mail notification ChangeSources, PBChangeSource, CVSToys - mail notification, Change Sources + at csindex buildbot.changes.mail.SyncmailMaildirSource + at csindex buildbot.changes.mail.BonsaiMaildirSource @subsection Other mail notification ChangeSources There are other types of maildir-watching ChangeSources, which only @@ -2640,6 +2660,7 @@ @code{BonsaiMaildirSource} parses messages sent out by Bonsai. @node PBChangeSource, P4Source, Other mail notification ChangeSources, Change Sources + at csindex buildbot.changes.pb.PBChangeSource @subsection PBChangeSource The last kind of ChangeSource actually listens on a TCP port for @@ -2693,6 +2714,7 @@ @node P4Source, , PBChangeSource, Change Sources + at csindex buildbot.changes.p4poller.P4Source @subsection P4Source The @code{P4Source} periodically polls a @uref{http://www.perforce.com/, @@ -2979,9 +3001,10 @@ @end menu @node CVS, SVN, Source Checkout, Source Checkout + at cindex CVS Checkout + at bsindex buildbot.process.step.CVS @subsubsection CVS - at cindex CVS Checkout The @code{CVS} build step performs a @uref{http://www.nongnu.org/cvs/, CVS} checkout or update. It takes the following arguments: @@ -3014,9 +3037,10 @@ @node SVN, Darcs, CVS, Source Checkout + at cindex SVN Checkout + at bsindex buildbot.process.step.SVN @subsubsection SVN - at cindex SVN Checkout The @code{SVN} build step performs a @uref{http://subversion.tigris.org, Subversion} checkout or update. @@ -3148,9 +3172,10 @@ ``features/newthing'' to get the svnurl for checkout. @node Darcs, Mercurial, SVN, Source Checkout + at cindex Darcs Checkout + at bsindex buildbot.process.step.Darcs @subsubsection Darcs - at cindex Darcs Checkout The @code{Darcs} build step performs a @uref{http://abridgegame.org/darcs/, Darcs} checkout or update. @@ -3190,9 +3215,10 @@ @end table @node Mercurial, Arch, Darcs, Source Checkout + at cindex Mercurial Checkout + at bsindex buildbot.process.step.Mercurial @subsubsection Mercurial - at cindex Mercurial Checkout The @code{Mercurial} build step performs a @uref{http://selenic.com/mercurial, Mercurial} (aka ``hg'') checkout @@ -3221,9 +3247,10 @@ @node Arch, Bazaar, Mercurial, Source Checkout + at cindex Arch Checkout + at bsindex buildbot.process.step.Arch @subsubsection Arch - at cindex Arch Checkout The @code{Arch} build step performs an @uref{http://gnuarch.org/, Arch} checkout or update using the @code{tla} client. It takes the @@ -3248,9 +3275,10 @@ @end table @node Bazaar, P4, Arch, Source Checkout + at cindex Bazaar Checkout + at bsindex buildbot.process.step.Bazaar @subsubsection Bazaar - at cindex Bazaar Checkout @code{Bazaar} is an alternate implementation of the Arch VC system, which uses a client named @code{baz}. The checkout semantics are just @@ -3264,9 +3292,11 @@ @node P4, , Bazaar, Source Checkout + at cindex Perforce Update + at bsindex buildbot.process.step.P4 + at c TODO @bsindex buildbot.process.step.P4Sync @subsubsection P4 - at cindex Perforce Update The @code{P4} build step creates a @uref{http://www.perforce.com/, Perforce} client specification and performs an update. @@ -3302,6 +3332,8 @@ @end table @node ShellCommand, Simple ShellCommand Subclasses, Source Checkout, Build Steps + at bsindex buildbot.process.step.ShellCommand + at c TODO @bsindex buildbot.process.step.TreeSize @subsection ShellCommand This is a useful base class for just about everything you might want @@ -3425,6 +3457,7 @@ @end menu @node Configure, Compile, Simple ShellCommand Subclasses, Simple ShellCommand Subclasses + at bsindex buildbot.process.step.Configure @subsubsection Configure This is intended to handle the @code{./configure} step from @@ -3433,6 +3466,7 @@ but you can change this by providing a @code{command=} parameter. @node Compile, Test, Configure, Simple ShellCommand Subclasses + at bsindex buildbot.process.step.Compile @subsubsection Compile This is meant to handle compiling or building a project written in C. The @@ -3442,6 +3476,7 @@ created). @node Test, Build Properties, Compile, Simple ShellCommand Subclasses + at bsindex buildbot.process.step.Test @subsubsection Test This is meant to handle unit tests. The default command is @code{make @@ -3721,9 +3756,10 @@ @node Interlocks, Build Factories, Build Steps, Build Process - at section Interlocks - @cindex locks + at slindex buildbot.locks.MasterLock + at slindex buildbot.locks.SlaveLock + at section Interlocks For various reasons, you may want to prevent certain Steps (or perhaps entire Builds) from running simultaneously. Limited CPU speed or @@ -3933,6 +3969,10 @@ @node BuildFactory, Process-Specific build factories, BuildStep Objects, Build Factories + at bfindex buildbot.process.factory.BuildFactory + at bfindex buildbot.process.factory.BasicBuildFactory + at c TODO: what is BasicSVN anyway? + at bfindex buildbot.process.factory.BasicSVN @subsection BuildFactory The default @code{BuildFactory}, provided in the @@ -4032,6 +4072,7 @@ @node Quick builds, , BuildFactory Attributes, BuildFactory + at bfindex buildbot.process.factory.QuickBuildFactory @subsubsection Quick builds The difference between a ``full build'' and a ``quick build'' is that @@ -4063,6 +4104,7 @@ @end menu @node GNUAutoconf, CPAN, Process-Specific build factories, Process-Specific build factories + at bfindex buildbot.process.factory.GNUAutoconf @subsubsection GNUAutoconf @uref{http://www.gnu.org/software/autoconf/, GNU Autoconf} is a @@ -4139,6 +4181,7 @@ @node CPAN, Python distutils, GNUAutoconf, Process-Specific build factories + at bfindex buildbot.process.factory.CPAN @subsubsection CPAN Most Perl modules available from the @uref{http://www.cpan.org/, CPAN} @@ -4172,6 +4215,7 @@ @node Python distutils, Python/Twisted/trial projects, CPAN, Process-Specific build factories + at bfindex buildbot.process.factory.Distutils @subsubsection Python distutils Most Python modules use the @code{distutils} package to provide @@ -4212,6 +4256,13 @@ @node Python/Twisted/trial projects, , Python distutils, Process-Specific build factories + at bfindex buildbot.process.factory.Trial + at c TODO: document these steps better + at bsindex buildbot.process.step_twisted.HLint + at bsindex buildbot.process.step_twisted.Trial + at bsindex buildbot.process.step_twisted.ProcessDocs + at bsindex buildbot.process.step_twisted.BuildDebs + at bsindex buildbot.process.step_twisted.RemovePYCs @subsubsection Python/Twisted/trial projects Twisted provides a unit test tool named @code{trial} which provides a @@ -4397,10 +4448,16 @@ * Writing New Status Plugins:: @end menu + at c @node Email Delivery, , Status Delivery, Status Delivery + at c @subsection Email Delivery + + at c DOCUMENT THIS + @node HTML Waterfall, IRC Bot, Status Delivery, Status Delivery + at cindex Waterfall + at stindex buildbot.status.html.Waterfall @section HTML Waterfall - at cindex Waterfall @example from buildbot.status import html @@ -4479,9 +4536,10 @@ @node IRC Bot, PBListener, HTML Waterfall, Status Delivery + at cindex IRC + at stindex buildbot.status.words.IRC @section IRC Bot - at cindex IRC The @code{buildbot.status.words.IRC} status target creates an IRC bot which will attach to certain channels and be available for status @@ -4549,9 +4607,10 @@ @end table @node PBListener, Writing New Status Plugins, IRC Bot, Status Delivery + at cindex PBListener + at stindex buildbot.status.client.PBListener @section PBListener - at cindex PBListener @example import buildbot.status.client @@ -4740,6 +4799,9 @@ @heading TryScheduler + at slindex buildbot.scheduler.Try_Jobdir + at slindex buildbot.scheduler.Try_Userpass + The buildmaster must have a @code{scheduler.Try} instance in the config file's @code{c['schedulers']} list. This lets the administrator control who may initiate these ``trial'' builds, which @@ -5257,130 +5319,30 @@ Classes that are not listed here are generally internal things that admins are unlikely to have much use for. - at heading Change Sources - - at table @code - at item buildbot.changes.freshcvs.FreshCVSSource - at ref{CVSToys - PBService} - - at item buildbot.changes.mail.FCMaildirSource - at ref{CVSToys - mail notification} - - at item buildbot.changes.mail.SyncmailMaildirSource - at ref{Other mail notification ChangeSources} - - at item buildbot.changes.mail.BonsaiMaildirSource - at ref{Other mail notification ChangeSources} - - at item buildbot.changes.p4poller.P4Source - at ref{P4Source} - - at item buildbot.changes.pb.PBChangeSource - at ref{PBChangeSource} - - at end table + at heading Change Sources + at printindex cs @heading Schedulers and Locks - at table @code - at item buildbot.scheduler.Scheduler - at ref{Scheduler Types} - - at item buildbot.scheduler.AnyBranchScheduler - at ref{Scheduler Types} - - at item buildbot.scheduler.Dependent - at ref{Build Dependencies} - - at item buildbot.scheduler.Periodic - at ref{Scheduler Types} - - at item buildbot.scheduler.Nightly - at ref{Scheduler Types} - - at item buildbot.scheduler.Try_Jobdir - at ref{try} - at item buildbot.scheduler.Try_Userpass - at ref{try} - - at item buildbot.locks.MasterLock - at ref{Interlocks} - - at item buildbot.locks.SlaveLock - at ref{Interlocks} - - at end table + at printindex sl @heading Build Factories - at table @code - at item buildbot.process.factory.BuildFactory - at ref{BuildFactory} - at item buildbot.process.factory.GNUAutoconf - at ref{GNUAutoconf} - at item buildbot.process.factory.CPAN - at ref{CPAN} - at item buildbot.process.factory.Distutils - at ref{Python distutils} - at item buildbot.process.factory.Trial - at ref{Python/Twisted/trial projects} - at item buildbot.process.factory.BasicBuildFactory - at item buildbot.process.factory.QuickBuildFactory - at item buildbot.process.factory.BasicSVN - at end table + at printindex bf @heading Build Steps - at table @code - at item buildbot.process.step.ShellCommand - at ref{ShellCommand} - at item buildbot.process.step.TreeSize - at item buildbot.process.step.CVS - at c TODO: this CVS link is broken, it goes to a CVSToys page for some reason - at ref{CVS} - at item buildbot.process.step.SVN - at ref{SVN} - at item buildbot.process.step.Darcs - at ref{Darcs} - at item buildbot.process.step.Git - at item buildbot.process.step.Arch - at ref{Arch} - at item buildbot.process.step.Bazaar - at ref{Bazaar} - at item buildbot.process.step.Mercurial - at ref{Mercurial} - at item buildbot.process.step.P4 - at c TODO: this P4 like goes to P4Source by mistake - at ref{P4} - at item buildbot.process.step.P4Sync - at item buildbot.process.step.Configure - at ref{Configure} - at item buildbot.process.step.Compile - at ref{Compile} - at item buildbot.process.step.Test - at ref{Test} - at item buildbot.process.step_twisted.HLint - at item buildbot.process.step_twisted.Trial - at item buildbot.process.step_twisted.ProcessDocs - at item buildbot.process.step_twisted.BuildDebs - at item buildbot.process.step_twisted.RemovePYCs - at item buildbot.process.maxq.MaxQ - at end table - - - at heading Status Targets - at table @code + at printindex bs - at item buildbot.status.html.Waterfall - at ref{HTML Waterfall} + at c undocumented steps + at bsindex buildbot.process.step.Git + at bsindex buildbot.process.maxq.MaxQ - at item buildbot.status.mail.MailNotifier - at item buildbot.status.words.IRC - at ref{IRC Bot} + at heading Status Targets + at printindex st - at item buildbot.status.client.PBListener - at ref{PBListener} + at c TODO: undocumented targets + at stindex buildbot.status.mail.MailNotifier - at end table @node Index, , Index of Useful Classes, Top @unnumbered Index From warner at users.sourceforge.net Fri Aug 11 07:31:05 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:05 +0000 Subject: [Buildbot-commits] buildbot/buildbot/scripts runner.py,1.43,1.44 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/scripts In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26041/buildbot/scripts Modified Files: runner.py Log Message: [project @ rename 'buildbot master' to 'buildbot create-master', likewise with create-slave] Original author: warner at lothar.com Date: 2006-08-11 04:45:26 Index: runner.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/scripts/runner.py,v retrieving revision 1.43 retrieving revision 1.44 diff -u -d -r1.43 -r1.44 --- runner.py 28 Jun 2006 17:42:16 -0000 1.43 +++ runner.py 11 Aug 2006 07:31:03 -0000 1.44 @@ -162,7 +162,7 @@ ["config", "c", "master.cfg", "name of the buildmaster config file"], ] def getSynopsis(self): - return "Usage: buildbot master [options] " + return "Usage: buildbot create-master [options] " longdesc = """ This command creates a buildmaster working directory and buildbot.tac @@ -229,7 +229,7 @@ """ def getSynopsis(self): - return "Usage: buildbot slave [options] " + return "Usage: buildbot create-slave [options] " def parseArgs(self, *args): if len(args) < 4: @@ -660,9 +660,9 @@ subCommands = [ # the following are all admin commands - ['master', None, MasterOptions, + ['create-master', None, MasterOptions, "Create and populate a directory for a new buildmaster"], - ['slave', None, SlaveOptions, + ['create-slave', None, SlaveOptions, "Create and populate a directory for a new buildslave"], ['start', None, StartOptions, "Start a buildmaster or buildslave"], ['stop', None, StopOptions, "Stop a buildmaster or buildslave"], @@ -720,9 +720,9 @@ command = config.subCommand so = config.subOptions - if command == "master": + if command == "create-master": createMaster(so) - elif command == "slave": + elif command == "create-slave": createSlave(so) elif command == "start": start(so) From warner at users.sourceforge.net Fri Aug 11 07:31:05 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:05 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.63,1.64 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26041/docs Modified Files: buildbot.texinfo Log Message: [project @ rename 'buildbot master' to 'buildbot create-master', likewise with create-slave] Original author: warner at lothar.com Date: 2006-08-11 04:45:26 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.63 retrieving revision 1.64 diff -u -d -r1.63 -r1.64 --- buildbot.texinfo 11 Aug 2006 07:30:49 -0000 1.63 +++ buildbot.texinfo 11 Aug 2006 07:31:03 -0000 1.64 @@ -346,10 +346,10 @@ responsible for everything). The buildmaster is usually fed @code{Changes} by some sort of version -control system @xref{Change Sources}, which may cause builds to be +control system (@pxref{Change Sources}), which may cause builds to be run. As the builds are performed, various status messages are produced, which are then sent to any registered Status Targets - at xref{Status Delivery}. +(@pxref{Status Delivery}). @c @image{FILENAME, WIDTH, HEIGHT, ALTTEXT, EXTENSION} @image{images/overview,,,Overview Diagram,} @@ -700,13 +700,14 @@ slaves, you may want a more distinctive name like @file{~/Buildbot/master/gnomovision} or @file{~/Buildmasters/fooproject}. If you are using a separate user -account, this might just be @file{~buildmaster/masters/fooprojects}. +account, this might just be @file{~buildmaster/masters/fooproject}. -Once you've picked a directory, use the @command{buildbot master} -command to create the directory and populate it with startup files: +Once you've picked a directory, use the @command{buildbot +create-master} command to create the directory and populate it with +startup files: @example -buildbot master @var{basedir} +buildbot create-master @var{basedir} @end example You will need to create a configuration file (@pxref{Configuration}) @@ -825,7 +826,7 @@ Now run the 'buildbot' command as follows: @example -buildbot slave @var{BASEDIR} @var{MASTERHOST}:@var{PORT} @var{SLAVENAME} @var{PASSWORD} +buildbot create-slave @var{BASEDIR} @var{MASTERHOST}:@var{PORT} @var{SLAVENAME} @var{PASSWORD} @end example This will create the base directory and a collection of files inside, @@ -867,13 +868,13 @@ @subsection Buildslave Options There are a handful of options you might want to use when creating the -buildslave with the @command{buildbot slave DIR } -command. You can type @command{buildbot slave --help} for a summary. -To use these, just include them on the @command{buildbot slave} +buildslave with the @command{buildbot create-slave DIR } +command. You can type @command{buildbot create-slave --help} for a summary. +To use these, just include them on the @command{buildbot create-slave} command line, like this: @example -buildbot slave --umask=022 ~/buildslave buildmaster.example.org:42012 myslavename mypasswd +buildbot create-slave --umask=022 ~/buildslave buildmaster.example.org:42012 myslavename mypasswd @end example @table @code @@ -1774,11 +1775,11 @@ The buildbot's behavior is defined by the ``config file'', which normally lives in the @file{master.cfg} file in the buildmaster's base directory (but this can be changed with an option to the - at code{buildbot master} command). This file completely specifies which -Builders are to be run, which slaves they should use, how Changes -should be tracked, and where the status information is to be sent. The -buildmaster's @file{buildbot.tac} file names the base directory; -everything else comes from the config file. + at code{buildbot create-master} command). This file completely specifies +which Builders are to be run, which slaves they should use, how +Changes should be tracked, and where the status information is to be +sent. The buildmaster's @file{buildbot.tac} file names the base +directory; everything else comes from the config file. A sample config file was installed for you when you created the buildmaster, but you will need to edit it before your buildbot will do @@ -3351,7 +3352,7 @@ All ShellCommands are run by default in the ``workdir'', which defaults to the ``@file{build}'' subdirectory of the slave builder's base directory. The absolute path of the workdir will thus be the -slave's basedir (set as an option to @code{buildbot slave}, +slave's basedir (set as an option to @code{buildbot create-slave}, @pxref{Creating a buildslave}) plus the builder's basedir (set in the builder's @code{c['builddir']} key in master.cfg) plus the workdir itself (a class-level attribute of the BuildFactory, defaults to @@ -4675,16 +4676,16 @@ The following @command{buildbot} sub-commands are intended for buildmaster administrators: - at heading master + at heading create-master This creates a new directory and populates it with files that allow it to be used as a buildmaster's base directory. @example -buildbot master BASEDIR +buildbot create-master BASEDIR @end example - at heading slave + at heading create-slave This creates a new directory and populates it with files that let it be used as a buildslave's base directory. You must provide several @@ -4692,7 +4693,7 @@ file. @example -buildbot slave @var{BASEDIR} @var{MASTERHOST}:@var{PORT} @var{SLAVENAME} @var{PASSWORD} +buildbot create-slave @var{BASEDIR} @var{MASTERHOST}:@var{PORT} @var{SLAVENAME} @var{PASSWORD} @end example @heading start From warner at users.sourceforge.net Fri Aug 11 07:31:05 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:05 +0000 Subject: [Buildbot-commits] buildbot ChangeLog, 1.701, 1.702 README, 1.31, 1.32 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26041 Modified Files: ChangeLog README Log Message: [project @ rename 'buildbot master' to 'buildbot create-master', likewise with create-slave] Original author: warner at lothar.com Date: 2006-08-11 04:45:26 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.701 retrieving revision 1.702 diff -u -d -r1.701 -r1.702 --- ChangeLog 11 Aug 2006 07:30:49 -0000 1.701 +++ ChangeLog 11 Aug 2006 07:31:03 -0000 1.702 @@ -1,5 +1,11 @@ 2006-08-10 Brian Warner + * buildbot/scripts/runner.py: rename 'buildbot master' to + 'buildbot create-master', and 'buildbot slave' to 'buildbot + create-slave' + * docs/buildbot.texinfo: same + * README: same + * docs/buildbot.texinfo: reimplement the "useful classes" index with actual texinfo indices. The .info rendering is a bit weird-looking but it works well, and the HTML rendering is quite Index: README =================================================================== RCS file: /cvsroot/buildbot/buildbot/README,v retrieving revision 1.31 retrieving revision 1.32 diff -u -d -r1.31 -r1.32 --- README 28 May 2006 07:56:08 -0000 1.31 +++ README 11 Aug 2006 07:31:03 -0000 1.32 @@ -130,7 +130,7 @@ Now run the 'buildbot' command as follows: - buildbot slave WORKDIR MASTERHOST:PORT SLAVENAME PASSWORD + buildbot create-slave WORKDIR MASTERHOST:PORT SLAVENAME PASSWORD This will create a file called "buildbot.tac", which bundles up all the state needed by the build slave application. Twisted has a tool called "twistd" @@ -176,9 +176,9 @@ SETTING UP A BUILD MASTER: Please read the user's manual for instructions. The short form is that you -use 'buildbot master MASTERDIR' to create the base directory, then you edit -the 'master.cfg' file to configure the buildmaster. Once this is ready, you -use 'buildbot START MASTERDIR' to launch it. +use 'buildbot create-master MASTERDIR' to create the base directory, then you +edit the 'master.cfg' file to configure the buildmaster. Once this is ready, +you use 'buildbot START MASTERDIR' to launch it. A sample configuration file will be created for you in WORKDIR/master.cfg . There are more examples in docs/examples/, and plenty of documentation in the From warner at users.sourceforge.net Fri Aug 11 07:31:25 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:25 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.702,1.703 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26391 Modified Files: ChangeLog Log Message: [project @ make sure images get built when rendering the manual] Original author: warner at lothar.com Date: 2006-08-11 04:47:29 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.702 retrieving revision 1.703 diff -u -d -r1.702 -r1.703 --- ChangeLog 11 Aug 2006 07:31:03 -0000 1.702 +++ ChangeLog 11 Aug 2006 07:31:23 -0000 1.703 @@ -1,5 +1,9 @@ 2006-08-10 Brian Warner + * docs/Makefile (images): make sure images get built when + rendering the manual + * docs/images/Makefile: same + * buildbot/scripts/runner.py: rename 'buildbot master' to 'buildbot create-master', and 'buildbot slave' to 'buildbot create-slave' From warner at users.sourceforge.net Fri Aug 11 07:31:26 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:26 +0000 Subject: [Buildbot-commits] buildbot/docs Makefile,1.2,1.3 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26391/docs Modified Files: Makefile Log Message: [project @ make sure images get built when rendering the manual] Original author: warner at lothar.com Date: 2006-08-11 04:47:29 Index: Makefile =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/Makefile,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- Makefile 19 Jul 2005 23:12:00 -0000 1.2 +++ Makefile 11 Aug 2006 07:31:23 -0000 1.3 @@ -2,13 +2,20 @@ buildbot.info: buildbot.texinfo makeinfo --fill-column=70 $< -buildbot.html: buildbot.texinfo +buildbot.html: buildbot.texinfo images-png makeinfo --no-split --html $< -buildbot.ps: buildbot.texinfo +buildbot.ps: buildbot.texinfo images-eps texi2dvi $< dvips buildbot.dvi rm buildbot.aux buildbot.cp buildbot.cps buildbot.fn buildbot.ky buildbot.log buildbot.pg buildbot.toc buildbot.tp buildbot.vr rm buildbot.dvi +.PHONY: images images-png images-eps +images: + $(MAKE) -C images all +images-png: + $(MAKE) -C images images-png +images-eps: + $(MAKE) -C images images-eps From warner at users.sourceforge.net Fri Aug 11 07:31:26 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:26 +0000 Subject: [Buildbot-commits] buildbot/docs/images Makefile,1.1,1.2 Message-ID: Update of /cvsroot/buildbot/buildbot/docs/images In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26391/docs/images Modified Files: Makefile Log Message: [project @ make sure images get built when rendering the manual] Original author: warner at lothar.com Date: 2006-08-11 04:47:29 Index: Makefile =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/images/Makefile,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- Makefile 31 Jul 2006 08:31:33 -0000 1.1 +++ Makefile 11 Aug 2006 07:31:24 -0000 1.2 @@ -1,5 +1,12 @@ -all: overview.png overview.eps slaves.png slaves.eps slavebuilder.png slavebuilder.eps master.png master.eps status.png status.eps +SOURCES = overview.svg slaves.svg slavebuilder.svg master.svg status.svg +PNGS = $(patsubst %.svg,%.png,$(SOURCES)) +EPSS = $(patsubst %.svg,%.eps,$(SOURCES)) + +.PHONY: images-png images-eps +all: $(PNGS) $(EPSS) +images-png: $(PNGS) +images-eps: $(EPSS) %.png: %.svg inkscape -b white --export-png $@ $< From warner at users.sourceforge.net Fri Aug 11 07:31:39 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:39 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.64,1.65 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26452/docs Modified Files: buildbot.texinfo Log Message: [project @ docs: add another index, this one of all master.cfg keys] Original author: warner at lothar.com Date: 2006-08-11 05:04:29 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.64 retrieving revision 1.65 diff -u -d -r1.64 -r1.65 --- buildbot.texinfo 11 Aug 2006 07:31:03 -0000 1.64 +++ buildbot.texinfo 11 Aug 2006 07:31:37 -0000 1.65 @@ -7,6 +7,7 @@ @defcodeindex bf @defcodeindex bs @defcodeindex st + at defcodeindex bc @c %**end of header @c these indices are for classes useful in a master.cfg config file @@ -16,6 +17,8 @@ @c @bsindex : Build Steps @c @stindex : Status Targets + at c @bcindex : keys that make up BuildmasterConfig + @copying This is the BuildBot manual. @@ -57,6 +60,7 @@ * Resources:: Getting help. * Developer's Appendix:: * Index of Useful Classes:: +* Index of master.cfg keys:: * Index:: Complete index. @detailmenu @@ -1898,18 +1902,18 @@ c['buildbotURL'] = "http://localhost:8010/" @end example - at cindex c['projectName'] + at bcindex c['projectName'] @code{projectName} is a short string will be used to describe the project that this buildbot is working on. For example, it is used as the title of the waterfall HTML page. - at cindex c['projectURL'] + at bcindex c['projectURL'] @code{projectURL} is a string that gives a URL for the project as a whole. HTML status displays will show @code{projectName} as a link to @code{projectURL}, to provide a link from buildbot HTML pages to your project's home page. - at cindex c['buildbotURL'] + at bcindex c['buildbotURL'] The @code{buildbotURL} string should point to the location where the buildbot's internal web server (usually the @code{html.Waterfall} page) is visible. This typically uses the port number set when you @@ -1926,7 +1930,7 @@ @node Listing Change Sources and Schedulers, Setting the slaveport, Defining the Project, Configuration @section Listing Change Sources and Schedulers - at cindex c['sources'] + at bcindex c['sources'] The @code{c['sources']} key is a list of ChangeSource instances at footnote{To be precise, it is a list of objects which all implement the @code{buildbot.interfaces.IChangeSource} Interface}. @@ -1939,7 +1943,7 @@ c['sources'] = [buildbot.changes.pb.PBChangeSource()] @end example - at cindex c['schedulers'] + at bcindex c['schedulers'] @code{c['schedulers']} is a list of Scheduler instances, each of which causes builds to be started on a particular set of Builders. The two basic Scheduler classes you are likely to start with are @@ -2111,7 +2115,7 @@ @node Setting the slaveport, Buildslave Specifiers, Listing Change Sources and Schedulers, Configuration @section Setting the slaveport - at cindex c['slavePortnum'] + at bcindex c['slavePortnum'] The buildmaster will listen on a TCP port of your choosing for connections from buildslaves. It can also use this port for @@ -2148,7 +2152,7 @@ @node Buildslave Specifiers, Defining Builders, Setting the slaveport, Configuration @section Buildslave Specifiers - at cindex c['bots'] + at bcindex c['bots'] The @code{c['bots']} key is a list of known buildslaves. Each buildslave is defined by a tuple of (slavename, slavepassword). These @@ -2174,7 +2178,7 @@ @node Defining Builders, Defining Status Targets, Buildslave Specifiers, Configuration @section Defining Builders - at cindex c['builders'] + at bcindex c['builders'] The @code{c['builders']} key is a list of dictionaries which specify the Builders. The Buildmaster runs a collection of Builders, each of @@ -2259,7 +2263,7 @@ in the configuration's @code{status} list. To add status targets, you just append more objects to this list: - at cindex c['status'] + at bcindex c['status'] @example c['status'] = [] @@ -2286,7 +2290,7 @@ @section Debug options - at cindex c['debugPassword'] + at bcindex c['debugPassword'] If you set @code{c['debugPassword']}, then you can connect to the buildmaster with the diagnostic tool launched by @code{buildbot debugclient MASTER:PORT}. From this tool, you can reload the config @@ -2300,7 +2304,7 @@ c['debugPassword'] = "debugpassword" @end example - at cindex c['manhole'] + at bcindex c['manhole'] If you set @code{c['manhole']} to an instance of one of the classes in @code{buildbot.manhole}, you can telnet or ssh into the buildmaster and get an interactive Python shell, which may be useful for debugging @@ -5312,7 +5316,7 @@ @code{.builders} dictionary. - at node Index of Useful Classes, Index, Developer's Appendix, Top + at node Index of Useful Classes, Index of master.cfg keys, Developer's Appendix, Top @unnumbered Index of Useful Classes This is a list of all user-visible classes. There are the ones that @@ -5344,11 +5348,30 @@ @c TODO: undocumented targets @stindex buildbot.status.mail.MailNotifier + at node Index of master.cfg keys, Index, Index of Useful Classes, Top + at unnumbered Index of master.cfg keys - at node Index, , Index of Useful Classes, Top +This is a list of all of the significant keys in master.cfg . Recall +that master.cfg is effectively a small python program one +responsibility: create a dictionary named @code{BuildmasterConfig}. +The keys of this dictionary are listed here. The beginning of the +master.cfg file typically starts with something like: + + at example +BuildmasterConfig = c = @{@} + at end example + +Therefore a config key of @code{sources} will usually appear in +master.cfg as @code{c['sources']}. + + at printindex bc + + + at node Index, , Index of master.cfg keys, Top @unnumbered Index @printindex cp + @bye From warner at users.sourceforge.net Fri Aug 11 07:31:39 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:39 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.703,1.704 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26452 Modified Files: ChangeLog Log Message: [project @ docs: add another index, this one of all master.cfg keys] Original author: warner at lothar.com Date: 2006-08-11 05:04:29 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.703 retrieving revision 1.704 diff -u -d -r1.703 -r1.704 --- ChangeLog 11 Aug 2006 07:31:23 -0000 1.703 +++ ChangeLog 11 Aug 2006 07:31:37 -0000 1.704 @@ -1,5 +1,8 @@ 2006-08-10 Brian Warner + * docs/buildbot.texinfo (Index of master.cfg keys): add another + index, this one of things like c['sources'] and c['schedulers'] + * docs/Makefile (images): make sure images get built when rendering the manual * docs/images/Makefile: same From warner at users.sourceforge.net Fri Aug 11 07:31:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:51 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.704,1.705 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26471 Modified Files: ChangeLog Log Message: [project @ docs: fix index tags to avoid confusing texinfo] Original author: warner at lothar.com Date: 2006-08-11 07:29:10 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.704 retrieving revision 1.705 diff -u -d -r1.704 -r1.705 --- ChangeLog 11 Aug 2006 07:31:37 -0000 1.704 +++ ChangeLog 11 Aug 2006 07:31:49 -0000 1.705 @@ -2,6 +2,14 @@ * docs/buildbot.texinfo (Index of master.cfg keys): add another index, this one of things like c['sources'] and c['schedulers'] + (indices): it looks like my clever idea of putting the @fooindex + commands in between the @node and the @subsection (to make the + HREF anchor jump to slightly above the section title, which works + much better in html) confused texinfo horribly, so I'm moving the + index tags back to be just after the @subsection marker. I also + added extra lines between the @node/@section paragraph and the + index tags, since I think maybe texinfo wants to see these be + separate paragraphs. * docs/Makefile (images): make sure images get built when rendering the manual From warner at users.sourceforge.net Fri Aug 11 07:31:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 11 Aug 2006 07:31:51 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.65,1.66 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv26471/docs Modified Files: buildbot.texinfo Log Message: [project @ docs: fix index tags to avoid confusing texinfo] Original author: warner at lothar.com Date: 2006-08-11 07:29:10 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.65 retrieving revision 1.66 diff -u -d -r1.65 -r1.66 --- buildbot.texinfo 11 Aug 2006 07:31:37 -0000 1.65 +++ buildbot.texinfo 11 Aug 2006 07:31:49 -0000 1.66 @@ -60,7 +60,7 @@ * Resources:: Getting help. * Developer's Appendix:: * Index of Useful Classes:: -* Index of master.cfg keys:: +* Index of master.cfg keys:: * Index:: Complete index. @detailmenu @@ -194,6 +194,11 @@ * BuildFactory:: * Process-Specific build factories:: +BuildStep Objects + +* BuildFactory Attributes:: +* Quick builds:: + BuildFactory * BuildFactory Attributes:: @@ -2013,11 +2018,12 @@ @end menu @node Scheduler Types, Build Dependencies, Listing Change Sources and Schedulers, Listing Change Sources and Schedulers + at subsection Scheduler Types + @slindex buildbot.scheduler.Scheduler @slindex buildbot.scheduler.AnyBranchScheduler @slindex buildbot.scheduler.Periodic @slindex buildbot.scheduler.Nightly - at subsection Scheduler Types Here is a brief catalog of the available Scheduler types. All these Schedulers are classes in @code{buildbot.scheduler}, and the @@ -2065,10 +2071,11 @@ @end table @node Build Dependencies, , Scheduler Types, Listing Change Sources and Schedulers + at subsection Build Dependencies + @cindex Dependent @cindex Dependencies @slindex buildbot.scheduler.Dependent - at subsection Build Dependencies It is common to wind up with one kind of build which should only be performed if the same source code was successfully handled by some @@ -2544,9 +2551,10 @@ @node CVSToys - PBService, CVSToys - mail notification, Choosing ChangeSources, Change Sources - at csindex buildbot.changes.freshcvs.FreshCVSSource @subsection CVSToys - PBService + at csindex buildbot.changes.freshcvs.FreshCVSSource + The @uref{http://purl.net/net/CVSToys, CVSToys} package provides a server which runs on the machine that hosts the CVS repository it watches. It has a variety of ways to distribute commit notifications, @@ -2617,9 +2625,10 @@ @node CVSToys - mail notification, Other mail notification ChangeSources, CVSToys - PBService, Change Sources - at csindex buildbot.changes.mail.FCMaildirSource @subsection CVSToys - mail notification + at csindex buildbot.changes.mail.FCMaildirSource + CVSToys also provides a @code{MailNotification} action which will send email to a list of recipients for each commit. This tends to work better than using @code{/bin/mail} from within the CVSROOT/loginfo @@ -2652,9 +2661,10 @@ directory name of the maildir root, and the prefix to strip. @node Other mail notification ChangeSources, PBChangeSource, CVSToys - mail notification, Change Sources + at subsection Other mail notification ChangeSources + @csindex buildbot.changes.mail.SyncmailMaildirSource @csindex buildbot.changes.mail.BonsaiMaildirSource - at subsection Other mail notification ChangeSources There are other types of maildir-watching ChangeSources, which only differ in the function used to parse the message body. @@ -2665,9 +2675,10 @@ @code{BonsaiMaildirSource} parses messages sent out by Bonsai. @node PBChangeSource, P4Source, Other mail notification ChangeSources, Change Sources - at csindex buildbot.changes.pb.PBChangeSource @subsection PBChangeSource + at csindex buildbot.changes.pb.PBChangeSource + The last kind of ChangeSource actually listens on a TCP port for clients to connect and push change notices @emph{into} the Buildmaster. This is used by the built-in @code{buildbot sendchange} @@ -2719,9 +2730,10 @@ @node P4Source, , PBChangeSource, Change Sources - at csindex buildbot.changes.p4poller.P4Source @subsection P4Source + at csindex buildbot.changes.p4poller.P4Source + The @code{P4Source} periodically polls a @uref{http://www.perforce.com/, Perforce} depot for changes. It accepts the following arguments: @@ -3006,9 +3018,10 @@ @end menu @node CVS, SVN, Source Checkout, Source Checkout + at subsubsection CVS + @cindex CVS Checkout @bsindex buildbot.process.step.CVS - at subsubsection CVS The @code{CVS} build step performs a @uref{http://www.nongnu.org/cvs/, @@ -3042,9 +3055,10 @@ @node SVN, Darcs, CVS, Source Checkout + at subsubsection SVN + @cindex SVN Checkout @bsindex buildbot.process.step.SVN - at subsubsection SVN The @code{SVN} build step performs a @@ -3177,9 +3191,10 @@ ``features/newthing'' to get the svnurl for checkout. @node Darcs, Mercurial, SVN, Source Checkout + at subsubsection Darcs + @cindex Darcs Checkout @bsindex buildbot.process.step.Darcs - at subsubsection Darcs The @code{Darcs} build step performs a @@ -3220,9 +3235,10 @@ @end table @node Mercurial, Arch, Darcs, Source Checkout + at subsubsection Mercurial + @cindex Mercurial Checkout @bsindex buildbot.process.step.Mercurial - at subsubsection Mercurial The @code{Mercurial} build step performs a @@ -3252,9 +3268,10 @@ @node Arch, Bazaar, Mercurial, Source Checkout + at subsubsection Arch + @cindex Arch Checkout @bsindex buildbot.process.step.Arch - at subsubsection Arch The @code{Arch} build step performs an @uref{http://gnuarch.org/, @@ -3280,9 +3297,10 @@ @end table @node Bazaar, P4, Arch, Source Checkout + at subsubsection Bazaar + @cindex Bazaar Checkout @bsindex buildbot.process.step.Bazaar - at subsubsection Bazaar @code{Bazaar} is an alternate implementation of the Arch VC system, @@ -3297,10 +3315,11 @@ @node P4, , Bazaar, Source Checkout + at subsubsection P4 + @cindex Perforce Update @bsindex buildbot.process.step.P4 @c TODO @bsindex buildbot.process.step.P4Sync - at subsubsection P4 The @code{P4} build step creates a @uref{http://www.perforce.com/, @@ -3337,9 +3356,10 @@ @end table @node ShellCommand, Simple ShellCommand Subclasses, Source Checkout, Build Steps + at subsection ShellCommand + @bsindex buildbot.process.step.ShellCommand @c TODO @bsindex buildbot.process.step.TreeSize - at subsection ShellCommand This is a useful base class for just about everything you might want to do during a build (except for the initial source checkout). It runs @@ -3462,18 +3482,20 @@ @end menu @node Configure, Compile, Simple ShellCommand Subclasses, Simple ShellCommand Subclasses - at bsindex buildbot.process.step.Configure @subsubsection Configure + at bsindex buildbot.process.step.Configure + This is intended to handle the @code{./configure} step from autoconf-style projects, or the @code{perl Makefile.PL} step from perl MakeMaker.pm-style modules. The default command is @code{./configure} but you can change this by providing a @code{command=} parameter. @node Compile, Test, Configure, Simple ShellCommand Subclasses - at bsindex buildbot.process.step.Compile @subsubsection Compile + at bsindex buildbot.process.step.Compile + This is meant to handle compiling or building a project written in C. The default command is @code{make all}. When the compile is finished, the log file is scanned for GCC error/warning messages and a summary log is @@ -3481,9 +3503,10 @@ created). @node Test, Build Properties, Compile, Simple ShellCommand Subclasses - at bsindex buildbot.process.step.Test @subsubsection Test + at bsindex buildbot.process.step.Test + This is meant to handle unit tests. The default command is @code{make test}, and the @code{warnOnFailure} flag is set. @@ -3761,10 +3784,11 @@ @node Interlocks, Build Factories, Build Steps, Build Process + at section Interlocks + @cindex locks @slindex buildbot.locks.MasterLock @slindex buildbot.locks.SlaveLock - at section Interlocks For various reasons, you may want to prevent certain Steps (or perhaps entire Builds) from running simultaneously. Limited CPU speed or @@ -3973,12 +3997,18 @@ to later builds. + at menu +* BuildFactory Attributes:: +* Quick builds:: + at end menu + @node BuildFactory, Process-Specific build factories, BuildStep Objects, Build Factories + at subsection BuildFactory + @bfindex buildbot.process.factory.BuildFactory @bfindex buildbot.process.factory.BasicBuildFactory @c TODO: what is BasicSVN anyway? @bfindex buildbot.process.factory.BasicSVN - at subsection BuildFactory The default @code{BuildFactory}, provided in the @code{buildbot.process.factory} module, is constructed with a list of @@ -4077,9 +4107,10 @@ @node Quick builds, , BuildFactory Attributes, BuildFactory - at bfindex buildbot.process.factory.QuickBuildFactory @subsubsection Quick builds + at bfindex buildbot.process.factory.QuickBuildFactory + The difference between a ``full build'' and a ``quick build'' is that quick builds are generally done incrementally, starting with the tree where the previous build was performed. That simply means that the @@ -4109,9 +4140,10 @@ @end menu @node GNUAutoconf, CPAN, Process-Specific build factories, Process-Specific build factories - at bfindex buildbot.process.factory.GNUAutoconf @subsubsection GNUAutoconf + at bfindex buildbot.process.factory.GNUAutoconf + @uref{http://www.gnu.org/software/autoconf/, GNU Autoconf} is a software portability tool, intended to make it possible to write programs in C (and other languages) which will run on a variety of @@ -4186,9 +4218,10 @@ @node CPAN, Python distutils, GNUAutoconf, Process-Specific build factories - at bfindex buildbot.process.factory.CPAN @subsubsection CPAN + at bfindex buildbot.process.factory.CPAN + Most Perl modules available from the @uref{http://www.cpan.org/, CPAN} archive use the @code{MakeMaker} module to provide configuration, build, and test services. The standard build routine for these modules @@ -4220,9 +4253,10 @@ @node Python distutils, Python/Twisted/trial projects, CPAN, Process-Specific build factories - at bfindex buildbot.process.factory.Distutils @subsubsection Python distutils + at bfindex buildbot.process.factory.Distutils + Most Python modules use the @code{distutils} package to provide configuration and build services. The standard build process looks like: @@ -4261,6 +4295,8 @@ @node Python/Twisted/trial projects, , Python distutils, Process-Specific build factories + at subsubsection Python/Twisted/trial projects + @bfindex buildbot.process.factory.Trial @c TODO: document these steps better @bsindex buildbot.process.step_twisted.HLint @@ -4268,7 +4304,6 @@ @bsindex buildbot.process.step_twisted.ProcessDocs @bsindex buildbot.process.step_twisted.BuildDebs @bsindex buildbot.process.step_twisted.RemovePYCs - at subsubsection Python/Twisted/trial projects Twisted provides a unit test tool named @code{trial} which provides a few improvements over Python's built-in @code{unittest} module. Many @@ -4459,9 +4494,10 @@ @c DOCUMENT THIS @node HTML Waterfall, IRC Bot, Status Delivery, Status Delivery + at section HTML Waterfall + @cindex Waterfall @stindex buildbot.status.html.Waterfall - at section HTML Waterfall @example @@ -4541,9 +4577,10 @@ @node IRC Bot, PBListener, HTML Waterfall, Status Delivery + at section IRC Bot + @cindex IRC @stindex buildbot.status.words.IRC - at section IRC Bot The @code{buildbot.status.words.IRC} status target creates an IRC bot @@ -4612,9 +4649,10 @@ @end table @node PBListener, Writing New Status Plugins, IRC Bot, Status Delivery + at section PBListener + @cindex PBListener @stindex buildbot.status.client.PBListener - at section PBListener @example From warner at users.sourceforge.net Mon Aug 14 03:46:29 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 14 Aug 2006 03:46:29 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.705,1.706 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27990 Modified Files: ChangeLog Log Message: [project @ document LogFiles better] Original author: warner at lothar.com Date: 2006-08-14 03:38:39 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.705 retrieving revision 1.706 diff -u -d -r1.705 -r1.706 --- ChangeLog 11 Aug 2006 07:31:49 -0000 1.705 +++ ChangeLog 14 Aug 2006 03:46:25 -0000 1.706 @@ -1,3 +1,7 @@ +2006-08-13 Brian Warner + + * docs/buildbot.texinfo (BuildStep LogFiles): document them + 2006-08-10 Brian Warner * docs/buildbot.texinfo (Index of master.cfg keys): add another From warner at users.sourceforge.net Mon Aug 14 03:46:29 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 14 Aug 2006 03:46:29 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.66,1.67 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27990/docs Modified Files: buildbot.texinfo Log Message: [project @ document LogFiles better] Original author: warner at lothar.com Date: 2006-08-14 03:38:39 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.66 retrieving revision 1.67 diff -u -d -r1.66 -r1.67 --- buildbot.texinfo 11 Aug 2006 07:31:49 -0000 1.66 +++ buildbot.texinfo 14 Aug 2006 03:46:27 -0000 1.67 @@ -186,6 +186,7 @@ Writing New BuildSteps +* BuildStep LogFiles:: * Adding LogObservers:: Build Factories @@ -3682,10 +3683,102 @@ TODO: add more description of BuildSteps. @menu +* BuildStep LogFiles:: * Adding LogObservers:: @end menu - at node Adding LogObservers, , Writing New BuildSteps, Writing New BuildSteps + at node BuildStep LogFiles, Adding LogObservers, Writing New BuildSteps, Writing New BuildSteps + at subsubsection BuildStep LogFiles + +Each BuildStep has a collection of ``logfiles''. Each one has a short +name, like ``stdio'' or ``warnings''. Each LogFile contains an +arbitrary amount of text, usually the contents of some output file +generated during a build or test step, or a record of everything that +was printed to stdout/stderr during the execution of some command. + +These LogFiles are stored to disk, so they can be retrieved later. + +Each can contain multiple ``channels'', generally limited to three +basic ones: stdout, stderr, and ``headers''. For example, when a +ShellCommand runs, it writes a few lines to the ``headers'' channel to +indicate the exact argv strings being run, which directory the command +is being executed in, and the contents of the current environment +variables. Then, as the command runs, it adds a lot of ``stdout'' and +``stderr'' messages. When the command finishes, a final ``header'' +line is added with the exit code of the process. + +Status display plugins can format these different channels in +different ways. For example, the web page shows LogFiles as text/html, +with header lines in blue text, stdout in black, and stderr in red. A +different URL is available which provides a text/plain format, in +which stdout and stderr are collapsed together, and header lines are +stripped completely. This latter option makes it easy to save the +results to a file and run @command{grep} or whatever against the +output. + +Each BuildStep contains a mapping (implemented in a python dictionary) +from LogFile name to the actual LogFile objects. Status plugins can +get a list of LogFiles to display, for example, a list of HREF links +that, when clicked, provide the full contents of the LogFile. + + at heading Using LogFiles in custom BuildSteps + +The most common way for a custom BuildStep to use a LogFile is to +summarize the results of a ShellCommand (after the command has +finished running). For example, a compile step with thousands of lines +of output might want to create a summary of just the warning messages. +If you were doing this from a shell, you would use something like: + + at example +grep "warning:" output.log >warnings.log + at end example + +In a custom BuildStep, you could instead create a ``warnings'' LogFile +that contained the same text. To do this, you would add code to your + at code{createSummary} method that pulls lines from the main output log +and creates a new LogFile with the results: + + at example + def createSummary(self, log): + warnings = [] + for line in log.readlines(): + if "warning:" in line: + warnings.append() + self.addCompleteLog('warnings', "".join(warnings)) + at end example + +This example uses the @code{addCompleteLog} method, which creates a +new LogFile, puts some text in it, and then ``closes'' it, meaning +that no further contents will be added. This LogFile will appear in +the HTML display under an HREF with the name ``warnings'', since that +is the name of the LogFile. + +You can also use @code{addHTMLLog} to create a complete (closed) +LogFile that contains HTML instead of plain text. The normal LogFile +will be HTML-escaped if presented through a web page, but the HTML +LogFile will not. At the moment this is only used to present a pretty +HTML representation of an otherwise ugly exception traceback when +something goes badly wrong during the BuildStep. + +In contrast, you might want to create a new LogFile at the beginning +of the step, and add text to it as the command runs. You can create +the LogFile and attach it to the build by calling @code{addLog}, which +returns the LogFile object. You then add text to this LogFile by +calling methods like @code{addStdout} and @code{addHeader}. When you +are done, you must call the @code{finish} method so the LogFile can be +closed. It may be useful to create and populate a LogFile like this +from a LogObserver method @xref{Adding LogObservers}. + +The @code{logfiles=} argument to @code{ShellCommand} (see + at pxref{ShellCommand}) creates new LogFiles and fills them in realtime +by asking the buildslave to watch a actual file on disk. The +buildslave will look for additions in the target file and report them +back to the BuildStep. These additions will be added to the LogFile by +calling @code{addStdout}. These secondary LogFiles can be used as the +source of a LogObserver just like the normal ``stdio'' LogFile. + + + at node Adding LogObservers, , BuildStep LogFiles, Writing New BuildSteps @subsubsection Adding LogObservers @cindex LogObserver @@ -3778,9 +3871,10 @@ self.addLogObserver('stdio', counter) @end example -This creates a TrialTestCaseCounter and tells the step to attach it to -the ``stdio'' log. The observer is automatically given a reference to -the step in its @code{.step} attribute. +This creates a TrialTestCaseCounter and tells the step that the +counter wants to watch the ``stdio'' log. The observer is +automatically given a reference to the step in its @code{.step} +attribute. @node Interlocks, Build Factories, Build Steps, Build Process From warner at users.sourceforge.net Sun Aug 20 22:25:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:25:37 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.706,1.707 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv14956 Modified Files: ChangeLog Log Message: [project @ add BuildStep URLs] Original author: warner at lothar.com Date: 2006-08-20 22:24:14 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.706 retrieving revision 1.707 diff -u -d -r1.706 -r1.707 --- ChangeLog 14 Aug 2006 03:46:25 -0000 1.706 +++ ChangeLog 20 Aug 2006 22:25:35 -0000 1.707 @@ -1,3 +1,25 @@ +2006-08-20 Brian Warner + + * docs/buildbot.texinfo (BuildStep URLs): document new feature: + per-step URLs that will be displayed on the waterfall display, + for things like the HTML output of code-coverage tools, when + the results are hosted elsewhere. + * buildbot/interfaces.py (IBuildStepStatus.getURLs): document the + way to retrieve these URLs + * buildbot/status/builder.py (BuildStepStatus.getURLs): implement + the method to retrieve these URLs. Also provide backwards + compatibility for saved BuildStepStatus instances that didn't have + the .urls attribute + * buildbot/process/step.py (BuildStep.addURL): method to set these + URLs from within a BuildStep + * buildbot/status/html.py (StepBox.getBox): emit links to the URLs + * buildbot/test/test_web.py (WaterfallSteps): test that we really + do emit those links + * buildbot/test/test_steps.py (Steps): test that we can all the + URLs. Also add a bunch of other tests on methods that can be + called from within BuildSteps + * buildbot/test/runutils.py (makeBuildStep): add utility function + 2006-08-13 Brian Warner * docs/buildbot.texinfo (BuildStep LogFiles): document them From warner at users.sourceforge.net Sun Aug 20 22:25:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:25:37 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.97,1.98 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv14956/buildbot/process Modified Files: step.py Log Message: [project @ add BuildStep URLs] Original author: warner at lothar.com Date: 2006-08-20 22:24:14 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.97 retrieving revision 1.98 diff -u -d -r1.97 -r1.98 --- step.py 6 Aug 2006 20:05:31 -0000 1.97 +++ step.py 20 Aug 2006 22:25:35 -0000 1.98 @@ -867,6 +867,16 @@ observer.setLog(current_logs[logname]) self._pendingLogObservers.remove((logname, observer)) + def addURL(self, name, url): + """Add a BuildStep URL to this step. + + An HREF to this URL will be added to any HTML representations of this + step. This allows a step to provide links to external web pages, + perhaps to provide detailed HTML code coverage results or other forms + of build status. + """ + self.step_status.addURL(name, url) + def runCommand(self, c): d = c.run(self, self.remote) return d From warner at users.sourceforge.net Sun Aug 20 22:25:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:25:37 +0000 Subject: [Buildbot-commits] buildbot/buildbot interfaces.py,1.44,1.45 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv14956/buildbot Modified Files: interfaces.py Log Message: [project @ add BuildStep URLs] Original author: warner at lothar.com Date: 2006-08-20 22:24:14 Index: interfaces.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/interfaces.py,v retrieving revision 1.44 retrieving revision 1.45 diff -u -d -r1.44 -r1.45 --- interfaces.py 15 Jul 2006 08:50:35 -0000 1.44 +++ interfaces.py 20 Aug 2006 22:25:35 -0000 1.45 @@ -472,6 +472,12 @@ may or may not equal 'target' (which is merely the expectation based upon previous builds).""" + def getURLs(): + """Returns a dictionary of URLs. Each key is a link name (a short + string, like 'results' or 'coverage'), and each value is a URL. These + links will be displayed along with the LogFiles. + """ + def getLogs(): """Returns a list of IStatusLog objects. If the step has not yet finished, this list may be incomplete (asking again later may give From warner at users.sourceforge.net Sun Aug 20 22:25:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:25:37 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status builder.py, 1.84, 1.85 html.py, 1.87, 1.88 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv14956/buildbot/status Modified Files: builder.py html.py Log Message: [project @ add BuildStep URLs] Original author: warner at lothar.com Date: 2006-08-20 22:24:14 Index: builder.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/builder.py,v retrieving revision 1.84 retrieving revision 1.85 diff -u -d -r1.84 -r1.85 --- builder.py 16 Jun 2006 05:28:13 -0000 1.84 +++ builder.py 20 Aug 2006 22:25:35 -0000 1.85 @@ -647,7 +647,7 @@ self.observers.remove(observer) -class BuildStepStatus: +class BuildStepStatus(styles.Versioned): """ I represent a collection of output status for a L{buildbot.process.step.BuildStep}. @@ -672,6 +672,7 @@ implements(interfaces.IBuildStepStatus, interfaces.IStatusEvent) else: __implements__ = interfaces.IBuildStepStatus, interfaces.IStatusEvent + persistenceVersion = 1 started = None finished = None @@ -688,6 +689,7 @@ assert interfaces.IBuildStatus(parent) self.build = parent self.logs = [] + self.urls = {} self.watchers = [] self.updates = {} self.finishedWatchers = [] @@ -718,6 +720,8 @@ def getLogs(self): return self.logs + def getURLs(self): + return self.urls.copy() def isFinished(self): return (self.finished is not None) @@ -846,6 +850,9 @@ for w in self.watchers: w.logFinished(self.build, self, log) + def addURL(self, name, url): + self.urls[name] = url + def setColor(self, color): self.color = color def setText(self, text): @@ -873,7 +880,7 @@ # persistence def __getstate__(self): - d = self.__dict__.copy() + d = styles.Versioned.__getstate__(self) del d['build'] # filled in when loading if d.has_key('progress'): del d['progress'] @@ -883,11 +890,15 @@ return d def __setstate__(self, d): - self.__dict__ = d + styles.Versioned.__setstate__(self, d) # self.build must be filled in by our parent for loog in self.logs: loog.step = self + def upgradeToVersion1(self): + if not hasattr(self, "urls"): + self.urls = {} + class BuildStatus(styles.Versioned): if implements: Index: html.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/html.py,v retrieving revision 1.87 retrieving revision 1.88 diff -u -d -r1.87 -r1.88 --- html.py 16 Jul 2006 00:58:31 -0000 1.87 +++ html.py 20 Aug 2006 22:25:35 -0000 1.88 @@ -958,6 +958,9 @@ text.append("%s" % (url, html.escape(name))) else: text.append(html.escape(name)) + urls = self.original.getURLs() + for name, target in urls.items(): + text.append('[%s]' % (target, html.escape(name))) color = self.original.getColor() class_ = "BuildStep " + build_get_class(self.original) return Box(text, color, class_=class_) From warner at users.sourceforge.net Sun Aug 20 22:25:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:25:37 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test runutils.py, 1.11, 1.12 test_steps.py, 1.23, 1.24 test_web.py, 1.31, 1.32 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv14956/buildbot/test Modified Files: runutils.py test_steps.py test_web.py Log Message: [project @ add BuildStep URLs] Original author: warner at lothar.com Date: 2006-08-20 22:24:14 Index: runutils.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/runutils.py,v retrieving revision 1.11 retrieving revision 1.12 diff -u -d -r1.11 -r1.12 --- runutils.py 20 Jun 2006 08:08:58 -0000 1.11 +++ runutils.py 20 Aug 2006 22:25:35 -0000 1.12 @@ -7,7 +7,9 @@ from buildbot import master, interfaces from buildbot.twcompat import maybeWait from buildbot.slave import bot -from buildbot.process.base import BuildRequest +from buildbot.process.builder import Builder +from buildbot.process.base import BuildRequest, Build +from buildbot.process.step import BuildStep from buildbot.sourcestamp import SourceStamp from buildbot.status import builder @@ -227,6 +229,22 @@ s3.stepStarted() return s3 +def makeBuildStep(basedir): + bss = setupBuildStepStatus(basedir) + + ss = SourceStamp() + setup = {'name': "builder1", "slavename": "bot1", + 'builddir': "builddir", 'factory': None} + b0 = Builder(setup, bss.getBuild().getBuilder()) + br = BuildRequest("reason", ss) + b = Build([br]) + b.setBuilder(b0) + s = BuildStep(b) + s.setStepStatus(bss) + b.setupStatus(bss.getBuild()) + return s + + def findDir(): # the same directory that holds this script return util.sibpath(__file__, ".") Index: test_steps.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_steps.py,v retrieving revision 1.23 retrieving revision 1.24 diff -u -d -r1.23 -r1.24 --- test_steps.py 6 Aug 2006 07:05:49 -0000 1.23 +++ test_steps.py 20 Aug 2006 22:25:35 -0000 1.24 @@ -25,6 +25,7 @@ from buildbot.process.step import ShellCommand #, ShellCommands from buildbot.status import builder from buildbot.test.runutils import RunMixin, rmtree, setupBuildStepStatus +from buildbot.test.runutils import makeBuildStep from buildbot.twcompat import maybeWait from buildbot.slave import commands @@ -160,6 +161,11 @@ self.assertEqual(self.results, 0) +class MyObserver(step.LogObserver): + out = "" + def outReceived(self, data): + self.out = self.out + data + class Steps(unittest.TestCase): def testMultipleStepInstances(self): steps = [ @@ -176,6 +182,84 @@ b = f.newBuild([req]) #for s in b.steps: print s.name + # test the various methods available to buildsteps + + def test_getProperty(self): + s = makeBuildStep("test_steps.Steps.test_getProperty") + bs = s.step_status.getBuild() + + s.setProperty("prop1", "value1") + s.setProperty("prop2", "value2") + self.failUnlessEqual(s.getProperty("prop1"), "value1") + self.failUnlessEqual(bs.getProperty("prop1"), "value1") + self.failUnlessEqual(s.getProperty("prop2"), "value2") + self.failUnlessEqual(bs.getProperty("prop2"), "value2") + s.setProperty("prop1", "value1a") + self.failUnlessEqual(s.getProperty("prop1"), "value1a") + self.failUnlessEqual(bs.getProperty("prop1"), "value1a") + + + def test_addURL(self): + s = makeBuildStep("test_steps.Steps.test_addURL") + s.addURL("coverage", "http://coverage.example.org/target") + s.addURL("icon", "http://coverage.example.org/icon.png") + bs = s.step_status + links = bs.getURLs() + expected = {"coverage": "http://coverage.example.org/target", + "icon": "http://coverage.example.org/icon.png", + } + self.failUnlessEqual(links, expected) + + def test_addLog(self): + s = makeBuildStep("test_steps.Steps.test_addLog") + l = s.addLog("newlog") + l.addStdout("some stdout here") + l.finish() + bs = s.step_status + logs = bs.getLogs() + self.failUnlessEqual(len(logs), 1) + l1 = logs[0] + self.failUnlessEqual(l1.getText(), "some stdout here") + + def test_addHTMLLog(self): + s = makeBuildStep("test_steps.Steps.test_addHTMLLog") + l = s.addHTMLLog("newlog", "some html here") + bs = s.step_status + logs = bs.getLogs() + self.failUnlessEqual(len(logs), 1) + l1 = logs[0] + self.failUnless(isinstance(l1, builder.HTMLLogFile)) + self.failUnlessEqual(l1.getText(), "some html here") + + def test_addCompleteLog(self): + s = makeBuildStep("test_steps.Steps.test_addCompleteLog") + l = s.addCompleteLog("newlog", "some stdout here") + bs = s.step_status + logs = bs.getLogs() + self.failUnlessEqual(len(logs), 1) + l1 = logs[0] + self.failUnlessEqual(l1.getText(), "some stdout here") + + def test_addLogObserver(self): + s = makeBuildStep("test_steps.Steps.test_addLogObserver") + bss = s.step_status + o1,o2,o3 = MyObserver(), MyObserver(), MyObserver() + + # add the log before the observer + l1 = s.addLog("one") + l1.addStdout("onestuff") + s.addLogObserver("one", o1) + self.failUnlessEqual(o1.out, "onestuff") + l1.addStdout(" morestuff") + self.failUnlessEqual(o1.out, "onestuff morestuff") + + # add the observer before the log + s.addLogObserver("two", o2) + l2 = s.addLog("two") + l2.addStdout("twostuff") + self.failUnlessEqual(o2.out, "twostuff") + + class VersionCheckingStep(step.BuildStep): def start(self): # give our test a chance to run. It is non-trivial for a buildstep to @@ -201,7 +285,7 @@ c['slavePortnum'] = 0 """ -class Version(RunMixin, unittest.TestCase): +class SlaveVersion(RunMixin, unittest.TestCase): def setUp(self): RunMixin.setUp(self) self.master.loadConfig(version_config) @@ -233,36 +317,10 @@ self.failIf(s.slaveVersionIsOlderThan("svn", "1.1")) self.failUnless(s.slaveVersionIsOlderThan("svn", cver + ".1")) + self.failUnlessEqual(s.getSlaveName(), "bot1") + def testCompare(self): self.master._checker = self.checkCompare d = self.doBuild("quick") return maybeWait(d) - -class MyObserver(step.LogObserver): - out = "" - def outReceived(self, data): - self.out = self.out + data - -class LogObserver(unittest.TestCase): - def testAdd(self): - bss = setupBuildStepStatus("logobserver") - build = None - s = step.BuildStep(build) - s.setStepStatus(bss) - o1,o2,o3 = MyObserver(), MyObserver(), MyObserver() - - # add the log before the observer - l1 = s.addLog("one") - l1.addStdout("onestuff") - s.addLogObserver("one", o1) - self.failUnlessEqual(o1.out, "onestuff") - l1.addStdout(" morestuff") - self.failUnlessEqual(o1.out, "onestuff morestuff") - - # add the observer before the log - s.addLogObserver("two", o2) - l2 = s.addLog("two") - l2.addStdout("twostuff") - self.failUnlessEqual(o2.out, "twostuff") - Index: test_web.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_web.py,v retrieving revision 1.31 retrieving revision 1.32 diff -u -d -r1.31 -r1.32 --- test_web.py 16 Jun 2006 05:28:13 -0000 1.31 +++ test_web.py 20 Aug 2006 22:25:35 -0000 1.32 @@ -16,6 +16,7 @@ from buildbot.status import html, builder from buildbot.changes.changes import Change from buildbot.process import step, base +from buildbot.test.runutils import setupBuildStepStatus class ConfiguredMaster(master.BuildMaster): """This BuildMaster variant has a static config file, provided as a @@ -249,6 +250,32 @@ def _test_waterfall_5(self, robotstxt): self.failUnless(robotstxt == self.robots_txt_contents) +class WaterfallSteps(unittest.TestCase): + + # failUnlessSubstring copied from twisted-2.1.0, because this helps us + # maintain compatibility with python2.2. + def failUnlessSubstring(self, substring, astring, msg=None): + """a python2.2 friendly test to assert that substring is found in + astring parameters follow the semantics of failUnlessIn + """ + if astring.find(substring) == -1: + raise self.failureException(msg or "%r not found in %r" + % (substring, astring)) + return substring + assertSubstring = failUnlessSubstring + + def test_urls(self): + s = setupBuildStepStatus("test_web.test_urls") + s.addURL("coverage", "http://coverage.example.org/target") + s.addURL("icon", "http://coverage.example.org/icon.png") + box = html.IBox(s).getBox() + td = box.td() + e1 = 'coverage' + self.failUnlessSubstring(e1, td) + e2 = 'icon' + self.failUnlessSubstring(e2, td) + + geturl_config = """ from buildbot.status import html From warner at users.sourceforge.net Sun Aug 20 22:25:38 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:25:38 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.67,1.68 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv14956/docs Modified Files: buildbot.texinfo Log Message: [project @ add BuildStep URLs] Original author: warner at lothar.com Date: 2006-08-20 22:24:14 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.67 retrieving revision 1.68 diff -u -d -r1.67 -r1.68 --- buildbot.texinfo 14 Aug 2006 03:46:27 -0000 1.67 +++ buildbot.texinfo 20 Aug 2006 22:25:35 -0000 1.68 @@ -188,6 +188,7 @@ * BuildStep LogFiles:: * Adding LogObservers:: +* BuildStep URLs:: Build Factories @@ -3685,6 +3686,7 @@ @menu * BuildStep LogFiles:: * Adding LogObservers:: +* BuildStep URLs:: @end menu @node BuildStep LogFiles, Adding LogObservers, Writing New BuildSteps, Writing New BuildSteps @@ -3778,7 +3780,7 @@ source of a LogObserver just like the normal ``stdio'' LogFile. - at node Adding LogObservers, , BuildStep LogFiles, Writing New BuildSteps + at node Adding LogObservers, BuildStep URLs, BuildStep LogFiles, Writing New BuildSteps @subsubsection Adding LogObservers @cindex LogObserver @@ -3876,6 +3878,83 @@ automatically given a reference to the step in its @code{.step} attribute. + at node BuildStep URLs, , Adding LogObservers, Writing New BuildSteps + at subsubsection BuildStep URLs + + at cindex links + at cindex BuildStep URLs + at cindex addURL + +Each BuildStep has a collection of ``links''. Like its collection of +LogFiles, each link has a name and a target URL. The web status page +creates HREFs for each link in the same box as it does for LogFiles, +except that the target of the link is the external URL instead of an +internal link to a page that shows the contents of the LogFile. + +These external links can be used to point at build information hosted +on other servers. For example, the test process might produce an +intricate description of which tests passed and failed, or some sort +of code coverage data in HTML form, or a PNG or GIF image with a graph +of memory usage over time. The external link can provide an easy way +for users to navigate from the buildbot's status page to these +external web sites or file servers. Note that the step itself is +responsible for insuring that there will be a document available at +the given URL (perhaps by using @command{scp} to copy the HTML output +to a @file{~/public_html/} directory on a remote web server). Calling + at code{addURL} does not magically populate a web server. + +To set one of these links, the BuildStep should call the @code{addURL} +method with the name of the link and the target URL. Multiple URLs can +be set. + +In this example, we assume that the @command{make test} command causes +a collection of HTML files to be created and put somewhere on the +coverage.example.org web server, in a filename that incorporates the +build number. + + at example +class TestWithCodeCoverage(BuildStep): + command = ["make", "test", + WithProperties("buildnum=%s" % "buildnumber")] + + def createSummary(self, log): + buildnumber = self.getProperty("buildnumber") + url = "http://coverage.example.org/builds/%s.html" % buildnumber + self.addURL("coverage", url) + at end example + +You might also want to extract the URL from some special message +output by the build process itself: + + at example +class TestWithCodeCoverage(BuildStep): + command = ["make", "test", + WithProperties("buildnum=%s" % "buildnumber")] + + def createSummary(self, log): + output = StringIO(log.getText()) + for line in output.readlines(): + if line.startswith("coverage-url:"): + url = line[len("coverage-url:"):].strip() + self.addURL("coverage", url) + return + at end example + +Note that a build process which emits both stdout and stderr might +cause this line to be split or interleaved between other lines. It +might be necessary to restrict the getText() call to only stdout with +something like this: + + at example + output = StringIO("".join([c[1] + for c in log.getChunks() + if c[0] == LOG_CHANNEL_STDOUT])) + at end example + +Of course if the build is run under a PTY, then stdout and stderr will +be merged before the buildbot ever sees them, so such interleaving +will be unavoidable. + @node Interlocks, Build Factories, Build Steps, Build Process @section Interlocks From warner at users.sourceforge.net Sun Aug 20 22:39:49 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:39:49 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.707,1.708 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20619 Modified Files: ChangeLog Log Message: [project @ give external BuildStep links a distinct CSS class] Original author: warner at lothar.com Date: 2006-08-20 22:37:46 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.707 retrieving revision 1.708 diff -u -d -r1.707 -r1.708 --- ChangeLog 20 Aug 2006 22:25:35 -0000 1.707 +++ ChangeLog 20 Aug 2006 22:39:47 -0000 1.708 @@ -13,6 +13,10 @@ * buildbot/process/step.py (BuildStep.addURL): method to set these URLs from within a BuildStep * buildbot/status/html.py (StepBox.getBox): emit links to the URLs + (StepBox.getBox): give these external links a distinct CSS class + named "BuildStep external" so a .css file can display them + differently + * buildbot/test/test_web.py (WaterfallSteps): test that we really do emit those links * buildbot/test/test_steps.py (Steps): test that we can all the From warner at users.sourceforge.net Sun Aug 20 22:39:49 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:39:49 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status html.py,1.88,1.89 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20619/buildbot/status Modified Files: html.py Log Message: [project @ give external BuildStep links a distinct CSS class] Original author: warner at lothar.com Date: 2006-08-20 22:37:46 Index: html.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/html.py,v retrieving revision 1.88 retrieving revision 1.89 diff -u -d -r1.88 -r1.89 --- html.py 20 Aug 2006 22:25:35 -0000 1.88 +++ html.py 20 Aug 2006 22:39:47 -0000 1.89 @@ -959,8 +959,10 @@ else: text.append(html.escape(name)) urls = self.original.getURLs() + ex_url_class = "BuildStep external" for name, target in urls.items(): - text.append('[%s]' % (target, html.escape(name))) + text.append('[%s]' % + (target, ex_url_class, html.escape(name))) color = self.original.getColor() class_ = "BuildStep " + build_get_class(self.original) return Box(text, color, class_=class_) From warner at users.sourceforge.net Sun Aug 20 22:51:39 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:51:39 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status sample1.css, NONE, 1.1 sample2.css, NONE, 1.1 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25406/buildbot/status Added Files: sample1.css sample2.css Log Message: [project @ Start collecting alternative CSS files for the waterfall status page] >From discussions on buildbot mailing lists, a few of us wanted to start collecting alternative CSS files. These would be included in future buildbot releases, and allow people to easily change from the default CSS used by the waterfall status page. This patch includes two .css files: one from me, and one from another developer on buildbot-dev mailing list. There is no change to the default CSS. Original author: john at oduinn.com Date: 2006-07-27 14:09:56 --- NEW FILE: sample1.css --- * { font-family: Verdana, Cursor; font-size: 10px; font-weight: bold; } a:link,a:visited,a:active { color: #666666; } a:hover { color: #FFFFFF; } .table { border-spacing: 2px; } td.Event, td.Activity, td.Change, td.Time, td.Builder { color: #333333; border: 1px solid #666666; background-color: #CCCCCC; } /* LastBuild, BuildStep states */ .success { color: #FFFFFF; border: 1px solid #2f8f0f; background-color: #8fdf5f; } .failure { color: #FFFFFF; border: 1px solid #f33636; background-color: #e98080; } .warnings { color: #FFFFFF; border: 1px solid #fc901f; background-color: #ffc343; } .exception, td.offline { color: #FFFFFF; border: 1px solid #8000c0; background-color: #e0b0ff; } .start,.running, td.building { color: #666666; border: 1px solid #ffff00; background-color: #fffc6c; } --- NEW FILE: sample2.css --- * { font-family: Verdana, Cursor; font-size: 12px; font-weight: bold; } a:link,a:visited,a:active { color: #666666; } a:hover { color: #FFFFFF; } .table { border-spacing: 2px; } td.Event, td.Activity, td.Change, td.Time, td.Builder { color: #333333; border: 1px solid #666666; background-color: #CCCCCC; } /* LastBuild, BuildStep states */ .success { color: #FFFFFF; border: 1px solid #2f8f0f; background-color: #72ff75; } .failure { color: #FFFFFF; border: 1px solid #f33636; background-color: red; } .warnings { color: #FFFFFF; border: 1px solid #fc901f; background-color: #ffc343; } .exception, td.offline { color: #FFFFFF; border: 1px solid #8000c0; background-color: red; } .start,.running, td.building { color: #666666; border: 1px solid #ffff00; background-color: yellow; } From warner at users.sourceforge.net Sun Aug 20 22:51:58 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:51:58 +0000 Subject: [Buildbot-commits] buildbot/contrib/CSS - New directory Message-ID: Update of /cvsroot/buildbot/buildbot/contrib/CSS In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25440/contrib/CSS Log Message: Directory /cvsroot/buildbot/buildbot/contrib/CSS added to the repository From warner at users.sourceforge.net Sun Aug 20 22:52:12 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:52:12 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.708,1.709 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25791 Modified Files: ChangeLog Log Message: [project @ add some contributed Waterfall stylesheets, in contrib/CSS/*.css] Original author: warner at lothar.com Date: 2006-08-20 22:47:45 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.708 retrieving revision 1.709 diff -u -d -r1.708 -r1.709 --- ChangeLog 20 Aug 2006 22:39:47 -0000 1.708 +++ ChangeLog 20 Aug 2006 22:52:10 -0000 1.709 @@ -1,5 +1,9 @@ 2006-08-20 Brian Warner + * contrib/CSS/*.css: add some contributed CSS stylesheets, to make + the Waterfall display a bit less ugly. Thanks to John O'Duinn for + collecting the files and creating the patch. + * docs/buildbot.texinfo (BuildStep URLs): document new feature: per-step URLs that will be displayed on the waterfall display, for things like the HTML output of code-coverage tools, when From warner at users.sourceforge.net Sun Aug 20 22:52:12 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:52:12 +0000 Subject: [Buildbot-commits] buildbot/contrib README.txt,1.3,1.4 Message-ID: Update of /cvsroot/buildbot/buildbot/contrib In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25791/contrib Modified Files: README.txt Log Message: [project @ add some contributed Waterfall stylesheets, in contrib/CSS/*.css] Original author: warner at lothar.com Date: 2006-08-20 22:47:45 Index: README.txt =================================================================== RCS file: /cvsroot/buildbot/buildbot/contrib/README.txt,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- README.txt 17 Apr 2006 18:31:41 -0000 1.3 +++ README.txt 20 Aug 2006 22:52:10 -0000 1.4 @@ -35,3 +35,7 @@ for information about new CVS changes, then delivers them over PB to a remote buildmaster's PBChangeSource. Contributed by Stephen Kennedy. + +CSS/*.css: alternative HTML stylesheets to make the Waterfall display look + prettier. Copy them somewhere, then pass the filename to the + css= argument of the Waterfall() constructor. From warner at users.sourceforge.net Sun Aug 20 22:52:12 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:52:12 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status sample1.css, 1.1, NONE sample2.css, 1.1, NONE Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25791/buildbot/status Removed Files: sample1.css sample2.css Log Message: [project @ add some contributed Waterfall stylesheets, in contrib/CSS/*.css] Original author: warner at lothar.com Date: 2006-08-20 22:47:45 --- sample1.css DELETED --- --- sample2.css DELETED --- From warner at users.sourceforge.net Sun Aug 20 22:52:12 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 20 Aug 2006 22:52:12 +0000 Subject: [Buildbot-commits] buildbot/contrib/CSS sample1.css, NONE, 1.1 sample2.css, NONE, 1.1 Message-ID: Update of /cvsroot/buildbot/buildbot/contrib/CSS In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25791/contrib/CSS Added Files: sample1.css sample2.css Log Message: [project @ add some contributed Waterfall stylesheets, in contrib/CSS/*.css] Original author: warner at lothar.com Date: 2006-08-20 22:47:45 --- NEW FILE: sample1.css --- * { font-family: Verdana, Cursor; font-size: 10px; font-weight: bold; } a:link,a:visited,a:active { color: #666666; } a:hover { color: #FFFFFF; } .table { border-spacing: 2px; } td.Event, td.Activity, td.Change, td.Time, td.Builder { color: #333333; border: 1px solid #666666; background-color: #CCCCCC; } /* LastBuild, BuildStep states */ .success { color: #FFFFFF; border: 1px solid #2f8f0f; background-color: #8fdf5f; } .failure { color: #FFFFFF; border: 1px solid #f33636; background-color: #e98080; } .warnings { color: #FFFFFF; border: 1px solid #fc901f; background-color: #ffc343; } .exception, td.offline { color: #FFFFFF; border: 1px solid #8000c0; background-color: #e0b0ff; } .start,.running, td.building { color: #666666; border: 1px solid #ffff00; background-color: #fffc6c; } --- NEW FILE: sample2.css --- * { font-family: Verdana, Cursor; font-size: 12px; font-weight: bold; } a:link,a:visited,a:active { color: #666666; } a:hover { color: #FFFFFF; } .table { border-spacing: 2px; } td.Event, td.Activity, td.Change, td.Time, td.Builder { color: #333333; border: 1px solid #666666; background-color: #CCCCCC; } /* LastBuild, BuildStep states */ .success { color: #FFFFFF; border: 1px solid #2f8f0f; background-color: #72ff75; } .failure { color: #FFFFFF; border: 1px solid #f33636; background-color: red; } .warnings { color: #FFFFFF; border: 1px solid #fc901f; background-color: #ffc343; } .exception, td.offline { color: #FFFFFF; border: 1px solid #8000c0; background-color: red; } .start,.running, td.building { color: #666666; border: 1px solid #ffff00; background-color: yellow; } From warner at users.sourceforge.net Mon Aug 21 00:43:22 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 00:43:22 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.709,1.710 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv4503 Modified Files: ChangeLog Log Message: [project @ accept either a single string or a list for ShellCommand description=] This fixes SF#1524659, thanks to Paul Winkler for the catch. Original author: warner at lothar.com Date: 2006-08-21 00:38:00 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.709 retrieving revision 1.710 diff -u -d -r1.709 -r1.710 --- ChangeLog 20 Aug 2006 22:52:10 -0000 1.709 +++ ChangeLog 21 Aug 2006 00:43:20 -0000 1.710 @@ -1,5 +1,18 @@ 2006-08-20 Brian Warner + * docs/buildbot.texinfo (ShellCommand.command=): explain why a + list of strings is preferred over a single string with embedded + spaces + (ShellCommand.description=): explain that either single strings or + a list of strings is acceptable, and why you might prefer one over + the other. Add an example. Fixes SF#1524659, thanks to Paul + Winkler for the catch. + * buildbot/process/step.py (ShellCommand): accept either a single + string or a list of strings in both description= and + descriptionDone= + * buildbot/test/test_steps.py (Steps.test_description): test it + * buildbot/test/runutils.py (makeBuildStep): support for that test + * contrib/CSS/*.css: add some contributed CSS stylesheets, to make the Waterfall display a bit less ugly. Thanks to John O'Duinn for collecting the files and creating the patch. From warner at users.sourceforge.net Mon Aug 21 00:43:22 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 00:43:22 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.98,1.99 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv4503/buildbot/process Modified Files: step.py Log Message: [project @ accept either a single string or a list for ShellCommand description=] This fixes SF#1524659, thanks to Paul Winkler for the catch. Original author: warner at lothar.com Date: 2006-08-21 00:38:00 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.98 retrieving revision 1.99 diff -u -d -r1.98 -r1.99 --- step.py 20 Aug 2006 22:25:35 -0000 1.98 +++ step.py 21 Aug 2006 00:43:20 -0000 1.99 @@ -1128,8 +1128,12 @@ self.workdir = workdir # required by RemoteShellCommand if description: self.description = description + if isinstance(self.description, str): + self.description = [self.description] if descriptionDone: self.descriptionDone = descriptionDone + if isinstance(self.descriptionDone, str): + self.descriptionDone = [self.descriptionDone] if command: self.command = command # merge a class-level 'logfiles' attribute with one passed in as an From warner at users.sourceforge.net Mon Aug 21 00:43:22 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 00:43:22 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test runutils.py, 1.12, 1.13 test_steps.py, 1.24, 1.25 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv4503/buildbot/test Modified Files: runutils.py test_steps.py Log Message: [project @ accept either a single string or a list for ShellCommand description=] This fixes SF#1524659, thanks to Paul Winkler for the catch. Original author: warner at lothar.com Date: 2006-08-21 00:38:00 Index: runutils.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/runutils.py,v retrieving revision 1.12 retrieving revision 1.13 diff -u -d -r1.12 -r1.13 --- runutils.py 20 Aug 2006 22:25:35 -0000 1.12 +++ runutils.py 21 Aug 2006 00:43:20 -0000 1.13 @@ -229,7 +229,7 @@ s3.stepStarted() return s3 -def makeBuildStep(basedir): +def makeBuildStep(basedir, step_class=BuildStep, **kwargs): bss = setupBuildStepStatus(basedir) ss = SourceStamp() @@ -239,7 +239,7 @@ br = BuildRequest("reason", ss) b = Build([br]) b.setBuilder(b0) - s = BuildStep(b) + s = step_class(build=b, **kwargs) s.setStepStatus(bss) b.setupStatus(bss.getBuild()) return s Index: test_steps.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_steps.py,v retrieving revision 1.24 retrieving revision 1.25 diff -u -d -r1.24 -r1.25 --- test_steps.py 20 Aug 2006 22:25:35 -0000 1.24 +++ test_steps.py 21 Aug 2006 00:43:20 -0000 1.25 @@ -259,6 +259,24 @@ l2.addStdout("twostuff") self.failUnlessEqual(o2.out, "twostuff") + # test more stuff about ShellCommands + + def test_description(self): + s = makeBuildStep("test_steps.Steps.test_description.1", + step_class=ShellCommand, + workdir="dummy", + description=["list", "of", "strings"], + descriptionDone=["another", "list"]) + self.failUnlessEqual(s.description, ["list", "of", "strings"]) + self.failUnlessEqual(s.descriptionDone, ["another", "list"]) + + s = makeBuildStep("test_steps.Steps.test_description.2", + step_class=ShellCommand, + workdir="dummy", + description="single string", + descriptionDone="another string") + self.failUnlessEqual(s.description, ["single string"]) + self.failUnlessEqual(s.descriptionDone, ["another string"]) class VersionCheckingStep(step.BuildStep): def start(self): From warner at users.sourceforge.net Mon Aug 21 00:43:23 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 00:43:23 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.68,1.69 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv4503/docs Modified Files: buildbot.texinfo Log Message: [project @ accept either a single string or a list for ShellCommand description=] This fixes SF#1524659, thanks to Paul Winkler for the catch. Original author: warner at lothar.com Date: 2006-08-21 00:38:00 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.68 retrieving revision 1.69 diff -u -d -r1.68 -r1.69 --- buildbot.texinfo 20 Aug 2006 22:25:35 -0000 1.68 +++ buildbot.texinfo 21 Aug 2006 00:43:20 -0000 1.69 @@ -3389,7 +3389,12 @@ @table @code @item command a list of strings (preferred) or single string (discouraged) which -specifies the command to be run +specifies the command to be run. A list of strings is preferred +because it can be used directly as an argv array. Using a single +string (with embedded spaces) requires the buildslave to pass the +string to /bin/sh for interpretation, which raises all sorts of +difficult questions about how to escape or interpret shell +metacharacters. @item env a dictionary of environment strings which will be added to the child @@ -3455,17 +3460,28 @@ @item description This will be used to describe the command (on the Waterfall display) while the command is still running. It should be a single -imperfect-tense verb, like ``compiling'' or ``testing''. +imperfect-tense verb, like ``compiling'' or ``testing''. The preferred +form is a list of short strings, which allows the HTML Waterfall +display to create narrower columns by emitting a
    tag between each +word. You may also provide a single string. @item descriptionDone This will be used to describe the command once it has finished. A -simple noun like ``compile'' or ``tests'' should be used. +simple noun like ``compile'' or ``tests'' should be used. Like + at code{description}, this may either be a list of short strings or a +single string. If neither @code{description} nor @code{descriptionDone} are set, the actual command arguments will be used to construct the description. This may be a bit too wide to fit comfortably on the Waterfall display. + at example +f.addStep(ShellCommand, command=["make", "test"], + description=["testing"], + descriptionDone=["tests"]) + at end example + @end table @node Simple ShellCommand Subclasses, Writing New BuildSteps, ShellCommand, Build Steps From warner at users.sourceforge.net Mon Aug 21 00:43:32 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 00:43:32 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.710,1.711 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv4538 Modified Files: ChangeLog Log Message: [project @ docs: update to use f.addStep rather than using s() and the constructor list] Original author: warner at lothar.com Date: 2006-08-21 00:41:56 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.710 retrieving revision 1.711 diff -u -d -r1.710 -r1.711 --- ChangeLog 21 Aug 2006 00:43:20 -0000 1.710 +++ ChangeLog 21 Aug 2006 00:43:29 -0000 1.711 @@ -7,6 +7,9 @@ a list of strings is acceptable, and why you might prefer one over the other. Add an example. Fixes SF#1524659, thanks to Paul Winkler for the catch. + (Build Steps): update to use f.addStep() rather than using s() + and the constructor list + * buildbot/process/step.py (ShellCommand): accept either a single string or a list of strings in both description= and descriptionDone= From warner at users.sourceforge.net Mon Aug 21 00:43:32 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 00:43:32 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.69,1.70 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv4538/docs Modified Files: buildbot.texinfo Log Message: [project @ docs: update to use f.addStep rather than using s() and the constructor list] Original author: warner at lothar.com Date: 2006-08-21 00:41:56 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.69 retrieving revision 1.70 diff -u -d -r1.69 -r1.70 --- buildbot.texinfo 21 Aug 2006 00:43:20 -0000 1.69 +++ buildbot.texinfo 21 Aug 2006 00:43:30 -0000 1.70 @@ -2821,21 +2821,17 @@ not actual steps, but rather a tuple of the @code{BuildStep} subclass to be created and a dictionary of arguments. (the actual @code{BuildStep} instances are not created until the Build is started, -so that each Build gets an independent copy of each BuildStep). There -is a convenience function named ``@code{s}'' in the - at code{buildbot.process.factory} module for creating these -specification tuples. It allows you to create a - at code{BuildFactory}-ready list like this: +so that each Build gets an independent copy of each BuildStep). The +preferred way to create these step specifications is with the + at code{BuildFactory}'s @code{addStep} method: @example from buildbot.process import step, factory -from buildbot.process.factory import s -steps = [s(step.SVN, svnurl="http://svn.example.org/Trunk/"), - s(step.ShellCommand, command=["make", "all"]), - s(step.ShellCommand, command=["make", "test"]), - ] -f = factory.BuildFactory(steps) +f = factory.BuildFactory() +f.addStep(step.SVN, svnurl="http://svn.example.org/Trunk/") +f.addStep(step.ShellCommand, command=["make", "all"]) +f.addStep(step.ShellCommand, command=["make", "test"]) @end example The rest of this section lists all the standard BuildStep objects @@ -3163,19 +3159,20 @@ from buildbot.changes.pb import PBChangeSource from buildbot.scheduler import AnyBranchScheduler from buildbot.process import step, factory -from buildbot.process.factory import s c['sources'] = [PBChangeSource()] s1 = AnyBranchScheduler('main', ['trunk', 'features/newthing', 'features/otherthing'], 10*60, ['test-i386', 'test-ppc']) c['schedulers'] = [s1] -source = s(step.SVN, mode='update', - baseURL='svn://svn.example.org/MyProject/', - defaultBranch='trunk') -f = factory.BuildFactory([source, - s(step.Compile, command="make all"), - s(step.Test, command="make test")]) + +f = factory.BuildFactory() +f.addStep(step.SVN, mode='update', + baseURL='svn://svn.example.org/MyProject/', + defaultBranch='trunk') +f.addStep(step.Compile, command="make all") +f.addStep(step.Test, command="make test") + c['builders'] = [ @{'name':'test-i386', 'slavename':'bot-i386', 'builddir':'test-i386', 'factory':f @}, @@ -3402,8 +3399,8 @@ language setting, you might use @example -s(ShellCommand, command=["make", "test"], - env=@{'LANG': 'fr_FR'@}) +f.addStep(ShellCommand, command=["make", "test"], + env=@{'LANG': 'fr_FR'@}) @end example These variable settings will override any existing ones in the @@ -3416,9 +3413,9 @@ following: @example -s(ShellCommand, command=["make", "test"], - env=@{'PYTHONPATH': ["/usr/local/lib/python2.3", - "/home/buildbot/lib/python"] @}) +f.addStep(ShellCommand, command=["make", "test"], + env=@{'PYTHONPATH': ["/usr/local/lib/python2.3", + "/home/buildbot/lib/python"] @}) @end example @item want_stdout @@ -3448,8 +3445,8 @@ the build runs, and any new text will be sent over to the buildmaster. @example -s(ShellCommand, command=["make", "test"], - logfiles=@{"triallog": "_trial_temp/test.log"@}) +f.addStep(ShellCommand, command=["make", "test"], + logfiles=@{"triallog": "_trial_temp/test.log"@}) @end example @@ -3577,11 +3574,10 @@ @example from buildbot.process.step import ShellCommand, WithProperties -s(ShellCommand, - command=["tar", "czf", - WithProperties("build-%s.tar.gz", "revision"), - "source"], - ) +f.addStep(ShellCommand, + command=["tar", "czf", + WithProperties("build-%s.tar.gz", "revision"), + "source"]) @end example If this BuildStep were used in a tree obtained from Subversion, it @@ -3600,11 +3596,10 @@ arguments: @example -s(ShellCommand, - command=["tar", "czf", - WithProperties("build-%(revision)s.tar.gz"), - "source"], - ) +f.addStep(ShellCommand, + command=["tar", "czf", + WithProperties("build-%(revision)s.tar.gz"), + "source"]) @end example Don't forget the extra ``s'' after the closing parenthesis! This is @@ -4027,14 +4022,13 @@ @example from buildbot import locks -from buildbot.process import s, step, factory +from buildbot.process import step, factory db_lock = locks.MasterLock("database") -steps = [s(step.SVN, svnurl="http://example.org/svn/Trunk"), - s(step.ShellCommand, command="make all"), - s(step.ShellCommand, command="make test", locks=[db_lock]), - ] -f = factory.BuildFactory(steps) +f = factory.BuildFactory() +f.addStep(step.SVN, svnurl="http://example.org/svn/Trunk") +f.addStep(step.ShellCommand, command="make all") +f.addStep(step.ShellCommand, command="make test", locks=[db_lock]) b1 = @{'name': 'full1', 'slavename': 'bot-1', builddir='f1', 'factory': f@} b2 = @{'name': 'full2', 'slavename': 'bot-2', builddir='f2', 'factory': f@} b3 = @{'name': 'full3', 'slavename': 'bot-3', builddir='f3', 'factory': f@} @@ -4078,24 +4072,23 @@ @example from buildbot import locks -from buildbot.process import s, step, factory +from buildbot.process import step, factory db_lock = locks.MasterLock("database") cpu_lock = locks.SlaveLock("cpu") -slow_steps = [s(step.SVN, svnurl="http://example.org/svn/Trunk"), - s(step.ShellCommand, command="make all", locks=[cpu_lock]), - s(step.ShellCommand, command="make test", locks=[cpu_lock]), - s(step.ShellCommand, command="make db-test", - locks=[db_lock, cpu_lock]), - ] -slow_f = factory.BuildFactory(slow_steps) -fast_steps = [s(step.SVN, svnurl="http://example.org/svn/Trunk"), - s(step.ShellCommand, command="make all", locks=[]), - s(step.ShellCommand, command="make test", locks=[]), - s(step.ShellCommand, command="make db-test", - locks=[db_lock]), - ] -fast_factory = factory.BuildFactory(fast_steps) +slow_f = factory.BuildFactory() +f.addStep(step.SVN, svnurl="http://example.org/svn/Trunk") +f.addStep(step.ShellCommand, command="make all", locks=[cpu_lock]) +f.addStep(step.ShellCommand, command="make test", locks=[cpu_lock]) +f.addStep(step.ShellCommand, command="make db-test", + locks=[db_lock, cpu_lock]) + +fast_factory = factory.BuildFactory() +f.addStep(step.SVN, svnurl="http://example.org/svn/Trunk") +f.addStep(step.ShellCommand, command="make all", locks=[]) +f.addStep(step.ShellCommand, command="make test", locks=[]) +f.addStep(step.ShellCommand, command="make db-test", locks=[db_lock]) + b1 = @{'name': 'full1', 'slavename': 'bot-slow', builddir='full1', 'factory': slow_factory@} b2 = @{'name': 'full2', 'slavename': 'bot-slow', builddir='full2', @@ -4200,14 +4193,28 @@ @bfindex buildbot.process.factory.BasicSVN The default @code{BuildFactory}, provided in the - at code{buildbot.process.factory} module, is constructed with a list of -``BuildStep specifications'': a list of @code{(step_class, kwargs)} -tuples for each. When asked to create a Build, it loads the list of -steps into the new Build object. When the Build is actually started, -these step specifications are used to create the actual set of -BuildSteps, which are then executed one at a time. For example, a -build which consists of a CVS checkout followed by a @code{make build} -would be constructed as follows: + at code{buildbot.process.factory} module, contains a list of ``BuildStep +specifications'': a list of @code{(step_class, kwargs)} tuples for +each. When asked to create a Build, it loads the list of steps into +the new Build object. When the Build is actually started, these step +specifications are used to create the actual set of BuildSteps, which +are then executed one at a time. For example, a build which consists +of a CVS checkout followed by a @code{make build} would be constructed +as follows: + + at example +from buildbot.process import step, factory + +f = factory.BuildFactory() +f.addStep(step.CVS, cvsroot=CVSROOT, cvsmodule="project", mode="update") +f.addStep(step.Compile, command=["make", "build"]) + at end example + +It is also possible to pass a list of step specifications into the + at code{BuildFactory} when it is created. Using @code{addStep} is +usually simpler, but there are cases where is is more convenient to +create the list of steps ahead of time. To make this approach easier, +a convenience function named @code{s} is available: @example from buildbot.process import step, factory @@ -4215,12 +4222,13 @@ # s is a convenience function, defined with: # def s(steptype, **kwargs): return (steptype, kwargs) -f = factory.BuildFactory([s(step.CVS, - cvsroot=CVSROOT, cvsmodule="project", - mode="update"), - s(step.Compile, command=["make", "build"])]) +all_steps = [s(step.CVS, cvsroot=CVSROOT, cvsmodule="project", mode="update"), + s(step.Compile, command=["make", "build"]), + ] +f = factory.BuildFactory(all_steps) @end example + Each step can affect the build process in the following ways: @itemize @bullet From warner at users.sourceforge.net Mon Aug 21 01:35:54 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 01:35:54 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_web.py,1.32,1.33 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv24858/buildbot/test Modified Files: test_web.py Log Message: [project @ test_web: oops, fix test case to match actual addURL output] Original author: warner at lothar.com Date: 2006-08-21 01:34:52 Index: test_web.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_web.py,v retrieving revision 1.32 retrieving revision 1.33 diff -u -d -r1.32 -r1.33 --- test_web.py 20 Aug 2006 22:25:35 -0000 1.32 +++ test_web.py 21 Aug 2006 01:35:52 -0000 1.33 @@ -270,9 +270,9 @@ s.addURL("icon", "http://coverage.example.org/icon.png") box = html.IBox(s).getBox() td = box.td() - e1 = 'coverage' + e1 = '[coverage]' self.failUnlessSubstring(e1, td) - e2 = 'icon' + e2 = '[icon]' self.failUnlessSubstring(e2, td) From warner at users.sourceforge.net Mon Aug 21 01:35:54 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 01:35:54 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.711,1.712 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv24858 Modified Files: ChangeLog Log Message: [project @ test_web: oops, fix test case to match actual addURL output] Original author: warner at lothar.com Date: 2006-08-21 01:34:52 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.711 retrieving revision 1.712 diff -u -d -r1.711 -r1.712 --- ChangeLog 21 Aug 2006 00:43:29 -0000 1.711 +++ ChangeLog 21 Aug 2006 01:35:52 -0000 1.712 @@ -1,5 +1,10 @@ 2006-08-20 Brian Warner + * buildbot/test/test_web.py (WaterfallSteps.test_urls): oops, + update test case to match new link text.. the HREF has both a + class= setting and an enclosing [] pair that I didn't match in the + test. + * docs/buildbot.texinfo (ShellCommand.command=): explain why a list of strings is preferred over a single string with embedded spaces From warner at users.sourceforge.net Mon Aug 21 03:37:52 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 03:37:52 +0000 Subject: [Buildbot-commits] buildbot/docs/examples twisted_master.cfg, 1.41, 1.42 Message-ID: Update of /cvsroot/buildbot/buildbot/docs/examples In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8286/docs/examples Modified Files: twisted_master.cfg Log Message: [project @ docs/examples/twisted_master.cfg: update from actual installation] Original author: warner at lothar.com Date: 2006-08-21 03:32:57 Index: twisted_master.cfg =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/examples/twisted_master.cfg,v retrieving revision 1.41 retrieving revision 1.42 diff -u -d -r1.41 -r1.42 --- twisted_master.cfg 7 Apr 2006 04:31:37 -0000 1.41 +++ twisted_master.cfg 21 Aug 2006 03:37:50 -0000 1.42 @@ -77,6 +77,16 @@ builders = [] + +b24compile_opts = [ + "-Wignore::PendingDeprecationWarning:distutils.command.build_py", + "-Wignore::PendingDeprecationWarning:distutils.command.build_ext", + ] + + +b25compile_opts = b24compile_opts # FIXME + + b1 = {'name': "quick", 'slavename': "bot1", 'builddir': "quick", @@ -101,10 +111,6 @@ } builders.append(b23) -b24compile_opts = [ - "-Wignore::PendingDeprecationWarning:distutils.command.build_py", - "-Wignore::PendingDeprecationWarning:distutils.command.build_ext", - ] b24 = {'name': "full-2.4", 'slavenames': ["bot-exarkun"], 'builddir': "full2.4", @@ -116,6 +122,28 @@ } builders.append(b24) +b25debian = { + 'name': 'full-2.5-debian', + 'slavenames': ['bot-idnar-debian'], + 'builddir': 'full2.5-debian', + 'factory': FullTwistedBuildFactory(source_copy, + python=["python2.5", "-Wall"], + compileOpts=b24compile_opts), + } +builders.append(b25debian) + + +b24debian64 = { + 'name': 'full-2.4-debian64', + 'slavenames': ['bot-idnar-debian64'], + 'builddir': 'full2.4-debian64', + 'factory': FullTwistedBuildFactory(source_copy, + python=["python2.4", "-Wall"], + compileOpts=b24compile_opts), + } +builders.append(b24debian64) + + # debuild is offline while we figure out how to build 2.0 .debs from SVN # b3 = {'name': "debuild", # 'slavename': "bot2", @@ -143,7 +171,7 @@ python="python2.4", reactors=['qt']), } -builders.append(bqt) +#builders.append(bqt) jf = TwistedReactorsBuildFactory(source_copy, python="python2.4", reactors=["default"]) @@ -169,8 +197,28 @@ compileOpts2=["-c","mingw32"], reactors=["default"]), } -builders.append(b24w32_select) +#builders.append(b24w32_select) + +b25suse = { + 'name': 'full-2.5-suse', + 'slavenames': ['bot-scmikes-2.5'], + 'builddir': 'bot-scmikes-2.5', + 'factory': FullTwistedBuildFactory(source_copy, + python=["python2.5", "-Wall"], + compileOpts=b24compile_opts), + } +builders.append(b25suse) +b24w32_scmikes_select = { + 'name': "win32-scmikes-select", + 'slavename': "bot-scmikes-win32", + 'builddir': "W32-full2.4-scmikes-select", + 'factory': TwistedReactorsBuildFactory(source_copy, + python="python", + compileOpts2=["-c","mingw32"], + reactors=["default"]), + } +builders.append(b24w32_scmikes_select) b24w32_win32er = { 'name': "win32-win32er", @@ -190,7 +238,7 @@ 'builddir': "W32-full2.4-iocp", 'factory': TwistedReactorsBuildFactory(source_copy, python="python", - compileOpts2=["-c","mingw32"], + compileOpts2=[], reactors=["iocp"]), } builders.append(b24w32_iocp) @@ -253,7 +301,8 @@ c['debugPassword'] = private.debugPassword #c['interlocks'] = [("do-deb", ["full-2.2"], ["debuild"])] if hasattr(private, "manhole"): - c['manhole'] = master.Manhole(*private.manhole) + from buildbot import manhole + c['manhole'] = manhole.PasswordManhole(*private.manhole) c['status'].append(client.PBListener(9936)) m = mail.MailNotifier(fromaddr="buildbot at twistedmatrix.com", builders=["quick", "full-2.3"], From warner at users.sourceforge.net Mon Aug 21 03:37:52 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 03:37:52 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.712,1.713 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8286 Modified Files: ChangeLog Log Message: [project @ docs/examples/twisted_master.cfg: update from actual installation] Original author: warner at lothar.com Date: 2006-08-21 03:32:57 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.712 retrieving revision 1.713 diff -u -d -r1.712 -r1.713 --- ChangeLog 21 Aug 2006 01:35:52 -0000 1.712 +++ ChangeLog 21 Aug 2006 03:37:50 -0000 1.713 @@ -1,5 +1,7 @@ 2006-08-20 Brian Warner + * docs/examples/twisted_master.cfg: update to actual practice + * buildbot/test/test_web.py (WaterfallSteps.test_urls): oops, update test case to match new link text.. the HREF has both a class= setting and an enclosing [] pair that I didn't match in the From warner at users.sourceforge.net Mon Aug 21 03:38:06 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 03:38:06 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.713,1.714 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8420 Modified Files: ChangeLog Log Message: [project @ PBChangeSource.prefix= : fix and simplify, just do a string comparison] This effectively resolves SF#1217699 and SF#1381867 Original author: warner at lothar.com Date: 2006-08-21 03:36:38 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.713 retrieving revision 1.714 diff -u -d -r1.713 -r1.714 --- ChangeLog 21 Aug 2006 03:37:50 -0000 1.713 +++ ChangeLog 21 Aug 2006 03:38:04 -0000 1.714 @@ -1,5 +1,23 @@ 2006-08-20 Brian Warner + * buildbot/changes/pb.py (PBChangeSource): fix and simplify + meaning of the prefix= argument. It is now just a string which is + stripped from the beginning of the filename. If prefix= is set but + not found on any given filename, that filename is ignored. If all + filenames in a Change are ignored, the Change is dropped. This is + much simpler than the previous sep= nonsense, and I should have + implemented it this way from the beginning. Effectively resolves + SF#1217699 and SF#1381867. Thanks to Gary Granger and Marius + Gedminas for the catch and suggested fixes. + (ChangePerspective.perspective_addChange): implement the actual + prefix comparison + * buildbot/test/test_changes.py (TestChangePerspective): test it + * docs/buildbot.texinfo (PBChangeSource): document it, explain + how to properly use prefix= + * docs/examples/twisted_master.cfg (source): update prefix= by + adding the trailing slash + + * docs/examples/twisted_master.cfg: update to actual practice * buildbot/test/test_web.py (WaterfallSteps.test_urls): oops, From warner at users.sourceforge.net Mon Aug 21 03:38:07 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 03:38:07 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_changes.py,1.7,1.8 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8420/buildbot/test Modified Files: test_changes.py Log Message: [project @ PBChangeSource.prefix= : fix and simplify, just do a string comparison] This effectively resolves SF#1217699 and SF#1381867 Original author: warner at lothar.com Date: 2006-08-21 03:36:38 Index: test_changes.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_changes.py,v retrieving revision 1.7 retrieving revision 1.8 diff -u -d -r1.7 -r1.8 --- test_changes.py 25 Oct 2005 01:26:10 -0000 1.7 +++ test_changes.py 21 Aug 2006 03:38:05 -0000 1.8 @@ -18,6 +18,9 @@ d3 = {'files': ["Project/baz.c", "OtherProject/bloo.c"], 'who': "alice", 'comments': "mixed changes"} +d4 = {'files': ["trunk/baz.c", "branches/foobranch/foo.c", "trunk/bar.c"], + 'who': "alice", + 'comments': "mixed changes"} class TestChangePerspective(unittest.TestCase): @@ -38,7 +41,7 @@ self.failUnlessEqual(c1.who, "marvin") def testPrefix(self): - p = pb.ChangePerspective(self, "Project") + p = pb.ChangePerspective(self, "Project/") p.perspective_addChange(d1) self.failUnlessEqual(len(self.changes), 1) @@ -58,6 +61,42 @@ self.failUnlessEqual(c3.comments, "mixed changes") self.failUnlessEqual(c3.who, "alice") + def testPrefix2(self): + p = pb.ChangePerspective(self, "Project/bar/") + + p.perspective_addChange(d1) + self.failUnlessEqual(len(self.changes), 1) + c1 = self.changes[-1] + self.failUnlessEqual(c1.files, ["boo.c"]) + self.failUnlessEqual(c1.comments, "Some changes in Project") + self.failUnlessEqual(c1.who, "marvin") + + p.perspective_addChange(d2) # should be ignored + self.failUnlessEqual(len(self.changes), 1) + + p.perspective_addChange(d3) # should ignore this too + self.failUnlessEqual(len(self.changes), 1) + + def testPrefix3(self): + p = pb.ChangePerspective(self, "trunk/") + + p.perspective_addChange(d4) + self.failUnlessEqual(len(self.changes), 1) + c1 = self.changes[-1] + self.failUnlessEqual(c1.files, ["baz.c", "bar.c"]) + self.failUnlessEqual(c1.comments, "mixed changes") + + def testPrefix4(self): + p = pb.ChangePerspective(self, "branches/foobranch/") + + p.perspective_addChange(d4) + self.failUnlessEqual(len(self.changes), 1) + c1 = self.changes[-1] + self.failUnlessEqual(c1.files, ["foo.c"]) + self.failUnlessEqual(c1.comments, "mixed changes") + + + config_empty = """ BuildmasterConfig = c = {} c['bots'] = [] From warner at users.sourceforge.net Mon Aug 21 03:38:07 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 03:38:07 +0000 Subject: [Buildbot-commits] buildbot/docs/examples twisted_master.cfg, 1.42, 1.43 Message-ID: Update of /cvsroot/buildbot/buildbot/docs/examples In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8420/docs/examples Modified Files: twisted_master.cfg Log Message: [project @ PBChangeSource.prefix= : fix and simplify, just do a string comparison] This effectively resolves SF#1217699 and SF#1381867 Original author: warner at lothar.com Date: 2006-08-21 03:36:38 Index: twisted_master.cfg =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/examples/twisted_master.cfg,v retrieving revision 1.42 retrieving revision 1.43 diff -u -d -r1.42 -r1.43 --- twisted_master.cfg 21 Aug 2006 03:37:50 -0000 1.42 +++ twisted_master.cfg 21 Aug 2006 03:38:05 -0000 1.43 @@ -50,7 +50,7 @@ # prefix, so that the Builders all see sensible pathnames (which means they # can do things like ignore the sandbox properly). -source = PBChangeSource(prefix="trunk") +source = PBChangeSource(prefix="trunk/") c['sources'].append(source) From warner at users.sourceforge.net Mon Aug 21 03:38:07 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 03:38:07 +0000 Subject: [Buildbot-commits] buildbot/buildbot/changes pb.py,1.10,1.11 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/changes In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8420/buildbot/changes Modified Files: pb.py Log Message: [project @ PBChangeSource.prefix= : fix and simplify, just do a string comparison] This effectively resolves SF#1217699 and SF#1381867 Original author: warner at lothar.com Date: 2006-08-21 03:36:38 Index: pb.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/pb.py,v retrieving revision 1.10 retrieving revision 1.11 diff -u -d -r1.10 -r1.11 --- pb.py 15 Nov 2005 08:38:05 -0000 1.10 +++ pb.py 21 Aug 2006 03:38:04 -0000 1.11 @@ -10,13 +10,9 @@ class ChangePerspective(NewCredPerspective): - def __init__(self, changemaster, prefix, sep="/"): + def __init__(self, changemaster, prefix): self.changemaster = changemaster self.prefix = prefix - # this is the separator as used by the VC system, not the local host. - # If for some reason you're running your CVS repository under - # windows, you'll need to use a PBChangeSource(sep="\\") - self.sep = sep def attached(self, mind): return self @@ -26,16 +22,13 @@ def perspective_addChange(self, changedict): log.msg("perspective_addChange called") pathnames = [] + prefixpaths = None for path in changedict['files']: if self.prefix: - bits = path.split(self.sep) - if bits[0] == self.prefix: - if bits[1:]: - path = self.sep.join(bits[1:]) - else: - path = '' - else: - break + if not path.startswith(self.prefix): + # this file does not start with the prefix, so ignore it + continue + path = path[len(self.prefix):] pathnames.append(path) if pathnames: @@ -48,10 +41,40 @@ self.changemaster.addChange(change) class PBChangeSource(base.ChangeSource): - compare_attrs = ["user", "passwd", "port", "prefix", "sep"] + compare_attrs = ["user", "passwd", "port", "prefix"] def __init__(self, user="change", passwd="changepw", port=None, - prefix=None, sep="/"): + prefix=None, sep=None): + """I listen on a TCP port for Changes from 'buildbot sendchange'. + + I am a ChangeSource which will accept Changes from a remote source. I + share a TCP listening port with the buildslaves. + + Both the 'buildbot sendchange' command and the + contrib/svn_buildbot.py tool know how to send changes to me. + + @type prefix: string (or None) + @param prefix: if set, I will ignore any filenames that do not start + with this string. Moreover I will remove this string + from all filenames before creating the Change object + and delivering it to the Schedulers. This is useful + for changes coming from version control systems that + represent branches as parent directories within the + repository (like SVN and Perforce). Use a prefix of + 'trunk/' or 'project/branches/foobranch/' to only + follow one branch and to get correct tree-relative + filenames. + + @param sep: DEPRECATED (with an axe). sep= was removed in + buildbot-0.7.4 . Instead of using it, you should use + prefix= with a trailing directory separator. This + docstring (and the better-than-nothing error message + which occurs when you use it) will be removed in 0.7.5 . + """ + + # sep= was removed in 0.7.4 . This more-helpful-than-nothing error + # message will be removed in 0.7.5 . + assert sep is None, "prefix= is now a complete string, do not use sep=" # TODO: current limitations assert user == "change" assert passwd == "changepw" @@ -60,7 +83,6 @@ self.passwd = passwd self.port = port self.prefix = prefix - self.sep = sep def describe(self): # TODO: when the dispatcher is fixed, report the specific port @@ -85,5 +107,5 @@ master.dispatcher.unregister(self.user) def getPerspective(self): - return ChangePerspective(self.parent, self.prefix, self.sep) + return ChangePerspective(self.parent, self.prefix) From warner at users.sourceforge.net Mon Aug 21 03:38:07 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 21 Aug 2006 03:38:07 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.70,1.71 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8420/docs Modified Files: buildbot.texinfo Log Message: [project @ PBChangeSource.prefix= : fix and simplify, just do a string comparison] This effectively resolves SF#1217699 and SF#1381867 Original author: warner at lothar.com Date: 2006-08-21 03:36:38 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.70 retrieving revision 1.71 diff -u -d -r1.70 -r1.71 --- buildbot.texinfo 21 Aug 2006 00:43:30 -0000 1.70 +++ buildbot.texinfo 21 Aug 2006 03:38:05 -0000 1.71 @@ -2710,8 +2710,8 @@ be removed in the future.}. -The @code{PBChangeSource} is created with the following -arguments: +The @code{PBChangeSource} is created with the following arguments. All +are optional. @table @samp @item @code{port} @@ -2720,14 +2720,29 @@ Implemented, always set to @code{None}}. @item @code{user} and @code{passwd} -the user/passwd account information that the client program must use +The user/passwd account information that the client program must use to connect. Defaults to @code{change} and @code{changepw}. @emph{Not Implemented, @code{user} is currently always set to @code{change}, @code{passwd} is always set to @code{changepw}}. @item @code{prefix} -the prefix to be found and stripped from filenames delivered over the -connection. +The prefix to be found and stripped from filenames delivered over the +connection. Any filenames which do not start with this prefix will be +removed. If all the filenames in a given Change are removed, the that +whole Change will be dropped. This string should probably end with a +directory separator. + +This is useful for changes coming from version control systems that +represent branches as parent directories within the repository (like +SVN and Perforce). Use a prefix of 'trunk/' or +'project/branches/foobranch/' to only follow one branch and to get +correct tree-relative filenames. Without a prefix, the PBChangeSource +will probably deliver Changes with filenames like @file{trunk/foo.c} +instead of just @file{foo.c}. Of course this also depends upon the +tool sending the Changes in (like @command{buildbot sendchange}) and +what filenames it is delivering: that tool may be filtering and +stripping prefixes at the sending end. + @end table From warner at users.sourceforge.net Wed Aug 23 04:02:20 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 04:02:20 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_web.py,1.33,1.34 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv11396/buildbot/test Modified Files: test_web.py Log Message: [project @ tests: remove test_web.GetURL.testBrokenStuff, it was broken anyway] Original author: warner at lothar.com Date: 2006-08-23 03:57:43 Index: test_web.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_web.py,v retrieving revision 1.33 retrieving revision 1.34 diff -u -d -r1.33 -r1.34 --- test_web.py 21 Aug 2006 01:35:52 -0000 1.33 +++ test_web.py 23 Aug 2006 04:02:18 -0000 1.34 @@ -352,14 +352,6 @@ builder = s.getBuilder("b1") self.assertURLEqual(builder, "b1") - def testBrokenStuff(self): - s = self.status - self.assertURLEqual(s.getSchedulers()[0], "schedulers/0") - self.assertURLEqual(s.getSlave("bot1"), "slaves/bot1") - # we didn't put a Change into the actual Build before, so this fails - #self.assertURLEqual(build.getChanges()[0], "changes/1") - testBrokenStuff.todo = "not implemented yet" - def testChange(self): s = self.status c = Change("user", ["foo.c"], "comments") From warner at users.sourceforge.net Wed Aug 23 04:02:20 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 04:02:20 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.714,1.715 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv11396 Modified Files: ChangeLog Log Message: [project @ tests: remove test_web.GetURL.testBrokenStuff, it was broken anyway] Original author: warner at lothar.com Date: 2006-08-23 03:57:43 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.714 retrieving revision 1.715 diff -u -d -r1.714 -r1.715 --- ChangeLog 21 Aug 2006 03:38:04 -0000 1.714 +++ ChangeLog 23 Aug 2006 04:02:18 -0000 1.715 @@ -1,3 +1,9 @@ +2006-08-22 Brian Warner + + * buildbot/test/test_web.py (GetURL.testBrokenStuff): delete this + test.. I think the web-parts effort will render it pointless well + before it ever actually starts to work. + 2006-08-20 Brian Warner * buildbot/changes/pb.py (PBChangeSource): fix and simplify From warner at users.sourceforge.net Wed Aug 23 04:02:32 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 04:02:32 +0000 Subject: [Buildbot-commits] buildbot ChangeLog, 1.715, 1.716 MANIFEST.in, 1.21, 1.22 Makefile, 1.19, 1.20 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv11434 Modified Files: ChangeLog MANIFEST.in Makefile Log Message: [project @ stop shipping the PyCon-2003 paper, start shipping .html manual w/images] Original author: warner at lothar.com Date: 2006-08-23 04:01:08 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.715 retrieving revision 1.716 diff -u -d -r1.715 -r1.716 --- ChangeLog 23 Aug 2006 04:02:18 -0000 1.715 +++ ChangeLog 23 Aug 2006 04:02:30 -0000 1.716 @@ -1,5 +1,11 @@ 2006-08-22 Brian Warner + * MANIFEST.in: stop shipping the old PyCon-2003 paper.. with the + new diagrams, the user's manual is more informative than it was. + Start shipping the .html user's manual (and generated .png + images). + * Makefile: update 'release' target to match + * buildbot/test/test_web.py (GetURL.testBrokenStuff): delete this test.. I think the web-parts effort will render it pointless well before it ever actually starts to work. Index: MANIFEST.in =================================================================== RCS file: /cvsroot/buildbot/buildbot/MANIFEST.in,v retrieving revision 1.21 retrieving revision 1.22 diff -u -d -r1.21 -r1.22 --- MANIFEST.in 18 Jun 2005 02:50:39 -0000 1.21 +++ MANIFEST.in 23 Aug 2006 04:02:30 -0000 1.22 @@ -1,10 +1,9 @@ include ChangeLog MANIFEST.in README README.w32 NEWS -include docs/PyCon-2003/buildbot.html docs/PyCon-2003/stylesheet.css -include docs/PyCon-2003/overview.png docs/PyCon-2003/slave.png -include docs/PyCon-2003/waterfall.png include docs/examples/*.cfg -include docs/buildbot.texinfo docs/buildbot.info +include docs/buildbot.texinfo +include docs/buildbot.info +include docs/buildbot.html docs/images/*.png include docs/epyrun docs/gen-reference include buildbot/test/mail/* buildbot/test/subdir/* include buildbot/scripts/sample.cfg @@ -12,6 +11,4 @@ include buildbot/clients/debug.glade include buildbot/buildbot.png -exclude buildbot/test/test_trial.py - include contrib/* contrib/windows/* Index: Makefile =================================================================== RCS file: /cvsroot/buildbot/buildbot/Makefile,v retrieving revision 1.19 retrieving revision 1.20 diff -u -d -r1.19 -r1.20 --- Makefile 16 Jul 2006 03:34:21 -0000 1.19 +++ Makefile 23 Aug 2006 04:02:30 -0000 1.20 @@ -30,9 +30,9 @@ "snapshot build" debuild binary -.PHONY: docs apidocs paper +.PHONY: docs apidocs some-apidocs paper docs: - $(MAKE) -C docs buildbot.info + $(MAKE) -C docs buildbot.info buildbot.html apidocs: PYTHONPATH=.:$(T) python docs/epyrun -o docs/reference @@ -41,12 +41,12 @@ paper: $(MAKE) -C docs/PyCon-2003 all -release: docs paper +release: docs chmod 0755 . find buildbot contrib docs -type d -exec chmod 0755 {} \; find bin buildbot contrib docs -type f -exec chmod 0644 {} \; chmod 0644 ChangeLog MANIFEST* NEWS README* setup.py - chmod a+x bin/buildbot contrib/*.py + chmod a+x bin/buildbot contrib/*.py contrib/windows/*.py rm -rf _trial_temp python ./setup.py clean rm -f MANIFEST From warner at users.sourceforge.net Wed Aug 23 07:04:38 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 07:04:38 +0000 Subject: [Buildbot-commits] buildbot CREDITS, NONE, 1.1 ChangeLog, 1.716, 1.717 MANIFEST.in, 1.22, 1.23 NEWS, 1.56, 1.57 README, 1.32, 1.33 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv18437 Modified Files: ChangeLog MANIFEST.in NEWS README Added Files: CREDITS Log Message: [project @ update README and NEWS, add CREDITS] Original author: warner at lothar.com Date: 2006-08-23 05:11:12 --- NEW FILE: CREDITS --- This is a list of everybody who has contributed to Buildbot in some way, in no particular order. Thanks everybody! Scott Lamb Olivier Bonnet Mark Hammond Gary Granger Marius Gedminas Paul Winkler John O'Duinn JP Calderone Zandr Milewski Niklaus Giger Tobi Vollebregt John Pye Neal Norwitz Anthony Baxter AllMyData.com Clement Stenac Kevin Turner Steven Walter Dobes Vandermeer Brad Hards Nathaniel Smith Mark Dillavou Thomas Vander Stichele Bear Brandon Philips Nick Trout Paul Warren Rene Rivera Baptiste Lepilleur Gerald Combs Yoz Grahame Alexander Staubo Elliot Murphy Stephen Davis Kirill Lapshin Dave Peticolas Andrew Bennetts Olly Betts Philipp Frauenfelder Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.716 retrieving revision 1.717 diff -u -d -r1.716 -r1.717 --- ChangeLog 23 Aug 2006 04:02:30 -0000 1.716 +++ ChangeLog 23 Aug 2006 07:04:36 -0000 1.717 @@ -1,5 +1,10 @@ 2006-08-22 Brian Warner + * README: update + + * CREDITS: new file, list of people who have helped. Thanks! + * MANIFEST.in: ship it + * MANIFEST.in: stop shipping the old PyCon-2003 paper.. with the new diagrams, the user's manual is more informative than it was. Start shipping the .html user's manual (and generated .png Index: MANIFEST.in =================================================================== RCS file: /cvsroot/buildbot/buildbot/MANIFEST.in,v retrieving revision 1.22 retrieving revision 1.23 diff -u -d -r1.22 -r1.23 --- MANIFEST.in 23 Aug 2006 04:02:30 -0000 1.22 +++ MANIFEST.in 23 Aug 2006 07:04:36 -0000 1.23 @@ -1,5 +1,5 @@ -include ChangeLog MANIFEST.in README README.w32 NEWS +include ChangeLog MANIFEST.in README README.w32 NEWS CREDITS include docs/examples/*.cfg include docs/buildbot.texinfo include docs/buildbot.info Index: NEWS =================================================================== RCS file: /cvsroot/buildbot/buildbot/NEWS,v retrieving revision 1.56 retrieving revision 1.57 diff -u -d -r1.56 -r1.57 --- NEWS 6 Aug 2006 21:14:44 -0000 1.56 +++ NEWS 23 Aug 2006 07:04:36 -0000 1.57 @@ -2,6 +2,20 @@ * Release ?.?.? (?) +** Things You Need To Know + +The PBChangeSource's prefix= argument has changed, you probably need to add a +slash now. This is mostly used by sites which use Subversion and +svn_buildbot.py. + +The subcommands that are used to create a buildmaster or a buildslave have +changed. They used to be called 'buildbot master' and 'buildbot slave'. Now +they are called 'buildbot create-master' and 'buildbot create-slave'. Zipf's +Law suggests that these are more appropriate names for these +infrequently-used commands. + +The syntax for the c['manhole'] feature has changed. + ** new features *** full Perforce support @@ -23,6 +37,32 @@ files that use them must be updated. The "Debug options" section in the user's manual provides a complete description. +*** Multiple Logfiles + +BuildSteps can watch multiple log files in realtime, not just stdout/stderr. +This works in a similar fashion to 'tail -f': the file is polled once per +second, and any new data is sent to the buildmaster. + +This requires a buildslave running 0.7.4 or later, and a warning message is +produced if used against an old buildslave (which will otherwise produce no +data). Use "logfiles={'name': 'filename'}" to take advantage of this feature +from master.cfg, and see the "ShellCommand" section of the user's manual for +full documentation. + +The 'Trial' buildstep has been updated to use this, to display +_trial_temp/test.log in realtime. It also knows to fall back to the previous +"cat" command if the buildslave is too old. + +*** BuildStep URLs + +BuildSteps can now add arbitrary URLs which will be displayed on the +Waterfall page in the same place that Logs are presented. This is intended to +provide a link to generated HTML pages, such as the output of a code coverage +tool. The step is responsible for somehow uploading the HTML to a web server: +this feature merely provides an easy way to present the HREF link to the +user. See the "BuildStep URLs" section of the user's manual for details and +examples. + *** LogObservers BuildSteps can now attach LogObservers to various logfiles, allowing them to @@ -35,30 +75,29 @@ The 'Trial' buildstep has been updated to use this for progress tracking, by counting how many test cases have run. -*** Multiple Logfiles - -BuildSteps can watch multiple log files in realtime, not just stdout/stderr. -This works in a similar fashion to 'tail -f': the file is polled once per -second, and any new data is sent to the buildmaster. - -This requires a buildslave running 0.7.4 or later, and a warning message is -produced if used against an old buildslave. Use "logfiles={'name': -'filename'}" to take advantage of this feature from master.cfg, and see the -"ShellCommand" section of the user's manual for full documentation. - -The 'Trial' buildstep has been updated to use this, to display -_trial_temp/test.log in realtime. - ** new documentation What classes are useful in your master.cfg file? A table of them has been added to the user's manual, in a section called "Index of Useful Classes". +Want a list of all the keys in master.cfg? Look in the "Index of master.cfg +keys" section. + A number of pretty diagrams have been added to the "System Architecture" portion of the manual, explaining how all the buildbot pieces fit together. +An HTML form of the user's manual is now shipped in the source tarball. This +makes it a bit bigger: sorry about that. The old PyCon-2003 paper has been +removed from the distribution, as it is mostly supplanted by the user's +manual by this point. + ** bugfixes +SF#1217699 + SF#1381867: The prefix= argument to PBChangeSource has been +changed: now it does just a simple string-prefix match and strip. The +previous behavior was buggy and unhelpful. NOTE: if you were using prefix= +before, you probably need to add a slash to the end of it. + SF#1398174: ignore SVN property changes better, fixed by Olivier Bonnet SF#1452801: don't double-escape the build URL, fixed by Olivier Bonnet @@ -68,8 +107,8 @@ reloading unchanged config files with WithProperties shouldn't change anything. All svn commands now include --non-interactive so they won't ask for -passwords. Instead the command fails if it cannot be performed without user -input. +passwords. Instead, the command will fail if it cannot be performed without +user input. Deprecation warnings with newer versions of Twisted have been hushed. @@ -93,6 +132,13 @@ will pop up an HTML tooltip to show the reason for the build. Thanks to Zandr Milewski for the suggestion. +contrib/CSS/*.css now contains several contributed stylesheets to make the +Waterfall display a bit less ugly. Thanks to John O'Duinn for gathering them. + +ShellCommand and its derivatives can now accept either a string or a list of +strings in the description= and descriptionDone= arguments. Thanks to Paul +Winkler for the catch. + * Release 0.7.3 (23 May 2006) Index: README =================================================================== RCS file: /cvsroot/buildbot/buildbot/README,v retrieving revision 1.32 retrieving revision 1.33 diff -u -d -r1.32 -r1.33 --- README 11 Aug 2006 07:31:03 -0000 1.32 +++ README 23 Aug 2006 07:04:36 -0000 1.33 @@ -52,21 +52,26 @@ Python: http://www.python.org Buildbot requires python-2.2 or later, and is primarily developed against - python-2.3. The buildmaster uses generators, a feature which is not - available in python-2.1, and both master and slave require a version of - Twisted which only works with python-2.2 or later. Certain features (like - the inclusion of build logs in status emails) require python-2.2.2 or - later, while the IRC 'force' command requires python-2.3 . + python-2.3. Specifically, it requires generators, and a version of Twisted + which only works with python-2.2 or later. Certain features (like the + inclusion of build logs in status emails) require python-2.2.2 or later, + while the IRC 'force' command requires python-2.3 . It has been tested + against python-2.4 . Twisted: http://twistedmatrix.com Both the buildmaster and the buildslaves require Twisted-2.0.x or later. - As always, the most recent version is recommended. + As always, the most recent version is recommended. It has been tested + against Twisted-2.4.0 and Twisted SVN as of the date of release. When using the split subpackages of Twisted-2.x.x, you'll need at least "Twisted" (the core package), and you'll also want TwistedMail, TwistedWeb, and TwistedWords (for sending email, serving a web status - page, and delivering build status via IRC, respectively). + page, and delivering build status via IRC, respectively). You might also + want TwistedConch (for the encrypted Manhole debug port). Note that recent + Twisteds require ZopeInterface to be installed as well. + +ACCESSORIES: CVSToys: http://purl.net/net/CVSToys @@ -78,9 +83,8 @@ INSTALLATION: -Please read the User's Manual in docs/buildbot.info (or in HTML form on the -buildbot web site) for complete instructions. This file only contains a brief -summary. +Please read the User's Manual in docs/buildbot.info or docs/buildbot.html for +complete instructions. This file only contains a brief summary. RUNNING THE UNIT TESTS @@ -88,11 +92,12 @@ PYTHONPATH=. trial buildbot.test -This should run up to 175 tests, depending upon what VC tools you have -installed. On my desktop machine it takes about four minutes to complete. -Nothing should fail, a few might be skipped. If any of the tests fail, you -should stop and investigate the cause before continuing the installation -process, as it will probably be easier to track down the bug early. +This should run up to 221 tests, depending upon what VC tools you have +installed. On my desktop machine it takes about six minutes to complete. +Nothing should fail (at least under unix), a few might be skipped. If any of +the tests fail, you should stop and investigate the cause before continuing +the installation process, as it will probably be easier to track down the bug +early. Neither CVS nor SVN support file based repositories on network filesystem (or network drives in Windows parlance). Therefore it is recommended to run @@ -108,11 +113,9 @@ To test this, shift to a different directory (like /tmp), and run: - pydoc buildbot + buildbot --version -If it shows you a brief description of the package and its contents, the -install went ok. If it says "no Python documentation found for 'buildbot'", -then something went wrong. +If it announces the versions of Buildbot and Twisted, the install went ok. SETTING UP A BUILD SLAVE: From warner at users.sourceforge.net Wed Aug 23 07:13:39 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 07:13:39 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.71,1.72 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22007/docs Modified Files: buildbot.texinfo Log Message: [project @ releasing buildbot-0.7.4] Original author: warner at lothar.com Date: 2006-08-23 07:10:32 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.71 retrieving revision 1.72 diff -u -d -r1.71 -r1.72 --- buildbot.texinfo 21 Aug 2006 03:38:05 -0000 1.71 +++ buildbot.texinfo 23 Aug 2006 07:13:36 -0000 1.72 @@ -1,7 +1,7 @@ \input texinfo @c -*-texinfo-*- @c %**start of header @setfilename buildbot.info - at settitle BuildBot Manual 0.7.3+ + at settitle BuildBot Manual 0.7.4 @defcodeindex cs @defcodeindex sl @defcodeindex bf From warner at users.sourceforge.net Wed Aug 23 07:13:38 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 07:13:38 +0000 Subject: [Buildbot-commits] buildbot/buildbot __init__.py,1.24,1.25 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22007/buildbot Modified Files: __init__.py Log Message: [project @ releasing buildbot-0.7.4] Original author: warner at lothar.com Date: 2006-08-23 07:10:32 Index: __init__.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/__init__.py,v retrieving revision 1.24 retrieving revision 1.25 diff -u -d -r1.24 -r1.25 --- __init__.py 28 May 2006 01:59:45 -0000 1.24 +++ __init__.py 23 Aug 2006 07:13:36 -0000 1.25 @@ -1,3 +1,3 @@ #! /usr/bin/python -version = "0.7.3+" +version = "0.7.4" From warner at users.sourceforge.net Wed Aug 23 07:13:38 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 07:13:38 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.717,1.718 NEWS,1.57,1.58 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22007 Modified Files: ChangeLog NEWS Log Message: [project @ releasing buildbot-0.7.4] Original author: warner at lothar.com Date: 2006-08-23 07:10:32 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.717 retrieving revision 1.718 diff -u -d -r1.717 -r1.718 --- ChangeLog 23 Aug 2006 07:04:36 -0000 1.717 +++ ChangeLog 23 Aug 2006 07:13:36 -0000 1.718 @@ -1,3 +1,11 @@ +2006-08-23 Brian Warner + + * buildbot/__init__.py (version): Releasing buildbot-0.7.4 + * docs/buildbot.texinfo: set version to match + * NEWS: update for 0.7.4 + * buildbot/slave/commands.py (command_version): mention that this + version (2.1) was released with buildbot-0.7.4 + 2006-08-22 Brian Warner * README: update Index: NEWS =================================================================== RCS file: /cvsroot/buildbot/buildbot/NEWS,v retrieving revision 1.57 retrieving revision 1.58 diff -u -d -r1.57 -r1.58 --- NEWS 23 Aug 2006 07:04:36 -0000 1.57 +++ NEWS 23 Aug 2006 07:13:36 -0000 1.58 @@ -1,6 +1,6 @@ User visible changes in Buildbot. -* Release ?.?.? (?) +* Release 0.7.4 (23 Aug 2006) ** Things You Need To Know From warner at users.sourceforge.net Wed Aug 23 07:13:38 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 07:13:38 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.60,1.61 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22007/buildbot/slave Modified Files: commands.py Log Message: [project @ releasing buildbot-0.7.4] Original author: warner at lothar.com Date: 2006-08-23 07:10:32 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.60 retrieving revision 1.61 diff -u -d -r1.60 -r1.61 --- commands.py 6 Aug 2006 23:30:28 -0000 1.60 +++ commands.py 23 Aug 2006 07:13:36 -0000 1.61 @@ -31,6 +31,7 @@ # (not externally visible: ShellCommandPP has writeStdin/closeStdin. # ShellCommand accepts new arguments (logfiles=, initialStdin=, # keepStdinOpen=) and no longer accepts stdin=) +# (release 0.7.4) class CommandInterrupted(Exception): pass From warner at users.sourceforge.net Wed Aug 23 22:41:19 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 22:41:19 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.72,1.73 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv13832/docs Modified Files: buildbot.texinfo Log Message: [project @ bump version to 0.7.4+ while between releases] Original author: warner at lothar.com Date: 2006-08-23 22:39:39 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.72 retrieving revision 1.73 diff -u -d -r1.72 -r1.73 --- buildbot.texinfo 23 Aug 2006 07:13:36 -0000 1.72 +++ buildbot.texinfo 23 Aug 2006 22:41:17 -0000 1.73 @@ -1,7 +1,7 @@ \input texinfo @c -*-texinfo-*- @c %**start of header @setfilename buildbot.info - at settitle BuildBot Manual 0.7.4 + at settitle BuildBot Manual 0.7.4+ @defcodeindex cs @defcodeindex sl @defcodeindex bf From warner at users.sourceforge.net Wed Aug 23 22:41:19 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 22:41:19 +0000 Subject: [Buildbot-commits] buildbot/buildbot __init__.py,1.25,1.26 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv13832/buildbot Modified Files: __init__.py Log Message: [project @ bump version to 0.7.4+ while between releases] Original author: warner at lothar.com Date: 2006-08-23 22:39:39 Index: __init__.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/__init__.py,v retrieving revision 1.25 retrieving revision 1.26 diff -u -d -r1.25 -r1.26 --- __init__.py 23 Aug 2006 07:13:36 -0000 1.25 +++ __init__.py 23 Aug 2006 22:41:17 -0000 1.26 @@ -1,3 +1,3 @@ #! /usr/bin/python -version = "0.7.4" +version = "0.7.4+" From warner at users.sourceforge.net Wed Aug 23 22:41:19 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 23 Aug 2006 22:41:19 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.718,1.719 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv13832 Modified Files: ChangeLog Log Message: [project @ bump version to 0.7.4+ while between releases] Original author: warner at lothar.com Date: 2006-08-23 22:39:39 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.718 retrieving revision 1.719 diff -u -d -r1.718 -r1.719 --- ChangeLog 23 Aug 2006 07:13:36 -0000 1.718 +++ ChangeLog 23 Aug 2006 22:41:17 -0000 1.719 @@ -1,5 +1,11 @@ 2006-08-23 Brian Warner + * buildbot/__init__.py (version): bump to 0.7.4+ while between + releases + * docs/buildbot.texinfo: same + +2006-08-23 Brian Warner + * buildbot/__init__.py (version): Releasing buildbot-0.7.4 * docs/buildbot.texinfo: set version to match * NEWS: update for 0.7.4 From warner at users.sourceforge.net Thu Aug 24 10:05:22 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Thu, 24 Aug 2006 10:05:22 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_locks.py,1.3,1.4 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27952/buildbot/test Modified Files: test_locks.py Log Message: [project @ locks: can now have multiple simultaneous owners. fixes SF#1434997] Original author: warner at lothar.com Date: 2006-08-24 10:03:52 Index: test_locks.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_locks.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- test_locks.py 26 Nov 2005 02:14:31 -0000 1.3 +++ test_locks.py 24 Aug 2006 10:05:20 -0000 1.4 @@ -1,14 +1,275 @@ # -*- test-case-name: buildbot.test.test_locks -*- +import random + from twisted.trial import unittest -from twisted.internet import defer +from twisted.internet import defer, reactor -from buildbot import interfaces +from buildbot import interfaces, master from buildbot.process import step from buildbot.sourcestamp import SourceStamp from buildbot.process.base import BuildRequest from buildbot.test.runutils import RunMixin from buildbot.twcompat import maybeWait +from buildbot import locks + +def claimHarder(lock, owner): + """Return a Deferred that will fire when the lock is claimed. Keep trying + until we succeed.""" + if lock.isAvailable(): + #print "claimHarder(%s): claiming" % owner + lock.claim(owner) + return defer.succeed(lock) + #print "claimHarder(%s): waiting" % owner + d = lock.waitUntilMaybeAvailable(owner) + d.addCallback(claimHarder, owner) + return d + +def hold(lock, owner, mode="now"): + if mode == "now": + lock.release(owner) + elif mode == "very soon": + reactor.callLater(0, lock.release, owner) + elif mode == "soon": + reactor.callLater(0.1, lock.release, owner) + + +class Unit(unittest.TestCase): + def testNow(self): + l = locks.BaseLock("name") + self.failUnless(l.isAvailable()) + l.claim("owner1") + self.failIf(l.isAvailable()) + l.release("owner1") + self.failUnless(l.isAvailable()) + + def testLater(self): + lock = locks.BaseLock("name") + d = claimHarder(lock, "owner1") + d.addCallback(lambda lock: lock.release("owner1")) + return d + + def testCompetition(self): + lock = locks.BaseLock("name") + d = claimHarder(lock, "owner1") + d.addCallback(self._claim1) + return d + def _claim1(self, lock): + # we should have claimed it by now + self.failIf(lock.isAvailable()) + # now set up two competing owners. We don't know which will get the + # lock first. + d2 = claimHarder(lock, "owner2") + d2.addCallback(hold, "owner2", "now") + d3 = claimHarder(lock, "owner3") + d3.addCallback(hold, "owner3", "soon") + dl = defer.DeferredList([d2,d3]) + dl.addCallback(self._cleanup, lock) + # and release the lock in a moment + reactor.callLater(0.1, lock.release, "owner1") + return dl + + def _cleanup(self, res, lock): + d = claimHarder(lock, "cleanup") + d.addCallback(lambda lock: lock.release("cleanup")) + return d + + def testRandom(self): + lock = locks.BaseLock("name") + dl = [] + for i in range(100): + owner = "owner%d" % i + mode = random.choice(["now", "very soon", "soon"]) + d = claimHarder(lock, owner) + d.addCallback(hold, owner, mode) + dl.append(d) + d = defer.DeferredList(dl) + d.addCallback(self._cleanup, lock) + return d + +class Multi(unittest.TestCase): + def testNow(self): + lock = locks.BaseLock("name", 2) + self.failUnless(lock.isAvailable()) + lock.claim("owner1") + self.failUnless(lock.isAvailable()) + lock.claim("owner2") + self.failIf(lock.isAvailable()) + lock.release("owner1") + self.failUnless(lock.isAvailable()) + lock.release("owner2") + self.failUnless(lock.isAvailable()) + + def testLater(self): + lock = locks.BaseLock("name", 2) + lock.claim("owner1") + lock.claim("owner2") + d = claimHarder(lock, "owner3") + d.addCallback(lambda lock: lock.release("owner3")) + lock.release("owner2") + lock.release("owner1") + return d + + def _cleanup(self, res, lock, count): + dl = [] + for i in range(count): + d = claimHarder(lock, "cleanup%d" % i) + dl.append(d) + d2 = defer.DeferredList(dl) + # once all locks are claimed, we know that any previous owners have + # been flushed out + def _release(res): + for i in range(count): + lock.release("cleanup%d" % i) + d2.addCallback(_release) + return d2 + + def testRandom(self): + COUNT = 5 + lock = locks.BaseLock("name", COUNT) + dl = [] + for i in range(100): + owner = "owner%d" % i + mode = random.choice(["now", "very soon", "soon"]) + d = claimHarder(lock, owner) + def _check(lock): + self.failIf(len(lock.owners) > COUNT) + return lock + d.addCallback(_check) + d.addCallback(hold, owner, mode) + dl.append(d) + d = defer.DeferredList(dl) + d.addCallback(self._cleanup, lock, COUNT) + return d + +class Dummy: + pass + +def slave(slavename): + slavebuilder = Dummy() + slavebuilder.slave = Dummy() + slavebuilder.slave.slavename = slavename + return slavebuilder + +class MakeRealLock(unittest.TestCase): + + def make(self, lockid): + return lockid.lockClass(lockid) + + def testMaster(self): + mid1 = locks.MasterLock("name1") + mid2 = locks.MasterLock("name1") + mid3 = locks.MasterLock("name3") + mid4 = locks.MasterLock("name1", 3) + self.failUnlessEqual(mid1, mid2) + self.failIfEqual(mid1, mid3) + # they should all be hashable + d = {mid1: 1, mid2: 2, mid3: 3, mid4: 4} + + l1 = self.make(mid1) + self.failUnlessEqual(l1.name, "name1") + self.failUnlessEqual(l1.maxCount, 1) + self.failUnlessIdentical(l1.getLock(slave("slave1")), l1) + l4 = self.make(mid4) + self.failUnlessEqual(l4.name, "name1") + self.failUnlessEqual(l4.maxCount, 3) + self.failUnlessIdentical(l4.getLock(slave("slave1")), l4) + + def testSlave(self): + sid1 = locks.SlaveLock("name1") + sid2 = locks.SlaveLock("name1") + sid3 = locks.SlaveLock("name3") + sid4 = locks.SlaveLock("name1", maxCount=3) + mcfs = {"bigslave": 4, "smallslave": 1} + sid5 = locks.SlaveLock("name1", maxCount=3, maxCountForSlave=mcfs) + mcfs2 = {"bigslave": 4, "smallslave": 1} + sid5a = locks.SlaveLock("name1", maxCount=3, maxCountForSlave=mcfs2) + mcfs3 = {"bigslave": 1, "smallslave": 99} + sid5b = locks.SlaveLock("name1", maxCount=3, maxCountForSlave=mcfs3) + self.failUnlessEqual(sid1, sid2) + self.failIfEqual(sid1, sid3) + self.failIfEqual(sid1, sid4) + self.failIfEqual(sid1, sid5) + self.failUnlessEqual(sid5, sid5a) + self.failIfEqual(sid5a, sid5b) + # they should all be hashable + d = {sid1: 1, sid2: 2, sid3: 3, sid4: 4, sid5: 5, sid5a: 6, sid5b: 7} + + l1 = self.make(sid1) + self.failUnlessEqual(l1.name, "name1") + self.failUnlessEqual(l1.maxCount, 1) + l1s1 = l1.getLock(slave("slave1")) + self.failIfIdentical(l1s1, l1) + + l4 = self.make(sid4) + self.failUnlessEqual(l4.maxCount, 3) + l4s1 = l4.getLock(slave("slave1")) + self.failUnlessEqual(l4s1.maxCount, 3) + + l5 = self.make(sid5) + l5s1 = l5.getLock(slave("bigslave")) + l5s2 = l5.getLock(slave("smallslave")) + l5s3 = l5.getLock(slave("unnamedslave")) + self.failUnlessEqual(l5s1.maxCount, 4) + self.failUnlessEqual(l5s2.maxCount, 1) + self.failUnlessEqual(l5s3.maxCount, 3) + +class GetLock(unittest.TestCase): + def testGet(self): + # the master.cfg file contains "lock ids", which are instances of + # MasterLock and SlaveLock but which are not actually Locks per se. + # When the build starts, these markers are turned into RealMasterLock + # and RealSlaveLock instances. This insures that any builds running + # on slaves that were unaffected by the config change are still + # referring to the same Lock instance as new builds by builders that + # *were* affected by the change. There have been bugs in the past in + # which this didn't happen, and the Locks were bypassed because half + # the builders were using one incarnation of the lock while the other + # half were using a separate (but equal) incarnation. + # + # Changing the lock id in any way should cause it to be replaced in + # the BotMaster. This will result in a couple of funky artifacts: + # builds in progress might pay attention to a different lock, so we + # might bypass the locking for the duration of a couple builds. + # There's also the problem of old Locks lingering around in + # BotMaster.locks, but they're small and shouldn't really cause a + # problem. + + b = master.BotMaster() + l1 = locks.MasterLock("one") + l1a = locks.MasterLock("one") + l2 = locks.MasterLock("one", maxCount=4) + + rl1 = b.getLockByID(l1) + rl2 = b.getLockByID(l1a) + self.failUnlessIdentical(rl1, rl2) + rl3 = b.getLockByID(l2) + self.failIfIdentical(rl1, rl3) + + s1 = locks.SlaveLock("one") + s1a = locks.SlaveLock("one") + s2 = locks.SlaveLock("one", maxCount=4) + s3 = locks.SlaveLock("one", maxCount=4, + maxCountForSlave={"a":1, "b":2}) + s3a = locks.SlaveLock("one", maxCount=4, + maxCountForSlave={"a":1, "b":2}) + s4 = locks.SlaveLock("one", maxCount=4, + maxCountForSlave={"a":4, "b":4}) + + rl1 = b.getLockByID(s1) + rl2 = b.getLockByID(s1a) + self.failUnlessIdentical(rl1, rl2) + rl3 = b.getLockByID(s2) + self.failIfIdentical(rl1, rl3) + rl4 = b.getLockByID(s3) + self.failIfIdentical(rl1, rl4) + self.failIfIdentical(rl3, rl4) + rl5 = b.getLockByID(s3a) + self.failUnlessIdentical(rl4, rl5) + rl6 = b.getLockByID(s4) + self.failIfIdentical(rl5, rl6) + + class LockStep(step.Dummy): def start(self): From warner at users.sourceforge.net Thu Aug 24 10:05:22 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Thu, 24 Aug 2006 10:05:22 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process base.py, 1.68, 1.69 step.py, 1.99, 1.100 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27952/buildbot/process Modified Files: base.py step.py Log Message: [project @ locks: can now have multiple simultaneous owners. fixes SF#1434997] Original author: warner at lothar.com Date: 2006-08-24 10:03:52 Index: base.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/base.py,v retrieving revision 1.68 retrieving revision 1.69 diff -u -d -r1.68 -r1.69 --- base.py 6 Aug 2006 20:05:21 -0000 1.68 +++ base.py 24 Aug 2006 10:05:16 -0000 1.69 @@ -332,7 +332,7 @@ for lock in self.locks: if not lock.isAvailable(): log.msg("Build %s waiting for lock %s" % (self, lock)) - d = lock.waitUntilAvailable(self) + d = lock.waitUntilMaybeAvailable(self) d.addCallback(self.acquireLocks) return d # all locks are available, claim them all Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.99 retrieving revision 1.100 diff -u -d -r1.99 -r1.100 --- step.py 21 Aug 2006 00:43:20 -0000 1.99 +++ step.py 24 Aug 2006 10:05:16 -0000 1.100 @@ -661,7 +661,7 @@ for lock in self.locks: if not lock.isAvailable(): log.msg("step %s waiting for lock %s" % (self, lock)) - d = lock.waitUntilAvailable(self) + d = lock.waitUntilMaybeAvailable(self) d.addCallback(self.acquireLocks) return d # all locks are available, claim them all From warner at users.sourceforge.net Thu Aug 24 10:05:22 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Thu, 24 Aug 2006 10:05:22 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.73,1.74 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27952/docs Modified Files: buildbot.texinfo Log Message: [project @ locks: can now have multiple simultaneous owners. fixes SF#1434997] Original author: warner at lothar.com Date: 2006-08-24 10:03:52 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.73 retrieving revision 1.74 diff -u -d -r1.73 -r1.74 --- buildbot.texinfo 23 Aug 2006 22:41:17 -0000 1.73 +++ buildbot.texinfo 24 Aug 2006 10:05:20 -0000 1.74 @@ -3994,14 +3994,27 @@ network bandwidth to the VC server, problems with simultaneous access to a database server used by unit tests, or multiple Builds which access shared state may all require some kind of interlock to prevent -corruption, confusion, or resource overload. +corruption, confusion, or resource overload. These resources might +require completely exclusive access, or it might be sufficient to +establish a limit of two or three simultaneous builds. @code{Locks} are the mechanism used to express these kinds of constraints on when Builds or Steps can be run. There are two kinds of - at code{Locks}, each with their own scope: @code{SlaveLock}s are scoped -to a single buildslave, while @code{MasterLock} instances are scoped -to the buildbot as a whole. Each @code{Lock} is created with a unique -name. + at code{Locks}, each with their own scope: @code{MasterLock} instances +are scoped to the buildbot as a whole, while @code{SlaveLock}s are +scoped to a single buildslave. This means that each buildslave has a +separate copy of each @code{SlaveLock}, which could enforce a +one-Build-at-a-time limit for each machine, but still allow as many +simultaneous builds as there are machines. + +Each @code{Lock} is created with a unique name. Each lock gets a count +of how many owners it may have: how many processes can claim it at ths +same time. This limit defaults to one, and is controllable through the + at code{maxCount} argument. On @code{SlaveLock}s you can set the owner +count on a per-slave basis by providing a dictionary (that maps from +slavename to maximum owner count) to its @code{maxCountForSlave} +argument. Any buildslaves that aren't mentioned in + at code{maxCountForSlave} get their owner count from @code{maxCount}. To use a lock, simply include it in the @code{locks=} argument of the @code{BuildStep} object that should obtain the lock before it runs. @@ -4053,27 +4066,31 @@ In the next example, we have one buildslave hosting three separate Builders (each running tests against a different version of Python). The machine which hosts this buildslave is not particularly fast, so -we want to prevent the builds from all happening at the same time. We -use a @code{SlaveLock} because the builds happening on the slow slave -do not affect builds running on other slaves, and we use the lock on -the build as a whole because the slave is so slow that even multiple -SVN checkouts would be taxing. +we want to prevent all three builds from all happening at the same +time. (Assume we've experimentally determined that one build leaves +unused CPU capacity, three builds causes a lot of disk thrashing, but +two builds at a time is Just Right). We use a @code{SlaveLock} because +the builds happening on this one slow slave should not affect builds +running on other slaves, and we use the lock on the build as a whole +because the slave is so slow that even multiple simultaneous SVN +checkouts would be too taxing. We set @code{maxCount=2} to achieve our +goal of two simultaneous builds per slave. @example from buildbot import locks from buildbot.process import s, step, factory -slow_lock = locks.SlaveLock("cpu") +slow_lock = locks.SlaveLock("cpu", maxCount=2) source = s(step.SVN, svnurl="http://example.org/svn/Trunk") f22 = factory.Trial(source, trialpython=["python2.2"]) f23 = factory.Trial(source, trialpython=["python2.3"]) f24 = factory.Trial(source, trialpython=["python2.4"]) b1 = @{'name': 'p22', 'slavename': 'bot-1', builddir='p22', 'factory': f22, - 'locks': [slow_lock] @} + 'locks': [slow_lock] @} b2 = @{'name': 'p23', 'slavename': 'bot-1', builddir='p23', 'factory': f23, - 'locks': [slow_lock] @} + 'locks': [slow_lock] @} b3 = @{'name': 'p24', 'slavename': 'bot-1', builddir='p24', 'factory': f24, - 'locks': [slow_lock] @} + 'locks': [slow_lock] @} c['builders'] = [b1, b2, b3] @end example @@ -4083,35 +4100,32 @@ been split into those which use the database and those which do not. In addition, two of the Builds run on a fast machine which does not need to worry about the cpu lock, but which still must be prevented -from simultaneous database access. +from simultaneous database access. We use @code{maxCountForSlave} to +limit the slow machine to one simultanous build, but allow practically +unlimited concurrent builds on the fast machine. @example from buildbot import locks from buildbot.process import step, factory db_lock = locks.MasterLock("database") -cpu_lock = locks.SlaveLock("cpu") -slow_f = factory.BuildFactory() +slavecounts = @{"bot-slow": 1, "bot-fast": 100@} +cpu_lock = locks.SlaveLock("cpu", maxCountForSlave=slavecounts) +f = factory.BuildFactory() f.addStep(step.SVN, svnurl="http://example.org/svn/Trunk") f.addStep(step.ShellCommand, command="make all", locks=[cpu_lock]) f.addStep(step.ShellCommand, command="make test", locks=[cpu_lock]) f.addStep(step.ShellCommand, command="make db-test", locks=[db_lock, cpu_lock]) -fast_factory = factory.BuildFactory() -f.addStep(step.SVN, svnurl="http://example.org/svn/Trunk") -f.addStep(step.ShellCommand, command="make all", locks=[]) -f.addStep(step.ShellCommand, command="make test", locks=[]) -f.addStep(step.ShellCommand, command="make db-test", locks=[db_lock]) - b1 = @{'name': 'full1', 'slavename': 'bot-slow', builddir='full1', - 'factory': slow_factory@} + 'factory': f@} b2 = @{'name': 'full2', 'slavename': 'bot-slow', builddir='full2', - 'factory': slow_factory@} + 'factory': f@} b3 = @{'name': 'full3', 'slavename': 'bot-fast', builddir='full3', - 'factory': fast_factory@} + 'factory': f@} b4 = @{'name': 'full4', 'slavename': 'bot-fast', builddir='full4', - 'factory': fast_factory@} + 'factory': f@} c['builders'] = [b1, b2, b3, b4] @end example From warner at users.sourceforge.net Thu Aug 24 10:05:18 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Thu, 24 Aug 2006 10:05:18 +0000 Subject: [Buildbot-commits] buildbot/buildbot locks.py, 1.4, 1.5 master.py, 1.94, 1.95 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27952/buildbot Modified Files: locks.py master.py Log Message: [project @ locks: can now have multiple simultaneous owners. fixes SF#1434997] Original author: warner at lothar.com Date: 2006-08-24 10:03:52 Index: locks.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/locks.py,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- locks.py 26 Nov 2005 02:14:31 -0000 1.4 +++ locks.py 24 Aug 2006 10:05:16 -0000 1.5 @@ -4,69 +4,90 @@ from twisted.internet import reactor, defer from buildbot import util +if False: # for debugging + def debuglog(msg): + log.msg(msg) +else: + def debuglog(msg): + pass + class BaseLock: - owner = None description = "" - def __init__(self, name): + def __init__(self, name, maxCount=1): self.name = name self.waiting = [] + self.owners = [] + self.maxCount=maxCount def __repr__(self): return self.description def isAvailable(self): - log.msg("%s isAvailable: self.owner=%s" % (self, self.owner)) - return not self.owner + debuglog("%s isAvailable: self.owners=%r" % (self, self.owners)) + return len(self.owners) < self.maxCount def claim(self, owner): - log.msg("%s claim(%s)" % (self, owner)) + debuglog("%s claim(%s)" % (self, owner)) assert owner is not None - self.owner = owner - log.msg(" %s is claimed" % (self,)) + assert len(self.owners) < self.maxCount, "ask for isAvailable() first" + self.owners.append(owner) + debuglog(" %s is claimed" % (self,)) def release(self, owner): - log.msg("%s release(%s)" % (self, owner)) - assert owner is self.owner - self.owner = None - reactor.callLater(0, self.nowAvailable) + debuglog("%s release(%s)" % (self, owner)) + assert owner in self.owners + self.owners.remove(owner) + # who can we wake up? + if self.waiting: + d = self.waiting.pop(0) + reactor.callLater(0, d.callback, self) - def waitUntilAvailable(self, owner): - log.msg("%s waitUntilAvailable(%s)" % (self, owner)) - assert self.owner, "You aren't supposed to call this on a free Lock" + def waitUntilMaybeAvailable(self, owner): + """Fire when the lock *might* be available. The caller will need to + check with isAvailable() when the deferred fires. This loose form is + used to avoid deadlocks. If we were interested in a stronger form, + this would be named 'waitUntilAvailable', and the deferred would fire + after the lock had been claimed. + """ + debuglog("%s waitUntilAvailable(%s)" % (self, owner)) + if self.isAvailable(): + return defer.succeed(self) d = defer.Deferred() - self.waiting.append((d, owner)) + self.waiting.append(d) return d - def nowAvailable(self): - log.msg("%s nowAvailable" % self) - assert not self.owner - if not self.waiting: - return - d,owner = self.waiting.pop(0) - d.callback(self) class RealMasterLock(BaseLock): - def __init__(self, name): - BaseLock.__init__(self, name) - self.description = "" % (name,) + def __init__(self, lockid): + BaseLock.__init__(self, lockid.name, lockid.maxCount) + self.description = "" % (self.name, self.maxCount) def getLock(self, slave): return self -class RealSlaveLock(BaseLock): - def __init__(self, name): - BaseLock.__init__(self, name) - self.description = "" % (name,) +class RealSlaveLock: + def __init__(self, lockid): + self.name = lockid.name + self.maxCount = lockid.maxCount + self.maxCountForSlave = lockid.maxCountForSlave + self.description = "" % (self.name, + self.maxCount, + self.maxCountForSlave) self.locks = {} + def __repr__(self): + return self.description + def getLock(self, slavebuilder): slavename = slavebuilder.slave.slavename if not self.locks.has_key(slavename): - lock = self.locks[slavename] = BaseLock(self.name) - lock.description = "" % (self.name, - slavename, - id(lock)) + maxCount = self.maxCountForSlave.get(slavename, + self.maxCount) + lock = self.locks[slavename] = BaseLock(self.name, maxCount) + desc = "" % (self.name, maxCount, + slavename, id(lock)) + lock.description = desc self.locks[slavename] = lock return self.locks[slavename] @@ -76,14 +97,52 @@ # via the BotMaster.getLockByID method. class MasterLock(util.ComparableMixin): - compare_attrs = ['name'] + """I am a semaphore that limits the number of simultaneous actions. + + Builds and BuildSteps can declare that they wish to claim me as they run. + Only a limited number of such builds or steps will be able to run + simultaneously. By default this number is one, but my maxCount parameter + can be raised to allow two or three or more operations to happen at the + same time. + + Use this to protect a resource that is shared among all builders and all + slaves, for example to limit the load on a common SVN repository. + """ + + compare_attrs = ['name', 'maxCount'] lockClass = RealMasterLock - def __init__(self, name): + def __init__(self, name, maxCount=1): self.name = name + self.maxCount = maxCount class SlaveLock(util.ComparableMixin): - compare_attrs = ['name'] + """I am a semaphore that limits simultaneous actions on each buildslave. + + Builds and BuildSteps can declare that they wish to claim me as they run. + Only a limited number of such builds or steps will be able to run + simultaneously on any given buildslave. By default this number is one, + but my maxCount parameter can be raised to allow two or three or more + operations to happen on a single buildslave at the same time. + + Use this to protect a resource that is shared among all the builds taking + place on each slave, for example to limit CPU or memory load on an + underpowered machine. + + Each buildslave will get an independent copy of this semaphore. By + default each copy will use the same owner count (set with maxCount), but + you can provide maxCountForSlave with a dictionary that maps slavename to + owner count, to allow some slaves more parallelism than others. + + """ + + compare_attrs = ['name', 'maxCount', '_maxCountForSlaveList'] lockClass = RealSlaveLock - def __init__(self, name): + def __init__(self, name, maxCount=1, maxCountForSlave={}): self.name = name - + self.maxCount = maxCount + self.maxCountForSlave = maxCountForSlave + # for comparison purposes, turn this dictionary into a stably-sorted + # list of tuples + self._maxCountForSlaveList = self.maxCountForSlave.items() + self._maxCountForSlaveList.sort() + self._maxCountForSlaveList = tuple(self._maxCountForSlaveList) Index: master.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/master.py,v retrieving revision 1.94 retrieving revision 1.95 diff -u -d -r1.94 -r1.95 --- master.py 24 Jul 2006 22:09:20 -0000 1.94 +++ master.py 24 Aug 2006 10:05:16 -0000 1.95 @@ -416,10 +416,13 @@ @param lockid: a locks.MasterLock or locks.SlaveLock instance @return: a locks.RealMasterLock or locks.RealSlaveLock instance """ - k = (lockid.__class__, lockid.name) - if not k in self.locks: - self.locks[k] = lockid.lockClass(lockid.name) - return self.locks[k] + if not lockid in self.locks: + self.locks[lockid] = lockid.lockClass(lockid) + # if the master.cfg file has changed maxCount= on the lock, the next + # time a build is started, they'll get a new RealLock instance. Note + # that this requires that MasterLock and SlaveLock (marker) instances + # be hashable and that they should compare properly. + return self.locks[lockid] ######################################## From warner at users.sourceforge.net Thu Aug 24 10:05:18 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Thu, 24 Aug 2006 10:05:18 +0000 Subject: [Buildbot-commits] buildbot CREDITS,1.1,1.2 ChangeLog,1.719,1.720 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27952 Modified Files: CREDITS ChangeLog Log Message: [project @ locks: can now have multiple simultaneous owners. fixes SF#1434997] Original author: warner at lothar.com Date: 2006-08-24 10:03:52 Index: CREDITS =================================================================== RCS file: /cvsroot/buildbot/buildbot/CREDITS,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- CREDITS 23 Aug 2006 07:04:36 -0000 1.1 +++ CREDITS 24 Aug 2006 10:05:15 -0000 1.2 @@ -40,3 +40,4 @@ Andrew Bennetts Olly Betts Philipp Frauenfelder +James Knight Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.719 retrieving revision 1.720 diff -u -d -r1.719 -r1.720 --- ChangeLog 23 Aug 2006 22:41:17 -0000 1.719 +++ ChangeLog 24 Aug 2006 10:05:15 -0000 1.720 @@ -1,3 +1,21 @@ +2006-08-24 Brian Warner + + * buildbot/locks.py (BaseLock): you can now configure Locks to + allow multiple simultaneous owners. They still default to + maxCount=1. Fixes SF#1434997. Thanks to James Knight (foom) for + the patch. + * docs/buildbot.texinfo (Interlocks): document the new options + * buildbot/test/test_locks.py: add a bunch of new unit tests + * buildbot/process/base.py (Build.acquireLocks): locks now offer + waitUntilMaybeAvailable, not waitUntilAvailable + * buildbot/process/step.py (BuildStep.acquireLocks): same + * buildbot/master.py (BotMaster.getLockByID): real locks now use + the whole lockid in their constructor, not just the name. Also, + keep track of which real locks we've handed out by the full + lockid, not just class+name, otherwise changing just the maxCount= + in the master.cfg file would not actually cause a behavioral + change + 2006-08-23 Brian Warner * buildbot/__init__.py (version): bump to 0.7.4+ while between From warner at users.sourceforge.net Fri Aug 25 21:54:45 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 25 Aug 2006 21:54:45 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.720,1.721 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25199 Modified Files: ChangeLog Log Message: [project @ make 'stdio' log come first, refactor setupLogfiles] Original author: warner at lothar.com Date: 2006-08-25 21:48:58 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.720 retrieving revision 1.721 diff -u -d -r1.720 -r1.721 --- ChangeLog 24 Aug 2006 10:05:15 -0000 1.720 +++ ChangeLog 25 Aug 2006 21:54:43 -0000 1.721 @@ -1,3 +1,23 @@ +2006-08-25 Brian Warner + + * buildbot/process/step.py (LoggingBuildStep.startCommand): set up + all logfiles= in startCommand(), rather than in start() . This + makes it easier to have the 'stdio' log come before any secondary + logfiles, which I feel makes the waterfall display more + understandable. + (LoggingBuildStep.setupLogfiles): move the addLog/cmd.useLog code + from ShellCommand up into LoggingBuildStep + (LoggingBuildStep.__init__): move the handling of logfiles= from + ShellCommand up to LoggingBuildStep, because startCommand is + provided by LoggingBuildStep, whereas start() was specific to + subclasses like ShellCommand and Source. This removes code + duplication in those subclasses. + (ShellCommand.__init__): same + (ShellCommand.checkForOldSlaveAndLogfiles): split out the check + for a slave that's too old to understand logfiles= into a separate + method, so it can live in ShellCommand. The rest of + setupLogfiles() can live in LoggingBuildStep. + 2006-08-24 Brian Warner * buildbot/locks.py (BaseLock): you can now configure Locks to From warner at users.sourceforge.net Fri Aug 25 21:54:45 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 25 Aug 2006 21:54:45 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.100,1.101 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25199/buildbot/process Modified Files: step.py Log Message: [project @ make 'stdio' log come first, refactor setupLogfiles] Original author: warner at lothar.com Date: 2006-08-25 21:48:58 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.100 retrieving revision 1.101 diff -u -d -r1.100 -r1.101 --- step.py 24 Aug 2006 10:05:16 -0000 1.100 +++ step.py 25 Aug 2006 21:54:43 -0000 1.101 @@ -898,9 +898,16 @@ """ progressMetrics = ('output',) + logfiles = {} - def __init__(self, *args, **kwargs): + parms = BuildStep.parms + ['logfiles'] + + def __init__(self, logfiles={}, *args, **kwargs): BuildStep.__init__(self, *args, **kwargs) + # merge a class-level 'logfiles' attribute with one passed in as an + # argument + self.logfiles = self.logfiles.copy() + self.logfiles.update(logfiles) self.addLogObserver('stdio', OutputProgressObserver("output")) def describe(self, done=False): @@ -909,17 +916,25 @@ def startCommand(self, cmd, errorMessages=[]): """ @param cmd: a suitable RemoteCommand which will be launched, with - all output being put into a LogFile named 'stdio' + all output being put into our self.stdio_log LogFile """ + log.msg("ShellCommand.startCommand(cmd=%s)", (cmd,)) self.cmd = cmd # so we can interrupt it self.step_status.setColor("yellow") self.step_status.setText(self.describe(False)) - stdio_log = self.addLog("stdio") - log.msg("ShellCommand.start using log", stdio_log) - log.msg(" for cmd", cmd) + + # stdio is the first log + self.stdio_log = stdio_log = self.addLog("stdio") cmd.useLog(stdio_log, True) for em in errorMessages: stdio_log.addHeader(em) + # TODO: consider setting up self.stdio_log earlier, and have the + # code that passes in errorMessages instead call + # self.stdio_log.addHeader() directly. + + # there might be other logs + self.setupLogfiles(cmd, self.logfiles) + d = self.runCommand(cmd) # might raise ConnectionLost d.addCallback(lambda res: self.commandComplete(cmd)) d.addCallback(lambda res: self.createSummary(cmd.logs['stdio'])) @@ -931,6 +946,15 @@ d.addCallbacks(self.finished, self.checkDisconnect) d.addErrback(self.failed) + def setupLogfiles(self, cmd, logfiles): + """Set up any additional logfiles= logs. + """ + for logname,remotefilename in logfiles.items(): + # tell the BuildStepStatus to add a LogFile + newlog = self.addLog(logname) + # and tell the LoggedRemoteCommand to feed it + cmd.useLog(newlog, True) + def interrupt(self, reason): # TODO: consider adding an INTERRUPTED or STOPPED status to use # instead of FAILURE, might make the text a bit more clear. @@ -1102,8 +1126,9 @@ @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs of their corresponding logfiles. The contents of the file - named FILENAME will be put into a LogFile named NAME, in - something approximating real-time. + named FILENAME will be put into a LogFile named NAME, ina + something approximating real-time. (note that logfiles= + is actually handled by our parent class LoggingBuildStep) """ @@ -1111,7 +1136,9 @@ description = None # set this to a list of short strings to override descriptionDone = None # alternate description when the step is complete command = None # set this to a command, or set in kwargs - logfiles = {} + # logfiles={} # you can also set 'logfiles' to a dictionary, and it + # will be merged with any logfiles= argument passed in + # to __init__ # override this on a specific ShellCommand if you want to let it fail # without dooming the entire build to a status of FAILURE @@ -1119,7 +1146,7 @@ def __init__(self, workdir, description=None, descriptionDone=None, - command=None, logfiles={}, + command=None, **kwargs): # most of our arguments get passed through to the RemoteShellCommand # that we create, but first strip out the ones that we pass to @@ -1136,12 +1163,8 @@ self.descriptionDone = [self.descriptionDone] if command: self.command = command - # merge a class-level 'logfiles' attribute with one passed in as an - # argument - self.logfiles = self.logfiles.copy() - self.logfiles.update(logfiles) - # pull out the ones that BuildStep wants, then upcall + # pull out the ones that LoggingBuildStep wants, then upcall buildstep_kwargs = {} for k in kwargs.keys()[:]: if k in self.__class__.parms: @@ -1216,32 +1239,32 @@ # note that each RemoteShellCommand gets its own copy of the # dictionary, so we shouldn't be affecting anyone but ourselves. - def setupLogfiles(self, cmd, logfiles): - if not logfiles: - return - if self.slaveVersionIsOlderThan("shell", "2.1"): - # this buildslave is too old and will ignore the 'logfiles' - # argument. You'll either have to pull the logfiles manually - # (say, by using 'cat' in a separate RemoteShellCommand) or - # upgrade the buildslave. - msg1 = ("Warning: buildslave %s is too old " - "to understand logfiles=, ignoring it." - % self.getSlaveName()) - msg2 = "You will have to pull this logfile (%s) manually." - log.msg(msg1) - for logname,remotefilename in logfiles.items(): - newlog = self.addLog(logname) - newlog.addHeader(msg1 + "\n") - newlog.addHeader(msg2 % remotefilename + "\n") - newlog.finish() - return - for logname,remotefilename in logfiles.items(): - # tell the BuildStepStatus to add a LogFile + def checkForOldSlaveAndLogfiles(self): + if not self.logfiles: + return # doesn't matter + if not self.slaveVersionIsOlderThan("shell", "2.1"): + return # slave is new enough + # this buildslave is too old and will ignore the 'logfiles' + # argument. You'll either have to pull the logfiles manually + # (say, by using 'cat' in a separate RemoteShellCommand) or + # upgrade the buildslave. + msg1 = ("Warning: buildslave %s is too old " + "to understand logfiles=, ignoring it." + % self.getSlaveName()) + msg2 = "You will have to pull this logfile (%s) manually." + log.msg(msg1) + for logname,remotefilename in self.logfiles.items(): newlog = self.addLog(logname) - # and tell the LoggedRemoteCommand to feed it - cmd.useLog(newlog, True) + newlog.addHeader(msg1 + "\n") + newlog.addHeader(msg2 % remotefilename + "\n") + newlog.finish() + # now prevent setupLogfiles() from adding them + self.logfiles = {} def start(self): + # this block is specific to ShellCommands. subclasses that don't need + # to set up an argv array, an environment, or extra logfiles= (like + # the Source subclasses) can just skip straight to startCommand() command = self._interpolateProperties(self.command) # create the actual RemoteShellCommand instance now kwargs = self.remote_kwargs @@ -1249,9 +1272,9 @@ kwargs['logfiles'] = self.logfiles cmd = RemoteShellCommand(**kwargs) self.setupEnvironment(cmd) - self.setupLogfiles(cmd, self.logfiles) - self.startCommand(cmd) + self.checkForOldSlaveAndLogfiles() + self.startCommand(cmd) From warner at users.sourceforge.net Fri Aug 25 21:54:53 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 25 Aug 2006 21:54:53 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.721,1.722 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25222 Modified Files: ChangeLog Log Message: [project @ fix typo in MailNotifier docstring] Original author: warner at lothar.com Date: 2006-08-25 21:53:07 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.721 retrieving revision 1.722 diff -u -d -r1.721 -r1.722 --- ChangeLog 25 Aug 2006 21:54:43 -0000 1.721 +++ ChangeLog 25 Aug 2006 21:54:51 -0000 1.722 @@ -1,5 +1,7 @@ 2006-08-25 Brian Warner + * buildbot/status/mail.py (MailNotifier.__init__): fix typo in docs + * buildbot/process/step.py (LoggingBuildStep.startCommand): set up all logfiles= in startCommand(), rather than in start() . This makes it easier to have the 'stdio' log come before any secondary From warner at users.sourceforge.net Fri Aug 25 21:54:54 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 25 Aug 2006 21:54:54 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status mail.py,1.25,1.26 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25222/buildbot/status Modified Files: mail.py Log Message: [project @ fix typo in MailNotifier docstring] Original author: warner at lothar.com Date: 2006-08-25 21:53:07 Index: mail.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/mail.py,v retrieving revision 1.25 retrieving revision 1.26 diff -u -d -r1.25 -r1.26 --- mail.py 3 Jun 2006 18:23:04 -0000 1.25 +++ mail.py 25 Aug 2006 21:54:51 -0000 1.26 @@ -95,7 +95,7 @@ @type subject: string @param subject: a string to be used as the subject line of the message. %(builder)s will be replaced with the name of the - %builder which provoked the message. + builder which provoked the message. @type mode: string (defaults to all) @param mode: one of: From warner at users.sourceforge.net Sat Aug 26 19:50:53 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 26 Aug 2006 19:50:53 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_locks.py,1.4,1.5 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv5653/buildbot/test Modified Files: test_locks.py Log Message: [project @ test_locks: make tests compatible with twisted-1.3.0] Original author: warner at lothar.com Date: 2006-08-26 19:49:05 Index: test_locks.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_locks.py,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- test_locks.py 24 Aug 2006 10:05:20 -0000 1.4 +++ test_locks.py 26 Aug 2006 19:50:50 -0000 1.5 @@ -47,13 +47,13 @@ lock = locks.BaseLock("name") d = claimHarder(lock, "owner1") d.addCallback(lambda lock: lock.release("owner1")) - return d + return maybeWait(d) def testCompetition(self): lock = locks.BaseLock("name") d = claimHarder(lock, "owner1") d.addCallback(self._claim1) - return d + return maybeWait(d) def _claim1(self, lock): # we should have claimed it by now self.failIf(lock.isAvailable()) @@ -85,7 +85,7 @@ dl.append(d) d = defer.DeferredList(dl) d.addCallback(self._cleanup, lock) - return d + return maybeWait(d) class Multi(unittest.TestCase): def testNow(self): @@ -108,7 +108,7 @@ d.addCallback(lambda lock: lock.release("owner3")) lock.release("owner2") lock.release("owner1") - return d + return maybeWait(d) def _cleanup(self, res, lock, count): dl = [] @@ -140,7 +140,7 @@ dl.append(d) d = defer.DeferredList(dl) d.addCallback(self._cleanup, lock, COUNT) - return d + return maybeWait(d) class Dummy: pass From warner at users.sourceforge.net Sat Aug 26 19:50:52 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 26 Aug 2006 19:50:52 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.722,1.723 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv5653 Modified Files: ChangeLog Log Message: [project @ test_locks: make tests compatible with twisted-1.3.0] Original author: warner at lothar.com Date: 2006-08-26 19:49:05 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.722 retrieving revision 1.723 diff -u -d -r1.722 -r1.723 --- ChangeLog 25 Aug 2006 21:54:51 -0000 1.722 +++ ChangeLog 26 Aug 2006 19:50:50 -0000 1.723 @@ -1,3 +1,9 @@ +2006-08-26 Brian Warner + + * buildbot/test/test_locks.py (Unit.testLater): make the tests + compatible with twisted-1.3.0, for some reason I just can't seem + to let go of the past. + 2006-08-25 Brian Warner * buildbot/status/mail.py (MailNotifier.__init__): fix typo in docs