[Buildbot-commits] buildbot/buildbot/process factory.py,1.7,1.8 step.py,1.60,1.61 base.py,1.44,1.45 interlock.py,1.4,1.5 builder.py,1.21,1.22 process_twisted.py,1.36,1.37 step_twisted.py,1.66,1.67
Brian Warner
warner at users.sourceforge.net
Mon Apr 18 00:26:59 UTC 2005
Update of /cvsroot/buildbot/buildbot/buildbot/process
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv8250/buildbot/process
Modified Files:
factory.py step.py base.py interlock.py builder.py
process_twisted.py step_twisted.py
Log Message:
Revision: arch at buildbot.sf.net--2004/buildbot--dev--0--patch-42
Creator: Brian Warner <warner at monolith.lothar.com>
merge doc updates [org.apestaart at thomas/buildbot--doc--0--patch-6]
Merge epydoc changes from Thomas, add some hacks of my own. No code
changes to the buildbot itself, although I hacked docs/epyrun a bit.
Index: base.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/process/base.py,v
retrieving revision 1.44
retrieving revision 1.45
diff -u -d -r1.44 -r1.45
--- base.py 3 Dec 2004 22:54:51 -0000 1.44
+++ base.py 18 Apr 2005 00:26:56 -0000 1.45
@@ -19,6 +19,19 @@
use subclasses of Build to hold status information unique to those build
processes.
+ I am responsible for two things:
+ 1. deciding B{when} a build should occur. This involves knowing
+ which file changes to ignore (documentation or comments files,
+ for example), and deciding how long to wait for the tree to
+ become stable before starting. The base class pays attention
+ to all files, and waits 10 seconds for a stable tree.
+
+ 2. controlling B{how} the build proceeds. The actual build is
+ broken up into a series of steps, saved in the .buildSteps[]
+ array as a list of L{buildbot.process.step.BuildStep}
+ objects. Each step is a single remote command, possibly a shell
+ command.
+
Before the build is started, I accumulate Changes and track the
tree-stable timers and interlocks necessary to decide when I ought to
start building.
@@ -28,7 +41,11 @@
After the build, I hold historical data about the build, like how long
it took, tree size, lines of code, etc. It is expected to be used to
generate graphs and quantify long-term trends. It does not hold any
- status events or build logs."""
+ status events or build logs.
+
+ I can be used by a factory by setting buildClass on
+ L{buildbot.process.factory.BuildFactory}
+ """
treeStableTimer = 10 #*60
workdir = "build"
@@ -54,6 +71,11 @@
self.currentStep = None
def setBuilder(self, builder):
+ """
+ Set the given builder as our builder.
+
+ @type builder: L{buildbot.process.builder.Builder}
+ """
self.builder = builder
def setSourceStamp(self, baserev, patch, reason="try"):
@@ -62,12 +84,24 @@
self.reason = reason
def isFileImportant(self, filename):
- """I return 1 if the given file is important enough to trigger a
- rebuild, 0 if it should be ignored. Override me to ignore unimporant
- files: documentation, .cvsignore files, etc. The timer is not
- restarted,, so a checkout may occur in the middle of a set of
- changes marked 'unimportant'. Also, the checkout may or may not pick
- up the 'unimportant' changes."""
+ """
+ I check if the given file is important enough to trigger a rebuild.
+
+ Override me to ignore unimporant files: documentation, .cvsignore
+ files, etc.
+
+ The timer is not restarted, so a checkout may occur in the middle of
+ a set of changes marked 'unimportant'. Also, the checkout may or may
+ not pick up the 'unimportant' changes. The implicit assumption is
+ that any file marked 'unimportant' is incapable of affecting the
+ results of the build.
+
+ @param filename: name of a file to check, relative to the VC base
+ @type filename: string
+
+ @rtype: 0 or 1
+ @returns: whether the change to this file should trigger a rebuild
+ """
return 1
def bumpMaxChangeNumber(self, change):
@@ -75,7 +109,14 @@
self.maxChangeNumber = change.number
if change.number > self.maxChangeNumber:
self.maxChangeNumber = change.number
+
def addChange(self, change):
+ """
+ Add the change, deciding if the change is important or not.
+ Called by L{buildbot.process.builder.filesChanged}
+
+ @type change: L{buildbot.changes.changes.Change}
+ """
important = 0
for filename in change.files:
if self.isFileImportant(filename):
Index: builder.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/process/builder.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- builder.py 9 Dec 2004 10:24:56 -0000 1.21
+++ builder.py 18 Apr 2005 00:26:56 -0000 1.22
@@ -12,36 +12,50 @@
class Builder(pb.Referenceable):
- """This class defines how and when a given kind of build is performed.
- Each Builder has an associated BuildProcess object. This class should be
- subclassed by the user to define how their build ought to work.
+ """I manage all Builds of a given type.
- The BuildProcess object is responsible for two things. The first is
- deciding *when* a build ought to occur. This involves knowing which file
- changes to ignore (documentation or comments files, for example), and
- deciding how long to wait for the tree to become stable before starting.
- The base class pays attention to all files, and waits 10 minutes for a
- stable tree:
+ Each Builder is created by an entry in the config file (the c['builders']
+ list), with a number of parameters.
- .fileChanged(filename, when) is called when a file has been modified.
+ One of these parameters is the L{buildbot.process.factory.BuildFactory}
+ object that is associated with this Builder. The factory is responsible
+ for creating new L{Build<buildbot.process.base.Build>} objects. Each
+ Build object defines when and how the build is performed, so a new
+ Factory or Builder should be defined to control this behavior.
- .startBuild(when) should be called when the build should begin. The
- build should use a tree checked out with a timestamp of 'when' to make
- sure no partial commits are picked up. 'when' should be in the middle
- of the 10-minute stable window.
+ The Builder holds on to a number of these Build
+ objects, in various slots like C{.waiting}, C{.interlocked},
+ C{.buildable}, and C{.currentBuild}. Incoming
+ L{Change<buildbot.change.changes.Change>} objects are passed to the
+ C{.waiting} build, and when it decides it is ready to go, I move it to
+ the C{.buildable} slot. When a slave becomes available, I move it to the
+ C{.currentBuild} slot and start it running.
- The second is controlling *how* the build proceeds. The actual build is
- broken up into a series of steps, saved in the .buildSteps[] array as a
- list of BuildStep objects. Each step is a single remote command, possibly
- a shell command.
+ The Builder is also the master-side representative for one of the
+ L{buildbot.slave.bot.SlaveBuilder} objects that lives in a remote
+ buildbot. When a remote builder connects, I query it for command versions
+ and then make it available to any Builds that are ready to run.
- """
+ I also manage Interlocks, periodic build timers, forced builds, progress
+ expectation (ETA) management, and some status delivery chores.
- """This is the master-side representative for one of the SlaveBuilder
- objects that lives in a remote buildbot. Change notifications are
- delivered to it with .fileChanged(), which influences the enclosed
- BuildProcess state machine. When a remote builder is available, this
- object sends it commands to be executed in the slave process."""
+ @type waiting: L{buildbot.process.base.Build}
+ @ivar waiting: a slot for a Build waiting for its 'tree stable' timer to
+ expire
+
+ @type interlocked: list of L{buildbot.process.base.Build}
+ @ivar interlocked: a slot for the Builds that are stable, but which must
+ wait for other Builds to complete successfully before
+ they can be run.
+
+ @type buildable: L{buildbot.process.base.Build}
+ @ivar buildable: a slot for a Build that is stable and ready to build,
+ but which is waiting for a buildslave to be available.
+
+ @type currentBuild: L{buildbot.process.base.Build}
+ @ivar currentBuild: a slot for the Build that actively running
+
+ """
remote = None
lastChange = None
@@ -55,6 +69,13 @@
expectations = None # this is created the first time we get a good build
def __init__(self, setup, builder_status):
+ """
+ @type setup: dict
+ @param setup: builder setup data, as stored in
+ BuildmasterConfig['builders']. Contains name,
+ slavename, builddir, factory.
+ @type builder_status: L{buildbot.status.builder.BuilderStatus}
+ """
self.name = setup['name']
self.slavename = setup['slavename']
self.builddir = setup['builddir']
@@ -93,6 +114,12 @@
return diffs
def newBuild(self):
+ """
+ Create a new build from our build factory and set ourself as the
+ builder.
+
+ @rtype: L{buildbot.process.base.Build}
+ """
b = self.buildFactory.newBuild()
b.setBuilder(self)
return b
@@ -222,6 +249,12 @@
self.builder_status.currentlyOffline()
def filesChanged(self, change):
+ """
+ Tell the waiting L{buildbot.process.base.Build} that files have
+ changed.
+
+ @type change: L{buildbot.changes.changes.Change}
+ """
# this is invoked by the BotMaster to distribute change notification
# we assume they are added in strictly increasing order
if not self.waiting:
Index: factory.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/process/factory.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- factory.py 23 Nov 2004 04:03:25 -0000 1.7
+++ factory.py 18 Apr 2005 00:26:56 -0000 1.8
@@ -10,6 +10,10 @@
return (steptype, kwargs)
class BuildFactory(util.ComparableMixin):
+ """
+ @cvar buildClass: class to use when creating builds
+ @type buildClass: L{buildbot.process.base.Build}
+ """
buildClass = Build
treeStableTimer = None
steps = []
Index: interlock.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/process/interlock.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- interlock.py 25 Sep 2003 19:09:48 -0000 1.4
+++ interlock.py 18 Apr 2005 00:26:56 -0000 1.5
@@ -19,7 +19,7 @@
self.feeders = {}
for n in feeders:
self.feeders[n] = (None, 0)
- # feeders.keys are .names of BuildProcess objects that we watch
+ # feeders.keys are .names of Build objects that we watch
# feeders.values are Change numbers that have been successfully built
self.watchers = {}
# watchers.keys are Change numbers that a builder cares about
Index: process_twisted.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/process/process_twisted.py,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- process_twisted.py 17 Apr 2005 12:17:51 -0000 1.36
+++ process_twisted.py 18 Apr 2005 00:26:56 -0000 1.37
@@ -1,6 +1,6 @@
#! /usr/bin/python
-# BuildProcesses specific to the Twisted codebase
+# Build classes specific to the Twisted codebase
from buildbot.process.base import Build
from buildbot.process.factory import BuildFactory, s
Index: step.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v
retrieving revision 1.60
retrieving revision 1.61
diff -u -d -r1.60 -r1.61
--- step.py 12 Apr 2005 00:28:30 -0000 1.60
+++ step.py 18 Apr 2005 00:26:56 -0000 1.61
@@ -13,30 +13,52 @@
from buildbot.status import progress, builder
from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED
-class RemoteCommand(pb.Referenceable):
+"""
+BuildStep and RemoteCommand classes for master-side representation of the
+build process
+"""
- """This base class represents a single command to be run on the slave. It
- handles the details of reliably gathering status updates from the slave
- (acknowledging each), and (eventually) recovering from interrupted
- builds.
+class RemoteCommand(pb.Referenceable):
+ """
+ I represent a single command to be run on the slave. I handle the details
+ of reliably gathering status updates from the slave (acknowledging each),
+ and (eventually, in a future release) recovering from interrupted builds.
+ This is the master-side object that is known to the slave-side
+ L{buildbot.slave.bot.SlaveBuilder}, to which status update are sent.
- The command should be started by calling .run(), which returns a Deferred
- that will fire when the command has finished, or will errback if an
- exception is raised. Typically __init__ or run() will set up
- self.remote_command to be a string which corresponds to one of the
- SlaveCommands registered in the buildslave, and self.args to a dictionary
- of arguments that will be passed to the SlaveCommand instance.
+ My command should be started by calling .run(), which returns a
+ Deferred that will fire when the command has finished, or will
+ errback if an exception is raised.
+
+ Typically __init__ or run() will set up self.remote_command to be a
+ string which corresponds to one of the SlaveCommands registered in
+ the buildslave, and self.args to a dictionary of arguments that will
+ be passed to the SlaveCommand instance.
start, remoteUpdate, and remoteComplete are available to be overridden
+ @type commandCounter: list of one int
+ @cvar commandCounter: provides a unique value for each
+ RemoteCommand executed across all slaves
+ @type active: boolean
+ @cvar active: whether the command is currently running
"""
-
- # this counter provides a unique value for each RemoteCommand executed
- # across all slaves. It is a class attribute.
- commandCounter = [0]
+ commandCounter = [0] # we use a list as a poor man's singleton
active = False
def __init__(self, remote_command, args):
+ """
+ @type remote_command: string
+ @param remote_command: remote command to start. This will be
+ passed to
+ L{buildbot.slave.bot.SlaveBuilder.remote_startCommand}
+ and needs to have been registered
+ slave-side by
+ L{buildbot.slave.registry.registerSlaveCommand}
+ @type args: dict
+ @param args: arguments to send to the remote command
+ """
+
self.remote_command = remote_command
self.args = args
@@ -74,6 +96,14 @@
return self.deferred
def start(self):
+ """
+ Tell the slave to start executing the remote command.
+
+ @rtype: L{twisted.internet.defer.Deferred}
+ @returns: a deferred that will fire when the remote command is
+ done (with None as the result)
+ """
+ # This method only initiates the remote command.
# We will receive remote_update messages as the command runs.
# We will get a single remote_complete when it finishes.
# We should fire self.deferred when the command is done.
@@ -115,6 +145,13 @@
return None
def remote_update(self, updates):
+ """
+ I am called by the slave's L{buildbot.slave.bot.SlaveBuilder} so
+ I can receive updates from the running remote command.
+
+ @type updates: list of [object, int]
+ @param updates: list of updates from the remote command
+ """
max_updatenum = 0
for (update, num) in updates:
#log.msg("update[%d]:" % num)
@@ -134,6 +171,14 @@
raise NotImplementedError("You must implement this in a subclass")
def remote_complete(self, failure=None):
+ """
+ Called by the slave's L{buildbot.slave.bot.SlaveBuilder} to
+ notify me the remote command has finished.
+
+ @type failure: L{twisted.python.failure.Failure} or None
+
+ @rtype: None
+ """
# call the real remoteComplete a moment later, but first return an
# acknowledgement so the slave can retire the completion message.
if self.active:
@@ -151,6 +196,8 @@
# arrange for the callback to get this RemoteCommand instance
# instead of just None
d.addCallback(lambda r: self)
+ # this fires the original deferred we returned from .run(),
+ # with self as the result, or a failure
d.addBoth(self.deferred.callback)
def remoteComplete(self, maybeFailure):
@@ -160,32 +207,32 @@
will be None if the command completed normally, or a Failure
instance in one of the following situations:
- # the slave was lost before the command was started
- # the slave didn't respond to the startCommand message
- # the slave raised an exception while starting the command
- # (bad command name, bad args, OSError from missing executable)
- # the slave raised an exception while finishing the command
- # (they send back a remote_complete message with a Failure payload)
- # and also (for now):
- # slave disconnected while the command was running
+ - the slave was lost before the command was started
+ - the slave didn't respond to the startCommand message
+ - the slave raised an exception while starting the command
+ (bad command name, bad args, OSError from missing executable)
+ - the slave raised an exception while finishing the command
+ (they send back a remote_complete message with a Failure payload)
+
+ and also (for now):
+ - slave disconnected while the command was running
This method should do cleanup, like closing log files. It should
normally return the 'failure' argument, so that any exceptions will
be propagated to the Step. If it wants to consume them, return None
instead."""
- return failure
+ return maybeFailure
class LoggedRemoteCommand(RemoteCommand):
- """This is a RemoteCommand which expects the slave to send back
- stdout/stderr/rc updates. It gathers these updates into a
- builder.LogFile named self.log . You can give it a LogFile to use by
- calling useLog(), or it will create its own when the command is started.
- Unless you tell it otherwise, it will close the log when the command is
- complete.
-
- The constructor takes an 'args' parameter which is a dictionary of
- arguments to send to the remote command."""
+ """
+ I am a L{RemoteCommand} which expects the slave to send back
+ stdout/stderr/rc updates. I gather these updates into a
+ L{buildbot.status.builder.LogFile} named C{self.log}. You can give me a
+ LogFile to use by calling useLog(), or I will create my own when the
+ command is started. Unless you tell me otherwise, I will close the log
+ when the command is complete.
+ """
log = None
closeWhenFinished = False
@@ -237,39 +284,50 @@
"""This class helps you run a shell command on the build slave. It will
accumulate all the command's output into a Log. When the command is
finished, it will fire a Deferred. You can then check the results of the
- command and parse the output however you like.
-
- @param workdir: the directory where the command ought to run, relative to
- the Builder's home directory. If missing or 'None', it will default to
- '.': the same as the Builder's homedir. This should probably be '.' for
- the initial 'cvs checkout' command (which creates a workdir), and the
- workdir for all subsequent commands (including compiles and 'cvs
- update').
-
- @param command: the shell command to run, like 'make all' or 'cvs
- update'. This should be a list or tuple which can be used directly as the
- argv array. For backwards compatibility, if this is a string, the text
- will be given to '/bin/sh -c %s'.
+ command and parse the output however you like."""
- @param env: a dict of environment variables to add or change, or None
- to leave the slave's environment alone. Each command gets a separate
- environment; all inherit the slave's initial one.
- TODO: make it possible to delete some or all of the slave's environment.
+ def __init__(self, workdir, command, env=None,
+ want_stdout=1, want_stderr=1,
+ timeout=20*60, **kwargs):
+ """
+ @type workdir: string
+ @param workdir: directory where the command ought to run,
+ relative to the Builder's home directory. Defaults to
+ '.': the same as the Builder's homedir. This should
+ probably be '.' for the initial 'cvs checkout'
+ command (which creates a workdir), and the Build-wide
+ workdir for all subsequent commands (including
+ compiles and 'cvs update').
- @param want_stdout: defaults to 1. Set to 0 if stdout should be thrown
- away. Do this to avoid storing or sending large amounts of useless data.
+ @type command: list of strings (or string)
+ @param command: the shell command to run, like 'make all' or
+ 'cvs update'. This should be a list or tuple
+ which can be used directly as the argv array.
+ For backwards compatibility, if this is a
+ string, the text will be given to '/bin/sh -c
+ %s'.
- @param want_stderr: Set to 0 if stderr should be thrown away.
+ @type env: dict of string->string
+ @param env: environment variables to add or change for the
+ slave. Each command gets a separate
+ environment; all inherit the slave's initial
+ one. TODO: make it possible to delete some or
+ all of the slave's environment.
- @param timeout: tell the remote that if the command fails to produce any
- output for this number of seconds, the command is hung and should be
- killed. Use None to disable the timeout.
+ @type want_stdout: bool
+ @param want_stdout: defaults to True. Set to False if stdout should
+ be thrown away. Do this to avoid storing or
+ sending large amounts of useless data.
- """
+ @type want_stderr: bool
+ @param want_stderr: False if stderr should be thrown away
- def __init__(self, workdir, command, env=None,
- want_stdout=1, want_stderr=1,
- timeout=20*60, **kwargs):
+ @type timeout: int
+ @param timeout: tell the remote that if the command fails to
+ produce any output for this number of seconds,
+ the command is hung and should be killed. Use
+ None to disable the timeout.
+ """
self.command = command # stash .command, set it later
if env is not None:
# avoid mutating the original master.cfg dictionary. Each
@@ -299,15 +357,16 @@
def __repr__(self):
return "<RemoteShellCommand '%s'>" % self.command
-
-
class BuildStep:
- """This class represents a single step of the build process. This step
- may involve multiple commands to be run in the build slave, as well as
- arbitrary processing on the master side.
+ """
+ I represent a single step of the build process. This step may involve
+ zero or more commands to be run in the build slave, as well as arbitrary
+ processing on the master side. Regardless of how many slave commands are
+ run, the BuildStep will result in a single status value.
The step is started by calling startStep(), which returns a Deferred that
- fires when the step finishes.
+ fires when the step finishes. See C{startStep} for a description of the
+ results provided by that Deferred.
__init__ and start are good methods to override. Don't forget to upcall
BuildStep.__init__ or bad things will happen.
@@ -315,20 +374,22 @@
To launch a RemoteCommand, pass it to .runCommand and wait on the
Deferred it returns.
- Each BuildStep has a collection of output status. These will all be put
- into the 'step_status' (a BuildStepStatus instance). 'results' is also
- passed to the first callback of the Deferred that is returned by
- .startStep():
+ Each BuildStep generates status as it runs. This status data is fed to
+ the L{buildbot.status.builder.BuildStepStatus} listener that sits in
+ C{self.step_status}. It can also feed progress data (like how much text
+ is output by a shell command) to the
+ L{buildbot.status.progress.StepProgress} object that lives in
+ C{self.progress}, by calling C{progress.setProgress(metric, value)} as it
+ runs.
- results: this is one of SUCCESS/WARNINGS/FAILURE/SKIPPED
- progress: a StepProgress instance, which tracks ETA
- logs={}: a set of named status.builder.LogFile objects, holding text
- color: a string indicating the color that this step feels best
- represents its current mood. yellow,green,red,orange are the
- most likely choices, although purple indicates an exception
- text=[]: short text strings that describe the command and its status
- text2=[]: short text that is added to the overall build description
+ @type build: L{buildbot.process.base.Build}
+ @ivar build: the parent Build which is executing this step
+ @type progress: L{buildbot.status.progress.StepProgress}
+ @ivar progress: tracks ETA for the step
+
+ @type step_status: L{buildbot.status.builder.BuildStepStatus}
+ @ivar step_status: collects output status
"""
# these parameters are used by the parent Build object to decide how to
@@ -380,15 +441,29 @@
return None
def startStep(self, remote):
- """Begin the step. This returns a Deferred that will fire with a
- constant of: SUCCESS, WARNINGS, FAILURE, SKIPPED. Any other status
- can be read out of our attributes.
+ """Begin the step. This returns a Deferred that will fire when the
+ step finishes.
- 'status' is a BuildStepStatus object to which I will send status
- updates. 'progress' is a StepProgress object: I will call
- progress.setProgress(metric, value) as I work. 'remote' is a
- RemoteReference to a buildslave that will execute any RemoteCommands
- I want to run."""
+ This deferred fires with a tuple of (result, [extra text]), although
+ older steps used to return just the 'result' value, so the receiving
+ L{base.Build} needs to be prepared to handle that too. C{result} is
+ one of the SUCCESS/WARNINGS/FAILURE/SKIPPED constants from
+ L{buildbot.status.builder}, and the extra text is a list of short
+ strings which should be appended to the Build's text results. This
+ text allows a test-case step which fails to append B{17 tests} to the
+ Build's status, in addition to marking the build as failing.
+
+ The deferred will errback if the step encounters an exception,
+ including an exception on the slave side (or if the slave goes away
+ altogether). Failures in shell commands (rc!=0) will B{not} cause an
+ errback, in general the BuildStep will evaluate the results and
+ decide whether to treat it as a WARNING or FAILURE.
+
+ @type remote: L{twisted.spread.pb.RemoteReference}
+ @param remote: a reference to the slave's
+ L{buildbot.slave.bot.SlaveBuilder} instance where any
+ RemoteCommands may be run
+ """
self.remote = remote
self.deferred = defer.Deferred()
@@ -405,39 +480,40 @@
return self.deferred
def start(self):
- """Begin the step. Add code here to do local processing, fire off
- remote commands, etc.
+ """Begin the step. Override this method and add code to do local
+ processing, fire off remote commands, etc.
To spawn a command in the buildslave, create a RemoteCommand instance
- and run it with self.runCommand:
+ and run it with self.runCommand::
- c = RemoteCommandFoo(args)
- d = self.runCommand(c)
- d.addCallback(self.fooDone).addErrback(self.failed)
+ c = RemoteCommandFoo(args)
+ d = self.runCommand(c)
+ d.addCallback(self.fooDone).addErrback(self.failed)
As the step runs, it should send status information to the
- BuildStepStatus.
+ BuildStepStatus::
- self.step_status.setColor('red')
- self.step_status.setText(['compile', 'failed'])
- self.step_status.setText2(['4', 'warnings'])
+ self.step_status.setColor('red')
+ self.step_status.setText(['compile', 'failed'])
+ self.step_status.setText2(['4', 'warnings'])
To add a LogFile, use self.addLog. Make sure it gets closed when it
finishes. When giving a Logfile to a RemoteShellCommand, just ask it
- to close the log when the command completes:
+ to close the log when the command completes::
- log = self.addLog('output')
- cmd = RemoteShellCommand
- cmd.useLog(log, closeWhenFinished=True)
+ log = self.addLog('output')
+ cmd = RemoteShellCommand(args)
+ cmd.useLog(log, closeWhenFinished=True)
You can also create complete Logfiles with generated text in a single
- step:
+ step::
- self.addCompleteLog('warnings', text)
+ self.addCompleteLog('warnings', text)
When the step is done, it should call self.finished(result). 'result'
- will be provided to the BuildProcess, and should one of the constants
- defined above: SUCCESS, WARNINGS, FAILURE, or SKIPPED.
+ will be provided to the L{buildbot.process.base.Build}, and should be
+ one of the constants defined above: SUCCESS, WARNINGS, FAILURE, or
+ SKIPPED.
If the step encounters an exception, it should call self.failed(why).
'why' should be a Failure object. This automatically fails the whole
@@ -518,7 +594,7 @@
class ShellCommand(BuildStep):
"""I run a single shell command on the buildslave. I return FAILURE if
the exit code of that command is non-zero, SUCCESS otherwise. To change
- this behavior, override my .commandFinished method.
+ this behavior, override my .evaluateCommand method.
I create a single Log named 'log' which contains the output of the
command. To create additional summary Logs, override my .createSummary
@@ -526,12 +602,9 @@
The shell command I run (a list of argv strings) can be provided in
several ways:
-
- a class-level .command attribute
-
- a command= parameter to my constructor (overrides .command)
-
- set explicitly with my .setCommand() method (overrides both)
+ - a class-level .command attribute
+ - a command= parameter to my constructor (overrides .command)
+ - set explicitly with my .setCommand() method (overrides both)
"""
@@ -559,16 +632,21 @@
self.cmd.command = command
def describe(self, done=False):
- """Return a list of short strings to describe this step. This uses
- the first few words of the shell command. You can replace this by
- setting .description in your subclass, or by overriding this method
- to describe the step better.
+ """Return a list of short strings to describe this step, for the
+ status display. This uses the first few words of the shell command.
+ You can replace this by setting .description in your subclass, or by
+ overriding this method to describe the step better.
- done=False is used to describe the step while it is running, so a
- single imperfect-tense verb is appropriate ('compiling', 'testing',
- etc). done=True is used when the step has finished, and the default
- getText() method adds some text, so a noun is appropriate
- ('compile', 'tests', etc)."""
+ @type done: boolean
+ @param done: whether the command is complete or not, to improve the
+ way the command is described. C{done=False} is used
+ while the command is still running, so a single
+ imperfect-tense verb is appropriate ('compiling',
+ 'testing', ...) C{done=True} is used when the command
+ has finished, and the default getText() method adds some
+ text, so a simple noun is appropriate ('compile',
+ 'tests' ...)
+ """
if done and self.descriptionDone is not None:
return self.descriptionDone
@@ -752,69 +830,83 @@
startVC(). The class as a whole builds up the self.args dictionary, then
starts a LoggedRemoteCommand with those arguments.
- @param workdir: a string giving the local directory (relative to the
- Builder's root) where the tree should be placed.
+ """
- @param mode: a string describing the kind of VC operation that is
- desired.
+ # if the checkout fails, there's no point in doing anything else
+ haltOnFailure = True
+ notReally = False
- 'update' specifies that the checkout/update should be performed
- directly into the workdir. Each build is performed in the same
- directory, allowing for incremental builds. This minimizes disk space,
- bandwidth, and CPU time. However, it may encounter problems if the
- build process does not handle dependencies properly (if you must
- sometimes do a 'clean build' to make sure everything gets compiled), or
- if source files are deleted but generated files can influence test
- behavior (e.g. python's .pyc files), or when source directories are
- deleted but generated files prevent CVS from removing them.
+ def __init__(self, workdir, mode='update', alwaysUseLatest=False,
+ timeout=20*60, **kwargs):
+ """
+ @type workdir: string
+ @param workdir: local directory (relative to the Builder's root)
+ where the tree should be placed
- 'copy' specifies that the source-controlled workspace should be
- maintained in a separate directory (called the 'copydir'), using
- checkout or update as necessary. For each build, a new workdir is
- created with a copy of the source tree (rm -rf workdir; cp -r copydir
- workdir). This doubles the disk space required, but keeps the bandwidth
- low (update instead of a full checkout). A full 'clean' build is
- performed each time. This avoids any generated-file build problems, but
- is still occasionally vulnerable to problems such as a CVS repository
- being manually rearranged (causing CVS errors on update) which are not
- an issue with a full checkout.
+ @type mode: string
+ @param mode: the kind of VC operation that is desired:
+ - 'update': specifies that the checkout/update should be
+ performed directly into the workdir. Each build is performed
+ in the same directory, allowing for incremental builds. This
+ minimizes disk space, bandwidth, and CPU time. However, it
+ may encounter problems if the build process does not handle
+ dependencies properly (if you must sometimes do a 'clean
+ build' to make sure everything gets compiled), or if source
+ files are deleted but generated files can influence test
+ behavior (e.g. python's .pyc files), or when source
+ directories are deleted but generated files prevent CVS from
+ removing them.
- 'clobber' specifes that the working directory should be deleted each
- time, necessitating a full checkout for each build. This insures a
- clean build off a complete checkout, avoiding any of the problems
- described above, but is bandwidth intensive, as the whole source tree
- must be pulled down for each build.
+ - 'copy': specifies that the source-controlled workspace
+ should be maintained in a separate directory (called the
+ 'copydir'), using checkout or update as necessary. For each
+ build, a new workdir is created with a copy of the source
+ tree (rm -rf workdir; cp -r copydir workdir). This doubles
+ the disk space required, but keeps the bandwidth low
+ (update instead of a full checkout). A full 'clean' build
+ is performed each time. This avoids any generated-file
+ build problems, but is still occasionally vulnerable to
+ problems such as a CVS repository being manually rearranged
+ (causing CVS errors on update) which are not an issue with
+ a full checkout.
- 'export' is like 'clobber', except that e.g. the 'cvs export' command
- is used to create the working directory. This command removes all VC
- metadata files (the CVS/.svn/{arch} directories) from the tree, which
- is sometimes useful for creating source tarballs (to avoid including
- the metadata in the tar file). Not all VC systems support export.
+ - 'clobber': specifies that the working directory should be
+ deleted each time, necessitating a full checkout for each
+ build. This insures a clean build off a complete checkout,
+ avoiding any of the problems described above, but is
+ bandwidth intensive, as the whole source tree must be
+ pulled down for each build.
- @param alwaysUseLatest: normally the Source step asks its Build for a
- list of all Changes that are supposed to go into the build, then
- computes a 'source stamp' (revision number or timestamp) that will
- cause exactly that set of changes to be present in the checked out
- tree. This is turned into, e.g., 'cvs update -D timestamp', or 'svn
- update -r revnum'. If alwaysUseLatest=True, bypass this computation and
- always update to the latest available sources for each build.
+ - 'export': is like 'clobber', except that e.g. the 'cvs
+ export' command is used to create the working directory.
+ This command removes all VC metadata files (the
+ CVS/.svn/{arch} directories) from the tree, which is
+ sometimes useful for creating source tarballs (to avoid
+ including the metadata in the tar file). Not all VC systems
+ support export.
- The source stamp helps avoid a race condition in which someone commits
- a change after the master has decided to start a build but before the
- slave finishes checking out the sources. At best this results in a
- build which contains more changes than the buildmaster thinks it has
- (possibly resulting in the wrong person taking the blame for any
- problems that result), at worst is can result in an incoherent set of
- sources (splitting a non-atomic commit) which may not build at all.
+ @type alwaysUseLatest: boolean
+ @param alwaysUseLatest: whether to always update to the most
+ recent available sources for this build.
- """
+ Normally the Source step asks its Build for a list of all
+ Changes that are supposed to go into the build, then computes a
+ 'source stamp' (revision number or timestamp) that will cause
+ exactly that set of changes to be present in the checked out
+ tree. This is turned into, e.g., 'cvs update -D timestamp', or
+ 'svn update -r revnum'. If alwaysUseLatest=True, bypass this
+ computation and always update to the latest available sources
+ for each build.
- # if the checkout fails, there's no point in doing anything else
- haltOnFailure = True
- notReally = False
+ The source stamp helps avoid a race condition in which someone
+ commits a change after the master has decided to start a build
+ but before the slave finishes checking out the sources. At best
+ this results in a build which contains more changes than the
+ buildmaster thinks it has (possibly resulting in the wrong
+ person taking the blame for any problems that result), at worst
+ is can result in an incoherent set of sources (splitting a
+ non-atomic commit) which may not build at all. """
- def __init__(self, workdir, mode='update', alwaysUseLatest=False,
- timeout=20*60, **kwargs):
BuildStep.__init__(self, **kwargs)
assert mode in ("update", "copy", "clobber", "export")
self.args = {'mode': mode,
@@ -868,38 +960,6 @@
class CVS(Source):
"""I do CVS checkout/update operations.
- @param cvsroot (required): a string which describes the CVS Repository
- from which the source tree should be obtained. '/home/warner/Repository'
- for local or NFS-reachable repositories, ':pserver:anon at foo.com:/cvs'
- for anonymous CVS, 'user at host.com:/cvs' for non-anonymous CVS or CVS
- over ssh. Lots of possibilities, check the CVS documentation for more.
-
- @param cvsmodule (required): a string giving the subdirectory of the CVS
- repository that should be retrieved.
-
- @param login: if not None, a string which will be provided as a password
- to the 'cvs login' command, used when a :pserver: method is used to
- access the repository. This login is only needed once, but must be run
- each time (just before the CVS operation) because there is no way for
- the buildslave to tell whether it was previously performed or not.
-
- @param branch: a string to be used in a '-r' argument to specify which
- named branch of the source tree should be used for this checkout.
- Defaults to 'HEAD'.
-
- @param checkoutDelay: if not None, the number of seconds to put between
- the last known Change and the timestamp given to the -D argument. This
- defaults to exactly half of the parent Build's .treeStableTimer, but it
- could be set to something else if your CVS change notification has
- particularly weird latency characteristics.
-
- @param global_options=[]: these arguments are inserted in the cvs
- command line, before the 'checkout'/'update' command word. See 'cvs
- --help-options' for a list of what may be accepted here. ['-r'] will
- make the checked out files read only. ['-r', '-R'] will also assume the
- repository is read-only (I assume this means it won't use locks to
- insure atomic access to the ,v files).
-
Note: if you are doing anonymous/pserver CVS operations, you will need
to manually do a 'cvs login' on each buildslave before the slave has any
hope of success. XXX: fix then, take a cvs password as an argument and
@@ -924,6 +984,56 @@
clobber=0, export=0, copydir=None,
**kwargs):
+ """
+ @type cvsroot: string
+ @param cvsroot: CVS Repository from which the source tree should
+ be obtained. '/home/warner/Repository' for local
+ or NFS-reachable repositories,
+ ':pserver:anon at foo.com:/cvs' for anonymous CVS,
+ 'user at host.com:/cvs' for non-anonymous CVS or
+ CVS over ssh. Lots of possibilities, check the
+ CVS documentation for more.
+
+ @type cvsmodule: string
+ @param cvsmodule: subdirectory of CVS repository that should be
+ retrieved
+
+ @type login: string or None
+ @param login: if not None, a string which will be provided as a
+ password to the 'cvs login' command, used when a
+ :pserver: method is used to access the repository.
+ This login is only needed once, but must be run
+ each time (just before the CVS operation) because
+ there is no way for the buildslave to tell whether
+ it was previously performed or not.
+
+ @type branch: string
+ @param branch: a string to be used in a '-r' argument to specify
+ which named branch of the source tree should be
+ used for this checkout. Defaults to 'HEAD'.
+
+ @type checkoutDelay: int or None
+ @param checkoutDelay: if not None, the number of seconds to put
+ between the last known Change and the
+ timestamp given to the -D argument. This
+ defaults to exactly half of the parent
+ Build's .treeStableTimer, but it could be
+ set to something else if your CVS change
+ notification has particularly weird
+ latency characteristics.
+
+ @type global_options: list of strings
+ @param global_options: these arguments are inserted in the cvs
+ command line, before the
+ 'checkout'/'update' command word. See
+ 'cvs --help-options' for a list of what
+ may be accepted here. ['-r'] will make
+ the checked out files read only. ['-r',
+ '-R'] will also assume the repository is
+ read-only (I assume this means it won't
+ use locks to insure atomic access to the
+ ,v files)."""
+
self.checkoutDelay = checkoutDelay
if not kwargs.has_key('mode') and (clobber or export or copydir):
@@ -983,17 +1093,19 @@
class SVN(Source):
- """I perform Subversion checkout/update operations.
-
- @param svnurl (required): the URL which points to the Subversion server.
- This one string combines the access method (HTTP, ssh, local file), the
- repository host/port, the repository path, the sub-tree within the
- repository, and which branch to check out.
- """
+ """I perform Subversion checkout/update operations."""
name = 'svn'
def __init__(self, svnurl, directory=None, **kwargs):
+ """
+ @type svnurl: string
+ @param svnurl: the URL which points to the Subversion server,
+ combining the access method (HTTP, ssh, local file),
+ the repository host/port, the repository path,
+ the sub-tree within the repository, and the branch
+ to check out.
+ """
if not kwargs.has_key('workdir') and directory is not None:
# deal with old configs
@@ -1044,12 +1156,16 @@
means the eXecute-bit will be cleared on all source files. As a result,
you may need to invoke configuration scripts with something like:
- s(step.Configure, command=['/bin/sh', './configure'])
+ C{s(step.Configure, command=['/bin/sh', './configure'])}
"""
name = "darcs"
def __init__(self, repourl, **kwargs):
+ """
+ @type repourl: string
+ @param repourl: the URL which points at the Darcs repository
+ """
assert kwargs['mode'] != "export", \
"Darcs does not have an 'export' mode"
Source.__init__(self, **kwargs)
@@ -1069,18 +1185,29 @@
This step will first register the archive, which requires a per-user
'archive name' to correspond to the URL from which the sources can be
fetched. The archive's default name will be used for this unless you
- override it by setting the 'archive' parameter. You might want to do
- this if, for some reason, you are hosting the archive on the same
- machine (and in the same account) as the build slave, and you don't want
- to confuse local access with remote access.
-
- [forgive the confusion expressed in the previous paragraph, I'm still
- trying to get my head around Arch.. -warner]
+ override it by setting the 'archive' parameter.
"""
name = "arch"
def __init__(self, url, version, archive=None, **kwargs):
+ """
+ @type url: string
+ @param url: the Arch coordinates of the repository. This is
+ typically an http:// URL, but could also be the absolute
+ pathname of a local directory instead.
+
+ @type version: string
+ @param version: the category--branch--version to check out
+
+ @type archive: string
+ @param archive: an optional archive name, to override the one
+ provided by the repository. You might want to do this
+ if, for some reason, you are hosting the archive on
+ the same machine (and in the same account) as the
+ build slave, and you don't want to confuse local
+ access with remote access.
+ """
Source.__init__(self, **kwargs)
self.args.update({'url': url,
'version': version,
@@ -1123,9 +1250,9 @@
Each slave needs the following environment:
- PATH: the 'p4' binary must be on the slave's PATH
- P4USER: each slave needs a distinct user account
- P4CLIENT: each slave needs a distinct client specification
+ - PATH: the 'p4' binary must be on the slave's PATH
+ - P4USER: each slave needs a distinct user account
+ - P4CLIENT: each slave needs a distinct client specification
You should use 'p4 client' (?) to set up a client view spec which maps
the desired files into $SLAVEBASE/$BUILDERBASE/source .
@@ -1152,14 +1279,18 @@
class Dummy(BuildStep):
- """I am a dummy no-op step that takes 5 seconds to complete.
- @param timeout: the number of seconds to delay
+ """I am a dummy no-op step, which runs entirely on the master, and simply
+ waits 5 seconds before finishing with SUCCESS
"""
haltOnFailure = True
name = "dummy"
def __init__(self, timeout=5, **kwargs):
+ """
+ @type timeout: int
+ @param timeout: the number of seconds to delay before completing
+ """
BuildStep.__init__(self, **kwargs)
self.timeout = timeout
self.timer = None
@@ -1182,9 +1313,8 @@
self.finished(SUCCESS)
class FailingDummy(Dummy):
- """I am a dummy step that raises an Exception after 5 seconds
- @param timeout: the number of seconds to delay
- """
+ """I am a dummy no-op step that 'runs' master-side and raises an
+ Exception after by default 5 seconds."""
name = "failing dummy"
@@ -1202,18 +1332,21 @@
f = Failure()
self.failed(f)
+# subclasses from Shell Command to get the output reporting
class RemoteDummy(ShellCommand):
- """I am a dummy no-op step that runs on the remote side and takes 5
- seconds to complete.
-
- @param timeout: the number of seconds to delay
- @param results: None
+ """I am a dummy no-op step that runs on the remote side and
+ simply waits 5 seconds before completing with success.
+ See L{buildbot.slave.commands.DummyCommand}
"""
haltOnFailure = True
name = "remote dummy"
def __init__(self, timeout=5, **kwargs):
+ """
+ @type timeout: int
+ @param timeout: the number of seconds to delay
+ """
BuildStep.__init__(self, **kwargs)
args = {'timeout': timeout}
self.cmd = LoggedRemoteCommand("dummy", args)
Index: step_twisted.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step_twisted.py,v
retrieving revision 1.66
retrieving revision 1.67
diff -u -d -r1.66 -r1.67
--- step_twisted.py 11 Apr 2005 19:30:30 -0000 1.66
+++ step_twisted.py 18 Apr 2005 00:26:57 -0000 1.67
@@ -31,15 +31,6 @@
warnings = 0
def __init__(self, python=None, **kwargs):
- """
- @type results: double of [int, string]
- @keyword results: [warnings, output].
- - warnings: the number of problems that were found
- in the XHTML files (equal to the number of lines of
- output that have colons in them), 0 if none were
- found.
- - output: string with all the warnings.
- """
ShellCommand.__init__(self, **kwargs)
self.python = python
@@ -657,22 +648,26 @@
class BuildDebs(ShellCommand):
- """I build the .deb packages.
-
- @param workdir: the workdir to start from: must be the base of the
- Twisted tree
-
- @param results: [rc, output]. rc==0 if all .debs were created
- successfully. 'output' is a string with any errors or warnings.
-
- """
-
+ """I build the .deb packages."""
+
name = "debuild"
flunkOnFailure = 1
command = ["debuild", "-uc", "-us"]
description = ["building", "debs"]
descriptionDone = ["debs"]
+ def __init__(self, **kwargs):
+ """
+ @type workdir: string
+ @keyword workdir: the workdir to start from (must be the base of the
+ Twisted tree)
+ @type results: double of [int, string]
+ @keyword results: [rc, output].
+ - rc == 0 if all .debs were created successfully
+ - output: string with any errors or warnings
+ """
+ ShellCommand.__init__(self, **kwargs)
+
def commandComplete(self, cmd):
errors, warnings = 0, 0
output = cmd.log.getText()
More information about the Commits
mailing list