[Buildbot-devel] [PATCH] 2/2: triggerable schedulers
Dustin J. Mitchell
dustin at zmanda.com
Mon Jul 2 16:11:37 UTC 2007
A pair of a Scheduler and a BuildStep, where the step can "trigger"
one or more schedulers, and optionally waits on their completion.
This allows a general form of subroutine call, but is more specifically
useful as a more powerful way to express dependent builds, particularly
when a build must take place cooperatively among a group of builders.
Comments are more than welcome here! In particular, I didn't spend much
time on the documentation, so any help there would be appreciated.
Dustin
--
Dustin J. Mitchell
Storage Software Engineer, Zmanda, Inc.
http://www.zmanda.com/
-------------- next part --------------
A pair of a Scheduler and a BuildStep, where the step can "trigger"
one or more schedulers, and optionally waits on their completion.
This allows a general form of subroutine call, but is more specifically
useful as a more powerful way to express dependent builds, particularly
when a buid must take place cooperatively among a group of builders.
Index: triggers/buildbot/scheduler.py
===================================================================
--- triggers.orig/buildbot/scheduler.py 2007-07-02 10:28:39.596836658 -0500
+++ triggers/buildbot/scheduler.py 2007-07-02 10:30:27.405240496 -0500
@@ -682,3 +682,28 @@
from buildbot.status.client import makeRemote
return makeRemote(bs.status)
+class TriggerableScheduler(BaseUpstreamScheduler):
+ """
+ This scheduler doesn't do anything until it is triggered by
+ a TriggerScheduler step in a factory.
+ """
+
+ def __init__(self, name, builderNames):
+ BaseUpstreamScheduler.__init__(self, name)
+ self.builderNames = builderNames
+
+ def listBuilderNames(self):
+ return self.builderNames
+
+ def getPendingBuildTimes(self):
+ return []
+
+ def trigger(self, ss):
+ """
+ Trigger this scheduler. Returns a deferred that will fire when the buildset
+ is finished.
+ """
+ bs = buildset.BuildSet(self.builderNames, ss)
+ d = bs.waitUntilFinished()
+ self.submit(bs)
+ return d
Index: triggers/buildbot/steps/trigger.py
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ triggers/buildbot/steps/trigger.py 2007-07-02 10:30:27.405240496 -0500
@@ -0,0 +1,102 @@
+from buildbot.process.buildstep import LoggingBuildStep, SUCCESS, FAILURE, EXCEPTION
+from buildbot.steps.shell import WithProperties
+from buildbot.sourcestamp import SourceStamp
+from buildbot.scheduler import TriggerableScheduler
+from twisted.internet import defer
+from twisted.python import log
+import os
+
+class TriggerScheduler(LoggingBuildStep):
+ """
+ I trigger a TriggerableScheduler. It's fun.
+ """
+ name = "trigger"
+
+ flunkOnFailure = True
+
+ def __init__(self,
+ schedulers=[],
+ updateSourceStamp=False,
+ waitForFinish=False,
+ **kwargs):
+ """
+ Trigger the given schedulers when this step is executed.
+
+ @var schedulers: list of schedulers' names that should be triggered. Schedulers
+ can be specified using WithProperties, if desired.
+
+ @var updateSourceStamp: should I update the source stamp to contain got_revision
+ from the triggering build?
+
+ @var waitForFinish: should I wait for all of the triggered schedulers to finish
+ their builds?
+ """
+ assert schedulers, "You must specify a scheduler to trigger"
+ self.schedulers = schedulers
+ self.updateSourceStamp = updateSourceStamp
+ self.waitForFinish = waitForFinish
+ self.running = False
+ LoggingBuildStep.__init__(self, **kwargs)
+
+ def interrupt(self, reason):
+ if self.running:
+ self.step_status.setColor("red")
+ self.step_status.setText(["interrupted"])
+
+ def start(self):
+ self.running = True
+ ss = self.build.getSourceStamp()
+ if self.updateSourceStamp:
+ ss = SourceStamp(ss.branch, self.build.getProperty('got_revision'), ss.patch)
+ # (is there an easier way to find the BuildMaster?)
+ all_schedulers = self.build.builder.botmaster.parent.allSchedulers()
+ all_schedulers = dict([(sch.name, sch) for sch in all_schedulers])
+ unknown_schedulers = []
+ triggered_schedulers = []
+
+ dl = []
+ for scheduler in self.schedulers:
+ if isinstance(scheduler, WithProperties):
+ scheduler = scheduler.render(self.build)
+ if all_schedulers.has_key(scheduler):
+ sch = all_schedulers[scheduler]
+ if isinstance(sch, TriggerableScheduler):
+ dl.append(sch.trigger(ss))
+ triggered_schedulers.append(scheduler)
+ else:
+ unknown_schedulers.append(scheduler)
+ else:
+ unknown_schedulers.append(scheduler)
+
+ if unknown_schedulers:
+ self.step_status.setColor("red")
+ self.step_status.setText(['no scheduler:'] + unknown_schedulers)
+ rc = FAILURE
+ else:
+ rc = SUCCESS
+ self.step_status.setText(['triggered'] + triggered_schedulers)
+ if self.waitForFinish:
+ self.step_status.setColor("yellow")
+ else:
+ self.step_status.setColor("green")
+
+ if self.waitForFinish:
+ d = defer.DeferredList(dl, consumeErrors=1)
+ else:
+ d = defer.succeed([])
+
+ def cb(rclist):
+ rc = SUCCESS
+ for was_cb, buildsetstatus in rclist:
+ # TODO: make this algo more configurable
+ if not was_cb:
+ rc = EXCEPTION
+ break
+ if buildsetstatus.getResults() == FAILURE:
+ rc = FAILURE
+ return self.finished(rc)
+
+ def eb(why):
+ return self.finished(FAILURE)
+
+ d.addCallbacks(cb, eb)
Index: triggers/buildbot/test/test_run.py
===================================================================
--- triggers.orig/buildbot/test/test_run.py 2007-07-02 10:28:39.596836658 -0500
+++ triggers/buildbot/test/test_run.py 2007-07-02 10:30:27.405240496 -0500
@@ -543,6 +543,77 @@
def _testTestFlag_2(self, res):
self.failUnlessEqual(self.getFlag('foo'), 'bar')
+class Triggers(RunMixin, TestFlagMixin, unittest.TestCase):
+ config_trigger = config_base + """
+from buildbot.scheduler import TriggerableScheduler, Scheduler
+from buildbot.steps.trigger import TriggerScheduler
+from buildbot.steps.dummy import Dummy
+from buildbot.test.runutils import SetTestFlagStep
+c['schedulers'] = [
+ Scheduler('triggerer', None, 0.1, ['triggerer']),
+ TriggerableScheduler('triggeree', ['triggeree'])
+]
+triggerer = factory.BuildFactory([
+ s(SetTestFlagStep, flagname='triggerer_started'),
+ s(TriggerScheduler, flunkOnFailure=True, @ARGS@),
+ s(SetTestFlagStep, flagname='triggerer_finished'),
+ ])
+triggeree = factory.BuildFactory([
+ s(SetTestFlagStep, flagname='triggeree_started'),
+ s(@DUMMYCLASS@),
+ s(SetTestFlagStep, flagname='triggeree_finished'),
+ ])
+c['builders'] = [{'name': 'triggerer', 'slavename': 'bot1',
+ 'builddir': 'triggerer', 'factory': triggerer},
+ {'name': 'triggeree', 'slavename': 'bot1',
+ 'builddir': 'triggeree', 'factory': triggeree}]
+"""
+
+ def mkConfig(self, args, dummyclass="Dummy"):
+ return self.config_trigger.replace("@ARGS@", args).replace("@DUMMYCLASS@", dummyclass)
+
+ def setupTest(self, args, dummyclass, checkFn):
+ self.clearFlags()
+ m = self.master
+ m.loadConfig(self.mkConfig(args, dummyclass))
+ m.readConfig = True
+ m.startService()
+
+ c = changes.Change("bob", ["Makefile", "foo/bar.c"], "changed stuff")
+ m.change_svc.addChange(c)
+
+ d = self.connectSlave(builders=['triggerer', 'triggeree'])
+ d.addCallback(self.startTimer, 0.5, checkFn)
+ return d
+
+ def startTimer(self, res, time, next_fn):
+ d = defer.Deferred()
+ reactor.callLater(time, d.callback, None)
+ d.addCallback(next_fn)
+ return d
+
+ def testTriggerBuild(self):
+ return self.setupTest("schedulers=['triggeree']",
+ "Dummy",
+ self._checkTriggerBuild)
+
+ def _checkTriggerBuild(self, res):
+ self.failIfFlagNotSet('triggerer_started')
+ self.failIfFlagNotSet('triggeree_started')
+ self.failIfFlagSet('triggeree_finished')
+ self.failIfFlagNotSet('triggerer_finished')
+
+ def testTriggerBuildWait(self):
+ return self.setupTest("schedulers=['triggeree'], waitForFinish=1",
+ "Dummy",
+ self._checkTriggerBuildWait)
+
+ def _checkTriggerBuildWait(self, res):
+ self.failIfFlagNotSet('triggerer_started')
+ self.failIfFlagNotSet('triggeree_started')
+ self.failIfFlagSet('triggeree_finished')
+ self.failIfFlagSet('triggerer_finished')
+
# TODO: test everything, from Change submission to Scheduler to Build to
# Status. Use all the status types. Specifically I want to catch recurrences
# of the bug where I forgot to make Waterfall inherit from StatusReceiver
Index: triggers/docs/buildbot.texinfo
===================================================================
--- triggers.orig/docs/buildbot.texinfo 2007-06-17 21:52:49.000000000 -0500
+++ triggers/docs/buildbot.texinfo 2007-07-02 11:10:47.566771703 -0500
@@ -2102,6 +2102,12 @@
section (@pxref{Build Dependencies}) describes this scheduler in more
detail.
+ at item TriggerableScheduler
+This scheduler does nothing until it is triggered by a TriggerScheduler
+step in another build. This facilitates a more general form of
+build dependencies, as described in the next section (@pxref{Build
+Dependencies}).
+
@item Periodic
This simple scheduler just triggers a build every N seconds.
@@ -2128,6 +2134,7 @@
@cindex Dependent
@cindex Dependencies
@slindex buildbot.scheduler.Dependent
+ at slindex buildbot.scheduler.TriggerableScheduler
It is common to wind up with one kind of build which should only be
performed if the same source code was successfully handled by some
@@ -2170,6 +2177,55 @@
@code{Scheduler} @emph{instance}, not a name. This makes it impossible
to create circular dependencies in the config file.
+A more general way to coordinate builds is by ``triggering'' schedulers
+from builds. The TriggerableScheduler waits to be triggered by a
+TriggerScheduler step in another build. That step can optionally
+wait for the scheduler's builds to complete. This provides two
+advantages over Dependent schedulers. First, the same scheduler
+can be triggered from multiple builds. Second, the ability to wait
+for a TriggerableScheduler's builds to complete provides a form of
+"subroutine call", where one or more builds can "call" a scheduler
+to perform some work for them, perhaps on other buildslaves.
+
+ at example
+from buildbot import scheduler
+from buildbot.steps import trigger
+checkin = scheduler.Scheduler("checkin", None, 5*60,
+ ["checkin"])
+nightly = scheduler.Scheduler("nightly", ...
+ ["nightly"])
+mktarball = scheduler.TriggerableScheduler("mktarball",
+ ["mktarball"])
+build = scheduler.TriggerableScheduler("build-all-platforms",
+ ["build-all-platforms"])
+test = scheduler.TriggerableScheduler("distributed-test",
+ ["distributed-test"])
+package = scheduler.TriggerableScheduler("package-all-platforms",
+ ["package-all-platforms"])
+c['schedulers'] = [checkin, nightly, build, test, package]
+
+checkin_factory = factory.BuildFactory()
+f.addStep(trigger.TriggerStep('mktarball',
+ schedulers=['mktarball'],
+ waitForFinish=1)
+f.addStep(trigger.TriggerStep('build',
+ schedulers=['build-all-platforms'],
+ waitForFinish=1)
+f.addStep(trigger.TriggerStep('test',
+ schedulers=['distributed-test'],
+ waitForFinish=1)
+
+nightly_factory = factory.BuildFactory()
+f.addStep(trigger.TriggerStep('mktarball',
+ schedulers=['mktarball'],
+ waitForFinish=1)
+f.addStep(trigger.TriggerStep('build',
+ schedulers=['build-all-platforms'],
+ waitForFinish=1)
+f.addStep(trigger.TriggerStep('package',
+ schedulers=['package-all-platforms'],
+ waitForFinish=1)
+ at end example
@node Setting the slaveport, Buildslave Specifiers, Listing Change Sources and Schedulers, Configuration
@section Setting the slaveport
@@ -3211,6 +3267,7 @@
* Simple ShellCommand Subclasses::
* Python BuildSteps::
* Transferring Files::
+* Triggering Schedulers::
* Writing New BuildSteps::
@end menu
@@ -4082,7 +4139,7 @@
@end table
- at node Python BuildSteps, Transferring Files, Simple ShellCommand Subclasses, Build Steps
+ at node Python BuildSteps, Triggering Schedulers, Simple ShellCommand Subclasses, Build Steps
@subsection Python BuildSteps
Here are some BuildSteps that are specifcally useful for projects
@@ -4158,7 +4215,7 @@
@end example
- at node Transferring Files, Writing New BuildSteps, Python BuildSteps, Build Steps
+ at node Transferring Files, Triggering Schedulers, Python BuildSteps, Build Steps
@subsection Transferring Files
@cindex File Transfer
@@ -4247,7 +4304,39 @@
creation time (@pxref{Buildslave Options}).
- at node Writing New BuildSteps, , Transferring Files, Build Steps
+ at node Triggering Schedulers, Writing New BuildSteps, Transferring Files, Build Steps
+ at subsection Triggering Schedulers
+
+The counterpart to the TriggerableScheduler described in section
+ at pxref{Build Dependencies} is the TriggerScheduler BuildStep.
+
+ at example
+from buildbot.steps.trigger import TriggerScheduler
+f.addStep(TriggerScheduler,
+ schedulers=['build-prep'],
+ waitForFinish=1,
+ updateSourceStamp=1)
+ at end example
+
+The @code{schedulers=} argument lists the TriggerableSchedulers
+that should be triggered when this step is executed. Note that
+it is possible, but not advisable, to create a cycle where a build
+continually triggers itself, because the schedulers are specified
+by name.
+
+If @code{waitForFinish} is true, then the step will not finish until
+all of the builds from the triggered schedulers have finished. If this
+argument is not given, then the buildstep succeeds immediately after
+triggering the schedulers.
+
+If @code{updateSourceStamp} is true, then step updates the SourceStamp
+given to the TriggerableSchedulers to include @code{got_revision}
+(the revision actually used in this build) as @code{revision} (the
+revision to use in the triggered builds). This is useful to ensure
+that all of the builds use exactly the same SourceStamp, even if
+other Changes have occurred while the build was running.
+
+ at node Writing New BuildSteps, , Triggering Schedulers, Build Steps
@subsection Writing New BuildSteps
While it is a good idea to keep your build process self-contained in
More information about the devel
mailing list