[Buildbot-devel] One way to manage multiple buildbots
Einar Karttunen
ekarttun at cs.helsinki.fi
Mon Sep 4 18:55:31 UTC 2006
Hello
Here is a description of the approach I took
with buildbot configuration. Most of the machines
are inside one LAN. There are many projects and
I want to keep the configuration as simple as possible.
The buildmaster host is buildmaster.aoinet which
runs Debian. It has a default buildbot installation
with user buildbot and home /var/run/buildbot.
Under /var/run/buildbot/ there is a file named
"global-config" (attached) that is global configuration
shared by buildbot master instances. It also specifies
passwords for the slave machines.
Projects are controlled via a file called
/etc/aoinet/buildslave.list (attached). This has a
simple syntax of:
project-name build-system slave-port http-port
There are three helper scripts (attached):
/etc/aoinet/bin/create-buildslave-user
* create buildslave user
/etc/aoinet/bin/update-buildmasters
* create the buildmaster instances from
buildslave.list missing from the buildmaster.
/etc/aoinet/bin/update-buildslaves
* create the buildmaster instances from
buildslave.list missing from this buildslave.
Common operations:
1) Create all missing slavebots on a buildslave host
su buildslave -c /etc/aoinet/bin/update-buildslaves
It helps to have a shellscript to run the same command
on multiple hosts.
2) Add a new slave host
add the slave name + password in global-config
/etc/aoinet/bin/create-buildslave-user on the slave host
and 1)
3) Add a new project
Add a line to /etc/aoinet/buildslave.list
on buildmaster /etc/aoinet/bin/update-buildmaster
on buildslaves /etc/aoinet/bin/update-buildslave
In addition I modified darcs_buildbot.py to take a
project name and infer the port from the buildslave.list.
Hope this helps someone.
- Einar Karttunen
-------------- next part --------------
# -*- python -*-
# ex: set syntax=python:
#project = '<FIXME>'
buildtype = ''
portslave = 0
porthttp = 0
n=0
for line in file("/etc/aoinet/buildslave.list"):
n=n+1
arr=line.split()
if arr[0] == project:
buildtype=arr[1]
portslave=arr[2]
porthttp =arr[3]
break
# configuration parameters:
# 'preconfigure' = shell command as list to run before configure
c = BuildmasterConfig = {}
####### BUILDSLAVES
c['slavePortnum'] = portslave
c['bots'] = [
("yui", "<omitted>"),
("casca","<omitted>"),
<omitted rest>
]
####### CHANGESOURCES
import buildbot.changes.pb
c['sources'] = [buildbot.changes.pb.PBChangeSource()]
####### SCHEDULERS
## configure the Schedulers
from buildbot.scheduler import Scheduler
from buildbot.scheduler import Try_Jobdir
c['schedulers'] = [Scheduler(name="all", branch=None, treeStableTimer=2*60, builderNames=["casca"]),
Scheduler(name="fast", branch=None, treeStableTimer=5, builderNames=["fast"]),
Try_Jobdir("try1", ["fast"], jobdir="jobdir")
]
####### BUILDERS
from buildbot import util
from buildbot.process.base import Build
from buildbot.process import step, factory
class Cabal(factory.BuildFactory):
def __init__(self, source):
assert isinstance(source, tuple)
assert issubclass(source[0], step.BuildStep)
factory.BuildFactory.__init__(self, [source])
if options.has_key('preconfigure'):
self.addStep(step.ShellCommand, command=options['preconfigure'], descriptionDone="preconfigure")
self.addStep(step.Configure, command=["runghc", "Setup", "configure"])
self.addStep(step.Compile, command=["runghc", "Setup", "build"])
self.addStep(step.Test, command=["runghc", "Setup", "test"])
def s(steptype, **kwargs): return (steptype, kwargs)
def f(m):
darcs = s(step.Darcs, repourl = ("http://www.cs.helsinki.fi/u/ekarttun/"+project+"/"+project), mode=m)
return {'cabal': Cabal(source=darcs),
'make' : factory.GNUAutoconf(source=darcs, configure="sh", configureFlags=["configure"])
}[buildtype]
def b(n, s, m):
return ({'name': n,
'slavename': s,
'builddir': n,
'factory': f(m)})
c['builders'] = [b("fast", "yui", "update"),
b("casca","casca", "copy")
]
####### STATUS TARGETS
from buildbot.status import html
c['status'] = [html.Waterfall(http_port=porthttp)]
####### PROJECT IDENTITY
c['projectName'] = project
c['projectURL'] = "http://www.cs.helsinki.fi/u/ekarttun/"+project
c['buildbotURL'] = "http://localhost:"+porthttp+"/"
-------------- next part --------------
System-Daemon cabal 7201 7301
network-alt cabal 7202 7302
JoinHs make 7203 7303
<omitted rest>
-------------- next part --------------
#!/bin/sh
set -e
set -v
HOST=`hostname`
BASE=/var/home/buildslave
umask 077
mkdir /var/home || true
useradd -m -d ${BASE} -s /bin/sh buildslave
chmod 711 /var/home
cd ${BASE}
mkdir .info
echo "Einar Karttunen <ekarttun at cs.helsinki.fi>" > .info/admin
uname -a > .info/host
pwd
echo "Enter password for buildslave ${HOST}"
read PASS
echo $PASS > .buildbot-password
FILES=".info/* .info .buildbot-password"
chown buildslave ${FILES}
(crontab -l && echo "@reboot /etc/aoinet/bin/run-buildslaves") | crontab -
-------------- next part --------------
#!/usr/bin/env perl
my $b = "/var/run/buildbot";
chdir $b;
my $host = `hostname -s`;
chomp($host);
open(F, "/etc/aoinet/buildslave.list") or die "No /etc/aoinet/buildslave.list!";
foreach(<F>) {
@a = split;
my $name = $a[0];
my $sname = $name;
$sname =~ s/\//_/g;
my $type = $a[1];
my $dir = "${b}/${sname}";
if(-d $dir) {
# print "Skipping existing build-master dir: ${dir}\n";
} else {
# print "Creating build-master dir: ${dir}\n";
mkdir $dir;
cmd("buildbot create-master $dir");
chdir $dir or die "chdir $dir failed!";
open(N, ">${dir}/master.cfg") or die "Cannot write ${dir}/master.cfg";
print N "project = '$name'\n";
print N "options = {}\n";
print N "execfile('../global-config')\n";
close N;
jdir("jobdir"); jdir("jobdir/new"); jdir("jobdir/tmp");
rename('Makefile.sample', 'Makefile') || die "Failed with Makefile";
cmd("make start");
}
}
sub jdir($) {
my $d = shift;
mkdir $d || die "mkdir $d failed";
cmd("chgrp buildbot-try $d");
chmod(0770, $d) || die "chmod $d failed!";
}
sub cmd($) {
$c = shift;
`$c && echo ok` || die "$c: failed!\n";
}
print "ok\n"
-------------- next part --------------
#!/usr/bin/env perl
my $b = $ENV{'HOME'};
chdir $b;
my $host = `hostname -s`;
chomp($host);
open(P, "${b}/.buildbot-password") or die "No buildbot password";
my $pass = <P>;
chomp $pass;
close P;
open(F, "/etc/aoinet/buildslave.list") or die "No /etc/aoinet/buildslave.list!";
foreach(<F>) {
s/\//_/g;
@a = split;
my $name = $a[0];
my $port = $a[2];
my $dir = "${b}/${name}";
if(-d $dir) {
# print "Skipping existing build-slave dir: ${dir}\n";
} else {
# print "Creating build-slave dir: ${dir}\n";
my $cmd = "buildbot create-slave $dir buildmaster.aoinet:$port $host '$pass'\n";
print $cmd; `$cmd` or die "Command execution failed!";
chdir $dir or die "chdir $dir failed!";
`rm -r info`;
symlink("../.info", "info") || die "Failed to link in real info directory";
rename('Makefile.sample', 'Makefile') || die "Failed with Makefile";
`make start` || die "Error starting the service";
}
}
print "ok\n"
-------------- next part --------------
A non-text attachment was scrubbed...
Name: darcs_buildbot.py
Type: text/x-python
Size: 3106 bytes
Desc: not available
URL: <http://buildbot.net/pipermail/devel/attachments/20060904/3d715ea7/attachment.py>
More information about the devel
mailing list