[Buildbot-devel] [PATCH] monotone support for buildbot
Nathaniel Smith
njs at pobox.com
Fri Jul 29 10:48:40 UTC 2005
On Fri, Jul 29, 2005 at 03:11:45AM -0700, Nathaniel Smith wrote:
> Attached. This has been in use at http://venge.net:9000 for a while
> now... (disregard all the failing builders, they're slaves that
> haven't been upgraded to a newer version of monotone).
Someday, I will become smart. In the mean time, have a patch.
-- Nathaniel
--
"Lull'd in the countless chambers of the brain,
Our thoughts are link'd by many a hidden chain:
Awake but one, and lo! what myriads rise!
Each stamps its image as the other flies"
-- Ann Ward Radcliffe, The Mysteries of Udolpho
-------------- next part --------------
New file: buildbot/changes/monotone.py
--- /dev/null 2005-06-15 19:13:09.000000000 -0700
+++ buildbot/changes/monotone.py 2005-07-16 00:49:47.000000000 -0700
@@ -0,0 +1,309 @@
+#! /usr/bin/python
+
+import tempfile
+import os
+import os.path
+import time
+from cStringIO import StringIO
+
+from twisted.python import log
+from twisted.application import service
+from twisted.internet import defer, protocol, error, reactor
+from twisted.internet.utils import getProcessOutput, getProcessValue
+from twisted.internet.task import LoopingCall
+
+from buildbot import util
+from buildbot.interfaces import IChangeSource
+from buildbot.changes.changes import Change
+
+class _MTProtocol(protocol.ProcessProtocol):
+
+ def __init__(self, deferred, cmdline):
+ self.cmdline = cmdline
+ self.deferred = deferred
+ self.s = StringIO()
+
+ def errReceived(self, text):
+ log.msg("stderr: %s" % text)
+
+ def outReceived(self, text):
+ log.msg("stdout: %s" % text)
+ self.s.write(text)
+
+ def processEnded(self, reason):
+ log.msg("Command %r exited with value %s" % (self.cmdline, reason))
+ if isinstance(reason.value, error.ProcessDone):
+ self.deferred.callback(self.s.getvalue())
+ else:
+ self.deferred.errback(reason)
+
+class Monotone:
+ """All methods of this class return a Deferred."""
+
+ def __init__(self, bin, db):
+ self.bin = bin
+ self.db = db
+
+ def _run_monotone(self, args):
+ d = defer.Deferred()
+ cmdline = (self.bin, "--db=" + self.db) + tuple(args)
+ p = _MTProtocol(d, cmdline)
+ log.msg("Running command: %r" % (cmdline,))
+ log.msg("wd: %s" % os.getcwd())
+ reactor.spawnProcess(p, self.bin, cmdline)
+ return d
+
+ def _process_revision_list(self, output):
+ if output:
+ return output.strip().split("\n")
+ else:
+ return []
+
+ def get_interface_version(self):
+ d = self._run_monotone(["automate", "interface_version"])
+ d.addCallback(self._process_interface_version)
+ return d
+
+ def _process_interface_version(self, output):
+ return tuple(map(int, output.strip().split(".")))
+
+ def db_init(self):
+ return self._run_monotone(["db", "init"])
+
+ def db_migrate(self):
+ return self._run_monotone(["db", "migrate"])
+
+ def pull(self, server, pattern):
+ return self._run_monotone(["pull", server, pattern])
+
+ def get_revision(self, rid):
+ return self._run_monotone(["cat", "revision", rid])
+
+ def get_heads(self, branch, rcfile=""):
+ cmd = ["automate", "heads", branch]
+ if rcfile:
+ cmd += ["--rcfile=" + rcfile]
+ d = self._run_monotone(cmd)
+ d.addCallback(self._process_revision_list)
+ return d
+
+ def erase_ancestors(self, revs):
+ d = self._run_monotone(["automate", "erase_ancestors"] + revs)
+ d.addCallback(self._process_revision_list)
+ return d
+
+ def ancestry_difference(self, new_rev, old_revs):
+ d = self._run_monotone(["automate", "ancestry_difference", new_rev]
+ + old_revs)
+ d.addCallback(self._process_revision_list)
+ return d
+
+ def descendents(self, rev):
+ d = self._run_monotone(["automate", "descendents", rev])
+ d.addCallback(self._process_revision_list)
+ return d
+
+ def log(self, rev, depth=None):
+ if depth is not None:
+ depth_arg = ["--last=%i" % (depth,)]
+ else:
+ depth_arg = []
+ return self._run_monotone(["log", "-r", rev] + depth_arg)
+
+
+class MonotoneSource(service.Service, util.ComparableMixin):
+ """This source will poll a monotone server for changes and submit them to
+ the change master.
+
+ @param server_addr: monotone server specification (host:portno)
+
+ @param branch: monotone branch to watch
+
+ @param trusted_keys: list of keys whose code you trust
+
+ @param db_path: path to monotone database to pull into
+
+ @param pollinterval: interval in seconds between polls, defaults to 10 minutes
+ @param monotone_exec: path to monotone executable, defaults to "monotone"
+ """
+
+ __implements__ = IChangeSource, service.Service.__implements__
+ compare_attrs = ["server_addr", "trusted_keys", "db_path",
+ "pollinterval", "branch", "monotone_exec"]
+
+ parent = None # filled in when we're added
+ done_revisions = []
+ last_revision = None
+ loop = None
+ d = None
+ tmpfile = None
+ monotone = None
+ volatile = ["loop", "d", "tmpfile", "monotone"]
+
+ def __init__(self, server_addr, branch, trusted_keys, db_path,
+ pollinterval=60 * 10, monotone_exec="monotone"):
+ self.server_addr = server_addr
+ self.branch = branch
+ self.trusted_keys = trusted_keys
+ self.db_path = db_path
+ self.pollinterval = pollinterval
+ self.monotone_exec = monotone_exec
+ self.monotone = Monotone(self.monotone_exec, self.db_path)
+
+ def startService(self):
+ self.loop = LoopingCall(self.start_poll)
+ self.loop.start(self.pollinterval)
+ service.Service.startService(self)
+
+ def stopService(self):
+ self.loop.stop()
+ return service.Service.stopService(self)
+
+ def describe(self):
+ return "monotone_source %s %s" % (self.server_addr,
+ self.branch)
+
+ def start_poll(self):
+ if self.d is not None:
+ log.msg("last poll still in progress, skipping next poll")
+ return
+ log.msg("starting poll")
+ self.d = self._maybe_init_db()
+ self.d.addCallback(self._do_netsync)
+ self.d.addCallback(self._get_changes)
+ self.d.addErrback(self._handle_error)
+
+ def _handle_error(self, failure):
+ log.err(failure)
+ self.d = None
+
+ def _maybe_init_db(self):
+ if not os.path.exists(self.db_path):
+ log.msg("init'ing db")
+ return self.monotone.db_init()
+ else:
+ log.msg("db already exists, migrating")
+ return self.monotone.db_migrate()
+
+ def _do_netsync(self, output):
+ return self.monotone.pull(self.server_addr, self.branch)
+
+ def _get_changes(self, output):
+ d = self._get_new_head()
+ d.addCallback(self._process_new_head)
+ return d
+
+ def _get_new_head(self):
+ # This function returns a deferred that resolves to a good pick of new
+ # head (or None if there is no good new head.)
+
+ # First need to get all new heads...
+ rcfile = """function get_revision_cert_trust(signers, id, name, val)
+ local trusted_signers = { %s }
+ local ts_table = {}
+ for k, v in pairs(trusted_signers) do ts_table[v] = 1 end
+ for k, v in pairs(signers) do
+ if ts_table[v] then
+ return true
+ end
+ end
+ return false
+ end
+ """
+ trusted_list = ", ".join(['"' + key + '"' for key in self.trusted_keys])
+ # mktemp is unsafe, but mkstemp is not 2.2 compatible.
+ tmpfile_name = tempfile.mktemp()
+ f = open(tmpfile_name, "w")
+ f.write(rcfile % trusted_list)
+ f.close()
+ d = self.monotone.get_heads(self.branch, tmpfile_name)
+ d.addCallback(self._find_new_head, tmpfile_name)
+ return d
+
+ def _find_new_head(self, new_heads, tmpfile_name):
+ os.unlink(tmpfile_name)
+ # Now get the old head's descendents...
+ if self.last_revision is not None:
+ d = self.monotone.descendents(self.last_revision)
+ else:
+ d = defer.succeed(new_heads)
+ d.addCallback(self._pick_new_head, new_heads)
+ return d
+
+ def _pick_new_head(self, old_head_descendents, new_heads):
+ for r in new_heads:
+ if r in old_head_descendents:
+ return r
+ return None
+
+ def _process_new_head(self, new_head):
+ if new_head is None:
+ log.msg("No new head")
+ self.d = None
+ return None
+ # Okay, we have a new head; we need to get all the revisions since
+ # then and create change objects for them.
+ # Step 1: simplify set of processed revisions.
+ d = self._simplify_revisions()
+ # Step 2: get the list of new revisions
+ d.addCallback(self._get_new_revisions, new_head)
+ # Step 3: add a change for each
+ d.addCallback(self._add_changes_for_revisions)
+ # Step 4: all done
+ d.addCallback(self._finish_changes, new_head)
+ return d
+
+ def _simplify_revisions(self):
+ d = self.monotone.erase_ancestors(self.done_revisions)
+ d.addCallback(self._reset_done_revisions)
+ return d
+
+ def _reset_done_revisions(self, new_done_revisions):
+ self.done_revisions = new_done_revisions
+ return None
+
+ def _get_new_revisions(self, blah, new_head):
+ if self.done_revisions:
+ return self.monotone.ancestry_difference(new_head,
+ self.done_revisions)
+ else:
+ # Don't force feed the builder with every change since the
+ # beginning of time when it's first started up.
+ return defer.succeed([new_head])
+
+ def _add_changes_for_revisions(self, revs):
+ d = defer.succeed(None)
+ for rid in revs:
+ d.addCallback(self._add_change_for_revision, rid)
+ return d
+
+ def _add_change_for_revision(self, blah, rid):
+ d = self.monotone.log(rid, 1)
+ d.addCallback(self._add_change_from_log, rid)
+ return d
+
+ def _add_change_from_log(self, log, rid):
+ d = self.monotone.get_revision(rid)
+ d.addCallback(self._add_change_from_log_and_revision, log, rid)
+ return d
+
+ def _add_change_from_log_and_revision(self, revision, log, rid):
+ # Stupid way to pull out everything inside quotes (which currently
+ # uniquely identifies filenames inside a changeset).
+ pieces = revision.split('"')
+ files = []
+ for i in range(len(pieces)):
+ if (i % 2) == 1:
+ files.append(pieces[i])
+ # Also pull out author key and date
+ author = "unknown author"
+ pieces = log.split('\n')
+ for p in pieces:
+ if p.startswith("Author:"):
+ author = p.split()[1]
+ self.parent.addChange(Change(author, files, log, revision=rid))
+
+ def _finish_changes(self, blah, new_head):
+ self.done_revisions.append(new_head)
+ self.last_revision = new_head
+ self.d = None
Index: buildbot/process/step.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v
retrieving revision 1.66
diff -u -r1.66 step.py
--- buildbot/process/step.py 17 May 2005 04:40:55 -0000 1.66
+++ buildbot/process/step.py 29 Jul 2005 09:42:41 -0000
@@ -1346,7 +1346,37 @@
self.cmd = LoggedRemoteCommand("bazaar", self.args)
ShellCommand.start(self)
+class Monotone(Source):
+ """Check out a revision from a monotone server at 'server_addr',
+ branch 'branch'. 'revision' specifies which revision id to check
+ out.
+
+ This step will first create a local database, if necessary, and then pull
+ the contents of the server into the database. Then it will do the
+ checkout/update from this database."""
+
+ name = "monotone"
+
+ def __init__(self, server_addr, branch, db_path="monotone.db",
+ monotone="monotone",
+ **kwargs):
+ Source.__init__(self, **kwargs)
+ self.args.update({"server_addr": server_addr,
+ "branch": branch,
+ "db_path": db_path,
+ "monotone": monotone})
+
+ def computeSourceRevision(self, changes):
+ if not changes:
+ return None
+ return changes[-1].revision
+ def startVC(self):
+ slavever = self.slaveVersion("monotone")
+ assert slavever, "slave is too old, does not know about monotone"
+ self.cmd = LoggedRemoteCommand("monotone", self.args)
+ ShellCommand.start(self)
+
class todo_P4(Source):
name = "p4"
Index: buildbot/slave/commands.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v
retrieving revision 1.35
diff -u -r1.35 commands.py
--- buildbot/slave/commands.py 17 May 2005 22:19:18 -0000 1.35
+++ buildbot/slave/commands.py 29 Jul 2005 09:42:42 -0000
@@ -916,6 +916,105 @@
registerSlaveCommand("darcs", Darcs, cvs_ver)
+class Monotone(SourceBase):
+ """Monotone-specific VC operation. In addition to the arguments handled
+ by SourceBase, this command reads the following keys:
+
+ ['server_addr'] (required): the address of the server to pull from
+ ['branch'] (required): the branch the revision is on
+ ['db_path'] (required): the local database path to use
+ ['revision'] (required): the revision to check out
+ ['monotone']: (required): path to monotone executable
+ """
+
+ header = "monotone operation"
+
+ def setup(self, args):
+ SourceBase.setup(self, args)
+ self.server_addr = args["server_addr"]
+ self.branch = args["branch"]
+ self.db_path = args["db_path"]
+ self.revision = args["revision"]
+ self.monotone = args["monotone"]
+ self._made_fulls = False
+ self._pull_timeout = args["timeout"]
+
+ def _makefulls(self):
+ if not self._made_fulls:
+ self.full_db_path = os.path.join(self.builder.basedir, self.db_path)
+ self.full_srcdir = os.path.join(self.builder.basedir, self.srcdir)
+ self._made_fulls = True
+
+ def sourcedirIsUpdateable(self):
+ self._makefulls()
+ if os.path.exists(os.path.join(self.full_srcdir,
+ ".buildbot_patched")):
+ return False
+ return (os.path.isfile(self.full_db_path)
+ and os.path.isdir(os.path.join(self.full_srcdir, "MT")))
+
+ def doVCUpdate(self):
+ return self._withFreshDb(self._doUpdate)
+
+ def _doUpdate(self):
+ # update: possible for mode in ('copy', 'update')
+ command = [self.monotone, "update", "-r", self.revision, "-b", self.branch]
+ c = ShellCommand(self.builder, command, self.full_srcdir,
+ sendRC=False, timeout=self.timeout)
+ self.command = c
+ return c.start()
+
+ def doVCFull(self):
+ return self._withFreshDb(self._doFull)
+
+ def _doFull(self):
+ command = [self.monotone, "--db=" + self.full_db_path,
+ "checkout", "-r", self.revision, "-b", self.branch,
+ self.full_srcdir]
+ c = ShellCommand(self.builder, command, self.builder.basedir,
+ sendRC=False, timeout=self.timeout)
+ self.command = c
+ return c.start()
+
+ def _withFreshDb(self, callback):
+ self._makefulls()
+ # first ensure the db exists and is usable
+ if os.path.isfile(self.full_db_path):
+ # already exists, so run 'db migrate' in case monotone has been
+ # upgraded under us
+ command = [self.monotone, "db", "migrate", "--db=" + self.full_db_path]
+ else:
+ # We'll be doing an initial pull, so up the timeout to 3 hours to
+ # make sure it will have time to complete.
+ self._pull_timeout = max(self._pull_timeout, 3 * 60 * 60)
+ self.sendStatus({"header": "creating database %s\n"
+ % (self.full_db_path,)})
+ command = [self.monotone, "db", "init", "--db=" + self.full_db_path]
+ c = ShellCommand(self.builder, command, self.builder.basedir,
+ sendRC=False, timeout=self.timeout)
+ self.command = c
+ d = c.start()
+ d.addCallback(self._abandonOnFailure)
+ d.addCallback(self._didDbInit)
+ d.addCallback(self._didPull, callback)
+ return d
+
+ def _didDbInit(self, res):
+ command = [self.monotone, "--db=" + self.full_db_path,
+ "pull", "--ticker=dot", self.server_addr, self.branch]
+ c = ShellCommand(self.builder, command, self.builder.basedir,
+ sendRC=False, timeout=self._pull_timeout)
+ self.sendStatus({"header": "pulling %s from %s\n"
+ % (self.branch, self.server_addr)})
+ self.command = c
+ return c.start()
+
+ def _didPull(self, res, callback):
+ return callback()
+
+registerSlaveCommand("monotone", Monotone, cvs_ver)
+
+
class Git(SourceBase):
"""Git specific VC operation. In addition to the arguments
handled by SourceBase, this command reads the following keys:
More information about the devel
mailing list