[Buildbot-commits] buildbot/buildbot/scripts runner.py,1.30,1.31 tryclient.py,1.1,1.2

Brian Warner warner at users.sourceforge.net
Wed Aug 10 08:15:46 UTC 2005


Update of /cvsroot/buildbot/buildbot/buildbot/scripts
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv8539/buildbot/scripts

Modified Files:
	runner.py tryclient.py 
Log Message:
Revision: arch at buildbot.sf.net--2004/buildbot--dev--0--patch-275
Creator:  Brian Warner <warner at lothar.com>

jobdir-style 'try' framework is now 90% implemented

	* buildbot/scripts/runner.py: Add 'buildbot try' command, jobdir
	style is 90% done, still missing status reporting or waiting for
	the buildsets to finish, and it is completely untested.

	* buildbot/trybuild.py: delete file, move contents to ..
	* buildbot/scripts/tryclient.py (getSourceStamp): .. here
	* buildbot/test/test_vc.py: match the move


Index: runner.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/scripts/runner.py,v
retrieving revision 1.30
retrieving revision 1.31
diff -u -d -r1.30 -r1.31
--- runner.py	19 Jul 2005 23:12:01 -0000	1.30
+++ runner.py	10 Aug 2005 08:15:44 -0000	1.31
@@ -560,6 +560,58 @@
     return d
 
 
+class TryOptions(usage.Options):
+    optParameters = [
+        ["connect", "c", None,
+         "how to reach the buildmaster, either 'ssh' or 'pb'"],
+        # for ssh, use --tryhost, --username, and --trydir
+        ["tryhost", None, None,
+         "the hostname (used by ssh) for the buildmaster"],
+        ["trydir", None, None,
+         "the directory (on the tryhost) where tryjobs are deposited"],
+        ["username", "u", None, "Username performing the trial build"],
+        # for PB, use --master, --username, and --passwd
+        ["passwd", None, None, "password for PB authentication"],
+        ["master", "m", None,
+         "Location of the buildmaster's PBListener (host:port)"],
+        
+        ["vc", None, None,
+         "The VC system in use, one of: cvs,svn,tla,baz,darcs"],
+        ]
+
+    optFlags = [
+        ["wait", None, "wait until the builds have finished"],
+        ]
+
+    def getSynopsis(self):
+        return "Usage:    buildbot try [options]"
+
+def doTry(config):
+    from buildbot.scripts import tryclient
+    tryclient.doTry(config)
+
+class TryServerOptions(usage.Options):
+    optParameters = [
+        ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"],
+        ]
+
+def doTryServer(config):
+    jobdir = os.path.expanduser(config["jobdir"])
+    job = sys.stdin.read()
+    # now do a 'safecat'-style write to jobdir/tmp, then move atomically to
+    # jobdir/new . I'm just going to MD5 the contents and prepend a
+    # timestamp.
+    timestring = "%d" % time.time()
+    jobhash = md5.new(job).hexdigest()
+    fn = "%s-%s" % (timestring, jobhash)
+    tmpfile = os.path.join(jobdir, "tmp", fn)
+    newfile = os.path.join(jobdir, "new", fn)
+    f = open(tmpfile, "w")
+    f.write(job)
+    f.close()
+    os.rename(tmpfile, newfile)
+
+
 class Options(usage.Options):
     synopsis = "Usage:    buildbot <command> [command options]"
 
@@ -585,7 +637,12 @@
         ['statusgui', None, StatusClientOptions,
          "Display a small window showing current builder status"],
 
-        # TODO: 'try', 'watch'
+        ['try', None, TryOptions,
+         "Run a build with your local changes"],
+        ['tryserver', None, TryServerOptions,
+         "buildmaster-side 'try' support function, not for users"],
+
+        # TODO: 'watch'
         ]
 
     def opt_version(self):
@@ -630,3 +687,9 @@
         statuslog(so)
     elif command == "statusgui":
         statusgui(so)
+    elif command == "try":
+        doTry(so)
+    elif command == "tryserver":
+        doTryServer(so)
+
+

Index: tryclient.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/scripts/tryclient.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- tryclient.py	10 Aug 2005 07:06:11 -0000	1.1
+++ tryclient.py	10 Aug 2005 08:15:44 -0000	1.2
@@ -1,17 +1,200 @@
-# -*- test-case-name: buildbot.test.test_scheduler -*-
+# -*- test-case-name: buildbot.test.test_scheduler,buildbot.test.test_vc -*-
+
+import os, re
+from twisted.internet import utils, protocol, defer
+
+from buildbot.scripts import runner
+
+class SourceStampExtractor:
+    def __init__(self, treetop):
+        self.treetop = treetop
+    def do(self, cmd):
+        return utils.getProcessOutput(cmd[0], cmd[1:], env=os.environ,
+                                      path=self.treetop)
+    def get(self):
+        d = self.getBaseRevision()
+        d.addCallback(self.getPatch)
+        d.addCallback(self.done)
+        return d
+    def readPatch(self, res, patchlevel):
+        self.patch = (patchlevel, res)
+    def done(self, res):
+        return (self.baserev, self.patch)
+
+class SVNExtractor(SourceStampExtractor):
+    patchlevel = 0
+    def getBaseRevision(self):
+        d = self.do(['svn', "status", "-u"])
+        d.addCallback(self.parseStatus)
+        return d
+    def parseStatus(self, res):
+        # svn shows the base revision for each file that has been modified or
+        # which needs an update. You can update each file to a different
+        # version, so each file is displayed with its individual base
+        # revision. It also shows the repository-wide latest revision number
+        # on the last line ("Status against revision: \d+").
+
+        # for our purposes, we use the latest revision number as the "base"
+        # revision, and get a diff against that. This means we will get
+        # reverse-diffs for local files that need updating, but the resulting
+        # tree will still be correct. The only weirdness is that the baserev
+        # that we emit may be different than the version of the tree that we
+        # first checked out.
+
+        # to do this differently would probably involve scanning the revision
+        # numbers to find the max (or perhaps the min) revision, and then
+        # using that as a base.
+
+        for line in res.split("\n"):
+            m = re.search(r'^Status against revision:\s+(\d+)', line)
+            if m:
+                num = m.group(1)
+                self.baserev = int(num)
+                return
+        raise IndexError("Could not find 'Status against revision' in "
+                         "SVN output: %s" % res)
+    def getPatch(self, res):
+        d = self.do(["svn", "diff", "-r%d" % self.baserev])
+        d.addCallback(self.readPatch, self.patchlevel)
+        return d
+
+class BazExtractor(SourceStampExtractor):
+    def getBaseRevision(self):
+        d = self.do(["baz", "tree-id"])
+        d.addCallback(self.parseStatus)
+        return d
+    def parseStatus(self, res):
+        self.baserev = res.strip()
+    def getPatch(self, res):
+        d = self.do(["baz", "diff"])
+        d.addCallback(self.readPatch, 1)
+        return d
+
+class TlaExtractor(SourceStampExtractor):
+    def getBaseRevision(self):
+        d = self.do(["tla", "logs", "--full", "--reverse"])
+        d.addCallback(self.parseStatus)
+        return d
+    def parseStatus(self, res):
+        lines = res.split("\n")
+        self.baserev = lines[0].strip()
+    def getPatch(self, res):
+        d = self.do(["tla", "changes", "--diffs"])
+        d.addCallback(self.readPatch, 1)
+        return d
+
+class DarcsExtractor(SourceStampExtractor):
+    patchlevel = 1
+    def getBaseRevision(self):
+        d = self.do(["darcs", "changes", "--context"])
+        d.addCallback(self.parseStatus)
+        return d
+    def parseStatus(self, res):
+        self.baserev = res # the whole context file
+    def getPatch(self, res):
+        d = self.do(["darcs", "diff", "-u"])
+        d.addCallback(self.readPatch, self.patchlevel)
+        return d
+
+def getSourceStamp(vctype, treetop):
+    if vctype == "cvs":
+        raise NotImplementedError("CVSExtractor not yet implemented")
+    elif vctype == "svn":
+        e = SVNExtractor(treetop)
+    elif vctype == "baz":
+        e = BazExtractor(treetop)
+    elif vctype == "tla":
+        e = TlaExtractor(treetop)
+    elif vctype == "darcs":
+        e = DarcsExtractor(treetop)
+    else:
+        raise KeyError("unknown vctype '%s'" % vctype)
+    return e.get()
+
 
 def ns(s):
     return "%d:%s," % (len(s), s)
 
-def createJob(bsid, branch, baserev, patchlevel, patch, builderNames):
+def createJob(bsid, branch, baserev, patchlevel, diff, builderNames):
     job = ""
     job += ns("1")
     job += ns(bsid)
     job += ns(branch)
     job += ns(baserev)
     job += ns("%d" % patchlevel)
-    job += ns(patch)
+    job += ns(diff)
     for bn in builderNames:
         job += ns(bn)
     return job
 
+def getTopdir(topfile, start=None):
+    if not start:
+        start = os.getcwd()
+    # TODO: now walk upwards from the current directory until we find this
+    # topfile
+    raise NotImplemented
+
+class RemoteTryPP(protocol.ProcessProtocol):
+    def __init__(self, job):
+        self.job = job
+        self.d = defer.Deferred()
+    def connectionMade(self):
+        self.transport.write(self.job)
+        self.transport.closeStdin()
+    def processEnded(self, status_object):
+        sig = status_object.value.signal
+        rc = status_object.value.exitCode
+        self.d.callback((sig, rc))
+
+class Try:
+    def run(self, config):
+        opts = runner.loadOptions()
+        # common options
+        vc = config.get("vc", opts.get("try_vc"))
+
+        if vc in ("cvs", "svn"):
+            # we need to find the tree-top
+            topdir = config.get("try_topdir", opts.get("try_topdir"))
+            if topdir:
+                treedir = os.path.expanduser(topdir)
+            else:
+                topfile = config.get("try-topfile", opts.get("try_topfile"))
+                treedir = getTopdir(topfile)
+        else:
+            treedir = os.getcwd()
+        ss = getSourceStamp(vc, treedir)
+        builderNames = [] # TODO??
+
+        wait = bool(config.get("wait", opts.get("try_wait")))
+        bsid = "buildsetID" # TODO: generate a random (unique) string
+
+
+        connect = config.get('connect', opts.get('try_connect'))
+        if connect == "ssh":
+            tryhost = config.get("tryhost", opts.get("try_host"))
+            tryuser = config.get("username", opts.get("try_username"))
+            trydir = config.get("trydir", opts.get("try_dir"))
+
+            patchlevel, diff = ss.patch
+            job = createJob(bsid, ss.branch or "", ss.revision, patchlevel, diff,
+                            builderNames)
+            argv = ["ssh", "-l", tryuser, tryhost,
+                    "buildbot", "tryserver", trydir]
+            # now run this command and feed the contents of 'job' into stdin
+
+            pp = RemoteTryPP(job)
+            p = reactor.spawnProcess(pp, argv[0], argv, os.environ)
+            d = pp.d
+            return d
+
+    def done(self, res):
+        reactor.stop()
+
+
+def doTry(config):
+    t = Try()
+    d = t.run(config)
+    d.addBoth(t.done)
+    reactor.run()
+
+    





More information about the Commits mailing list