From warner at users.sourceforge.net Wed Sep 6 00:41:57 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 06 Sep 2006 00:41:57 +0000 Subject: [Buildbot-commits] buildbot/buildbot/scripts runner.py,1.44,1.45 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/scripts In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8799/buildbot/scripts Modified Files: runner.py Log Message: [project @ remove a lot of unused imports, marked by pyflakes] Original author: warner at lothar.com Date: 2006-09-06 00:35:06 Index: runner.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/scripts/runner.py,v retrieving revision 1.44 retrieving revision 1.45 diff -u -d -r1.44 -r1.45 --- runner.py 11 Aug 2006 07:31:03 -0000 1.44 +++ runner.py 6 Sep 2006 00:41:55 -0000 1.45 @@ -2,7 +2,7 @@ # N.B.: don't import anything that might pull in a reactor yet. Some of our # subcommands want to load modules that need the gtk reactor. -import os, os.path, sys, shutil, stat, re, time +import os, sys, stat, re, time from twisted.python import usage, util, runtime # this is mostly just a front-end for mktap, twistd, and kill(1), but in the From warner at users.sourceforge.net Wed Sep 6 00:41:57 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 06 Sep 2006 00:41:57 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave bot.py, 1.19, 1.20 commands.py, 1.61, 1.62 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8799/buildbot/slave Modified Files: bot.py commands.py Log Message: [project @ remove a lot of unused imports, marked by pyflakes] Original author: warner at lothar.com Date: 2006-09-06 00:35:06 Index: bot.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/bot.py,v retrieving revision 1.19 retrieving revision 1.20 diff -u -d -r1.19 -r1.20 --- bot.py 28 Jun 2006 17:42:28 -0000 1.19 +++ bot.py 6 Sep 2006 00:41:55 -0000 1.20 @@ -1,9 +1,9 @@ #! /usr/bin/python -import time, os, os.path, re, sys +import os.path from twisted.spread import pb -from twisted.python import log, usage, failure +from twisted.python import log from twisted.internet import reactor, defer from twisted.application import service, internet from twisted.cred import credentials Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.61 retrieving revision 1.62 diff -u -d -r1.61 -r1.62 --- commands.py 23 Aug 2006 07:13:36 -0000 1.61 +++ commands.py 6 Sep 2006 00:41:55 -0000 1.62 @@ -1,6 +1,6 @@ # -*- test-case-name: buildbot.test.test_slavecommand -*- -import os, os.path, re, signal, shutil, types, time +import os, re, signal, shutil, types, time from stat import ST_CTIME, ST_MTIME, ST_SIZE from twisted.internet.protocol import ProcessProtocol From warner at users.sourceforge.net Wed Sep 6 00:41:57 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 06 Sep 2006 00:41:57 +0000 Subject: [Buildbot-commits] buildbot/buildbot/clients debug.py, 1.3, 1.4 sendchange.py, 1.3, 1.4 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/clients In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8799/buildbot/clients Modified Files: debug.py sendchange.py Log Message: [project @ remove a lot of unused imports, marked by pyflakes] Original author: warner at lothar.com Date: 2006-09-06 00:35:06 Index: debug.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/clients/debug.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- debug.py 28 Oct 2005 00:02:49 -0000 1.3 +++ debug.py 6 Sep 2006 00:41:55 -0000 1.4 @@ -7,7 +7,7 @@ from twisted.spread import pb from twisted.cred import credentials import gtk, gtk.glade, gnome.ui -import os, sys, re +import sys, re class DebugWidget: def __init__(self, master="localhost:8007", passwd="debugpw"): Index: sendchange.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/clients/sendchange.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- sendchange.py 25 Oct 2005 01:26:11 -0000 1.3 +++ sendchange.py 6 Sep 2006 00:41:55 -0000 1.4 @@ -2,7 +2,6 @@ from twisted.spread import pb from twisted.cred import credentials from twisted.internet import reactor -from twisted.python import log class Sender: def __init__(self, master, user): From warner at users.sourceforge.net Wed Sep 6 00:41:56 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 06 Sep 2006 00:41:56 +0000 Subject: [Buildbot-commits] buildbot ChangeLog, 1.723, 1.724 Makefile, 1.20, 1.21 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8799 Modified Files: ChangeLog Makefile Log Message: [project @ remove a lot of unused imports, marked by pyflakes] Original author: warner at lothar.com Date: 2006-09-06 00:35:06 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.723 retrieving revision 1.724 diff -u -d -r1.723 -r1.724 --- ChangeLog 26 Aug 2006 19:50:50 -0000 1.723 +++ ChangeLog 6 Sep 2006 00:41:54 -0000 1.724 @@ -1,3 +1,12 @@ +2006-09-05 Brian Warner + + * lots: run pyflakes, removed a lot of unused imports, changed the + form of some conditional imports to remove false pyflakes + warnings. There are still a number of warnings left, mostly from + imports that are done for their side-effects. + * buildbot/test/test_vc.py: import twisted.python.failure, since it + was missing + 2006-08-26 Brian Warner * buildbot/test/test_locks.py (Unit.testLater): make the tests Index: Makefile =================================================================== RCS file: /cvsroot/buildbot/buildbot/Makefile,v retrieving revision 1.20 retrieving revision 1.21 diff -u -d -r1.20 -r1.21 --- Makefile 23 Aug 2006 04:02:30 -0000 1.20 +++ Makefile 6 Sep 2006 00:41:54 -0000 1.21 @@ -51,3 +51,7 @@ python ./setup.py clean rm -f MANIFEST python ./setup.py sdist + +FLAKES=buildbot +pyflakes: + pyflakes $(FLAKES) From warner at users.sourceforge.net Wed Sep 6 00:41:57 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 06 Sep 2006 00:41:57 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process base.py, 1.69, 1.70 builder.py, 1.36, 1.37 step.py, 1.101, 1.102 step_twisted.py, 1.83, 1.84 step_twisted2.py, 1.3, 1.4 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8799/buildbot/process Modified Files: base.py builder.py step.py step_twisted.py step_twisted2.py Log Message: [project @ remove a lot of unused imports, marked by pyflakes] Original author: warner at lothar.com Date: 2006-09-06 00:35:06 Index: base.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/base.py,v retrieving revision 1.69 retrieving revision 1.70 diff -u -d -r1.69 -r1.70 --- base.py 24 Aug 2006 10:05:16 -0000 1.69 +++ base.py 6 Sep 2006 00:41:55 -0000 1.70 @@ -1,16 +1,13 @@ # -*- test-case-name: buildbot.test.test_step -*- -import types, time -from StringIO import StringIO +import types -from twisted.python import log, components +from twisted.python import log from twisted.python.failure import Failure from twisted.internet import reactor, defer, error -from twisted.spread import pb from buildbot import interfaces from buildbot.twcompat import implements -from buildbot.util import now from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION from buildbot.status.builder import Results, BuildRequestStatus from buildbot.status.progress import BuildProgress Index: builder.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/builder.py,v retrieving revision 1.36 retrieving revision 1.37 diff -u -d -r1.36 -r1.37 --- builder.py 17 Apr 2006 19:22:34 -0000 1.36 +++ builder.py 6 Sep 2006 00:41:55 -0000 1.37 @@ -2,14 +2,13 @@ import warnings -from twisted.python import log, components, failure +from twisted.python import log, components from twisted.spread import pb from twisted.internet import reactor, defer from buildbot import interfaces, sourcestamp from buildbot.twcompat import implements from buildbot.status.progress import Expectations -from buildbot.status import builder from buildbot.util import now from buildbot.process import base Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.101 retrieving revision 1.102 diff -u -d -r1.101 -r1.102 --- step.py 25 Aug 2006 21:54:43 -0000 1.101 +++ step.py 6 Sep 2006 00:41:55 -0000 1.102 @@ -1,6 +1,6 @@ # -*- test-case-name: buildbot.test.test_steps -*- -import time, random, types, re, warnings +import types, re, warnings from email.Utils import formatdate from twisted.internet import reactor, defer, error @@ -14,8 +14,7 @@ from buildbot.twcompat import implements, providedBy from buildbot import util from buildbot.interfaces import BuildSlaveTooOldError -from buildbot.util import now -from buildbot.status import progress, builder +from buildbot.status import progress from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, \ EXCEPTION Index: step_twisted.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step_twisted.py,v retrieving revision 1.83 retrieving revision 1.84 diff -u -d -r1.83 -r1.84 --- step_twisted.py 6 Aug 2006 20:05:38 -0000 1.83 +++ step_twisted.py 6 Sep 2006 00:41:55 -0000 1.84 @@ -1,18 +1,18 @@ # -*- test-case-name: buildbot.test.test_twisted -*- -from twisted.python import log, failure +from twisted.python import log -from buildbot import interfaces from buildbot.status import tests, builder from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED from buildbot.process import step -from buildbot.process.step import BuildStep, ShellCommand +from buildbot.process.step import ShellCommand try: - import cStringIO as StringIO + import cStringIO + StringIO = cStringIO except ImportError: import StringIO -import os, re, types +import re # BuildSteps that are specific to the Twisted source tree Index: step_twisted2.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step_twisted2.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- step_twisted2.py 16 Apr 2006 23:05:40 -0000 1.3 +++ step_twisted2.py 6 Sep 2006 00:41:55 -0000 1.4 @@ -1,15 +1,13 @@ #! /usr/bin/python from buildbot.status import tests -from buildbot.process.step import SUCCESS, FAILURE, WARNINGS, SKIPPED, \ - BuildStep +from buildbot.process.step import SUCCESS, FAILURE, BuildStep from buildbot.process.step_twisted import RunUnitTests from zope.interface import implements from twisted.python import log, failure from twisted.spread import jelly from twisted.pb.tokens import BananaError -from twisted.web.util import formatFailure from twisted.web.html import PRE from twisted.web.error import NoResource From warner at users.sourceforge.net Wed Sep 6 00:41:57 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 06 Sep 2006 00:41:57 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status base.py, 1.4, 1.5 builder.py, 1.85, 1.86 client.py, 1.26, 1.27 html.py, 1.89, 1.90 mail.py, 1.26, 1.27 words.py, 1.46, 1.47 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8799/buildbot/status Modified Files: base.py builder.py client.py html.py mail.py words.py Log Message: [project @ remove a lot of unused imports, marked by pyflakes] Original author: warner at lothar.com Date: 2006-09-06 00:35:06 Index: base.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/base.py,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- base.py 7 Oct 2005 18:45:42 -0000 1.4 +++ base.py 6 Sep 2006 00:41:55 -0000 1.5 @@ -1,14 +1,7 @@ #! /usr/bin/python from twisted.application import service -from twisted.python import components - -try: - from zope.interface import implements -except ImportError: - implements = None -if not hasattr(components, "interface"): - implements = None # nope +from buildbot.twcompat import implements from buildbot.interfaces import IStatusReceiver from buildbot import util, pbutil Index: builder.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/builder.py,v retrieving revision 1.85 retrieving revision 1.86 diff -u -d -r1.85 -r1.86 --- builder.py 20 Aug 2006 22:25:35 -0000 1.85 +++ builder.py 6 Sep 2006 00:41:55 -0000 1.86 @@ -7,9 +7,10 @@ from twisted.internet import reactor, defer from twisted.protocols import basic -import time, os, os.path, shutil, sys, re, urllib +import os, shutil, sys, re, urllib try: - import cPickle as pickle + import cPickle + pickle = cPickle except ImportError: import pickle Index: client.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/client.py,v retrieving revision 1.26 retrieving revision 1.27 diff -u -d -r1.26 -r1.27 --- client.py 8 Jun 2006 21:42:34 -0000 1.26 +++ client.py 6 Sep 2006 00:41:55 -0000 1.27 @@ -2,12 +2,11 @@ from twisted.spread import pb from twisted.python import log, components -from twisted.python.failure import Failure -from twisted.internet import defer, reactor -from twisted.application import service, strports +from twisted.internet import reactor +from twisted.application import strports from twisted.cred import portal, checkers -from buildbot import util, interfaces +from buildbot import interfaces from buildbot.twcompat import Interface, implements from buildbot.status import builder, base from buildbot.changes import changes Index: html.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/html.py,v retrieving revision 1.89 retrieving revision 1.90 diff -u -d -r1.89 -r1.90 --- html.py 20 Aug 2006 22:39:47 -0000 1.89 +++ html.py 6 Sep 2006 00:41:55 -0000 1.90 @@ -3,7 +3,6 @@ from __future__ import generators from twisted.python import log, components -from twisted.python.util import sibpath import urllib, re from twisted.internet import defer, reactor Index: mail.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/mail.py,v retrieving revision 1.26 retrieving revision 1.27 diff -u -d -r1.26 -r1.27 --- mail.py 25 Aug 2006 21:54:51 -0000 1.26 +++ mail.py 6 Sep 2006 00:41:55 -0000 1.27 @@ -13,7 +13,6 @@ import urllib from twisted.internet import defer -from twisted.application import service try: from twisted.mail.smtp import sendmail # Twisted-2.0 except ImportError: Index: words.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/words.py,v retrieving revision 1.46 retrieving revision 1.47 diff -u -d -r1.46 -r1.47 --- words.py 13 Mar 2006 09:29:57 -0000 1.46 +++ words.py 6 Sep 2006 00:41:55 -0000 1.47 @@ -3,7 +3,7 @@ # code to deliver build status through twisted.words (instant messaging # protocols: irc, etc) -import traceback, StringIO, re, shlex +import re, shlex from twisted.internet import protocol, reactor try: From warner at users.sourceforge.net Wed Sep 6 00:41:56 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 06 Sep 2006 00:41:56 +0000 Subject: [Buildbot-commits] buildbot/buildbot buildset.py, 1.4, 1.5 dnotify.py, 1.1, 1.2 master.py, 1.95, 1.96 scheduler.py, 1.17, 1.18 twcompat.py, 1.7, 1.8 util.py, 1.9, 1.10 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8799/buildbot Modified Files: buildset.py dnotify.py master.py scheduler.py twcompat.py util.py Log Message: [project @ remove a lot of unused imports, marked by pyflakes] Original author: warner at lothar.com Date: 2006-09-06 00:35:06 Index: buildset.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/buildset.py,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- buildset.py 12 Mar 2006 11:28:04 -0000 1.4 +++ buildset.py 6 Sep 2006 00:41:54 -0000 1.5 @@ -1,6 +1,4 @@ -from twisted.internet import defer - from buildbot.process import base from buildbot.status import builder Index: dnotify.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/dnotify.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- dnotify.py 12 Feb 2003 21:56:57 -0000 1.1 +++ dnotify.py 6 Sep 2006 00:41:54 -0000 1.2 @@ -73,7 +73,6 @@ def test_dnotify1(): d = DNotify(".") - import time while 1: signal.pause() @@ -95,7 +94,6 @@ d1.callback = fire1 def fire2(): print "foo/ changed!" d2 = DNotify("foo", fire2) - import time while 1: signal.pause() Index: master.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/master.py,v retrieving revision 1.95 retrieving revision 1.96 diff -u -d -r1.95 -r1.96 --- master.py 24 Aug 2006 10:05:16 -0000 1.95 +++ master.py 6 Sep 2006 00:41:54 -0000 1.96 @@ -1,17 +1,19 @@ # -*- test-case-name: buildbot.test.test_run -*- from __future__ import generators -import string, sys, os, time, warnings +import string, os +signal = None try: import signal except ImportError: - signal = None + pass try: - import cPickle as pickle + import cPickle + pickle = cPickle except ImportError: import pickle -from twisted.python import log, usage, components +from twisted.python import log, components from twisted.internet import defer, reactor from twisted.spread import pb from twisted.cred import portal, checkers @@ -19,12 +21,11 @@ from twisted.persisted import styles # sibling imports -from buildbot import util from buildbot.twcompat import implements from buildbot.util import now from buildbot.pbutil import NewCredPerspective from buildbot.process.builder import Builder, IDLE -from buildbot.status.builder import BuilderStatus, SlaveStatus, Status +from buildbot.status.builder import SlaveStatus, Status from buildbot.changes.changes import Change, ChangeMaster from buildbot import interfaces Index: scheduler.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/scheduler.py,v retrieving revision 1.17 retrieving revision 1.18 diff -u -d -r1.17 -r1.18 --- scheduler.py 7 May 2006 18:36:41 -0000 1.17 +++ scheduler.py 6 Sep 2006 00:41:54 -0000 1.18 @@ -10,7 +10,6 @@ from twisted.spread import pb from buildbot import interfaces, buildset, util, pbutil -from buildbot.util import now from buildbot.status import builder from buildbot.twcompat import implements, providedBy from buildbot.sourcestamp import SourceStamp Index: twcompat.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/twcompat.py,v retrieving revision 1.7 retrieving revision 1.8 diff -u -d -r1.7 -r1.8 --- twcompat.py 16 Jul 2006 00:59:14 -0000 1.7 +++ twcompat.py 6 Sep 2006 00:41:54 -0000 1.8 @@ -23,12 +23,13 @@ assert providedBy(obj, IFoo) """ -import os, os.path +import os from twisted.copyright import version from twisted.python import components # does our Twisted use zope.interface? +implements = None if hasattr(components, "interface"): # yes from zope.interface import implements @@ -37,7 +38,6 @@ return iface.providedBy(obj) else: # nope - implements = None from twisted.python.components import Interface providedBy = components.implements @@ -216,7 +216,8 @@ from twisted.internet import reactor, protocol _callProtocolWithDeferred = utils._callProtocolWithDeferred try: - import cStringIO as StringIO + import cStringIO + StringIO = cStringIO except ImportError: import StringIO @@ -279,7 +280,8 @@ result.append(pext) return result +which = _which try: from twisted.python.procutils import which except ImportError: - which = _which + pass Index: util.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/util.py,v retrieving revision 1.9 retrieving revision 1.10 diff -u -d -r1.9 -r1.10 --- util.py 5 Nov 2005 22:52:59 -0000 1.9 +++ util.py 6 Sep 2006 00:41:54 -0000 1.10 @@ -1,7 +1,6 @@ # -*- test-case-name: buildbot.test.test_util -*- from twisted.internet.defer import Deferred -from twisted.python import log from twisted.spread import pb import time From warner at users.sourceforge.net Wed Sep 6 00:41:57 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 06 Sep 2006 00:41:57 +0000 Subject: [Buildbot-commits] buildbot/buildbot/changes base.py, 1.1, 1.2 changes.py, 1.27, 1.28 dnotify.py, 1.3, 1.4 mail.py, 1.20, 1.21 maildir.py, 1.7, 1.8 pb.py, 1.11, 1.12 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/changes In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8799/buildbot/changes Modified Files: base.py changes.py dnotify.py mail.py maildir.py pb.py Log Message: [project @ remove a lot of unused imports, marked by pyflakes] Original author: warner at lothar.com Date: 2006-09-06 00:35:06 Index: base.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/base.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- base.py 17 May 2005 10:14:09 -0000 1.1 +++ base.py 6 Sep 2006 00:41:54 -0000 1.2 @@ -1,7 +1,6 @@ #! /usr/bin/python from twisted.application import service -from twisted.python import components from buildbot.twcompat import implements from buildbot.interfaces import IChangeSource Index: changes.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/changes.py,v retrieving revision 1.27 retrieving revision 1.28 diff -u -d -r1.27 -r1.28 --- changes.py 5 Nov 2005 22:52:59 -0000 1.27 +++ changes.py 6 Sep 2006 00:41:54 -0000 1.28 @@ -1,17 +1,16 @@ #! /usr/bin/python from __future__ import generators -import string, sys, os, os.path, time, types +import sys, os, time try: - import cPickle as pickle + import cPickle + pickle = cPickle except ImportError: import pickle -from twisted.python import log, components +from twisted.python import log from twisted.internet import defer -from twisted.spread import pb from twisted.application import service -from twisted.cred import portal from twisted.web import html from buildbot import interfaces, util Index: dnotify.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/dnotify.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- dnotify.py 4 May 2005 07:04:18 -0000 1.3 +++ dnotify.py 6 Sep 2006 00:41:54 -0000 1.4 @@ -71,7 +71,6 @@ def test_dnotify1(): d = DNotify(".") - import time while 1: signal.pause() @@ -93,7 +92,6 @@ d1.callback = fire1 def fire2(): print "foo/ changed!" d2 = DNotify("foo", fire2) - import time while 1: signal.pause() Index: mail.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/mail.py,v retrieving revision 1.20 retrieving revision 1.21 diff -u -d -r1.20 -r1.21 --- mail.py 19 Jul 2005 19:49:36 -0000 1.20 +++ mail.py 6 Sep 2006 00:41:54 -0000 1.21 @@ -3,7 +3,7 @@ """ Parse various kinds of 'CVS notify' email. """ -import os, os.path, re +import os, re from rfc822 import Message from buildbot import util Index: maildir.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/maildir.py,v retrieving revision 1.7 retrieving revision 1.8 diff -u -d -r1.7 -r1.8 --- maildir.py 2 Oct 2005 23:25:10 -0000 1.7 +++ maildir.py 6 Sep 2006 00:41:54 -0000 1.8 @@ -16,7 +16,7 @@ have_dnotify = 1 except: have_dnotify = 0 -import os, os.path +import os class Maildir: """This is a class which watches a maildir for new messages. Once Index: pb.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/pb.py,v retrieving revision 1.11 retrieving revision 1.12 diff -u -d -r1.11 -r1.12 --- pb.py 21 Aug 2006 03:38:04 -0000 1.11 +++ pb.py 6 Sep 2006 00:41:54 -0000 1.12 @@ -1,8 +1,5 @@ # -*- test-case-name: buildbot.test.test_changes -*- -import os, os.path - -from twisted.application import service from twisted.python import log from buildbot.pbutil import NewCredPerspective From warner at users.sourceforge.net Wed Sep 6 00:41:58 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 06 Sep 2006 00:41:58 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_buildreq.py, 1.4, 1.5 test_changes.py, 1.8, 1.9 test_config.py, 1.36, 1.37 test_control.py, 1.10, 1.11 test_dependencies.py, 1.3, 1.4 test_locks.py, 1.5, 1.6 test_mailparse.py, 1.4, 1.5 test_p4poller.py, 1.2, 1.3 test_run.py, 1.38, 1.39 test_runner.py, 1.12, 1.13 test_shell.py, 1.3, 1.4 test_slavecommand.py, 1.22, 1.23 test_status.py, 1.31, 1.32 test_steps.py, 1.25, 1.26 test_vc.py, 1.66, 1.67 test_web.py, 1.34, 1.35 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv8799/buildbot/test Modified Files: test_buildreq.py test_changes.py test_config.py test_control.py test_dependencies.py test_locks.py test_mailparse.py test_p4poller.py test_run.py test_runner.py test_shell.py test_slavecommand.py test_status.py test_steps.py test_vc.py test_web.py Log Message: [project @ remove a lot of unused imports, marked by pyflakes] Original author: warner at lothar.com Date: 2006-09-06 00:35:06 Index: test_buildreq.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_buildreq.py,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- test_buildreq.py 14 Oct 2005 19:32:55 -0000 1.4 +++ test_buildreq.py 6 Sep 2006 00:41:55 -0000 1.5 @@ -3,7 +3,6 @@ from twisted.trial import unittest from buildbot import buildset, interfaces, sourcestamp -from buildbot.twcompat import maybeWait from buildbot.process import base from buildbot.status import builder from buildbot.changes.changes import Change Index: test_changes.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_changes.py,v retrieving revision 1.8 retrieving revision 1.9 diff -u -d -r1.8 -r1.9 --- test_changes.py 21 Aug 2006 03:38:05 -0000 1.8 +++ test_changes.py 6 Sep 2006 00:41:55 -0000 1.9 @@ -2,7 +2,6 @@ from twisted.trial import unittest from twisted.internet import defer, reactor -from twisted.python import log from buildbot import master from buildbot.twcompat import maybeWait Index: test_config.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_config.py,v retrieving revision 1.36 retrieving revision 1.37 diff -u -d -r1.36 -r1.37 --- test_config.py 24 Jul 2006 22:09:20 -0000 1.36 +++ test_config.py 6 Sep 2006 00:41:55 -0000 1.37 @@ -1,22 +1,22 @@ # -*- test-case-name: buildbot.test.test_config -*- from __future__ import generators -import os, os.path +import os from twisted.trial import unittest -from twisted.python import components, failure +from twisted.python import failure from twisted.internet import defer +cvstoys = None try: import cvstoys from buildbot.changes.freshcvs import FreshCVSSource except ImportError: - cvstoys = None + pass from buildbot.twcompat import providedBy, maybeWait from buildbot.master import BuildMaster from buildbot import scheduler -from buildbot import interfaces as ibb from twisted.application import service, internet from twisted.spread import pb from twisted.web.server import Site @@ -24,15 +24,12 @@ from buildbot.process.builder import Builder from buildbot.process.factory import BasicBuildFactory from buildbot.process import step -from buildbot.status import html, builder, base +from buildbot.status import base +words = None try: from buildbot.status import words except ImportError: - words = None - -import sys -from twisted.python import log -#log.startLogging(sys.stdout) + pass emptyCfg = \ """ Index: test_control.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_control.py,v retrieving revision 1.10 retrieving revision 1.11 diff -u -d -r1.10 -r1.11 --- test_control.py 20 Jun 2006 08:08:17 -0000 1.10 +++ test_control.py 6 Sep 2006 00:41:55 -0000 1.11 @@ -1,18 +1,17 @@ # -*- test-case-name: buildbot.test.test_control -*- -import sys, os, signal, shutil, time, errno +import os from twisted.trial import unittest -from twisted.internet import defer, reactor +from twisted.internet import defer from buildbot import master, interfaces from buildbot.sourcestamp import SourceStamp from buildbot.twcompat import providedBy, maybeWait from buildbot.slave import bot -from buildbot.status import builder from buildbot.status.builder import SUCCESS from buildbot.process import base -from buildbot.test.runutils import SignalMixin, rmtree +from buildbot.test.runutils import rmtree config = """ from buildbot.process import factory, step Index: test_dependencies.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_dependencies.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- test_dependencies.py 3 Jan 2006 09:26:41 -0000 1.3 +++ test_dependencies.py 6 Sep 2006 00:41:55 -0000 1.4 @@ -4,10 +4,6 @@ from twisted.internet import reactor, defer -from buildbot import interfaces -from buildbot.process import step -from buildbot.sourcestamp import SourceStamp -from buildbot.process.base import BuildRequest from buildbot.test.runutils import RunMixin from buildbot.twcompat import maybeWait from buildbot.status import base Index: test_locks.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_locks.py,v retrieving revision 1.5 retrieving revision 1.6 diff -u -d -r1.5 -r1.6 --- test_locks.py 26 Aug 2006 19:50:50 -0000 1.5 +++ test_locks.py 6 Sep 2006 00:41:55 -0000 1.6 @@ -5,7 +5,7 @@ from twisted.trial import unittest from twisted.internet import defer, reactor -from buildbot import interfaces, master +from buildbot import master from buildbot.process import step from buildbot.sourcestamp import SourceStamp from buildbot.process.base import BuildRequest Index: test_mailparse.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_mailparse.py,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- test_mailparse.py 18 May 2005 07:49:30 -0000 1.4 +++ test_mailparse.py 6 Sep 2006 00:41:55 -0000 1.5 @@ -1,6 +1,5 @@ # -*- test-case-name: buildbot.test.test_mailparse -*- -import os.path from twisted.trial import unittest from twisted.python import util from buildbot.changes.mail import parseFreshCVSMail, parseSyncmail Index: test_p4poller.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_p4poller.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- test_p4poller.py 14 Jun 2006 06:49:19 -0000 1.2 +++ test_p4poller.py 6 Sep 2006 00:41:55 -0000 1.3 @@ -1,7 +1,6 @@ -import sys import time -from twisted.python import log, failure +from twisted.python import failure from twisted.internet import defer from twisted.trial import unittest @@ -9,8 +8,6 @@ from buildbot.changes.changes import Change from buildbot.changes.p4poller import P4Source, get_simple_split -#log.startLogging(sys.stderr) - first_p4changes = \ """Change 1 on 2006/04/13 by slamb at testclient 'first rev' """ Index: test_run.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_run.py,v retrieving revision 1.38 retrieving revision 1.39 diff -u -d -r1.38 -r1.39 --- test_run.py 20 Jun 2006 08:08:23 -0000 1.38 +++ test_run.py 6 Sep 2006 00:41:55 -0000 1.39 @@ -2,13 +2,10 @@ from twisted.trial import unittest from twisted.internet import reactor, defer -from twisted.python import log -import sys, os, os.path, shutil, time, errno -#log.startLogging(sys.stderr) +import os from buildbot import master, interfaces from buildbot.sourcestamp import SourceStamp -from buildbot.slave import bot from buildbot.changes import changes from buildbot.status import builder from buildbot.process.base import BuildRequest Index: test_runner.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_runner.py,v retrieving revision 1.12 retrieving revision 1.13 diff -u -d -r1.12 -r1.13 --- test_runner.py 22 Oct 2005 23:22:10 -0000 1.12 +++ test_runner.py 6 Sep 2006 00:41:55 -0000 1.13 @@ -2,8 +2,8 @@ # this file tests the 'buildbot' command, with its various sub-commands from twisted.trial import unittest -from twisted.python import runtime, usage -import os, os.path, shutil, shlex +from twisted.python import usage +import os, shutil, shlex from buildbot.scripts import runner, tryclient Index: test_shell.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_shell.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- test_shell.py 6 Aug 2006 22:37:20 -0000 1.3 +++ test_shell.py 6 Sep 2006 00:41:55 -0000 1.4 @@ -6,7 +6,6 @@ from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python import util -from buildbot.process.step import ShellCommand from buildbot.slave.commands import SlaveShellCommand from buildbot.twcompat import maybeWait from buildbot.test.runutils import SlaveCommandTestBase Index: test_slavecommand.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_slavecommand.py,v retrieving revision 1.22 retrieving revision 1.23 diff -u -d -r1.22 -r1.23 --- test_slavecommand.py 20 Jun 2006 08:08:51 -0000 1.22 +++ test_slavecommand.py 6 Sep 2006 00:41:55 -0000 1.23 @@ -5,12 +5,12 @@ from twisted.python import runtime, failure, util from buildbot.twcompat import maybeWait -import os, re, sys +import os, sys from buildbot.slave import commands SlaveShellCommand = commands.SlaveShellCommand -from buildbot.test.runutils import findDir, SignalMixin, FakeSlaveBuilder +from buildbot.test.runutils import SignalMixin, FakeSlaveBuilder # test slavecommand.py by running the various commands with a fake # SlaveBuilder object that logs the calls to sendUpdate() Index: test_status.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_status.py,v retrieving revision 1.31 retrieving revision 1.32 diff -u -d -r1.31 -r1.32 --- test_status.py 15 Jun 2006 05:47:35 -0000 1.31 +++ test_status.py 6 Sep 2006 00:41:55 -0000 1.32 @@ -10,10 +10,12 @@ from buildbot.process.base import BuildRequest from buildbot.twcompat import implements, providedBy, maybeWait from buildbot.status import builder, base + +mail = None try: from buildbot.status import mail except ImportError: - mail = None + pass from buildbot.status import progress, client # NEEDS COVERAGE from buildbot.test.runutils import RunMixin Index: test_steps.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_steps.py,v retrieving revision 1.25 retrieving revision 1.26 diff -u -d -r1.25 -r1.26 --- test_steps.py 21 Aug 2006 00:43:20 -0000 1.25 +++ test_steps.py 6 Sep 2006 00:41:55 -0000 1.26 @@ -14,7 +14,7 @@ # todo: test batched updates, by invoking remote_update(updates) instead of # statusUpdate(update). Also involves interrupted builds. -import os, sys, time +import os, time from twisted.trial import unittest from twisted.internet import reactor @@ -24,13 +24,11 @@ from buildbot.process import step, base, factory from buildbot.process.step import ShellCommand #, ShellCommands from buildbot.status import builder -from buildbot.test.runutils import RunMixin, rmtree, setupBuildStepStatus +from buildbot.test.runutils import RunMixin, rmtree from buildbot.test.runutils import makeBuildStep from buildbot.twcompat import maybeWait from buildbot.slave import commands -from twisted.python import log -#log.startLogging(sys.stdout) class MyShellCommand(ShellCommand): started = False Index: test_vc.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_vc.py,v retrieving revision 1.66 retrieving revision 1.67 diff -u -d -r1.66 -r1.67 --- test_vc.py 6 Aug 2006 23:30:00 -0000 1.66 +++ test_vc.py 6 Sep 2006 00:41:55 -0000 1.67 @@ -2,11 +2,12 @@ from __future__ import generators -import sys, os, signal, shutil, time, re +import sys, os, time, re from email.Utils import mktime_tz, parsedate_tz from twisted.trial import unittest from twisted.internet import defer, reactor, utils, protocol, error +from twisted.python import failure #defer.Deferred.debug = True @@ -53,7 +54,8 @@ # all-out: bind the listen socket ourselves and pretend to be inetd. try: - import cStringIO as StringIO + import cStringIO + StringIO = cStringIO except ImportError: import StringIO Index: test_web.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_web.py,v retrieving revision 1.34 retrieving revision 1.35 diff -u -d -r1.34 -r1.35 --- test_web.py 23 Aug 2006 04:02:18 -0000 1.34 +++ test_web.py 6 Sep 2006 00:41:55 -0000 1.35 @@ -1,8 +1,7 @@ # -*- test-case-name: buildbot.test.test_web -*- -import sys, os, os.path, time, shutil -from twisted.python import log, components, util -#log.startLogging(sys.stderr) +import os, time, shutil +from twisted.python import components from twisted.trial import unittest from buildbot.test.runutils import RunMixin @@ -11,7 +10,7 @@ from twisted.internet.interfaces import IReactorUNIX from twisted.web import client -from buildbot import master, interfaces, buildset, sourcestamp +from buildbot import master, interfaces, sourcestamp from buildbot.twcompat import providedBy, maybeWait from buildbot.status import html, builder from buildbot.changes.changes import Change From warner at users.sourceforge.net Sat Sep 9 06:20:43 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 09 Sep 2006 06:20:43 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps - New directory Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12149/buildbot/steps Log Message: Directory /cvsroot/buildbot/buildbot/buildbot/steps added to the repository From warner at users.sourceforge.net Sat Sep 9 06:20:52 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 09 Sep 2006 06:20:52 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.102,1.103 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12175/buildbot/process Modified Files: step.py Log Message: [project @ split user-visible BuildSteps out to separate buildbot/steps/* files] Original author: warner at lothar.com Date: 2006-09-08 20:22:35 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.102 retrieving revision 1.103 diff -u -d -r1.102 -r1.103 --- step.py 6 Sep 2006 00:41:55 -0000 1.102 +++ step.py 9 Sep 2006 06:20:50 -0000 1.103 @@ -1,8 +1,5 @@ # -*- test-case-name: buildbot.test.test_steps -*- -import types, re, warnings -from email.Utils import formatdate - from twisted.internet import reactor, defer, error from twisted.protocols import basic from twisted.spread import pb @@ -13,7 +10,6 @@ from buildbot import interfaces from buildbot.twcompat import implements, providedBy [...1227 lines suppressed...] - OFFprogressMetrics = ('output',) - # things to track: number of files compiled, number of directories - # traversed (assuming 'make' is being used) +# legacy compatibility - def createSummary(self, cmd): - # TODO: grep for the characteristic GCC warning/error lines and - # assemble them into a pair of buffers - pass +from buildbot.steps.shell import ShellCommand, WithProperties, TreeSize, Configure, Compile, Test +from buildbot.steps.source import CVS, SVN, Darcs, Git, Arch, Bazaar, Mercurial, P4, P4Sync +from buildbot.steps.dummy import Dummy, FailingDummy, RemoteDummy -class Test(ShellCommand): - name = "test" - warnOnFailure = 1 - description = ["testing"] - descriptionDone = ["test"] - command = ["make", "test"] From warner at users.sourceforge.net Sat Sep 9 06:20:52 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 09 Sep 2006 06:20:52 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_steps.py,1.26,1.27 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12175/buildbot/test Modified Files: test_steps.py Log Message: [project @ split user-visible BuildSteps out to separate buildbot/steps/* files] Original author: warner at lothar.com Date: 2006-09-08 20:22:35 Index: test_steps.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_steps.py,v retrieving revision 1.26 retrieving revision 1.27 diff -u -d -r1.26 -r1.27 --- test_steps.py 6 Sep 2006 00:41:55 -0000 1.26 +++ test_steps.py 9 Sep 2006 06:20:50 -0000 1.27 @@ -340,3 +340,26 @@ d = self.doBuild("quick") return maybeWait(d) + +class ReorgCompatibility(unittest.TestCase): + def testCompat(self): + from buildbot.process.step import LogObserver, LogLineObserver + from buildbot.process.step import RemoteShellCommand + from buildbot.process.step import BuildStep, LoggingBuildStep + from buildbot.process.step import ShellCommand, WithProperties + from buildbot.process.step import TreeSize + from buildbot.process.step import Configure + from buildbot.process.step import Compile + from buildbot.process.step import Test + from buildbot.process.step import CVS + from buildbot.process.step import SVN + from buildbot.process.step import Darcs + from buildbot.process.step import Git + from buildbot.process.step import Arch + from buildbot.process.step import Bazaar + from buildbot.process.step import Mercurial + from buildbot.process.step import P4 + from buildbot.process.step import P4Sync + from buildbot.process.step import Dummy + from buildbot.process.step import FailingDummy + from buildbot.process.step import RemoteDummy From warner at users.sourceforge.net Sat Sep 9 06:20:53 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 09 Sep 2006 06:20:53 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps __init__.py, NONE, 1.1 dummy.py, NONE, 1.1 shell.py, NONE, 1.1 source.py, NONE, 1.1 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12175/buildbot/steps Added Files: __init__.py dummy.py shell.py source.py Log Message: [project @ split user-visible BuildSteps out to separate buildbot/steps/* files] Original author: warner at lothar.com Date: 2006-09-08 20:22:35 --- NEW FILE: __init__.py --- --- NEW FILE: dummy.py --- from twisted.internet import reactor from buildbot.process.step import BuildStep, LoggingBuildStep from buildbot.process.step import LoggedRemoteCommand from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED class Dummy(BuildStep): """I am a dummy no-op step, which runs entirely on the master, and simply waits 5 seconds before finishing with SUCCESS """ haltOnFailure = True name = "dummy" def __init__(self, timeout=5, **kwargs): """ @type timeout: int @param timeout: the number of seconds to delay before completing """ BuildStep.__init__(self, **kwargs) self.timeout = timeout self.timer = None def start(self): self.step_status.setColor("yellow") self.step_status.setText(["delay", "%s secs" % self.timeout]) self.timer = reactor.callLater(self.timeout, self.done) def interrupt(self, reason): if self.timer: self.timer.cancel() self.timer = None self.step_status.setColor("red") self.step_status.setText(["delay", "interrupted"]) self.finished(FAILURE) def done(self): self.step_status.setColor("green") self.finished(SUCCESS) class FailingDummy(Dummy): """I am a dummy no-op step that 'runs' master-side and finishes (with a FAILURE status) after 5 seconds.""" name = "failing dummy" def start(self): self.step_status.setColor("yellow") self.step_status.setText(["boom", "%s secs" % self.timeout]) self.timer = reactor.callLater(self.timeout, self.done) def done(self): self.step_status.setColor("red") self.finished(FAILURE) class RemoteDummy(LoggingBuildStep): """I am a dummy no-op step that runs on the remote side and simply waits 5 seconds before completing with success. See L{buildbot.slave.commands.DummyCommand} """ haltOnFailure = True name = "remote dummy" def __init__(self, timeout=5, **kwargs): """ @type timeout: int @param timeout: the number of seconds to delay """ LoggingBuildStep.__init__(self, **kwargs) self.timeout = timeout self.description = ["remote", "delay", "%s secs" % timeout] def describe(self, done=False): return self.description def start(self): args = {'timeout': self.timeout} cmd = LoggedRemoteCommand("dummy", args) self.startCommand(cmd) --- NEW FILE: shell.py --- # -*- test-case-name: buildbot.test.test_steps -*- # -*- test-case-name: buildbot.test.test_properties -*- import types, re from twisted.python import log from buildbot import util from buildbot.process.step import LoggingBuildStep, RemoteShellCommand from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED class _BuildPropertyDictionary: def __init__(self, build): self.build = build def __getitem__(self, name): p = self.build.getProperty(name) if p is None: p = "" return p class WithProperties(util.ComparableMixin): """This is a marker class, used in ShellCommand's command= argument to indicate that we want to interpolate a build property. """ compare_attrs = ('fmtstring', 'args') def __init__(self, fmtstring, *args): self.fmtstring = fmtstring self.args = args def render(self, build): if self.args: strings = [] for name in self.args: p = build.getProperty(name) if p is None: p = "" strings.append(p) s = self.fmtstring % tuple(strings) else: s = self.fmtstring % _BuildPropertyDictionary(build) return s class ShellCommand(LoggingBuildStep): """I run a single shell command on the buildslave. I return FAILURE if the exit code of that command is non-zero, SUCCESS otherwise. To change this behavior, override my .evaluateCommand method. By default, a failure of this step will mark the whole build as FAILURE. To override this, give me an argument of flunkOnFailure=False . I create a single Log named 'log' which contains the output of the command. To create additional summary Logs, override my .createSummary method. The shell command I run (a list of argv strings) can be provided in several ways: - a class-level .command attribute - a command= parameter to my constructor (overrides .command) - set explicitly with my .setCommand() method (overrides both) @ivar command: a list of argv strings (or WithProperties instances). This will be used by start() to create a RemoteShellCommand instance. @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs of their corresponding logfiles. The contents of the file named FILENAME will be put into a LogFile named NAME, ina something approximating real-time. (note that logfiles= is actually handled by our parent class LoggingBuildStep) """ name = "shell" description = None # set this to a list of short strings to override descriptionDone = None # alternate description when the step is complete command = None # set this to a command, or set in kwargs # logfiles={} # you can also set 'logfiles' to a dictionary, and it # will be merged with any logfiles= argument passed in # to __init__ # override this on a specific ShellCommand if you want to let it fail # without dooming the entire build to a status of FAILURE flunkOnFailure = True def __init__(self, workdir, description=None, descriptionDone=None, command=None, **kwargs): # most of our arguments get passed through to the RemoteShellCommand # that we create, but first strip out the ones that we pass to # BuildStep (like haltOnFailure and friends), and a couple that we # consume ourselves. self.workdir = workdir # required by RemoteShellCommand if description: self.description = description if isinstance(self.description, str): self.description = [self.description] if descriptionDone: self.descriptionDone = descriptionDone if isinstance(self.descriptionDone, str): self.descriptionDone = [self.descriptionDone] if command: self.command = command # pull out the ones that LoggingBuildStep wants, then upcall buildstep_kwargs = {} for k in kwargs.keys()[:]: if k in self.__class__.parms: buildstep_kwargs[k] = kwargs[k] del kwargs[k] LoggingBuildStep.__init__(self, **buildstep_kwargs) # everything left over goes to the RemoteShellCommand kwargs['workdir'] = workdir # including a copy of 'workdir' self.remote_kwargs = kwargs def setCommand(self, command): self.command = command def describe(self, done=False): """Return a list of short strings to describe this step, for the status display. This uses the first few words of the shell command. You can replace this by setting .description in your subclass, or by overriding this method to describe the step better. @type done: boolean @param done: whether the command is complete or not, to improve the way the command is described. C{done=False} is used while the command is still running, so a single imperfect-tense verb is appropriate ('compiling', 'testing', ...) C{done=True} is used when the command has finished, and the default getText() method adds some text, so a simple noun is appropriate ('compile', 'tests' ...) """ if done and self.descriptionDone is not None: return self.descriptionDone if self.description is not None: return self.description words = self.command # TODO: handle WithProperties here if isinstance(words, types.StringTypes): words = words.split() if len(words) < 1: return ["???"] if len(words) == 1: return ["'%s'" % words[0]] if len(words) == 2: return ["'%s" % words[0], "%s'" % words[1]] return ["'%s" % words[0], "%s" % words[1], "...'"] def _interpolateProperties(self, command): # interpolate any build properties into our command if not isinstance(command, (list, tuple)): return command command_argv = [] for argv in command: if isinstance(argv, WithProperties): command_argv.append(argv.render(self.build)) else: command_argv.append(argv) return command_argv def setupEnvironment(self, cmd): # merge in anything from Build.slaveEnvironment . Earlier steps # (perhaps ones which compile libraries or sub-projects that need to # be referenced by later steps) can add keys to # self.build.slaveEnvironment to affect later steps. slaveEnv = self.build.slaveEnvironment if slaveEnv: if cmd.args['env'] is None: cmd.args['env'] = {} cmd.args['env'].update(slaveEnv) # note that each RemoteShellCommand gets its own copy of the # dictionary, so we shouldn't be affecting anyone but ourselves. def checkForOldSlaveAndLogfiles(self): if not self.logfiles: return # doesn't matter if not self.slaveVersionIsOlderThan("shell", "2.1"): return # slave is new enough # this buildslave is too old and will ignore the 'logfiles' # argument. You'll either have to pull the logfiles manually # (say, by using 'cat' in a separate RemoteShellCommand) or # upgrade the buildslave. msg1 = ("Warning: buildslave %s is too old " "to understand logfiles=, ignoring it." % self.getSlaveName()) msg2 = "You will have to pull this logfile (%s) manually." log.msg(msg1) for logname,remotefilename in self.logfiles.items(): newlog = self.addLog(logname) newlog.addHeader(msg1 + "\n") newlog.addHeader(msg2 % remotefilename + "\n") newlog.finish() # now prevent setupLogfiles() from adding them self.logfiles = {} def start(self): # this block is specific to ShellCommands. subclasses that don't need # to set up an argv array, an environment, or extra logfiles= (like # the Source subclasses) can just skip straight to startCommand() command = self._interpolateProperties(self.command) # create the actual RemoteShellCommand instance now kwargs = self.remote_kwargs kwargs['command'] = command kwargs['logfiles'] = self.logfiles cmd = RemoteShellCommand(**kwargs) self.setupEnvironment(cmd) self.checkForOldSlaveAndLogfiles() self.startCommand(cmd) class TreeSize(ShellCommand): name = "treesize" command = ["du", "-s", "."] kb = None def commandComplete(self, cmd): out = cmd.log.getText() m = re.search(r'^(\d+)', out) if m: self.kb = int(m.group(1)) def evaluateCommand(self, cmd): if cmd.rc != 0: return FAILURE if self.kb is None: return WARNINGS # not sure how 'du' could fail, but whatever return SUCCESS def getText(self, cmd, results): if self.kb is not None: return ["treesize", "%d kb" % self.kb] return ["treesize", "unknown"] class Configure(ShellCommand): name = "configure" haltOnFailure = 1 description = ["configuring"] descriptionDone = ["configure"] command = ["./configure"] class Compile(ShellCommand): name = "compile" haltOnFailure = 1 description = ["compiling"] descriptionDone = ["compile"] command = ["make", "all"] OFFprogressMetrics = ('output',) # things to track: number of files compiled, number of directories # traversed (assuming 'make' is being used) def createSummary(self, cmd): # TODO: grep for the characteristic GCC warning/error lines and # assemble them into a pair of buffers pass class Test(ShellCommand): name = "test" warnOnFailure = 1 description = ["testing"] descriptionDone = ["test"] command = ["make", "test"] --- NEW FILE: source.py --- # -*- test-case-name: buildbot.test.test_vc -*- import warnings from email.Utils import formatdate from twisted.python import log from buildbot.process.step import LoggingBuildStep, LoggedRemoteCommand from buildbot.interfaces import BuildSlaveTooOldError from buildbot.status.builder import SKIPPED class Source(LoggingBuildStep): """This is a base class to generate a source tree in the buildslave. Each version control system has a specialized subclass, and is expected to override __init__ and implement computeSourceRevision() and startVC(). The class as a whole builds up the self.args dictionary, then starts a LoggedRemoteCommand with those arguments. """ # if the checkout fails, there's no point in doing anything else haltOnFailure = True notReally = False branch = None # the default branch, should be set in __init__ def __init__(self, workdir, mode='update', alwaysUseLatest=False, timeout=20*60, retry=None, **kwargs): """ @type workdir: string @param workdir: local directory (relative to the Builder's root) where the tree should be placed @type mode: string @param mode: the kind of VC operation that is desired: - 'update': specifies that the checkout/update should be performed directly into the workdir. Each build is performed in the same directory, allowing for incremental builds. This minimizes disk space, bandwidth, and CPU time. However, it may encounter problems if the build process does not handle dependencies properly (if you must sometimes do a 'clean build' to make sure everything gets compiled), or if source files are deleted but generated files can influence test behavior (e.g. python's .pyc files), or when source directories are deleted but generated files prevent CVS from removing them. - 'copy': specifies that the source-controlled workspace should be maintained in a separate directory (called the 'copydir'), using checkout or update as necessary. For each build, a new workdir is created with a copy of the source tree (rm -rf workdir; cp -r copydir workdir). This doubles the disk space required, but keeps the bandwidth low (update instead of a full checkout). A full 'clean' build is performed each time. This avoids any generated-file build problems, but is still occasionally vulnerable to problems such as a CVS repository being manually rearranged (causing CVS errors on update) which are not an issue with a full checkout. - 'clobber': specifies that the working directory should be deleted each time, necessitating a full checkout for each build. This insures a clean build off a complete checkout, avoiding any of the problems described above, but is bandwidth intensive, as the whole source tree must be pulled down for each build. - 'export': is like 'clobber', except that e.g. the 'cvs export' command is used to create the working directory. This command removes all VC metadata files (the CVS/.svn/{arch} directories) from the tree, which is sometimes useful for creating source tarballs (to avoid including the metadata in the tar file). Not all VC systems support export. @type alwaysUseLatest: boolean @param alwaysUseLatest: whether to always update to the most recent available sources for this build. Normally the Source step asks its Build for a list of all Changes that are supposed to go into the build, then computes a 'source stamp' (revision number or timestamp) that will cause exactly that set of changes to be present in the checked out tree. This is turned into, e.g., 'cvs update -D timestamp', or 'svn update -r revnum'. If alwaysUseLatest=True, bypass this computation and always update to the latest available sources for each build. The source stamp helps avoid a race condition in which someone commits a change after the master has decided to start a build but before the slave finishes checking out the sources. At best this results in a build which contains more changes than the buildmaster thinks it has (possibly resulting in the wrong person taking the blame for any problems that result), at worst is can result in an incoherent set of sources (splitting a non-atomic commit) which may not build at all. @type retry: tuple of ints (delay, repeats) (or None) @param retry: if provided, VC update failures are re-attempted up to REPEATS times, with DELAY seconds between each attempt. Some users have slaves with poor connectivity to their VC repository, and they say that up to 80% of their build failures are due to transient network failures that could be handled by simply retrying a couple times. """ LoggingBuildStep.__init__(self, **kwargs) assert mode in ("update", "copy", "clobber", "export") if retry: delay, repeats = retry assert isinstance(repeats, int) assert repeats > 0 self.args = {'mode': mode, 'workdir': workdir, 'timeout': timeout, 'retry': retry, 'patch': None, # set during .start } self.alwaysUseLatest = alwaysUseLatest # Compute defaults for descriptions: description = ["updating"] descriptionDone = ["update"] if mode == "clobber": description = ["checkout"] # because checkingouting takes too much space descriptionDone = ["checkout"] elif mode == "export": description = ["exporting"] descriptionDone = ["export"] self.description = description self.descriptionDone = descriptionDone def describe(self, done=False): if done: return self.descriptionDone return self.description def computeSourceRevision(self, changes): """Each subclass must implement this method to do something more precise than -rHEAD every time. For version control systems that use repository-wide change numbers (SVN, P4), this can simply take the maximum such number from all the changes involved in this build. For systems that do not (CVS), it needs to create a timestamp based upon the latest Change, the Build's treeStableTimer, and an optional self.checkoutDelay value.""" return None def start(self): if self.notReally: log.msg("faking %s checkout/update" % self.name) self.step_status.setColor("green") self.step_status.setText(["fake", self.name, "successful"]) self.addCompleteLog("log", "Faked %s checkout/update 'successful'\n" \ % self.name) return SKIPPED # what source stamp would this build like to use? s = self.build.getSourceStamp() # if branch is None, then use the Step's "default" branch branch = s.branch or self.branch # if revision is None, use the latest sources (-rHEAD) revision = s.revision if not revision and not self.alwaysUseLatest: revision = self.computeSourceRevision(s.changes) # if patch is None, then do not patch the tree after checkout # 'patch' is None or a tuple of (patchlevel, diff) patch = s.patch self.startVC(branch, revision, patch) def commandComplete(self, cmd): got_revision = None if cmd.updates.has_key("got_revision"): got_revision = cmd.updates["got_revision"][-1] self.setProperty("got_revision", got_revision) class CVS(Source): """I do CVS checkout/update operations. Note: if you are doing anonymous/pserver CVS operations, you will need to manually do a 'cvs login' on each buildslave before the slave has any hope of success. XXX: fix then, take a cvs password as an argument and figure out how to do a 'cvs login' on each build """ name = "cvs" #progressMetrics = ('output',) # # additional things to track: update gives one stderr line per directory # (starting with 'cvs server: Updating ') (and is fairly stable if files # is empty), export gives one line per directory (starting with 'cvs # export: Updating ') and another line per file (starting with U). Would # be nice to track these, requires grepping LogFile data for lines, # parsing each line. Might be handy to have a hook in LogFile that gets # called with each complete line. def __init__(self, cvsroot, cvsmodule, global_options=[], branch=None, checkoutDelay=None, login=None, clobber=0, export=0, copydir=None, **kwargs): """ @type cvsroot: string @param cvsroot: CVS Repository from which the source tree should be obtained. '/home/warner/Repository' for local or NFS-reachable repositories, ':pserver:anon at foo.com:/cvs' for anonymous CVS, 'user at host.com:/cvs' for non-anonymous CVS or CVS over ssh. Lots of possibilities, check the CVS documentation for more. @type cvsmodule: string @param cvsmodule: subdirectory of CVS repository that should be retrieved @type login: string or None @param login: if not None, a string which will be provided as a password to the 'cvs login' command, used when a :pserver: method is used to access the repository. This login is only needed once, but must be run each time (just before the CVS operation) because there is no way for the buildslave to tell whether it was previously performed or not. @type branch: string @param branch: the default branch name, will be used in a '-r' argument to specify which branch of the source tree should be used for this checkout. Defaults to None, which means to use 'HEAD'. @type checkoutDelay: int or None @param checkoutDelay: if not None, the number of seconds to put between the last known Change and the timestamp given to the -D argument. This defaults to exactly half of the parent Build's .treeStableTimer, but it could be set to something else if your CVS change notification has particularly weird latency characteristics. @type global_options: list of strings @param global_options: these arguments are inserted in the cvs command line, before the 'checkout'/'update' command word. See 'cvs --help-options' for a list of what may be accepted here. ['-r'] will make the checked out files read only. ['-r', '-R'] will also assume the repository is read-only (I assume this means it won't use locks to insure atomic access to the ,v files).""" self.checkoutDelay = checkoutDelay self.branch = branch if not kwargs.has_key('mode') and (clobber or export or copydir): # deal with old configs warnings.warn("Please use mode=, not clobber/export/copydir", DeprecationWarning) if export: kwargs['mode'] = "export" elif clobber: kwargs['mode'] = "clobber" elif copydir: kwargs['mode'] = "copy" else: kwargs['mode'] = "update" Source.__init__(self, **kwargs) self.args.update({'cvsroot': cvsroot, 'cvsmodule': cvsmodule, 'global_options': global_options, 'login': login, }) def computeSourceRevision(self, changes): if not changes: return None lastChange = max([c.when for c in changes]) if self.checkoutDelay is not None: when = lastChange + self.checkoutDelay else: lastSubmit = max([r.submittedAt for r in self.build.requests]) when = (lastChange + lastSubmit) / 2 return formatdate(when) def startVC(self, branch, revision, patch): if self.slaveVersionIsOlderThan("cvs", "1.39"): # the slave doesn't know to avoid re-using the same sourcedir # when the branch changes. We have no way of knowing which branch # the last build used, so if we're using a non-default branch and # either 'update' or 'copy' modes, it is safer to refuse to # build, and tell the user they need to upgrade the buildslave. if (branch != self.branch and self.args['mode'] in ("update", "copy")): m = ("This buildslave (%s) does not know about multiple " "branches, and using mode=%s would probably build the " "wrong tree. " "Refusing to build. Please upgrade the buildslave to " "buildbot-0.7.0 or newer." % (self.build.slavename, self.args['mode'])) log.msg(m) raise BuildSlaveTooOldError(m) if branch is None: branch = "HEAD" self.args['branch'] = branch self.args['revision'] = revision self.args['patch'] = patch if self.args['branch'] == "HEAD" and self.args['revision']: # special case. 'cvs update -r HEAD -D today' gives no files # TODO: figure out why, see if it applies to -r BRANCH self.args['branch'] = None # deal with old slaves warnings = [] slavever = self.slaveVersion("cvs", "old") if slavever == "old": # 0.5.0 if self.args['mode'] == "export": self.args['export'] = 1 elif self.args['mode'] == "clobber": self.args['clobber'] = 1 elif self.args['mode'] == "copy": self.args['copydir'] = "source" self.args['tag'] = self.args['branch'] assert not self.args['patch'] # 0.5.0 slave can't do patch cmd = LoggedRemoteCommand("cvs", self.args) self.startCommand(cmd, warnings) class SVN(Source): """I perform Subversion checkout/update operations.""" name = 'svn' def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, directory=None, **kwargs): """ @type svnurl: string @param svnurl: the URL which points to the Subversion server, combining the access method (HTTP, ssh, local file), the repository host/port, the repository path, the sub-tree within the repository, and the branch to check out. Using C{svnurl} does not enable builds of alternate branches: use C{baseURL} to enable this. Use exactly one of C{svnurl} and C{baseURL}. @param baseURL: if branches are enabled, this is the base URL to which a branch name will be appended. It should probably end in a slash. Use exactly one of C{svnurl} and C{baseURL}. @param defaultBranch: if branches are enabled, this is the branch to use if the Build does not specify one explicitly. It will simply be appended to C{baseURL} and the result handed to the SVN command. """ if not kwargs.has_key('workdir') and directory is not None: # deal with old configs warnings.warn("Please use workdir=, not directory=", DeprecationWarning) kwargs['workdir'] = directory self.svnurl = svnurl self.baseURL = baseURL self.branch = defaultBranch Source.__init__(self, **kwargs) if not svnurl and not baseURL: raise ValueError("you must use exactly one of svnurl and baseURL") def computeSourceRevision(self, changes): if not changes: return None lastChange = max([int(c.revision) for c in changes]) return lastChange def startVC(self, branch, revision, patch): # handle old slaves warnings = [] slavever = self.slaveVersion("svn", "old") if not slavever: m = "slave does not have the 'svn' command" raise BuildSlaveTooOldError(m) if self.slaveVersionIsOlderThan("svn", "1.39"): # the slave doesn't know to avoid re-using the same sourcedir # when the branch changes. We have no way of knowing which branch # the last build used, so if we're using a non-default branch and # either 'update' or 'copy' modes, it is safer to refuse to # build, and tell the user they need to upgrade the buildslave. if (branch != self.branch and self.args['mode'] in ("update", "copy")): m = ("This buildslave (%s) does not know about multiple " "branches, and using mode=%s would probably build the " "wrong tree. " "Refusing to build. Please upgrade the buildslave to " "buildbot-0.7.0 or newer." % (self.build.slavename, self.args['mode'])) raise BuildSlaveTooOldError(m) if slavever == "old": # 0.5.0 compatibility if self.args['mode'] in ("clobber", "copy"): # TODO: use some shell commands to make up for the # deficiency, by blowing away the old directory first (thus # forcing a full checkout) warnings.append("WARNING: this slave can only do SVN updates" ", not mode=%s\n" % self.args['mode']) log.msg("WARNING: this slave only does mode=update") if self.args['mode'] == "export": raise BuildSlaveTooOldError("old slave does not have " "mode=export") self.args['directory'] = self.args['workdir'] if revision is not None: # 0.5.0 can only do HEAD. We have no way of knowing whether # the requested revision is HEAD or not, and for # slowly-changing trees this will probably do the right # thing, so let it pass with a warning m = ("WARNING: old slave can only update to HEAD, not " "revision=%s" % revision) log.msg(m) warnings.append(m + "\n") revision = "HEAD" # interprets this key differently if patch: raise BuildSlaveTooOldError("old slave can't do patch") if self.svnurl: assert not branch # we need baseURL= to use branches self.args['svnurl'] = self.svnurl else: self.args['svnurl'] = self.baseURL + branch self.args['revision'] = revision self.args['patch'] = patch revstuff = [] if branch is not None and branch != self.branch: revstuff.append("[branch]") if revision is not None: revstuff.append("r%s" % revision) self.description.extend(revstuff) self.descriptionDone.extend(revstuff) cmd = LoggedRemoteCommand("svn", self.args) self.startCommand(cmd, warnings) class Darcs(Source): """Check out a source tree from a Darcs repository at 'repourl'. To the best of my knowledge, Darcs has no concept of file modes. This means the eXecute-bit will be cleared on all source files. As a result, you may need to invoke configuration scripts with something like: C{s(step.Configure, command=['/bin/sh', './configure'])} """ name = "darcs" def __init__(self, repourl=None, baseURL=None, defaultBranch=None, **kwargs): """ @type repourl: string @param repourl: the URL which points at the Darcs repository. This is used as the default branch. Using C{repourl} does not enable builds of alternate branches: use C{baseURL} to enable this. Use either C{repourl} or C{baseURL}, not both. @param baseURL: if branches are enabled, this is the base URL to which a branch name will be appended. It should probably end in a slash. Use exactly one of C{repourl} and C{baseURL}. @param defaultBranch: if branches are enabled, this is the branch to use if the Build does not specify one explicitly. It will simply be appended to C{baseURL} and the result handed to the 'darcs pull' command. """ self.repourl = repourl self.baseURL = baseURL self.branch = defaultBranch Source.__init__(self, **kwargs) assert kwargs['mode'] != "export", \ "Darcs does not have an 'export' mode" if (not repourl and not baseURL) or (repourl and baseURL): raise ValueError("you must provide exactly one of repourl and" " baseURL") def startVC(self, branch, revision, patch): slavever = self.slaveVersion("darcs") if not slavever: m = "slave is too old, does not know about darcs" raise BuildSlaveTooOldError(m) if self.slaveVersionIsOlderThan("darcs", "1.39"): if revision: # TODO: revisit this once we implement computeSourceRevision m = "0.6.6 slaves can't handle args['revision']" raise BuildSlaveTooOldError(m) # the slave doesn't know to avoid re-using the same sourcedir # when the branch changes. We have no way of knowing which branch # the last build used, so if we're using a non-default branch and # either 'update' or 'copy' modes, it is safer to refuse to # build, and tell the user they need to upgrade the buildslave. if (branch != self.branch and self.args['mode'] in ("update", "copy")): m = ("This buildslave (%s) does not know about multiple " "branches, and using mode=%s would probably build the " "wrong tree. " "Refusing to build. Please upgrade the buildslave to " "buildbot-0.7.0 or newer." % (self.build.slavename, self.args['mode'])) raise BuildSlaveTooOldError(m) if self.repourl: assert not branch # we need baseURL= to use branches self.args['repourl'] = self.repourl else: self.args['repourl'] = self.baseURL + branch self.args['revision'] = revision self.args['patch'] = patch revstuff = [] if branch is not None and branch != self.branch: revstuff.append("[branch]") self.description.extend(revstuff) self.descriptionDone.extend(revstuff) cmd = LoggedRemoteCommand("darcs", self.args) self.startCommand(cmd) class Git(Source): """Check out a source tree from a git repository 'repourl'.""" name = "git" def __init__(self, repourl, **kwargs): """ @type repourl: string @param repourl: the URL which points at the git repository """ self.branch = None # TODO Source.__init__(self, **kwargs) self.args['repourl'] = repourl def startVC(self, branch, revision, patch): self.args['branch'] = branch self.args['revision'] = revision self.args['patch'] = patch slavever = self.slaveVersion("git") if not slavever: raise BuildSlaveTooOldError("slave is too old, does not know " "about git") cmd = LoggedRemoteCommand("git", self.args) self.startCommand(cmd) class Arch(Source): """Check out a source tree from an Arch repository named 'archive' available at 'url'. 'version' specifies which version number (development line) will be used for the checkout: this is mostly equivalent to a branch name. This version uses the 'tla' tool to do the checkout, to use 'baz' see L{Bazaar} instead. """ name = "arch" # TODO: slaves >0.6.6 will accept args['build-config'], so use it def __init__(self, url, version, archive=None, **kwargs): """ @type url: string @param url: the Arch coordinates of the repository. This is typically an http:// URL, but could also be the absolute pathname of a local directory instead. @type version: string @param version: the category--branch--version to check out. This is the default branch. If a build specifies a different branch, it will be used instead of this. @type archive: string @param archive: The archive name. If provided, it must match the one that comes from the repository. If not, the repository's default will be used. """ self.branch = version Source.__init__(self, **kwargs) self.args.update({'url': url, 'archive': archive, }) def computeSourceRevision(self, changes): # in Arch, fully-qualified revision numbers look like: # arch at buildbot.sourceforge.net--2004/buildbot--dev--0--patch-104 # For any given builder, all of this is fixed except the patch-104. # The Change might have any part of the fully-qualified string, so we # just look for the last part. We return the "patch-NN" string. if not changes: return None lastChange = None for c in changes: if not c.revision: continue if c.revision.endswith("--base-0"): rev = 0 else: i = c.revision.rindex("patch") rev = int(c.revision[i+len("patch-"):]) lastChange = max(lastChange, rev) if lastChange is None: return None if lastChange == 0: return "base-0" return "patch-%d" % lastChange def checkSlaveVersion(self, cmd, branch): warnings = [] slavever = self.slaveVersion(cmd) if not slavever: m = "slave is too old, does not know about %s" % cmd raise BuildSlaveTooOldError(m) # slave 1.28 and later understand 'revision' if self.slaveVersionIsOlderThan(cmd, "1.28"): if not self.alwaysUseLatest: # we don't know whether our requested revision is the latest # or not. If the tree does not change very quickly, this will # probably build the right thing, so emit a warning rather # than refuse to build at all m = "WARNING, buildslave is too old to use a revision" log.msg(m) warnings.append(m + "\n") if self.slaveVersionIsOlderThan(cmd, "1.39"): # the slave doesn't know to avoid re-using the same sourcedir # when the branch changes. We have no way of knowing which branch # the last build used, so if we're using a non-default branch and # either 'update' or 'copy' modes, it is safer to refuse to # build, and tell the user they need to upgrade the buildslave. if (branch != self.branch and self.args['mode'] in ("update", "copy")): m = ("This buildslave (%s) does not know about multiple " "branches, and using mode=%s would probably build the " "wrong tree. " "Refusing to build. Please upgrade the buildslave to " "buildbot-0.7.0 or newer." % (self.build.slavename, self.args['mode'])) log.msg(m) raise BuildSlaveTooOldError(m) return warnings def startVC(self, branch, revision, patch): self.args['version'] = branch self.args['revision'] = revision self.args['patch'] = patch warnings = self.checkSlaveVersion("arch", branch) revstuff = [] if branch is not None and branch != self.branch: revstuff.append("[branch]") if revision is not None: revstuff.append("patch%s" % revision) self.description.extend(revstuff) self.descriptionDone.extend(revstuff) cmd = LoggedRemoteCommand("arch", self.args) self.startCommand(cmd, warnings) class Bazaar(Arch): """Bazaar is an alternative client for Arch repositories. baz is mostly compatible with tla, but archive registration is slightly different.""" # TODO: slaves >0.6.6 will accept args['build-config'], so use it def __init__(self, url, version, archive, **kwargs): """ @type url: string @param url: the Arch coordinates of the repository. This is typically an http:// URL, but could also be the absolute pathname of a local directory instead. @type version: string @param version: the category--branch--version to check out @type archive: string @param archive: The archive name (required). This must always match the one that comes from the repository, otherwise the buildslave will attempt to get sources from the wrong archive. """ self.branch = version Source.__init__(self, **kwargs) self.args.update({'url': url, 'archive': archive, }) def startVC(self, branch, revision, patch): self.args['version'] = branch self.args['revision'] = revision self.args['patch'] = patch warnings = self.checkSlaveVersion("bazaar", branch) revstuff = [] if branch is not None and branch != self.branch: revstuff.append("[branch]") if revision is not None: revstuff.append("patch%s" % revision) self.description.extend(revstuff) self.descriptionDone.extend(revstuff) cmd = LoggedRemoteCommand("bazaar", self.args) self.startCommand(cmd, warnings) class Mercurial(Source): """Check out a source tree from a mercurial repository 'repourl'.""" name = "hg" def __init__(self, repourl=None, baseURL=None, defaultBranch=None, **kwargs): """ @type repourl: string @param repourl: the URL which points at the Mercurial repository. This is used as the default branch. Using C{repourl} does not enable builds of alternate branches: use C{baseURL} to enable this. Use either C{repourl} or C{baseURL}, not both. @param baseURL: if branches are enabled, this is the base URL to which a branch name will be appended. It should probably end in a slash. Use exactly one of C{repourl} and C{baseURL}. @param defaultBranch: if branches are enabled, this is the branch to use if the Build does not specify one explicitly. It will simply be appended to C{baseURL} and the result handed to the 'hg clone' command. """ self.repourl = repourl self.baseURL = baseURL self.branch = defaultBranch Source.__init__(self, **kwargs) if (not repourl and not baseURL) or (repourl and baseURL): raise ValueError("you must provide exactly one of repourl and" " baseURL") def startVC(self, branch, revision, patch): slavever = self.slaveVersion("hg") if not slavever: raise BuildSlaveTooOldError("slave is too old, does not know " "about hg") if self.repourl: assert not branch # we need baseURL= to use branches self.args['repourl'] = self.repourl else: self.args['repourl'] = self.baseURL + branch self.args['revision'] = revision self.args['patch'] = patch revstuff = [] if branch is not None and branch != self.branch: revstuff.append("[branch]") self.description.extend(revstuff) self.descriptionDone.extend(revstuff) cmd = LoggedRemoteCommand("hg", self.args) self.startCommand(cmd) class P4(Source): """ P4 is a class for accessing perforce revision control""" name = "p4" def __init__(self, p4base, defaultBranch=None, p4port=None, p4user=None, p4passwd=None, p4extra_views=[], p4client='buildbot_%(slave)s_%(builder)s', **kwargs): """ @type p4base: string @param p4base: A view into a perforce depot, typically "//depot/proj/" @type defaultBranch: string @param defaultBranch: Identify a branch to build by default. Perforce is a view based branching system. So, the branch is normally the name after the base. For example, branch=1.0 is view=//depot/proj/1.0/... branch=1.1 is view=//depot/proj/1.1/... @type p4port: string @param p4port: Specify the perforce server to connection in the format :. Example "perforce.example.com:1666" @type p4user: string @param p4user: The perforce user to run the command as. @type p4passwd: string @param p4passwd: The password for the perforce user. @type p4extra_views: list of tuples @param p4extra_views: Extra views to be added to the client that is being used. @type p4client: string @param p4client: The perforce client to use for this buildslave. """ self.branch = defaultBranch Source.__init__(self, **kwargs) self.args['p4port'] = p4port self.args['p4user'] = p4user self.args['p4passwd'] = p4passwd self.args['p4base'] = p4base self.args['p4extra_views'] = p4extra_views self.args['p4client'] = p4client % { 'slave': self.build.slavename, 'builder': self.build.builder.name, } def computeSourceRevision(self, changes): if not changes: return None lastChange = max([int(c.revision) for c in changes]) return lastChange def startVC(self, branch, revision, patch): slavever = self.slaveVersion("p4") assert slavever, "slave is too old, does not know about p4" args = dict(self.args) args['branch'] = branch or self.branch args['revision'] = revision args['patch'] = patch cmd = LoggedRemoteCommand("p4", args) self.startCommand(cmd) class P4Sync(Source): """This is a partial solution for using a P4 source repository. You are required to manually set up each build slave with a useful P4 environment, which means setting various per-slave environment variables, and creating a P4 client specification which maps the right files into the slave's working directory. Once you have done that, this step merely performs a 'p4 sync' to update that workspace with the newest files. Each slave needs the following environment: - PATH: the 'p4' binary must be on the slave's PATH - P4USER: each slave needs a distinct user account - P4CLIENT: each slave needs a distinct client specification You should use 'p4 client' (?) to set up a client view spec which maps the desired files into $SLAVEBASE/$BUILDERBASE/source . """ name = "p4sync" def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs): assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy" self.branch = None Source.__init__(self, **kwargs) self.args['p4port'] = p4port self.args['p4user'] = p4user self.args['p4passwd'] = p4passwd self.args['p4client'] = p4client def computeSourceRevision(self, changes): if not changes: return None lastChange = max([int(c.revision) for c in changes]) return lastChange def startVC(self, branch, revision, patch): slavever = self.slaveVersion("p4sync") assert slavever, "slave is too old, does not know about p4" cmd = LoggedRemoteCommand("p4sync", self.args) self.startCommand(cmd) From warner at users.sourceforge.net Sat Sep 9 06:20:52 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 09 Sep 2006 06:20:52 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.724,1.725 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12175 Modified Files: ChangeLog Log Message: [project @ split user-visible BuildSteps out to separate buildbot/steps/* files] Original author: warner at lothar.com Date: 2006-09-08 20:22:35 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.724 retrieving revision 1.725 diff -u -d -r1.724 -r1.725 --- ChangeLog 6 Sep 2006 00:41:54 -0000 1.724 +++ ChangeLog 9 Sep 2006 06:20:50 -0000 1.725 @@ -1,3 +1,11 @@ +2006-09-08 Brian Warner + + * buildbot/process/step.py: split the user-visible BuildSteps into + separate files, all under buildbot/steps/ + * buildbot/steps/source.py: this holds VC-checkout steps like SVN + * buildbot/steps/shell.py: this holds ShellCommand and friends + * buildbot/steps/dummy.py: this holds the testing steps like Dummy + 2006-09-05 Brian Warner * lots: run pyflakes, removed a lot of unused imports, changed the From warner at users.sourceforge.net Fri Sep 15 14:47:05 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:47:05 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.725,1.726 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20440 Modified Files: ChangeLog Log Message: move more BuildSteps into buildbot/steps/: step_twisted.py and maxq Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.725 retrieving revision 1.726 diff -u -d -r1.725 -r1.726 --- ChangeLog 9 Sep 2006 06:20:50 -0000 1.725 +++ ChangeLog 15 Sep 2006 14:47:03 -0000 1.726 @@ -1,5 +1,13 @@ 2006-09-08 Brian Warner + * buildbot/process/step_twisted.py: move definition to.. + * buildbot/steps/python_twisted.py: .. here, unfortunately python's + relative-import mechanisms prevent this from being named 'twisted' + or 'python/twisted' as I would have preferred. + + * buildbot/process/maxq.py: move definition to.. + * buildbot/steps/maxq.py: .. here, leave a compatibility import + * buildbot/process/step.py: split the user-visible BuildSteps into separate files, all under buildbot/steps/ * buildbot/steps/source.py: this holds VC-checkout steps like SVN From warner at users.sourceforge.net Fri Sep 15 14:47:05 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:47:05 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process maxq.py, 1.5, NONE step_twisted.py, 1.84, NONE Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20440/buildbot/process Removed Files: maxq.py step_twisted.py Log Message: move more BuildSteps into buildbot/steps/: step_twisted.py and maxq --- maxq.py DELETED --- --- step_twisted.py DELETED --- From warner at users.sourceforge.net Fri Sep 15 14:47:06 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:47:06 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps maxq.py, NONE, 1.1 python_twisted.py, NONE, 1.1 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20440/buildbot/steps Added Files: maxq.py python_twisted.py Log Message: move more BuildSteps into buildbot/steps/: step_twisted.py and maxq --- NEW FILE: maxq.py --- from buildbot.steps.shell import ShellCommand from buildbot.status import event, builder class MaxQ(ShellCommand): flunkOnFailure = True name = "maxq" def __init__(self, testdir=None, **kwargs): if not testdir: raise TypeError("please pass testdir") command = 'run_maxq.py %s' % (testdir,) ShellCommand.__init__(self, command=command, **kwargs) def startStatus(self): evt = event.Event("yellow", ['running', 'maxq', 'tests'], files={'log': self.log}) self.setCurrentActivity(evt) def finished(self, rc): self.failures = 0 if rc: self.failures = 1 output = self.log.getAll() self.failures += output.count('\nTEST FAILURE:') result = (builder.SUCCESS, ['maxq']) if self.failures: result = (builder.FAILURE, [str(self.failures), 'maxq', 'failures']) return self.stepComplete(result) def finishStatus(self, result): if self.failures: color = "red" text = ["maxq", "failed"] else: color = "green" text = ['maxq', 'tests'] self.updateCurrentActivity(color=color, text=text) self.finishStatusSummary() self.finishCurrentActivity() --- NEW FILE: python_twisted.py --- # -*- test-case-name: buildbot.test.test_twisted -*- from twisted.python import log from buildbot.status import tests, builder from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED from buildbot.process.step import LogLineObserver, OutputProgressObserver from buildbot.process.step import RemoteShellCommand from buildbot.steps.shell import ShellCommand try: import cStringIO StringIO = cStringIO except ImportError: import StringIO import re # BuildSteps that are specific to the Twisted source tree class HLint(ShellCommand): """I run a 'lint' checker over a set of .xhtml files. Any deviations from recommended style is flagged and put in the output log. This step looks at .changes in the parent Build to extract a list of Lore XHTML files to check.""" name = "hlint" description = ["running", "hlint"] descriptionDone = ["hlint"] warnOnWarnings = True warnOnFailure = True # TODO: track time, but not output warnings = 0 def __init__(self, python=None, **kwargs): ShellCommand.__init__(self, **kwargs) self.python = python def start(self): # create the command htmlFiles = {} for f in self.build.allFiles(): if f.endswith(".xhtml") and not f.startswith("sandbox/"): htmlFiles[f] = 1 # remove duplicates hlintTargets = htmlFiles.keys() hlintTargets.sort() if not hlintTargets: return SKIPPED self.hlintFiles = hlintTargets c = [] if self.python: c.append(self.python) c += ["bin/lore", "-p", "--output", "lint"] + self.hlintFiles self.setCommand(c) # add an extra log file to show the .html files we're checking self.addCompleteLog("files", "\n".join(self.hlintFiles)+"\n") ShellCommand.start(self) def commandComplete(self, cmd): # TODO: remove the 'files' file (a list of .xhtml files that were # submitted to hlint) because it is available in the logfile and # mostly exists to give the user an idea of how long the step will # take anyway). lines = cmd.logs['stdio'].getText().split("\n") warningLines = filter(lambda line:':' in line, lines) if warningLines: self.addCompleteLog("warnings", "".join(warningLines)) warnings = len(warningLines) self.warnings = warnings def evaluateCommand(self, cmd): # warnings are in stdout, rc is always 0, unless the tools break if cmd.rc != 0: return FAILURE if self.warnings: return WARNINGS return SUCCESS def getText2(self, cmd, results): if cmd.rc != 0: return ["hlint"] return ["%d hlin%s" % (self.warnings, self.warnings == 1 and 't' or 'ts')] def countFailedTests(output): # start scanning 10kb from the end, because there might be a few kb of # import exception tracebacks between the total/time line and the errors # line chunk = output[-10000:] lines = chunk.split("\n") lines.pop() # blank line at end # lines[-3] is "Ran NN tests in 0.242s" # lines[-2] is blank # lines[-1] is 'OK' or 'FAILED (failures=1, errors=12)' # or 'FAILED (failures=1)' # or "PASSED (skips=N, successes=N)" (for Twisted-2.0) # there might be other lines dumped here. Scan all the lines. res = {'total': None, 'failures': 0, 'errors': 0, 'skips': 0, 'expectedFailures': 0, 'unexpectedSuccesses': 0, } for l in lines: out = re.search(r'Ran (\d+) tests', l) if out: res['total'] = int(out.group(1)) if (l.startswith("OK") or l.startswith("FAILED ") or l.startswith("PASSED")): # the extra space on FAILED_ is to distinguish the overall # status from an individual test which failed. The lack of a # space on the OK is because it may be printed without any # additional text (if there are no skips,etc) out = re.search(r'failures=(\d+)', l) if out: res['failures'] = int(out.group(1)) out = re.search(r'errors=(\d+)', l) if out: res['errors'] = int(out.group(1)) out = re.search(r'skips=(\d+)', l) if out: res['skips'] = int(out.group(1)) out = re.search(r'expectedFailures=(\d+)', l) if out: res['expectedFailures'] = int(out.group(1)) out = re.search(r'unexpectedSuccesses=(\d+)', l) if out: res['unexpectedSuccesses'] = int(out.group(1)) # successes= is a Twisted-2.0 addition, and is not currently used out = re.search(r'successes=(\d+)', l) if out: res['successes'] = int(out.group(1)) return res class TrialTestCaseCounter(LogLineObserver): _line_re = re.compile(r'^([\w\.]+) \.\.\. \[([^\]]+)\]$') numTests = 0 finished = False def outLineReceived(self, line): # different versions of Twisted emit different per-test lines with # the bwverbose reporter. # 2.0.0: testSlave (buildbot.test.test_runner.Create) ... [OK] # 2.1.0: buildbot.test.test_runner.Create.testSlave ... [OK] # 2.4.0: buildbot.test.test_runner.Create.testSlave ... [OK] # Let's just handle the most recent version, since it's the easiest. if self.finished: return if line.startswith("=" * 40): self.finished = True return m = self._line_re.search(line.strip()) if m: testname, result = m.groups() self.numTests += 1 self.step.setProgress('tests', self.numTests) UNSPECIFIED=() # since None is a valid choice class Trial(ShellCommand): """I run a unit test suite using 'trial', a unittest-like testing framework that comes with Twisted. Trial is used to implement Twisted's own unit tests, and is the unittest-framework of choice for many projects that use Twisted internally. Projects that use trial typically have all their test cases in a 'test' subdirectory of their top-level library directory. I.e. for my package 'petmail', the tests are in 'petmail/test/test_*.py'. More complicated packages (like Twisted itself) may have multiple test directories, like 'twisted/test/test_*.py' for the core functionality and 'twisted/mail/test/test_*.py' for the email-specific tests. To run trial tests, you run the 'trial' executable and tell it where the test cases are located. The most common way of doing this is with a module name. For petmail, I would run 'trial petmail.test' and it would locate all the test_*.py files under petmail/test/, running every test case it could find in them. Unlike the unittest.py that comes with Python, you do not run the test_foo.py as a script; you always let trial do the importing and running. The 'tests' parameter controls which tests trial will run: it can be a string or a list of strings. You can also use a higher-level module name and pass the --recursive flag to trial: this will search recursively within the named module to find all test cases. For large multiple-test-directory projects like Twisted, this means you can avoid specifying all the test directories explicitly. Something like 'trial --recursive twisted' will pick up everything. To find these test cases, you must set a PYTHONPATH that allows something like 'import petmail.test' to work. For packages that don't use a separate top-level 'lib' directory, PYTHONPATH=. will work, and will use the test cases (and the code they are testing) in-place. PYTHONPATH=build/lib or PYTHONPATH=build/lib.$ARCH are also useful when you do a'setup.py build' step first. The 'testpath' attribute of this class controls what PYTHONPATH= is set to. Trial has the ability (through the --testmodule flag) to run only the set of test cases named by special 'test-case-name' tags in source files. We can get the list of changed source files from our parent Build and provide them to trial, thus running the minimal set of test cases needed to cover the Changes. This is useful for quick builds, especially in trees with a lot of test cases. The 'testChanges' parameter controls this feature: if set, it will override 'tests'. The trial executable itself is typically just 'trial' (which is usually found on your $PATH as /usr/bin/trial), but it can be overridden with the 'trial' parameter. This is useful for Twisted's own unittests, which want to use the copy of bin/trial that comes with the sources. (when bin/trial discovers that it is living in a subdirectory named 'Twisted', it assumes it is being run from the source tree and adds that parent directory to PYTHONPATH. Therefore the canonical way to run Twisted's own unittest suite is './bin/trial twisted.test' rather than 'PYTHONPATH=. /usr/bin/trial twisted.test', especially handy when /usr/bin/trial has not yet been installed). To influence the version of python being used for the tests, or to add flags to the command, set the 'python' parameter. This can be a string (like 'python2.2') or a list (like ['python2.3', '-Wall']). Trial creates and switches into a directory named _trial_temp/ before running the tests, and sends the twisted log (which includes all exceptions) to a file named test.log . This file will be pulled up to the master where it can be seen as part of the status output. There are some class attributes which may be usefully overridden by subclasses. 'trialMode' and 'trialArgs' can influence the trial command line. """ name = "trial" progressMetrics = ('output', 'tests', 'test.log') # note: the slash only works on unix buildslaves, of course, but we have # no way to know what the buildslave uses as a separator. TODO: figure # out something clever. logfiles = {"test.log": "_trial_temp/test.log"} # we use test.log to track Progress at the end of __init__() flunkOnFailure = True python = None trial = "trial" trialMode = ["--reporter=bwverbose"] # requires Twisted-2.1.0 or newer # for Twisted-2.0.0 or 1.3.0, use ["-o"] instead trialArgs = [] testpath = UNSPECIFIED # required (but can be None) testChanges = False # TODO: needs better name recurse = False reactor = None randomly = False tests = None # required def __init__(self, reactor=UNSPECIFIED, python=None, trial=None, testpath=UNSPECIFIED, tests=None, testChanges=None, recurse=None, randomly=None, trialMode=None, trialArgs=None, **kwargs): """ @type testpath: string @param testpath: use in PYTHONPATH when running the tests. If None, do not set PYTHONPATH. Setting this to '.' will cause the source files to be used in-place. @type python: string (without spaces) or list @param python: which python executable to use. Will form the start of the argv array that will launch trial. If you use this, you should set 'trial' to an explicit path (like /usr/bin/trial or ./bin/trial). Defaults to None, which leaves it out entirely (running 'trial args' instead of 'python ./bin/trial args'). Likely values are 'python', ['python2.2'], ['python', '-Wall'], etc. @type trial: string @param trial: which 'trial' executable to run. Defaults to 'trial', which will cause $PATH to be searched and probably find /usr/bin/trial . If you set 'python', this should be set to an explicit path (because 'python2.3 trial' will not work). @type trialMode: list of strings @param trialMode: a list of arguments to pass to trial, specifically to set the reporting mode. This defaults to ['-to'] which means 'verbose colorless output' to the trial that comes with Twisted-2.0.x and at least -2.1.0 . Newer versions of Twisted may come with a trial that prefers ['--reporter=bwverbose']. @type trialArgs: list of strings @param trialArgs: a list of arguments to pass to trial, available to turn on any extra flags you like. Defaults to []. @type tests: list of strings @param tests: a list of test modules to run, like ['twisted.test.test_defer', 'twisted.test.test_process']. If this is a string, it will be converted into a one-item list. @type testChanges: boolean @param testChanges: if True, ignore the 'tests' parameter and instead ask the Build for all the files that make up the Changes going into this build. Pass these filenames to trial and ask it to look for test-case-name tags, running just the tests necessary to cover the changes. @type recurse: boolean @param recurse: If True, pass the --recurse option to trial, allowing test cases to be found in deeper subdirectories of the modules listed in 'tests'. This does not appear to be necessary when using testChanges. @type reactor: string @param reactor: which reactor to use, like 'gtk' or 'java'. If not provided, the Twisted's usual platform-dependent default is used. @type randomly: boolean @param randomly: if True, add the --random=0 argument, which instructs trial to run the unit tests in a random order each time. This occasionally catches problems that might be masked when one module always runs before another (like failing to make registerAdapter calls before lookups are done). @type kwargs: dict @param kwargs: parameters. The following parameters are inherited from L{ShellCommand} and may be useful to set: workdir, haltOnFailure, flunkOnWarnings, flunkOnFailure, warnOnWarnings, warnOnFailure, want_stdout, want_stderr, timeout. """ ShellCommand.__init__(self, **kwargs) if python: self.python = python if self.python is not None: if type(self.python) is str: self.python = [self.python] for s in self.python: if " " in s: # this is not strictly an error, but I suspect more # people will accidentally try to use python="python2.3 # -Wall" than will use embedded spaces in a python flag log.msg("python= component '%s' has spaces") log.msg("To add -Wall, use python=['python', '-Wall']") why = "python= value has spaces, probably an error" raise ValueError(why) if trial: self.trial = trial if " " in self.trial: raise ValueError("trial= value has spaces") if trialMode is not None: self.trialMode = trialMode if trialArgs is not None: self.trialArgs = trialArgs if testpath is not UNSPECIFIED: self.testpath = testpath if self.testpath is UNSPECIFIED: raise ValueError("You must specify testpath= (it can be None)") assert isinstance(self.testpath, str) or self.testpath is None if reactor is not UNSPECIFIED: self.reactor = reactor if tests is not None: self.tests = tests if type(self.tests) is str: self.tests = [self.tests] if testChanges is not None: self.testChanges = testChanges #self.recurse = True # not sure this is necessary if not self.testChanges and self.tests is None: raise ValueError("Must either set testChanges= or provide tests=") if recurse is not None: self.recurse = recurse if randomly is not None: self.randomly = randomly # build up most of the command, then stash it until start() command = [] if self.python: command.extend(self.python) command.append(self.trial) command.extend(self.trialMode) if self.recurse: command.append("--recurse") if self.reactor: command.append("--reactor=%s" % reactor) if self.randomly: command.append("--random=0") command.extend(self.trialArgs) self.command = command if self.reactor: self.description = ["testing", "(%s)" % self.reactor] self.descriptionDone = ["tests"] # commandComplete adds (reactorname) to self.text else: self.description = ["testing"] self.descriptionDone = ["tests"] # this counter will feed Progress along the 'test cases' metric self.addLogObserver('stdio', TrialTestCaseCounter()) # this one just measures bytes of output in _trial_temp/test.log self.addLogObserver('test.log', OutputProgressObserver('test.log')) def setupEnvironment(self, cmd): ShellCommand.setupEnvironment(self, cmd) if self.testpath != None: e = cmd.args['env'] if e is None: cmd.args['env'] = {'PYTHONPATH': self.testpath} else: # TODO: somehow, each build causes another copy of # self.testpath to get prepended if e.get('PYTHONPATH', "") == "": e['PYTHONPATH'] = self.testpath else: e['PYTHONPATH'] = self.testpath + ":" + e['PYTHONPATH'] try: p = cmd.args['env']['PYTHONPATH'] if type(p) is not str: log.msg("hey, not a string:", p) assert False except (KeyError, TypeError): # KeyError if args doesn't have ['env'] # KeyError if args['env'] doesn't have ['PYTHONPATH'] # TypeError if args is None pass def start(self): # now that self.build.allFiles() is nailed down, finish building the # command if self.testChanges: for f in self.build.allFiles(): if f.endswith(".py"): self.command.append("--testmodule=%s" % f) else: self.command.extend(self.tests) log.msg("Trial.start: command is", self.command) # if our slave is too old to understand logfiles=, fetch them # manually. This is a fallback for the Twisted buildbot and some old # buildslaves. self._needToPullTestDotLog = False if self.slaveVersionIsOlderThan("shell", "2.1"): log.msg("Trial: buildslave %s is too old to accept logfiles=" % self.getSlaveName()) log.msg(" falling back to 'cat _trial_temp/test.log' instead") self.logfiles = {} self._needToPullTestDotLog = True ShellCommand.start(self) def commandComplete(self, cmd): if not self._needToPullTestDotLog: return self._gotTestDotLog(cmd) # if the buildslave was too old, pull test.log now catcmd = ["cat", "_trial_temp/test.log"] c2 = RemoteShellCommand(command=catcmd, workdir=self.workdir) loog = self.addLog("test.log") c2.useLog(loog, True, logfileName="stdio") self.cmd = c2 # to allow interrupts d = c2.run(self, self.remote) d.addCallback(lambda res: self._gotTestDotLog(cmd)) return d def rtext(self, fmt='%s'): if self.reactor: rtext = fmt % self.reactor return rtext.replace("reactor", "") return "" def _gotTestDotLog(self, cmd): # figure out all status, then let the various hook functions return # different pieces of it # 'cmd' is the original trial command, so cmd.logs['stdio'] is the # trial output. We don't have access to test.log from here. output = cmd.logs['stdio'].getText() counts = countFailedTests(output) total = counts['total'] failures, errors = counts['failures'], counts['errors'] parsed = (total != None) text = [] text2 = "" if cmd.rc == 0: if parsed: results = SUCCESS if total: text += ["%d %s" % \ (total, total == 1 and "test" or "tests"), "passed"] else: text += ["no tests", "run"] else: results = FAILURE text += ["testlog", "unparseable"] text2 = "tests" else: # something failed results = FAILURE if parsed: text.append("tests") if failures: text.append("%d %s" % \ (failures, failures == 1 and "failure" or "failures")) if errors: text.append("%d %s" % \ (errors, errors == 1 and "error" or "errors")) count = failures + errors text2 = "%d tes%s" % (count, (count == 1 and 't' or 'ts')) else: text += ["tests", "failed"] text2 = "tests" if counts['skips']: text.append("%d %s" % \ (counts['skips'], counts['skips'] == 1 and "skip" or "skips")) if counts['expectedFailures']: text.append("%d %s" % \ (counts['expectedFailures'], counts['expectedFailures'] == 1 and "todo" or "todos")) if 0: # TODO results = WARNINGS if not text2: text2 = "todo" if 0: # ignore unexpectedSuccesses for now, but it should really mark # the build WARNING if counts['unexpectedSuccesses']: text.append("%d surprises" % counts['unexpectedSuccesses']) results = WARNINGS if not text2: text2 = "tests" if self.reactor: text.append(self.rtext('(%s)')) if text2: text2 = "%s %s" % (text2, self.rtext('(%s)')) self.results = results self.text = text self.text2 = [text2] def addTestResult(self, testname, results, text, tlog): if self.reactor is not None: testname = (self.reactor,) + testname tr = builder.TestResult(testname, results, text, logs={'log': tlog}) #self.step_status.build.addTestResult(tr) self.build.build_status.addTestResult(tr) def createSummary(self, loog): output = loog.getText() problems = "" sio = StringIO.StringIO(output) warnings = {} while 1: line = sio.readline() if line == "": break if line.find(" exceptions.DeprecationWarning: ") != -1: # no source warning = line # TODO: consider stripping basedir prefix here warnings[warning] = warnings.get(warning, 0) + 1 elif (line.find(" DeprecationWarning: ") != -1 or line.find(" UserWarning: ") != -1): # next line is the source warning = line + sio.readline() warnings[warning] = warnings.get(warning, 0) + 1 elif line.find("Warning: ") != -1: warning = line warnings[warning] = warnings.get(warning, 0) + 1 if line.find("=" * 60) == 0 or line.find("-" * 60) == 0: problems += line problems += sio.read() break if problems: self.addCompleteLog("problems", problems) # now parse the problems for per-test results pio = StringIO.StringIO(problems) pio.readline() # eat the first separator line testname = None done = False while not done: while 1: line = pio.readline() if line == "": done = True break if line.find("=" * 60) == 0: break if line.find("-" * 60) == 0: # the last case has --- as a separator before the # summary counts are printed done = True break if testname is None: # the first line after the === is like: # EXPECTED FAILURE: testLackOfTB (twisted.test.test_failure.FailureTestCase) # SKIPPED: testRETR (twisted.test.test_ftp.TestFTPServer) # FAILURE: testBatchFile (twisted.conch.test.test_sftp.TestOurServerBatchFile) r = re.search(r'^([^:]+): (\w+) \(([\w\.]+)\)', line) if not r: # TODO: cleanup, if there are no problems, # we hit here continue result, name, case = r.groups() testname = tuple(case.split(".") + [name]) results = {'SKIPPED': SKIPPED, 'EXPECTED FAILURE': SUCCESS, 'UNEXPECTED SUCCESS': WARNINGS, 'FAILURE': FAILURE, 'ERROR': FAILURE, 'SUCCESS': SUCCESS, # not reported }.get(result, WARNINGS) text = result.lower().split() loog = line # the next line is all dashes loog += pio.readline() else: # the rest goes into the log loog += line if testname: self.addTestResult(testname, results, text, loog) testname = None if warnings: lines = warnings.keys() lines.sort() self.addCompleteLog("warnings", "".join(lines)) def evaluateCommand(self, cmd): return self.results def getText(self, cmd, results): return self.text def getText2(self, cmd, results): return self.text2 class ProcessDocs(ShellCommand): """I build all docs. This requires some LaTeX packages to be installed. It will result in the full documentation book (dvi, pdf, etc). """ name = "process-docs" warnOnWarnings = 1 command = ["admin/process-docs"] description = ["processing", "docs"] descriptionDone = ["docs"] # TODO: track output and time def __init__(self, **kwargs): """ @type workdir: string @keyword workdir: the workdir to start from: must be the base of the Twisted tree @type results: triple of (int, int, string) @keyword results: [rc, warnings, output] - rc==0 if all files were converted successfully. - warnings is a count of hlint warnings. - output is the verbose output of the command. """ ShellCommand.__init__(self, **kwargs) def createSummary(self, log): output = log.getText() # hlint warnings are of the format: 'WARNING: file:line:col: stuff # latex warnings start with "WARNING: LaTeX Warning: stuff", but # sometimes wrap around to a second line. lines = output.split("\n") warningLines = [] wantNext = False for line in lines: wantThis = wantNext wantNext = False if line.startswith("WARNING: "): wantThis = True wantNext = True if wantThis: warningLines.append(line) if warningLines: self.addCompleteLog("warnings", "\n".join(warningLines) + "\n") self.warnings = len(warningLines) def evaluateCommand(self, cmd): if cmd.rc != 0: return FAILURE if self.warnings: return WARNINGS return SUCCESS def getText(self, cmd, results): if results == SUCCESS: return ["docs", "successful"] if results == WARNINGS: return ["docs", "%d warnin%s" % (self.warnings, self.warnings == 1 and 'g' or 'gs')] if results == FAILURE: return ["docs", "failed"] def getText2(self, cmd, results): if results == WARNINGS: return ["%d do%s" % (self.warnings, self.warnings == 1 and 'c' or 'cs')] return ["docs"] class BuildDebs(ShellCommand): """I build the .deb packages.""" name = "debuild" flunkOnFailure = 1 command = ["debuild", "-uc", "-us"] description = ["building", "debs"] descriptionDone = ["debs"] def __init__(self, **kwargs): """ @type workdir: string @keyword workdir: the workdir to start from (must be the base of the Twisted tree) @type results: double of [int, string] @keyword results: [rc, output]. - rc == 0 if all .debs were created successfully - output: string with any errors or warnings """ ShellCommand.__init__(self, **kwargs) def commandComplete(self, cmd): errors, warnings = 0, 0 output = cmd.logs['stdio'].getText() summary = "" sio = StringIO.StringIO(output) for line in sio.readlines(): if line.find("E: ") == 0: summary += line errors += 1 if line.find("W: ") == 0: summary += line warnings += 1 if summary: self.addCompleteLog("problems", summary) self.errors = errors self.warnings = warnings def evaluateCommand(self, cmd): if cmd.rc != 0: return FAILURE if self.errors: return FAILURE if self.warnings: return WARNINGS return SUCCESS def getText(self, cmd, results): text = ["debuild"] if cmd.rc != 0: text.append("failed") errors, warnings = self.errors, self.warnings if warnings or errors: text.append("lintian:") if warnings: text.append("%d warnin%s" % (warnings, warnings == 1 and 'g' or 'gs')) if errors: text.append("%d erro%s" % (errors, errors == 1 and 'r' or 'rs')) return text def getText2(self, cmd, results): if cmd.rc != 0: return ["debuild"] if self.errors or self.warnings: return ["%d lintian" % (self.errors + self.warnings)] return [] class RemovePYCs(ShellCommand): name = "remove-.pyc" command = 'find . -name "*.pyc" | xargs rm' description = ["removing", ".pyc", "files"] descriptionDone = ["remove", ".pycs"] From warner at users.sourceforge.net Fri Sep 15 14:47:06 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:47:06 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_twisted.py,1.7,1.8 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20440/buildbot/test Modified Files: test_twisted.py Log Message: move more BuildSteps into buildbot/steps/: step_twisted.py and maxq Index: test_twisted.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_twisted.py,v retrieving revision 1.7 retrieving revision 1.8 diff -u -d -r1.7 -r1.8 --- test_twisted.py 15 Jun 2006 05:47:48 -0000 1.7 +++ test_twisted.py 15 Sep 2006 14:47:04 -0000 1.8 @@ -3,8 +3,8 @@ from twisted.trial import unittest from buildbot import interfaces -from buildbot.process import step_twisted -from buildbot.process.step_twisted import countFailedTests, Trial +from buildbot.steps.python_twisted import countFailedTests +from buildbot.steps.python_twisted import Trial, TrialTestCaseCounter from buildbot.status import builder noisy = 0 @@ -140,7 +140,7 @@ def testCounter(self): self.progress = (None,None) - c = step_twisted.TrialTestCaseCounter() + c = TrialTestCaseCounter() c.setStep(self) STDOUT = interfaces.LOG_CHANNEL_STDOUT def add(text): From warner at users.sourceforge.net Fri Sep 15 14:47:29 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:47:29 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process maxq.py, 1.6, 1.7 step_twisted.py, 1.85, 1.86 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20668/buildbot/process Added Files: maxq.py step_twisted.py Log Message: [project @ move more BuildSteps into buildbot/steps/: step_twisted.py and maxq] Original author: warner at lothar.com Date: 2006-09-08 20:44:00 --- NEW FILE: maxq.py --- # legacy compatibility from buildbot.steps.maxq import MaxQ --- NEW FILE: step_twisted.py --- # -*- test-case-name: buildbot.test.test_twisted -*- # legacy compatibility from buildbot.steps.python_twisted import HLint, Trial, ProcessDocs, BuildDebs from buildbot.steps.python_twisted import RemovePYCs From warner at users.sourceforge.net Fri Sep 15 14:47:43 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:47:43 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.726,1.727 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20685 Modified Files: ChangeLog Log Message: [project @ update docs, factories, and tests to use new buildbot/steps/* definitions] Original author: warner at lothar.com Date: 2006-09-08 21:25:42 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.726 retrieving revision 1.727 diff -u -d -r1.726 -r1.727 --- ChangeLog 15 Sep 2006 14:47:03 -0000 1.726 +++ ChangeLog 15 Sep 2006 14:47:41 -0000 1.727 @@ -1,5 +1,10 @@ 2006-09-08 Brian Warner + * docs/buildbot.texinfo: update to match changes + * buildbot/process/factory.py: stop using old definitions + * buildbot/process/process_twisted.py: same + * buildbot/test/test_*.py: same + * buildbot/process/step_twisted.py: move definition to.. * buildbot/steps/python_twisted.py: .. here, unfortunately python's relative-import mechanisms prevent this from being named 'twisted' From warner at users.sourceforge.net Fri Sep 15 14:47:43 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:47:43 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process factory.py, 1.13, 1.14 process_twisted.py, 1.46, 1.47 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20685/buildbot/process Modified Files: factory.py process_twisted.py Log Message: [project @ update docs, factories, and tests to use new buildbot/steps/* definitions] Original author: warner at lothar.com Date: 2006-09-08 21:25:42 Index: factory.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/factory.py,v retrieving revision 1.13 retrieving revision 1.14 diff -u -d -r1.13 -r1.14 --- factory.py 17 Apr 2006 19:22:33 -0000 1.13 +++ factory.py 15 Sep 2006 14:47:41 -0000 1.14 @@ -2,7 +2,9 @@ from buildbot import util from buildbot.process.base import Build -from buildbot.process import step +from buildbot.process.step import BuildStep +from buildbot.steps.source import CVS, SVN +from buildbot.steps.shell import Configure, Compile, Test # deprecated, use BuildFactory.addStep def s(steptype, **kwargs): @@ -46,7 +48,7 @@ compile=["make", "all"], test=["make", "check"]): assert isinstance(source, tuple) - assert issubclass(source[0], step.BuildStep) + assert issubclass(source[0], BuildStep) BuildFactory.__init__(self, [source]) if configure is not None: # we either need to wind up with a string (which will be @@ -61,29 +63,29 @@ else: assert isinstance(configure, (list, tuple)) command = configure + configureFlags - self.addStep(step.Configure, command=command, env=configureEnv) + self.addStep(Configure, command=command, env=configureEnv) if compile is not None: - self.addStep(step.Compile, command=compile) + self.addStep(Compile, command=compile) if test is not None: - self.addStep(step.Test, command=test) + self.addStep(Test, command=test) class CPAN(BuildFactory): def __init__(self, source, perl="perl"): assert isinstance(source, tuple) - assert issubclass(source[0], step.BuildStep) + assert issubclass(source[0], BuildStep) BuildFactory.__init__(self, [source]) - self.addStep(step.Configure, command=[perl, "Makefile.PL"]) - self.addStep(step.Compile, command=["make"]) - self.addStep(step.Test, command=["make", "test"]) + self.addStep(Configure, command=[perl, "Makefile.PL"]) + self.addStep(Compile, command=["make"]) + self.addStep(Test, command=["make", "test"]) class Distutils(BuildFactory): def __init__(self, source, python="python", test=None): assert isinstance(source, tuple) - assert issubclass(source[0], step.BuildStep) + assert issubclass(source[0], BuildStep) BuildFactory.__init__(self, [source]) - self.addStep(step.Compile, command=[python, "./setup.py", "build"]) + self.addStep(Compile, command=[python, "./setup.py", "build"]) if test is not None: - self.addStep(step.Test, command=test) + self.addStep(Test, command=test) class Trial(BuildFactory): """Build a python module that uses distutils and trial. Set 'tests' to @@ -106,7 +108,7 @@ tests=None, useTestCaseNames=False, env=None): BuildFactory.__init__(self, [source]) assert isinstance(source, tuple) - assert issubclass(source[0], step.BuildStep) + assert issubclass(source[0], BuildStep) assert tests or useTestCaseNames, "must use one or the other" if trial is not None: self.trial = trial @@ -115,10 +117,10 @@ if recurse is not None: self.recurse = recurse - from buildbot.process import step_twisted + from buildbot.steps.python_twisted import Trial buildcommand = buildpython + ["./setup.py", "build"] - self.addStep(step.Compile, command=buildcommand, env=env) - self.addStep(step_twisted.Trial, + self.addStep(Compile, command=buildcommand, env=env) + self.addStep(Trial, python=trialpython, trial=self.trial, testpath=testpath, tests=tests, testChanges=useTestCaseNames, @@ -144,7 +146,7 @@ mode = "clobber" if cvsCopy: mode = "copy" - source = s(step.CVS, cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) + source = s(CVS, cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, @@ -158,7 +160,7 @@ compile="make all", test="make check", cvsCopy=False): mode = "update" - source = s(step.CVS, cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) + source = s(CVS, cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, @@ -170,7 +172,7 @@ configure=None, configureEnv={}, compile="make all", test="make check"): - source = s(step.SVN, svnurl=svnurl, mode="update") + source = s(SVN, svnurl=svnurl, mode="update") GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, Index: process_twisted.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/process_twisted.py,v retrieving revision 1.46 retrieving revision 1.47 diff -u -d -r1.46 -r1.47 --- process_twisted.py 21 May 2006 19:51:35 -0000 1.46 +++ process_twisted.py 15 Sep 2006 14:47:41 -0000 1.47 @@ -4,8 +4,8 @@ from buildbot.process.base import Build from buildbot.process.factory import BuildFactory -from buildbot.process import step -from buildbot.process.step_twisted import HLint, ProcessDocs, BuildDebs, \ +from buildbot.steps import shell +from buildbot.steps.python_twisted import HLint, ProcessDocs, BuildDebs, \ Trial, RemovePYCs class TwistedBuild(Build): @@ -47,7 +47,7 @@ self.addStep(RemovePYCs) for p in python: cmd = [p, "setup.py", "build_ext", "-i"] - self.addStep(step.Compile, command=cmd, flunkOnFailure=True) + self.addStep(shell.Compile, command=cmd, flunkOnFailure=True) self.addStep(TwistedTrial, python=p, testChanges=True) class FullTwistedBuildFactory(TwistedBaseFactory): @@ -67,7 +67,7 @@ cmd = (python + compileOpts + ["setup.py", "build_ext"] + compileOpts2 + ["-i"]) - self.addStep(step.Compile, command=cmd, flunkOnFailure=True) + self.addStep(shell.Compile, command=cmd, flunkOnFailure=True) self.addStep(RemovePYCs) self.addStep(TwistedTrial, python=python, randomly=runTestsRandomly) @@ -94,7 +94,7 @@ cmd = (python + compileOpts + ["setup.py", "build_ext"] + compileOpts2 + ["-i"]) - self.addStep(step.Compile, command=cmd, warnOnFailure=True) + self.addStep(shell.Compile, command=cmd, warnOnFailure=True) if reactors == None: reactors = [ From warner at users.sourceforge.net Fri Sep 15 14:47:44 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:47:44 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_config.py, 1.37, 1.38 test_control.py, 1.11, 1.12 test_dependencies.py, 1.4, 1.5 test_locks.py, 1.6, 1.7 test_properties.py, 1.4, 1.5 test_run.py, 1.39, 1.40 test_slaves.py, 1.3, 1.4 test_status.py, 1.32, 1.33 test_steps.py, 1.27, 1.28 test_vc.py, 1.67, 1.68 test_web.py, 1.35, 1.36 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20685/buildbot/test Modified Files: test_config.py test_control.py test_dependencies.py test_locks.py test_properties.py test_run.py test_slaves.py test_status.py test_steps.py test_vc.py test_web.py Log Message: [project @ update docs, factories, and tests to use new buildbot/steps/* definitions] Original author: warner at lothar.com Date: 2006-09-08 21:25:42 Index: test_config.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_config.py,v retrieving revision 1.37 retrieving revision 1.38 diff -u -d -r1.37 -r1.38 --- test_config.py 6 Sep 2006 00:41:55 -0000 1.37 +++ test_config.py 15 Sep 2006 14:47:41 -0000 1.38 @@ -23,7 +23,8 @@ from twisted.web.distrib import ResourcePublisher from buildbot.process.builder import Builder from buildbot.process.factory import BasicBuildFactory -from buildbot.process import step +from buildbot.steps.source import CVS +from buildbot.steps.shell import Compile, Test from buildbot.status import base words = None try: @@ -80,19 +81,19 @@ wpCfg1 = buildersCfg + \ """ -from buildbot.process import step +from buildbot.steps import shell f1 = BasicBuildFactory('cvsroot', 'cvsmodule') -f1.addStep(step.ShellCommand, command=[step.WithProperties('echo')]) +f1.addStep(shell.ShellCommand, command=[shell.WithProperties('echo')]) c['builders'] = [{'name':'builder1', 'slavename':'bot1', 'builddir':'workdir1', 'factory': f1}] """ wpCfg2 = buildersCfg + \ """ -from buildbot.process import step +from buildbot.steps import shell f1 = BasicBuildFactory('cvsroot', 'cvsmodule') -f1.addStep(step.ShellCommand, - command=[step.WithProperties('echo %s', 'revision')]) +f1.addStep(shell.ShellCommand, + command=[shell.WithProperties('echo %s', 'revision')]) c['builders'] = [{'name':'builder1', 'slavename':'bot1', 'builddir':'workdir1', 'factory': f1}] """ @@ -176,7 +177,7 @@ lockCfgBad1 = \ """ -from buildbot.process.step import Dummy +from buildbot.steps.dummy import Dummy from buildbot.process.factory import BuildFactory, s from buildbot.locks import MasterLock c = {} @@ -198,7 +199,7 @@ lockCfgBad2 = \ """ -from buildbot.process.step import Dummy +from buildbot.steps.dummy import Dummy from buildbot.process.factory import BuildFactory, s from buildbot.locks import MasterLock, SlaveLock c = {} @@ -220,7 +221,7 @@ lockCfgBad3 = \ """ -from buildbot.process.step import Dummy +from buildbot.steps.dummy import Dummy from buildbot.process.factory import BuildFactory, s from buildbot.locks import MasterLock c = {} @@ -286,7 +287,7 @@ # test out step Locks lockCfg2a = \ """ -from buildbot.process.step import Dummy +from buildbot.steps.dummy import Dummy from buildbot.process.factory import BuildFactory, s from buildbot.locks import MasterLock c = {} @@ -310,7 +311,7 @@ lockCfg2b = \ """ -from buildbot.process.step import Dummy +from buildbot.steps.dummy import Dummy from buildbot.process.factory import BuildFactory, s from buildbot.locks import MasterLock c = {} @@ -334,7 +335,7 @@ lockCfg2c = \ """ -from buildbot.process.step import Dummy +from buildbot.steps.dummy import Dummy from buildbot.process.factory import BuildFactory, s from buildbot.locks import MasterLock c = {} @@ -661,13 +662,13 @@ self.failUnless(isinstance(f1, BasicBuildFactory)) steps = f1.steps self.failUnlessEqual(len(steps), 3) - self.failUnlessEqual(steps[0], (step.CVS, + self.failUnlessEqual(steps[0], (CVS, {'cvsroot': 'cvsroot', 'cvsmodule': 'cvsmodule', 'mode': 'clobber'})) - self.failUnlessEqual(steps[1], (step.Compile, + self.failUnlessEqual(steps[1], (Compile, {'command': 'make all'})) - self.failUnlessEqual(steps[2], (step.Test, + self.failUnlessEqual(steps[2], (Test, {'command': 'make check'})) Index: test_control.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_control.py,v retrieving revision 1.11 retrieving revision 1.12 diff -u -d -r1.11 -r1.12 --- test_control.py 6 Sep 2006 00:41:55 -0000 1.11 +++ test_control.py 15 Sep 2006 14:47:41 -0000 1.12 @@ -14,13 +14,14 @@ from buildbot.test.runutils import rmtree config = """ -from buildbot.process import factory, step +from buildbot.process import factory +from buildbot.steps import dummy def s(klass, **kwargs): return (klass, kwargs) f1 = factory.BuildFactory([ - s(step.Dummy, timeout=1), + s(dummy.Dummy, timeout=1), ]) c = {} c['bots'] = [['bot1', 'sekrit']] Index: test_dependencies.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_dependencies.py,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- test_dependencies.py 6 Sep 2006 00:41:55 -0000 1.4 +++ test_dependencies.py 15 Sep 2006 14:47:41 -0000 1.5 @@ -10,7 +10,8 @@ config_1 = """ from buildbot import scheduler -from buildbot.process import step, factory +from buildbot.process import factory +from buildbot.steps import dummy s = factory.s from buildbot.test.test_locks import LockStep @@ -33,9 +34,9 @@ s5 = scheduler.Dependent('downstream5', s4, ['b5']) c['schedulers'] = [s1, s2, s3, s4, s5] -f_fastpass = factory.BuildFactory([s(step.Dummy, timeout=1)]) -f_slowpass = factory.BuildFactory([s(step.Dummy, timeout=2)]) -f_fastfail = factory.BuildFactory([s(step.FailingDummy, timeout=1)]) +f_fastpass = factory.BuildFactory([s(dummy.Dummy, timeout=1)]) +f_slowpass = factory.BuildFactory([s(dummy.Dummy, timeout=2)]) +f_fastfail = factory.BuildFactory([s(dummy.FailingDummy, timeout=1)]) def builder(name, f): d = {'name': name, 'slavename': 'bot1', 'builddir': name, 'factory': f} Index: test_locks.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_locks.py,v retrieving revision 1.6 retrieving revision 1.7 diff -u -d -r1.6 -r1.7 --- test_locks.py 6 Sep 2006 00:41:55 -0000 1.6 +++ test_locks.py 15 Sep 2006 14:47:41 -0000 1.7 @@ -6,7 +6,7 @@ from twisted.internet import defer, reactor from buildbot import master -from buildbot.process import step +from buildbot.steps import dummy from buildbot.sourcestamp import SourceStamp from buildbot.process.base import BuildRequest from buildbot.test.runutils import RunMixin @@ -271,19 +271,19 @@ -class LockStep(step.Dummy): +class LockStep(dummy.Dummy): def start(self): number = self.build.requests[0].number self.build.requests[0].events.append(("start", number)) - step.Dummy.start(self) + dummy.Dummy.start(self) def done(self): number = self.build.requests[0].number self.build.requests[0].events.append(("done", number)) - step.Dummy.done(self) + dummy.Dummy.done(self) config_1 = """ from buildbot import locks -from buildbot.process import step, factory +from buildbot.process import factory s = factory.s from buildbot.test.test_locks import LockStep Index: test_properties.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_properties.py,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- test_properties.py 29 May 2006 00:10:38 -0000 1.4 +++ test_properties.py 15 Sep 2006 14:47:41 -0000 1.5 @@ -7,7 +7,7 @@ from buildbot.twcompat import maybeWait from buildbot.sourcestamp import SourceStamp from buildbot.process import base -from buildbot.process.step import ShellCommand, WithProperties +from buildbot.steps.shell import ShellCommand, WithProperties from buildbot.status import builder from buildbot.slave.commands import rmdirRecursive from buildbot.test.runutils import RunMixin @@ -118,8 +118,8 @@ run_config = """ -from buildbot.process import step, factory -from buildbot.process.step import ShellCommand, WithProperties +from buildbot.process import factory +from buildbot.steps.shell import ShellCommand, WithProperties s = factory.s BuildmasterConfig = c = {} @@ -134,7 +134,7 @@ # zombie and the step never completes. To keep this from messing up the unit # tests too badly, this step runs with a reduced timeout. -f1 = factory.BuildFactory([s(step.ShellCommand, +f1 = factory.BuildFactory([s(ShellCommand, flunkOnFailure=True, command=['touch', WithProperties('%s-slave', 'slavename'), Index: test_run.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_run.py,v retrieving revision 1.39 retrieving revision 1.40 diff -u -d -r1.39 -r1.40 --- test_run.py 6 Sep 2006 00:41:55 -0000 1.39 +++ test_run.py 15 Sep 2006 14:47:41 -0000 1.40 @@ -14,14 +14,15 @@ from buildbot.test.runutils import RunMixin, rmtree config_base = """ -from buildbot.process import factory, step +from buildbot.process import factory +from buildbot.steps import dummy s = factory.s f1 = factory.QuickBuildFactory('fakerep', 'cvsmodule', configure=None) f2 = factory.BuildFactory([ - s(step.Dummy, timeout=1), - s(step.RemoteDummy, timeout=2), + s(dummy.Dummy, timeout=1), + s(dummy.RemoteDummy, timeout=2), ]) BuildmasterConfig = c = {} Index: test_slaves.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_slaves.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- test_slaves.py 20 Oct 2005 22:25:29 -0000 1.3 +++ test_slaves.py 15 Sep 2006 14:47:41 -0000 1.4 @@ -10,7 +10,8 @@ from buildbot.status.builder import SUCCESS config_1 = """ -from buildbot.process import step, factory +from buildbot.process import factory +from buildbot.steps import dummy s = factory.s BuildmasterConfig = c = {} @@ -20,7 +21,7 @@ c['slavePortnum'] = 0 c['schedulers'] = [] -f = factory.BuildFactory([s(step.RemoteDummy, timeout=1)]) +f = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)]) c['builders'] = [ {'name': 'b1', 'slavenames': ['bot1','bot2','bot3'], Index: test_status.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_status.py,v retrieving revision 1.32 retrieving revision 1.33 diff -u -d -r1.32 -r1.33 --- test_status.py 6 Sep 2006 00:41:55 -0000 1.32 +++ test_status.py 15 Sep 2006 14:47:41 -0000 1.33 @@ -699,14 +699,15 @@ testLargeSummary.timeout = 5 config_base = """ -from buildbot.process import factory, step +from buildbot.process import factory +from buildbot.steps import dummy s = factory.s f1 = factory.QuickBuildFactory('fakerep', 'cvsmodule', configure=None) f2 = factory.BuildFactory([ - s(step.Dummy, timeout=1), - s(step.RemoteDummy, timeout=2), + s(dummy.Dummy, timeout=1), + s(dummy.RemoteDummy, timeout=2), ]) BuildmasterConfig = c = {} Index: test_steps.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_steps.py,v retrieving revision 1.27 retrieving revision 1.28 diff -u -d -r1.27 -r1.28 --- test_steps.py 9 Sep 2006 06:20:50 -0000 1.27 +++ test_steps.py 15 Sep 2006 14:47:41 -0000 1.28 @@ -22,7 +22,7 @@ from buildbot.sourcestamp import SourceStamp from buildbot.process import step, base, factory -from buildbot.process.step import ShellCommand #, ShellCommands +from buildbot.steps import shell, source from buildbot.status import builder from buildbot.test.runutils import RunMixin, rmtree from buildbot.test.runutils import makeBuildStep @@ -30,12 +30,12 @@ from buildbot.slave import commands -class MyShellCommand(ShellCommand): +class MyShellCommand(shell.ShellCommand): started = False def runCommand(self, c): self.started = True self.rc = c - return ShellCommand.runCommand(self, c) + return shell.ShellCommand.runCommand(self, c) class FakeBuild: pass @@ -167,13 +167,13 @@ class Steps(unittest.TestCase): def testMultipleStepInstances(self): steps = [ - (step.CVS, {'cvsroot': "root", 'cvsmodule': "module"}), - (step.Configure, {'command': "./configure"}), - (step.Compile, {'command': "make"}), - (step.Compile, {'command': "make more"}), - (step.Compile, {'command': "make evenmore"}), - (step.Test, {'command': "make test"}), - (step.Test, {'command': "make testharder"}), + (source.CVS, {'cvsroot': "root", 'cvsmodule': "module"}), + (shell.Configure, {'command': "./configure"}), + (shell.Compile, {'command': "make"}), + (shell.Compile, {'command': "make more"}), + (shell.Compile, {'command': "make evenmore"}), + (shell.Test, {'command': "make test"}), + (shell.Test, {'command': "make testharder"}), ] f = factory.ConfigurableBuildFactory(steps) req = base.BuildRequest("reason", SourceStamp()) @@ -261,7 +261,7 @@ def test_description(self): s = makeBuildStep("test_steps.Steps.test_description.1", - step_class=ShellCommand, + step_class=shell.ShellCommand, workdir="dummy", description=["list", "of", "strings"], descriptionDone=["another", "list"]) @@ -269,7 +269,7 @@ self.failUnlessEqual(s.descriptionDone, ["another", "list"]) s = makeBuildStep("test_steps.Steps.test_description.2", - step_class=ShellCommand, + step_class=shell.ShellCommand, workdir="dummy", description="single string", descriptionDone="another string") @@ -287,7 +287,7 @@ self.finished(step.SUCCESS) version_config = """ -from buildbot.process import factory, step +from buildbot.process import factory from buildbot.test.test_steps import VersionCheckingStep BuildmasterConfig = c = {} f1 = factory.BuildFactory([ Index: test_vc.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_vc.py,v retrieving revision 1.67 retrieving revision 1.68 diff -u -d -r1.67 -r1.68 --- test_vc.py 6 Sep 2006 00:41:55 -0000 1.67 +++ test_vc.py 15 Sep 2006 14:47:41 -0000 1.68 @@ -18,7 +18,8 @@ from buildbot.slave import bot, commands from buildbot.slave.commands import rmdirRecursive from buildbot.status.builder import SUCCESS, FAILURE -from buildbot.process import step, base +from buildbot.process import base +from buildbot.steps import source from buildbot.changes import changes from buildbot.sourcestamp import SourceStamp from buildbot.twcompat import maybeWait, which @@ -95,7 +96,8 @@ return d config_vc = """ -from buildbot.process import factory, step +from buildbot.process import factory +from buildbot.steps import source s = factory.s f1 = factory.BuildFactory([ @@ -1183,7 +1185,7 @@ vc_name = "cvs" metadir = "CVS" - vctype = "step.CVS" + vctype = "source.CVS" vctype_try = "cvs" # CVS gives us got_revision, but it is based entirely upon the local # clock, which means it is unlikely to match the timestamp taken earlier. @@ -1326,7 +1328,7 @@ vc_name = "svn" metadir = ".svn" - vctype = "step.SVN" + vctype = "source.SVN" vctype_try = "svn" has_got_revision = True has_got_revision_branches_are_merged = True @@ -1495,7 +1497,7 @@ class P4(VCBase, unittest.TestCase): metadir = None - vctype = "step.P4" + vctype = "source.P4" vc_name = "p4" def tearDownClass(self): @@ -1624,7 +1626,7 @@ # Darcs has a metadir="_darcs", but it does not have an 'export' # mode metadir = None - vctype = "step.Darcs" + vctype = "source.Darcs" vctype_try = "darcs" has_got_revision = True @@ -1832,7 +1834,7 @@ # build will re-register the correct one for whichever test is # currently being run. - # except, that step.Bazaar really doesn't like it when the archive + # except, that source.Bazaar really doesn't like it when the archive # gets unregistered behind its back. The slave tries to do a 'baz # replay' in a tree with an archive that is no longer recognized, and # baz aborts with a botched invariant exception. This causes @@ -1911,7 +1913,7 @@ metadir = None # Arch has a metadir="{arch}", but it does not have an 'export' mode. - vctype = "step.Arch" + vctype = "source.Arch" vctype_try = "tla" has_got_revision = True @@ -1981,7 +1983,7 @@ class Bazaar(Arch): vc_name = "bazaar" - vctype = "step.Bazaar" + vctype = "source.Bazaar" vctype_try = "baz" has_got_revision = True @@ -2045,7 +2047,7 @@ self.site.resource = self.root def testRetry(self): - # we want to verify that step.Source(retry=) works, and the easiest + # we want to verify that source.Source(retry=) works, and the easiest # way to make VC updates break (temporarily) is to break the HTTP # server that's providing the repository. Anything else pretty much # requires mutating the (read-only) BUILDBOT_TEST_VC repository, or @@ -2220,7 +2222,7 @@ # Mercurial has a metadir=".hg", but it does not have an 'export' mode. metadir = None - vctype = "step.Mercurial" + vctype = "source.Mercurial" vctype_try = "hg" has_got_revision = True @@ -2275,7 +2277,7 @@ def testCVS1(self): r = base.BuildRequest("forced build", SourceStamp()) b = base.Build([r]) - s = step.CVS(cvsroot=None, cvsmodule=None, workdir=None, build=b) + s = source.CVS(cvsroot=None, cvsmodule=None, workdir=None, build=b) self.failUnlessEqual(s.computeSourceRevision(b.allChanges()), None) def testCVS2(self): @@ -2287,7 +2289,7 @@ submitted = "Wed, 08 Sep 2004 09:04:00 -0700" r.submittedAt = mktime_tz(parsedate_tz(submitted)) b = base.Build([r]) - s = step.CVS(cvsroot=None, cvsmodule=None, workdir=None, build=b) + s = source.CVS(cvsroot=None, cvsmodule=None, workdir=None, build=b) self.failUnlessEqual(s.computeSourceRevision(b.allChanges()), "Wed, 08 Sep 2004 16:03:00 -0000") @@ -2300,8 +2302,8 @@ submitted = "Wed, 08 Sep 2004 09:04:00 -0700" r.submittedAt = mktime_tz(parsedate_tz(submitted)) b = base.Build([r]) - s = step.CVS(cvsroot=None, cvsmodule=None, workdir=None, build=b, - checkoutDelay=10) + s = source.CVS(cvsroot=None, cvsmodule=None, workdir=None, build=b, + checkoutDelay=10) self.failUnlessEqual(s.computeSourceRevision(b.allChanges()), "Wed, 08 Sep 2004 16:02:10 -0000") @@ -2321,14 +2323,14 @@ r2.submittedAt = mktime_tz(parsedate_tz(submitted)) b = base.Build([r1, r2]) - s = step.CVS(cvsroot=None, cvsmodule=None, workdir=None, build=b) + s = source.CVS(cvsroot=None, cvsmodule=None, workdir=None, build=b) self.failUnlessEqual(s.computeSourceRevision(b.allChanges()), "Wed, 08 Sep 2004 16:06:00 -0000") def testSVN1(self): r = base.BuildRequest("forced", SourceStamp()) b = base.Build([r]) - s = step.SVN(svnurl="dummy", workdir=None, build=b) + s = source.SVN(svnurl="dummy", workdir=None, build=b) self.failUnlessEqual(s.computeSourceRevision(b.allChanges()), None) def testSVN2(self): @@ -2338,7 +2340,7 @@ c.append(self.makeChange(revision=67)) r = base.BuildRequest("forced", SourceStamp(changes=c)) b = base.Build([r]) - s = step.SVN(svnurl="dummy", workdir=None, build=b) + s = source.SVN(svnurl="dummy", workdir=None, build=b) self.failUnlessEqual(s.computeSourceRevision(b.allChanges()), 67) class Patch(VCBase, unittest.TestCase): Index: test_web.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_web.py,v retrieving revision 1.35 retrieving revision 1.36 diff -u -d -r1.35 -r1.36 --- test_web.py 6 Sep 2006 00:41:55 -0000 1.35 +++ test_web.py 15 Sep 2006 14:47:41 -0000 1.36 @@ -14,7 +14,8 @@ from buildbot.twcompat import providedBy, maybeWait from buildbot.status import html, builder from buildbot.changes.changes import Change -from buildbot.process import step, base +from buildbot.process import base +from buildbot.process.step import BuildStep from buildbot.test.runutils import setupBuildStepStatus class ConfiguredMaster(master.BuildMaster): @@ -279,7 +280,8 @@ geturl_config = """ from buildbot.status import html from buildbot.changes import mail -from buildbot.process import step, factory +from buildbot.process import factory +from buildbot.steps import dummy from buildbot.scheduler import Scheduler from buildbot.changes.base import ChangeSource s = factory.s @@ -297,7 +299,7 @@ c['slavePortnum'] = 0 c['status'] = [html.Waterfall(http_port=0)] -f = factory.BuildFactory([s(step.RemoteDummy, timeout=1)]) +f = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)]) c['builders'] = [ {'name': 'b1', 'slavenames': ['bot1','bot2'], @@ -413,7 +415,7 @@ bs.setReason("reason") bs.buildStarted(build1) - step1 = step.BuildStep(build=build1, name="setup") + step1 = BuildStep(build=build1, name="setup") bss = bs.addStepWithName("setup") step1.setStepStatus(bss) bss.stepStarted() From warner at users.sourceforge.net Fri Sep 15 14:47:44 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:47:44 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.74,1.75 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20685/docs Modified Files: buildbot.texinfo Log Message: [project @ update docs, factories, and tests to use new buildbot/steps/* definitions] Original author: warner at lothar.com Date: 2006-09-08 21:25:42 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.74 retrieving revision 1.75 diff -u -d -r1.74 -r1.75 --- buildbot.texinfo 24 Aug 2006 10:05:20 -0000 1.74 +++ buildbot.texinfo 15 Sep 2006 14:47:42 -0000 1.75 @@ -1400,7 +1400,7 @@ @code{filename.endswith(".c")} to only run a build if a C file were checked in. Certain BuildSteps can also use the list of changed files to run a more targeted series of tests, e.g. the - at code{step_twisted.Trial} step can run just the unit tests that + at code{python_twisted.Trial} step can run just the unit tests that provide coverage for the modified .py files instead of running the full test suite. @@ -2841,12 +2841,13 @@ @code{BuildFactory}'s @code{addStep} method: @example -from buildbot.process import step, factory +from buildbot.steps import source, shell +from buildbot.process import factory f = factory.BuildFactory() -f.addStep(step.SVN, svnurl="http://svn.example.org/Trunk/") -f.addStep(step.ShellCommand, command=["make", "all"]) -f.addStep(step.ShellCommand, command=["make", "test"]) +f.addStep(source.SVN, svnurl="http://svn.example.org/Trunk/") +f.addStep(shell.ShellCommand, command=["make", "all"]) +f.addStep(shell.ShellCommand, command=["make", "test"]) @end example The rest of this section lists all the standard BuildStep objects @@ -3034,7 +3035,7 @@ @subsubsection CVS @cindex CVS Checkout - at bsindex buildbot.process.step.CVS + at bsindex buildbot.steps.source.CVS The @code{CVS} build step performs a @uref{http://www.nongnu.org/cvs/, @@ -3071,7 +3072,7 @@ @subsubsection SVN @cindex SVN Checkout - at bsindex buildbot.process.step.SVN + at bsindex buildbot.steps.source.SVN The @code{SVN} build step performs a @@ -3173,7 +3174,8 @@ @example from buildbot.changes.pb import PBChangeSource from buildbot.scheduler import AnyBranchScheduler -from buildbot.process import step, factory +from buildbot.process import source, factory +from buildbot.steps import source, shell c['sources'] = [PBChangeSource()] s1 = AnyBranchScheduler('main', @@ -3182,11 +3184,11 @@ c['schedulers'] = [s1] f = factory.BuildFactory() -f.addStep(step.SVN, mode='update', +f.addStep(source.SVN, mode='update', baseURL='svn://svn.example.org/MyProject/', defaultBranch='trunk') -f.addStep(step.Compile, command="make all") -f.addStep(step.Test, command="make test") +f.addStep(shell.Compile, command="make all") +f.addStep(shell.Test, command="make test") c['builders'] = [ @{'name':'test-i386', 'slavename':'bot-i386', 'builddir':'test-i386', @@ -3208,7 +3210,7 @@ @subsubsection Darcs @cindex Darcs Checkout - at bsindex buildbot.process.step.Darcs + at bsindex buildbot.steps.source.Darcs The @code{Darcs} build step performs a @@ -3252,7 +3254,7 @@ @subsubsection Mercurial @cindex Mercurial Checkout - at bsindex buildbot.process.step.Mercurial + at bsindex buildbot.steps.source.Mercurial The @code{Mercurial} build step performs a @@ -3285,7 +3287,7 @@ @subsubsection Arch @cindex Arch Checkout - at bsindex buildbot.process.step.Arch + at bsindex buildbot.steps.source.Arch The @code{Arch} build step performs an @uref{http://gnuarch.org/, @@ -3314,7 +3316,7 @@ @subsubsection Bazaar @cindex Bazaar Checkout - at bsindex buildbot.process.step.Bazaar + at bsindex buildbot.steps.source.Bazaar @code{Bazaar} is an alternate implementation of the Arch VC system, @@ -3332,8 +3334,8 @@ @subsubsection P4 @cindex Perforce Update - at bsindex buildbot.process.step.P4 - at c TODO @bsindex buildbot.process.step.P4Sync + at bsindex buildbot.steps.source.P4 + at c TODO @bsindex buildbot.steps.source.P4Sync The @code{P4} build step creates a @uref{http://www.perforce.com/, @@ -3372,8 +3374,8 @@ @node ShellCommand, Simple ShellCommand Subclasses, Source Checkout, Build Steps @subsection ShellCommand - at bsindex buildbot.process.step.ShellCommand - at c TODO @bsindex buildbot.process.step.TreeSize + at bsindex buildbot.steps.shell.ShellCommand + at c TODO @bsindex buildbot.steps.shell.TreeSize This is a useful base class for just about everything you might want to do during a build (except for the initial source checkout). It runs @@ -3514,7 +3516,7 @@ @node Configure, Compile, Simple ShellCommand Subclasses, Simple ShellCommand Subclasses @subsubsection Configure - at bsindex buildbot.process.step.Configure + at bsindex buildbot.steps.shell.Configure This is intended to handle the @code{./configure} step from autoconf-style projects, or the @code{perl Makefile.PL} step from perl @@ -3524,7 +3526,7 @@ @node Compile, Test, Configure, Simple ShellCommand Subclasses @subsubsection Compile - at bsindex buildbot.process.step.Compile + at bsindex buildbot.steps.shell.Compile This is meant to handle compiling or building a project written in C. The default command is @code{make all}. When the compile is finished, the @@ -3535,7 +3537,7 @@ @node Test, Build Properties, Compile, Simple ShellCommand Subclasses @subsubsection Test - at bsindex buildbot.process.step.Test + at bsindex buildbot.steps.shell.Test This is meant to handle unit tests. The default command is @code{make test}, and the @code{warnOnFailure} flag is set. @@ -3571,12 +3573,12 @@ class instances.} object. For example: @example -class MakeTarball(step.ShellCommand): +class MakeTarball(ShellCommand): def start(self): self.setCommand(["tar", "czf", "build-%s.tar.gz" % self.getProperty("revision"), "source"]) - step.ShellCommand.start(self) + ShellCommand.start(self) @end example @cindex WithProperties @@ -3587,7 +3589,7 @@ generated shell command. @example -from buildbot.process.step import ShellCommand, WithProperties +from buildbot.steps.shell import ShellCommand, WithProperties f.addStep(ShellCommand, command=["tar", "czf", @@ -4050,13 +4052,14 @@ @example from buildbot import locks -from buildbot.process import step, factory +from buildbot.steps import source, shell +from buildbot.process import factory db_lock = locks.MasterLock("database") f = factory.BuildFactory() -f.addStep(step.SVN, svnurl="http://example.org/svn/Trunk") -f.addStep(step.ShellCommand, command="make all") -f.addStep(step.ShellCommand, command="make test", locks=[db_lock]) +f.addStep(source.SVN, svnurl="http://example.org/svn/Trunk") +f.addStep(shell.ShellCommand, command="make all") +f.addStep(shell.ShellCommand, command="make test", locks=[db_lock]) b1 = @{'name': 'full1', 'slavename': 'bot-1', builddir='f1', 'factory': f@} b2 = @{'name': 'full2', 'slavename': 'bot-2', builddir='f2', 'factory': f@} b3 = @{'name': 'full3', 'slavename': 'bot-3', builddir='f3', 'factory': f@} @@ -4078,10 +4081,11 @@ @example from buildbot import locks -from buildbot.process import s, step, factory +from buildbot.steps import source +from buildbot.process import s, factory slow_lock = locks.SlaveLock("cpu", maxCount=2) -source = s(step.SVN, svnurl="http://example.org/svn/Trunk") +source = s(source.SVN, svnurl="http://example.org/svn/Trunk") f22 = factory.Trial(source, trialpython=["python2.2"]) f23 = factory.Trial(source, trialpython=["python2.3"]) f24 = factory.Trial(source, trialpython=["python2.4"]) @@ -4106,17 +4110,18 @@ @example from buildbot import locks -from buildbot.process import step, factory +from buildbot.steps import source, shell +from buildbot.process import factory db_lock = locks.MasterLock("database") slavecounts = @{"bot-slow": 1, "bot-fast": 100@} cpu_lock = locks.SlaveLock("cpu", maxCountForSlave=slavecounts) f = factory.BuildFactory() -f.addStep(step.SVN, svnurl="http://example.org/svn/Trunk") -f.addStep(step.ShellCommand, command="make all", locks=[cpu_lock]) -f.addStep(step.ShellCommand, command="make test", locks=[cpu_lock]) -f.addStep(step.ShellCommand, command="make db-test", - locks=[db_lock, cpu_lock]) +f.addStep(source.SVN, svnurl="http://example.org/svn/Trunk") +f.addStep(shell.ShellCommand, command="make all", locks=[cpu_lock]) +f.addStep(shell.ShellCommand, command="make test", locks=[cpu_lock]) +f.addStep(shell.ShellCommand, command="make db-test", + locks=[db_lock, cpu_lock]) b1 = @{'name': 'full1', 'slavename': 'bot-slow', builddir='full1', 'factory': f@} @@ -4232,11 +4237,12 @@ as follows: @example -from buildbot.process import step, factory +from buildbot.steps import source, shell +from buildbot.process import factory f = factory.BuildFactory() -f.addStep(step.CVS, cvsroot=CVSROOT, cvsmodule="project", mode="update") -f.addStep(step.Compile, command=["make", "build"]) +f.addStep(source.CVS, cvsroot=CVSROOT, cvsmodule="project", mode="update") +f.addStep(shell.Compile, command=["make", "build"]) @end example It is also possible to pass a list of step specifications into the @@ -4246,13 +4252,15 @@ a convenience function named @code{s} is available: @example -from buildbot.process import step, factory +from buildbot.steps import source, shell +from buildbot.process import factory from buildbot.factory import s # s is a convenience function, defined with: # def s(steptype, **kwargs): return (steptype, kwargs) -all_steps = [s(step.CVS, cvsroot=CVSROOT, cvsmodule="project", mode="update"), - s(step.Compile, command=["make", "build"]), +all_steps = [s(source.CVS, cvsroot=CVSROOT, cvsmodule="project", + mode="update"), + s(shell.Compile, command=["make", "build"]), ] f = factory.BuildFactory(all_steps) @end example @@ -4525,11 +4533,11 @@ @bfindex buildbot.process.factory.Trial @c TODO: document these steps better - at bsindex buildbot.process.step_twisted.HLint - at bsindex buildbot.process.step_twisted.Trial - at bsindex buildbot.process.step_twisted.ProcessDocs - at bsindex buildbot.process.step_twisted.BuildDebs - at bsindex buildbot.process.step_twisted.RemovePYCs + at bsindex buildbot.steps.python_twisted.HLint + at bsindex buildbot.steps.python_twisted.Trial + at bsindex buildbot.steps.python_twisted.ProcessDocs + at bsindex buildbot.steps.python_twisted.BuildDebs + at bsindex buildbot.steps.python_twisted.RemovePYCs Twisted provides a unit test tool named @code{trial} which provides a few improvements over Python's built-in @code{unittest} module. Many @@ -5602,8 +5610,8 @@ @printindex bs @c undocumented steps - at bsindex buildbot.process.step.Git - at bsindex buildbot.process.maxq.MaxQ + at bsindex buildbot.steps.source.Git + at bsindex buildbot.steps.maxq.MaxQ @heading Status Targets From warner at users.sourceforge.net Fri Sep 15 14:48:46 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:48:46 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_status.py, 1.33, 1.34 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21118/buildbot/test Modified Files: test_status.py Log Message: [project @ add IStatusLog.readlines()] Original author: warner at lothar.com Date: 2006-09-08 21:42:21 Index: test_status.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_status.py,v retrieving revision 1.33 retrieving revision 1.34 diff -u -d -r1.33 -r1.34 --- test_status.py 15 Sep 2006 14:47:41 -0000 1.33 +++ test_status.py 15 Sep 2006 14:48:44 -0000 1.34 @@ -488,6 +488,29 @@ l.finish() self.failUnlessEqual(l.getText(), 160*"a") + def testReadlines(self): + l = MyLog(self.basedir, "chunks") + l.addHeader("HEADER\n") # should be ignored + l.addStdout("Some text\n") + l.addStdout("Some More Text\nAnd Some More\n") + l.addStderr("Some Stderr\n") + l.addStdout("Last line\n") + l.finish() + alllines = list(l.readlines()) + self.failUnlessEqual(len(alllines), 5) + self.failUnlessEqual(alllines[0], "Some text\n") + self.failUnlessEqual(alllines[3], "Some Stderr\n") + self.failUnlessEqual(alllines[4], "Last line\n") + lines = l.readlines() + if False: # TODO: l.readlines() is not yet an iterator + # verify that it really is an iterator + line0 = lines.next() + self.failUnlessEqual(line0, "Some text\n") + line1 = lines.next() + line2 = lines.next() + self.failUnlessEqual(line2, "And Some More\n") + + def testChunks(self): l = MyLog(self.basedir, "chunks") c1 = l.getChunks() From warner at users.sourceforge.net Fri Sep 15 14:48:46 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:48:46 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.727,1.728 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21118 Modified Files: ChangeLog Log Message: [project @ add IStatusLog.readlines()] Original author: warner at lothar.com Date: 2006-09-08 21:42:21 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.727 retrieving revision 1.728 diff -u -d -r1.727 -r1.728 --- ChangeLog 15 Sep 2006 14:47:41 -0000 1.727 +++ ChangeLog 15 Sep 2006 14:48:44 -0000 1.728 @@ -1,5 +1,15 @@ 2006-09-08 Brian Warner + * buildbot/interfaces.py (IStatusLog.readlines): make it easier to + walk through StatusLogs one line at a time, mostly for the benefit of + ShellCommand.createSummary methods. + * buildbot/status/builder.py (LogFile.readlines): implement it. + Note that this is not yet memory-efficient, it just pulls the + whole file into RAM and then splits it up with a StringIO. + Eventually this should be a generator that only pulls chunks from + disk as necessary. + * buildbot/test/test_status.py (Log.testReadlines): test it + * docs/buildbot.texinfo: update to match changes * buildbot/process/factory.py: stop using old definitions * buildbot/process/process_twisted.py: same From warner at users.sourceforge.net Fri Sep 15 14:48:46 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:48:46 +0000 Subject: [Buildbot-commits] buildbot/buildbot interfaces.py,1.45,1.46 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21118/buildbot Modified Files: interfaces.py Log Message: [project @ add IStatusLog.readlines()] Original author: warner at lothar.com Date: 2006-09-08 21:42:21 Index: interfaces.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/interfaces.py,v retrieving revision 1.45 retrieving revision 1.46 diff -u -d -r1.45 -r1.46 --- interfaces.py 20 Aug 2006 22:25:35 -0000 1.45 +++ interfaces.py 15 Sep 2006 14:48:44 -0000 1.46 @@ -661,6 +661,10 @@ """Return one big string with the contents of the Log. This merges all non-header chunks together.""" + def readlines(): + """Return a list (really an iterator) of newline-terminated lines, + excluding header chunks.""" + def getTextWithHeaders(): """Return one big string with the contents of the Log. This merges all chunks (including headers) together.""" From warner at users.sourceforge.net Fri Sep 15 14:48:46 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:48:46 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status builder.py,1.86,1.87 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21118/buildbot/status Modified Files: builder.py Log Message: [project @ add IStatusLog.readlines()] Original author: warner at lothar.com Date: 2006-09-08 21:42:21 Index: builder.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/builder.py,v retrieving revision 1.86 retrieving revision 1.87 diff -u -d -r1.86 -r1.87 --- builder.py 6 Sep 2006 00:41:55 -0000 1.86 +++ builder.py 15 Sep 2006 14:48:44 -0000 1.87 @@ -13,6 +13,10 @@ pickle = cPickle except ImportError: import pickle +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO # sibling imports from buildbot import interfaces, util, sourcestamp @@ -334,6 +338,15 @@ else: yield leftover + def readlines(self): + """Return an iterator that produces newline-terminated lines, + excluding header chunks.""" + # TODO: make this memory-efficient, by turning it into a generator + # that retrieves chunks as necessary, like a pull-driven version of + # twisted.protocols.basic.LineReceiver + io = StringIO(self.getText()) + return io.readlines() + def subscribe(self, receiver, catchup): if self.finished: return From warner at users.sourceforge.net Fri Sep 15 14:48:55 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:48:55 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status builder.py,1.87,1.88 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21152/buildbot/status Modified Files: builder.py Log Message: [project @ enhance IStatusLog.readlines to accept a channel= argument] Original author: warner at lothar.com Date: 2006-09-08 21:55:02 Index: builder.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/builder.py,v retrieving revision 1.87 retrieving revision 1.88 diff -u -d -r1.87 -r1.88 --- builder.py 15 Sep 2006 14:48:44 -0000 1.87 +++ builder.py 15 Sep 2006 14:48:53 -0000 1.88 @@ -338,13 +338,14 @@ else: yield leftover - def readlines(self): + def readlines(self, channel=STDOUT): """Return an iterator that produces newline-terminated lines, excluding header chunks.""" # TODO: make this memory-efficient, by turning it into a generator # that retrieves chunks as necessary, like a pull-driven version of # twisted.protocols.basic.LineReceiver - io = StringIO(self.getText()) + alltext = "".join(self.getChunks([channel], onlyText=True)) + io = StringIO(alltext) return io.readlines() def subscribe(self, receiver, catchup): From warner at users.sourceforge.net Fri Sep 15 14:48:55 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:48:55 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.728,1.729 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21152 Modified Files: ChangeLog Log Message: [project @ enhance IStatusLog.readlines to accept a channel= argument] Original author: warner at lothar.com Date: 2006-09-08 21:55:02 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.728 retrieving revision 1.729 diff -u -d -r1.728 -r1.729 --- ChangeLog 15 Sep 2006 14:48:44 -0000 1.728 +++ ChangeLog 15 Sep 2006 14:48:53 -0000 1.729 @@ -1,8 +1,10 @@ 2006-09-08 Brian Warner * buildbot/interfaces.py (IStatusLog.readlines): make it easier to - walk through StatusLogs one line at a time, mostly for the benefit of - ShellCommand.createSummary methods. + walk through StatusLogs one line at a time, mostly for the benefit + of ShellCommand.createSummary methods. You can either walk through + STDOUT or STDERR, but the default is STDOUT. + * buildbot/status/builder.py (LogFile.readlines): implement it. Note that this is not yet memory-efficient, it just pulls the whole file into RAM and then splits it up with a StringIO. From warner at users.sourceforge.net Fri Sep 15 14:48:55 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:48:55 +0000 Subject: [Buildbot-commits] buildbot/buildbot interfaces.py,1.46,1.47 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21152/buildbot Modified Files: interfaces.py Log Message: [project @ enhance IStatusLog.readlines to accept a channel= argument] Original author: warner at lothar.com Date: 2006-09-08 21:55:02 Index: interfaces.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/interfaces.py,v retrieving revision 1.46 retrieving revision 1.47 diff -u -d -r1.46 -r1.47 --- interfaces.py 15 Sep 2006 14:48:44 -0000 1.46 +++ interfaces.py 15 Sep 2006 14:48:53 -0000 1.47 @@ -557,6 +557,11 @@ """Returns a single string with the color that should be used to display this event. 'red' and 'yellow' are the most likely ones.""" + +LOG_CHANNEL_STDOUT = 0 +LOG_CHANNEL_STDERR = 1 +LOG_CHANNEL_HEADER = 2 + class IStatusLog(Interface): """I represent a single Log, which is a growing list of text items that contains some kind of output for a single BuildStep. I might be finished, @@ -661,9 +666,11 @@ """Return one big string with the contents of the Log. This merges all non-header chunks together.""" - def readlines(): - """Return a list (really an iterator) of newline-terminated lines, - excluding header chunks.""" + def readlines(channel=LOG_CHANNEL_STDOUT): + """Read lines from one channel of the logfile. This returns an + iterator that will provide single lines of text (including the + trailing newline). + """ def getTextWithHeaders(): """Return one big string with the contents of the Log. This merges @@ -674,10 +681,6 @@ 0 for stdout, 1 for stderr, 2 for header. (note that stderr is merged into stdout if PTYs are in use).""" -LOG_CHANNEL_STDOUT = 0 -LOG_CHANNEL_STDERR = 1 -LOG_CHANNEL_HEADER = 2 - class IStatusLogConsumer(Interface): """I am an object which can be passed to IStatusLog.subscribeConsumer(). I represent a target for writing the contents of an IStatusLog. This From warner at users.sourceforge.net Fri Sep 15 14:48:55 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:48:55 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_status.py, 1.34, 1.35 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21152/buildbot/test Modified Files: test_status.py Log Message: [project @ enhance IStatusLog.readlines to accept a channel= argument] Original author: warner at lothar.com Date: 2006-09-08 21:55:02 Index: test_status.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_status.py,v retrieving revision 1.34 retrieving revision 1.35 diff -u -d -r1.34 -r1.35 --- test_status.py 15 Sep 2006 14:48:44 -0000 1.34 +++ test_status.py 15 Sep 2006 14:48:53 -0000 1.35 @@ -497,10 +497,13 @@ l.addStdout("Last line\n") l.finish() alllines = list(l.readlines()) - self.failUnlessEqual(len(alllines), 5) + self.failUnlessEqual(len(alllines), 4) self.failUnlessEqual(alllines[0], "Some text\n") - self.failUnlessEqual(alllines[3], "Some Stderr\n") - self.failUnlessEqual(alllines[4], "Last line\n") + self.failUnlessEqual(alllines[2], "And Some More\n") + self.failUnlessEqual(alllines[3], "Last line\n") + stderr = list(l.readlines(interfaces.LOG_CHANNEL_STDERR)) + self.failUnlessEqual(len(stderr), 1) + self.failUnlessEqual(stderr[0], "Some Stderr\n") lines = l.readlines() if False: # TODO: l.readlines() is not yet an iterator # verify that it really is an iterator From warner at users.sourceforge.net Fri Sep 15 14:49:02 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:02 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.62,1.63 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21206/buildbot/slave Modified Files: commands.py Log Message: [project @ initial checkin of Albert's file-upload/download patch] Original author: warner at lothar.com Date: 2006-06-16 01:21:37 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.62 retrieving revision 1.63 diff -u -d -r1.62 -r1.63 --- commands.py 6 Sep 2006 00:41:55 -0000 1.62 +++ commands.py 15 Sep 2006 14:49:00 -0000 1.63 @@ -666,6 +666,219 @@ return None + +class SlaveFileUploadCommand(Command): + """ + Upload a file from slave to build master + Arguments: + + - ['workdir']: directory to use + - ['slavesrc']: name of the file to upload to the buildmaster + - ['writer']: object for remote writing + - ['maxsize']: max size (in bytes) of file to write + - ['blocksize']: max size for one data block + + """ + + def setup(self,args): + self.workdir = args['workdir'] + self.filename = os.path.basename(args['slavesrc']) + self.writer = args['writer'] + self.maxsize = args['maxsize'] + self.blocksize = args['blocksize'] + self.stderr = None + self.rc = 0 + + self.debug = 0 + if self.debug: log.msg('SlaveFileUploadCommand started') + + # Open file + self.path = os.path.join(self.builder.basedir,self.workdir,self.filename) + try: + self.fp = open(self.path, 'r') + if self.debug: log.msg('Opened %r for upload' % self.path) + except: + self.fp = None + self.stderr = 'Cannot open file %r for upload' % self.path + self.rc = 1 + if self.debug: log.msg('Cannot open file %r for upload' % self.path) + + + def start(self): + self.cmd = defer.Deferred() + reactor.callLater(0, self._writeBlock) + + return self.cmd + + def _writeBlock(self): + """ + Write a block of data to the remote writer + """ + if self.interrupted or self.fp is None: + if self.debug: log.msg('SlaveFileUploadCommand._writeBlock(): end') + d = self.writer.callRemote('close') + d.addCallback(lambda _: self.finished()) + return + + length = self.blocksize + if self.maxsize is not None and length > self.maxsize: + length = self.maxsize + + if length <= 0: + if self.stderr is None: + self.stderr = 'Maximum filesize reached, truncating file %r' \ + % self.path + self.rc = 1 + data = '' + else: + data = self.fp.read(length) + + if self.debug: log.msg('SlaveFileUploadCommand._writeBlock(): '+ + 'allowed=%d readlen=%d' % (length,len(data))) + if len(data) == 0: + d = self.writer.callRemote('close') + d.addCallback(lambda _: self.finished()) + else: + if self.maxsize is not None: + self.maxsize = self.maxsize - len(data) + assert self.maxsize >= 0 + d = self.writer.callRemote('write',data) + d.addCallback(lambda _: self._writeBlock()) + + + def interrupt(self): + if self.debug: log.msg('interrupted') + if self.interrupted: + return + if self.stderr is None: + self.stderr = 'Upload of %r interrupted' % self.path + self.rc = 1 + self.interrupted = True + self.finished() + + + def finished(self): + if self.debug: log.msg('finished: stderr=%r, rc=%r' % (self.stderr,self.rc)) + if self.stderr is None: + self.sendStatus({'rc':self.rc}) + else: + self.sendStatus({'stderr':self.stderr, 'rc':self.rc}) + self.cmd.callback(0) + +registerSlaveCommand("uploadFile", SlaveFileUploadCommand, command_version) + + +class SlaveFileDownloadCommand(Command): + """ + Download a file from master to slave + Arguments: + + - ['workdir']: directory to use + - ['slavedest']: name of the file to upload to the buildmaster + - ['reader']: object for remote writing + - ['maxsize']: max size (in bytes) of file to write + - ['blocksize']: max size for one data block + + """ + + def setup(self,args): + self.workdir = args['workdir'] + self.filename = os.path.basename(args['slavedest']) + self.reader = args['reader'] + self.maxsize = args['maxsize'] + self.blocksize = args['blocksize'] + self.stderr = None + self.rc = 0 + + self.debug = 0 + if self.debug: log.msg('SlaveFileDownloadCommand started') + + # Open file + self.path = os.path.join(self.builder.basedir,self.workdir,self.filename) + try: + self.fp = open(self.path, 'w') + if self.debug: log.msg('Opened %r for download' % self.path) + except: + self.fp = None + self.stderr = 'Cannot open file %r for download' % self.path + self.rc = 1 + if self.debug: + log.msg('Cannot open file %r for download' % self.path) + + + def start(self): + self.cmd = defer.Deferred() + reactor.callLater(0, self._readBlock) + + return self.cmd + + def _readBlock(self): + """ + Read a block of data from the remote reader + """ + if self.interrupted or self.fp is None: + if self.debug: log.msg('SlaveFileDownloadCommand._readBlock(): end') + d = self.reader.callRemote('close') + d.addCallback(lambda _: self.finished()) + return + + length = self.blocksize + if self.maxsize is not None and length > self.maxsize: + length = self.maxsize + + if length <= 0: + if self.stderr is None: + self.stderr = 'Maximum filesize reached, truncating file %r' \ + % self.path + self.rc = 1 + d = self.reader.callRemote('close') + d.addCallback(lambda _: self.finished()) + else: + d = self.reader.callRemote('read', length) + d.addCallback(self._writeData) + + def _writeData(self,data): + if self.debug: log.msg('SlaveFileDownloadCommand._readBlock(): '+ + 'readlen=%d' % len(data)) + if len(data) == 0: + d = self.reader.callRemote('close') + d.addCallback(lambda _: self.finished()) + else: + if self.maxsize is not None: + self.maxsize = self.maxsize - len(data) + assert self.maxsize >= 0 + self.fp.write(data) + self._readBlock() # setup call back for next block (or finish) + + + def interrupt(self): + if self.debug: log.msg('interrupted') + if self.interrupted: + return + if self.stderr is None: + self.stderr = 'Download of %r interrupted' % self.path + self.rc = 1 + self.interrupted = True + self.finished() + + + def finished(self): + if self.fp is not None: + self.fp.close() + + if self.debug: log.msg('finished: stderr=%r, rc=%r' + % (self.stderr,self.rc)) + if self.stderr is None: + self.sendStatus({'rc':self.rc}) + else: + self.sendStatus({'stderr':self.stderr, 'rc':self.rc}) + self.cmd.callback(0) + + +registerSlaveCommand("downloadFile", SlaveFileDownloadCommand, command_version) + + + class SlaveShellCommand(Command): """This is a Command which runs a shell command. The args dict contains the following keys: From warner at users.sourceforge.net Fri Sep 15 14:49:02 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:02 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.103,1.104 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21206/buildbot/process Modified Files: step.py Log Message: [project @ initial checkin of Albert's file-upload/download patch] Original author: warner at lothar.com Date: 2006-06-16 01:21:37 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.103 retrieving revision 1.104 diff -u -d -r1.103 -r1.104 --- step.py 9 Sep 2006 06:20:50 -0000 1.103 +++ step.py 15 Sep 2006 14:49:00 -0000 1.104 @@ -1,5 +1,8 @@ # -*- test-case-name: buildbot.test.test_steps -*- +import time, random, types, re, warnings +from email.Utils import formatdate + from twisted.internet import reactor, defer, error from twisted.protocols import basic from twisted.spread import pb @@ -1061,6 +1064,226 @@ self.step_status.setText(self.getText(cmd, results)) self.step_status.setText2(self.maybeGetText2(cmd, results)) +# +# ================================================================= +# + +class FileIO(pb.Referenceable): + """ + Helper base class that acts as remote-accessible file-object + """ + + def __init__(self,fp): + self.fp = fp + + def remote_close(self): + """ + Called by remote slave to state that no more data will be transfered + """ + if self.fp is not None: + self.fp.close() + self.fp = None + +class FileWriter(FileIO): + """ + Helper class that acts as a file-object with write access + """ + + def __init__(self,fp, maxsize=None): + FileIO.__init__(self,fp) + self.maxsize = maxsize + + def remote_write(self,data): + """ + Called from remote slave to write L{data} to L{fp} within boundaries + of L{maxsize} + + @type data: C{string} + @param data: String of data to write + """ + if self.fp is not None: + if self.maxsize is not None: + if len(data) > self.maxsize: + data = data[:self.maxsize] + self.fp.write(data) + self.maxsize = self.maxsize - len(data) + else: + self.fp.write(data) + +class FileReader(FileIO): + """ + Helper class that acts as a file-object with read access + """ + + def remote_read(self,maxlength): + """ + Called from remote slave to read at most L{maxlength} bytes of data + + @type maxlength: C{integer} + @param maxlength: Maximum number of data bytes that can be returned + + @return: Data read from L{fp} + @rtype: C{string} of bytes read from file + """ + if self.fp is None: + return '' + + data = self.fp.read(maxlength) + return data + + +class StatusRemoteCommand(RemoteCommand): + def __init__(self, remote_command, args): + RemoteCommand.__init__(self, remote_command, args) + + self.rc = None + self.stderr = '' + + def remoteUpdate(self, update): + #log.msg('StatusRemoteCommand: update=%r' % update) + if 'rc' in update: + self.rc = update['rc'] + if 'stderr' in update: + self.stderr = self.stderr + update['stderr'] + '\n' + + +class FileUpload(BuildStep): + """ + Build step to upload a file + arguments: + + ['slavesrc'] filename of source file at slave + ['masterdest'] filename of destination file at master + ['workdir'] string with slave working directory relative to builder + base dir, default 'build' + ['maxsize'] maximum size of the file, default None (=unlimited) + ['blocksize'] maximum size of each block being transfered + + """ + + name = 'upload' + + def __init__(self,build, **kwargs): + buildstep_kwargs = {} + for k in kwargs.keys()[:]: + if k in BuildStep.parms: + buildstep_kwargs[k] = kwargs[k] + del kwargs[k] + BuildStep.__init__(self,build,**buildstep_kwargs) + + self.args = kwargs + self.fileWriter = None + + def start(self): + log.msg("FileUpload started, from slave %r to master %r" + % (self.args['slavesrc'],self.args['masterdest'])) + + self.step_status.setColor('yellow') + self.step_status.setText(['uploading', self.args['slavesrc']]) + + fp = open(self.args['masterdest'],'w') + self.fileWriter = FileWriter(fp) + + a = self.args.copy() + a['writer'] = self.fileWriter + + # add defaults for optional settings + for k,dv in [('maxsize',None),('blocksize',16*1024),('workdir','build')]: + if k not in a: + a[k] = dv + + self.cmd = StatusRemoteCommand('uploadFile', a) + d = self.runCommand(self.cmd) + d.addCallback(self.finished).addErrback(self.failed) + + def finished(self,result): + if self.cmd.stderr != '': + self.addCompleteLog('stderr', self.cmd.stderr) + + self.fileWriter = None + + if self.cmd.rc is None or self.cmd.rc == 0: + self.step_status.setColor('green') + return BuildStep.finished(self,SUCCESS) + self.step_status.setColor('red') + return BuildStep.finished(self,FAILURE) + +class FileDownload(BuildStep): + """ + Build step to download a file + arguments: + + ['mastersrc'] filename of source file at master + ['slavedest'] filename of destination file at slave + ['workdir'] string with slave working directory relative to builder + base dir, default 'build' + ['maxsize'] maximum size of the file, default None (=unlimited) + ['blocksize'] maximum size of each block being transfered + + """ + + name = 'download' + + def __init__(self,build, **kwargs): + buildstep_kwargs = {} + for k in kwargs.keys()[:]: + if k in BuildStep.parms: + buildstep_kwargs[k] = kwargs[k] + del kwargs[k] + BuildStep.__init__(self,build,**buildstep_kwargs) + + self.args = kwargs + self.fileReader = None + + def start(self): + log.msg("FileDownload started, from master %r to slave %r" + % (self.args['mastersrc'],self.args['slavedest'])) + + self.step_status.setColor('yellow') + self.step_status.setText(['downloading', self.args['slavedest']]) + + # If file does not exist, bail out with an error + if not os.path.isfile(self.args['mastersrc']): + self.addCompleteLog('stderr', + 'File %r not available at master' % self.args['mastersrc']) + reactor.callLater(0, self.reportFail) + return + + # setup structures for reading the file + fp = open(self.args['mastersrc'],'r') + self.fileReader = FileReader(fp) + + a = self.args.copy() + a['reader'] = self.fileReader + + # add defaults for optional settings + for k,dv in [('maxsize',None),('blocksize',16*1024),('workdir','build')]: + if k not in a: + a[k] = dv + + self.cmd = StatusRemoteCommand('downloadFile', a) + d = self.runCommand(self.cmd) + d.addCallback(self.finished).addErrback(self.failed) + + def finished(self,result): + if self.cmd.stderr != '': + self.addCompleteLog('stderr', self.cmd.stderr) + + self.fileReader = None + + if self.cmd.rc is None or self.cmd.rc == 0: + self.step_status.setColor('green') + return BuildStep.finished(self,SUCCESS) + return self.reportFail() + + def reportFail(self): + self.step_status.setColor('red') + return BuildStep.finished(self,FAILURE) + +# +# ================================================================= +# + # legacy compatibility From warner at users.sourceforge.net Fri Sep 15 14:49:08 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:08 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.63,1.64 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21539/buildbot/slave Modified Files: commands.py Log Message: [project @ File Upload/Download: minor cosmetic edits] Original author: warner at lothar.com Date: 2006-06-16 01:24:13 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.63 retrieving revision 1.64 diff -u -d -r1.63 -r1.64 --- commands.py 15 Sep 2006 14:49:00 -0000 1.63 +++ commands.py 15 Sep 2006 14:49:06 -0000 1.64 @@ -719,7 +719,7 @@ d = self.writer.callRemote('close') d.addCallback(lambda _: self.finished()) return - + length = self.blocksize if self.maxsize is not None and length > self.maxsize: length = self.maxsize @@ -821,7 +821,7 @@ d = self.reader.callRemote('close') d.addCallback(lambda _: self.finished()) return - + length = self.blocksize if self.maxsize is not None and length > self.maxsize: length = self.maxsize @@ -836,7 +836,7 @@ else: d = self.reader.callRemote('read', length) d.addCallback(self._writeData) - + def _writeData(self,data): if self.debug: log.msg('SlaveFileDownloadCommand._readBlock(): '+ 'readlen=%d' % len(data)) From warner at users.sourceforge.net Fri Sep 15 14:49:08 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:08 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.104,1.105 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21539/buildbot/process Modified Files: step.py Log Message: [project @ File Upload/Download: minor cosmetic edits] Original author: warner at lothar.com Date: 2006-06-16 01:24:13 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.104 retrieving revision 1.105 diff -u -d -r1.104 -r1.105 --- step.py 15 Sep 2006 14:49:00 -0000 1.104 +++ step.py 15 Sep 2006 14:49:06 -0000 1.105 @@ -1075,7 +1075,7 @@ def __init__(self,fp): self.fp = fp - + def remote_close(self): """ Called by remote slave to state that no more data will be transfered @@ -1092,7 +1092,7 @@ def __init__(self,fp, maxsize=None): FileIO.__init__(self,fp) self.maxsize = maxsize - + def remote_write(self,data): """ Called from remote slave to write L{data} to L{fp} within boundaries @@ -1127,7 +1127,7 @@ """ if self.fp is None: return '' - + data = self.fp.read(maxlength) return data @@ -1275,7 +1275,7 @@ self.step_status.setColor('green') return BuildStep.finished(self,SUCCESS) return self.reportFail() - + def reportFail(self): self.step_status.setColor('red') return BuildStep.finished(self,FAILURE) From warner at users.sourceforge.net Fri Sep 15 14:49:14 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:14 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.105,1.106 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21595/buildbot/process Modified Files: step.py Log Message: [project @ filetransfer: some stylistic changes] Original author: warner at lothar.com Date: 2006-09-08 21:44:26 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.105 retrieving revision 1.106 diff -u -d -r1.105 -r1.106 --- step.py 15 Sep 2006 14:49:06 -0000 1.105 +++ step.py 15 Sep 2006 14:49:12 -0000 1.106 @@ -1149,21 +1149,22 @@ class FileUpload(BuildStep): """ - Build step to upload a file + Build step to transfer a file from the slave to the master. + arguments: - ['slavesrc'] filename of source file at slave - ['masterdest'] filename of destination file at master - ['workdir'] string with slave working directory relative to builder - base dir, default 'build' - ['maxsize'] maximum size of the file, default None (=unlimited) - ['blocksize'] maximum size of each block being transfered + - ['slavesrc'] filename of source file at slave, relative to workdir + - ['masterdest'] filename of destination file at master + - ['workdir'] string with slave working directory relative to builder + base dir, default 'build' + - ['maxsize'] maximum size of the file, default None (=unlimited) + - ['blocksize'] maximum size of each block being transfered """ name = 'upload' - def __init__(self,build, **kwargs): + def __init__(self, build, **kwargs): buildstep_kwargs = {} for k in kwargs.keys()[:]: if k in BuildStep.parms: @@ -1184,15 +1185,16 @@ fp = open(self.args['masterdest'],'w') self.fileWriter = FileWriter(fp) - a = self.args.copy() - a['writer'] = self.fileWriter - - # add defaults for optional settings - for k,dv in [('maxsize',None),('blocksize',16*1024),('workdir','build')]: - if k not in a: - a[k] = dv + # default arguments + args = { + 'maxsize': None, + 'blocksize': 16*1024, + 'workdir': 'build', + } + args.update(self.args) + args['writer'] = self.fileWriter - self.cmd = StatusRemoteCommand('uploadFile', a) + self.cmd = StatusRemoteCommand('uploadFile', args) d = self.runCommand(self.cmd) d.addCallback(self.finished).addErrback(self.failed) From warner at users.sourceforge.net Fri Sep 15 14:49:20 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:20 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.106,1.107 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21610/buildbot/process Modified Files: step.py Log Message: [project @ filetransfer: merge from trunk, resolve conflict] Original author: warner at lothar.com Date: 2006-09-08 21:46:41 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.106 retrieving revision 1.107 diff -u -d -r1.106 -r1.107 --- step.py 15 Sep 2006 14:49:12 -0000 1.106 +++ step.py 15 Sep 2006 14:49:18 -0000 1.107 @@ -1,8 +1,5 @@ # -*- test-case-name: buildbot.test.test_steps -*- -import time, random, types, re, warnings -from email.Utils import formatdate - from twisted.internet import reactor, defer, error from twisted.protocols import basic from twisted.spread import pb From warner at users.sourceforge.net Fri Sep 15 14:49:27 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:27 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps transfer.py,NONE,1.1 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21626/buildbot/steps Added Files: transfer.py Log Message: [project @ filetransfer: move out to buildbot/steps/transfer.py] Original author: warner at lothar.com Date: 2006-09-08 21:49:07 --- NEW FILE: transfer.py --- from twisted.internet import reactor from twisted.spread import pb from buildbot.process.step import RemoteCommand, BuildStep from buildbot.process.step import SUCCESS, FAILURE class FileIO(pb.Referenceable): """ Helper base class that acts as remote-accessible file-object """ def __init__(self,fp): self.fp = fp def remote_close(self): """ Called by remote slave to state that no more data will be transfered """ if self.fp is not None: self.fp.close() self.fp = None class FileWriter(FileIO): """ Helper class that acts as a file-object with write access """ def __init__(self,fp, maxsize=None): FileIO.__init__(self,fp) self.maxsize = maxsize def remote_write(self,data): """ Called from remote slave to write L{data} to L{fp} within boundaries of L{maxsize} @type data: C{string} @param data: String of data to write """ if self.fp is not None: if self.maxsize is not None: if len(data) > self.maxsize: data = data[:self.maxsize] self.fp.write(data) self.maxsize = self.maxsize - len(data) else: self.fp.write(data) class FileReader(FileIO): """ Helper class that acts as a file-object with read access """ def remote_read(self,maxlength): """ Called from remote slave to read at most L{maxlength} bytes of data @type maxlength: C{integer} @param maxlength: Maximum number of data bytes that can be returned @return: Data read from L{fp} @rtype: C{string} of bytes read from file """ if self.fp is None: return '' data = self.fp.read(maxlength) return data class StatusRemoteCommand(RemoteCommand): def __init__(self, remote_command, args): RemoteCommand.__init__(self, remote_command, args) self.rc = None self.stderr = '' def remoteUpdate(self, update): #log.msg('StatusRemoteCommand: update=%r' % update) if 'rc' in update: self.rc = update['rc'] if 'stderr' in update: self.stderr = self.stderr + update['stderr'] + '\n' class FileUpload(BuildStep): """ Build step to transfer a file from the slave to the master. arguments: - ['slavesrc'] filename of source file at slave, relative to workdir - ['masterdest'] filename of destination file at master - ['workdir'] string with slave working directory relative to builder base dir, default 'build' - ['maxsize'] maximum size of the file, default None (=unlimited) - ['blocksize'] maximum size of each block being transfered """ name = 'upload' def __init__(self, build, **kwargs): buildstep_kwargs = {} for k in kwargs.keys()[:]: if k in BuildStep.parms: buildstep_kwargs[k] = kwargs[k] del kwargs[k] BuildStep.__init__(self,build,**buildstep_kwargs) self.args = kwargs self.fileWriter = None def start(self): log.msg("FileUpload started, from slave %r to master %r" % (self.args['slavesrc'],self.args['masterdest'])) self.step_status.setColor('yellow') self.step_status.setText(['uploading', self.args['slavesrc']]) fp = open(self.args['masterdest'],'w') self.fileWriter = FileWriter(fp) # default arguments args = { 'maxsize': None, 'blocksize': 16*1024, 'workdir': 'build', } args.update(self.args) args['writer'] = self.fileWriter self.cmd = StatusRemoteCommand('uploadFile', args) d = self.runCommand(self.cmd) d.addCallback(self.finished).addErrback(self.failed) def finished(self,result): if self.cmd.stderr != '': self.addCompleteLog('stderr', self.cmd.stderr) self.fileWriter = None if self.cmd.rc is None or self.cmd.rc == 0: self.step_status.setColor('green') return BuildStep.finished(self,SUCCESS) self.step_status.setColor('red') return BuildStep.finished(self,FAILURE) class FileDownload(BuildStep): """ Build step to download a file arguments: ['mastersrc'] filename of source file at master ['slavedest'] filename of destination file at slave ['workdir'] string with slave working directory relative to builder base dir, default 'build' ['maxsize'] maximum size of the file, default None (=unlimited) ['blocksize'] maximum size of each block being transfered """ name = 'download' def __init__(self,build, **kwargs): buildstep_kwargs = {} for k in kwargs.keys()[:]: if k in BuildStep.parms: buildstep_kwargs[k] = kwargs[k] del kwargs[k] BuildStep.__init__(self,build,**buildstep_kwargs) self.args = kwargs self.fileReader = None def start(self): log.msg("FileDownload started, from master %r to slave %r" % (self.args['mastersrc'],self.args['slavedest'])) self.step_status.setColor('yellow') self.step_status.setText(['downloading', self.args['slavedest']]) # If file does not exist, bail out with an error if not os.path.isfile(self.args['mastersrc']): self.addCompleteLog('stderr', 'File %r not available at master' % self.args['mastersrc']) reactor.callLater(0, self.reportFail) return # setup structures for reading the file fp = open(self.args['mastersrc'],'r') self.fileReader = FileReader(fp) a = self.args.copy() a['reader'] = self.fileReader # add defaults for optional settings for k,dv in [('maxsize',None),('blocksize',16*1024),('workdir','build')]: if k not in a: a[k] = dv self.cmd = StatusRemoteCommand('downloadFile', a) d = self.runCommand(self.cmd) d.addCallback(self.finished).addErrback(self.failed) def finished(self,result): if self.cmd.stderr != '': self.addCompleteLog('stderr', self.cmd.stderr) self.fileReader = None if self.cmd.rc is None or self.cmd.rc == 0: self.step_status.setColor('green') return BuildStep.finished(self,SUCCESS) return self.reportFail() def reportFail(self): self.step_status.setColor('red') return BuildStep.finished(self,FAILURE) From warner at users.sourceforge.net Fri Sep 15 14:49:27 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:27 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.107,1.108 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21626/buildbot/process Modified Files: step.py Log Message: [project @ filetransfer: move out to buildbot/steps/transfer.py] Original author: warner at lothar.com Date: 2006-09-08 21:49:07 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.107 retrieving revision 1.108 diff -u -d -r1.107 -r1.108 --- step.py 15 Sep 2006 14:49:18 -0000 1.107 +++ step.py 15 Sep 2006 14:49:25 -0000 1.108 @@ -1061,228 +1061,6 @@ self.step_status.setText(self.getText(cmd, results)) self.step_status.setText2(self.maybeGetText2(cmd, results)) -# -# ================================================================= -# - -class FileIO(pb.Referenceable): - """ - Helper base class that acts as remote-accessible file-object - """ - - def __init__(self,fp): - self.fp = fp - - def remote_close(self): - """ - Called by remote slave to state that no more data will be transfered - """ - if self.fp is not None: - self.fp.close() - self.fp = None - -class FileWriter(FileIO): - """ - Helper class that acts as a file-object with write access - """ - - def __init__(self,fp, maxsize=None): - FileIO.__init__(self,fp) - self.maxsize = maxsize - - def remote_write(self,data): - """ - Called from remote slave to write L{data} to L{fp} within boundaries - of L{maxsize} - - @type data: C{string} - @param data: String of data to write - """ - if self.fp is not None: - if self.maxsize is not None: - if len(data) > self.maxsize: - data = data[:self.maxsize] - self.fp.write(data) - self.maxsize = self.maxsize - len(data) - else: - self.fp.write(data) - -class FileReader(FileIO): - """ - Helper class that acts as a file-object with read access - """ - - def remote_read(self,maxlength): - """ - Called from remote slave to read at most L{maxlength} bytes of data - - @type maxlength: C{integer} - @param maxlength: Maximum number of data bytes that can be returned - - @return: Data read from L{fp} - @rtype: C{string} of bytes read from file - """ - if self.fp is None: - return '' - - data = self.fp.read(maxlength) - return data - - -class StatusRemoteCommand(RemoteCommand): - def __init__(self, remote_command, args): - RemoteCommand.__init__(self, remote_command, args) - - self.rc = None - self.stderr = '' - - def remoteUpdate(self, update): - #log.msg('StatusRemoteCommand: update=%r' % update) - if 'rc' in update: - self.rc = update['rc'] - if 'stderr' in update: - self.stderr = self.stderr + update['stderr'] + '\n' - - -class FileUpload(BuildStep): - """ - Build step to transfer a file from the slave to the master. - - arguments: - - - ['slavesrc'] filename of source file at slave, relative to workdir - - ['masterdest'] filename of destination file at master - - ['workdir'] string with slave working directory relative to builder - base dir, default 'build' - - ['maxsize'] maximum size of the file, default None (=unlimited) - - ['blocksize'] maximum size of each block being transfered - - """ - - name = 'upload' - - def __init__(self, build, **kwargs): - buildstep_kwargs = {} - for k in kwargs.keys()[:]: - if k in BuildStep.parms: - buildstep_kwargs[k] = kwargs[k] - del kwargs[k] - BuildStep.__init__(self,build,**buildstep_kwargs) - - self.args = kwargs - self.fileWriter = None - - def start(self): - log.msg("FileUpload started, from slave %r to master %r" - % (self.args['slavesrc'],self.args['masterdest'])) - - self.step_status.setColor('yellow') - self.step_status.setText(['uploading', self.args['slavesrc']]) - - fp = open(self.args['masterdest'],'w') - self.fileWriter = FileWriter(fp) - - # default arguments - args = { - 'maxsize': None, - 'blocksize': 16*1024, - 'workdir': 'build', - } - args.update(self.args) - args['writer'] = self.fileWriter - - self.cmd = StatusRemoteCommand('uploadFile', args) - d = self.runCommand(self.cmd) - d.addCallback(self.finished).addErrback(self.failed) - - def finished(self,result): - if self.cmd.stderr != '': - self.addCompleteLog('stderr', self.cmd.stderr) - - self.fileWriter = None - - if self.cmd.rc is None or self.cmd.rc == 0: - self.step_status.setColor('green') - return BuildStep.finished(self,SUCCESS) - self.step_status.setColor('red') - return BuildStep.finished(self,FAILURE) - -class FileDownload(BuildStep): - """ - Build step to download a file - arguments: - - ['mastersrc'] filename of source file at master - ['slavedest'] filename of destination file at slave - ['workdir'] string with slave working directory relative to builder - base dir, default 'build' - ['maxsize'] maximum size of the file, default None (=unlimited) - ['blocksize'] maximum size of each block being transfered - - """ - - name = 'download' - - def __init__(self,build, **kwargs): - buildstep_kwargs = {} - for k in kwargs.keys()[:]: - if k in BuildStep.parms: - buildstep_kwargs[k] = kwargs[k] - del kwargs[k] - BuildStep.__init__(self,build,**buildstep_kwargs) - - self.args = kwargs - self.fileReader = None - - def start(self): - log.msg("FileDownload started, from master %r to slave %r" - % (self.args['mastersrc'],self.args['slavedest'])) - - self.step_status.setColor('yellow') - self.step_status.setText(['downloading', self.args['slavedest']]) - - # If file does not exist, bail out with an error - if not os.path.isfile(self.args['mastersrc']): - self.addCompleteLog('stderr', - 'File %r not available at master' % self.args['mastersrc']) - reactor.callLater(0, self.reportFail) - return - - # setup structures for reading the file - fp = open(self.args['mastersrc'],'r') - self.fileReader = FileReader(fp) - - a = self.args.copy() - a['reader'] = self.fileReader - - # add defaults for optional settings - for k,dv in [('maxsize',None),('blocksize',16*1024),('workdir','build')]: - if k not in a: - a[k] = dv - - self.cmd = StatusRemoteCommand('downloadFile', a) - d = self.runCommand(self.cmd) - d.addCallback(self.finished).addErrback(self.failed) - - def finished(self,result): - if self.cmd.stderr != '': - self.addCompleteLog('stderr', self.cmd.stderr) - - self.fileReader = None - - if self.cmd.rc is None or self.cmd.rc == 0: - self.step_status.setColor('green') - return BuildStep.finished(self,SUCCESS) - return self.reportFail() - - def reportFail(self): - self.step_status.setColor('red') - return BuildStep.finished(self,FAILURE) - -# -# ================================================================= -# - # legacy compatibility From warner at users.sourceforge.net Fri Sep 15 14:49:35 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:35 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_transfer.py, NONE, 1.1 runutils.py, 1.13, 1.14 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21663/buildbot/test Modified Files: runutils.py Added Files: test_transfer.py Log Message: [project @ filetransfer: add a unit test, docs, do some cleanup] Original author: warner at lothar.com Date: 2006-09-09 06:11:59 Index: runutils.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/runutils.py,v retrieving revision 1.13 retrieving revision 1.14 diff -u -d -r1.13 -r1.14 --- runutils.py 21 Aug 2006 00:43:20 -0000 1.13 +++ runutils.py 15 Sep 2006 14:49:33 -0000 1.14 @@ -319,3 +319,94 @@ if "stderr" in u: print u["stderr"] +# ---------------------------------------- +from twisted.spread.util import LocalAsRemote + +class LocalWrapper: + # r = pb.Referenceable() + # w = LocalWrapper(r) + # now you can do things like w.callRemote() + def __init__(self, target): + self.target = target + + def callRemote(self, name, *args, **kwargs): + d = defer.maybeDeferred(self._callRemote, name, *args, **kwargs) + return d + + def _callRemote(self, name, *args, **kwargs): + method = getattr(self.target, "remote_"+name) + return method(*args, **kwargs) + + def notifyOnDisconnect(self, observer): + pass + def dontNotifyOnDisconnect(self, observer): + pass + + +class LocalSlaveBuilder(bot.SlaveBuilder): + """I am object that behaves like a pb.RemoteReference, but in fact I + invoke methods locally.""" + _arg_filter = None + + def setArgFilter(self, filter): + self._arg_filter = filter + + def remote_startCommand(self, stepref, stepId, command, args): + if self._arg_filter: + args = self._arg_filter(args) + # stepref should be a RemoteReference to the RemoteCommand + return bot.SlaveBuilder.remote_startCommand(self, + LocalWrapper(stepref), + stepId, command, args) + +class StepTester: + """Utility class to exercise BuildSteps and RemoteCommands, without + really using a Build or a Bot. No networks are used. + + Use this as follows:: + + class MyTest(StepTester, unittest.TestCase): + def testOne(self): + self.slavebase = 'testOne.slave' + self.masterbase = 'testOne.master' + sb = self.makeSlaveBuilder() + step = self.makeStep(stepclass, **kwargs) + d = self.runStep(step) + d.addCallback(_checkResults) + return d + """ + + #slavebase = "slavebase" + slavebuilderbase = "slavebuilderbase" + #masterbase = "masterbase" + + def makeSlaveBuilder(self): + os.mkdir(self.slavebase) + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase)) + b = bot.Bot(self.slavebase, False) + b.startService() + sb = LocalSlaveBuilder("slavebuildername", False) + sb.setArgFilter(self.filterArgs) + sb.usePTY = False + sb.setServiceParent(b) + sb.setBuilddir(self.slavebuilderbase) + self.remote = LocalWrapper(sb) + return sb + + workdir = "build" + def makeStep(self, factory, **kwargs): + if not kwargs.has_key("workdir"): + kwargs['workdir'] = self.workdir + step = makeBuildStep(self.masterbase, factory, **kwargs) + return step + + def runStep(self, step): + d = defer.maybeDeferred(step.startStep, self.remote) + return d + + def wrap(self, target): + return LocalWrapper(target) + + def filterArgs(self, args): + # this can be overridden + return args --- NEW FILE: test_transfer.py --- # -*- test-case-name: buildbot.test.test_transfer -*- import os from twisted.trial import unittest from twisted.internet import defer from buildbot.twcompat import maybeWait from buildbot.steps.transfer import FileUpload, FileDownload from buildbot.test.runutils import StepTester from buildbot.status.builder import SUCCESS, FAILURE # these steps pass a pb.Referenceable inside their arguments, so we have to # catch and wrap them. If the LocalAsRemote wrapper were a proper membrane, # we wouldn't have to do this. class Upload(StepTester, unittest.TestCase): def filterArgs(self, args): if "writer" in args: args["writer"] = self.wrap(args["writer"]) return args def testSuccess(self): self.slavebase = "testUpload.slave" self.masterbase = "testUpload.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) # the buildmaster normally runs chdir'ed into masterbase, so uploaded # files will appear there. Under trial, we're chdir'ed into # _trial_temp instead, so use a different masterdest= to keep the # uploaded file in a test-local directory masterdest = os.path.join(self.masterbase, "dest.text") step = self.makeStep(FileUpload, slavesrc="source.txt", masterdest=masterdest) slavesrc = os.path.join(self.slavebase, self.slavebuilderbase, "build", "source.txt") contents = "this is the source file\n" open(slavesrc, "w").write(contents) f = open(masterdest, "w") f.write("overwrite me\n") f.close() d = self.runStep(step) def _checkUpload(results): step_status = step.step_status #l = step_status.getLogs() #if l: # logtext = l[0].getText() # print logtext self.failUnlessEqual(results, SUCCESS) self.failUnless(os.path.exists(masterdest)) masterdest_contents = open(masterdest, "r").read() self.failUnlessEqual(masterdest_contents, contents) d.addCallback(_checkUpload) return maybeWait(d) def testMissingFile(self): self.slavebase = "testUploadMissingFile.slave" self.masterbase = "testUploadMissingFile.master" sb = self.makeSlaveBuilder() step = self.makeStep(FileUpload, slavesrc="MISSING.txt", masterdest="dest.txt") masterdest = os.path.join(self.masterbase, "dest.txt") d = self.runStep(step) def _checkUpload(results): step_status = step.step_status self.failUnlessEqual(results, FAILURE) self.failIf(os.path.exists(masterdest)) l = step_status.getLogs() logtext = l[0].getText().strip() self.failUnless(logtext.startswith("Cannot open file")) self.failUnless(logtext.endswith("for upload")) d.addCallback(_checkUpload) return maybeWait(d) class Download(StepTester, unittest.TestCase): def filterArgs(self, args): if "reader" in args: args["reader"] = self.wrap(args["reader"]) return args def testSuccess(self): self.slavebase = "testDownload.slave" self.masterbase = "testDownload.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) mastersrc = os.path.join(self.masterbase, "source.text") slavedest = os.path.join(self.slavebase, self.slavebuilderbase, "build", "dest.txt") step = self.makeStep(FileDownload, mastersrc=mastersrc, slavedest="dest.txt") contents = "this is the source file\n" open(mastersrc, "w").write(contents) f = open(slavedest, "w") f.write("overwrite me\n") f.close() d = self.runStep(step) def _checkDownload(results): step_status = step.step_status self.failUnlessEqual(results, SUCCESS) self.failUnless(os.path.exists(slavedest)) slavedest_contents = open(slavedest, "r").read() self.failUnlessEqual(slavedest_contents, contents) d.addCallback(_checkDownload) return maybeWait(d) def testMissingFile(self): self.slavebase = "testDownloadMissingFile.slave" self.masterbase = "testDownloadMissingFile.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) mastersrc = os.path.join(self.masterbase, "MISSING.text") slavedest = os.path.join(self.slavebase, self.slavebuilderbase, "build", "dest.txt") step = self.makeStep(FileDownload, mastersrc=mastersrc, slavedest="dest.txt") d = self.runStep(step) def _checkDownload(results): step_status = step.step_status self.failUnlessEqual(results, FAILURE) self.failIf(os.path.exists(slavedest)) l = step_status.getLogs() logtext = l[0].getText().strip() self.failUnless(logtext.endswith(" not available at master")) d.addCallback(_checkDownload) return maybeWait(d) # TODO: # test relative paths # need to implement expanduser() for slave-side # test error message when master-side file is in a missing directory # remove workdir= default? # clean up command start/finish code From warner at users.sourceforge.net Fri Sep 15 14:49:35 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:35 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps transfer.py,1.1,1.2 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21663/buildbot/steps Modified Files: transfer.py Log Message: [project @ filetransfer: add a unit test, docs, do some cleanup] Original author: warner at lothar.com Date: 2006-09-09 06:11:59 Index: transfer.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/transfer.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- transfer.py 15 Sep 2006 14:49:25 -0000 1.1 +++ transfer.py 15 Sep 2006 14:49:33 -0000 1.2 @@ -1,11 +1,14 @@ +# -*- test-case-name: buildbot.test.test_transfer -*- +import os.path from twisted.internet import reactor from twisted.spread import pb +from twisted.python import log from buildbot.process.step import RemoteCommand, BuildStep from buildbot.process.step import SUCCESS, FAILURE -class FileIO(pb.Referenceable): +class _FileIO(pb.Referenceable): """ Helper base class that acts as remote-accessible file-object """ @@ -21,13 +24,13 @@ self.fp.close() self.fp = None -class FileWriter(FileIO): +class _FileWriter(_FileIO): """ Helper class that acts as a file-object with write access """ def __init__(self,fp, maxsize=None): - FileIO.__init__(self,fp) + _FileIO.__init__(self,fp) self.maxsize = maxsize def remote_write(self,data): @@ -47,7 +50,7 @@ else: self.fp.write(data) -class FileReader(FileIO): +class _FileReader(_FileIO): """ Helper class that acts as a file-object with read access """ @@ -107,20 +110,27 @@ if k in BuildStep.parms: buildstep_kwargs[k] = kwargs[k] del kwargs[k] - BuildStep.__init__(self,build,**buildstep_kwargs) + BuildStep.__init__(self, build, **buildstep_kwargs) self.args = kwargs self.fileWriter = None def start(self): + source = self.args['slavesrc'] + masterdest = self.args['masterdest'] + # we rely upon the fact that the buildmaster runs chdir'ed into its + # basedir to make sure that relative paths in masterdest are expanded + # properly. TODO: maybe pass the master's basedir all the way down + # into the BuildStep so we can do this better. + target = os.path.expanduser(masterdest) log.msg("FileUpload started, from slave %r to master %r" - % (self.args['slavesrc'],self.args['masterdest'])) + % (source, target)) self.step_status.setColor('yellow') - self.step_status.setText(['uploading', self.args['slavesrc']]) + self.step_status.setText(['uploading', source]) fp = open(self.args['masterdest'],'w') - self.fileWriter = FileWriter(fp) + self.fileWriter = _FileWriter(fp) # default arguments args = { @@ -143,9 +153,9 @@ if self.cmd.rc is None or self.cmd.rc == 0: self.step_status.setColor('green') - return BuildStep.finished(self,SUCCESS) + return BuildStep.finished(self, SUCCESS) self.step_status.setColor('red') - return BuildStep.finished(self,FAILURE) + return BuildStep.finished(self, FAILURE) class FileDownload(BuildStep): """ @@ -169,38 +179,41 @@ if k in BuildStep.parms: buildstep_kwargs[k] = kwargs[k] del kwargs[k] - BuildStep.__init__(self,build,**buildstep_kwargs) + BuildStep.__init__(self, build, **buildstep_kwargs) self.args = kwargs self.fileReader = None def start(self): - log.msg("FileDownload started, from master %r to slave %r" - % (self.args['mastersrc'],self.args['slavedest'])) + source = os.path.expanduser(self.args['mastersrc']) + slavedest = self.args['slavedest'] + log.msg("FileDownload started, from master %r to slave %r" % + (source, slavedest)) self.step_status.setColor('yellow') - self.step_status.setText(['downloading', self.args['slavedest']]) + self.step_status.setText(['downloading', slavedest]) # If file does not exist, bail out with an error - if not os.path.isfile(self.args['mastersrc']): + if not os.path.isfile(source): self.addCompleteLog('stderr', - 'File %r not available at master' % self.args['mastersrc']) + 'File %r not available at master' % source) reactor.callLater(0, self.reportFail) return # setup structures for reading the file - fp = open(self.args['mastersrc'],'r') - self.fileReader = FileReader(fp) - - a = self.args.copy() - a['reader'] = self.fileReader + fp = open(source, 'r') + self.fileReader = _FileReader(fp) - # add defaults for optional settings - for k,dv in [('maxsize',None),('blocksize',16*1024),('workdir','build')]: - if k not in a: - a[k] = dv + # default arguments + args = { + 'maxsize': None, + 'blocksize': 16*1024, + 'workdir': 'build', + } + args.update(self.args) + args['reader'] = self.fileReader - self.cmd = StatusRemoteCommand('downloadFile', a) + self.cmd = StatusRemoteCommand('downloadFile', args) d = self.runCommand(self.cmd) d.addCallback(self.finished).addErrback(self.failed) @@ -212,10 +225,10 @@ if self.cmd.rc is None or self.cmd.rc == 0: self.step_status.setColor('green') - return BuildStep.finished(self,SUCCESS) + return BuildStep.finished(self, SUCCESS) return self.reportFail() def reportFail(self): self.step_status.setColor('red') - return BuildStep.finished(self,FAILURE) + return BuildStep.finished(self, FAILURE) From warner at users.sourceforge.net Fri Sep 15 14:49:35 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:35 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.64,1.65 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21663/buildbot/slave Modified Files: commands.py Log Message: [project @ filetransfer: add a unit test, docs, do some cleanup] Original author: warner at lothar.com Date: 2006-09-09 06:11:59 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.64 retrieving revision 1.65 diff -u -d -r1.64 -r1.65 --- commands.py 15 Sep 2006 14:49:06 -0000 1.64 +++ commands.py 15 Sep 2006 14:49:33 -0000 1.65 @@ -680,6 +680,8 @@ """ + debug = False + def setup(self,args): self.workdir = args['workdir'] self.filename = os.path.basename(args['slavesrc']) @@ -689,19 +691,23 @@ self.stderr = None self.rc = 0 - self.debug = 0 - if self.debug: log.msg('SlaveFileUploadCommand started') + if self.debug: + log.msg('SlaveFileUploadCommand started') # Open file - self.path = os.path.join(self.builder.basedir,self.workdir,self.filename) + self.path = os.path.join(self.builder.basedir, + self.workdir, + self.filename) try: self.fp = open(self.path, 'r') - if self.debug: log.msg('Opened %r for upload' % self.path) + if self.debug: + log.msg('Opened %r for upload' % self.path) except: self.fp = None self.stderr = 'Cannot open file %r for upload' % self.path self.rc = 1 - if self.debug: log.msg('Cannot open file %r for upload' % self.path) + if self.debug: + log.msg('Cannot open file %r for upload' % self.path) def start(self): @@ -715,7 +721,8 @@ Write a block of data to the remote writer """ if self.interrupted or self.fp is None: - if self.debug: log.msg('SlaveFileUploadCommand._writeBlock(): end') + if self.debug: + log.msg('SlaveFileUploadCommand._writeBlock(): end') d = self.writer.callRemote('close') d.addCallback(lambda _: self.finished()) return @@ -733,8 +740,9 @@ else: data = self.fp.read(length) - if self.debug: log.msg('SlaveFileUploadCommand._writeBlock(): '+ - 'allowed=%d readlen=%d' % (length,len(data))) + if self.debug: + log.msg('SlaveFileUploadCommand._writeBlock(): '+ + 'allowed=%d readlen=%d' % (length,len(data))) if len(data) == 0: d = self.writer.callRemote('close') d.addCallback(lambda _: self.finished()) @@ -747,7 +755,8 @@ def interrupt(self): - if self.debug: log.msg('interrupted') + if self.debug: + log.msg('interrupted') if self.interrupted: return if self.stderr is None: @@ -758,7 +767,8 @@ def finished(self): - if self.debug: log.msg('finished: stderr=%r, rc=%r' % (self.stderr,self.rc)) + if self.debug: + log.msg('finished: stderr=%r, rc=%r' % (self.stderr,self.rc)) if self.stderr is None: self.sendStatus({'rc':self.rc}) else: @@ -780,6 +790,7 @@ - ['blocksize']: max size for one data block """ + debug = False def setup(self,args): self.workdir = args['workdir'] @@ -790,14 +801,17 @@ self.stderr = None self.rc = 0 - self.debug = 0 - if self.debug: log.msg('SlaveFileDownloadCommand started') + if self.debug: + log.msg('SlaveFileDownloadCommand started') # Open file - self.path = os.path.join(self.builder.basedir,self.workdir,self.filename) + self.path = os.path.join(self.builder.basedir, + self.workdir, + self.filename) try: self.fp = open(self.path, 'w') - if self.debug: log.msg('Opened %r for download' % self.path) + if self.debug: + log.msg('Opened %r for download' % self.path) except: self.fp = None self.stderr = 'Cannot open file %r for download' % self.path @@ -817,7 +831,8 @@ Read a block of data from the remote reader """ if self.interrupted or self.fp is None: - if self.debug: log.msg('SlaveFileDownloadCommand._readBlock(): end') + if self.debug: + log.msg('SlaveFileDownloadCommand._readBlock(): end') d = self.reader.callRemote('close') d.addCallback(lambda _: self.finished()) return @@ -838,8 +853,9 @@ d.addCallback(self._writeData) def _writeData(self,data): - if self.debug: log.msg('SlaveFileDownloadCommand._readBlock(): '+ - 'readlen=%d' % len(data)) + if self.debug: + log.msg('SlaveFileDownloadCommand._readBlock(): '+ + 'readlen=%d' % len(data)) if len(data) == 0: d = self.reader.callRemote('close') d.addCallback(lambda _: self.finished()) @@ -852,7 +868,8 @@ def interrupt(self): - if self.debug: log.msg('interrupted') + if self.debug: + log.msg('interrupted') if self.interrupted: return if self.stderr is None: @@ -866,8 +883,8 @@ if self.fp is not None: self.fp.close() - if self.debug: log.msg('finished: stderr=%r, rc=%r' - % (self.stderr,self.rc)) + if self.debug: + log.msg('finished: stderr=%r, rc=%r' % (self.stderr,self.rc)) if self.stderr is None: self.sendStatus({'rc':self.rc}) else: From warner at users.sourceforge.net Fri Sep 15 14:49:35 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:35 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.75,1.76 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21663/docs Modified Files: buildbot.texinfo Log Message: [project @ filetransfer: add a unit test, docs, do some cleanup] Original author: warner at lothar.com Date: 2006-09-09 06:11:59 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.75 retrieving revision 1.76 diff -u -d -r1.75 -r1.76 --- buildbot.texinfo 15 Sep 2006 14:47:42 -0000 1.75 +++ buildbot.texinfo 15 Sep 2006 14:49:33 -0000 1.76 @@ -165,6 +165,7 @@ * Source Checkout:: * ShellCommand:: * Simple ShellCommand Subclasses:: +* Transferring Files:: * Writing New BuildSteps:: Source Checkout @@ -2859,6 +2860,7 @@ * Source Checkout:: * ShellCommand:: * Simple ShellCommand Subclasses:: +* Transferring Files:: * Writing New BuildSteps:: @end menu @@ -3498,7 +3500,7 @@ @end table - at node Simple ShellCommand Subclasses, Writing New BuildSteps, ShellCommand, Build Steps + at node Simple ShellCommand Subclasses, Transferring Files, ShellCommand, Build Steps @subsection Simple ShellCommand Subclasses Several subclasses of ShellCommand are provided as starting points for @@ -3692,8 +3694,64 @@ @end table + at node Transferring Files, Writing New BuildSteps, Simple ShellCommand Subclasses, Build Steps + at subsection Transferring Files - at node Writing New BuildSteps, , Simple ShellCommand Subclasses, Build Steps + at cindex File Transfer + at bsindex buildbot.steps.transfer.FileUpload + at bsindex buildbot.steps.transfer.FileDownload + +Most of the work involved in a build will take place on the +buildslave. But occasionally it is useful to do some work on the +buildmaster side. The most basic way to involve the buildmaster is +simply to move a file from the slave to the master, or vice versa. +There are a pair of BuildSteps named @code{FileUpload} and + at code{FileDownload} to provide this functionality. + +As an example, let's assume that there is a step which produces an +HTML file within the source tree that contains some sort of generated +project documentation. We want to move this file to the buildmaster, +into a @file{~/public_html} directory, so it can be visible to +developers. This file will wind up in the slave-side working directory +under the name @file{docs/reference.html}. We want to put it into the +master-side @file{~/public_html/ref.html}. + + at example +from buildbot.steps.shell import ShellCommand +from buildbot.steps.transfer import FileUpload + +f.addStep(ShellCommand, command=["make", "docs"]) +f.addStep(FileUpload, + slavesrc="docs/reference.html", + masterdest="~/public_html/ref.html") + at end example + +The @code{masterdest=} argument will be passed to os.path.expanduser, +so things like ``~'' will be expanded properly. Non-absolute paths +will be interpreted relative to the buildmaster's base directory. +Likewise, the @code{slavesrc=} argument will be expanded and +interpreted relative to the builder's working directory. + + +To move a file from the master to the slave, use the + at code{FileDownload} command. For example, let's assume that some step +requires a configuration file that, for whatever reason, could not be +recorded in the source code repository or generated on the buildslave +side: + + at example +from buildbot.steps.shell import ShellCommand +from buildbot.steps.transfer import FileUpload + +f.addStep(FileDownload + mastersrc="~/todays_build_config.txt", + slavedest="build_config.txt") +f.addStep(ShellCommand, command=["make", "config"]) + at end example + + + + at node Writing New BuildSteps, , Transferring Files, Build Steps @subsection Writing New BuildSteps While it is a good idea to keep your build process self-contained in From warner at users.sourceforge.net Fri Sep 15 14:49:42 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:42 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_steps.py,1.28,1.29 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21701/buildbot/test Modified Files: test_steps.py Log Message: [project @ test_steps: validate new just-run-the-steps StepTester utility class] Original author: warner at lothar.com Date: 2006-09-09 06:12:41 Index: test_steps.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_steps.py,v retrieving revision 1.28 retrieving revision 1.29 diff -u -d -r1.28 -r1.29 --- test_steps.py 15 Sep 2006 14:47:41 -0000 1.28 +++ test_steps.py 15 Sep 2006 14:49:39 -0000 1.29 @@ -17,17 +17,17 @@ import os, time from twisted.trial import unittest -from twisted.internet import reactor -from twisted.internet.defer import Deferred +from twisted.internet import reactor, defer from buildbot.sourcestamp import SourceStamp from buildbot.process import step, base, factory from buildbot.steps import shell, source from buildbot.status import builder +from buildbot.status.builder import SUCCESS from buildbot.test.runutils import RunMixin, rmtree -from buildbot.test.runutils import makeBuildStep +from buildbot.test.runutils import makeBuildStep, StepTester from buildbot.twcompat import maybeWait -from buildbot.slave import commands +from buildbot.slave import commands, registry class MyShellCommand(shell.ShellCommand): @@ -57,7 +57,7 @@ ## if self.callRemoteNotifier: ## reactor.callLater(0, self.callRemoteNotifier, event) self.remoteCalls += 1 - self.deferred = Deferred() + self.deferred = defer.Deferred() return self.deferred def notifyOnDisconnect(self, callback): pass @@ -363,3 +363,32 @@ from buildbot.process.step import Dummy from buildbot.process.step import FailingDummy from buildbot.process.step import RemoteDummy + + +class _SimpleBuildStep(step.BuildStep): + def start(self): + args = {"arg1": "value"} + cmd = step.RemoteCommand("simple", args) + d = self.runCommand(cmd) + d.addCallback(lambda res: self.finished(SUCCESS)) + +class _SimpleCommand(commands.Command): + def start(self): + self.builder.flag = True + self.builder.flag_args = self.args + return defer.succeed(None) + +class CheckStepTester(StepTester, unittest.TestCase): + def testSimple(self): + self.slavebase = "testSimple.slave" + self.masterbase = "testSimple.master" + sb = self.makeSlaveBuilder() + sb.flag = False + registry.registerSlaveCommand("simple", _SimpleCommand, "1") + step = self.makeStep(_SimpleBuildStep) + d = self.runStep(step) + def _checkSimple(results): + self.failUnless(sb.flag) + self.failUnlessEqual(sb.flag_args, {"arg1": "value"}) + d.addCallback(_checkSimple) + return maybeWait(d) From warner at users.sourceforge.net Fri Sep 15 14:49:48 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:48 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.65,1.66 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21714/buildbot/slave Modified Files: commands.py Log Message: [project @ file transfer: minor changes, add ChangeLog] Original author: warner at lothar.com Date: 2006-09-09 06:25:25 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.65 retrieving revision 1.66 diff -u -d -r1.65 -r1.66 --- commands.py 15 Sep 2006 14:49:33 -0000 1.65 +++ commands.py 15 Sep 2006 14:49:46 -0000 1.66 @@ -785,7 +785,7 @@ - ['workdir']: directory to use - ['slavedest']: name of the file to upload to the buildmaster - - ['reader']: object for remote writing + - ['reader']: object for remote reading - ['maxsize']: max size (in bytes) of file to write - ['blocksize']: max size for one data block From warner at users.sourceforge.net Fri Sep 15 14:49:48 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:48 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.729,1.730 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21714 Modified Files: ChangeLog Log Message: [project @ file transfer: minor changes, add ChangeLog] Original author: warner at lothar.com Date: 2006-09-09 06:25:25 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.729 retrieving revision 1.730 diff -u -d -r1.729 -r1.730 --- ChangeLog 15 Sep 2006 14:48:53 -0000 1.729 +++ ChangeLog 15 Sep 2006 14:49:46 -0000 1.730 @@ -1,5 +1,19 @@ 2006-09-08 Brian Warner + * buildbot/steps/transfer.py (FileUpload,FileDownload): new + BuildStep which lets you transfer files from the master to the + slave or vice versa. Thanks to Albert Hofkamp for the original + patch. Fixes SF#1504631. + * buildbot/slave/commands.py (SlaveFileUploadCommand): slave-side + support for it + (SlaveFileDownloadCommand): same + * docs/buildbot.texinfo (Transferring Files): document it + * buildbot/test/test_transfer.py: test it + * buildbot/test/runutils.py (StepTester): new utility class for + testing BuildSteps and RemoteCommands without Builds or Bots or PB + * buildbot/test/test_steps.py (CheckStepTester): validate that the + utility class works + * buildbot/interfaces.py (IStatusLog.readlines): make it easier to walk through StatusLogs one line at a time, mostly for the benefit of ShellCommand.createSummary methods. You can either walk through From warner at users.sourceforge.net Fri Sep 15 14:49:48 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:48 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test runutils.py, 1.14, 1.15 test_transfer.py, 1.1, 1.2 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21714/buildbot/test Modified Files: runutils.py test_transfer.py Log Message: [project @ file transfer: minor changes, add ChangeLog] Original author: warner at lothar.com Date: 2006-09-09 06:25:25 Index: runutils.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/runutils.py,v retrieving revision 1.14 retrieving revision 1.15 diff -u -d -r1.14 -r1.15 --- runutils.py 15 Sep 2006 14:49:33 -0000 1.14 +++ runutils.py 15 Sep 2006 14:49:46 -0000 1.15 @@ -320,7 +320,6 @@ print u["stderr"] # ---------------------------------------- -from twisted.spread.util import LocalAsRemote class LocalWrapper: # r = pb.Referenceable() Index: test_transfer.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_transfer.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- test_transfer.py 15 Sep 2006 14:49:33 -0000 1.1 +++ test_transfer.py 15 Sep 2006 14:49:46 -0000 1.2 @@ -150,6 +150,8 @@ # test relative paths # need to implement expanduser() for slave-side # test error message when master-side file is in a missing directory +# test maxsize= # remove workdir= default? # clean up command start/finish code +# detect slave-too-old From warner at users.sourceforge.net Fri Sep 15 14:49:49 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:49 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.76,1.77 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21714/docs Modified Files: buildbot.texinfo Log Message: [project @ file transfer: minor changes, add ChangeLog] Original author: warner at lothar.com Date: 2006-09-09 06:25:25 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.76 retrieving revision 1.77 diff -u -d -r1.76 -r1.77 --- buildbot.texinfo 15 Sep 2006 14:49:33 -0000 1.76 +++ buildbot.texinfo 15 Sep 2006 14:49:46 -0000 1.77 @@ -3706,7 +3706,9 @@ buildmaster side. The most basic way to involve the buildmaster is simply to move a file from the slave to the master, or vice versa. There are a pair of BuildSteps named @code{FileUpload} and - at code{FileDownload} to provide this functionality. + at code{FileDownload} to provide this functionality. @code{FileUpload} +moves a file @emph{up to} the master, while @code{FileDownload} moves +a file @emph{down from} the master. As an example, let's assume that there is a step which produces an HTML file within the source tree that contains some sort of generated @@ -3749,6 +3751,7 @@ f.addStep(ShellCommand, command=["make", "config"]) @end example + at c TODO: document other parameters @node Writing New BuildSteps, , Transferring Files, Build Steps From warner at users.sourceforge.net Fri Sep 15 14:49:54 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:49:54 +0000 Subject: [Buildbot-commits] buildbot CREDITS,1.2,1.3 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21753 Modified Files: CREDITS Log Message: [project @ add Albert Hofkamp to CREDITS] Original author: warner at lothar.com Date: 2006-09-09 06:30:20 Index: CREDITS =================================================================== RCS file: /cvsroot/buildbot/buildbot/CREDITS,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- CREDITS 24 Aug 2006 10:05:15 -0000 1.2 +++ CREDITS 15 Sep 2006 14:49:52 -0000 1.3 @@ -41,3 +41,4 @@ Olly Betts Philipp Frauenfelder James Knight +Albert Hofkamp From warner at users.sourceforge.net Fri Sep 15 14:50:00 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:50:00 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.730,1.731 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21767 Modified Files: ChangeLog Log Message: [project @ minor changes: clean up test-case-name line, remove extra imports] Original author: warner at lothar.com Date: 2006-09-12 17:19:02 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.730 retrieving revision 1.731 diff -u -d -r1.730 -r1.731 --- ChangeLog 15 Sep 2006 14:49:46 -0000 1.730 +++ ChangeLog 15 Sep 2006 14:49:58 -0000 1.731 @@ -1,3 +1,9 @@ +2006-09-12 Brian Warner + + * buildbot/steps/shell.py: clean up test-case-name line, remove some + unnecessary imports + * buildbot/steps/dummy.py: same + 2006-09-08 Brian Warner * buildbot/steps/transfer.py (FileUpload,FileDownload): new From warner at users.sourceforge.net Fri Sep 15 14:50:00 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:50:00 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps dummy.py, 1.1, 1.2 shell.py, 1.1, 1.2 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv21767/buildbot/steps Modified Files: dummy.py shell.py Log Message: [project @ minor changes: clean up test-case-name line, remove extra imports] Original author: warner at lothar.com Date: 2006-09-12 17:19:02 Index: dummy.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/dummy.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- dummy.py 9 Sep 2006 06:20:50 -0000 1.1 +++ dummy.py 15 Sep 2006 14:49:58 -0000 1.2 @@ -2,7 +2,7 @@ from twisted.internet import reactor from buildbot.process.step import BuildStep, LoggingBuildStep from buildbot.process.step import LoggedRemoteCommand -from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED +from buildbot.status.builder import SUCCESS, FAILURE class Dummy(BuildStep): """I am a dummy no-op step, which runs entirely on the master, and simply Index: shell.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/shell.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- shell.py 9 Sep 2006 06:20:51 -0000 1.1 +++ shell.py 15 Sep 2006 14:49:58 -0000 1.2 @@ -1,12 +1,10 @@ -# -*- test-case-name: buildbot.test.test_steps -*- - -# -*- test-case-name: buildbot.test.test_properties -*- +# -*- test-case-name: buildbot.test.test_steps,buildbot.test.test_properties -*- import types, re from twisted.python import log from buildbot import util from buildbot.process.step import LoggingBuildStep, RemoteShellCommand -from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED +from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE class _BuildPropertyDictionary: def __init__(self, build): From warner at users.sourceforge.net Fri Sep 15 14:50:11 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:50:11 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.77,1.78 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22052/docs Modified Files: buildbot.texinfo Log Message: [project @ add new python-specific steps: BuildEPYDoc and PyFlakes] Original author: warner at lothar.com Date: 2006-09-12 17:20:32 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.77 retrieving revision 1.78 diff -u -d -r1.77 -r1.78 --- buildbot.texinfo 15 Sep 2006 14:49:46 -0000 1.77 +++ buildbot.texinfo 15 Sep 2006 14:50:09 -0000 1.78 @@ -165,6 +165,7 @@ * Source Checkout:: * ShellCommand:: * Simple ShellCommand Subclasses:: +* Python BuildSteps:: * Transferring Files:: * Writing New BuildSteps:: @@ -185,6 +186,11 @@ * Test:: * Build Properties:: +Python BuildSteps + +* BuildEPYDoc:: +* PyFlakes:: + Writing New BuildSteps * BuildStep LogFiles:: @@ -2860,6 +2866,7 @@ * Source Checkout:: * ShellCommand:: * Simple ShellCommand Subclasses:: +* Python BuildSteps:: * Transferring Files:: * Writing New BuildSteps:: @end menu @@ -3500,7 +3507,7 @@ @end table - at node Simple ShellCommand Subclasses, Transferring Files, ShellCommand, Build Steps + at node Simple ShellCommand Subclasses, Python BuildSteps, ShellCommand, Build Steps @subsection Simple ShellCommand Subclasses Several subclasses of ShellCommand are provided as starting points for @@ -3694,7 +3701,83 @@ @end table - at node Transferring Files, Writing New BuildSteps, Simple ShellCommand Subclasses, Build Steps + at node Python BuildSteps, Transferring Files, Simple ShellCommand Subclasses, Build Steps + at subsection Python BuildSteps + +Here are some BuildSteps that are specifcally useful for projects +implemented in Python. + + at menu +* BuildEPYDoc:: +* PyFlakes:: + at end menu + + at node BuildEPYDoc, PyFlakes, Python BuildSteps, Python BuildSteps + at subsubsection BuildEPYDoc + + at bsindex buildbot.steps.python.BuildEPYDoc + + at url{http://epydoc.sourceforge.net/, epydoc} is a tool for generating +API documentation for Python modules from their docstrings. It reads +all the .py files from your source tree, processes the docstrings +therein, and creates a large tree of .html files (or a single .pdf +file). + +The @code{buildbot.steps.python.BuildEPYDoc} step will run + at command{epydoc} to produce this API documentation, and will count the +errors and warnings from its output. + +You must supply the command line to be used. The default is + at command{make epydocs}, which assumes that your project has a Makefile +with an ``epydocs'' target. You might wish to use something like + at command{epydoc -o apiref source/PKGNAME} instead. You might also want +to add @command{--pdf} to generate a PDF file instead of a large tree +of HTML files. + +The API docs are generated in-place in the build tree (under the +workdir, in the subdirectory controlled by the ``-o'' argument). To +make them useful, you will probably have to copy them to somewhere +they can be read. A command like @command{rsync -ad apiref/ +dev.example.com:~public_html/current-apiref/} might be useful. You +might instead want to bundle them into a tarball and publish it in the +same place where the generated install tarball is placed. + + at example +from buildbot.steps.python import BuildEPYDoc + +... +f.addStep(BuildEPYDoc, command=["epydoc", "-o", "apiref", "source/mypkg"]) + at end example + + + at node PyFlakes, , BuildEPYDoc, Python BuildSteps + at subsubsection PyFlakes + + at bsindex buildbot.steps.python.PyFlakes + + at url{http://divmod.org/trac/wiki/DivmodPyflakes, PyFlakes} is a tool +to perform basic static analysis of Python code to look for simple +errors, like missing imports and references of undefined names. It is +like a fast and simple form of the C ``lint'' program. Other tools +(like pychecker) provide more detailed results but take longer to run. + +The @code{buildbot.steps.python.PyFlakes} step will run pyflakes and +count the various kinds of errors and warnings it detects. + +You must supply the command line to be used. The default is + at command{make pyflakes}, which assumes you have a top-level Makefile +with a ``pyflakes'' target. You might want to use something like + at command{pyflakes .} or @command{pyflakes src}. + + at example +from buildbot.steps.python import PyFlakes + +... +f.addStep(PyFlakes, command=["pyflakes", "src"]) + at end example + + + at node Transferring Files, Writing New BuildSteps, Python BuildSteps, Build Steps @subsection Transferring Files @cindex File Transfer From warner at users.sourceforge.net Fri Sep 15 14:50:11 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:50:11 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.731,1.732 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22052 Modified Files: ChangeLog Log Message: [project @ add new python-specific steps: BuildEPYDoc and PyFlakes] Original author: warner at lothar.com Date: 2006-09-12 17:20:32 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.731 retrieving revision 1.732 diff -u -d -r1.731 -r1.732 --- ChangeLog 15 Sep 2006 14:49:58 -0000 1.731 +++ ChangeLog 15 Sep 2006 14:50:09 -0000 1.732 @@ -1,5 +1,9 @@ 2006-09-12 Brian Warner + * buildbot/steps/python.py (BuildEPYDoc, PyFlakes): add new steps. No + tests yet, alas. + * docs/buildbot.texinfo (Python BuildSteps): document them + * buildbot/steps/shell.py: clean up test-case-name line, remove some unnecessary imports * buildbot/steps/dummy.py: same From warner at users.sourceforge.net Fri Sep 15 14:50:11 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:50:11 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps python.py,NONE,1.1 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22052/buildbot/steps Added Files: python.py Log Message: [project @ add new python-specific steps: BuildEPYDoc and PyFlakes] Original author: warner at lothar.com Date: 2006-09-12 17:20:32 --- NEW FILE: python.py --- from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS from buildbot.steps.shell import ShellCommand try: import cStringIO StringIO = cStringIO except ImportError: import StringIO class BuildEPYDoc(ShellCommand): name = "epydoc" command = ["make", "epydocs"] description = ["building", "epydocs"] descriptionDone = ["epydoc"] def createSummary(self, log): import_errors = 0 warnings = 0 errors = 0 for line in StringIO(log.getText()): if line.startswith("Error importing "): import_errors += 1 if line.find("Warning: ") != -1: warnings += 1 if line.find("Error: ") != -1: errors += 1 self.descriptionDone = self.descriptionDone[:] if import_errors: self.descriptionDone.append("ierr=%d" % import_errors) if warnings: self.descriptionDone.append("warn=%d" % warnings) if errors: self.descriptionDone.append("err=%d" % errors) self.import_errors = import_errors self.warnings = warnings self.errors = errors def evaluateCommand(self, cmd): if cmd.rc != 0: return FAILURE if self.warnings or self.errors: return WARNINGS return SUCCESS class PyFlakes(ShellCommand): name = "pyflakes" command = ["make", "pyflakes"] description = ["running", "pyflakes"] descriptionDone = ["pyflakes"] flunkOnFailure = False def createSummary(self, log): unused_imports = 0 undefined_names = 0 redefinition_of_unused = 0 star_import = 0 misc = 0 total = 0 for line in StringIO(log.getText()).readlines(): if "imported but unused" in line: unused_imports += 1 elif "undefined name" in line: undefined_names += 1 elif "redefinition of unused" in line: redefinition_of_unused += 1 elif "*' used; unable to detect undefined names": star_import += 1 else: misc += 1 total += 1 self.descriptionDone = self.descriptionDone[:] if unused_imports: self.descriptionDone.append("unused=%d" % unused_imports) if undefined_names: self.descriptionDone.append("undefined=%s" % undefined_names) if redefinition_of_unused: self.descriptionDone.append("redefs=%s" % redefinition_of_unused) if star_import: self.descriptionDone.append("import*=%s" % star_import) if misc: self.descriptionDone.append("misc=%s" % misc) self.num_warnings = total self.setProperty("pyflakes-unused", unused_imports) self.setProperty("pyflakes-undefined", undefined_names) self.setProperty("pyflakes-redefinitions", redefinition_of_unused) self.setProperty("pyflakes-import*", star_import) self.setProperty("pyflakes-misc", misc) self.setProperty("pyflakes-total", total) def evaluateCommand(self, cmd): if cmd.rc != 0: return FAILURE if self.num_warnings: return WARNINGS return SUCCESS From warner at users.sourceforge.net Fri Sep 15 14:50:19 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:50:19 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.732,1.733 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22231 Modified Files: ChangeLog Log Message: [project @ docs: include a link from 'buildbot sendchange' to PBChangeSource] Original author: warner at lothar.com Date: 2006-09-12 17:44:01 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.732 retrieving revision 1.733 diff -u -d -r1.732 -r1.733 --- ChangeLog 15 Sep 2006 14:50:09 -0000 1.732 +++ ChangeLog 15 Sep 2006 14:50:17 -0000 1.733 @@ -3,6 +3,7 @@ * buildbot/steps/python.py (BuildEPYDoc, PyFlakes): add new steps. No tests yet, alas. * docs/buildbot.texinfo (Python BuildSteps): document them + (sendchange): include a link to PBChangeSource, since you need one * buildbot/steps/shell.py: clean up test-case-name line, remove some unnecessary imports From warner at users.sourceforge.net Fri Sep 15 14:50:19 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:50:19 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.78,1.79 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22231/docs Modified Files: buildbot.texinfo Log Message: [project @ docs: include a link from 'buildbot sendchange' to PBChangeSource] Original author: warner at lothar.com Date: 2006-09-12 17:44:01 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.78 retrieving revision 1.79 diff -u -d -r1.78 -r1.79 --- buildbot.texinfo 15 Sep 2006 14:50:09 -0000 1.78 +++ buildbot.texinfo 15 Sep 2006 14:50:17 -0000 1.79 @@ -5544,7 +5544,10 @@ This command is used to tell the buildmaster about source changes. It is intended to be used from within a commit script, installed on the -VC server. +VC server. It requires that you have a PBChangeSource +(@pxref{PBChangeSource}) running in the buildmaster (by being included +in the @code{c['sources']} list). + @example buildbot sendchange --master @var{MASTERHOST}:@var{PORT} --username @var{USER} @var{FILENAMES..} From warner at users.sourceforge.net Fri Sep 15 14:50:25 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:50:25 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.733,1.734 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22262 Modified Files: ChangeLog Log Message: [project @ scheduler: add more asserts] Original author: warner at lothar.com Date: 2006-09-12 18:55:51 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.733 retrieving revision 1.734 diff -u -d -r1.733 -r1.734 --- ChangeLog 15 Sep 2006 14:50:17 -0000 1.733 +++ ChangeLog 15 Sep 2006 14:50:23 -0000 1.734 @@ -1,5 +1,11 @@ 2006-09-12 Brian Warner + * buildbot/scheduler.py (Scheduler.__init__): make sure that + builderNames= is actually a sequence, since if you happen to give + it a single builder-specification dictionary instead, it won't get + caught by the existing assert. Thanks to Brett Neely for the + catch. + * buildbot/steps/python.py (BuildEPYDoc, PyFlakes): add new steps. No tests yet, alas. * docs/buildbot.texinfo (Python BuildSteps): document them From warner at users.sourceforge.net Fri Sep 15 14:50:25 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 15 Sep 2006 14:50:25 +0000 Subject: [Buildbot-commits] buildbot/buildbot scheduler.py,1.18,1.19 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv22262/buildbot Modified Files: scheduler.py Log Message: [project @ scheduler: add more asserts] Original author: warner at lothar.com Date: 2006-09-12 18:55:51 Index: scheduler.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/scheduler.py,v retrieving revision 1.18 retrieving revision 1.19 diff -u -d -r1.18 -r1.19 --- scheduler.py 6 Sep 2006 00:41:54 -0000 1.18 +++ scheduler.py 15 Sep 2006 14:50:23 -0000 1.19 @@ -106,6 +106,7 @@ BaseUpstreamScheduler.__init__(self, name) self.treeStableTimer = treeStableTimer + assert isinstance(builderNames, (list, tuple)) for b in builderNames: assert isinstance(b, str) self.builderNames = builderNames From warner at users.sourceforge.net Sun Sep 17 20:35:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:35:51 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.734,1.735 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25940 Modified Files: ChangeLog Log Message: [project @ move b.p.step to b.p.buildstep, leaving step.py just for backwards compability] Original author: warner at lothar.com Date: 2006-09-17 19:47:55 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.734 retrieving revision 1.735 diff -u -d -r1.734 -r1.735 --- ChangeLog 15 Sep 2006 14:50:23 -0000 1.734 +++ ChangeLog 17 Sep 2006 20:35:49 -0000 1.735 @@ -1,3 +1,12 @@ +2006-09-17 Brian Warner + + * buildbot/process/buildstep.py: rename step.py to buildstep.py . + The idea is that all the base classes (like BuildStep and + RemoteCommand and LogObserver) live in b.p.buildstep, and b.p.step + will be a leftover backwards-compatibility file that only contains + aliases for the steps that were moved out to buildbot.steps.* + * lots: change imports to match + 2006-09-12 Brian Warner * buildbot/scheduler.py (Scheduler.__init__): make sure that From warner at users.sourceforge.net Sun Sep 17 20:35:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:35:51 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process buildstep.py, NONE, 1.1 factory.py, 1.14, 1.15 step.py, 1.108, NONE Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25940/buildbot/process Modified Files: factory.py Added Files: buildstep.py Removed Files: step.py Log Message: [project @ move b.p.step to b.p.buildstep, leaving step.py just for backwards compability] Original author: warner at lothar.com Date: 2006-09-17 19:47:55 --- NEW FILE: buildstep.py --- # -*- test-case-name: buildbot.test.test_steps -*- from twisted.internet import reactor, defer, error from twisted.protocols import basic from twisted.spread import pb from twisted.python import log from twisted.python.failure import Failure from twisted.web.util import formatFailure from buildbot import interfaces from buildbot.twcompat import implements, providedBy from buildbot import util from buildbot.status import progress from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, \ EXCEPTION """ BuildStep and RemoteCommand classes for master-side representation of the build process [...1032 lines suppressed...] elif results == WARNINGS: return "orange" else: return "red" def setStatus(self, cmd, results): # this is good enough for most steps, but it can be overridden to # get more control over the displayed text self.step_status.setColor(self.getColor(cmd, results)) self.step_status.setText(self.getText(cmd, results)) self.step_status.setText2(self.maybeGetText2(cmd, results)) # legacy compatibility from buildbot.steps.shell import ShellCommand, WithProperties, TreeSize, Configure, Compile, Test from buildbot.steps.source import CVS, SVN, Darcs, Git, Arch, Bazaar, Mercurial, P4, P4Sync from buildbot.steps.dummy import Dummy, FailingDummy, RemoteDummy Index: factory.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/factory.py,v retrieving revision 1.14 retrieving revision 1.15 diff -u -d -r1.14 -r1.15 --- factory.py 15 Sep 2006 14:47:41 -0000 1.14 +++ factory.py 17 Sep 2006 20:35:49 -0000 1.15 @@ -2,7 +2,7 @@ from buildbot import util from buildbot.process.base import Build -from buildbot.process.step import BuildStep +from buildbot.process.buildstep import BuildStep from buildbot.steps.source import CVS, SVN from buildbot.steps.shell import Configure, Compile, Test --- step.py DELETED --- From warner at users.sourceforge.net Sun Sep 17 20:35:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:35:51 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps dummy.py, 1.2, 1.3 python_twisted.py, 1.1, 1.2 shell.py, 1.2, 1.3 source.py, 1.1, 1.2 transfer.py, 1.2, 1.3 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25940/buildbot/steps Modified Files: dummy.py python_twisted.py shell.py source.py transfer.py Log Message: [project @ move b.p.step to b.p.buildstep, leaving step.py just for backwards compability] Original author: warner at lothar.com Date: 2006-09-17 19:47:55 Index: dummy.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/dummy.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- dummy.py 15 Sep 2006 14:49:58 -0000 1.2 +++ dummy.py 17 Sep 2006 20:35:49 -0000 1.3 @@ -1,7 +1,7 @@ from twisted.internet import reactor -from buildbot.process.step import BuildStep, LoggingBuildStep -from buildbot.process.step import LoggedRemoteCommand +from buildbot.process.buildstep import BuildStep, LoggingBuildStep +from buildbot.process.buildstep import LoggedRemoteCommand from buildbot.status.builder import SUCCESS, FAILURE class Dummy(BuildStep): Index: python_twisted.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/python_twisted.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- python_twisted.py 15 Sep 2006 14:47:03 -0000 1.1 +++ python_twisted.py 17 Sep 2006 20:35:49 -0000 1.2 @@ -4,8 +4,8 @@ from buildbot.status import tests, builder from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED -from buildbot.process.step import LogLineObserver, OutputProgressObserver -from buildbot.process.step import RemoteShellCommand +from buildbot.process.buildstep import LogLineObserver, OutputProgressObserver +from buildbot.process.buildstep import RemoteShellCommand from buildbot.steps.shell import ShellCommand try: Index: shell.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/shell.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- shell.py 15 Sep 2006 14:49:58 -0000 1.2 +++ shell.py 17 Sep 2006 20:35:49 -0000 1.3 @@ -3,7 +3,7 @@ import types, re from twisted.python import log from buildbot import util -from buildbot.process.step import LoggingBuildStep, RemoteShellCommand +from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE class _BuildPropertyDictionary: Index: source.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/source.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- source.py 9 Sep 2006 06:20:51 -0000 1.1 +++ source.py 17 Sep 2006 20:35:49 -0000 1.2 @@ -3,7 +3,7 @@ import warnings from email.Utils import formatdate from twisted.python import log -from buildbot.process.step import LoggingBuildStep, LoggedRemoteCommand +from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand from buildbot.interfaces import BuildSlaveTooOldError from buildbot.status.builder import SKIPPED Index: transfer.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/transfer.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- transfer.py 15 Sep 2006 14:49:33 -0000 1.2 +++ transfer.py 17 Sep 2006 20:35:49 -0000 1.3 @@ -4,8 +4,8 @@ from twisted.internet import reactor from twisted.spread import pb from twisted.python import log -from buildbot.process.step import RemoteCommand, BuildStep -from buildbot.process.step import SUCCESS, FAILURE +from buildbot.process.buildstep import RemoteCommand, BuildStep +from buildbot.process.buildstep import SUCCESS, FAILURE class _FileIO(pb.Referenceable): From warner at users.sourceforge.net Sun Sep 17 20:35:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:35:51 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test runutils.py, 1.15, 1.16 test_steps.py, 1.29, 1.30 test_web.py, 1.36, 1.37 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25940/buildbot/test Modified Files: runutils.py test_steps.py test_web.py Log Message: [project @ move b.p.step to b.p.buildstep, leaving step.py just for backwards compability] Original author: warner at lothar.com Date: 2006-09-17 19:47:55 Index: runutils.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/runutils.py,v retrieving revision 1.15 retrieving revision 1.16 diff -u -d -r1.15 -r1.16 --- runutils.py 15 Sep 2006 14:49:46 -0000 1.15 +++ runutils.py 17 Sep 2006 20:35:49 -0000 1.16 @@ -9,7 +9,7 @@ from buildbot.slave import bot from buildbot.process.builder import Builder from buildbot.process.base import BuildRequest, Build -from buildbot.process.step import BuildStep +from buildbot.process.buildstep import BuildStep from buildbot.sourcestamp import SourceStamp from buildbot.status import builder Index: test_steps.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_steps.py,v retrieving revision 1.29 retrieving revision 1.30 diff -u -d -r1.29 -r1.30 --- test_steps.py 15 Sep 2006 14:49:39 -0000 1.29 +++ test_steps.py 17 Sep 2006 20:35:49 -0000 1.30 @@ -20,7 +20,7 @@ from twisted.internet import reactor, defer from buildbot.sourcestamp import SourceStamp -from buildbot.process import step, base, factory +from buildbot.process import buildstep, base, factory from buildbot.steps import shell, source from buildbot.status import builder from buildbot.status.builder import SUCCESS @@ -98,7 +98,7 @@ cmd = "argle bargle" dir = "murkle" expectedEvents = [] - step.RemoteCommand.commandCounter[0] = 3 + buildstep.RemoteCommand.commandCounter[0] = 3 c = MyShellCommand(workdir=dir, command=cmd, build=self.build, timeout=10) self.assertEqual(self.remote.events, expectedEvents) @@ -159,7 +159,7 @@ self.assertEqual(self.results, 0) -class MyObserver(step.LogObserver): +class MyObserver(buildstep.LogObserver): out = "" def outReceived(self, data): self.out = self.out + data @@ -276,7 +276,7 @@ self.failUnlessEqual(s.description, ["single string"]) self.failUnlessEqual(s.descriptionDone, ["another string"]) -class VersionCheckingStep(step.BuildStep): +class VersionCheckingStep(buildstep.BuildStep): def start(self): # give our test a chance to run. It is non-trivial for a buildstep to # claw its way back out to the test case which is currently running. @@ -284,7 +284,7 @@ checker = master._checker checker(self) # then complete - self.finished(step.SUCCESS) + self.finished(buildstep.SUCCESS) version_config = """ from buildbot.process import factory @@ -365,10 +365,10 @@ from buildbot.process.step import RemoteDummy -class _SimpleBuildStep(step.BuildStep): +class _SimpleBuildStep(buildstep.BuildStep): def start(self): args = {"arg1": "value"} - cmd = step.RemoteCommand("simple", args) + cmd = buildstep.RemoteCommand("simple", args) d = self.runCommand(cmd) d.addCallback(lambda res: self.finished(SUCCESS)) Index: test_web.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_web.py,v retrieving revision 1.36 retrieving revision 1.37 diff -u -d -r1.36 -r1.37 --- test_web.py 15 Sep 2006 14:47:41 -0000 1.36 +++ test_web.py 17 Sep 2006 20:35:49 -0000 1.37 @@ -15,7 +15,7 @@ from buildbot.status import html, builder from buildbot.changes.changes import Change from buildbot.process import base -from buildbot.process.step import BuildStep +from buildbot.process.buildstep import BuildStep from buildbot.test.runutils import setupBuildStepStatus class ConfiguredMaster(master.BuildMaster): From warner at users.sourceforge.net Sun Sep 17 20:39:32 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:39:32 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.109,1.110 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27536/buildbot/process Added Files: step.py Log Message: [project @ phase 2 of moving step to buildstep, adding back in step.py] Original author: warner at lothar.com Date: 2006-09-17 19:49:36 --- NEW FILE: step.py --- # -*- test-case-name: buildbot.test.test_steps.ReorgCompatibility -*- # legacy compatibility from buildbot.steps.shell import ShellCommand, WithProperties, TreeSize, Configure, Compile, Test from buildbot.steps.source import CVS, SVN, Darcs, Git, Arch, Bazaar, Mercurial, P4, P4Sync from buildbot.steps.dummy import Dummy, FailingDummy, RemoteDummy From warner at users.sourceforge.net Sun Sep 17 20:39:39 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:39:39 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.735,1.736 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27550 Modified Files: ChangeLog Log Message: [project @ add DeprecationWarning to b.p.step, add some missing base classes] Original author: warner at lothar.com Date: 2006-09-17 20:04:53 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.735 retrieving revision 1.736 diff -u -d -r1.735 -r1.736 --- ChangeLog 17 Sep 2006 20:35:49 -0000 1.735 +++ ChangeLog 17 Sep 2006 20:39:36 -0000 1.736 @@ -6,6 +6,8 @@ will be a leftover backwards-compatibility file that only contains aliases for the steps that were moved out to buildbot.steps.* * lots: change imports to match + * buildbot/process/step.py: add a DeprecationWarning if it ever + gets imported 2006-09-12 Brian Warner From warner at users.sourceforge.net Sun Sep 17 20:39:39 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:39:39 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.110,1.111 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27550/buildbot/process Modified Files: step.py Log Message: [project @ add DeprecationWarning to b.p.step, add some missing base classes] Original author: warner at lothar.com Date: 2006-09-17 20:04:53 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.110 retrieving revision 1.111 diff -u -d -r1.110 -r1.111 --- step.py 17 Sep 2006 20:39:30 -0000 1.110 +++ step.py 17 Sep 2006 20:39:37 -0000 1.111 @@ -2,8 +2,16 @@ # legacy compatibility +import warnings +warnings.warn("buildbot.process.step is deprecated. Please import things like ShellCommand from one of the buildbot.steps.* modules instead.", + DeprecationWarning) + from buildbot.steps.shell import ShellCommand, WithProperties, TreeSize, Configure, Compile, Test from buildbot.steps.source import CVS, SVN, Darcs, Git, Arch, Bazaar, Mercurial, P4, P4Sync from buildbot.steps.dummy import Dummy, FailingDummy, RemoteDummy +from buildbot.process.buildstep import LogObserver, LogLineObserver +from buildbot.process.buildstep import RemoteShellCommand +from buildbot.process.buildstep import BuildStep, LoggingBuildStep +from buildbot.process.buildstep import ShellCommand, WithProperties From warner at users.sourceforge.net Sun Sep 17 20:39:45 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:39:45 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process buildstep.py,1.1,1.2 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27578/buildbot/process Modified Files: buildstep.py Log Message: [project @ duh, actually remove the old imports from buildstep.py] Original author: warner at lothar.com Date: 2006-09-17 20:07:12 Index: buildstep.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/buildstep.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- buildstep.py 17 Sep 2006 20:35:49 -0000 1.1 +++ buildstep.py 17 Sep 2006 20:39:43 -0000 1.2 @@ -1061,11 +1061,3 @@ self.step_status.setText(self.getText(cmd, results)) self.step_status.setText2(self.maybeGetText2(cmd, results)) - -# legacy compatibility - -from buildbot.steps.shell import ShellCommand, WithProperties, TreeSize, Configure, Compile, Test -from buildbot.steps.source import CVS, SVN, Darcs, Git, Arch, Bazaar, Mercurial, P4, P4Sync -from buildbot.steps.dummy import Dummy, FailingDummy, RemoteDummy - - From warner at users.sourceforge.net Sun Sep 17 20:43:41 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:43:41 +0000 Subject: [Buildbot-commits] buildbot/buildbot master.py, 1.96, 1.97 scheduler.py, 1.19, 1.20 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29142/buildbot Modified Files: master.py scheduler.py Log Message: [project @ improve error messages when c['schedulers'] are broken] Original author: warner at lothar.com Date: 2006-09-17 20:32:53 Index: master.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/master.py,v retrieving revision 1.96 retrieving revision 1.97 diff -u -d -r1.96 -r1.97 --- master.py 6 Sep 2006 00:41:54 -0000 1.96 +++ master.py 17 Sep 2006 20:43:39 -0000 1.97 @@ -733,9 +733,10 @@ assert interfaces.IChangeSource(s, None) # this assertion catches c['schedulers'] = Scheduler(), since # Schedulers are service.MultiServices and thus iterable. - assert isinstance(schedulers, (list, tuple)) + errmsg = "c['schedulers'] must be a list of Scheduler instances" + assert isinstance(schedulers, (list, tuple)), errmsg for s in schedulers: - assert interfaces.IScheduler(s, None) + assert interfaces.IScheduler(s, None), errmsg assert isinstance(status, (list, tuple)) for s in status: assert interfaces.IStatusReceiver(s, None) @@ -774,7 +775,7 @@ # Manhole, the ChangeMaster, and the BotMaster (although most # of these don't have names) msg = ("Schedulers must have unique names, but " - "'%s' was a duplicate" + s.name) + "'%s' was a duplicate" % (s.name,)) raise ValueError(msg) schedulernames.append(s.name) Index: scheduler.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/scheduler.py,v retrieving revision 1.19 retrieving revision 1.20 diff -u -d -r1.19 -r1.20 --- scheduler.py 15 Sep 2006 14:50:23 -0000 1.19 +++ scheduler.py 17 Sep 2006 20:43:39 -0000 1.20 @@ -106,9 +106,12 @@ BaseUpstreamScheduler.__init__(self, name) self.treeStableTimer = treeStableTimer - assert isinstance(builderNames, (list, tuple)) + errmsg = ("The builderNames= argument to Scheduler must be a list " + "of Builder description names (i.e. the 'name' key of the " + "Builder specification dictionary)") + assert isinstance(builderNames, (list, tuple)), errmsg for b in builderNames: - assert isinstance(b, str) + assert isinstance(b, str), errmsg self.builderNames = builderNames self.branch = branch if fileIsImportant: From warner at users.sourceforge.net Sun Sep 17 20:43:41 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:43:41 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.736,1.737 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29142 Modified Files: ChangeLog Log Message: [project @ improve error messages when c['schedulers'] are broken] Original author: warner at lothar.com Date: 2006-09-17 20:32:53 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.736 retrieving revision 1.737 diff -u -d -r1.736 -r1.737 --- ChangeLog 17 Sep 2006 20:39:36 -0000 1.736 +++ ChangeLog 17 Sep 2006 20:43:39 -0000 1.737 @@ -1,5 +1,12 @@ 2006-09-17 Brian Warner + * buildbot/master.py (BuildMaster.loadConfig): improve the error + message logged when c['schedulers'] is not right + * buildbot/scheduler.py (Scheduler.__init__): improve error + message when a Scheduler() is created with the wrong arguments + * buildbot/test/test_config.py (ConfigTest.testSchedulerErrors): + verify that these error messages are emitted + * buildbot/process/buildstep.py: rename step.py to buildstep.py . The idea is that all the base classes (like BuildStep and RemoteCommand and LogObserver) live in b.p.buildstep, and b.p.step From warner at users.sourceforge.net Sun Sep 17 20:43:41 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:43:41 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_config.py, 1.38, 1.39 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29142/buildbot/test Modified Files: test_config.py Log Message: [project @ improve error messages when c['schedulers'] are broken] Original author: warner at lothar.com Date: 2006-09-17 20:32:53 Index: test_config.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_config.py,v retrieving revision 1.38 retrieving revision 1.39 diff -u -d -r1.38 -r1.39 --- test_config.py 15 Sep 2006 14:47:41 -0000 1.38 +++ test_config.py 17 Sep 2006 20:43:39 -0000 1.39 @@ -357,6 +357,25 @@ BuildmasterConfig = c """ +schedulersCfg = \ +""" +from buildbot.scheduler import Scheduler, Dependent +from buildbot.process.factory import BasicBuildFactory +c = {} +c['bots'] = [('bot1', 'pw1')] +c['sources'] = [] +f1 = BasicBuildFactory('cvsroot', 'cvsmodule') +b1 = {'name':'builder1', 'slavename':'bot1', + 'builddir':'workdir', 'factory':f1} +c['builders'] = [b1] +c['schedulers'] = [Scheduler('full', None, 60, ['builder1'])] +c['slavePortnum'] = 9999 +c['projectName'] = 'dummy project' +c['projectURL'] = 'http://dummy.example.com' +c['buildbotURL'] = 'http://dummy.example.com/buildbot' +BuildmasterConfig = c +""" + class ConfigTest(unittest.TestCase): def setUp(self): self.buildmaster = BuildMaster(".") @@ -533,64 +552,96 @@ res.trap(*expected) return None # all is good - def testSchedulers(self): + def testSchedulerErrors(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(master.allSchedulers(), []) - self.schedulersCfg = \ -""" -from buildbot.scheduler import Scheduler, Dependent -from buildbot.process.factory import BasicBuildFactory -c = {} -c['bots'] = [('bot1', 'pw1')] -c['sources'] = [] -c['schedulers'] = [Scheduler('full', None, 60, ['builder1'])] -f1 = BasicBuildFactory('cvsroot', 'cvsmodule') -c['builders'] = [{'name':'builder1', 'slavename':'bot1', - 'builddir':'workdir', 'factory':f1}] -c['slavePortnum'] = 9999 -c['projectName'] = 'dummy project' -c['projectURL'] = 'http://dummy.example.com' -c['buildbotURL'] = 'http://dummy.example.com/buildbot' -BuildmasterConfig = c -""" + def _shouldBeFailure(res, hint=None): + self.shouldBeFailure(res, AssertionError, ValueError) + if hint: + self.failUnless(str(res).find(hint) != -1) + + def _loadConfig(res, newcfg): + return self.buildmaster.loadConfig(newcfg) + d = defer.succeed(None) # c['schedulers'] must be a list - badcfg = self.schedulersCfg + \ + badcfg = schedulersCfg + \ """ c['schedulers'] = Scheduler('full', None, 60, ['builder1']) """ - d = defer.maybeDeferred(self.buildmaster.loadConfig, badcfg) - d.addBoth(self._testSchedulers_1) - return maybeWait(d) - def _testSchedulers_1(self, res): - self.shouldBeFailure(res, AssertionError) + d.addCallback(_loadConfig, badcfg) + d.addBoth(_shouldBeFailure, + "c['schedulers'] must be a list of Scheduler instances") + # c['schedulers'] must be a list of IScheduler objects - badcfg = self.schedulersCfg + \ + badcfg = schedulersCfg + \ """ c['schedulers'] = ['oops', 'problem'] """ - d = defer.maybeDeferred(self.buildmaster.loadConfig, badcfg) - d.addBoth(self._testSchedulers_2) - return d - def _testSchedulers_2(self, res): - self.shouldBeFailure(res, AssertionError) + d.addCallback(_loadConfig, badcfg) + d.addBoth(_shouldBeFailure, + "c['schedulers'] must be a list of Scheduler instances") + # c['schedulers'] must point at real builders - badcfg = self.schedulersCfg + \ + badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('full', None, 60, ['builder-bogus'])] """ - d = defer.maybeDeferred(self.buildmaster.loadConfig, badcfg) - d.addBoth(self._testSchedulers_3) - return d - def _testSchedulers_3(self, res): - self.shouldBeFailure(res, AssertionError) - d = self.buildmaster.loadConfig(self.schedulersCfg) - d.addCallback(self._testSchedulers_4) - return d - def _testSchedulers_4(self, res): + d.addCallback(_loadConfig, badcfg) + d.addBoth(_shouldBeFailure, "uses unknown builder") + + # builderNames= must be a list + badcfg = schedulersCfg + \ +""" +c['schedulers'] = [Scheduler('full', None, 60, 'builder1')] +""" + d.addCallback(_loadConfig, badcfg) + d.addBoth(_shouldBeFailure, + "must be a list of Builder description names") + + # builderNames= must be a list of strings, not dicts + badcfg = schedulersCfg + \ +""" +c['schedulers'] = [Scheduler('full', None, 60, [b1])] +""" + d.addCallback(_loadConfig, badcfg) + d.addBoth(_shouldBeFailure, + "must be a list of Builder description names") + + # builderNames= must be a list of strings, not a dict + badcfg = schedulersCfg + \ +""" +c['schedulers'] = [Scheduler('full', None, 60, b1)] +""" + d.addCallback(_loadConfig, badcfg) + d.addBoth(_shouldBeFailure, + "must be a list of Builder description names") + + # each Scheduler must have a unique name + badcfg = schedulersCfg + \ +""" +c['schedulers'] = [Scheduler('dup', None, 60, []), + Scheduler('dup', None, 60, [])] +""" + d.addCallback(_loadConfig, badcfg) + d.addBoth(_shouldBeFailure, "Schedulers must have unique names") + + return maybeWait(d) + + def testSchedulers(self): + master = self.buildmaster + master.loadChanges() + master.loadConfig(emptyCfg) + self.failUnlessEqual(master.allSchedulers(), []) + + d = self.buildmaster.loadConfig(schedulersCfg) + d.addCallback(self._testSchedulers_1) + return maybeWait(d) + + def _testSchedulers_1(self, res): sch = self.buildmaster.allSchedulers() self.failUnlessEqual(len(sch), 1) s = sch[0] @@ -600,15 +651,15 @@ self.failUnlessEqual(s.treeStableTimer, 60) self.failUnlessEqual(s.builderNames, ['builder1']) - newcfg = self.schedulersCfg + \ + newcfg = schedulersCfg + \ """ s1 = Scheduler('full', None, 60, ['builder1']) c['schedulers'] = [s1, Dependent('downstream', s1, ['builder1'])] """ d = self.buildmaster.loadConfig(newcfg) - d.addCallback(self._testSchedulers_5, newcfg) + d.addCallback(self._testSchedulers_2, newcfg) return d - def _testSchedulers_5(self, res, newcfg): + def _testSchedulers_2(self, res, newcfg): sch = self.buildmaster.allSchedulers() self.failUnlessEqual(len(sch), 2) s = sch[0] @@ -620,9 +671,9 @@ # reloading the same config file should leave the schedulers in place d = self.buildmaster.loadConfig(newcfg) - d.addCallback(self._testSchedulers_6, sch) + d.addCallback(self._testSchedulers_3, sch) return d - def _testSchedulers_6(self, res, sch1): + def _testSchedulers_3(self, res, sch1): sch2 = self.buildmaster.allSchedulers() self.failUnlessEqual(len(sch2), 2) sch1.sort() @@ -632,16 +683,6 @@ self.failUnlessIdentical(sch1[1], sch2[1]) self.failUnlessIdentical(sch1[0].parent, self.buildmaster) self.failUnlessIdentical(sch1[1].parent, self.buildmaster) - badcfg = self.schedulersCfg + \ -""" -c['schedulers'] = [Scheduler('dup', None, 60, []), - Scheduler('dup', None, 60, [])] -""" - d = defer.maybeDeferred(self.buildmaster.loadConfig, badcfg) - d.addBoth(self._testSchedulers_7) - return d - def _testSchedulers_7(self, res): - self.shouldBeFailure(res, ValueError) From warner at users.sourceforge.net Sun Sep 17 20:49:33 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:49:33 +0000 Subject: [Buildbot-commits] buildbot CREDITS,1.3,1.4 ChangeLog,1.737,1.738 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv31473 Modified Files: CREDITS ChangeLog Log Message: [project @ remove a stray ^M from the output of 'p4 describe', fixes SF#1555985] Original author: warner at lothar.com Date: 2006-09-17 20:48:47 Index: CREDITS =================================================================== RCS file: /cvsroot/buildbot/buildbot/CREDITS,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- CREDITS 15 Sep 2006 14:49:52 -0000 1.3 +++ CREDITS 17 Sep 2006 20:49:31 -0000 1.4 @@ -42,3 +42,5 @@ Philipp Frauenfelder James Knight Albert Hofkamp +Brett Neely +Wade Brainerd Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.737 retrieving revision 1.738 diff -u -d -r1.737 -r1.738 --- ChangeLog 17 Sep 2006 20:43:39 -0000 1.737 +++ ChangeLog 17 Sep 2006 20:49:31 -0000 1.738 @@ -1,5 +1,10 @@ 2006-09-17 Brian Warner + * buildbot/changes/p4poller.py (P4Source._process_describe): do an + rstrip() on the first line coming out of the 'p4 describe' + process, to remove the stray ^M that Wade Brainerd reports seeing + in the 'when' field. Fixes SF#1555985. + * buildbot/master.py (BuildMaster.loadConfig): improve the error message logged when c['schedulers'] is not right * buildbot/scheduler.py (Scheduler.__init__): improve error From warner at users.sourceforge.net Sun Sep 17 20:49:33 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 20:49:33 +0000 Subject: [Buildbot-commits] buildbot/buildbot/changes p4poller.py,1.6,1.7 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/changes In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv31473/buildbot/changes Modified Files: p4poller.py Log Message: [project @ remove a stray ^M from the output of 'p4 describe', fixes SF#1555985] Original author: warner at lothar.com Date: 2006-09-17 20:48:47 Index: p4poller.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/p4poller.py,v retrieving revision 1.6 retrieving revision 1.7 diff -u -d -r1.6 -r1.7 --- p4poller.py 12 Jun 2006 08:36:08 -0000 1.6 +++ p4poller.py 17 Sep 2006 20:49:31 -0000 1.7 @@ -166,6 +166,9 @@ def _process_describe(self, result, num): lines = result.split('\n') + # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date + # field. The rstrip() is intended to remove that. + lines[0] = lines[0].rstrip() m = self.describe_header_re.match(lines[0]) assert m, "Unexpected 'p4 describe -s' result: %r" % result who = m.group('who') From warner at users.sourceforge.net Sun Sep 17 21:05:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 21:05:37 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process step.py,1.111,1.112 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv5290/buildbot/process Modified Files: step.py Log Message: [project @ step.py: accidentally added some extra imports] Original author: warner at lothar.com Date: 2006-09-17 21:05:06 Index: step.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/step.py,v retrieving revision 1.111 retrieving revision 1.112 diff -u -d -r1.111 -r1.112 --- step.py 17 Sep 2006 20:39:37 -0000 1.111 +++ step.py 17 Sep 2006 21:05:35 -0000 1.112 @@ -13,5 +13,4 @@ from buildbot.process.buildstep import LogObserver, LogLineObserver from buildbot.process.buildstep import RemoteShellCommand from buildbot.process.buildstep import BuildStep, LoggingBuildStep -from buildbot.process.buildstep import ShellCommand, WithProperties From warner at users.sourceforge.net Sun Sep 17 21:05:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sun, 17 Sep 2006 21:05:37 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.738,1.739 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv5290 Modified Files: ChangeLog Log Message: [project @ step.py: accidentally added some extra imports] Original author: warner at lothar.com Date: 2006-09-17 21:05:06 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.738 retrieving revision 1.739 diff -u -d -r1.738 -r1.739 --- ChangeLog 17 Sep 2006 20:49:31 -0000 1.738 +++ ChangeLog 17 Sep 2006 21:05:35 -0000 1.739 @@ -1,5 +1,7 @@ 2006-09-17 Brian Warner + * buildbot/process/step.py: oops, added extra imports by mistake + * buildbot/changes/p4poller.py (P4Source._process_describe): do an rstrip() on the first line coming out of the 'p4 describe' process, to remove the stray ^M that Wade Brainerd reports seeing From warner at users.sourceforge.net Mon Sep 18 02:14:47 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 18 Sep 2006 02:14:47 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_vc.py,1.68,1.69 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29116/buildbot/test Modified Files: test_vc.py Log Message: [project @ fix test failure against SVN-1.4.0, because it is cleverer than older SVNs] Original author: warner at lothar.com Date: 2006-09-18 02:09:21 Index: test_vc.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_vc.py,v retrieving revision 1.68 retrieving revision 1.69 diff -u -d -r1.68 -r1.69 --- test_vc.py 15 Sep 2006 14:47:41 -0000 1.68 +++ test_vc.py 18 Sep 2006 02:14:45 -0000 1.69 @@ -499,6 +499,15 @@ self.fail("build did not succeed") return bs + def printLogs(self, bs): + for s in bs.getSteps(): + for l in s.getLogs(): + print "--- START step %s / log %s ---" % (s.getName(), + l.getName()) + print l.getTextWithHeaders() + print "--- STOP ---" + print + def touch(self, d, f): open(os.path.join(d,f),"w").close() def shouldExist(self, *args): @@ -666,7 +675,12 @@ d.addCallback(self._do_vctest_update_retry_1) return d def _do_vctest_update_retry_1(self, bs): - self.shouldNotExist(self.workdir, "newfile") + # SVN-1.4.0 doesn't seem to have any problem with the + # file-turned-directory issue (although older versions did). So don't + # actually check that the tree was clobbered.. as long as the update + # succeeded (checked by doBuild), that should be good enough. + #self.shouldNotExist(self.workdir, "newfile") + pass def _do_vctest_copy(self, res): d = self.doBuild() # copy rebuild clobbers new files From warner at users.sourceforge.net Mon Sep 18 02:14:47 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 18 Sep 2006 02:14:47 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.739,1.740 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29116 Modified Files: ChangeLog Log Message: [project @ fix test failure against SVN-1.4.0, because it is cleverer than older SVNs] Original author: warner at lothar.com Date: 2006-09-18 02:09:21 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.739 retrieving revision 1.740 diff -u -d -r1.739 -r1.740 --- ChangeLog 17 Sep 2006 21:05:35 -0000 1.739 +++ ChangeLog 18 Sep 2006 02:14:45 -0000 1.740 @@ -1,5 +1,13 @@ 2006-09-17 Brian Warner + * buildbot/test/test_vc.py (VCBase._do_vctest_update_retry_1): it + turns out that SVN-1.4.0 doesn't fail to update once you've + replaced a file with a directory, unlike older versions of SVN and + pretty much every other VC tool we support. Since what we really + care about is that the update succeeds anyway, stop checking that + the tree got clobbered and just assert that the build succeeded. + (VCBase.printLogs): add a utility function for debugging + * buildbot/process/step.py: oops, added extra imports by mistake * buildbot/changes/p4poller.py (P4Source._process_describe): do an From warner at users.sourceforge.net Mon Sep 18 20:35:42 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 18 Sep 2006 20:35:42 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.740,1.741 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv9489 Modified Files: ChangeLog Log Message: [project @ steps.python: fix import of StringIO] Original author: warner at lothar.com Date: 2006-09-18 20:29:12 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.740 retrieving revision 1.741 diff -u -d -r1.740 -r1.741 --- ChangeLog 18 Sep 2006 02:14:45 -0000 1.740 +++ ChangeLog 18 Sep 2006 20:35:40 -0000 1.741 @@ -1,3 +1,7 @@ +2006-09-18 Brian Warner + + * buildbot/steps/python.py: oops, fix import of StringIO + 2006-09-17 Brian Warner * buildbot/test/test_vc.py (VCBase._do_vctest_update_retry_1): it From warner at users.sourceforge.net Mon Sep 18 20:35:42 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 18 Sep 2006 20:35:42 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps python.py,1.1,1.2 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv9489/buildbot/steps Modified Files: python.py Log Message: [project @ steps.python: fix import of StringIO] Original author: warner at lothar.com Date: 2006-09-18 20:29:12 Index: python.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/python.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- python.py 15 Sep 2006 14:50:09 -0000 1.1 +++ python.py 18 Sep 2006 20:35:40 -0000 1.2 @@ -4,9 +4,9 @@ try: import cStringIO - StringIO = cStringIO + StringIO = cStringIO.StringIO except ImportError: - import StringIO + from StringIO import StringIO class BuildEPYDoc(ShellCommand): From warner at users.sourceforge.net Tue Sep 19 18:46:57 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 19 Sep 2006 18:46:57 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_steps.py,1.30,1.31 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv30314/buildbot/test Modified Files: test_steps.py Log Message: [project @ improve PyFlakes, add a unit test] Original author: warner at lothar.com Date: 2006-09-19 18:07:47 Index: test_steps.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_steps.py,v retrieving revision 1.30 retrieving revision 1.31 diff -u -d -r1.30 -r1.31 --- test_steps.py 17 Sep 2006 20:35:49 -0000 1.30 +++ test_steps.py 19 Sep 2006 18:46:55 -0000 1.31 @@ -21,9 +21,9 @@ from buildbot.sourcestamp import SourceStamp from buildbot.process import buildstep, base, factory -from buildbot.steps import shell, source +from buildbot.steps import shell, source, python from buildbot.status import builder -from buildbot.status.builder import SUCCESS +from buildbot.status.builder import SUCCESS, FAILURE from buildbot.test.runutils import RunMixin, rmtree from buildbot.test.runutils import makeBuildStep, StepTester from buildbot.twcompat import maybeWait @@ -392,3 +392,49 @@ self.failUnlessEqual(sb.flag_args, {"arg1": "value"}) d.addCallback(_checkSimple) return maybeWait(d) + +class Python(StepTester, unittest.TestCase): + def testPyFlakes(self): + self.masterbase = "Python.master" + step = self.makeStep(python.PyFlakes) + output = \ +"""buildbot/changes/freshcvsmail.py:5: 'FCMaildirSource' imported but unused +buildbot/clients/debug.py:9: redefinition of unused 'gtk' from line 9 +buildbot/clients/debug.py:9: 'gnome' imported but unused +buildbot/scripts/runner.py:323: redefinition of unused 'run' from line 321 +buildbot/scripts/runner.py:325: redefinition of unused 'run' from line 323 +buildbot/scripts/imaginary.py:12: undefined name 'size' +buildbot/scripts/imaginary.py:18: 'from buildbot import *' used; unable to detect undefined names +""" + log = step.addLog("stdio") + log.addStdout(output) + log.finish() + step.createSummary(log) + desc = step.descriptionDone + self.failUnless("unused=2" in desc) + self.failUnless("undefined=1" in desc) + self.failUnless("redefs=3" in desc) + self.failUnless("import*=1" in desc) + + self.failUnlessEqual(step.getProperty("pyflakes-unused"), 2) + self.failUnlessEqual(step.getProperty("pyflakes-undefined"), 1) + self.failUnlessEqual(step.getProperty("pyflakes-redefs"), 3) + self.failUnlessEqual(step.getProperty("pyflakes-import*"), 1) + self.failUnlessEqual(step.getProperty("pyflakes-misc"), 0) + self.failUnlessEqual(step.getProperty("pyflakes-total"), 7) + + logs = {} + for log in step.step_status.getLogs(): + logs[log.getName()] = log + + for name in ["unused", "undefined", "redefs", "import*"]: + self.failUnless(name in logs) + self.failIf("misc" in logs) + lines = logs["unused"].readlines() + self.failUnlessEqual(len(lines), 2) + self.failUnlessEqual(lines[0], "buildbot/changes/freshcvsmail.py:5: 'FCMaildirSource' imported but unused\n") + + cmd = buildstep.RemoteCommand(None, {}) + cmd.rc = 0 + results = step.evaluateCommand(cmd) + self.failUnlessEqual(results, FAILURE) # because of the 'undefined' From warner at users.sourceforge.net Tue Sep 19 18:46:57 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 19 Sep 2006 18:46:57 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.741,1.742 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv30314 Modified Files: ChangeLog Log Message: [project @ improve PyFlakes, add a unit test] Original author: warner at lothar.com Date: 2006-09-19 18:07:47 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.741 retrieving revision 1.742 diff -u -d -r1.741 -r1.742 --- ChangeLog 18 Sep 2006 20:35:40 -0000 1.741 +++ ChangeLog 19 Sep 2006 18:46:55 -0000 1.742 @@ -1,3 +1,9 @@ +2006-09-19 Brian Warner + + * buildbot/steps/python.py (PyFlakes): refactor, add summary logs + * buildbot/test/test_steps.py (Python.testPyFlakes): add a test + for at least the output-parsing parts of PyFlakes + 2006-09-18 Brian Warner * buildbot/steps/python.py: oops, fix import of StringIO From warner at users.sourceforge.net Tue Sep 19 18:46:57 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 19 Sep 2006 18:46:57 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps python.py,1.2,1.3 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv30314/buildbot/steps Modified Files: python.py Log Message: [project @ improve PyFlakes, add a unit test] Original author: warner at lothar.com Date: 2006-09-19 18:07:47 Index: python.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/python.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- python.py 18 Sep 2006 20:35:40 -0000 1.2 +++ python.py 19 Sep 2006 18:46:55 -0000 1.3 @@ -54,53 +54,47 @@ description = ["running", "pyflakes"] descriptionDone = ["pyflakes"] flunkOnFailure = False + flunkingIssues = ["undefined"] # any pyflakes lines like this cause FAILURE + + MESSAGES = ("unused", "undefined", "redefs", "import*", "misc") def createSummary(self, log): - unused_imports = 0 - undefined_names = 0 - redefinition_of_unused = 0 - star_import = 0 - misc = 0 - total = 0 + counts = {} + summaries = {} + for m in self.MESSAGES: + counts[m] = 0 + summaries[m] = [] for line in StringIO(log.getText()).readlines(): if "imported but unused" in line: - unused_imports += 1 + m = "unused" + elif "*' used; unable to detect undefined names" in line: + m = "import*" elif "undefined name" in line: - undefined_names += 1 + m = "undefined" elif "redefinition of unused" in line: - redefinition_of_unused += 1 - elif "*' used; unable to detect undefined names": - star_import += 1 + m = "redefs" else: - misc += 1 - total += 1 + m = "misc" + summaries[m].append(line) + counts[m] += 1 self.descriptionDone = self.descriptionDone[:] - if unused_imports: - self.descriptionDone.append("unused=%d" % unused_imports) - if undefined_names: - self.descriptionDone.append("undefined=%s" % undefined_names) - if redefinition_of_unused: - self.descriptionDone.append("redefs=%s" % redefinition_of_unused) - if star_import: - self.descriptionDone.append("import*=%s" % star_import) - if misc: - self.descriptionDone.append("misc=%s" % misc) - self.num_warnings = total - - self.setProperty("pyflakes-unused", unused_imports) - self.setProperty("pyflakes-undefined", undefined_names) - self.setProperty("pyflakes-redefinitions", redefinition_of_unused) - self.setProperty("pyflakes-import*", star_import) - self.setProperty("pyflakes-misc", misc) - self.setProperty("pyflakes-total", total) + for m in self.MESSAGES: + if counts[m]: + self.descriptionDone.append("%s=%d" % (m, counts[m])) + self.addCompleteLog(m, "".join(summaries[m])) + self.setProperty("pyflakes-%s" % m, counts[m]) + self.setProperty("pyflakes-total", sum(counts.values())) def evaluateCommand(self, cmd): if cmd.rc != 0: return FAILURE - if self.num_warnings: + for m in self.flunkingIssues: + if self.getProperty("pyflakes-%s" % m): + return FAILURE + if self.getProperty("pyflakes-total"): return WARNINGS return SUCCESS From warner at users.sourceforge.net Wed Sep 20 06:33:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 20 Sep 2006 06:33:37 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps python.py,1.3,1.4 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29813/buildbot/steps Modified Files: python.py Log Message: [project @ steps.python.PyFlakes: make it compatible with python-2.2] Original author: warner at lothar.com Date: 2006-09-20 06:31:29 Index: python.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/python.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- python.py 19 Sep 2006 18:46:55 -0000 1.3 +++ python.py 20 Sep 2006 06:33:35 -0000 1.4 @@ -66,13 +66,13 @@ summaries[m] = [] for line in StringIO(log.getText()).readlines(): - if "imported but unused" in line: + if line.find("imported but unused") != -1: m = "unused" - elif "*' used; unable to detect undefined names" in line: + elif line.find("*' used; unable to detect undefined names") != -1: m = "import*" - elif "undefined name" in line: + elif line.find("undefined name") != -1: m = "undefined" - elif "redefinition of unused" in line: + elif line.find("redefinition of unused") != -1: m = "redefs" else: m = "misc" From warner at users.sourceforge.net Wed Sep 20 06:33:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Wed, 20 Sep 2006 06:33:37 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.742,1.743 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29813 Modified Files: ChangeLog Log Message: [project @ steps.python.PyFlakes: make it compatible with python-2.2] Original author: warner at lothar.com Date: 2006-09-20 06:31:29 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.742 retrieving revision 1.743 diff -u -d -r1.742 -r1.743 --- ChangeLog 19 Sep 2006 18:46:55 -0000 1.742 +++ ChangeLog 20 Sep 2006 06:33:35 -0000 1.743 @@ -1,6 +1,8 @@ 2006-09-19 Brian Warner * buildbot/steps/python.py (PyFlakes): refactor, add summary logs + (PyFlakes.createSummary): make it compatible with python-2.2 + * buildbot/test/test_steps.py (Python.testPyFlakes): add a test for at least the output-parsing parts of PyFlakes From warner at users.sourceforge.net Fri Sep 22 05:46:30 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 22 Sep 2006 05:46:30 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.743,1.744 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25701 Modified Files: ChangeLog Log Message: [project @ update example config files to match buildbot.steps.* move] Original author: warner at lothar.com Date: 2006-09-22 05:44:48 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.743 retrieving revision 1.744 diff -u -d -r1.743 -r1.744 --- ChangeLog 20 Sep 2006 06:33:35 -0000 1.743 +++ ChangeLog 22 Sep 2006 05:46:28 -0000 1.744 @@ -1,3 +1,11 @@ +2006-09-21 Brian Warner + + * buildbot/scripts/sample.cfg: update to use new BuildStep classes + from buildbot.steps + * docs/examples/glib_master.cfg: same + * docs/examples/hello.cfg: same + * docs/examples/twisted_master.cfg: same + 2006-09-19 Brian Warner * buildbot/steps/python.py (PyFlakes): refactor, add summary logs From warner at users.sourceforge.net Fri Sep 22 05:46:30 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 22 Sep 2006 05:46:30 +0000 Subject: [Buildbot-commits] buildbot/buildbot/scripts sample.cfg,1.11,1.12 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/scripts In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25701/buildbot/scripts Modified Files: sample.cfg Log Message: [project @ update example config files to match buildbot.steps.* move] Original author: warner at lothar.com Date: 2006-09-22 05:44:48 Index: sample.cfg =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/scripts/sample.cfg,v retrieving revision 1.11 retrieving revision 1.12 diff -u -d -r1.11 -r1.12 --- sample.cfg 21 Jul 2006 07:40:42 -0000 1.11 +++ sample.cfg 22 Sep 2006 05:46:28 -0000 1.12 @@ -93,13 +93,16 @@ builders = [] -from buildbot.process import step, factory +from buildbot.process import factory +from buildbot.steps.source import CVS +from buildbot.steps.shell import Compile +from buildbot.steps.python_twisted import Trial f1 = factory.BuildFactory() -f1.addStep(step.CVS, +f1.addStep(CVS, cvsroot=cvsroot, cvsmodule=cvsmodule, login="", mode="copy") -f1.addStep(step.Compile, command=["./setup.py", "build"]) -f1.addStep(step_twisted.Trial, testpath=".") +f1.addStep(Compile, command=["./setup.py", "build"]) +f1.addStep(Trial, testpath=".") b1 = {'name': "buildbot-full", 'slavename': "bot1name", From warner at users.sourceforge.net Fri Sep 22 05:46:30 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 22 Sep 2006 05:46:30 +0000 Subject: [Buildbot-commits] buildbot/docs/examples glib_master.cfg, 1.4, 1.5 hello.cfg, 1.9, 1.10 twisted_master.cfg, 1.43, 1.44 Message-ID: Update of /cvsroot/buildbot/buildbot/docs/examples In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25701/docs/examples Modified Files: glib_master.cfg hello.cfg twisted_master.cfg Log Message: [project @ update example config files to match buildbot.steps.* move] Original author: warner at lothar.com Date: 2006-09-22 05:44:48 Index: glib_master.cfg =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/examples/glib_master.cfg,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- glib_master.cfg 1 Oct 2004 01:56:29 -0000 1.4 +++ glib_master.cfg 22 Sep 2006 05:46:28 -0000 1.5 @@ -1,7 +1,7 @@ #! /usr/bin/python from buildbot.changes.freshcvs import FreshCVSSource -from buildbot.process.step import CVS +from buildbot.steps.source import CVS from buildbot.process.factory import GNUAutoconf, s from buildbot.status import html Index: hello.cfg =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/examples/hello.cfg,v retrieving revision 1.9 retrieving revision 1.10 diff -u -d -r1.9 -r1.10 --- hello.cfg 1 Oct 2004 01:56:29 -0000 1.9 +++ hello.cfg 22 Sep 2006 05:46:28 -0000 1.10 @@ -1,7 +1,9 @@ #! /usr/bin/python from buildbot import master -from buildbot.process import factory, step +from buildbot.process import factory +from buildbot.steps.source import CVS, SVN, Darcs, Arch +from buildbot.steps.shell import Configure, Compile, Test from buildbot.status import html, client from buildbot.changes.pb import PBChangeSource s = factory.s @@ -16,16 +18,16 @@ if 1: steps = [ - s(step.CVS, + s(CVS, cvsroot="/usr/home/warner/stuff/Projects/BuildBot/demo/Repository", cvsmodule="hello", mode="clobber", checkoutDelay=6, alwaysUseLatest=True, ), - s(step.Configure), - s(step.Compile), - s(step.Test, command=["make", "check"]), + s(Configure), + s(Compile), + s(Test, command=["make", "check"]), ] b1 = {"name": "cvs-hello", "slavename": "bot1", @@ -37,13 +39,13 @@ if 1: svnrep="file:///usr/home/warner/stuff/Projects/BuildBot/demo/SVN-Repository" steps = [ - s(step.SVN, + s(SVN, svnurl=svnrep+"/hello", mode="update", ), - s(step.Configure), - s(step.Compile), - s(step.Test, command=["make", "check"]), + s(Configure), + s(Compile), + s(Test, command=["make", "check"]), ] b1 = {"name": "svn-hello", "slavename": "bot1", @@ -54,13 +56,13 @@ if 1: steps = [ - s(step.Darcs, + s(Darcs, repourl="http://localhost/~warner/hello-darcs", mode="copy", ), - s(step.Configure, command=["/bin/sh", "./configure"]), - s(step.Compile), - s(step.Test, command=["make", "check"]), + s(Configure, command=["/bin/sh", "./configure"]), + s(Compile), + s(Test, command=["make", "check"]), ] b1 = {"name": "darcs-hello", "slavename": "bot1", @@ -71,14 +73,14 @@ if 1: steps = [ - s(step.Arch, + s(Arch, url="http://localhost/~warner/hello-arch", version="gnu-hello--release--2.1.1", mode="copy", ), - s(step.Configure), - s(step.Compile), - s(step.Test, command=["make", "check"]), + s(Configure), + s(Compile), + s(Test, command=["make", "check"]), ] b1 = {"name": "arch-hello", "slavename": "bot1", Index: twisted_master.cfg =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/examples/twisted_master.cfg,v retrieving revision 1.43 retrieving revision 1.44 diff -u -d -r1.43 -r1.44 --- twisted_master.cfg 21 Aug 2006 03:38:05 -0000 1.43 +++ twisted_master.cfg 22 Sep 2006 05:46:28 -0000 1.44 @@ -14,7 +14,8 @@ from buildbot import master from buildbot.changes.pb import PBChangeSource from buildbot.scheduler import Scheduler, Try_Userpass -from buildbot.process import step +from buildbot.steps.source import SVN +from buildbot.steps.shell import ShellCommand from buildbot.process.factory import s from buildbot.process.process_twisted import \ QuickTwistedBuildFactory, \ @@ -59,18 +60,18 @@ if 0: # always build on trunk svnurl = "svn://svn.twistedmatrix.com/svn/Twisted/trunk" - source_update = s(step.SVN, svnurl=svnurl, mode="update") - source_copy = s(step.SVN, svnurl=svnurl, mode="copy") - source_export = s(step.SVN, svnurl=svnurl, mode="export") + source_update = s(SVN, svnurl=svnurl, mode="update") + source_copy = s(SVN, svnurl=svnurl, mode="copy") + source_export = s(SVN, svnurl=svnurl, mode="export") else: # for build-on-branch, we use these instead baseURL = "svn://svn.twistedmatrix.com/svn/Twisted/" defaultBranch = "trunk" - source_update = s(step.SVN, baseURL=baseURL, defaultBranch=defaultBranch, + source_update = s(SVN, baseURL=baseURL, defaultBranch=defaultBranch, mode="update") - source_copy = s(step.SVN, baseURL=baseURL, defaultBranch=defaultBranch, + source_copy = s(SVN, baseURL=baseURL, defaultBranch=defaultBranch, mode="copy") - source_export = s(step.SVN, baseURL=baseURL, defaultBranch=defaultBranch, + source_export = s(SVN, baseURL=baseURL, defaultBranch=defaultBranch, mode="export") @@ -175,7 +176,7 @@ jf = TwistedReactorsBuildFactory(source_copy, python="python2.4", reactors=["default"]) -jf.steps.insert(0, s(step.ShellCommand, workdir=".", +jf.steps.insert(0, s(ShellCommand, workdir=".", command=["ktrace", "rm", "-rf", "Twisted"])) b24osx = {'name': "OS-X", 'slavename': "bot-jerub", From warner at users.sourceforge.net Fri Sep 22 05:58:01 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 22 Sep 2006 05:58:01 +0000 Subject: [Buildbot-commits] buildbot/docs/examples twisted_master.cfg, 1.44, 1.45 Message-ID: Update of /cvsroot/buildbot/buildbot/docs/examples In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29931/docs/examples Modified Files: twisted_master.cfg Log Message: [project @ update twisted_master.cfg to current usage] Original author: warner at lothar.com Date: 2006-09-22 05:57:24 Index: twisted_master.cfg =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/examples/twisted_master.cfg,v retrieving revision 1.44 retrieving revision 1.45 diff -u -d -r1.44 -r1.45 --- twisted_master.cfg 22 Sep 2006 05:46:28 -0000 1.44 +++ twisted_master.cfg 22 Sep 2006 05:57:58 -0000 1.45 @@ -123,6 +123,26 @@ } builders.append(b24) +bosx24 = {'name': 'osx-2.4', + 'slavenames': ['bot-exarkun-osx'], + 'builddir': 'full2.4-exarkun-osx', + 'factory': FullTwistedBuildFactory(source_copy, + python=["python2.4", "-Wall"], + # use -Werror soon + compileOpts=b24compile_opts, + runTestsRandomly=1), + } +builders.append(bosx24) + +bpollosx24 = {'name': 'poll-osx-2.4', + 'slavenames': ['bot-exarkun-osx-2'], + 'builddir': 'full2.4-poll-exarkun-osx', + 'factory': TwistedReactorsBuildFactory(source_copy, + python="python2.4", + reactors=['poll']), + } +builders.append(bpollosx24) + b25debian = { 'name': 'full-2.5-debian', 'slavenames': ['bot-idnar-debian'], @@ -257,6 +277,14 @@ } builders.append(b24freebsd) +bpypyc = {'name': 'pypy-c', + 'slavename': 'bot-jerub-pypy', + 'builddir': 'pypy-c', + 'factory': TwistedReactorsBuildFactory(source_copy, + python="pypy-c", + reactors=["default"])} +builders.append(bpypyc) + # b24threadless = {'name': 'threadless', # 'slavename': 'bot-threadless', # 'builddir': 'debian-threadless-2.4', @@ -294,10 +322,12 @@ c['status'].append(html.Waterfall(distrib_port=p)) else: c['status'].append(html.Waterfall(http_port=9988)) -if really: - c['status'].append(words.IRC(host="irc.us.freenode.net", - nick='buildbot', - channels=["twisted"])) +# IRC disabled until I figure out why it won't connect successfully +# -warner 22-Aug-2006 +#if really: +# c['status'].append(words.IRC(host="irc.us.freenode.net", +# nick='buildbot', +# channels=["twisted"])) c['debugPassword'] = private.debugPassword #c['interlocks'] = [("do-deb", ["full-2.2"], ["debuild"])] From warner at users.sourceforge.net Fri Sep 22 05:58:00 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 22 Sep 2006 05:58:00 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.744,1.745 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29931 Modified Files: ChangeLog Log Message: [project @ update twisted_master.cfg to current usage] Original author: warner at lothar.com Date: 2006-09-22 05:57:24 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.744 retrieving revision 1.745 diff -u -d -r1.744 -r1.745 --- ChangeLog 22 Sep 2006 05:46:28 -0000 1.744 +++ ChangeLog 22 Sep 2006 05:57:58 -0000 1.745 @@ -4,7 +4,7 @@ from buildbot.steps * docs/examples/glib_master.cfg: same * docs/examples/hello.cfg: same - * docs/examples/twisted_master.cfg: same + * docs/examples/twisted_master.cfg: same, update to current usage 2006-09-19 Brian Warner From warner at users.sourceforge.net Mon Sep 25 02:43:58 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 02:43:58 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.745,1.746 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25647 Modified Files: ChangeLog Log Message: [project @ improve recording/announcing the buildslave's name for each build] Original author: warner at lothar.com Date: 2006-09-25 02:40:54 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.745 retrieving revision 1.746 diff -u -d -r1.745 -r1.746 --- ChangeLog 22 Sep 2006 05:57:58 -0000 1.745 +++ ChangeLog 25 Sep 2006 02:43:56 -0000 1.746 @@ -1,3 +1,15 @@ +2006-09-24 Brian Warner + + * buildbot/process/base.py (Build.setupSlaveBuilder): tell our + BuildStatus about the buildslave name at the *beginning* of the + build, rather than at the end. Thanks to Alexander Lorenz for the + patch. + * buildbot/status/html.py (StatusResourceBuild.body): always + include the slavename in the build page, not just when the build + has finished. + * buildbot/status/mail.py (MailNotifier.buildMessage): include the + slavename in the email message + 2006-09-21 Brian Warner * buildbot/scripts/sample.cfg: update to use new BuildStep classes From warner at users.sourceforge.net Mon Sep 25 02:43:58 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 02:43:58 +0000 Subject: [Buildbot-commits] buildbot/buildbot/process base.py,1.70,1.71 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/process In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25647/buildbot/process Modified Files: base.py Log Message: [project @ improve recording/announcing the buildslave's name for each build] Original author: warner at lothar.com Date: 2006-09-25 02:40:54 Index: base.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/process/base.py,v retrieving revision 1.70 retrieving revision 1.71 diff -u -d -r1.70 -r1.71 --- base.py 6 Sep 2006 00:41:55 -0000 1.70 +++ base.py 25 Sep 2006 02:43:56 -0000 1.71 @@ -273,6 +273,7 @@ def setupSlaveBuilder(self, slavebuilder): self.slavebuilder = slavebuilder self.slavename = slavebuilder.slave.slavename + self.build_status.setSlavename(self.slavename) self.setProperty("slavename", self.slavename) def startBuild(self, build_status, expectations, slavebuilder): @@ -543,7 +544,6 @@ self.results = results log.msg(" %s: build finished" % self) - self.build_status.setSlavename(self.slavename) self.build_status.setText(text) self.build_status.setColor(color) self.build_status.setResults(results) From warner at users.sourceforge.net Mon Sep 25 02:43:59 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 02:43:59 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status html.py, 1.90, 1.91 mail.py, 1.27, 1.28 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv25647/buildbot/status Modified Files: html.py mail.py Log Message: [project @ improve recording/announcing the buildslave's name for each build] Original author: warner at lothar.com Date: 2006-09-25 02:40:54 Index: html.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/html.py,v retrieving revision 1.90 retrieving revision 1.91 diff -u -d -r1.90 -r1.91 --- html.py 6 Sep 2006 00:41:55 -0000 1.90 +++ html.py 25 Sep 2006 02:43:56 -0000 1.91 @@ -324,10 +324,10 @@ projectName) # the color in the following line gives python-mode trouble data += ("

Build %s:#%d

\n" - "

Reason:

\n%s\n" % (self.status.getURLForThing(b.getBuilder()), - b.getBuilder().getName(), b.getNumber(), - html.escape(b.getReason()))) + b.getBuilder().getName(), b.getNumber())) + data += "

Buildslave:

\n %s\n" % html.escape(b.getSlavename()) + data += "

Reason:

\n%s\n" % html.escape(b.getReason()) branch, revision, patch = b.getSourceStamp() data += "

SourceStamp:

\n" @@ -345,7 +345,6 @@ data += "
  • build of most recent revision
    • \n" data += " \n" if b.isFinished(): - data += "

    Buildslave: %s

    \n" % html.escape(b.getSlavename()) data += "

    Results:

    \n" data += " ".join(b.getText()) + "\n" if b.getTestResults(): Index: mail.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/mail.py,v retrieving revision 1.27 retrieving revision 1.28 diff -u -d -r1.27 -r1.28 --- mail.py 6 Sep 2006 00:41:55 -0000 1.27 +++ mail.py 25 Sep 2006 02:43:56 -0000 1.28 @@ -239,6 +239,7 @@ if url: text += "Buildbot URL: %s\n\n" % urllib.quote(url, '/:') + text += "Buildslave for this Build: %s\n\n" % build.getSlavename() text += "Build Reason: %s\n" % build.getReason() patch = None From warner at users.sourceforge.net Mon Sep 25 06:51:38 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 06:51:38 +0000 Subject: [Buildbot-commits] buildbot/buildbot/changes mail.py,1.21,1.22 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/changes In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv28646/buildbot/changes Modified Files: mail.py Log Message: [project @ fix the bonsai mail parser] Original author: warner at lothar.com Date: 2006-09-25 06:50:24 Index: mail.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/mail.py,v retrieving revision 1.21 retrieving revision 1.22 diff -u -d -r1.21 -r1.22 --- mail.py 6 Sep 2006 00:41:54 -0000 1.21 +++ mail.py 25 Sep 2006 06:51:36 -0000 1.22 @@ -229,7 +229,7 @@ # actually make a new directory part of the build process. That's my story # and I'm sticking to it. -def parseBonsaiMail(self, fd, prefix=None): +def parseBonsaiMail(self, fd, prefix=None, sep="/"): """Parse mail sent by the Bonsai cvs loginfo script.""" msg = Message(fd) @@ -274,6 +274,8 @@ if module and file: path = "%s/%s" % (module, file) files.append(path) + sticky = items[7] + branch = items[8] # if no files changed, return nothing if not files: @@ -289,7 +291,8 @@ comments = comments.rstrip() + "\n" # return buildbot Change object - return changes.Change(who, files, comments, when=timestamp) + return changes.Change(who, files, comments, when=timestamp, branch=branch) + class MaildirSource(maildirtwisted.MaildirTwisted, base.ChangeSource): From warner at users.sourceforge.net Mon Sep 25 06:51:38 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 06:51:38 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.746,1.747 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv28646 Modified Files: ChangeLog Log Message: [project @ fix the bonsai mail parser] Original author: warner at lothar.com Date: 2006-09-25 06:50:24 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.746 retrieving revision 1.747 diff -u -d -r1.746 -r1.747 --- ChangeLog 25 Sep 2006 02:43:56 -0000 1.746 +++ ChangeLog 25 Sep 2006 06:51:36 -0000 1.747 @@ -1,5 +1,8 @@ 2006-09-24 Brian Warner + * buildbot/changes/mail.py (parseBonsaiMail): fix the parser. + Thanks to Robert Helmer for the patch. + * buildbot/process/base.py (Build.setupSlaveBuilder): tell our BuildStatus about the buildslave name at the *beginning* of the build, rather than at the end. Thanks to Alexander Lorenz for the From warner at users.sourceforge.net Mon Sep 25 07:04:26 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:04:26 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.747,1.748 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv1222 Modified Files: ChangeLog Log Message: [project @ Tinderbox-compatible email-sending status plugin, from Ben Hearsum] Original author: warner at lothar.com Date: 2006-09-25 07:03:39 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.747 retrieving revision 1.748 diff -u -d -r1.747 -r1.748 --- ChangeLog 25 Sep 2006 06:51:36 -0000 1.747 +++ ChangeLog 25 Sep 2006 07:04:23 -0000 1.748 @@ -1,3 +1,10 @@ +2006-09-25 Brian Warner + + * buildbot/status/tinderbox.py (TinderboxMailNotifier): Ben + Hearsum contributed a status plugin which sends email in the same + format that Tinderbox does: this allows a number of tinderbox + tools to be driven by Buildbot instead. Thanks Ben! + 2006-09-24 Brian Warner * buildbot/changes/mail.py (parseBonsaiMail): fix the parser. From warner at users.sourceforge.net Mon Sep 25 07:04:26 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:04:26 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status tinderbox.py,NONE,1.1 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv1222/buildbot/status Added Files: tinderbox.py Log Message: [project @ Tinderbox-compatible email-sending status plugin, from Ben Hearsum] Original author: warner at lothar.com Date: 2006-09-25 07:03:39 --- NEW FILE: tinderbox.py --- from email.Message import Message from email.Utils import formatdate from twisted.internet import defer from buildbot import interfaces from buildbot.twcompat import implements from buildbot.status import base, mail from buildbot.status.builder import SUCCESS, WARNINGS import zlib, bz2, base64 # TODO: docs, maybe a test of some sort just to make sure it actually imports # and can format email without raising an exception. class TinderboxMailNotifier(mail.MailNotifier): """This is a Tinderbox status notifier. It can send e-mail to a number of different tinderboxes or people. E-mails are sent at the beginning and upon completion of each build. It can be configured to send out e-mails for only certain builds. The most basic usage is as follows:: TinderboxMailNotifier(fromaddr="buildbot at localhost", tree="MyTinderboxTree", extraRecipients=["tinderboxdaemon at host.org"]) The builder name (as specified in master.cfg) is used as the "build" tinderbox option. """ if implements: implements(interfaces.IEmailSender) else: __implements__ = (interfaces.IEmailSender, base.StatusReceiverMultiService.__implements__) compare_attrs = ["extraRecipients", "fromaddr", "categories", "builders", "addLogs", "relayhost", "subject", "binaryURL", "tree", "logCompression"] def __init__(self, fromaddr, tree, extraRecipients, categories=None, builders=None, relayhost="localhost", subject="buildbot %(result)s in %(builder)s", binaryURL="", logCompression=""): """ @type fromaddr: string @param fromaddr: the email address to be used in the 'From' header. @type tree: string @param tree: The Tinderbox tree to post to. @type extraRecipients: tuple of string @param extraRecipients: E-mail addresses of recipients. This should at least include the tinderbox daemon. @type categories: list of strings @param categories: a list of category names to serve status information for. Defaults to None (all categories). Use either builders or categories, but not both. @type builders: list of strings @param builders: a list of builder names for which mail should be sent. Defaults to None (send mail for all builds). Use either builders or categories, but not both. @type relayhost: string @param relayhost: the host to which the outbound SMTP connection should be made. Defaults to 'localhost' @type subject: string @param subject: a string to be used as the subject line of the message. %(builder)s will be replaced with the name of the %builder which provoked the message. This parameter is not significant for the tinderbox daemon. @type binaryURL: string @param binaryURL: If specified, this should be the location where final binary for a build is located. (ie. http://www.myproject.org/nightly/08-08-2006.tgz) It will be posted to the Tinderbox. @type logCompression: string @param logCompression: The type of compression to use on the log. Valid options are"bzip2" and "gzip". gzip is only known to work on Python 2.4 and above. """ mail.MailNotifier.__init__(self, fromaddr, categories=categories, builders=builders, relayhost=relayhost, subject=subject, extraRecipients=extraRecipients, sendToInterestedUsers=False) self.tree = tree self.binaryURL = binaryURL self.logCompression = logCompression def buildStarted(self, name, build): self.buildMessage(name, build, "building") def buildMessage(self, name, build, results): text = "" # shortform t = "tinderbox:" text += "%s tree: %s\n" % (t, self.tree) # the start time # getTimes() returns a fractioned time that tinderbox doesn't understand text += "%s builddate: %s\n" % (t, int(build.getTimes()[0])) text += "%s status: " % t if results == "building": text += "building\n" elif results == WARNINGS or results == SUCCESS: text += "success\n" else: text += "busted\n" text += "%s build: %s\n" % (t, name) text += "%s errorparser: unix\n" % t # always use the unix errorparser # if the build just started... if results == "building": text += "%s END\n" % t # if the build finished... else: text += "%s binaryurl: %s\n" % (t, self.binaryURL) text += "%s logcompression: %s\n" % (t, self.logCompression) text += "%s logencoding: base64\n" % t text += "%s END\n\n" % t # logs will always be appended for log in build.getLogs(): l = "" if self.logCompression == "bzip2": l = bz2.compress(log.getText()) elif self.logCompression == "gzip": l = zlib.compress(log.getText()) else: l = log.getText() text += base64.encodestring(l) text += "\n" m = Message() m.set_payload(text) m['Date'] = formatdate(localtime=True) m['Subject'] = self.subject m['From'] = self.fromaddr # m['To'] is added later d = defer.DeferredList([]) d.addCallback(self._gotRecipients, self.extraRecipients, m) return d From warner at users.sourceforge.net Mon Sep 25 07:21:04 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:21:04 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.748,1.749 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv7627 Modified Files: ChangeLog Log Message: [project @ add Ben Hearsum's BonsaiPoller changesource] Original author: warner at lothar.com Date: 2006-09-25 07:20:26 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.748 retrieving revision 1.749 diff -u -d -r1.748 -r1.749 --- ChangeLog 25 Sep 2006 07:04:23 -0000 1.748 +++ ChangeLog 25 Sep 2006 07:21:02 -0000 1.749 @@ -1,5 +1,9 @@ 2006-09-25 Brian Warner + * buildbot/changes/bonsaipoller.py (BonsaiPoller): Ben also + contributed a Change Source that polls a Bonsai server (a + kind of web-based viewcvs CGI script). + * buildbot/status/tinderbox.py (TinderboxMailNotifier): Ben Hearsum contributed a status plugin which sends email in the same format that Tinderbox does: this allows a number of tinderbox From warner at users.sourceforge.net Mon Sep 25 07:21:04 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:21:04 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.79,1.80 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv7627/docs Modified Files: buildbot.texinfo Log Message: [project @ add Ben Hearsum's BonsaiPoller changesource] Original author: warner at lothar.com Date: 2006-09-25 07:20:26 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.79 retrieving revision 1.80 diff -u -d -r1.79 -r1.80 --- buildbot.texinfo 15 Sep 2006 14:50:17 -0000 1.79 +++ buildbot.texinfo 25 Sep 2006 07:21:02 -0000 1.80 @@ -152,6 +152,7 @@ * Other mail notification ChangeSources:: * PBChangeSource:: * P4Source:: +* BonsaiPoller:: Build Process @@ -2531,6 +2532,7 @@ * Other mail notification ChangeSources:: * PBChangeSource:: * P4Source:: +* BonsaiPoller:: @end menu @node Choosing ChangeSources, CVSToys - PBService, Change Sources, Change Sources @@ -2753,7 +2755,7 @@ @end table - at node P4Source, , PBChangeSource, Change Sources + at node P4Source, BonsaiPoller, PBChangeSource, Change Sources @subsection P4Source @csindex buildbot.changes.p4poller.P4Source @@ -2804,6 +2806,23 @@ )) @end example + at node BonsaiPoller, , P4Source, Change Sources + at subsection BonsaiPoller + + at csindex buildbot.changes.bonsaipoller.BonsaiPoller + +The @code{BonsaiPoller} periodically polls a Bonsai server. This is a +CGI script accessed through a web server that provides information +about a CVS tree, for example the Mozilla bonsai server at + at uref{http://bonsai.mozilla.org}. Bonsai servers are usable by both +humans and machines. In this case, the buildbot's change source forms +a query which asks about any files in the specified branch which have +changed since the last query. + +Please take a look at the BonsaiPoller docstring for details about the +arguments it accepts. + + @node Build Process, Status Delivery, Getting Source Code Changes, Top @chapter Build Process From warner at users.sourceforge.net Mon Sep 25 07:21:04 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:21:04 +0000 Subject: [Buildbot-commits] buildbot/buildbot/changes bonsaipoller.py, NONE, 1.1 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/changes In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv7627/buildbot/changes Added Files: bonsaipoller.py Log Message: [project @ add Ben Hearsum's BonsaiPoller changesource] Original author: warner at lothar.com Date: 2006-09-25 07:20:26 --- NEW FILE: bonsaipoller.py --- import time from urllib import urlopen from xml.dom import minidom, Node from twisted.python import log, failure from twisted.internet import defer, reactor from twisted.internet.task import LoopingCall from buildbot.changes import base, changes class BonsaiResult: """I hold a list of CiNodes""" def __init__(self): self.nodes = [] class CiNode: """I hold information about one Ci node, including a list of files""" def __init__(self): self.log = "" self.who = "" self.date = "" self.files = [] class FileNode: """I hold information about one file node""" def __init__(self): self.revision = "" self.filename = "" class BonsaiParser: """I parse the XML result from a Bonsai cvsquery. Typical usage is as follows:: bp = BonsaiParser(urlopen(bonsaiURL)) data = bp.getData() for cinode in data.nodes: print cinode.who, cinode.log, cinode.date for file in cinode.files: print file.filename, file.revision """ def __init__(self, bonsaiQuery): try: self.dom = minidom.parse(bonsaiQuery) except: self.dom = None return self.currentCiNode = None self.currentFileNode = None def getData(self): """I return data from a Bonsai cvsquery""" data = BonsaiResult() while self._nextCiNode(): ci = CiNode() ci.log = self._getLog() ci.who = self._getWho() ci.date = self._getDate() while self._nextFileNode(): fn = FileNode() fn.revision = self._getRevision() fn.filename = self._getFilename() ci.files.append(fn) data.nodes.append(ci) return data def _nextCiNode(self): try: # first node? if not self.currentCiNode: # every other sibling is a , so jump 2 ahead self.currentCiNode = self.dom.getElementsByTagName("ci")[0] else: self.currentCiNode = self.currentCiNode.nextSibling.nextSibling except (AttributeError,IndexError): self.currentCiNode = None if self.currentCiNode: return True else: return False def _getLog(self): log = "" for child in self.currentCiNode.childNodes: if child.nodeType == Node.ELEMENT_NODE and child.tagName == "log": log = child.firstChild.data return str(log) def _getWho(self): """Returns the e-mail address of the commit'er""" return str(self.currentCiNode.getAttribute("who").replace("%", "@")) def _getDate(self): """Returns the date (unix time) of the commit""" return int(self.currentCiNode.getAttribute("date")) def _firstFileNode(self): for child in self.currentCiNode.childNodes: if child.nodeType == Node.ELEMENT_NODE and child.tagName == "files": # child is now the element for c in child.childNodes: if c.nodeType == Node.ELEMENT_NODE and c.tagName == "f": return c def _nextFileNode(self): # every other sibling is a , so go two past the current one try: # first node? if not self.currentFileNode: self.currentFileNode = self._firstFileNode() else: self.currentFileNode = self.currentFileNode.nextSibling.nextSibling except AttributeError: self.currentFileNode = None if self.currentFileNode: return True else: return False def _getFilename(self): return str(self.currentFileNode.firstChild.data) def _getRevision(self): return float(self.currentFileNode.getAttribute("rev")) class BonsaiPoller(base.ChangeSource): """This source will poll a bonsai server for changes and submit them to the change master.""" compare_attrs = ["bonsaiURL", "pollInterval", "tree", "module", "branch", "cvsroot"] parent = None # filled in when we're added loop = None volatile = ['loop'] working = False def __init__(self, bonsaiURL, module, branch, tree="default", cvsroot="/cvsroot", pollInterval=30): """ @type bonsaiURL: string @param bonsaiURL: The base URL of the Bonsai server (ie. http://bonsai.mozilla.org) @type module: string @param module: The module to look for changes in. Commonly this is 'all' @type branch: string @param branch: The branch to look for changes in. This must match the 'branch' option for the Scheduler. @type tree: string @param tree: The tree to look for changes in. Commonly this is 'all' @type cvsroot: string @param cvsroot: The cvsroot of the repository. Usually this is '/cvsroot' @type pollInterval: int @param pollInterval: The time (in seconds) between queries for changes """ self.bonsaiURL = bonsaiURL self.module = module self.branch = branch self.tree = tree self.cvsroot = cvsroot self.pollInterval = pollInterval self.lastChange = time.time() self.lastPoll = time.time() def startService(self): self.loop = LoopingCall(self.poll) base.ChangeSource.startService(self) reactor.callLater(0, self.loop.start, self.pollInterval) def stopService(self): self.loop.stop() return base.ChangeSource.stopService(self) def describe(self): str = "" str += "Getting changes from the Bonsai service running at %s " \ % self.bonsaiURL str += "
    Using tree: %s, branch: %s, and module: %s" % (self.tree, \ self.branch, self.module) return str def poll(self): if self.working: log.msg("Not polling Bonsai because last poll is still working") else: self.working = True d = self._get_changes() d.addCallback(self._process_changes) d.addBoth(self._finished) return def _finished(self, res): assert self.working self.working = False # check for failure if isinstance(res, failure.Failure): log.msg("Bonsai poll failed: %s" % res) return res def _get_changes(self): args = ["treeid=%s" % self.tree, "module=%s" % self.module, "branch=%s" % self.branch, "branchtype=match", "sortby=Date", "date=explicit", "mindate=%d" % self.lastChange, "maxdate=%d" % int(time.time()), "cvsroot=%s" % self.cvsroot, "xml=1"] # build the bonsai URL url = self.bonsaiURL url += "/cvsquery.cgi?" url += "&".join(args) log.msg("Polling Bonsai tree at %s" % url) self.lastPoll = time.time() # get the page, in XML format return defer.maybeDeferred(urlopen, url) def _process_changes(self, query): bp = BonsaiParser(query) files = [] data = bp.getData() for cinode in data.nodes: for file in cinode.files: files.append(file) c = changes.Change(who = cinode.who, files = files, comments = cinode.log, when = cinode.date, branch = self.branch) self.parent.addChange(c) self.lastChange = self.lastPoll From warner at users.sourceforge.net Mon Sep 25 07:46:58 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:46:58 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.66,1.67 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17754/buildbot/slave Modified Files: commands.py Log Message: [project @ initial support for Monotone, by Nathaniel Smith] Original author: warner at lothar.com Date: 2006-09-25 07:45:16 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.66 retrieving revision 1.67 diff -u -d -r1.66 -r1.67 --- commands.py 15 Sep 2006 14:49:46 -0000 1.66 +++ commands.py 25 Sep 2006 07:46:56 -0000 1.67 @@ -14,7 +14,7 @@ # this used to be a CVS $-style "Revision" auto-updated keyword, but since I # moved to Darcs as the primary repository, this is updated manually each # time this file is changed. The last cvs_ver that was here was 1.51 . -command_version = "2.1" +command_version = "2.2" # version history: # >=1.17: commands are interruptable @@ -32,6 +32,7 @@ # ShellCommand accepts new arguments (logfiles=, initialStdin=, # keepStdinOpen=) and no longer accepts stdin=) # (release 0.7.4) +# >= 2.2: added monotone, uploadFile, and downloadFile class CommandInterrupted(Exception): pass @@ -1530,6 +1531,112 @@ registerSlaveCommand("darcs", Darcs, command_version) +class Monotone(SourceBase): + """Monotone-specific VC operation. In addition to the arguments handled + by SourceBase, this command reads the following keys: + + ['server_addr'] (required): the address of the server to pull from + ['branch'] (required): the branch the revision is on + ['db_path'] (required): the local database path to use + ['revision'] (required): the revision to check out + ['monotone']: (required): path to monotone executable + """ + + header = "monotone operation" + + def setup(self, args): + SourceBase.setup(self, args) + self.server_addr = args["server_addr"] + self.branch = args["branch"] + self.db_path = args["db_path"] + self.revision = args["revision"] + self.monotone = args["monotone"] + self._made_fulls = False + self._pull_timeout = args["timeout"] + + def _makefulls(self): + if not self._made_fulls: + basedir = self.builder.basedir + self.full_db_path = os.path.join(basedir, self.db_path) + self.full_srcdir = os.path.join(basedir, self.srcdir) + self._made_fulls = True + + def sourcedirIsUpdateable(self): + self._makefulls() + if os.path.exists(os.path.join(self.full_srcdir, + ".buildbot_patched")): + return False + return (os.path.isfile(self.full_db_path) + and os.path.isdir(os.path.join(self.full_srcdir, "MT"))) + + def doVCUpdate(self): + return self._withFreshDb(self._doUpdate) + + def _doUpdate(self): + # update: possible for mode in ('copy', 'update') + command = [self.monotone, "update", + "-r", self.revision, + "-b", self.branch] + c = ShellCommand(self.builder, command, self.full_srcdir, + sendRC=False, timeout=self.timeout) + self.command = c + return c.start() + + def doVCFull(self): + return self._withFreshDb(self._doFull) + + def _doFull(self): + command = [self.monotone, "--db=" + self.full_db_path, + "checkout", + "-r", self.revision, + "-b", self.branch, + self.full_srcdir] + c = ShellCommand(self.builder, command, self.builder.basedir, + sendRC=False, timeout=self.timeout) + self.command = c + return c.start() + + def _withFreshDb(self, callback): + self._makefulls() + # first ensure the db exists and is usable + if os.path.isfile(self.full_db_path): + # already exists, so run 'db migrate' in case monotone has been + # upgraded under us + command = [self.monotone, "db", "migrate", + "--db=" + self.full_db_path] + else: + # We'll be doing an initial pull, so up the timeout to 3 hours to + # make sure it will have time to complete. + self._pull_timeout = max(self._pull_timeout, 3 * 60 * 60) + self.sendStatus({"header": "creating database %s\n" + % (self.full_db_path,)}) + command = [self.monotone, "db", "init", + "--db=" + self.full_db_path] + c = ShellCommand(self.builder, command, self.builder.basedir, + sendRC=False, timeout=self.timeout) + self.command = c + d = c.start() + d.addCallback(self._abandonOnFailure) + d.addCallback(self._didDbInit) + d.addCallback(self._didPull, callback) + return d + + def _didDbInit(self, res): + command = [self.monotone, "--db=" + self.full_db_path, + "pull", "--ticker=dot", self.server_addr, self.branch] + c = ShellCommand(self.builder, command, self.builder.basedir, + sendRC=False, timeout=self._pull_timeout) + self.sendStatus({"header": "pulling %s from %s\n" + % (self.branch, self.server_addr)}) + self.command = c + return c.start() + + def _didPull(self, res, callback): + return callback() + +registerSlaveCommand("monotone", Monotone, command_version) + + class Git(SourceBase): """Git specific VC operation. In addition to the arguments handled by SourceBase, this command reads the following keys: From warner at users.sourceforge.net Mon Sep 25 07:46:58 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:46:58 +0000 Subject: [Buildbot-commits] buildbot/buildbot/changes monotone.py,NONE,1.1 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/changes In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17754/buildbot/changes Added Files: monotone.py Log Message: [project @ initial support for Monotone, by Nathaniel Smith] Original author: warner at lothar.com Date: 2006-09-25 07:45:16 --- NEW FILE: monotone.py --- import tempfile import os import os.path from cStringIO import StringIO from twisted.python import log from twisted.application import service from twisted.internet import defer, protocol, error, reactor from twisted.internet.task import LoopingCall from buildbot import util from buildbot.interfaces import IChangeSource from buildbot.changes.changes import Change class _MTProtocol(protocol.ProcessProtocol): def __init__(self, deferred, cmdline): self.cmdline = cmdline self.deferred = deferred self.s = StringIO() def errReceived(self, text): log.msg("stderr: %s" % text) def outReceived(self, text): log.msg("stdout: %s" % text) self.s.write(text) def processEnded(self, reason): log.msg("Command %r exited with value %s" % (self.cmdline, reason)) if isinstance(reason.value, error.ProcessDone): self.deferred.callback(self.s.getvalue()) else: self.deferred.errback(reason) class Monotone: """All methods of this class return a Deferred.""" def __init__(self, bin, db): self.bin = bin self.db = db def _run_monotone(self, args): d = defer.Deferred() cmdline = (self.bin, "--db=" + self.db) + tuple(args) p = _MTProtocol(d, cmdline) log.msg("Running command: %r" % (cmdline,)) log.msg("wd: %s" % os.getcwd()) reactor.spawnProcess(p, self.bin, cmdline) return d def _process_revision_list(self, output): if output: return output.strip().split("\n") else: return [] def get_interface_version(self): d = self._run_monotone(["automate", "interface_version"]) d.addCallback(self._process_interface_version) return d def _process_interface_version(self, output): return tuple(map(int, output.strip().split("."))) def db_init(self): return self._run_monotone(["db", "init"]) def db_migrate(self): return self._run_monotone(["db", "migrate"]) def pull(self, server, pattern): return self._run_monotone(["pull", server, pattern]) def get_revision(self, rid): return self._run_monotone(["cat", "revision", rid]) def get_heads(self, branch, rcfile=""): cmd = ["automate", "heads", branch] if rcfile: cmd += ["--rcfile=" + rcfile] d = self._run_monotone(cmd) d.addCallback(self._process_revision_list) return d def erase_ancestors(self, revs): d = self._run_monotone(["automate", "erase_ancestors"] + revs) d.addCallback(self._process_revision_list) return d def ancestry_difference(self, new_rev, old_revs): d = self._run_monotone(["automate", "ancestry_difference", new_rev] + old_revs) d.addCallback(self._process_revision_list) return d def descendents(self, rev): d = self._run_monotone(["automate", "descendents", rev]) d.addCallback(self._process_revision_list) return d def log(self, rev, depth=None): if depth is not None: depth_arg = ["--last=%i" % (depth,)] else: depth_arg = [] return self._run_monotone(["log", "-r", rev] + depth_arg) class MonotoneSource(service.Service, util.ComparableMixin): """This source will poll a monotone server for changes and submit them to the change master. @param server_addr: monotone server specification (host:portno) @param branch: monotone branch to watch @param trusted_keys: list of keys whose code you trust @param db_path: path to monotone database to pull into @param pollinterval: interval in seconds between polls, defaults to 10 minutes @param monotone_exec: path to monotone executable, defaults to "monotone" """ __implements__ = IChangeSource, service.Service.__implements__ compare_attrs = ["server_addr", "trusted_keys", "db_path", "pollinterval", "branch", "monotone_exec"] parent = None # filled in when we're added done_revisions = [] last_revision = None loop = None d = None tmpfile = None monotone = None volatile = ["loop", "d", "tmpfile", "monotone"] def __init__(self, server_addr, branch, trusted_keys, db_path, pollinterval=60 * 10, monotone_exec="monotone"): self.server_addr = server_addr self.branch = branch self.trusted_keys = trusted_keys self.db_path = db_path self.pollinterval = pollinterval self.monotone_exec = monotone_exec self.monotone = Monotone(self.monotone_exec, self.db_path) def startService(self): self.loop = LoopingCall(self.start_poll) self.loop.start(self.pollinterval) service.Service.startService(self) def stopService(self): self.loop.stop() return service.Service.stopService(self) def describe(self): return "monotone_source %s %s" % (self.server_addr, self.branch) def start_poll(self): if self.d is not None: log.msg("last poll still in progress, skipping next poll") return log.msg("starting poll") self.d = self._maybe_init_db() self.d.addCallback(self._do_netsync) self.d.addCallback(self._get_changes) self.d.addErrback(self._handle_error) def _handle_error(self, failure): log.err(failure) self.d = None def _maybe_init_db(self): if not os.path.exists(self.db_path): log.msg("init'ing db") return self.monotone.db_init() else: log.msg("db already exists, migrating") return self.monotone.db_migrate() def _do_netsync(self, output): return self.monotone.pull(self.server_addr, self.branch) def _get_changes(self, output): d = self._get_new_head() d.addCallback(self._process_new_head) return d def _get_new_head(self): # This function returns a deferred that resolves to a good pick of new # head (or None if there is no good new head.) # First need to get all new heads... rcfile = """function get_revision_cert_trust(signers, id, name, val) local trusted_signers = { %s } local ts_table = {} for k, v in pairs(trusted_signers) do ts_table[v] = 1 end for k, v in pairs(signers) do if ts_table[v] then return true end end return false end """ trusted_list = ", ".join(['"' + key + '"' for key in self.trusted_keys]) # mktemp is unsafe, but mkstemp is not 2.2 compatible. tmpfile_name = tempfile.mktemp() f = open(tmpfile_name, "w") f.write(rcfile % trusted_list) f.close() d = self.monotone.get_heads(self.branch, tmpfile_name) d.addCallback(self._find_new_head, tmpfile_name) return d def _find_new_head(self, new_heads, tmpfile_name): os.unlink(tmpfile_name) # Now get the old head's descendents... if self.last_revision is not None: d = self.monotone.descendents(self.last_revision) else: d = defer.succeed(new_heads) d.addCallback(self._pick_new_head, new_heads) return d def _pick_new_head(self, old_head_descendents, new_heads): for r in new_heads: if r in old_head_descendents: return r return None def _process_new_head(self, new_head): if new_head is None: log.msg("No new head") self.d = None return None # Okay, we have a new head; we need to get all the revisions since # then and create change objects for them. # Step 1: simplify set of processed revisions. d = self._simplify_revisions() # Step 2: get the list of new revisions d.addCallback(self._get_new_revisions, new_head) # Step 3: add a change for each d.addCallback(self._add_changes_for_revisions) # Step 4: all done d.addCallback(self._finish_changes, new_head) return d def _simplify_revisions(self): d = self.monotone.erase_ancestors(self.done_revisions) d.addCallback(self._reset_done_revisions) return d def _reset_done_revisions(self, new_done_revisions): self.done_revisions = new_done_revisions return None def _get_new_revisions(self, blah, new_head): if self.done_revisions: return self.monotone.ancestry_difference(new_head, self.done_revisions) else: # Don't force feed the builder with every change since the # beginning of time when it's first started up. return defer.succeed([new_head]) def _add_changes_for_revisions(self, revs): d = defer.succeed(None) for rid in revs: d.addCallback(self._add_change_for_revision, rid) return d def _add_change_for_revision(self, blah, rid): d = self.monotone.log(rid, 1) d.addCallback(self._add_change_from_log, rid) return d def _add_change_from_log(self, log, rid): d = self.monotone.get_revision(rid) d.addCallback(self._add_change_from_log_and_revision, log, rid) return d def _add_change_from_log_and_revision(self, revision, log, rid): # Stupid way to pull out everything inside quotes (which currently # uniquely identifies filenames inside a changeset). pieces = revision.split('"') files = [] for i in range(len(pieces)): if (i % 2) == 1: files.append(pieces[i]) # Also pull out author key and date author = "unknown author" pieces = log.split('\n') for p in pieces: if p.startswith("Author:"): author = p.split()[1] self.parent.addChange(Change(author, files, log, revision=rid)) def _finish_changes(self, blah, new_head): self.done_revisions.append(new_head) self.last_revision = new_head self.d = None From warner at users.sourceforge.net Mon Sep 25 07:46:58 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:46:58 +0000 Subject: [Buildbot-commits] buildbot/buildbot/steps source.py,1.2,1.3 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/steps In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17754/buildbot/steps Modified Files: source.py Log Message: [project @ initial support for Monotone, by Nathaniel Smith] Original author: warner at lothar.com Date: 2006-09-25 07:45:16 Index: source.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/source.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- source.py 17 Sep 2006 20:35:49 -0000 1.2 +++ source.py 25 Sep 2006 07:46:56 -0000 1.3 @@ -899,3 +899,34 @@ cmd = LoggedRemoteCommand("p4sync", self.args) self.startCommand(cmd) +class Monotone(Source): + """Check out a revision from a monotone server at 'server_addr', + branch 'branch'. 'revision' specifies which revision id to check + out. + + This step will first create a local database, if necessary, and then pull + the contents of the server into the database. Then it will do the + checkout/update from this database.""" + + name = "monotone" + + def __init__(self, server_addr, branch, db_path="monotone.db", + monotone="monotone", + **kwargs): + Source.__init__(self, **kwargs) + self.args.update({"server_addr": server_addr, + "branch": branch, + "db_path": db_path, + "monotone": monotone}) + + def computeSourceRevision(self, changes): + if not changes: + return None + return changes[-1].revision + + def startVC(self): + slavever = self.slaveVersion("monotone") + assert slavever, "slave is too old, does not know about monotone" + cmd = LoggedRemoteCommand("monotone", self.args) + self.startCommand(cmd) + From warner at users.sourceforge.net Mon Sep 25 07:46:58 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:46:58 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.749,1.750 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17754 Modified Files: ChangeLog Log Message: [project @ initial support for Monotone, by Nathaniel Smith] Original author: warner at lothar.com Date: 2006-09-25 07:45:16 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.749 retrieving revision 1.750 diff -u -d -r1.749 -r1.750 --- ChangeLog 25 Sep 2006 07:21:02 -0000 1.749 +++ ChangeLog 25 Sep 2006 07:46:56 -0000 1.750 @@ -1,14 +1,3 @@ -2006-09-25 Brian Warner - - * buildbot/changes/bonsaipoller.py (BonsaiPoller): Ben also - contributed a Change Source that polls a Bonsai server (a - kind of web-based viewcvs CGI script). - - * buildbot/status/tinderbox.py (TinderboxMailNotifier): Ben - Hearsum contributed a status plugin which sends email in the same - format that Tinderbox does: this allows a number of tinderbox - tools to be driven by Buildbot instead. Thanks Ben! - 2006-09-24 Brian Warner * buildbot/changes/mail.py (parseBonsaiMail): fix the parser. From warner at users.sourceforge.net Mon Sep 25 07:47:04 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 07:47:04 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.750,1.751 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17854 Modified Files: ChangeLog Log Message: [project @ whoops, forgot to pull before modifying the ChangeLog.. merge conflict] Original author: warner at lothar.com Date: 2006-09-25 07:46:22 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.750 retrieving revision 1.751 diff -u -d -r1.750 -r1.751 --- ChangeLog 25 Sep 2006 07:46:56 -0000 1.750 +++ ChangeLog 25 Sep 2006 07:47:02 -0000 1.751 @@ -1,3 +1,22 @@ +2006-09-25 Brian Warner + + * buildbot/steps/source.py (Monotone): initial Monotone support, + contributed by Nathaniel Smith. Still needs docs and tests, but + this code has been in use on the Monotone buildbot for a long + time now. + * buildbot/slave/commands.py (Monotone): slave-side support + * buildbot/changes/monotone.py (MonotoneSource): polling change + source + + * buildbot/changes/bonsaipoller.py (BonsaiPoller): Ben also + contributed a Change Source that polls a Bonsai server (a + kind of web-based viewcvs CGI script). + + * buildbot/status/tinderbox.py (TinderboxMailNotifier): Ben + Hearsum contributed a status plugin which sends email in the same + format that Tinderbox does: this allows a number of tinderbox + tools to be driven by Buildbot instead. Thanks Ben! + 2006-09-24 Brian Warner * buildbot/changes/mail.py (parseBonsaiMail): fix the parser. From warner at users.sourceforge.net Mon Sep 25 08:01:25 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 08:01:25 +0000 Subject: [Buildbot-commits] buildbot/contrib svn_buildbot.py,1.12,1.13 Message-ID: Update of /cvsroot/buildbot/buildbot/contrib In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv24670/contrib Modified Files: svn_buildbot.py Log Message: [project @ svn_buildbot.py: strip *4* chars, not 6. Closes SF#1545146] Original author: warner at lothar.com Date: 2006-09-25 08:00:49 Index: svn_buildbot.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/contrib/svn_buildbot.py,v retrieving revision 1.12 retrieving revision 1.13 diff -u -d -r1.12 -r1.13 --- svn_buildbot.py 2 Jun 2006 15:54:56 -0000 1.12 +++ svn_buildbot.py 25 Sep 2006 08:01:23 -0000 1.13 @@ -145,8 +145,8 @@ changed = commands.getoutput('svnlook changed %s "%s"' % (rev_arg, repo) ).split('\n') - # the first 6 columns can contain status information - changed = [x[6:].strip() for x in changed] + # the first 4 columns can contain status information + changed = [x[4:] for x in changed] message = commands.getoutput('svnlook log %s "%s"' % (rev_arg, repo)) who = commands.getoutput('svnlook author %s "%s"' % (rev_arg, repo)) From warner at users.sourceforge.net Mon Sep 25 08:01:25 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 08:01:25 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.751,1.752 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv24670 Modified Files: ChangeLog Log Message: [project @ svn_buildbot.py: strip *4* chars, not 6. Closes SF#1545146] Original author: warner at lothar.com Date: 2006-09-25 08:00:49 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.751 retrieving revision 1.752 diff -u -d -r1.751 -r1.752 --- ChangeLog 25 Sep 2006 07:47:02 -0000 1.751 +++ ChangeLog 25 Sep 2006 08:01:23 -0000 1.752 @@ -1,5 +1,11 @@ 2006-09-25 Brian Warner + * contrib/svn_buildbot.py (ChangeSender.getChanges): the first *4* + columns of 'svnlook changed' output contain status information, so + strip [:4] instead of [:6]. Depending upon what the status flags + were, this would sometimes lead to mangled filenames. Thanks to + Riccardo Magliocchetti for the patch. Closes SF#1545146. + * buildbot/steps/source.py (Monotone): initial Monotone support, contributed by Nathaniel Smith. Still needs docs and tests, but this code has been in use on the Monotone buildbot for a long From warner at users.sourceforge.net Mon Sep 25 08:12:18 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 08:12:18 +0000 Subject: [Buildbot-commits] buildbot ChangeLog, 1.752, 1.753 setup.py, 1.38, 1.39 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv28916 Modified Files: ChangeLog setup.py Log Message: [project @ the new buildbot.steps module wasn't installed. Fixes SF#1560631] Original author: warner at lothar.com Date: 2006-09-25 08:04:38 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.752 retrieving revision 1.753 diff -u -d -r1.752 -r1.753 --- ChangeLog 25 Sep 2006 08:01:23 -0000 1.752 +++ ChangeLog 25 Sep 2006 08:12:16 -0000 1.753 @@ -1,5 +1,8 @@ 2006-09-25 Brian Warner + * setup.py: the new buildbot.steps module wasn't being installed. + Thanks to Jose Dapena Paz for the catch, fixes SF#1560631. + * contrib/svn_buildbot.py (ChangeSender.getChanges): the first *4* columns of 'svnlook changed' output contain status information, so strip [:4] instead of [:6]. Depending upon what the status flags Index: setup.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/setup.py,v retrieving revision 1.38 retrieving revision 1.39 diff -u -d -r1.38 -r1.39 --- setup.py 16 Jul 2006 03:34:13 -0000 1.38 +++ setup.py 25 Sep 2006 08:12:16 -0000 1.39 @@ -59,11 +59,13 @@ packages=["buildbot", "buildbot.status", "buildbot.changes", + "buildbot.steps", "buildbot.process", "buildbot.clients", "buildbot.slave", "buildbot.scripts", - "buildbot.test"], + "buildbot.test", + ], data_files=[("buildbot", ["buildbot/buildbot.png"]), ("buildbot/clients", ["buildbot/clients/debug.glade"]), ("buildbot/status", ["buildbot/status/classic.css"]), From warner at users.sourceforge.net Mon Sep 25 08:12:24 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Mon, 25 Sep 2006 08:12:24 +0000 Subject: [Buildbot-commits] buildbot ChangeLog, 1.753, 1.754 setup.py, 1.39, 1.40 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv28932 Modified Files: ChangeLog setup.py Log Message: [project @ add the extra stuff from buildbot/test/* so unit tests will run on installed copies, not just in the source tree] Original author: warner at lothar.com Date: 2006-09-25 08:11:30 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.753 retrieving revision 1.754 diff -u -d -r1.753 -r1.754 --- ChangeLog 25 Sep 2006 08:12:16 -0000 1.753 +++ ChangeLog 25 Sep 2006 08:12:22 -0000 1.754 @@ -2,6 +2,9 @@ * setup.py: the new buildbot.steps module wasn't being installed. Thanks to Jose Dapena Paz for the catch, fixes SF#1560631. + (testmsgs): add the extra stuff from buildbot/test/* so you can + run unit tests on an installed copy of buildbot, not just from + the source tree. * contrib/svn_buildbot.py (ChangeSender.getChanges): the first *4* columns of 'svnlook changed' output contain status information, so Index: setup.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/setup.py,v retrieving revision 1.39 retrieving revision 1.40 diff -u -d -r1.39 -r1.40 --- setup.py 25 Sep 2006 08:12:16 -0000 1.39 +++ setup.py 25 Sep 2006 08:12:22 -0000 1.40 @@ -1,6 +1,6 @@ #! /usr/bin/python -import sys +import sys, os from distutils.core import setup from buildbot import version @@ -37,6 +37,12 @@ scripts.append("contrib/windows/buildbot.bat") scripts.append("contrib/windows/buildbot_service.py") +testmsgs = [] +for f in os.listdir("buildbot/test/mail"): + if f.endswith("~"): + continue + testmsgs.append("buildbot/test/mail/%s" % f) + setup(name="buildbot", version=version, description="BuildBot build automation system", @@ -65,11 +71,14 @@ "buildbot.slave", "buildbot.scripts", "buildbot.test", + "buildbot.test.subdir", ], data_files=[("buildbot", ["buildbot/buildbot.png"]), ("buildbot/clients", ["buildbot/clients/debug.glade"]), ("buildbot/status", ["buildbot/status/classic.css"]), - ("buildbot/scripts", ["buildbot/scripts/sample.cfg"]),], + ("buildbot/scripts", ["buildbot/scripts/sample.cfg"]), + ("buildbot/test/mail", testmsgs), + ], scripts = scripts, cmdclass={'install_data': install_data_twisted}, ) From warner at users.sourceforge.net Fri Sep 29 07:04:23 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 29 Sep 2006 07:04:23 +0000 Subject: [Buildbot-commits] buildbot/buildbot/status tinderbox.py,1.1,1.2 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/status In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17729/buildbot/status Modified Files: tinderbox.py Log Message: [project @ TinderboxMailNotifier: updates from Robert Helmer] Original author: warner at lothar.com Date: 2006-09-29 07:03:32 Index: tinderbox.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/status/tinderbox.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- tinderbox.py 25 Sep 2006 07:04:24 -0000 1.1 +++ tinderbox.py 29 Sep 2006 07:04:21 -0000 1.2 @@ -102,6 +102,7 @@ def buildMessage(self, name, build, results): text = "" + res = "" # shortform t = "tinderbox:" @@ -112,11 +113,16 @@ text += "%s status: " % t if results == "building": - text += "building\n" + res = "building" + text += res elif results == WARNINGS or results == SUCCESS: - text += "success\n" + res = "success" + text += res else: - text += "busted\n" + res += "busted" + text += res + + text += "\n"; text += "%s build: %s\n" % (t, name) text += "%s errorparser: unix\n" % t # always use the unix errorparser @@ -128,26 +134,36 @@ else: text += "%s binaryurl: %s\n" % (t, self.binaryURL) text += "%s logcompression: %s\n" % (t, self.logCompression) - text += "%s logencoding: base64\n" % t - text += "%s END\n\n" % t # logs will always be appended + tinderboxLogs = "" for log in build.getLogs(): l = "" + logEncoding = "" if self.logCompression == "bzip2": - l = bz2.compress(log.getText()) + compressedLog = bz2.compress(log.getText()) + l = base64.encodestring(compressedLog) + logEncoding = "base64"; elif self.logCompression == "gzip": - l = zlib.compress(log.getText()) + compressedLog = zlib.compress(log.getText()) + l = base64.encodestring(compressedLog) + logEncoding = "base64"; else: l = log.getText() - text += base64.encodestring(l) - text += "\n" + tinderboxLogs += l + + text += "%s logencoding: %s\n" % (t, logEncoding) + text += "%s END\n\n" % t + text += tinderboxLogs + text += "\n" m = Message() m.set_payload(text) m['Date'] = formatdate(localtime=True) - m['Subject'] = self.subject + m['Subject'] = self.subject % { 'result': res, + 'builder': name, + } m['From'] = self.fromaddr # m['To'] is added later From warner at users.sourceforge.net Fri Sep 29 07:04:23 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Fri, 29 Sep 2006 07:04:23 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.754,1.755 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv17729 Modified Files: ChangeLog Log Message: [project @ TinderboxMailNotifier: updates from Robert Helmer] Original author: warner at lothar.com Date: 2006-09-29 07:03:32 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.754 retrieving revision 1.755 diff -u -d -r1.754 -r1.755 --- ChangeLog 25 Sep 2006 08:12:22 -0000 1.754 +++ ChangeLog 29 Sep 2006 07:04:20 -0000 1.755 @@ -1,3 +1,8 @@ +2006-09-29 Brian Warner + + * buildbot/status/tinderbox.py (TinderboxMailNotifier): updates + from Robert Helmer + 2006-09-25 Brian Warner * setup.py: the new buildbot.steps module wasn't being installed. From warner at users.sourceforge.net Sat Sep 30 19:32:51 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 19:32:51 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.67,1.68 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29161/buildbot/slave Modified Files: commands.py Log Message: [project @ LogFileWatcher: explicitly close the filehandle to fix SF#1568415] Original author: warner at lothar.com Date: 2006-09-30 19:32:09 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.67 retrieving revision 1.68 diff -u -d -r1.67 -r1.68 --- commands.py 25 Sep 2006 07:46:56 -0000 1.67 +++ commands.py 30 Sep 2006 19:32:48 -0000 1.68 @@ -174,6 +174,8 @@ self.poll() if self.poller is not None: self.poller.stop() + if self.started: + self.f.close() def statFile(self): if os.path.exists(self.logfile): From warner at users.sourceforge.net Sat Sep 30 19:32:50 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 19:32:50 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.755,1.756 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv29161 Modified Files: ChangeLog Log Message: [project @ LogFileWatcher: explicitly close the filehandle to fix SF#1568415] Original author: warner at lothar.com Date: 2006-09-30 19:32:09 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.755 retrieving revision 1.756 diff -u -d -r1.755 -r1.756 --- ChangeLog 29 Sep 2006 07:04:20 -0000 1.755 +++ ChangeLog 30 Sep 2006 19:32:48 -0000 1.756 @@ -1,3 +1,14 @@ +2006-09-30 Brian Warner + + * buildbot/slave/commands.py (LogFileWatcher.stop): explicitly + close the filehandle when we stop watching the file. Before, the + filehandle was only closed when the LogFileWatcher was + garbage-collected, which could be quite a while in the future. If + it was still open by the time the next build started, windows will + refuse to let the new build delete the old build/ directory. Fixes + SF#1568415, thanks to , , and on + #twisted for the catch. + 2006-09-29 Brian Warner * buildbot/status/tinderbox.py (TinderboxMailNotifier): updates From warner at users.sourceforge.net Sat Sep 30 20:20:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 20:20:37 +0000 Subject: [Buildbot-commits] buildbot/buildbot/changes bonsaipoller.py, 1.1, 1.2 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/changes In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv15240/buildbot/changes Modified Files: bonsaipoller.py Log Message: [project @ bonsaipoller: more updates from Robert Helmer] Original author: warner at lothar.com Date: 2006-09-30 20:19:58 Index: bonsaipoller.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/bonsaipoller.py,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- bonsaipoller.py 25 Sep 2006 07:21:02 -0000 1.1 +++ bonsaipoller.py 30 Sep 2006 20:20:35 -0000 1.2 @@ -130,7 +130,7 @@ return str(self.currentFileNode.firstChild.data) def _getRevision(self): - return float(self.currentFileNode.getAttribute("rev")) + return str(self.currentFileNode.getAttribute("rev")) @@ -238,7 +238,7 @@ data = bp.getData() for cinode in data.nodes: for file in cinode.files: - files.append(file) + files.append(file.filename+' (revision '+file.revision+')') c = changes.Change(who = cinode.who, files = files, comments = cinode.log, From warner at users.sourceforge.net Sat Sep 30 20:20:37 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 20:20:37 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.756,1.757 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv15240 Modified Files: ChangeLog Log Message: [project @ bonsaipoller: more updates from Robert Helmer] Original author: warner at lothar.com Date: 2006-09-30 20:19:58 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.756 retrieving revision 1.757 diff -u -d -r1.756 -r1.757 --- ChangeLog 30 Sep 2006 19:32:48 -0000 1.756 +++ ChangeLog 30 Sep 2006 20:20:35 -0000 1.757 @@ -1,5 +1,9 @@ 2006-09-30 Brian Warner + * buildbot/changes/bonsaipoller.py (BonsaiParser): more updates + from Robert Helmer + (BonsaiPoller): same + * buildbot/slave/commands.py (LogFileWatcher.stop): explicitly close the filehandle when we stop watching the file. Before, the filehandle was only closed when the LogFileWatcher was From warner at users.sourceforge.net Sat Sep 30 21:12:10 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 21:12:10 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.757,1.758 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv3157 Modified Files: ChangeLog Log Message: [project @ try: update docs, remove some dead code] Original author: warner at lothar.com Date: 2006-09-30 21:11:24 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.757 retrieving revision 1.758 diff -u -d -r1.757 -r1.758 --- ChangeLog 30 Sep 2006 20:20:35 -0000 1.757 +++ ChangeLog 30 Sep 2006 21:12:08 -0000 1.758 @@ -1,5 +1,10 @@ 2006-09-30 Brian Warner + * docs/buildbot.texinfo (.buildbot config directory): add more + docs on the .buildbot/options keys used by "buildbot try" + * buildbot/scripts/tryclient.py (Try.createJob): remove dead code + (Try.deliverJob): same + * buildbot/changes/bonsaipoller.py (BonsaiParser): more updates from Robert Helmer (BonsaiPoller): same From warner at users.sourceforge.net Sat Sep 30 21:12:10 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 21:12:10 +0000 Subject: [Buildbot-commits] buildbot/buildbot/scripts tryclient.py, 1.15, 1.16 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/scripts In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv3157/buildbot/scripts Modified Files: tryclient.py Log Message: [project @ try: update docs, remove some dead code] Original author: warner at lothar.com Date: 2006-09-30 21:11:24 Index: tryclient.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/scripts/tryclient.py,v retrieving revision 1.15 retrieving revision 1.16 diff -u -d -r1.15 -r1.16 --- tryclient.py 19 May 2006 07:44:21 -0000 1.15 +++ tryclient.py 30 Sep 2006 21:12:08 -0000 1.16 @@ -301,7 +301,6 @@ def createJob(self): # returns a Deferred which fires when the job parameters have been # created - config = self.config opts = self.opts # generate a random (unique) string. It would make sense to add a # hostname and process ID here, but a) I suspect that would cause @@ -336,7 +335,6 @@ def deliverJob(self): # returns a Deferred that fires when the job has been delivered - config = self.config opts = self.opts if self.connect == "ssh": From warner at users.sourceforge.net Sat Sep 30 21:12:10 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 21:12:10 +0000 Subject: [Buildbot-commits] buildbot/docs buildbot.texinfo,1.80,1.81 Message-ID: Update of /cvsroot/buildbot/buildbot/docs In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv3157/docs Modified Files: buildbot.texinfo Log Message: [project @ try: update docs, remove some dead code] Original author: warner at lothar.com Date: 2006-09-30 21:11:24 Index: buildbot.texinfo =================================================================== RCS file: /cvsroot/buildbot/buildbot/docs/buildbot.texinfo,v retrieving revision 1.80 retrieving revision 1.81 diff -u -d -r1.80 -r1.81 --- buildbot.texinfo 25 Sep 2006 07:21:02 -0000 1.80 +++ buildbot.texinfo 30 Sep 2006 21:12:08 -0000 1.81 @@ -5720,6 +5720,46 @@ @end table +The following options are used by the @code{buildbot try} command +(@pxref{try}): + + at table @code + at item try_connect +This specifies how the ``try'' command should deliver its request to +the buildmaster. The currently accepted values are ``ssh'' and ``pb''. + at item try_builders +Which builders should be used for the ``try'' build. + at item try_vc +This specifies the version control system being used. + at item try_branch +This indicates that the current tree is on a non-trunk branch. + at item try_topdir + at item try_topfile +Use @code{try_topdir} to explicitly indicate the top of your working +tree, or @code{try_topfile} to name a file that will only be found in +that top-most directory. + + at item try_host + at item try_username + at item try_dir +When try_connect is ``ssh'', the command will pay attention to + at code{try_host}, @code{try_username}, and @code{try_dir}. + + at item try_username + at item try_password + at item try_master +Instead, when @code{try_connect} is ``pb'', the command will pay +attention to @code{try_username}, @code{try_password}, and + at code{try_master}. + + at item try_wait + at item masterstatus + at code{try_wait} and @code{masterstatus} are used to ask the ``try'' +command to wait for the requested build to complete. + + at end table + + @node Resources, Developer's Appendix, Command-line tool, Top @chapter Resources From warner at users.sourceforge.net Sat Sep 30 22:36:01 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 22:36:01 +0000 Subject: [Buildbot-commits] buildbot/buildbot/changes p4poller.py,1.7,1.8 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/changes In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv2971/buildbot/changes Modified Files: p4poller.py Log Message: [project @ changes/p4poller.py: make some minor stylistic changes] Original author: warner at lothar.com Date: 2006-09-24 01:12:32 Index: p4poller.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/changes/p4poller.py,v retrieving revision 1.7 retrieving revision 1.8 diff -u -d -r1.7 -r1.8 --- p4poller.py 17 Sep 2006 20:49:31 -0000 1.7 +++ p4poller.py 30 Sep 2006 22:35:59 -0000 1.8 @@ -40,7 +40,6 @@ parent = None # filled in when we're added last_change = None loop = None - volatile = ['loop'] working = False def __init__(self, p4port=None, p4user=None, p4passwd=None, @@ -75,9 +74,9 @@ self.split_file = split_file self.pollinterval = pollinterval self.histmax = histmax + self.loop = LoopingCall(self.checkp4) def startService(self): - self.loop = LoopingCall(self.checkp4) base.ChangeSource.startService(self) # Don't start the loop just yet because the reactor isn't running. From warner at users.sourceforge.net Sat Sep 30 22:36:01 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 22:36:01 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.758,1.759 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv2971 Modified Files: ChangeLog Log Message: [project @ changes/p4poller.py: make some minor stylistic changes] Original author: warner at lothar.com Date: 2006-09-24 01:12:32 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.758 retrieving revision 1.759 diff -u -d -r1.758 -r1.759 --- ChangeLog 30 Sep 2006 21:12:08 -0000 1.758 +++ ChangeLog 30 Sep 2006 22:35:59 -0000 1.759 @@ -1,74 +1,3 @@ -2006-09-30 Brian Warner - - * docs/buildbot.texinfo (.buildbot config directory): add more - docs on the .buildbot/options keys used by "buildbot try" - * buildbot/scripts/tryclient.py (Try.createJob): remove dead code - (Try.deliverJob): same - - * buildbot/changes/bonsaipoller.py (BonsaiParser): more updates - from Robert Helmer - (BonsaiPoller): same - - * buildbot/slave/commands.py (LogFileWatcher.stop): explicitly - close the filehandle when we stop watching the file. Before, the - filehandle was only closed when the LogFileWatcher was - garbage-collected, which could be quite a while in the future. If - it was still open by the time the next build started, windows will - refuse to let the new build delete the old build/ directory. Fixes - SF#1568415, thanks to , , and on - #twisted for the catch. - -2006-09-29 Brian Warner - - * buildbot/status/tinderbox.py (TinderboxMailNotifier): updates - from Robert Helmer - -2006-09-25 Brian Warner - - * setup.py: the new buildbot.steps module wasn't being installed. - Thanks to Jose Dapena Paz for the catch, fixes SF#1560631. - (testmsgs): add the extra stuff from buildbot/test/* so you can - run unit tests on an installed copy of buildbot, not just from - the source tree. - - * contrib/svn_buildbot.py (ChangeSender.getChanges): the first *4* - columns of 'svnlook changed' output contain status information, so - strip [:4] instead of [:6]. Depending upon what the status flags - were, this would sometimes lead to mangled filenames. Thanks to - Riccardo Magliocchetti for the patch. Closes SF#1545146. - - * buildbot/steps/source.py (Monotone): initial Monotone support, - contributed by Nathaniel Smith. Still needs docs and tests, but - this code has been in use on the Monotone buildbot for a long - time now. - * buildbot/slave/commands.py (Monotone): slave-side support - * buildbot/changes/monotone.py (MonotoneSource): polling change - source - - * buildbot/changes/bonsaipoller.py (BonsaiPoller): Ben also - contributed a Change Source that polls a Bonsai server (a - kind of web-based viewcvs CGI script). - - * buildbot/status/tinderbox.py (TinderboxMailNotifier): Ben - Hearsum contributed a status plugin which sends email in the same - format that Tinderbox does: this allows a number of tinderbox - tools to be driven by Buildbot instead. Thanks Ben! - -2006-09-24 Brian Warner - - * buildbot/changes/mail.py (parseBonsaiMail): fix the parser. - Thanks to Robert Helmer for the patch. - - * buildbot/process/base.py (Build.setupSlaveBuilder): tell our - BuildStatus about the buildslave name at the *beginning* of the - build, rather than at the end. Thanks to Alexander Lorenz for the - patch. - * buildbot/status/html.py (StatusResourceBuild.body): always - include the slavename in the build page, not just when the build - has finished. - * buildbot/status/mail.py (MailNotifier.buildMessage): include the - slavename in the email message - 2006-09-21 Brian Warner * buildbot/scripts/sample.cfg: update to use new BuildStep classes From warner at users.sourceforge.net Sat Sep 30 22:36:12 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 22:36:12 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.759,1.760 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv3339 Modified Files: ChangeLog Log Message: [project @ p4poller.py: minor stylistic changes] Original author: warner at lothar.com Date: 2006-09-30 21:22:50 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.759 retrieving revision 1.760 diff -u -d -r1.759 -r1.760 --- ChangeLog 30 Sep 2006 22:35:59 -0000 1.759 +++ ChangeLog 30 Sep 2006 22:36:09 -0000 1.760 @@ -1,3 +1,77 @@ +2006-09-30 Brian Warner + + * buildbot/changes/p4poller.py (P4Source): some minor stylistic + changes: set self.loop in __init__, remove unused volatile= + + * docs/buildbot.texinfo (.buildbot config directory): add more + docs on the .buildbot/options keys used by "buildbot try" + * buildbot/scripts/tryclient.py (Try.createJob): remove dead code + (Try.deliverJob): same + + * buildbot/changes/bonsaipoller.py (BonsaiParser): more updates + from Robert Helmer + (BonsaiPoller): same + + * buildbot/slave/commands.py (LogFileWatcher.stop): explicitly + close the filehandle when we stop watching the file. Before, the + filehandle was only closed when the LogFileWatcher was + garbage-collected, which could be quite a while in the future. If + it was still open by the time the next build started, windows will + refuse to let the new build delete the old build/ directory. Fixes + SF#1568415, thanks to , , and on + #twisted for the catch. + +2006-09-29 Brian Warner + + * buildbot/status/tinderbox.py (TinderboxMailNotifier): updates + from Robert Helmer + +2006-09-25 Brian Warner + + * setup.py: the new buildbot.steps module wasn't being installed. + Thanks to Jose Dapena Paz for the catch, fixes SF#1560631. + (testmsgs): add the extra stuff from buildbot/test/* so you can + run unit tests on an installed copy of buildbot, not just from + the source tree. + + * contrib/svn_buildbot.py (ChangeSender.getChanges): the first *4* + columns of 'svnlook changed' output contain status information, so + strip [:4] instead of [:6]. Depending upon what the status flags + were, this would sometimes lead to mangled filenames. Thanks to + Riccardo Magliocchetti for the patch. Closes SF#1545146. + + * buildbot/steps/source.py (Monotone): initial Monotone support, + contributed by Nathaniel Smith. Still needs docs and tests, but + this code has been in use on the Monotone buildbot for a long + time now. + * buildbot/slave/commands.py (Monotone): slave-side support + * buildbot/changes/monotone.py (MonotoneSource): polling change + source + + * buildbot/changes/bonsaipoller.py (BonsaiPoller): Ben also + contributed a Change Source that polls a Bonsai server (a + kind of web-based viewcvs CGI script). + + * buildbot/status/tinderbox.py (TinderboxMailNotifier): Ben + Hearsum contributed a status plugin which sends email in the same + format that Tinderbox does: this allows a number of tinderbox + tools to be driven by Buildbot instead. Thanks Ben! + +2006-09-24 Brian Warner + + * buildbot/changes/mail.py (parseBonsaiMail): fix the parser. + Thanks to Robert Helmer for the patch. + + * buildbot/process/base.py (Build.setupSlaveBuilder): tell our + BuildStatus about the buildslave name at the *beginning* of the + build, rather than at the end. Thanks to Alexander Lorenz for the + patch. + * buildbot/status/html.py (StatusResourceBuild.body): always + include the slavename in the build page, not just when the build + has finished. + * buildbot/status/mail.py (MailNotifier.buildMessage): include the + slavename in the email message + 2006-09-21 Brian Warner * buildbot/scripts/sample.cfg: update to use new BuildStep classes From warner at users.sourceforge.net Sat Sep 30 22:36:24 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 22:36:24 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.760,1.761 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv3381 Modified Files: ChangeLog Log Message: [project @ Periodic/Nightly: add a reason= to the BuildSet, to explain why it happened] Original author: warner at lothar.com Date: 2006-09-30 22:34:46 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.760 retrieving revision 1.761 diff -u -d -r1.760 -r1.761 --- ChangeLog 30 Sep 2006 22:36:09 -0000 1.760 +++ ChangeLog 30 Sep 2006 22:36:22 -0000 1.761 @@ -1,5 +1,11 @@ 2006-09-30 Brian Warner + * buildbot/scheduler.py (Periodic): submit a reason= to the + BuildSet to indicate which Scheduler triggered the build. Thanks + to Mateusz Loskot for the suggestion. + (Nightly): same + * buildbot/test/test_scheduler.py (Scheduling.testPeriodic1): test it + * buildbot/changes/p4poller.py (P4Source): some minor stylistic changes: set self.loop in __init__, remove unused volatile= From warner at users.sourceforge.net Sat Sep 30 22:36:24 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 22:36:24 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_scheduler.py, 1.10, 1.11 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv3381/buildbot/test Modified Files: test_scheduler.py Log Message: [project @ Periodic/Nightly: add a reason= to the BuildSet, to explain why it happened] Original author: warner at lothar.com Date: 2006-09-30 22:34:46 Index: test_scheduler.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_scheduler.py,v retrieving revision 1.10 retrieving revision 1.11 diff -u -d -r1.10 -r1.11 --- test_scheduler.py 1 May 2006 00:10:57 -0000 1.10 +++ test_scheduler.py 30 Sep 2006 22:36:22 -0000 1.11 @@ -45,6 +45,7 @@ self.failUnless(len(self.master.sets) > 1) s1 = self.master.sets[0] self.failUnlessEqual(s1.builderNames, ["a","b"]) + self.failUnlessEqual(s1.reason, "The Periodic scheduler named 'quickly' triggered this build") def testNightly(self): # now == 15-Nov-2005, 00:05:36 AM . By using mktime, this is From warner at users.sourceforge.net Sat Sep 30 22:36:24 2006 From: warner at users.sourceforge.net (Brian Warner) Date: Sat, 30 Sep 2006 22:36:24 +0000 Subject: [Buildbot-commits] buildbot/buildbot scheduler.py,1.20,1.21 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv3381/buildbot Modified Files: scheduler.py Log Message: [project @ Periodic/Nightly: add a reason= to the BuildSet, to explain why it happened] Original author: warner at lothar.com Date: 2006-09-30 22:34:46 Index: scheduler.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/scheduler.py,v retrieving revision 1.20 retrieving revision 1.21 diff -u -d -r1.20 -r1.21 --- scheduler.py 17 Sep 2006 20:43:39 -0000 1.20 +++ scheduler.py 30 Sep 2006 22:36:22 -0000 1.21 @@ -334,6 +334,8 @@ self.builderNames = builderNames self.periodicBuildTimer = periodicBuildTimer self.branch = branch + self.reason = ("The Periodic scheduler named '%s' triggered this build" + % name) self.timer = internet.TimerService(self.periodicBuildTimer, self.doPeriodicBuild) self.timer.setServiceParent(self) @@ -348,7 +350,8 @@ def doPeriodicBuild(self): bs = buildset.BuildSet(self.builderNames, - SourceStamp(branch=self.branch)) + SourceStamp(branch=self.branch), + self.reason) self.submit(bs) @@ -409,6 +412,8 @@ self.branch = branch self.delayedRun = None self.nextRunTime = None + self.reason = ("The Nightly scheduler named '%s' triggered this build" + % name) def addTime(self, timetuple, secs): return time.localtime(time.mktime(timetuple)+secs) @@ -501,7 +506,8 @@ # And trigger a build bs = buildset.BuildSet(self.builderNames, - SourceStamp(branch=self.branch)) + SourceStamp(branch=self.branch), + self.reason) self.submit(bs) def addChange(self, change):