[Buildbot-commits] buildbot/buildbot/steps transfer.py,1.5,1.6
Brian Warner
warner at users.sourceforge.net
Sat Dec 9 09:16:38 UTC 2006
Update of /cvsroot/buildbot/buildbot/buildbot/steps
In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv28528/buildbot/steps
Modified Files:
transfer.py
Log Message:
[project @ cleanup/enhance transfer.FileUpload/FileDownload]
Original author: warner at lothar.com
Date: 2006-12-09 09:13:31
Index: transfer.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/steps/transfer.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- transfer.py 13 Oct 2006 09:08:25 -0000 1.5
+++ transfer.py 9 Dec 2006 09:16:36 -0000 1.6
@@ -6,34 +6,22 @@
from twisted.python import log
from buildbot.process.buildstep import RemoteCommand, BuildStep
from buildbot.process.buildstep import SUCCESS, FAILURE
+from buildbot.interfaces import BuildSlaveTooOldError
-class _FileIO(pb.Referenceable):
- """
- Helper base class that acts as remote-accessible file-object
- """
-
- def __init__(self,fp):
- self.fp = fp
-
- def remote_close(self):
- """
- Called by remote slave to state that no more data will be transfered
- """
- if self.fp is not None:
- self.fp.close()
- self.fp = None
-
-class _FileWriter(_FileIO):
+class _FileWriter(pb.Referenceable):
"""
Helper class that acts as a file-object with write access
"""
- def __init__(self,fp, maxsize=None):
- _FileIO.__init__(self,fp)
- self.maxsize = maxsize
+ def __init__(self, destfile, maxsize, mode):
+ self.destfile = destfile
+ self.fp = open(destfile, "w")
+ if mode is not None:
+ os.chmod(destfile, mode)
+ self.remaining = maxsize
- def remote_write(self,data):
+ def remote_write(self, data):
"""
Called from remote slave to write L{data} to L{fp} within boundaries
of L{maxsize}
@@ -41,35 +29,27 @@
@type data: C{string}
@param data: String of data to write
"""
- if self.fp is not None:
- if self.maxsize is not None:
- if len(data) > self.maxsize:
- data = data[:self.maxsize]
- self.fp.write(data)
- self.maxsize = self.maxsize - len(data)
- else:
- self.fp.write(data)
-
-class _FileReader(_FileIO):
- """
- Helper class that acts as a file-object with read access
- """
-
- def remote_read(self,maxlength):
- """
- Called from remote slave to read at most L{maxlength} bytes of data
-
- @type maxlength: C{integer}
- @param maxlength: Maximum number of data bytes that can be returned
+ if self.remaining is not None:
+ if len(data) > self.remaining:
+ data = data[:self.remaining]
+ self.fp.write(data)
+ self.remaining = self.remaining - len(data)
+ else:
+ self.fp.write(data)
- @return: Data read from L{fp}
- @rtype: C{string} of bytes read from file
- """
- if self.fp is None:
- return ''
+ def remote_close(self):
+ """
+ Called by remote slave to state that no more data will be transfered
+ """
+ self.fp.close()
+ self.fp = None
- data = self.fp.read(maxlength)
- return data
+ def __del__(self):
+ # unclean shutdown, the file is probably truncated, so delete it
+ # altogether rather than deliver a corrupted file
+ if self.fp is not None:
+ self.fp.close()
+ os.unlink(self.destfile)
class StatusRemoteCommand(RemoteCommand):
@@ -99,13 +79,16 @@
base dir, default 'build'
- ['maxsize'] maximum size of the file, default None (=unlimited)
- ['blocksize'] maximum size of each block being transfered
+ - ['mode'] file access mode for the resulting master-side file.
+ The default (=None) is to leave it up to the umask of
+ the buildmaster process.
"""
name = 'upload'
def __init__(self, build, slavesrc, masterdest,
- workdir="build", maxsize=None, blocksize=16*1024,
+ workdir="build", maxsize=None, blocksize=16*1024, mode=None,
**buildstep_kwargs):
BuildStep.__init__(self, build, **buildstep_kwargs)
@@ -114,8 +97,15 @@
self.workdir = workdir
self.maxsize = maxsize
self.blocksize = blocksize
+ assert isinstance(mode, (int, type(None)))
+ self.mode = mode
def start(self):
+ version = self.slaveVersion("uploadFile")
+ if not version:
+ m = "slave is too old, does not know about uploadFile"
+ raise BuildSlaveTooOldError(m)
+
source = self.slavesrc
masterdest = self.masterdest
# we rely upon the fact that the buildmaster runs chdir'ed into its
@@ -127,54 +117,98 @@
% (source, target))
self.step_status.setColor('yellow')
- self.step_status.setText(['uploading', source])
+ self.step_status.setText(['uploading', os.path.basename(source)])
- fp = open(self.masterdest, 'w')
- self.fileWriter = _FileWriter(fp)
+ # we use maxsize to limit the amount of data on both sides
+ fileWriter = _FileWriter(self.masterdest, self.maxsize, self.mode)
# default arguments
args = {
'slavesrc': source,
'workdir': self.workdir,
- 'writer': self.fileWriter,
+ 'writer': fileWriter,
'maxsize': self.maxsize,
- 'blocksize': self.blocksize
+ 'blocksize': self.blocksize,
}
self.cmd = StatusRemoteCommand('uploadFile', args)
d = self.runCommand(self.cmd)
d.addCallback(self.finished).addErrback(self.failed)
- def finished(self,result):
+ def finished(self, result):
if self.cmd.stderr != '':
self.addCompleteLog('stderr', self.cmd.stderr)
- self.fileWriter = None
-
if self.cmd.rc is None or self.cmd.rc == 0:
self.step_status.setColor('green')
return BuildStep.finished(self, SUCCESS)
self.step_status.setColor('red')
return BuildStep.finished(self, FAILURE)
+
+
+
+
+class _FileReader(pb.Referenceable):
+ """
+ Helper class that acts as a file-object with read access
+ """
+
+ def __init__(self, fp):
+ self.fp = fp
+
+ def remote_read(self, maxlength):
+ """
+ Called from remote slave to read at most L{maxlength} bytes of data
+
+ @type maxlength: C{integer}
+ @param maxlength: Maximum number of data bytes that can be returned
+
+ @return: Data read from L{fp}
+ @rtype: C{string} of bytes read from file
+ """
+ if self.fp is None:
+ return ''
+
+ data = self.fp.read(maxlength)
+ return data
+
+ def remote_close(self):
+ """
+ Called by remote slave to state that no more data will be transfered
+ """
+ if self.fp is not None:
+ self.fp.close()
+ self.fp = None
+
+
class FileDownload(BuildStep):
"""
- Build step to download a file
- arguments:
+ Download the first 'maxsize' bytes of a file, from the buildmaster to the
+ buildslave. Set the mode of the file
- ['mastersrc'] filename of source file at master
- ['slavedest'] filename of destination file at slave
- ['workdir'] string with slave working directory relative to builder
- base dir, default 'build'
- ['maxsize'] maximum size of the file, default None (=unlimited)
- ['blocksize'] maximum size of each block being transfered
+ Arguments::
+
+ ['mastersrc'] filename of source file at master
+ ['slavedest'] filename of destination file at slave
+ ['workdir'] string with slave working directory relative to builder
+ base dir, default 'build'
+ ['maxsize'] maximum size of the file, default None (=unlimited)
+ ['blocksize'] maximum size of each block being transfered
+ ['mode'] use this to set the access permissions of the resulting
+ buildslave-side file. This is traditionally an octal
+ integer, like 0644 to be world-readable (but not
+ world-writable), or 0600 to only be readable by
+ the buildslave account, or 0755 to be world-executable.
+ The default (=None) is to leave it up to the umask of
+ the buildslave process.
"""
name = 'download'
def __init__(self, build, mastersrc, slavedest,
- workdir="build", maxsize=None, blocksize=16*1024,
+ workdir="build", maxsize=None, blocksize=16*1024, mode=None,
**buildstep_kwargs):
BuildStep.__init__(self, build, **buildstep_kwargs)
@@ -183,52 +217,60 @@
self.workdir = workdir
self.maxsize = maxsize
self.blocksize = blocksize
+ assert isinstance(mode, (int, type(None)))
+ self.mode = mode
def start(self):
+ version = self.slaveVersion("downloadFile")
+ if not version:
+ m = "slave is too old, does not know about downloadFile"
+ raise BuildSlaveTooOldError(m)
+
+ # we are currently in the buildmaster's basedir, so any non-absolute
+ # paths will be interpreted relative to that
source = os.path.expanduser(self.mastersrc)
slavedest = self.slavedest
log.msg("FileDownload started, from master %r to slave %r" %
(source, slavedest))
self.step_status.setColor('yellow')
- self.step_status.setText(['downloading', slavedest])
+ self.step_status.setText(['downloading', "to",
+ os.path.basename(slavedest)])
- # If file does not exist, bail out with an error
- if not os.path.isfile(source):
+ # setup structures for reading the file
+ try:
+ fp = open(source, 'r')
+ except IOError:
+ # if file does not exist, bail out with an error
self.addCompleteLog('stderr',
'File %r not available at master' % source)
- reactor.callLater(0, self.reportFail)
+ # TODO: once BuildStep.start() gets rewritten to use
+ # maybeDeferred, just re-raise the exception here.
+ reactor.callLater(0, BuildStep.finished, self, FAILURE)
return
-
- # setup structures for reading the file
- fp = open(source, 'r')
- self.fileReader = _FileReader(fp)
+ fileReader = _FileReader(fp)
# default arguments
args = {
'slavedest': self.slavedest,
'maxsize': self.maxsize,
- 'reader': self.fileReader,
+ 'reader': fileReader,
'blocksize': self.blocksize,
'workdir': self.workdir,
+ 'mode': self.mode,
}
self.cmd = StatusRemoteCommand('downloadFile', args)
d = self.runCommand(self.cmd)
d.addCallback(self.finished).addErrback(self.failed)
- def finished(self,result):
+ def finished(self, result):
if self.cmd.stderr != '':
self.addCompleteLog('stderr', self.cmd.stderr)
- self.fileReader = None
-
if self.cmd.rc is None or self.cmd.rc == 0:
self.step_status.setColor('green')
return BuildStep.finished(self, SUCCESS)
- return self.reportFail()
-
- def reportFail(self):
self.step_status.setColor('red')
return BuildStep.finished(self, FAILURE)
More information about the Commits
mailing list