[Buildbot-commits] buildbot/buildbot/slave commands.py,1.71,1.72

Brian Warner warner at users.sourceforge.net
Sat Dec 9 09:16:38 UTC 2006


Update of /cvsroot/buildbot/buildbot/buildbot/slave
In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv28528/buildbot/slave

Modified Files:
	commands.py 
Log Message:
[project @ cleanup/enhance transfer.FileUpload/FileDownload]

Original author: warner at lothar.com
Date: 2006-12-09 09:13:31

Index: commands.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v
retrieving revision 1.71
retrieving revision 1.72
diff -u -d -r1.71 -r1.72
--- commands.py	24 Nov 2006 07:16:35 -0000	1.71
+++ commands.py	9 Dec 2006 09:16:36 -0000	1.72
@@ -684,41 +684,44 @@
     """
     debug = False
 
-    def setup(self,args):
+    def setup(self, args):
         self.workdir = args['workdir']
-        self.filename = os.path.basename(args['slavesrc'])
+        self.filename = args['slavesrc']
         self.writer = args['writer']
-        self.maxsize = args['maxsize']
+        self.remaining = args['maxsize']
         self.blocksize = args['blocksize']
         self.stderr = None
         self.rc = 0
 
+    def start(self):
 	if self.debug:
             log.msg('SlaveFileUploadCommand started')
 
         # Open file
         self.path = os.path.join(self.builder.basedir,
                                  self.workdir,
-                                 self.filename)
+                                 os.path.expanduser(self.filename))
         try:
             self.fp = open(self.path, 'r')
             if self.debug:
                 log.msg('Opened %r for upload' % self.path)
         except:
+            # TODO: this needs cleanup
             self.fp = None
             self.stderr = 'Cannot open file %r for upload' % self.path
             self.rc = 1
             if self.debug:
                 log.msg('Cannot open file %r for upload' % self.path)
 
+        self.sendStatus({'header': "sending %s" % self.path})
 
-    def start(self):
-        self.cmd = defer.Deferred()
-        reactor.callLater(0, self._writeBlock)
-
-        return self.cmd
+        d = defer.Deferred()
+        d.addCallback(self._writeBlock)
+        d.addBoth(self.finished)
+        reactor.callLater(0, d.callback, None)
+        return d
 
-    def _writeBlock(self):
+    def _writeBlock(self, res):
         """
         Write a block of data to the remote writer
         """
@@ -726,12 +729,11 @@
             if self.debug:
                 log.msg('SlaveFileUploadCommand._writeBlock(): end')
             d = self.writer.callRemote('close')
-            d.addCallback(lambda _: self.finished())
-            return
+            return d
 
         length = self.blocksize
-        if self.maxsize is not None and length > self.maxsize:
-            length = self.maxsize
+        if self.remaining is not None and length > self.remaining:
+            length = self.remaining
 
         if length <= 0:
             if self.stderr is None:
@@ -744,17 +746,17 @@
 
         if self.debug:
             log.msg('SlaveFileUploadCommand._writeBlock(): '+
-                    'allowed=%d readlen=%d' % (length,len(data)))
+                    'allowed=%d readlen=%d' % (length, len(data)))
         if len(data) == 0:
             d = self.writer.callRemote('close')
-            d.addCallback(lambda _: self.finished())
-        else:
-            if self.maxsize is not None:
-                self.maxsize = self.maxsize - len(data)
-                assert self.maxsize >= 0
-            d = self.writer.callRemote('write',data)
-            d.addCallback(lambda _: self._writeBlock())
+            return d
 
+        if self.remaining is not None:
+            self.remaining = self.remaining - len(data)
+            assert self.remaining >= 0
+        d = self.writer.callRemote('write', data)
+        d.addCallback(self._writeBlock)
+        return d
 
     def interrupt(self):
         if self.debug:
@@ -765,17 +767,16 @@
             self.stderr = 'Upload of %r interrupted' % self.path
             self.rc = 1
         self.interrupted = True
-        self.finished()
-
+        # the next _writeBlock call will notice the .interrupted flag
 
-    def finished(self):
+    def finished(self, res):
         if self.debug:
-            log.msg('finished: stderr=%r, rc=%r' % (self.stderr,self.rc))
+            log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc))
         if self.stderr is None:
-            self.sendStatus({'rc':self.rc})
+            self.sendStatus({'rc': self.rc})
         else:
-            self.sendStatus({'stderr':self.stderr, 'rc':self.rc})
-        self.cmd.callback(0)
+            self.sendStatus({'stderr': self.stderr, 'rc': self.rc})
+        return res
 
 registerSlaveCommand("uploadFile", SlaveFileUploadCommand, command_version)
 
@@ -790,44 +791,56 @@
         - ['reader']:    RemoteReference to a transfer._FileReader object
         - ['maxsize']:   max size (in bytes) of file to write
         - ['blocksize']: max size for each data block
+        - ['mode']:      access mode for the new file
     """
     debug = False
 
-    def setup(self,args):
+    def setup(self, args):
         self.workdir = args['workdir']
-        self.filename = os.path.basename(args['slavedest'])
+        self.filename = args['slavedest']
         self.reader = args['reader']
-        self.maxsize = args['maxsize']
+        self.bytes_remaining = args['maxsize']
         self.blocksize = args['blocksize']
+        self.mode = args['mode']
         self.stderr = None
         self.rc = 0
 
+    def start(self):
 	if self.debug:
-            log.msg('SlaveFileDownloadCommand started')
+            log.msg('SlaveFileDownloadCommand starting')
 
         # Open file
         self.path = os.path.join(self.builder.basedir,
                                  self.workdir,
-                                 self.filename)
+                                 os.path.expanduser(self.filename))
         try:
             self.fp = open(self.path, 'w')
             if self.debug:
                 log.msg('Opened %r for download' % self.path)
+            if self.mode is not None:
+                # note: there is a brief window during which the new file
+                # will have the buildslave's default (umask) mode before we
+                # set the new one. Don't use this mode= feature to keep files
+                # private: use the buildslave's umask for that instead. (it
+                # is possible to call os.umask() before and after the open()
+                # call, but cleaning up from exceptions properly is more of a
+                # nuisance that way).
+                os.chmod(self.path, self.mode)
         except IOError:
+            # TODO: this still needs cleanup
             self.fp = None
             self.stderr = 'Cannot open file %r for download' % self.path
             self.rc = 1
             if self.debug:
                 log.msg('Cannot open file %r for download' % self.path)
 
+        d = defer.Deferred()
+        d.addCallback(self._readBlock)
+        d.addBoth(self.finished)
+        reactor.callLater(0, d.callback, None)
+        return d
 
-    def start(self):
-        self.cmd = defer.Deferred()
-        reactor.callLater(0, self._readBlock)
-
-        return self.cmd
-
-    def _readBlock(self):
+    def _readBlock(self, res):
         """
         Read a block of data from the remote reader
         """
@@ -835,12 +848,11 @@
             if self.debug:
                 log.msg('SlaveFileDownloadCommand._readBlock(): end')
             d = self.reader.callRemote('close')
-            d.addCallback(lambda _: self.finished())
-            return
+            return d
 
         length = self.blocksize
-        if self.maxsize is not None and length > self.maxsize:
-            length = self.maxsize
+        if self.bytes_remaining is not None and length > self.bytes_remaining:
+            length = self.bytes_remaining
 
         if length <= 0:
             if self.stderr is None:
@@ -848,25 +860,25 @@
                                 % self.path
                 self.rc = 1
             d = self.reader.callRemote('close')
-            d.addCallback(lambda _: self.finished())
         else:
             d = self.reader.callRemote('read', length)
             d.addCallback(self._writeData)
+        return d
 
-    def _writeData(self,data):
+    def _writeData(self, data):
         if self.debug:
-            log.msg('SlaveFileDownloadCommand._readBlock(): '+
-                    'readlen=%d' % len(data))
+            log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' %
+                    len(data))
         if len(data) == 0:
             d = self.reader.callRemote('close')
-            d.addCallback(lambda _: self.finished())
-        else:
-            if self.maxsize is not None:
-                self.maxsize = self.maxsize - len(data)
-                assert self.maxsize >= 0
-            self.fp.write(data)
-            self._readBlock()   # setup call back for next block (or finish)
+            return d
 
+        if self.bytes_remaining is not None:
+            self.bytes_remaining = self.bytes_remaining - len(data)
+            assert self.bytes_remaining >= 0
+        self.fp.write(data)
+        d = self._readBlock(None) # setup call back for next block (or finish)
+        return d
 
     def interrupt(self):
         if self.debug:
@@ -877,21 +889,20 @@
             self.stderr = 'Download of %r interrupted' % self.path
             self.rc = 1
         self.interrupted = True
-        self.finished()
-
+        # now we wait for the next read request to return. _readBlock will
+        # abandon the file when it sees self.interrupted set.
 
-    def finished(self):
+    def finished(self, res):
         if self.fp is not None:
             self.fp.close()
 
         if self.debug:
-            log.msg('finished: stderr=%r, rc=%r' % (self.stderr,self.rc))
+            log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc))
         if self.stderr is None:
-            self.sendStatus({'rc':self.rc})
+            self.sendStatus({'rc': self.rc})
         else:
-            self.sendStatus({'stderr':self.stderr, 'rc':self.rc})
-        self.cmd.callback(0)
-
+            self.sendStatus({'stderr': self.stderr, 'rc': self.rc})
+        return res
 
 registerSlaveCommand("downloadFile", SlaveFileDownloadCommand, command_version)
 





More information about the Commits mailing list