From warner at users.sourceforge.net Tue Jan 1 02:46:40 2008 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 01 Jan 2008 02:46:40 +0000 Subject: [Buildbot-commits] buildbot/buildbot/slave commands.py,1.91,1.92 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/slave In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv28197/buildbot/slave Modified Files: commands.py Log Message: [project @ file transfer: avoid tail recursion, to fix stack-depth errors. Closes #106.] Original author: warner at lothar.com Date: 2008-01-01 02:44:42+00:00 Index: commands.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/slave/commands.py,v retrieving revision 1.91 retrieving revision 1.92 diff -u -d -r1.91 -r1.92 --- commands.py 22 Dec 2007 08:54:49 -0000 1.91 +++ commands.py 1 Jan 2008 02:46:37 -0000 1.92 @@ -748,20 +748,36 @@ self.sendStatus({'header': "sending %s" % self.path}) d = defer.Deferred() - d.addCallback(self._writeBlock) + reactor.callLater(0, self._loop, d) + def _close(res): + # close the file, but pass through any errors from _loop + d1 = self.writer.callRemote("close") + d1.addErrback(log.err) + d1.addCallback(lambda ignored: res) + return d1 + d.addBoth(_close) d.addBoth(self.finished) - reactor.callLater(0, d.callback, None) return d - def _writeBlock(self, res): - """ - Write a block of data to the remote writer - """ + def _loop(self, fire_when_done): + d = defer.maybeDeferred(self._writeBlock) + def _done(finished): + if finished: + fire_when_done.callback(None) + else: + self._loop(fire_when_done) + def _err(why): + fire_when_done.errback(why) + d.addCallbacks(_done, _err) + return None + + def _writeBlock(self): + """Write a block of data to the remote writer""" + if self.interrupted or self.fp is None: if self.debug: log.msg('SlaveFileUploadCommand._writeBlock(): end') - d = self.writer.callRemote('close') - return d + return True length = self.blocksize if self.remaining is not None and length > self.remaining: @@ -780,14 +796,14 @@ log.msg('SlaveFileUploadCommand._writeBlock(): '+ 'allowed=%d readlen=%d' % (length, len(data))) if len(data) == 0: - d = self.writer.callRemote('close') - return d + log.msg("EOF: callRemote(close)") + return True if self.remaining is not None: self.remaining = self.remaining - len(data) assert self.remaining >= 0 d = self.writer.callRemote('write', data) - d.addCallback(self._writeBlock) + d.addCallback(lambda res: False) return d def interrupt(self): @@ -867,20 +883,36 @@ log.msg('Cannot open file %r for download' % self.path) d = defer.Deferred() - d.addCallback(self._readBlock) + reactor.callLater(0, self._loop, d) + def _close(res): + # close the file, but pass through any errors from _loop + d1 = self.reader.callRemote('close') + d1.addErrback(log.err) + d1.addCallback(lambda ignored: res) + return d1 + d.addBoth(_close) d.addBoth(self.finished) - reactor.callLater(0, d.callback, None) return d - def _readBlock(self, res): - """ - Read a block of data from the remote reader - """ + def _loop(self, fire_when_done): + d = defer.maybeDeferred(self._readBlock) + def _done(finished): + if finished: + fire_when_done.callback(None) + else: + self._loop(fire_when_done) + def _err(why): + fire_when_done.errback(why) + d.addCallbacks(_done, _err) + return None + + def _readBlock(self): + """Read a block of data from the remote reader.""" + if self.interrupted or self.fp is None: if self.debug: log.msg('SlaveFileDownloadCommand._readBlock(): end') - d = self.reader.callRemote('close') - return d + return True length = self.blocksize if self.bytes_remaining is not None and length > self.bytes_remaining: @@ -891,26 +923,24 @@ self.stderr = 'Maximum filesize reached, truncating file %r' \ % self.path self.rc = 1 - d = self.reader.callRemote('close') + return True else: d = self.reader.callRemote('read', length) d.addCallback(self._writeData) - return d + return d def _writeData(self, data): if self.debug: log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' % len(data)) if len(data) == 0: - d = self.reader.callRemote('close') - return d + return True if self.bytes_remaining is not None: self.bytes_remaining = self.bytes_remaining - len(data) assert self.bytes_remaining >= 0 self.fp.write(data) - d = self._readBlock(None) # setup call back for next block (or finish) - return d + return False def interrupt(self): if self.debug: From warner at users.sourceforge.net Tue Jan 1 02:46:40 2008 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 01 Jan 2008 02:46:40 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test runutils.py, 1.22, 1.23 test_transfer.py, 1.5, 1.6 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv28197/buildbot/test Modified Files: runutils.py test_transfer.py Log Message: [project @ file transfer: avoid tail recursion, to fix stack-depth errors. Closes #106.] Original author: warner at lothar.com Date: 2008-01-01 02:44:42+00:00 Index: runutils.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/runutils.py,v retrieving revision 1.22 retrieving revision 1.23 diff -u -d -r1.22 -r1.23 --- runutils.py 12 Aug 2007 22:22:41 -0000 1.22 +++ runutils.py 1 Jan 2008 02:46:38 -0000 1.23 @@ -340,7 +340,10 @@ self.target = target def callRemote(self, name, *args, **kwargs): - d = defer.maybeDeferred(self._callRemote, name, *args, **kwargs) + # callRemote is not allowed to fire its Deferred in the same turn + d = defer.Deferred() + d.addCallback(self._callRemote, *args, **kwargs) + reactor.callLater(0, d.callback, name) return d def _callRemote(self, name, *args, **kwargs): Index: test_transfer.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_transfer.py,v retrieving revision 1.5 retrieving revision 1.6 diff -u -d -r1.5 -r1.6 --- test_transfer.py 11 Dec 2006 08:23:29 -0000 1.5 +++ test_transfer.py 1 Jan 2008 02:46:38 -0000 1.6 @@ -7,7 +7,6 @@ from buildbot.test.runutils import StepTester from buildbot.status.builder import SUCCESS, FAILURE - # these steps pass a pb.Referenceable inside their arguments, so we have to # catch and wrap them. If the LocalAsRemote wrapper were a proper membrane, # we wouldn't have to do this. @@ -154,7 +153,47 @@ d.addCallback(_checkUpload) return d - + def testLotsOfBlocks(self): + self.slavebase = "Upload.testLotsOfBlocks.slave" + self.masterbase = "Upload.testLotsOfBlocks.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + # the buildmaster normally runs chdir'ed into masterbase, so uploaded + # files will appear there. Under trial, we're chdir'ed into + # _trial_temp instead, so use a different masterdest= to keep the + # uploaded file in a test-local directory + masterdest = os.path.join(self.masterbase, "dest.text") + step = self.makeStep(FileUpload, + slavesrc="source.txt", + masterdest=masterdest, + blocksize=15) + slavesrc = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "source.txt") + contents = "".join(["this is the source file #%d\n" % i + for i in range(1000)]) + open(slavesrc, "w").write(contents) + f = open(masterdest, "w") + f.write("overwrite me\n") + f.close() + + d = self.runStep(step) + def _checkUpload(results): + step_status = step.step_status + #l = step_status.getLogs() + #if l: + # logtext = l[0].getText() + # print logtext + self.failUnlessEqual(results, SUCCESS) + self.failUnless(os.path.exists(masterdest)) + masterdest_contents = open(masterdest, "r").read() + self.failUnlessEqual(masterdest_contents, contents) + d.addCallback(_checkUpload) + return d + testLotsOfBlocks.timeout = 20 + class Download(StepTester, unittest.TestCase): @@ -289,6 +328,39 @@ return d + def testLotsOfBlocks(self): + self.slavebase = "Download.testLotsOfBlocks.slave" + self.masterbase = "Download.testLotsOfBlocks.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + mastersrc = os.path.join(self.masterbase, "source.text") + slavedest = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "dest.txt") + step = self.makeStep(FileDownload, + mastersrc=mastersrc, + slavedest="dest.txt", + blocksize=15) + contents = "".join(["this is the source file #%d\n" % i + for i in range(1000)]) + open(mastersrc, "w").write(contents) + f = open(slavedest, "w") + f.write("overwrite me\n") + f.close() + + d = self.runStep(step) + def _checkDownload(results): + step_status = step.step_status + self.failUnlessEqual(results, SUCCESS) + self.failUnless(os.path.exists(slavedest)) + slavedest_contents = open(slavedest, "r").read() + self.failUnlessEqual(slavedest_contents, contents) + d.addCallback(_checkDownload) + return d + testLotsOfBlocks.timeout = 20 + # TODO: # test relative paths, ~/paths From warner at users.sourceforge.net Tue Jan 1 02:46:40 2008 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 01 Jan 2008 02:46:40 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.1013,1.1014 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv28197 Modified Files: ChangeLog Log Message: [project @ file transfer: avoid tail recursion, to fix stack-depth errors. Closes #106.] Original author: warner at lothar.com Date: 2008-01-01 02:44:42+00:00 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.1013 retrieving revision 1.1014 diff -u -d -r1.1013 -r1.1014 --- ChangeLog 23 Dec 2007 08:35:31 -0000 1.1013 +++ ChangeLog 1 Jan 2008 02:46:37 -0000 1.1014 @@ -1,3 +1,15 @@ +2007-12-31 Brian Warner + + * buildbot/slave/commands.py (SlaveFileUploadCommand.start): + rewrite the main loop to avoid tail recursion, since Deferreds + don't optimize those out. Without this, at the end of the transfer + we'd hit a stack depth of about two or three frames per block + transferred, and for large transfers this would run into a "stack + depth exceeded" error. Closes #106. + (SlaveFileDownloadCommand.start): same, for downloads. + * buildbot/test/test_transfer.py (Upload.testLotsOfBlocks): test it + (Download.testLotsOfBlocks): same + 2007-12-22 Brian Warner * buildbot/steps/shell.py (TreeSize): This never worked properly. From warner at users.sourceforge.net Tue Jan 1 03:23:55 2008 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 01 Jan 2008 03:23:55 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.1014,1.1015 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv10827 Modified Files: ChangeLog Log Message: [project @ on windows, make 'buildbot reconfig' print a warning instead of hanging. For #105.] Original author: warner at lothar.com Date: 2008-01-01 03:23:19+00:00 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.1014 retrieving revision 1.1015 diff -u -d -r1.1014 -r1.1015 --- ChangeLog 1 Jan 2008 02:46:37 -0000 1.1014 +++ ChangeLog 1 Jan 2008 03:23:53 -0000 1.1015 @@ -1,5 +1,11 @@ 2007-12-31 Brian Warner + * buildbot/scripts/reconfig.py (Reconfigurator.run): if we're on + windows, make 'buildbot reconfig' emit a warning message (since + there's no SIGHUP to be had) rather than just hanging. Addresses + #105, but we still need to document this somewhere. Thanks to + Ben Hearsum for the patch. + * buildbot/slave/commands.py (SlaveFileUploadCommand.start): rewrite the main loop to avoid tail recursion, since Deferreds don't optimize those out. Without this, at the end of the transfer From warner at users.sourceforge.net Tue Jan 1 03:23:55 2008 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 01 Jan 2008 03:23:55 +0000 Subject: [Buildbot-commits] buildbot/buildbot/scripts reconfig.py,1.3,1.4 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/scripts In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv10827/buildbot/scripts Modified Files: reconfig.py Log Message: [project @ on windows, make 'buildbot reconfig' print a warning instead of hanging. For #105.] Original author: warner at lothar.com Date: 2008-01-01 03:23:19+00:00 Index: reconfig.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/scripts/reconfig.py,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- reconfig.py 29 Sep 2007 21:23:12 -0000 1.3 +++ reconfig.py 1 Jan 2008 03:23:53 -0000 1.4 @@ -1,5 +1,5 @@ -import os, signal +import os, signal, platform from twisted.internet import reactor from buildbot.scripts.logwatcher import LogWatcher, BuildmasterTimeoutError, \ @@ -7,6 +7,12 @@ class Reconfigurator: def run(self, config): + # Returns "Microsoft" for Vista and "Windows" for other versions + if platform.system() in ("Windows", "Microsoft"): + print "Reconfig (through SIGHUP) is not supported on Windows." + print "The 'buildbot debugclient' tool can trigger a reconfig" + print "remotely, but requires Gtk+ libraries to run." + return basedir = config['basedir'] quiet = config['quiet'] From warner at users.sourceforge.net Tue Jan 1 03:27:39 2008 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 01 Jan 2008 03:27:39 +0000 Subject: [Buildbot-commits] buildbot ChangeLog,1.1015,1.1016 Message-ID: Update of /cvsroot/buildbot/buildbot In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12443 Modified Files: ChangeLog Log Message: [project @ test_transfer: remove the short timeouts, now that the patch is done] Original author: warner at lothar.com Date: 2008-01-01 03:27:16+00:00 Index: ChangeLog =================================================================== RCS file: /cvsroot/buildbot/buildbot/ChangeLog,v retrieving revision 1.1015 retrieving revision 1.1016 diff -u -d -r1.1015 -r1.1016 --- ChangeLog 1 Jan 2008 03:23:53 -0000 1.1015 +++ ChangeLog 1 Jan 2008 03:27:37 -0000 1.1016 @@ -1,5 +1,10 @@ 2007-12-31 Brian Warner + * buildbot/test/test_transfer.py (Upload.testLotsOfBlocks): remove + the short timeouts, these can cause false test failures on slow + systems, and were only needed while developing the recent patch + (Download.testLotsOfBlocks): same + * buildbot/scripts/reconfig.py (Reconfigurator.run): if we're on windows, make 'buildbot reconfig' emit a warning message (since there's no SIGHUP to be had) rather than just hanging. Addresses From warner at users.sourceforge.net Tue Jan 1 03:27:39 2008 From: warner at users.sourceforge.net (Brian Warner) Date: Tue, 01 Jan 2008 03:27:39 +0000 Subject: [Buildbot-commits] buildbot/buildbot/test test_transfer.py, 1.6, 1.7 Message-ID: Update of /cvsroot/buildbot/buildbot/buildbot/test In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv12443/buildbot/test Modified Files: test_transfer.py Log Message: [project @ test_transfer: remove the short timeouts, now that the patch is done] Original author: warner at lothar.com Date: 2008-01-01 03:27:16+00:00 Index: test_transfer.py =================================================================== RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_transfer.py,v retrieving revision 1.6 retrieving revision 1.7 diff -u -d -r1.6 -r1.7 --- test_transfer.py 1 Jan 2008 02:46:38 -0000 1.6 +++ test_transfer.py 1 Jan 2008 03:27:37 -0000 1.7 @@ -192,7 +192,6 @@ self.failUnlessEqual(masterdest_contents, contents) d.addCallback(_checkUpload) return d - testLotsOfBlocks.timeout = 20 class Download(StepTester, unittest.TestCase): @@ -359,7 +358,6 @@ self.failUnlessEqual(slavedest_contents, contents) d.addCallback(_checkDownload) return d - testLotsOfBlocks.timeout = 20 # TODO: