[Buildbot-commits] buildbot/buildbot/test test_web.py,1.20,1.21 test_status.py,1.22,1.23
Brian Warner
warner at users.sourceforge.net
Tue Aug 9 00:43:37 UTC 2005
Update of /cvsroot/buildbot/buildbot/buildbot/test
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv26164/buildbot/test
Modified Files:
test_web.py test_status.py
Log Message:
Revision: arch at buildbot.sf.net--2004/buildbot--dev--0--patch-268
Creator: Brian Warner <warner at monolith.lothar.com>
fix the large-logfile-hang against twisted-1.3.0
* buildbot/status/builder.py (LogFileProducer.resumeProducing):
put off the actual resumeProducing for a moment with
reactor.callLater(0). This works around a twisted-1.3.0 bug which
causes large logfiles to hang midway through.
* buildbot/test/test_status.py (Log.testMerge3): update to match new
addEntry merging (>=chunkSize) behavior
(Log.testConsumer): update to handle new callLater(0) behavior
* buildbot/test/test_web.py: rearrange tests a bit, add test for
both the MAX_LENGTH bugfix and the resumeProducing hang.
--This line, and those below, will be ignored--
Files to commit:
<can't compute list>
This list might be incomplete or outdated if editing the log
message was not invoked from an up-to-date changes buffer!
Index: test_status.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_status.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- test_status.py 19 Jul 2005 23:11:59 -0000 1.22
+++ test_status.py 9 Aug 2005 00:43:35 -0000 1.23
@@ -8,7 +8,7 @@
from buildbot import interfaces
from buildbot.sourcestamp import SourceStamp
-from buildbot.twcompat import implements, providedBy
+from buildbot.twcompat import implements, providedBy, maybeWait
from buildbot.status import builder
try:
from buildbot.status import mail
@@ -21,6 +21,18 @@
def getName(self):
return "step"
+class MyLogFileProducer(builder.LogFileProducer):
+ # The reactor.callLater(0) in LogFileProducer.resumeProducing is a bit of
+ # a nuisance from a testing point of view. This subclass adds a Deferred
+ # to that call so we can find out when it is complete.
+ def resumeProducing(self):
+ d = defer.Deferred()
+ reactor.callLater(0, self._resumeProducing, d)
+ return d
+ def _resumeProducing(self, d):
+ builder.LogFileProducer._resumeProducing(self)
+ reactor.callLater(0, d.callback, None)
+
class MyLog(builder.LogFile):
def __init__(self, basedir, name, text=None, step=None):
self.fakeBuilderBasedir = basedir
@@ -33,6 +45,11 @@
def getFilename(self):
return os.path.join(self.fakeBuilderBasedir, self.name)
+ def subscribeConsumer(self, consumer):
+ p = MyLogFileProducer(self, consumer)
+ d = p.resumeProducing()
+ return d
+
class MyHTMLLog(builder.HTMLLogFile):
def __init__(self, basedir, name, html):
step = MyStep()
@@ -439,8 +456,8 @@
l.addStdout(10*"a")
self.failUnlessEqual(list(l.getChunks()),
[(builder.HEADER, "HEADER\n"),
- (builder.STDOUT, 110*"a"),
- (builder.STDOUT, 50*"a")])
+ (builder.STDOUT, 100*"a"),
+ (builder.STDOUT, 60*"a")])
l.finish()
self.failUnlessEqual(l.getText(), 160*"a")
@@ -557,7 +574,10 @@
self.failUnless(l1.isFinished())
s = MyLogConsumer()
- l1.subscribeConsumer(s)
+ d = l1.subscribeConsumer(s)
+ d.addCallback(self._testConsumer_1, s)
+ return maybeWait(d, 5)
+ def _testConsumer_1(self, res, s):
self.failIf(s.chunks)
self.failUnless(s.finished)
self.failIf(s.producer) # producer should be registered and removed
@@ -568,7 +588,10 @@
self.failUnless(l2.isFinished())
s = MyLogConsumer()
- l2.subscribeConsumer(s)
+ d = l2.subscribeConsumer(s)
+ d.addCallback(self._testConsumer_2, s)
+ return d
+ def _testConsumer_2(self, res, s):
self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n")])
self.failUnless(s.finished)
self.failIf(s.producer) # producer should be registered and removed
@@ -584,15 +607,24 @@
l2.addStdout(200*"c") # HEADER,1600*a,1600*b on disk,200*c in memory
s = MyLogConsumer(limit=1)
- l2.subscribeConsumer(s)
+ d = l2.subscribeConsumer(s)
+ d.addCallback(self._testConsumer_3, l2, s)
+ return d
+ def _testConsumer_3(self, res, l2, s):
self.failUnless(s.streaming)
self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n")])
s.limit = 1
- s.producer.resumeProducing()
+ d = s.producer.resumeProducing()
+ d.addCallback(self._testConsumer_4, l2, s)
+ return d
+ def _testConsumer_4(self, res, l2, s):
self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
(builder.STDOUT, 1600*"a")])
s.limit = None
- s.producer.resumeProducing()
+ d = s.producer.resumeProducing()
+ d.addCallback(self._testConsumer_5, l2, s)
+ return d
+ def _testConsumer_5(self, res, l2, s):
self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
(builder.STDOUT, 1600*"a"),
(builder.STDOUT, 1600*"b"),
Index: test_web.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_web.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -d -r1.20 -r1.21
--- test_web.py 19 Jul 2005 23:51:52 -0000 1.20
+++ test_web.py 9 Aug 2005 00:43:35 -0000 1.21
@@ -1,12 +1,12 @@
# -*- test-case-name: buildbot.test.test_web -*-
-import sys, os, os.path, time
+import sys, os, os.path, time, shutil
from twisted.python import log, components, util
#log.startLogging(sys.stderr)
from twisted.trial import unittest
-from twisted.internet import reactor, defer
+from twisted.internet import reactor, defer, protocol
from twisted.internet.interfaces import IReactorUNIX
from twisted.web import client
@@ -71,8 +71,35 @@
def _shutdown_1(self, res):
return self.r.publisher.broker.transport.loseConnection()
+class SlowReader(protocol.Protocol):
+ didPause = False
+ count = 0
+ data = ""
+ def __init__(self, req):
+ self.req = req
+ self.d = defer.Deferred()
+ def connectionMade(self):
+ self.transport.write(self.req)
+ def dataReceived(self, data):
+ self.data += data
+ self.count += len(data)
+ if not self.didPause and self.count > 10*1000:
+ self.didPause = True
+ self.transport.pauseProducing()
+ reactor.callLater(2, self.resume)
+ def resume(self):
+ self.transport.resumeProducing()
+ def connectionLost(self, why):
+ self.d.callback(None)
-class WebTest(unittest.TestCase):
+class CFactory(protocol.ClientFactory):
+ def __init__(self, p):
+ self.p = p
+ def buildProtocol(self, addr):
+ self.p.factory = self
+ return self.p
+
+class BaseWeb:
master = None
def failUnlessIn(self, substr, string):
@@ -93,6 +120,8 @@
return filter(lambda child: isinstance(child, html.Waterfall),
list(master))
+class Ports(BaseWeb, unittest.TestCase):
+
def test_webPortnum(self):
# run a regular web server on a TCP socket
config = base_config + "c['status'] = [html.Waterfall(http_port=0)]\n"
@@ -151,6 +180,8 @@
self.failUnlessIn("BuildBot", page)
return p.shutdown()
+
+class Waterfall(BaseWeb, unittest.TestCase):
def test_waterfall(self):
# this is the right way to configure the Waterfall status
config1 = \
@@ -200,7 +231,8 @@
self.failUnlessIn("<li>Syncmail mailing list in maildir " +
"my-maildir</li>", changes)
- def test_logfile(self):
+class Logfile(BaseWeb, unittest.TestCase):
+ def setUp(self):
config = """
from buildbot.status import html
from buildbot.process.factory import BasicBuildFactory
@@ -215,11 +247,14 @@
'status': [html.Waterfall(http_port=0)],
}
"""
- os.mkdir("test_web5")
- self.master = m = ConfiguredMaster("test_web5", config)
+ if os.path.exists("test_logfile"):
+ shutil.rmtree("test_logfile")
+ os.mkdir("test_logfile")
+ self.master = m = ConfiguredMaster("test_logfile", config)
m.startService()
# hack to find out what randomly-assigned port it is listening on
port = list(self.find_waterfall(m)[0])[0]._port.getHost().port
+ self.port = port
# insert an event
s = m.status.getBuilder("builder1")
@@ -238,33 +273,85 @@
log2 = step1.addHTMLLog("error", "<html>ouch</html>")
+ log3 = step1.addLog("big")
+ log3.addStdout("big log\n")
+ for i in range(1000):
+ log3.addStdout("a" * 500)
+ log3.addStderr("b" * 500)
+ log3.finish()
+
+ log4 = step1.addCompleteLog("bigcomplete",
+ "big2 log\n" + "a" * 1*1000*1000)
+
step1.step_status.stepFinished(builder.SUCCESS)
bs.buildFinished()
- d = client.getPage("http://localhost:%d/" % port)
- d.addCallback(self._test_logfile_1, port)
+
+ def test_logfile1(self):
+ d = client.getPage("http://localhost:%d/" % self.port)
+ d.addCallback(self._test_logfile1_1)
return maybeWait(d)
- test_logfile.timeout = 10
- def _test_logfile_1(self, page, port):
+ test_logfile1.timeout = 20
+ def _test_logfile1_1(self, page):
self.failUnless(page)
- logurl = "http://localhost:%d/builder1/builds/0/setup/0" % port
+ def test_logfile2(self):
+ logurl = "http://localhost:%d/builder1/builds/0/setup/0" % self.port
d = client.getPage(logurl)
- d.addCallback(self._test_logfile_2, port)
- return d
- def _test_logfile_2(self, logbody, port):
+ d.addCallback(self._test_logfile2_1)
+ return maybeWait(d)
+ def _test_logfile2_1(self, logbody):
self.failUnless(logbody)
- logurl = "http://localhost:%d/builder1/builds/0/setup/0" % port
+
+ def test_logfile3(self):
+ logurl = "http://localhost:%d/builder1/builds/0/setup/0" % self.port
d = client.getPage(logurl + "/text")
- d.addCallback(self._test_logfile_3, port)
- return d
- def _test_logfile_3(self, logtext, port):
+ d.addCallback(self._test_logfile3_1)
+ return maybeWait(d)
+ def _test_logfile3_1(self, logtext):
self.failUnlessEqual(logtext, "some stdout\n")
- logurl = "http://localhost:%d/builder1/builds/0/setup/1" % port
+ def test_logfile4(self):
+ logurl = "http://localhost:%d/builder1/builds/0/setup/1" % self.port
d = client.getPage(logurl)
- d.addCallback(self._test_logfile_4)
- return d
- def _test_logfile_4(self, logbody):
+ d.addCallback(self._test_logfile4_1)
+ return maybeWait(d)
+ def _test_logfile4_1(self, logbody):
self.failUnlessEqual(logbody, "<html>ouch</html>")
+ def test_logfile5(self):
+ # this is log3, which is about 1MB in size, made up of alternating
+ # stdout/stderr chunks. buildbot-0.6.6, when run against
+ # twisted-1.3.0, fails to resume sending chunks after the client
+ # stalls for a few seconds, because of a recursive doWrite() call
+ # that was fixed in twisted-2.0.0
+ p = SlowReader("GET /builder1/builds/0/setup/2 HTTP/1.0\r\n\r\n")
+ f = CFactory(p)
+ c = reactor.connectTCP("localhost", self.port, f)
+ d = p.d
+ d.addCallback(self._test_logfile5_1, p)
+ return maybeWait(d, 10)
+ test_logfile5.timeout = 10
+ def _test_logfile5_1(self, res, p):
+ self.failUnlessIn("big log", p.data)
+ self.failUnlessIn("a"*100, p.data)
+ self.failUnless(p.count > 1*1000*1000)
+
+ def test_logfile6(self):
+ # this is log4, which is about 1MB in size, one big chunk.
+ # buildbot-0.6.6 dies as the NetstringReceiver barfs on the
+ # saved logfile, because it was using one big chunk and exceeding
+ # NetstringReceiver.MAX_LENGTH
+ p = SlowReader("GET /builder1/builds/0/setup/3 HTTP/1.0\r\n\r\n")
+ f = CFactory(p)
+ c = reactor.connectTCP("localhost", self.port, f)
+ d = p.d
+ d.addCallback(self._test_logfile6_1, p)
+ return maybeWait(d, 10)
+ test_logfile6.timeout = 10
+ def _test_logfile6_1(self, res, p):
+ self.failUnlessIn("big2 log", p.data)
+ self.failUnlessIn("a"*100, p.data)
+ self.failUnless(p.count > 1*1000*1000)
+
+
More information about the Commits
mailing list