[Buildbot-commits] buildbot/buildbot/test test_status.py,1.16,1.17 test_twisted.py,1.5,1.6
Brian Warner
warner at users.sourceforge.net
Sun May 15 23:43:59 UTC 2005
Update of /cvsroot/buildbot/buildbot/buildbot/test
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv688/buildbot/test
Modified Files:
test_status.py test_twisted.py
Log Message:
Revision: arch at buildbot.sf.net--2004/buildbot--dev--0--patch-171
Creator: Brian Warner <warner at monolith.lothar.com>
handle large logfiles without consuming lots of memory
Merged from warner at monolith.lothar.com--2005 (patch 19-25)
Patches applied:
* warner at monolith.lothar.com--2005/buildbot--dev--0--patch-19
Merged from arch at buildbot.sf.net--2004 (patch 159-160)
* warner at monolith.lothar.com--2005/buildbot--dev--0--patch-20
Merged from arch at buildbot.sf.net--2004 (patch 161-166)
* warner at monolith.lothar.com--2005/buildbot--dev--0--patch-21
Merged from arch at buildbot.sf.net--2004 (patch 167)
* warner at monolith.lothar.com--2005/buildbot--dev--0--patch-22
Merged from arch at buildbot.sf.net--2004 (patch 168)
* warner at monolith.lothar.com--2005/buildbot--dev--0--patch-23
Merged from arch at buildbot.sf.net--2004 (patch 169)
* warner at monolith.lothar.com--2005/buildbot--dev--0--patch-24
Merged from arch at buildbot.sf.net--2004 (patch 170)
* warner at monolith.lothar.com--2005/buildbot--dev--0--patch-25
handle large log files without using lots of memory
Index: test_status.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_status.py,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -d -r1.16 -r1.17
--- test_status.py 6 May 2005 06:40:04 -0000 1.16
+++ test_status.py 15 May 2005 23:43:57 -0000 1.17
@@ -12,16 +12,46 @@
from buildbot.status import progress, client # NEEDS COVERAGE
class MyStep:
+ build = None
def getName(self):
return "step"
class MyLog(builder.LogFile):
- def __init__(self, name, text):
- builder.LogFile.__init__(self, None)
- self.name = name
- self.addStdout(text)
- self.finish()
- self.step = MyStep()
+ def __init__(self, basedir, name, text=None, step=None):
+ self.fakeBuilderBasedir = basedir
+ if not step:
+ step = MyStep()
+ builder.LogFile.__init__(self, step, name, name)
+ if text:
+ self.addStdout(text)
+ self.finish()
+ def getFilename(self):
+ return os.path.join(self.fakeBuilderBasedir, self.name)
+
+class MyLogSubscriber:
+ def __init__(self):
+ self.chunks = []
+ def logChunk(self, build, step, log, channel, text):
+ self.chunks.append((channel, text))
+
+class MyLogConsumer:
+ def __init__(self, limit=None):
+ self.chunks = []
+ self.finished = False
+ self.limit = limit
+ def registerProducer(self, producer, streaming):
+ self.producer = producer
+ self.streaming = streaming
+ def unregisterProducer(self):
+ self.producer = None
+ def writeChunk(self, chunk):
+ self.chunks.append(chunk)
+ if self.limit:
+ self.limit -= 1
+ if self.limit == 0:
+ self.producer.pauseProducing()
+ def finish(self):
+ self.finished = True
class MyMailer(mail.MailNotifier):
def sendMessage(self, m, recipients):
@@ -223,6 +253,8 @@
"recip2 at example.com", "recip at example.com"])
def testLogs(self):
+ basedir = "test_status_logs"
+ os.mkdir(basedir)
mailer = MyMailer(fromaddr="buildbot at example.com", addLogs=True,
extraRecipients=["recip at example.com",
"recip2 at example.com"])
@@ -231,8 +263,9 @@
self.messages = []
b1 = self.makeBuild(3, builder.WARNINGS)
- b1.testlogs = [MyLog('compile', "Compile log here\n"),
- MyLog('test', "Test log here\nTest 4 failed\n"),
+ b1.testlogs = [MyLog(basedir, 'compile', "Compile log here\n"),
+ MyLog(basedir,
+ 'test', "Test log here\nTest 4 failed\n"),
]
b1.text = ["unusual", "gnarzzler", "output"]
mailer.buildFinished("builder1", b1, b1.results)
@@ -254,6 +287,8 @@
self.failUnlessIn("Test log here\n", p[2].get_payload())
def testMail(self):
+ basedir = "test_status_mail"
+ os.mkdir(basedir)
dest = os.environ.get("BUILDBOT_TEST_MAIL")
if not dest:
raise unittest.SkipTest("define BUILDBOT_TEST_MAIL=dest to run this")
@@ -265,8 +300,9 @@
mailer.status = s
b1 = self.makeBuild(3, builder.SUCCESS)
- b1.testlogs = [MyLog('compile', "Compile log here\n"),
- MyLog('test', "Test log here\nTest 4 failed\n"),
+ b1.testlogs = [MyLog(basedir, 'compile', "Compile log here\n"),
+ MyLog(basedir,
+ 'test', "Test log here\nTest 4 failed\n"),
]
print "sending mail to", dest
@@ -309,10 +345,12 @@
self.failUnlessEqual(t.getLogs(), {'output': ""})
class Log(unittest.TestCase):
+ def setUpClass(self):
+ self.basedir = "status_log_add"
+ os.mkdir(self.basedir)
+
def testAdd(self):
- l = builder.LogFile(None)
- l.name = "compile"
- l.step = 13
+ l = MyLog(self.basedir, "compile", step=13)
self.failUnlessEqual(l.getName(), "compile")
self.failUnlessEqual(l.getStep(), 13)
l.addHeader("HEADER\n")
@@ -327,10 +365,10 @@
self.failUnlessEqual(l.getTextWithHeaders(),
"HEADER\n" +
"Some text\nSome error\nSome more text\n")
- self.failUnlessEqual(len(l.getChunks()), 4)
+ self.failUnlessEqual(len(list(l.getChunks())), 4)
- def testMerge(self):
- l = builder.LogFile(None)
+ def testMerge1(self):
+ l = MyLog(self.basedir, "merge1")
l.addHeader("HEADER\n")
l.addStdout("Some text\n")
l.addStdout("Some more text\n")
@@ -341,10 +379,10 @@
self.failUnlessEqual(l.getTextWithHeaders(),
"HEADER\n" +
"Some text\nSome more text\nmore\n")
- self.failUnlessEqual(len(l.getChunks()), 2)
+ self.failUnlessEqual(len(list(l.getChunks())), 2)
def testMerge2(self):
- l = builder.LogFile(None)
+ l = MyLog(self.basedir, "merge2")
l.addHeader("HEADER\n")
for i in xrange(1000):
l.addStdout("aaaa")
@@ -352,26 +390,191 @@
l.addStderr("bbbb")
for i in xrange(10):
l.addStdout("cc")
- self.failUnlessEqual(l.getText(),
- 1000*"aaaa" + 30 * "bbbb" + 10 * "cc")
+ target = 1000*"aaaa" + 30 * "bbbb" + 10 * "cc"
+ self.failUnlessEqual(len(l.getText()), len(target))
+ self.failUnlessEqual(l.getText(), target)
+ l.finish()
+ self.failUnlessEqual(len(l.getText()), len(target))
+ self.failUnlessEqual(l.getText(), target)
+ self.failUnlessEqual(len(list(l.getChunks())), 4)
+
+ def testMerge3(self):
+ l = MyLog(self.basedir, "merge3")
+ l.chunkSize = 100
+ l.addHeader("HEADER\n")
+ for i in xrange(8):
+ l.addStdout(10*"a")
+ for i in xrange(8):
+ l.addStdout(10*"a")
+ self.failUnlessEqual(list(l.getChunks()),
+ [(builder.HEADER, "HEADER\n"),
+ (builder.STDOUT, 110*"a"),
+ (builder.STDOUT, 50*"a")])
+ l.finish()
+ self.failUnlessEqual(l.getText(), 160*"a")
+
+ def testChunks(self):
+ l = MyLog(self.basedir, "chunks")
+ c1 = l.getChunks()
+ l.addHeader("HEADER\n")
+ l.addStdout("Some text\n")
+ self.failUnlessEqual("".join(l.getChunks(onlyText=True)),
+ "HEADER\nSome text\n")
+ c2 = l.getChunks()
+
+ l.addStdout("Some more text\n")
+ self.failUnlessEqual("".join(l.getChunks(onlyText=True)),
+ "HEADER\nSome text\nSome more text\n")
+ c3 = l.getChunks()
+
+ l.addStdout("more\n")
l.finish()
+
+ self.failUnlessEqual(list(c1), [])
+ self.failUnlessEqual(list(c2), [(builder.HEADER, "HEADER\n"),
+ (builder.STDOUT, "Some text\n")])
+ self.failUnlessEqual(list(c3), [(builder.HEADER, "HEADER\n"),
+ (builder.STDOUT,
+ "Some text\nSome more text\n")])
+
self.failUnlessEqual(l.getText(),
- 1000*"aaaa" + 30 * "bbbb" + 10 * "cc")
- self.failUnlessEqual(len(l.getChunks()), 4)
+ "Some text\nSome more text\nmore\n")
+ self.failUnlessEqual(l.getTextWithHeaders(),
+ "HEADER\n" +
+ "Some text\nSome more text\nmore\n")
+ self.failUnlessEqual(len(list(l.getChunks())), 2)
- def testStubify(self):
- l = builder.LogFile(None)
- l.name = "compile"
- l.step = 13
+ def testUpgrade(self):
+ l = MyLog(self.basedir, "upgrade")
l.addHeader("HEADER\n")
l.addStdout("Some text\n")
l.addStdout("Some more text\n")
l.addStdout("more\n")
- stub = l.stubify()
- self.failUnless(components.implements(stub,
- interfaces.IStatusLogStub))
- self.failUnlessEqual(stub.getName(), l.getName())
- self.failUnlessEqual(stub.getStep(), l.getStep())
+ l.finish()
+ # now doctor it to look like a 0.6.4-era non-upgraded logfile
+ l.entries = list(l.getChunks())
+ del l.filename
+ os.unlink(l.getFilename())
+ # now make sure we can upgrade it
+ l.upgrade("upgrade")
+ self.failUnlessEqual(l.getText(),
+ "Some text\nSome more text\nmore\n")
+ self.failUnlessEqual(len(list(l.getChunks())), 2)
+ self.failIf(l.entries)
+
+ # now, do it again, but make it look like an upgraded 0.6.4 logfile
+ # (i.e. l.filename is missing, but the contents are there on disk)
+ l.entries = list(l.getChunks())
+ del l.filename
+ l.upgrade("upgrade")
+ self.failUnlessEqual(l.getText(),
+ "Some text\nSome more text\nmore\n")
+ self.failUnlessEqual(len(list(l.getChunks())), 2)
+ self.failIf(l.entries)
+
+ def testSubscribe(self):
+ l1 = MyLog(self.basedir, "subscribe1")
+ l1.finish()
+ self.failUnless(l1.isFinished())
+
+ s = MyLogSubscriber()
+ l1.subscribe(s, True)
+ l1.unsubscribe(s)
+ self.failIf(s.chunks)
+
+ s = MyLogSubscriber()
+ l1.subscribe(s, False)
+ l1.unsubscribe(s)
+ self.failIf(s.chunks)
+
+ finished = []
+ l2 = MyLog(self.basedir, "subscribe2")
+ l2.waitUntilFinished().addCallback(finished.append)
+ l2.addHeader("HEADER\n")
+ s1 = MyLogSubscriber()
+ l2.subscribe(s1, True)
+ s2 = MyLogSubscriber()
+ l2.subscribe(s2, False)
+ self.failUnlessEqual(s1.chunks, [(builder.HEADER, "HEADER\n")])
+ self.failUnlessEqual(s2.chunks, [])
+
+ l2.addStdout("Some text\n")
+ self.failUnlessEqual(s1.chunks, [(builder.HEADER, "HEADER\n"),
+ (builder.STDOUT, "Some text\n")])
+ self.failUnlessEqual(s2.chunks, [(builder.STDOUT, "Some text\n")])
+ l2.unsubscribe(s1)
+
+ l2.addStdout("Some more text\n")
+ self.failUnlessEqual(s1.chunks, [(builder.HEADER, "HEADER\n"),
+ (builder.STDOUT, "Some text\n")])
+ self.failUnlessEqual(s2.chunks, [(builder.STDOUT, "Some text\n"),
+ (builder.STDOUT, "Some more text\n"),
+ ])
+ self.failIf(finished)
+ l2.finish()
+ self.failUnlessEqual(finished, [l2])
+
+ def testConsumer(self):
+ l1 = MyLog(self.basedir, "consumer1")
+ l1.finish()
+ self.failUnless(l1.isFinished())
+
+ s = MyLogConsumer()
+ l1.subscribeConsumer(s)
+ self.failIf(s.chunks)
+ self.failUnless(s.finished)
+ self.failIf(s.producer) # producer should be registered and removed
+
+ l2 = MyLog(self.basedir, "consumer2")
+ l2.addHeader("HEADER\n")
+ l2.finish()
+ self.failUnless(l2.isFinished())
+
+ s = MyLogConsumer()
+ l2.subscribeConsumer(s)
+ self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n")])
+ self.failUnless(s.finished)
+ self.failIf(s.producer) # producer should be registered and removed
+
+
+ l2 = MyLog(self.basedir, "consumer3")
+ l2.chunkSize = 1000
+ l2.addHeader("HEADER\n")
+ l2.addStdout(800*"a")
+ l2.addStdout(800*"a") # should now have two chunks on disk
+ l2.addStdout(800*"b") # HEADER,1600*a on disk, 800*a in memory
+ l2.addStdout(800*"b") # HEADER,1600*a,1600*b on disk
+ l2.addStdout(200*"c") # HEADER,1600*a,1600*b on disk,200*c in memory
+
+ s = MyLogConsumer(limit=1)
+ l2.subscribeConsumer(s)
+ self.failUnless(s.streaming)
+ self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n")])
+ s.limit = 1
+ s.producer.resumeProducing()
+ self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
+ (builder.STDOUT, 1600*"a")])
+ s.limit = None
+ s.producer.resumeProducing()
+ self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
+ (builder.STDOUT, 1600*"a"),
+ (builder.STDOUT, 1600*"b"),
+ (builder.STDOUT, 200*"c")])
+ l2.addStdout(1000*"c") # HEADER,1600*a,1600*b,1200*c on disk
+ self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
+ (builder.STDOUT, 1600*"a"),
+ (builder.STDOUT, 1600*"b"),
+ (builder.STDOUT, 200*"c"),
+ (builder.STDOUT, 1000*"c")])
+ l2.finish()
+ self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
+ (builder.STDOUT, 1600*"a"),
+ (builder.STDOUT, 1600*"b"),
+ (builder.STDOUT, 200*"c"),
+ (builder.STDOUT, 1000*"c")])
+ self.failIf(s.producer)
+ self.failUnless(s.finished)
+
class Client(unittest.TestCase):
def testAdaptation(self):
Index: test_twisted.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_twisted.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- test_twisted.py 26 Apr 2005 09:43:19 -0000 1.5
+++ test_twisted.py 15 May 2005 23:43:57 -0000 1.6
@@ -98,6 +98,12 @@
def addCompleteLog(self, name, log):
pass
+class MyLogFile:
+ def __init__(self, text):
+ self.text = text
+ def getText(self):
+ return self.text
+
class Count(unittest.TestCase):
@@ -132,8 +138,7 @@
def testParse(self):
t = MyTrial(build=None, workdir=".", testpath=None, testChanges=True)
t.results = []
- log = builder.LogFile(None)
- log.addStdout(out6)
+ log = MyLogFile(out6)
t.createSummary(log)
self.failUnlessEqual(len(t.results), 4)
More information about the Commits
mailing list