[Buildbot-commits] buildbot/buildbot/status html.py,1.61,1.62 builder.py,1.53,1.54

[Brian Warner]

Update of /cvsroot/buildbot/buildbot/buildbot/status
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv15619/buildbot/status

Modified Files:
	html.py builder.py 
Log Message:
Revision: arch at buildbot.sf.net--2004/buildbot--dev--0--patch-165
Creator:  Brian Warner <warner at monolith.lothar.com>

fix OfflineLogFile to work with html.Waterfall

	* buildbot/status/builder.py (OfflineLogFile.getChunks): have this
	method generate chunks instead of returning a big list. This
	allows the same method to be used for both old LogFile and new
	OfflineLogFile.
	(OfflineLogFile.getText): use the generator
	(OfflineLogFile.subscribe): same
	* buildbot/status/html.py (TextLog.resumeProducing): same
	* buildbot/interfaces.py (IStatusLog.getChunks): document it


Index: builder.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/status/builder.py,v
retrieving revision 1.53
retrieving revision 1.54
diff -u -d -r1.53 -r1.54
--- builder.py	6 May 2005 06:40:05 -0000	1.53
+++ builder.py	12 May 2005 21:55:57 -0000	1.54
@@ -118,7 +118,7 @@
             return
         self.watchers.append(receiver)
         if catchup:
-            for channel, text in self.entries + self.runEntries:
+            for channel, text in self.getChunks():
                 # TODO: add logChunks(), to send over everything at once?
                 receiver.logChunk(self.step.build, self.step, self,
                                   channel, text)
@@ -202,13 +202,13 @@
         self.finished = True
 
 class OfflineLogFileScanner(basic.NetstringReceiver):
-    def __init__(self, line_cb, channels=[]):
-        self.line_cb = line_cb
+    def __init__(self, chunk_cb, channels=[]):
+        self.chunk_cb = chunk_cb
         self.channels = channels
     def stringReceived(self, line):
         channel = int(line[0])
         if not self.channels or (channel in self.channels):
-            self.line_cb(channel, line[1:])
+            self.chunk_cb((channel, line[1:]))
 
 class OfflineLogFile(LogFile):
     """An OfflineLogFile keeps all of its contents on disk, in a non-pickle
@@ -235,44 +235,51 @@
             self.openfile = f
         return self.openfile
 
-    def scanEntries(self, cb, channels=[]):
+    def getChunks(self, channels=[], onlyText=False):
         if self.openfile:
             # we must close it so we can read it properly. It will be opened
             # again the next time we try to write to it.
             self.openfile.close()
             self.openfile = None
         f = open(self.step.build.getLogfileNamed(self.filename), "r")
-        p = OfflineLogFileScanner(cb, channels)
+        chunks = []
+        p = OfflineLogFileScanner(chunks.append, channels)
+        # TODO: if merge() is called while we're paused, we might miss some
+        # data.
         data = f.read(2000)
         while data:
             p.dataReceived(data)
+            for c in chunks:
+                if onlyText:
+                    yield c[1]
+                else:
+                    yield c
+            chunks = []
             data = f.read(2000)
         f.close()
+        # TODO: if merge() is called while we're paused, we'll lose some data
         for channel, text in self.runEntries:
             if not channels or (channel in channels):
-                cb(channel, text)
+                if onlyText:
+                    yield text
+                else:
+                    yield (channel, text)
 
     def getText(self):
         # this produces one ginormous string
-        chunks = []
-        self.scanEntries(lambda channel,chunk: chunks.append(chunk),
-                         (STDOUT, STDERR))
-        return "".join(chunks)
+        return "".join(self.getChunks([STDOUT, STDERR], onlyText=True))
 
     def getTextWithHeaders(self):
-        chunks = []
-        self.scanEntries(lambda channel,chunk: chunks.append(chunk))
-        return "".join(chunks)
+        return "".join(self.getChunks(onlyText=True))
 
     def subscribe(self, receiver, catchup):
         if self.finished:
             return
         self.watchers.append(receiver)
         if catchup:
-            self.scanEntries(lambda channel,chunk:
-                             receiver.logChunk(self.step.build,
-                                               self.step, self,
-                                               channel, chunk))
+            for channel,chunk in self.getChunks():
+                receiver.logChunk(self.step.build, self.step, self,
+                                  channel, chunk)
 
     def merge(self):
         # merge all .runEntries (which are all of the same type) into a

Index: html.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/status/html.py,v
retrieving revision 1.61
retrieving revision 1.62
diff -u -d -r1.61 -r1.62
--- html.py	12 May 2005 20:10:11 -0000	1.61
+++ html.py	12 May 2005 21:55:57 -0000	1.62
@@ -533,6 +533,7 @@
     # it, so we can afford to track the request in the Resource.
     __implements__ = IHTMLLog,
     asText = False
+    subscribed = False
 
     def __init__(self, original):
         Resource.__init__(self)
@@ -583,20 +584,16 @@
         return ''
 
     def resumeProducing(self):
-        while self.chunkNumber < len(self.original.entries):
-            chunk = self.original.entries[self.chunkNumber]
-            self.chunkNumber += 1
+        try:
+            chunk = self.chunkGenerator.next()
             data = self.content([chunk])
             if data:
                 self.req.write(data)
-                return
-        # TODO: I've seen this double-print a line
-        # now send all of .runEntries in a batch
-        data = self.content(self.original.runEntries)
-        if data:
-            self.req.write(data)
+            return
+        except StopIteration:
+            pass
         self.req.unregisterProducer()
-        # then see if there is more to come
+        # now subscribe to anything that might happen later
         self.original.subscribe(self, False)
         self.subscribed = True
         d = self.original.waitUntilFinished()
@@ -622,8 +619,7 @@
         if not self.asText:
             req.write(self.htmlHeader(req))
 
-        self.chunkNumber = 0
-        self.subscribed = False
+        self.chunkGenerator = self.original.getChunks()
         req.registerProducer(self, False)
         d = req.notifyFinish()
         d.addErrback(self.stop)