[Buildbot-devel] big custom logs horking the status web server?
Stephen Davis
stephendavis at mac.com
Wed Aug 3 09:14:18 UTC 2005
I've got a custom compile class that overrides createSummary() to
generate separate logs for the warnings and errors from a given
build. This was an attempt to pare the full build log (which is
multiple MB) down into something that's actually useful. My custom
class looks like:
class XcodeCompile( Compile ):
"""A compile class which filters the xcodebuild output into
something more manageable"""
def __init__( self, configuration=None, **kwargs ):
# set our step name to the build configuration being built
if configuration:
self.description = [ configuration ]
self.descriptionDone = [ configuration ]
Compile.__init__( self, **kwargs )
def createSummary( self, log ):
# build up regular expressions once for the whole loop
targetRegex = re.compile( r"=== BUILDING" );
warningRegex = re.compile( r"^/.*/+(.*warning:)" )
errRegex = re.compile( r"^/.*/+(.*:\d+: error:.*)" )
missingRegex = re.compile( r"Missing" )
linkerRegex = re.compile( r"^/usr/bin/ld: Undefined|_|
collect2" )
rezRegex = re.compile( r": ###" )
# [3086553] gcc issues spurious warnings for a legal AltiVec
construct and it clouds the log so ignore it
gccBugRegex = re.compile( r".*operand of comma
expression has no effect.*" )
warnings = ""
errors = ""
sio = StringIO.StringIO( log.getText() )
for line in sio.readlines():
if targetRegex.match( line ):
warnings += line
errors += line
elif warningRegex.match( line ):
if not gccBugRegex.match( line ):
line = warningRegex.sub( r"\1", line )
warnings += line
elif errRegex.match( line ):
line = errRegex.sub( r"\1", line )
errors += line
elif missingRegex.match( line ):
errors += line
elif linkerRegex.match( line ):
errors += line
elif rezRegex.match( line ):
errors += line
# add warnings and/or error summaries
if warnings:
self.addCompleteLog( 'warnings', warnings )
if errors:
self.addCompleteLog( 'errors', errors )
---
This has been working fine for a while but I just enabled every
warning I could and now the webserver status page is choking when I
click on the "warnings" log. The error page generated by the
webserver is at the end of this email and it's pretty long.
Anyway, I don't see this with small logs but I have seen the exact
same thing with a VisualStudioCompile custom compile class that looks
almost exactly the same as the XcodeCompile class (the regex's are
different but it's otherwise identical). The VisualStudio build has
always spit out a boatload of warnings and the warnings log has
always horked when clicked.
I'm thinking that something is going wrong with really big custom
logs but I'm not sure where to start poking around to see what's
breaking. I looked at the code referenced by the error page but it
was all gibberish to me.
The original multi-megabyte full log file has no trouble transferring
(ignoring the lack of a gzip option which would make it a whole lot
smaller) so it's something specific to the custom summary/log that
gets created.
Thoughts?
thanks,
stephen
P.S. Buildbot 0.6.6, Twisted 2.0.1 Core, Web, Mail
--
web.Server Traceback (most recent call last):
exceptions.AttributeError: 'NoneType' object has no attribute
'loseConnection'
/System/Library/Frameworks/Python.framework/Versions/2.3/lib/
python2.3/site-packages/twisted/web/server.py, line 153 in process
151 try:
152 resrc = self.site.getResourceFor(self)
153 self.render(resrc)
154 except:
Self
site
twisted.web.server.Site instance @ 0x7bf490
<twisted.web.server.Site instance at 0x7bf490>
Locals
resrc
buildbot.status.html.TextLog instance @ 0x166fb48
<buildbot.status.html.TextLog instance at 0x166fb48>
self
twisted.web.server.Request instance @ 0x166f4e0
<GET /MyApp-XC/builds/104/compile/1/text HTTP/1.1>
/System/Library/Frameworks/Python.framework/Versions/2.3/lib/
python2.3/site-packages/twisted/web/server.py, line 160 in render
158 def render(self, resrc):
159 try:
160 body = resrc.render(self)
161 except UnsupportedMethod, e:
Locals
resrc
buildbot.status.html.TextLog instance @ 0x166fb48
<buildbot.status.html.TextLog instance at 0x166fb48>
self
twisted.web.server.Request instance @ 0x166f4e0
<GET /MyApp-XC/builds/104/compile/1/text HTTP/1.1>
Globals
UnsupportedMethod
<class twisted.web.server.UnsupportedMethod at 0x6e3e10>
/System/Library/Frameworks/Python.framework/Versions/2.3/lib/
python2.3/site-packages/twisted/web/resource.py, line 189 in render
187 from twisted.web.server import UnsupportedMethod
188 raise UnsupportedMethod(getattr(self, 'allowedMethods', ()))
189 return m(request)
190
Locals
m
<bound method TextLog.render_GET of <buildbot.status.html.TextLog
instance at 0x166fb48>>
self
buildbot.status.html.TextLog instance @ 0x166fb48
<buildbot.status.html.TextLog instance at 0x166fb48>
request
twisted.web.server.Request instance @ 0x166f4e0
<GET /MyApp-XC/builds/104/compile/1/text HTTP/1.1>
/System/Library/Frameworks/Python.framework/Versions/2.3/lib/
python2.3/site-packages/buildbot/status/html.py, line 632 in render_GET
630 req.write(self.htmlHeader(req))
631
632 self.original.subscribeConsumer(ChunkConsumer(req, self))
633 return server.NOT_DONE_YET
Self
original
buildbot.status.builder.LogFile instance @ 0x1640990
<buildbot.status.builder.LogFile instance at 0x1640990>
Locals
self
buildbot.status.html.TextLog instance @ 0x166fb48
<buildbot.status.html.TextLog instance at 0x166fb48>
req
twisted.web.server.Request instance @ 0x166f4e0
<GET /MyApp-XC/builds/104/compile/1/text HTTP/1.1>
Globals
ChunkConsumer
<class buildbot.status.html.ChunkConsumer at 0x750270>
server
<module 'twisted.web.server' from '/System/Library/Frameworks/
Python.framework/Versions/2.3/lib/python2.3/site-packages/twisted/web/
server.pyc'>
/System/Library/Frameworks/Python.framework/Versions/2.3/lib/
python2.3/site-packages/buildbot/status/builder.py, line 334 in
subscribeConsumer
332 def subscribeConsumer(self, consumer):
333 p = LogFileProducer(self, consumer)
334 p.resumeProducing()
335
Locals
p
buildbot.status.builder.LogFileProducer instance @ 0x166fd50
<buildbot.status.builder.LogFileProducer instance at 0x166fd50>
self
buildbot.status.builder.LogFile instance @ 0x1640990
<buildbot.status.builder.LogFile instance at 0x1640990>
consumer
buildbot.status.html.ChunkConsumer instance @ 0x166fdf0
<buildbot.status.html.ChunkConsumer instance at 0x166fdf0>
Globals
LogFileProducer
<class buildbot.status.builder.LogFileProducer at 0x5f0ba0>
/System/Library/Frameworks/Python.framework/Versions/2.3/lib/
python2.3/site-packages/buildbot/status/builder.py, line 157 in
resumeProducing
155 try:
156 while not self.paused:
157 chunk = self.chunkGenerator.next()
158 self.consumer.writeChunk(chunk)
Self
chunkGenerator
<generator object at 0x166feb8>
paused
False
consumer
buildbot.status.html.ChunkConsumer instance @ 0x166fdf0
<buildbot.status.html.ChunkConsumer instance at 0x166fdf0>
Locals
self
buildbot.status.builder.LogFileProducer instance @ 0x166fd50
<buildbot.status.builder.LogFileProducer instance at 0x166fd50>
/System/Library/Frameworks/Python.framework/Versions/2.3/lib/
python2.3/site-packages/buildbot/status/builder.py, line 109 in
getChunks
107 offset = f.tell()
108 while data:
109 p.dataReceived(data)
110 while chunks:
Locals
p
buildbot.status.builder.LogFileScanner instance @ 0x166ff58
<buildbot.status.builder.LogFileScanner instance at 0x166ff58>
f
<open file '/Users/buildbot/Documents/master/MyApp-XC/104-log-compile-
warnings', mode 'rt' at 0x166d060>
chunks
List instance @ 0x1645ff0
offset
2048L
data
"[... snipped ...]"
/System/Library/Frameworks/Python.framework/Versions/2.3/lib/
python2.3/site-packages/twisted/protocols/basic.py, line 107 in
dataReceived
105 raise RuntimeError, "mode is not DATA, COMMA or LENGTH"
106 except NetstringParseError:
107 self.transport.loseConnection()
108 self.brokenPeer = 1
Locals
self
buildbot.status.builder.LogFileScanner instance @ 0x166ff58
<buildbot.status.builder.LogFileScanner instance at 0x166ff58>
Globals
DATA
1
NetstringParseError
<class twisted.protocols.basic.NetstringParseError at 0x5f0810>
LENGTH
0
COMMA
2
exceptions.AttributeError: 'NoneType' object has no attribute
'loseConnection'
More information about the devel
mailing list