[users at bb.net] Buildbot becoming unresponsive

Will Newton will.newton at gmail.com
Fri Mar 23 09:47:28 UTC 2018 

Hi,

I have a build that has some quite long-running and CPU intensive
steps in it (building Android images) and it seems like I often run
into issues with the buildbot UI becoming unresponsive.

The home page will load but it will read:

    Welcome to buildbot
    0 build running currently
    0 recent builds

Which I know is wrong (on both counts). If I look in the twisted log
for the build master I see the below errors. I'm not sure what
triggers them, maybe refreshing the page, but as you can see from the
timestamps the sync job has been running for some time at this point.

Does anyone have any suggestions as to how I can debug this?

Thanks,

2018-03-23 09:29:58+0000 [-] <RemoteShellCommand '['repo', 'sync',
'--force-sync', '-j6', '-c', '-q']'>: RemoteCommand.run [38]
2018-03-23 09:29:58+0000 [-] command '['repo', 'sync', '--force-sync',
'-j6', '-c', '-q']' in dir 'build'
2018-03-23 09:41:42+0000 [-] while handling API request
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1442, in gotResult
        _inlineCallbacks(r, g, deferred)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1386, in _inlineCallbacks
        result = g.send(result)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__
        self.gen.throw(type, value, traceback)
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 141, in handleErrors
        yield
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/server.py",
line 234, in write
        http.Request.write(self, data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/http.py",
line 1069, in write
        self.channel.writeHeaders(version, code, reason, headers)
    exceptions.AttributeError: 'NoneType' object has no attribute 'writeHeaders'

2018-03-23 09:41:42+0000 [-] While rendering resource:
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1442, in gotResult
        _inlineCallbacks(r, g, deferred)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1432, in _inlineCallbacks
        deferred.errback()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 500, in errback
        self._startRunCallbacks(fail)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 567, in _startRunCallbacks
        self._runCallbacks()
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 653, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/resource.py",
line 91, in failHttpError
        f.trap(Error)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 359, in trap
        self.raiseException()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 653, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/resource.py",
line 84, in failHttpRedirect
        f.trap(Redirect)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 359, in trap
        self.raiseException()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1384, in _inlineCallbacks
        result = result.throwExceptionIntoGenerator(g)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 408, in throwExceptionIntoGenerator
        return g.throw(self.type, self.value, self.tb)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 528, in asyncRender
        res = yield self.renderRest(request)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1386, in _inlineCallbacks
        result = g.send(result)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__
        self.gen.throw(type, value, traceback)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 172, in handleErrors
        jsonrpccode=JSONRPC_CODES["internal_error"])
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 375, in writeError
        request.write(data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/server.py",
line 234, in write
        http.Request.write(self, data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/http.py",
line 1086, in write
        self.channel.write(data)
    exceptions.AttributeError: 'NoneType' object has no attribute 'write'

2018-03-23 09:41:49+0000 [-] while handling API request
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1442, in gotResult
        _inlineCallbacks(r, g, deferred)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1386, in _inlineCallbacks
        result = g.send(result)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__
        self.gen.throw(type, value, traceback)
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 141, in handleErrors
        yield
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/server.py",
line 234, in write
        http.Request.write(self, data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/http.py",
line 1069, in write
        self.channel.writeHeaders(version, code, reason, headers)
    exceptions.AttributeError: 'NoneType' object has no attribute 'writeHeaders'

2018-03-23 09:41:49+0000 [-] While rendering resource:
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1442, in gotResult
        _inlineCallbacks(r, g, deferred)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1432, in _inlineCallbacks
        deferred.errback()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 500, in errback
        self._startRunCallbacks(fail)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 567, in _startRunCallbacks
        self._runCallbacks()
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 653, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/resource.py",
line 91, in failHttpError
        f.trap(Error)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 359, in trap
        self.raiseException()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 653, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/resource.py",
line 84, in failHttpRedirect
        f.trap(Redirect)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 359, in trap
        self.raiseException()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1384, in _inlineCallbacks
        result = result.throwExceptionIntoGenerator(g)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 408, in throwExceptionIntoGenerator
        return g.throw(self.type, self.value, self.tb)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 528, in asyncRender
        res = yield self.renderRest(request)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1386, in _inlineCallbacks
        result = g.send(result)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__
        self.gen.throw(type, value, traceback)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 172, in handleErrors
        jsonrpccode=JSONRPC_CODES["internal_error"])
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 375, in writeError
        request.write(data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/server.py",
line 234, in write
        http.Request.write(self, data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/http.py",
line 1086, in write
        self.channel.write(data)
    exceptions.AttributeError: 'NoneType' object has no attribute 'write'

2018-03-23 09:41:51+0000 [-] while handling API request
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1442, in gotResult
        _inlineCallbacks(r, g, deferred)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1386, in _inlineCallbacks
        result = g.send(result)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__
        self.gen.throw(type, value, traceback)
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 141, in handleErrors
        yield
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/server.py",
line 234, in write
        http.Request.write(self, data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/http.py",
line 1069, in write
        self.channel.writeHeaders(version, code, reason, headers)
    exceptions.AttributeError: 'NoneType' object has no attribute 'writeHeaders'

2018-03-23 09:41:51+0000 [-] While rendering resource:
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1442, in gotResult
        _inlineCallbacks(r, g, deferred)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1432, in _inlineCallbacks
        deferred.errback()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 500, in errback
        self._startRunCallbacks(fail)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 567, in _startRunCallbacks
        self._runCallbacks()
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 653, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/resource.py",
line 91, in failHttpError
        f.trap(Error)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 359, in trap
        self.raiseException()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 653, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/resource.py",
line 84, in failHttpRedirect
        f.trap(Redirect)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 359, in trap
        self.raiseException()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1384, in _inlineCallbacks
        result = result.throwExceptionIntoGenerator(g)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 408, in throwExceptionIntoGenerator
        return g.throw(self.type, self.value, self.tb)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 528, in asyncRender
        res = yield self.renderRest(request)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1386, in _inlineCallbacks
        result = g.send(result)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__
        self.gen.throw(type, value, traceback)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 172, in handleErrors
        jsonrpccode=JSONRPC_CODES["internal_error"])
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 375, in writeError
        request.write(data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/server.py",
line 234, in write
        http.Request.write(self, data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/http.py",
line 1086, in write
        self.channel.write(data)
    exceptions.AttributeError: 'NoneType' object has no attribute 'write'

2018-03-23 09:41:54+0000 [-] while handling API request
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1442, in gotResult
        _inlineCallbacks(r, g, deferred)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1386, in _inlineCallbacks
        result = g.send(result)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__
        self.gen.throw(type, value, traceback)
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 141, in handleErrors
        yield
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/server.py",
line 234, in write
        http.Request.write(self, data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/http.py",
line 1069, in write
        self.channel.writeHeaders(version, code, reason, headers)
    exceptions.AttributeError: 'NoneType' object has no attribute 'writeHeaders'

2018-03-23 09:41:54+0000 [-] While rendering resource:
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1442, in gotResult
        _inlineCallbacks(r, g, deferred)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1432, in _inlineCallbacks
        deferred.errback()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 500, in errback
        self._startRunCallbacks(fail)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 567, in _startRunCallbacks
        self._runCallbacks()
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 653, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/resource.py",
line 91, in failHttpError
        f.trap(Error)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 359, in trap
        self.raiseException()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 653, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/resource.py",
line 84, in failHttpRedirect
        f.trap(Redirect)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 359, in trap
        self.raiseException()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1384, in _inlineCallbacks
        result = result.throwExceptionIntoGenerator(g)
      File "/usr/local/lib/python2.7/dist-packages/twisted/python/failure.py",
line 408, in throwExceptionIntoGenerator
        return g.throw(self.type, self.value, self.tb)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 528, in asyncRender
        res = yield self.renderRest(request)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py",
line 1386, in _inlineCallbacks
        result = g.send(result)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 453, in renderRest
        request.write(data)
      File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__
        self.gen.throw(type, value, traceback)
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 172, in handleErrors
        jsonrpccode=JSONRPC_CODES["internal_error"])
      File "/usr/local/lib/python2.7/dist-packages/buildbot-1.1.0-py2.7.egg/buildbot/www/rest.py",
line 375, in writeError
        request.write(data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/server.py",
line 234, in write
        http.Request.write(self, data)
      File "/usr/local/lib/python2.7/dist-packages/twisted/web/http.py",
line 1086, in write
        self.channel.write(data)
    exceptions.AttributeError: 'NoneType' object has no attribute 'write'

More information about the users mailing list