[Buildbot] #3602: hyperworker should react appropriately in case of no more quota

[Buildbot trac]

#3602: hyperworker should react appropriately in case of no more quota
----------------------+------------------------
Reporter:  tardyp     |       Owner:
    Type:  undecided  |      Status:  new
Priority:  major      |   Milestone:  undecided
 Version:  0.8.12     |  Resolution:
Keywords:             |
----------------------+------------------------
Description changed by tardyp:

Old description:

> It is possible that buildbot also does not rm the container and only
> stops it.
>
> Maybe we can automatically rm an old container in case we get this error.
>

> 0Traceback (most recent call last):
> 1  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/internet/defer.py", line 588, in _runCallbacks
> 2    current.result = callback(current.result, *args, **kw)
> 3  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/internet/defer.py", line 1184, in gotResult
> 4    _inlineCallbacks(r, g, deferred)
> 5  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
> 6    result = result.throwExceptionIntoGenerator(g)
> 7  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/python/failure.py", line 389, in
> throwExceptionIntoGenerator
> 8    return g.throw(self.type, self.value, self.tb)
> 9--- <exception caught here> ---
> 10  File "/data/buildbot/buildbot/master/buildbot/process/build.py", line
> 288, in startBuild
> 11    ready_or_failure = yield workerforbuilder.prepare(self)
> 12  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
> 13    result = result.throwExceptionIntoGenerator(g)
> 14  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/python/failure.py", line 389, in
> throwExceptionIntoGenerator
> 15    return g.throw(self.type, self.value, self.tb)
> 16  File "/data/buildbot/buildbot/master/buildbot/worker/hyper.py", line
> 121, in start_instance
> 17    res = yield threads.deferToThreadPool(self.reactor,
> self.threadPool, self._thd_start_instance, image)
> 18  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/python/threadpool.py", line 246, in inContext
> 19    result = inContext.theWork()
> 20  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/python/threadpool.py", line 262, in <lambda>
> 21    inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
> 22  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/python/context.py", line 118, in callWithContext
> 23    return self.currentContext().callWithContext(ctx, func, *args,
> **kw)
> 24  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/twisted/python/context.py", line 81, in callWithContext
> 25    return func(*args,**kw)
> 26  File "/data/buildbot/buildbot/master/buildbot/worker/hyper.py", line
> 131, in _thd_start_instance
> 27    name=('%s%s' % (self.workername, id(self))).replace("_", "-")
> 28  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/docker/api/container.py", line 120, in create_container
> 29    return self.create_container_from_config(config, name)
> 30  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/docker/api/container.py", line 131, in
> create_container_from_config
> 31    return self._result(res, True)
> 32  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/docker/client.py", line 176, in _result
> 33    self._raise_for_status(response)
> 34  File "/data/buildbot/sandbox/local/lib/python2.7/site-
> packages/docker/client.py", line 172, in _raise_for_status
> 35    raise errors.APIError(e, response, explanation=explanation)
> 36docker.errors.APIError: 500 Server Error: Internal Server Error ("No
> more quota for use, please delete some, or apply more")

New description:

 It is possible that buildbot also does not rm the container and only stops
 it.

 Maybe we can automatically rm an old container in case we get this error.

 {{{
 0Traceback (most recent call last):
 1  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/internet/defer.py", line 588, in _runCallbacks
 2    current.result = callback(current.result, *args, **kw)
 3  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/internet/defer.py", line 1184, in gotResult
 4    _inlineCallbacks(r, g, deferred)
 5  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
 6    result = result.throwExceptionIntoGenerator(g)
 7  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/python/failure.py", line 389, in
 throwExceptionIntoGenerator
 8    return g.throw(self.type, self.value, self.tb)
 9--- <exception caught here> ---
 10  File "/data/buildbot/buildbot/master/buildbot/process/build.py", line
 288, in startBuild
 11    ready_or_failure = yield workerforbuilder.prepare(self)
 12  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
 13    result = result.throwExceptionIntoGenerator(g)
 14  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/python/failure.py", line 389, in
 throwExceptionIntoGenerator
 15    return g.throw(self.type, self.value, self.tb)
 16  File "/data/buildbot/buildbot/master/buildbot/worker/hyper.py", line
 121, in start_instance
 17    res = yield threads.deferToThreadPool(self.reactor, self.threadPool,
 self._thd_start_instance, image)
 18  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/python/threadpool.py", line 246, in inContext
 19    result = inContext.theWork()
 20  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/python/threadpool.py", line 262, in <lambda>
 21    inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
 22  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/python/context.py", line 118, in callWithContext
 23    return self.currentContext().callWithContext(ctx, func, *args, **kw)
 24  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/twisted/python/context.py", line 81, in callWithContext
 25    return func(*args,**kw)
 26  File "/data/buildbot/buildbot/master/buildbot/worker/hyper.py", line
 131, in _thd_start_instance
 27    name=('%s%s' % (self.workername, id(self))).replace("_", "-")
 28  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/docker/api/container.py", line 120, in create_container
 29    return self.create_container_from_config(config, name)
 30  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/docker/api/container.py", line 131, in
 create_container_from_config
 31    return self._result(res, True)
 32  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/docker/client.py", line 176, in _result
 33    self._raise_for_status(response)
 34  File "/data/buildbot/sandbox/local/lib/python2.7/site-
 packages/docker/client.py", line 172, in _raise_for_status
 35    raise errors.APIError(e, response, explanation=explanation)
 36docker.errors.APIError: 500 Server Error: Internal Server Error ("No
 more quota for use, please delete some, or apply more")
 }}}

--

--
Ticket URL: <http://trac.buildbot.net/ticket/3602#comment:1>
Buildbot <http://buildbot.net/>
Buildbot: build/test automation