[Buildbot-commits] [Buildbot] #2427: Master requires restart sometimes if slave connection lost during graceful shutdown

[Buildbot]

#2427: Master requires restart sometimes if slave connection lost during graceful
shutdown
-------------------+--------------------
Reporter:  dank    |       Owner:
    Type:  defect  |      Status:  new
Priority:  minor   |   Milestone:  0.8.8
 Version:  0.8.7   |  Resolution:
Keywords:          |
-------------------+--------------------

Comment (by dustin):

 {{{
 !patch
 diff --git a/master/buildbot/process/builder.py
 b/master/buildbot/process/builder.py
 index bf94377..ef3b9fe 100644
 --- a/master/buildbot/process/builder.py
 +++ b/master/buildbot/process/builder.py
 @@ -282,27 +282,34 @@ class Builder(config.ReconfigurableServiceMixin,
          # the last cleanup we want to perform is to update the big
          # status based on any other cleanup
          cleanups.append(lambda : self.updateBigStatus())
 +        print "_sBF 1"

          build = self.config.factory.newBuild(buildrequests)
          build.setBuilder(self)
          log.msg("starting build %s using slave %s" % (build,
 slavebuilder))
 +        print "_sBF 2"

          # set up locks
          build.setLocks(self.config.locks)
          cleanups.append(lambda : slavebuilder.slave.releaseLocks())
 +        print "_sBF 3"

          if len(self.config.env) > 0:
              build.setSlaveEnvironment(self.config.env)
 +        print "_sBF 4"

          # append the build to self.building
          self.building.append(build)
          cleanups.append(lambda : self.building.remove(build))
 +        print "_sBF 5"

          # update the big status accordingly
          self.updateBigStatus()
 +        print "_sBF 6"

          try:
              ready = yield slavebuilder.prepare(self.builder_status,
 build)
 +            print "_sBF 7"
          except:
              log.err(failure.Failure(), 'while preparing slavebuilder:')
              ready = False
 @@ -310,9 +317,11 @@ class Builder(config.ReconfigurableServiceMixin,
          # If prepare returns True then it is ready and we start a build
          # If it returns false then we don't start a new build.
          if not ready:
 +            print "_sBF 8"
              log.msg("slave %s can't build %s after all; re-queueing the "
                      "request" % (build, slavebuilder))
              run_cleanups()
 +            print "_sBF 9"
              defer.returnValue(False)
              return

 @@ -328,33 +337,45 @@ class Builder(config.ReconfigurableServiceMixin,
                  % (build, slavebuilder))
          try:
              ping_success = yield slavebuilder.ping()
 +            print "_sBF 10"
          except:
              log.err(failure.Failure(), 'while pinging slave before
 build:')
 +            print "_sBF 11"
              ping_success = False

          if not ping_success:
              log.msg("slave ping failed; re-queueing the request")
 +            print "_sBF 12"
              run_cleanups()
 +            print "_sBF 13"
              defer.returnValue(False)
              return

          # The buildslave is ready to go. slavebuilder.buildStarted() sets
 its
          # state to BUILDING (so we won't try to use it for any other
 builds).
          # This gets set back to IDLE by the Build itself when it
 finishes.
 +        print "_sBF 14"
          slavebuilder.buildStarted()
 +        print "_sBF 15"
          cleanups.append(lambda : slavebuilder.buildFinished())

          # tell the remote that it's starting a build, too
          try:
 +            print "_sBF 16"
              yield slavebuilder.remote.callRemote("startBuild")
 +            print "_sBF 17"
          except:
              log.err(failure.Failure(), 'while calling remote
 startBuild:')
 +            print "_sBF 18"
              run_cleanups()
 +            print "_sBF 19"
              defer.returnValue(False)
              return

          # create the BuildStatus object that goes with the Build
 +        print "_sBF 20"
          bs = self.builder_status.newBuild()
 +        print "_sBF 21"

          # record the build in the db - one row per buildrequest
          try:
 @@ -369,7 +390,9 @@ class Builder(config.ReconfigurableServiceMixin,
              return

          # let status know
 +        print "_sBF 22"
          self.master.status.build_started(req.id, self.name, bs)
 +        print "_sBF 23"

          # start the build. This will first set up the steps, then tell
 the
          # BuildStatus that it has started, which will announce it to the
 world
 @@ -377,13 +400,17 @@ class Builder(config.ReconfigurableServiceMixin,
          # will start the actual build process.  This is done with a fresh
          # Deferred since _startBuildFor should not wait until the build
 is
          # finished.
 +        print "_sBF 24"
          d = build.startBuild(bs, self.expectations, slavebuilder)
 +        print "_sBF 25"
          d.addCallback(self.buildFinished, slavebuilder, bids)
 +        print "_sBF 26"
          # this shouldn't happen. if it does, the slave will be wedged
          d.addErrback(log.err)

          # make sure the builder's status is represented correctly
          self.updateBigStatus()
 +        print "_sBF 27"

          defer.returnValue(True)

 }}}
 might be a good start

-- 
Ticket URL: <http://trac.buildbot.net/ticket/2427#comment:4>
Buildbot <http://buildbot.net/>
Buildbot: build/test automation