[Buildbot-commits] buildbot/buildbot/test test_run.py,1.20,1.21

[Brian Warner]

Update of /cvsroot/buildbot/buildbot/buildbot/test
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv3363/buildbot/test

Modified Files:
	test_run.py 
Log Message:
* buildbot/slave/bot.py (BotFactory): revamp keepalive/lost-master detection
code. Require some sign of life from the buildmaster every
BotFactory.keepaliveInterval seconds. Provoke this indication at
BotFactory.keepaliveTimeout seconds before the deadline by sending a
keepalive request. We don't actually care if that request is answered in a
timely fashion, what we care about is that .activity() is called before the
deadline. .activity() is triggered by any PB message from the master
(including an ack to one of the slave's status-update messages). With this
new scheme, large status messages over slow pipes are OK, as long as any
given message can be sent (and thus acked) within .keepaliveTimeout seconds
(which defaults to 30).
(SlaveBuilder.remote_startCommand): record activity
(SlaveBuilder.ackUpdate): same
(SlaveBuilder.ackComplete): same
(BotFactory.gotPerspective): same
* buildbot/test/test_run.py (Disconnect.testSlaveTimeout): test it


Index: test_run.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/test/test_run.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -d -r1.20 -r1.21
--- test_run.py	8 Dec 2004 03:54:56 -0000	1.20
+++ test_run.py	11 Dec 2004 11:12:33 -0000	1.21
@@ -8,6 +8,7 @@
 #log.startLogging(sys.stderr)
 
 from buildbot import master, interfaces
+from buildbot.util import now
 from buildbot.slave import bot
 from buildbot.changes.changes import Change
 from buildbot.status import builder
@@ -157,12 +158,26 @@
         self.slave2 = slave
         slave.startService()
 
+    def connectSlave3(self):
+        # this slave has a very fast keepalive timeout
+        port = self.master.slavePort._port.getHost().port
+        os.mkdir("slavebase")
+        slave = MyBuildSlave("localhost", port, "bot1", "sekrit",
+                             "slavebase", keepalive=2, usePTY=1,
+                             keepaliveTimeout=1)
+        slave.info = {"admin": "one"}
+        self.slave = slave
+        slave.startService()
+        d = self.master.botmaster.waitUntilBuilderAttached("dummy")
+        dr(d)
+
     def tearDown(self):
         log.msg("doing tearDown")
         self.shutdownSlave()
         if self.master:
             dr(defer.maybeDeferred(self.master.stopService))
             self.master = None
+        log.msg("tearDown done")
 
     # various forms of slave death
 
@@ -342,7 +357,7 @@
 
 class Disconnect(RunMixin, unittest.TestCase):
 
-    def disconnectSetup(self):
+    def disconnectSetupMaster(self):
         # verify that disconnecting the slave during a build properly
         # terminates the build
         m = self.master
@@ -360,11 +375,21 @@
         self.failUnlessEqual(s1.getCurrentBuild(), None)
         self.failUnlessEqual(s1.getLastFinishedBuild(), None)
         self.failUnlessEqual(s1.getBuild(-1), None)
+        return m,s,c,s1
 
+    def disconnectSetup(self):
+        m,s,c,s1 = self.disconnectSetupMaster()
         self.connectSlave()
         self.failUnlessEqual(s1.getState(), ("idle", None, None))
         return m,s,c,s1
 
+    def disconnectSetup2(self):
+        m,s,c,s1 = self.disconnectSetupMaster()
+        self.connectSlave3()
+        self.failUnlessEqual(s1.getState(), ("idle", None, None))
+        return m,s,c,s1
+
+
     def verifyDisconnect(self, bs):
         self.failUnless(bs.isFinished())
 
@@ -413,6 +438,41 @@
         print bs.getText()
     testIdle2.skip = "short timeout not yet implemented"
 
+    def testSlaveTimeout(self):
+        m,s,c,s1 = self.disconnectSetup2() # fast timeout
+
+        # now suppose the slave goes missing. We want to find out when it
+        # creates a new Broker, so we reach inside and mark it with the
+        # well-known sigil of impending messy death.
+        bd = self.slave.getServiceNamed("bot").builders["dummy"]
+        broker = bd.remote.broker
+        broker.redshirt = 1
+
+        # make sure the keepalives will keep the connection up
+        later = now() + 5
+        while 1:
+            if now() > later:
+                break
+            bd = self.slave.getServiceNamed("bot").builders["dummy"]
+            if not bd.remote or not hasattr(bd.remote.broker, "redshirt"):
+                self.fail("slave disconnected when it shouldn't have")
+            reactor.iterate(0.01)
+
+        d = self.master.botmaster.waitUntilBuilderDetached("dummy")
+        # whoops! how careless of me.
+        self.disappearSlave()
+
+        # the slave will realize the connection is lost within 2 seconds, and
+        # reconnect.
+        dr(d, 5)
+        d = self.master.botmaster.waitUntilBuilderAttached("dummy")
+        dr(d, 5)
+        # make sure it is a new connection (i.e. a new Broker)
+        bd = self.slave.getServiceNamed("bot").builders["dummy"]
+        self.failUnless(bd.remote, "hey, slave isn't really connected")
+        self.failIf(hasattr(bd.remote.broker, "redshirt"),
+                    "hey, slave's Broker is still marked for death")
+
     def testBuild1(self):
         m,s,c,s1 = self.disconnectSetup()
         # this next sequence is timing-dependent. The dummy build takes at