[Buildbot-commits] buildbot/buildbot locks.py, 1.4, 1.5 master.py, 1.94, 1.95
Brian Warner
warner at users.sourceforge.net
Thu Aug 24 10:05:18 UTC 2006
Update of /cvsroot/buildbot/buildbot/buildbot
In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv27952/buildbot
Modified Files:
locks.py master.py
Log Message:
[project @ locks: can now have multiple simultaneous owners. fixes SF#1434997]
Original author: warner at lothar.com
Date: 2006-08-24 10:03:52
Index: locks.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/locks.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- locks.py 26 Nov 2005 02:14:31 -0000 1.4
+++ locks.py 24 Aug 2006 10:05:16 -0000 1.5
@@ -4,69 +4,90 @@
from twisted.internet import reactor, defer
from buildbot import util
+if False: # for debugging
+ def debuglog(msg):
+ log.msg(msg)
+else:
+ def debuglog(msg):
+ pass
+
class BaseLock:
- owner = None
description = "<BaseLock>"
- def __init__(self, name):
+ def __init__(self, name, maxCount=1):
self.name = name
self.waiting = []
+ self.owners = []
+ self.maxCount=maxCount
def __repr__(self):
return self.description
def isAvailable(self):
- log.msg("%s isAvailable: self.owner=%s" % (self, self.owner))
- return not self.owner
+ debuglog("%s isAvailable: self.owners=%r" % (self, self.owners))
+ return len(self.owners) < self.maxCount
def claim(self, owner):
- log.msg("%s claim(%s)" % (self, owner))
+ debuglog("%s claim(%s)" % (self, owner))
assert owner is not None
- self.owner = owner
- log.msg(" %s is claimed" % (self,))
+ assert len(self.owners) < self.maxCount, "ask for isAvailable() first"
+ self.owners.append(owner)
+ debuglog(" %s is claimed" % (self,))
def release(self, owner):
- log.msg("%s release(%s)" % (self, owner))
- assert owner is self.owner
- self.owner = None
- reactor.callLater(0, self.nowAvailable)
+ debuglog("%s release(%s)" % (self, owner))
+ assert owner in self.owners
+ self.owners.remove(owner)
+ # who can we wake up?
+ if self.waiting:
+ d = self.waiting.pop(0)
+ reactor.callLater(0, d.callback, self)
- def waitUntilAvailable(self, owner):
- log.msg("%s waitUntilAvailable(%s)" % (self, owner))
- assert self.owner, "You aren't supposed to call this on a free Lock"
+ def waitUntilMaybeAvailable(self, owner):
+ """Fire when the lock *might* be available. The caller will need to
+ check with isAvailable() when the deferred fires. This loose form is
+ used to avoid deadlocks. If we were interested in a stronger form,
+ this would be named 'waitUntilAvailable', and the deferred would fire
+ after the lock had been claimed.
+ """
+ debuglog("%s waitUntilAvailable(%s)" % (self, owner))
+ if self.isAvailable():
+ return defer.succeed(self)
d = defer.Deferred()
- self.waiting.append((d, owner))
+ self.waiting.append(d)
return d
- def nowAvailable(self):
- log.msg("%s nowAvailable" % self)
- assert not self.owner
- if not self.waiting:
- return
- d,owner = self.waiting.pop(0)
- d.callback(self)
class RealMasterLock(BaseLock):
- def __init__(self, name):
- BaseLock.__init__(self, name)
- self.description = "<MasterLock(%s)>" % (name,)
+ def __init__(self, lockid):
+ BaseLock.__init__(self, lockid.name, lockid.maxCount)
+ self.description = "<MasterLock(%s, %s)>" % (self.name, self.maxCount)
def getLock(self, slave):
return self
-class RealSlaveLock(BaseLock):
- def __init__(self, name):
- BaseLock.__init__(self, name)
- self.description = "<SlaveLock(%s)>" % (name,)
+class RealSlaveLock:
+ def __init__(self, lockid):
+ self.name = lockid.name
+ self.maxCount = lockid.maxCount
+ self.maxCountForSlave = lockid.maxCountForSlave
+ self.description = "<SlaveLock(%s, %s, %s)>" % (self.name,
+ self.maxCount,
+ self.maxCountForSlave)
self.locks = {}
+ def __repr__(self):
+ return self.description
+
def getLock(self, slavebuilder):
slavename = slavebuilder.slave.slavename
if not self.locks.has_key(slavename):
- lock = self.locks[slavename] = BaseLock(self.name)
- lock.description = "<SlaveLock(%s)[%s] %d>" % (self.name,
- slavename,
- id(lock))
+ maxCount = self.maxCountForSlave.get(slavename,
+ self.maxCount)
+ lock = self.locks[slavename] = BaseLock(self.name, maxCount)
+ desc = "<SlaveLock(%s, %s)[%s] %d>" % (self.name, maxCount,
+ slavename, id(lock))
+ lock.description = desc
self.locks[slavename] = lock
return self.locks[slavename]
@@ -76,14 +97,52 @@
# via the BotMaster.getLockByID method.
class MasterLock(util.ComparableMixin):
- compare_attrs = ['name']
+ """I am a semaphore that limits the number of simultaneous actions.
+
+ Builds and BuildSteps can declare that they wish to claim me as they run.
+ Only a limited number of such builds or steps will be able to run
+ simultaneously. By default this number is one, but my maxCount parameter
+ can be raised to allow two or three or more operations to happen at the
+ same time.
+
+ Use this to protect a resource that is shared among all builders and all
+ slaves, for example to limit the load on a common SVN repository.
+ """
+
+ compare_attrs = ['name', 'maxCount']
lockClass = RealMasterLock
- def __init__(self, name):
+ def __init__(self, name, maxCount=1):
self.name = name
+ self.maxCount = maxCount
class SlaveLock(util.ComparableMixin):
- compare_attrs = ['name']
+ """I am a semaphore that limits simultaneous actions on each buildslave.
+
+ Builds and BuildSteps can declare that they wish to claim me as they run.
+ Only a limited number of such builds or steps will be able to run
+ simultaneously on any given buildslave. By default this number is one,
+ but my maxCount parameter can be raised to allow two or three or more
+ operations to happen on a single buildslave at the same time.
+
+ Use this to protect a resource that is shared among all the builds taking
+ place on each slave, for example to limit CPU or memory load on an
+ underpowered machine.
+
+ Each buildslave will get an independent copy of this semaphore. By
+ default each copy will use the same owner count (set with maxCount), but
+ you can provide maxCountForSlave with a dictionary that maps slavename to
+ owner count, to allow some slaves more parallelism than others.
+
+ """
+
+ compare_attrs = ['name', 'maxCount', '_maxCountForSlaveList']
lockClass = RealSlaveLock
- def __init__(self, name):
+ def __init__(self, name, maxCount=1, maxCountForSlave={}):
self.name = name
-
+ self.maxCount = maxCount
+ self.maxCountForSlave = maxCountForSlave
+ # for comparison purposes, turn this dictionary into a stably-sorted
+ # list of tuples
+ self._maxCountForSlaveList = self.maxCountForSlave.items()
+ self._maxCountForSlaveList.sort()
+ self._maxCountForSlaveList = tuple(self._maxCountForSlaveList)
Index: master.py
===================================================================
RCS file: /cvsroot/buildbot/buildbot/buildbot/master.py,v
retrieving revision 1.94
retrieving revision 1.95
diff -u -d -r1.94 -r1.95
--- master.py 24 Jul 2006 22:09:20 -0000 1.94
+++ master.py 24 Aug 2006 10:05:16 -0000 1.95
@@ -416,10 +416,13 @@
@param lockid: a locks.MasterLock or locks.SlaveLock instance
@return: a locks.RealMasterLock or locks.RealSlaveLock instance
"""
- k = (lockid.__class__, lockid.name)
- if not k in self.locks:
- self.locks[k] = lockid.lockClass(lockid.name)
- return self.locks[k]
+ if not lockid in self.locks:
+ self.locks[lockid] = lockid.lockClass(lockid)
+ # if the master.cfg file has changed maxCount= on the lock, the next
+ # time a build is started, they'll get a new RealLock instance. Note
+ # that this requires that MasterLock and SlaveLock (marker) instances
+ # be hashable and that they should compare properly.
+ return self.locks[lockid]
########################################
More information about the Commits
mailing list