[Buildbot-commits] buildbot/contrib/windows buildbot_service.py, NONE, 1.1 setup.py, NONE, 1.1
Brian Warner
warner at users.sourceforge.net
Sat Jun 3 20:21:06 UTC 2006
Update of /cvsroot/buildbot/buildbot/contrib/windows
In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv15165/contrib/windows
Added Files:
buildbot_service.py setup.py
Log Message:
[project @ add py2exe support for windows, SF#1401121]
Original author: warner at lothar.com
Date: 2006-06-03 20:18:42
--- NEW FILE: buildbot_service.py ---
# Runs the build-bot as a Windows service.
# To use:
# * Install and configure buildbot as per normal (ie, running
# 'setup.py install' from the source directory).
#
# * Configure any number of build-bot directories (slaves or masters), as
# per the buildbot instructions. Test these directories normally by
# using the (possibly modified) "buildbot.bat" file and ensure everything
# is working as expected.
#
# * Install the buildbot service. Execute the command:
# % python buildbot_service.py
# To see installation options. You probably want to specify:
# + --username and --password options to specify the user to run the
# + --startup auto to have the service start at boot time.
#
# For example:
# % python buildbot_service.py --user mark --password secret \
# --startup auto install
# Alternatively, you could execute:
# % python buildbot_service.py install
# to install the service with default options, then use Control Panel
# to configure it.
#
# * Start the service specifying the name of all buildbot directories as
# service args. This can be done one of 2 ways:
# - Execute the command:
# % python buildbot_service.py start "dir_name1" "dir_name2"
# or:
# - Start Control Panel->Administrative Tools->Services
# - Locate the previously installed buildbot service.
# - Open the "properties" for the service.
# - Enter the directory names into the "Start Parameters" textbox. The
# directory names must be fully qualified, and surrounded in quotes if
# they include spaces.
# - Press the "Start"button.
# Note that the service will automatically use the previously specified
# directories if no arguments are specified. This means the directories
# need only be specified when the directories to use have changed (and
# therefore also the first time buildbot is configured)
#
# * The service should now be running. You should check the Windows
# event log. If all goes well, you should see some information messages
# telling you the buildbot has successfully started.
#
# * If you change the buildbot configuration, you must restart the service.
# There is currently no way to ask a running buildbot to reload the
# config. You can restart by executing:
# % python buildbot_service.py restart
#
# Troubleshooting:
# * Check the Windows event log for any errors.
# * Check the "twistd.log" file in your buildbot directories - once each
# bot has been started it just writes to this log as normal.
# * Try executing:
# % python buildbot_service.py debug
# This will execute the buildbot service in "debug" mode, and allow you to
# see all messages etc generated. If the service works in debug mode but
# not as a real service, the error probably relates to the environment or
# permissions of the user configured to run the service (debug mode runs as
# the currently logged in user, not the service user)
# * Ensure you have the latest pywin32 build available, at least version 206.
# Written by Mark Hammond, 2006.
import sys, os, threading
import pywintypes
import winerror, win32con
import win32api, win32event, win32file, win32pipe, win32process, win32security
import win32service, win32serviceutil, servicemanager
# Are we running in a py2exe environment?
is_frozen = hasattr(sys, "frozen")
# Taken from the Zope service support - each "child" is run as a sub-process
# (trying to run multiple twisted apps in the same process is likely to screw
# stdout redirection etc).
# Note that unlike the Zope service, we do *not* attempt to detect a failed
# client and perform restarts - buildbot itself does a good job
# at reconnecting, and Windows itself provides restart semantics should
# everything go pear-shaped.
# We execute a new thread that captures the tail of the output from our child
# process. If the child fails, it is written to the event log.
# This process is unconditional, and the output is never written to disk
# (except obviously via the event log entry)
# Size of the blocks we read from the child process's output.
CHILDCAPTURE_BLOCK_SIZE = 80
# The number of BLOCKSIZE blocks we keep as process output.
CHILDCAPTURE_MAX_BLOCKS = 200
class BBService(win32serviceutil.ServiceFramework):
_svc_name_ = 'BuildBot'
_svc_display_name_ = _svc_name_
_svc_description_ = 'Manages local buildbot slaves and masters - ' \
'see http://buildbot.sourceforge.net'
def __init__(self, args):
win32serviceutil.ServiceFramework.__init__(self, args)
# Create an event which we will use to wait on. The "service stop"
# request will set this event.
# * We must make it inheritable so we can pass it to the child
# process via the cmd-line
# * Must be manual reset so each child process and our service
# all get woken from a single set of the event.
sa = win32security.SECURITY_ATTRIBUTES()
sa.bInheritHandle = True
self.hWaitStop = win32event.CreateEvent(sa, True, False, None)
self.args = args
self.dirs = None
self.runner_prefix = None
# Patch up the service messages file in a frozen exe.
# (We use the py2exe option that magically bundles the .pyd files
# into the .zip file - so servicemanager.pyd doesn't exist.)
if is_frozen and servicemanager.RunningAsService():
msg_file = os.path.join(os.path.dirname(sys.executable),
"buildbot.msg")
if os.path.isfile(msg_file):
servicemanager.Initialize("BuildBot", msg_file)
else:
self.warning("Strange - '%s' does not exist" % (msg_file,))
def _checkConfig(self):
# Locate our child process runner (but only when run from source)
if not is_frozen:
# Running from source
python_exe = os.path.join(sys.prefix, "python.exe")
if not os.path.isfile(python_exe):
# for ppl who build Python itself from source.
python_exe = os.path.join(sys.prefix, "PCBuild", "python.exe")
if not os.path.isfile(python_exe):
self.error("Can not find python.exe to spawn subprocess")
return False
me = __file__
if me.endswith(".pyc") or me.endswith(".pyo"):
me = me[:-1]
self.runner_prefix = '"%s" "%s"' % (python_exe, me)
else:
# Running from a py2exe built executable - our child process is
# us (but with the funky cmdline args!)
self.runner_prefix = '"' + sys.executable + '"'
# Now our arg processing - this may be better handled by a
# twisted/buildbot style config file - but as of time of writing,
# MarkH is clueless about such things!
# Note that the "arguments" you type into Control Panel for the
# service do *not* persist - they apply only when you click "start"
# on the service. When started by Windows, args are never presented.
# Thus, it is the responsibility of the service to persist any args.
# so, when args are presented, we save them as a "custom option". If
# they are not presented, we load them from the option.
self.dirs = []
if len(self.args) > 1:
dir_string = os.pathsep.join(self.args[1:])
save_dirs = True
else:
dir_string = win32serviceutil.GetServiceCustomOption(self,
"directories")
save_dirs = False
if not dir_string:
self.error("You must specify the buildbot directories as "
"parameters to the service.\nStopping the service.")
return False
dirs = dir_string.split(os.pathsep)
for d in dirs:
d = os.path.abspath(d)
sentinal = os.path.join(d, "buildbot.tac")
if os.path.isfile(sentinal):
self.dirs.append(d)
else:
msg = "Directory '%s' is not a buildbot dir - ignoring" \
% (d,)
self.warning(msg)
if not self.dirs:
self.error("No valid buildbot directories were specified.\n"
"Stopping the service.")
return False
if save_dirs:
dir_string = os.pathsep.join(self.dirs).encode("mbcs")
win32serviceutil.SetServiceCustomOption(self, "directories",
dir_string)
return True
def SvcStop(self):
# Tell the SCM we are starting the stop process.
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
# Set the stop event - the main loop takes care of termination.
win32event.SetEvent(self.hWaitStop)
# SvcStop only gets triggered when the user explictly stops (or restarts)
# the service. To shut the service down cleanly when Windows is shutting
# down, we also need to hook SvcShutdown.
SvcShutdown = SvcStop
def SvcDoRun(self):
if not self._checkConfig():
# stopped status set by caller.
return
self.logmsg(servicemanager.PYS_SERVICE_STARTED)
child_infos = []
for bbdir in self.dirs:
self.info("Starting BuildBot in directory '%s'" % (bbdir,))
hstop = self.hWaitStop
cmd = '%s --spawn %d start %s' % (self.runner_prefix, hstop, bbdir)
#print "cmd is", cmd
h, t, output = self.createProcess(cmd)
child_infos.append((bbdir, h, t, output))
while child_infos:
handles = [self.hWaitStop] + [i[1] for i in child_infos]
rc = win32event.WaitForMultipleObjects(handles,
0, # bWaitAll
win32event.INFINITE)
if rc == win32event.WAIT_OBJECT_0:
# user sent a stop service request
break
else:
# A child process died. For now, just log the output
# and forget the process.
index = rc - win32event.WAIT_OBJECT_0 - 1
bbdir, dead_handle, dead_thread, output_blocks = \
child_infos[index]
status = win32process.GetExitCodeProcess(dead_handle)
output = "".join(output_blocks)
if not output:
output = "The child process generated no output. " \
"Please check the twistd.log file in the " \
"indicated directory."
self.warning("BuildBot for directory %r terminated with "
"exit code %d.\n%s" % (bbdir, status, output))
del child_infos[index]
if not child_infos:
self.warning("All BuildBot child processes have "
"terminated. Service stopping.")
# Either no child processes left, or stop event set.
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
# The child processes should have also seen our stop signal
# so wait for them to terminate.
for bbdir, h, t, output in child_infos:
for i in range(10): # 30 seconds to shutdown...
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
rc = win32event.WaitForSingleObject(h, 3000)
if rc == win32event.WAIT_OBJECT_0:
break
# Process terminated - no need to try harder.
if rc == win32event.WAIT_OBJECT_0:
break
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
# If necessary, kill it
if win32process.GetExitCodeProcess(h)==win32con.STILL_ACTIVE:
self.warning("BuildBot process at %r failed to terminate - "
"killing it" % (bbdir,))
win32api.TerminateProcess(h, 3)
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
# Wait for the redirect thread - it should have died as the remote
# process terminated.
# As we are shutting down, we do the join with a little more care,
# reporting progress as we wait (even though we never will <wink>)
for i in range(5):
t.join(1)
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
if not t.isAlive():
break
else:
self.warning("Redirect thread did not stop!")
# All done.
self.logmsg(servicemanager.PYS_SERVICE_STOPPED)
#
# Error reporting/logging functions.
#
def logmsg(self, event):
# log a service event using servicemanager.LogMsg
try:
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
event,
(self._svc_name_,
" (%s)" % self._svc_display_name_))
except win32api.error, details:
# Failed to write a log entry - most likely problem is
# that the event log is full. We don't want this to kill us
try:
print "FAILED to write INFO event", event, ":", details
except IOError:
# No valid stdout! Ignore it.
pass
def _dolog(self, func, msg):
try:
func(msg)
except win32api.error, details:
# Failed to write a log entry - most likely problem is
# that the event log is full. We don't want this to kill us
try:
print "FAILED to write event log entry:", details
print msg
except IOError:
pass
def info(self, s):
self._dolog(servicemanager.LogInfoMsg, s)
def warning(self, s):
self._dolog(servicemanager.LogWarningMsg, s)
def error(self, s):
self._dolog(servicemanager.LogErrorMsg, s)
# Functions that spawn a child process, redirecting any output.
# Although builtbot itself does this, it is very handy to debug issues
# such as ImportErrors that happen before buildbot has redirected.
def createProcess(self, cmd):
hInputRead, hInputWriteTemp = self.newPipe()
hOutReadTemp, hOutWrite = self.newPipe()
pid = win32api.GetCurrentProcess()
# This one is duplicated as inheritable.
hErrWrite = win32api.DuplicateHandle(pid, hOutWrite, pid, 0, 1,
win32con.DUPLICATE_SAME_ACCESS)
# These are non-inheritable duplicates.
hOutRead = self.dup(hOutReadTemp)
hInputWrite = self.dup(hInputWriteTemp)
# dup() closed hOutReadTemp, hInputWriteTemp
si = win32process.STARTUPINFO()
si.hStdInput = hInputRead
si.hStdOutput = hOutWrite
si.hStdError = hErrWrite
si.dwFlags = win32process.STARTF_USESTDHANDLES | \
win32process.STARTF_USESHOWWINDOW
si.wShowWindow = win32con.SW_HIDE
# pass True to allow handles to be inherited. Inheritance is
# problematic in general, but should work in the controlled
# circumstances of a service process.
create_flags = win32process.CREATE_NEW_CONSOLE
# info is (hProcess, hThread, pid, tid)
info = win32process.CreateProcess(None, cmd, None, None, True,
create_flags, None, None, si)
# (NOTE: these really aren't necessary for Python - they are closed
# as soon as they are collected)
hOutWrite.Close()
hErrWrite.Close()
hInputRead.Close()
# We don't use stdin
hInputWrite.Close()
# start a thread collecting output
blocks = []
t = threading.Thread(target=self.redirectCaptureThread,
args = (hOutRead,blocks))
t.start()
return info[0], t, blocks
def redirectCaptureThread(self, handle, captured_blocks):
# One of these running per child process we are watching. It
# handles both stdout and stderr on a single handle. The read data is
# never referenced until the thread dies - so no need for locks
# around self.captured_blocks.
#self.info("Redirect thread starting")
while 1:
try:
ec, data = win32file.ReadFile(handle, CHILDCAPTURE_BLOCK_SIZE)
except pywintypes.error, err:
# ERROR_BROKEN_PIPE means the child process closed the
# handle - ie, it terminated.
if err[0] != winerror.ERROR_BROKEN_PIPE:
self.warning("Error reading output from process: %s" % err)
break
captured_blocks.append(data)
del captured_blocks[CHILDCAPTURE_MAX_BLOCKS:]
handle.Close()
#self.info("Redirect capture thread terminating")
def newPipe(self):
sa = win32security.SECURITY_ATTRIBUTES()
sa.bInheritHandle = True
return win32pipe.CreatePipe(sa, 0)
def dup(self, pipe):
# create a duplicate handle that is not inherited, so that
# it can be closed in the parent. close the original pipe in
# the process.
pid = win32api.GetCurrentProcess()
dup = win32api.DuplicateHandle(pid, pipe, pid, 0, 0,
win32con.DUPLICATE_SAME_ACCESS)
pipe.Close()
return dup
# Service registration and startup
def RegisterWithFirewall(exe_name, description):
# Register our executable as an exception with Windows Firewall.
# taken from http://msdn.microsoft.com/library/default.asp?url=/library/en-us/ics/ics/wf_adding_an_application.asp
from win32com.client import Dispatch
# Set constants
NET_FW_PROFILE_DOMAIN = 0
NET_FW_PROFILE_STANDARD = 1
# Scope
NET_FW_SCOPE_ALL = 0
# IP Version - ANY is the only allowable setting for now
NET_FW_IP_VERSION_ANY = 2
fwMgr = Dispatch("HNetCfg.FwMgr")
# Get the current profile for the local firewall policy.
profile = fwMgr.LocalPolicy.CurrentProfile
app = Dispatch("HNetCfg.FwAuthorizedApplication")
app.ProcessImageFileName = exe_name
app.Name = description
app.Scope = NET_FW_SCOPE_ALL
# Use either Scope or RemoteAddresses, but not both
#app.RemoteAddresses = "*"
app.IpVersion = NET_FW_IP_VERSION_ANY
app.Enabled = True
# Use this line if you want to add the app, but disabled.
#app.Enabled = False
profile.AuthorizedApplications.Add(app)
# A custom install function.
def CustomInstall(opts):
# Register this process with the Windows Firewaall
import pythoncom
try:
RegisterWithFirewall(sys.executable, "BuildBot")
except pythoncom.com_error, why:
print "FAILED to register with the Windows firewall"
print why
#
# Magic code to allow shutdown. Note that this code is executed in
# the *child* process, by way of the service process executing us with
# special cmdline args (which includes the service stop handle!)
def _RunChild():
del sys.argv[1] # The --spawn arg.
# Create a new thread that just waits for the event to be signalled.
t = threading.Thread(target=_WaitForShutdown,
args = (int(sys.argv[1]),)
)
del sys.argv[1] # The stop handle
# This child process will be sent a console handler notification as
# users log off, or as the system shuts down. We want to ignore these
# signals as the service parent is responsible for our shutdown.
def ConsoleHandler(what):
# We can ignore *everything* - ctrl+c will never be sent as this
# process is never attached to a console the user can press the
# key in!
return True
win32api.SetConsoleCtrlHandler(ConsoleHandler, True)
t.setDaemon(True) # we don't want to wait for this to stop!
t.start()
if hasattr(sys, "frozen"):
# py2exe sets this env vars that may screw our child process - reset
del os.environ["PYTHONPATH"]
# Start the buildbot app
from buildbot.scripts import runner
runner.run()
print "Service child process terminating normally."
def _WaitForShutdown(h):
win32event.WaitForSingleObject(h, win32event.INFINITE)
print "Shutdown requested"
from twisted.internet import reactor
reactor.callLater(0, reactor.stop)
# This function is also called by the py2exe startup code.
def HandleCommandLine():
if len(sys.argv)>1 and sys.argv[1] == "--spawn":
# Special command-line created by the service to execute the
# child-process.
# First arg is the handle to wait on
_RunChild()
else:
win32serviceutil.HandleCommandLine(BBService,
customOptionHandler=CustomInstall)
if __name__ == '__main__':
HandleCommandLine()
--- NEW FILE: setup.py ---
# setup.py
# A distutils setup script to create py2exe binaries for buildbot.
# Both a service and standard executable are created.
# Usage:
# % setup.py py2exe
import sys, os, tempfile, shutil
from os.path import dirname, join, abspath, exists, splitext
this_dir = abspath(dirname(__file__))
bb_root_dir = abspath(join(this_dir, "..", ".."))
from distutils.core import setup
import py2exe
includes = []
# We try and bundle *all* modules in the following packages:
for package in ["buildbot.changes", "buildbot.process", "buildbot.status"]:
__import__(package)
p = sys.modules[package]
for fname in os.listdir(p.__path__[0]):
base, ext = splitext(fname)
if not fname.startswith("_") and ext == ".py":
includes.append(p.__name__ + "." + base)
# Other misc modules dynamically imported, so missed by py2exe
includes.extend("""
buildbot.scheduler
buildbot.slave.bot
buildbot.master
twisted.internet.win32eventreactor
twisted.web.resource""".split())
# Turn into "," sep py2exe requires
includes = ",".join(includes)
py2exe_options = {"bundle_files": 1,
"includes": includes,
}
# Each "target" executable we create
buildbot_target = {
"script": join(bb_root_dir, "bin", "buildbot")
}
# Due to the way py2exe works, we need to rebuild the service code as a
# normal console process - this will be executed by the service itself.
service_target = {
"modules": ["buildbot_service"],
"cmdline_style": "custom",
}
# We use the py2exe "bundle" option, so servicemanager.pyd
# (which has the message resources) does not exist. Take a copy
# of it with a "friendlier" name. The service runtime arranges for this
# to be used.
import servicemanager
msg_file = join(tempfile.gettempdir(), "buildbot.msg")
shutil.copy(servicemanager.__file__, msg_file)
data_files = [
["", [msg_file]],
["", [join(bb_root_dir, "buildbot", "status", "classic.css")]],
["", [join(bb_root_dir, "buildbot", "buildbot.png")]],
]
try:
setup(name="buildbot",
# The buildbot script as a normal executable
console=[buildbot_target],
service=[service_target],
options={'py2exe': py2exe_options},
data_files = data_files,
zipfile = "buildbot.library", # 'library.zip' invites trouble :)
)
finally:
os.unlink(msg_file)
More information about the Commits
mailing list