[users at bb.net] Buildbot Hanging
Chris Spencer
chrisspen at gmail.com
Thu Apr 6 13:54:41 UTC 2017
I'm continuing to see some strange issues with Buildbot hanging. I tried
using a crontab to see if the worker or master had crashed and restart
them, but it looks like they're still running just fine. Buildbot's even
showing pending builds, but it's not ignoring them.
If I manually restart them, it immediately begins working again, but I have
no way to detect it being hung except to login and check if there are
pending builds with nothing being processed.
How do I diagnose and fix this?
My master's log is just a bunch of:
2017-04-04 18:15:41-0400 [-] gitpoller: processing changes from
"git at bitbucket.org:myorg/myproject.git"
2017-04-04 18:16:41-0400 [-] gitpoller: processing changes from
"git at bitbucket.org:myorg/myproject.git"
2017-04-04 18:17:41-0400 [-] gitpoller: processing changes from
"git at bitbucket.org:myorg/myproject.git"
2017-04-04 18:18:41-0400 [-] gitpoller: processing changes from
"git at bitbucket.org:myorg/myproject.git"
and my worker's log is just a bunch of:
2017-04-04 16:13:02-0400 [-] sending app-level keepalive
2017-04-04 16:23:02-0400 [-] sending app-level keepalive
2017-04-04 16:33:02-0400 [-] sending app-level keepalive
2017-04-04 16:43:02-0400 [-] sending app-level keepalive
2017-04-04 16:53:02-0400 [-] sending app-level keepalive
2017-04-04 17:03:03-0400 [-] sending app-level keepalive
2017-04-04 17:13:03-0400 [-] sending app-level keepalive
This is my master.cfg:
# -*- python -*-
# ex: set syntax=python:
import os
from buildbot.plugins import util, reporters, worker, changes,
schedulers, steps
from buildbot.reporters.bitbucket import BitbucketStatusPush
from bitbucket_sensitive import BITBUCKET_OATH_KEY,
BITBUCKET_OATH_SECRET # pylint: disable=import-error,no-name-in-module
import master_sensitive # pylint: disable=import-error
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
HOME_DIR = '/var/lib/buildbot'
c = BuildmasterConfig = {}
c['workers'] = [worker.Worker("example-worker", "pass")]
c['protocols'] = {'pb': {'port': 9989}}
c['change_source'] = []
c['change_source'].append(changes.GitPoller(
'git at bitbucket.org:myorg/myproject.git',
workdir='gitpoller-workdir',
branches=True,
pollinterval=60, # interval in seconds between polls
))
c['schedulers'] = []
c['schedulers'].append(schedulers.AnyBranchScheduler(
name="all",
treeStableTimer=60,
builderNames=["myproject_runtests"],
))
c['schedulers'].append(schedulers.ForceScheduler(
name="force",
builderNames=["myproject_runtests"]))
factory = util.BuildFactory()
factory.addStep(steps.Git(
repourl='git at bitbucket.org:myorg/myproject.git',
mode='incremental',
))
factory.addStep(steps.ShellCommand(
command="rm -rf $(find . -name '*.pyc' 2>/dev/null)",
workdir="/usr/local/myproject",
haltOnFailure=True,
env={'HOME': HOME_DIR},
))
factory.addStep(steps.ShellCommand(
command=['git', 'fetch', 'origin', 'master'],
workdir="build",
haltOnFailure=True,
env={'HOME': HOME_DIR},
))
factory.addStep(steps.ShellCommand(
command=['git', 'merge', 'origin/master', '--no-edit'],
workdir="build",
haltOnFailure=True,
env={'HOME': HOME_DIR},
))
factory.addStep(steps.ShellCommand(
command=['git', 'merge', 'origin/staging', '--no-edit'],
workdir="build",
haltOnFailure=True,
env={'HOME': HOME_DIR},
))
factory.addStep(steps.ShellCommand(
command=['./init_virtualenv.sh'],
workdir="build/src/scripts",
haltOnFailure=True,
env={'HOME': HOME_DIR},
))
factory.addStep(steps.ShellCommand(
command=['./checkjs.sh'],
workdir="build",
haltOnFailure=True,
env={'HOME': HOME_DIR},
))
factory.addStep(steps.ShellCommand(
command=['./pep8.sh'],
workdir="build",
haltOnFailure=True,
env={'HOME': HOME_DIR},
))
factory.addStep(steps.ShellCommand(
command=["./run_buildbot_tests"],
workdir="build/src/myproject",
env={'HOME': HOME_DIR},
))
c['builders'] = []
c['builders'].append(
util.BuilderConfig(
name="myproject_runtests",
workernames=["example-worker"],
collapseRequests=True,
factory=factory))
c['status'] = []
authz = util.Authz(
allowRules=[
util.StopBuildEndpointMatcher(role="admins"),
util.ForceBuildEndpointMatcher(role="admins"),
util.RebuildBuildEndpointMatcher(role="admins")
],
roleMatchers=[
util.RolesFromEmails(admins=["myuser at myorg.com"]),
util.RolesFromUsername(roles=["admins"], usernames=["myuser"]),
]
)
c['www'] = dict(
port=8010,
plugins=dict(waterfall_view={}, console_view={}),
#auth=util.HTPasswdAuth(os.path.join(BASE_DIR, '../htpasswd')),
auth=util.RemoteUserAuth(),
authz=authz,
)
mn = reporters.MailNotifier(
fromaddr="buildbot at myproject.com",
sendToInterestedUsers=True,
useTls=True,
relayhost="smtp.myproject.com",
smtpPort=587,
smtpUser="buildbot at myproject.com",
smtpPassword=master_sensitive.EMAIL_HOST_PASSWORD,
extraRecipients=[
'developers at myproject.com',
],
)
c['services'] = []
c['services'].append(mn)
bs = BitbucketStatusPush(BITBUCKET_OATH_KEY, BITBUCKET_OATH_SECRET)
c['services'].append(bs)
c['title'] = "MyProject"
c['buildbotURL'] = "http://buildbot.myproject.com/"
c['db'] = {
'db_url' : "sqlite:///state.sqlite",
}
c['changeHorizon'] = 200
c['buildHorizon'] = 100
c['logHorizon'] = 40
c['buildCacheSize'] = 15
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.buildbot.net/pipermail/users/attachments/20170406/758e22f6/attachment.html>
More information about the users
mailing list