[users at bb.net] Buildbot Hanging

[Chris Spencer]

I'm continuing to see some strange issues with Buildbot hanging. I tried
using a crontab to see if the worker or master had crashed and restart
them, but it looks like they're still running just fine. Buildbot's even
showing pending builds, but it's not ignoring them.

If I manually restart them, it immediately begins working again, but I have
no way to detect it being hung except to login and check if there are
pending builds with nothing being processed.

How do I diagnose and fix this?

My master's log is just a bunch of:

    2017-04-04 18:15:41-0400 [-] gitpoller: processing changes from
"git at bitbucket.org:myorg/myproject.git"
    2017-04-04 18:16:41-0400 [-] gitpoller: processing changes from
"git at bitbucket.org:myorg/myproject.git"
    2017-04-04 18:17:41-0400 [-] gitpoller: processing changes from
"git at bitbucket.org:myorg/myproject.git"
    2017-04-04 18:18:41-0400 [-] gitpoller: processing changes from
"git at bitbucket.org:myorg/myproject.git"

and my worker's log is just a bunch of:

    2017-04-04 16:13:02-0400 [-] sending app-level keepalive
    2017-04-04 16:23:02-0400 [-] sending app-level keepalive
    2017-04-04 16:33:02-0400 [-] sending app-level keepalive
    2017-04-04 16:43:02-0400 [-] sending app-level keepalive
    2017-04-04 16:53:02-0400 [-] sending app-level keepalive
    2017-04-04 17:03:03-0400 [-] sending app-level keepalive
    2017-04-04 17:13:03-0400 [-] sending app-level keepalive

This is my master.cfg:

    # -*- python -*-
    # ex: set syntax=python:
    import os

    from buildbot.plugins import util, reporters, worker, changes,
schedulers, steps
    from buildbot.reporters.bitbucket import BitbucketStatusPush

    from bitbucket_sensitive import BITBUCKET_OATH_KEY,
BITBUCKET_OATH_SECRET # pylint: disable=import-error,no-name-in-module
    import master_sensitive # pylint: disable=import-error

    BASE_DIR = os.path.abspath(os.path.dirname(__file__))

    HOME_DIR = '/var/lib/buildbot'

    c = BuildmasterConfig = {}

    c['workers'] = [worker.Worker("example-worker", "pass")]
    c['protocols'] = {'pb': {'port': 9989}}
    c['change_source'] = []
    c['change_source'].append(changes.GitPoller(
            'git at bitbucket.org:myorg/myproject.git',
            workdir='gitpoller-workdir',
            branches=True,
            pollinterval=60, # interval in seconds between polls
    ))

    c['schedulers'] = []
    c['schedulers'].append(schedulers.AnyBranchScheduler(
        name="all",
        treeStableTimer=60,
        builderNames=["myproject_runtests"],
    ))
    c['schedulers'].append(schedulers.ForceScheduler(
        name="force",
        builderNames=["myproject_runtests"]))

    factory = util.BuildFactory()
    factory.addStep(steps.Git(
        repourl='git at bitbucket.org:myorg/myproject.git',
        mode='incremental',
    ))
    factory.addStep(steps.ShellCommand(
        command="rm -rf $(find . -name '*.pyc' 2>/dev/null)",
        workdir="/usr/local/myproject",
        haltOnFailure=True,
        env={'HOME': HOME_DIR},
    ))
    factory.addStep(steps.ShellCommand(
        command=['git', 'fetch', 'origin', 'master'],
        workdir="build",
        haltOnFailure=True,
        env={'HOME': HOME_DIR},
    ))
    factory.addStep(steps.ShellCommand(
        command=['git', 'merge', 'origin/master', '--no-edit'],
        workdir="build",
        haltOnFailure=True,
        env={'HOME': HOME_DIR},
    ))
    factory.addStep(steps.ShellCommand(
        command=['git', 'merge', 'origin/staging', '--no-edit'],
        workdir="build",
        haltOnFailure=True,
        env={'HOME': HOME_DIR},
    ))
    factory.addStep(steps.ShellCommand(
        command=['./init_virtualenv.sh'],
        workdir="build/src/scripts",
        haltOnFailure=True,
        env={'HOME': HOME_DIR},
    ))
    factory.addStep(steps.ShellCommand(
        command=['./checkjs.sh'],
        workdir="build",
        haltOnFailure=True,
        env={'HOME': HOME_DIR},
    ))
    factory.addStep(steps.ShellCommand(
        command=['./pep8.sh'],
        workdir="build",
        haltOnFailure=True,
        env={'HOME': HOME_DIR},
    ))
    factory.addStep(steps.ShellCommand(
        command=["./run_buildbot_tests"],
        workdir="build/src/myproject",
        env={'HOME': HOME_DIR},
    ))

    c['builders'] = []
    c['builders'].append(
        util.BuilderConfig(
            name="myproject_runtests",
            workernames=["example-worker"],
            collapseRequests=True,
            factory=factory))

    c['status'] = []

    authz = util.Authz(
      allowRules=[
        util.StopBuildEndpointMatcher(role="admins"),
        util.ForceBuildEndpointMatcher(role="admins"),
        util.RebuildBuildEndpointMatcher(role="admins")
      ],
      roleMatchers=[
        util.RolesFromEmails(admins=["myuser at myorg.com"]),
        util.RolesFromUsername(roles=["admins"], usernames=["myuser"]),
      ]
    )
    c['www'] = dict(
        port=8010,
        plugins=dict(waterfall_view={}, console_view={}),
        #auth=util.HTPasswdAuth(os.path.join(BASE_DIR, '../htpasswd')),
        auth=util.RemoteUserAuth(),
        authz=authz,
    )

    mn = reporters.MailNotifier(
        fromaddr="buildbot at myproject.com",
        sendToInterestedUsers=True,
        useTls=True,
        relayhost="smtp.myproject.com",
        smtpPort=587,
        smtpUser="buildbot at myproject.com",
        smtpPassword=master_sensitive.EMAIL_HOST_PASSWORD,
        extraRecipients=[
            'developers at myproject.com',
        ],
    )
    c['services'] = []
    c['services'].append(mn)

    bs = BitbucketStatusPush(BITBUCKET_OATH_KEY, BITBUCKET_OATH_SECRET)
    c['services'].append(bs)

    c['title'] = "MyProject"

    c['buildbotURL'] = "http://buildbot.myproject.com/"

    c['db'] = {
        'db_url' : "sqlite:///state.sqlite",
    }

    c['changeHorizon'] = 200
    c['buildHorizon'] = 100
    c['logHorizon'] = 40
    c['buildCacheSize'] = 15
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.buildbot.net/pipermail/users/attachments/20170406/758e22f6/attachment.html>