yellow team mailing list archive

[Brad Crittenden <bac@xxxxxxxxxxxxx>]

Brad Crittenden has proposed merging lp:~bac/charms/oneiric/buildbot-slave/dynamic-relationship into lp:~yellow/charms/oneiric/buildbot-slave/trunk.

Requested reviews:
  Launchpad Yellow Squad (yellow): code

For more details, see:
https://code.launchpad.net/~bac/charms/oneiric/buildbot-slave/dynamic-relationship/+merge/91739

Clean up.  Merged Francesco's clean up branch.

The files helpers.py and local.py no longer live in the buildbot slave.  The One True Version is in the buildbot master.  To get access to them, in the buildbot-slave/hooks directory create a symbolic link to the version in the master.  This is not ideal and is temporary until we create a branch just for them or a PPA.
-- 
https://code.launchpad.net/~bac/charms/oneiric/buildbot-slave/dynamic-relationship/+merge/91739
Your team Launchpad Yellow Squad is requested to review the proposed merge of lp:~bac/charms/oneiric/buildbot-slave/dynamic-relationship into lp:~yellow/charms/oneiric/buildbot-slave/trunk.

=== added file '.bzrignore'
--- .bzrignore	1970-01-01 00:00:00 +0000
+++ .bzrignore	2012-02-06 23:23:59 +0000
@@ -0,0 +1,3 @@
+revision
+hooks/helpers.py
+hooks/local.py

=== added file 'config.setuplxc.yaml'
--- config.setuplxc.yaml	1970-01-01 00:00:00 +0000
+++ config.setuplxc.yaml	2012-02-06 23:23:59 +0000
@@ -0,0 +1,8 @@
+buildbot-slave:
+  builders: lucid_lp,lucid_db_lp
+  script-retrieval-method: bzr_cat
+  script-url: "http://bazaar.launchpad.net/~gmb/launchpad/neuter-setuplxc/utilities/setuplxc.py";
+  script-path: setuplxc.py
+  script-args: "-u launchpad-pqm -e launchpad-pqm@xxxxxxxxxxxxx -f 'Launchpad PQM' /home/launchpad-pqm/launchpad/"
+  extra-repository: deb http://us.archive.ubuntu.com/ubuntu/ lucid main universe
+  buildbot-pkg: buildbot/lucid

=== modified file 'config.yaml'
--- config.yaml	2012-01-30 19:09:50 +0000
+++ config.yaml	2012-02-06 23:23:59 +0000
@@ -1,17 +1,43 @@
 options:
+  script-retrieval-method:
+    description: |
+      How the script is retrieved. You can use bzr, brz_cat, git,
+      mercurial or wget.
+    type: string
+    default: bzr
+  script-url:
+    description: |
+      The url where this script lives.
+    type: string
+  script-path:
+    description: |
+      The path of the script that will be executed.
+    type: string
+  script-args:
+    description: |
+      The script arguments.
+    type: string
+  builders:
+    description: |
+      The builders that this slave should be a part of. Builders must be
+      comma-separated.
+    type: string
+  buildbot-pkg:
+    description: |
+      The package name, possibly with versioning information, to be
+      installed for buildbot.  Example values are 'buildbot',
+      'buildbot/lucid', or 'buildbot=0.7.9'.
+    type: string
+    default: buildbot
+  extra-repository:
+    description: |
+      The full line to be inserted into an apt sources.list for a repository.
+      If called multiple times the new repository will be added but
+      ones added previously will not be removed. An example would be:
+      deb http://us.archive.ubuntu.com/ubuntu/ lucid main universe
+    type: string
   installdir:
     description: |
       The directory where the Buildbot slave will be installed.
     type: string
-    default: /tmp/buildbot
-  name:
-    description: |
-      The name of this slave.
-    type: string
-    default: example-slave
-  passwd:
-    description: |
-      The password of this slave.
-    type: string
-    default: pass
-
+    default: /tmp/buildslave

=== added file 'copyright'
--- copyright	1970-01-01 00:00:00 +0000
+++ copyright	2012-02-06 23:23:59 +0000
@@ -0,0 +1,17 @@
+Format: http://dep.debian.net/deps/dep5/
+
+Files: *
+Copyright: Copyright 2012, Canonical Ltd., All Rights Reserved.
+License: GPL-3
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <http://www.gnu.org/licenses/>.

=== modified file 'hooks/buildbot-relation-changed'
--- hooks/buildbot-relation-changed	2012-01-31 01:50:27 +0000
+++ hooks/buildbot-relation-changed	2012-02-06 23:23:59 +0000
@@ -1,26 +1,40 @@
-#!/bin/bash
-# This must be renamed to the name of the relation. The goal here is to
-# affect any change needed by relationships being formed, modified, or broken
-# This script should be idempotent.
-juju-log $JUJU_REMOTE_UNIT modified its settings
-
-BUILDBOT_DIR=`config-get installdir`
-NAME=`config-get name`
-PASSWD=`config-get passwd`
-HOST=`relation-get private-address`
-
-juju-log "--> stop"
-# The exit code is 0 even if the buildslave was not running.
-buildslave stop $BUILDBOT_DIR
-juju-log "<-- stop"
-
-juju-log "--> create .tac"
-if [ -e $BUILDBOT_DIR/buildbot.tac ]; then
-    rm $BUILDBOT_DIR/buildbot.tac
-fi
-buildslave create-slave $BUILDBOT_DIR $HOST $NAME $PASSWD
-juju-log "<-- create .tac"
-
-juju-log "--> start"
-buildslave start $BUILDBOT_DIR
-juju-log "<-- start"
+#!/usr/bin/env python
+
+import sys
+
+from helpers import (
+    get_config,
+    log,
+    relation_get,
+    )
+from local import (
+    buildslave_start,
+    buildslave_stop,
+    create_slave,
+    )
+
+
+def restart_buildslave(name, passwd, host, buildbot_dir):
+    log('Stopping buildbot slave.')
+    buildslave_stop()
+    log('Creating buildbot slave: {}@{} (passwd={}).'.format(
+        name, host, passwd))
+    create_slave(name, passwd, host=host, buildbot_dir=buildbot_dir)
+    log('Starting buildbot slave.')
+    return buildslave_start()
+
+
+def main():
+    name = relation_get('name')
+    passwd = relation_get('passwd')
+    if name and passwd:
+        # We heard back from the master and we are ready to start.
+        config = get_config()
+        buildbot_dir = config.get('installdir')
+        host = relation_get('private-address')
+        sys.exit(restart_buildslave(name, passwd, host, buildbot_dir))
+
+
+if __name__ == '__main__':
+    log('BUILDBOT-RELATION-CHANGED HOOK:')
+    main()

=== added file 'hooks/buildbot-relation-joined'
--- hooks/buildbot-relation-joined	1970-01-01 00:00:00 +0000
+++ hooks/buildbot-relation-joined	2012-02-06 23:23:59 +0000
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+from helpers import (
+    get_config,
+    log,
+    relation_set,
+    )
+
+
+def send_builders(builders):
+    log('Sending builders to the master: {}'.format(builders))
+    relation_set(builders=builders)
+
+
+def main():
+    config = get_config()
+    builders = config.get('builders')
+    if builders:
+         # This is the first step of the handshake.
+         send_builders(builders)
+
+
+if __name__ == '__main__':
+    log('BUILDBOT-RELATION-JOINED HOOK:')
+    main()

=== added file 'hooks/config-changed'
--- hooks/config-changed	1970-01-01 00:00:00 +0000
+++ hooks/config-changed	2012-02-06 23:23:59 +0000
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+from helpers import (
+    apt_get_install,
+    command,
+    DictDiffer,
+    get_config,
+    install_extra_repository,
+    load_pickle,
+    log,
+    save_pickle,
+    )
+from local import create_slave
+
+
+CONFIG_PICKLE = "slave_config.pkl"
+
+
+def main():
+
+    config = get_config()
+    prev_config = load_pickle(CONFIG_PICKLE)
+
+    diff = DictDiffer(config, prev_config)
+    log("Differences:")
+    log(str(diff))
+
+    buildbot_pkg = config.get('buildbot-pkg')
+    extra_repo = config.get('extra-repository')
+    buildbot_dir = config.get('installdir')
+
+    if extra_repo and 'extra-repository' in diff.added_or_changed:
+        install_extra_repository(extra_repo)
+
+    if buildbot_pkg and 'buildbot-pkg' not in diff.unchanged:
+        log('Installing ' + buildbot_pkg)
+        apt_get_install(buildbot_pkg)
+        log('Creating initial buildbot slave in ' + buildbot_dir)
+        create_slave('temporary', 'temporary', buildbot_dir=buildbot_dir)
+
+    save_pickle(config, CONFIG_PICKLE)
+
+
+if __name__ == '__main__':
+    log('CONFIG-CHANGED HOOK:')
+    main()

=== removed file 'hooks/config-changed'
--- hooks/config-changed	2012-01-30 19:09:50 +0000
+++ hooks/config-changed	1970-01-01 00:00:00 +0000
@@ -1,2 +0,0 @@
-#!/bin/bash
-# Hook for handling config changes.

=== modified file 'hooks/install'
--- hooks/install	2012-01-31 01:50:27 +0000
+++ hooks/install	2012-02-06 23:23:59 +0000
@@ -1,27 +1,90 @@
-#!/bin/bash
-# Here do anything needed to install the service
-# i.e. apt-get install -y foo  or  bzr branch http://myserver/mycode /srv/webroot
-set -eux # -x for verbose logging to juju debug-log
-
-juju-log "--> install packages"
-BUILDBOT_DIR=`config-get installdir`
-apt-get install -y buildbot
-mkdir -p $BUILDBOT_DIR
-juju-log "<-- install packages"
-
-juju-log "--> create slave"
-NAME=`config-get name`
-PASSWD=`config-get passwd`
-
-# We set HOST to localhost for now because we can't get the actual host
-# until we're joined to the buildbot-master instance.
-HOST="localhost"
-
-juju-log "buildslave create-slave $BUILDBOT_DIR $HOST $NAME $PASSWD"
-buildslave create-slave $BUILDBOT_DIR $HOST $NAME $PASSWD
-juju-log "<-- create slave"
-
-# This is where we would set info/admin and info/host.  Not sure what
-# is pertinent, but the default is a bit lame in the juju case.
-
-# This is where we would run something like setuplxc.
+#!/usr/bin/env python
+
+from helpers import (
+    apt_get_install,
+    command,
+    get_config,
+    log,
+    run,
+    )
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+
+
+log('INSTALL HOOK:')
+config = get_config()
+
+bzr_ = command('bzr')
+
+buildbot_dir = config.get('installdir')
+
+def bzr(source, path):
+    apt_get_install('bzr')
+    target = tempfile.mktemp()
+    bzr('branch', source, target)
+    return os.path.join(target, path)
+
+
+def bzr_cat(source, path):
+    apt_get_install('bzr')
+    content = bzr_('--no-plugins', 'cat', source)
+    target = os.path.join('/tmp', path)
+    with open(target, 'w') as f:
+        f.write(content)
+    return target
+
+
+def wget(source, path):
+    target = os.path.join('/tmp', path)
+    command('wget', '-O', target, source)
+    return target
+
+
+def hg(source, target):
+    apt_get_install('mercurial')
+    target = tempfile.mktemp()
+    command('hg', 'clone', source, target)
+    return os.path.join(target, path)
+
+
+def git(source, target):
+    apt_get_install('git')
+    target = tempfile.mktemp()
+    command('git', 'clone', source, target)
+    return os.path.join(target, path)
+
+
+METHODS = {
+    'bzr': bzr,
+    'bzr_cat': bzr_cat,
+    'wget': wget,
+    'hg': hg,
+    'git': git,
+    }
+
+def handle_script(retrieve, url, path, args):
+    log('Retrieving script: {}.'.format(url))
+    script = retrieve(url, path)
+    log('Changing script mode.')
+    command('chmod', '+x', script)
+    log('Executing script: {}.'.format(script))
+    return subprocess.call([script] + shlex.split(str(args)))
+
+
+def main():
+    config = get_config()
+    method = config.get('script-retrieval-method')
+    url = config.get('script-url')
+    path = config.get('script-path')
+    args = config.get('script-args')
+    retrieve = METHODS.get(method)
+    if retrieve and url and path:
+        sys.exit(handle_script(retrieve, url, path, args))
+
+
+if __name__ == '__main__':
+    log('INSTALL HOOK:')
+    main()

=== removed file 'hooks/start'
--- hooks/start	2012-01-30 19:09:50 +0000
+++ hooks/start	1970-01-01 00:00:00 +0000
@@ -1,5 +0,0 @@
-#!/bin/bash
-# Here put anything that is needed to start the service.
-# Note that currently this is run directly after install
-# i.e. 'service apache2 start'
-

=== removed file 'hooks/stop'
--- hooks/stop	2012-01-30 19:09:50 +0000
+++ hooks/stop	1970-01-01 00:00:00 +0000
@@ -1,7 +0,0 @@
-#!/bin/bash
-# This will be run when the service is being torn down, allowing you to disable
-# it in various ways..
-# For example, if your web app uses a text file to signal to the load balancer
-# that it is live... you could remove it and sleep for a bit to allow the load
-# balancer to stop sending traffic.
-# rm /srv/webroot/server-live.txt && sleep 30

=== added file 'hooks/tests.py'
--- hooks/tests.py	1970-01-01 00:00:00 +0000
+++ hooks/tests.py	2012-02-06 23:23:59 +0000
@@ -0,0 +1,47 @@
+import unittest
+from subprocess import CalledProcessError
+from helpers import command
+
+
+class testCommand(unittest.TestCase):
+
+    def testSimpleCommand(self):
+        # Creating a simple command (ls) works and running the command
+        # produces a string.
+        ls = command('/bin/ls')
+        self.assertIsInstance(ls(), str)
+
+    def testArguments(self):
+        # Arguments can be passed to commands.
+        ls = command('/bin/ls')
+        self.assertIn('Usage:', ls('--help'))
+
+    def testMissingExecutable(self):
+        # If the command does not exist, an OSError (No such file or
+        # directory) is raised.
+        bad = command('this command does not exist')
+        with self.assertRaises(OSError) as info:
+            bad()
+        self.assertEqual(2, info.exception.errno)
+
+    def testError(self):
+        # If the command returns a non-zero exit code, an exception is raised.
+        ls = command('/bin/ls')
+        with self.assertRaises(CalledProcessError):
+            ls('--not a valid switch')
+
+    def testBakedInArguments(self):
+        # Arguments can be passed when creating the command as well as when
+        # executing it.
+        ll = command('/bin/ls', '-l')
+        self.assertIn('rw', ll()) # Assumes a file is r/w in the pwd.
+        self.assertIn('Usage:', ll('--help'))
+
+    def testQuoting(self):
+        # There is no need to quote special shell characters in commands.
+        ls = command('/bin/ls')
+        ls('--help', '>')
+
+
+if __name__ == '__main__':
+    unittest.main()

=== removed file 'revision'
--- revision	2012-01-31 01:50:27 +0000
+++ revision	1970-01-01 00:00:00 +0000
@@ -1,1 +0,0 @@
-19

=== added directory 'tests'
=== added file 'tests/100_buildbot-slave.config.test'
--- tests/100_buildbot-slave.config.test	1970-01-01 00:00:00 +0000
+++ tests/100_buildbot-slave.config.test	2012-02-06 23:23:59 +0000
@@ -0,0 +1,69 @@
+#!/bin/sh
+
+set -e
+
+teardown() {
+    juju destroy-service buildbot-slave
+    if [ -n "$datadir" ] ; then
+        if [ -f $datadir/passed ]; then
+            rm -r $datadir
+        else
+            echo $datadir preserved
+        fi
+    fi
+}
+trap teardown EXIT
+
+juju deploy --repository=$PWD/../ local:buildbot-slave
+juju expose buildbot-slave
+
+for try in `seq 1 600` ; do
+    slave_host=`juju status | tests/get-unit-info buildbot-slave public-address`
+    if [ -z "$slave_host" ] ; then
+        sleep 1
+    else
+        break
+    fi
+done
+
+if [ -z "$slave_host" ] ; then
+    echo ERROR: status timed out 
+    exit 1
+fi
+
+datadir=`mktemp -d ${TMPDIR:-/tmp}/wget.test.XXXXXXX`
+echo INFO: datadir=$datadir
+
+#slave_connected=$(
+#    wget --tries=100 --timeout=6 http://$slave_host:9000 -O - \
+#     -a $datadir/wget.log | grep -q 'UP!')
+
+assert_is_listening() {
+    local port=$1
+    listening=""
+    for try in `seq 1 10` ; do
+        if ! nc $slave_host $port < /dev/null ; then
+            continue
+        fi
+        listening="yes"
+        break
+    done
+
+    if [ -z "$listening" ] ; then
+       echo "FAIL: not listening on port $port after 10 retries"
+       return 1
+    else
+       echo "PASS: listening on port $port"
+       return 0
+    fi
+}
+
+assert_is_listening 9000
+
+touch $datadir/passed
+
+trap - EXIT
+teardown
+
+echo INFO: PASS
+exit 0

=== added file 'tests/200_buildbot-slave.test'
--- tests/200_buildbot-slave.test	1970-01-01 00:00:00 +0000
+++ tests/200_buildbot-slave.test	2012-02-06 23:23:59 +0000
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+set -e
+
+teardown() {
+    juju destroy-service buildbot-slave
+    if [ -n "$datadir" ] ; then
+        if [ -f $datadir/passed ]; then
+            rm -r $datadir
+        else
+            echo $datadir preserved
+        fi
+    fi
+}
+trap teardown EXIT
+
+
+juju deploy buildbot-master
+juju deploy buildbot-slave
+juju add-relation buildbot-slave buildbot-master
+juju expose buildbot-master
+
+for try in `seq 1 600` ; do
+    master_host=`juju status | tests/get-unit-info buildbot-master public-address`
+    if [ -z "$master_host" ] ; then
+        sleep 1 
+    else
+        break
+    fi
+done
+
+if [ -z "$master_host" ] ; then
+    echo ERROR: status timed out 
+    exit 1
+fi
+
+datadir=`mktemp -d ${TMPDIR:-/tmp}/wget.test.XXXXXXX`
+echo INFO: datadir=$datadir
+
+slave_connected=$(
+    wget --tries=100 --timeout=6 http://$master_host:8010/buildslaves -O - \
+     -a $datadir/wget.log | grep -q 'Idle')
+
+if [ -z $slave_connected ]; then
+    echo "ERROR: The slave is not connected after 600 seconds."
+    exit 1
+fi
+
+touch $datadir/passed
+
+trap - EXIT
+teardown
+
+echo INFO: PASS
+exit 0

=== added file 'tests/config.test.yaml'
--- tests/config.test.yaml	1970-01-01 00:00:00 +0000
+++ tests/config.test.yaml	2012-02-06 23:23:59 +0000
@@ -0,0 +1,6 @@
+buildbot-slave:
+  installdir: /tmp/buildbot
+  name: example-slave
+  passwd: pass
+  script-retrieval-method: bzr
+  script-url: "http://bazaar.launchpad.net/~gmb/charms/oneiric/buildbot-slave/look-we-have-tests/tests/open-port.py";

=== added file 'tests/get-unit-info'
--- tests/get-unit-info	1970-01-01 00:00:00 +0000
+++ tests/get-unit-info	2012-02-06 23:23:59 +0000
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+# machines:
+#   0: {dns-name: ec2-50-17-84-127.compute-1.amazonaws.com, instance-id: i-8c5c3fec}
+#   1: {dns-name: ec2-184-73-102-113.compute-1.amazonaws.com, instance-id: i-14a2c174}
+#   2: {dns-name: ec2-75-101-184-93.compute-1.amazonaws.com, instance-id: i-e0a2c180}
+# services:
+#   mysql:
+#     charm: local:mysql-11
+#     relations: {db: wordpress}
+#     units:
+#       mysql/0:
+#         machine: 2
+#         relations:
+#           db: {state: up}
+#         state: started
+#   wordpress:
+#     charm: local:wordpress-31
+#     exposed: true
+#     relations: {db: mysql}
+#     units:
+#       wordpress/0:
+#         machine: 1
+#         open-ports: []
+#         relations: {}
+#         state: null
+
+import yaml
+import sys
+from subprocess import Popen, PIPE
+
+
+def main():
+    d = yaml.safe_load(Popen(['juju','status'],stdout=PIPE).stdout)
+    srv = d.get("services", {}).get(sys.argv[1])
+    if srv is None:
+        return
+    units = srv.get("units", {})
+    if units is None:
+        return
+    item = units.items()[0][1].get(sys.argv[2])
+    if item is None:
+        return
+    print item
+
+if __name__ == "__main__":
+    main()

=== added file 'tests/helpers.py'
--- tests/helpers.py	1970-01-01 00:00:00 +0000
+++ tests/helpers.py	2012-02-06 23:23:59 +0000
@@ -0,0 +1,25 @@
+from collections import defaultdict
+import subprocess
+
+def command(*base_args):
+    def callable_command(*args):
+        return subprocess.check_output(base_args+args, shell=False)
+
+    return callable_command
+
+
+def run(*args):
+    return subprocess.check_output(args, shell=False)
+
+
+class Config(defaultdict):
+
+    def __init__(self):
+        super(defaultdict, self).__init__()
+        self.config_get = command('config-get')
+
+    def __missing__(self, key):
+        return self.config_get(key).strip()
+
+    def __setitem__(self, key, value):
+        raise RuntimeError('configuration is read-only')

=== added file 'tests/open-port.py'
--- tests/open-port.py	1970-01-01 00:00:00 +0000
+++ tests/open-port.py	2012-02-06 23:23:59 +0000
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# A little python script to open a port on the buildslave so that we can
+# test that it's actually deployed successfully.
+
+import subprocess
+import SimpleHTTPServer
+import SocketServer
+
+PORT = 9000
+
+simple_html = """
+<html>
+    <head>
+        <title>Here's some HTML</title>
+    </head>
+    <body><p>The slave is UP!</p></body>
+</html>
+"""
+
+with open('/tmp/index.html', 'w') as index_file:
+    index_file.write(simple_html)
+
+subprocess.call(['open-port', '9000/TCP'])
+Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
+httpd = SocketServer.TCPServer(("", PORT), Handler)
+#httpd.serve_forever()
+print "HERE I AM!!!"