yellow team mailing list archive

[Brad Crittenden <bac@xxxxxxxxxxxxx>]

Brad Crittenden has proposed merging lp:~bac/charms/oneiric/buildbot-master/history-s3 into lp:~yellow/charms/oneiric/buildbot-master/trunk.

Requested reviews:
  Launchpad Yellow Squad (yellow): code

For more details, see:
https://code.launchpad.net/~bac/charms/oneiric/buildbot-master/history-s3/+merge/94262

Add ability to store buildbot data files to S3 and retrieve them the next time the master is started.

The configuration settings for access-key and secret-key need to be passed, like so:

juju set buildbot-master access-key='<your AWS key>' secret-key='<your AWS secret key>'

Unfortunately the 'stop' hook is not being called (see bug 872264) so automatically saving the history when bringing down the juju service does not work.  As a work-around a new config value is provided which causes the files to get pushed.  Setting that config variable to a different value will cause it to save again, e.g.

juju set buildbot-master save-history-now='`date`'
-- 
https://code.launchpad.net/~bac/charms/oneiric/buildbot-master/history-s3/+merge/94262
Your team Launchpad Yellow Squad is requested to review the proposed merge of lp:~bac/charms/oneiric/buildbot-master/history-s3 into lp:~yellow/charms/oneiric/buildbot-master/trunk.

=== modified file 'config.yaml'
--- config.yaml	2012-02-10 21:19:58 +0000
+++ config.yaml	2012-02-22 19:51:18 +0000
@@ -52,3 +52,25 @@
       install the newly specified packages while leaving the previous
       ones installed.
     type: string
+  access-key:
+    description: |
+      Access key for EC2.
+    type: string
+  secret-key:
+    description: |
+      Secret key for EC2.
+    type: string
+  bucket-name:
+    description: |
+      The bucket used to store buildbot history.  If not provided a
+      default based on the access-key will be used.
+    type: string
+  save-history-now:
+    description: |
+      Configuration hack to fire off the saving of the buildbot master
+      history.  Normally this would be done in the stop hook but due to
+      Bug 872264 that hook is not firing properly.  The value of the
+      setting is not important but it must change between invocations
+      or the event will not be recogized.  Monotonically increasing
+      integer values would be a good choice.  Or a time string.
+    type: string

=== modified file 'examples/pyflakes.yaml'
--- examples/pyflakes.yaml	2012-02-10 21:19:58 +0000
+++ examples/pyflakes.yaml	2012-02-22 19:51:18 +0000
@@ -1,5 +1,5 @@
 buildbot-master:
-  extra-packages: git
+  extra-packages: git python-sqlalchemy python-migrate
   installdir: /tmp/buildbot
   config-file: |
     # -*- python -*-

=== modified file 'hooks/config-changed'
--- hooks/config-changed	2012-02-14 15:51:49 +0000
+++ hooks/config-changed	2012-02-22 19:51:18 +0000
@@ -3,10 +3,12 @@
 # Copyright 2012 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
+import base64
 import json
 import os
 import os.path
 import shutil
+import subprocess
 import sys
 
 from helpers import (
@@ -27,6 +29,9 @@
     buildbot_reconfig,
     config_json,
     generate_string,
+    get_bucket,
+    get_key,
+    put_history,
     slave_json,
     )
 
@@ -36,7 +41,6 @@
     ]
 SUPPORTED_TRANSPORTS = ['bzr']
 
-
 bzr = command('bzr')
 
 
@@ -58,7 +62,6 @@
 def handle_config_changes(config, diff):
     log('Updating buildbot configuration.')
     log('Configuration changes seen:')
-    log(str(diff))
 
     buildbot_pkg = config.get('buildbot-pkg')
     extra_repo = config.get('extra-repository')
@@ -91,7 +94,6 @@
 
     # Write the buildbot config to disk (fetching it if necessary).
     added_or_changed = diff.added_or_changed
-    log("CONFIG FILE: {}".format(config_file))
     log("ADDED OR CHANGED: {}".format(added_or_changed))
     if config_file and 'config-file' in added_or_changed:
         buildbot_create(buildbot_dir)
@@ -108,7 +110,7 @@
         # gpg-agent needs to send the key over and the bzr launchpad-login
         # needs to be set.
         with su('buildbot'):
-            lp_id = config.get('config-usr')
+            lp_id = config.get('config-user')
             if lp_id:
                 bzr('launchpad-login', lp_id)
             private_key = config.get('config-private-key')
@@ -134,6 +136,44 @@
     return restart_required
 
 
+def fetch_history(config, diff):
+    """Fetch the buildbot history from an external store."""
+    # Currently only S3 is supported.
+    restart_required = False
+
+    log("fetching history")
+    if 'secret-key' not in diff.added_or_changed:
+        log("skipping fetch of history")
+        return restart_required
+
+    bucket_name = config.get('bucket-name')
+    bucket = get_bucket(config, bucket_name)
+
+    if bucket:
+        key = get_key(bucket)
+        if key.exists():
+            target = '/tmp/history-fetched.tgz'
+            key.get_contents_to_filename(target)
+            cwd = os.getcwd()
+            os.chdir(config['installdir'])
+            with su('buildbot'):
+                try:
+                    run('tar', 'xzf', target)
+                except subprocess.CalledProcessError as e:
+                    print e
+                    print e.output
+                    raise
+            os.chdir(cwd)
+            os.unlink(target)
+            log("History fetched from S3.")
+            restart_required = True
+        else:
+            log("Key does not exist: " + key.key)
+    else:
+        log("Bucket not found: " + bucket_name)
+    return restart_required
+
+
 def initialize_buildbot(config):
     # Initialize the buildbot directory and (re)start buildbot.
     log("Initializing buildbot")
@@ -153,12 +193,21 @@
         if not os.path.exists(placeholder_path):
             with open(placeholder_path, 'w') as f:
                 json.dump(generate_string("temporary-placeholder-"), f)
-    buildbot_reconfig()
+    try:
+        buildbot_reconfig()
+    except subprocess.CalledProcessError as e:
+        print e
+        print e.output
+        raise
+
+
+def conditionally_save_history(config, diff):
+    if 'save-history-now' in diff.added_or_changed:
+        put_history(config)
 
 
 def main():
     config = get_config()
-    log(str(config))
     if not check_config(config):
         log("Configuration not valid.")
         sys.exit(1)
@@ -178,6 +227,7 @@
             os.makedirs(buildbot_dir)
 
     restart_required |= configure_buildbot(config, diff)
+    restart_required |= fetch_history(config, diff)
 
     master_cfg_path = os.path.join(buildbot_dir, 'master.cfg')
     if restart_required and os.path.exists(master_cfg_path):
@@ -185,6 +235,8 @@
     else:
         log("Configuration changed but didn't require restarting.")
 
+    conditionally_save_history(config, diff)
+
     config_json.set(config)
 
 

=== modified file 'hooks/install'
--- hooks/install	2012-02-14 16:54:21 +0000
+++ hooks/install	2012-02-22 19:51:18 +0000
@@ -22,7 +22,7 @@
 
 
 def cleanup(buildbot_dir):
-    apt_get_install('bzr')
+    apt_get_install('bzr', 'python-boto')
     # Since we may be installing into a pre-existing service, ensure the
     # buildbot directory is removed.
     if os.path.exists(buildbot_dir):

=== modified file 'hooks/local.py'
--- hooks/local.py	2012-02-10 01:05:09 +0000
+++ hooks/local.py	2012-02-22 19:51:18 +0000
@@ -11,11 +11,15 @@
     'config_json',
     'create_slave',
     'generate_string',
+    'get_bucket',
+    'get_key',
     'HTTP_PORT_PROTOCOL',
+    'put_history',
     'slave_json',
     ]
 
 import os
+import re
 import subprocess
 import uuid
 
@@ -157,3 +161,84 @@
 
 slave_json = Serializer('/tmp/slave_info.json')
 config_json = Serializer('/tmp/config.json')
+
+
+def get_bucket(config, bucket_name=None):
+    """Return an S3 bucket or None."""
+    # Late import to ensure python-boto package has been installed.
+    import boto
+    access_key = config.get('access-key')
+    secret_key = config.get('secret-key')
+    bucket = None
+    if access_key and secret_key:
+        if bucket_name is None:
+            bucket_name = str(access_key + '-buildbot-history').lower()
+        conn = boto.connect_s3(access_key, secret_key)
+        bucket = conn.create_bucket(bucket_name)
+        log("Using bucket: " + bucket.name)
+    return bucket
+
+
+def get_key(bucket):
+    """Return an S3 key for the bucket."""
+    # Late import to ensure python-boto package has been installed.
+    from boto.s3.key import Key
+    key = Key(bucket)
+    key.key = os.environ['JUJU_UNIT_NAME']
+    log("Using key: " + key.key)
+    return key
+
+
+VERSION_TO_STORE = {
+    '0.7': "*/builder",
+    '0.8': "state.sqlite",
+    }
+
+
+def get_buildbot_version():
+    """Get the major version (x.y) of buildbot and return as a string.
+
+    Return None if the output from buildbot cannot be parsed.
+    """
+    version = None
+    output = run('buildbot', '--version')
+    match = re.search('Buildbot version: (\d+\.\d+)(\.\d+)*\n', output)
+    if match and len(match.groups()) > 0:
+        version = match.group(1)
+    return version
+
+
+def put_history(config):
+    """Put the buildbot history to an external store, if set up."""
+    log("put_history called")
+    bucket_name = config.get('bucket-name')
+    bucket = get_bucket(config, bucket_name)
+    success = False
+    if bucket:
+        key = get_key(bucket)
+        target = '/tmp/history-put.tgz'
+        version = get_buildbot_version()
+        store_pattern = VERSION_TO_STORE.get(version)
+        assert store_pattern is not None, (
+            "Buildbot version not supported: {}".format(version))
+        cwd = os.getcwd()
+        os.chdir(config['installdir'])
+        with su('buildbot'):
+            try:
+                run('tar', 'czf', target, store_pattern)
+                key.set_contents_from_filename(target)
+                success = True
+                # If would be natural to just log the success here, but we are
+                # su-ed to the buildbot user and that causes permission
+                # problems.
+                # log("History stored to S3.")
+            except subprocess.CalledProcessError as e:
+                print e
+                print e.output
+                raise
+            os.unlink(target)
+        os.chdir(cwd)
+    else:
+        log("Bucket not found: " + bucket_name)
+    if success:
+        log("History stored to S3.")

=== modified file 'hooks/start'
--- hooks/start	2012-02-08 14:26:58 +0000
+++ hooks/start	2012-02-22 19:51:18 +0000
@@ -4,7 +4,6 @@
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 from helpers import (
-    log,
     log_entry,
     log_exit,
     run,

=== modified file 'hooks/stop'
--- hooks/stop	2012-02-08 14:26:58 +0000
+++ hooks/stop	2012-02-22 19:51:18 +0000
@@ -10,14 +10,17 @@
     log_exit,
     run,
     )
-from local import HTTP_PORT_PROTOCOL
+from local import (
+    HTTP_PORT_PROTOCOL,
+    put_history,
+    )
 
 
 def main():
     config = get_config()
     log('Stopping buildbot in {}'.format(config['installdir']))
+    put_history(config)
     run('close-port', HTTP_PORT_PROTOCOL)
-
     log('Finished stopping buildbot')
 
 

=== modified file 'hooks/tests.py'
--- hooks/tests.py	2012-02-10 19:43:46 +0000
+++ hooks/tests.py	2012-02-22 19:51:18 +0000
@@ -10,7 +10,6 @@
     unit_info,
     )
 
-from subprocess import CalledProcessError
 
 class TestRun(unittest.TestCase):
 
@@ -24,7 +23,7 @@
         # produces a string.
         self.assertIn('Usage:', run('/bin/ls', '--help'))
 
-    def testSimpleCommand(self):
+    def testCalledProcessErrorRaised(self):
         # If an error occurs a CalledProcessError is raised with the return
         # code, command executed, and the output of the command.
         with self.assertRaises(CalledProcessError) as info: