yellow team mailing list archive
-
yellow team
-
Mailing list archive
-
Message #00580
[Merge] lp:~frankban/lpsetup/split-files into lp:lpsetup
Francesco Banconi has proposed merging lp:~frankban/lpsetup/split-files into lp:lpsetup.
Requested reviews:
Launchpad Yellow Squad (yellow)
For more details, see:
https://code.launchpad.net/~frankban/lpsetup/split-files/+merge/97028
== Changes ==
- Created an lpsetup project (personal, but this should change).
- Splitted the code present in `lpsetup.py` into smaller files.
- Added distribution files.
- The script now uses helpers from `python-shell-toolbox`.
- Fixed some doctests.
- Fixed the way `file_append` and `file_prepend` are used.
This is an incremental change, not tested, but required to start working on
lpsetup as a project and not just a standalone script.
--
https://code.launchpad.net/~frankban/lpsetup/split-files/+merge/97028
Your team Launchpad Yellow Squad is requested to review the proposed merge of lp:~frankban/lpsetup/split-files into lp:lpsetup.
=== added file '.bzrignore'
--- .bzrignore 1970-01-01 00:00:00 +0000
+++ .bzrignore 2012-03-12 14:27:19 +0000
@@ -0,0 +1,3 @@
+build
+dist
+MANIFEST
=== added file 'INSTALL'
--- INSTALL 1970-01-01 00:00:00 +0000
+++ INSTALL 2012-03-12 14:27:19 +0000
@@ -0,0 +1,4 @@
+To install lpsetup, run the following command inside this directory:
+
+ python setup.py install
+
=== added file 'MANIFEST.in'
--- MANIFEST.in 1970-01-01 00:00:00 +0000
+++ MANIFEST.in 2012-03-12 14:27:19 +0000
@@ -0,0 +1,5 @@
+include INSTALL
+include LICENSE
+include README.rst
+include MANIFEST.in
+recursive-include lpsetup/tests *
=== added file 'README.rst'
--- README.rst 1970-01-01 00:00:00 +0000
+++ README.rst 2012-03-12 14:27:19 +0000
@@ -0,0 +1,34 @@
+lpsetup
+=======
+
+This project provides the `lp-setup` command, that can be used to create
+and update a Launchpad development or testing environment.
+
+This project also allows user to set up Launchpad inside an LXC container.
+
+
+Requirements
+~~~~~~~~~~~~
+
+lp-setup requires Python >= 2.7 and `python-shell-toolbox`.
+The package `python-argparse` is required to run lp-setup using Python 2.6.
+
+
+Getting started
+~~~~~~~~~~~~~~~
+
+Install the Launchpad environment::
+
+ lp-setup install
+
+Install the Launchpad environment inside an LXC::
+
+ lp-setup lxc-install
+
+Update Launchpad::
+
+ lp-setup update
+
+Help::
+
+ lp-setup help [command]
=== added file 'lp-setup'
--- lp-setup 1970-01-01 00:00:00 +0000
+++ lp-setup 2012-03-12 14:27:19 +0000
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Command line interface entry point for lpsetup."""
+
+from lpsetup import cli
+
+
+if __name__ == '__main__':
+ cli.main()
=== added directory 'lpsetup'
=== removed file 'lpsetup.py'
--- lpsetup.py 2012-03-09 09:57:35 +0000
+++ lpsetup.py 1970-01-01 00:00:00 +0000
@@ -1,1730 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2012 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Create and update Launchpad development and testing environments."""
-
-__metaclass__ = type
-__all__ = [
- 'ActionsBasedSubCommand',
- 'apt_get_install',
- 'ArgumentParser',
- 'BaseSubCommand',
- 'bzr_whois',
- 'call',
- 'cd',
- 'check_output',
- 'CommandError',
- 'create_lxc',
- 'environ',
- 'file_append',
- 'file_prepend',
- 'generate_ssh_keys',
- 'get_container_path',
- 'get_lxc_gateway',
- 'get_su_command',
- 'get_user_home',
- 'get_user_ids',
- 'initialize',
- 'InstallSubCommand',
- 'join_command',
- 'LaunchpadError',
- 'link_sourcecode_in_branches',
- 'LXCInstallSubCommand',
- 'lxc_in_state',
- 'lxc_running',
- 'lxc_stopped',
- 'make_launchpad',
- 'mkdirs',
- 'setup_apt',
- 'setup_codebase',
- 'setup_external_sourcecode',
- 'setup_launchpad',
- 'setup_launchpad_lxc',
- 'ssh',
- 'SSHError',
- 'start_lxc',
- 'stop_lxc',
- 'su',
- 'this_command',
- 'update_launchpad',
- 'user_exists',
- 'ValidationError',
- 'wait_for_lxc',
- ]
-
-# To run doctests: python -m doctest -v lpsetup.py
-
-from collections import namedtuple
-from contextlib import contextmanager
-from email.Utils import parseaddr, formataddr
-from functools import partial
-import argparse
-import errno
-import os
-import pipes
-import platform
-import pwd
-import shutil
-import subprocess
-import sys
-import time
-
-
-APT_REPOSITORIES = (
- 'deb http://archive.ubuntu.com/ubuntu {distro} multiverse',
- 'deb http://archive.ubuntu.com/ubuntu {distro}-updates multiverse',
- 'deb http://archive.ubuntu.com/ubuntu {distro}-security multiverse',
- 'ppa:launchpad/ppa',
- 'ppa:bzr/ppa',
- )
-BASE_PACKAGES = ['ssh', 'bzr', 'language-pack-en']
-CHECKOUT_DIR = '~/launchpad/branches'
-DEPENDENCIES_DIR = '~/launchpad/dependencies'
-DHCP_FILE = '/etc/dhcp/dhclient.conf'
-HOSTS_CONTENT = (
- ('127.0.0.88',
- 'launchpad.dev answers.launchpad.dev archive.launchpad.dev '
- 'api.launchpad.dev bazaar-internal.launchpad.dev beta.launchpad.dev '
- 'blueprints.launchpad.dev bugs.launchpad.dev code.launchpad.dev '
- 'feeds.launchpad.dev id.launchpad.dev keyserver.launchpad.dev '
- 'lists.launchpad.dev openid.launchpad.dev '
- 'ubuntu-openid.launchpad.dev ppa.launchpad.dev '
- 'private-ppa.launchpad.dev testopenid.dev translations.launchpad.dev '
- 'xmlrpc-private.launchpad.dev xmlrpc.launchpad.dev'),
- ('127.0.0.99', 'bazaar.launchpad.dev'),
- )
-HOSTS_FILE = '/etc/hosts'
-LP_APACHE_MODULES = 'proxy proxy_http rewrite ssl deflate headers'
-LP_APACHE_ROOTS = (
- '/var/tmp/bazaar.launchpad.dev/static',
- '/var/tmp/bazaar.launchpad.dev/mirrors',
- '/var/tmp/archive',
- '/var/tmp/ppa',
- )
-LP_BZR_LOCATIONS = (
- ('submit_branch', '{checkout_dir}'),
- ('public_branch', 'bzr+ssh://bazaar.launchpad.net/~{lpuser}/launchpad'),
- ('public_branch:policy', 'appendpath'),
- ('push_location', 'lp:~{lpuser}/launchpad'),
- ('push_location:policy', 'appendpath'),
- ('merge_target', '{checkout_dir}'),
- ('submit_to', 'merge@xxxxxxxxxxxxxxxxxx'),
- )
-LP_CHECKOUT = 'devel'
-LP_PACKAGES = [
- 'bzr', 'launchpad-developer-dependencies', 'apache2',
- 'apache2-mpm-worker', 'libapache2-mod-wsgi'
- ]
-LP_REPOS = (
- 'http://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel',
- 'lp:launchpad',
- )
-LP_SOURCE_DEPS = (
- 'http://bazaar.launchpad.net/~launchpad/lp-source-dependencies/trunk')
-LXC_CONFIG_TEMPLATE = '/etc/lxc/local.conf'
-LXC_GUEST_ARCH = 'i386'
-LXC_GUEST_CHOICES = ('lucid', 'oneiric', 'precise')
-LXC_GUEST_OS = LXC_GUEST_CHOICES[0]
-LXC_NAME = 'lptests'
-LXC_OPTIONS = """
-lxc.network.type = veth
-lxc.network.link = {interface}
-lxc.network.flags = up
-"""
-LXC_PACKAGES = ['lxc', 'libvirt-bin']
-LXC_PATH = '/var/lib/lxc/'
-RESOLV_FILE = '/etc/resolv.conf'
-
-
-Env = namedtuple('Env', 'uid gid home')
-
-
-class LaunchpadError(Exception):
- """Base exception for lpsetup."""
-
-
-class CommandError(LaunchpadError):
- """Errors occurred running shell commands."""
-
-
-class SSHError(LaunchpadError):
- """Errors occurred during SSH connection."""
-
-
-class ValidationError(LaunchpadError):
- """Argparse invalid arguments."""
-
-
-def apt_get_install(*args):
- """Install given packages using apt."""
- with environ(DEBIAN_FRONTEND='noninteractive'):
- cmd = ('apt-get', '-y', 'install') + args
- return call(*cmd)
-
-
-def join_command(args):
- """Return a valid Unix command line from `args`.
-
- >>> join_command(['ls', '-l'])
- 'ls -l'
-
- Arguments containing spaces and empty args are correctly quoted::
-
- >>> join_command(['command', 'arg1', 'arg containing spaces', ''])
- "command arg1 'arg containing spaces' ''"
- """
- return ' '.join(pipes.quote(arg) for arg in args)
-
-
-def bzr_whois(user):
- """Return full name and email of bzr `user`.
-
- Return None if the given `user` does not have a bzr user id.
- """
- with su(user):
- try:
- whoami = check_output('bzr', 'whoami')
- except (CommandError, OSError):
- return None
- return parseaddr(whoami)
-
-
-@contextmanager
-def cd(directory):
- """A context manager to temporarily change current working dir, e.g.::
-
- >>> import os
- >>> os.chdir('/tmp')
- >>> with cd('/bin'): print os.getcwd()
- /bin
- >>> print os.getcwd()
- /tmp
- """
- cwd = os.getcwd()
- os.chdir(directory)
- try:
- yield
- finally:
- os.chdir(cwd)
-
-
-def check_output(*args, **kwargs):
- r"""Run the command with the given arguments.
-
- The first argument is the path to the command to run.
- Subsequent arguments are command-line arguments to be passed.
-
- Usually the output is captured by a pipe and returned::
-
- >>> check_output('echo', 'output')
- 'output\n'
-
- A `CommandError` exception is raised if the return code is not zero::
-
- >>> check_output('ls', '--not a valid switch', stderr=subprocess.PIPE)
- Traceback (most recent call last):
- CommandError: Error running: ls '--not a valid switch'
-
- None arguments are ignored::
-
- >>> check_output(None, 'echo', None, 'output')
- 'output\n'
- """
- args = [i for i in args if i is not None]
- process = subprocess.Popen(
- args, stdout=kwargs.pop('stdout', subprocess.PIPE),
- close_fds=True, **kwargs)
- stdout, stderr = process.communicate()
- retcode = process.poll()
- if retcode:
- raise CommandError('Error running: ' + join_command(args))
- return stdout
-
-
-call = partial(check_output, stdout=None)
-
-
-@contextmanager
-def environ(**kwargs):
- """A context manager to temporarily change environment variables.
-
- If an existing environment variable is changed, it is restored during
- context cleanup::
-
- >>> import os
- >>> os.environ['MY_VARIABLE'] = 'foo'
- >>> with environ(MY_VARIABLE='bar'): print os.getenv('MY_VARIABLE')
- bar
- >>> print os.getenv('MY_VARIABLE')
- foo
- >>> del os.environ['MY_VARIABLE']
-
- If we are adding environment variables, they are removed during context
- cleanup::
-
- >>> import os
- >>> with environ(MY_VAR1='foo', MY_VAR2='bar'):
- ... print os.getenv('MY_VAR1'), os.getenv('MY_VAR2')
- foo bar
- >>> os.getenv('MY_VAR1') == os.getenv('MY_VAR2') == None
- True
- """
- backup = {}
- for key, value in kwargs.items():
- backup[key] = os.getenv(key)
- os.environ[key] = value
- try:
- yield
- finally:
- for key, value in backup.items():
- if value is None:
- del os.environ[key]
- else:
- os.environ[key] = value
-
-
-def file_append(filename, line):
- r"""Append given `line`, if not present, at the end of `filename`.
-
- Usage example::
-
- >>> import tempfile
- >>> f = tempfile.NamedTemporaryFile('w', delete=False)
- >>> f.write('line1\n')
- >>> f.close()
- >>> file_append(f.name, 'new line\n')
- >>> open(f.name).read()
- 'line1\nnew line\n'
-
- Nothing happens if the file already contains the given `line`::
-
- >>> file_append(f.name, 'new line\n')
- >>> open(f.name).read()
- 'line1\nnew line\n'
-
- A new line is automatically added before the given `line` if it is not
- present at the end of current file content::
-
- >>> import tempfile
- >>> f = tempfile.NamedTemporaryFile('w', delete=False)
- >>> f.write('line1')
- >>> f.close()
- >>> file_append(f.name, 'new line\n')
- >>> open(f.name).read()
- 'line1\nnew line\n'
-
- The file is created if it does not exist::
-
- >>> import tempfile
- >>> filename = tempfile.mktemp()
- >>> file_append(filename, 'line1\n')
- >>> open(filename).read()
- 'line1\n'
- """
- with open(filename, 'a+') as f:
- content = f.read()
- if line not in content:
- if content.endswith('\n') or not content:
- f.write(line)
- else:
- f.write('\n' + line)
-
-
-def file_prepend(filename, line):
- r"""Insert given `line`, if not present, at the beginning of `filename`.
-
- Usage example::
-
- >>> import tempfile
- >>> f = tempfile.NamedTemporaryFile('w', delete=False)
- >>> f.write('line1\n')
- >>> f.close()
- >>> file_prepend(f.name, 'line0\n')
- >>> open(f.name).read()
- 'line0\nline1\n'
-
- If the file starts with the given `line`, nothing happens::
-
- >>> file_prepend(f.name, 'line0\n')
- >>> open(f.name).read()
- 'line0\nline1\n'
-
- If the file contains the given `line`, but not at the beginning,
- the line is moved on top::
-
- >>> file_prepend(f.name, 'line1\n')
- >>> open(f.name).read()
- 'line1\nline0\n'
- """
- with open(filename, 'r+') as f:
- lines = f.readlines()
- if lines[0] != line:
- if line in lines:
- lines.remove(line)
- lines.insert(0, line)
- f.seek(0)
- f.writelines(lines)
-
-
-def generate_ssh_keys(directory, filename='id_rsa'):
- """Generate ssh key pair, saving them inside the given `directory`.
-
- >>> generate_ssh_keys('/tmp/')
- >>> open('/tmp/id_rsa').readlines()[0].strip()
- '-----BEGIN RSA PRIVATE KEY-----'
- >>> open('/tmp/id_rsa.pub').read().startswith('ssh-rsa')
- True
- >>> os.remove('/tmp/id_rsa')
- >>> os.remove('/tmp/id_rsa.pub')
-
- The key filename can be changed using the `filename` keyword argument
- (default is 'id_rsa')::
-
- >>> generate_ssh_keys('/tmp/', 'custom_key')
- >>> os.path.exists('/tmp/custom_key')
- True
- >>> os.path.exists('/tmp/id_rsa')
- False
-
- >>> os.remove('/tmp/custom_key')
- >>> os.remove('/tmp/custom_key.pub')
- """
- path = os.path.join(directory, filename)
- return call('ssh-keygen', '-q', '-t', 'rsa', '-N', '', '-f', path)
-
-
-def get_container_path(lxc_name, path='', base_path=LXC_PATH):
- """Return the path of LXC container called `lxc_name`.
-
- If a `path` is given, return that path inside the container, e.g.::
-
- >>> get_container_path('mycontainer')
- '/var/lib/lxc/mycontainer/rootfs/'
- >>> get_container_path('mycontainer', '/etc/apt/')
- '/var/lib/lxc/mycontainer/rootfs/etc/apt/'
- >>> get_container_path('mycontainer', 'home')
- '/var/lib/lxc/mycontainer/rootfs/home'
- """
- return os.path.join(base_path, lxc_name, 'rootfs', path.lstrip('/'))
-
-
-def get_lxc_gateway():
- """Return a tuple of gateway name and address.
-
- The gateway name and address will change depending on which version
- of Ubuntu the script is running on.
- """
- release_name = platform.linux_distribution()[2]
- if release_name == 'oneiric':
- return 'virbr0', '192.168.122.1'
- else:
- return 'lxcbr0', '10.0.3.1'
-
-
-def get_su_command(user, args):
- """Return a command line as a sequence, prepending "su" if necessary.
-
- This can be used together with `call` or `check_output` when the `su`
- context manager is not enaugh (e.g. an external program uses uid rather
- than euid).
-
- >>> import getpass
- >>> current_user = getpass.getuser()
-
- If the su is requested as current user, the arguments are returned as
- given::
-
- >>> get_su_command(current_user, ('ls', '-l'))
- ('ls', '-l')
-
- Otherwise, "su" is prepended::
-
- >>> get_su_command('nobody', ('ls', '-l', 'my file'))
- ('su', 'nobody', '-c', "ls -l 'my file'")
- """
- if get_user_ids(user)[0] != os.getuid():
- args = [i for i in args if i is not None]
- return ('su', user, '-c', join_command(args))
- return args
-
-
-def get_user_home(user):
- """Return the home directory of the given `user`.
-
- >>> get_user_home('root')
- '/root'
-
- If the user does not exist, return a default /home/[username] home::
-
- >>> get_user_home('_this_user_does_not_exist_')
- '/home/_this_user_does_not_exist_'
- """
- try:
- return pwd.getpwnam(user).pw_dir
- except KeyError:
- return os.path.join(os.path.sep, 'home', user)
-
-
-def get_user_ids(user):
- """Return the uid and gid of given `user`, e.g.::
-
- >>> get_user_ids('root')
- (0, 0)
- """
- userdata = pwd.getpwnam(user)
- return userdata.pw_uid, userdata.pw_gid
-
-
-def lxc_in_state(state, lxc_name, timeout=30):
- """Return True if the LXC named `lxc_name` is in state `state`.
-
- Return False otherwise.
- """
- while timeout:
- try:
- output = check_output(
- 'lxc-info', '-n', lxc_name, stderr=subprocess.STDOUT)
- except CommandError:
- pass
- else:
- if state in output:
- return True
- timeout -= 1
- time.sleep(1)
- return False
-
-
-lxc_running = partial(lxc_in_state, 'RUNNING')
-lxc_stopped = partial(lxc_in_state, 'STOPPED')
-
-
-def mkdirs(*args):
- """Create leaf directories (given as `args`) and all intermediate ones.
-
- >>> import tempfile
- >>> base_dir = tempfile.mktemp(suffix='/')
- >>> dir1 = tempfile.mktemp(prefix=base_dir)
- >>> dir2 = tempfile.mktemp(prefix=base_dir)
- >>> mkdirs(dir1, dir2)
- >>> os.path.isdir(dir1)
- True
- >>> os.path.isdir(dir2)
- True
-
- If the leaf directory already exists the function returns without errors::
-
- >>> mkdirs(dir1)
-
- An `OSError` is raised if the leaf path exists and it is a file::
-
- >>> f = tempfile.NamedTemporaryFile(
- ... 'w', delete=False, prefix=base_dir)
- >>> f.close()
- >>> mkdirs(f.name) # doctest: +ELLIPSIS
- Traceback (most recent call last):
- OSError: ...
- """
- for directory in args:
- try:
- os.makedirs(directory)
- except OSError as err:
- if err.errno != errno.EEXIST or os.path.isfile(directory):
- raise
-
-
-def ssh(location, user=None, caller=subprocess.call):
- """Return a callable that can be used to run ssh shell commands.
-
- The ssh `location` and, optionally, `user` must be given.
- If the user is None then the current user is used for the connection.
-
- The callable internally uses the given `caller`::
-
- >>> def caller(cmd):
- ... print cmd
- >>> sshcall = ssh('example.com', 'myuser', caller=caller)
- >>> root_sshcall = ssh('example.com', caller=caller)
- >>> sshcall('ls -l') # doctest: +ELLIPSIS
- ('ssh', '-t', ..., 'myuser@xxxxxxxxxxx', '--', 'ls -l')
- >>> root_sshcall('ls -l') # doctest: +ELLIPSIS
- ('ssh', '-t', ..., 'example.com', '--', 'ls -l')
-
- If the ssh command exits with an error code, an `SSHError` is raised::
-
- >>> ssh('loc', caller=lambda cmd: 1)('ls -l') # doctest: +ELLIPSIS
- Traceback (most recent call last):
- SSHError: ...
-
- If ignore_errors is set to True when executing the command, no error
- will be raised, even if the command itself returns an error code.
-
- >>> sshcall = ssh('loc', caller=lambda cmd: 1)
- >>> sshcall('ls -l', ignore_errors=True)
- """
- if user is not None:
- location = '{user}@{location}'.format(user=user, location=location)
-
- def _sshcall(cmd, ignore_errors=False):
- sshcmd = (
- 'ssh',
- '-t',
- '-t', # Yes, this second -t is deliberate. See `man ssh`.
- '-o', 'StrictHostKeyChecking=no',
- '-o', 'UserKnownHostsFile=/dev/null',
- location,
- '--', cmd,
- )
- if caller(sshcmd) and not ignore_errors:
- raise SSHError('Error running command: ' + ' '.join(sshcmd))
-
- return _sshcall
-
-
-@contextmanager
-def su(user):
- """A context manager to temporarily run the script as a different user."""
- uid, gid = get_user_ids(user)
- os.setegid(gid)
- os.seteuid(uid)
- home = get_user_home(user)
- with environ(HOME=home):
- try:
- yield Env(uid, gid, home)
- finally:
- os.setegid(os.getgid())
- os.seteuid(os.getuid())
-
-
-def this_command(directory, args):
- """Return a command line to re-launch this script using given `args`.
-
- The command returned will execute this script from `directory`::
-
- >>> import os
- >>> script_name = os.path.basename(__file__)
-
- >>> cmd = this_command('/home/user/lp/branches', ['install'])
- >>> cmd == ('cd /home/user/lp/branches && devel/utilities/' +
- ... script_name + ' install')
- True
-
- Arguments are correctly quoted::
-
- >>> cmd = this_command('/path', ['-arg', 'quote me'])
- >>> cmd == ('cd /path && devel/utilities/' +
- ... script_name + " -arg 'quote me'")
- True
- """
- script = join_command([
- os.path.join(LP_CHECKOUT, 'utilities', os.path.basename(__file__)),
- ] + list(args))
- return join_command(['cd', directory]) + ' && ' + script
-
-
-def user_exists(username):
- """Return True if given `username` exists, e.g.::
-
- >>> user_exists('root')
- True
- >>> user_exists('_this_user_does_not_exist_')
- False
- """
- try:
- pwd.getpwnam(username)
- except KeyError:
- return False
- return True
-
-
-def setup_codebase(user, valid_ssh_keys, checkout_dir, dependencies_dir):
- """Set up Launchpad repository and source dependencies.
-
- Return True if new changes are pulled from bzr repository.
- """
- # Using real su because bzr uses uid.
- if os.path.exists(checkout_dir):
- # Pull the repository.
- revno_args = ('bzr', 'revno', checkout_dir)
- revno = check_output(*revno_args)
- call(*get_su_command(user, ['bzr', 'pull', '-d', checkout_dir]))
- changed = revno != check_output(*revno_args)
- else:
- # Branch the repository.
- cmd = ('bzr', 'branch',
- LP_REPOS[1] if valid_ssh_keys else LP_REPOS[0], checkout_dir)
- call(*get_su_command(user, cmd))
- changed = True
- # Check repository integrity.
- if subprocess.call(['bzr', 'status', '-q', checkout_dir]):
- raise LaunchpadError(
- 'Repository {0} is corrupted.'.format(checkout_dir))
- # Set up source dependencies.
- with su(user):
- for subdir in ('eggs', 'yui', 'sourcecode'):
- mkdirs(os.path.join(dependencies_dir, subdir))
- download_cache = os.path.join(dependencies_dir, 'download-cache')
- if os.path.exists(download_cache):
- call('bzr', 'up', download_cache)
- else:
- call('bzr', 'co', '--lightweight', LP_SOURCE_DEPS, download_cache)
- return changed
-
-
-def setup_external_sourcecode(
- user, valid_ssh_keys, checkout_dir, dependencies_dir):
- """Update and link external sourcecode."""
- cmd = (
- 'utilities/update-sourcecode',
- None if valid_ssh_keys else '--use-http',
- os.path.join(dependencies_dir, 'sourcecode'),
- )
- with cd(checkout_dir):
- # Using real su because update-sourcecode uses uid.
- call(*get_su_command(user, cmd))
- with su(user):
- call('utilities/link-external-sourcecode', dependencies_dir)
-
-
-def make_launchpad(user, checkout_dir, install=False):
- """Make and optionally install Launchpad."""
- # Using real su because mailman make script uses uid.
- call(*get_su_command(user, ['make', '-C', checkout_dir]))
- if install:
- call('make', '-C', checkout_dir, 'install')
-
-
-def initialize(
- user, full_name, email, lpuser, private_key, public_key, valid_ssh_keys,
- dependencies_dir, directory):
- """Initialize host machine."""
- # Install necessary deb packages. This requires Oneiric or later.
- call('apt-get', 'update')
- apt_get_install(*BASE_PACKAGES)
- # Create the user (if he does not exist).
- if not user_exists(user):
- call('useradd', '-m', '-s', '/bin/bash', '-U', user)
- # Generate root ssh keys if they do not exist.
- if not os.path.exists('/root/.ssh/id_rsa.pub'):
- generate_ssh_keys('/root/.ssh/')
- with su(user) as env:
- # Set up the user's ssh directory. The ssh key must be associated
- # with the lpuser's Launchpad account.
- ssh_dir = os.path.join(env.home, '.ssh')
- mkdirs(ssh_dir)
- # Generate user ssh keys if none are supplied.
- if not valid_ssh_keys:
- generate_ssh_keys(ssh_dir)
- private_key = open(os.path.join(ssh_dir, 'id_rsa')).read()
- public_key = open(os.path.join(ssh_dir, 'id_rsa.pub')).read()
- priv_file = os.path.join(ssh_dir, 'id_rsa')
- pub_file = os.path.join(ssh_dir, 'id_rsa.pub')
- auth_file = os.path.join(ssh_dir, 'authorized_keys')
- known_hosts = os.path.join(ssh_dir, 'known_hosts')
- known_host_content = check_output(
- 'ssh-keyscan', '-t', 'rsa', 'bazaar.launchpad.net')
- for filename, contents, mode in [
- (priv_file, private_key, 'w'),
- (pub_file, public_key, 'w'),
- (auth_file, public_key, 'a'),
- (known_hosts, known_host_content, 'a'),
- ]:
- with open(filename, mode) as f:
- f.write(contents + '\n')
- os.chmod(filename, 0644)
- os.chmod(priv_file, 0600)
- # Set up bzr and Launchpad authentication.
- call('bzr', 'whoami', formataddr([full_name, email]))
- if valid_ssh_keys:
- subprocess.call(['bzr', 'lp-login', lpuser])
- # Set up the repository.
- mkdirs(directory)
- call('bzr', 'init-repo', '--2a', directory)
- # Set up the codebase.
- checkout_dir = os.path.join(directory, LP_CHECKOUT)
- setup_codebase(user, valid_ssh_keys, checkout_dir, dependencies_dir)
- # Set up bzr locations
- tmpl = ''.join('{0} = {1}\n'.format(k, v) for k, v in LP_BZR_LOCATIONS)
- locations = tmpl.format(checkout_dir=checkout_dir, lpuser=lpuser)
- contents = '[{0}]\n'.format(directory) + locations
- with su(user) as env:
- bzr_locations = os.path.join(env.home, '.bazaar', 'locations.conf')
- file_append(bzr_locations, contents)
-
-
-def setup_apt(no_repositories=True):
- """Setup, update and upgrade deb packages."""
- if not no_repositories:
- distro = check_output('lsb_release', '-cs').strip()
- # APT repository update.
- for reposirory in APT_REPOSITORIES:
- assume_yes = None if distro == 'lucid' else '-y'
- call('add-apt-repository', assume_yes,
- reposirory.format(distro=distro))
- # XXX frankban 2012-01-13 - Bug 892892: upgrading mountall in LXC
- # containers currently does not work.
- call("echo 'mountall hold' | dpkg --set-selections", shell=True)
- call('apt-get', 'update')
- # Install base and Launchpad deb packages.
- apt_get_install(*LP_PACKAGES)
-
-
-def setup_launchpad(user, dependencies_dir, directory, valid_ssh_keys):
- """Set up the Launchpad environment."""
- # User configuration.
- subprocess.call(['adduser', user, 'sudo'])
- gid = pwd.getpwnam(user).pw_gid
- subprocess.call(['addgroup', '--gid', str(gid), user])
- # Set up Launchpad dependencies.
- checkout_dir = os.path.join(directory, LP_CHECKOUT)
- setup_external_sourcecode(
- user, valid_ssh_keys, checkout_dir, dependencies_dir)
- with su(user):
- # Create Apache document roots, to avoid warnings.
- mkdirs(*LP_APACHE_ROOTS)
- # Set up Apache modules.
- for module in LP_APACHE_MODULES.split():
- call('a2enmod', module)
- with cd(checkout_dir):
- # Launchpad database setup.
- call('utilities/launchpad-database-setup', user)
- # Make and install launchpad.
- make_launchpad(user, checkout_dir, install=True)
- # Set up container hosts file.
- lines = ['{0}\t{1}'.format(ip, names) for ip, names in HOSTS_CONTENT]
- file_append(HOSTS_FILE, '\n'.join(lines))
-
-
-def update_launchpad(user, valid_ssh_keys, dependencies_dir, directory, apt):
- """Update the Launchpad environment."""
- if apt:
- setup_apt(no_repositories=True)
- checkout_dir = os.path.join(directory, LP_CHECKOUT)
- # Update the Launchpad codebase.
- changed = setup_codebase(
- user, valid_ssh_keys, checkout_dir, dependencies_dir)
- setup_external_sourcecode(
- user, valid_ssh_keys, checkout_dir, dependencies_dir)
- if changed:
- make_launchpad(user, checkout_dir, install=False)
-
-
-def link_sourcecode_in_branches(user, dependencies_dir, directory):
- """Link external sourcecode for all branches in the project."""
- checkout_dir = os.path.join(directory, LP_CHECKOUT)
- cmd = os.path.join(checkout_dir, 'utilities', 'link-external-sourcecode')
- with su(user):
- for dirname in os.listdir(directory):
- branch = os.path.join(directory, dirname)
- sourcecode_dir = os.path.join(branch, 'sourcecode')
- if (branch != checkout_dir and
- os.path.exists(sourcecode_dir) and
- os.path.isdir(sourcecode_dir)):
- call(cmd, '--parent', dependencies_dir, '--target', branch)
-
-
-def create_lxc(user, lxc_name, lxc_arch, lxc_os):
- """Create the LXC named `lxc_name` sharing `user` home directory.
-
- The container will be used as development environment or as base template
- for parallel testing using ephemeral instances.
- """
- # Install necessary deb packages.
- apt_get_install(*LXC_PACKAGES)
- # XXX 2012-02-02 gmb bug=925024:
- # These calls need to be removed once the lxc vs. apparmor bug
- # is resolved, since having apparmor enabled for lxc is very
- # much a Good Thing.
- # Disable the apparmor profiles for lxc so that we don't have
- # problems installing postgres.
- call('ln', '-s',
- '/etc/apparmor.d/usr.bin.lxc-start', '/etc/apparmor.d/disable/')
- call('apparmor_parser', '-R', '/etc/apparmor.d/usr.bin.lxc-start')
- # Update resolv file in order to get the ability to ssh into the LXC
- # container using its name.
- lxc_gateway_name, lxc_gateway_address = get_lxc_gateway()
- file_prepend(RESOLV_FILE, 'nameserver {0}\n'.format(lxc_gateway_address))
- file_append(
- DHCP_FILE,
- 'prepend domain-name-servers {0};\n'.format(lxc_gateway_address))
- # Container configuration template.
- content = LXC_OPTIONS.format(interface=lxc_gateway_name)
- with open(LXC_CONFIG_TEMPLATE, 'w') as f:
- f.write(content)
- # Creating container.
- call(
- 'lxc-create',
- '-t', 'ubuntu',
- '-n', lxc_name,
- '-f', LXC_CONFIG_TEMPLATE,
- '--',
- '-r {os} -a {arch} -b {user}'.format(
- os=lxc_os, arch=lxc_arch, user=user),
- )
- # Set up root ssh key.
- user_authorized_keys = os.path.expanduser(
- '~' + user + '/.ssh/authorized_keys')
- with open(user_authorized_keys, 'a') as f:
- f.write(open('/root/.ssh/id_rsa.pub').read())
- dst = get_container_path(lxc_name, '/root/.ssh/')
- mkdirs(dst)
- shutil.copy(user_authorized_keys, dst)
-
-
-def initialize_lxc(lxc_name, lxc_os):
- """Initialize LXC container."""
- base_packages = list(BASE_PACKAGES)
- if lxc_os == 'lucid':
- # Install argparse to be able to run this script inside a lucid lxc.
- base_packages.append('python-argparse')
- ssh(lxc_name)(
- 'DEBIAN_FRONTEND=noninteractive '
- 'apt-get install -y ' + ' '.join(base_packages))
-
-
-def setup_launchpad_lxc(
- user, dependencies_dir, directory, valid_ssh_keys, lxc_name):
- """Set up the Launchpad environment inside an LXC."""
- # Use ssh to call this script from inside the container.
- args = [
- 'install', '-u', user, '-a', 'setup_apt', 'setup_launchpad',
- '-d', dependencies_dir, '-c', directory
- ]
- cmd = this_command(directory, args)
- ssh(lxc_name)(cmd)
-
-
-def start_lxc(lxc_name):
- """Start the lxc instance named `lxc_name`."""
- call('lxc-start', '-n', lxc_name, '-d')
-
-
-def stop_lxc(lxc_name):
- """Stop the lxc instance named `lxc_name`."""
- ssh(lxc_name)('poweroff')
- if not lxc_stopped(lxc_name):
- subprocess.call(['lxc-stop', '-n', lxc_name])
-
-
-def wait_for_lxc(lxc_name, trials=60, sleep_seconds=1):
- """Try to ssh as `user` into the LXC container named `lxc_name`."""
- sshcall = ssh(lxc_name)
- while True:
- trials -= 1
- try:
- sshcall('true')
- except SSHError:
- if not trials:
- raise
- time.sleep(sleep_seconds)
- else:
- break
-
-
-def handle_user(namespace):
- """Handle user argument.
-
- This validator populates namespace with `home_dir` name::
-
- >>> import getpass
- >>> username = getpass.getuser()
-
- >>> namespace = argparse.Namespace(user=username)
-
- >>> handle_user(namespace)
- >>> namespace.home_dir == '/home/' + username
- True
-
- The validation fails if the current user is root and no user is provided::
-
- >>> namespace = argparse.Namespace(user=None, euid=0)
- >>> handle_user(namespace) # doctest: +ELLIPSIS
- Traceback (most recent call last):
- ValidationError: argument user ...
- """
- if namespace.user is None:
- if not namespace.euid:
- raise ValidationError('argument user can not be omitted if '
- 'the script is run as root.')
- namespace.user = pwd.getpwuid(namespace.euid).pw_name
- namespace.home_dir = get_user_home(namespace.user)
-
-
-def handle_lpuser(namespace):
- """Handle lpuser argument.
-
- If lpuser is not provided by namespace, the user name is used::
-
- >>> import getpass
- >>> username = getpass.getuser()
-
- >>> namespace = argparse.Namespace(user=username, lpuser=None)
- >>> handle_lpuser(namespace)
- >>> namespace.lpuser == username
- True
- """
- if namespace.lpuser is None:
- namespace.lpuser = namespace.user
-
-
-def handle_userdata(namespace, whois=bzr_whois):
- """Handle full_name and email arguments.
-
- If they are not provided, this function tries to obtain them using
- the given `whois` callable::
-
- >>> namespace = argparse.Namespace(
- ... full_name=None, email=None, user='root')
- >>> email = 'email@xxxxxxxxxxx'
- >>> handle_userdata(namespace, lambda user: (user, email))
- >>> namespace.full_name == namespace.user
- True
- >>> namespace.email == email
- True
-
- The validation fails if full_name or email are not provided and
- they can not be obtained using the `whois` callable::
-
- >>> namespace = argparse.Namespace(
- ... full_name=None, email=None, user='root')
- >>> handle_userdata(namespace, lambda user: None) # doctest: +ELLIPSIS
- Traceback (most recent call last):
- ValidationError: arguments full-name ...
-
- >>> namespace = argparse.Namespace(
- ... full_name=None, email=None, user='this_user_does_not_exist')
- >>> handle_userdata(namespace) # doctest: +ELLIPSIS
- Traceback (most recent call last):
- ValidationError: arguments full-name ...
-
- It does not make sense to provide only one argument::
-
- >>> namespace = argparse.Namespace(full_name='Foo Bar', email=None)
- >>> handle_userdata(namespace) # doctest: +ELLIPSIS
- Traceback (most recent call last):
- ValidationError: arguments full-name ...
- """
- args = (namespace.full_name, namespace.email)
- if not all(args):
- if any(args):
- raise ValidationError(
- 'arguments full-name and email: '
- 'either none or both must be provided.')
- if user_exists(namespace.user):
- userdata = whois(namespace.user)
- if userdata is None:
- raise ValidationError(
- 'arguments full-name and email are required: '
- 'bzr user id not found.')
- namespace.full_name, namespace.email = userdata
- else:
- raise ValidationError(
- 'arguments full-name and email are required: '
- 'system user not found.')
-
-
-def handle_ssh_keys(namespace):
- r"""Handle private and public ssh keys.
-
- Keys contained in the namespace are escaped::
-
- >>> private = r'PRIVATE\nKEY'
- >>> public = r'PUBLIC\nKEY'
- >>> namespace = argparse.Namespace(
- ... private_key=private, public_key=public)
- >>> handle_ssh_keys(namespace)
- >>> namespace.private_key == private.decode('string-escape')
- True
- >>> namespace.public_key == public.decode('string-escape')
- True
- >>> namespace.valid_ssh_keys
- True
-
- Keys are None if they are not provided and can not be found in the
- current home directory::
-
- >>> namespace = argparse.Namespace(
- ... private_key=None, home_dir='/tmp/__does_not_exists__')
- >>> handle_ssh_keys(namespace) # doctest: +ELLIPSIS
- >>> print namespace.private_key
- None
- >>> print namespace.public_key
- None
- >>> namespace.valid_ssh_keys
- False
-
- If only one of private_key and public_key is provided, a
- ValidationError will be raised.
-
- >>> namespace = argparse.Namespace(
- ... private_key=private, public_key=None,
- ... home_dir='/tmp/__does_not_exists__')
- >>> handle_ssh_keys(namespace) # doctest: +ELLIPSIS
- Traceback (most recent call last):
- ValidationError: arguments private-key...
- """
- namespace.valid_ssh_keys = True
- for attr, filename in (
- ('private_key', 'id_rsa'),
- ('public_key', 'id_rsa.pub')):
- value = getattr(namespace, attr, None)
- if value:
- setattr(namespace, attr, value.decode('string-escape'))
- else:
- path = os.path.join(namespace.home_dir, '.ssh', filename)
- try:
- value = open(path).read()
- except IOError:
- value = None
- namespace.valid_ssh_keys = False
- setattr(namespace, attr, value)
- if bool(namespace.private_key) != bool(namespace.public_key):
- raise ValidationError(
- "arguments private-key and public-key: "
- "both must be provided or neither must be provided.")
-
-
-def handle_directories(namespace):
- """Handle checkout and dependencies directories.
-
- The ~ construction is automatically expanded::
-
- >>> namespace = argparse.Namespace(
- ... directory='~/launchpad', dependencies_dir='~/launchpad/deps',
- ... home_dir='/home/foo')
- >>> handle_directories(namespace)
- >>> namespace.directory
- '/home/foo/launchpad'
- >>> namespace.dependencies_dir
- '/home/foo/launchpad/deps'
-
- The validation fails for directories not residing inside the home::
-
- >>> namespace = argparse.Namespace(
- ... directory='/tmp/launchpad',
- ... dependencies_dir='~/launchpad/deps',
- ... home_dir='/home/foo')
- >>> handle_directories(namespace) # doctest: +ELLIPSIS
- Traceback (most recent call last):
- ValidationError: argument directory ...
-
- The validation fails if the directory contains spaces::
-
- >>> namespace = argparse.Namespace(directory='my directory')
- >>> handle_directories(namespace) # doctest: +ELLIPSIS
- Traceback (most recent call last):
- ValidationError: argument directory ...
- """
- if ' ' in namespace.directory:
- raise ValidationError('argument directory can not contain spaces.')
- for attr in ('directory', 'dependencies_dir'):
- directory = getattr(
- namespace, attr).replace('~', namespace.home_dir)
- if not directory.startswith(namespace.home_dir + os.path.sep):
- raise ValidationError(
- 'argument {0} does not reside under the home '
- 'directory of the system user.'.format(attr))
- setattr(namespace, attr, directory)
-
-
-class ArgumentParser(argparse.ArgumentParser):
- """A customized argument parser for `argparse`."""
-
- def __init__(self, *args, **kwargs):
- self.actions = []
- self.subparsers = None
- super(ArgumentParser, self).__init__(*args, **kwargs)
-
- def add_argument(self, *args, **kwargs):
- """Override to store actions in a "public" instance attribute.
-
- >>> parser = ArgumentParser()
- >>> parser.add_argument('arg1')
- >>> parser.add_argument('arg2')
- >>> [action.dest for action in parser.actions]
- ['help', 'arg1', 'arg2']
- """
- action = super(ArgumentParser, self).add_argument(*args, **kwargs)
- self.actions.append(action)
-
- def register_subcommand(self, name, subcommand_class, handler=None):
- """Add a subcommand to this parser.
-
- A sub command is registered giving the sub command `name` and class::
-
- >>> parser = ArgumentParser()
-
- >>> class SubCommand(BaseSubCommand):
- ... def handle(self, namespace):
- ... return self.name
-
- >>> parser = ArgumentParser()
- >>> _ = parser.register_subcommand('foo', SubCommand)
-
- The `main` method of the subcommand class is added to namespace, and
- can be used to actually handle the sub command execution.
-
- >>> namespace = parser.parse_args(['foo'])
- >>> namespace.main(namespace)
- 'foo'
-
- A `handler` callable can also be provided to handle the subcommand
- execution::
-
- >>> handler = lambda namespace: 'custom handler'
-
- >>> parser = ArgumentParser()
- >>> _ = parser.register_subcommand(
- ... 'bar', SubCommand, handler=handler)
-
- >>> namespace = parser.parse_args(['bar'])
- >>> namespace.main(namespace)
- 'custom handler'
- """
- if self.subparsers is None:
- self.subparsers = self.add_subparsers(
- title='subcommands',
- help='Each subcommand accepts --h or --help to describe it.')
- subcommand = subcommand_class(name, handler=handler)
- parser = self.subparsers.add_parser(
- subcommand.name, help=subcommand.help)
- subcommand.add_arguments(parser)
- parser.set_defaults(main=subcommand.main, get_parser=lambda: parser)
- return subcommand
-
- def get_args_from_namespace(self, namespace):
- """Return a list of arguments taking values from `namespace`.
-
- Having a parser defined as usual::
-
- >>> parser = ArgumentParser()
- >>> _ = parser.add_argument('--foo')
- >>> _ = parser.add_argument('bar')
- >>> namespace = parser.parse_args('--foo eggs spam'.split())
-
- It is possible to recreate the argument list taking values from
- a different namespace::
-
- >>> namespace.foo = 'changed'
- >>> parser.get_args_from_namespace(namespace)
- ['--foo', 'changed', 'spam']
- """
- args = []
- for action in self.actions:
- dest = action.dest
- option_strings = action.option_strings
- value = getattr(namespace, dest, None)
- if value:
- if option_strings:
- args.append(option_strings[0])
- if isinstance(value, list):
- args.extend(value)
- elif not isinstance(value, bool):
- args.append(value)
- return args
-
- def _handle_help(self, namespace):
- """Help sub command handler."""
- command = namespace.command
- help = self.prefix_chars + 'h'
- args = [help] if command is None else [command, help]
- self.parse_args(args)
-
- def _add_help_subcommand(self):
- """Add an help sub command to this parser."""
- name = 'help'
- choices = self.subparsers.choices.keys()
- if name not in choices:
- choices.append(name)
- parser = self.subparsers.add_parser(
- name, help='More help on a command.')
- parser.add_argument('command', nargs='?', choices=choices)
- parser.set_defaults(main=self._handle_help)
-
- def parse_args(self, *args, **kwargs):
- """Override to add an help sub command.
-
- The help sub command is added only if other sub commands exist::
-
- >>> stderr, sys.stderr = sys.stderr, sys.stdout
- >>> parser = ArgumentParser()
- >>> parser.parse_args(['help'])
- Traceback (most recent call last):
- SystemExit: 2
- >>> sys.stderr = stderr
-
- >>> class SubCommand(BaseSubCommand): pass
- >>> _ = parser.register_subcommand('command', SubCommand)
- >>> namespace = parser.parse_args(['help'])
- >>> namespace.main(namespace)
- Traceback (most recent call last):
- SystemExit: 0
- """
- if self.subparsers is not None:
- self._add_help_subcommand()
- return super(ArgumentParser, self).parse_args(*args, **kwargs)
-
-
-class BaseSubCommand(object):
- """Base class defining a sub command.
-
- Objects of this class can be used to easily add sub commands to this
- script as plugins, providing arguments, validating them, restarting
- as root if needed.
-
- Override `add_arguments()` to add arguments, `validators` to add
- namespace validators, and `handle()` to manage sub command execution::
-
- >>> def validator(namespace):
- ... namespace.bar = True
-
- >>> class SubCommand(BaseSubCommand):
- ... help = 'Sub command example.'
- ... validators = (validator,)
- ...
- ... def add_arguments(self, parser):
- ... super(SubCommand, self).add_arguments(parser)
- ... parser.add_argument('--foo')
- ...
- ... def handle(self, namespace):
- ... return namespace
-
- Register the sub command using `ArgumentParser.register_subcommand`::
-
- >>> parser = ArgumentParser()
- >>> sub_command = parser.register_subcommand('spam', SubCommand)
-
- Now the subcommand has a name::
-
- >>> sub_command.name
- 'spam'
-
- The sub command handler can be called using `namespace.main()`::
-
- >>> namespace = parser.parse_args('spam --foo eggs'.split())
- >>> namespace = namespace.main(namespace)
- >>> namespace.foo
- 'eggs'
- >>> namespace.bar
- True
-
- The help attribute of sub command instances is used to generate
- the command usage message::
- >>> help = parser.format_help()
- >>> 'spam' in help
- True
- >>> 'Sub command example.' in help
- True
- """
-
- help = ''
- needs_root = False
- validators = ()
-
- def __init__(self, name, handler=None):
- self.name = name
- self.handler = handler or self.handle
-
- def __repr__(self):
- return '<{klass}: {name}>'.format(
- klass=self.__class__.__name__, name=self.name)
-
- def init_namespace(self, namespace):
- """Add `run_as_root` and `euid` names to the given `namespace`."""
- euid = os.geteuid()
- namespace.euid, namespace.run_as_root = euid, not euid
-
- def get_needs_root(self, namespace):
- """Return True if root is needed to run this subcommand.
-
- Subclasses can override this to dynamically change this value
- based on the given `namespace`.
- """
- return self.needs_root
-
- def get_validators(self, namespace):
- """Return an iterable of namespace validators for this sub command.
-
- Subclasses can override this to dynamically change validators
- based on the given `namespace`.
- """
- return self.validators
-
- def validate(self, parser, namespace):
- """Validate the current namespace.
-
- The method `self.get_validators` can contain an iterable of objects
- that are called once the arguments namespace is fully populated.
- This allows cleaning and validating arguments that depend on
- each other, or on the current environment.
-
- Each validator is a callable object, takes the current namespace
- and can raise ValidationError if the arguments are not valid::
-
- >>> import sys
- >>> stderr, sys.stderr = sys.stderr, sys.stdout
- >>> def validator(namespace):
- ... raise ValidationError('nothing is going on')
- >>> sub_command = BaseSubCommand('foo')
- >>> sub_command.validators = [validator]
- >>> sub_command.validate(ArgumentParser(), argparse.Namespace())
- Traceback (most recent call last):
- SystemExit: 2
- >>> sys.stderr = stderr
- """
- for validator in self.get_validators(namespace):
- try:
- validator(namespace)
- except ValidationError as err:
- parser.error(err)
-
- def restart_as_root(self, parser, namespace):
- """Restart this script using *sudo*.
-
- The arguments are recreated using the given `namespace`.
- """
- args = parser.get_args_from_namespace(namespace)
- return subprocess.call(['sudo', sys.argv[0], self.name] + args)
-
- def main(self, namespace):
- """Entry point for subcommand execution.
-
- This method takes care of:
-
- - current argparse subparser retrieval
- - namespace initialization
- - namespace validation
- - script restart as root (if this sub command needs to be run as root)
-
- If everything is ok the sub command handler is called passing
- the validated namespace.
- """
- parser = namespace.get_parser()
- self.init_namespace(namespace)
- self.validate(parser, namespace)
- if self.get_needs_root(namespace) and not namespace.run_as_root:
- return self.restart_as_root(parser, namespace)
- return self.handler(namespace)
-
- def handle(self, namespace):
- """Default sub command handler.
-
- Subclasses must either implement this method or provide another
- callable handler during sub command registration.
- """
- raise NotImplementedError
-
- def add_arguments(self, parser):
- """Here subclasses can add arguments to the subparser."""
- pass
-
-
-class ActionsBasedSubCommand(BaseSubCommand):
- """A sub command that uses "actions" to handle its execution.
-
- Actions are callables stored in the `actions` attribute, together
- with the arguments they expect. Those arguments are strings
- representing attributes of the argparse namespace::
-
- >>> trace = []
-
- >>> def action1(foo):
- ... trace.append('action1 received ' + foo)
-
- >>> def action2(foo, bar):
- ... trace.append('action2 received {0} and {1}'.format(foo, bar))
-
- >>> class SubCommand(ActionsBasedSubCommand):
- ... actions = (
- ... (action1, 'foo'),
- ... (action2, 'foo', 'bar'),
- ... )
- ...
- ... def add_arguments(self, parser):
- ... super(SubCommand, self).add_arguments(parser)
- ... parser.add_argument('--foo')
- ... parser.add_argument('--bar')
-
- This class implements an handler method that executes actions in the
- order they are provided::
-
- >>> parser = ArgumentParser()
- >>> _ = parser.register_subcommand('sub', SubCommand)
- >>> namespace = parser.parse_args('sub --foo eggs --bar spam'.split())
- >>> namespace.main(namespace)
- >>> trace
- ['action1 received eggs', 'action2 received eggs and spam']
-
- A special argument `-a` or `--actions` is automatically added to the
- parser. It can be used to execute only one or a subset of actions::
-
- >>> trace = []
-
- >>> namespace = parser.parse_args('sub --foo eggs -a action1'.split())
- >>> namespace.main(namespace)
- >>> trace
- ['action1 received eggs']
-
- A special argument `--skip-actions` is automatically added to the
- parser. It can be used to skip one or more actions::
-
- >>> trace = []
-
- >>> namespace = parser.parse_args(
- ... 'sub --foo eggs --skip-actions action1'.split())
- >>> namespace.main(namespace)
- >>> trace
- ['action2 received eggs and None']
-
- The actions execution is stopped if an action raises `LaunchpadError`.
- In that case, the error is returned by the handler.
-
- >>> trace = []
-
- >>> def erroneous_action(foo):
- ... raise LaunchpadError('error')
-
- >>> class SubCommandWithErrors(SubCommand):
- ... actions = (
- ... (action1, 'foo'),
- ... (erroneous_action, 'foo'),
- ... (action2, 'foo', 'bar'),
- ... )
-
- >>> parser = ArgumentParser()
- >>> _ = parser.register_subcommand('sub', SubCommandWithErrors)
- >>> namespace = parser.parse_args('sub --foo eggs'.split())
- >>> error = namespace.main(namespace)
- >>> error.message
- 'error'
-
- The action `action2` is not executed::
-
- >>> trace
- ['action1 received eggs']
- """
-
- actions = ()
-
- def __init__(self, *args, **kwargs):
- super(ActionsBasedSubCommand, self).__init__(*args, **kwargs)
- self._action_names = []
- self._actions = {}
- for action_args in self.actions:
- action, args = action_args[0], action_args[1:]
- action_name = self._get_action_name(action)
- self._action_names.append(action_name)
- self._actions[action_name] = (action, args)
-
- def _get_action_name(self, action):
- """Return the string representation of an action callable.
-
- The name is retrieved using attributes lookup for `action_name`
- and then `__name__`::
-
- >>> def action1():
- ... pass
- >>> action1.action_name = 'myaction'
-
- >>> def action2():
- ... pass
-
- >>> sub_command = ActionsBasedSubCommand('foo')
- >>> sub_command._get_action_name(action1)
- 'myaction'
- >>> sub_command._get_action_name(action2)
- 'action2'
- """
- try:
- return action.action_name
- except AttributeError:
- return action.__name__
-
- def add_arguments(self, parser):
- super(ActionsBasedSubCommand, self).add_arguments(parser)
- parser.add_argument(
- '-a', '--actions', nargs='+', choices=self._action_names,
- help='Call one or more internal functions.')
- parser.add_argument(
- '--skip-actions', nargs='+', choices=self._action_names,
- help='Skip one or more internal functions.')
-
- def handle(self, namespace):
- skip_actions = namespace.skip_actions or []
- action_names = filter(
- lambda action_name: action_name not in skip_actions,
- namespace.actions or self._action_names)
- for action_name in action_names:
- action, arg_names = self._actions[action_name]
- args = [getattr(namespace, i) for i in arg_names]
- try:
- action(*args)
- except LaunchpadError as err:
- return err
-
-
-class InstallSubCommand(ActionsBasedSubCommand):
- """Install the Launchpad environment."""
-
- actions = (
- (initialize,
- 'user', 'full_name', 'email', 'lpuser', 'private_key',
- 'public_key', 'valid_ssh_keys', 'dependencies_dir', 'directory'),
- (setup_apt, 'no_repositories'),
- (setup_launchpad,
- 'user', 'dependencies_dir', 'directory', 'valid_ssh_keys'),
- )
- help = __doc__
- needs_root = True
- validators = (
- handle_user,
- handle_lpuser,
- handle_userdata,
- handle_ssh_keys,
- handle_directories,
- )
-
- def add_arguments(self, parser):
- super(InstallSubCommand, self).add_arguments(parser)
- parser.add_argument(
- '-u', '--user',
- help='The name of the system user to be created or updated. '
- 'The current user is used if this script is not run as '
- 'root and this argument is omitted.')
- parser.add_argument(
- '-e', '--email',
- help='The email of the user, used for bzr whoami. This argument '
- 'can be omitted if a bzr id exists for current user.')
- parser.add_argument(
- '-f', '--full-name',
- help='The full name of the user, used for bzr whoami. '
- 'This argument can be omitted if a bzr id exists for '
- 'current user.')
- parser.add_argument(
- '-l', '--lpuser',
- help='The name of the Launchpad user that will be used to '
- 'check out dependencies. If not provided, the system '
- 'user name is used.')
- parser.add_argument(
- '-v', '--private-key',
- help='The SSH private key for the Launchpad user (without '
- 'passphrase). If this argument is omitted and a keypair is '
- 'not found in the home directory of the system user a new '
- 'SSH keypair will be generated and the checkout of the '
- 'Launchpad code will use HTTP rather than bzr+ssh.')
- parser.add_argument(
- '-b', '--public-key',
- help='The SSH public key for the Launchpad user. '
- 'If this argument is omitted and a keypair is not found '
- 'in the home directory of the system user a new SSH '
- 'keypair will be generated and the checkout of the '
- 'Launchpad code will use HTTP rather than bzr+ssh.')
- parser.add_argument(
- '-d', '--dependencies-dir', default=DEPENDENCIES_DIR,
- help='The directory of the Launchpad dependencies to be created. '
- 'The directory must reside under the home directory of the '
- 'given user (see -u argument). '
- '[DEFAULT={0}]'.format(DEPENDENCIES_DIR))
- parser.add_argument(
- '-c', '--directory', default=CHECKOUT_DIR,
- help='The directory of the Launchpad repository to be created. '
- 'The directory must reside under the home directory of the '
- 'given user (see -u argument). '
- '[DEFAULT={0}]'.format(CHECKOUT_DIR))
- parser.add_argument(
- '-N', '--no-repositories', action='store_true',
- help='Do not add APT repositories.')
-
-
-class UpdateSubCommand(ActionsBasedSubCommand):
- """Update the Launchpad environment to latest version."""
-
- actions = (
- (update_launchpad,
- 'user', 'valid_ssh_keys', 'dependencies_dir', 'directory', 'apt'),
- (link_sourcecode_in_branches,
- 'user', 'dependencies_dir', 'directory'),
- )
- help = __doc__
- validators = (
- handle_user,
- handle_ssh_keys,
- handle_directories,
- )
-
- def get_needs_root(self, namespace):
- # Root is needed only if an apt update/upgrade is requested.
- return namespace.apt
-
- def add_arguments(self, parser):
- super(UpdateSubCommand, self).add_arguments(parser)
- parser.add_argument(
- '-u', '--user',
- help='The name of the system user used to update Launchpad. '
- 'The current user is used if this script is not run as '
- 'root and this argument is omitted.')
- parser.add_argument(
- '-d', '--dependencies-dir', default=DEPENDENCIES_DIR,
- help='The directory of the Launchpad dependencies to be updated. '
- 'The directory must reside under the home directory of the '
- 'given user (see -u argument). '
- '[DEFAULT={0}]'.format(DEPENDENCIES_DIR))
- parser.add_argument(
- '-c', '--directory', default=CHECKOUT_DIR,
- help='The directory of the Launchpad repository to be updated. '
- 'The directory must reside under the home directory of the '
- 'given user (see -u argument). '
- '[DEFAULT={0}]'.format(CHECKOUT_DIR))
- parser.add_argument(
- '-D', '--apt', action='store_true',
- help='Also update deb packages.')
-
-
-class LXCInstallSubCommand(InstallSubCommand):
- """Install the Launchpad environment inside an LXC."""
-
- actions = (
- (initialize,
- 'user', 'full_name', 'email', 'lpuser', 'private_key',
- 'public_key', 'valid_ssh_keys', 'dependencies_dir', 'directory'),
- (create_lxc,
- 'user', 'lxc_name', 'lxc_arch', 'lxc_os'),
- (start_lxc, 'lxc_name'),
- (wait_for_lxc, 'lxc_name'),
- (initialize_lxc,
- 'lxc_name', 'lxc_os'),
- (setup_launchpad_lxc,
- 'user', 'dependencies_dir', 'directory', 'valid_ssh_keys',
- 'lxc_name'),
- (stop_lxc, 'lxc_name'),
- )
- help = __doc__
-
- def add_arguments(self, parser):
- super(LXCInstallSubCommand, self).add_arguments(parser)
- parser.add_argument(
- '-n', '--lxc-name', default=LXC_NAME,
- help='The LXC container name to setup. '
- '[DEFAULT={0}]'.format(LXC_NAME))
- parser.add_argument(
- '-A', '--lxc-arch', default=LXC_GUEST_ARCH,
- help='The LXC container architecture. '
- '[DEFAULT={0}]'.format(LXC_GUEST_ARCH))
- parser.add_argument(
- '-r', '--lxc-os', default=LXC_GUEST_OS,
- choices=LXC_GUEST_CHOICES,
- help='The LXC container distro codename. '
- '[DEFAULT={0}]'.format(LXC_GUEST_OS))
-
-
-def main():
- parser = ArgumentParser(description=__doc__)
- parser.register_subcommand('install', InstallSubCommand)
- parser.register_subcommand('update', UpdateSubCommand)
- parser.register_subcommand('lxc-install', LXCInstallSubCommand)
- args = parser.parse_args()
- return args.main(args)
-
-
-if __name__ == '__main__':
- sys.exit(main())
=== added file 'lpsetup/__init__.py'
--- lpsetup/__init__.py 1970-01-01 00:00:00 +0000
+++ lpsetup/__init__.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Create and update Launchpad development and testing environments."""
+
+__metaclass__ = type
+__all__ = [
+ 'get_version',
+ ]
+
+VERSION = (0, 1, 0)
+
+
+def get_version():
+ return '.'.join(map(str, VERSION))
=== added file 'lpsetup/argparser.py'
--- lpsetup/argparser.py 1970-01-01 00:00:00 +0000
+++ lpsetup/argparser.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,456 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Argparse wrapper to add support for sub commands."""
+
+__metaclass__ = type
+__all__ = [
+ 'ActionsBasedSubCommand',
+ 'ArgumentParser',
+ 'BaseSubCommand',
+ ]
+
+import argparse
+import os
+import subprocess
+import sys
+
+from lpsetup.exceptions import ValidationError
+
+
+class ArgumentParser(argparse.ArgumentParser):
+ """A customized argument parser for `argparse`."""
+
+ def __init__(self, *args, **kwargs):
+ self.actions = []
+ self.subparsers = None
+ super(ArgumentParser, self).__init__(*args, **kwargs)
+
+ def add_argument(self, *args, **kwargs):
+ """Override to store actions in a "public" instance attribute.
+
+ >>> parser = ArgumentParser()
+ >>> parser.add_argument('arg1')
+ >>> parser.add_argument('arg2')
+ >>> [action.dest for action in parser.actions]
+ ['help', 'arg1', 'arg2']
+ """
+ action = super(ArgumentParser, self).add_argument(*args, **kwargs)
+ self.actions.append(action)
+
+ def register_subcommand(self, name, subcommand_class, handler=None):
+ """Add a subcommand to this parser.
+
+ A sub command is registered giving the sub command `name` and class::
+
+ >>> parser = ArgumentParser()
+
+ >>> class SubCommand(BaseSubCommand):
+ ... def handle(self, namespace):
+ ... return self.name
+
+ >>> parser = ArgumentParser()
+ >>> _ = parser.register_subcommand('foo', SubCommand)
+
+ The `main` method of the subcommand class is added to namespace, and
+ can be used to actually handle the sub command execution.
+
+ >>> namespace = parser.parse_args(['foo'])
+ >>> namespace.main(namespace)
+ 'foo'
+
+ A `handler` callable can also be provided to handle the subcommand
+ execution::
+
+ >>> handler = lambda namespace: 'custom handler'
+
+ >>> parser = ArgumentParser()
+ >>> _ = parser.register_subcommand(
+ ... 'bar', SubCommand, handler=handler)
+
+ >>> namespace = parser.parse_args(['bar'])
+ >>> namespace.main(namespace)
+ 'custom handler'
+ """
+ if self.subparsers is None:
+ self.subparsers = self.add_subparsers(
+ title='subcommands',
+ help='Each subcommand accepts --h or --help to describe it.')
+ subcommand = subcommand_class(name, handler=handler)
+ parser = self.subparsers.add_parser(
+ subcommand.name, help=subcommand.help)
+ subcommand.add_arguments(parser)
+ parser.set_defaults(main=subcommand.main, get_parser=lambda: parser)
+ return subcommand
+
+ def get_args_from_namespace(self, namespace):
+ """Return a list of arguments taking values from `namespace`.
+
+ Having a parser defined as usual::
+
+ >>> parser = ArgumentParser()
+ >>> _ = parser.add_argument('--foo')
+ >>> _ = parser.add_argument('bar')
+ >>> namespace = parser.parse_args('--foo eggs spam'.split())
+
+ It is possible to recreate the argument list taking values from
+ a different namespace::
+
+ >>> namespace.foo = 'changed'
+ >>> parser.get_args_from_namespace(namespace)
+ ['--foo', 'changed', 'spam']
+ """
+ args = []
+ for action in self.actions:
+ dest = action.dest
+ option_strings = action.option_strings
+ value = getattr(namespace, dest, None)
+ if value:
+ if option_strings:
+ args.append(option_strings[0])
+ if isinstance(value, list):
+ args.extend(value)
+ elif not isinstance(value, bool):
+ args.append(value)
+ return args
+
+ def _handle_help(self, namespace):
+ """Help sub command handler."""
+ command = namespace.command
+ help = self.prefix_chars + 'h'
+ args = [help] if command is None else [command, help]
+ self.parse_args(args)
+
+ def _add_help_subcommand(self):
+ """Add an help sub command to this parser."""
+ name = 'help'
+ choices = self.subparsers.choices.keys()
+ if name not in choices:
+ choices.append(name)
+ parser = self.subparsers.add_parser(
+ name, help='More help on a command.')
+ parser.add_argument('command', nargs='?', choices=choices)
+ parser.set_defaults(main=self._handle_help)
+
+ def parse_args(self, *args, **kwargs):
+ """Override to add an help sub command.
+
+ The help sub command is added only if other sub commands exist::
+
+ >>> stderr, sys.stderr = sys.stderr, sys.stdout
+ >>> parser = ArgumentParser()
+ >>> parser.parse_args(['help'])
+ Traceback (most recent call last):
+ SystemExit: 2
+ >>> sys.stderr = stderr
+
+ >>> class SubCommand(BaseSubCommand): pass
+ >>> _ = parser.register_subcommand('command', SubCommand)
+ >>> namespace = parser.parse_args(['help'])
+ >>> namespace.main(namespace)
+ Traceback (most recent call last):
+ SystemExit: 0
+ """
+ if self.subparsers is not None:
+ self._add_help_subcommand()
+ return super(ArgumentParser, self).parse_args(*args, **kwargs)
+
+
+class BaseSubCommand(object):
+ """Base class defining a sub command.
+
+ Objects of this class can be used to easily add sub commands to this
+ script as plugins, providing arguments, validating them, restarting
+ as root if needed.
+
+ Override `add_arguments()` to add arguments, `validators` to add
+ namespace validators, and `handle()` to manage sub command execution::
+
+ >>> def validator(namespace):
+ ... namespace.bar = True
+
+ >>> class SubCommand(BaseSubCommand):
+ ... help = 'Sub command example.'
+ ... validators = (validator,)
+ ...
+ ... def add_arguments(self, parser):
+ ... super(SubCommand, self).add_arguments(parser)
+ ... parser.add_argument('--foo')
+ ...
+ ... def handle(self, namespace):
+ ... return namespace
+
+ Register the sub command using `ArgumentParser.register_subcommand`::
+
+ >>> parser = ArgumentParser()
+ >>> sub_command = parser.register_subcommand('spam', SubCommand)
+
+ Now the subcommand has a name::
+
+ >>> sub_command.name
+ 'spam'
+
+ The sub command handler can be called using `namespace.main()`::
+
+ >>> namespace = parser.parse_args('spam --foo eggs'.split())
+ >>> namespace = namespace.main(namespace)
+ >>> namespace.foo
+ 'eggs'
+ >>> namespace.bar
+ True
+
+ The help attribute of sub command instances is used to generate
+ the command usage message::
+ >>> help = parser.format_help()
+ >>> 'spam' in help
+ True
+ >>> 'Sub command example.' in help
+ True
+ """
+
+ help = ''
+ needs_root = False
+ validators = ()
+
+ def __init__(self, name, handler=None):
+ self.name = name
+ self.handler = handler or self.handle
+
+ def __repr__(self):
+ return '<{klass}: {name}>'.format(
+ klass=self.__class__.__name__, name=self.name)
+
+ def init_namespace(self, namespace):
+ """Add `run_as_root` and `euid` names to the given `namespace`."""
+ euid = os.geteuid()
+ namespace.euid, namespace.run_as_root = euid, not euid
+
+ def get_needs_root(self, namespace):
+ """Return True if root is needed to run this subcommand.
+
+ Subclasses can override this to dynamically change this value
+ based on the given `namespace`.
+ """
+ return self.needs_root
+
+ def get_validators(self, namespace):
+ """Return an iterable of namespace validators for this sub command.
+
+ Subclasses can override this to dynamically change validators
+ based on the given `namespace`.
+ """
+ return self.validators
+
+ def validate(self, parser, namespace):
+ """Validate the current namespace.
+
+ The method `self.get_validators` can contain an iterable of objects
+ that are called once the arguments namespace is fully populated.
+ This allows cleaning and validating arguments that depend on
+ each other, or on the current environment.
+
+ Each validator is a callable object, takes the current namespace
+ and can raise ValidationError if the arguments are not valid::
+
+ >>> import sys
+ >>> stderr, sys.stderr = sys.stderr, sys.stdout
+ >>> def validator(namespace):
+ ... raise ValidationError('nothing is going on')
+ >>> sub_command = BaseSubCommand('foo')
+ >>> sub_command.validators = [validator]
+ >>> sub_command.validate(ArgumentParser(), argparse.Namespace())
+ Traceback (most recent call last):
+ SystemExit: 2
+ >>> sys.stderr = stderr
+ """
+ for validator in self.get_validators(namespace):
+ try:
+ validator(namespace)
+ except ValidationError as err:
+ parser.error(err)
+
+ def restart_as_root(self, parser, namespace):
+ """Restart this script using *sudo*.
+
+ The arguments are recreated using the given `namespace`.
+ """
+ args = parser.get_args_from_namespace(namespace)
+ return subprocess.call(['sudo', sys.argv[0], self.name] + args)
+
+ def main(self, namespace):
+ """Entry point for subcommand execution.
+
+ This method takes care of:
+
+ - current argparse subparser retrieval
+ - namespace initialization
+ - namespace validation
+ - script restart as root (if this sub command needs to be run as root)
+
+ If everything is ok the sub command handler is called passing
+ the validated namespace.
+ """
+ parser = namespace.get_parser()
+ self.init_namespace(namespace)
+ self.validate(parser, namespace)
+ if self.get_needs_root(namespace) and not namespace.run_as_root:
+ return self.restart_as_root(parser, namespace)
+ return self.handler(namespace)
+
+ def handle(self, namespace):
+ """Default sub command handler.
+
+ Subclasses must either implement this method or provide another
+ callable handler during sub command registration.
+ """
+ raise NotImplementedError
+
+ def add_arguments(self, parser):
+ """Here subclasses can add arguments to the subparser."""
+ pass
+
+
+class ActionsBasedSubCommand(BaseSubCommand):
+ """A sub command that uses "actions" to handle its execution.
+
+ Actions are callables stored in the `actions` attribute, together
+ with the arguments they expect. Those arguments are strings
+ representing attributes of the argparse namespace::
+
+ >>> trace = []
+
+ >>> def action1(foo):
+ ... trace.append('action1 received ' + foo)
+
+ >>> def action2(foo, bar):
+ ... trace.append('action2 received {0} and {1}'.format(foo, bar))
+
+ >>> class SubCommand(ActionsBasedSubCommand):
+ ... actions = (
+ ... (action1, 'foo'),
+ ... (action2, 'foo', 'bar'),
+ ... )
+ ...
+ ... def add_arguments(self, parser):
+ ... super(SubCommand, self).add_arguments(parser)
+ ... parser.add_argument('--foo')
+ ... parser.add_argument('--bar')
+
+ This class implements an handler method that executes actions in the
+ order they are provided::
+
+ >>> parser = ArgumentParser()
+ >>> _ = parser.register_subcommand('sub', SubCommand)
+ >>> namespace = parser.parse_args('sub --foo eggs --bar spam'.split())
+ >>> namespace.main(namespace)
+ >>> trace
+ ['action1 received eggs', 'action2 received eggs and spam']
+
+ A special argument `-a` or `--actions` is automatically added to the
+ parser. It can be used to execute only one or a subset of actions::
+
+ >>> trace = []
+
+ >>> namespace = parser.parse_args('sub --foo eggs -a action1'.split())
+ >>> namespace.main(namespace)
+ >>> trace
+ ['action1 received eggs']
+
+ A special argument `--skip-actions` is automatically added to the
+ parser. It can be used to skip one or more actions::
+
+ >>> trace = []
+
+ >>> namespace = parser.parse_args(
+ ... 'sub --foo eggs --skip-actions action1'.split())
+ >>> namespace.main(namespace)
+ >>> trace
+ ['action2 received eggs and None']
+
+ The actions execution is stopped if an action raises
+ `subprocess.CalledProcessError`.
+ In that case, the error is returned by the handler.
+
+ >>> trace = []
+
+ >>> def erroneous_action(foo):
+ ... raise subprocess.CalledProcessError(1, 'command')
+
+ >>> class SubCommandWithErrors(SubCommand):
+ ... actions = (
+ ... (action1, 'foo'),
+ ... (erroneous_action, 'foo'),
+ ... (action2, 'foo', 'bar'),
+ ... )
+
+ >>> parser = ArgumentParser()
+ >>> _ = parser.register_subcommand('sub', SubCommandWithErrors)
+ >>> namespace = parser.parse_args('sub --foo eggs'.split())
+ >>> error = namespace.main(namespace)
+ >>> str(error)
+ "Command 'command' returned non-zero exit status 1"
+
+ The action `action2` is not executed::
+
+ >>> trace
+ ['action1 received eggs']
+ """
+
+ actions = ()
+
+ def __init__(self, *args, **kwargs):
+ super(ActionsBasedSubCommand, self).__init__(*args, **kwargs)
+ self._action_names = []
+ self._actions = {}
+ for action_args in self.actions:
+ action, args = action_args[0], action_args[1:]
+ action_name = self._get_action_name(action)
+ self._action_names.append(action_name)
+ self._actions[action_name] = (action, args)
+
+ def _get_action_name(self, action):
+ """Return the string representation of an action callable.
+
+ The name is retrieved using attributes lookup for `action_name`
+ and then `__name__`::
+
+ >>> def action1():
+ ... pass
+ >>> action1.action_name = 'myaction'
+
+ >>> def action2():
+ ... pass
+
+ >>> sub_command = ActionsBasedSubCommand('foo')
+ >>> sub_command._get_action_name(action1)
+ 'myaction'
+ >>> sub_command._get_action_name(action2)
+ 'action2'
+ """
+ try:
+ return action.action_name
+ except AttributeError:
+ return action.__name__
+
+ def add_arguments(self, parser):
+ super(ActionsBasedSubCommand, self).add_arguments(parser)
+ parser.add_argument(
+ '-a', '--actions', nargs='+', choices=self._action_names,
+ help='Call one or more internal functions.')
+ parser.add_argument(
+ '--skip-actions', nargs='+', choices=self._action_names,
+ help='Skip one or more internal functions.')
+
+ def handle(self, namespace):
+ skip_actions = namespace.skip_actions or []
+ action_names = filter(
+ lambda action_name: action_name not in skip_actions,
+ namespace.actions or self._action_names)
+ for action_name in action_names:
+ action, arg_names = self._actions[action_name]
+ args = [getattr(namespace, i) for i in arg_names]
+ try:
+ action(*args)
+ except subprocess.CalledProcessError as err:
+ return err
=== added file 'lpsetup/cli.py'
--- lpsetup/cli.py 1970-01-01 00:00:00 +0000
+++ lpsetup/cli.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Command line interface set up."""
+
+__metaclass__ = type
+__all__ = [
+ 'main',
+ ]
+
+from lpsetup import (
+ __doc__ as description,
+ argparser,
+ )
+from lpsetup.subcommands import (
+ install,
+ lxcinstall,
+ update,
+ )
+
+
+def main():
+ parser = argparser.ArgumentParser(description=description)
+ parser.register_subcommand('install', install.SubCommand)
+ parser.register_subcommand('update', update.SubCommand)
+ parser.register_subcommand('lxc-install', lxcinstall.SubCommand)
+ args = parser.parse_args()
+ return args.main(args)
=== added file 'lpsetup/exceptions.py'
--- lpsetup/exceptions.py 1970-01-01 00:00:00 +0000
+++ lpsetup/exceptions.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Exceptions used during subcommand execution and args validation."""
+
+__metaclass__ = type
+__all__ = [
+ 'LaunchpadError',
+ 'ValidationError',
+ ]
+
+
+class LaunchpadError(Exception):
+ """Base exception for lpsetup."""
+
+
+class ValidationError(LaunchpadError):
+ """Argparse invalid arguments."""
=== added file 'lpsetup/handlers.py'
--- lpsetup/handlers.py 1970-01-01 00:00:00 +0000
+++ lpsetup/handlers.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Sub command arguments validation."""
+
+__metaclass__ = type
+__all__ = [
+ 'handle_directories',
+ 'handle_lpuser',
+ 'handle_ssh_keys',
+ 'handle_user',
+ 'handle_userdata',
+ ]
+
+import os
+import pwd
+
+from shelltoolbox import (
+ bzr_whois,
+ get_user_home,
+ user_exists,
+ )
+
+from lpsetup.exceptions import ValidationError
+
+
+def handle_user(namespace):
+ """Handle user argument.
+
+ This validator populates namespace with `home_dir` name::
+
+ >>> import getpass
+ >>> username = getpass.getuser()
+
+ >>> import argparse
+ >>> namespace = argparse.Namespace(user=username)
+
+ >>> handle_user(namespace)
+ >>> namespace.home_dir == '/home/' + username
+ True
+
+ The validation fails if the current user is root and no user is provided::
+
+ >>> namespace = argparse.Namespace(user=None, euid=0)
+ >>> handle_user(namespace) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ValidationError: argument user ...
+ """
+ if namespace.user is None:
+ if not namespace.euid:
+ raise ValidationError('argument user can not be omitted if '
+ 'the script is run as root.')
+ namespace.user = pwd.getpwuid(namespace.euid).pw_name
+ namespace.home_dir = get_user_home(namespace.user)
+
+
+def handle_lpuser(namespace):
+ """Handle lpuser argument.
+
+ If lpuser is not provided by namespace, the user name is used::
+
+ >>> import getpass
+ >>> username = getpass.getuser()
+
+ >>> import argparse
+ >>> namespace = argparse.Namespace(user=username, lpuser=None)
+ >>> handle_lpuser(namespace)
+ >>> namespace.lpuser == username
+ True
+ """
+ if namespace.lpuser is None:
+ namespace.lpuser = namespace.user
+
+
+def handle_userdata(namespace, whois=bzr_whois):
+ """Handle full_name and email arguments.
+
+ If they are not provided, this function tries to obtain them using
+ the given `whois` callable::
+
+ >>> import argparse
+ >>> namespace = argparse.Namespace(
+ ... full_name=None, email=None, user='root')
+ >>> email = 'email@xxxxxxxxxxx'
+ >>> handle_userdata(namespace, lambda user: (user, email))
+ >>> namespace.full_name == namespace.user
+ True
+ >>> namespace.email == email
+ True
+
+ The validation fails if full_name or email are not provided and
+ they can not be obtained using the `whois` callable::
+
+ >>> namespace = argparse.Namespace(
+ ... full_name=None, email=None, user='root')
+ >>> handle_userdata(namespace, lambda user: None) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ValidationError: arguments full-name ...
+
+ >>> namespace = argparse.Namespace(
+ ... full_name=None, email=None, user='this_user_does_not_exist')
+ >>> handle_userdata(namespace) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ValidationError: arguments full-name ...
+
+ It does not make sense to provide only one argument::
+
+ >>> namespace = argparse.Namespace(full_name='Foo Bar', email=None)
+ >>> handle_userdata(namespace) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ValidationError: arguments full-name ...
+ """
+ args = (namespace.full_name, namespace.email)
+ if not all(args):
+ if any(args):
+ raise ValidationError(
+ 'arguments full-name and email: '
+ 'either none or both must be provided.')
+ if user_exists(namespace.user):
+ userdata = whois(namespace.user)
+ if userdata is None:
+ raise ValidationError(
+ 'arguments full-name and email are required: '
+ 'bzr user id not found.')
+ namespace.full_name, namespace.email = userdata
+ else:
+ raise ValidationError(
+ 'arguments full-name and email are required: '
+ 'system user not found.')
+
+
+def handle_ssh_keys(namespace):
+ r"""Handle private and public ssh keys.
+
+ Keys contained in the namespace are escaped::
+
+ >>> import argparse
+ >>> private = r'PRIVATE\nKEY'
+ >>> public = r'PUBLIC\nKEY'
+ >>> namespace = argparse.Namespace(
+ ... private_key=private, public_key=public)
+ >>> handle_ssh_keys(namespace)
+ >>> namespace.private_key == private.decode('string-escape')
+ True
+ >>> namespace.public_key == public.decode('string-escape')
+ True
+ >>> namespace.valid_ssh_keys
+ True
+
+ Keys are None if they are not provided and can not be found in the
+ current home directory::
+
+ >>> namespace = argparse.Namespace(
+ ... private_key=None, home_dir='/tmp/__does_not_exists__')
+ >>> handle_ssh_keys(namespace) # doctest: +ELLIPSIS
+ >>> print namespace.private_key
+ None
+ >>> print namespace.public_key
+ None
+ >>> namespace.valid_ssh_keys
+ False
+
+ If only one of private_key and public_key is provided, a
+ ValidationError will be raised.
+
+ >>> namespace = argparse.Namespace(
+ ... private_key=private, public_key=None,
+ ... home_dir='/tmp/__does_not_exists__')
+ >>> handle_ssh_keys(namespace) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ValidationError: arguments private-key...
+ """
+ namespace.valid_ssh_keys = True
+ for attr, filename in (
+ ('private_key', 'id_rsa'),
+ ('public_key', 'id_rsa.pub')):
+ value = getattr(namespace, attr, None)
+ if value:
+ setattr(namespace, attr, value.decode('string-escape'))
+ else:
+ path = os.path.join(namespace.home_dir, '.ssh', filename)
+ try:
+ value = open(path).read()
+ except IOError:
+ value = None
+ namespace.valid_ssh_keys = False
+ setattr(namespace, attr, value)
+ if bool(namespace.private_key) != bool(namespace.public_key):
+ raise ValidationError(
+ "arguments private-key and public-key: "
+ "both must be provided or neither must be provided.")
+
+
+def handle_directories(namespace):
+ """Handle checkout and dependencies directories.
+
+ The ~ construction is automatically expanded::
+
+ >>> import argparse
+ >>> namespace = argparse.Namespace(
+ ... directory='~/launchpad', dependencies_dir='~/launchpad/deps',
+ ... home_dir='/home/foo')
+ >>> handle_directories(namespace)
+ >>> namespace.directory
+ '/home/foo/launchpad'
+ >>> namespace.dependencies_dir
+ '/home/foo/launchpad/deps'
+
+ The validation fails for directories not residing inside the home::
+
+ >>> namespace = argparse.Namespace(
+ ... directory='/tmp/launchpad',
+ ... dependencies_dir='~/launchpad/deps',
+ ... home_dir='/home/foo')
+ >>> handle_directories(namespace) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ValidationError: argument directory ...
+
+ The validation fails if the directory contains spaces::
+
+ >>> namespace = argparse.Namespace(directory='my directory')
+ >>> handle_directories(namespace) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ValidationError: argument directory ...
+ """
+ if ' ' in namespace.directory:
+ raise ValidationError('argument directory can not contain spaces.')
+ for attr in ('directory', 'dependencies_dir'):
+ directory = getattr(
+ namespace, attr).replace('~', namespace.home_dir)
+ if not directory.startswith(namespace.home_dir + os.path.sep):
+ raise ValidationError(
+ 'argument {0} does not reside under the home '
+ 'directory of the system user.'.format(attr))
+ setattr(namespace, attr, directory)
=== added file 'lpsetup/settings.py'
--- lpsetup/settings.py 1970-01-01 00:00:00 +0000
+++ lpsetup/settings.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Global settings and defaults for lpsetup."""
+
+
+APT_REPOSITORIES = (
+ 'deb http://archive.ubuntu.com/ubuntu {distro} multiverse',
+ 'deb http://archive.ubuntu.com/ubuntu {distro}-updates multiverse',
+ 'deb http://archive.ubuntu.com/ubuntu {distro}-security multiverse',
+ 'ppa:launchpad/ppa',
+ 'ppa:bzr/ppa',
+ )
+BASE_PACKAGES = ['ssh', 'bzr', 'language-pack-en']
+CHECKOUT_DIR = '~/launchpad/branches'
+DEPENDENCIES_DIR = '~/launchpad/dependencies'
+DHCP_FILE = '/etc/dhcp/dhclient.conf'
+HOSTS_CONTENT = (
+ ('127.0.0.88',
+ 'launchpad.dev answers.launchpad.dev archive.launchpad.dev '
+ 'api.launchpad.dev bazaar-internal.launchpad.dev beta.launchpad.dev '
+ 'blueprints.launchpad.dev bugs.launchpad.dev code.launchpad.dev '
+ 'feeds.launchpad.dev id.launchpad.dev keyserver.launchpad.dev '
+ 'lists.launchpad.dev openid.launchpad.dev '
+ 'ubuntu-openid.launchpad.dev ppa.launchpad.dev '
+ 'private-ppa.launchpad.dev testopenid.dev translations.launchpad.dev '
+ 'xmlrpc-private.launchpad.dev xmlrpc.launchpad.dev'),
+ ('127.0.0.99', 'bazaar.launchpad.dev'),
+ )
+HOSTS_FILE = '/etc/hosts'
+LP_APACHE_MODULES = 'proxy proxy_http rewrite ssl deflate headers'
+LP_APACHE_ROOTS = (
+ '/var/tmp/bazaar.launchpad.dev/static',
+ '/var/tmp/bazaar.launchpad.dev/mirrors',
+ '/var/tmp/archive',
+ '/var/tmp/ppa',
+ )
+LP_BZR_LOCATIONS = (
+ ('submit_branch', '{checkout_dir}'),
+ ('public_branch', 'bzr+ssh://bazaar.launchpad.net/~{lpuser}/launchpad'),
+ ('public_branch:policy', 'appendpath'),
+ ('push_location', 'lp:~{lpuser}/launchpad'),
+ ('push_location:policy', 'appendpath'),
+ ('merge_target', '{checkout_dir}'),
+ ('submit_to', 'merge@xxxxxxxxxxxxxxxxxx'),
+ )
+LP_CHECKOUT = 'devel'
+LP_PACKAGES = [
+ 'bzr', 'launchpad-developer-dependencies', 'apache2',
+ 'apache2-mpm-worker', 'libapache2-mod-wsgi'
+ ]
+LP_REPOS = (
+ 'http://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel',
+ 'lp:launchpad',
+ )
+LP_SOURCE_DEPS = (
+ 'http://bazaar.launchpad.net/~launchpad/lp-source-dependencies/trunk')
+LXC_CONFIG_TEMPLATE = '/etc/lxc/local.conf'
+LXC_GUEST_ARCH = 'i386'
+LXC_GUEST_CHOICES = ('lucid', 'oneiric', 'precise')
+LXC_GUEST_OS = LXC_GUEST_CHOICES[0]
+LXC_NAME = 'lptests'
+LXC_OPTIONS = """
+lxc.network.type = veth
+lxc.network.link = {interface}
+lxc.network.flags = up
+"""
+LXC_PACKAGES = ['lxc', 'libvirt-bin']
+LXC_PATH = '/var/lib/lxc/'
+RESOLV_FILE = '/etc/resolv.conf'
=== added directory 'lpsetup/subcommands'
=== added file 'lpsetup/subcommands/__init__.py'
=== added file 'lpsetup/subcommands/install.py'
--- lpsetup/subcommands/install.py 1970-01-01 00:00:00 +0000
+++ lpsetup/subcommands/install.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,288 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Install subcommand: create a Launchpad development environment."""
+
+__metaclass__ = type
+__all__ = [
+ 'make_launchpad',
+ 'initialize',
+ 'SubCommand',
+ 'setup_apt',
+ 'setup_codebase',
+ 'setup_external_sourcecode',
+ 'setup_launchpad',
+ ]
+
+from email.Utils import formataddr
+import os
+import pwd
+import subprocess
+
+from shelltoolbox import (
+ apt_get_install,
+ cd,
+ generate_ssh_keys,
+ get_su_command,
+ file_append,
+ mkdirs,
+ run,
+ su,
+ user_exists,
+ )
+
+from lpsetup import (
+ argparser,
+ handlers,
+ )
+from lpsetup.settings import (
+ APT_REPOSITORIES,
+ BASE_PACKAGES,
+ CHECKOUT_DIR,
+ DEPENDENCIES_DIR,
+ HOSTS_CONTENT,
+ HOSTS_FILE,
+ LP_APACHE_MODULES,
+ LP_APACHE_ROOTS,
+ LP_BZR_LOCATIONS,
+ LP_CHECKOUT,
+ LP_PACKAGES,
+ LP_REPOS,
+ LP_SOURCE_DEPS,
+ )
+from lpsetup.utils import call
+
+
+def setup_codebase(user, valid_ssh_keys, checkout_dir, dependencies_dir):
+ """Set up Launchpad repository and source dependencies.
+
+ Return True if new changes are pulled from bzr repository.
+ """
+ # Using real su because bzr uses uid.
+ if os.path.exists(checkout_dir):
+ # Pull the repository.
+ revno_args = ('bzr', 'revno', checkout_dir)
+ revno = run(*revno_args)
+ call(*get_su_command(user, ['bzr', 'pull', '-d', checkout_dir]))
+ changed = revno != run(*revno_args)
+ else:
+ # Branch the repository.
+ cmd = ('bzr', 'branch',
+ LP_REPOS[1] if valid_ssh_keys else LP_REPOS[0], checkout_dir)
+ call(*get_su_command(user, cmd))
+ changed = True
+ # Check repository integrity.
+ if subprocess.call(['bzr', 'status', '-q', checkout_dir]):
+ raise subprocess.CalledProcessError(
+ 'Repository {0} is corrupted.'.format(checkout_dir))
+ # Set up source dependencies.
+ with su(user):
+ for subdir in ('eggs', 'yui', 'sourcecode'):
+ mkdirs(os.path.join(dependencies_dir, subdir))
+ download_cache = os.path.join(dependencies_dir, 'download-cache')
+ if os.path.exists(download_cache):
+ call('bzr', 'up', download_cache)
+ else:
+ call('bzr', 'co', '--lightweight', LP_SOURCE_DEPS, download_cache)
+ return changed
+
+
+def setup_external_sourcecode(
+ user, valid_ssh_keys, checkout_dir, dependencies_dir):
+ """Update and link external sourcecode."""
+ cmd = (
+ 'utilities/update-sourcecode',
+ None if valid_ssh_keys else '--use-http',
+ os.path.join(dependencies_dir, 'sourcecode'),
+ )
+ with cd(checkout_dir):
+ # Using real su because update-sourcecode uses uid.
+ call(*get_su_command(user, cmd))
+ with su(user):
+ call('utilities/link-external-sourcecode', dependencies_dir)
+
+
+def make_launchpad(user, checkout_dir, install=False):
+ """Make and optionally install Launchpad."""
+ # Using real su because mailman make script uses uid.
+ call(*get_su_command(user, ['make', '-C', checkout_dir]))
+ if install:
+ call('make', '-C', checkout_dir, 'install')
+
+
+
+def initialize(
+ user, full_name, email, lpuser, private_key, public_key, valid_ssh_keys,
+ dependencies_dir, directory):
+ """Initialize host machine."""
+ # Install necessary deb packages. This requires Oneiric or later.
+ call('apt-get', 'update')
+ apt_get_install(*BASE_PACKAGES)
+ # Create the user (if he does not exist).
+ if not user_exists(user):
+ call('useradd', '-m', '-s', '/bin/bash', '-U', user)
+ # Generate root ssh keys if they do not exist.
+ if not os.path.exists('/root/.ssh/id_rsa.pub'):
+ generate_ssh_keys('/root/.ssh/')
+ with su(user) as env:
+ # Set up the user's ssh directory. The ssh key must be associated
+ # with the lpuser's Launchpad account.
+ ssh_dir = os.path.join(env.home, '.ssh')
+ mkdirs(ssh_dir)
+ # Generate user ssh keys if none are supplied.
+ if not valid_ssh_keys:
+ generate_ssh_keys(ssh_dir)
+ private_key = open(os.path.join(ssh_dir, 'id_rsa')).read()
+ public_key = open(os.path.join(ssh_dir, 'id_rsa.pub')).read()
+ priv_file = os.path.join(ssh_dir, 'id_rsa')
+ pub_file = os.path.join(ssh_dir, 'id_rsa.pub')
+ auth_file = os.path.join(ssh_dir, 'authorized_keys')
+ known_hosts = os.path.join(ssh_dir, 'known_hosts')
+ known_host_content = run(
+ 'ssh-keyscan', '-t', 'rsa', 'bazaar.launchpad.net')
+ for filename, contents, mode in [
+ (priv_file, private_key, 'w'),
+ (pub_file, public_key, 'w'),
+ (auth_file, public_key, 'a'),
+ (known_hosts, known_host_content, 'a'),
+ ]:
+ with open(filename, mode) as f:
+ f.write(contents + '\n')
+ os.chmod(filename, 0644)
+ os.chmod(priv_file, 0600)
+ # Set up bzr and Launchpad authentication.
+ call('bzr', 'whoami', formataddr([full_name, email]))
+ if valid_ssh_keys:
+ subprocess.call(['bzr', 'lp-login', lpuser])
+ # Set up the repository.
+ mkdirs(directory)
+ call('bzr', 'init-repo', '--2a', directory)
+ # Set up the codebase.
+ checkout_dir = os.path.join(directory, LP_CHECKOUT)
+ setup_codebase(user, valid_ssh_keys, checkout_dir, dependencies_dir)
+ # Set up bzr locations
+ with su(user) as env:
+ bzr_locations = os.path.join(env.home, '.bazaar', 'locations.conf')
+ file_append(bzr_locations, '[{0}]\n'.format(directory))
+ lines = ['{0} = {1}\n'.format(k, v) for k, v in LP_BZR_LOCATIONS]
+ for line in lines:
+ location = line.format(checkout_dir=checkout_dir, lpuser=lpuser)
+ file_append(bzr_locations, location)
+
+
+def setup_apt(no_repositories=True):
+ """Setup, update and upgrade deb packages."""
+ if not no_repositories:
+ distro = run('lsb_release', '-cs').strip()
+ # APT repository update.
+ for reposirory in APT_REPOSITORIES:
+ assume_yes = None if distro == 'lucid' else '-y'
+ call('add-apt-repository', assume_yes,
+ reposirory.format(distro=distro))
+ # XXX frankban 2012-01-13 - Bug 892892: upgrading mountall in LXC
+ # containers currently does not work.
+ call("echo 'mountall hold' | dpkg --set-selections", shell=True)
+ call('apt-get', 'update')
+ # Install base and Launchpad deb packages.
+ apt_get_install(*LP_PACKAGES)
+
+
+def setup_launchpad(user, dependencies_dir, directory, valid_ssh_keys):
+ """Set up the Launchpad environment."""
+ # User configuration.
+ subprocess.call(['adduser', user, 'sudo'])
+ gid = pwd.getpwnam(user).pw_gid
+ subprocess.call(['addgroup', '--gid', str(gid), user])
+ # Set up Launchpad dependencies.
+ checkout_dir = os.path.join(directory, LP_CHECKOUT)
+ setup_external_sourcecode(
+ user, valid_ssh_keys, checkout_dir, dependencies_dir)
+ with su(user):
+ # Create Apache document roots, to avoid warnings.
+ mkdirs(*LP_APACHE_ROOTS)
+ # Set up Apache modules.
+ for module in LP_APACHE_MODULES.split():
+ call('a2enmod', module)
+ with cd(checkout_dir):
+ # Launchpad database setup.
+ call('utilities/launchpad-database-setup', user)
+ # Make and install launchpad.
+ make_launchpad(user, checkout_dir, install=True)
+ # Set up container hosts file.
+ lines = ['{0}\t{1}\n'.format(ip, names) for ip, names in HOSTS_CONTENT]
+ for line in lines:
+ file_append(HOSTS_FILE, line)
+
+
+class SubCommand(argparser.ActionsBasedSubCommand):
+ """Install the Launchpad environment."""
+
+ actions = (
+ (initialize,
+ 'user', 'full_name', 'email', 'lpuser', 'private_key',
+ 'public_key', 'valid_ssh_keys', 'dependencies_dir', 'directory'),
+ (setup_apt, 'no_repositories'),
+ (setup_launchpad,
+ 'user', 'dependencies_dir', 'directory', 'valid_ssh_keys'),
+ )
+ help = __doc__
+ needs_root = True
+ validators = (
+ handlers.handle_user,
+ handlers.handle_lpuser,
+ handlers.handle_userdata,
+ handlers.handle_ssh_keys,
+ handlers.handle_directories,
+ )
+
+ def add_arguments(self, parser):
+ super(SubCommand, self).add_arguments(parser)
+ parser.add_argument(
+ '-u', '--user',
+ help='The name of the system user to be created or updated. '
+ 'The current user is used if this script is not run as '
+ 'root and this argument is omitted.')
+ parser.add_argument(
+ '-e', '--email',
+ help='The email of the user, used for bzr whoami. This argument '
+ 'can be omitted if a bzr id exists for current user.')
+ parser.add_argument(
+ '-f', '--full-name',
+ help='The full name of the user, used for bzr whoami. '
+ 'This argument can be omitted if a bzr id exists for '
+ 'current user.')
+ parser.add_argument(
+ '-l', '--lpuser',
+ help='The name of the Launchpad user that will be used to '
+ 'check out dependencies. If not provided, the system '
+ 'user name is used.')
+ parser.add_argument(
+ '-v', '--private-key',
+ help='The SSH private key for the Launchpad user (without '
+ 'passphrase). If this argument is omitted and a keypair is '
+ 'not found in the home directory of the system user a new '
+ 'SSH keypair will be generated and the checkout of the '
+ 'Launchpad code will use HTTP rather than bzr+ssh.')
+ parser.add_argument(
+ '-b', '--public-key',
+ help='The SSH public key for the Launchpad user. '
+ 'If this argument is omitted and a keypair is not found '
+ 'in the home directory of the system user a new SSH '
+ 'keypair will be generated and the checkout of the '
+ 'Launchpad code will use HTTP rather than bzr+ssh.')
+ parser.add_argument(
+ '-d', '--dependencies-dir', default=DEPENDENCIES_DIR,
+ help='The directory of the Launchpad dependencies to be created. '
+ 'The directory must reside under the home directory of the '
+ 'given user (see -u argument). '
+ '[DEFAULT={0}]'.format(DEPENDENCIES_DIR))
+ parser.add_argument(
+ '-c', '--directory', default=CHECKOUT_DIR,
+ help='The directory of the Launchpad repository to be created. '
+ 'The directory must reside under the home directory of the '
+ 'given user (see -u argument). '
+ '[DEFAULT={0}]'.format(CHECKOUT_DIR))
+ parser.add_argument(
+ '-N', '--no-repositories', action='store_true',
+ help='Do not add APT repositories.')
=== added file 'lpsetup/subcommands/lxcinstall.py'
--- lpsetup/subcommands/lxcinstall.py 1970-01-01 00:00:00 +0000
+++ lpsetup/subcommands/lxcinstall.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""LXC install subcommand: create a Launchpad environment inside an LXC."""
+
+__metaclass__ = type
+__all__ = [
+ 'create_lxc',
+ 'initialize_lxc',
+ 'SubCommand',
+ 'setup_launchpad_lxc',
+ 'start_lxc',
+ 'stop_lxc',
+ 'wait_for_lxc',
+ ]
+
+import os
+import shutil
+import subprocess
+import time
+
+from shelltoolbox import (
+ apt_get_install,
+ file_append,
+ file_prepend,
+ mkdirs,
+ ssh,
+ )
+
+from lpsetup.settings import (
+ BASE_PACKAGES,
+ DHCP_FILE,
+ LXC_CONFIG_TEMPLATE,
+ LXC_GUEST_ARCH,
+ LXC_GUEST_CHOICES,
+ LXC_GUEST_OS,
+ LXC_NAME,
+ LXC_OPTIONS,
+ LXC_PACKAGES,
+ RESOLV_FILE,
+ )
+from lpsetup.subcommands import install
+from lpsetup.utils import (
+ call,
+ get_container_path,
+ get_lxc_gateway,
+ lxc_stopped,
+ this_command,
+ )
+
+
+def create_lxc(user, lxc_name, lxc_arch, lxc_os):
+ """Create the LXC named `lxc_name` sharing `user` home directory.
+
+ The container will be used as development environment or as base template
+ for parallel testing using ephemeral instances.
+ """
+ # Install necessary deb packages.
+ apt_get_install(*LXC_PACKAGES)
+ # XXX 2012-02-02 gmb bug=925024:
+ # These calls need to be removed once the lxc vs. apparmor bug
+ # is resolved, since having apparmor enabled for lxc is very
+ # much a Good Thing.
+ # Disable the apparmor profiles for lxc so that we don't have
+ # problems installing postgres.
+ call('ln', '-s',
+ '/etc/apparmor.d/usr.bin.lxc-start', '/etc/apparmor.d/disable/')
+ call('apparmor_parser', '-R', '/etc/apparmor.d/usr.bin.lxc-start')
+ # Update resolv file in order to get the ability to ssh into the LXC
+ # container using its name.
+ lxc_gateway_name, lxc_gateway_address = get_lxc_gateway()
+ file_prepend(RESOLV_FILE, 'nameserver {0}\n'.format(lxc_gateway_address))
+ file_append(
+ DHCP_FILE,
+ 'prepend domain-name-servers {0};\n'.format(lxc_gateway_address))
+ # Container configuration template.
+ content = LXC_OPTIONS.format(interface=lxc_gateway_name)
+ with open(LXC_CONFIG_TEMPLATE, 'w') as f:
+ f.write(content)
+ # Creating container.
+ call(
+ 'lxc-create',
+ '-t', 'ubuntu',
+ '-n', lxc_name,
+ '-f', LXC_CONFIG_TEMPLATE,
+ '--',
+ '-r {os} -a {arch} -b {user}'.format(
+ os=lxc_os, arch=lxc_arch, user=user),
+ )
+ # Set up root ssh key.
+ user_authorized_keys = os.path.expanduser(
+ '~' + user + '/.ssh/authorized_keys')
+ with open(user_authorized_keys, 'a') as f:
+ f.write(open('/root/.ssh/id_rsa.pub').read())
+ dst = get_container_path(lxc_name, '/root/.ssh/')
+ mkdirs(dst)
+ shutil.copy(user_authorized_keys, dst)
+
+
+def start_lxc(lxc_name):
+ """Start the lxc instance named `lxc_name`."""
+ call('lxc-start', '-n', lxc_name, '-d')
+
+
+def wait_for_lxc(lxc_name, trials=60, sleep_seconds=1):
+ """Try to ssh as `user` into the LXC container named `lxc_name`."""
+ sshcall = ssh(lxc_name)
+ while True:
+ trials -= 1
+ try:
+ sshcall('true')
+ except subprocess.CalledProcessError:
+ if not trials:
+ raise
+ time.sleep(sleep_seconds)
+ else:
+ break
+
+
+def initialize_lxc(lxc_name, lxc_os):
+ """Initialize LXC container."""
+ base_packages = list(BASE_PACKAGES)
+ if lxc_os == 'lucid':
+ # Install argparse to be able to run this script inside a lucid lxc.
+ base_packages.append('python-argparse')
+ ssh(lxc_name)(
+ 'DEBIAN_FRONTEND=noninteractive '
+ 'apt-get install -y ' + ' '.join(base_packages))
+
+
+def setup_launchpad_lxc(
+ user, dependencies_dir, directory, valid_ssh_keys, lxc_name):
+ """Set up the Launchpad environment inside an LXC."""
+ # Use ssh to call this script from inside the container.
+ args = [
+ 'install', '-u', user, '-a', 'setup_apt', 'setup_launchpad',
+ '-d', dependencies_dir, '-c', directory
+ ]
+ cmd = this_command(directory, args)
+ ssh(lxc_name)(cmd)
+
+
+def stop_lxc(lxc_name):
+ """Stop the lxc instance named `lxc_name`."""
+ ssh(lxc_name)('poweroff')
+ if not lxc_stopped(lxc_name):
+ subprocess.call(['lxc-stop', '-n', lxc_name])
+
+
+class SubCommand(install.SubCommand):
+ """Install the Launchpad environment inside an LXC."""
+
+ actions = (
+ (install.initialize,
+ 'user', 'full_name', 'email', 'lpuser', 'private_key',
+ 'public_key', 'valid_ssh_keys', 'dependencies_dir', 'directory'),
+ (create_lxc,
+ 'user', 'lxc_name', 'lxc_arch', 'lxc_os'),
+ (start_lxc, 'lxc_name'),
+ (wait_for_lxc, 'lxc_name'),
+ (initialize_lxc,
+ 'lxc_name', 'lxc_os'),
+ (setup_launchpad_lxc,
+ 'user', 'dependencies_dir', 'directory', 'valid_ssh_keys',
+ 'lxc_name'),
+ (stop_lxc, 'lxc_name'),
+ )
+ help = __doc__
+
+ def add_arguments(self, parser):
+ super(SubCommand, self).add_arguments(parser)
+ parser.add_argument(
+ '-n', '--lxc-name', default=LXC_NAME,
+ help='The LXC container name to setup. '
+ '[DEFAULT={0}]'.format(LXC_NAME))
+ parser.add_argument(
+ '-A', '--lxc-arch', default=LXC_GUEST_ARCH,
+ help='The LXC container architecture. '
+ '[DEFAULT={0}]'.format(LXC_GUEST_ARCH))
+ parser.add_argument(
+ '-r', '--lxc-os', default=LXC_GUEST_OS,
+ choices=LXC_GUEST_CHOICES,
+ help='The LXC container distro codename. '
+ '[DEFAULT={0}]'.format(LXC_GUEST_OS))
=== added file 'lpsetup/subcommands/update.py'
--- lpsetup/subcommands/update.py 1970-01-01 00:00:00 +0000
+++ lpsetup/subcommands/update.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Update subcommand: update a Launchpad environment, pulling changes."""
+
+__metaclass__ = type
+__all__ = [
+ 'link_sourcecode_in_branches',
+ 'update_launchpad',
+ 'SubCommand',
+ ]
+
+import os
+
+from shelltoolbox import su
+
+from lpsetup import (
+ argparser,
+ handlers,
+ )
+from lpsetup.settings import (
+ CHECKOUT_DIR,
+ DEPENDENCIES_DIR,
+ LP_CHECKOUT,
+ )
+from lpsetup.subcommands import install
+from lpsetup.utils import call
+
+
+def update_launchpad(user, valid_ssh_keys, dependencies_dir, directory, apt):
+ """Update the Launchpad environment."""
+ if apt:
+ install.setup_apt(no_repositories=True)
+ checkout_dir = os.path.join(directory, LP_CHECKOUT)
+ # Update the Launchpad codebase.
+ changed = install.setup_codebase(
+ user, valid_ssh_keys, checkout_dir, dependencies_dir)
+ install.setup_external_sourcecode(
+ user, valid_ssh_keys, checkout_dir, dependencies_dir)
+ if changed:
+ install.make_launchpad(user, checkout_dir, install=False)
+
+
+def link_sourcecode_in_branches(user, dependencies_dir, directory):
+ """Link external sourcecode for all branches in the project."""
+ checkout_dir = os.path.join(directory, LP_CHECKOUT)
+ cmd = os.path.join(checkout_dir, 'utilities', 'link-external-sourcecode')
+ with su(user):
+ for dirname in os.listdir(directory):
+ branch = os.path.join(directory, dirname)
+ sourcecode_dir = os.path.join(branch, 'sourcecode')
+ if (branch != checkout_dir and
+ os.path.exists(sourcecode_dir) and
+ os.path.isdir(sourcecode_dir)):
+ call(cmd, '--parent', dependencies_dir, '--target', branch)
+
+
+
+class SubCommand(argparser.ActionsBasedSubCommand):
+ """Update the Launchpad environment to latest version."""
+
+ actions = (
+ (update_launchpad,
+ 'user', 'valid_ssh_keys', 'dependencies_dir', 'directory', 'apt'),
+ (link_sourcecode_in_branches,
+ 'user', 'dependencies_dir', 'directory'),
+ )
+ help = __doc__
+ validators = (
+ handlers.handle_user,
+ handlers.handle_ssh_keys,
+ handlers.handle_directories,
+ )
+
+ def get_needs_root(self, namespace):
+ # Root is needed only if an apt update/upgrade is requested.
+ return namespace.apt
+
+ def add_arguments(self, parser):
+ super(SubCommand, self).add_arguments(parser)
+ parser.add_argument(
+ '-u', '--user',
+ help='The name of the system user used to update Launchpad. '
+ 'The current user is used if this script is not run as '
+ 'root and this argument is omitted.')
+ parser.add_argument(
+ '-d', '--dependencies-dir', default=DEPENDENCIES_DIR,
+ help='The directory of the Launchpad dependencies to be updated. '
+ 'The directory must reside under the home directory of the '
+ 'given user (see -u argument). '
+ '[DEFAULT={0}]'.format(DEPENDENCIES_DIR))
+ parser.add_argument(
+ '-c', '--directory', default=CHECKOUT_DIR,
+ help='The directory of the Launchpad repository to be updated. '
+ 'The directory must reside under the home directory of the '
+ 'given user (see -u argument). '
+ '[DEFAULT={0}]'.format(CHECKOUT_DIR))
+ parser.add_argument(
+ '-D', '--apt', action='store_true',
+ help='Also update deb packages.')
=== added directory 'lpsetup/tests'
=== added file 'lpsetup/tests/__init__.py'
=== added file 'lpsetup/utils.py'
--- lpsetup/utils.py 1970-01-01 00:00:00 +0000
+++ lpsetup/utils.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""A collection of helper functions."""
+
+__metaclass__ = type
+__all__ = [
+ 'call',
+ 'get_container_path',
+ 'get_lxc_gateway',
+ 'lxc_in_state',
+ 'lxc_running',
+ 'lxc_stopped',
+ 'this_command',
+ ]
+
+from functools import partial
+import os
+import platform
+import subprocess
+import time
+
+from shelltoolbox import (
+ join_command,
+ run,
+ )
+
+from lpsetup.settings import (
+ LP_CHECKOUT,
+ LXC_PATH,
+ )
+
+
+call = partial(run, stdout=None)
+
+
+def get_container_path(lxc_name, path='', base_path=LXC_PATH):
+ """Return the path of LXC container called `lxc_name`.
+
+ If a `path` is given, return that path inside the container, e.g.::
+
+ >>> get_container_path('mycontainer')
+ '/var/lib/lxc/mycontainer/rootfs/'
+ >>> get_container_path('mycontainer', '/etc/apt/')
+ '/var/lib/lxc/mycontainer/rootfs/etc/apt/'
+ >>> get_container_path('mycontainer', 'home')
+ '/var/lib/lxc/mycontainer/rootfs/home'
+ """
+ return os.path.join(base_path, lxc_name, 'rootfs', path.lstrip('/'))
+
+
+def get_lxc_gateway():
+ """Return a tuple of gateway name and address.
+
+ The gateway name and address will change depending on which version
+ of Ubuntu the script is running on.
+ """
+ release_name = platform.linux_distribution()[2]
+ if release_name == 'oneiric':
+ return 'virbr0', '192.168.122.1'
+ else:
+ return 'lxcbr0', '10.0.3.1'
+
+
+def lxc_in_state(state, lxc_name, timeout=30):
+ """Return True if the LXC named `lxc_name` is in state `state`.
+
+ Return False otherwise.
+ """
+ while timeout:
+ try:
+ output = run('lxc-info', '-n', lxc_name, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError:
+ pass
+ else:
+ if state in output:
+ return True
+ timeout -= 1
+ time.sleep(1)
+ return False
+
+
+lxc_running = partial(lxc_in_state, 'RUNNING')
+lxc_stopped = partial(lxc_in_state, 'STOPPED')
+
+
+def this_command(directory, args):
+ """Return a command line to re-launch this script using given `args`.
+
+ The command returned will execute this script from `directory`::
+
+ >>> import os
+ >>> script_name = os.path.basename(__file__)
+
+ >>> cmd = this_command('/home/user/lp/branches', ['install'])
+ >>> cmd == ('cd /home/user/lp/branches && devel/utilities/' +
+ ... script_name + ' install')
+ True
+
+ Arguments are correctly quoted::
+
+ >>> cmd = this_command('/path', ['-arg', 'quote me'])
+ >>> cmd == ('cd /path && devel/utilities/' +
+ ... script_name + " -arg 'quote me'")
+ True
+ """
+ script = join_command([
+ os.path.join(LP_CHECKOUT, 'utilities', os.path.basename(__file__)),
+ ] + list(args))
+ return join_command(['cd', directory]) + ' && ' + script
=== added file 'setup.py'
--- setup.py 1970-01-01 00:00:00 +0000
+++ setup.py 2012-03-12 14:27:19 +0000
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from distutils.core import setup
+import os
+
+
+project_name = 'lpsetup'
+
+root_dir = os.path.dirname(__file__)
+if root_dir:
+ os.chdir(root_dir)
+
+data_files = []
+for dirpath, dirnames, filenames in os.walk(project_name):
+ for i, dirname in enumerate(dirnames):
+ if dirname.startswith('.'): del dirnames[i]
+ if '__init__.py' in filenames:
+ continue
+ elif filenames:
+ for f in filenames:
+ data_files.append(os.path.join(dirpath[len(project_name)+1:], f))
+
+project = __import__(project_name)
+long_description = open(os.path.join(root_dir, 'README.rst')).read()
+
+setup(
+ name=project_name,
+ version=project.get_version(),
+ description=project.__doc__,
+ long_description=long_description,
+ author='Francesco Banconi',
+ author_email='francesco.banconi@xxxxxxxxxxxxx',
+ url='https://launchpad.net/lpsetup',
+ scripts=['lp-setup'],
+ packages=[
+ 'lpsetup',
+ 'lpsetup.subcommands',
+ ],
+ package_data={project_name: data_files},
+ # install_requires=['python-shell-toolbox'],
+ classifiers=[
+ 'Development Status :: 3 - Alpha',
+ 'Environment :: Console',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: GNU Affero General Public License v3',
+ 'Operating System :: POSIX :: Linux',
+ 'Programming Language :: Python',
+ 'Topic :: Utilities'
+ ],
+
+ )
