launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #05553
[Merge] lp:~mbp/launchpad/ec2-region into lp:launchpad
Martin Pool has proposed merging lp:~mbp/launchpad/ec2-region into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~mbp/launchpad/ec2-region/+merge/82487
This adds a --region option to ec2, so that if you live in the asia-pacific or europe you can get lower latency, or if the us-east-1 zone is on fire you can work somewhere else.
This also includes changes to move some data onto tmpfs, in the hope that will make things faster. It's still running the first test but seems to be working ok, and not doing any hard io.
the devscripts/ec2 is a bit hacky. I don't know if this makes it too much worse.
--
https://code.launchpad.net/~mbp/launchpad/ec2-region/+merge/82487
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~mbp/launchpad/ec2-region into lp:launchpad.
=== modified file 'lib/devscripts/ec2test/builtins.py'
--- lib/devscripts/ec2test/builtins.py 2011-09-07 13:38:46 +0000
+++ lib/devscripts/ec2test/builtins.py 2011-11-17 04:15:30 +0000
@@ -129,6 +129,14 @@
"and --file."))
+region_option = Option(
+ 'region',
+ type=str,
+ help=("Name of the AWS region in which to run the instance. "
+ "Must be the same as the region holding the image file. "
+ "For example, 'us-west-1'."))
+
+
def filename_type(filename):
"""An option validator for filenames.
@@ -219,6 +227,7 @@
trunk_option,
machine_id_option,
instance_type_option,
+ region_option,
Option(
'file', short_name='f', type=filename_type,
help=('Store abridged test results in FILE.')),
@@ -285,6 +294,7 @@
noemail=False, submit_pqm_message=None, pqm_public_location=None,
pqm_submit_location=None, pqm_email=None, postmortem=False,
attached=False, debug=False, open_browser=False,
+ region=None,
include_download_cache_changes=False):
set_trace_if(debug)
if branch is None:
@@ -313,7 +323,8 @@
"supported")
session_name = EC2SessionName.make(EC2TestRunner.name)
- instance = EC2Instance.make(session_name, instance_type, machine)
+ instance = EC2Instance.make(session_name, instance_type, machine,
+ region=region)
runner = EC2TestRunner(
test_branch, email=email, file=file,
@@ -334,6 +345,7 @@
takes_options = [
debug_option,
+ instance_type_option,
Option('dry-run', help="Just print the equivalent ec2 test command."),
Option('print-commit', help="Print the full commit message."),
Option(
@@ -501,6 +513,7 @@
postmortem_option,
debug_option,
include_download_cache_changes_option,
+ region_option,
ListOption(
'demo', type=str,
help="Allow this netmask to connect to the instance."),
@@ -569,6 +582,7 @@
instance_type_option,
postmortem_option,
debug_option,
+ region_option,
ListOption(
'extra-update-image-command', type=str,
help=('Run this command (with an ssh agent) on the image before '
@@ -585,6 +599,7 @@
def run(self, ami_name, machine=None, instance_type='m1.large',
debug=False, postmortem=False, extra_update_image_command=None,
+ region=None,
public=False):
set_trace_if(debug)
@@ -598,17 +613,14 @@
for variable in ['LANG', 'LC_ALL', 'LC_TIME']:
os.environ.pop(variable, None)
- credentials = EC2Credentials.load_from_file()
-
session_name = EC2SessionName.make(EC2TestRunner.name)
instance = EC2Instance.make(
session_name, instance_type, machine,
- credentials=credentials)
- instance.check_bundling_prerequisites(
- ami_name, credentials)
+ region=region)
+ instance.check_bundling_prerequisites(ami_name)
instance.set_up_and_run(
postmortem, True, self.update_image, instance,
- extra_update_image_command, ami_name, credentials, public)
+ extra_update_image_command, ami_name, instance._credentials, public)
def update_image(self, instance, extra_update_image_command, ami_name,
credentials, public):
@@ -668,9 +680,13 @@
The first in the list is the default image.
"""
- def run(self):
- credentials = EC2Credentials.load_from_file()
+ takes_options = [
+ region_option,
+ ]
+
+ def run(self, region=None):
session_name = EC2SessionName.make(EC2TestRunner.name)
+ credentials = EC2Credentials.load_from_file(region_name=region)
account = credentials.connect(session_name)
format = "%5s %-12s %-12s %-12s %s\n"
self.outf.write(
@@ -697,6 +713,7 @@
aliases = ["ls"]
takes_options = [
+ region_option,
Option('show-urls',
help="Include more information about each instance"),
Option('all', short_name='a',
@@ -765,9 +782,9 @@
': '.join((state, str(num)))
for (state, num) in sorted(list(by_state.items())))
- def run(self, show_urls=False, all=False):
- credentials = EC2Credentials.load_from_file()
+ def run(self, show_urls=False, all=False, region=None):
session_name = EC2SessionName.make(EC2TestRunner.name)
+ credentials = EC2Credentials.load_from_file(region_name=region)
account = credentials.connect(session_name)
instances = list(self.iter_instances(account))
if len(instances) == 0:
=== modified file 'lib/devscripts/ec2test/credentials.py'
--- lib/devscripts/ec2test/credentials.py 2011-05-27 19:57:02 +0000
+++ lib/devscripts/ec2test/credentials.py 2011-11-17 04:15:30 +0000
@@ -12,6 +12,7 @@
import os
import boto
+import boto.ec2
from bzrlib.errors import BzrCommandError
from devscripts.ec2test.account import EC2Account
@@ -32,12 +33,13 @@
DEFAULT_CREDENTIALS_FILE = '~/.ec2/aws_id'
- def __init__(self, identifier, secret):
+ def __init__(self, identifier, secret, region_name):
self.identifier = identifier
self.secret = secret
+ self.region_name = region_name or 'us-east-1'
@classmethod
- def load_from_file(cls, filename=None):
+ def load_from_file(cls, filename=None, region_name=None):
"""Load the EC2 credentials from 'filename'."""
if filename is None:
filename = os.path.expanduser(cls.DEFAULT_CREDENTIALS_FILE)
@@ -50,15 +52,18 @@
secret = aws_file.readline().strip()
finally:
aws_file.close()
- return cls(identifier, secret)
+ return cls(identifier, secret, region_name)
def connect(self, name):
"""Connect to EC2 with these credentials.
- :param name: ???
+ :param name: Arbitrary local name for the object.
:return: An `EC2Account` connected to EC2 with these credentials.
"""
- conn = boto.connect_ec2(self.identifier, self.secret)
+ conn = boto.ec2.connect_to_region(
+ self.region_name,
+ aws_access_key_id=self.identifier,
+ aws_secret_access_key=self.secret)
return EC2Account(name, conn)
def connect_s3(self):
=== modified file 'lib/devscripts/ec2test/instance.py'
--- lib/devscripts/ec2test/instance.py 2011-11-13 21:23:00 +0000
+++ lib/devscripts/ec2test/instance.py 2011-11-17 04:15:30 +0000
@@ -25,7 +25,9 @@
DEFAULT_INSTANCE_TYPE = 'c1.xlarge'
-AVAILABLE_INSTANCE_TYPES = ('m1.large', 'm1.xlarge', 'c1.xlarge')
+AVAILABLE_INSTANCE_TYPES = (
+ 'm1.large', 'm1.xlarge', 'm2.xlarge', 'm2.2xlarge', 'm2.4xlarge',
+ 'c1.xlarge', 'cc1.4xlarge', 'cc1.8xlarge')
class AcceptAllPolicy:
@@ -71,17 +73,30 @@
# -e Exit immediately if a command exits with a non-zero status.
set -xe
+# They end up as just one stream; this avoids ordering problems.
+exec 2>&1
+
sed -ie 's/main universe/main universe multiverse/' /etc/apt/sources.list
. /etc/lsb-release
+mount -o remount,data=writeback,commit=3600,async,relatime /
+
cat >> /etc/apt/sources.list << EOF
deb http://ppa.launchpad.net/launchpad/ubuntu $DISTRIB_CODENAME main
deb http://ppa.launchpad.net/bzr/ubuntu $DISTRIB_CODENAME main
-deb http://us.ec2.archive.ubuntu.com/ubuntu/ $DISTRIB_CODENAME multiverse
-deb-src http://us.ec2.archive.ubuntu.com/ubuntu/ $DISTRIB_CODENAME main
EOF
+export DEBIAN_FRONTEND=noninteractive
+
+# PPA keys
+apt-key adv --recv-keys --keyserver pool.sks-keyservers.net 2af499cb24ac5f65461405572d1ffb6c0a5174af # launchpad
+apt-key adv --recv-keys --keyserver pool.sks-keyservers.net ece2800bacf028b31ee3657cd702bf6b8c6c1efd # bzr
+
+aptitude update
+LANG=C aptitude -y install language-pack-en # Do this first so later things don't complain about locales
+aptitude -y full-upgrade
+
# This next part is cribbed from rocketfuel-setup
dev_host() {
sed -i \"s/^127.0.0.88.*$/&\ ${hostname}/\" /etc/hosts
@@ -124,15 +139,7 @@
127.0.0.99 bazaar.launchpad.dev
' >> /etc/hosts
-# Add the keys for the three PPAs added to sources.list above.
-apt-key adv --recv-keys --keyserver pool.sks-keyservers.net 2af499cb24ac5f65461405572d1ffb6c0a5174af
-apt-key adv --recv-keys --keyserver pool.sks-keyservers.net ece2800bacf028b31ee3657cd702bf6b8c6c1efd
-apt-key adv --recv-keys --keyserver pool.sks-keyservers.net cbede690576d1e4e813f6bb3ebaf723d37b19b80
-
-aptitude update
-aptitude -y full-upgrade
-
-DEBIAN_FRONTEND=noninteractive apt-get -y install launchpad-developer-dependencies apache2 apache2-mpm-worker
+apt-get -y install launchpad-developer-dependencies apache2 apache2-mpm-worker
# Create the ec2test user, give them passwordless sudo.
adduser --gecos "" --disabled-password ec2test
@@ -155,6 +162,9 @@
# -e Exit immediately if a command exits with a non-zero status.
set -xe
+# They end up as just one stream; this avoids ordering problems.
+exec 2>&1
+
bzr launchpad-login %(launchpad-login)s
bzr init-repo --2a /var/launchpad
bzr branch lp:~launchpad-pqm/launchpad/devel /var/launchpad/test
@@ -181,7 +191,7 @@
@classmethod
def make(cls, name, instance_type, machine_id, demo_networks=None,
- credentials=None):
+ credentials=None, region=None):
"""Construct an `EC2Instance`.
:param name: The name to use for the key pair and security group for
@@ -196,6 +206,7 @@
:param demo_networks: A list of networks to add to the security group
to allow access to the instance.
:param credentials: An `EC2Credentials` object.
+ :param region: A string region name eg 'us-east-1'.
"""
# This import breaks in the test environment. Do it here so
# that unit tests (which don't use this factory) can still
@@ -217,7 +228,7 @@
user_key = get_user_key()
if credentials is None:
- credentials = EC2Credentials.load_from_file()
+ credentials = EC2Credentials.load_from_file(region_name=region)
# Make the EC2 connection.
account = credentials.connect(name)
@@ -245,12 +256,14 @@
raise BzrCommandError(
'you must have set your launchpad login in bzr.')
- return EC2Instance(
+ instance = EC2Instance(
name, image, instance_type, demo_networks, account,
- from_scratch, user_key, login)
+ from_scratch, user_key, login, region)
+ instance._credentials = credentials
+ return instance
def __init__(self, name, image, instance_type, demo_networks, account,
- from_scratch, user_key, launchpad_login):
+ from_scratch, user_key, launchpad_login, region):
self._name = name
self._image = image
self._account = account
@@ -260,6 +273,7 @@
self._from_scratch = from_scratch
self._user_key = user_key
self._launchpad_login = launchpad_login
+ self._region = region
def log(self, msg):
"""Log a message on stdout, flushing afterwards."""
@@ -296,7 +310,8 @@
self._ec2test_user_has_keys = False
else:
raise BzrCommandError(
- 'failed to start: %s\n' % self._boto_instance.state)
+ 'failed to start: %s: %r\n' % (
+ self._boto_instance.state, self._boto_instance.state_reason))
def shutdown(self):
"""Shut down the instance."""
@@ -305,9 +320,10 @@
return
self._boto_instance.update()
if self._boto_instance.state not in ('shutting-down', 'terminated'):
- # terminate instance
- self._boto_instance.stop()
+ self.log("terminating %s..." % self._boto_instance)
+ self._boto_instance.terminate()
self._boto_instance.update()
+ self.log(" done\n")
self.log('instance %s\n' % (self._boto_instance.state,))
@property
@@ -320,24 +336,28 @@
"""Connect to the instance as `user`. """
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(AcceptAllPolicy())
+ self.log('ssh connect to %s: ' % self.hostname)
connect_args = {
'username': username,
'pkey': self.private_key,
'allow_agent': False,
'look_for_keys': False,
}
- for count in range(10):
+ for count in range(20):
try:
ssh.connect(self.hostname, **connect_args)
- except (socket.error, paramiko.AuthenticationException), e:
- self.log('_connect: %r\n' % (e,))
+ except (socket.error, paramiko.AuthenticationException, EOFError), e:
+ self.log('.')
+ if getattr(e, 'errno', None) not in (
+ errno.ECONNREFUSED, errno.ETIMEDOUT, errno.EHOSTUNREACH):
+ self.log('ssh _connect: %r\n' % (e,))
if count < 9:
time.sleep(5)
- self.log('retrying...')
else:
raise
else:
break
+ self.log(' ok!\n')
return EC2InstanceConnection(self, username, ssh)
def _upload_local_key(self, conn, remote_filename):
@@ -399,6 +419,7 @@
from_scratch_ec2test
% {'launchpad-login': self._launchpad_login})
self._from_scratch = False
+ self.log('done running from_scratch setup\n')
return conn
self._ensure_ec2test_user_has_keys()
return self._connect('ec2test')
@@ -492,13 +513,9 @@
'%r must match a single %s file' % (pattern, file_kind))
return matches[0]
- def check_bundling_prerequisites(self, name, credentials):
+ def check_bundling_prerequisites(self, name):
"""Check, as best we can, that all the files we need to bundle exist.
"""
- if subprocess.call(['which', 'ec2-register']):
- raise BzrCommandError(
- '`ec2-register` command not found. '
- 'Try `sudo apt-get install ec2-api-tools`.')
local_ec2_dir = os.path.expanduser('~/.ec2')
if not os.path.exists(local_ec2_dir):
raise BzrCommandError(
@@ -516,12 +533,13 @@
# The bucket `name` needs to exist and be accessible. We create it
# here to reserve the name. If the bucket already exists and conforms
# to the above requirements, this is a no-op.
- credentials.connect_s3().create_bucket(name)
+ self._credentials.connect_s3().create_bucket(
+ name, location=self._credentials.region_name)
def bundle(self, name, credentials):
"""Bundle, upload and register the instance as a new AMI.
- :param name: The name-to-be of the new AMI.
+ :param name: The name-to-be of the new AMI, eg 'launchpad-ec2test500'.
:param credentials: An `EC2Credentials` object.
"""
connection = self.connect()
@@ -571,21 +589,20 @@
mfilename = os.path.basename(manifest)
manifest_path = os.path.join(name, mfilename)
- env = os.environ.copy()
- if 'JAVA_HOME' not in os.environ:
- env['JAVA_HOME'] = '/usr/lib/jvm/default-java'
now = datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S UTC")
- description = "Created %s" % now
- cmd = [
- 'ec2-register',
- '--private-key=%s' % self.local_pk,
- '--cert=%s' % self.local_cert,
- '--name=%s' % (name,),
- '--description=%s' % description,
- manifest_path,
- ]
- self.log("Executing command: %s" % ' '.join(cmd))
- subprocess.check_call(cmd, env=env)
+ description = "launchpad ec2test created %s by %r on %s" % (
+ now,
+ os.environ.get('EMAIL', '<unknown>'),
+ socket.gethostname())
+
+ self.log('registering image: ')
+ image_id = credentials.connect('bundle').conn.register_image(
+ name=name,
+ description=description,
+ image_location=manifest_path,
+ )
+ self.log('ok\n')
+ self.log('** new instance: %r\n' % (image_id,))
class EC2InstanceConnection:
@@ -601,6 +618,8 @@
def sftp(self):
if self._sftp is None:
self._sftp = self._ssh.open_sftp()
+ if self._sftp is None:
+ raise AssertionError("failed to open sftp connection")
return self._sftp
def perform(self, cmd, ignore_failure=False, out=None, err=None):
=== modified file 'lib/devscripts/ec2test/testrunner.py'
--- lib/devscripts/ec2test/testrunner.py 2011-11-14 18:14:15 +0000
+++ lib/devscripts/ec2test/testrunner.py 2011-11-17 04:15:30 +0000
@@ -323,6 +323,15 @@
# really wrong with the server or suite.
user_connection.perform("sudo shutdown -P +%d &" % self.timeout)
as_user = user_connection.perform
+ as_user("sudo mount -o remount,data=writeback,commit=3600,async,relatime /")
+ for d in ['/tmp', '/var/tmp']:
+ as_user('sudo mkdir -p %s && sudo mount -t tmpfs none %s' % (d, d))
+ as_user("sudo service postgresql-8.4 stop"
+ "; sudo mv /var/lib/postgresql /tmp/postgresql-tmp"
+ "&& sudo mkdir /var/lib/postgresql"
+ "&& sudo mount -t tmpfs none /var/lib/postgresql"
+ "&& sudo mv /tmp/postgresql-tmp/* /var/lib/postgresql"
+ "&& sudo service postgresql-8.4 start")
as_user("sudo add-apt-repository ppa:bzr")
as_user("sudo add-apt-repository ppa:launchpad")
as_user("sudo aptitude update")
@@ -358,9 +367,11 @@
user_connection = self._instance.connect()
# Clean up the test branch left in the instance image.
user_connection.perform('rm -rf /var/launchpad/test')
+ user_connection.perform('sudo mkdir /var/launchpad/test '
+ '&& sudo mount -t tmpfs none /var/launchpad/test')
# Get trunk.
user_connection.run_with_ssh_agent(
- 'bzr branch %s /var/launchpad/test' % (self._trunk_branch,))
+ 'bzr branch --use-existing-dir %s /var/launchpad/test' % (self._trunk_branch,))
# Merge the branch in.
if self._branch is not None:
user_connection.run_with_ssh_agent(
@@ -378,8 +389,9 @@
'bzr branch --standalone %s %s' % (src, fulldest))
# prepare fresh copy of sourcecode and buildout sources for building
p = user_connection.perform
- p('rm -rf /var/launchpad/tmp')
- p('mkdir /var/launchpad/tmp')
+ p('rm -rf /var/launchpad/tmp'
+ '&& mkdir /var/launchpad/tmp '
+ '&& sudo mount -t tmpfs none /var/launchpad/tmp')
p('mv /var/launchpad/sourcecode /var/launchpad/tmp/sourcecode')
p('mkdir /var/launchpad/tmp/eggs')
p('mkdir /var/launchpad/tmp/yui')
Follow ups
