launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #05536
[Merge] lp:~mbp/launchpad/ec2-region into lp:launchpad
Martin Pool has proposed merging lp:~mbp/launchpad/ec2-region into lp:launchpad with lp:~mbp/launchpad/ec2-tmpfs as a prerequisite.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~mbp/launchpad/ec2-region/+merge/82373
This adds a --region option to ec2, so that if you live in the asia-pacific or europe you can get lower latency, or if the us-east-1 zone is on fire you can work somewhere else.
Needs more testing.
ec2 is a bit hacky. I don't know if this makes it too much worse.
--
https://code.launchpad.net/~mbp/launchpad/ec2-region/+merge/82373
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~mbp/launchpad/ec2-region into lp:launchpad.
=== modified file 'lib/devscripts/ec2test/builtins.py'
--- lib/devscripts/ec2test/builtins.py 2011-11-16 09:49:32 +0000
+++ lib/devscripts/ec2test/builtins.py 2011-11-16 09:49:33 +0000
@@ -129,6 +129,14 @@
"and --file."))
+region_option = Option(
+ 'region',
+ type=str,
+ help=("Name of the AWS region in which to run the instance. "
+ "Must be the same as the region holding the image file. "
+ "For example, 'us-west-1'."))
+
+
def filename_type(filename):
"""An option validator for filenames.
@@ -502,6 +510,7 @@
postmortem_option,
debug_option,
include_download_cache_changes_option,
+ region_option,
ListOption(
'demo', type=str,
help="Allow this netmask to connect to the instance."),
@@ -570,6 +579,7 @@
instance_type_option,
postmortem_option,
debug_option,
+ region_option,
ListOption(
'extra-update-image-command', type=str,
help=('Run this command (with an ssh agent) on the image before '
@@ -586,6 +596,7 @@
def run(self, ami_name, machine=None, instance_type='m1.large',
debug=False, postmortem=False, extra_update_image_command=None,
+ region=None,
public=False):
set_trace_if(debug)
@@ -599,17 +610,14 @@
for variable in ['LANG', 'LC_ALL', 'LC_TIME']:
os.environ.pop(variable, None)
- credentials = EC2Credentials.load_from_file()
-
session_name = EC2SessionName.make(EC2TestRunner.name)
instance = EC2Instance.make(
session_name, instance_type, machine,
- credentials=credentials)
- instance.check_bundling_prerequisites(
- ami_name, credentials)
+ region=region)
+ instance.check_bundling_prerequisites(ami_name)
instance.set_up_and_run(
postmortem, True, self.update_image, instance,
- extra_update_image_command, ami_name, credentials, public)
+ extra_update_image_command, ami_name, instance._credentials, public)
def update_image(self, instance, extra_update_image_command, ami_name,
credentials, public):
@@ -669,9 +677,13 @@
The first in the list is the default image.
"""
- def run(self):
- credentials = EC2Credentials.load_from_file()
+ takes_options = [
+ region_option,
+ ]
+
+ def run(self, region=None):
session_name = EC2SessionName.make(EC2TestRunner.name)
+ credentials = EC2Credentials.load_from_file(region_name=region)
account = credentials.connect(session_name)
format = "%5s %-12s %-12s %-12s %s\n"
self.outf.write(
@@ -698,6 +710,7 @@
aliases = ["ls"]
takes_options = [
+ region_option,
Option('show-urls',
help="Include more information about each instance"),
Option('all', short_name='a',
@@ -766,9 +779,9 @@
': '.join((state, str(num)))
for (state, num) in sorted(list(by_state.items())))
- def run(self, show_urls=False, all=False):
- credentials = EC2Credentials.load_from_file()
+ def run(self, show_urls=False, all=False, region=None):
session_name = EC2SessionName.make(EC2TestRunner.name)
+ credentials = EC2Credentials.load_from_file(region_name=region)
account = credentials.connect(session_name)
instances = list(self.iter_instances(account))
if len(instances) == 0:
=== modified file 'lib/devscripts/ec2test/credentials.py'
--- lib/devscripts/ec2test/credentials.py 2011-05-27 19:57:02 +0000
+++ lib/devscripts/ec2test/credentials.py 2011-11-16 09:49:33 +0000
@@ -12,6 +12,7 @@
import os
import boto
+import boto.ec2
from bzrlib.errors import BzrCommandError
from devscripts.ec2test.account import EC2Account
@@ -32,12 +33,13 @@
DEFAULT_CREDENTIALS_FILE = '~/.ec2/aws_id'
- def __init__(self, identifier, secret):
+ def __init__(self, identifier, secret, region_name):
self.identifier = identifier
self.secret = secret
+ self.region_name = region_name or 'us-east-1'
@classmethod
- def load_from_file(cls, filename=None):
+ def load_from_file(cls, filename=None, region_name=None):
"""Load the EC2 credentials from 'filename'."""
if filename is None:
filename = os.path.expanduser(cls.DEFAULT_CREDENTIALS_FILE)
@@ -50,15 +52,18 @@
secret = aws_file.readline().strip()
finally:
aws_file.close()
- return cls(identifier, secret)
+ return cls(identifier, secret, region_name)
def connect(self, name):
"""Connect to EC2 with these credentials.
- :param name: ???
+ :param name: Arbitrary local name for the object.
:return: An `EC2Account` connected to EC2 with these credentials.
"""
- conn = boto.connect_ec2(self.identifier, self.secret)
+ conn = boto.ec2.connect_to_region(
+ self.region_name,
+ aws_access_key_id=self.identifier,
+ aws_secret_access_key=self.secret)
return EC2Account(name, conn)
def connect_s3(self):
=== modified file 'lib/devscripts/ec2test/instance.py'
--- lib/devscripts/ec2test/instance.py 2011-11-16 09:49:32 +0000
+++ lib/devscripts/ec2test/instance.py 2011-11-16 09:49:33 +0000
@@ -85,8 +85,6 @@
cat >> /etc/apt/sources.list << EOF
deb http://ppa.launchpad.net/launchpad/ubuntu $DISTRIB_CODENAME main
deb http://ppa.launchpad.net/bzr/ubuntu $DISTRIB_CODENAME main
-deb http://us.ec2.archive.ubuntu.com/ubuntu/ $DISTRIB_CODENAME multiverse
-deb-src http://us.ec2.archive.ubuntu.com/ubuntu/ $DISTRIB_CODENAME main
EOF
export DEBIAN_FRONTEND=noninteractive
@@ -113,7 +111,7 @@
apt-key adv --recv-keys --keyserver pool.sks-keyservers.net cbede690576d1e4e813f6bb3ebaf723d37b19b80
aptitude update
-aptitude -y install language-pack-en # Do this first so later things don't complain about locales
+LANG=C aptitude -y install language-pack-en # Do this first so later things don't complain about locales
aptitude -y full-upgrade
# This next part is cribbed from rocketfuel-setup
@@ -210,7 +208,7 @@
@classmethod
def make(cls, name, instance_type, machine_id, demo_networks=None,
- credentials=None):
+ credentials=None, region=None):
"""Construct an `EC2Instance`.
:param name: The name to use for the key pair and security group for
@@ -225,6 +223,7 @@
:param demo_networks: A list of networks to add to the security group
to allow access to the instance.
:param credentials: An `EC2Credentials` object.
+ :param region: A string region name eg 'us-east-1'.
"""
# This import breaks in the test environment. Do it here so
# that unit tests (which don't use this factory) can still
@@ -246,7 +245,7 @@
user_key = get_user_key()
if credentials is None:
- credentials = EC2Credentials.load_from_file()
+ credentials = EC2Credentials.load_from_file(region_name=region)
# Make the EC2 connection.
account = credentials.connect(name)
@@ -274,12 +273,14 @@
raise BzrCommandError(
'you must have set your launchpad login in bzr.')
- return EC2Instance(
+ instance = EC2Instance(
name, image, instance_type, demo_networks, account,
- from_scratch, user_key, login)
+ from_scratch, user_key, login, region)
+ instance._credentials = credentials
+ return instance
def __init__(self, name, image, instance_type, demo_networks, account,
- from_scratch, user_key, launchpad_login):
+ from_scratch, user_key, launchpad_login, region):
self._name = name
self._image = image
self._account = account
@@ -289,6 +290,7 @@
self._from_scratch = from_scratch
self._user_key = user_key
self._launchpad_login = launchpad_login
+ self._region = region
def log(self, msg):
"""Log a message on stdout, flushing afterwards."""
@@ -363,7 +365,7 @@
ssh.connect(self.hostname, **connect_args)
except (socket.error, paramiko.AuthenticationException, EOFError), e:
self.log('.')
- if getattr(e, 'errno') == errno.ECONNREFUSED:
+ if getattr(e, 'errno', None) == errno.ECONNREFUSED:
# Pretty normal if the machine has started but sshd isn't
# up yet. Don't make a fuss.
time.sleep(1)
@@ -531,13 +533,9 @@
'%r must match a single %s file' % (pattern, file_kind))
return matches[0]
- def check_bundling_prerequisites(self, name, credentials):
+ def check_bundling_prerequisites(self, name):
"""Check, as best we can, that all the files we need to bundle exist.
"""
- if subprocess.call(['which', 'ec2-register']):
- raise BzrCommandError(
- '`ec2-register` command not found. '
- 'Try `sudo apt-get install ec2-api-tools`.')
local_ec2_dir = os.path.expanduser('~/.ec2')
if not os.path.exists(local_ec2_dir):
raise BzrCommandError(
@@ -555,7 +553,8 @@
# The bucket `name` needs to exist and be accessible. We create it
# here to reserve the name. If the bucket already exists and conforms
# to the above requirements, this is a no-op.
- credentials.connect_s3().create_bucket(name)
+ self._credentials.connect_s3().create_bucket(
+ name, location=self._credentials.region_name)
def bundle(self, name, credentials):
"""Bundle, upload and register the instance as a new AMI.
@@ -610,21 +609,20 @@
mfilename = os.path.basename(manifest)
manifest_path = os.path.join(name, mfilename)
- env = os.environ.copy()
- if 'JAVA_HOME' not in os.environ:
- env['JAVA_HOME'] = '/usr/lib/jvm/default-java'
now = datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S UTC")
- description = "Created %s" % now
- cmd = [
- 'ec2-register',
- '--private-key=%s' % self.local_pk,
- '--cert=%s' % self.local_cert,
- '--name=%s' % (name,),
- '--description=%s' % description,
- manifest_path,
- ]
- self.log("Executing command: %s" % ' '.join(cmd))
- subprocess.check_call(cmd, env=env)
+ description = "Created %s by %r on %s" % (
+ now,
+ os.environ.get('EMAIL'),
+ socket.gethostname())
+
+ self.log('registering image: ')
+ credentials.connect('bundle').register_image(
+ name=name,
+ description=description,
+ image_location=manifest_path,
+ )
+ self.log('ok\n')
+
class EC2InstanceConnection:
