ubuntu-server-ec2-testing-dev team mailing list archive

[Daniel Watkins <daniel.watkins@xxxxxxxxxxxxx>]

Daniel Watkins has proposed merging lp:~daniel-thewatkins/ubuntu-server-ec2-testing/use-sstreams into lp:ubuntu-server-ec2-testing.

Requested reviews:
  Ben Howard (utlemming)

For more details, see:
https://code.launchpad.net/~daniel-thewatkins/ubuntu-server-ec2-testing/use-sstreams/+merge/235932

Replace the current query format with simplestreams.

This doesn't change the CLI interface, it maps between 'query' names and simplestreams names.

I've tested that this finds the same AMIs for all of the tests we run for 14.10 (and, furthermore, that actually running those tests succeeds).
-- 
https://code.launchpad.net/~daniel-thewatkins/ubuntu-server-ec2-testing/use-sstreams/+merge/235932
Your team Ubuntu Server ec2 Testing Developers is subscribed to branch lp:ubuntu-server-ec2-testing.

=== modified file 'src/execute_ubuntu_ec2_test.py'
--- src/execute_ubuntu_ec2_test.py	2014-09-24 08:32:14 +0000
+++ src/execute_ubuntu_ec2_test.py	2014-09-25 10:34:04 +0000
@@ -24,7 +24,7 @@
 import os
 import yaml
 import sys
-from ubuntu.ec2.testing import TestCase,TestCaseExecutor,retrieve_mi
+from ubuntu.ec2.testing import MachineImage, TestCase, TestCaseExecutor
 from ubuntu.ec2 import settings
 
 # Defaults
@@ -144,13 +144,12 @@
                       options.region,options.storage,
                       options.build_date)
 
-        l_mi = retrieve_mi(release=options.release,
-                           variant=options.variant,
-                           arch=options.arch,
-                           region=options.region,
-                           instance_type=options.storage,
-                           release_date=options.build_date,
-                           virt_type=options.virt_type)
+        l_mi = MachineImage.from_simple_stream(release=options.release,
+                                               arch=options.arch,
+                                               region=options.region,
+                                               instance_type=options.storage,
+                                               release_date=options.build_date,
+                                               virt_type=options.virt_type)
 
         if l_mi is None:
             logging.error("Could not find machine image for testing")

=== modified file 'src/ubuntu/ec2/testing.py'
--- src/ubuntu/ec2/testing.py	2014-09-24 08:32:45 +0000
+++ src/ubuntu/ec2/testing.py	2014-09-25 10:34:04 +0000
@@ -36,18 +36,18 @@
 '''
 
 import gzip
+import json
 import logging
 import os
 import pipes
-import re
 import stat
 import subprocess
 import time
-import urllib
 import uuid
 from email import MIMEBase
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
+from operator import itemgetter
 from StringIO import StringIO
 
 import boto.ec2
@@ -58,12 +58,9 @@
 from ubuntu.ec2 import settings
 
 # Note optionality on final \t and match due to change in query format
-QUERY_REGEX = '(\S*)\t(\S*)\t(\S*)\t(\S*)\t(\S*)\t(\S*)\t(\S*)\t(\S*)\t(\S*)\t(\S*)\t*(\S*)$'
-UEC_IMAGES_LATEST_URL = 'http://uec-images.ubuntu.com/query/%s/%s/daily.current.txt'
-UEC_IMAGES_ALL_URL = 'http://uec-images.ubuntu.com/query/%s/%s/daily.txt'
 LATEST_RELEASE = 'current'
 DEFAULT_TEST_DIR = os.path.expanduser('~/tests')
-UUID_PREFIX = "uec2"
+UUID_PREFIX = "cloud"
 
 CI_SCRIPT = "text/x-shellscript"
 CI_SCRIPT_START = "#!"
@@ -86,6 +83,7 @@
     ''' Marker exception for any issues during test execution '''
     pass
 
+
 class MachineImage:
     '''
     This class encapsulates a Machine Image
@@ -104,20 +102,38 @@
             - virt_type:     %s
         '''
 
-    def __init__ (self):
+    # Our names -> simplestreams names
+    name_conversions = {
+        'virt_type': {
+            'paravirtual': 'pv',
+        },
+        'instance_type': {
+            'ebs-io1': 'io1',
+            'ebs-ssd': 'ssd',
+            'instance-store': 'instance',
+        }
+    }
+    reverse_name_conversions = {
+        attr: {v: k for k, v in mappings.items()}
+        for attr, mappings in name_conversions.items()
+    }
+
+    def __init__ (self, release=None, release_type=None, release_date=None,
+                  instance_type=None, arch=None, region=None, ami=None,
+                  virt_type=None):
         '''
         Constructs an empty MachineImage
         '''
-        self.release = None
+        self.release = release
         self.variant = None
-        self.release_type = None
-        self.release_date = None
-        self.instance_type = None
-        self.arch = None
-        self.region = None
-        self.ami = None
+        self.release_type = release_type
+        self.release_date = release_date
+        self.instance_type = instance_type
+        self.arch = arch
+        self.region = region
+        self.ami = ami
         self.aki = None
-        self.virt_type = None
+        self.virt_type = virt_type
 
 
     def __str__(self):
@@ -155,6 +171,64 @@
             if len(t_raw_data) > 10:
                 self.virt_type = t_raw_data[10]
 
+    @classmethod
+    def from_simple_stream(cls, release='natty', arch='i386',
+                           region='eu-west-1', instance_type='instance-store',
+                           release_date='current', virt_type='paravirtual'):
+        logging.debug("Looking for release:%s arch:%s region:%s"
+                      " instance_type:%s release_date:%s virt_type:%s",
+                      release, arch, region, instance_type, release_date,
+                      virt_type)
+
+        def _get_filter_strings():
+            filters = {
+                'release': release,
+                'arch': arch,
+                'region': region,
+                'root_store': cls.name_conversions['instance_type'].get(
+                    instance_type, instance_type),
+                'virt': cls.name_conversions['virt_type'].get(virt_type,
+                                                              virt_type),
+            }
+            if release_date != LATEST_RELEASE:
+                filters['version_name'] = release_date
+            return ['{}={}'.format(k, v) for k, v in filters.items()]
+
+        def _get_result_dicts(filter_strings):
+            out = subprocess.check_output(
+                ['sstream-query', 'http://cloud-images.ubuntu.com/daily',
+                 'datatype=image-ids'] + filter_strings)
+            return [json.loads(line.replace("'", '"'))
+                    for line in out.split('\n') if line]
+
+        results = _get_result_dicts(_get_filter_strings())
+        if len(results) == 0:
+            return None
+        if release_date == LATEST_RELEASE:
+            product_dict = sorted(results, key=itemgetter('version_name'))[-1]
+        else:
+            if len(results) > 1:
+                raise TestException('More than one AMI found for parameters.')
+            product_dict = results[0]
+        logging.info("Found AMI %s ", product_dict['id'])
+        return cls.from_product_dict(product_dict)
+
+    @classmethod
+    def from_product_dict(cls, product_dict):
+        mi = cls(
+            release=product_dict['release'],
+            release_type=product_dict['label'],
+            release_date=product_dict['version_name'],
+            instance_type=cls.reverse_name_conversions['instance_type'].get(
+                product_dict['root_store'], product_dict['root_store']),
+            arch=product_dict['arch'],
+            region=product_dict['region'],
+            ami=product_dict['id'],
+            virt_type=cls.reverse_name_conversions['virt_type'].get(
+                product_dict['virt'], product_dict['virt']),
+        )
+        logging.debug(mi)
+        return mi
 
 class TestCase:
     '''
@@ -980,67 +1054,6 @@
         return self.complete
 
 
-def retrieve_mi(release='natty', variant='server', arch='i386',
-                region='eu-west-1', instance_type='instance-store',
-                release_date=LATEST_RELEASE, virt_type='paravirtual'):
-    """
-    This function uses uec-images.ubuntu.com to retrieve the latest AMI for
-    the specified combination of parameters.
-    """
-    # FIXME: Cache lookups for speed - good for multi testing
-    l_url = None
-    # Check to see which file to parse
-    if release_date == LATEST_RELEASE:
-        l_url = UEC_IMAGES_LATEST_URL % (release, variant)
-    else:
-        # If not the latest then will be a '20100102' type image
-        l_url = UEC_IMAGES_ALL_URL % (release, variant)
-
-    logging.info("Retieving ec2 AMI information from %s", l_url)
-    l_fh = urllib.urlopen(l_url)
-    l_regex = re.compile(QUERY_REGEX)
-    l_mi = MachineImage()
-    l_ami_found = False
-
-    logging.debug("Looking for release:%s variant:%s arch:%s region:%s "
-                  "instance_type:%s release_date:%s virt_type:%s",
-                  release, variant, arch, region, instance_type,
-                  release_date, virt_type)
-    for ami in l_fh:
-        logging.debug(ami)
-        l_raw = l_regex.findall(ami)
-        l_mi.parse_mi(l_raw)
-        if (release_date == LATEST_RELEASE):
-            if ((l_mi.release == release) and
-                (l_mi.variant == variant) and
-                (l_mi.arch == arch) and
-                (l_mi.region == region) and
-                (l_mi.instance_type == instance_type) and
-                (l_mi.virt_type == virt_type)):
-                logging.info("Found AMI %s ", l_mi.ami)
-                logging.debug(l_mi)
-                l_ami_found = True
-                break
-        else:
-            if ((l_mi.release == release) and
-                (l_mi.variant == variant) and
-                (l_mi.arch == arch) and
-                (l_mi.region == region) and
-                (l_mi.instance_type == instance_type) and
-                (l_mi.release_date == release_date) and
-                (l_mi.virt_type == virt_type)):
-                logging.info("Found AMI %s ", l_mi.ami)
-                logging.debug(l_mi)
-                l_ami_found = True
-                break
-
-    l_fh.close()
-
-    if l_ami_found == True:
-        return l_mi
-    else:
-        return None
-
 # These two functions where borrowed from awstrial
 # see http://launchpad.net/awstrial
 def part_item(part_content, part_type=None, filename=None):
@@ -1083,4 +1096,3 @@
         return(ofile.getvalue())
     else:
         return(outer.as_string())
-