← Back to team overview

txawsteam team mailing list archive

[Merge] lp:~therve/txaws/ebs-support into lp:txaws

 

Thomas Herve has proposed merging lp:~therve/txaws/ebs-support into lp:txaws.

Requested reviews:
    txAWS Team (txawsteam)

This is ready to review in the attached branch. I also took the opportunity to remove the need of namespaces, as it feels useless to me.
-- 
https://code.launchpad.net/~therve/txaws/ebs-support/+merge/10742
Your team txAWS Team is subscribed to branch lp:txaws.
=== modified file 'txaws/ec2/client.py'
--- txaws/ec2/client.py	2009-08-21 03:26:35 +0000
+++ txaws/ec2/client.py	2009-08-26 13:50:25 +0000
@@ -3,7 +3,7 @@
 
 """EC2 client support."""
 
-from base64 import b64encode
+from datetime import datetime
 from urllib import quote
 
 from twisted.web.client import getPage
@@ -19,7 +19,7 @@
     """An Amazon EC2 Reservation.
 
     @attrib reservation_id: Unique ID of the reservation.
-    @attrib owner_id: AWS Access Key ID of the user who owns the reservation. 
+    @attrib owner_id: AWS Access Key ID of the user who owns the reservation.
     @attrib groups: A list of security groups.
     """
     def __init__(self, reservation_id, owner_id, groups=None):
@@ -72,11 +72,32 @@
         self.reservation = reservation
 
 
+class Volume(object):
+    """An EBS volume instance."""
+
+    def __init__(self, id, size, status, create_time):
+        self.id = id
+        self.size = size
+        self.status = status
+        self.create_time = create_time
+        self.attachments = []
+
+
+class Attachment(object):
+    """An attachment of a L{Volume}."""
+
+    def __init__(self, instance_id, snapshot_id, availability_zone, status,
+                 attach_time):
+        self.instance_id = instance_id
+        self.snapshot_id = snapshot_id
+        self.availability_zone = availability_zone
+        self.status = status
+        self.attach_time = attach_time
+
+
 class EC2Client(object):
     """A client for EC2."""
 
-    name_space = '{http://ec2.amazonaws.com/doc/2008-12-01/}'
-
     def __init__(self, creds=None, query_factory=None):
         """Create an EC2Client.
 
@@ -102,7 +123,7 @@
         """
         Parse the reservations XML payload that is returned from an AWS
         describeInstances API call.
-       
+
         Instead of returning the reservations as the "top-most" object, we
         return the object that most developers and their code will be
         interested in: the instances. In instances reservation is available on
@@ -111,53 +132,38 @@
         root = XML(xml_bytes)
         results = []
         # May be a more elegant way to do this:
-        for reservation_data in root.find(self.name_space + 'reservationSet'):
+        for reservation_data in root.find("reservationSet"):
             # Get the security group information.
             groups = []
-            for group_data in reservation_data.find(
-                self.name_space + 'groupSet'):
-                group_id = group_data.findtext(self.name_space + 'groupId')
+            for group_data in reservation_data.find("groupSet"):
+                group_id = group_data.findtext("groupId")
                 groups.append(group_id)
             # Create a reservation object with the parsed data.
             reservation = Reservation(
-                reservation_id=reservation_data.findtext(
-                    self.name_space + 'reservationId'),
-                owner_id=reservation_data.findtext(
-                    self.name_space + 'ownerId'),
+                reservation_id=reservation_data.findtext("reservationId"),
+                owner_id=reservation_data.findtext("ownerId"),
                 groups=groups)
             # Get the list of instances.
             instances = []
-            for instance_data in reservation_data.find(
-                self.name_space + 'instancesSet'):
-                instance_id = instance_data.findtext(
-                    self.name_space + 'instanceId')
+            for instance_data in reservation_data.find("instancesSet"):
+                instance_id = instance_data.findtext("instanceId")
                 instance_state = instance_data.find(
-                    self.name_space + 'instanceState').findtext(
-                        self.name_space + 'name')
-                instance_type = instance_data.findtext(
-                    self.name_space + 'instanceType')
-                image_id = instance_data.findtext(self.name_space + 'imageId')
-                private_dns_name = instance_data.findtext(
-                    self.name_space + 'privateDnsName')
-                dns_name = instance_data.findtext(self.name_space + 'dnsName')
-                key_name = instance_data.findtext(self.name_space + 'keyName')
-                ami_launch_index = instance_data.findtext(
-                    self.name_space + 'amiLaunchIndex')
-                launch_time = instance_data.findtext(
-                    self.name_space + 'launchTime')
-                placement = instance_data.find(
-                    self.name_space + 'placement').findtext(
-                        self.name_space + 'availabilityZone')
+                    "instanceState").findtext("name")
+                instance_type = instance_data.findtext("instanceType")
+                image_id = instance_data.findtext("imageId")
+                private_dns_name = instance_data.findtext("privateDnsName")
+                dns_name = instance_data.findtext("dnsName")
+                key_name = instance_data.findtext("keyName")
+                ami_launch_index = instance_data.findtext("amiLaunchIndex")
+                launch_time = instance_data.findtext("launchTime")
+                placement = instance_data.find("placement").findtext(
+                    "availabilityZone")
                 products = []
-                for product_data in instance_data.find(
-                    self.name_space + 'productCodesSet'):
-                    product_code = product_data.findtext(
-                        self.name_space + 'productCode')
+                for product_data in instance_data.find("productCodesSet"):
+                    product_code = product_data.findtext("productCode")
                     products.append(product_code)
-                kernel_id = instance_data.findtext(
-                    self.name_space + 'kernelId')
-                ramdisk_id = instance_data.findtext(
-                    self.name_space + 'ramdiskId')
+                kernel_id = instance_data.findtext("kernelId")
+                ramdisk_id = instance_data.findtext("ramdiskId")
                 instance = Instance(
                     instance_id, instance_state, instance_type, image_id,
                     private_dns_name, dns_name, key_name, ami_launch_index,
@@ -169,7 +175,7 @@
 
     def terminate_instances(self, *instance_ids):
         """Terminate some instances.
-        
+
         @param instance_ids: The ids of the instances to terminate.
         @return: A deferred which on success gives an iterable of
             (id, old-state, new-state) tuples.
@@ -185,17 +191,48 @@
         root = XML(xml_bytes)
         result = []
         # May be a more elegant way to do this:
-        for instance in root.find(self.name_space + 'instancesSet'):
-            instanceId = instance.findtext(self.name_space + 'instanceId')
-            previousState = instance.find(
-                self.name_space + 'previousState').findtext(
-                    self.name_space + 'name')
-            shutdownState = instance.find(
-                self.name_space + 'shutdownState').findtext(
-                    self.name_space + 'name')
+        for instance in root.find("instancesSet"):
+            instanceId = instance.findtext("instanceId")
+            previousState = instance.find("previousState").findtext(
+                "name")
+            shutdownState = instance.find("shutdownState").findtext(
+                "name")
             result.append((instanceId, previousState, shutdownState))
         return result
 
+    def describe_volumes(self):
+        """Describe available volumes."""
+        q = self.query_factory("DescribeVolumes", self.creds)
+        d = q.submit()
+        return d.addCallback(self._parse_volumes)
+
+    def _parse_volumes(self, xml_bytes):
+        root = XML(xml_bytes)
+        result = []
+        for volume_data in root.find("volumeSet"):
+            volume_id = volume_data.findtext("volumeId")
+            size = int(volume_data.findtext("size"))
+            status = volume_data.findtext("status")
+            create_time = volume_data.findtext("createTime")
+            create_time = datetime.strptime(
+                create_time[:19], "%Y-%m-%dT%H:%M:%S")
+            volume = Volume(volume_id, size, status, create_time)
+            result.append(volume)
+            for attachment_data in volume_data.find("attachmentSet"):
+                instance_id = attachment_data.findtext("instanceId")
+                snapshot_id = attachment_data.findtext("snapshotId")
+                availability_zone = attachment_data.findtext(
+                    "availabilityZone")
+                status = attachment_data.findtext("status")
+                attach_time = attachment_data.findtext("attachTime")
+                attach_time = datetime.strptime(
+                    attach_time[:19], "%Y-%m-%dT%H:%M:%S")
+                attachment = Attachment(
+                    instance_id, snapshot_id, availability_zone, status,
+                    attach_time)
+                volume.attachments.append(attachment)
+        return result
+
 
 class Query(object):
     """A query that may be submitted to EC2."""
@@ -204,7 +241,7 @@
         """Create a Query to submit to EC2."""
         # Require params (2008-12-01 API):
         # Version, SignatureVersion, SignatureMethod, Action, AWSAccessKeyId,
-        # Timestamp || Expires, Signature, 
+        # Timestamp || Expires, Signature,
         self.params = {'Version': '2008-12-01',
             'SignatureVersion': '2',
             'SignatureMethod': 'HmacSHA1',
@@ -242,7 +279,7 @@
 
     def sign(self):
         """Sign this query using its built in credentials.
-        
+
         This prepares it to be sent, and should be done as the last step before
         submitting the query. Signing is done automatically - this is a public
         method to facilitate testing.

=== modified file 'txaws/ec2/tests/test_client.py'
--- txaws/ec2/tests/test_client.py	2009-08-21 03:26:35 +0000
+++ txaws/ec2/tests/test_client.py	2009-08-26 14:31:01 +0000
@@ -1,6 +1,7 @@
 # Copyright (C) 2009 Robert Collins <robertc@xxxxxxxxxxxxxxxxx>
 # Licenced under the txaws licence available at /LICENSE in the txaws source.
 
+from datetime import datetime
 import os
 
 from twisted.internet.defer import succeed
@@ -86,6 +87,31 @@
 """
 
 
+sample_describe_volumes_result = """<?xml version="1.0"?>
+<DescribeVolumesResponse xmlns="http://ec2.amazonaws.com/doc/2008-12-01/";>
+  <volumeSet>
+    <item>
+      <volumeId>vol-4282672b</volumeId>
+      <size>800</size>
+      <status>in-use</status>
+      <createTime>2008-05-07T11:51:50.000Z</createTime>
+      <attachmentSet>
+        <item>
+          <volumeId>vol-4282672b</volumeId>
+          <instanceId>i-6058a509</instanceId>
+          <size>800</size>
+          <snapshotId>snap-12345678</snapshotId>
+          <availabilityZone>us-east-1a</availabilityZone>
+          <status>attached</status>
+          <attachTime>2008-05-07T12:51:50.000Z</attachTime>
+        </item>
+      </attachmentSet>
+    </item>
+  </volumeSet>
+</DescribeVolumesResponse>
+"""
+
+
 class ReservationTestCase(TXAWSTestCase):
 
     def test_reservation_creation(self):
@@ -118,7 +144,7 @@
 
 
 class TestEC2Client(TXAWSTestCase):
-    
+
     def test_init_no_creds(self):
         os.environ['AWS_SECRET_ACCESS_KEY'] = 'foo'
         os.environ['AWS_ACCESS_KEY_ID'] = 'bar'
@@ -160,7 +186,6 @@
         self.assertEquals(instance.kernel_id, "aki-b51cf9dc")
         self.assertEquals(instance.ramdisk_id, "ari-b31cf9da")
 
-
     def test_parse_reservation(self):
         ec2 = client.EC2Client(creds='foo')
         results = ec2._parse_instances(sample_describe_instances_result)
@@ -286,3 +311,38 @@
         query.sign()
         self.assertEqual('4hEtLuZo9i6kuG3TOXvRQNOrE/U=',
             query.params['Signature'])
+
+
+class TestEBS(TXAWSTestCase):
+
+    def test_describe_volumes(self):
+
+        class StubQuery(object):
+            def __init__(stub, action, creds):
+                self.assertEqual(action, "DescribeVolumes")
+                self.assertEqual("foo", creds)
+
+            def submit(self):
+                return succeed(sample_describe_volumes_result)
+
+        def check_parsed_volumes(volumes):
+            self.assertEquals(len(volumes), 1)
+            volume = volumes[0]
+            self.assertEquals(volume.id, "vol-4282672b")
+            self.assertEquals(volume.size, 800)
+            self.assertEquals(volume.status, "in-use")
+            create_time = datetime(2008, 05, 07, 11, 51, 50)
+            self.assertEquals(volume.create_time, create_time)
+            self.assertEquals(len(volume.attachments), 1)
+            attachment = volume.attachments[0]
+            self.assertEquals(attachment.instance_id, "i-6058a509")
+            self.assertEquals(attachment.snapshot_id, "snap-12345678")
+            self.assertEquals(attachment.availability_zone, "us-east-1a")
+            self.assertEquals(attachment.status, "attached")
+            attach_time = datetime(2008, 05, 07, 12, 51, 50)
+            self.assertEquals(attachment.attach_time, attach_time)
+
+        ec2 = client.EC2Client(creds="foo", query_factory=StubQuery)
+        d = ec2.describe_volumes()
+        d.addCallback(check_parsed_volumes)
+        return d

=== modified file 'txaws/storage/client.py'
--- txaws/storage/client.py	2009-08-20 12:15:12 +0000
+++ txaws/storage/client.py	2009-08-26 13:50:25 +0000
@@ -19,9 +19,6 @@
 from txaws.util import XML, calculate_md5
 
 
-name_space = '{http://s3.amazonaws.com/doc/2006-03-01/}'
-
-
 class S3Request(object):
 
     def __init__(self, verb, bucket=None, object_name=None, data='',
@@ -114,10 +111,10 @@
         Parse XML bucket list response.
         """
         root = XML(response)
-        for bucket in root.find(name_space + 'Buckets'):
-            timeText = bucket.findtext(name_space + 'CreationDate')
+        for bucket in root.find("Buckets"):
+            timeText = bucket.findtext("CreationDate")
             yield {
-                'name': bucket.findtext(name_space + 'Name'),
+                'name': bucket.findtext("Name"),
                 'created': Time.fromISO8601TimeAndDate(timeText),
                 }
 

=== modified file 'txaws/util.py'
--- txaws/util.py	2009-08-20 12:15:12 +0000
+++ txaws/util.py	2009-08-26 13:50:25 +0000
@@ -9,14 +9,14 @@
 import hmac
 import time
 
-# Import XML from somwhere; here in one place to prevent duplication.
+# Import XMLTreeBuilder from somwhere; here in one place to prevent duplication.
 try:
-    from xml.etree.ElementTree import XML
+    from xml.etree.ElementTree import XMLTreeBuilder
 except ImportError:
-    from elementtree.ElementTree import XML
-
-
-__all__ = ['hmac_sha1', 'iso8601time']
+    from elementtree.ElementTree import XMLTreeBuilder
+
+
+__all__ = ["hmac_sha1", "iso8601time", "XML"]
 
 
 def calculate_md5(data):
@@ -38,3 +38,17 @@
         return time.strftime("%Y-%m-%dT%H:%M:%SZ", time_tuple)
     else:
         return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+
+class NamespaceFixXmlTreeBuilder(XMLTreeBuilder):
+
+    def _fixname(self, key):
+        if "}" in key:
+            key = key.split("}", 1)[1]
+        return key
+
+
+def XML(text):
+    parser = NamespaceFixXmlTreeBuilder()
+    parser.feed(text)
+    return parser.close()


Follow ups