cloud-init-dev team mailing list archive

[Joshua Harlow <harlowja@xxxxxxxxx>]

Joshua Harlow has proposed merging lp:~harlowja/cloud-init/boto-metadata-fixings into lp:cloud-init.

Requested reviews:
  cloud init development team (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~harlowja/cloud-init/boto-metadata-fixings/+merge/131301
-- 
https://code.launchpad.net/~harlowja/cloud-init/boto-metadata-fixings/+merge/131301
Your team cloud init development team is requested to review the proposed merge of lp:~harlowja/cloud-init/boto-metadata-fixings into lp:cloud-init.

=== added file 'cloudinit/ec2_utils.py'
--- cloudinit/ec2_utils.py	1970-01-01 00:00:00 +0000
+++ cloudinit/ec2_utils.py	2012-10-25 03:34:18 +0000
@@ -0,0 +1,65 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Joshua Harlow <harlowja@xxxxxxxxxxxxx>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pkg_resources
+from pkg_resources import parse_version
+
+import cloudinit.util as util
+import cloudinit.url_helper as uh
+
+import boto.utils as boto_utils
+
+
+BOTO_LAZY = False
+try:
+    _boto_lib = pkg_resources.get_distribution('boto')
+    if _boto_lib.parsed_version > parse_version("2.5.2"):
+        BOTO_LAZY = True
+except pkg_resources.DistributionNotFound:
+    pass
+
+
+# Versions of boto >= 2.6.0 try to lazily load
+# the metadata backing, which doesn't work so well
+# in cloud-init especially since the metadata is
+# serialized and actions are performed where the
+# metadata server may be blocked (thus the datasource
+# will start failing) resulting in url exceptions
+# when fields that do exist (or would have existed)
+# do not exist due to the blocking that occurred.
+def _unlazy_dict(mp):
+    if not isinstance(mp, (dict)):
+        return mp
+    if not BOTO_LAZY:
+        return mp
+    for (k, v) in mp.items():
+        _unlazy_dict(v)
+
+
+def get_instance_userdata(api_version, metadata_address):
+    ud = boto_utils.get_instance_userdata(api_version, None, metadata_address)
+    if not ud:
+        ud = ''
+    return ud
+
+
+def get_instance_metadata(api_version, metadata_address):
+    metadata = boto_utils.get_instance_metadata(api_version, metadata_address)
+    if not isinstance(metadata, (dict)):
+        metadata = {}
+    return _unlazy_dict(metadata)

=== modified file 'cloudinit/sources/DataSourceCloudStack.py'
--- cloudinit/sources/DataSourceCloudStack.py	2012-08-22 18:26:35 +0000
+++ cloudinit/sources/DataSourceCloudStack.py	2012-10-25 03:34:18 +0000
@@ -26,8 +26,7 @@
 import os
 import time
 
-import boto.utils as boto_utils
-
+from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
@@ -116,10 +115,10 @@
             if not self.wait_for_metadata_service():
                 return False
             start_time = time.time()
-            self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver,
-                None, self.metadata_address)
-            self.metadata = boto_utils.get_instance_metadata(self.api_ver,
-                self.metadata_address)
+            self.userdata_raw = ec2.get_instance_userdata(self.api_ver,
+                                                          self.metadata_address)
+            self.metadata = ec2.get_instance_metadata(self.api_ver,
+                                                      self.metadata_address)
             LOG.debug("Crawl of metadata service took %s seconds",
                       int(time.time() - start_time))
             return True

=== modified file 'cloudinit/sources/DataSourceEc2.py'
--- cloudinit/sources/DataSourceEc2.py	2012-10-05 20:38:54 +0000
+++ cloudinit/sources/DataSourceEc2.py	2012-10-25 03:34:18 +0000
@@ -23,8 +23,7 @@
 import os
 import time
 
-import boto.utils as boto_utils
-
+from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
@@ -53,6 +52,10 @@
     def __str__(self):
         return util.obj_name(self)
 
+    def __getstate__(self):
+        # Versions of boto
+        pass
+
     def get_data(self):
         seed_ret = {}
         if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
@@ -65,10 +68,10 @@
             if not self.wait_for_metadata_service():
                 return False
             start_time = time.time()
-            self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver,
-                None, self.metadata_address)
-            self.metadata = boto_utils.get_instance_metadata(self.api_ver,
-                self.metadata_address)
+            self.userdata_raw = ec2.get_instance_userdata(self.api_ver,
+                                                          self.metadata_address)
+            self.metadata = ec2.get_instance_metadata(self.api_ver,
+                                                      self.metadata_address)
             LOG.debug("Crawl of metadata service took %s seconds",
                        int(time.time() - start_time))
             return True