cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #00360
[Merge] lp:~harlowja/cloud-init/boto-ec2-replace into lp:cloud-init
Joshua Harlow has proposed merging lp:~harlowja/cloud-init/boto-ec2-replace into lp:cloud-init.
Requested reviews:
cloud init development team (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~harlowja/cloud-init/boto-ec2-replace/+merge/201269
Not done yet!!
--
https://code.launchpad.net/~harlowja/cloud-init/boto-ec2-replace/+merge/201269
Your team cloud init development team is requested to review the proposed merge of lp:~harlowja/cloud-init/boto-ec2-replace into lp:cloud-init.
=== modified file 'cloudinit/ec2_utils.py'
--- cloudinit/ec2_utils.py 2013-03-20 12:34:19 +0000
+++ cloudinit/ec2_utils.py 2014-01-11 00:17:31 +0000
@@ -16,48 +16,154 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import boto.utils as boto_utils
-
-# Versions of boto >= 2.6.0 (and possibly 2.5.2)
-# try to lazily load the metadata backing, which
-# doesn't work so well in cloud-init especially
-# since the metadata is serialized and actions are
-# performed where the metadata server may be blocked
-# (thus the datasource will start failing) resulting
-# in url exceptions when fields that do exist (or
-# would have existed) do not exist due to the blocking
-# that occurred.
-
-# TODO(harlowja): https://github.com/boto/boto/issues/1401
-# When boto finally moves to using requests, we should be able
-# to provide it ssl details, it does not yet, so we can't provide them...
-
-
-def _unlazy_dict(mp):
- if not isinstance(mp, (dict)):
- return mp
- # Walk over the keys/values which
- # forces boto to unlazy itself and
- # has no effect on dictionaries that
- # already have there items.
- for (_k, v) in mp.items():
- _unlazy_dict(v)
- return mp
-
-
-def get_instance_userdata(api_version, metadata_address):
- # Note: boto.utils.get_instance_metadata returns '' for empty string
- # so the change from non-true to '' is not specifically necessary, but
- # this way cloud-init will get consistent behavior even if boto changed
- # in the future to return a None on "no user-data provided".
- ud = boto_utils.get_instance_userdata(api_version, None, metadata_address)
- if not ud:
- ud = ''
- return ud
-
-
-def get_instance_metadata(api_version, metadata_address):
- metadata = boto_utils.get_instance_metadata(api_version, metadata_address)
- if not isinstance(metadata, (dict)):
- metadata = {}
- return _unlazy_dict(metadata)
+from urlparse import (urlparse, urlunparse)
+
+import functools
+import json
+import urllib
+
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+def combine_url(base, add_on):
+ base_parsed = list(urlparse(base))
+ path = base_parsed[2]
+ if path and not path.endswith("/"):
+ path += "/"
+ path += urllib.quote(str(add_on), safe="/:")
+ base_parsed[2] = path
+ return urlunparse(base_parsed)
+
+
+# See: http://bit.ly/TyoUQs
+#
+# Since boto metadata reader uses the old urllib which does not
+# support ssl, we need to ahead and create our own reader which
+# works the same as the boto one (for now).
+class MetadataMaterializer(object):
+ def __init__(self, blob, base_url, caller):
+ self._blob = blob
+ self._md = None
+ self._base_url = base_url
+ self._caller = caller
+
+ def _parse(self, blob):
+ leaves = {}
+ children = []
+ if not blob:
+ return (leaves, children)
+
+ def has_children(item):
+ if item.endswith("/"):
+ return True
+ else:
+ return False
+
+ def get_name(item):
+ if item.endswith("/"):
+ return item.rstrip("/")
+ return item
+
+ for field in blob.splitlines():
+ field = field.strip()
+ field_name = get_name(field)
+ if not field or not field_name:
+ continue
+ if has_children(field):
+ if field_name not in children:
+ children.append(field_name)
+ else:
+ contents = field.split("=", 1)
+ resource = field_name
+ if len(contents) > 1:
+ # What a PITA...
+ (ident, sub_contents) = contents
+ checked_ident = util.safe_int(ident)
+ if checked_ident is not None:
+ resource = "%s/openssh-key" % (checked_ident)
+ field_name = sub_contents
+ leaves[field_name] = resource
+ return (leaves, children)
+
+ def materialize(self):
+ if self._md is not None:
+ return self._md
+ self._md = self._materialize(self._blob, self._base_url)
+ return self._md
+
+ def _decode_leaf_blob(self, blob):
+ if not blob:
+ return blob
+ stripped_blob = blob.strip()
+ if stripped_blob.startswith("{") and stripped_blob.endswith("}"):
+ # Assume and try with json
+ try:
+ return json.loads(blob)
+ except (ValueError, TypeError):
+ pass
+ if blob.find("\n") != -1:
+ return blob.splitlines()
+ return blob
+
+ def _materialize(self, blob, base_url):
+ (leaves, children) = self._parse(blob)
+ child_contents = {}
+ for c in children:
+ child_url = combine_url(base_url, c)
+ if not child_url.endswith("/"):
+ child_url += "/"
+ child_blob = str(self._caller(child_url))
+ child_contents[c] = self._materialize(child_blob, child_url)
+ leaf_contents = {}
+ for (field, resource) in leaves.items():
+ leaf_url = combine_url(base_url, resource)
+ leaf_blob = str(self._caller(leaf_url))
+ leaf_contents[field] = self._decode_leaf_blob(leaf_blob)
+ joined = {}
+ joined.update(child_contents)
+ for field in leaf_contents.keys():
+ if field in joined:
+ LOG.warn("Duplicate key found in results from %s", base_url)
+ else:
+ joined[field] = leaf_contents[field]
+ return joined
+
+
+def get_instance_userdata(api_version='latest',
+ metadata_address='http://169.254.169.254',
+ ssl_details=None, timeout=5, retries=5):
+ ud_url = combine_url(metadata_address, api_version)
+ ud_url = combine_url(ud_url, 'user-data')
+ try:
+ response = util.read_file_or_url(ud_url,
+ ssl_details=ssl_details,
+ timeout=timeout,
+ retries=retries)
+ return str(response)
+ except Exception:
+ util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
+ return None
+
+
+def get_instance_metadata(api_version='latest',
+ metadata_address='http://169.254.169.254',
+ ssl_details=None, timeout=5, retries=5):
+ md_url = combine_url(metadata_address, api_version)
+ md_url = combine_url(md_url, 'meta-data')
+ caller = functools.partial(util.read_file_or_url,
+ ssl_details=ssl_details, timeout=timeout,
+ retries=retries)
+
+ try:
+ response = caller(md_url)
+ materializer = MetadataMaterializer(str(response), md_url, caller)
+ md = materializer.materialize()
+ if not isinstance(md, (dict)):
+ md = {}
+ return md
+ except Exception:
+ util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
+ return {}
Follow ups