cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #01062
[Merge] lp:~utlemming/cloud-init/digitalocean into lp:cloud-init
Ben Howard has proposed merging lp:~utlemming/cloud-init/digitalocean into lp:cloud-init.
Requested reviews:
cloud init development team (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~utlemming/cloud-init/digitalocean/+merge/301123
This request comprises two commits to the DigitalOcean provider.
Commit 1259: Update to the JSON meta-data endpoint
- Per the API documentation, the JSON endpoint is the preferred method of getting the metadata.
- Use the SMBIOS information to determine if the droplet ID has changed. In certain circumstances a droplet that is either migrated or snapshotted may not have metadata via the metadata URL. The droplets identity is authoritatively embedded in the SMBIOS.
- Only hit the Metadata URL if the droplet identifies as being on DigitalOcean
Commit 1260: Update the tests cases.
--
Your team cloud init development team is requested to review the proposed merge of lp:~utlemming/cloud-init/digitalocean into lp:cloud-init.
=== modified file 'cloudinit/sources/DataSourceDigitalOcean.py'
--- cloudinit/sources/DataSourceDigitalOcean.py 2016-05-12 17:56:26 +0000
+++ cloudinit/sources/DataSourceDigitalOcean.py 2016-07-25 23:38:22 +0000
@@ -1,6 +1,7 @@
# vi: ts=4 expandtab
#
# Author: Neal Shrader <neal@xxxxxxxxxxxxxxxx>
+# Author: Ben Howard <bh@xxxxxxxxxxxxxxxx>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
@@ -14,22 +15,27 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from cloudinit import ec2_utils
+# DigitalOcean Droplet API:
+# https://developers.digitalocean.com/documentation/metadata/
+
from cloudinit import log as logging
+from cloudinit import util
from cloudinit import sources
-from cloudinit import util
-
-import functools
-
+from cloudinit import url_helper
+
+import json
+import os
LOG = logging.getLogger(__name__)
BUILTIN_DS_CONFIG = {
- 'metadata_url': 'http://169.254.169.254/metadata/v1/',
- 'mirrors_url': 'http://mirrors.digitalocean.com/'
+ 'metadata_url': 'http://169.254.169.254/metadata/v1.json',
}
-MD_RETRIES = 0
-MD_TIMEOUT = 1
+
+# Wait for a up to a minute, retrying the meta-data server
+# every 2 seconds.
+MD_RETRIES = 30
+MD_TIMEOUT = 2
class DataSourceDigitalOcean(sources.DataSource):
@@ -40,43 +46,67 @@
util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}),
BUILTIN_DS_CONFIG])
self.metadata_address = self.ds_cfg['metadata_url']
-
- if self.ds_cfg.get('retries'):
- self.retries = self.ds_cfg['retries']
- else:
- self.retries = MD_RETRIES
-
- if self.ds_cfg.get('timeout'):
- self.timeout = self.ds_cfg['timeout']
- else:
- self.timeout = MD_TIMEOUT
-
- def get_data(self):
- caller = functools.partial(util.read_file_or_url,
- timeout=self.timeout, retries=self.retries)
-
- def mcaller(url):
- return caller(url).contents
-
- md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address),
- base_url=self.metadata_address,
- caller=mcaller)
-
- self.metadata = md.materialize()
-
- if self.metadata.get('id'):
- return True
- else:
+ self.retries = self.ds_cfg.get('retries', MD_RETRIES)
+ self.timeout = self.ds_cfg.get('timeout', MD_TIMEOUT)
+
+ def _get_sysinfo(self):
+ # DigitalOcean embeds vendor ID and instance/droplet_id in the
+ # SMBIOS information
+
+ # DigitalOcean only supports i386/x86_64 hardware
+ uname_arch = os.uname()[4]
+ if uname_arch not in ('i386', 'i686', 'x86_64'):
+ return (None, None)
+ LOG.debug("checking if instance is a DigitalOcean droplet")
+
+ is_do = False
+ droplet_id = None
+
+ # Detect if we are on DigitalOcean and return the Droplet's ID
+ vendor_name = util.read_dmi_data("system-manufacturer")
+ if vendor_name == "DigitalOcean":
+ is_do = True
+ LOG.info("running on DigitalOcean")
+
+ droplet_id = util.read_dmi_data("system-serial-number")
+
+ if droplet_id:
+ LOG.debug(("system identified via SMBIOS as DigitalOcean "
+ "Droplet {}").format(droplet_id))
+ else:
+ LOG.critical(("system identified via SMBIOS as a DigitalOcean "
+ "Droplet, but not provide an ID. "
+ "Please file a support ticket at: "
+ "https://cloud.digitalocean.com/support/tickets";
+ "/new"))
+
+ return (is_do, droplet_id)
+
+ def get_data(self, apply_filter=False):
+ (is_do, droplet_id) = self._get_sysinfo()
+
+ # only proceed if we know we are on DigitalOcean
+ if not is_do:
return False
- def get_userdata_raw(self):
- return "\n".join(self.metadata['user-data'])
-
- def get_vendordata_raw(self):
- return "\n".join(self.metadata['vendor-data'])
+ LOG.debug("reading metadata from {}".format(self.metadata_address))
+ response = url_helper.readurl(self.metadata_address,
+ timeout=self.timeout,
+ sec_between=self.timeout,
+ retries=self.retries)
+
+ contents = util.decode_binary(response.contents)
+ decoded = json.loads(contents)
+
+ self.metadata = decoded
+ self.metadata['instance-id'] = decoded.get('droplet_id', droplet_id)
+ self.metadata['local-hostname'] = decoded.get('hostname', droplet_id)
+ self.vendordata_raw = decoded.get("vendor_data", None)
+ self.userdata_raw = decoded.get("user_data", None)
+ return True
def get_public_ssh_keys(self):
- public_keys = self.metadata['public-keys']
+ public_keys = self.metadata.get('public_keys', [])
if isinstance(public_keys, list):
return public_keys
else:
@@ -84,21 +114,17 @@
@property
def availability_zone(self):
- return self.metadata['region']
-
- def get_instance_id(self):
- return self.metadata['id']
-
- def get_hostname(self, fqdn=False, resolve_ip=False):
- return self.metadata['hostname']
-
- def get_package_mirror_info(self):
- return self.ds_cfg['mirrors_url']
+ return self.metadata.get('region', 'default')
@property
def launch_index(self):
return None
+ def check_instance_id(self, sys_cfg):
+ (_, droplet_id) = self._get_sysinfo()
+ return droplet_id
+
+
# Used to match classes to dependencies
datasources = [
(DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
=== modified file 'tests/unittests/test_datasource/test_digitalocean.py'
--- tests/unittests/test_datasource/test_digitalocean.py 2016-05-12 20:43:11 +0000
+++ tests/unittests/test_datasource/test_digitalocean.py 2016-07-25 23:38:22 +0000
@@ -15,68 +15,58 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import re
-
-from six.moves.urllib_parse import urlparse
+import json
from cloudinit import helpers
from cloudinit import settings
from cloudinit.sources import DataSourceDigitalOcean
from .. import helpers as test_helpers
+from ..helpers import HttprettyTestCase
httpretty = test_helpers.import_httpretty()
-# Abbreviated for the test
-DO_INDEX = """id
- hostname
- user-data
- vendor-data
- public-keys
- region"""
-
-DO_MULTIPLE_KEYS = """ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@xxxxxxxxxxxxxxxx
- ssh-rsa AAAAB3NzaC1yc2EAAAA... neal2@xxxxxxxxxxxxxxxx"""
-DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@xxxxxxxxxxxxxxxx"
+DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@xxxxx",
+ "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@xxxxx"]
+DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... test@xxxxx"
DO_META = {
- '': DO_INDEX,
- 'user-data': '#!/bin/bash\necho "user-data"',
- 'vendor-data': '#!/bin/bash\necho "vendor-data"',
- 'public-keys': DO_SINGLE_KEY,
+ 'user_data': 'user_data_here',
+ 'vendor_data': 'vendor_data_here',
+ 'public_keys': DO_SINGLE_KEY,
'region': 'nyc3',
'id': '2000000',
'hostname': 'cloudinit-test',
}
-MD_URL_RE = re.compile(r'http://169.254.169.254/metadata/v1/.*')
+MD_URL = 'http://169.254.169.254/metadata/v1.json'
+
+
+def _mock_dmi():
+ return (True, DO_META.get('id'))
def _request_callback(method, uri, headers):
- url_path = urlparse(uri).path
- if url_path.startswith('/metadata/v1/'):
- path = url_path.split('/metadata/v1/')[1:][0]
- else:
- path = None
- if path in DO_META:
- return (200, headers, DO_META.get(path))
- else:
- return (404, headers, '')
-
-
-class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
+ return (200, headers, json.dumps(DO_META))
+
+
+class TestDataSourceDigitalOcean(HttprettyTestCase):
+ """
+ Test reading the meta-data
+ """
def setUp(self):
self.ds = DataSourceDigitalOcean.DataSourceDigitalOcean(
settings.CFG_BUILTIN, None,
helpers.Paths({}))
+ self.ds._get_sysinfo = _mock_dmi
super(TestDataSourceDigitalOcean, self).setUp()
@httpretty.activate
def test_connection(self):
httpretty.register_uri(
- httpretty.GET, MD_URL_RE,
- body=_request_callback)
+ httpretty.GET, MD_URL,
+ body=json.dumps(DO_META))
success = self.ds.get_data()
self.assertTrue(success)
@@ -84,14 +74,14 @@
@httpretty.activate
def test_metadata(self):
httpretty.register_uri(
- httpretty.GET, MD_URL_RE,
+ httpretty.GET, MD_URL,
body=_request_callback)
self.ds.get_data()
- self.assertEqual(DO_META.get('user-data'),
+ self.assertEqual(DO_META.get('user_data'),
self.ds.get_userdata_raw())
- self.assertEqual(DO_META.get('vendor-data'),
+ self.assertEqual(DO_META.get('vendor_data'),
self.ds.get_vendordata_raw())
self.assertEqual(DO_META.get('region'),
@@ -103,11 +93,8 @@
self.assertEqual(DO_META.get('hostname'),
self.ds.get_hostname())
- self.assertEqual('http://mirrors.digitalocean.com/',
- self.ds.get_package_mirror_info())
-
# Single key
- self.assertEqual([DO_META.get('public-keys')],
+ self.assertEqual([DO_META.get('public_keys')],
self.ds.get_public_ssh_keys())
self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
@@ -116,12 +103,12 @@
def test_multiple_ssh_keys(self):
DO_META['public_keys'] = DO_MULTIPLE_KEYS
httpretty.register_uri(
- httpretty.GET, MD_URL_RE,
+ httpretty.GET, MD_URL,
body=_request_callback)
self.ds.get_data()
# Multiple keys
- self.assertEqual(DO_META.get('public-keys').splitlines(),
+ self.assertEqual(DO_META.get('public_keys'),
self.ds.get_public_ssh_keys())
self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
Follow ups
