cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #01120
[Merge] cloud-init:digitalocean into cloud-init:master
Ben Howard has proposed merging cloud-init:digitalocean into cloud-init:master.
Requested reviews:
cloud init development team (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~cloud-init-dev/cloud-init/+git/cloud-init/+merge/302715
Update the DigitalOcean data-source. Incorporate feedback from https://code.launchpad.net/~utlemming/cloud-init/digitalocean/+merge/301123.
This is the same commit, just rebased on 0.7.8.
--
Your team cloud init development team is requested to review the proposed merge of cloud-init:digitalocean into cloud-init:master.
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index a352960..48136f7 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -110,12 +110,6 @@ class DataSourceAltCloud(sources.DataSource):
'''
- uname_arch = os.uname()[4]
- if uname_arch.startswith("arm") or uname_arch == "aarch64":
- # Disabling because dmi data is not available on ARM processors
- LOG.debug("Disabling AltCloud datasource on arm (LP: #1243287)")
- return 'UNKNOWN'
-
system_name = util.read_dmi_data("system-product-name")
if not system_name:
return 'UNKNOWN'
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index d1f806d..be74503 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -16,7 +16,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from base64 import b64decode
-import os
import re
from cloudinit.cs_utils import Cepko
@@ -45,11 +44,6 @@ class DataSourceCloudSigma(sources.DataSource):
Uses dmi data to detect if this instance of cloud-init is running
in the CloudSigma's infrastructure.
"""
- uname_arch = os.uname()[4]
- if uname_arch.startswith("arm") or uname_arch == "aarch64":
- # Disabling because dmi data on ARM processors
- LOG.debug("Disabling CloudSigma datasource on arm (LP: #1243287)")
- return False
LOG.debug("determining hypervisor product name via dmi data")
sys_product_name = util.read_dmi_data("system-product-name")
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
index 44a17a0..ba1992e 100644
--- a/cloudinit/sources/DataSourceDigitalOcean.py
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -1,6 +1,7 @@
# vi: ts=4 expandtab
#
# Author: Neal Shrader <neal@xxxxxxxxxxxxxxxx>
+# Author: Ben Howard <bh@xxxxxxxxxxxxxxxx>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
@@ -14,22 +15,27 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from cloudinit import ec2_utils
+# DigitalOcean Droplet API:
+# https://developers.digitalocean.com/documentation/metadata/
+
+import json
+
from cloudinit import log as logging
from cloudinit import sources
+from cloudinit import url_helper
from cloudinit import util
-import functools
-
-
LOG = logging.getLogger(__name__)
BUILTIN_DS_CONFIG = {
- 'metadata_url': 'http://169.254.169.254/metadata/v1/',
- 'mirrors_url': 'http://mirrors.digitalocean.com/'
+ 'metadata_url': 'http://169.254.169.254/metadata/v1.json',
}
-MD_RETRIES = 0
-MD_TIMEOUT = 1
+
+# Wait for a up to a minute, retrying the meta-data server
+# every 2 seconds.
+MD_RETRIES = 30
+MD_TIMEOUT = 2
+MD_WAIT_RETRY = 2
class DataSourceDigitalOcean(sources.DataSource):
@@ -40,43 +46,61 @@ class DataSourceDigitalOcean(sources.DataSource):
util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}),
BUILTIN_DS_CONFIG])
self.metadata_address = self.ds_cfg['metadata_url']
+ self.retries = self.ds_cfg.get('retries', MD_RETRIES)
+ self.timeout = self.ds_cfg.get('timeout', MD_TIMEOUT)
+ self.wait_retry = self.ds_cfg.get('wait_retry', MD_WAIT_RETRY)
- if self.ds_cfg.get('retries'):
- self.retries = self.ds_cfg['retries']
- else:
- self.retries = MD_RETRIES
+ def _get_sysinfo(self):
+ # DigitalOcean embeds vendor ID and instance/droplet_id in the
+ # SMBIOS information
- if self.ds_cfg.get('timeout'):
- self.timeout = self.ds_cfg['timeout']
- else:
- self.timeout = MD_TIMEOUT
+ LOG.debug("checking if instance is a DigitalOcean droplet")
+
+ # Detect if we are on DigitalOcean and return the Droplet's ID
+ vendor_name = util.read_dmi_data("system-manufacturer")
+ if vendor_name != "DigitalOcean":
+ return (False, None)
- def get_data(self):
- caller = functools.partial(util.read_file_or_url,
- timeout=self.timeout, retries=self.retries)
+ LOG.info("running on DigitalOcean")
- def mcaller(url):
- return caller(url).contents
+ droplet_id = util.read_dmi_data("system-serial-number")
+ if droplet_id:
+ LOG.debug(("system identified via SMBIOS as DigitalOcean Droplet"
+ "{}").format(droplet_id))
+ else:
+ LOG.critical(("system identified via SMBIOS as a DigitalOcean "
+ "Droplet, but did not provide an ID. Please file a "
+ "support ticket at: "
+ "https://cloud.digitalocean.com/support/tickets/";
+ "new"))
- md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address),
- base_url=self.metadata_address,
- caller=mcaller)
+ return (True, droplet_id)
- self.metadata = md.materialize()
+ def get_data(self, apply_filter=False):
+ (is_do, droplet_id) = self._get_sysinfo()
- if self.metadata.get('id'):
- return True
- else:
+ # only proceed if we know we are on DigitalOcean
+ if not is_do:
return False
- def get_userdata_raw(self):
- return "\n".join(self.metadata['user-data'])
+ LOG.debug("reading metadata from {}".format(self.metadata_address))
+ response = url_helper.readurl(self.metadata_address,
+ timeout=self.timeout,
+ sec_between=self.wait_retry,
+ retries=self.retries)
- def get_vendordata_raw(self):
- return "\n".join(self.metadata['vendor-data'])
+ contents = util.decode_binary(response.contents)
+ decoded = json.loads(contents)
+
+ self.metadata = decoded
+ self.metadata['instance-id'] = decoded.get('droplet_id', droplet_id)
+ self.metadata['local-hostname'] = decoded.get('hostname', droplet_id)
+ self.vendordata_raw = decoded.get("vendor_data", None)
+ self.userdata_raw = decoded.get("user_data", None)
+ return True
def get_public_ssh_keys(self):
- public_keys = self.metadata['public-keys']
+ public_keys = self.metadata.get('public_keys', [])
if isinstance(public_keys, list):
return public_keys
else:
@@ -84,21 +108,17 @@ class DataSourceDigitalOcean(sources.DataSource):
@property
def availability_zone(self):
- return self.metadata['region']
-
- def get_instance_id(self):
- return self.metadata['id']
-
- def get_hostname(self, fqdn=False, resolve_ip=False):
- return self.metadata['hostname']
-
- def get_package_mirror_info(self):
- return self.ds_cfg['mirrors_url']
+ return self.metadata.get('region', 'default')
@property
def launch_index(self):
return None
+ def check_instance_id(self, sys_cfg):
+ return sources.instance_id_matches_system_uuid(
+ self.get_instance_id(), 'system-serial-number')
+
+
# Used to match classes to dependencies
datasources = [
(DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 39e7bbd..143ab36 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -653,14 +653,8 @@ def write_boot_content(content, content_f, link=None, shebang=False,
util.logexc(LOG, "failed establishing content link: %s", e)
-def get_smartos_environ(uname_version=None, product_name=None,
- uname_arch=None):
+def get_smartos_environ(uname_version=None, product_name=None):
uname = os.uname()
- if uname_arch is None:
- uname_arch = uname[4]
-
- if uname_arch.startswith("arm") or uname_arch == "aarch64":
- return None
# SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
# report 'BrandZ virtual linux' as the kernel version
diff --git a/cloudinit/util.py b/cloudinit/util.py
index e5dd61a..60f1617 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2227,6 +2227,12 @@ def read_dmi_data(key):
If all of the above fail to find a value, None will be returned.
"""
+
+ uname_arch = os.uname()[4]
+ if uname_arch.startswith("arm") or uname_arch == "aarch64":
+ LOG.debug("dmidata is not supported on {0}".format(uname_arch))
+ return None
+
syspath_value = _read_dmi_syspath(key)
if syspath_value is not None:
return syspath_value
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
index 8936a1e..f5d2ef3 100644
--- a/tests/unittests/test_datasource/test_digitalocean.py
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -15,68 +15,58 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import re
-
-from six.moves.urllib_parse import urlparse
+import json
from cloudinit import helpers
from cloudinit import settings
from cloudinit.sources import DataSourceDigitalOcean
from .. import helpers as test_helpers
+from ..helpers import HttprettyTestCase
httpretty = test_helpers.import_httpretty()
-# Abbreviated for the test
-DO_INDEX = """id
- hostname
- user-data
- vendor-data
- public-keys
- region"""
-
-DO_MULTIPLE_KEYS = """ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@xxxxxxxxxxxxxxxx
- ssh-rsa AAAAB3NzaC1yc2EAAAA... neal2@xxxxxxxxxxxxxxxx"""
-DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@xxxxxxxxxxxxxxxx"
+DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@xxxxx",
+ "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@xxxxx"]
+DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... test@xxxxx"
DO_META = {
- '': DO_INDEX,
- 'user-data': '#!/bin/bash\necho "user-data"',
- 'vendor-data': '#!/bin/bash\necho "vendor-data"',
- 'public-keys': DO_SINGLE_KEY,
+ 'user_data': 'user_data_here',
+ 'vendor_data': 'vendor_data_here',
+ 'public_keys': DO_SINGLE_KEY,
'region': 'nyc3',
'id': '2000000',
'hostname': 'cloudinit-test',
}
-MD_URL_RE = re.compile(r'http://169.254.169.254/metadata/v1/.*')
+MD_URL = 'http://169.254.169.254/metadata/v1.json'
+
+
+def _mock_dmi():
+ return (True, DO_META.get('id'))
def _request_callback(method, uri, headers):
- url_path = urlparse(uri).path
- if url_path.startswith('/metadata/v1/'):
- path = url_path.split('/metadata/v1/')[1:][0]
- else:
- path = None
- if path in DO_META:
- return (200, headers, DO_META.get(path))
- else:
- return (404, headers, '')
+ return (200, headers, json.dumps(DO_META))
-class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
+class TestDataSourceDigitalOcean(HttprettyTestCase):
+ """
+ Test reading the meta-data
+ """
def setUp(self):
self.ds = DataSourceDigitalOcean.DataSourceDigitalOcean(
settings.CFG_BUILTIN, None,
helpers.Paths({}))
+ self.ds._get_sysinfo = _mock_dmi
super(TestDataSourceDigitalOcean, self).setUp()
@httpretty.activate
def test_connection(self):
httpretty.register_uri(
- httpretty.GET, MD_URL_RE,
- body=_request_callback)
+ httpretty.GET, MD_URL,
+ body=json.dumps(DO_META))
success = self.ds.get_data()
self.assertTrue(success)
@@ -84,14 +74,14 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
@httpretty.activate
def test_metadata(self):
httpretty.register_uri(
- httpretty.GET, MD_URL_RE,
+ httpretty.GET, MD_URL,
body=_request_callback)
self.ds.get_data()
- self.assertEqual(DO_META.get('user-data'),
+ self.assertEqual(DO_META.get('user_data'),
self.ds.get_userdata_raw())
- self.assertEqual(DO_META.get('vendor-data'),
+ self.assertEqual(DO_META.get('vendor_data'),
self.ds.get_vendordata_raw())
self.assertEqual(DO_META.get('region'),
@@ -103,11 +93,8 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
self.assertEqual(DO_META.get('hostname'),
self.ds.get_hostname())
- self.assertEqual('http://mirrors.digitalocean.com/',
- self.ds.get_package_mirror_info())
-
# Single key
- self.assertEqual([DO_META.get('public-keys')],
+ self.assertEqual([DO_META.get('public_keys')],
self.ds.get_public_ssh_keys())
self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
@@ -116,12 +103,12 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
def test_multiple_ssh_keys(self):
DO_META['public_keys'] = DO_MULTIPLE_KEYS
httpretty.register_uri(
- httpretty.GET, MD_URL_RE,
+ httpretty.GET, MD_URL,
body=_request_callback)
self.ds.get_data()
# Multiple keys
- self.assertEqual(DO_META.get('public-keys').splitlines(),
+ self.assertEqual(DO_META.get('public_keys'),
self.ds.get_public_ssh_keys())
self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
