cloud-init-dev team mailing list archive

[Markus Schade <launchpad@xxxxxxxxxxxxxxxx>]

Markus Schade has proposed merging ~lp-markusschade/cloud-init:hetznercloud_ds into cloud-init:master.

Requested reviews:
  cloud-init commiters (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~lp-markusschade/cloud-init/+git/cloud-init/+merge/338439

Add Hetzner Cloud DataSource
    
The Hetzner Cloud metadata service is an AWS-style service available
over HTTP via the link local address 169.254.169.254.
    
https://hetzner.com/cloud
https://docs.hetzner.cloud/

-- 
Your team cloud-init commiters is requested to review the proposed merge of ~lp-markusschade/cloud-init:hetznercloud_ds into cloud-init:master.

diff --git a/cloudinit/apport.py b/cloudinit/apport.py
index 221f341..618b016 100644
--- a/cloudinit/apport.py
+++ b/cloudinit/apport.py
@@ -14,9 +14,9 @@ except ImportError:
 
 KNOWN_CLOUD_NAMES = [
     'Amazon - Ec2', 'AliYun', 'AltCloud', 'Azure', 'Bigstep', 'CloudSigma',
-    'CloudStack', 'DigitalOcean', 'GCE - Google Compute Engine', 'MAAS',
-    'NoCloud', 'OpenNebula', 'OpenStack', 'OVF', 'Scaleway', 'SmartOS',
-    'VMware', 'Other']
+    'CloudStack', 'DigitalOcean', 'GCE - Google Compute Engine',
+    'Hetzner Cloud', 'MAAS', 'NoCloud', 'OpenNebula', 'OpenStack', 'OVF',
+    'Scaleway', 'SmartOS', 'VMware', 'Other']
 
 # Potentially clear text collected logs
 CLOUDINIT_LOG = '/var/log/cloud-init.log'
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index c120498..5fe749d 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -36,6 +36,7 @@ CFG_BUILTIN = {
         'SmartOS',
         'Bigstep',
         'Scaleway',
+        'Hetzner',
         # At the end to act as a 'catch' when none of the above work...
         'None',
     ],
diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py
new file mode 100644
index 0000000..ea81a0c
--- /dev/null
+++ b/cloudinit/sources/DataSourceHetzner.py
@@ -0,0 +1,95 @@
+# Author: Jonas Keidel <jonas.keidel@xxxxxxxxxxx>
+# Author: Markus Schade <markus.schade@xxxxxxxxxxx>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+#
+# Hetzner Cloud API:
+# https://docs.hetzner.cloud/
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import util
+
+import cloudinit.sources.helpers.hetzner as ho_helper
+
+LOG = logging.getLogger(__name__)
+
+BUILTIN_DS_CONFIG = {
+    'metadata_url': 'http://169.254.169.254/hetzner/v1/metadata',
+    'userdata_url': 'http://169.254.169.254/hetzner/v1/userdata',
+}
+
+MD_RETRIES = 60
+MD_TIMEOUT = 2
+MD_WAIT_RETRY = 2
+
+
+class DataSourceHetzner(sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.distro = distro
+        self.metadata = dict()
+        self.ds_cfg = util.mergemanydict([
+            util.get_cfg_by_path(sys_cfg, ["datasource", "Hetzner"], {}),
+            BUILTIN_DS_CONFIG])
+        self.metadata_address = self.ds_cfg['metadata_url']
+        self.userdata_address = self.ds_cfg['userdata_url']
+        self.retries = self.ds_cfg.get('retries', MD_RETRIES)
+        self.timeout = self.ds_cfg.get('timeout', MD_TIMEOUT)
+        self.wait_retry = self.ds_cfg.get('wait_retry', MD_WAIT_RETRY)
+        self._network_config = None
+        self.dsmode = sources.DSMODE_NETWORK
+
+    def get_data(self):
+        local_ip_nic = ho_helper.add_local_ip()
+
+        md = ho_helper.read_metadata(
+            self.metadata_address, timeout=self.timeout,
+            sec_between=self.wait_retry, retries=self.retries)
+
+        ud = ho_helper.read_userdata(
+            self.userdata_address, timeout=self.timeout,
+            sec_between=self.wait_retry, retries=self.retries)
+        self.userdata_raw = ud
+
+        self.metadata_full = md
+        self.metadata['instance-id'] = md.get('instance-id', None)
+        self.metadata['local-hostname'] = md.get('hostname', None)
+        self.metadata['network-config'] = md.get('network-config', None)
+        self.metadata['public-keys'] = md.get('public-keys', None)
+        self.vendordata_raw = md.get("vendor_data", None)
+
+        ho_helper.remove_local_ip(local_ip_nic)
+
+        return True
+
+    @property
+    def network_config(self):
+        """Configure the networking. This needs to be done each boot, since
+           the IP information may have changed due to snapshot and/or
+           migration.
+        """
+
+        if self._network_config:
+            return self._network_config
+
+        _net_config = self.metadata['network-config']
+        if not _net_config:
+            raise Exception("Unable to get meta-data from server....")
+
+        self._network_config = _net_config
+
+        return self._network_config
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceHetzner, (sources.DEP_FILESYSTEM, )),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py
new file mode 100644
index 0000000..e48e21f
--- /dev/null
+++ b/cloudinit/sources/helpers/hetzner.py
@@ -0,0 +1,74 @@
+# Author: Jonas Keidel <jonas.keidel@xxxxxxxxxxx>
+# Author: Markus Schade <markus.schade@xxxxxxxxxxx>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit import log as logging
+from cloudinit import net as cloudnet
+from cloudinit import url_helper
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+def read_metadata(url, timeout=2, sec_between=2, retries=30):
+    response = url_helper.readurl(url, timeout=timeout,
+                                  sec_between=sec_between, retries=retries)
+    if not response.ok():
+        raise RuntimeError("unable to read metadata at %s" % url)
+    return util.load_yaml(response.contents.decode())
+
+
+def read_userdata(url, timeout=2, sec_between=2, retries=30):
+    response = url_helper.readurl(url, timeout=timeout,
+                                  sec_between=sec_between, retries=retries)
+    if not response.ok():
+        raise RuntimeError("unable to read metadata at %s" % url)
+    return response.contents.decode()
+
+
+def add_local_ip(nic=None):
+    nic = get_local_ip_nic()
+    LOG.debug("selected interface '%s' for reading metadata", nic)
+
+    if not nic:
+        raise RuntimeError("unable to find interfaces to access the"
+                           "meta-data server. This droplet is broken.")
+
+    ip_addr_cmd = ['ip', 'addr', 'add', '169.254.0.1/16', 'dev', nic]
+    ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'up']
+
+    if not util.which('ip'):
+        raise RuntimeError("No 'ip' command available to configure local ip "
+                           "address")
+
+    try:
+        (result, _err) = util.subp(ip_addr_cmd)
+        LOG.debug("assigned local ip to '%s'", nic)
+
+        (result, _err) = util.subp(ip_link_cmd)
+        LOG.debug("brought device '%s' up", nic)
+    except Exception:
+        util.logexc(LOG, "local ip address assignment to '%s' failed.", nic)
+        raise
+
+    return nic
+
+
+def get_local_ip_nic():
+    nics = [f for f in cloudnet.get_devicelist() if cloudnet.is_physical(f)]
+    if not nics:
+        return None
+    return min(nics, key=lambda d: cloudnet.read_sys_net_int(d, 'ifindex'))
+
+
+def remove_local_ip(nic=None):
+    ip_addr_cmd = ['ip', 'addr', 'flush', 'dev', nic]
+    ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'down']
+    try:
+        (result, _err) = util.subp(ip_addr_cmd)
+        LOG.debug("removed all addresses from %s", nic)
+        (result, _err) = util.subp(ip_link_cmd)
+        LOG.debug("brought device '%s' down", nic)
+    except Exception as e:
+        util.logexc(LOG, "failed to remove all address from '%s'.", nic, e)
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 80b9c65..6d2dc5b 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -14,6 +14,7 @@ from cloudinit.sources import (
     DataSourceDigitalOcean as DigitalOcean,
     DataSourceEc2 as Ec2,
     DataSourceGCE as GCE,
+    DataSourceHetzner as Hetzner,
     DataSourceMAAS as MAAS,
     DataSourceNoCloud as NoCloud,
     DataSourceOpenNebula as OpenNebula,
@@ -31,6 +32,7 @@ DEFAULT_LOCAL = [
     CloudSigma.DataSourceCloudSigma,
     ConfigDrive.DataSourceConfigDrive,
     DigitalOcean.DataSourceDigitalOcean,
+    Hetzner.DataSourceHetzner,
     NoCloud.DataSourceNoCloud,
     OpenNebula.DataSourceOpenNebula,
     OVF.DataSourceOVF,
diff --git a/tools/ds-identify b/tools/ds-identify
index 5f76243..55fde2a 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -114,7 +114,7 @@ DI_DSNAME=""
 # be searched if there is no setting found in config.
 DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
 CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
-OVF SmartOS Scaleway"
+OVF SmartOS Scaleway Hetzner"
 DI_DSLIST=""
 DI_MODE=""
 DI_ON_FOUND=""
@@ -975,6 +975,11 @@ dscheck_Scaleway() {
     return ${DS_NOT_FOUND}
 }
 
+dscheck_Hetzner() {
+    dmi_sys_vendor_is Hetzner && return ${DS_FOUND}
+    return ${DS_NOT_FOUND}
+}
+
 collect_info() {
     read_virt
     read_pid1_product_name