cloud-init-dev team mailing list archive

[edouardb <ebonlieusubs@xxxxxxxxx>]

edouardb has proposed merging lp:~edouardb/cloud-init/scaleway-datasource into lp:cloud-init.

Requested reviews:
  cloud init development team (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~edouardb/cloud-init/scaleway-datasource/+merge/274861

Add Datasource for Scaleway's metadata service
-- 
Your team cloud init development team is requested to review the proposed merge of lp:~edouardb/cloud-init/scaleway-datasource into lp:cloud-init.

=== added file 'cloudinit/sources/DataSourceScaleway.py'
--- cloudinit/sources/DataSourceScaleway.py	1970-01-01 00:00:00 +0000
+++ cloudinit/sources/DataSourceScaleway.py	2015-10-19 09:05:49 +0000
@@ -0,0 +1,149 @@
+# vi: ts=4 expandtab
+#
+#    Author: Edouard Bonlieu <ebonlieu@xxxxxxxxxxxxxx>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import functools
+import json
+
+from requests.packages.urllib3.poolmanager import PoolManager
+import requests
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import url_helper
+from cloudinit import util
+
+
+LOG = logging.getLogger(__name__)
+
+BUILTIN_DS_CONFIG = {
+    'metadata_url': 'http://169.254.42.42/conf?format=json',
+    'userdata_url': 'http://169.254.42.42/user_data/cloud-init'
+}
+
+DEF_MD_RETRIES = 5
+DEF_MD_TIMEOUT = 10
+
+
+class SourceAddressAdapter(requests.adapters.HTTPAdapter):
+    """ Adapter for requests to choose the local address to bind to.
+    """
+
+    def __init__(self, source_address, **kwargs):
+        self.source_address = source_address
+        super(SourceAddressAdapter, self).__init__(**kwargs)
+
+    def init_poolmanager(self, connections, maxsize, block=False):
+        self.poolmanager = PoolManager(num_pools=connections,
+                                       maxsize=maxsize,
+                                       block=block,
+                                       source_address=self.source_address)
+
+
+class DataSourceScaleway(sources.DataSource):
+
+    def __init__(self, sys_cfg, distro, paths):
+        LOG.debug('Init scw')
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+
+        self.metadata = {}
+        self.ds_cfg = util.mergemanydict([
+            util.get_cfg_by_path(sys_cfg, ["datasource", "Scaleway"], {}),
+            BUILTIN_DS_CONFIG
+        ])
+
+        self.metadata_address = self.ds_cfg['metadata_url']
+        self.userdata_address = self.ds_cfg['userdata_url']
+
+        self.retries = self.ds_cfg.get('retries', DEF_MD_RETRIES)
+        self.timeout = self.ds_cfg.get('timeout', DEF_MD_TIMEOUT)
+
+    def _get_metadata(self):
+        resp = url_helper.readurl(
+            self.metadata_address,
+            data=None,
+            timeout=self.timeout,
+            retries=self.retries
+        )
+        metadata = json.loads(util.decode_binary(resp.contents))
+        LOG.debug('metadata downloaded')
+
+        # try to make a request on the first privileged port available
+        for port in range(1, 1024):
+            try:
+                session = requests.Session()
+                session.mount(
+                    'http://',
+                    SourceAddressAdapter(source_address=('0.0.0.0', port))
+                )
+                resp = url_helper.readurl(
+                    self.userdata_address,
+                    data=None,
+                    timeout=self.timeout,
+                    retries=self.retries,
+                    session=session
+                )
+                user_data = util.decode_binary(resp.contents)
+                LOG.debug('user-data downloaded')
+                return metadata, user_data
+
+            except url_helper.UrlError:  # try next port
+                pass
+
+    def get_data(self):
+        metadata, metadata['user-data'] = self._get_metadata()
+        self.metadata = {
+            'id': metadata['id'],
+            'hostname': metadata['hostname'],
+            'user-data': metadata['user-data'],
+            'ssh_public_keys': [
+                key['key'] for key in metadata['ssh_public_keys']
+            ]
+        }
+        return True
+
+    @property
+    def launch_index(self):
+        return None
+
+    def get_instance_id(self):
+        return self.metadata['id']
+
+    def get_public_ssh_keys(self):
+        return self.metadata['ssh_public_keys']
+
+    def get_hostname(self, fqdn=False, resolve_ip=False):
+        return self.metadata['hostname']
+
+    def get_userdata_raw(self):
+        return self.metadata['user-data']
+
+    @property
+    def availability_zone(self):
+        return None
+
+    @property
+    def region(self):
+        return None
+
+
+datasources = [
+    (DataSourceScaleway, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
+

=== modified file 'cloudinit/url_helper.py'
--- cloudinit/url_helper.py	2015-09-29 21:17:49 +0000
+++ cloudinit/url_helper.py	2015-10-19 09:05:49 +0000
@@ -181,9 +181,10 @@
     return ssl_args
 
 
-def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
-            headers=None, headers_cb=None, ssl_details=None,
-            check_status=True, allow_redirects=True, exception_cb=None):
+def readurl(url, data=none, timeout=none, retries=0, sec_between=1,
+            headers=none, headers_cb=none, ssl_details=none,
+            check_status=true, allow_redirects=true, exception_cb=none,
+            session=none):
     url = _cleanurl(url)
     req_args = {
         'url': url,
@@ -242,7 +243,9 @@
             LOG.debug("[%s/%s] open '%s' with %s configuration", i,
                       manual_tries, url, filtered_req_args)
 
-            r = requests.request(**req_args)
+            if session is None:
+                session = requests.Session()
+            r = session.request(**req_args)
             if check_status:
                 r.raise_for_status()
             LOG.debug("Read from %s (%s, %sb) after %s attempts", url,
@@ -258,6 +261,8 @@
                 excps.append(UrlError(e, code=e.response.status_code,
                                       headers=e.response.headers,
                                       url=url))
+                if e.response.status_code == 404:
+                    return None
             else:
                 excps.append(UrlError(e, url=url))
                 if SSL_ENABLED and isinstance(e, exceptions.SSLError):