cloud-init-dev team mailing list archive

[Scott Moser <smoser@xxxxxxxxxx>]

two nitpicks.  but this looks good other than failing tests.
b

Diff comments:

> === added file 'cloudinit/sources/DataSourceScaleway.py'
> --- cloudinit/sources/DataSourceScaleway.py	1970-01-01 00:00:00 +0000
> +++ cloudinit/sources/DataSourceScaleway.py	2015-10-28 09:51:17 +0000
> @@ -0,0 +1,216 @@
> +# vi: ts=4 expandtab
> +#
> +#    Author: Edouard Bonlieu <ebonlieu@xxxxxxxxxxxxxx>
> +#
> +#    This program is free software: you can redistribute it and/or modify
> +#    it under the terms of the GNU General Public License version 3, as
> +#    published by the Free Software Foundation.
> +#
> +#    This program is distributed in the hope that it will be useful,
> +#    but WITHOUT ANY WARRANTY; without even the implied warranty of
> +#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +#    GNU General Public License for more details.
> +#
> +#    You should have received a copy of the GNU General Public License
> +#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
> +
> +import functools
> +import errno
> +import json
> +import time
> +
> +from requests.packages.urllib3.poolmanager import PoolManager
> +import requests
> +
> +from cloudinit import log as logging
> +from cloudinit import sources
> +from cloudinit import url_helper
> +from cloudinit import util
> +
> +
> +LOG = logging.getLogger(__name__)
> +
> +BUILTIN_DS_CONFIG = {
> +    'metadata_url': 'http://169.254.42.42/conf?format=json',
> +    'userdata_url': 'http://169.254.42.42/user_data/cloud-init'
> +}
> +
> +DEF_MD_RETRIES = 5
> +DEF_MD_TIMEOUT = 10
> +
> +
> +def on_scaleway(user_data_url, retries=5):
> +    """ Check if we are on Scaleway.
> +
> +    If Scaleway's user-data API isn't queried from a privileged source port
> +    (ie. below 1024), it returns HTTP/403.
> +    """
> +    for _ in range(retries):
> +        try:
> +            code = requests.head(user_data_url).status_code
> +            if code not in (403, 429) and code < 500:
> +                return False
> +            if code == 403:
> +                return True
> +        except (requests.exceptions.ConnectionError,
> +                requests.exceptions.Timeout):
> +            return False
> +
> +        time.sleep(1)  # be nice, and wait a bit before retrying
> +    return False
> +
> +
> +class SourceAddressAdapter(requests.adapters.HTTPAdapter):
> +    """ Adapter for requests to choose the local address to bind to.
> +    """
> +
> +    def __init__(self, source_address, **kwargs):
> +        self.source_address = source_address
> +        super(SourceAddressAdapter, self).__init__(**kwargs)
> +
> +    def init_poolmanager(self, connections, maxsize, block=False):
> +        self.poolmanager = PoolManager(num_pools=connections,
> +                                       maxsize=maxsize,
> +                                       block=block,
> +                                       source_address=self.source_address)
> +
> +
> +def _get_user_data(userdata_address, timeout, retries, session):
> +    """ Retrieve user data.
> +
> +    Scaleway userdata API returns HTTP/404 if user data is not set.
> +
> +    This function wraps `url_helper.readurl` but instead of considering
> +    HTTP/404 as an error that requires a retry, it considers it as empty user
> +    data.
> +
> +    Also, user data API require the source port to be below 1024. If requests
> +    raises ConnectionError (EADDRINUSE), we raise immediately instead of
> +    retrying. This way, the caller can retry to call this function on an other
> +    port.
> +    """
> +    try:
> +        # exception_cb is used to re-raise the exception if the API responds
> +        # HTTP/404.
> +        resp = url_helper.readurl(
> +            userdata_address,
> +            data=None,
> +            timeout=timeout,
> +            retries=retries,
> +            session=session,
> +            exception_cb=lambda _, exc: exc.code == 404 or isinstance(
> +                exc.cause, requests.exceptions.ConnectionError
> +            )
> +        )
> +        return util.decode_binary(resp.contents)
> +    except url_helper.UrlError as exc:
> +        # Empty user data
> +        if exc.code == 404:
> +            return None
> +
> +        # `retries` is reached, re-raise
> +        raise
> +
> +
> +class DataSourceScaleway(sources.DataSource):
> +
> +    def __init__(self, sys_cfg, distro, paths):
> +        LOG.debug('Init scw')
> +        sources.DataSource.__init__(self, sys_cfg, distro, paths)
> +
> +        self.metadata = {}
> +        self.ds_cfg = util.mergemanydict([
> +            util.get_cfg_by_path(sys_cfg, ["datasource", "Scaleway"], {}),
> +            BUILTIN_DS_CONFIG
> +        ])
> +
> +        self.metadata_address = self.ds_cfg['metadata_url']
> +        self.userdata_address = self.ds_cfg['userdata_url']
> +
> +        self.retries = self.ds_cfg.get('retries', DEF_MD_RETRIES)
> +        self.timeout = self.ds_cfg.get('timeout', DEF_MD_TIMEOUT)
> +
> +    def _get_metadata(self):
> +        resp = url_helper.readurl(
> +            self.metadata_address,
> +            timeout=self.timeout,
> +            retries=self.retries
> +        )
> +        metadata = json.loads(util.decode_binary(resp.contents))
> +        LOG.debug('metadata downloaded')
> +
> +        # try to make a request on the first privileged port available
> +        for port in range(1, 1024):
> +            try:
> +                LOG.debug(
> +                    'Trying to get user data (bind on port %d)...' % port

this seems a bit verbose even for 'debug'.

> +                )
> +                session = requests.Session()
> +                session.mount(
> +                    'http://',
> +                    SourceAddressAdapter(source_address=('0.0.0.0', port))
> +                )
> +                user_data = _get_user_data(
> +                    self.userdata_address,
> +                    timeout=self.timeout,
> +                    retries=self.retries,
> +                    session=session
> +                )
> +                LOG.debug('user-data downloaded')
> +                return metadata, user_data
> +
> +            except url_helper.UrlError as exc:

so in the case that port '1' does not work, do we iterate quickly through these ?

> +                # local port already in use, try next port
> +                if isinstance(exc.cause, requests.exceptions.ConnectionError):
> +                    continue
> +
> +                # unexpected exception
> +                raise
> +
> +    def get_data(self):
> +        if on_scaleway(self.ds_cfg['userdata_url'], self.retries) is False:
> +            return False
> +
> +        metadata, metadata['user-data'] = self._get_metadata()
> +        self.metadata = {
> +            'id': metadata['id'],
> +            'hostname': metadata['hostname'],
> +            'user-data': metadata['user-data'],
> +            'ssh_public_keys': [
> +                key['key'] for key in metadata['ssh_public_keys']
> +            ]
> +        }
> +        return True
> +
> +    @property
> +    def launch_index(self):
> +        return None
> +
> +    def get_instance_id(self):
> +        return self.metadata['id']
> +
> +    def get_public_ssh_keys(self):
> +        return self.metadata['ssh_public_keys']
> +
> +    def get_hostname(self, fqdn=False, resolve_ip=False):
> +        return self.metadata['hostname']
> +
> +    def get_userdata_raw(self):
> +        return self.metadata['user-data']
> +
> +    @property
> +    def availability_zone(self):
> +        return None
> +
> +    @property
> +    def region(self):
> +        return None
> +
> +
> +datasources = [
> +    (DataSourceScaleway, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
> +]
> +
> +
> +def get_datasource_list(depends):
> +    return sources.list_from_depends(depends, datasources)


-- 
https://code.launchpad.net/~edouardb/cloud-init/scaleway-datasource/+merge/274861
Your team cloud init development team is requested to review the proposed merge of lp:~edouardb/cloud-init/scaleway-datasource into lp:cloud-init.