cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #05480
[Merge] ~raharper/cloud-init:ubuntu/devel/newupstream-20180914 into cloud-init:ubuntu/devel
Ryan Harper has proposed merging ~raharper/cloud-init:ubuntu/devel/newupstream-20180914 into cloud-init:ubuntu/devel.
Commit message:
cloud-init (18.3-44-g84bf2482-0ubuntu1) cosmic; urgency=medium
* New upstream snapshot.
- bash_completion/cloud-init: fix shell syntax error.
- EphemeralIPv4Network: Be more explicit when adding default route.
- OpenStack: support reading of newer versions of metdata.
- OpenStack: fix bug causing 'latest' version to be used from network.
- user-data: jinja template to render instance-data.json in cloud-config
-- Ryan Harper <ryan.harper@xxxxxxxxxxxxx> Fri, 14 Sep 2018 14:06:29 -0500
Requested reviews:
cloud-init commiters (cloud-init-dev)
Related bugs:
Bug #1791781 in cloud-init: "instance-data.json not persisted across reboot."
https://bugs.launchpad.net/cloud-init/+bug/1791781
Bug #1792157 in cloud-init (Ubuntu): "cloud-init uses openstack latest version due to decoding bug"
https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1792157
Bug #1792415 in cloud-init: "WARNINGs and failures in log on OVH public cloud"
https://bugs.launchpad.net/cloud-init/+bug/1792415
For more details, see:
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/354969
--
Your team cloud-init commiters is requested to review the proposed merge of ~raharper/cloud-init:ubuntu/devel/newupstream-20180914 into cloud-init:ubuntu/devel.
diff --git a/bash_completion/cloud-init b/bash_completion/cloud-init
index f38164b..6d01bf3 100644
--- a/bash_completion/cloud-init
+++ b/bash_completion/cloud-init
@@ -62,6 +62,8 @@ _cloudinit_complete()
net-convert)
COMPREPLY=($(compgen -W "--help --network-data --kind --directory --output-kind" -- $cur_word))
;;
+ render)
+ COMPREPLY=($(compgen -W "--help --instance-data --debug" -- $cur_word));;
schema)
COMPREPLY=($(compgen -W "--help --config-file --doc --annotate" -- $cur_word))
;;
diff --git a/cloudinit/cmd/devel/__init__.py b/cloudinit/cmd/devel/__init__.py
index e69de29..3ae28b6 100644
--- a/cloudinit/cmd/devel/__init__.py
+++ b/cloudinit/cmd/devel/__init__.py
@@ -0,0 +1,25 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Common cloud-init devel commandline utility functions."""
+
+
+import logging
+
+from cloudinit import log
+from cloudinit.stages import Init
+
+
+def addLogHandlerCLI(logger, log_level):
+ """Add a commandline logging handler to emit messages to stderr."""
+ formatter = logging.Formatter('%(levelname)s: %(message)s')
+ log.setupBasicLogging(log_level, formatter=formatter)
+ return logger
+
+
+def read_cfg_paths():
+ """Return a Paths object based on the system configuration on disk."""
+ init = Init(ds_deps=[])
+ init.read_cfg()
+ return init.paths
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py
index 40a4b01..99a234c 100644
--- a/cloudinit/cmd/devel/parser.py
+++ b/cloudinit/cmd/devel/parser.py
@@ -8,6 +8,7 @@ import argparse
from cloudinit.config import schema
from . import net_convert
+from . import render
def get_parser(parser=None):
@@ -22,7 +23,9 @@ def get_parser(parser=None):
('schema', 'Validate cloud-config files for document schema',
schema.get_parser, schema.handle_schema_args),
(net_convert.NAME, net_convert.__doc__,
- net_convert.get_parser, net_convert.handle_args)
+ net_convert.get_parser, net_convert.handle_args),
+ (render.NAME, render.__doc__,
+ render.get_parser, render.handle_args)
]
for (subcmd, helpmsg, get_parser, handler) in subcmds:
parser = subparsers.add_parser(subcmd, help=helpmsg)
diff --git a/cloudinit/cmd/devel/render.py b/cloudinit/cmd/devel/render.py
new file mode 100755
index 0000000..e85933d
--- /dev/null
+++ b/cloudinit/cmd/devel/render.py
@@ -0,0 +1,90 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Debug jinja template rendering of user-data."""
+
+import argparse
+import os
+import sys
+
+from cloudinit.handlers.jinja_template import render_jinja_payload_from_file
+from cloudinit import log
+from cloudinit.sources import INSTANCE_JSON_FILE
+from cloudinit import util
+from . import addLogHandlerCLI, read_cfg_paths
+
+NAME = 'render'
+DEFAULT_INSTANCE_DATA = '/run/cloud-init/instance-data.json'
+
+LOG = log.getLogger(NAME)
+
+
+def get_parser(parser=None):
+ """Build or extend and arg parser for jinja render utility.
+
+ @param parser: Optional existing ArgumentParser instance representing the
+ subcommand which will be extended to support the args of this utility.
+
+ @returns: ArgumentParser with proper argument configuration.
+ """
+ if not parser:
+ parser = argparse.ArgumentParser(prog=NAME, description=__doc__)
+ parser.add_argument(
+ 'user_data', type=str, help='Path to the user-data file to render')
+ parser.add_argument(
+ '-i', '--instance-data', type=str,
+ help=('Optional path to instance-data.json file. Defaults to'
+ ' /run/cloud-init/instance-data.json'))
+ parser.add_argument('-d', '--debug', action='store_true', default=False,
+ help='Add verbose messages during template render')
+ return parser
+
+
+def handle_args(name, args):
+ """Render the provided user-data template file using instance-data values.
+
+ Also setup CLI log handlers to report to stderr since this is a development
+ utility which should be run by a human on the CLI.
+
+ @return 0 on success, 1 on failure.
+ """
+ addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING)
+ if not args.instance_data:
+ paths = read_cfg_paths()
+ instance_data_fn = os.path.join(
+ paths.run_dir, INSTANCE_JSON_FILE)
+ else:
+ instance_data_fn = args.instance_data
+ try:
+ with open(instance_data_fn) as stream:
+ instance_data = stream.read()
+ instance_data = util.load_json(instance_data)
+ except IOError:
+ LOG.error('Missing instance-data.json file: %s', instance_data_fn)
+ return 1
+ try:
+ with open(args.user_data) as stream:
+ user_data = stream.read()
+ except IOError:
+ LOG.error('Missing user-data file: %s', args.user_data)
+ return 1
+ rendered_payload = render_jinja_payload_from_file(
+ payload=user_data, payload_fn=args.user_data,
+ instance_data_file=instance_data_fn,
+ debug=True if args.debug else False)
+ if not rendered_payload:
+ LOG.error('Unable to render user-data file: %s', args.user_data)
+ return 1
+ sys.stdout.write(rendered_payload)
+ return 0
+
+
+def main():
+ args = get_parser().parse_args()
+ return(handle_args(NAME, args))
+
+
+if __name__ == '__main__':
+ sys.exit(main())
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/tests/test_render.py b/cloudinit/cmd/devel/tests/test_render.py
new file mode 100644
index 0000000..fc5d2c0
--- /dev/null
+++ b/cloudinit/cmd/devel/tests/test_render.py
@@ -0,0 +1,101 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from six import StringIO
+import os
+
+from collections import namedtuple
+from cloudinit.cmd.devel import render
+from cloudinit.helpers import Paths
+from cloudinit.sources import INSTANCE_JSON_FILE
+from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJinja
+from cloudinit.util import ensure_dir, write_file
+
+
+class TestRender(CiTestCase):
+
+ with_logs = True
+
+ args = namedtuple('renderargs', 'user_data instance_data debug')
+
+ def setUp(self):
+ super(TestRender, self).setUp()
+ self.tmp = self.tmp_dir()
+
+ def test_handle_args_error_on_missing_user_data(self):
+ """When user_data file path does not exist, log an error."""
+ absent_file = self.tmp_path('user-data', dir=self.tmp)
+ instance_data = self.tmp_path('instance-data', dir=self.tmp)
+ write_file(instance_data, '{}')
+ args = self.args(
+ user_data=absent_file, instance_data=instance_data, debug=False)
+ with mock.patch('sys.stderr', new_callable=StringIO):
+ self.assertEqual(1, render.handle_args('anyname', args))
+ self.assertIn(
+ 'Missing user-data file: %s' % absent_file,
+ self.logs.getvalue())
+
+ def test_handle_args_error_on_missing_instance_data(self):
+ """When instance_data file path does not exist, log an error."""
+ user_data = self.tmp_path('user-data', dir=self.tmp)
+ absent_file = self.tmp_path('instance-data', dir=self.tmp)
+ args = self.args(
+ user_data=user_data, instance_data=absent_file, debug=False)
+ with mock.patch('sys.stderr', new_callable=StringIO):
+ self.assertEqual(1, render.handle_args('anyname', args))
+ self.assertIn(
+ 'Missing instance-data.json file: %s' % absent_file,
+ self.logs.getvalue())
+
+ def test_handle_args_defaults_instance_data(self):
+ """When no instance_data argument, default to configured run_dir."""
+ user_data = self.tmp_path('user-data', dir=self.tmp)
+ run_dir = self.tmp_path('run_dir', dir=self.tmp)
+ ensure_dir(run_dir)
+ paths = Paths({'run_dir': run_dir})
+ self.add_patch('cloudinit.cmd.devel.render.read_cfg_paths', 'm_paths')
+ self.m_paths.return_value = paths
+ args = self.args(
+ user_data=user_data, instance_data=None, debug=False)
+ with mock.patch('sys.stderr', new_callable=StringIO):
+ self.assertEqual(1, render.handle_args('anyname', args))
+ json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
+ self.assertIn(
+ 'Missing instance-data.json file: %s' % json_file,
+ self.logs.getvalue())
+
+ @skipUnlessJinja()
+ def test_handle_args_renders_instance_data_vars_in_template(self):
+ """If user_data file is a jinja template render instance-data vars."""
+ user_data = self.tmp_path('user-data', dir=self.tmp)
+ write_file(user_data, '##template: jinja\nrendering: {{ my_var }}')
+ instance_data = self.tmp_path('instance-data', dir=self.tmp)
+ write_file(instance_data, '{"my-var": "jinja worked"}')
+ args = self.args(
+ user_data=user_data, instance_data=instance_data, debug=True)
+ with mock.patch('sys.stderr', new_callable=StringIO) as m_console_err:
+ with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+ self.assertEqual(0, render.handle_args('anyname', args))
+ self.assertIn(
+ 'DEBUG: Converted jinja variables\n{', self.logs.getvalue())
+ self.assertIn(
+ 'DEBUG: Converted jinja variables\n{', m_console_err.getvalue())
+ self.assertEqual('rendering: jinja worked', m_stdout.getvalue())
+
+ @skipUnlessJinja()
+ def test_handle_args_warns_and_gives_up_on_invalid_jinja_operation(self):
+ """If user_data file has invalid jinja operations log warnings."""
+ user_data = self.tmp_path('user-data', dir=self.tmp)
+ write_file(user_data, '##template: jinja\nrendering: {{ my-var }}')
+ instance_data = self.tmp_path('instance-data', dir=self.tmp)
+ write_file(instance_data, '{"my-var": "jinja worked"}')
+ args = self.args(
+ user_data=user_data, instance_data=instance_data, debug=True)
+ with mock.patch('sys.stderr', new_callable=StringIO):
+ self.assertEqual(1, render.handle_args('anyname', args))
+ self.assertIn(
+ 'WARNING: Ignoring jinja template for %s: Undefined jinja'
+ ' variable: "my-var". Jinja tried subtraction. Perhaps you meant'
+ ' "my_var"?' % user_data,
+ self.logs.getvalue())
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 4ea4fe7..0eee583 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -348,6 +348,7 @@ def main_init(name, args):
LOG.debug("[%s] barreling on in force mode without datasource",
mode)
+ _maybe_persist_instance_data(init)
# Stage 6
iid = init.instancify()
LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s",
@@ -490,6 +491,7 @@ def main_modules(action_name, args):
print_exc(msg)
if not args.force:
return [(msg)]
+ _maybe_persist_instance_data(init)
# Stage 3
mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
# Stage 4
@@ -541,6 +543,7 @@ def main_single(name, args):
" likely bad things to come!"))
if not args.force:
return 1
+ _maybe_persist_instance_data(init)
# Stage 3
mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
mod_args = args.module_args
@@ -688,6 +691,15 @@ def status_wrapper(name, args, data_d=None, link_d=None):
return len(v1[mode]['errors'])
+def _maybe_persist_instance_data(init):
+ """Write instance-data.json file if absent and datasource is restored."""
+ if init.ds_restored:
+ instance_data_file = os.path.join(
+ init.paths.run_dir, sources.INSTANCE_JSON_FILE)
+ if not os.path.exists(instance_data_file):
+ init.datasource.persist_instance_data()
+
+
def _maybe_set_hostname(init, stage, retry_stage):
"""Call set-hostname if metadata, vendordata or userdata provides it.
@@ -887,6 +899,8 @@ def main(sysv_args=None):
if __name__ == '__main__':
if 'TZ' not in os.environ:
os.environ['TZ'] = ":/etc/localtime"
- main(sys.argv)
+ return_value = main(sys.argv)
+ if return_value:
+ sys.exit(return_value)
# vi: ts=4 expandtab
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index c3576c0..0db75af 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -41,7 +41,7 @@ PART_HANDLER_FN_TMPL = 'part-handler-%03d'
# For parts without filenames
PART_FN_TPL = 'part-%03d'
-# Different file beginnings to there content type
+# Different file beginnings to their content type
INCLUSION_TYPES_MAP = {
'#include': 'text/x-include-url',
'#include-once': 'text/x-include-once-url',
@@ -52,6 +52,7 @@ INCLUSION_TYPES_MAP = {
'#cloud-boothook': 'text/cloud-boothook',
'#cloud-config-archive': 'text/cloud-config-archive',
'#cloud-config-jsonp': 'text/cloud-config-jsonp',
+ '## template: jinja': 'text/jinja2',
}
# Sorted longest first
@@ -69,9 +70,13 @@ class Handler(object):
def __repr__(self):
return "%s: [%s]" % (type_utils.obj_name(self), self.list_types())
- @abc.abstractmethod
def list_types(self):
- raise NotImplementedError()
+ # Each subclass must define the supported content prefixes it handles.
+ if not hasattr(self, 'prefixes'):
+ raise NotImplementedError('Missing prefixes subclass attribute')
+ else:
+ return [INCLUSION_TYPES_MAP[prefix]
+ for prefix in getattr(self, 'prefixes')]
@abc.abstractmethod
def handle_part(self, *args, **kwargs):
diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py
index 057b4db..dca50a4 100644
--- a/cloudinit/handlers/boot_hook.py
+++ b/cloudinit/handlers/boot_hook.py
@@ -17,10 +17,13 @@ from cloudinit import util
from cloudinit.settings import (PER_ALWAYS)
LOG = logging.getLogger(__name__)
-BOOTHOOK_PREFIX = "#cloud-boothook"
class BootHookPartHandler(handlers.Handler):
+
+ # The content prefixes this handler understands.
+ prefixes = ['#cloud-boothook']
+
def __init__(self, paths, datasource, **_kwargs):
handlers.Handler.__init__(self, PER_ALWAYS)
self.boothook_dir = paths.get_ipath("boothooks")
@@ -28,16 +31,11 @@ class BootHookPartHandler(handlers.Handler):
if datasource:
self.instance_id = datasource.get_instance_id()
- def list_types(self):
- return [
- handlers.type_from_starts_with(BOOTHOOK_PREFIX),
- ]
-
def _write_part(self, payload, filename):
filename = util.clean_filename(filename)
filepath = os.path.join(self.boothook_dir, filename)
contents = util.strip_prefix_suffix(util.dos2unix(payload),
- prefix=BOOTHOOK_PREFIX)
+ prefix=self.prefixes[0])
util.write_file(filepath, contents.lstrip(), 0o700)
return filepath
diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index 178a5b9..99bf0e6 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -42,14 +42,12 @@ DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()')
CLOUD_PREFIX = "#cloud-config"
JSONP_PREFIX = "#cloud-config-jsonp"
-# The file header -> content types this module will handle.
-CC_TYPES = {
- JSONP_PREFIX: handlers.type_from_starts_with(JSONP_PREFIX),
- CLOUD_PREFIX: handlers.type_from_starts_with(CLOUD_PREFIX),
-}
-
class CloudConfigPartHandler(handlers.Handler):
+
+ # The content prefixes this handler understands.
+ prefixes = [CLOUD_PREFIX, JSONP_PREFIX]
+
def __init__(self, paths, **_kwargs):
handlers.Handler.__init__(self, PER_ALWAYS, version=3)
self.cloud_buf = None
@@ -58,9 +56,6 @@ class CloudConfigPartHandler(handlers.Handler):
self.cloud_fn = paths.get_ipath(_kwargs["cloud_config_path"])
self.file_names = []
- def list_types(self):
- return list(CC_TYPES.values())
-
def _write_cloud_config(self):
if not self.cloud_fn:
return
@@ -138,7 +133,7 @@ class CloudConfigPartHandler(handlers.Handler):
# First time through, merge with an empty dict...
if self.cloud_buf is None or not self.file_names:
self.cloud_buf = {}
- if ctype == CC_TYPES[JSONP_PREFIX]:
+ if ctype == handlers.INCLUSION_TYPES_MAP[JSONP_PREFIX]:
self._merge_patch(payload)
else:
self._merge_part(payload, headers)
diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py
new file mode 100644
index 0000000..3fa4097
--- /dev/null
+++ b/cloudinit/handlers/jinja_template.py
@@ -0,0 +1,137 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import os
+import re
+
+try:
+ from jinja2.exceptions import UndefinedError as JUndefinedError
+except ImportError:
+ # No jinja2 dependency
+ JUndefinedError = Exception
+
+from cloudinit import handlers
+from cloudinit import log as logging
+from cloudinit.sources import INSTANCE_JSON_FILE
+from cloudinit.templater import render_string, MISSING_JINJA_PREFIX
+from cloudinit.util import b64d, load_file, load_json, json_dumps
+
+from cloudinit.settings import PER_ALWAYS
+
+LOG = logging.getLogger(__name__)
+
+
+class JinjaTemplatePartHandler(handlers.Handler):
+
+ prefixes = ['## template: jinja']
+
+ def __init__(self, paths, **_kwargs):
+ handlers.Handler.__init__(self, PER_ALWAYS, version=3)
+ self.paths = paths
+ self.sub_handlers = {}
+ for handler in _kwargs.get('sub_handlers', []):
+ for ctype in handler.list_types():
+ self.sub_handlers[ctype] = handler
+
+ def handle_part(self, data, ctype, filename, payload, frequency, headers):
+ if ctype in handlers.CONTENT_SIGNALS:
+ return
+ jinja_json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
+ rendered_payload = render_jinja_payload_from_file(
+ payload, filename, jinja_json_file)
+ if not rendered_payload:
+ return
+ subtype = handlers.type_from_starts_with(rendered_payload)
+ sub_handler = self.sub_handlers.get(subtype)
+ if not sub_handler:
+ LOG.warning(
+ 'Ignoring jinja template for %s. Could not find supported'
+ ' sub-handler for type %s', filename, subtype)
+ return
+ if sub_handler.handler_version == 3:
+ sub_handler.handle_part(
+ data, ctype, filename, rendered_payload, frequency, headers)
+ elif sub_handler.handler_version == 2:
+ sub_handler.handle_part(
+ data, ctype, filename, rendered_payload, frequency)
+
+
+def render_jinja_payload_from_file(
+ payload, payload_fn, instance_data_file, debug=False):
+ """Render a jinja template payload sourcing variables from jinja_vars_path.
+
+ @param payload: String of jinja template content. Should begin with
+ ## template: jinja\n.
+ @param payload_fn: String representing the filename from which the payload
+ was read used in error reporting. Generally in part-handling this is
+ 'part-##'.
+ @param instance_data_file: A path to a json file containing variables that
+ will be used as jinja template variables.
+
+ @return: A string of jinja-rendered content with the jinja header removed.
+ Returns None on error.
+ """
+ instance_data = {}
+ rendered_payload = None
+ if not os.path.exists(instance_data_file):
+ raise RuntimeError(
+ 'Cannot render jinja template vars. Instance data not yet'
+ ' present at %s' % instance_data_file)
+ instance_data = load_json(load_file(instance_data_file))
+ rendered_payload = render_jinja_payload(
+ payload, payload_fn, instance_data, debug)
+ if not rendered_payload:
+ return None
+ return rendered_payload
+
+
+def render_jinja_payload(payload, payload_fn, instance_data, debug=False):
+ instance_jinja_vars = convert_jinja_instance_data(
+ instance_data,
+ decode_paths=instance_data.get('base64-encoded-keys', []))
+ if debug:
+ LOG.debug('Converted jinja variables\n%s',
+ json_dumps(instance_jinja_vars))
+ try:
+ rendered_payload = render_string(payload, instance_jinja_vars)
+ except (TypeError, JUndefinedError) as e:
+ LOG.warning(
+ 'Ignoring jinja template for %s: %s', payload_fn, str(e))
+ return None
+ warnings = [
+ "'%s'" % var.replace(MISSING_JINJA_PREFIX, '')
+ for var in re.findall(
+ r'%s[^\s]+' % MISSING_JINJA_PREFIX, rendered_payload)]
+ if warnings:
+ LOG.warning(
+ "Could not render jinja template variables in file '%s': %s",
+ payload_fn, ', '.join(warnings))
+ return rendered_payload
+
+
+def convert_jinja_instance_data(data, prefix='', sep='/', decode_paths=()):
+ """Process instance-data.json dict for use in jinja templates.
+
+ Replace hyphens with underscores for jinja templates and decode any
+ base64_encoded_keys.
+ """
+ result = {}
+ decode_paths = [path.replace('-', '_') for path in decode_paths]
+ for key, value in sorted(data.items()):
+ if '-' in key:
+ # Standardize keys for use in #cloud-config/shell templates
+ key = key.replace('-', '_')
+ key_path = '{0}{1}{2}'.format(prefix, sep, key) if prefix else key
+ if key_path in decode_paths:
+ value = b64d(value)
+ if isinstance(value, dict):
+ result[key] = convert_jinja_instance_data(
+ value, key_path, sep=sep, decode_paths=decode_paths)
+ if re.match(r'v\d+', key):
+ # Copy values to top-level aliases
+ for subkey, subvalue in result[key].items():
+ result[subkey] = subvalue
+ else:
+ result[key] = value
+ return result
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py
index e4945a2..214714b 100644
--- a/cloudinit/handlers/shell_script.py
+++ b/cloudinit/handlers/shell_script.py
@@ -17,21 +17,18 @@ from cloudinit import util
from cloudinit.settings import (PER_ALWAYS)
LOG = logging.getLogger(__name__)
-SHELL_PREFIX = "#!"
class ShellScriptPartHandler(handlers.Handler):
+
+ prefixes = ['#!']
+
def __init__(self, paths, **_kwargs):
handlers.Handler.__init__(self, PER_ALWAYS)
self.script_dir = paths.get_ipath_cur('scripts')
if 'script_path' in _kwargs:
self.script_dir = paths.get_ipath_cur(_kwargs['script_path'])
- def list_types(self):
- return [
- handlers.type_from_starts_with(SHELL_PREFIX),
- ]
-
def handle_part(self, data, ctype, filename, payload, frequency):
if ctype in handlers.CONTENT_SIGNALS:
# TODO(harlowja): maybe delete existing things here
diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py
index dc33876..83fb072 100644
--- a/cloudinit/handlers/upstart_job.py
+++ b/cloudinit/handlers/upstart_job.py
@@ -18,19 +18,16 @@ from cloudinit import util
from cloudinit.settings import (PER_INSTANCE)
LOG = logging.getLogger(__name__)
-UPSTART_PREFIX = "#upstart-job"
class UpstartJobPartHandler(handlers.Handler):
+
+ prefixes = ['#upstart-job']
+
def __init__(self, paths, **_kwargs):
handlers.Handler.__init__(self, PER_INSTANCE)
self.upstart_dir = paths.upstart_conf_d
- def list_types(self):
- return [
- handlers.type_from_starts_with(UPSTART_PREFIX),
- ]
-
def handle_part(self, data, ctype, filename, payload, frequency):
if ctype in handlers.CONTENT_SIGNALS:
return
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 1979cd9..3cc1fb1 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -449,4 +449,8 @@ class DefaultingConfigParser(RawConfigParser):
contents = '\n'.join([header, contents, ''])
return contents
+
+def identity(object):
+ return object
+
# vi: ts=4 expandtab
diff --git a/cloudinit/log.py b/cloudinit/log.py
index 1d75c9f..5ae312b 100644
--- a/cloudinit/log.py
+++ b/cloudinit/log.py
@@ -38,10 +38,18 @@ DEF_CON_FORMAT = '%(asctime)s - %(filename)s[%(levelname)s]: %(message)s'
logging.Formatter.converter = time.gmtime
-def setupBasicLogging(level=DEBUG):
+def setupBasicLogging(level=DEBUG, formatter=None):
+ if not formatter:
+ formatter = logging.Formatter(DEF_CON_FORMAT)
root = logging.getLogger()
+ for handler in root.handlers:
+ if hasattr(handler, 'stream') and hasattr(handler.stream, 'name'):
+ if handler.stream.name == '<stderr>':
+ handler.setLevel(level)
+ return
+ # Didn't have an existing stderr handler; create a new handler
console = logging.StreamHandler(sys.stderr)
- console.setFormatter(logging.Formatter(DEF_CON_FORMAT))
+ console.setFormatter(formatter)
console.setLevel(level)
root.addHandler(console)
root.setLevel(level)
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 3ffde52..5e87bca 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -698,6 +698,13 @@ class EphemeralIPv4Network(object):
self.interface, out.strip())
return
util.subp(
+ ['ip', '-4', 'route', 'add', self.router, 'dev', self.interface,
+ 'src', self.ip], capture=True)
+ self.cleanup_cmds.insert(
+ 0,
+ ['ip', '-4', 'route', 'del', self.router, 'dev', self.interface,
+ 'src', self.ip])
+ util.subp(
['ip', '-4', 'route', 'add', 'default', 'via', self.router,
'dev', self.interface], capture=True)
self.cleanup_cmds.insert(
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 8b444f1..58e0a59 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -515,12 +515,17 @@ class TestEphemeralIPV4Network(CiTestCase):
capture=True),
mock.call(
['ip', 'route', 'show', '0.0.0.0/0'], capture=True),
+ mock.call(['ip', '-4', 'route', 'add', '192.168.2.1',
+ 'dev', 'eth0', 'src', '192.168.2.2'], capture=True),
mock.call(
['ip', '-4', 'route', 'add', 'default', 'via',
'192.168.2.1', 'dev', 'eth0'], capture=True)]
- expected_teardown_calls = [mock.call(
- ['ip', '-4', 'route', 'del', 'default', 'dev', 'eth0'],
- capture=True)]
+ expected_teardown_calls = [
+ mock.call(['ip', '-4', 'route', 'del', 'default', 'dev', 'eth0'],
+ capture=True),
+ mock.call(['ip', '-4', 'route', 'del', '192.168.2.1',
+ 'dev', 'eth0', 'src', '192.168.2.2'], capture=True),
+ ]
with net.EphemeralIPv4Network(**params):
self.assertEqual(expected_setup_calls, m_subp.call_args_list)
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 41fde9b..a775f1a 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -58,22 +58,27 @@ class InvalidMetaDataException(Exception):
pass
-def process_base64_metadata(metadata, key_path=''):
- """Strip ci-b64 prefix and return metadata with base64-encoded-keys set."""
+def process_instance_metadata(metadata, key_path=''):
+ """Process all instance metadata cleaning it up for persisting as json.
+
+ Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list
+
+ @return Dict copy of processed metadata.
+ """
md_copy = copy.deepcopy(metadata)
- md_copy['base64-encoded-keys'] = []
+ md_copy['base64_encoded_keys'] = []
for key, val in metadata.items():
if key_path:
sub_key_path = key_path + '/' + key
else:
sub_key_path = key
if isinstance(val, str) and val.startswith('ci-b64:'):
- md_copy['base64-encoded-keys'].append(sub_key_path)
+ md_copy['base64_encoded_keys'].append(sub_key_path)
md_copy[key] = val.replace('ci-b64:', '')
if isinstance(val, dict):
- return_val = process_base64_metadata(val, sub_key_path)
- md_copy['base64-encoded-keys'].extend(
- return_val.pop('base64-encoded-keys'))
+ return_val = process_instance_metadata(val, sub_key_path)
+ md_copy['base64_encoded_keys'].extend(
+ return_val.pop('base64_encoded_keys'))
md_copy[key] = return_val
return md_copy
@@ -180,15 +185,24 @@ class DataSource(object):
"""
self._dirty_cache = True
return_value = self._get_data()
- json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
if not return_value:
return return_value
+ self.persist_instance_data()
+ return return_value
+
+ def persist_instance_data(self):
+ """Process and write INSTANCE_JSON_FILE with all instance metadata.
+ Replace any hyphens with underscores in key names for use in template
+ processing.
+
+ @return True on successful write, False otherwise.
+ """
instance_data = {
'ds': {
- 'meta-data': self.metadata,
- 'user-data': self.get_userdata_raw(),
- 'vendor-data': self.get_vendordata_raw()}}
+ 'meta_data': self.metadata,
+ 'user_data': self.get_userdata_raw(),
+ 'vendor_data': self.get_vendordata_raw()}}
if hasattr(self, 'network_json'):
network_json = getattr(self, 'network_json')
if network_json != UNSET:
@@ -202,16 +216,17 @@ class DataSource(object):
try:
# Process content base64encoding unserializable values
content = util.json_dumps(instance_data)
- # Strip base64: prefix and return base64-encoded-keys
- processed_data = process_base64_metadata(json.loads(content))
+ # Strip base64: prefix and set base64_encoded_keys list.
+ processed_data = process_instance_metadata(json.loads(content))
except TypeError as e:
LOG.warning('Error persisting instance-data.json: %s', str(e))
- return return_value
+ return False
except UnicodeDecodeError as e:
LOG.warning('Error persisting instance-data.json: %s', str(e))
- return return_value
+ return False
+ json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
write_json(json_file, processed_data, mode=0o600)
- return return_value
+ return True
def _get_data(self):
"""Walk metadata sources, process crawled data and save attributes."""
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 8f9c144..d6f39a1 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -38,21 +38,38 @@ KEY_COPIES = (
('local-hostname', 'hostname', False),
('instance-id', 'uuid', True),
)
+
+# Versions and names taken from nova source nova/api/metadata/base.py
OS_LATEST = 'latest'
OS_FOLSOM = '2012-08-10'
OS_GRIZZLY = '2013-04-04'
OS_HAVANA = '2013-10-17'
OS_LIBERTY = '2015-10-15'
+# NEWTON_ONE adds 'devices' to md (sriov-pf-passthrough-neutron-port-vlan)
+OS_NEWTON_ONE = '2016-06-30'
+# NEWTON_TWO adds vendor_data2.json (vendordata-reboot)
+OS_NEWTON_TWO = '2016-10-06'
+# OS_OCATA adds 'vif' field to devices (sriov-pf-passthrough-neutron-port-vlan)
+OS_OCATA = '2017-02-22'
+# OS_ROCKY adds a vf_trusted field to devices (sriov-trusted-vfs)
+OS_ROCKY = '2018-08-27'
+
+
# keep this in chronological order. new supported versions go at the end.
OS_VERSIONS = (
OS_FOLSOM,
OS_GRIZZLY,
OS_HAVANA,
OS_LIBERTY,
+ OS_NEWTON_ONE,
+ OS_NEWTON_TWO,
+ OS_OCATA,
+ OS_ROCKY,
)
PHYSICAL_TYPES = (
None,
+ 'bgpovs', # not present in OpenStack upstream but used on OVH cloud.
'bridge',
'dvs',
'ethernet',
@@ -439,7 +456,7 @@ class MetadataReader(BaseReader):
return self._versions
found = []
version_path = self._path_join(self.base_path, "openstack")
- content = self._path_read(version_path)
+ content = self._path_read(version_path, decode=True)
for line in content.splitlines():
line = line.strip()
if not line:
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index 9e939c1..8299af2 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -20,10 +20,12 @@ class DataSourceTestSubclassNet(DataSource):
dsname = 'MyTestSubclass'
url_max_wait = 55
- def __init__(self, sys_cfg, distro, paths, custom_userdata=None):
+ def __init__(self, sys_cfg, distro, paths, custom_userdata=None,
+ get_data_retval=True):
super(DataSourceTestSubclassNet, self).__init__(
sys_cfg, distro, paths)
self._custom_userdata = custom_userdata
+ self._get_data_retval = get_data_retval
def _get_cloud_name(self):
return 'SubclassCloudName'
@@ -37,7 +39,7 @@ class DataSourceTestSubclassNet(DataSource):
else:
self.userdata_raw = 'userdata_raw'
self.vendordata_raw = 'vendordata_raw'
- return True
+ return self._get_data_retval
class InvalidDataSourceTestSubclassNet(DataSource):
@@ -264,7 +266,18 @@ class TestDataSource(CiTestCase):
self.assertEqual('fqdnhostname.domain.com',
datasource.get_hostname(fqdn=True))
- def test_get_data_write_json_instance_data(self):
+ def test_get_data_does_not_write_instance_data_on_failure(self):
+ """get_data does not write INSTANCE_JSON_FILE on get_data False."""
+ tmp = self.tmp_dir()
+ datasource = DataSourceTestSubclassNet(
+ self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
+ get_data_retval=False)
+ self.assertFalse(datasource.get_data())
+ json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
+ self.assertFalse(
+ os.path.exists(json_file), 'Found unexpected file %s' % json_file)
+
+ def test_get_data_writes_json_instance_data_on_success(self):
"""get_data writes INSTANCE_JSON_FILE to run_dir as readonly root."""
tmp = self.tmp_dir()
datasource = DataSourceTestSubclassNet(
@@ -273,7 +286,7 @@ class TestDataSource(CiTestCase):
json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
content = util.load_file(json_file)
expected = {
- 'base64-encoded-keys': [],
+ 'base64_encoded_keys': [],
'v1': {
'availability-zone': 'myaz',
'cloud-name': 'subclasscloudname',
@@ -281,11 +294,12 @@ class TestDataSource(CiTestCase):
'local-hostname': 'test-subclass-hostname',
'region': 'myregion'},
'ds': {
- 'meta-data': {'availability_zone': 'myaz',
+ 'meta_data': {'availability_zone': 'myaz',
'local-hostname': 'test-subclass-hostname',
'region': 'myregion'},
- 'user-data': 'userdata_raw',
- 'vendor-data': 'vendordata_raw'}}
+ 'user_data': 'userdata_raw',
+ 'vendor_data': 'vendordata_raw'}}
+ self.maxDiff = None
self.assertEqual(expected, util.load_json(content))
file_stat = os.stat(json_file)
self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode))
@@ -296,7 +310,7 @@ class TestDataSource(CiTestCase):
datasource = DataSourceTestSubclassNet(
self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
custom_userdata={'key1': 'val1', 'key2': {'key2.1': self.paths}})
- self.assertTrue(datasource.get_data())
+ datasource.get_data()
json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
content = util.load_file(json_file)
expected_userdata = {
@@ -306,7 +320,40 @@ class TestDataSource(CiTestCase):
" 'cloudinit.helpers.Paths'>"}}
instance_json = util.load_json(content)
self.assertEqual(
- expected_userdata, instance_json['ds']['user-data'])
+ expected_userdata, instance_json['ds']['user_data'])
+
+ def test_persist_instance_data_writes_ec2_metadata_when_set(self):
+ """When ec2_metadata class attribute is set, persist to json."""
+ tmp = self.tmp_dir()
+ datasource = DataSourceTestSubclassNet(
+ self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+ datasource.ec2_metadata = UNSET
+ datasource.get_data()
+ json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
+ instance_data = util.load_json(util.load_file(json_file))
+ self.assertNotIn('ec2_metadata', instance_data['ds'])
+ datasource.ec2_metadata = {'ec2stuff': 'is good'}
+ datasource.persist_instance_data()
+ instance_data = util.load_json(util.load_file(json_file))
+ self.assertEqual(
+ {'ec2stuff': 'is good'},
+ instance_data['ds']['ec2_metadata'])
+
+ def test_persist_instance_data_writes_network_json_when_set(self):
+ """When network_data.json class attribute is set, persist to json."""
+ tmp = self.tmp_dir()
+ datasource = DataSourceTestSubclassNet(
+ self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+ datasource.get_data()
+ json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
+ instance_data = util.load_json(util.load_file(json_file))
+ self.assertNotIn('network_json', instance_data['ds'])
+ datasource.network_json = {'network_json': 'is good'}
+ datasource.persist_instance_data()
+ instance_data = util.load_json(util.load_file(json_file))
+ self.assertEqual(
+ {'network_json': 'is good'},
+ instance_data['ds']['network_json'])
@skipIf(not six.PY3, "json serialization on <= py2.7 handles bytes")
def test_get_data_base64encodes_unserializable_bytes(self):
@@ -320,11 +367,11 @@ class TestDataSource(CiTestCase):
content = util.load_file(json_file)
instance_json = util.load_json(content)
self.assertEqual(
- ['ds/user-data/key2/key2.1'],
- instance_json['base64-encoded-keys'])
+ ['ds/user_data/key2/key2.1'],
+ instance_json['base64_encoded_keys'])
self.assertEqual(
{'key1': 'val1', 'key2': {'key2.1': 'EjM='}},
- instance_json['ds']['user-data'])
+ instance_json['ds']['user_data'])
@skipIf(not six.PY2, "json serialization on <= py2.7 handles bytes")
def test_get_data_handles_bytes_values(self):
@@ -337,10 +384,10 @@ class TestDataSource(CiTestCase):
json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
content = util.load_file(json_file)
instance_json = util.load_json(content)
- self.assertEqual([], instance_json['base64-encoded-keys'])
+ self.assertEqual([], instance_json['base64_encoded_keys'])
self.assertEqual(
{'key1': 'val1', 'key2': {'key2.1': '\x123'}},
- instance_json['ds']['user-data'])
+ instance_json['ds']['user_data'])
@skipIf(not six.PY2, "Only python2 hits UnicodeDecodeErrors on non-utf8")
def test_non_utf8_encoding_logs_warning(self):
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 8874d40..ef5c699 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -17,10 +17,11 @@ from cloudinit.settings import (
from cloudinit import handlers
# Default handlers (used if not overridden)
-from cloudinit.handlers import boot_hook as bh_part
-from cloudinit.handlers import cloud_config as cc_part
-from cloudinit.handlers import shell_script as ss_part
-from cloudinit.handlers import upstart_job as up_part
+from cloudinit.handlers.boot_hook import BootHookPartHandler
+from cloudinit.handlers.cloud_config import CloudConfigPartHandler
+from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler
+from cloudinit.handlers.shell_script import ShellScriptPartHandler
+from cloudinit.handlers.upstart_job import UpstartJobPartHandler
from cloudinit.event import EventType
@@ -413,12 +414,17 @@ class Init(object):
'datasource': self.datasource,
})
# TODO(harlowja) Hmmm, should we dynamically import these??
+ cloudconfig_handler = CloudConfigPartHandler(**opts)
+ shellscript_handler = ShellScriptPartHandler(**opts)
def_handlers = [
- cc_part.CloudConfigPartHandler(**opts),
- ss_part.ShellScriptPartHandler(**opts),
- bh_part.BootHookPartHandler(**opts),
- up_part.UpstartJobPartHandler(**opts),
+ cloudconfig_handler,
+ shellscript_handler,
+ BootHookPartHandler(**opts),
+ UpstartJobPartHandler(**opts),
]
+ opts.update(
+ {'sub_handlers': [cloudconfig_handler, shellscript_handler]})
+ def_handlers.append(JinjaTemplatePartHandler(**opts))
return def_handlers
def _default_userdata_handlers(self):
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index 7e7acb8..b668674 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -13,6 +13,7 @@
import collections
import re
+
try:
from Cheetah.Template import Template as CTemplate
CHEETAH_AVAILABLE = True
@@ -20,23 +21,44 @@ except (ImportError, AttributeError):
CHEETAH_AVAILABLE = False
try:
- import jinja2
+ from jinja2.runtime import implements_to_string
from jinja2 import Template as JTemplate
+ from jinja2 import DebugUndefined as JUndefined
JINJA_AVAILABLE = True
except (ImportError, AttributeError):
+ from cloudinit.helpers import identity
+ implements_to_string = identity
JINJA_AVAILABLE = False
+ JUndefined = object
from cloudinit import log as logging
from cloudinit import type_utils as tu
from cloudinit import util
+
LOG = logging.getLogger(__name__)
TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I)
BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)')
+MISSING_JINJA_PREFIX = u'CI_MISSING_JINJA_VAR/'
+
+
+@implements_to_string # Needed for python2.7. Otherwise cached super.__str__
+class UndefinedJinjaVariable(JUndefined):
+ """Class used to represent any undefined jinja template varible."""
+
+ def __str__(self):
+ return u'%s%s' % (MISSING_JINJA_PREFIX, self._undefined_name)
+
+ def __sub__(self, other):
+ other = str(other).replace(MISSING_JINJA_PREFIX, '')
+ raise TypeError(
+ 'Undefined jinja variable: "{this}-{other}". Jinja tried'
+ ' subtraction. Perhaps you meant "{this}_{other}"?'.format(
+ this=self._undefined_name, other=other))
def basic_render(content, params):
- """This does simple replacement of bash variable like templates.
+ """This does sumple replacement of bash variable like templates.
It identifies patterns like ${a} or $a and can also identify patterns like
${a.b} or $a.b which will look for a key 'b' in the dictionary rooted
@@ -82,7 +104,7 @@ def detect_template(text):
# keep_trailing_newline is in jinja2 2.7+, not 2.6
add = "\n" if content.endswith("\n") else ""
return JTemplate(content,
- undefined=jinja2.StrictUndefined,
+ undefined=UndefinedJinjaVariable,
trim_blocks=True).render(**params) + add
if text.find("\n") != -1:
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 42f56c2..2eb7b0c 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -32,6 +32,7 @@ from cloudinit import cloud
from cloudinit import distros
from cloudinit import helpers as ch
from cloudinit.sources import DataSourceNone
+from cloudinit.templater import JINJA_AVAILABLE
from cloudinit import util
_real_subp = util.subp
@@ -518,6 +519,14 @@ def skipUnlessJsonSchema():
_missing_jsonschema_dep, "No python-jsonschema dependency present.")
+def skipUnlessJinja():
+ return skipIf(not JINJA_AVAILABLE, "No jinja dependency present.")
+
+
+def skipIfJinja():
+ return skipIf(JINJA_AVAILABLE, "Jinja dependency present.")
+
+
# older versions of mock do not have the useful 'assert_not_called'
if not hasattr(mock.Mock, 'assert_not_called'):
def __mock_assert_not_called(mmock):
diff --git a/debian/changelog b/debian/changelog
index 6b1a35f..4ad8877 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,14 @@
+cloud-init (18.3-44-g84bf2482-0ubuntu1) cosmic; urgency=medium
+
+ * New upstream snapshot.
+ - bash_completion/cloud-init: fix shell syntax error.
+ - EphemeralIPv4Network: Be more explicit when adding default route.
+ - OpenStack: support reading of newer versions of metdata.
+ - OpenStack: fix bug causing 'latest' version to be used from network.
+ - user-data: jinja template to render instance-data.json in cloud-config
+
+ -- Ryan Harper <ryan.harper@xxxxxxxxxxxxx> Fri, 14 Sep 2018 14:06:29 -0500
+
cloud-init (18.3-39-g757247f9-0ubuntu1) cosmic; urgency=medium
* New upstream snapshot.
diff --git a/doc/rtd/topics/capabilities.rst b/doc/rtd/topics/capabilities.rst
index 3e2c9e3..2d8e253 100644
--- a/doc/rtd/topics/capabilities.rst
+++ b/doc/rtd/topics/capabilities.rst
@@ -16,13 +16,15 @@ User configurability
`Cloud-init`_ 's behavior can be configured via user-data.
- User-data can be given by the user at instance launch time.
+ User-data can be given by the user at instance launch time. See
+ :ref:`user_data_formats` for acceptable user-data content.
+
This is done via the ``--user-data`` or ``--user-data-file`` argument to
ec2-run-instances for example.
-* Check your local clients documentation for how to provide a `user-data`
- string or `user-data` file for usage by cloud-init on instance creation.
+* Check your local client's documentation for how to provide a `user-data`
+ string or `user-data` file to cloud-init on instance creation.
Feature detection
@@ -166,6 +168,13 @@ likely be promoted to top-level subcommands when stable.
validation is work in progress and supports a subset of cloud-config
modules.
+ * ``cloud-init devel render``: Use cloud-init's jinja template render to
+ process **#cloud-config** or **custom-scripts**, injecting any variables
+ from ``/run/cloud-init/instance-data.json``. It accepts a user-data file
+ containing the jinja template header ``## template: jinja`` and renders
+ that content with any instance-data.json variables present.
+
+
.. _cli_clean:
cloud-init clean
diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
index 8303458..14432e6 100644
--- a/doc/rtd/topics/datasources.rst
+++ b/doc/rtd/topics/datasources.rst
@@ -18,6 +18,8 @@ single way to access the different cloud systems methods to provide this data
through the typical usage of subclasses.
+.. _instance_metadata:
+
instance-data
-------------
For reference, cloud-init stores all the metadata, vendordata and userdata
@@ -110,6 +112,51 @@ Below is an instance-data.json example from an OpenStack instance:
}
}
+
+As of cloud-init v. 18.4, any values present in
+``/run/cloud-init/instance-data.json`` can be used in cloud-init user data
+scripts or cloud config data. This allows consumers to use cloud-init's
+vendor-neutral, standardized metadata keys as well as datasource-specific
+content for any scripts or cloud-config modules they are using.
+
+To use instance-data.json values in scripts and **#config-config** files the
+user-data will need to contain the following header as the first line **## template: jinja**. Cloud-init will source all variables defined in
+``/run/cloud-init/instance-data.json`` and allow scripts or cloud-config files
+to reference those paths. Below are two examples::
+
+ * Cloud config calling home with the ec2 public hostname and avaliability-zone
+ ```
+ ## template: jinja
+ #cloud-config
+ runcmd:
+ - echo 'EC2 public hostname allocated to instance: {{ ds.meta_data.public_hostname }}' > /tmp/instance_metadata
+ - echo 'EC2 avaiability zone: {{ v1.availability_zone }}' >> /tmp/instance_metadata
+ - curl -X POST -d '{"hostname": "{{ds.meta_data.public_hostname }}", "availability-zone": "{{ v1.availability_zone }}"}' https://example.com.com
+ ```
+
+ * Custom user script performing different operations based on region
+ ```
+ ## template: jinja
+ #!/bin/bash
+ {% if v1.region == 'us-east-2' -%}
+ echo 'Installing custom proxies for {{ v1.region }}
+ sudo apt-get install my-xtra-fast-stack
+ {%- endif %}
+ ...
+
+ ```
+
+.. note::
+ Trying to reference jinja variables that don't exist in
+ instance-data.json will result in warnings in ``/var/log/cloud-init.log``
+ and the following string in your rendered user-data:
+ ``CI_MISSING_JINJA_VAR/<your_varname>``.
+
+.. note::
+ To save time designing your user-data for a specific cloud's
+ instance-data.json, use the 'render' cloud-init command on an
+ instance booted on your favorite cloud. See :ref:`cli_devel` for more
+ information.
Datasource API
diff --git a/doc/rtd/topics/format.rst b/doc/rtd/topics/format.rst
index 1b0ff36..15234d2 100644
--- a/doc/rtd/topics/format.rst
+++ b/doc/rtd/topics/format.rst
@@ -1,6 +1,8 @@
-*******
-Formats
-*******
+.. _user_data_formats:
+
+*****************
+User-Data Formats
+*****************
User data that will be acted upon by cloud-init must be in one of the following types.
@@ -65,6 +67,11 @@ Typically used by those who just want to execute a shell script.
Begins with: ``#!`` or ``Content-Type: text/x-shellscript`` when using a MIME archive.
+.. note::
+ New in cloud-init v. 18.4: User-data scripts can also render cloud instance
+ metadata variables using jinja templating. See
+ :ref:`instance_metadata` for more information.
+
Example
-------
@@ -103,12 +110,18 @@ These things include:
- certain ssh keys should be imported
- *and many more...*
-**Note:** The file must be valid yaml syntax.
+.. note::
+ This file must be valid yaml syntax.
See the :ref:`yaml_examples` section for a commented set of examples of supported cloud config formats.
Begins with: ``#cloud-config`` or ``Content-Type: text/cloud-config`` when using a MIME archive.
+.. note::
+ New in cloud-init v. 18.4: Cloud config dta can also render cloud instance
+ metadata variables using jinja templating. See
+ :ref:`instance_metadata` for more information.
+
Upstart Job
===========
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 696db8d..2745827 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -168,7 +168,7 @@ class CloudTestCase(unittest.TestCase):
' OS: %s not bionic or newer' % self.os_name)
instance_data = json.loads(out)
self.assertEqual(
- ['ds/user-data'], instance_data['base64-encoded-keys'])
+ ['ds/user_data'], instance_data['base64_encoded_keys'])
ds = instance_data.get('ds', {})
v1_data = instance_data.get('v1', {})
metadata = ds.get('meta-data', {})
@@ -214,8 +214,8 @@ class CloudTestCase(unittest.TestCase):
instance_data = json.loads(out)
v1_data = instance_data.get('v1', {})
self.assertEqual(
- ['ds/user-data', 'ds/vendor-data'],
- sorted(instance_data['base64-encoded-keys']))
+ ['ds/user_data', 'ds/vendor_data'],
+ sorted(instance_data['base64_encoded_keys']))
self.assertEqual('nocloud', v1_data['cloud-name'])
self.assertIsNone(
v1_data['availability-zone'],
@@ -249,7 +249,7 @@ class CloudTestCase(unittest.TestCase):
instance_data = json.loads(out)
v1_data = instance_data.get('v1', {})
self.assertEqual(
- ['ds/user-data'], instance_data['base64-encoded-keys'])
+ ['ds/user_data'], instance_data['base64_encoded_keys'])
self.assertEqual('nocloud', v1_data['cloud-name'])
self.assertIsNone(
v1_data['availability-zone'],
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index 9751ed9..abe820e 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -2,27 +2,34 @@
"""Tests of the built-in user data handlers."""
+import copy
import os
import shutil
import tempfile
+from textwrap import dedent
-try:
- from unittest import mock
-except ImportError:
- import mock
-from cloudinit.tests import helpers as test_helpers
+from cloudinit.tests.helpers import (
+ FilesystemMockingTestCase, CiTestCase, mock, skipUnlessJinja)
from cloudinit import handlers
from cloudinit import helpers
from cloudinit import util
-from cloudinit.handlers import upstart_job
+from cloudinit.handlers.cloud_config import CloudConfigPartHandler
+from cloudinit.handlers.jinja_template import (
+ JinjaTemplatePartHandler, convert_jinja_instance_data,
+ render_jinja_payload)
+from cloudinit.handlers.shell_script import ShellScriptPartHandler
+from cloudinit.handlers.upstart_job import UpstartJobPartHandler
from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE)
-class TestBuiltins(test_helpers.FilesystemMockingTestCase):
+class TestUpstartJobPartHandler(FilesystemMockingTestCase):
+
+ mpath = 'cloudinit.handlers.upstart_job.'
+
def test_upstart_frequency_no_out(self):
c_root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, c_root)
@@ -32,14 +39,13 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase):
'cloud_dir': c_root,
'upstart_dir': up_root,
})
- freq = PER_ALWAYS
- h = upstart_job.UpstartJobPartHandler(paths)
+ h = UpstartJobPartHandler(paths)
# No files should be written out when
# the frequency is ! per-instance
h.handle_part('', handlers.CONTENT_START,
None, None, None)
h.handle_part('blah', 'text/upstart-job',
- 'test.conf', 'blah', freq)
+ 'test.conf', 'blah', frequency=PER_ALWAYS)
h.handle_part('', handlers.CONTENT_END,
None, None, None)
self.assertEqual(0, len(os.listdir(up_root)))
@@ -48,7 +54,6 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase):
# files should be written out when frequency is ! per-instance
new_root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, new_root)
- freq = PER_INSTANCE
self.patchOS(new_root)
self.patchUtils(new_root)
@@ -56,22 +61,297 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase):
'upstart_dir': "/etc/upstart",
})
- upstart_job.SUITABLE_UPSTART = True
util.ensure_dir("/run")
util.ensure_dir("/etc/upstart")
- with mock.patch.object(util, 'subp') as mockobj:
- h = upstart_job.UpstartJobPartHandler(paths)
- h.handle_part('', handlers.CONTENT_START,
- None, None, None)
- h.handle_part('blah', 'text/upstart-job',
- 'test.conf', 'blah', freq)
- h.handle_part('', handlers.CONTENT_END,
- None, None, None)
+ with mock.patch(self.mpath + 'SUITABLE_UPSTART', return_value=True):
+ with mock.patch.object(util, 'subp') as m_subp:
+ h = UpstartJobPartHandler(paths)
+ h.handle_part('', handlers.CONTENT_START,
+ None, None, None)
+ h.handle_part('blah', 'text/upstart-job',
+ 'test.conf', 'blah', frequency=PER_INSTANCE)
+ h.handle_part('', handlers.CONTENT_END,
+ None, None, None)
- self.assertEqual(len(os.listdir('/etc/upstart')), 1)
+ self.assertEqual(len(os.listdir('/etc/upstart')), 1)
- mockobj.assert_called_once_with(
+ m_subp.assert_called_once_with(
['initctl', 'reload-configuration'], capture=False)
+
+class TestJinjaTemplatePartHandler(CiTestCase):
+
+ with_logs = True
+
+ mpath = 'cloudinit.handlers.jinja_template.'
+
+ def setUp(self):
+ super(TestJinjaTemplatePartHandler, self).setUp()
+ self.tmp = self.tmp_dir()
+ self.run_dir = os.path.join(self.tmp, 'run_dir')
+ util.ensure_dir(self.run_dir)
+ self.paths = helpers.Paths({
+ 'cloud_dir': self.tmp, 'run_dir': self.run_dir})
+
+ def test_jinja_template_part_handler_defaults(self):
+ """On init, paths are saved and subhandler types are empty."""
+ h = JinjaTemplatePartHandler(self.paths)
+ self.assertEqual(['## template: jinja'], h.prefixes)
+ self.assertEqual(3, h.handler_version)
+ self.assertEqual(self.paths, h.paths)
+ self.assertEqual({}, h.sub_handlers)
+
+ def test_jinja_template_part_handler_looks_up_sub_handler_types(self):
+ """When sub_handlers are passed, init lists types of subhandlers."""
+ script_handler = ShellScriptPartHandler(self.paths)
+ cloudconfig_handler = CloudConfigPartHandler(self.paths)
+ h = JinjaTemplatePartHandler(
+ self.paths, sub_handlers=[script_handler, cloudconfig_handler])
+ self.assertItemsEqual(
+ ['text/cloud-config', 'text/cloud-config-jsonp',
+ 'text/x-shellscript'],
+ h.sub_handlers)
+
+ def test_jinja_template_part_handler_looks_up_subhandler_types(self):
+ """When sub_handlers are passed, init lists types of subhandlers."""
+ script_handler = ShellScriptPartHandler(self.paths)
+ cloudconfig_handler = CloudConfigPartHandler(self.paths)
+ h = JinjaTemplatePartHandler(
+ self.paths, sub_handlers=[script_handler, cloudconfig_handler])
+ self.assertItemsEqual(
+ ['text/cloud-config', 'text/cloud-config-jsonp',
+ 'text/x-shellscript'],
+ h.sub_handlers)
+
+ def test_jinja_template_handle_noop_on_content_signals(self):
+ """Perform no part handling when content type is CONTENT_SIGNALS."""
+ script_handler = ShellScriptPartHandler(self.paths)
+
+ h = JinjaTemplatePartHandler(
+ self.paths, sub_handlers=[script_handler])
+ with mock.patch.object(script_handler, 'handle_part') as m_handle_part:
+ h.handle_part(
+ data='data', ctype=handlers.CONTENT_START, filename='part-1',
+ payload='## template: jinja\n#!/bin/bash\necho himom',
+ frequency='freq', headers='headers')
+ m_handle_part.assert_not_called()
+
+ @skipUnlessJinja()
+ def test_jinja_template_handle_subhandler_v2_with_clean_payload(self):
+ """Call version 2 subhandler.handle_part with stripped payload."""
+ script_handler = ShellScriptPartHandler(self.paths)
+ self.assertEqual(2, script_handler.handler_version)
+
+ # Create required instance-data.json file
+ instance_json = os.path.join(self.run_dir, 'instance-data.json')
+ instance_data = {'topkey': 'echo himom'}
+ util.write_file(instance_json, util.json_dumps(instance_data))
+ h = JinjaTemplatePartHandler(
+ self.paths, sub_handlers=[script_handler])
+ with mock.patch.object(script_handler, 'handle_part') as m_part:
+ # ctype with leading '!' not in handlers.CONTENT_SIGNALS
+ h.handle_part(
+ data='data', ctype="!" + handlers.CONTENT_START,
+ filename='part01',
+ payload='## template: jinja \t \n#!/bin/bash\n{{ topkey }}',
+ frequency='freq', headers='headers')
+ m_part.assert_called_once_with(
+ 'data', '!__begin__', 'part01', '#!/bin/bash\necho himom', 'freq')
+
+ @skipUnlessJinja()
+ def test_jinja_template_handle_subhandler_v3_with_clean_payload(self):
+ """Call version 3 subhandler.handle_part with stripped payload."""
+ cloudcfg_handler = CloudConfigPartHandler(self.paths)
+ self.assertEqual(3, cloudcfg_handler.handler_version)
+
+ # Create required instance-data.json file
+ instance_json = os.path.join(self.run_dir, 'instance-data.json')
+ instance_data = {'topkey': {'sub': 'runcmd: [echo hi]'}}
+ util.write_file(instance_json, util.json_dumps(instance_data))
+ h = JinjaTemplatePartHandler(
+ self.paths, sub_handlers=[cloudcfg_handler])
+ with mock.patch.object(cloudcfg_handler, 'handle_part') as m_part:
+ # ctype with leading '!' not in handlers.CONTENT_SIGNALS
+ h.handle_part(
+ data='data', ctype="!" + handlers.CONTENT_END,
+ filename='part01',
+ payload='## template: jinja\n#cloud-config\n{{ topkey.sub }}',
+ frequency='freq', headers='headers')
+ m_part.assert_called_once_with(
+ 'data', '!__end__', 'part01', '#cloud-config\nruncmd: [echo hi]',
+ 'freq', 'headers')
+
+ def test_jinja_template_handle_errors_on_missing_instance_data_json(self):
+ """If instance-data is absent, raise an error from handle_part."""
+ script_handler = ShellScriptPartHandler(self.paths)
+ h = JinjaTemplatePartHandler(
+ self.paths, sub_handlers=[script_handler])
+ with self.assertRaises(RuntimeError) as context_manager:
+ h.handle_part(
+ data='data', ctype="!" + handlers.CONTENT_START,
+ filename='part01',
+ payload='## template: jinja \n#!/bin/bash\necho himom',
+ frequency='freq', headers='headers')
+ script_file = os.path.join(script_handler.script_dir, 'part01')
+ self.assertEqual(
+ 'Cannot render jinja template vars. Instance data not yet present'
+ ' at {}/instance-data.json'.format(
+ self.run_dir), str(context_manager.exception))
+ self.assertFalse(
+ os.path.exists(script_file),
+ 'Unexpected file created %s' % script_file)
+
+ @skipUnlessJinja()
+ def test_jinja_template_handle_renders_jinja_content(self):
+ """When present, render jinja variables from instance-data.json."""
+ script_handler = ShellScriptPartHandler(self.paths)
+ instance_json = os.path.join(self.run_dir, 'instance-data.json')
+ instance_data = {'topkey': {'subkey': 'echo himom'}}
+ util.write_file(instance_json, util.json_dumps(instance_data))
+ h = JinjaTemplatePartHandler(
+ self.paths, sub_handlers=[script_handler])
+ h.handle_part(
+ data='data', ctype="!" + handlers.CONTENT_START,
+ filename='part01',
+ payload=(
+ '## template: jinja \n'
+ '#!/bin/bash\n'
+ '{{ topkey.subkey|default("nosubkey") }}'),
+ frequency='freq', headers='headers')
+ script_file = os.path.join(script_handler.script_dir, 'part01')
+ self.assertNotIn(
+ 'Instance data not yet present at {}/instance-data.json'.format(
+ self.run_dir),
+ self.logs.getvalue())
+ self.assertEqual(
+ '#!/bin/bash\necho himom', util.load_file(script_file))
+
+ @skipUnlessJinja()
+ def test_jinja_template_handle_renders_jinja_content_missing_keys(self):
+ """When specified jinja variable is undefined, log a warning."""
+ script_handler = ShellScriptPartHandler(self.paths)
+ instance_json = os.path.join(self.run_dir, 'instance-data.json')
+ instance_data = {'topkey': {'subkey': 'echo himom'}}
+ util.write_file(instance_json, util.json_dumps(instance_data))
+ h = JinjaTemplatePartHandler(
+ self.paths, sub_handlers=[script_handler])
+ h.handle_part(
+ data='data', ctype="!" + handlers.CONTENT_START,
+ filename='part01',
+ payload='## template: jinja \n#!/bin/bash\n{{ goodtry }}',
+ frequency='freq', headers='headers')
+ script_file = os.path.join(script_handler.script_dir, 'part01')
+ self.assertTrue(
+ os.path.exists(script_file),
+ 'Missing expected file %s' % script_file)
+ self.assertIn(
+ "WARNING: Could not render jinja template variables in file"
+ " 'part01': 'goodtry'\n",
+ self.logs.getvalue())
+
+
+class TestConvertJinjaInstanceData(CiTestCase):
+
+ def test_convert_instance_data_hyphens_to_underscores(self):
+ """Replace hyphenated keys with underscores in instance-data."""
+ data = {'hyphenated-key': 'hyphenated-val',
+ 'underscore_delim_key': 'underscore_delimited_val'}
+ expected_data = {'hyphenated_key': 'hyphenated-val',
+ 'underscore_delim_key': 'underscore_delimited_val'}
+ self.assertEqual(
+ expected_data,
+ convert_jinja_instance_data(data=data))
+
+ def test_convert_instance_data_promotes_versioned_keys_to_top_level(self):
+ """Any versioned keys are promoted as top-level keys
+
+ This provides any cloud-init standardized keys up at a top-level to
+ allow ease of reference for users. Intsead of v1.availability_zone,
+ the name availability_zone can be used in templates.
+ """
+ data = {'ds': {'dskey1': 1, 'dskey2': 2},
+ 'v1': {'v1key1': 'v1.1'},
+ 'v2': {'v2key1': 'v2.1'}}
+ expected_data = copy.deepcopy(data)
+ expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'})
+
+ converted_data = convert_jinja_instance_data(data=data)
+ self.assertItemsEqual(
+ ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys())
+ self.assertEqual(
+ expected_data,
+ converted_data)
+
+ def test_convert_instance_data_most_recent_version_of_promoted_keys(self):
+ """The most-recent versioned key value is promoted to top-level."""
+ data = {'v1': {'key1': 'old v1 key1', 'key2': 'old v1 key2'},
+ 'v2': {'key1': 'newer v2 key1', 'key3': 'newer v2 key3'},
+ 'v3': {'key1': 'newest v3 key1'}}
+ expected_data = copy.deepcopy(data)
+ expected_data.update(
+ {'key1': 'newest v3 key1', 'key2': 'old v1 key2',
+ 'key3': 'newer v2 key3'})
+
+ converted_data = convert_jinja_instance_data(data=data)
+ self.assertEqual(
+ expected_data,
+ converted_data)
+
+ def test_convert_instance_data_decodes_decode_paths(self):
+ """Any decode_paths provided are decoded by convert_instance_data."""
+ data = {'key1': {'subkey1': 'aGkgbW9t'}, 'key2': 'aGkgZGFk'}
+ expected_data = copy.deepcopy(data)
+ expected_data['key1']['subkey1'] = 'hi mom'
+
+ converted_data = convert_jinja_instance_data(
+ data=data, decode_paths=('key1/subkey1',))
+ self.assertEqual(
+ expected_data,
+ converted_data)
+
+
+class TestRenderJinjaPayload(CiTestCase):
+
+ with_logs = True
+
+ @skipUnlessJinja()
+ def test_render_jinja_payload_logs_jinja_vars_on_debug(self):
+ """When debug is True, log jinja varables available."""
+ payload = (
+ '## template: jinja\n#!/bin/sh\necho hi from {{ v1.hostname }}')
+ instance_data = {'v1': {'hostname': 'foo'}, 'instance-id': 'iid'}
+ expected_log = dedent("""\
+ DEBUG: Converted jinja variables
+ {
+ "hostname": "foo",
+ "instance_id": "iid",
+ "v1": {
+ "hostname": "foo"
+ }
+ }
+ """)
+ self.assertEqual(
+ render_jinja_payload(
+ payload=payload, payload_fn='myfile',
+ instance_data=instance_data, debug=True),
+ '#!/bin/sh\necho hi from foo')
+ self.assertEqual(expected_log, self.logs.getvalue())
+
+ @skipUnlessJinja()
+ def test_render_jinja_payload_replaces_missing_variables_and_warns(self):
+ """Warn on missing jinja variables and replace the absent variable."""
+ payload = (
+ '## template: jinja\n#!/bin/sh\necho hi from {{ NOTHERE }}')
+ instance_data = {'v1': {'hostname': 'foo'}, 'instance-id': 'iid'}
+ self.assertEqual(
+ render_jinja_payload(
+ payload=payload, payload_fn='myfile',
+ instance_data=instance_data),
+ '#!/bin/sh\necho hi from CI_MISSING_JINJA_VAR/NOTHERE')
+ expected_log = (
+ 'WARNING: Could not render jinja template variables in file'
+ " 'myfile': 'NOTHERE'")
+ self.assertIn(expected_log, self.logs.getvalue())
+
# vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 6e1e971..a731f1e 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -12,7 +12,7 @@ import re
from cloudinit.tests import helpers as test_helpers
from six.moves.urllib.parse import urlparse
-from six import StringIO
+from six import StringIO, text_type
from cloudinit import helpers
from cloudinit import settings
@@ -555,4 +555,94 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
m_proc_env.assert_called_with(1)
+class TestMetadataReader(test_helpers.HttprettyTestCase):
+ """Test the MetadataReader."""
+ burl = 'http://169.254.169.254/'
+ md_base = {
+ 'availability_zone': 'myaz1',
+ 'hostname': 'sm-foo-test.novalocal',
+ "keys": [{"data": PUBKEY, "name": "brickies", "type": "ssh"}],
+ 'launch_index': 0,
+ 'name': 'sm-foo-test',
+ 'public_keys': {'mykey': PUBKEY},
+ 'project_id': '6a103f813b774b9fb15a4fcd36e1c056',
+ 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'}
+
+ def register(self, path, body=None, status=200):
+ content = (body if not isinstance(body, text_type)
+ else body.encode('utf-8'))
+ hp.register_uri(
+ hp.GET, self.burl + "openstack" + path, status=status,
+ body=content)
+
+ def register_versions(self, versions):
+ self.register("", '\n'.join(versions))
+ self.register("/", '\n'.join(versions))
+
+ def register_version(self, version, data):
+ content = '\n'.join(sorted(data.keys()))
+ self.register(version, content)
+ self.register(version + "/", content)
+ for path, content in data.items():
+ self.register("/%s/%s" % (version, path), content)
+ self.register("/%s/%s" % (version, path), content)
+ if 'user_data' not in data:
+ self.register("/%s/user_data" % version, "nodata", status=404)
+
+ def test__find_working_version(self):
+ """Test a working version ignores unsupported."""
+ unsup = "2016-11-09"
+ self.register_versions(
+ [openstack.OS_FOLSOM, openstack.OS_LIBERTY, unsup,
+ openstack.OS_LATEST])
+ self.assertEqual(
+ openstack.OS_LIBERTY,
+ openstack.MetadataReader(self.burl)._find_working_version())
+
+ def test__find_working_version_uses_latest(self):
+ """'latest' should be used if no supported versions."""
+ unsup1, unsup2 = ("2016-11-09", '2017-06-06')
+ self.register_versions([unsup1, unsup2, openstack.OS_LATEST])
+ self.assertEqual(
+ openstack.OS_LATEST,
+ openstack.MetadataReader(self.burl)._find_working_version())
+
+ def test_read_v2_os_ocata(self):
+ """Validate return value of read_v2 for os_ocata data."""
+ md = copy.deepcopy(self.md_base)
+ md['devices'] = []
+ network_data = {'links': [], 'networks': [], 'services': []}
+ vendor_data = {}
+ vendor_data2 = {"static": {}}
+
+ data = {
+ 'meta_data.json': json.dumps(md),
+ 'network_data.json': json.dumps(network_data),
+ 'vendor_data.json': json.dumps(vendor_data),
+ 'vendor_data2.json': json.dumps(vendor_data2),
+ }
+
+ self.register_versions([openstack.OS_OCATA, openstack.OS_LATEST])
+ self.register_version(openstack.OS_OCATA, data)
+
+ mock_read_ec2 = test_helpers.mock.MagicMock(
+ return_value={'instance-id': 'unused-ec2'})
+ expected_md = copy.deepcopy(md)
+ expected_md.update(
+ {'instance-id': md['uuid'], 'local-hostname': md['hostname']})
+ expected = {
+ 'userdata': '', # Annoying, no user-data results in empty string.
+ 'version': 2,
+ 'metadata': expected_md,
+ 'vendordata': vendor_data,
+ 'networkdata': network_data,
+ 'ec2-metadata': mock_read_ec2.return_value,
+ 'files': {},
+ }
+ reader = openstack.MetadataReader(self.burl)
+ reader._read_ec2_metadata = mock_read_ec2
+ self.assertEqual(expected, reader.read_v2())
+ self.assertEqual(1, mock_read_ec2.call_count)
+
+
# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_etc_hosts.py b/tests/unittests/test_handler/test_handler_etc_hosts.py
index ced05a8..d854afc 100644
--- a/tests/unittests/test_handler/test_handler_etc_hosts.py
+++ b/tests/unittests/test_handler/test_handler_etc_hosts.py
@@ -49,6 +49,7 @@ class TestHostsFile(t_help.FilesystemMockingTestCase):
if '192.168.1.1\tblah.blah.us\tblah' not in contents:
self.assertIsNone('Default etc/hosts content modified')
+ @t_help.skipUnlessJinja()
def test_write_etc_hosts_suse_template(self):
cfg = {
'manage_etc_hosts': 'template',
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 6fe3659..0f22e57 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -3,6 +3,7 @@
from cloudinit.config import cc_ntp
from cloudinit.sources import DataSourceNone
from cloudinit import (distros, helpers, cloud, util)
+
from cloudinit.tests.helpers import (
CiTestCase, FilesystemMockingTestCase, mock, skipUnlessJsonSchema)
diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py
index 20c87ef..c36e6eb 100644
--- a/tests/unittests/test_templating.py
+++ b/tests/unittests/test_templating.py
@@ -21,6 +21,9 @@ except ImportError:
class TestTemplates(test_helpers.CiTestCase):
+
+ with_logs = True
+
jinja_utf8 = b'It\xe2\x80\x99s not ascii, {{name}}\n'
jinja_utf8_rbob = b'It\xe2\x80\x99s not ascii, bob\n'.decode('utf-8')
@@ -124,6 +127,13 @@ $a,$b'''
self.add_header("jinja", self.jinja_utf8), {"name": "bob"}),
self.jinja_utf8_rbob)
+ def test_jinja_nonascii_render_undefined_variables_to_default_py3(self):
+ """Test py3 jinja render_to_string with undefined variable default."""
+ self.assertEqual(
+ templater.render_string(
+ self.add_header("jinja", self.jinja_utf8), {}),
+ self.jinja_utf8_rbob.replace('bob', 'CI_MISSING_JINJA_VAR/name'))
+
def test_jinja_nonascii_render_to_file(self):
"""Test jinja render_to_file of a filename with non-ascii content."""
tmpl_fn = self.tmp_path("j-render-to-file.template")
@@ -144,5 +154,18 @@ $a,$b'''
result = templater.render_from_file(tmpl_fn, {"name": "bob"})
self.assertEqual(result, self.jinja_utf8_rbob)
+ @test_helpers.skipIfJinja()
+ def test_jinja_warns_on_missing_dep_and_uses_basic_renderer(self):
+ """Test jinja render_from_file will fallback to basic renderer."""
+ tmpl_fn = self.tmp_path("j-render-from-file.template")
+ write_file(tmpl_fn, omode="wb",
+ content=self.add_header(
+ "jinja", self.jinja_utf8).encode('utf-8'))
+ result = templater.render_from_file(tmpl_fn, {"name": "bob"})
+ self.assertEqual(result, self.jinja_utf8.decode())
+ self.assertIn(
+ 'WARNING: Jinja not available as the selected renderer for desired'
+ ' template, reverting to the basic renderer.',
+ self.logs.getvalue())
# vi: ts=4 expandtab
Follow ups
