cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #04362
Re: [Merge] ~chad.smith/cloud-init:feature/snap-module into cloud-init:master
Diff comments:
> diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py
> new file mode 100644
> index 0000000..3210d6d
> --- /dev/null
> +++ b/cloudinit/config/cc_snap.py
> @@ -0,0 +1,272 @@
> +# Copyright (C) 2018 Canonical Ltd.
> +#
> +# This file is part of cloud-init. See LICENSE file for license information.
> +
> +"""Snap: Install, configure and manage snapd and snap packages."""
> +
> +import sys
> +from textwrap import dedent
> +
> +from cloudinit import log as logging
> +from cloudinit.config.schema import (
> + get_schema_doc, validate_cloudconfig_schema)
> +from cloudinit.settings import PER_INSTANCE
> +from cloudinit import util
> +
> +
> +distros = ['ubuntu']
> +frequency = PER_INSTANCE
> +
> +LOG = logging.getLogger(__name__)
> +
> +schema = {
> + 'id': 'cc_snap',
> + 'name': 'Snap',
> + 'title': 'Install, configure and manage snapd and snap packages',
> + 'description': dedent("""\
> + This module provides a simple configuration namespace in cloud-init to
> + both setup snapd and install snaps.
> +
> + .. note::
> + Both ``assertions`` and ``commands`` values can be either a
> + dictionary or a list. If these configs are provided as a
> + dictionary, the keys are only used to order the execution of the
> + assertions or commands and the dictionary is merged with any
> + vendor-data snap configuration provided. If a list is provided by
> + the user instead of a dict, any vendor-data snap configuration is
> + ignored.
> +
> + The ``assertions`` configuration option is a dictionary or list of
> + properly-signed snap assertions which will run before any snap
> + ``commands``. They will be added to snapd's assertion database by
> + invoking ``snap ack <aggregate_assertion_file>``.
> +
> + Snap ``commands`` is a dictionary or list of individual snap
> + commands to run on the target system. These commands can be used to
> + create snap users, install snaps and provide snap configuration.
> +
> + .. note::
> + If 'side-loading' private/unpublished snaps on an instance, it is
> + best to create a snap seed directory and seed.yaml manifest in
> + **/var/lib/snapd/seed/** which snapd automatically installs on
> + startup.
> +
> + **Development only**: The ``squashfuse_in_container`` boolean can be
> + set true to install squashfuse package when in a container to enable
> + snap installs. Default is false.
> + """),
> + 'distros': distros,
> + 'examples': [dedent("""\
> + snap:
> + assertions:
> + 00: |
> + signed_assertion_blob_here
> + 02: |
> + signed_assertion_blob_here
> + commands:
> + 00: snap create-user --sudoer --known <snap-user>@mydomain.com
> + 01: snap install canonical-livepatch
> + 02: canonical-livepatch enable <AUTH_TOKEN>
> + """), dedent("""\
> + # LXC-based containers require squashfuse before snaps can be installed
> + snap:
> + commands:
> + 00: apt-get install squashfuse -y
> + 11: snap install emoj
> +
> + """), dedent("""\
> + # Convenience: the snap command can be ommited when specifying commands
fixed
> + # as a list and 'snap' will automatically be prepended.
> + # The following commands are equivalent:
> + snap:
> + commands:
> + 00: ['install', 'vlc']
> + 01: ['snap', 'install', 'vlc']
> + 02: snap install vlc
> + 03: 'snap install vlc'
> + """)],
> + 'frequency': PER_INSTANCE,
> + 'type': 'object',
> + 'properties': {
> + 'snap': {
> + 'type': 'object',
> + 'properties': {
> + 'assertions': {
> + 'type': ['object', 'array'], # Array of strings or dict
> + 'items': {'type': 'string'},
> + 'additionalItems': False, # Reject items non-string
> + 'minItems': 1,
> + 'minProperties': 1,
> + 'uniqueItems': True
> + },
> + 'commands': {
> + 'type': ['object', 'array'], # Array of strings or dict
> + 'items': {
> + 'oneOf': [
> + {'type': 'array', 'items': {'type': 'string'}},
> + {'type': 'string'}]
> + },
> + 'additionalItems': False, # Reject non-string & non-list
> + 'minItems': 1,
> + 'minProperties': 1,
> + 'uniqueItems': True
> + },
> + 'squashfuse_in_container': {
> + 'type': 'boolean'
> + }
> + },
> + 'additionalProperties': False, # Reject keys not in schema
> + 'required': [],
> + 'minProperties': 1
> + }
> + }
> +}
> +
> +# TODO schema for 'assertions' and 'commands' are too permissive at the moment.
> +# Once python-jsonschema supports schema draft 6 add support for arbitrary
> +# object keys with 'patternProperties' constraint to validate string values.
> +
> +__doc__ = get_schema_doc(schema) # Supplement python help()
> +
> +SNAP_CMD = "snap"
> +ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions"
> +
> +
> +def add_assertions(assertions):
> + """Import list of assertions.
> +
> + Import assertions by concatenating each assertion into a
> + string separated by a '\n'. Write this string to a instance file and
> + then invoke `snap ack /path/to/file` and check for errors.
> + If snap exits 0, then all assertions are imported.
> + """
> + if not assertions:
> + return
> + LOG.debug('Importing user-provided snap assertions')
> + if isinstance(assertions, dict):
> + assertions = assertions.values()
> + elif not isinstance(assertions, list):
> + raise TypeError(
> + 'assertion parameter was not a list or dict: {assertions}'.format(
> + assertions=assertions))
> +
> + snap_cmd = [SNAP_CMD, 'ack']
> + combined = "\n".join(assertions)
> +
> + for asrt in assertions:
> + LOG.debug('Snap acking: %s', asrt.split('\n')[0:2])
> +
> + util.write_file(ASSERTIONS_FILE, combined.encode('utf-8'))
> + util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True)
> +
> +
> +def prepend_snap_commands(commands):
> + """Ensure user-provided commands start with SNAP_CMD, warn otherwise.
> +
> + Each command is either a list or string. Perform the following:
> + - When the command is a list, pop the first element if it is None
> + - When the command is a list, insert SNAP_CMD as the first element if
> + not present.
> + - When the command is a string containing a non-snap command, warn.
> +
> + Support cut-n-paste snap command sets from public snappy documentation.
> + Allow flexibility to provide non-snap environment/config setup if neeeded.
fixed
> +
> + @commands: List of commands. Each command element is a list or string.
> +
> + @return: List of 'fixed up' snap commands.
> + @raise: TypeError on invalid config item type.
> + """
> + warnings = []
> + errors = []
> + fixed_commands = []
> + for command in commands:
> + if isinstance(command, list):
> + if command[0] is None: # Avoid warnings by specifying None
> + command = command[1:]
> + elif command[0] != SNAP_CMD: # Automatically prepend SNAP_CMD
> + command.insert(0, SNAP_CMD)
> + elif isinstance(command, str):
> + if not command.startswith('%s ' % SNAP_CMD):
> + warnings.append(command)
> + else:
> + errors.append(str(command))
> + continue
> + fixed_commands.append(command)
> +
> + if warnings:
> + LOG.warning(
> + 'Non-snap commands in snap config:\n%s', '\n'.join(warnings))
> + if errors:
> + raise TypeError(
> + 'Invalid snap config.'
> + ' These commands are not a string or list:\n' + '\n'.join(errors))
> + return fixed_commands
> +
> +
> +def run_commands(commands):
> + """Run the provided commands provided in snap:commands configuration.
> +
> + Commands are run individually. Any errors are collected and reported
> + after attempting all commands.
> +
> + @param commands: A list or dict containing commands to run. Keys of a
> + dict will be used to order the commands provided as dict values.
> + """
> + if not commands:
> + return
> + LOG.debug('Running user-provided snap commands')
> + if isinstance(commands, dict):
> + # Sort commands based on dictionary key
> + commands = [v for _, v in sorted(commands.items())]
> + elif not isinstance(commands, list):
> + raise TypeError(
> + 'commands parameter was not a list or dict: {commands}'.format(
> + commands=commands))
> +
> + fixed_snap_commands = prepend_snap_commands(commands)
> +
> + cmd_failures = []
> + for command in fixed_snap_commands:
> + shell = isinstance(command, str)
> + try:
> + util.subp(command, shell=shell, status_cb=sys.stderr.write)
> + except util.ProcessExecutionError as e:
> + cmd_failures.append(str(e))
> + if cmd_failures:
> + msg = 'Failures running snap commands:\n{cmd_failures}'.format(
> + cmd_failures=cmd_failures)
> + util.logexc(LOG, msg)
> + raise RuntimeError(msg)
> +
> +
> +def maybe_install_squashfuse(cloud):
> + """Install squashfuse if we are in a container."""
> + if not util.is_container():
> + return
> + try:
> + cloud.distro.update_package_sources()
> + except Exception as e:
> + util.logexc(LOG, "Package update failed")
> + raise
> + try:
> + cloud.distro.install_packages(['squashfuse'])
> + except Exception as e:
> + util.logexc(LOG, "Failed to install squashfuse")
> + raise
> +
> +
> +def handle(name, cfg, cloud, log, args):
> + cfgin = cfg.get('snap', {})
> + if not cfgin:
> + LOG.debug(("Skipping module named %s,"
> + " no 'snap' key in configuration"), name)
> + return
> +
> + validate_cloudconfig_schema(cfg, schema)
> + if util.is_true(cfgin.get('squashfuse_in_container', False)):
> + maybe_install_squashfuse(cloud)
> + add_assertions(cfgin.get('assertions', []))
> + run_commands(cfgin.get('commands', []))
> +
> +# vi: ts=4 expandtab
> diff --git a/tests/cloud_tests/testcases/modules/snap.py b/tests/cloud_tests/testcases/modules/snap.py
> new file mode 100644
> index 0000000..c9c56ad
> --- /dev/null
> +++ b/tests/cloud_tests/testcases/modules/snap.py
> @@ -0,0 +1,135 @@
> +# This file is part of cloud-init. See LICENSE file for license information.
> +
> +"""cloud-init Integration Test Verify Script"""
> +import json
> +import re
> +
> +from tests.cloud_tests.testcases import base
> +
> +
> +class TestSnap(base.CloudTestCase):
> + """Test snap module"""
> +
> + def test_snappy_version(self):
> + """Expect hello-world and core snaps are installed."""
> + out = self.get_data_file('snaplist')
> + self.assertIn('core', out)
> + self.assertIn('hello-world', out)
> +
> + def test_instance_data_json_ec2(self):
> + """Validate instance-data.json content by ec2 platform.
> +
> + This content is sourced by snapd when determining snapstore endpoints.
> + We validate expected values per cloud type to ensure we don't break
> + snapd.
> + """
> + if self.platform != 'ec2':
> + raise base.SkipTest(
> + 'Skipping ec2 instance-data.json on %s' % self.platform)
> + out = self.get_data_file('instance-data.json')
> + if not out:
> + if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0:
> + raise AssertionError(
> + 'No instance-data.json found on %s' % self.os_name)
> + raise base.SkipTest(
> + 'Skipping instance-data.json test.'
> + ' OS: %s not bionic or newer' % self.os_name)
> + instance_data = json.loads(out)
> + self.assertEqual(
> + ['ds/user-data'], instance_data['base64-encoded-keys'])
> + ds = instance_data.get('ds', {})
> + macs = ds.get('network', {}).get('interfaces', {}).get('macs', {})
> + if not macs:
> + raise AssertionError('No network data from EC2 meta-data')
> + # Check meta-data items we depend on
> + expected_net_keys = [
> + 'public-ipv4s', 'ipv4-associations', 'local-hostname',
> + 'public-hostname']
> + for mac, mac_data in macs.items():
> + for key in expected_net_keys:
> + self.assertIn(key, mac_data)
> + self.assertIsNotNone(
> + ds.get('placement', {}).get('availability-zone'),
> + 'Could not determine EC2 Availability zone placement')
> + ds = instance_data.get('ds', {})
> + v1_data = instance_data.get('v1', {})
> + self.assertIsNotNone(
> + v1_data['availability-zone'], 'expected ec2 availability-zone')
> + self.assertEqual('aws', v1_data['cloud-name'])
> + self.assertIn('i-', v1_data['instance-id'])
> + self.assertIn('ip-', v1_data['local-hostname'])
> + self.assertIsNotNone(v1_data['region'], 'expected ec2 region')
> +
> + def test_instance_data_json_lxd(self):
> + """Validate instance-data.json content by lxd platform.
> +
> + This content is sourced by snapd when determining snapstore endpoints.
> + We validate expected values per cloud type to ensure we don't break
> + snapd.
> + """
> + if self.platform != 'lxd':
> + raise base.SkipTest(
> + 'Skipping lxd instance-data.json on %s' % self.platform)
> + out = self.get_data_file('instance-data.json')
> + if not out:
> + if self.os_version_cmp('bionic') >= 0:
oops +1 fixed.
> + raise AssertionError(
> + 'No instance-data.json found on %s' % self.os_name)
> + raise base.SkipTest(
> + 'Skipping instance-data.json test.'
> + ' OS: %s not bionic or newer' % self.os_name)
> + instance_data = json.loads(out)
> + v1_data = instance_data.get('v1', {})
> + self.assertEqual(
> + ['ds/user-data', 'ds/vendor-data'],
> + instance_data['base64-encoded-keys'])
> + self.assertEqual('nocloud', v1_data['cloud-name'])
> + self.assertIsNone(
> + v1_data['availability-zone'],
> + 'found unexpected lxd availability-zone %s' %
> + v1_data['availability-zone'])
> + self.assertIn('cloud-test', v1_data['instance-id'])
> + self.assertIn('cloud-test', v1_data['local-hostname'])
> + self.assertIsNone(
> + v1_data['region'],
> + 'found unexpected lxd region %s' % v1_data['region'])
> +
> + def test_instance_data_json_kvm(self):
> + """Validate instance-data.json content by nocloud-kvm platform.
> +
> + This content is sourced by snapd when determining snapstore endpoints.
> + We validate expected values per cloud type to ensure we don't break
> + snapd.
> + """
> + if self.platform != 'nocloud-kvm':
> + raise base.SkipTest(
> + 'Skipping nocloud-kvm instance-data.json on %s' %
> + self.platform)
> + out = self.get_data_file('instance-data.json')
> + if not out:
> + if self.os_version_cmp('bionic') >= 0:
+1 fixed here too thanks for catching this.
> + raise AssertionError(
> + 'No instance-data.json found on %s' % self.os_name)
> + raise base.SkipTest(
> + 'Skipping instance-data.json test.'
> + ' OS: %s not bionic or newer' % self.os_name)
> + instance_data = json.loads(out)
> + v1_data = instance_data.get('v1', {})
> + self.assertEqual(
> + ['ds/user-data'], instance_data['base64-encoded-keys'])
> + self.assertEqual('nocloud', v1_data['cloud-name'])
> + self.assertIsNone(
> + v1_data['availability-zone'],
> + 'found unexpected kvm availability-zone %s' %
> + v1_data['availability-zone'])
> + self.assertIsNotNone(
> + re.match('[\da-f]{8}(-[\da-f]{4}){3}-[\da-f]{12}',
> + v1_data['instance-id']),
> + 'kvm instance-id is not a UUID: %s' % v1_data['instance-id'])
> + self.assertIn('ubuntu', v1_data['local-hostname'])
> + self.assertIsNone(
> + v1_data['region'],
> + 'found unexpected lxd region %s' % v1_data['region'])
> +
> +
> +# vi: ts=4 expandtab
--
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/338366
Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:feature/snap-module into cloud-init:master.
