cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #03630
[Merge] ~raharper/cloud-init:ubuntu-devel-new-bionic-release-v1 into cloud-init:ubuntu/devel
Ryan Harper has proposed merging ~raharper/cloud-init:ubuntu-devel-new-bionic-release-v1 into cloud-init:ubuntu/devel.
Requested reviews:
cloud-init commiters (cloud-init-dev)
Related bugs:
Bug #1724354 in cloud-init: "WARNING in logs due to missing python-jsonschema"
https://bugs.launchpad.net/cloud-init/+bug/1724354
Bug #1724951 in cloud-init: "Ntp schema definition permits empty ntp cloud-config, but code disallows"
https://bugs.launchpad.net/cloud-init/+bug/1724951
Bug #1725067 in cloud-init: "cloud-init resizefs fails when booting with root=PARTUUID="
https://bugs.launchpad.net/cloud-init/+bug/1725067
For more details, see:
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/332722
New Upload for Bionic Release
--
Your team cloud-init commiters is requested to review the proposed merge of ~raharper/cloud-init:ubuntu-devel-new-bionic-release-v1 into cloud-init:ubuntu/devel.
diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py
index e6262f8..09374d2 100644
--- a/cloudinit/config/cc_lxd.py
+++ b/cloudinit/config/cc_lxd.py
@@ -72,7 +72,7 @@ def handle(name, cfg, cloud, log, args):
type(init_cfg))
init_cfg = {}
- bridge_cfg = lxd_cfg.get('bridge')
+ bridge_cfg = lxd_cfg.get('bridge', {})
if not isinstance(bridge_cfg, dict):
log.warn("lxd/bridge config must be a dictionary. found a '%s'",
type(bridge_cfg))
diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py
index 15ae1ec..d43d060 100644
--- a/cloudinit/config/cc_ntp.py
+++ b/cloudinit/config/cc_ntp.py
@@ -100,7 +100,9 @@ def handle(name, cfg, cloud, log, _args):
LOG.debug(
"Skipping module named %s, not present or disabled by cfg", name)
return
- ntp_cfg = cfg.get('ntp', {})
+ ntp_cfg = cfg['ntp']
+ if ntp_cfg is None:
+ ntp_cfg = {} # Allow empty config which will install the package
# TODO drop this when validate_cloudconfig_schema is strict=True
if not isinstance(ntp_cfg, (dict)):
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index f774baa..0d282e6 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -145,25 +145,6 @@ RESIZE_FS_PRECHECK_CMDS = {
}
-def rootdev_from_cmdline(cmdline):
- found = None
- for tok in cmdline.split():
- if tok.startswith("root="):
- found = tok[5:]
- break
- if found is None:
- return None
-
- if found.startswith("/dev/"):
- return found
- if found.startswith("LABEL="):
- return "/dev/disk/by-label/" + found[len("LABEL="):]
- if found.startswith("UUID="):
- return "/dev/disk/by-uuid/" + found[len("UUID="):]
-
- return "/dev/" + found
-
-
def can_skip_resize(fs_type, resize_what, devpth):
fstype_lc = fs_type.lower()
for i, func in RESIZE_FS_PRECHECK_CMDS.items():
@@ -172,14 +153,15 @@ def can_skip_resize(fs_type, resize_what, devpth):
return False
-def is_device_path_writable_block(devpath, info, log):
- """Return True if devpath is a writable block device.
+def maybe_get_writable_device_path(devpath, info, log):
+ """Return updated devpath if the devpath is a writable block device.
- @param devpath: Path to the root device we want to resize.
+ @param devpath: Requested path to the root device we want to resize.
@param info: String representing information about the requested device.
@param log: Logger to which logs will be added upon error.
- @returns Boolean True if block device is writable
+ @returns devpath or updated devpath per kernel commandline if the device
+ path is a writable block device, returns None otherwise.
"""
container = util.is_container()
@@ -189,12 +171,12 @@ def is_device_path_writable_block(devpath, info, log):
devpath = util.rootdev_from_cmdline(util.get_cmdline())
if devpath is None:
log.warn("Unable to find device '/dev/root'")
- return False
+ return None
log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath)
if devpath == 'overlayroot':
log.debug("Not attempting to resize devpath '%s': %s", devpath, info)
- return False
+ return None
try:
statret = os.stat(devpath)
@@ -207,7 +189,7 @@ def is_device_path_writable_block(devpath, info, log):
devpath, info)
else:
raise exc
- return False
+ return None
if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
if container:
@@ -216,8 +198,8 @@ def is_device_path_writable_block(devpath, info, log):
else:
log.warn("device '%s' not a block device. cannot resize: %s" %
(devpath, info))
- return False
- return True
+ return None
+ return devpath # The writable block devpath
def handle(name, cfg, _cloud, log, args):
@@ -242,8 +224,9 @@ def handle(name, cfg, _cloud, log, args):
info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)
log.debug("resize_info: %s" % info)
- if not is_device_path_writable_block(devpth, info, log):
- return
+ devpth = maybe_get_writable_device_path(devpth, info, log)
+ if not devpth:
+ return # devpath was not a writable block device
resizer = None
if can_skip_resize(fs_type, resize_what, devpth):
diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py
index b80d1d3..f363000 100644
--- a/cloudinit/config/cc_users_groups.py
+++ b/cloudinit/config/cc_users_groups.py
@@ -15,7 +15,8 @@ options, see the ``Including users and groups`` config example.
Groups to add to the system can be specified as a list under the ``groups``
key. Each entry in the list should either contain a the group name as a string,
or a dictionary with the group name as the key and a list of users who should
-be members of the group as the value.
+be members of the group as the value. **Note**: Groups are added before users,
+so any users in a group list must already exist on the system.
The ``users`` config key takes a list of users to configure. The first entry in
this list is used as the default user for the system. To preserve the standard
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index bb291ff..ca7d0d5 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -74,7 +74,7 @@ def validate_cloudconfig_schema(config, schema, strict=False):
try:
from jsonschema import Draft4Validator, FormatChecker
except ImportError:
- logging.warning(
+ logging.debug(
'Ignoring schema validation. python-jsonschema is not present')
return
validator = Draft4Validator(schema, format_checker=FormatChecker())
diff --git a/debian/changelog b/debian/changelog
index 26d1d45..bede4fe 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,19 @@
+cloud-init (17.1-25-g17a15f9e-0ubuntu1) bionic; urgency=medium
+
+ * New upstream snapshot.
+ - resizefs: Fix regression when system booted with root=PARTUUID=
+ [Chad Smith] (LP: #1725067)
+ - tools: make yum package installation more reliable
+ - citest: fix remaining warnings raised by integration tests.
+ - citest: show the class actual class name in results.
+ - ntp: fix config module schema to allow empty ntp config
+ [Chad Smith] (LP: #1724951)
+ - tools: disable fastestmirror if using proxy [Joshua Powers]
+ - schema: Log debug instead of warning when jsonschema is not available.
+ (LP: #1724354)
+
+ -- Ryan Harper <ryan.harper@xxxxxxxxxxxxx> Tue, 24 Oct 2017 10:40:00 -0500
+
cloud-init (17.1-18-gd4f70470-0ubuntu1) artful; urgency=medium
* New upstream snapshot.
diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt
index 9c5202f..0554d1f 100644
--- a/doc/examples/cloud-config-user-groups.txt
+++ b/doc/examples/cloud-config-user-groups.txt
@@ -1,8 +1,8 @@
# Add groups to the system
-# The following example adds the ubuntu group with members foo and bar and
-# the group cloud-users.
+# The following example adds the ubuntu group with members 'root' and 'sys'
+# and the empty group cloud-users.
groups:
- - ubuntu: [foo,bar]
+ - ubuntu: [root,sys]
- cloud-users
# Add users to the system. Users are added after groups are added.
diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py
index 47217ce..a29a092 100644
--- a/tests/cloud_tests/testcases/__init__.py
+++ b/tests/cloud_tests/testcases/__init__.py
@@ -5,6 +5,7 @@
import importlib
import inspect
import unittest
+from unittest.util import strclass
from tests.cloud_tests import config
from tests.cloud_tests.testcases.base import CloudTestCase as base_test
@@ -37,6 +38,12 @@ def get_suite(test_name, data, conf):
class tmp(test_class):
+ _realclass = test_class
+
+ def __str__(self):
+ return "%s (%s)" % (self._testMethodName,
+ strclass(self._realclass))
+
@classmethod
def setUpClass(cls):
cls.data = data
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index bb545ab..1706f59 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -16,10 +16,6 @@ class CloudTestCase(unittest.TestCase):
conf = None
_cloud_config = None
- def shortDescription(self):
- """Prevent nose from using docstrings."""
- return None
-
@property
def cloud_config(self):
"""Get the cloud-config used by the test."""
@@ -72,6 +68,14 @@ class CloudTestCase(unittest.TestCase):
result = self.get_status_data(self.get_data_file('result.json'))
self.assertEqual(len(result['errors']), 0)
+ def test_no_warnings_in_log(self):
+ """Warnings should not be found in the log."""
+ self.assertEqual(
+ [],
+ [l for l in self.get_data_file('cloud-init.log').splitlines()
+ if 'WARN' in l],
+ msg="'WARN' found inside cloud-init.log")
+
class PasswordListTest(CloudTestCase):
"""Base password test case class."""
diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.py b/tests/cloud_tests/testcases/examples/including_user_groups.py
index 67af527..93b7a82 100644
--- a/tests/cloud_tests/testcases/examples/including_user_groups.py
+++ b/tests/cloud_tests/testcases/examples/including_user_groups.py
@@ -40,4 +40,10 @@ class TestUserGroups(base.CloudTestCase):
out = self.get_data_file('user_cloudy')
self.assertRegex(out, r'cloudy:x:[0-9]{3,4}:')
+ def test_user_root_in_secret(self):
+ """Test root user is in 'secret' group."""
+ user, _, groups = self.get_data_file('root_groups').partition(":")
+ self.assertIn("secret", groups.split(),
+ msg="User root is not in group 'secret'")
+
# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.yaml b/tests/cloud_tests/testcases/examples/including_user_groups.yaml
index 0aa7ad2..469d03c 100644
--- a/tests/cloud_tests/testcases/examples/including_user_groups.yaml
+++ b/tests/cloud_tests/testcases/examples/including_user_groups.yaml
@@ -8,7 +8,7 @@ cloud_config: |
#cloud-config
# Add groups to the system
groups:
- - secret: [foobar,barfoo]
+ - secret: [root]
- cloud-users
# Add users to the system. Users are added after groups are added.
@@ -24,7 +24,7 @@ cloud_config: |
- name: barfoo
gecos: Bar B. Foo
sudo: ALL=(ALL) NOPASSWD:ALL
- groups: cloud-users
+ groups: [cloud-users, secret]
lock_passwd: true
- name: cloudy
gecos: Magic Cloud App Daemon User
@@ -49,5 +49,8 @@ collect_scripts:
user_cloudy: |
#!/bin/bash
getent passwd cloudy
+ root_groups: |
+ #!/bin/bash
+ groups root
# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/main/command_output_simple.py b/tests/cloud_tests/testcases/main/command_output_simple.py
index fe4c767..857881c 100644
--- a/tests/cloud_tests/testcases/main/command_output_simple.py
+++ b/tests/cloud_tests/testcases/main/command_output_simple.py
@@ -15,4 +15,20 @@ class TestCommandOutputSimple(base.CloudTestCase):
data.splitlines()[-1].strip())
# TODO: need to test that all stages redirected here
+ def test_no_warnings_in_log(self):
+ """Warnings should not be found in the log.
+
+ This class redirected stderr and stdout, so it expects to find
+ a warning in cloud-init.log to that effect."""
+ redirect_msg = 'Stdout, stderr changing to'
+ warnings = [
+ l for l in self.get_data_file('cloud-init.log').splitlines()
+ if 'WARN' in l]
+ self.assertEqual(
+ [], [w for w in warnings if redirect_msg not in w],
+ msg="'WARN' found inside cloud-init.log")
+ self.assertEqual(
+ 1, len(warnings),
+ msg="Did not find %s in cloud-init.log" % redirect_msg)
+
# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/ntp.yaml b/tests/cloud_tests/testcases/modules/ntp.yaml
index fbef431..2530d72 100644
--- a/tests/cloud_tests/testcases/modules/ntp.yaml
+++ b/tests/cloud_tests/testcases/modules/ntp.yaml
@@ -4,8 +4,8 @@
cloud_config: |
#cloud-config
ntp:
- pools: {}
- servers: {}
+ pools: []
+ servers: []
collect_scripts:
ntp_installed: |
#!/bin/bash
diff --git a/tests/cloud_tests/testcases/modules/user_groups.py b/tests/cloud_tests/testcases/modules/user_groups.py
index 67af527..93b7a82 100644
--- a/tests/cloud_tests/testcases/modules/user_groups.py
+++ b/tests/cloud_tests/testcases/modules/user_groups.py
@@ -40,4 +40,10 @@ class TestUserGroups(base.CloudTestCase):
out = self.get_data_file('user_cloudy')
self.assertRegex(out, r'cloudy:x:[0-9]{3,4}:')
+ def test_user_root_in_secret(self):
+ """Test root user is in 'secret' group."""
+ user, _, groups = self.get_data_file('root_groups').partition(":")
+ self.assertIn("secret", groups.split(),
+ msg="User root is not in group 'secret'")
+
# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/user_groups.yaml b/tests/cloud_tests/testcases/modules/user_groups.yaml
index 71cc9da..22b5d70 100644
--- a/tests/cloud_tests/testcases/modules/user_groups.yaml
+++ b/tests/cloud_tests/testcases/modules/user_groups.yaml
@@ -7,7 +7,7 @@ cloud_config: |
#cloud-config
# Add groups to the system
groups:
- - secret: [foobar,barfoo]
+ - secret: [root]
- cloud-users
# Add users to the system. Users are added after groups are added.
@@ -23,7 +23,7 @@ cloud_config: |
- name: barfoo
gecos: Bar B. Foo
sudo: ALL=(ALL) NOPASSWD:ALL
- groups: cloud-users
+ groups: [cloud-users, secret]
lock_passwd: true
- name: cloudy
gecos: Magic Cloud App Daemon User
@@ -48,5 +48,8 @@ collect_scripts:
user_cloudy: |
#!/bin/bash
getent passwd cloudy
+ root_groups: |
+ #!/bin/bash
+ groups root
# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py
index f132a77..e0d9ab6 100644
--- a/tests/unittests/test_handler/test_handler_lxd.py
+++ b/tests/unittests/test_handler/test_handler_lxd.py
@@ -5,17 +5,16 @@ from cloudinit.sources import DataSourceNoCloud
from cloudinit import (distros, helpers, cloud)
from cloudinit.tests import helpers as t_help
-import logging
-
try:
from unittest import mock
except ImportError:
import mock
-LOG = logging.getLogger(__name__)
+class TestLxd(t_help.CiTestCase):
+
+ with_logs = True
-class TestLxd(t_help.TestCase):
lxd_cfg = {
'lxd': {
'init': {
@@ -41,7 +40,7 @@ class TestLxd(t_help.TestCase):
def test_lxd_init(self, mock_util):
cc = self._get_cloud('ubuntu')
mock_util.which.return_value = True
- cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, [])
+ cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, [])
self.assertTrue(mock_util.which.called)
init_call = mock_util.subp.call_args_list[0][0][0]
self.assertEqual(init_call,
@@ -55,7 +54,8 @@ class TestLxd(t_help.TestCase):
cc = self._get_cloud('ubuntu')
cc.distro = mock.MagicMock()
mock_util.which.return_value = None
- cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, [])
+ cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, [])
+ self.assertNotIn('WARN', self.logs.getvalue())
self.assertTrue(cc.distro.install_packages.called)
install_pkg = cc.distro.install_packages.call_args_list[0][0][0]
self.assertEqual(sorted(install_pkg), ['lxd', 'zfs'])
@@ -64,7 +64,7 @@ class TestLxd(t_help.TestCase):
def test_no_init_does_nothing(self, mock_util):
cc = self._get_cloud('ubuntu')
cc.distro = mock.MagicMock()
- cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, LOG, [])
+ cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, self.logger, [])
self.assertFalse(cc.distro.install_packages.called)
self.assertFalse(mock_util.subp.called)
@@ -72,7 +72,7 @@ class TestLxd(t_help.TestCase):
def test_no_lxd_does_nothing(self, mock_util):
cc = self._get_cloud('ubuntu')
cc.distro = mock.MagicMock()
- cc_lxd.handle('cc_lxd', {'package_update': True}, cc, LOG, [])
+ cc_lxd.handle('cc_lxd', {'package_update': True}, cc, self.logger, [])
self.assertFalse(cc.distro.install_packages.called)
self.assertFalse(mock_util.subp.called)
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 4f29124..3abe578 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -293,23 +293,24 @@ class TestNtp(FilesystemMockingTestCase):
def test_ntp_handler_schema_validation_allows_empty_ntp_config(self):
"""Ntp schema validation allows for an empty ntp: configuration."""
- invalid_config = {'ntp': {}}
+ valid_empty_configs = [{'ntp': {}}, {'ntp': None}]
distro = 'ubuntu'
cc = self._get_cloud(distro)
ntp_conf = os.path.join(self.new_root, 'ntp.conf')
with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream:
stream.write(NTP_TEMPLATE)
- with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf):
- cc_ntp.handle('cc_ntp', invalid_config, cc, None, [])
+ for valid_empty_config in valid_empty_configs:
+ with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf):
+ cc_ntp.handle('cc_ntp', valid_empty_config, cc, None, [])
+ with open(ntp_conf) as stream:
+ content = stream.read()
+ default_pools = [
+ "{0}.{1}.pool.ntp.org".format(x, distro)
+ for x in range(0, cc_ntp.NR_POOL_SERVERS)]
+ self.assertEqual(
+ "servers []\npools {0}\n".format(default_pools),
+ content)
self.assertNotIn('Invalid config:', self.logs.getvalue())
- with open(ntp_conf) as stream:
- content = stream.read()
- default_pools = [
- "{0}.{1}.pool.ntp.org".format(x, distro)
- for x in range(0, cc_ntp.NR_POOL_SERVERS)]
- self.assertEqual(
- "servers []\npools {0}\n".format(default_pools),
- content)
@skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
def test_ntp_handler_schema_validation_warns_non_string_item_type(self):
diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py
index 3e5d436..29d5574 100644
--- a/tests/unittests/test_handler/test_handler_resizefs.py
+++ b/tests/unittests/test_handler/test_handler_resizefs.py
@@ -1,9 +1,9 @@
# This file is part of cloud-init. See LICENSE file for license information.
from cloudinit.config.cc_resizefs import (
- can_skip_resize, handle, is_device_path_writable_block,
- rootdev_from_cmdline)
+ can_skip_resize, handle, maybe_get_writable_device_path)
+from collections import namedtuple
import logging
import textwrap
@@ -138,47 +138,48 @@ class TestRootDevFromCmdline(CiTestCase):
invalid_cases = [
'BOOT_IMAGE=/adsf asdfa werasef root adf', 'BOOT_IMAGE=/adsf', '']
for case in invalid_cases:
- self.assertIsNone(rootdev_from_cmdline(case))
+ self.assertIsNone(util.rootdev_from_cmdline(case))
def test_rootdev_from_cmdline_with_root_startswith_dev(self):
"""Return the cmdline root when the path starts with /dev."""
self.assertEqual(
- '/dev/this', rootdev_from_cmdline('asdf root=/dev/this'))
+ '/dev/this', util.rootdev_from_cmdline('asdf root=/dev/this'))
def test_rootdev_from_cmdline_with_root_without_dev_prefix(self):
"""Add /dev prefix to cmdline root when the path lacks the prefix."""
- self.assertEqual('/dev/this', rootdev_from_cmdline('asdf root=this'))
+ self.assertEqual(
+ '/dev/this', util.rootdev_from_cmdline('asdf root=this'))
def test_rootdev_from_cmdline_with_root_with_label(self):
"""When cmdline root contains a LABEL, our root is disk/by-label."""
self.assertEqual(
'/dev/disk/by-label/unique',
- rootdev_from_cmdline('asdf root=LABEL=unique'))
+ util.rootdev_from_cmdline('asdf root=LABEL=unique'))
def test_rootdev_from_cmdline_with_root_with_uuid(self):
"""When cmdline root contains a UUID, our root is disk/by-uuid."""
self.assertEqual(
'/dev/disk/by-uuid/adsfdsaf-adsf',
- rootdev_from_cmdline('asdf root=UUID=adsfdsaf-adsf'))
+ util.rootdev_from_cmdline('asdf root=UUID=adsfdsaf-adsf'))
-class TestIsDevicePathWritableBlock(CiTestCase):
+class TestMaybeGetDevicePathAsWritableBlock(CiTestCase):
with_logs = True
- def test_is_device_path_writable_block_false_on_overlayroot(self):
+ def test_maybe_get_writable_device_path_none_on_overlayroot(self):
"""When devpath is overlayroot (on MAAS), is_dev_writable is False."""
info = 'does not matter'
- is_writable = wrap_and_call(
+ devpath = wrap_and_call(
'cloudinit.config.cc_resizefs.util',
{'is_container': {'return_value': False}},
- is_device_path_writable_block, 'overlayroot', info, LOG)
- self.assertFalse(is_writable)
+ maybe_get_writable_device_path, 'overlayroot', info, LOG)
+ self.assertIsNone(devpath)
self.assertIn(
"Not attempting to resize devpath 'overlayroot'",
self.logs.getvalue())
- def test_is_device_path_writable_block_warns_missing_cmdline_root(self):
+ def test_maybe_get_writable_device_path_warns_missing_cmdline_root(self):
"""When root does not exist isn't in the cmdline, log warning."""
info = 'does not matter'
@@ -190,43 +191,43 @@ class TestIsDevicePathWritableBlock(CiTestCase):
exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists'
with mock.patch(exists_mock_path) as m_exists:
m_exists.return_value = False
- is_writable = wrap_and_call(
+ devpath = wrap_and_call(
'cloudinit.config.cc_resizefs.util',
{'is_container': {'return_value': False},
'get_mount_info': {'side_effect': fake_mount_info},
'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}},
- is_device_path_writable_block, '/dev/root', info, LOG)
- self.assertFalse(is_writable)
+ maybe_get_writable_device_path, '/dev/root', info, LOG)
+ self.assertIsNone(devpath)
logs = self.logs.getvalue()
self.assertIn("WARNING: Unable to find device '/dev/root'", logs)
- def test_is_device_path_writable_block_does_not_exist(self):
+ def test_maybe_get_writable_device_path_does_not_exist(self):
"""When devpath does not exist, a warning is logged."""
info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none'
- is_writable = wrap_and_call(
+ devpath = wrap_and_call(
'cloudinit.config.cc_resizefs.util',
{'is_container': {'return_value': False}},
- is_device_path_writable_block, '/I/dont/exist', info, LOG)
- self.assertFalse(is_writable)
+ maybe_get_writable_device_path, '/I/dont/exist', info, LOG)
+ self.assertIsNone(devpath)
self.assertIn(
"WARNING: Device '/I/dont/exist' did not exist."
' cannot resize: %s' % info,
self.logs.getvalue())
- def test_is_device_path_writable_block_does_not_exist_in_container(self):
+ def test_maybe_get_writable_device_path_does_not_exist_in_container(self):
"""When devpath does not exist in a container, log a debug message."""
info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none'
- is_writable = wrap_and_call(
+ devpath = wrap_and_call(
'cloudinit.config.cc_resizefs.util',
{'is_container': {'return_value': True}},
- is_device_path_writable_block, '/I/dont/exist', info, LOG)
- self.assertFalse(is_writable)
+ maybe_get_writable_device_path, '/I/dont/exist', info, LOG)
+ self.assertIsNone(devpath)
self.assertIn(
"DEBUG: Device '/I/dont/exist' did not exist in container."
' cannot resize: %s' % info,
self.logs.getvalue())
- def test_is_device_path_writable_block_raises_oserror(self):
+ def test_maybe_get_writable_device_path_raises_oserror(self):
"""When unexpected OSError is raises by os.stat it is reraised."""
info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none'
with self.assertRaises(OSError) as context_manager:
@@ -234,41 +235,63 @@ class TestIsDevicePathWritableBlock(CiTestCase):
'cloudinit.config.cc_resizefs',
{'util.is_container': {'return_value': True},
'os.stat': {'side_effect': OSError('Something unexpected')}},
- is_device_path_writable_block, '/I/dont/exist', info, LOG)
+ maybe_get_writable_device_path, '/I/dont/exist', info, LOG)
self.assertEqual(
'Something unexpected', str(context_manager.exception))
- def test_is_device_path_writable_block_non_block(self):
+ def test_maybe_get_writable_device_path_non_block(self):
"""When device is not a block device, emit warning return False."""
fake_devpath = self.tmp_path('dev/readwrite')
util.write_file(fake_devpath, '', mode=0o600) # read-write
info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath)
- is_writable = wrap_and_call(
+ devpath = wrap_and_call(
'cloudinit.config.cc_resizefs.util',
{'is_container': {'return_value': False}},
- is_device_path_writable_block, fake_devpath, info, LOG)
- self.assertFalse(is_writable)
+ maybe_get_writable_device_path, fake_devpath, info, LOG)
+ self.assertIsNone(devpath)
self.assertIn(
"WARNING: device '{0}' not a block device. cannot resize".format(
fake_devpath),
self.logs.getvalue())
- def test_is_device_path_writable_block_non_block_on_container(self):
+ def test_maybe_get_writable_device_path_non_block_on_container(self):
"""When device is non-block device in container, emit debug log."""
fake_devpath = self.tmp_path('dev/readwrite')
util.write_file(fake_devpath, '', mode=0o600) # read-write
info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath)
- is_writable = wrap_and_call(
+ devpath = wrap_and_call(
'cloudinit.config.cc_resizefs.util',
{'is_container': {'return_value': True}},
- is_device_path_writable_block, fake_devpath, info, LOG)
- self.assertFalse(is_writable)
+ maybe_get_writable_device_path, fake_devpath, info, LOG)
+ self.assertIsNone(devpath)
self.assertIn(
"DEBUG: device '{0}' not a block device in container."
' cannot resize'.format(fake_devpath),
self.logs.getvalue())
+ def test_maybe_get_writable_device_path_returns_cmdline_root(self):
+ """When root device is UUID in kernel commandline, update devpath."""
+ # XXX Long-term we want to use FilesystemMocking test to avoid
+ # touching os.stat.
+ FakeStat = namedtuple(
+ 'FakeStat', ['st_mode', 'st_size', 'st_mtime']) # minimal def.
+ info = 'dev=/dev/root mnt_point=/ path=/does/not/matter'
+ devpath = wrap_and_call(
+ 'cloudinit.config.cc_resizefs',
+ {'util.get_cmdline': {'return_value': 'asdf root=UUID=my-uuid'},
+ 'util.is_container': False,
+ 'os.path.exists': False, # /dev/root doesn't exist
+ 'os.stat': {
+ 'return_value': FakeStat(25008, 0, 1)} # char block device
+ },
+ maybe_get_writable_device_path, '/dev/root', info, LOG)
+ self.assertEqual('/dev/disk/by-uuid/my-uuid', devpath)
+ self.assertIn(
+ "DEBUG: Converted /dev/root to '/dev/disk/by-uuid/my-uuid'"
+ " per kernel cmdline",
+ self.logs.getvalue())
+
# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index b8fc893..648573f 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -4,11 +4,12 @@ from cloudinit.config.schema import (
CLOUD_CONFIG_HEADER, SchemaValidationError, annotated_cloudconfig_file,
get_schema_doc, get_schema, validate_cloudconfig_file,
validate_cloudconfig_schema, main)
-from cloudinit.util import write_file
+from cloudinit.util import subp, write_file
from cloudinit.tests.helpers import CiTestCase, mock, skipIf
from copy import copy
+import os
from six import StringIO
from textwrap import dedent
from yaml import safe_load
@@ -364,4 +365,38 @@ class MainTest(CiTestCase):
self.assertIn(
'Valid cloud-config file {0}'.format(myyaml), m_stdout.getvalue())
+
+class CloudTestsIntegrationTest(CiTestCase):
+ """Validate all cloud-config yaml schema provided in integration tests.
+
+ It is less expensive to have unittests validate schema of all cloud-config
+ yaml provided to integration tests, than to run an integration test which
+ raises Warnings or errors on invalid cloud-config schema.
+ """
+
+ @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+ def test_all_integration_test_cloud_config_schema(self):
+ """Validate schema of cloud_tests yaml files looking for warnings."""
+ schema = get_schema()
+ testsdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+ integration_testdir = os.path.sep.join(
+ [testsdir, 'cloud_tests', 'testcases'])
+ errors = []
+ out, _ = subp(['find', integration_testdir, '-name', '*yaml'])
+ for filename in out.splitlines():
+ test_cfg = safe_load(open(filename))
+ cloud_config = test_cfg.get('cloud_config')
+ if cloud_config:
+ cloud_config = safe_load(
+ cloud_config.replace("#cloud-config\n", ""))
+ try:
+ validate_cloudconfig_schema(
+ cloud_config, schema, strict=True)
+ except SchemaValidationError as e:
+ errors.append(
+ '{0}: {1}'.format(
+ filename, e))
+ if errors:
+ raise AssertionError(', '.join(errors))
+
# vi: ts=4 expandtab syntax=python
diff --git a/tools/read-dependencies b/tools/read-dependencies
index 2a64868..421f470 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -30,9 +30,35 @@ DISTRO_PKG_TYPE_MAP = {
'suse': 'suse'
}
-DISTRO_INSTALL_PKG_CMD = {
+MAYBE_RELIABLE_YUM_INSTALL = [
+ 'sh', '-c',
+ """
+ error() { echo "$@" 1>&2; }
+ n=0; max=10;
+ bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1"
+ while n=$(($n+1)); do
+ error ":: running $bcmd $* [$n/$max]"
+ $bcmd "$@"
+ r=$?
+ [ $r -eq 0 ] && break
+ [ $n -ge $max ] && { error "gave up on $bcmd"; exit $r; }
+ nap=$(($n*5))
+ error ":: failed [$r] ($n/$max). sleeping $nap."
+ sleep $nap
+ done
+ error ":: running yum install --cacheonly --assumeyes $*"
+ yum install --cacheonly --assumeyes "$@"
+ """,
+ 'reliable-yum-install']
+
+DRY_DISTRO_INSTALL_PKG_CMD = {
'centos': ['yum', 'install', '--assumeyes'],
'redhat': ['yum', 'install', '--assumeyes'],
+}
+
+DISTRO_INSTALL_PKG_CMD = {
+ 'centos': MAYBE_RELIABLE_YUM_INSTALL,
+ 'redhat': MAYBE_RELIABLE_YUM_INSTALL,
'debian': ['apt', 'install', '-y'],
'ubuntu': ['apt', 'install', '-y'],
'opensuse': ['zypper', 'install'],
@@ -80,8 +106,8 @@ def get_parser():
help='Additionally install continuous integration system packages '
'required for build and test automation.')
parser.add_argument(
- '-v', '--python-version', type=str, dest='python_version', default=None,
- choices=["2", "3"],
+ '-v', '--python-version', type=str, dest='python_version',
+ default=None, choices=["2", "3"],
help='Override the version of python we want to generate system '
'package dependencies for. Defaults to the version of python '
'this script is called with')
@@ -219,10 +245,15 @@ def pkg_install(pkg_list, distro, test_distro=False, dry_run=False):
'(dryrun)' if dry_run else '', ' '.join(pkg_list)))
install_cmd = []
if dry_run:
- install_cmd.append('echo')
+ install_cmd.append('echo')
if os.geteuid() != 0:
install_cmd.append('sudo')
- install_cmd.extend(DISTRO_INSTALL_PKG_CMD[distro])
+
+ cmd = DISTRO_INSTALL_PKG_CMD[distro]
+ if dry_run and distro in DRY_DISTRO_INSTALL_PKG_CMD:
+ cmd = DRY_DISTRO_INSTALL_PKG_CMD[distro]
+ install_cmd.extend(cmd)
+
if distro in ['centos', 'redhat']:
# CentOS and Redhat need epel-release to access oauthlib and jsonschema
subprocess.check_call(install_cmd + ['epel-release'])
diff --git a/tools/run-centos b/tools/run-centos
index d44d514..d58ef3e 100755
--- a/tools/run-centos
+++ b/tools/run-centos
@@ -123,7 +123,22 @@ prep() {
return 0
fi
error "Installing prep packages: ${needed}"
- yum install --assumeyes ${needed}
+ set -- $needed
+ local n max r
+ n=0; max=10;
+ bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1"
+ while n=$(($n+1)); do
+ error ":: running $bcmd $* [$n/$max]"
+ $bcmd "$@"
+ r=$?
+ [ $r -eq 0 ] && break
+ [ $n -ge $max ] && { error "gave up on $bcmd"; exit $r; }
+ nap=$(($n*5))
+ error ":: failed [$r] ($n/$max). sleeping $nap."
+ sleep $nap
+ done
+ error ":: running yum install --cacheonly --assumeyes $*"
+ yum install --cacheonly --assumeyes "$@"
}
start_container() {
@@ -153,6 +168,7 @@ start_container() {
if [ ! -z "${http_proxy-}" ]; then
debug 1 "configuring proxy ${http_proxy}"
inside "$name" sh -c "echo proxy=$http_proxy >> /etc/yum.conf"
+ inside "$name" sed -i s/enabled=1/enabled=0/ /etc/yum/pluginconf.d/fastestmirror.conf
fi
}
Follow ups
