cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #06360
[Merge] ~raharper/cloud-init:feature/disk_setup_async into cloud-init:master
Ryan Harper has proposed merging ~raharper/cloud-init:feature/disk_setup_async into cloud-init:master.
Commit message:
Implement async mount via fstab entry
On systemd enabled systems, cloud-init can make use of systemd-makefs
service via use of 'x-systemd.makefs' fs_mntop entry. This results in
the creation of a mount unit and mkfs service including dependencies to
ensure that the target is formatted and mounted by systemd prior to the
start of cloud-config.service.
This feature is enabled by specifying 'x-systemd.makefs' in a mount
cloud-config entry. If enabled, cc_disk_setup will skip filesystem
creation deferring to systemd-makefs, if the specified device in the
mount entry matches an entry in fs_setup.
Lastly, cc_mounts now supports specifying 'cidefaults' as a fs_mntop
value which will replace the string 'cidefaults' with the default system
mount options; either cloud-init defaults or via cloud-config when users
specify 'default_mount_options'.
Also included in this branch:
- Add eventReportStack context managers for timing disk_setup and
fs_setup separately
- fixes to cloud-init analyze parsing for sub-stage events
Requested reviews:
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/369362
--
Your team cloud-init commiters is requested to review the proposed merge of ~raharper/cloud-init:feature/disk_setup_async into cloud-init:master.
diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py
index 3e778b8..b15cd2c 100644
--- a/cloudinit/analyze/show.py
+++ b/cloudinit/analyze/show.py
@@ -94,6 +94,10 @@ def event_parent(event):
return None
+def event_is_stage(event):
+ return '/' not in event_name(event)
+
+
def event_timestamp(event):
return float(event.get('timestamp'))
@@ -146,7 +150,9 @@ def generate_records(events, blame_sort=False,
next_evt = None
if event_type(event) == 'start':
- if event.get('name') in stages_seen:
+ stage_name = event_parent(event)
+ if stage_name == event_name(event) and stage_name in stages_seen:
+ # new boot record
records.append(total_time_record(total_time))
boot_records.append(records)
records = []
@@ -166,19 +172,26 @@ def generate_records(events, blame_sort=False,
event,
next_evt)))
else:
- # This is a parent event
- records.append("Starting stage: %s" % event.get('name'))
- unprocessed.append(event)
- stages_seen.append(event.get('name'))
- continue
+ if event_is_stage(event):
+ records.append("Starting stage: %s" % event.get('name'))
+ unprocessed.append(event)
+ stages_seen.append(event.get('name'))
+ else:
+ # Start of a substage event
+ records.append(format_record(print_format,
+ event_record(start_time,
+ event,
+ next_evt)))
+
else:
prev_evt = unprocessed.pop()
if event_name(event) == event_name(prev_evt):
- record = event_record(start_time, prev_evt, event)
- records.append(format_record("Finished stage: "
- "(%n) %d seconds ",
- record) + "\n")
- total_time += record.get('delta')
+ if event_is_stage(event):
+ record = event_record(start_time, prev_evt, event)
+ records.append(format_record("Finished stage: "
+ "(%n) %d seconds ",
+ record) + "\n")
+ total_time += record.get('delta')
else:
# not a match, put it back
unprocessed.append(prev_evt)
diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index 29e192e..00d9930 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -71,6 +71,11 @@ specified using ``filesystem``.
.. note::
``replace_fs`` is ignored unless ``partition`` is ``auto`` or ``any``.
+.. note::
+ Filesystem creation can be deferred to systemd-makefs if a specified
+ device is present in ``mounts`` cloud-config and includes the
+ ``x-systemd-makefs`` in the fs_mntopts field.
+
**Internal name:** ``cc_disk_setup``
**Module frequency:** per instance
@@ -99,6 +104,7 @@ specified using ``filesystem``.
from cloudinit.settings import PER_INSTANCE
from cloudinit import util
+from cloudinit.reporting import events
import logging
import os
import shlex
@@ -126,9 +132,23 @@ def handle(_name, cfg, cloud, log, _args):
"""
disk_setup = cfg.get("disk_setup")
if isinstance(disk_setup, dict):
- update_disk_setup_devices(disk_setup, cloud.device_name_to_device)
- log.debug("Partitioning disks: %s", str(disk_setup))
- for disk, definition in disk_setup.items():
+ _handle_disk_setup(disk_setup, cloud, log)
+
+ fs_setup = cfg.get("fs_setup")
+ mounts = cfg.get("mounts", [])
+ if isinstance(fs_setup, list):
+ _handle_fs_setup(fs_setup, mounts, cloud, log)
+
+
+def _handle_disk_setup(disk_setup, cloud, log):
+ update_disk_setup_devices(disk_setup, cloud.device_name_to_device)
+ log.debug("Partitioning disks: %s", str(disk_setup))
+ for disk, definition in disk_setup.items():
+ disk_str = disk.lstrip('/').replace('/', '-')
+ run_name = 'config-disk_setup/partition-' + disk_str
+ with events.ReportEventStack(name=run_name,
+ description='Partitioning ' + disk_str,
+ parent=cloud.reporter):
if not isinstance(definition, dict):
log.warning("Invalid disk definition for %s" % disk)
continue
@@ -141,18 +161,32 @@ def handle(_name, cfg, cloud, log, _args):
except Exception as e:
util.logexc(LOG, "Failed partitioning operation\n%s" % e)
- fs_setup = cfg.get("fs_setup")
- if isinstance(fs_setup, list):
- log.debug("setting up filesystems: %s", str(fs_setup))
- update_fs_setup_devices(fs_setup, cloud.device_name_to_device)
- for definition in fs_setup:
+
+def _handle_fs_setup(fs_setup, mounts, cloud, log):
+ log.debug("setting up filesystems: %s", str(fs_setup))
+ update_fs_setup_devices(fs_setup, cloud.device_name_to_device)
+
+ # list of devices which use systemd-makefs to format
+ skip_mkfs = [mnt[0] for mnt in mounts
+ if len(mnt) > 3 and 'x-systemd.makefs' in mnt[3]]
+
+ for idx, definition in enumerate(fs_setup):
+ run_name = 'config-disk_setup/mkfs-entry-%s' % idx
+ desc = 'Create Filesystem for entry %s' % idx
+ with events.ReportEventStack(name=run_name, description=desc,
+ parent=cloud.reporter):
if not isinstance(definition, dict):
log.warning("Invalid file system definition: %s" % definition)
continue
try:
- log.debug("Creating new filesystem.")
device = definition.get('device')
+ og_dev = definition.get('_origname')
+ if set(skip_mkfs).intersection(set([device, og_dev])):
+ log.debug('Deferring mkfs of %s to systemd-make-fs',
+ device)
+ continue
+ log.debug("Creating new filesystem.")
util.log_time(logfunc=LOG.debug,
msg="Creating fs for %s" % device,
func=mkfs, args=(definition,))
diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index 123ffb8..c227253 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -38,6 +38,10 @@ On a systemd booted system that default is the mostly equivalent::
Note that `nobootwait` is an upstart specific boot option that somewhat
equates to the more standard `nofail`.
+When specifying mount opts, using ``cidefaults`` will expand into the
+the mount_default_fields values for fs_mntops. This is useful for
+appending non-standard values without having to track the default values.
+
Swap files can be configured by setting the path to the swap file to create
with ``filename``, the size of the swap file with ``size`` maximum size of
the swap file if using an ``size: auto`` with ``maxsize``. By default no
@@ -55,6 +59,7 @@ swap file is created.
- [ /dev/ephemeral0, /mnt, auto, "defaults,noexec" ]
- [ sdc, /opt/data ]
- [ xvdh, /opt/data, "auto", "defaults,nofail", "0", "0" ]
+ - [ ephemeral0.1, ext4, cidefaults,x-systemd.makefs, none, none ]
mount_default_fields: [None, None, "auto", "defaults,nofail", "0", "2"]
swap:
filename: <file>
@@ -77,6 +82,9 @@ DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER)
WS = re.compile("[%s]+" % (whitespace))
FSTAB_PATH = "/etc/fstab"
MNT_COMMENT = "comment=cloudconfig"
+MNT_BEFORE_CC = "x-systemd.before=cloud-config.service"
+DEF_MNT_OPTS = "defaults,nobootwait"
+SYSTEMD_DEF_MNT_OPTS = "defaults,nofail,x-systemd.requires=cloud-init.service"
LOG = logging.getLogger(__name__)
@@ -310,16 +318,16 @@ def handle_swapcfg(swapcfg):
def handle(_name, cfg, cloud, log, _args):
# fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
- def_mnt_opts = "defaults,nobootwait"
+ def_mnt_opts = DEF_MNT_OPTS
uses_systemd = cloud.distro.uses_systemd()
if uses_systemd:
- def_mnt_opts = "defaults,nofail,x-systemd.requires=cloud-init.service"
+ def_mnt_opts = SYSTEMD_DEF_MNT_OPTS
defvals = [None, None, "auto", def_mnt_opts, "0", "2"]
defvals = cfg.get("mount_default_fields", defvals)
# these are our default set of mounts
- defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"],
+ defmnts = [["ephemeral0", "/mnt", defvals[2], defvals[3], "0", "2"],
["swap", "none", "swap", "sw", "0", "0"]]
cfgmnt = []
@@ -390,6 +398,12 @@ def handle(_name, cfg, cloud, log, _args):
if cfgmnt[j][0] == cfgmnt[i][0]:
cfgmnt[j][1] = None
+ # expand fs_mntopts 'cidefaults' to defvals fs_mntopts
+ if 'cidefaults' in cfgmnt[i][3]:
+ log.debug("expanding fs_mntops keyword 'cidefaults' => %s",
+ defvals[3])
+ cfgmnt[i][3] = cfgmnt[i][3].replace('cidefaults', defvals[3])
+
# for each of the "default" mounts, add them only if no other
# entry has the same device name
for defmnt in defmnts:
@@ -441,7 +455,15 @@ def handle(_name, cfg, cloud, log, _args):
needswap = False
need_mount_all = False
dirs = []
+ systemd_mounts = []
for line in actlist:
+ # if user specifies a x-systemd.makefs, extract the mount point
+ # which is the name of the systemd mount unit we need to enqueue
+ if 'x-systemd.makefs' in line[3]:
+ # transform mount point entry to systemd mount unit name
+ systemd_mounts.append(line[1].lstrip('/').replace('/', '-'))
+ # ensure this unit is mounted before cloud-config.target
+ line[3] = "%s,%s" % (line[3], MNT_BEFORE_CC)
# write 'comment' in the fs_mntops, entry, claiming this
line[3] = "%s,%s" % (line[3], MNT_COMMENT)
if line[2] == "swap":
@@ -474,19 +496,22 @@ def handle(_name, cfg, cloud, log, _args):
util.write_file(FSTAB_PATH, contents)
activate_cmds = []
- if needswap:
- activate_cmds.append(["swapon", "-a"])
-
if len(sops) == 0:
log.debug("No changes to /etc/fstab made.")
else:
log.debug("Changes to fstab: %s", sops)
need_mount_all = True
+ if needswap:
+ activate_cmds.append(["swapon", "-a"])
if need_mount_all:
- activate_cmds.append(["mount", "-a"])
if uses_systemd:
activate_cmds.append(["systemctl", "daemon-reload"])
+ for sd_mount in systemd_mounts:
+ activate_cmds.append(['systemctl', 'start', '--no-block',
+ '%s.mount' % sd_mount])
+ if len(systemd_mounts) == 0:
+ activate_cmds.append(["mount", "-a"])
fmt = "Activating swap and mounts with: %s"
for cmd in activate_cmds:
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index d2fad9b..21dc0f0 100755
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -232,6 +232,9 @@ BUILTIN_CLOUD_CONFIG = {
},
'fs_setup': [{'filesystem': DEFAULT_FS,
'device': 'ephemeral0.1'}],
+ 'mounts': [
+ ['ephemeral0.1', '/mnt', DEFAULT_FS, 'cidefaults,x-systemd.makefs']
+ ],
}
DS_CFG_PATH = ['datasource', DS_NAME]
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index f41180f..f088bab 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -6,7 +6,9 @@ import functools
import httpretty
import logging
import os
+import random
import shutil
+import string
import sys
import tempfile
import time
@@ -122,6 +124,12 @@ class TestCase(unittest2.TestCase):
parser.readfp(contents)
return parser
+ @classmethod
+ def random_string(cls, length=8):
+ """ return a random lowercase string with default length of 8"""
+ return ''.join(
+ random.choice(string.ascii_lowercase) for _ in range(length))
+
class CiTestCase(TestCase):
"""This is the preferred test case base class unless user
diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py
index 0fb160b..4c7470d 100644
--- a/tests/unittests/test_handler/test_handler_mounts.py
+++ b/tests/unittests/test_handler/test_handler_mounts.py
@@ -255,7 +255,52 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase):
self.assertEqual(fstab_expected_content, fstab_new_content)
cc_mounts.handle(None, cc, self.mock_cloud, self.mock_log, [])
self.m_util_subp.assert_has_calls([
+ mock.call(['swapon', '-a']),
+ mock.call(['systemctl', 'daemon-reload']),
mock.call(['mount', '-a']),
- mock.call(['systemctl', 'daemon-reload'])])
+ ])
+
+ def test_ephemeral_systemd_makefs(self):
+ self.maxDiff = None
+ dev = '/dev/disk/cloud/azure_resource-part1'
+ def_opts = "defaults,nofail,x-systemd.requires=cloud-init.service"
+ make_opts = ',x-systemd.makefs'
+ before_opts = ',x-systemd.before=cloud-config.service'
+ comment_opts = ',comment=cloudconfig'
+ opts = def_opts + make_opts + before_opts + comment_opts
+
+ fstab_original_content = (
+ 'LABEL=cloudimg-rootfs / ext4 defaults 0 0\n'
+ 'LABEL=UEFI /boot/efi vfat defaults 0 0\n'
+ )
+ fstab_expected_content = (
+ fstab_original_content +
+ '%s /mnt ext4 %s 0 2\n' % (dev, opts)
+ )
+
+ with open(cc_mounts.FSTAB_PATH, 'w') as fd:
+ fd.write(fstab_original_content)
+
+ # enable systemd
+ self.mock_cloud.distro.uses_systemd.return_value = True
+
+ # return devname
+ name_mock = mock.Mock()
+ name_mock.return_value = dev
+ self.mock_cloud.device_name_to_device = name_mock
+
+ cc_opts = def_opts + make_opts
+ cc = {'mount_default_fields': [None, None, 'ext4', cc_opts, "0", "2"]}
+ cc_mounts.handle(None, cc, self.mock_cloud, self.mock_log, [])
+
+ self.m_util_subp.assert_has_calls([
+ mock.call(['systemctl', 'daemon-reload']),
+ mock.call(['systemctl', 'start', '--no-block', 'mnt.mount']),
+ ])
+
+ with open(cc_mounts.FSTAB_PATH, 'r') as fd:
+ fstab_new_content = fd.read()
+ print('\n%s' % fstab_new_content)
+ self.assertEqual(fstab_expected_content, fstab_new_content)
# vi: ts=4 expandtab
Follow ups
