cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #00224
[Merge] lp:~smoser/cloud-init/growroot into lp:cloud-init
Scott Moser has proposed merging lp:~smoser/cloud-init/growroot into lp:cloud-init.
Requested reviews:
cloud init development team (cloud-init-dev)
Related bugs:
Bug #1136936 in cloud-init: "growpart and cloud-utils should support growing mounted filesystem"
https://bugs.launchpad.net/cloud-init/+bug/1136936
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/growroot/+merge/151817
--
https://code.launchpad.net/~smoser/cloud-init/growroot/+merge/151817
Your team cloud init development team is requested to review the proposed merge of lp:~smoser/cloud-init/growroot into lp:cloud-init.
=== added file 'cloudinit/config/cc_growpart.py'
--- cloudinit/config/cc_growpart.py 1970-01-01 00:00:00 +0000
+++ cloudinit/config/cc_growpart.py 2013-03-05 18:39:22 +0000
@@ -0,0 +1,238 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2011 Canonical Ltd.
+#
+# Author: Scott Moser <scott.moser@xxxxxxxxxxxxx>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os.path
+import os
+import re
+import stat
+
+from cloudinit.settings import PER_ALWAYS
+from cloudinit import log as logging
+from cloudinit import util
+
+frequency = PER_ALWAYS
+
+DEFAULT_CONFIG = {
+ 'mode': 'auto',
+ 'devices': ['/'],
+}
+
+LOG = logging.getLogger(__name__)
+
+def resizer_factory(mode):
+ resize_class = None
+ if mode == "auto":
+ for (_name, resizer) in RESIZERS:
+ cur = resizer()
+ if cur.available():
+ resize_class = cur
+ break
+
+ if not resize_class:
+ raise ValueError("No resizers available")
+
+ else:
+ mmap = {}
+ for (k, v) in RESIZERS:
+ mmap[k] = v
+
+ if mode not in mmap:
+ raise TypeError("unknown resize mode %s" % mode)
+
+ mclass = mmap[mode]()
+ if mclass.available():
+ resize_class = mclass
+
+ if not resize_class:
+ raise ValueError("mode %s not available" % mode)
+
+ return resize_class
+
+
+class ResizeFailedException(Exception):
+ pass
+
+
+class ResizeParted(object):
+ def available(self):
+ myenv = os.environ.copy()
+ myenv['LANG'] = 'C'
+
+ try:
+ (out, _err) = util.subp(["parted", "--help"], env=myenv)
+ if re.search("COMMAND.*resizepart\s+", out, re.DOTALL):
+ return True
+
+ except util.ProcessExecutionError:
+ pass
+ return False
+
+ def resize(self, blockdev, part):
+ try:
+ util.subp(["parted", "resizepart", blockdev, part])
+ except util.ProcessExecutionError as e:
+ raise ResizeFailedException(e)
+
+
+class ResizeGrowPart(object):
+ def available(self):
+ myenv = os.environ.copy()
+ myenv['LANG'] = 'C'
+
+ try:
+ (out, _err) = util.subp(["growpart", "--help"], env=myenv)
+ if re.search("--update\s+", out, re.DOTALL):
+ return True
+
+ except util.ProcessExecutionError:
+ pass
+ return False
+
+ def resize(self, blockdev, part):
+ try:
+ util.subp(["growpart", '--dry-run', blockdev, part])
+ except util.ProcessExecutionError as e:
+ if e.exit_code != 1:
+ logexc(LOG, ("Failed growpart --dry-run for (%s, %s)" %
+ (blockdev, part)))
+ raise ResizeFailedException(e)
+ LOG.debug("no change necessary on (%s,%s)" % (blockdev, part))
+ return
+
+ try:
+ util.subp(["growpart", blockdev, part])
+ except util.ProcessExecutionError as e:
+ logexc(LOG, "Failed: growpart %s %s" % (blockdev, part))
+ raise ResizeFailedException(e)
+
+
+def device_part_info(devpath):
+ # convert an entry in /dev/ to parent disk and partition number
+
+ # input of /dev/vdb or /dev/disk/by-label/foo
+ # rpath is hopefully a real-ish path in /dev (vda, sdb..)
+ rpath = os.path.realpath(devpath)
+
+ bname = os.path.basename(rpath)
+ syspath = "/sys/class/block/%s" % bname
+
+ if not os.path.exists(syspath):
+ raise ValueError("%s had no syspath (%s)" % (devpath, syspath))
+
+ ptpath = os.path.join(syspath, "partition")
+ if not os.path.exists(ptpath):
+ raise TypeError("%s not a partition" % devpath)
+
+ ptnum = util.load_file(ptpath).rstrip()
+
+ # for a partition, real syspath is something like:
+ # /sys/devices/pci0000:00/0000:00:04.0/virtio1/block/vda/vda1
+ rsyspath = os.path.realpath(syspath)
+ disksyspath = os.path.dirname(rsyspath)
+
+ diskmajmin = util.load_file(os.path.join(disksyspath, "dev")).rstrip()
+ diskdevpath = os.path.realpath("/dev/block/%s" % diskmajmin)
+
+ # diskdevpath has something like 253:0
+ # and udev has put links in /dev/block/253:0 to the device name in /dev/
+ return (diskdevpath, ptnum)
+
+
+def devent2dev(devent):
+ if devent.startswith("/dev/"):
+ return devent
+ else:
+ result = util.get_mount_info(devent)
+ if not result:
+ raise ValueError("Could not determine device of '%s' % dev_ent")
+ return result[0]
+
+
+def resize_devices(resizer, devices):
+ resized = []
+ for devent in devices:
+ try:
+ blockdev = devent2dev(devent)
+ except ValueError as e:
+ LOG.debug("unable to turn %s into device: %s" % (devent, e))
+ continue
+
+ try:
+ statret = os.stat(blockdev)
+ except OSError as e:
+ LOG.debug("device '%s' for '%s' failed stat" %
+ (blockdev, devent))
+ continue
+
+ if not stat.S_ISBLK(statret.st_mode):
+ LOG.debug("device '%s' for '%s' is not a block device" %
+ (blockdev, devent))
+ continue
+
+ try:
+ (disk, ptnum) = device_part_info(blockdev)
+ except (TypeError, ValueError) as e:
+ LOG.debug("failed to get part_info for (%s, %s): %s" %
+ (devent, blockdev, e))
+ continue
+
+ try:
+ resizer.resize(disk, ptnum)
+ except ResizeFailedException as e:
+ LOG.warn("failed to resize: devent=%s, disk=%s, ptnum=%s: %s",
+ devent, disk, ptnum, e)
+
+ resized.append(devent)
+
+ return resized
+
+
+def handle(name, cfg, _cloud, log, _args):
+ if 'growpart' not in cfg:
+ log.debug("No 'growpart' entry in cfg. Using default: %s" %
+ DEFAULT_CONFIG)
+ cfg['growpart'] = DEFAULT_CONFIG
+
+ mycfg = cfg.get('growpart')
+ if not isinstance(mycfg, dict):
+ log.warn("'growpart' in config was not a dict")
+ return
+
+ mode = mycfg.get('mode')
+ if util.is_false(mode):
+ log.debug("growpart disabled: mode=%s" % mode)
+ return
+
+ devices = util.get_cfg_option_list(cfg, "devices", ["/"])
+ if not len(devices):
+ log.debug("growpart: empty device list")
+ return
+
+ try:
+ resizer = resizer_factory(mode)
+ except (ValueError, TypeError) as e:
+ log.debug("growpart unable to find resizer for '%s': %s" % (mode, e))
+ if mode != "auto":
+ raise e
+ return
+
+ resized = resize_devices(resizer, devices)
+ log.debug("resized: %s" % resized)
+
+RESIZERS = (('parted', ResizeParted), ('growpart', ResizeGrowPart))
+
=== modified file 'cloudinit/config/cc_resizefs.py'
--- cloudinit/config/cc_resizefs.py 2013-03-01 05:28:35 +0000
+++ cloudinit/config/cc_resizefs.py 2013-03-05 18:39:22 +0000
@@ -51,89 +51,6 @@
NOBLOCK = "noblock"
-def get_mount_info(path, log):
- # Use /proc/$$/mountinfo to find the device where path is mounted.
- # This is done because with a btrfs filesystem using os.stat(path)
- # does not return the ID of the device.
- #
- # Here, / has a device of 18 (decimal).
- #
- # $ stat /
- # File: '/'
- # Size: 234 Blocks: 0 IO Block: 4096 directory
- # Device: 12h/18d Inode: 256 Links: 1
- # Access: (0755/drwxr-xr-x) Uid: ( 0/ root) Gid: ( 0/ root)
- # Access: 2013-01-13 07:31:04.358011255 +0000
- # Modify: 2013-01-13 18:48:25.930011255 +0000
- # Change: 2013-01-13 18:48:25.930011255 +0000
- # Birth: -
- #
- # Find where / is mounted:
- #
- # $ mount | grep ' / '
- # /dev/vda1 on / type btrfs (rw,subvol=@,compress=lzo)
- #
- # And the device ID for /dev/vda1 is not 18:
- #
- # $ ls -l /dev/vda1
- # brw-rw---- 1 root disk 253, 1 Jan 13 08:29 /dev/vda1
- #
- # So use /proc/$$/mountinfo to find the device underlying the
- # input path.
- path_elements = [e for e in path.split('/') if e]
- devpth = None
- fs_type = None
- match_mount_point = None
- match_mount_point_elements = None
- mountinfo_path = '/proc/%s/mountinfo' % os.getpid()
- for line in util.load_file(mountinfo_path).splitlines():
- parts = line.split()
-
- mount_point = parts[4]
- mount_point_elements = [e for e in mount_point.split('/') if e]
-
- # Ignore mounts deeper than the path in question.
- if len(mount_point_elements) > len(path_elements):
- continue
-
- # Ignore mounts where the common path is not the same.
- l = min(len(mount_point_elements), len(path_elements))
- if mount_point_elements[0:l] != path_elements[0:l]:
- continue
-
- # Ignore mount points higher than an already seen mount
- # point.
- if (match_mount_point_elements is not None and
- len(match_mount_point_elements) > len(mount_point_elements)):
- continue
-
- # Find the '-' which terminates a list of optional columns to
- # find the filesystem type and the path to the device. See
- # man 5 proc for the format of this file.
- try:
- i = parts.index('-')
- except ValueError:
- log.debug("Did not find column named '-' in %s",
- mountinfo_path)
- return None
-
- # Get the path to the device.
- try:
- fs_type = parts[i + 1]
- devpth = parts[i + 2]
- except IndexError:
- log.debug("Too few columns in %s after '-' column", mountinfo_path)
- return None
-
- match_mount_point = mount_point
- match_mount_point_elements = mount_point_elements
-
- if devpth and fs_type and match_mount_point:
- return (devpth, fs_type, match_mount_point)
- else:
- return None
-
-
def handle(name, cfg, _cloud, log, args):
if len(args) != 0:
resize_root = args[0]
@@ -150,7 +67,7 @@
# TODO(harlowja): allow what is to be resized to be configurable??
resize_what = "/"
- result = get_mount_info(resize_what, log)
+ result = util.get_mount_info(resize_what, log)
if not result:
log.warn("Could not determine filesystem type of %s", resize_what)
return
=== modified file 'cloudinit/util.py'
--- cloudinit/util.py 2013-01-31 00:21:37 +0000
+++ cloudinit/util.py 2013-03-05 18:39:22 +0000
@@ -1586,3 +1586,86 @@
raise RuntimeError("Invalid package type.")
return pkglist
+
+
+def get_mount_info(path, log=LOG):
+ # Use /proc/$$/mountinfo to find the device where path is mounted.
+ # This is done because with a btrfs filesystem using os.stat(path)
+ # does not return the ID of the device.
+ #
+ # Here, / has a device of 18 (decimal).
+ #
+ # $ stat /
+ # File: '/'
+ # Size: 234 Blocks: 0 IO Block: 4096 directory
+ # Device: 12h/18d Inode: 256 Links: 1
+ # Access: (0755/drwxr-xr-x) Uid: ( 0/ root) Gid: ( 0/ root)
+ # Access: 2013-01-13 07:31:04.358011255 +0000
+ # Modify: 2013-01-13 18:48:25.930011255 +0000
+ # Change: 2013-01-13 18:48:25.930011255 +0000
+ # Birth: -
+ #
+ # Find where / is mounted:
+ #
+ # $ mount | grep ' / '
+ # /dev/vda1 on / type btrfs (rw,subvol=@,compress=lzo)
+ #
+ # And the device ID for /dev/vda1 is not 18:
+ #
+ # $ ls -l /dev/vda1
+ # brw-rw---- 1 root disk 253, 1 Jan 13 08:29 /dev/vda1
+ #
+ # So use /proc/$$/mountinfo to find the device underlying the
+ # input path.
+ path_elements = [e for e in path.split('/') if e]
+ devpth = None
+ fs_type = None
+ match_mount_point = None
+ match_mount_point_elements = None
+ mountinfo_path = '/proc/%s/mountinfo' % os.getpid()
+ for line in load_file(mountinfo_path).splitlines():
+ parts = line.split()
+
+ mount_point = parts[4]
+ mount_point_elements = [e for e in mount_point.split('/') if e]
+
+ # Ignore mounts deeper than the path in question.
+ if len(mount_point_elements) > len(path_elements):
+ continue
+
+ # Ignore mounts where the common path is not the same.
+ l = min(len(mount_point_elements), len(path_elements))
+ if mount_point_elements[0:l] != path_elements[0:l]:
+ continue
+
+ # Ignore mount points higher than an already seen mount
+ # point.
+ if (match_mount_point_elements is not None and
+ len(match_mount_point_elements) > len(mount_point_elements)):
+ continue
+
+ # Find the '-' which terminates a list of optional columns to
+ # find the filesystem type and the path to the device. See
+ # man 5 proc for the format of this file.
+ try:
+ i = parts.index('-')
+ except ValueError:
+ log.debug("Did not find column named '-' in %s",
+ mountinfo_path)
+ return None
+
+ # Get the path to the device.
+ try:
+ fs_type = parts[i + 1]
+ devpth = parts[i + 2]
+ except IndexError:
+ log.debug("Too few columns in %s after '-' column", mountinfo_path)
+ return None
+
+ match_mount_point = mount_point
+ match_mount_point_elements = mount_point_elements
+
+ if devpth and fs_type and match_mount_point:
+ return (devpth, fs_type, match_mount_point)
+ else:
+ return None
=== modified file 'config/cloud.cfg'
--- config/cloud.cfg 2012-11-19 14:25:09 +0000
+++ config/cloud.cfg 2013-03-05 18:39:22 +0000
@@ -26,6 +26,7 @@
- migrator
- bootcmd
- write-files
+ - growpart
- resizefs
- set_hostname
- update_hostname
=== added file 'tests/unittests/test_handler/test_handler_growpart.py'
--- tests/unittests/test_handler/test_handler_growpart.py 1970-01-01 00:00:00 +0000
+++ tests/unittests/test_handler/test_handler_growpart.py 2013-03-05 18:39:22 +0000
@@ -0,0 +1,239 @@
+from mocker import MockerTestCase
+
+from cloudinit import cloud
+from cloudinit import helpers
+from cloudinit import util
+
+from cloudinit.config import cc_growpart
+
+import errno
+import logging
+import os
+import mocker
+import re
+import stat
+
+# growpart:
+# mode: auto # off, on, auto, 'growpart', 'parted'
+# devices: ['root']
+
+HELP_PARTED_NO_RESIZE = """
+Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...]
+Apply COMMANDs with PARAMETERS to DEVICE. If no COMMAND(s) are given, run in
+interactive mode.
+
+OPTIONs:
+<SNIP>
+
+COMMANDs:
+<SNIP>
+ quit exit program
+ rescue START END rescue a lost partition near START
+ and END
+ resize NUMBER START END resize partition NUMBER and its file
+ system
+ rm NUMBER delete partition NUMBER
+<SNIP>
+Report bugs to bug-parted@xxxxxxx
+"""
+
+HELP_PARTED_RESIZE = """
+Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...]
+Apply COMMANDs with PARAMETERS to DEVICE. If no COMMAND(s) are given, run in
+interactive mode.
+
+OPTIONs:
+<SNIP>
+
+COMMANDs:
+<SNIP>
+ quit exit program
+ rescue START END rescue a lost partition near START
+ and END
+ resize NUMBER START END resize partition NUMBER and its file
+ system
+ resizepart NUMBER END resize partition NUMBER
+ rm NUMBER delete partition NUMBER
+<SNIP>
+Report bugs to bug-parted@xxxxxxx
+"""
+
+HELP_GROWPART_RESIZE = """
+growpart disk partition
+ rewrite partition table so that partition takes up all the space it can
+ options:
+ -h | --help print Usage and exit
+<SNIP>
+ -u | --update R update the the kernel partition table info after growing
+ this requires kernel support and 'partx --update'
+ R is one of:
+ - 'auto' : [default] update partition if possible
+<SNIP>
+ Example:
+ - growpart /dev/sda 1
+ Resize partition 1 on /dev/sda
+"""
+
+HELP_GROWPART_NO_RESIZE = """
+growpart disk partition
+ rewrite partition table so that partition takes up all the space it can
+ options:
+ -h | --help print Usage and exit
+<SNIP>
+ Example:
+ - growpart /dev/sda 1
+ Resize partition 1 on /dev/sda
+"""
+
+class TestDisabled(MockerTestCase):
+ def setUp(self):
+ super(TestDisabled, self).setUp()
+ self.name = "growpart"
+ self.cloud_init = None
+ self.log = logging.getLogger("TestDisabled")
+ self.args = []
+
+ self.handle = cc_growpart.handle
+
+ def test_mode_off(self):
+ #Test that nothing is done if mode is off.
+
+ # this really only verifies that resizer_factory isn't called
+ config = {'growpart': {'mode': 'off'}}
+ self.mocker.replace(cc_growpart.resizer_factory,
+ passthrough=False)
+ self.mocker.replay()
+
+ self.handle(self.name, config, self.cloud_init, self.log, self.args)
+
+class TestConfig(MockerTestCase):
+ def setUp(self):
+ super(TestConfig, self).setUp()
+ self.name = "growpart"
+ self.paths = None
+ self.cloud = cloud.Cloud(None, self.paths, None, None, None)
+ self.log = logging.getLogger("TestConfig")
+ self.args = []
+ os.environ = {}
+
+ self.cloud_init = None
+ self.handle = cc_growpart.handle
+
+ # Order must be correct
+ self.mocker.order()
+
+ def test_no_resizers_auto_is_fine(self):
+ subp = self.mocker.replace(util.subp, passthrough=False)
+ subp(['parted', '--help'], env={'LANG': 'C'})
+ self.mocker.result((HELP_PARTED_NO_RESIZE,""))
+ subp(['growpart', '--help'], env={'LANG': 'C'})
+ self.mocker.result((HELP_GROWPART_NO_RESIZE,""))
+ self.mocker.replay()
+
+ config = {'growpart': {'mode': 'auto'}}
+ self.handle(self.name, config, self.cloud_init, self.log, self.args)
+
+ def test_no_resizers_mode_growpart_is_exception(self):
+ subp = self.mocker.replace(util.subp, passthrough=False)
+ subp(['growpart', '--help'], env={'LANG': 'C'})
+ self.mocker.result((HELP_GROWPART_NO_RESIZE,""))
+ self.mocker.replay()
+
+ config = {'growpart': {'mode': "growpart"}}
+ self.assertRaises(ValueError, self.handle, self.name, config,
+ self.cloud_init, self.log, self.args)
+
+ def test_mode_auto_prefers_parted(self):
+ subp = self.mocker.replace(util.subp, passthrough=False)
+ subp(['parted', '--help'], env={'LANG': 'C'})
+ self.mocker.result((HELP_PARTED_RESIZE,""))
+ self.mocker.replay()
+
+ ret = cc_growpart.resizer_factory(mode="auto")
+ self.assertTrue(isinstance(ret, cc_growpart.ResizeParted))
+
+ def test_handle_with_no_growpart_entry(self):
+ #if no 'growpart' entry in config, then mode=auto should be used
+
+ myresizer = object()
+
+ factory = self.mocker.replace(cc_growpart.resizer_factory,
+ passthrough=False)
+ rsdevs = self.mocker.replace(cc_growpart.resize_devices,
+ passthrough=False)
+ factory("auto")
+ self.mocker.result(myresizer)
+ rsdevs(myresizer, ["/"])
+ self.mocker.result(["/"])
+ self.mocker.replay()
+
+ try:
+ orig_resizers = cc_growpart.RESIZERS
+ cc_growpart.RESIZERS = (('mysizer', object),)
+ self.handle(self.name, {}, self.cloud_init, self.log, self.args)
+ finally:
+ cc_growpart.RESIZERS = orig_resizers
+
+
+class TestResize(MockerTestCase):
+ def setUp(self):
+ super(TestResize, self).setUp()
+ self.name = "growpart"
+ self.log = logging.getLogger("TestResize")
+
+ # Order must be correct
+ self.mocker.order()
+
+ def test_simple_devices(self):
+ #test simple device list
+ # this patches out devent2dev, os.stat, and device_part_info
+ # so in the end, doesn't test a lot
+ devs = ["/dev/XXda1", "/dev/YYda2"]
+ devstat_ret = Bunch(st_mode=25008, st_ino=6078, st_dev=5L,
+ st_nlink=1, st_uid=0, st_gid=6, st_size=0,
+ st_atime=0, st_mtime=0, st_ctime=0)
+ enoent = ["/dev/NOENT"]
+ real_stat = os.stat
+ resize_calls = []
+
+ class myresizer():
+ def resize(self, dev, part):
+ resize_calls.append((dev, part,))
+ return
+
+ def mystat(path):
+ if path in devs:
+ return devstat_ret
+ if path in enoent:
+ e = OSError("%s: does not exist" % path)
+ e.errno = errno.ENOENT
+ raise e
+ return real_stat(path)
+
+ try:
+ opinfo = cc_growpart.device_part_info
+ cc_growpart.device_part_info = simple_device_part_info
+ os.stat = mystat
+
+ resized = cc_growpart.resize_devices(myresizer(), devs + enoent)
+
+ self.assertEqual(devs, resized)
+ self.assertEqual(resize_calls,
+ [("/dev/XXda", "1",), ("/dev/YYda", "2",)])
+ finally:
+ cc_growpart.device_part_info = opinfo
+ os.stat = real_stat
+
+
+def simple_device_part_info(devpath):
+ # simple stupid return (/dev/vda, 1) for /dev/vda
+ ret = re.search("([^0-9]*)([0-9]*)$", devpath)
+ x = (ret.group(1), ret.group(2))
+ return x
+
+class Bunch:
+ def __init__(self, **kwds):
+ self.__dict__.update(kwds)
+
+
+# vi: ts=4 expandtab
Follow ups
