← Back to team overview

cloud-init-dev team mailing list archive

[Merge] lp:~smoser/cloud-init/growroot into lp:cloud-init

 

Scott Moser has proposed merging lp:~smoser/cloud-init/growroot into lp:cloud-init.

Requested reviews:
  cloud init development team (cloud-init-dev)
Related bugs:
  Bug #1136936 in cloud-init: "growpart and cloud-utils should support growing mounted filesystem"
  https://bugs.launchpad.net/cloud-init/+bug/1136936

For more details, see:
https://code.launchpad.net/~smoser/cloud-init/growroot/+merge/151817
-- 
https://code.launchpad.net/~smoser/cloud-init/growroot/+merge/151817
Your team cloud init development team is requested to review the proposed merge of lp:~smoser/cloud-init/growroot into lp:cloud-init.
=== added file 'cloudinit/config/cc_growpart.py'
--- cloudinit/config/cc_growpart.py	1970-01-01 00:00:00 +0000
+++ cloudinit/config/cc_growpart.py	2013-03-05 18:39:22 +0000
@@ -0,0 +1,238 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@xxxxxxxxxxxxx>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os.path
+import os
+import re
+import stat
+
+from cloudinit.settings import PER_ALWAYS
+from cloudinit import log as logging
+from cloudinit import util
+
+frequency = PER_ALWAYS
+
+DEFAULT_CONFIG = {
+   'mode': 'auto',
+   'devices': ['/'],
+}
+
+LOG = logging.getLogger(__name__)
+
+def resizer_factory(mode):
+    resize_class = None
+    if mode == "auto":
+        for (_name, resizer) in RESIZERS:
+            cur = resizer()
+            if cur.available():
+                resize_class = cur
+                break
+
+        if not resize_class:
+            raise ValueError("No resizers available")
+
+    else:
+        mmap = {}
+        for (k, v) in RESIZERS:
+            mmap[k] = v
+
+        if mode not in mmap:
+            raise TypeError("unknown resize mode %s" % mode)
+
+        mclass = mmap[mode]()
+        if mclass.available():
+            resize_class = mclass
+
+        if not resize_class:
+            raise ValueError("mode %s not available" % mode)
+
+    return resize_class
+
+
+class ResizeFailedException(Exception):
+    pass
+
+
+class ResizeParted(object):
+    def available(self):
+        myenv = os.environ.copy()
+        myenv['LANG'] = 'C'
+
+        try:
+            (out, _err) = util.subp(["parted", "--help"], env=myenv)
+            if re.search("COMMAND.*resizepart\s+", out, re.DOTALL):
+                return True
+
+        except util.ProcessExecutionError:
+            pass
+        return False
+
+    def resize(self, blockdev, part):
+        try:
+            util.subp(["parted", "resizepart", blockdev, part])
+        except util.ProcessExecutionError as e:
+            raise ResizeFailedException(e)
+
+
+class ResizeGrowPart(object):
+    def available(self):
+        myenv = os.environ.copy()
+        myenv['LANG'] = 'C'
+
+        try:
+            (out, _err) = util.subp(["growpart", "--help"], env=myenv)
+            if re.search("--update\s+", out, re.DOTALL):
+                return True
+
+        except util.ProcessExecutionError:
+            pass
+        return False
+
+    def resize(self, blockdev, part):
+        try:
+            util.subp(["growpart", '--dry-run', blockdev, part])
+        except util.ProcessExecutionError as e:
+            if e.exit_code != 1:
+                logexc(LOG, ("Failed growpart --dry-run for (%s, %s)" %
+                             (blockdev, part)))
+                raise ResizeFailedException(e)
+            LOG.debug("no change necessary on (%s,%s)" % (blockdev, part))
+            return
+
+        try:
+            util.subp(["growpart", blockdev, part])
+        except util.ProcessExecutionError as e:
+            logexc(LOG, "Failed: growpart %s %s" % (blockdev, part))
+            raise ResizeFailedException(e)
+
+
+def device_part_info(devpath):
+    # convert an entry in /dev/ to parent disk and partition number
+
+    # input of /dev/vdb or /dev/disk/by-label/foo
+    # rpath is hopefully a real-ish path in /dev (vda, sdb..)
+    rpath = os.path.realpath(devpath)
+
+    bname = os.path.basename(rpath)
+    syspath = "/sys/class/block/%s" % bname
+
+    if not os.path.exists(syspath):
+        raise ValueError("%s had no syspath (%s)" % (devpath, syspath))
+
+    ptpath = os.path.join(syspath, "partition")
+    if not os.path.exists(ptpath):
+        raise TypeError("%s not a partition" % devpath)
+
+    ptnum = util.load_file(ptpath).rstrip()
+
+    # for a partition, real syspath is something like:
+    # /sys/devices/pci0000:00/0000:00:04.0/virtio1/block/vda/vda1
+    rsyspath = os.path.realpath(syspath)
+    disksyspath = os.path.dirname(rsyspath)
+
+    diskmajmin = util.load_file(os.path.join(disksyspath, "dev")).rstrip()
+    diskdevpath = os.path.realpath("/dev/block/%s" % diskmajmin)
+
+    # diskdevpath has something like 253:0
+    # and udev has put links in /dev/block/253:0 to the device name in /dev/
+    return (diskdevpath, ptnum)
+
+
+def devent2dev(devent):
+    if devent.startswith("/dev/"):
+        return devent
+    else:
+        result = util.get_mount_info(devent)
+        if not result:
+            raise ValueError("Could not determine device of '%s' % dev_ent")
+        return result[0]
+
+
+def resize_devices(resizer, devices):
+    resized = []
+    for devent in devices:
+        try:
+            blockdev = devent2dev(devent)
+        except ValueError as e:
+            LOG.debug("unable to turn %s into device: %s" % (devent, e))
+            continue
+
+        try:
+            statret = os.stat(blockdev)
+        except OSError as e:
+            LOG.debug("device '%s' for '%s' failed stat" %
+                      (blockdev, devent))
+            continue
+            
+        if not stat.S_ISBLK(statret.st_mode):
+            LOG.debug("device '%s' for '%s' is not a block device" %
+                      (blockdev, devent))
+            continue
+
+        try:
+            (disk, ptnum) = device_part_info(blockdev)
+        except (TypeError, ValueError) as e:
+            LOG.debug("failed to get part_info for (%s, %s): %s" %
+                      (devent, blockdev, e))
+            continue
+
+        try:
+            resizer.resize(disk, ptnum)
+        except ResizeFailedException as e:
+            LOG.warn("failed to resize: devent=%s, disk=%s, ptnum=%s: %s",
+                     devent, disk, ptnum, e)
+
+        resized.append(devent)
+
+    return resized
+
+
+def handle(name, cfg, _cloud, log, _args):
+    if 'growpart' not in cfg:
+        log.debug("No 'growpart' entry in cfg.  Using default: %s" %
+                  DEFAULT_CONFIG)
+        cfg['growpart'] = DEFAULT_CONFIG
+
+    mycfg = cfg.get('growpart')
+    if not isinstance(mycfg, dict):
+        log.warn("'growpart' in config was not a dict")
+        return
+
+    mode = mycfg.get('mode')
+    if util.is_false(mode):
+        log.debug("growpart disabled: mode=%s" % mode)
+        return
+
+    devices = util.get_cfg_option_list(cfg, "devices", ["/"])
+    if not len(devices):
+        log.debug("growpart: empty device list")
+        return
+
+    try:
+        resizer = resizer_factory(mode)
+    except (ValueError, TypeError) as e:
+        log.debug("growpart unable to find resizer for '%s': %s" % (mode, e))
+        if mode != "auto":
+            raise e
+        return
+
+    resized = resize_devices(resizer, devices)
+    log.debug("resized: %s" % resized)
+
+RESIZERS = (('parted', ResizeParted), ('growpart', ResizeGrowPart))
+

=== modified file 'cloudinit/config/cc_resizefs.py'
--- cloudinit/config/cc_resizefs.py	2013-03-01 05:28:35 +0000
+++ cloudinit/config/cc_resizefs.py	2013-03-05 18:39:22 +0000
@@ -51,89 +51,6 @@
 NOBLOCK = "noblock"
 
 
-def get_mount_info(path, log):
-    # Use /proc/$$/mountinfo to find the device where path is mounted.
-    # This is done because with a btrfs filesystem using os.stat(path)
-    # does not return the ID of the device.
-    #
-    # Here, / has a device of 18 (decimal).
-    #
-    # $ stat /
-    #   File: '/'
-    #   Size: 234               Blocks: 0          IO Block: 4096   directory
-    # Device: 12h/18d   Inode: 256         Links: 1
-    # Access: (0755/drwxr-xr-x)  Uid: (    0/    root)   Gid: (    0/    root)
-    # Access: 2013-01-13 07:31:04.358011255 +0000
-    # Modify: 2013-01-13 18:48:25.930011255 +0000
-    # Change: 2013-01-13 18:48:25.930011255 +0000
-    #  Birth: -
-    #
-    # Find where / is mounted:
-    #
-    # $ mount | grep ' / '
-    # /dev/vda1 on / type btrfs (rw,subvol=@,compress=lzo)
-    #
-    # And the device ID for /dev/vda1 is not 18:
-    #
-    # $ ls -l /dev/vda1
-    # brw-rw---- 1 root disk 253, 1 Jan 13 08:29 /dev/vda1
-    #
-    # So use /proc/$$/mountinfo to find the device underlying the
-    # input path.
-    path_elements = [e for e in path.split('/') if e]
-    devpth = None
-    fs_type = None
-    match_mount_point = None
-    match_mount_point_elements = None
-    mountinfo_path = '/proc/%s/mountinfo' % os.getpid()
-    for line in util.load_file(mountinfo_path).splitlines():
-        parts = line.split()
-
-        mount_point = parts[4]
-        mount_point_elements = [e for e in mount_point.split('/') if e]
-
-        # Ignore mounts deeper than the path in question.
-        if len(mount_point_elements) > len(path_elements):
-            continue
-
-        # Ignore mounts where the common path is not the same.
-        l = min(len(mount_point_elements), len(path_elements))
-        if mount_point_elements[0:l] != path_elements[0:l]:
-            continue
-
-        # Ignore mount points higher than an already seen mount
-        # point.
-        if (match_mount_point_elements is not None and
-            len(match_mount_point_elements) > len(mount_point_elements)):
-            continue
-
-        # Find the '-' which terminates a list of optional columns to
-        # find the filesystem type and the path to the device.  See
-        # man 5 proc for the format of this file.
-        try:
-            i = parts.index('-')
-        except ValueError:
-            log.debug("Did not find column named '-' in %s",
-                      mountinfo_path)
-            return None
-
-        # Get the path to the device.
-        try:
-            fs_type = parts[i + 1]
-            devpth = parts[i + 2]
-        except IndexError:
-            log.debug("Too few columns in %s after '-' column", mountinfo_path)
-            return None
-
-        match_mount_point = mount_point
-        match_mount_point_elements = mount_point_elements
-
-    if devpth and fs_type and match_mount_point:
-        return (devpth, fs_type, match_mount_point)
-    else:
-        return None
-
-
 def handle(name, cfg, _cloud, log, args):
     if len(args) != 0:
         resize_root = args[0]
@@ -150,7 +67,7 @@
 
     # TODO(harlowja): allow what is to be resized to be configurable??
     resize_what = "/"
-    result = get_mount_info(resize_what, log)
+    result = util.get_mount_info(resize_what, log)
     if not result:
         log.warn("Could not determine filesystem type of %s", resize_what)
         return

=== modified file 'cloudinit/util.py'
--- cloudinit/util.py	2013-01-31 00:21:37 +0000
+++ cloudinit/util.py	2013-03-05 18:39:22 +0000
@@ -1586,3 +1586,86 @@
             raise RuntimeError("Invalid package type.")
 
     return pkglist
+
+
+def get_mount_info(path, log=LOG):
+    # Use /proc/$$/mountinfo to find the device where path is mounted.
+    # This is done because with a btrfs filesystem using os.stat(path)
+    # does not return the ID of the device.
+    #
+    # Here, / has a device of 18 (decimal).
+    #
+    # $ stat /
+    #   File: '/'
+    #   Size: 234               Blocks: 0          IO Block: 4096   directory
+    # Device: 12h/18d   Inode: 256         Links: 1
+    # Access: (0755/drwxr-xr-x)  Uid: (    0/    root)   Gid: (    0/    root)
+    # Access: 2013-01-13 07:31:04.358011255 +0000
+    # Modify: 2013-01-13 18:48:25.930011255 +0000
+    # Change: 2013-01-13 18:48:25.930011255 +0000
+    #  Birth: -
+    #
+    # Find where / is mounted:
+    #
+    # $ mount | grep ' / '
+    # /dev/vda1 on / type btrfs (rw,subvol=@,compress=lzo)
+    #
+    # And the device ID for /dev/vda1 is not 18:
+    #
+    # $ ls -l /dev/vda1
+    # brw-rw---- 1 root disk 253, 1 Jan 13 08:29 /dev/vda1
+    #
+    # So use /proc/$$/mountinfo to find the device underlying the
+    # input path.
+    path_elements = [e for e in path.split('/') if e]
+    devpth = None
+    fs_type = None
+    match_mount_point = None
+    match_mount_point_elements = None
+    mountinfo_path = '/proc/%s/mountinfo' % os.getpid()
+    for line in load_file(mountinfo_path).splitlines():
+        parts = line.split()
+
+        mount_point = parts[4]
+        mount_point_elements = [e for e in mount_point.split('/') if e]
+
+        # Ignore mounts deeper than the path in question.
+        if len(mount_point_elements) > len(path_elements):
+            continue
+
+        # Ignore mounts where the common path is not the same.
+        l = min(len(mount_point_elements), len(path_elements))
+        if mount_point_elements[0:l] != path_elements[0:l]:
+            continue
+
+        # Ignore mount points higher than an already seen mount
+        # point.
+        if (match_mount_point_elements is not None and
+            len(match_mount_point_elements) > len(mount_point_elements)):
+            continue
+
+        # Find the '-' which terminates a list of optional columns to
+        # find the filesystem type and the path to the device.  See
+        # man 5 proc for the format of this file.
+        try:
+            i = parts.index('-')
+        except ValueError:
+            log.debug("Did not find column named '-' in %s",
+                      mountinfo_path)
+            return None
+
+        # Get the path to the device.
+        try:
+            fs_type = parts[i + 1]
+            devpth = parts[i + 2]
+        except IndexError:
+            log.debug("Too few columns in %s after '-' column", mountinfo_path)
+            return None
+
+        match_mount_point = mount_point
+        match_mount_point_elements = mount_point_elements
+
+    if devpth and fs_type and match_mount_point:
+        return (devpth, fs_type, match_mount_point)
+    else:
+        return None

=== modified file 'config/cloud.cfg'
--- config/cloud.cfg	2012-11-19 14:25:09 +0000
+++ config/cloud.cfg	2013-03-05 18:39:22 +0000
@@ -26,6 +26,7 @@
  - migrator
  - bootcmd
  - write-files
+ - growpart
  - resizefs
  - set_hostname
  - update_hostname

=== added file 'tests/unittests/test_handler/test_handler_growpart.py'
--- tests/unittests/test_handler/test_handler_growpart.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_handler/test_handler_growpart.py	2013-03-05 18:39:22 +0000
@@ -0,0 +1,239 @@
+from mocker import MockerTestCase
+
+from cloudinit import cloud
+from cloudinit import helpers
+from cloudinit import util
+
+from cloudinit.config import cc_growpart
+
+import errno
+import logging
+import os
+import mocker
+import re
+import stat
+
+# growpart:
+#   mode: auto  # off, on, auto, 'growpart', 'parted'
+#   devices: ['root']
+
+HELP_PARTED_NO_RESIZE = """
+Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...]
+Apply COMMANDs with PARAMETERS to DEVICE.  If no COMMAND(s) are given, run in
+interactive mode.
+
+OPTIONs:
+<SNIP>
+
+COMMANDs:
+<SNIP>
+  quit                                     exit program
+  rescue START END                         rescue a lost partition near START
+        and END
+  resize NUMBER START END                  resize partition NUMBER and its file
+        system
+  rm NUMBER                                delete partition NUMBER
+<SNIP>
+Report bugs to bug-parted@xxxxxxx
+"""
+
+HELP_PARTED_RESIZE = """
+Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...]
+Apply COMMANDs with PARAMETERS to DEVICE.  If no COMMAND(s) are given, run in
+interactive mode.
+
+OPTIONs:
+<SNIP>
+
+COMMANDs:
+<SNIP>
+  quit                                     exit program
+  rescue START END                         rescue a lost partition near START
+        and END
+  resize NUMBER START END                  resize partition NUMBER and its file
+        system
+  resizepart NUMBER END                    resize partition NUMBER
+  rm NUMBER                                delete partition NUMBER
+<SNIP>
+Report bugs to bug-parted@xxxxxxx
+"""
+
+HELP_GROWPART_RESIZE = """
+growpart disk partition
+   rewrite partition table so that partition takes up all the space it can
+   options:
+    -h | --help       print Usage and exit
+<SNIP>
+    -u | --update  R  update the the kernel partition table info after growing
+                      this requires kernel support and 'partx --update'
+                      R is one of:
+                       - 'auto'  : [default] update partition if possible
+<SNIP>
+   Example:
+    - growpart /dev/sda 1
+      Resize partition 1 on /dev/sda
+"""
+
+HELP_GROWPART_NO_RESIZE = """
+growpart disk partition
+   rewrite partition table so that partition takes up all the space it can
+   options:
+    -h | --help       print Usage and exit
+<SNIP>
+   Example:
+    - growpart /dev/sda 1
+      Resize partition 1 on /dev/sda
+"""
+
+class TestDisabled(MockerTestCase):
+    def setUp(self):
+        super(TestDisabled, self).setUp()
+        self.name = "growpart"
+        self.cloud_init = None
+        self.log = logging.getLogger("TestDisabled")
+        self.args = []
+
+        self.handle = cc_growpart.handle
+
+    def test_mode_off(self):
+        #Test that nothing is done if mode is off.
+
+        # this really only verifies that resizer_factory isn't called
+        config = {'growpart': {'mode': 'off'}}
+        self.mocker.replace(cc_growpart.resizer_factory,
+                            passthrough=False)
+        self.mocker.replay()
+
+        self.handle(self.name, config, self.cloud_init, self.log, self.args)
+
+class TestConfig(MockerTestCase):
+    def setUp(self):
+        super(TestConfig, self).setUp()
+        self.name = "growpart"
+        self.paths = None
+        self.cloud = cloud.Cloud(None, self.paths, None, None, None)
+        self.log = logging.getLogger("TestConfig")
+        self.args = []
+        os.environ = {}
+
+        self.cloud_init = None
+        self.handle = cc_growpart.handle
+
+        # Order must be correct
+        self.mocker.order()
+
+    def test_no_resizers_auto_is_fine(self):
+        subp = self.mocker.replace(util.subp, passthrough=False)
+        subp(['parted', '--help'], env={'LANG': 'C'})
+        self.mocker.result((HELP_PARTED_NO_RESIZE,""))
+        subp(['growpart', '--help'], env={'LANG': 'C'})
+        self.mocker.result((HELP_GROWPART_NO_RESIZE,""))
+        self.mocker.replay()
+
+        config = {'growpart': {'mode': 'auto'}}
+        self.handle(self.name, config, self.cloud_init, self.log, self.args)
+
+    def test_no_resizers_mode_growpart_is_exception(self):
+        subp = self.mocker.replace(util.subp, passthrough=False)
+        subp(['growpart', '--help'], env={'LANG': 'C'})
+        self.mocker.result((HELP_GROWPART_NO_RESIZE,""))
+        self.mocker.replay()
+
+        config = {'growpart': {'mode': "growpart"}}
+        self.assertRaises(ValueError, self.handle, self.name, config,
+                          self.cloud_init, self.log, self.args)
+
+    def test_mode_auto_prefers_parted(self):
+        subp = self.mocker.replace(util.subp, passthrough=False)
+        subp(['parted', '--help'], env={'LANG': 'C'})
+        self.mocker.result((HELP_PARTED_RESIZE,""))
+        self.mocker.replay()
+
+        ret = cc_growpart.resizer_factory(mode="auto")
+        self.assertTrue(isinstance(ret, cc_growpart.ResizeParted))
+
+    def test_handle_with_no_growpart_entry(self):
+        #if no 'growpart' entry in config, then mode=auto should be used
+
+        myresizer = object()
+
+        factory = self.mocker.replace(cc_growpart.resizer_factory,
+                                      passthrough=False)
+        rsdevs = self.mocker.replace(cc_growpart.resize_devices,
+                                     passthrough=False)
+        factory("auto")
+        self.mocker.result(myresizer)
+        rsdevs(myresizer, ["/"])
+        self.mocker.result(["/"])
+        self.mocker.replay()
+
+        try:
+            orig_resizers = cc_growpart.RESIZERS
+            cc_growpart.RESIZERS = (('mysizer', object),)
+            self.handle(self.name, {}, self.cloud_init, self.log, self.args)
+        finally:
+            cc_growpart.RESIZERS = orig_resizers
+            
+
+class TestResize(MockerTestCase):
+    def setUp(self):
+        super(TestResize, self).setUp()
+        self.name = "growpart"
+        self.log = logging.getLogger("TestResize")
+
+        # Order must be correct
+        self.mocker.order()
+
+    def test_simple_devices(self):
+        #test simple device list
+        # this patches out devent2dev, os.stat, and device_part_info
+        # so in the end, doesn't test a lot
+        devs = ["/dev/XXda1", "/dev/YYda2"]
+        devstat_ret = Bunch(st_mode=25008, st_ino=6078, st_dev=5L,
+                            st_nlink=1, st_uid=0, st_gid=6, st_size=0,
+                            st_atime=0, st_mtime=0, st_ctime=0)
+        enoent = ["/dev/NOENT"]
+        real_stat = os.stat
+        resize_calls = []
+
+        class myresizer():
+            def resize(self, dev, part):
+                resize_calls.append((dev, part,))
+                return
+
+        def mystat(path):
+            if path in devs:
+                return devstat_ret
+            if path in enoent:
+                e = OSError("%s: does not exist" % path)
+                e.errno = errno.ENOENT
+                raise e
+            return real_stat(path)
+
+        try:
+            opinfo = cc_growpart.device_part_info
+            cc_growpart.device_part_info = simple_device_part_info
+            os.stat = mystat
+
+            resized = cc_growpart.resize_devices(myresizer(), devs + enoent)
+
+            self.assertEqual(devs, resized)
+            self.assertEqual(resize_calls,
+                             [("/dev/XXda", "1",), ("/dev/YYda", "2",)])
+        finally:
+            cc_growpart.device_part_info = opinfo
+            os.stat = real_stat
+
+
+def simple_device_part_info(devpath):
+    # simple stupid return (/dev/vda, 1) for /dev/vda
+    ret = re.search("([^0-9]*)([0-9]*)$", devpath)
+    x = (ret.group(1), ret.group(2))
+    return x
+        
+class Bunch:
+    def __init__(self, **kwds):
+        self.__dict__.update(kwds)
+
+
+# vi: ts=4 expandtab


Follow ups