curtin-dev team mailing list archive

[Dan Bungert <mp+417829@xxxxxxxxxxxxxxxxxx>]

Dan Bungert has proposed merging ~dbungert/curtin:resize into curtin:master.

Commit message:
Draft of resize support

Requested reviews:
  curtin developers (curtin-dev)

For more details, see:
https://code.launchpad.net/~dbungert/curtin/+git/curtin/+merge/417829
-- 
Your team curtin developers is requested to review the proposed merge of ~dbungert/curtin:resize into curtin:master.

diff --git a/curtin/block/schemas.py b/curtin/block/schemas.py
index 0a6e305..1834343 100644
--- a/curtin/block/schemas.py
+++ b/curtin/block/schemas.py
@@ -290,6 +290,7 @@ PARTITION = {
                  'pattern': _path_dev},
         'name': {'$ref': '#/definitions/name'},
         'offset': {'$ref': '#/definitions/size'},  # XXX: This is not used
+        'resize': {'type': 'boolean'},
         'preserve': {'$ref': '#/definitions/preserve'},
         'size': {'$ref': '#/definitions/size'},
         'uuid': {'$ref': '#/definitions/uuid'},    # XXX: This is not used
diff --git a/curtin/commands/block_meta.py b/curtin/commands/block_meta.py
index cdf30c5..297b967 100644
--- a/curtin/commands/block_meta.py
+++ b/curtin/commands/block_meta.py
@@ -779,12 +779,17 @@ def verify_exists(devpath):
         raise RuntimeError("Device %s does not exist" % devpath)
 
 
-def verify_size(devpath, expected_size_bytes, part_info):
+def get_part_size_bytes(devpath, part_info):
     (found_type, _code) = ptable_uuid_to_flag_entry(part_info.get('type'))
     if found_type == 'extended':
         found_size_bytes = int(part_info['size']) * 512
     else:
         found_size_bytes = block.read_sys_block_size_bytes(devpath)
+    return found_size_bytes
+
+
+def verify_size(devpath, expected_size_bytes, part_info):
+    found_size_bytes = get_part_size_bytes(devpath, part_info)
     msg = (
         'Verifying %s size, expecting %s bytes, found %s bytes' % (
          devpath, expected_size_bytes, found_size_bytes))
diff --git a/curtin/commands/block_meta_v2.py b/curtin/commands/block_meta_v2.py
index 051649b..1879957 100644
--- a/curtin/commands/block_meta_v2.py
+++ b/curtin/commands/block_meta_v2.py
@@ -1,5 +1,6 @@
 # This file is part of curtin. See LICENSE file for copyright and license info.
 
+import os
 from typing import Optional
 
 import attr
@@ -7,10 +8,12 @@ import attr
 from curtin import (block, util)
 from curtin.commands.block_meta import (
     disk_handler as disk_handler_v1,
+    get_part_size_bytes,
     get_path_to_storage_volume,
     make_dname,
     partition_handler as partition_handler_v1,
-    partition_verify_sfdisk,
+    verify_ptable_flag,
+    verify_size,
     )
 from curtin.log import LOG
 from curtin.storage_config import (
@@ -214,6 +217,52 @@ def _wipe_for_action(action):
     return 'superblock'
 
 
+def partition_verify_sfdisk_v2(part_action, label, sfdisk_part_info):
+    devpath = os.path.realpath(sfdisk_part_info['node'])
+    if not part_action.get('resize', False):
+        verify_size(devpath, int(util.human2bytes(part_action['size'])),
+                    sfdisk_part_info)
+    expected_flag = part_action.get('flag')
+    if expected_flag:
+        verify_ptable_flag(devpath, expected_flag, label, sfdisk_part_info)
+
+
+def process_resize(part_action, storage_config):
+    if not part_action.get('resize', False):
+        return
+    disk_id = part_action['device']
+    disk_action = storage_config[disk_id]
+    part_num = part_action['number']
+    disk_dev = disk_action['dev']
+    part_dev = f'{disk_dev}p{part_num}'
+
+    start_size = get_part_size_bytes(part_dev, part_action)
+    end_size = util.human2bytes(part_action['size'])
+    offset = part_action['offset']
+    end_position = end_size + offset
+    end_size_k = end_size // 1024
+    if start_size == end_size:
+        LOG.debug('Skipping resize of %s - partition is already target size',
+                  part_action['id'])
+    elif start_size < end_size:
+        LOG.debug('Resizing %s upward from %s to %s',
+                  part_action['id'], start_size, end_size)
+        util.subp(['parted', disk_dev, '--script', 'resizepart',
+                   str(part_num), f'{end_position}B'])
+        util.subp(['e2fsck', '-p', '-f', part_dev])
+        util.subp(['resize2fs', part_dev, f'{end_size_k}k'])
+    else:
+        LOG.debug('Resizing %s downward from %s to %s',
+                  part_dev, start_size, end_size)
+        util.subp(['e2fsck', '-p', '-f', part_dev])
+        util.subp(['resize2fs', part_dev, f'{end_size_k}k'])
+        # Shrink step has an extra prompt, stdin Yes is the response.
+        # Also needs this super-cool triple dash argument.
+        cmd = ['parted', '---pretend-input-tty', disk_dev, 'resizepart',
+               str(part_num), f'{end_position}B']
+        util.subp(cmd, data='Yes\n'.encode())
+
+
 def disk_handler_v2(info, storage_config, handlers):
     disk_handler_v1(info, storage_config, handlers)
 
@@ -251,7 +300,8 @@ def disk_handler_v2(info, storage_config, handlers):
                 # vmtest infrastructure unhappy.
                 sfdisk_info = block.sfdisk_info(disk)
             part_info = _find_part_info(sfdisk_info, entry.start)
-            partition_verify_sfdisk(action, sfdisk_info['label'], part_info)
+            partition_verify_sfdisk_v2(action, sfdisk_info['label'], part_info)
+            process_resize(action, storage_config)
             preserved_offsets.add(entry.start)
         wipe = wipes[entry.start] = _wipe_for_action(action)
         if wipe is not None:
diff --git a/curtin/util.py b/curtin/util.py
index 5b66b55..d3c3b66 100644
--- a/curtin/util.py
+++ b/curtin/util.py
@@ -501,6 +501,13 @@ def chdir(dirname):
         os.chdir(curdir)
 
 
+@contextmanager
+def mount(src, target):
+    do_mount(src, target)
+    yield
+    do_umount(target)
+
+
 def do_mount(src, target, opts=None):
     # mount src at target with opts and return True
     # if already mounted, return False
diff --git a/doc/topics/storage.rst b/doc/topics/storage.rst
index 5fae90f..ab89b16 100644
--- a/doc/topics/storage.rst
+++ b/doc/topics/storage.rst
@@ -429,6 +429,16 @@ filesystem or be mounted anywhere on the system.
 
 If the preserve flag is set to true, curtin will verify that the partition
 exists and that  the ``size`` and ``flag`` match the configuration provided.
+See also the resize flag, which adjust this behavior.
+
+**resize**: *true, false*
+
+Only applicable to v2 storage configuration.
+If the preserve flag is set to false, this value is not applicable.
+If the preserve flag is set to true, curtin will adjust the size of the
+partition to the new size.  When adjusting smaller, the size of the contents
+must permit that.  When adjusting larger, there must already be a gap beyond
+the partition in question.
 
 **name**: *<name>*
 
diff --git a/tests/integration/test_block_meta.py b/tests/integration/test_block_meta.py
index 0c74cd6..79cf20b 100644
--- a/tests/integration/test_block_meta.py
+++ b/tests/integration/test_block_meta.py
@@ -34,6 +34,18 @@ def loop_dev(image, sector_size=512):
 PartData = namedtuple("PartData", ('number', 'offset', 'size'))
 
 
+def _get_filesystem_size(dev, part_action):
+    num = part_action['number']
+    cmd = ['dumpe2fs', '-h', f'{dev}p{num}']
+    out, _ = util.subp(cmd, capture=True)
+    for line in out.splitlines():
+        if line.startswith('Block count'):
+            block_count = line.split(':')[1].strip()
+        if line.startswith('Block size'):
+            block_size = line.split(':')[1].strip()
+    return int(block_count) * int(block_size)
+
+
 def summarize_partitions(dev):
     # We don't care about the kname
     return sorted(
@@ -55,33 +67,28 @@ class StorageConfigBuilder:
                 },
             }
 
-    def add_image(self, *, path, size, create=False, **kw):
-        action = {
-            'type': 'image',
-            'id': 'id' + str(len(self.config)),
-            'path': path,
-            'size': size,
-            }
-        action.update(**kw)
-        self.cur_image = action['id']
+    def _add(self, *, type, **kw):
+        if type != 'image' and self.cur_image is None:
+            raise Exception("no current image")
+        action = {'id': 'id' + str(len(self.config))}
+        action.update(type=type, **kw)
         self.config.append(action)
+        return action
+
+    def add_image(self, *, path, size, create=False, **kw):
         if create:
             with open(path, "wb") as f:
                 f.write(b"\0" * int(util.human2bytes(size)))
+        action = self._add(type='image', path=path, size=size, **kw)
+        self.cur_image = action['id']
         return action
 
     def add_part(self, *, size, **kw):
-        if self.cur_image is None:
-            raise Exception("no current image")
-        action = {
-            'type': 'partition',
-            'id': 'id' + str(len(self.config)),
-            'device': self.cur_image,
-            'size': size,
-            }
-        action.update(**kw)
-        self.config.append(action)
-        return action
+        return self._add(type='partition', device=self.cur_image, size=size,
+                         **kw)
+
+    def add_format(self, *, part, fstype='ext4', **kw):
+        return self._add(type='format', volume=part['id'], fstype=fstype, **kw)
 
     def set_preserve(self):
         for action in self.config:
@@ -89,8 +96,14 @@ class StorageConfigBuilder:
 
 
 class TestBlockMeta(IntegrationTestCase):
-
-    def run_bm(self, config, *args, **kwargs):
+    @contextlib.contextmanager
+    def mount(self, dev, partition_cfg):
+        mnt_point = self.tmp_dir()
+        num = partition_cfg['number']
+        with util.mount(f'{dev}p{num}', mnt_point):
+            yield mnt_point
+
+    def run_bm(self, config, *args, fake=False, **kwargs):
         config_path = self.tmp_path('config.yaml')
         with open(config_path, 'w') as fp:
             yaml.dump(config, fp)
@@ -111,6 +124,17 @@ class TestBlockMeta(IntegrationTestCase):
             '-c', config_path, 'block-meta', '--testmode', 'custom',
             *args,
             ]
+        if fake:
+            script = '/tmp/curtin.sh'
+            with open(script, 'w') as fp:
+                fp.write('#!/bin/bash\n')
+                fp.writelines([
+                    f'export {var}="{val}"\n' for var, val in cmd_env.items()])
+                fp.write(' '.join(cmd))
+                fp.write('\n')
+            print(script)
+            breakpoint()
+            raise Exception('aborting test')
         util.subp(cmd, env=cmd_env, **kwargs)
 
     def _test_default_offsets(self, ptable, version, sector_size=512):
@@ -415,3 +439,127 @@ class TestBlockMeta(IntegrationTestCase):
                     )
         finally:
             server.stop()
+
+    def _do_test_resize(self, start, end):
+        start <<= 20
+        end <<= 20
+        img = self.tmp_path('image.img')
+        config = StorageConfigBuilder(version=2)
+        config.add_image(path=img, size='200M', ptable='gpt')
+        p1 = config.add_part(size=start, offset=1 << 20, number=1)
+        config.add_format(part=p1)
+        self.run_bm(config.render())
+
+        expected = 'preserve my data!'
+        with loop_dev(img) as dev:
+            self.assertEqual(
+                summarize_partitions(dev), [
+                    PartData(number=1, offset=1 << 20, size=start),
+                ])
+            with self.mount(dev, p1) as mnt_point:
+                with open(f'{mnt_point}/data.txt', 'w') as fp:
+                    fp.write(expected)
+            orig_fs_size = _get_filesystem_size(dev, p1)
+            self.assertEqual(start, orig_fs_size)
+
+        config.set_preserve()
+        p1['resize'] = True
+        p1['size'] = end
+        self.run_bm(config.render())
+
+        with loop_dev(img) as dev:
+            self.assertEqual(
+                summarize_partitions(dev), [
+                    PartData(number=1, offset=1 << 20, size=end),
+                ])
+            with self.mount(dev, p1) as mnt_point:
+                with open(f'{mnt_point}/data.txt', 'r') as fp:
+                    self.assertEqual(expected, fp.read())
+            resized_fs_size = _get_filesystem_size(dev, p1)
+            self.assertEqual(end, resized_fs_size)
+
+    def test_resize_up(self):
+        self._do_test_resize(40, 80)
+
+    def test_resize_down(self):
+        self._do_test_resize(80, 40)
+
+    def test_sizes(self):
+        for size in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048):
+            size <<= 20
+            img = self.tmp_path('image.img')
+            config = StorageConfigBuilder(version=2)
+            config.add_image(path=img, size='2100M', ptable='gpt')
+            p1 = config.add_part(size=size, offset=1 << 20, number=1)
+            config.add_format(part=p1)
+            self.run_bm(config.render())
+
+            with loop_dev(img) as dev:
+                actual = _get_filesystem_size(dev, p1)
+                self.assertEqual(size, actual)
+
+    def test_resizes_up(self):
+        start = 1 << 20
+        for end in (2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048):
+            end <<= 20
+            img = self.tmp_path('image.img')
+            config = StorageConfigBuilder(version=2)
+            config.add_image(path=img, size='2100M', ptable='gpt')
+            p1 = config.add_part(size=start, offset=1 << 20, number=1)
+            config.add_format(part=p1)
+            self.run_bm(config.render())
+
+            with loop_dev(img) as dev:
+                self.assertEqual(
+                    summarize_partitions(dev), [
+                        PartData(number=1, offset=1 << 20, size=start),
+                    ])
+                actual = _get_filesystem_size(dev, p1)
+                self.assertEqual(start, actual, 'initial size')
+
+            config.set_preserve()
+            p1['resize'] = True
+            p1['size'] = end
+            self.run_bm(config.render())
+
+            with loop_dev(img) as dev:
+                self.assertEqual(
+                    summarize_partitions(dev), [
+                        PartData(number=1, offset=1 << 20, size=end),
+                    ])
+                actual = _get_filesystem_size(dev, p1)
+                self.assertEqual(end, actual, 'resized size')
+
+    def test_resizes_down(self):
+        start = 2048 << 20
+        for end in (128, 256, 512, 1024):
+            # resize2fs doesn't want to resize smaller than 128 for a 2048
+            # start
+            end <<= 20
+            img = self.tmp_path('image.img')
+            config = StorageConfigBuilder(version=2)
+            config.add_image(path=img, size='2100M', ptable='gpt')
+            p1 = config.add_part(size=start, offset=1 << 20, number=1)
+            config.add_format(part=p1)
+            self.run_bm(config.render())
+
+            with loop_dev(img) as dev:
+                self.assertEqual(
+                    summarize_partitions(dev), [
+                        PartData(number=1, offset=1 << 20, size=start),
+                    ])
+                actual = _get_filesystem_size(dev, p1)
+                self.assertEqual(start, actual, 'initial size')
+
+            config.set_preserve()
+            p1['resize'] = True
+            p1['size'] = end
+            self.run_bm(config.render())
+
+            with loop_dev(img) as dev:
+                self.assertEqual(
+                    summarize_partitions(dev), [
+                        PartData(number=1, offset=1 << 20, size=end),
+                    ])
+                actual = _get_filesystem_size(dev, p1)
+                self.assertEqual(end, actual, 'resized size')