curtin-dev team mailing list archive

[Olivier Gayot <mp+427671@xxxxxxxxxxxxxxxxxx>]

Olivier Gayot has proposed merging ~ogayot/curtin:install-step-by-step into curtin:master.

Commit message:
install: allow to perform an install in incremental invocations

Requested reviews:
  Michael Hudson-Doyle (mwhudson)

For more details, see:
https://code.launchpad.net/~ogayot/curtin/+git/curtin/+merge/427671

install: allow to perform an install in incremental invocations

In order to perform a more "step-by-step" install, we want the ability to run `curtin install` multiple times with different stage(s) specified.

We now support a mechanism to "resume" an installation, through two new configuration fields:

 * export_resume_data
 * import_resume_data

If more than one invocation of curtin install is planned, the export_resume_data field should be specified. curtin install will then create the file and fill it with internal data that will be needed to resume the installation later.

In subsequent invocations of curtin, the export_resume_data should be specified to load the internal data and resume the installation.
-- 
Your team curtin developers is subscribed to branch curtin:master.

diff --git a/curtin/commands/install.py b/curtin/commands/install.py
index 7d108d1..b029377 100644
--- a/curtin/commands/install.py
+++ b/curtin/commands/install.py
@@ -110,7 +110,39 @@ def writeline(fname, output):
 
 
 class WorkingDir(object):
-    def __init__(self, config):
+    @classmethod
+    def import_existing(cls, config):
+        """ Import the data from the directory specified and create the
+        WorkingDirectory instance. """
+        resume_data_path = config["install"]["import_resume_data"]
+        with open(resume_data_path, mode="r") as resume_data_file:
+            resume_data = json.load(resume_data_file)
+
+        target = resume_data["target"]
+
+        # When resuming, the name of the target directory should be retrieved
+        # from the exported data. If the user supplies a name explicitly in the
+        # config, make sure it matches.
+        if config["install"].get("target"):
+            if config["install"]["target"] != target:
+                raise ValueError(
+                    "Attempting to resume from a different target directory"
+                    " '%s' vs '%s'" % (config["install"]["target"], target))
+
+        if not os.path.exists(target):
+            raise ValueError(
+                "Attempting to resume an installation but the target directory"
+                " does not exist.")
+
+        with open(resume_data["config_file"], "w") as fp:
+            json.dump(config, fp)
+
+        return cls(**resume_data)
+
+    @classmethod
+    def create(cls, config):
+        """ Create the needed directories and create the associated
+        WorkingDirectory instance. """
         top_d = tempfile.mkdtemp()
         state_d = os.path.join(top_d, 'state')
         scratch_d = os.path.join(top_d, 'scratch')
@@ -144,15 +176,35 @@ class WorkingDir(object):
             with open(f, "ab") as fp:
                 pass
 
-        self.scratch = scratch_d
-        self.target = target_d
-        self.top = top_d
-        self.interfaces = interfaces_f
-        self.netconf = netconf_f
-        self.netstate = netstate_f
-        self.fstab = fstab_f
-        self.config = config
-        self.config_file = config_f
+        return cls(scratch=scratch_d,
+                   target=target_d,
+                   top=top_d,
+                   interfaces=interfaces_f,
+                   netconf=netconf_f,
+                   netstate=netstate_f,
+                   fstab=fstab_f,
+                   config_file=config_f)
+
+    @classmethod
+    def create_or_import(cls, config):
+        """ Create or import the WorkingDirectory instance based on the config
+        supplied. """
+        if config.get("install", {}).get("import_resume_data"):
+            return cls.import_existing(config)
+        else:
+            return cls.create(config)
+
+    def __init__(self, scratch, target, top, interfaces, netconf, netstate,
+                 fstab, config_file):
+        """ Initialize the directory from existing files and directories. """
+        self.target = target
+        self.top = top
+        self.scratch = scratch
+        self.interfaces = interfaces
+        self.netconf = netconf
+        self.netstate = netstate
+        self.fstab = fstab
+        self.config_file = config_file
 
     def env(self):
         return ({'WORKING_DIR': self.scratch, 'OUTPUT_FSTAB': self.fstab,
@@ -162,6 +214,11 @@ class WorkingDir(object):
                  'TARGET_MOUNT_POINT': self.target,
                  'CONFIG': self.config_file})
 
+    def export(self, path):
+        """ Export all attributes so that they can be loaded again later. """
+        with open(path, mode="w") as fh:
+            json.dump(self.__dict__, fh)
+
 
 class Stage(object):
 
@@ -440,7 +497,13 @@ def cmd_install(args):
     args.reportstack.post_files = post_files
     workingd = None
     try:
-        workingd = WorkingDir(cfg)
+        workingd = WorkingDir.create_or_import(cfg)
+        # Export the resume data if requested
+        export_path = cfg.get("install", {}).get("export_resume_data")
+        if export_path:
+            LOG.debug("Exporting resume data to %s so that further stages"
+                      " can be executed in a later invocation.", export_path)
+            workingd.export(export_path)
         dd_images = util.get_dd_images(cfg.get('sources', {}))
         if len(dd_images) > 1:
             raise ValueError("You may not use more than one disk image")
diff --git a/doc/topics/config.rst b/doc/topics/config.rst
index ff766c7..9e3eab5 100644
--- a/doc/topics/config.rst
+++ b/doc/topics/config.rst
@@ -363,6 +363,17 @@ If this key is set to the string 'disabled' then curtin will not
 unmount the target filesystem when install is complete.  This
 skips unmounting in all cases of install success or failure.
 
+**export_resume_data**: *<path to where to export the data needed to resume>*
+
+When specified, curtin will export in the file specified the data required to
+resume the installation and run further stages.
+
+**import_resume_data**: *<path from where to import the data needed to resume>*
+
+When specified, curtin will consider that the installation has already been
+initiated and will load the data from the file specified. The target directory
+will not be expected to be empty.
+
 **Example**::
 
   install:
diff --git a/tests/unittests/test_commands_install.py b/tests/unittests/test_commands_install.py
index 9a64a79..0e22ad3 100644
--- a/tests/unittests/test_commands_install.py
+++ b/tests/unittests/test_commands_install.py
@@ -1,7 +1,9 @@
 # This file is part of curtin. See LICENSE file for copyright and license info.
 
 import copy
+import json
 import mock
+import os
 
 from curtin import config
 from curtin.commands import install
@@ -201,7 +203,8 @@ class TestWorkingDir(CiTestCase):
         ensure_dir(target_d)
         with mock.patch("curtin.commands.install.tempfile.mkdtemp",
                         return_value=work_d) as m_mkdtemp:
-            workingdir = install.WorkingDir({'install': {'target': target_d}})
+            workingdir = install.WorkingDir.create(
+                    {'install': {'target': target_d}})
         self.assertEqual(1, m_mkdtemp.call_count)
         self.assertEqual(target_d, workingdir.target)
         self.assertEqual(target_d, workingdir.env().get('TARGET_MOUNT_POINT'))
@@ -217,7 +220,7 @@ class TestWorkingDir(CiTestCase):
         with mock.patch("curtin.commands.install.tempfile.mkdtemp",
                         return_value=work_d):
             with self.assertRaises(ValueError):
-                install.WorkingDir({'install': {'target': target_d}})
+                install.WorkingDir.create({'install': {'target': target_d}})
 
     def test_target_dir_by_default_is_under_workd(self):
         """WorkingDir does not require target in config."""
@@ -226,6 +229,48 @@ class TestWorkingDir(CiTestCase):
         ensure_dir(work_d)
         with mock.patch("curtin.commands.install.tempfile.mkdtemp",
                         return_value=work_d) as m_mkdtemp:
-            wd = install.WorkingDir({})
+            wd = install.WorkingDir.create({})
         self.assertEqual(1, m_mkdtemp.call_count)
         self.assertTrue(wd.target.startswith(work_d + "/"))
+
+    def test_import_target_dir_exists(self):
+        tmp_d = self.tmp_dir()
+        target_d = self.tmp_path("target_d", tmp_d)
+        ensure_dir(target_d)
+        resume_data_path = os.path.join(tmp_d, "resume_data")
+        with open(resume_data_path, mode="w") as fh:
+            json.dump({"target": target_d,
+                       "top": "/dev/zero",
+                       "scratch": "/dev/zero",
+                       "interfaces": "/dev/zero",
+                       "netconf": "/dev/zero",
+                       "netstate": "/dev/zero",
+                       "fstab": "/dev/zero",
+                       "config_file": "/dev/zero"}, fh)
+
+        work_d = self.tmp_path("work_d", tmp_d)
+        ensure_dir(work_d)
+        with mock.patch("curtin.commands.install.tempfile.mkdtemp",
+                        return_value=work_d) as m_mkdtemp:
+            install.WorkingDir.import_existing(
+                    {"install": {"import_resume_data": resume_data_path}})
+
+    def test_import_target_dir_does_not_exist(self):
+        tmp_d = self.tmp_dir()
+        resume_data_path = os.path.join(tmp_d, "resume_data")
+        with open(resume_data_path, mode="w") as fh:
+            json.dump({"target": "/inexistent",
+                       "top": "/dev/zero",
+                       "scratch": "/dev/zero",
+                       "interfaces": "/dev/zero",
+                       "netconf": "/dev/zero",
+                       "netstate": "/dev/zero",
+                       "fstab": "/dev/zero",
+                       "config_file": "/dev/zero"}, fh)
+        work_d = self.tmp_path("work_d", tmp_d)
+        ensure_dir(work_d)
+        with mock.patch("curtin.commands.install.tempfile.mkdtemp",
+                        return_value=work_d) as m_mkdtemp:
+            with self.assertRaises(ValueError):
+                install.WorkingDir.import_existing(
+                        {"install": {"import_resume_data": resume_data_path}})