cloud-init-dev team mailing list archive

[Chad Smith <chad.smith@xxxxxxxxxxxxx>]

Chad Smith has proposed merging ~chad.smith/cloud-init:bug/1840080-ubuntu-drivers-emit-latelink-v2 into cloud-init:master.

Commit message:
ubuntu-drivers: call db_x_loadtemplatefile to accept NVIDIA EULA

Emit a script allowing cloud-init to set linux/nvidia/latelink
debconf selection to true. This avoids having to call
debconf-set-selections and allows cloud-init to pre-confgure
linux-restricted-modules to link NVIDIA drivers to the running kernel.

Cloud-init loads this debconf template and sets the value to true in the
debconf database by sourcing debconf's /usr/share/debconf/confmodule and
uses db_x_loadtemplatefile to register cloud-init's setting for
linux/nvidia/latelink.

LP: #1840080

Requested reviews:
  cloud-init commiters (cloud-init-dev)
Related bugs:
  Bug #1840080 in cloud-init (Ubuntu): "cloud-init cc_ubuntu_drivers does not set up /etc/default/linux-modules-nvidia"
  https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1840080

For more details, see:
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/371546
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:bug/1840080-ubuntu-drivers-emit-latelink-v2 into cloud-init:master.

diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py
index 4da34ee..44a2bfe 100644
--- a/cloudinit/config/cc_ubuntu_drivers.py
+++ b/cloudinit/config/cc_ubuntu_drivers.py
@@ -4,11 +4,11 @@
 
 from textwrap import dedent
 
-from cloudinit.config import cc_apt_configure
 from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import temp_utils
 from cloudinit import type_utils
 from cloudinit import util
 
@@ -65,6 +65,39 @@ OLD_UBUNTU_DRIVERS_STDERR_NEEDLE = (
 __doc__ = get_schema_doc(schema)  # Supplement python help()
 
 
+# debconf template to allow cloud-init pre-configure the global debconf
+# variable linux/nvidia/latelink to true, allowing linux-restricted-modules
+# to accept the NVIDIA EULA and automatically link drivers to the running
+# kernel.
+# EOL_XENIAL: can then drop this script and use python3-debconf which is only
+# available in Bionic and later. Can't use python3-debconf currently as it
+# isn't in Xenial and doesn't yet support X_LOADTEMPLATEFILE debconf command.
+
+NVIDIA_DRIVER_LATELINK_DEBCONF_TMPL = """\
+#/bin/sh
+# Allow cloud-init to trigger EULA acceptance via registering a debconf
+# template to set linux/nvidia/latelink true
+
+. /usr/share/debconf/confmodule
+
+SCRIPT=$(readlink -f "$0")
+DIRNAME=$(dirname "$SCRIPT")
+tmpfile=$(mktemp -p ${DIRNAME} -t "cloud-init-ubuntu-drivers-XXXXXX.template")
+cat > "$tmpfile" << EOF
+Template: linux/nvidia/latelink
+Type: boolean
+Default: true
+Description: Late-link NVIDIA kernel modules?
+ Enable this to link the NVIDIA kernel modules in cloud-init and
+ make them available for use.
+EOF
+echo BEFORE $tmpfile
+db_x_loadtemplatefile "$tmpfile" cloud-init
+echo AFTER $tmpfile
+rm "$tmpfile"
+"""
+
+
 def install_drivers(cfg, pkg_install_func):
     if not isinstance(cfg, dict):
         raise TypeError(
@@ -90,17 +123,22 @@ def install_drivers(cfg, pkg_install_func):
     if version_cfg:
         driver_arg += ':{}'.format(version_cfg)
 
-    LOG.debug("Installing NVIDIA drivers (%s=%s, version=%s)",
+    LOG.debug("Installing and activating NVIDIA drivers (%s=%s, version=%s)",
               cfgpath, nv_acc, version_cfg if version_cfg else 'latest')
 
-    # Setting NVIDIA latelink confirms acceptance of EULA for the package
-    # linux-restricted-modules
-    # Reference code defining debconf variable is here
-    # https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/
-    # linux-restricted-modules/+git/eoan/tree/debian/templates/
-    # nvidia.templates.in
-    selections = b'linux-restricted-modules linux/nvidia/latelink boolean true'
-    cc_apt_configure.debconf_set_selections(selections)
+    # Register and set debconf selection linux/nvidia/latelink = true
+    with temp_utils.ExtendedTemporaryFile(
+            suffix=".sh", needs_exe=True) as tmpf:
+        try:
+            tmpf.write(util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_TMPL))
+            tmpf.flush()
+            util.chmod(tmpf.name, 0o755)
+            util.subp([tmpf.name])
+        except Exception as e:
+            util.logexc(
+                LOG,
+                "Failed to register NVIDIA debconf template: %s", str(e))
+            raise
 
     try:
         util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg])
diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py
index 6a763bd..ca44d63 100644
--- a/cloudinit/config/tests/test_ubuntu_drivers.py
+++ b/cloudinit/config/tests/test_ubuntu_drivers.py
@@ -9,11 +9,20 @@ from cloudinit.config import cc_ubuntu_drivers as drivers
 from cloudinit.util import ProcessExecutionError
 
 MPATH = "cloudinit.config.cc_ubuntu_drivers."
+M_TMP_PATH = MPATH + "temp_utils._tempfile_dir_arg"
 OLD_UBUNTU_DRIVERS_ERROR_STDERR = (
     "ubuntu-drivers: error: argument <command>: invalid choice: 'install' "
     "(choose from 'list', 'autoinstall', 'devices', 'debug')\n")
 
 
+class AnySingleTempScriptCmdInDir(str):
+    def __eq__(self, cmd):
+        if not len(cmd) == 1:
+            return False
+        cmd = cmd[0]
+        return bool(cmd.startswith(self) and cmd.endswith('.sh'))
+
+
 class TestUbuntuDrivers(CiTestCase):
     cfg_accepted = {'drivers': {'nvidia': {'license-accepted': True}}}
     install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia']
@@ -28,20 +37,22 @@ class TestUbuntuDrivers(CiTestCase):
                 {'drivers': {'nvidia': {'license-accepted': "TRUE"}}},
                 schema=drivers.schema, strict=True)
 
+    @mock.patch(M_TMP_PATH)
     @mock.patch(MPATH + "cc_apt_configure.debconf_set_selections")
     @mock.patch(MPATH + "util.subp", return_value=('', ''))
     @mock.patch(MPATH + "util.which", return_value=False)
     def _assert_happy_path_taken(
-            self, config, m_which, m_subp, m_debconf_set_selections):
+            self, config, m_which, m_subp, m_debconf_set_selections, m_tmp):
         """Positive path test through handle. Package should be installed."""
+        tdir = self.tmp_dir()
+        m_tmp.return_value = tdir
         myCloud = mock.MagicMock()
         drivers.handle('ubuntu_drivers', config, myCloud, None, None)
         self.assertEqual([mock.call(['ubuntu-drivers-common'])],
                          myCloud.distro.install_packages.call_args_list)
-        self.assertEqual([mock.call(self.install_gpgpu)],
+        self.assertEqual([mock.call(AnySingleTempScriptCmdInDir(tdir)),
+                          mock.call(self.install_gpgpu)],
                          m_subp.call_args_list)
-        m_debconf_set_selections.assert_called_with(
-            b'linux-restricted-modules linux/nvidia/latelink boolean true')
 
     def test_handle_does_package_install(self):
         self._assert_happy_path_taken(self.cfg_accepted)
@@ -52,19 +63,30 @@ class TestUbuntuDrivers(CiTestCase):
             new_config['drivers']['nvidia']['license-accepted'] = true_value
             self._assert_happy_path_taken(new_config)
 
-    @mock.patch(MPATH + "cc_apt_configure.debconf_set_selections")
-    @mock.patch(MPATH + "util.subp", side_effect=ProcessExecutionError(
-        stdout='No drivers found for installation.\n', exit_code=1))
+    @mock.patch(M_TMP_PATH)
+    @mock.patch(MPATH + "util.subp")
     @mock.patch(MPATH + "util.which", return_value=False)
-    def test_handle_raises_error_if_no_drivers_found(self, m_which, m_subp, _):
+    def test_handle_raises_error_if_no_drivers_found(
+            self, m_which, m_subp, m_tmp):
         """If ubuntu-drivers doesn't install any drivers, raise an error."""
+        tdir = self.tmp_dir()
+        m_tmp.return_value = tdir
         myCloud = mock.MagicMock()
+
+        def fake_subp(cmd):
+            if cmd[0].startswith(tdir):
+                return
+            raise ProcessExecutionError(
+                stdout='No drivers found for installation.\n', exit_code=1)
+        m_subp.side_effect = fake_subp
+
         with self.assertRaises(Exception):
             drivers.handle(
                 'ubuntu_drivers', self.cfg_accepted, myCloud, None, None)
         self.assertEqual([mock.call(['ubuntu-drivers-common'])],
                          myCloud.distro.install_packages.call_args_list)
-        self.assertEqual([mock.call(self.install_gpgpu)],
+        self.assertEqual([mock.call(AnySingleTempScriptCmdInDir(tdir)),
+                          mock.call(self.install_gpgpu)],
                          m_subp.call_args_list)
         self.assertIn('ubuntu-drivers found no drivers for installation',
                       self.logs.getvalue())
@@ -113,18 +135,22 @@ class TestUbuntuDrivers(CiTestCase):
                       myLog.debug.call_args_list[0][0][0])
         self.assertEqual(0, m_install_drivers.call_count)
 
-    @mock.patch(MPATH + "cc_apt_configure.debconf_set_selections")
+    @mock.patch(M_TMP_PATH)
     @mock.patch(MPATH + "util.subp", return_value=('', ''))
     @mock.patch(MPATH + "util.which", return_value=True)
-    def test_install_drivers_no_install_if_present(self, m_which, m_subp, _):
+    def test_install_drivers_no_install_if_present(
+            self, m_which, m_subp, m_tmp):
         """If 'ubuntu-drivers' is present, no package install should occur."""
+        tdir = self.tmp_dir()
+        m_tmp.return_value = tdir
         pkg_install = mock.MagicMock()
         drivers.install_drivers(self.cfg_accepted['drivers'],
                                 pkg_install_func=pkg_install)
         self.assertEqual(0, pkg_install.call_count)
         self.assertEqual([mock.call('ubuntu-drivers')],
                          m_which.call_args_list)
-        self.assertEqual([mock.call(self.install_gpgpu)],
+        self.assertEqual([mock.call(AnySingleTempScriptCmdInDir(tdir)),
+                          mock.call(self.install_gpgpu)],
                          m_subp.call_args_list)
 
     def test_install_drivers_rejects_invalid_config(self):
@@ -134,20 +160,30 @@ class TestUbuntuDrivers(CiTestCase):
             drivers.install_drivers("mystring", pkg_install_func=pkg_install)
         self.assertEqual(0, pkg_install.call_count)
 
-    @mock.patch(MPATH + "cc_apt_configure.debconf_set_selections")
-    @mock.patch(MPATH + "util.subp", side_effect=ProcessExecutionError(
-        stderr=OLD_UBUNTU_DRIVERS_ERROR_STDERR, exit_code=2))
+    @mock.patch(M_TMP_PATH)
+    @mock.patch(MPATH + "util.subp")
     @mock.patch(MPATH + "util.which", return_value=False)
     def test_install_drivers_handles_old_ubuntu_drivers_gracefully(
-            self, m_which, m_subp, _):
+            self, m_which, m_subp, m_tmp):
         """Older ubuntu-drivers versions should emit message and raise error"""
+        tdir = self.tmp_dir()
+        m_tmp.return_value = tdir
         myCloud = mock.MagicMock()
+
+        def fake_subp(cmd):
+            if cmd[0].startswith(tdir):
+                return
+            raise ProcessExecutionError(
+                stderr=OLD_UBUNTU_DRIVERS_ERROR_STDERR, exit_code=2)
+        m_subp.side_effect = fake_subp
+
         with self.assertRaises(Exception):
             drivers.handle(
                 'ubuntu_drivers', self.cfg_accepted, myCloud, None, None)
         self.assertEqual([mock.call(['ubuntu-drivers-common'])],
                          myCloud.distro.install_packages.call_args_list)
-        self.assertEqual([mock.call(self.install_gpgpu)],
+        self.assertEqual([mock.call(AnySingleTempScriptCmdInDir(tdir)),
+                          mock.call(self.install_gpgpu)],
                          m_subp.call_args_list)
         self.assertIn('WARNING: the available version of ubuntu-drivers is'
                       ' too old to perform requested driver installation',
@@ -160,17 +196,20 @@ class TestUbuntuDriversWithVersion(TestUbuntuDrivers):
         'drivers': {'nvidia': {'license-accepted': True, 'version': '123'}}}
     install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia:123']
 
-    @mock.patch(MPATH + "cc_apt_configure.debconf_set_selections")
+    @mock.patch(M_TMP_PATH)
     @mock.patch(MPATH + "util.subp", return_value=('', ''))
     @mock.patch(MPATH + "util.which", return_value=False)
-    def test_version_none_uses_latest(self, m_which, m_subp, _):
+    def test_version_none_uses_latest(self, m_which, m_subp, m_tmp):
+        tdir = self.tmp_dir()
+        m_tmp.return_value = tdir
         myCloud = mock.MagicMock()
         version_none_cfg = {
             'drivers': {'nvidia': {'license-accepted': True, 'version': None}}}
         drivers.handle(
             'ubuntu_drivers', version_none_cfg, myCloud, None, None)
         self.assertEqual(
-            [mock.call(['ubuntu-drivers', 'install', '--gpgpu', 'nvidia'])],
+            [mock.call(AnySingleTempScriptCmdInDir(tdir)),
+             mock.call(['ubuntu-drivers', 'install', '--gpgpu', 'nvidia'])],
             m_subp.call_args_list)
 
     def test_specifying_a_version_doesnt_override_license_acceptance(self):