cloud-init-dev team mailing list archive

[Dan Watkins <daniel.watkins@xxxxxxxxxxxxx>]

Dan Watkins has proposed merging ~daniel-thewatkins/cloud-init/+git/cloud-init:feature/driver-enablement into cloud-init:master.

Commit message:
Add ubuntu_drivers config module

The ubuntu_drivers config module enables usage of the 'ubuntu-drivers'
command. At this point it only serves as a way of installing NVIDIA
drivers for general purpose graphics processing unit (GPGPU)
functionality.

Also, a small usability improvement to get_cfg_by_path to allow it to
take a string for the key path
  "toplevel/second/mykey"
in addition to the original:
  ("toplevel", "second", "mykey")

Requested reviews:
  cloud-init commiters (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~daniel-thewatkins/cloud-init/+git/cloud-init/+merge/363992
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~daniel-thewatkins/cloud-init/+git/cloud-init:feature/driver-enablement into cloud-init:master.

diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py
new file mode 100644
index 0000000..08e124f
--- /dev/null
+++ b/cloudinit/config/cc_ubuntu_drivers.py
@@ -0,0 +1,87 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Ubuntu Drivers: Interact with third party drivers in Ubuntu."""
+
+from textwrap import dedent
+
+from cloudinit.config.schema import (
+    get_schema_doc, validate_cloudconfig_schema)
+from cloudinit import log as logging
+from cloudinit.settings import PER_INSTANCE
+from cloudinit import type_utils
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+frequency = PER_INSTANCE
+distros = ['ubuntu']
+schema = {
+    'id': 'cc_ubuntu_drivers',
+    'name': 'Ubuntu Drivers',
+    'title': 'Interact with third party drivers in Ubuntu.',
+    'description': dedent("""\
+        This module interacts with 'ubuntu-drivers' to install third party
+        driver packages."""),
+    'distros': distros,
+    'examples': [dedent("""\
+        drivers:
+          nvidia:
+            license-accepted: true
+        """)],
+    'frequency': frequency,
+    'type': 'object',
+    'properties': {
+        'drivers': {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'nvidia': {
+                    'type': 'object',
+                    'additionalProperties': False,
+                    'required': ['license-accepted'],
+                    'properties': {
+                        'license-accepted': {
+                            'type': 'boolean',
+                            'description': "Is the license agreed to?",
+                        },
+                    },
+                },
+            },
+        },
+    },
+}
+
+__doc__ = get_schema_doc(schema)  # Supplement python help()
+
+
+def install_drivers(cfg, pkg_install_func):
+    if not isinstance(cfg, dict):
+        raise TypeError(
+            "'drivers' config expected dict, found '%s': %s" %
+            (type_utils.obj_name(cfg), cfg))
+
+    cfgpath = 'nvidia/license-accepted'
+    nv_acc = util.get_cfg_by_path(cfg, cfgpath)
+    # We want to explicitly only accept "true" values here, not any True-ish
+    if nv_acc is not True:
+        LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc)
+        return
+
+    if not util.which('ubuntu-drivers'):
+        LOG.debug("'ubuntu-drivers' command not available.  "
+                  "Installing ubuntu-drivers-common")
+        pkg_install_func(['ubuntu-drivers-common'])
+
+    LOG.debug("Installing NVIDIA drivers. %s=%s", cfgpath, nv_acc)
+    out, _ = util.subp(['ubuntu-drivers', 'install', '--gpgpu', 'nvidia'])
+    if 'No drivers found for installation.' in out:
+        LOG.info('ubuntu-drivers found no drivers for installation')
+
+
+def handle(name, cfg, cloud, log, _args):
+    if "drivers" not in cfg:
+        log.debug("Skipping module named %s, no 'drivers' key in config", name)
+        return
+
+    validate_cloudconfig_schema(cfg, schema)
+    install_drivers(cfg['drivers'], cloud.distro.install_packages)
diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py
new file mode 100644
index 0000000..2623e80
--- /dev/null
+++ b/cloudinit/config/tests/test_ubuntu_drivers.py
@@ -0,0 +1,110 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.tests.helpers import CiTestCase, skipUnlessJsonSchema, mock
+from cloudinit.config.schema import (
+    SchemaValidationError, validate_cloudconfig_schema)
+from cloudinit.config import cc_ubuntu_drivers as drivers
+
+MPATH = "cloudinit.config.cc_ubuntu_drivers."
+
+
+class TestUbuntuDrivers(CiTestCase):
+    cfg_accepted = {'drivers': {'nvidia': {'license-accepted': True}}}
+    install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia']
+
+    with_logs = True
+
+    @skipUnlessJsonSchema()
+    def test_schema_requires_boolean_for_license_accepted(self):
+        with self.assertRaisesRegex(
+                SchemaValidationError, ".*license-accepted.*TRUE.*boolean"):
+            validate_cloudconfig_schema(
+                {'drivers': {'nvidia': {'license-accepted': "TRUE"}}},
+                schema=drivers.schema, strict=True)
+
+    @mock.patch(MPATH + "util.subp", return_value=('', ''))
+    @mock.patch(MPATH + "util.which", return_value=False)
+    def test_handle_does_package_install(self, m_which, m_subp):
+        """Positive path test through handle. Package should be installed."""
+        myCloud = mock.MagicMock()
+        drivers.handle(
+            'ubuntu_drivers', self.cfg_accepted, myCloud, None, None)
+        self.assertEqual([mock.call(['ubuntu-drivers-common'])],
+                         myCloud.distro.install_packages.call_args_list)
+        self.assertEqual([mock.call(self.install_gpgpu)],
+                         m_subp.call_args_list)
+
+    @mock.patch(MPATH + "util.subp", return_value=('', ''))
+    @mock.patch(MPATH + "util.which", return_value=False)
+    def test_handle_emits_msg_if_no_drivers_found(self, m_which, m_subp):
+        """If ubuntu-drivers doesn't install any drivers, emit a warning."""
+        m_subp.return_value = ('No drivers found for installation.\n', '')
+        myCloud = mock.MagicMock()
+        drivers.handle(
+            'ubuntu_drivers', self.cfg_accepted, myCloud, None, None)
+        self.assertEqual([mock.call(['ubuntu-drivers-common'])],
+                         myCloud.distro.install_packages.call_args_list)
+        self.assertEqual([mock.call(self.install_gpgpu)],
+                         m_subp.call_args_list)
+        self.assertIn('ubuntu-drivers found no drivers for installation',
+                      self.logs.getvalue())
+
+    @mock.patch(MPATH + "util.subp", return_value=('', ''))
+    @mock.patch(MPATH + "util.which", return_value=False)
+    def _assert_inert_with_config(self, config, m_which, m_subp):
+        """Helper to reduce repetition when testing negative cases"""
+        myCloud = mock.MagicMock()
+        drivers.handle('ubuntu_drivers', config, myCloud, None, None)
+        self.assertEqual(0, myCloud.distro.install_packages.call_count)
+        self.assertEqual(0, m_subp.call_count)
+
+    def test_handle_inert_if_license_not_accepted(self):
+        """Ensure we don't do anything if the license is rejected."""
+        self._assert_inert_with_config(
+            {'drivers': {'nvidia': {'license-accepted': False}}})
+
+    def test_handle_inert_if_garbage_in_license_field(self):
+        """Ensure we don't do anything if the license is rejected."""
+        self._assert_inert_with_config(
+            {'drivers': {'nvidia': {'license-accepted': 'garbage'}}})
+
+    def test_handle_inert_if_no_license_key(self):
+        """Ensure we don't do anything if no license key."""
+        self._assert_inert_with_config({'drivers': {'nvidia': {}}})
+
+    def test_handle_inert_if_no_nvidia_key(self):
+        """Ensure we don't do anything if other license accepted."""
+        self._assert_inert_with_config(
+            {'drivers': {'acme': {'license-accepted': True}}})
+
+    @mock.patch(MPATH + "install_drivers")
+    def test_handle_no_drivers_does_nothing(self, m_install_drivers):
+        """If no 'drivers' key in the config, nothing should be done."""
+        myCloud = mock.MagicMock()
+        myLog = mock.MagicMock()
+        drivers.handle('ubuntu_drivers', {'foo': 'bzr'}, myCloud, myLog, None)
+        self.assertIn('Skipping module named',
+                      myLog.debug.call_args_list[0][0][0])
+        self.assertEqual(0, m_install_drivers.call_count)
+
+    @mock.patch(MPATH + "util.subp", return_value=('', ''))
+    @mock.patch(MPATH + "util.which", return_value=True)
+    def test_install_drivers_no_install_if_present(self, m_which, m_subp):
+        """If 'ubuntu-drivers' is present, no package install should occur."""
+        pkg_install = mock.MagicMock()
+        drivers.install_drivers(self.cfg_accepted['drivers'],
+                                pkg_install_func=pkg_install)
+        self.assertEqual(0, pkg_install.call_count)
+        self.assertEqual([mock.call('ubuntu-drivers')],
+                         m_which.call_args_list)
+        self.assertEqual([mock.call(self.install_gpgpu)],
+                         m_subp.call_args_list)
+
+    def test_install_drivers_raises_type_error(self):
+        pkg_install = mock.MagicMock()
+        with self.assertRaisesRegex(TypeError, ".*expected dict.*"):
+            drivers.install_drivers("mystring", pkg_install_func=pkg_install)
+        self.assertEqual(0, pkg_install.call_count)
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/util.py b/cloudinit/util.py
index a192091..385f231 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -703,6 +703,21 @@ def get_cfg_option_list(yobj, key, default=None):
 # get a cfg entry by its path array
 # for f['a']['b']: get_cfg_by_path(mycfg,('a','b'))
 def get_cfg_by_path(yobj, keyp, default=None):
+    """Return the value of the item at path C{keyp} in C{yobj}.
+
+    example:
+      get_cfg_by_path({'a': {'b': {'num': 4}}}, 'a/b/num') == 4
+      get_cfg_by_path({'a': {'b': {'num': 4}}}, 'c/d') == None
+
+    @param yobj: A dictionary.
+    @param keyp: A path inside yobj.  it can be a '/' delimited string,
+                 or an iterable.
+    @param default: The default to return if the path does not exist.
+    @return: The value of the item at keyp."
+        is not found."""
+
+    if isinstance(keyp, six.string_types):
+        keyp = keyp.split("/")
     cur = yobj
     for tok in keyp:
         if tok not in cur:
diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl
index 7513176..25db43e 100644
--- a/config/cloud.cfg.tmpl
+++ b/config/cloud.cfg.tmpl
@@ -112,6 +112,9 @@ cloud_final_modules:
  - landscape
  - lxd
 {% endif %}
+{% if variant in ["ubuntu", "unknown"] %}
+ - ubuntu-drivers
+{% endif %}
 {% if variant not in ["freebsd"] %}
  - puppet
  - chef
diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst
index d9720f6..3dcdd3b 100644
--- a/doc/rtd/topics/modules.rst
+++ b/doc/rtd/topics/modules.rst
@@ -54,6 +54,7 @@ Modules
 .. automodule:: cloudinit.config.cc_ssh_import_id
 .. automodule:: cloudinit.config.cc_timezone
 .. automodule:: cloudinit.config.cc_ubuntu_advantage
+.. automodule:: cloudinit.config.cc_ubuntu_drivers
 .. automodule:: cloudinit.config.cc_update_etc_hosts
 .. automodule:: cloudinit.config.cc_update_hostname
 .. automodule:: cloudinit.config.cc_users_groups
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 1bad07f..e69a47a 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -28,6 +28,7 @@ class GetSchemaTest(CiTestCase):
                 'cc_runcmd',
                 'cc_snap',
                 'cc_ubuntu_advantage',
+                'cc_ubuntu_drivers',
                 'cc_zypper_add_repo'
             ],
             [subschema['id'] for subschema in schema['allOf']])