curtin-dev team mailing list archive

[Olivier Gayot <mp+437494@xxxxxxxxxxxxxxxxxx>]

Olivier Gayot has proposed merging ~ogayot/curtin:curthooks-fix-apt-config-translation into curtin:master.

Commit message:
apt-config: fix curthooks unconditionally triggering apt-config

The following commit:

  1257a38f translate old curtin apt features to new format

introduced the ability to translate old top level keys related to apt
features (e.g., debconf_selections, apt-proxy, apt-mirrors) to the new
format where they are stored as children of 'apt'.

                              apt:
debconf_selections:    =>       debconf_selections:
  foobar                          foobar

Sadly, by doing so, it introduced a regression, making curthooks call
apt_config.handle_apt unconditionally.

The curthooks.do_apt_config function only calls apt_config.handle_apt if
the configuration does not contain the 'apt' key or if the 'apt' key has
the value None (i.e., null in YAML). When implementing the translation
from old to new configuration format, we accidentally forced the 'apt'
key to exist and coerced it to dictionary.

This effectively defeats the condition in curthooks.do_apt_config.

Fixed by making sure we only create the apt key if needed when doing the
translation.


Requested reviews:
  curtin developers (curtin-dev)

For more details, see:
https://code.launchpad.net/~ogayot/curtin/+git/curtin/+merge/437494

The conditional execution of apt_config.handle_apt from curthooks builtins has been broken for a while. A regression was introduced in 2016 when adding a mechanism to translate things like:

                                  apt:
    debconf_selections:    =>       debconf_selections:
      foobar                          foobar

The function responsible for calling handle_apt is as follows:

def do_apt_config(cfg, target):
    cfg = apt_config.translate_old_apt_features(cfg)
    apt_cfg = cfg.get("apt")
    if apt_cfg is not None:
        LOG.info("curthooks handling apt to target %s with config %s",
                 target, apt_cfg)
        apt_config.handle_apt(apt_cfg, target)
    else:
        LOG.info("No apt config provided, skipping")

Sadly, the implementation of translate_old_apt_features made is so that the `if apt_cfg is not None:` condition is always true.

This first patch fixes the issue by making sure the translate_old_apt_features function only creates an 'apt' key and assign it a dictionary, if necessary. This also covers the functions with unit tests.

The second patch also fixes an obvious mistake in a debug log.
-- 
Your team curtin developers is requested to review the proposed merge of ~ogayot/curtin:curthooks-fix-apt-config-translation into curtin:master.

diff --git a/curtin/commands/apt_config.py b/curtin/commands/apt_config.py
index f9cce34..d8956e2 100644
--- a/curtin/commands/apt_config.py
+++ b/curtin/commands/apt_config.py
@@ -6,6 +6,7 @@ Handle the setup of apt related tasks like proxies, mirrors, repositories.
 """
 
 import argparse
+import collections
 import glob
 import os
 import re
@@ -685,12 +686,10 @@ def apt_command(args):
     sys.exit(0)
 
 
-def translate_old_apt_features(cfg):
+def translate_old_apt_features(old_cfg):
     """translate the few old apt related features into the new config format"""
-    predef_apt_cfg = cfg.get("apt")
-    if predef_apt_cfg is None:
-        cfg['apt'] = {}
-        predef_apt_cfg = cfg.get("apt")
+    cfg = collections.defaultdict(dict, old_cfg)
+    predef_apt_cfg = cfg.get("apt", {})
 
     if cfg.get('apt_proxy') is not None:
         if predef_apt_cfg.get('proxy') is not None:
@@ -702,7 +701,7 @@ def translate_old_apt_features(cfg):
 
         cfg['apt']['proxy'] = cfg.get('apt_proxy')
         LOG.debug("Transferred %s into new format: %s", cfg.get('apt_proxy'),
-                  cfg.get('apte'))
+                  cfg.get('apt'))
         del cfg['apt_proxy']
 
     if cfg.get('apt_mirrors') is not None:
@@ -747,7 +746,7 @@ def translate_old_apt_features(cfg):
                  cfg.get('apt'))
         del cfg['debconf_selections']
 
-    return cfg
+    return dict(cfg)
 
 
 CMD_ARGUMENTS = (
diff --git a/tests/unittests/test_apt_source.py b/tests/unittests/test_apt_source.py
index 9fb3fc5..8609c61 100644
--- a/tests/unittests/test_apt_source.py
+++ b/tests/unittests/test_apt_source.py
@@ -678,6 +678,38 @@ Pin-Priority: -1
 
         mockobj.assert_called_with("preferencesfn", expected_content)
 
+    def test_translate_old_apt_features(self):
+        apt_config.translate_old_apt_features({})
+
+        self.assertEqual(
+                apt_config.translate_old_apt_features({
+                    "debconf_selections": "foo"}),
+                {"apt": {"debconf_selections": "foo"}}
+        )
+
+        self.assertEqual(
+                apt_config.translate_old_apt_features({
+                    "apt_proxy": {"http_proxy": "http://proxy:3128"}}),
+                {"apt": {
+                    "proxy": {"http_proxy": "http://proxy:3128"},
+                }}
+        )
+
+    def test_translate_old_apt_features_conflicts(self):
+        with self.assertRaisesRegex(ValueError, 'mutually exclusive'):
+            apt_config.translate_old_apt_features({
+                "debconf_selections": "foo",
+                "apt": {
+                    "debconf_selections": "bar",
+                }})
+
+        with self.assertRaisesRegex(ValueError, 'mutually exclusive'):
+            apt_config.translate_old_apt_features({
+                "apt_proxy": {"http_proxy": "http://proxy:3128"},
+                "apt": {
+                    "proxy": {"http_proxy": "http://proxy:3128"},
+                }})
+
     def test_mirror(self):
         """test_mirror - Test defining a mirror"""
         pmir = "http://us.archive.ubuntu.com/ubuntu/";
diff --git a/tests/unittests/test_curthooks.py b/tests/unittests/test_curthooks.py
index c13434c..9e4fa87 100644
--- a/tests/unittests/test_curthooks.py
+++ b/tests/unittests/test_curthooks.py
@@ -2268,4 +2268,37 @@ class TestUefiFindGrubDeviceIds(CiTestCase):
                              self._sconfig(cfg)))
 
 
+class TestDoAptConfig(CiTestCase):
+    def setUp(self):
+        super(TestDoAptConfig, self).setUp()
+        self.handle_apt_sym = 'curtin.commands.curthooks.apt_config.handle_apt'
+
+    def test_no_apt_config(self):
+        with patch(self.handle_apt_sym) as m_handle_apt:
+            curthooks.do_apt_config({}, target="/")
+        m_handle_apt.assert_not_called()
+
+    def test_apt_config_none(self):
+        with patch(self.handle_apt_sym) as m_handle_apt:
+            curthooks.do_apt_config({"apt": None}, target="/")
+        m_handle_apt.assert_not_called()
+
+    def test_apt_config_dict(self):
+        with patch(self.handle_apt_sym) as m_handle_apt:
+            curthooks.do_apt_config({"apt": {}}, target="/")
+        m_handle_apt.assert_called()
+
+    def test_with_apt_config(self):
+        with patch(self.handle_apt_sym) as m_handle_apt:
+            curthooks.do_apt_config(
+                    {"apt": {"proxy": {"http_proxy": "http://proxy:3128"}}},
+                    target="/")
+        m_handle_apt.assert_called_once()
+
+    def test_with_debconf_selections(self):
+        # debconf_selections are translated to apt config
+        with patch(self.handle_apt_sym) as m_handle_apt:
+            curthooks.do_apt_config({"debconf_selections": "foo"}, target="/")
+        m_handle_apt.assert_called_once()
+
 # vi: ts=4 expandtab syntax=python