cloud-init-dev team mailing list archive

[Conrad Hoffmann <ch@xxxxxxxxxxxxx>]

Conrad Hoffmann has proposed merging ~bitfehler/cloud-init:bitfehler/load_seed into cloud-init:master.

Requested reviews:
  cloud-init commiters (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~bitfehler/cloud-init/+git/cloud-init/+merge/369814
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~bitfehler/cloud-init:bitfehler/load_seed into cloud-init:master.

diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index f185dc7..3605076 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -110,12 +110,15 @@ class DataSourceCloudStack(sources.DataSource):
         return self.cfg
 
     def _get_data(self):
-        seed_ret = {}
-        if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
-            self.userdata_raw = seed_ret['user-data']
-            self.metadata = seed_ret['meta-data']
+        try:
+            # Using `required` wrapped in try because we want both or none
+            seed = util.load_seed(self.seed_dir, ['meta-data', 'user-data'])
+            self.userdata_raw = seed['user-data']
+            self.metadata = seed['meta-data']
             LOG.debug("Using seeded cloudstack data from: %s", self.seed_dir)
             return True
+        except Exception:
+            pass
         try:
             if not self.wait_for_metadata_service():
                 return False
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 8a9e5dd..da92875 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -165,13 +165,11 @@ class DataSourceNoCloud(sources.DataSource):
 
             # This could throw errors, but the user told us to do it
             # so if errors are raised, let them raise
-            (md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
+            seed = util.load_seed(seedfrom, timeout=None, **pp2d_kwargs)
             LOG.debug("Using seeded cache data from %s", seedfrom)
 
             # Values in the command line override those from the seed
-            mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
-                                                      md_seed])
-            mydata['user-data'] = ud
+            mydata = _merge_new_seed(mydata, seed)
             found.append(seedfrom)
 
         # Now that we have exhausted any other places merge in the defaults
@@ -357,8 +355,10 @@ def _merge_new_seed(cur, seeded):
     ret['meta-data'] = util.mergemanydict([cur['meta-data'], newmd])
 
     if seeded.get('network-config'):
-        ret['network-config'] = _maybe_remove_top_network(
-            util.load_yaml(seeded.get('network-config')))
+        newnc = seeded['network-config']
+        if not isinstance(newnc, dict):
+            newnc = util.load_yaml(seeded['network-config'])
+        ret['network-config'] = _maybe_remove_top_network(newnc)
 
     if 'user-data' in seeded:
         ret['user-data'] = seeded['user-data']
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 70e7a5c..dd9b994 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -262,10 +262,10 @@ class DataSourceOVF(sources.DataSource):
                           seedfrom, self)
                 return False
 
-            (md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
+            seed = util.load_seed(seedfrom, ['meta-data'], timeout=None)
             LOG.debug("Using seeded cache data from %s", seedfrom)
 
-            md = util.mergemanydict([md, md_seed])
+            md = util.mergemanydict([md, seed.get('meta-data', {})])
             found.append(seedfrom)
 
         # Now that we have exhausted any other places merge in the defaults
diff --git a/cloudinit/util.py b/cloudinit/util.py
index aa23b3f..43bb0e8 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -892,22 +892,6 @@ def runparts(dirp, skip_no_exist=True, exe_prefix=None):
                            % (len(failed), len(attempted)))
 
 
-# read_optional_seed
-# returns boolean indicating success or failure (presense of files)
-# if files are present, populates 'fill' dictionary with 'user-data' and
-# 'meta-data' entries
-def read_optional_seed(fill, base="", ext="", timeout=5):
-    try:
-        (md, ud) = read_seeded(base, ext, timeout)
-        fill['user-data'] = ud
-        fill['meta-data'] = md
-        return True
-    except url_helper.UrlError as e:
-        if e.code == url_helper.NOT_FOUND:
-            return False
-        raise
-
-
 def fetch_ssl_details(paths=None):
     ssl_details = {}
     # Lookup in these locations for ssl key/cert files
@@ -974,34 +958,31 @@ def load_yaml(blob, default=None, allowed=(dict,)):
     return loaded
 
 
-def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
+def load_seed(base='', required=None, optional=None, timeout=5):
+    result = {}
+    if required is None:
+        required = []
+    if optional is None:
+        optional = []
     if base.startswith("/"):
         base = "file://%s" % base
 
-    # default retries for file is 0. for network is 10
-    if base.startswith("file://"):
-        retries = file_retries
+    for item in required + optional:
+        try:
+            url = url_helper.combine_url(base, item)
+            resp = url_helper.read_file_or_url(url, timeout)
+            if resp.ok():
+                if item in ['meta-data', 'network-config']:
+                    result[item] = load_yaml(decode_binary(resp.contents))
+                else:
+                    result[item] = resp.contents
+        except url_helper.UrlError as e:
+            if item in optional and e.code == url_helper.NOT_FOUND:
+                pass
+            else:
+                raise e
 
-    if base.find("%s") >= 0:
-        ud_url = base % ("user-data" + ext)
-        md_url = base % ("meta-data" + ext)
-    else:
-        ud_url = "%s%s%s" % (base, "user-data", ext)
-        md_url = "%s%s%s" % (base, "meta-data", ext)
-
-    md_resp = url_helper.read_file_or_url(md_url, timeout, retries,
-                                          file_retries)
-    md = None
-    if md_resp.ok():
-        md = load_yaml(decode_binary(md_resp.contents), default={})
-
-    ud_resp = url_helper.read_file_or_url(ud_url, timeout, retries,
-                                          file_retries)
-    ud = None
-    if ud_resp.ok():
-        ud = ud_resp.contents
-
-    return (md, ud)
+    return result
 
 
 def read_conf_d(confd):
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 0e71db8..a31031a 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -724,21 +724,52 @@ class TestMessageFromString(helpers.TestCase):
         self.assertNotIn('\x00', roundtripped)
 
 
-class TestReadSeeded(helpers.TestCase):
+class TestLoadSeed(helpers.TestCase):
     def setUp(self):
-        super(TestReadSeeded, self).setUp()
+        super(TestLoadSeed, self).setUp()
         self.tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmp)
 
-    def test_unicode_not_messed_up(self):
+    def test_simple_seed(self):
         ud = b"userdatablob"
         helpers.populate_dir(
             self.tmp, {'meta-data': "key1: val1", 'user-data': ud})
-        sdir = self.tmp + os.path.sep
-        (found_md, found_ud) = util.read_seeded(sdir)
+        seed = util.load_seed(self.tmp, required=['meta-data', 'user-data'])
+        found_md = seed.get('meta-data', {})
+        found_ud = seed.get('user-data', '')
+
+        self.assertEqual(found_md, {'key1': 'val1'})
+        self.assertEqual(found_ud, ud)
+
+    def test_ignores_optional(self):
+        ud = b"userdatablob"
+        helpers.populate_dir(
+            self.tmp, {'meta-data': 'key1: val1', 'user-data': ud})
+        seed = util.load_seed(
+            self.tmp, ['meta-data', 'user-data'], ['network-config'])
+        found_md = seed.get('meta-data', {})
+        found_ud = seed.get('user-data', '')
+
+        self.assertEqual(found_md, {'key1': 'val1'})
+        self.assertEqual(found_ud, ud)
+
+    def test_loads_optional(self):
+        ud = b"userdatablob"
+        helpers.populate_dir(
+            self.tmp, {
+                'meta-data': 'key1: val1',
+                'user-data': ud,
+                'network-config': 'key: val'
+            })
+        seed = util.load_seed(
+            self.tmp, ['meta-data', 'user-data'], ['network-config'])
+        found_md = seed.get('meta-data', {})
+        found_ud = seed.get('user-data', '')
+        found_nc = seed.get('network-config', {})
 
         self.assertEqual(found_md, {'key1': 'val1'})
         self.assertEqual(found_ud, ud)
+        self.assertEqual(found_nc, {'key': 'val'})
 
 
 class TestSubp(helpers.CiTestCase):