cloud-init-dev team mailing list archive

[Joshua Harlow <harlowja@xxxxxxxxx>]

Joshua Harlow has proposed merging lp:~harlowja/cloud-init/fix-resets into lp:cloud-init.

Requested reviews:
  cloud init development team (cloud-init-dev)
Related bugs:
  Bug #1076811 in cloud-init: "Cloud-init modules do not reflect loaded config"
  https://bugs.launchpad.net/cloud-init/+bug/1076811

For more details, see:
https://code.launchpad.net/~harlowja/cloud-init/fix-resets/+merge/133603

Tried test on non-patched branch. It fails. On new fixed 'reset' branch it works.
-- 
https://code.launchpad.net/~harlowja/cloud-init/fix-resets/+merge/133603
Your team cloud init development team is requested to review the proposed merge of lp:~harlowja/cloud-init/fix-resets into lp:cloud-init.

=== modified file 'cloudinit/stages.py'
--- cloudinit/stages.py	2012-09-24 18:45:31 +0000
+++ cloudinit/stages.py	2012-11-09 00:35:23 +0000
@@ -36,6 +36,8 @@
 from cloudinit.handlers import shell_script as ss_part
 from cloudinit.handlers import upstart_job as up_part
 
+from cloudinit.sources import DataSourceNone
+
 from cloudinit import cloud
 from cloudinit import config
 from cloudinit import distros
@@ -58,8 +60,16 @@
         self._cfg = None
         self._paths = None
         self._distro = None
-        # Created only when a fetch occurs
-        self.datasource = None
+        # Changed only when a fetch occurs
+        self.datasource = DataSourceNone.DataSourceNone({}, None, None)
+
+    def _reset(self, ds=False):
+        # Recreated on access
+        self._cfg = None
+        self._paths = None
+        self._distro = None
+        if ds:
+            self.datasource = DataSourceNone.DataSourceNone({}, None, None)
 
     @property
     def distro(self):
@@ -236,7 +246,7 @@
         self.datasource = ds
         # Ensure we adjust our path members datasource
         # now that we have one (thus allowing ipath to be used)
-        self.paths.datasource = ds
+        self._reset()
         return ds
 
     def _get_instance_subdirs(self):
@@ -296,6 +306,10 @@
         util.write_file(iid_fn, "%s\n" % iid)
         util.write_file(os.path.join(dp, 'previous-instance-id'),
                         "%s\n" % (previous_iid))
+        # Ensure needed components are regenerated
+        # after change of instance which may cause
+        # change of configuration
+        self._reset()
         return iid
 
     def fetch(self):
@@ -409,6 +423,17 @@
             handlers.call_end(mod, data, frequency)
             called.append(mod)
 
+        # Perform post-consumption adjustments so that
+        # modules that run during the init stage reflect
+        # this consumed set.
+        #
+        # They will be recreated on future access...
+        self._reset()
+        # Note(harlowja): the 'active' datasource will have
+        # references to the previous config, distro, paths
+        # objects before the load of the userdata happened,
+        # this is expected.
+
 
 class Modules(object):
     def __init__(self, init, cfg_files=None):

=== added file 'tests/data/user_data.1.txt'
--- tests/data/user_data.1.txt	1970-01-01 00:00:00 +0000
+++ tests/data/user_data.1.txt	2012-11-09 00:35:23 +0000
@@ -0,0 +1,15 @@
+#cloud-config
+write_files:
+-   content: blah
+    path: /etc/blah.ini
+    permissions: 493
+
+system_info:
+ package_mirrors:
+   - arches: [i386, amd64, blah]
+     failsafe:
+      primary: http://my.archive.mydomain.com/ubuntu
+      security: http://my.security.mydomain.com/ubuntu
+     search:
+      primary: []
+      security: []

=== added file 'tests/unittests/test_merging.py'
--- tests/unittests/test_merging.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_merging.py	2012-11-09 00:35:23 +0000
@@ -0,0 +1,62 @@
+from mocker import MockerTestCase
+
+from cloudinit import util
+
+
+class TestMergeDict(MockerTestCase):
+    def test_simple_merge(self):
+        """Test simple non-conflict merge."""
+        source = {"key1": "value1"}
+        candidate = {"key2": "value2"}
+        result = util.mergedict(source, candidate)
+        self.assertEqual({"key1": "value1", "key2": "value2"}, result)
+
+    def test_nested_merge(self):
+        """Test nested merge."""
+        source = {"key1": {"key1.1": "value1.1"}}
+        candidate = {"key1": {"key1.2": "value1.2"}}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(
+            {"key1": {"key1.1": "value1.1", "key1.2": "value1.2"}}, result)
+
+    def test_merge_does_not_override(self):
+        """Test that candidate doesn't override source."""
+        source = {"key1": "value1", "key2": "value2"}
+        candidate = {"key1": "value2", "key2": "NEW VALUE"}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)
+
+    def test_empty_candidate(self):
+        """Test empty candidate doesn't change source."""
+        source = {"key": "value"}
+        candidate = {}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)
+
+    def test_empty_source(self):
+        """Test empty source is replaced by candidate."""
+        source = {}
+        candidate = {"key": "value"}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(candidate, result)
+
+    def test_non_dict_candidate(self):
+        """Test non-dict candidate is discarded."""
+        source = {"key": "value"}
+        candidate = "not a dict"
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)
+
+    def test_non_dict_source(self):
+        """Test non-dict source is not modified with a dict candidate."""
+        source = "not a dict"
+        candidate = {"key": "value"}
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)
+
+    def test_neither_dict(self):
+        """Test if neither candidate or source is dict source wins."""
+        source = "source"
+        candidate = "candidate"
+        result = util.mergedict(source, candidate)
+        self.assertEqual(source, result)

=== added file 'tests/unittests/test_runs/test_merge_run.py'
--- tests/unittests/test_runs/test_merge_run.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_runs/test_merge_run.py	2012-11-09 00:35:23 +0000
@@ -0,0 +1,52 @@
+import os
+
+from tests.unittests import helpers
+
+from cloudinit.settings import (PER_INSTANCE)
+from cloudinit import stages
+from cloudinit import util
+
+
+class TestSimpleRun(helpers.FilesystemMockingTestCase):
+    def _patchIn(self, root):
+        self.restore()
+        self.patchOS(root)
+        self.patchUtils(root)
+
+    def test_none_ds(self):
+        new_root = self.makeDir()
+        self.replicateTestRoot('simple_ubuntu', new_root)
+        cfg = {
+            'datasource_list': ['None'],
+            'cloud_init_modules': ['write-files'],
+        }
+        ud = self.readResource('user_data.1.txt')
+        cloud_cfg = util.yaml_dumps(cfg)
+        util.ensure_dir(os.path.join(new_root, 'etc', 'cloud'))
+        util.write_file(os.path.join(new_root, 'etc',
+                                     'cloud', 'cloud.cfg'), cloud_cfg)
+        self._patchIn(new_root)
+
+        # Now start verifying whats created
+        initer = stages.Init()
+        initer.read_cfg()
+        initer.initialize()
+        initer.fetch()
+        initer.datasource.userdata_raw = ud
+        iid = initer.instancify()
+        initer.update()
+        initer.cloudify().run('consume_userdata',
+                              initer.consume_userdata,
+                              args=[PER_INSTANCE],
+                              freq=PER_INSTANCE)
+        mirrors = initer.distro.get_option('package_mirrors')
+        self.assertEquals(1, len(mirrors))
+        mirror = mirrors[0]
+        self.assertEquals(mirror['arches'], ['i386', 'amd64', 'blah'])
+        mods = stages.Modules(initer)
+        (which_ran, failures) = mods.run_section('cloud_init_modules')
+        self.assertTrue(len(failures) == 0)
+        self.assertTrue(os.path.exists('/etc/blah.ini'))
+        self.assertIn('write-files', which_ran)
+        contents = util.load_file('/etc/blah.ini')
+        self.assertEquals(contents, 'blah')

=== modified file 'tests/unittests/test_util.py'
--- tests/unittests/test_util.py	2012-09-28 20:31:50 +0000
+++ tests/unittests/test_util.py	2012-11-09 00:35:23 +0000
@@ -28,65 +28,6 @@
         self.restored.append(path)
 
 
-class TestMergeDict(MockerTestCase):
-    def test_simple_merge(self):
-        """Test simple non-conflict merge."""
-        source = {"key1": "value1"}
-        candidate = {"key2": "value2"}
-        result = util.mergedict(source, candidate)
-        self.assertEqual({"key1": "value1", "key2": "value2"}, result)
-
-    def test_nested_merge(self):
-        """Test nested merge."""
-        source = {"key1": {"key1.1": "value1.1"}}
-        candidate = {"key1": {"key1.2": "value1.2"}}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(
-            {"key1": {"key1.1": "value1.1", "key1.2": "value1.2"}}, result)
-
-    def test_merge_does_not_override(self):
-        """Test that candidate doesn't override source."""
-        source = {"key1": "value1", "key2": "value2"}
-        candidate = {"key1": "value2", "key2": "NEW VALUE"}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-    def test_empty_candidate(self):
-        """Test empty candidate doesn't change source."""
-        source = {"key": "value"}
-        candidate = {}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-    def test_empty_source(self):
-        """Test empty source is replaced by candidate."""
-        source = {}
-        candidate = {"key": "value"}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(candidate, result)
-
-    def test_non_dict_candidate(self):
-        """Test non-dict candidate is discarded."""
-        source = {"key": "value"}
-        candidate = "not a dict"
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-    def test_non_dict_source(self):
-        """Test non-dict source is not modified with a dict candidate."""
-        source = "not a dict"
-        candidate = {"key": "value"}
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-    def test_neither_dict(self):
-        """Test if neither candidate or source is dict source wins."""
-        source = "source"
-        candidate = "candidate"
-        result = util.mergedict(source, candidate)
-        self.assertEqual(source, result)
-
-
 class TestGetCfgOptionListOrStr(TestCase):
     def test_not_found_no_default(self):
         """None is returned if key is not found and no default given."""

=== modified file 'tools/run-pep8'
--- tools/run-pep8	2012-10-23 16:58:32 +0000
+++ tools/run-pep8	2012-11-09 00:35:23 +0000
@@ -21,10 +21,21 @@
     base=`pwd`/tools/
 fi
 
+IGNORE="E501" # Line too long (these are caught by pylint)
+
+# King Arthur: Be quiet! ... Be Quiet! I Order You to Be Quiet.
+IGNORE="$IGNORE,E121" # Continuation line indentation is not a multiple of four
+IGNORE="$IGNORE,E123" # Closing bracket does not match indentation of opening bracket's line
+IGNORE="$IGNORE,E124" # Closing bracket missing visual indentation
+IGNORE="$IGNORE,E125" # Continuation line does not distinguish itself from next logical line
+IGNORE="$IGNORE,E126" # Continuation line over-indented for hanging indent
+IGNORE="$IGNORE,E127" # Continuation line over-indented for visual indent
+IGNORE="$IGNORE,E128" # Continuation line under-indented for visual indent
+
 cmd=(
     ${base}/hacking.py
 
-    --ignore=E501 # Line too long (these are caught by pylint)
+    --ignore="$IGNORE"
 
     "${files[@]}"
 )