cloud-init-dev team mailing list archive

[Chad Smith <chad.smith@xxxxxxxxxxxxx>]

Chad Smith has proposed merging ~chad.smith/cloud-init:bug/ec2-tests-unmocked-metadata into cloud-init:master.

Requested reviews:
  cloud-init commiters (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/330040

ec2 tests: Stop leaking calls through unmocked metadata addresses

DataSourceEc2 behavior changed to first check a minimum acceptable
metadata version uri http://169.154.169.254/<min_version>/instance-id,
retrying on 404, until the metadata service is available. After the
metadata service is up, the datasource inspects preferred
extended_metadata_versions for availability.  Unit tests only mocked the
preferred extended_metadata_version so all Ec2 tests were retrying
attempts against
http://169.254.169.254/meta-data/<min-version>/instance-id adding a lot of
time cost to the unit test runs.

This branch uses httpretty to properly mock the following:
  - 404s from metadata on undesired extended_metadata_version test routes
    - https://169.254.169.254/meta-data/2016-09-02/instance-id
  - full metadata dictionary represented on min_metadata_version
    - https://169.254.169.254/meta-data/2016-09-02/*

The branch also tightens httpretty to raise a MockError for any URL
which isn't mocked via httpretty.HTTPretty.allow_net_connect=False.
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:bug/ec2-tests-unmocked-metadata into cloud-init:master.

diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index e1ce644..5f920fd 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -3,6 +3,7 @@
 import copy
 import httpretty
 import mock
+import re
 
 from .. import helpers as test_helpers
 from cloudinit import helpers
@@ -116,6 +117,9 @@ def register_mock_metaserver(base_url, data):
     In the index, references to lists or dictionaries have a trailing /.
     """
     def register_helper(register, base_url, body):
+        if not isinstance(base_url, str):
+            register(base_url, body)
+            return
         base_url = base_url.rstrip("/")
         if isinstance(body, str):
             register(base_url, body)
@@ -138,7 +142,7 @@ def register_mock_metaserver(base_url, data):
             register(base_url, '\n'.join(vals) + '\n')
             register(base_url + '/', '\n'.join(vals) + '\n')
         elif body is None:
-            register(base_url, 'not found', status_code=404)
+            register(base_url, 'not found', status=404)
 
     def myreg(*argc, **kwargs):
         # print("register_url(%s, %s)" % (argc, kwargs))
@@ -161,38 +165,47 @@ class TestEc2(test_helpers.HttprettyTestCase):
         self.datasource = ec2.DataSourceEc2
         self.metadata_addr = self.datasource.metadata_urls[0]
 
-    @property
-    def metadata_url(self):
-        return '/'.join([
-            self.metadata_addr,
-            self.datasource.min_metadata_version, 'meta-data', ''])
-
-    @property
-    def userdata_url(self):
-        return '/'.join([
-            self.metadata_addr,
-            self.datasource.min_metadata_version, 'user-data'])
+    def data_url(self, version):
+        """Return a metadata url based on the version provided."""
+        return '/'.join([self.metadata_addr, version, 'meta-data', ''])
 
     def _patch_add_cleanup(self, mpath, *args, **kwargs):
         p = mock.patch(mpath, *args, **kwargs)
         p.start()
         self.addCleanup(p.stop)
 
-    def _setup_ds(self, sys_cfg, platform_data, md, ud=None):
+    def _setup_ds(self, sys_cfg, platform_data, md, md_version=None):
+        self.uris = []
         distro = {}
         paths = helpers.Paths({})
         if sys_cfg is None:
             sys_cfg = {}
         ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        if not md_version:
+            md_version = ds.min_metadata_version
         if platform_data is not None:
             self._patch_add_cleanup(
                 "cloudinit.sources.DataSourceEc2._collect_platform_data",
                 return_value=platform_data)
 
         if md:
-            register_mock_metaserver(self.metadata_url, md)
-            register_mock_metaserver(self.userdata_url, ud)
-
+            httpretty.HTTPretty.allow_net_connect = False
+            all_versions = (
+                [ds.min_metadata_version] + ds.extended_metadata_versions)
+            for version in all_versions:
+                metadata_url = self.data_url(version)
+                if version == md_version:
+                    # Register all metadata for desired version
+                    register_mock_metaserver(metadata_url, md)
+                else:
+                    instance_id_url = metadata_url + 'instance-id'
+                    if version == ds.min_metadata_version:
+                       # Add min_metadata_version service availability check
+                        register_mock_metaserver(
+                            instance_id_url, DEFAULT_METADATA['instance-id'])
+                    else:
+                       # Register 404s for all unrequested extended versions
+                       register_mock_metaserver(instance_id_url, None)
         return ds
 
     @httpretty.activate
@@ -297,6 +310,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         Then the metadata services is crawled for more network config info.
         When the platform data is valid, return True.
         """
+
         m_is_bsd.return_value = False
         m_dhcp.return_value = [{
             'interface': 'eth9', 'fixed-address': '192.168.2.9',
@@ -307,6 +321,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
+
         ret = ds.get_data()
         self.assertTrue(ret)
         m_dhcp.assert_called_once_with()