← Back to team overview

cloud-init-dev team mailing list archive

  1. cloud-init-dev team
  2. Mailing list archive
  3. Message #04886

[Merge] ~smoser/cloud-init:fix/opensuse-skip-on-httpretty-ssl-error into cloud-init:master

  Thread Previous • Date Previous • • Thread Next 
Scott Moser has proposed merging ~smoser/cloud-init:fix/opensuse-skip-on-httpretty-ssl-error into cloud-init:master.

Commit message:
tests: SkipTest some errors found in httpretty in OpenSuSE 42.3.

In OpenSuSE 42.3, with
   python-httpretty=0.8.8-7.1
   python-ndg-httpsclient=0.4.0-3.2
   python-pyOpenSSL=16.0.0-4.1
We hit what looks like the bug described at
   https://github.com/gabrielfalcao/HTTPretty/issues/242

This just skipTest if we hit that scenario.

Requested reviews:
  Server Team CI bot (server-team-bot): continuous-integration
  cloud-init commiters (cloud-init-dev)
Related bugs:
  Bug #1771659 in cloud-init: "unittests fail in OpenSuSE 42.3 with httpretty issues"
  https://bugs.launchpad.net/cloud-init/+bug/1771659

For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/345630

see commit message
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/opensuse-skip-on-httpretty-ssl-error into cloud-init:master.

diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 117a9cf..ff48068 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -3,6 +3,7 @@
 from __future__ import print_function
 
 import functools
+import httpretty
 import logging
 import os
 import shutil
@@ -303,14 +304,21 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
 class HttprettyTestCase(CiTestCase):
     # necessary as http_proxy gets in the way of httpretty
     # https://github.com/gabrielfalcao/HTTPretty/issues/122
+    # Also make sure that allow_net_connect is set to False.
+    # And make sure reset and enable/disable are done.
 
     def setUp(self):
         self.restore_proxy = os.environ.get('http_proxy')
         if self.restore_proxy is not None:
             del os.environ['http_proxy']
         super(HttprettyTestCase, self).setUp()
+        httpretty.HTTPretty.allow_net_connect = False
+        httpretty.reset()
+        httpretty.enable()
 
     def tearDown(self):
+        httpretty.disable()
+        httpretty.reset()
         if self.restore_proxy:
             os.environ['http_proxy'] = self.restore_proxy
         super(HttprettyTestCase, self).tearDown()
diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index 275b16d..91d35cb 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -524,7 +524,17 @@ c: 4
         self.assertEqual(cfg.get('password'), 'gocubs')
         self.assertEqual(cfg.get('locale'), 'chicago')
 
-    @httpretty.activate
+
+class TestConsumeUserDataHttp(TestConsumeUserData, helpers.HttprettyTestCase):
+
+    def setUp(self):
+        TestConsumeUserData.setUp(self)
+        helpers.HttprettyTestCase.setUp(self)
+
+    def tearDown(self):
+        TestConsumeUserData.tearDown(self)
+        helpers.HttprettyTestCase.tearDown(self)
+
     @mock.patch('cloudinit.url_helper.time.sleep')
     def test_include(self, mock_sleep):
         """Test #include."""
@@ -543,7 +553,6 @@ c: 4
         cc = util.load_yaml(cc_contents)
         self.assertTrue(cc.get('included'))
 
-    @httpretty.activate
     @mock.patch('cloudinit.url_helper.time.sleep')
     def test_include_bad_url(self, mock_sleep):
         """Test #include with a bad URL."""
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 4fa9616..1e77842 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -130,7 +130,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
                          self.ds.get_hostname())
 
     @mock.patch("cloudinit.sources.DataSourceAliYun._is_aliyun")
-    @httpretty.activate
     def test_with_mock_server(self, m_is_aliyun):
         m_is_aliyun.return_value = True
         self.regist_default_server()
@@ -143,7 +142,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
         self._test_host_name()
 
     @mock.patch("cloudinit.sources.DataSourceAliYun._is_aliyun")
-    @httpretty.activate
     def test_returns_false_when_not_on_aliyun(self, m_is_aliyun):
         """If is_aliyun returns false, then get_data should return False."""
         m_is_aliyun.return_value = False
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index dff8b1e..497e761 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -191,7 +191,6 @@ def register_mock_metaserver(base_url, data):
             register(base_url, 'not found', status=404)
 
     def myreg(*argc, **kwargs):
-        # print("register_url(%s, %s)" % (argc, kwargs))
         return httpretty.register_uri(httpretty.GET, *argc, **kwargs)
 
     register_helper(myreg, base_url, data)
@@ -236,7 +235,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
                 return_value=platform_data)
 
         if md:
-            httpretty.HTTPretty.allow_net_connect = False
             all_versions = (
                 [ds.min_metadata_version] + ds.extended_metadata_versions)
             for version in all_versions:
@@ -255,7 +253,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
                         register_mock_metaserver(instance_id_url, None)
         return ds
 
-    @httpretty.activate
     def test_network_config_property_returns_version_1_network_data(self):
         """network_config property returns network version 1 for metadata.
 
@@ -288,7 +285,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
                     m_get_mac.return_value = mac1
                     self.assertEqual(expected, ds.network_config)
 
-    @httpretty.activate
     def test_network_config_property_set_dhcp4_on_private_ipv4(self):
         """network_config property configures dhcp4 on private ipv4 nics.
 
@@ -330,7 +326,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
         ds._network_config = {'cached': 'data'}
         self.assertEqual({'cached': 'data'}, ds.network_config)
 
-    @httpretty.activate
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     def test_network_config_cached_property_refreshed_on_upgrade(self, m_dhcp):
         """Refresh the network_config Ec2 cache if network key is absent.
@@ -364,7 +359,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
              'type': 'physical'}]}
         self.assertEqual(expected, ds.network_config)
 
-    @httpretty.activate
     def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self):
         """get_instance-id gets DataSourceEc2Local.identity if not present.
 
@@ -397,7 +391,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
         ds.metadata = DEFAULT_METADATA
         self.assertEqual('my-identity-id', ds.get_instance_id())
 
-    @httpretty.activate
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     def test_valid_platform_with_strict_true(self, m_dhcp):
         """Valid platform data should return true with strict_id true."""
@@ -409,7 +402,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
         self.assertTrue(ret)
         self.assertEqual(0, m_dhcp.call_count)
 
-    @httpretty.activate
     def test_valid_platform_with_strict_false(self):
         """Valid platform data should return true with strict_id false."""
         ds = self._setup_ds(
@@ -419,7 +411,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
         ret = ds.get_data()
         self.assertTrue(ret)
 
-    @httpretty.activate
     def test_unknown_platform_with_strict_true(self):
         """Unknown platform data with strict_id true should return False."""
         uuid = 'ab439480-72bf-11d3-91fc-b8aded755F9a'
@@ -430,7 +421,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
         ret = ds.get_data()
         self.assertFalse(ret)
 
-    @httpretty.activate
     def test_unknown_platform_with_strict_false(self):
         """Unknown platform data with strict_id false should return True."""
         uuid = 'ab439480-72bf-11d3-91fc-b8aded755F9a'
@@ -462,7 +452,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
                     ' not {0}'.format(platform_name))
                 self.assertIn(message, self.logs.getvalue())
 
-    @httpretty.activate
     @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
     def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
         """DataSourceEc2Local returns False on BSD.
@@ -481,7 +470,6 @@ class TestEc2(test_helpers.HttprettyTestCase):
             "FreeBSD doesn't support running dhclient with -sf",
             self.logs.getvalue())
 
-    @httpretty.activate
     @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
     @mock.patch('cloudinit.net.find_fallback_nic')
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index eb3cec4..41176c6 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -78,7 +78,6 @@ def _set_mock_metadata(gce_meta=None):
             return (404, headers, '')
 
     # reset is needed. https://github.com/gabrielfalcao/HTTPretty/issues/316
-    httpretty.reset()
     httpretty.register_uri(httpretty.GET, MD_URL_RE, body=_request_callback)
 
 
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 42c3155..bb180c0 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -135,7 +135,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         super(TestOpenStackDataSource, self).setUp()
         self.tmp = self.tmp_dir()
 
-    @hp.activate
     def test_successful(self):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
         f = _read_metadata_service()
@@ -157,7 +156,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEqual('b0fa911b-69d4-4476-bbe2-1c92bff6535c',
                          metadata.get('instance-id'))
 
-    @hp.activate
     def test_no_ec2(self):
         _register_uris(self.VERSION, {}, {}, OS_FILES)
         f = _read_metadata_service()
@@ -168,7 +166,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEqual({}, f.get('ec2-metadata'))
         self.assertEqual(2, f.get('version'))
 
-    @hp.activate
     def test_bad_metadata(self):
         os_files = copy.deepcopy(OS_FILES)
         for k in list(os_files.keys()):
@@ -177,7 +174,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(openstack.NonReadable, _read_metadata_service)
 
-    @hp.activate
     def test_bad_uuid(self):
         os_files = copy.deepcopy(OS_FILES)
         os_meta = copy.deepcopy(OSTACK_META)
@@ -188,7 +184,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
 
-    @hp.activate
     def test_userdata_empty(self):
         os_files = copy.deepcopy(OS_FILES)
         for k in list(os_files.keys()):
@@ -201,7 +196,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
         self.assertFalse(f.get('userdata'))
 
-    @hp.activate
     def test_vendordata_empty(self):
         os_files = copy.deepcopy(OS_FILES)
         for k in list(os_files.keys()):
@@ -213,7 +207,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
         self.assertFalse(f.get('vendordata'))
 
-    @hp.activate
     def test_vendordata_invalid(self):
         os_files = copy.deepcopy(OS_FILES)
         for k in list(os_files.keys()):
@@ -222,7 +215,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
 
-    @hp.activate
     def test_metadata_invalid(self):
         os_files = copy.deepcopy(OS_FILES)
         for k in list(os_files.keys()):
@@ -231,7 +223,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
 
-    @hp.activate
     def test_datasource(self):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
         ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
@@ -251,7 +242,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEqual(VENDOR_DATA, ds_os.vendordata_pure)
         self.assertIsNone(ds_os.vendordata_raw)
 
-    @hp.activate
     def test_bad_datasource_meta(self):
         os_files = copy.deepcopy(OS_FILES)
         for k in list(os_files.keys()):
@@ -266,7 +256,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
 
-    @hp.activate
     def test_no_datasource(self):
         os_files = copy.deepcopy(OS_FILES)
         for k in list(os_files.keys()):
@@ -285,7 +274,6 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
 
-    @hp.activate
     def test_disabled_datasource(self):
         os_files = copy.deepcopy(OS_FILES)
         os_meta = copy.deepcopy(OSTACK_META)
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
index 8dec06b..e4e9bb2 100644
--- a/tests/unittests/test_datasource/test_scaleway.py
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -176,7 +176,6 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.vendordata_url = \
             DataSourceScaleway.BUILTIN_DS_CONFIG['vendordata_url']
 
-    @httpretty.activate
     @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
                 get_source_address_adapter)
     @mock.patch('cloudinit.util.get_cmdline')
@@ -212,7 +211,6 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.assertIsNone(self.datasource.region)
         self.assertEqual(sleep.call_count, 0)
 
-    @httpretty.activate
     @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
                 get_source_address_adapter)
     @mock.patch('cloudinit.util.get_cmdline')
@@ -236,7 +234,6 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.assertIsNone(self.datasource.get_vendordata_raw())
         self.assertEqual(sleep.call_count, 0)
 
-    @httpretty.activate
     @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
                 get_source_address_adapter)
     @mock.patch('cloudinit.util.get_cmdline')
diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py
index af78997..3f50f57 100644
--- a/tests/unittests/test_ec2_util.py
+++ b/tests/unittests/test_ec2_util.py
@@ -11,7 +11,6 @@ from cloudinit import url_helper as uh
 class TestEc2Util(helpers.HttprettyTestCase):
     VERSION = 'latest'
 
-    @hp.activate
     def test_userdata_fetch(self):
         hp.register_uri(hp.GET,
                         'http://169.254.169.254/%s/user-data' % (self.VERSION),
@@ -20,7 +19,6 @@ class TestEc2Util(helpers.HttprettyTestCase):
         userdata = eu.get_instance_userdata(self.VERSION)
         self.assertEqual('stuff', userdata.decode('utf-8'))
 
-    @hp.activate
     def test_userdata_fetch_fail_not_found(self):
         hp.register_uri(hp.GET,
                         'http://169.254.169.254/%s/user-data' % (self.VERSION),
@@ -28,7 +26,6 @@ class TestEc2Util(helpers.HttprettyTestCase):
         userdata = eu.get_instance_userdata(self.VERSION, retries=0)
         self.assertEqual('', userdata)
 
-    @hp.activate
     def test_userdata_fetch_fail_server_dead(self):
         hp.register_uri(hp.GET,
                         'http://169.254.169.254/%s/user-data' % (self.VERSION),
@@ -36,7 +33,6 @@ class TestEc2Util(helpers.HttprettyTestCase):
         userdata = eu.get_instance_userdata(self.VERSION, retries=0)
         self.assertEqual('', userdata)
 
-    @hp.activate
     def test_userdata_fetch_fail_server_not_found(self):
         hp.register_uri(hp.GET,
                         'http://169.254.169.254/%s/user-data' % (self.VERSION),
@@ -44,7 +40,6 @@ class TestEc2Util(helpers.HttprettyTestCase):
         userdata = eu.get_instance_userdata(self.VERSION)
         self.assertEqual('', userdata)
 
-    @hp.activate
     def test_metadata_fetch_no_keys(self):
         base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION)
         hp.register_uri(hp.GET, base_url, status=200,
@@ -62,7 +57,6 @@ class TestEc2Util(helpers.HttprettyTestCase):
         self.assertEqual(md['instance-id'], '123')
         self.assertEqual(md['ami-launch-index'], '1')
 
-    @hp.activate
     def test_metadata_fetch_key(self):
         base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION)
         hp.register_uri(hp.GET, base_url, status=200,
@@ -83,7 +77,6 @@ class TestEc2Util(helpers.HttprettyTestCase):
         self.assertEqual(md['instance-id'], '123')
         self.assertEqual(1, len(md['public-keys']))
 
-    @hp.activate
     def test_metadata_fetch_with_2_keys(self):
         base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION)
         hp.register_uri(hp.GET, base_url, status=200,
@@ -108,7 +101,6 @@ class TestEc2Util(helpers.HttprettyTestCase):
         self.assertEqual(md['instance-id'], '123')
         self.assertEqual(2, len(md['public-keys']))
 
-    @hp.activate
     def test_metadata_fetch_bdm(self):
         base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION)
         hp.register_uri(hp.GET, base_url, status=200,
@@ -140,7 +132,6 @@ class TestEc2Util(helpers.HttprettyTestCase):
         self.assertEqual(bdm['ami'], 'sdb')
         self.assertEqual(bdm['ephemeral0'], 'sdc')
 
-    @hp.activate
     def test_metadata_no_security_credentials(self):
         base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION)
         hp.register_uri(hp.GET, base_url, status=200,
diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py
index 0136a93..f4bbd66 100644
--- a/tests/unittests/test_handler/test_handler_chef.py
+++ b/tests/unittests/test_handler/test_handler_chef.py
@@ -14,19 +14,27 @@ from cloudinit.sources import DataSourceNone
 from cloudinit import util
 
 from cloudinit.tests.helpers import (
-    CiTestCase, FilesystemMockingTestCase, mock, skipIf)
+    HttprettyTestCase, FilesystemMockingTestCase, mock, skipIf)
 
 LOG = logging.getLogger(__name__)
 
 CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"])
 
+# This is adjusted to use http because using with https causes issue
+# in some openssl/httpretty combinations.
+#   https://github.com/gabrielfalcao/HTTPretty/issues/242
+# We saw issue in opensuse 42.3 with
+#    httpretty=0.8.8-7.1 ndg-httpsclient=0.4.0-3.2 pyOpenSSL=16.0.0-4.1
+OMNIBUS_URL_HTTP = cc_chef.OMNIBUS_URL.replace("https:", "http:")
 
-class TestInstallChefOmnibus(CiTestCase):
+
+class TestInstallChefOmnibus(HttprettyTestCase):
 
     def setUp(self):
+        super(TestInstallChefOmnibus, self).setUp()
         self.new_root = self.tmp_dir()
 
-    @httpretty.activate
+    @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", OMNIBUS_URL_HTTP)
     def test_install_chef_from_omnibus_runs_chef_url_content(self):
         """install_chef_from_omnibus runs downloaded OMNIBUS_URL as script."""
         chef_outfile = self.tmp_path('chef.out', self.new_root)
@@ -65,7 +73,7 @@ class TestInstallChefOmnibus(CiTestCase):
             expected_subp_kwargs,
             m_subp_blob.call_args_list[0][1])
 
-    @httpretty.activate
+    @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", OMNIBUS_URL_HTTP)
     @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile')
     def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob):
         """install_chef_from_omnibus provides version arg to OMNIBUS_URL."""
diff --git a/tools/read-dependencies b/tools/read-dependencies
index 421f470..b4656e6 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -51,6 +51,10 @@ MAYBE_RELIABLE_YUM_INSTALL = [
     """,
     'reliable-yum-install']
 
+ZYPPER_INSTALL = [
+    'zypper', '--non-interactive', '--gpg-auto-import-keys', 'install',
+    '--auto-agree-with-licenses']
+
 DRY_DISTRO_INSTALL_PKG_CMD = {
     'centos': ['yum', 'install', '--assumeyes'],
     'redhat': ['yum', 'install', '--assumeyes'],
@@ -61,8 +65,8 @@ DISTRO_INSTALL_PKG_CMD = {
     'redhat': MAYBE_RELIABLE_YUM_INSTALL,
     'debian': ['apt', 'install', '-y'],
     'ubuntu': ['apt', 'install', '-y'],
-    'opensuse': ['zypper', 'install'],
-    'suse': ['zypper', 'install']
+    'opensuse': ZYPPER_INSTALL,
+    'suse': ZYPPER_INSTALL,
 }

Follow ups

  Thread Previous • Date Previous • • Thread Next