cloud-init-dev team mailing list archive

[Chad Smith <chad.smith@xxxxxxxxxxxxx>]

Chad Smith has proposed merging ~chad.smith/cloud-init:unify-datasource-get-data into cloud-init:master with ~chad.smith/cloud-init:feature/move-base-testcase as a prerequisite.

Requested reviews:
  cloud-init commiters (cloud-init-dev)
  Server Team CI bot (server-team-bot): continuous-integration

For more details, see:
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/330115

Datasources: Create DataSource.get_data method in parent and write json
metadata

Each DataSource subclass must define it's own get_data method. This branch
formalizes our DataSource class to require that subclasses override the
_get_data method or a NotImplementedError is raised. The branch also
introduces a json data file cache at /run/cloud-init/instance-data.json.
This file caches all meta-data, user-data and vendor-data in a json blob
which other utilities with root-access could make use of. Becase the some
meta-data or user-data is potentially  the file is only readable by root.
Subsequent branches will attempt to structure some standardized content
across all datasources since meta-data and user-data formats may differ.
If some instance data content is not json serializable, a warning is
emmittted and no instance-data.json is written.
~                                                   
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:unify-datasource-get-data into cloud-init:master.

diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index ed1d691..98c4206 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -112,7 +112,7 @@ class DataSourceAltCloud(sources.DataSource):
 
         return 'UNKNOWN'
 
-    def get_data(self):
+    def _get_data(self):
         '''
         Description:
             User Data is passed to the launching instance which
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index b5a95a1..b646a08 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -327,7 +327,7 @@ class DataSourceAzure(sources.DataSource):
         metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files)
         return metadata
 
-    def get_data(self):
+    def _get_data(self):
         # azure removes/ejects the cdrom containing the ovf-env.xml
         # file on reboot.  So, in order to successfully reboot we
         # need to look in the datadir and consider that valid
diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py
index d7fcd45..b441812 100644
--- a/cloudinit/sources/DataSourceBigstep.py
+++ b/cloudinit/sources/DataSourceBigstep.py
@@ -22,7 +22,7 @@ class DataSourceBigstep(sources.DataSource):
         self.vendordata_raw = ""
         self.userdata_raw = ""
 
-    def get_data(self, apply_filter=False):
+    def _get_data(self, apply_filter=False):
         url = get_url_from_file()
         if url is None:
             return False
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index 19df16b..f1cafde 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -46,7 +46,7 @@ class DataSourceCloudSigma(sources.DataSource):
         LOG.warning("failed to query dmi data for system product name")
         return False
 
-    def get_data(self):
+    def _get_data(self):
         """
         Metadata is the whole server context and /meta/cloud-config is used
         as userdata.
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 0188d89..3124ec6 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -116,7 +116,7 @@ class DataSourceCloudStack(sources.DataSource):
     def get_config_obj(self):
         return self.cfg
 
-    def get_data(self):
+    def _get_data(self):
         seed_ret = {}
         if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
             self.userdata_raw = seed_ret['user-data']
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index ef374f3..63a261a 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -50,7 +50,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
         mstr += "[source=%s]" % (self.source)
         return mstr
 
-    def get_data(self):
+    def _get_data(self):
         found = None
         md = {}
         results = {}
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
index 5e7e66b..cedc075 100644
--- a/cloudinit/sources/DataSourceDigitalOcean.py
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -44,7 +44,7 @@ class DataSourceDigitalOcean(sources.DataSource):
     def _get_sysinfo(self):
         return do_helper.read_sysinfo()
 
-    def get_data(self):
+    def _get_data(self):
         (is_do, droplet_id) = self._get_sysinfo()
 
         # only proceed if we know we are on DigitalOcean
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 07c12bb..a521f7d 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -67,7 +67,7 @@ class DataSourceEc2(sources.DataSource):
         self.metadata_address = None
         self.seed_dir = os.path.join(paths.seed_dir, "ec2")
 
-    def get_data(self):
+    def _get_data(self):
         seed_ret = {}
         if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
             self.userdata_raw = seed_ret['user-data']
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 94484d6..c7d3627 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -50,7 +50,7 @@ class DataSourceGCE(sources.DataSource):
             BUILTIN_DS_CONFIG])
         self.metadata_address = self.ds_cfg['metadata_url']
 
-    def get_data(self):
+    def _get_data(self):
         ret = util.log_time(
             LOG.debug, 'Crawl of GCE metadata service',
             read_md, kwargs={'address': self.metadata_address})
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 77df5a5..c194b40 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -62,7 +62,7 @@ class DataSourceMAAS(sources.DataSource):
         root = sources.DataSource.__str__(self)
         return "%s [%s]" % (root, self.base_url)
 
-    def get_data(self):
+    def _get_data(self):
         mcfg = self.ds_cfg
 
         try:
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index e641244..990a01b 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -32,7 +32,7 @@ class DataSourceNoCloud(sources.DataSource):
         root = sources.DataSource.__str__(self)
         return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode)
 
-    def get_data(self):
+    def _get_data(self):
         defaults = {
             "instance-id": "nocloud",
             "dsmode": self.dsmode,
diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py
index 906bb27..feaf7c2 100644
--- a/cloudinit/sources/DataSourceNone.py
+++ b/cloudinit/sources/DataSourceNone.py
@@ -16,7 +16,7 @@ class DataSourceNone(sources.DataSource):
         self.metadata = {}
         self.userdata_raw = ''
 
-    def get_data(self):
+    def _get_data(self):
         # If the datasource config has any provided 'fallback'
         # userdata or metadata, use it...
         if 'userdata_raw' in self.ds_cfg:
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 73d3877..b8dc519 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -56,7 +56,7 @@ class DataSourceOVF(sources.DataSource):
         root = sources.DataSource.__str__(self)
         return "%s [seed=%s]" % (root, self.seed)
 
-    def get_data(self):
+    def _get_data(self):
         found = []
         md = {}
         ud = ""
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 5fdac19..acb0886 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -40,7 +40,7 @@ class DataSourceOpenNebula(sources.DataSource):
         root = sources.DataSource.__str__(self)
         return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode)
 
-    def get_data(self):
+    def _get_data(self):
         defaults = {"instance-id": DEFAULT_IID}
         results = None
         seed = None
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index b64a7f2..e4ab2eb 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -96,7 +96,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         self.metadata_address = url2base.get(avail_url)
         return bool(avail_url)
 
-    def get_data(self):
+    def _get_data(self):
         try:
             if not self.wait_for_metadata_service():
                 return False
diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py
index 3a8a8e8..9e07e58 100644
--- a/cloudinit/sources/DataSourceScaleway.py
+++ b/cloudinit/sources/DataSourceScaleway.py
@@ -184,7 +184,7 @@ class DataSourceScaleway(sources.DataSource):
         self.retries = int(self.ds_cfg.get('retries', DEF_MD_RETRIES))
         self.timeout = int(self.ds_cfg.get('timeout', DEF_MD_TIMEOUT))
 
-    def get_data(self):
+    def _get_data(self):
         if not on_scaleway():
             return False
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 6c6902f..94484d9 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -211,7 +211,7 @@ class DataSourceSmartOS(sources.DataSource):
             os.rename('/'.join([svc_path, 'provisioning']),
                       '/'.join([svc_path, 'provision_success']))
 
-    def get_data(self):
+    def _get_data(self):
         self._init()
 
         md = {}
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 9a43fbe..c08aa51 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -10,6 +10,7 @@
 
 import abc
 import copy
+import json
 import os
 import six
 
@@ -33,6 +34,9 @@ DEP_FILESYSTEM = "FILESYSTEM"
 DEP_NETWORK = "NETWORK"
 DS_PREFIX = 'DataSource'
 
+# Directory in which instance meta-data, user-data and vendor-data is written
+INSTANCE_JSON_FILE = 'instance-data.json'
+
 LOG = logging.getLogger(__name__)
 
 
@@ -78,6 +82,32 @@ class DataSource(object):
     def __str__(self):
         return type_utils.obj_name(self)
 
+    def get_data(self):
+        """Datasources implement _get_data to setup metadata and userdata_raw.
+
+        Minimally, the datasource should return a boolean True on success.
+        """
+        return_value = self._get_data()
+        json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
+        if return_value:
+            instance_data = {
+                'meta-data': self.metadata,
+                'user-data': self.get_userdata_raw(),
+                'vendor-data': self.get_vendordata_raw()}
+            LOG.info('Persisting instance data JSON: %s', json_file)
+            try:
+                content = json.dumps(instance_data)
+            except TypeError as e:
+                LOG.warning('Error persisting instance-data.json: %s', str(e))
+                return return_value
+            util.write_file(json_file, content, mode=0o600)
+        return return_value
+
+    def _get_data(self):
+        raise NotImplementedError(
+            'Subclasses of DataSource must implement _get_data which'
+            ' sets self.metadata, vendordata_raw and userdata_raw.')
+
     def get_userdata(self, apply_filter=False):
         if self.userdata is None:
             self.userdata = self.ud_proc.process(self.get_userdata_raw())
diff --git a/cloudinit/sources/tests/__init__.py b/cloudinit/sources/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cloudinit/sources/tests/__init__.py
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
new file mode 100644
index 0000000..92ce7ed
--- /dev/null
+++ b/cloudinit/sources/tests/test_init.py
@@ -0,0 +1,128 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import json
+import os
+import stat
+
+from cloudinit.helpers import Paths
+from cloudinit.sources import INSTANCE_JSON_FILE, DataSource
+from cloudinit.tests.helpers import CiTestCase
+from cloudinit.user_data import UserDataProcessor
+from cloudinit import util
+
+
+class DataSourceTestSubclassNet(DataSource):
+
+    def __init__(self, sys_cfg, distro, paths, custom_userdata=None):
+        super(DataSourceTestSubclassNet, self).__init__(
+            sys_cfg, distro, paths)
+        self._custom_userdata = custom_userdata
+
+    def _get_data(self):
+        self.metadata = {'DataSourceTestSubclassNet': 'was here'}
+        if self._custom_userdata:
+            self.userdata_raw = self._custom_userdata
+        else:
+            self.userdata_raw = 'userdata_raw'
+        self.vendordata_raw = 'vendordata_raw'
+        return True
+
+
+class InvalidDataSourceTestSubclassNet(DataSource):
+    pass
+
+
+class TestDataSource(CiTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestDataSource, self).setUp()
+        self.sys_cfg = {'datasource': {'': {'key1': False}}}
+        self.distro = 'distrotest'  # generally should be a Distro object
+        self.paths = Paths({})
+        self.datasource = DataSource(self.sys_cfg, self.distro, self.paths)
+
+    def test_datasource_init(self):
+        """DataSource initializes metadata attributes, ds_cfg and ud_proc."""
+        self.assertEqual(self.paths, self.datasource.paths)
+        self.assertEqual(self.sys_cfg, self.datasource.sys_cfg)
+        self.assertEqual(self.distro, self.datasource.distro)
+        self.assertIsNone(self.datasource.userdata)
+        self.assertEqual({}, self.datasource.metadata)
+        self.assertIsNone(self.datasource.userdata_raw)
+        self.assertIsNone(self.datasource.vendordata)
+        self.assertIsNone(self.datasource.vendordata_raw)
+        self.assertEqual({'key1': False}, self.datasource.ds_cfg)
+        self.assertIsInstance(self.datasource.ud_proc, UserDataProcessor)
+
+    def test_datasource_init_strips_classname_for_ds_cfg(self):
+        """Init strips DataSource prefix and Net suffix for ds_cfg."""
+        sys_cfg = {'datasource': {'TestSubclass': {'key2': False}}}
+        distro = 'distrotest'  # generally should be a Distro object
+        paths = Paths({})
+        datasource = DataSourceTestSubclassNet(sys_cfg, distro, paths)
+        self.assertEqual({'key2': False}, datasource.ds_cfg)
+
+    def test_str_is_classname(self):
+        """The string representation of the datasource is the classname."""
+        self.assertEqual('DataSource', str(self.datasource))
+        self.assertEqual(
+            'DataSourceTestSubclassNet',
+            str(DataSourceTestSubclassNet('', '', self.paths)))
+
+    def test__get_data_unimplemented(self):
+        """Raise an error when _get_data is not implemented."""
+        with self.assertRaises(NotImplementedError) as context_manager:
+            self.datasource.get_data()
+        self.assertIn(
+            'Subclasses of DataSource must implement _get_data',
+            str(context_manager.exception))
+        datasource2 = InvalidDataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, self.paths)
+        with self.assertRaises(NotImplementedError) as context_manager:
+            datasource2.get_data()
+        self.assertIn(
+            'Subclasses of DataSource must implement _get_data',
+            str(context_manager.exception))
+
+    def test_get_data_calls_subclass__get_data(self):
+        """Datasource.get_data uses the subclass' version of _get_data."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+        self.assertTrue(datasource.get_data())
+        self.assertEqual(
+            {'DataSourceTestSubclassNet': 'was here'},
+            datasource.metadata)
+        self.assertEqual('userdata_raw', datasource.userdata_raw)
+        self.assertEqual('vendordata_raw', datasource.vendordata_raw)
+
+    def test_get_data_write_json_instance_data(self):
+        """get_data writes INSTANCE_JSON_FILE to run_dir as readonly root."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+        datasource.get_data()
+        json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
+        content = util.load_file(json_file)
+        expected = {
+            'meta-data': {'DataSourceTestSubclassNet': 'was here'},
+            'user-data': 'userdata_raw',
+            'vendor-data': 'vendordata_raw'}
+        self.assertEqual(expected, json.loads(content))
+        file_stat = os.stat(json_file)
+        self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode))
+
+    def test_get_data_log_warning_on_non_json_instance_data(self):
+        """get_data succeeds but warns on non-json serialization content."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
+            custom_userdata=self.paths)
+        self.assertTrue(datasource.get_data())
+        json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
+        self.assertFalse(os.path.exists(json_file))
+        self.assertIn(
+            'WARNING: Error persisting instance-data.json',
+            self.logs.getvalue())
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 82ee971..714f5da 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -67,7 +67,7 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
         super(TestAliYunDatasource, self).setUp()
         cfg = {'datasource': {'AliYun': {'timeout': '1', 'max_wait': '1'}}}
         distro = {}
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'run_dir': self.tmp_dir()})
         self.ds = ay.DataSourceAliYun(cfg, distro, paths)
         self.metadata_address = self.ds.metadata_urls[0]
 
diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py
index 3b274d9..18c85b6 100644
--- a/tests/unittests/test_datasource/test_altcloud.py
+++ b/tests/unittests/test_datasource/test_altcloud.py
@@ -18,7 +18,7 @@ import tempfile
 from cloudinit import helpers
 from cloudinit import util
 
-from cloudinit.tests.helpers import TestCase
+from cloudinit.tests.helpers import CiTestCase
 
 import cloudinit.sources.DataSourceAltCloud as dsac
 
@@ -97,7 +97,7 @@ def _dmi_data(expected):
     return _data
 
 
-class TestGetCloudType(TestCase):
+class TestGetCloudType(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.get_cloud_type()
     '''
@@ -143,14 +143,16 @@ class TestGetCloudType(TestCase):
         self.assertEqual('UNKNOWN', dsrc.get_cloud_type())
 
 
-class TestGetDataCloudInfoFile(TestCase):
+class TestGetDataCloudInfoFile(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.get_data()
     With a contrived CLOUD_INFO_FILE
     '''
     def setUp(self):
         '''Set up.'''
-        self.paths = helpers.Paths({'cloud_dir': '/tmp'})
+        self.tmp = self.tmp_dir()
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
         self.cloud_info_file = tempfile.mkstemp()[1]
         self.dmi_data = util.read_dmi_data
         dsac.CLOUD_INFO_FILE = self.cloud_info_file
@@ -207,14 +209,16 @@ class TestGetDataCloudInfoFile(TestCase):
         self.assertEqual(False, dsrc.get_data())
 
 
-class TestGetDataNoCloudInfoFile(TestCase):
+class TestGetDataNoCloudInfoFile(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.get_data()
     Without a CLOUD_INFO_FILE
     '''
     def setUp(self):
         '''Set up.'''
-        self.paths = helpers.Paths({'cloud_dir': '/tmp'})
+        self.tmp = self.tmp_dir()
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
         self.dmi_data = util.read_dmi_data
         dsac.CLOUD_INFO_FILE = \
             'no such file'
@@ -254,7 +258,7 @@ class TestGetDataNoCloudInfoFile(TestCase):
         self.assertEqual(False, dsrc.get_data())
 
 
-class TestUserDataRhevm(TestCase):
+class TestUserDataRhevm(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.user_data_rhevm()
     '''
@@ -320,7 +324,7 @@ class TestUserDataRhevm(TestCase):
         self.assertEqual(False, dsrc.user_data_rhevm())
 
 
-class TestUserDataVsphere(TestCase):
+class TestUserDataVsphere(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.user_data_vsphere()
     '''
@@ -368,7 +372,7 @@ class TestUserDataVsphere(TestCase):
         self.assertEqual(1, m_mount_cb.call_count)
 
 
-class TestReadUserDataCallback(TestCase):
+class TestReadUserDataCallback(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.read_user_data_callback()
     '''
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 0a11777..eeef70f 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -11,9 +11,7 @@ from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
 
 import crypt
 import os
-import shutil
 import stat
-import tempfile
 import xml.etree.ElementTree as ET
 import yaml
 
@@ -84,11 +82,11 @@ class TestAzureDataSource(CiTestCase):
         super(TestAzureDataSource, self).setUp()
         if PY26:
             raise SkipTest("Does not work on python 2.6")
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
+        self.tmp = self.tmp_dir()
 
         # patch cloud_dir, so our 'seed_dir' is guaranteed empty
-        self.paths = helpers.Paths({'cloud_dir': self.tmp})
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
         self.waagent_d = os.path.join(self.tmp, 'var', 'lib', 'waagent')
 
         self.patches = ExitStack()
@@ -641,7 +639,7 @@ fdescfs            /dev/fd          fdescfs rw              0 0
         self.assertEqual(netconfig, expected_config)
 
 
-class TestAzureBounce(TestCase):
+class TestAzureBounce(CiTestCase):
 
     def mock_out_azure_moving_parts(self):
         self.patches.enter_context(
@@ -668,10 +666,10 @@ class TestAzureBounce(TestCase):
 
     def setUp(self):
         super(TestAzureBounce, self).setUp()
-        self.tmp = tempfile.mkdtemp()
+        self.tmp = self.tmp_dir()
         self.waagent_d = os.path.join(self.tmp, 'var', 'lib', 'waagent')
-        self.paths = helpers.Paths({'cloud_dir': self.tmp})
-        self.addCleanup(shutil.rmtree, self.tmp)
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
         dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d
         self.patches = ExitStack()
         self.mock_out_azure_moving_parts()
@@ -713,21 +711,24 @@ class TestAzureBounce(TestCase):
 
     def test_disabled_bounce_does_not_change_hostname(self):
         cfg = {'hostname_bounce': {'policy': 'off'}}
-        self._get_ds(self.get_ovf_env_with_dscfg('test-host', cfg)).get_data()
+        ds = self._get_ds(self.get_ovf_env_with_dscfg('test-host', cfg))
+        ds.get_data()
         self.assertEqual(0, self.set_hostname.call_count)
 
     @mock.patch('cloudinit.sources.DataSourceAzure.perform_hostname_bounce')
     def test_disabled_bounce_does_not_perform_bounce(
             self, perform_hostname_bounce):
         cfg = {'hostname_bounce': {'policy': 'off'}}
-        self._get_ds(self.get_ovf_env_with_dscfg('test-host', cfg)).get_data()
+        ds = self._get_ds(self.get_ovf_env_with_dscfg('test-host', cfg))
+        ds.get_data()
         self.assertEqual(0, perform_hostname_bounce.call_count)
 
     def test_same_hostname_does_not_change_hostname(self):
         host_name = 'unchanged-host-name'
         self.get_hostname.return_value = host_name
         cfg = {'hostname_bounce': {'policy': 'yes'}}
-        self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg)).get_data()
+        ds = self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg))
+        ds.get_data()
         self.assertEqual(0, self.set_hostname.call_count)
 
     @mock.patch('cloudinit.sources.DataSourceAzure.perform_hostname_bounce')
@@ -736,7 +737,8 @@ class TestAzureBounce(TestCase):
         host_name = 'unchanged-host-name'
         self.get_hostname.return_value = host_name
         cfg = {'hostname_bounce': {'policy': 'yes'}}
-        self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg)).get_data()
+        ds = self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg))
+        ds.get_data()
         self.assertEqual(0, perform_hostname_bounce.call_count)
 
     @mock.patch('cloudinit.sources.DataSourceAzure.perform_hostname_bounce')
diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py
index e4c5990..f6a59b6 100644
--- a/tests/unittests/test_datasource/test_cloudsigma.py
+++ b/tests/unittests/test_datasource/test_cloudsigma.py
@@ -3,6 +3,7 @@
 import copy
 
 from cloudinit.cs_utils import Cepko
+from cloudinit import helpers
 from cloudinit import sources
 from cloudinit.sources import DataSourceCloudSigma
 
@@ -38,10 +39,12 @@ class CepkoMock(Cepko):
         return self
 
 
-class DataSourceCloudSigmaTest(test_helpers.TestCase):
+class DataSourceCloudSigmaTest(test_helpers.CiTestCase):
     def setUp(self):
         super(DataSourceCloudSigmaTest, self).setUp()
-        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "")
+        self.paths = helpers.Paths({'run_dir': self.tmp_dir()})
+        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma(
+            "", "", paths=self.paths)
         self.datasource.is_running_in_cloudsigma = lambda: True
         self.datasource.cepko = CepkoMock(SERVER_CONTEXT)
         self.datasource.get_data()
@@ -85,7 +88,8 @@ class DataSourceCloudSigmaTest(test_helpers.TestCase):
     def test_lack_of_vendor_data(self):
         stripped_context = copy.deepcopy(SERVER_CONTEXT)
         del stripped_context["vendor_data"]
-        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "")
+        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma(
+            "", "", paths=self.paths)
         self.datasource.cepko = CepkoMock(stripped_context)
         self.datasource.get_data()
 
@@ -94,7 +98,8 @@ class DataSourceCloudSigmaTest(test_helpers.TestCase):
     def test_lack_of_cloudinit_key_in_vendor_data(self):
         stripped_context = copy.deepcopy(SERVER_CONTEXT)
         del stripped_context["vendor_data"]["cloudinit"]
-        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "")
+        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma(
+            "", "", paths=self.paths)
         self.datasource.cepko = CepkoMock(stripped_context)
         self.datasource.get_data()
 
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index 2dc9030..5345c37 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -3,10 +3,10 @@
 from cloudinit import helpers
 from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack
 
-from cloudinit.tests.helpers import TestCase, mock, ExitStack
+from cloudinit.tests.helpers import CiTestCase, mock, ExitStack
 
 
-class TestCloudStackPasswordFetching(TestCase):
+class TestCloudStackPasswordFetching(CiTestCase):
 
     def setUp(self):
         super(TestCloudStackPasswordFetching, self).setUp()
@@ -25,6 +25,7 @@ class TestCloudStackPasswordFetching(TestCase):
         self.patches.enter_context(mock.patch(
             'cloudinit.sources.DataSourceCloudStack.get_default_gateway',
             get_default_gw))
+        self.tmp = self.tmp_dir()
 
     def _set_password_server_response(self, response_string):
         subp = mock.MagicMock(return_value=(response_string, ''))
@@ -35,26 +36,30 @@ class TestCloudStackPasswordFetching(TestCase):
 
     def test_empty_password_doesnt_create_config(self):
         self._set_password_server_response('')
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertEqual({}, ds.get_config_obj())
 
     def test_saved_password_doesnt_create_config(self):
         self._set_password_server_response('saved_password')
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertEqual({}, ds.get_config_obj())
 
     def test_password_sets_password(self):
         password = 'SekritSquirrel'
         self._set_password_server_response(password)
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertEqual(password, ds.get_config_obj()['password'])
 
     def test_bad_request_doesnt_stop_ds_from_working(self):
         self._set_password_server_response('bad_request')
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         self.assertTrue(ds.get_data())
 
     def assertRequestTypesSent(self, subp, expected_request_types):
@@ -69,14 +74,16 @@ class TestCloudStackPasswordFetching(TestCase):
     def test_valid_response_means_password_marked_as_saved(self):
         password = 'SekritSquirrel'
         subp = self._set_password_server_response(password)
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertRequestTypesSent(subp,
                                     ['send_my_password', 'saved_password'])
 
     def _check_password_not_saved_for(self, response_string):
         subp = self._set_password_server_response(response_string)
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertRequestTypesSent(subp, ['send_my_password'])
 
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 237c189..9849788 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -725,8 +725,9 @@ class TestConvertNetworkData(TestCase):
 
 
 def cfg_ds_from_dir(seed_d):
+    tmp = tempfile.mkdtemp()
     cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None,
-                                      helpers.Paths({}))
+                                      helpers.Paths({'run_dir': tmp}))
     cfg_ds.seed_dir = seed_d
     cfg_ds.known_macs = KNOWN_MACS.copy()
     if not cfg_ds.get_data():
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
index f264f36..ec32173 100644
--- a/tests/unittests/test_datasource/test_digitalocean.py
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -13,7 +13,7 @@ from cloudinit import settings
 from cloudinit.sources import DataSourceDigitalOcean
 from cloudinit.sources.helpers import digitalocean
 
-from cloudinit.tests.helpers import mock, TestCase
+from cloudinit.tests.helpers import mock, CiTestCase
 
 DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@xxxxx",
                     "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@xxxxx"]
@@ -135,14 +135,17 @@ def _mock_dmi():
     return (True, DO_META.get('id'))
 
 
-class TestDataSourceDigitalOcean(TestCase):
+class TestDataSourceDigitalOcean(CiTestCase):
     """
     Test reading the meta-data
     """
+    def setUp(self):
+        super(TestDataSourceDigitalOcean, self).setUp()
+        self.tmp = self.tmp_dir()
 
     def get_ds(self, get_sysinfo=_mock_dmi):
         ds = DataSourceDigitalOcean.DataSourceDigitalOcean(
-            settings.CFG_BUILTIN, None, helpers.Paths({}))
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
         ds.use_ip4LL = False
         if get_sysinfo is not None:
             ds._get_sysinfo = get_sysinfo
@@ -194,7 +197,7 @@ class TestDataSourceDigitalOcean(TestCase):
         self.assertIsInstance(ds.get_public_ssh_keys(), list)
 
 
-class TestNetworkConvert(TestCase):
+class TestNetworkConvert(CiTestCase):
 
     @mock.patch('cloudinit.net.get_interfaces_by_mac')
     def _get_networking(self, m_get_by_mac):
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 9fb9048..ae4faf9 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -163,6 +163,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         super(TestEc2, self).setUp()
         self.datasource = ec2.DataSourceEc2
         self.metadata_addr = self.datasource.metadata_urls[0]
+        self.tmp = self.tmp_dir()
 
     def data_url(self, version):
         """Return a metadata url based on the version provided."""
@@ -176,7 +177,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
     def _setup_ds(self, sys_cfg, platform_data, md, md_version=None):
         self.uris = []
         distro = {}
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'run_dir': self.tmp})
         if sys_cfg is None:
             sys_cfg = {}
         ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index 50e49a1..a2e3330 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -69,9 +69,10 @@ def _set_mock_metadata(gce_meta=None):
 class TestDataSourceGCE(test_helpers.HttprettyTestCase):
 
     def setUp(self):
+        tmp = self.tmp_dir()
         self.ds = DataSourceGCE.DataSourceGCE(
             settings.CFG_BUILTIN, None,
-            helpers.Paths({}))
+            helpers.Paths({'run_dir': tmp}))
         ppatch = self.m_platform_reports_gce = mock.patch(
             'cloudinit.sources.DataSourceGCE.platform_reports_gce')
         self.m_platform_reports_gce = ppatch.start()
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index fea9156..70d50de 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -3,22 +3,20 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceNoCloud
 from cloudinit import util
-from cloudinit.tests.helpers import TestCase, populate_dir, mock, ExitStack
+from cloudinit.tests.helpers import CiTestCase, populate_dir, mock, ExitStack
 
 import os
-import shutil
-import tempfile
 import textwrap
 import yaml
 
 
-class TestNoCloudDataSource(TestCase):
+class TestNoCloudDataSource(CiTestCase):
 
     def setUp(self):
         super(TestNoCloudDataSource, self).setUp()
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
-        self.paths = helpers.Paths({'cloud_dir': self.tmp})
+        self.tmp = self.tmp_dir()
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
 
         self.cmdline = "root=TESTCMDLINE"
 
@@ -215,7 +213,7 @@ class TestNoCloudDataSource(TestCase):
         self.assertNotIn(gateway, str(dsrc.network_config))
 
 
-class TestParseCommandLineData(TestCase):
+class TestParseCommandLineData(CiTestCase):
 
     def test_parse_cmdline_data_valid(self):
         ds_id = "ds=nocloud"
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index e7d5569..2326dd5 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -3,12 +3,10 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
-from cloudinit.tests.helpers import mock, populate_dir, TestCase
+from cloudinit.tests.helpers import mock, populate_dir, CiTestCase
 
 import os
 import pwd
-import shutil
-import tempfile
 import unittest
 
 
@@ -36,14 +34,14 @@ PUBLIC_IP = '10.0.0.3'
 DS_PATH = "cloudinit.sources.DataSourceOpenNebula"
 
 
-class TestOpenNebulaDataSource(TestCase):
+class TestOpenNebulaDataSource(CiTestCase):
     parsed_user = None
 
     def setUp(self):
         super(TestOpenNebulaDataSource, self).setUp()
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
-        self.paths = helpers.Paths({'cloud_dir': self.tmp})
+        self.tmp = self.tmp_dir()
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
 
         # defaults for few tests
         self.ds = ds.DataSourceOpenNebula
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 177e980..f4d262f 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -128,6 +128,10 @@ def _read_metadata_service():
 class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
     VERSION = 'latest'
 
+    def setUp(self):
+        super(TestOpenStackDataSource, self).setUp()
+        self.tmp = self.tmp_dir()
+
     @hp.activate
     def test_successful(self):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
@@ -229,7 +233,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
         ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
                                        None,
-                                       helpers.Paths({}))
+                                       helpers.Paths({'run_dir': self.tmp}))
         self.assertIsNone(ds_os.version)
         found = ds_os.get_data()
         self.assertTrue(found)
@@ -253,7 +257,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
                                        None,
-                                       helpers.Paths({}))
+                                       helpers.Paths({'run_dir': self.tmp}))
         self.assertIsNone(ds_os.version)
         found = ds_os.get_data()
         self.assertFalse(found)
@@ -268,7 +272,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
                                        None,
-                                       helpers.Paths({}))
+                                       helpers.Paths({'run_dir': self.tmp}))
         ds_os.ds_cfg = {
             'max_wait': 0,
             'timeout': 0,
@@ -291,7 +295,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
                                        None,
-                                       helpers.Paths({}))
+                                       helpers.Paths({'run_dir': self.tmp}))
         ds_os.ds_cfg = {
             'max_wait': 0,
             'timeout': 0,
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
index 436df9e..8dec06b 100644
--- a/tests/unittests/test_datasource/test_scaleway.py
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -9,7 +9,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceScaleway
 
-from cloudinit.tests.helpers import mock, HttprettyTestCase, TestCase
+from cloudinit.tests.helpers import mock, HttprettyTestCase, CiTestCase
 
 
 class DataResponses(object):
@@ -63,7 +63,11 @@ class MetadataResponses(object):
         return 200, headers, json.dumps(cls.FAKE_METADATA)
 
 
-class TestOnScaleway(TestCase):
+class TestOnScaleway(CiTestCase):
+
+    def setUp(self):
+        super(TestOnScaleway, self).setUp()
+        self.tmp = self.tmp_dir()
 
     def install_mocks(self, fake_dmi, fake_file_exists, fake_cmdline):
         mock, faked = fake_dmi
@@ -91,7 +95,7 @@ class TestOnScaleway(TestCase):
 
         # When not on Scaleway, get_data() returns False.
         datasource = DataSourceScaleway.DataSourceScaleway(
-            settings.CFG_BUILTIN, None, helpers.Paths({})
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})
         )
         self.assertFalse(datasource.get_data())
 
@@ -159,8 +163,9 @@ def get_source_address_adapter(*args, **kwargs):
 class TestDataSourceScaleway(HttprettyTestCase):
 
     def setUp(self):
+        tmp = self.tmp_dir()
         self.datasource = DataSourceScaleway.DataSourceScaleway(
-            settings.CFG_BUILTIN, None, helpers.Paths({})
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': tmp})
         )
         super(TestDataSourceScaleway, self).setUp()
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 933d5b6..88bae5f 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -359,7 +359,8 @@ class TestSmartOSDataSource(FilesystemMockingTestCase):
 
         self.tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmp)
-        self.paths = c_helpers.Paths({'cloud_dir': self.tmp})
+        self.paths = c_helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
 
         self.legacy_user_d = os.path.join(self.tmp, 'legacy_user_tmp')
         os.mkdir(self.legacy_user_d)