cloud-init-dev team mailing list archive

[Chad Smith <chad.smith@xxxxxxxxxxxxx>]

Chad Smith has proposed merging ~chad.smith/cloud-init:feature/maintain-network-on-boot into cloud-init:master.

Commit message:
work-in-progress Strawman for MaintenanceEvent discussion.


Base-level MaintenanceEvent class and DataSource.maintain_metadata behavior to allow clearing cached instance data and re-crawling all metadata sources.



I'll build this out today and review any comments/suggestions folks want to attach while I'm developing it.

Requested reviews:
  cloud-init commiters (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/348000
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:feature/maintain-network-on-boot into cloud-init:master.

diff --git a/cloudinit/hotplug.py b/cloudinit/hotplug.py
new file mode 100644
index 0000000..c5ba1af
--- /dev/null
+++ b/cloudinit/hotplug.py
@@ -0,0 +1,15 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Classes and functions related to hotplug and eventing."""
+
+# Maintenance events describing the source generating a maintenance request.
+class MaintenanceEvent(object):
+    NONE = 0x0           # React to no maintenance events
+    BOOT = 0x1           # Any system boot or reboot event
+    DEVICE_ADD = 0x2     # Any new device added
+    DEVICE_REMOVE = 0x4  # Any device removed
+    DEVICE_CHANGE = 0x8  # Any device metadata change
+    ANY = 0xF            # Match any defined MaintenanceEvents
+
+MAINTENANCE_EVENT_STR = dict(
+    (attr, getattr(MaintenanceEvent, attr))
+    for attr in MaintenanceEvent.__dict__.keys() if attr.isupper())
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 24fd65f..cd6ab17 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -114,7 +114,7 @@ class DataSourceAltCloud(sources.DataSource):
 
         return 'UNKNOWN'
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         '''
         Description:
             User Data is passed to the launching instance which
diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py
index 699a85b..bc0d53a 100644
--- a/cloudinit/sources/DataSourceBigstep.py
+++ b/cloudinit/sources/DataSourceBigstep.py
@@ -25,7 +25,7 @@ class DataSourceBigstep(sources.DataSource):
         self.vendordata_raw = ""
         self.userdata_raw = ""
 
-    def _get_data(self, apply_filter=False):
+    def _get_data(self, clear_cache=False, apply_filter=False):
         url = get_url_from_file()
         if url is None:
             return False
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index c816f34..e67ff7c 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -49,7 +49,7 @@ class DataSourceCloudSigma(sources.DataSource):
         LOG.warning("failed to query dmi data for system product name")
         return False
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         """
         Metadata is the whole server context and /meta/cloud-config is used
         as userdata.
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index d4b758f..3912bc8 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -109,7 +109,7 @@ class DataSourceCloudStack(sources.DataSource):
     def get_config_obj(self):
         return self.cfg
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         seed_ret = {}
         if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
             self.userdata_raw = seed_ret['user-data']
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 4cb2897..99ff30c 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -54,7 +54,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
         mstr += "[source=%s]" % (self.source)
         return mstr
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         found = None
         md = {}
         results = {}
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
index e0ef665..25e8c87 100644
--- a/cloudinit/sources/DataSourceDigitalOcean.py
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -47,7 +47,7 @@ class DataSourceDigitalOcean(sources.DataSource):
     def _get_sysinfo(self):
         return do_helper.read_sysinfo()
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         (is_do, droplet_id) = self._get_sysinfo()
 
         # only proceed if we know we are on DigitalOcean
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 968ab3f..5c82994 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -77,7 +77,7 @@ class DataSourceEc2(sources.DataSource):
         """Return the cloud name as identified during _get_data."""
         return self.cloud_platform
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         seed_ret = {}
         if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
             self.userdata_raw = seed_ret['user-data']
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index d816262..0fd9c77 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -63,7 +63,7 @@ class DataSourceGCE(sources.DataSource):
             BUILTIN_DS_CONFIG])
         self.metadata_address = self.ds_cfg['metadata_url']
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         ret = util.log_time(
             LOG.debug, 'Crawl of GCE metadata service',
             read_md, kwargs={'address': self.metadata_address})
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index 01106ec..2e0f22f 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -136,7 +136,7 @@ class DataSourceIBMCloud(sources.DataSource):
         mstr = "%s [%s %s]" % (root, self.platform, self.source)
         return mstr
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         results = read_md()
         if results is None:
             return False
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index bcb3854..d511daa 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -61,7 +61,7 @@ class DataSourceMAAS(sources.DataSource):
         root = sources.DataSource.__str__(self)
         return "%s [%s]" % (root, self.base_url)
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         mcfg = self.ds_cfg
 
         try:
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 2daea59..1577745 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -35,7 +35,7 @@ class DataSourceNoCloud(sources.DataSource):
         root = sources.DataSource.__str__(self)
         return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode)
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         defaults = {
             "instance-id": "nocloud",
             "dsmode": self.dsmode,
diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py
index e63a7e3..44c0638 100644
--- a/cloudinit/sources/DataSourceNone.py
+++ b/cloudinit/sources/DataSourceNone.py
@@ -19,7 +19,7 @@ class DataSourceNone(sources.DataSource):
         self.metadata = {}
         self.userdata_raw = ''
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         # If the datasource config has any provided 'fallback'
         # userdata or metadata, use it...
         if 'userdata_raw' in self.ds_cfg:
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 178ccb0..7539c8a 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -65,7 +65,7 @@ class DataSourceOVF(sources.DataSource):
         root = sources.DataSource.__str__(self)
         return "%s [seed=%s]" % (root, self.seed)
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         found = []
         md = {}
         ud = ""
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 16c1078..a4d9ae1 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -44,7 +44,7 @@ class DataSourceOpenNebula(sources.DataSource):
         root = sources.DataSource.__str__(self)
         return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode)
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         defaults = {"instance-id": DEFAULT_IID}
         results = None
         seed = None
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 1a12a3f..9957b59 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -107,7 +107,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
             self.network_json, known_macs=None)
         return self._network_config
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         """Crawl metadata, parse and persist that data for this instance.
 
         @return: True when metadata discovered indicates OpenStack datasource.
diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py
index e2502b0..33f206e 100644
--- a/cloudinit/sources/DataSourceScaleway.py
+++ b/cloudinit/sources/DataSourceScaleway.py
@@ -186,7 +186,7 @@ class DataSourceScaleway(sources.DataSource):
         self.retries = int(self.ds_cfg.get('retries', DEF_MD_RETRIES))
         self.timeout = int(self.ds_cfg.get('timeout', DEF_MD_TIMEOUT))
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         if not on_scaleway():
             return False
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index f92e8b5..5242795 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -216,7 +216,7 @@ class DataSourceSmartOS(sources.DataSource):
             os.rename('/'.join([svc_path, 'provisioning']),
                       '/'.join([svc_path, 'provision_success']))
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         self._init()
 
         md = {}
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 90d7457..af876ad 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -19,6 +19,7 @@ from cloudinit.atomic_helper import write_json
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import net
+from cloudinit.hotplug import MaintenanceEvent
 from cloudinit import type_utils
 from cloudinit import user_data as ud
 from cloudinit import util
@@ -102,6 +103,13 @@ class DataSource(object):
     url_timeout = 10    # timeout for each metadata url read attempt
     url_retries = 5     # number of times to retry url upon 404
 
+    # Subclasses can define a mask of supported MaintenanceEvents during
+    # which the datasource will regenerate network_configuration. For example:
+    # network_maintenance_mask = MEvent.BOOT|MEvent.DEVICE_ADD
+
+    # Default behavior, perform no network update for any maintenance event
+    network_maintenance_mask = MaintenanceEvent.NONE
+
     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
         self.distro = distro
@@ -134,12 +142,25 @@ class DataSource(object):
             'region': self.region,
             'availability-zone': self.availability_zone}}
 
-    def get_data(self):
+    def get_data(self, clear_cache=False):
         """Datasources implement _get_data to setup metadata and userdata_raw.
 
         Minimally, the datasource should return a boolean True on success.
+        @param use_cache: Boolean set true to re-use data cache if present.
+           Value of False, will clear any cached data, re-crawling all
+           instance metadata.
         """
-        return_value = self._get_data()
+        if clear_cache:
+            if hasattr(self, '_network_config'):
+                # Clear network config property so it is regenerated from md.
+                setattr(self, '_network_config', None)
+            self.userdata = None
+            self.metadata = {}
+            self.userdata_raw = None
+            self.vendordata = None
+            self.vendordata_raw = None
+
+        return_value = self._get_data(clear_cache=clear_cache)
         json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
         if not return_value:
             return return_value
@@ -173,10 +194,13 @@ class DataSource(object):
         write_json(json_file, processed_data, mode=0o600)
         return return_value
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         raise NotImplementedError(
             'Subclasses of DataSource must implement _get_data which'
-            ' sets self.metadata, vendordata_raw and userdata_raw.')
+            ' sets self.metadata, vendordata_raw and userdata_raw. _get_data'
+            ' must accept an optional clear_cache boolean which will clear'
+            ' any cached metadata, vendordata, userdata etc to ensure a fresh'
+            ' crawl of new metadata')
 
     def get_url_params(self):
         """Return the Datasource's prefered url_read parameters.
@@ -416,6 +440,32 @@ class DataSource(object):
     def get_package_mirror_info(self):
         return self.distro.get_package_mirror_info(data_source=self)
 
+    def maintain_metadata(self, maintenance_event):
+        """Refresh cached metadata if the datasource handles this event.
+
+        The datasource defines a network_maintenance_mask attribute which
+        authorizes refreshing all cached metadata due to any number of
+        supported MaintenenanceEvent types.
+
+        @param maintenance_event: The source MaintenanceEvent type
+            observed to which the datasource may react.
+
+        @return True if the datasource has updated cached metadata due to the
+           the provided maintenance_event type. MaintenanceEvents will be
+           something like boot, configchange, device_add, device_remove etc.
+        """
+        if bool(maintenance_event & self.network_maintenance_mask):
+            LOG.debug(
+                "Re-crawling datasource metadata due to maintenance event: '%s'",
+                MAINTENANCE_EVENT_STR.get(maintenance_event, maintenance_event))
+            result = self.get_data(clear_cache=True)
+            if result:
+                return True
+            else:
+                LOG.warning(
+                    'Re-crawling metadata reported invalid datasource type')
+        return False
+
     def check_instance_id(self, sys_cfg):
         # quickly (local check only) if self.instance_id is still
         return False
@@ -442,7 +492,7 @@ class DataSource(object):
         return default
 
     @property
-    def network_config(self):
+    def network_config(self, regenerate=False):
         return None
 
     @property
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index d5bc98a..4c50854 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -27,7 +27,7 @@ class DataSourceTestSubclassNet(DataSource):
     def _get_cloud_name(self):
         return 'SubclassCloudName'
 
-    def _get_data(self):
+    def _get_data(self, clear_cache=False):
         self.metadata = {'availability_zone': 'myaz',
                          'local-hostname': 'test-subclass-hostname',
                          'region': 'myregion'}
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 3998cf6..f79e3b1 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -648,10 +648,15 @@ class Init(object):
         except Exception as e:
             LOG.warning("Failed to rename devices: %s", e)
 
-        if (self.datasource is not NULL_DATA_SOURCE and
-                not self.is_new_instance()):
             LOG.debug("not a new instance. network config is not applied.")
-            return
+
+        if self.datasource is not NULL_DATA_SOURCE:
+            if not self.is_new_instance():
+                if not self.datasource.maintain_metadata(MaintenanceEvent.boot):
+                   LOG.debug(
+                       'No network config applied. Neither a new instance nor'
+                       ' datasource network maintenance per boot')
+                   return
 
         LOG.info("Applying network configuration from %s bringup=%s: %s",
                  src, bring_up, netcfg)