cloud-init-dev team mailing list archive

[Scott Moser <smoser@xxxxxxxxxx>]

Scott Moser has proposed merging lp:~harlowja/cloud-init/launch-index-work into lp:cloud-init.

Requested reviews:
  cloud init development team (cloud-init-dev)
Related bugs:
  Bug #1023177 in cloud-init: "cloud-config/cloud-init should support launch-index"
  https://bugs.launchpad.net/cloud-init/+bug/1023177

For more details, see:
https://code.launchpad.net/~harlowja/cloud-init/launch-index-work/+merge/121423
-- 
https://code.launchpad.net/~harlowja/cloud-init/launch-index-work/+merge/121423
Your team cloud init development team is requested to review the proposed merge of lp:~harlowja/cloud-init/launch-index-work into lp:cloud-init.

=== modified file 'cloudinit/cloud.py'
--- cloudinit/cloud.py	2012-08-22 04:28:29 +0000
+++ cloudinit/cloud.py	2012-08-27 13:50:24 +0000
@@ -70,12 +70,15 @@
         return fn
 
     # The rest of thes are just useful proxies
-    def get_userdata(self):
-        return self.datasource.get_userdata()
+    def get_userdata(self, apply_filter=True):
+        return self.datasource.get_userdata(apply_filter)
 
     def get_instance_id(self):
         return self.datasource.get_instance_id()
 
+    def get_launch_index(self):
+        return self.datasource.get_launch_index()
+
     def get_public_ssh_keys(self):
         return self.datasource.get_public_ssh_keys()
 

=== modified file 'cloudinit/sources/DataSourceEc2.py'
--- cloudinit/sources/DataSourceEc2.py	2012-08-22 04:28:29 +0000
+++ cloudinit/sources/DataSourceEc2.py	2012-08-27 13:50:24 +0000
@@ -77,6 +77,9 @@
                         self.metadata_address)
             return False
 
+    def get_launch_index(self):
+        return self.metadata.get('ami-launch-index')
+
     def get_instance_id(self):
         return self.metadata['instance-id']
 

=== modified file 'cloudinit/sources/__init__.py'
--- cloudinit/sources/__init__.py	2012-08-22 15:38:20 +0000
+++ cloudinit/sources/__init__.py	2012-08-27 13:50:24 +0000
@@ -20,6 +20,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from email.mime.multipart import MIMEMultipart
+
 import abc
 
 from cloudinit import importer
@@ -59,12 +61,60 @@
         else:
             self.ud_proc = ud_proc
 
-    def get_userdata(self):
+    def get_userdata(self, apply_filter=False):
         if self.userdata is None:
-            raw_data = self.get_userdata_raw()
-            self.userdata = self.ud_proc.process(raw_data)
+            self.userdata = self.ud_proc.process(self.get_userdata_raw())
+        if apply_filter:
+            return self._filter_userdata(self.userdata)
         return self.userdata
 
+    def get_launch_index(self):
+        return None
+
+    def _filter_userdata(self, processed_ud):
+        if not processed_ud:
+            return processed_ud
+        idx = self.get_launch_index()
+        if idx is None:
+            return processed_ud
+        # First do a scan to see if any one with launch-index
+        # headers, if not just skip this....
+        launch_idxs = 0
+        for part in processed_ud.walk():
+            if ud.is_skippable(part):
+                continue
+            launch_idx_h = part.get('Launch-Index', None)
+            if launch_idx_h is not None:
+                launch_idxs += 1
+        if not launch_idxs:
+            return processed_ud
+        # Reform a new message with those that either have
+        # no launch index or ones that have our launch index or ones
+        # that have some other garbage that we don't know what to do with
+        accumulating_msg = MIMEMultipart()
+        tot_attached = 0
+        tot_processed = 0
+        for part in processed_ud.walk():
+            if ud.is_skippable(part):
+                continue
+            try:
+                tot_processed += 1
+                launch_idx_h = part.get('Launch-Index', None)
+                if launch_idx_h is None or int(launch_idx_h) == int(idx):
+                    accumulating_msg.attach(part)
+                    tot_attached += 1
+                else:
+                    LOG.debug(("Discarding multipart message %s, "
+                               "launch-index provided destined for %s "
+                               "and not %s"),
+                               tot_processed, launch_idx_h, idx)
+            except (TypeError, ValueError):
+                # If any int conversion fails keep the message
+                accumulating_msg.attach(part)
+                tot_attached += 1
+        accumulating_msg[ud.ATTACHMENT_FIELD] = str(tot_attached)
+        return accumulating_msg
+
     @property
     def is_disconnected(self):
         return False

=== modified file 'cloudinit/stages.py'
--- cloudinit/stages.py	2012-08-22 18:12:32 +0000
+++ cloudinit/stages.py	2012-08-27 13:50:24 +0000
@@ -347,7 +347,7 @@
             sys.path.insert(0, idir)
 
         # Ensure datasource fetched before activation (just incase)
-        user_data_msg = self.datasource.get_userdata()
+        user_data_msg = self.datasource.get_userdata(True)
 
         # This keeps track of all the active handlers
         c_handlers = helpers.ContentHandlers()

=== modified file 'cloudinit/user_data.py'
--- cloudinit/user_data.py	2012-08-22 18:12:32 +0000
+++ cloudinit/user_data.py	2012-08-27 13:50:24 +0000
@@ -52,21 +52,22 @@
 # Msg header used to track attachments
 ATTACHMENT_FIELD = 'Number-Attachments'
 
+# Only the following content types can have there launch index examined
+CAN_HAVE_LAUNCH_INDEX = ["text/cloud-config", "text/cloud-config-archive"]
+
 
 class UserDataProcessor(object):
     def __init__(self, paths):
         self.paths = paths
 
     def process(self, blob):
-        base_msg = convert_string(blob)
-        process_msg = MIMEMultipart()
-        self._process_msg(base_msg, process_msg)
-        return process_msg
+        accumulating_msg = MIMEMultipart()
+        self._process_msg(convert_string(blob), accumulating_msg)
+        return accumulating_msg
 
     def _process_msg(self, base_msg, append_msg):
         for part in base_msg.walk():
-            # multipart/* are just containers
-            if part.get_content_maintype() == 'multipart':
+            if is_skippable(part):
                 continue
 
             ctype = None
@@ -97,11 +98,41 @@
 
             self._attach_part(append_msg, part)
 
+    def _attach_launch_index(self, msg):
+        header_idx = msg.get('Launch-Index', None)
+        payload_idx = None
+        if msg.get_content_type() in CAN_HAVE_LAUNCH_INDEX:
+            try:
+                # See if it has a launch-index field
+                # that might affect the final header
+                payload = util.load_yaml(msg.get_payload(decode=True))
+                if payload:
+                    payload_idx = payload.get('launch-index')
+            except:
+                pass
+        # Header overrides contents, for now (?) or the other way around?
+        if header_idx is not None:
+            payload_idx = header_idx
+        # Nothing found in payload, use header (if anything there)
+        if payload_idx is None:
+            payload_idx = header_idx
+        if payload_idx is not None:
+            try:
+                msg.add_header('Launch-Index', str(int(payload_idx)))
+            except (ValueError, TypeError):
+                pass
+
     def _get_include_once_filename(self, entry):
         entry_fn = util.hash_blob(entry, 'md5', 64)
         return os.path.join(self.paths.get_ipath_cur('data'),
                             'urlcache', entry_fn)
 
+    def _process_before_attach(self, msg, attached_id):
+        if not msg.get_filename():
+            msg.add_header('Content-Disposition',
+                           'attachment', filename=PART_FN_TPL % (attached_id))
+        self._attach_launch_index(msg)
+
     def _do_include(self, content, append_msg):
         # Include a list of urls, one per line
         # also support '#include <url here>'
@@ -178,9 +209,11 @@
             if 'filename' in ent:
                 msg.add_header('Content-Disposition',
                                'attachment', filename=ent['filename'])
+            if 'launch-index' in ent:
+                msg.add_header('Launch-Index', str(ent['launch-index']))
 
             for header in list(ent.keys()):
-                if header in ('content', 'filename', 'type'):
+                if header in ('content', 'filename', 'type', 'launch-index'):
                     continue
                 msg.add_header(header, ent['header'])
 
@@ -204,21 +237,23 @@
             outer_msg.replace_header(ATTACHMENT_FIELD, str(fetched_count))
         return fetched_count
 
-    def _part_filename(self, _unnamed_part, count):
-        return PART_FN_TPL % (count + 1)
-
     def _attach_part(self, outer_msg, part):
         """
-        Attach an part to an outer message. outermsg must be a MIMEMultipart.
-        Modifies a header in the message to keep track of number of attachments.
+        Attach a message to an outer message. outermsg must be a MIMEMultipart.
+        Modifies a header in the outer message to keep track of number of attachments.
         """
-        cur_c = self._multi_part_count(outer_msg)
-        if not part.get_filename():
-            fn = self._part_filename(part, cur_c)
-            part.add_header('Content-Disposition',
-                            'attachment', filename=fn)
+        part_count = self._multi_part_count(outer_msg)
+        self._process_before_attach(part, part_count + 1)
         outer_msg.attach(part)
-        self._multi_part_count(outer_msg, cur_c + 1)
+        self._multi_part_count(outer_msg, part_count + 1)
+
+
+def is_skippable(part):
+    # multipart/* are just containers
+    part_maintype = part.get_content_maintype() or ''
+    if part_maintype.lower() == 'multipart':
+        return True
+    return False
 
 
 # Coverts a raw string into a mime message