cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #00164
[Merge] lp:~harlowja/cloud-init/merge-types-differently into lp:cloud-init
Joshua Harlow has proposed merging lp:~harlowja/cloud-init/merge-types-differently into lp:cloud-init.
Requested reviews:
cloud init development team (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~harlowja/cloud-init/merge-types-differently/+merge/135310
Not done yet, just an initial little work on it.
--
https://code.launchpad.net/~harlowja/cloud-init/merge-types-differently/+merge/135310
Your team cloud init development team is requested to review the proposed merge of lp:~harlowja/cloud-init/merge-types-differently into lp:cloud-init.
=== modified file 'cloudinit/handlers/__init__.py'
--- cloudinit/handlers/__init__.py 2012-11-06 22:27:56 +0000
+++ cloudinit/handlers/__init__.py 2012-11-21 04:06:19 +0000
@@ -69,7 +69,6 @@
class Handler(object):
-
__metaclass__ = abc.ABCMeta
def __init__(self, frequency, version=2):
@@ -83,15 +82,12 @@
def list_types(self):
raise NotImplementedError()
- def handle_part(self, data, ctype, filename, payload, frequency):
- return self._handle_part(data, ctype, filename, payload, frequency)
-
@abc.abstractmethod
- def _handle_part(self, data, ctype, filename, payload, frequency):
+ def handle_part(self, *args, **kwargs):
raise NotImplementedError()
-def run_part(mod, data, ctype, filename, payload, frequency):
+def run_part(mod, data, filename, payload, headers, frequency):
mod_freq = mod.frequency
if not (mod_freq == PER_ALWAYS or
(frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)):
@@ -102,19 +98,25 @@
mod_ver = int(mod_ver)
except:
mod_ver = 1
+ content_type = headers['Content-Type']
try:
LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s",
- mod, ctype, filename, mod_ver, frequency)
- if mod_ver >= 2:
+ mod, content_type, filename, mod_ver, frequency)
+ if mod_ver == 3:
+ # Treat as v. 3 which does get a frequency + headers
+ mod.handle_part(data, content_type, filename,
+ payload, frequency, headers)
+ elif mod_ver == 2:
# Treat as v. 2 which does get a frequency
- mod.handle_part(data, ctype, filename, payload, frequency)
+ mod.handle_part(data, content_type, filename,
+ payload, frequency)
else:
# Treat as v. 1 which gets no frequency
- mod.handle_part(data, ctype, filename, payload)
+ mod.handle_part(data, content_type, filename, payload)
except:
util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)"
" with frequency %s"),
- mod, ctype, filename,
+ mod, content_type, filename,
mod_ver, frequency)
@@ -173,26 +175,27 @@
return text
-def walker_callback(pdata, ctype, filename, payload):
- if ctype in PART_CONTENT_TYPES:
- walker_handle_handler(pdata, ctype, filename, payload)
+def walker_callback(data, filename, payload, headers):
+ content_type = headers['Content-Type']
+ if content_type in PART_CONTENT_TYPES:
+ walker_handle_handler(data, content_type, filename, payload)
return
- handlers = pdata['handlers']
- if ctype in pdata['handlers']:
- run_part(handlers[ctype], pdata['data'], ctype, filename,
- payload, pdata['frequency'])
+ handlers = data['handlers']
+ if content_type in handlers:
+ run_part(handlers[content_type], data['data'], filename,
+ payload, headers, data['frequency'])
elif payload:
# Extract the first line or 24 bytes for displaying in the log
start = _extract_first_or_bytes(payload, 24)
details = "'%s...'" % (_escape_string(start))
if ctype == NOT_MULTIPART_TYPE:
LOG.warning("Unhandled non-multipart (%s) userdata: %s",
- ctype, details)
+ content_type, details)
else:
LOG.warning("Unhandled unknown content-type (%s) userdata: %s",
- ctype, details)
+ content_type, details)
else:
- LOG.debug("empty payload of type %s" % ctype)
+ LOG.debug("Empty payload of type %s", content_type)
# Callback is a function that will be called with
@@ -212,7 +215,9 @@
if not filename:
filename = PART_FN_TPL % (partnum)
- callback(data, ctype, filename, part.get_payload(decode=True))
+ callback(data, ctype, filename,
+ part.get_payload(decode=True),
+ dict(part))
partnum = partnum + 1
=== modified file 'cloudinit/handlers/boot_hook.py'
--- cloudinit/handlers/boot_hook.py 2012-06-16 20:23:32 +0000
+++ cloudinit/handlers/boot_hook.py 2012-11-21 04:06:19 +0000
@@ -56,7 +56,7 @@
util.write_file(filepath, contents, 0700)
return filepath
- def _handle_part(self, _data, ctype, filename, payload, _frequency):
+ def handle_part(self, _data, ctype, filename, payload, _frequency):
if ctype in handlers.CONTENT_SIGNALS:
return
=== modified file 'cloudinit/handlers/cloud_config.py'
--- cloudinit/handlers/cloud_config.py 2012-06-16 17:56:08 +0000
+++ cloudinit/handlers/cloud_config.py 2012-11-21 04:06:19 +0000
@@ -22,6 +22,7 @@
from cloudinit import handlers
from cloudinit import log as logging
+from cloudinit import mergers
from cloudinit import util
from cloudinit.settings import (PER_ALWAYS)
@@ -31,8 +32,8 @@
class CloudConfigPartHandler(handlers.Handler):
def __init__(self, paths, **_kwargs):
- handlers.Handler.__init__(self, PER_ALWAYS)
- self.cloud_buf = []
+ handlers.Handler.__init__(self, PER_ALWAYS, version=3)
+ self.cloud_buf = {}
self.cloud_fn = paths.get_ipath("cloud_config")
def list_types(self):
@@ -43,20 +44,17 @@
def _write_cloud_config(self, buf):
if not self.cloud_fn:
return
- lines = [str(b) for b in buf]
- payload = "\n".join(lines)
+ payload = util.yaml_dumps(self.cloud_buf)
util.write_file(self.cloud_fn, payload, 0600)
- def _handle_part(self, _data, ctype, filename, payload, _frequency):
+ def handle_part(self, _data, ctype, filename, payload, _frequency, headers):
if ctype == handlers.CONTENT_START:
- self.cloud_buf = []
+ self.cloud_buf = {}
return
if ctype == handlers.CONTENT_END:
self._write_cloud_config(self.cloud_buf)
- self.cloud_buf = []
+ self.cloud_buf = {}
return
-
- filename = util.clean_filename(filename)
- if not filename:
- filename = '??'
- self.cloud_buf.extend(["#%s" % (filename), str(payload)])
+ merge_how = headers.get("Merge-Type", 'list+dict+str')
+ merger = mergers.construct(merge_how)
+ self.cloud_buf = merger.merge(self.cloud_buf, util.load_yaml(payload))
=== modified file 'cloudinit/handlers/shell_script.py'
--- cloudinit/handlers/shell_script.py 2012-08-22 18:12:32 +0000
+++ cloudinit/handlers/shell_script.py 2012-11-21 04:06:19 +0000
@@ -41,7 +41,7 @@
handlers.type_from_starts_with("#!"),
]
- def _handle_part(self, _data, ctype, filename, payload, _frequency):
+ def handle_part(self, _data, ctype, filename, payload, _frequency):
if ctype in handlers.CONTENT_SIGNALS:
# TODO(harlowja): maybe delete existing things here
return
=== modified file 'cloudinit/handlers/upstart_job.py'
--- cloudinit/handlers/upstart_job.py 2012-06-29 18:41:43 +0000
+++ cloudinit/handlers/upstart_job.py 2012-11-21 04:06:19 +0000
@@ -42,7 +42,7 @@
handlers.type_from_starts_with("#upstart-job"),
]
- def _handle_part(self, _data, ctype, filename, payload, frequency):
+ def handle_part(self, _data, ctype, filename, payload, frequency):
if ctype in handlers.CONTENT_SIGNALS:
return
=== added directory 'cloudinit/mergers'
=== added file 'cloudinit/mergers/__init__.py'
--- cloudinit/mergers/__init__.py 1970-01-01 00:00:00 +0000
+++ cloudinit/mergers/__init__.py 2012-11-21 04:06:19 +0000
@@ -0,0 +1,104 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@xxxxxxxxxxxxx>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+from cloudinit import importer
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class UnknownMerger(object):
+ # Named differently so auto-method finding
+ # doesn't pick this up if there is ever a type
+ # named "unknown"
+ def _handle_unknown(self, meth_wanted, value, merge_with):
+ return value
+
+ def merge(self, source, merge_with):
+ type_name = util.obj_name(source)
+ type_name = type_name.lower()
+ method_name = "_on_%s" % (type_name)
+ meth = None
+ args = [source, merge_with]
+ if hasattr(self, method_name):
+ meth = getattr(self, method_name)
+ if not meth:
+ meth = self._handle_unknown
+ args.insert(0, method_name)
+ return meth(*args)
+
+
+class LookupMerger(UnknownMerger):
+ def __init__(self, lookups=None):
+ UnknownMerger.__init__(self)
+ if lookups is None:
+ self._lookups = []
+ else:
+ self._lookups = lookups
+
+ def _handle_unknown(self, meth_wanted, value, merge_with):
+ meth = None
+ for merger in self._lookups:
+ if hasattr(merger, meth_wanted):
+ # First one that has that method/attr gets to be
+ # the one that will be called
+ meth = getattr(merger, meth_wanted)
+ break
+ if not meth:
+ return UnknownMerger._handle_unknown(self, meth_wanted,
+ value, merge_with)
+ return meth(value, merge_with)
+
+
+def _extract_merger_names(merge_how):
+ names = []
+ for m_name in merge_how.split("+"):
+ # Canonicalize the name (so that it can be found
+ # even when users alter it in various ways...
+ m_name = m_name.lower().strip()
+ m_name = m_name.replace(" ", "_")
+ m_name = m_name.replace("\t", "_")
+ m_name = m_name.replace("-", "_")
+ if not m_name:
+ continue
+ names.append(m_name)
+ return names
+
+
+def construct(merge_how, default_classes=None):
+ mergers = []
+ merger_classes = []
+ root = LookupMerger(mergers)
+ for m_name in _extract_merger_names(merge_how):
+ merger_locs = importer.find_module(m_name,
+ [__name__],
+ ['Merger'])
+ if not merger_locs:
+ msg = "Could not find merger named %s" % (m_name)
+ raise ImportError(msg)
+ else:
+ mod = importer.import_module(merger_locs[0])
+ cls = getattr(mod, 'Merger')
+ merger_classes.append(cls)
+ if not merger_classes and default_classes:
+ merger_classes = default_classes
+ for m_class in merger_classes:
+ mergers.append(m_class(root))
+ return root
\ No newline at end of file
=== added file 'cloudinit/mergers/dict.py'
--- cloudinit/mergers/dict.py 1970-01-01 00:00:00 +0000
+++ cloudinit/mergers/dict.py 2012-11-21 04:06:19 +0000
@@ -0,0 +1,33 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@xxxxxxxxxxxxx>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+ def __init__(self, merger):
+ self._merger = merger
+
+ def _on_dict(self, value, merge_with):
+ if not isinstance(merge_with, (dict)):
+ return value
+ merged = dict(value)
+ for (k, v) in merge_with.items():
+ if k in merged:
+ merged[k] = self._merger.merge(merged[k], v)
+ else:
+ merged[k] = v
+ return merged
=== added file 'cloudinit/mergers/list.py'
--- cloudinit/mergers/list.py 1970-01-01 00:00:00 +0000
+++ cloudinit/mergers/list.py 2012-11-21 04:06:19 +0000
@@ -0,0 +1,41 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@xxxxxxxxxxxxx>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+ def __init__(self, merger):
+ self._merger = merger
+
+ def _on_tuple(self, value, merge_with):
+ return self._on_list(list(value), merge_with)
+
+ def _on_list(self, value, merge_with):
+ if isinstance(merge_with, (tuple, list)):
+ new_value = list(value)
+ for m_v in merge_with:
+ m_am = 0
+ for (i, o_v) in enumerate(new_value):
+ if m_v == o_v:
+ new_value[i] = self._merger.merge(o_v, m_v)
+ m_am += 1
+ if m_am == 0:
+ new_value.append(m_v)
+ else:
+ new_value = list(value)
+ new_value.append(merge_with)
+ return new_value
=== added file 'cloudinit/mergers/str.py'
--- cloudinit/mergers/str.py 1970-01-01 00:00:00 +0000
+++ cloudinit/mergers/str.py 2012-11-21 04:06:19 +0000
@@ -0,0 +1,28 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@xxxxxxxxxxxxx>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+ def __init__(self, merger):
+ pass
+
+ def _on_unicode(self, value, merge_with):
+ return self._on_str(value, merge_with)
+
+ def _on_str(self, value, merge_with):
+ return value
