cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #00296
[Merge] lp:~utlemming/cloud-init/smartos_base64 into lp:cloud-init
You have been requested to review the proposed merge of lp:~utlemming/cloud-init/smartos_base64 into lp:cloud-init.
For more details, see:
https://code.launchpad.net/~utlemming/cloud-init/smartos_base64/+merge/177716
Added base64 support for parsing meta-data values. SmartOS CLI tools will support the ability to send over base64 encoded shortly, and the API supports it now.
--
https://code.launchpad.net/~utlemming/cloud-init/smartos_base64/+merge/177716
Your team cloud init development team is requested to review the proposed merge of lp:~utlemming/cloud-init/smartos_base64 into lp:cloud-init.
=== modified file 'cloudinit/sources/DataSourceSmartOS.py'
--- cloudinit/sources/DataSourceSmartOS.py 2013-07-24 14:44:21 +0000
+++ cloudinit/sources/DataSourceSmartOS.py 2013-07-30 23:04:24 +0000
@@ -27,6 +27,7 @@
#
+import base64
from cloudinit import log as logging
from cloudinit import sources
from cloudinit import util
@@ -49,6 +50,10 @@
'motd_sys_info': ('motd_sys_info', True),
}
+# These are values which will never be base64 encoded.
+SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info',
+ 'iptables_disable']
+
class DataSourceSmartOS(sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
@@ -56,6 +61,10 @@
self.seed_dir = os.path.join(paths.seed_dir, 'sdc')
self.is_smartdc = None
self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC)
+ self.all_base64 = self.sys_cfg.get("decode_base64", False)
+ self.base_64_encoded = []
+ self.smartos_no_base64 = self.sys_cfg.get("no_base64_decode",
+ SMARTOS_NO_BASE64)
self.seed_timeout = self.sys_cfg.get("serial_timeout",
DEF_TTY_TIMEOUT)
@@ -84,17 +93,41 @@
self.is_smartdc = True
md['instance-id'] = system_uuid
+ self.base_64_encoded = query_data('base_64_enocded',
+ self.seed,
+ self.seed_timeout,
+ strip=True)
+ if self.base_64_encoded:
+ self.base_64_encoded = str(self.base_64_encoded).split(',')
+ else:
+ self.base_64_encoded = []
+
+ if not self.all_base64:
+ self.all_base64 = util.is_true(query_data('meta_encoded_base64',
+ self.seed,
+ self.seed_timeout,
+ strip=True))
+
for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems():
smartos_noun, strip = attribute
+
+ b64encoded = False
+ if self.all_base64 and \
+ (smartos_noun not in self.smartos_no_base64 and \
+ ci_noun not in self.smartos_no_base64):
+ b64encoded = True
+
md[ci_noun] = query_data(smartos_noun, self.seed,
- self.seed_timeout, strip=strip)
+ self.seed_timeout, strip=strip,
+ b64encoded=b64encoded)
if not md['local-hostname']:
md['local-hostname'] = system_uuid
+ ud = None
if md['user-data']:
ud = md['user-data']
- else:
+ elif md['user-script']:
ud = md['user-script']
self.metadata = md
@@ -124,12 +157,17 @@
return ser
-def query_data(noun, seed_device, seed_timeout, strip=False):
+def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
"""Makes a request to via the serial console via "GET <NOUN>"
In the response, the first line is the status, while subsequent lines
are is the value. A blank line with a "." is used to indicate end of
response.
+
+ If the response is expected to be base64 encoded, then set b64encoded
+ to true. Unfortantely, there is no way to know if something is 100%
+ encoded, so this method relies on being told if the data is base64 or
+ not.
"""
if not noun:
@@ -153,12 +191,22 @@
response.append(m)
ser.close()
+
+ resp = None
if not strip:
- return "".join(response)
+ resp = "".join(response)
+ elif b64encoded:
+ resp = "".join(response).rstrip()
else:
- return "".join(response).rstrip()
-
- return None
+ resp = "".join(response).rstrip()
+
+ if b64encoded:
+ try:
+ return base64.b64decode(resp)
+ except TypeError:
+ return resp
+
+ return resp
def dmi_data():
=== added directory 'doc/sources/smartos'
=== added file 'doc/sources/smartos/README.rst'
--- doc/sources/smartos/README.rst 1970-01-01 00:00:00 +0000
+++ doc/sources/smartos/README.rst 2013-07-30 23:04:24 +0000
@@ -0,0 +1,66 @@
+==================
+SmartOS Datasource
+==================
+
+This datasource finds metadata and user-data from the SmartOS virtualization
+platform (i.e. Joyent).
+
+SmartOS Platform
+----------------
+The SmartOS virtualization platform meta-data to the instance via the second
+serial console. On Linux, this is /dev/ttyS1. The data is a provided via a
+simple protocol, where something queries for the userdata, where the console
+responds with the status and if "SUCCESS" returns until a single ".\n".
+
+The format is lossy. As such, new versions of the SmartOS tooling will include
+support for base64 encoded data.
+
+Userdata
+--------
+
+In SmartOS parlance, user-data is a actually meta-data. This userdata can be
+provided a key-value pairs.
+
+Cloud-init supports reading the traditional meta-data fields supported by the
+SmartOS tools. These are:
+ * root_authorized_keys
+ * hostname
+ * enable_motd_sys_info
+ * iptables_disable
+
+Note: At this time iptables_disable and enable_motd_sys_info are read but
+ are not actioned.
+
+user-script
+-----------
+
+SmartOS traditionally supports sending over a user-script for execution at the
+rc.local level. Cloud-init supports running user-scripts as if they were
+cloud-init user-data. In this sense, anything with a shell interpetter
+directive will run
+
+user-data and user-script
+-------------------------
+
+In the event that a user defines the meta-data key of "user-data" it will
+always supercede any user-script data. This is for consistency.
+
+base64
+------
+
+In order to provide a lossy format, all data except for:
+ * root_authorized_keys
+ * enable_motd_sys_info
+ * iptables_disable
+
+This means that user-script and user-data as well as other values can be
+base64 encoded to provide a lossy format. Since Cloud-init can only guess
+as to whether or not something is truly base64 encoded, the following
+meta-data keys are hints as to whether or not to base64 decode something:
+ * decode_base64: Except for excluded keys, attempt to base64 decode
+ the values. If the value fails to decode properly, it will be
+ returned in its text
+ * base_64_encoded: A comma deliminated list of which values are base64
+ encoded.
+ * no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d)
+ that sets which values should not be base64 decoded.
=== modified file 'tests/unittests/test_datasource/test_smartos.py'
--- tests/unittests/test_datasource/test_smartos.py 2013-07-23 22:33:46 +0000
+++ tests/unittests/test_datasource/test_smartos.py 2013-07-30 23:04:24 +0000
@@ -22,6 +22,7 @@
# return responses.
#
+import base64
from cloudinit import helpers
from cloudinit.sources import DataSourceSmartOS
@@ -35,7 +36,7 @@
'enable_motd_sys_info': None,
'system_uuid': str(uuid.uuid4()),
'smartdc': 'smartdc',
- 'userdata': """
+ 'user-data': """
#!/bin/sh
/bin/true
""",
@@ -48,12 +49,14 @@
port = None
- def __init__(self):
+ def __init__(self, b64encode=False):
self.last = None
self.last = None
self.new = True
self.count = 0
self.mocked_out = []
+ self.b64encode = b64encode
+ self.b64excluded = DataSourceSmartOS.SMARTOS_NO_BASE64
def open(self):
return True
@@ -87,11 +90,17 @@
def _format_out(self):
if self.last in mock_returns:
- try:
- for l in mock_returns[self.last].splitlines():
- yield "%s\n" % l
- except:
- yield "%s\n" % mock_returns[self.last]
+ _mret = mock_returns[self.last]
+ if self.b64encode and \
+ self.last not in self.b64excluded:
+ yield base64.b64encode(_mret)
+
+ else:
+ try:
+ for l in _mret.splitlines():
+ yield "%s\n" % l.rstrip()
+ except:
+ yield "%s\n" % _mret.rstrip()
yield '\n'
yield '.'
@@ -116,16 +125,19 @@
ret = apply_patches(patches)
self.unapply += ret
- def _get_ds(self):
+ def _get_ds(self, b64encode=False, sys_cfg=None):
+ mod = DataSourceSmartOS
def _get_serial(*_):
- return MockSerial()
+ return MockSerial(b64encode=b64encode)
def _dmi_data():
return mock_returns['system_uuid'], 'smartdc'
- data = {'sys_cfg': {}}
- mod = DataSourceSmartOS
+ if not sys_cfg:
+ sys_cfg = {}
+
+ data = {'sys_cfg': sys_cfg}
self.apply_patches([(mod, 'get_serial', _get_serial)])
self.apply_patches([(mod, 'dmi_data', _dmi_data)])
dsrc = mod.DataSourceSmartOS(
@@ -158,6 +170,13 @@
self.assertEquals(mock_returns['root_authorized_keys'],
dsrc.metadata['public-keys'])
+ def test_hostname_b64(self):
+ dsrc = self._get_ds(b64encode=True)
+ ret = dsrc.get_data()
+ self.assertTrue(ret)
+ self.assertEquals(base64.b64encode(mock_returns['hostname']),
+ dsrc.metadata['local-hostname'])
+
def test_hostname(self):
dsrc = self._get_ds()
ret = dsrc.get_data()
@@ -165,6 +184,32 @@
self.assertEquals(mock_returns['hostname'],
dsrc.metadata['local-hostname'])
+ def test_base64(self):
+ """This tests to make sure that SmartOS system key/value pairs
+ are not interpetted as being base64 encoded, while making
+ sure that the others are when 'decode_base64' is set"""
+ dsrc = self._get_ds(sys_cfg={'decode_base64': True},
+ b64encode=True)
+ ret = dsrc.get_data()
+ self.assertTrue(ret)
+ self.assertEquals(mock_returns['hostname'],
+ dsrc.metadata['local-hostname'])
+ self.assertEquals("%s" % mock_returns['user-data'],
+ dsrc.userdata_raw)
+ self.assertEquals(mock_returns['root_authorized_keys'],
+ dsrc.metadata['public-keys'])
+ self.assertEquals(mock_returns['disable_iptables_flag'],
+ dsrc.metadata['iptables_disable'])
+ self.assertEquals(mock_returns['enable_motd_sys_info'],
+ dsrc.metadata['motd_sys_info'])
+
+ def test_userdata(self):
+ dsrc = self._get_ds()
+ ret = dsrc.get_data()
+ self.assertTrue(ret)
+ self.assertEquals("%s\n" % mock_returns['user-data'],
+ dsrc.userdata_raw)
+
def test_disable_iptables_flag(self):
dsrc = self._get_ds()
ret = dsrc.get_data()
