cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #00035
[Merge] lp:~smoser/cloud-init/cfg-drive-2 into lp:cloud-init
Scott Moser has proposed merging lp:~smoser/cloud-init/cfg-drive-2 into lp:cloud-init.
Requested reviews:
cloud init development team (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/cfg-drive-2/+merge/121280
Adds support for config-drive-v2
config-drive-v2 of openstack was implemented at https://review.openstack.org/#/c/11184/ .
This should maintain backwards compatibility for the config-drive-v1 also.
--
https://code.launchpad.net/~smoser/cloud-init/cfg-drive-2/+merge/121280
Your team cloud init development team is requested to review the proposed merge of lp:~smoser/cloud-init/cfg-drive-2 into lp:cloud-init.
=== modified file 'cloudinit/sources/DataSourceConfigDrive.py'
--- cloudinit/sources/DataSourceConfigDrive.py 2012-08-22 18:12:32 +0000
+++ cloudinit/sources/DataSourceConfigDrive.py 2012-08-24 20:34:19 +0000
@@ -30,88 +30,119 @@
# Various defaults/constants...
DEFAULT_IID = "iid-dsconfigdrive"
DEFAULT_MODE = 'pass'
-CFG_DRIVE_FILES = [
+CFG_DRIVE_FILES_V1 = [
"etc/network/interfaces",
"root/.ssh/authorized_keys",
"meta.js",
]
DEFAULT_METADATA = {
"instance-id": DEFAULT_IID,
- "dsmode": DEFAULT_MODE,
}
-CFG_DRIVE_DEV_ENV = 'CLOUD_INIT_CONFIG_DRIVE_DEVICE'
+VALID_DSMODES = ("local", "net", "pass", "disabled")
class DataSourceConfigDrive(sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
- self.seed = None
- self.cfg = {}
+ self.source = None
self.dsmode = 'local'
self.seed_dir = os.path.join(paths.seed_dir, 'config_drive')
+ self.version = None
def __str__(self):
- mstr = "%s [%s]" % (util.obj_name(self), self.dsmode)
- mstr += "[seed=%s]" % (self.seed)
+ mstr = "%s [%s,ver=%s]" % (util.obj_name(self), self.dsmode,
+ self.version)
+ mstr += "[source=%s]" % (self.source)
return mstr
def get_data(self):
found = None
md = {}
- ud = ""
+ results = {}
if os.path.isdir(self.seed_dir):
try:
- (md, ud) = read_config_drive_dir(self.seed_dir)
+ results = read_config_drive_dir(self.seed_dir)
found = self.seed_dir
except NonConfigDriveDir:
util.logexc(LOG, "Failed reading config drive from %s",
self.seed_dir)
if not found:
- dev = find_cfg_drive_device()
- if dev:
+ devlist = find_candidate_devs()
+ for dev in devlist:
try:
- (md, ud) = util.mount_cb(dev, read_config_drive_dir)
+ results = util.mount_cb(dev, read_config_drive_dir)
found = dev
+ break
except (NonConfigDriveDir, util.MountFailedError):
pass
+ except BrokenConfigDriveDir:
+ util.logexc(LOG, "broken config drive: %s", dev)
if not found:
return False
- if 'dsconfig' in md:
- self.cfg = md['dscfg']
-
+ md = results['metadata']
md = util.mergedict(md, DEFAULT_METADATA)
- # Update interfaces and ifup only on the local datasource
- # this way the DataSourceConfigDriveNet doesn't do it also.
- if 'network-interfaces' in md and self.dsmode == "local":
+ user_dsmode = results.get('dsmode', None)
+ if user_dsmode not in VALID_DSMODES + (None,):
+ LOG.warn("user specified invalid mode: %s" % user_dsmode)
+ user_dsmode = None
+
+ dsmode = get_ds_mode(cfgdrv_ver=results['cfgdrive_ver'],
+ ds_cfg=self.ds_cfg.get('dsmode'),
+ user=user_dsmode)
+
+ if dsmode == "disabled":
+ # most likely user specified
+ return False
+
+ # TODO(smoser): fix this, its dirty.
+ # we want to do some things (writing files and network config)
+ # only on first boot, and even then, we want to do so in the
+ # local datasource (so they happen earlier) even if the configured
+ # dsmode is 'net' or 'pass'. To do this, we check the previous
+ # instance-id
+ prev_iid = get_previous_iid(self.paths)
+ cur_iid = md['instance-id']
+
+ if ('network_config' in results and self.dsmode == "local" and
+ prev_iid != cur_iid):
LOG.debug("Updating network interfaces from config drive (%s)",
- md['dsmode'])
- self.distro.apply_network(md['network-interfaces'])
-
- self.seed = found
+ dsmode)
+ self.distro.apply_network(results['network_config'])
+
+ # file writing occurs in local mode (to be as early as possible)
+ if self.dsmode == "local" and prev_iid != cur_iid and results['files']:
+ LOG.debug("writing injected files")
+ try:
+ write_files(results['files'])
+ except:
+ util.logexc(LOG, "Failed writing files")
+
+ # dsmode != self.dsmode here if:
+ # * dsmode = "pass", pass means it should only copy files and then
+ # pass to another datasource
+ # * dsmode = "net" and self.dsmode = "local"
+ # so that user boothooks would be applied with network, the
+ # local datasource just gets out of the way, and lets the net claim
+ if dsmode != self.dsmode:
+ LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
+ return False
+
+ self.source = found
self.metadata = md
- self.userdata_raw = ud
-
- if md['dsmode'] == self.dsmode:
- return True
-
- LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode'])
- return False
+ self.userdata_raw = results.get('userdata')
+ self.version = results['cfgdrive_ver']
+
+ return True
def get_public_ssh_keys(self):
if not 'public-keys' in self.metadata:
return []
return self.metadata['public-keys']
- # The data sources' config_obj is a cloud-config formated
- # object that came to it from ways other than cloud-config
- # because cloud-config content would be handled elsewhere
- def get_config_obj(self):
- return self.cfg
-
class DataSourceConfigDriveNet(DataSourceConfigDrive):
def __init__(self, sys_cfg, distro, paths):
@@ -123,48 +154,132 @@
pass
-def find_cfg_drive_device():
- """Get the config drive device. Return a string like '/dev/vdb'
- or None (if there is no non-root device attached). This does not
- check the contents, only reports that if there *were* a config_drive
- attached, it would be this device.
- Note: per config_drive documentation, this is
- "associated as the last available disk on the instance"
+class BrokenConfigDriveDir(Exception):
+ pass
+
+
+def find_candidate_devs():
+ """Return a list of devices that may contain the config drive.
+
+ The returned list is sorted by search order where the first item has
+ should be searched first (highest priority)
+
+ config drive v1:
+ Per documentation, this is "associated as the last available disk on the
+ instance", and should be VFAT.
+ Currently, we do not restrict search list to "last available disk"
+
+ config drive v2:
+ Disk should be:
+ * either vfat or iso9660 formated
+ * labeled with 'config-2'
"""
- # This seems to be for debugging??
- if CFG_DRIVE_DEV_ENV in os.environ:
- return os.environ[CFG_DRIVE_DEV_ENV]
-
- # We are looking for a raw block device (sda, not sda1) with a vfat
- # filesystem on it....
- letters = "abcdefghijklmnopqrstuvwxyz"
- devs = util.find_devs_with("TYPE=vfat")
-
- # Filter out anything not ending in a letter (ignore partitions)
- devs = [f for f in devs if f[-1] in letters]
-
- # Sort them in reverse so "last" device is first
- devs.sort(reverse=True)
-
- if devs:
- return devs[0]
-
- return None
+ by_fstype = (util.find_devs_with("TYPE=vfat") +
+ util.find_devs_with("TYPE=iso9660"))
+ by_label = util.find_devs_with("LABEL=config-2")
+
+ # give preference to "last available disk" (vdb over vda)
+ # note, this is not a perfect rendition of that.
+ by_fstype.sort(reverse=True)
+ by_label.sort(reverse=True)
+
+ # combine list of items by putting by-label items first
+ # followed by fstype items, but with dupes removed
+ combined = (by_label + [d for d in by_fstype if d not in by_label])
+
+ # We are looking for block device (sda, not sda1), ignore partitions
+ combined = [d for d in combined if d[-1] not in "0123456789"]
+
+ return combined
def read_config_drive_dir(source_dir):
- """
- read_config_drive_dir(source_dir):
- read source_dir, and return a tuple with metadata dict and user-data
- string populated. If not a valid dir, raise a NonConfigDriveDir
- """
-
- # TODO(harlowja): fix this for other operating systems...
- # Ie: this is where https://fedorahosted.org/netcf/ or similar should
- # be hooked in... (or could be)
+ last_e = NonConfigDriveDir("Not found")
+ for finder in (read_config_drive_dir_v2, read_config_drive_dir_v1):
+ try:
+ data = finder(source_dir)
+ return data
+ except NonConfigDriveDir as exc:
+ last_e = exc
+ raise last_e
+
+
+def read_config_drive_dir_v2(source_dir, version="latest"):
+ datafiles = (
+ ('metadata',
+ "openstack/%s/meta_data.json" % version, True, json.loads),
+ ('userdata', "openstack/%s/user_data" % version, False, None),
+ ('ec2-metadata', "ec2/latest/metadata.json", False, json.loads),
+ )
+
+ results = {'userdata': None}
+ for (name, path, required, process) in datafiles:
+ fpath = os.path.join(source_dir, path)
+ data = None
+ found = False
+ if os.path.isfile(fpath):
+ try:
+ with open(fpath) as fp:
+ data = fp.read()
+ except Exception as exc:
+ raise BrokenConfigDriveDir("failed to read: %s" % fpath)
+ found = True
+ elif required:
+ raise NonConfigDriveDir("missing mandatory %s" % fpath)
+
+ if found and process:
+ try:
+ data = process(data)
+ except Exception as exc:
+ raise BrokenConfigDriveDir("failed to process: %s" % fpath)
+
+ if found:
+ results[name] = data
+
+ def read_content_path(item):
+ # do not use os.path.join here, as content_path starts with /
+ cpath = os.path.sep.join((source_dir, "openstack",
+ "./%s" % item['content_path']))
+ with open(cpath) as fp:
+ return(fp.read())
+
+ files = {}
+ try:
+ for item in results['metadata'].get('files', {}):
+ files[item['path']] = read_content_path(item)
+
+ # the 'network_config' item in metadata is a content pointer
+ # to the network config that should be applied.
+ # in folsom, it is just a '/etc/network/interfaces' file.
+ item = results['metadata'].get("network_config", None)
+ if item:
+ results['network_config'] = read_content_path(item)
+ except Exception as exc:
+ raise BrokenConfigDriveDir("failed to read file %s: %s" % (item, exc))
+
+ # to openstack, user can specify meta ('nova boot --meta=key=value') and
+ # those will appear under metadata['meta'].
+ # if they specify 'dsmode' they're indicating the mode that they intend
+ # for this datasource to operate in.
+ try:
+ results['dsmode'] = results['metadata']['meta']['dsmode']
+ except KeyError:
+ pass
+
+ results['files'] = files
+ results['cfgdrive_ver'] = 2
+ return results
+
+
+def read_config_drive_dir_v1(source_dir):
+ """
+ read source_dir, and return a tuple with metadata dict, user-data,
+ files and version (1). If not a valid dir, raise a NonConfigDriveDir
+ """
+
found = {}
- for af in CFG_DRIVE_FILES:
+ for af in CFG_DRIVE_FILES_V1:
fn = os.path.join(source_dir, af)
if os.path.isfile(fn):
found[af] = fn
@@ -173,11 +288,10 @@
raise NonConfigDriveDir("%s: %s" % (source_dir, "no files found"))
md = {}
- ud = ""
keydata = ""
if "etc/network/interfaces" in found:
fn = found["etc/network/interfaces"]
- md['network-interfaces'] = util.load_file(fn)
+ md['network_config'] = util.load_file(fn)
if "root/.ssh/authorized_keys" in found:
fn = found["root/.ssh/authorized_keys"]
@@ -197,21 +311,75 @@
(source_dir, "invalid json in meta.js", e))
md['meta_js'] = content
- # Key data override??
+ # keydata in meta_js is preferred over "injected"
keydata = meta_js.get('public-keys', keydata)
if keydata:
lines = keydata.splitlines()
md['public-keys'] = [l for l in lines
if len(l) and not l.startswith("#")]
- for copy in ('dsmode', 'instance-id', 'dscfg'):
- if copy in meta_js:
- md[copy] = meta_js[copy]
-
- if 'user-data' in meta_js:
- ud = meta_js['user-data']
-
- return (md, ud)
+ # config-drive-v1 has no way for openstack to provide the instance-id
+ # so we copy that into metadata from the user input
+ if 'instance-id' in meta_js:
+ md['instance-id'] = meta_js['instance-id']
+
+ results = {'cfgdrive_ver': 1, 'metadata': md}
+
+ # allow the user to specify 'dsmode' in a meta tag
+ if 'dsmode' in meta_js:
+ results['dsmode'] = meta_js['dsmode']
+
+ # config-drive-v1 has no way of specifying user-data, so the user has
+ # to cheat and stuff it in a meta tag also.
+ results['userdata'] = meta_js.get('user-data')
+
+ # this implementation does not support files
+ # (other than network/interfaces and authorized_keys)
+ results['files'] = []
+
+ return results
+
+
+def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None):
+ """Determine what mode should be used.
+ valid values are 'pass', 'disabled', 'local', 'net'
+ """
+ # user passed data trumps everything
+ if user is not None:
+ return user
+
+ if ds_cfg is not None:
+ return ds_cfg
+
+ # at config-drive version 1, the default behavior was pass. That
+ # meant to not use use it as primary data source, but expect a ec2 metadata
+ # source. for version 2, we default to 'net', which means
+ # the DataSourceConfigDriveNet, would be used.
+ #
+ # this could change in the future. If there was definitive metadata
+ # that indicated presense of an openstack metadata service, then
+ # we could change to 'pass' by default also. The motivation for that
+ # would be 'cloud-init query' as the web service could be more dynamic
+ if cfgdrv_ver == 1:
+ return "pass"
+ return "net"
+
+
+def get_previous_iid(paths):
+ fname = os.path.join(paths.get_cpath('data'),
+ 'previous-instance-id')
+ try:
+ with open(fname) as fp:
+ return fp.read()
+ except IOError:
+ return None
+
+
+def write_files(files):
+ for (name, content) in files.iteritems():
+ if name[0] != os.sep:
+ name = os.sep + name
+ util.write_file(name, content, mode=0660)
# Used to match classes to dependencies
=== added file 'tests/unittests/test_datasource/test_configdrive.py'
--- tests/unittests/test_datasource/test_configdrive.py 1970-01-01 00:00:00 +0000
+++ tests/unittests/test_datasource/test_configdrive.py 2012-08-24 20:34:19 +0000
@@ -0,0 +1,170 @@
+from copy import copy
+import json
+import os
+import os.path
+import shutil
+import tempfile
+from unittest import TestCase
+
+from cloudinit.sources import DataSourceConfigDrive as ds
+from cloudinit import util
+
+
+PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n'
+EC2_META = {
+ 'ami-id': 'ami-00000001',
+ 'ami-launch-index': 0,
+ 'ami-manifest-path': 'FIXME',
+ 'block-device-mapping': {
+ 'ami': 'sda1',
+ 'ephemeral0': 'sda2',
+ 'root': '/dev/sda1',
+ 'swap': 'sda3'},
+ 'hostname': 'sm-foo-test.novalocal',
+ 'instance-action': 'none',
+ 'instance-id': 'i-00000001',
+ 'instance-type': 'm1.tiny',
+ 'local-hostname': 'sm-foo-test.novalocal',
+ 'local-ipv4': None,
+ 'placement': {'availability-zone': 'nova'},
+ 'public-hostname': 'sm-foo-test.novalocal',
+ 'public-ipv4': '',
+ 'public-keys': {'0': {'openssh-key': PUBKEY}},
+ 'reservation-id': 'r-iru5qm4m',
+ 'security-groups': ['default']
+}
+USER_DATA = '#!/bin/sh\necho This is user data\n'
+OSTACK_META = {
+ 'availability_zone': 'nova',
+ 'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'},
+ {'content_path': '/content/0001', 'path': '/etc/bar/bar.cfg'}],
+ 'hostname': 'sm-foo-test.novalocal',
+ 'meta': {'dsmode': 'local', 'my-meta': 'my-value'},
+ 'name': 'sm-foo-test',
+ 'public_keys': {'mykey': PUBKEY},
+ 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'}
+
+CONTENT_0 = 'This is contents of /etc/foo.cfg\n'
+CONTENT_1 = '# this is /etc/bar/bar.cfg\n'
+
+CFG_DRIVE_FILES_V2 = {
+ 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META),
+ 'ec2/2009-04-04/user-data': USER_DATA,
+ 'ec2/latest/meta-data.json': json.dumps(EC2_META),
+ 'ec2/latest/user-data': USER_DATA,
+ 'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META),
+ 'openstack/2012-08-10/user_data': USER_DATA,
+ 'openstack/content/0000': CONTENT_0,
+ 'openstack/content/0001': CONTENT_1,
+ 'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
+ 'openstack/latest/user_data': USER_DATA}
+
+
+class TestConfigDriveDataSource(TestCase):
+
+ def setUp(self):
+ super(TestConfigDriveDataSource, self).setUp()
+ self.tmp = tempfile.mkdtemp()
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(self.tmp)
+ except OSError:
+ pass
+
+ def test_dir_valid(self):
+ """Verify a dir is read as such."""
+
+ populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
+
+ found = ds.read_config_drive_dir(self.tmp)
+
+ self.assertEqual(USER_DATA, found['userdata'])
+ self.assertEqual(OSTACK_META, found['metadata'])
+ self.assertEqual(found['files']['/etc/foo.cfg'], CONTENT_0)
+ self.assertEqual(found['files']['/etc/bar/bar.cfg'], CONTENT_1)
+
+ def test_seed_dir_valid_extra(self):
+ """Verify extra files do not affect datasource validity."""
+
+ data = copy(CFG_DRIVE_FILES_V2)
+ data["myfoofile.txt"] = "myfoocontent"
+ data["openstack/latest/random-file.txt"] = "random-content"
+
+ populate_dir(self.tmp, data)
+
+ found = ds.read_config_drive_dir(self.tmp)
+ self.assertEqual(OSTACK_META, found['metadata'])
+
+ def test_seed_dir_bad_json_metadata(self):
+ """Verify that bad json in metadata raises BrokenConfigDriveDir."""
+ data = copy(CFG_DRIVE_FILES_V2)
+
+ data["openstack/2012-08-10/meta_data.json"] = "non-json garbage {}"
+ data["openstack/latest/meta_data.json"] = "non-json garbage {}"
+
+ populate_dir(self.tmp, data)
+
+ self.assertRaises(ds.BrokenConfigDriveDir,
+ ds.read_config_drive_dir, self.tmp)
+
+ def test_seed_dir_no_configdrive(self):
+ """Verify that no metadata raises NonConfigDriveDir."""
+
+ my_d = os.path.join(self.tmp, "non-configdrive")
+ data = copy(CFG_DRIVE_FILES_V2)
+ data["myfoofile.txt"] = "myfoocontent"
+ data["openstack/latest/random-file.txt"] = "random-content"
+ data["content/foo"] = "foocontent"
+
+ self.assertRaises(ds.NonConfigDriveDir,
+ ds.read_config_drive_dir, my_d)
+
+ def test_seed_dir_missing(self):
+ """Verify that missing seed_dir raises NonConfigDriveDir."""
+ my_d = os.path.join(self.tmp, "nonexistantdirectory")
+ self.assertRaises(ds.NonConfigDriveDir,
+ ds.read_config_drive_dir, my_d)
+
+ def test_find_candidates(self):
+ devs_with_answers = {
+ "TYPE=vfat": [],
+ "TYPE=iso9660": ["/dev/vdb"],
+ "LABEL=config-2": ["/dev/vdb"],
+ }
+
+ def my_devs_with(criteria):
+ return devs_with_answers[criteria]
+
+ try:
+ orig_find_devs_with = util.find_devs_with
+ util.find_devs_with = my_devs_with
+
+ self.assertEqual(["/dev/vdb"], ds.find_candidate_devs())
+
+ # add a vfat item
+ # zdd reverse sorts after vdb, but config-2 label is preferred
+ devs_with_answers['TYPE=vfat'] = ["/dev/zdd"]
+ self.assertEqual(["/dev/vdb", "/dev/zdd"],
+ ds.find_candidate_devs())
+
+ # verify that partitions are not considered
+ devs_with_answers = {"TYPE=vfat": ["/dev/sda1"],
+ "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]}
+ self.assertEqual([], ds.find_candidate_devs())
+
+ finally:
+ util.find_devs_with = orig_find_devs_with
+
+
+def populate_dir(seed_dir, files):
+ for (name, content) in files.iteritems():
+ path = os.path.join(seed_dir, name)
+ dirname = os.path.dirname(path)
+ if not os.path.isdir(dirname):
+ os.makedirs(dirname)
+ with open(path, "w") as fp:
+ fp.write(content)
+ fp.close()
+
+# vi: ts=4 expandtab
Follow ups
