cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #00897
[Merge] lp:~smoser/cloud-init/trunk.fix-networking into lp:cloud-init
Scott Moser has proposed merging lp:~smoser/cloud-init/trunk.fix-networking into lp:cloud-init.
Requested reviews:
cloud init development team (cloud-init-dev)
Related bugs:
Bug #1577844 in ifupdown (Ubuntu): "Drop unnecessary blocking of all net udev rules"
https://bugs.launchpad.net/ubuntu/+source/ifupdown/+bug/1577844
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/trunk.fix-networking/+merge/296272
--
Your team cloud init development team is requested to review the proposed merge of lp:~smoser/cloud-init/trunk.fix-networking into lp:cloud-init.
=== modified file 'bin/cloud-init'
--- bin/cloud-init 2016-04-15 17:54:05 +0000
+++ bin/cloud-init 2016-06-02 00:36:33 +0000
@@ -211,27 +211,27 @@
util.logexc(LOG, "Failed to initialize, likely bad things to come!")
# Stage 4
path_helper = init.paths
- if not args.local:
+ mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK
+
+ if mode == sources.DSMODE_NETWORK:
existing = "trust"
sys.stderr.write("%s\n" % (netinfo.debug_info()))
LOG.debug(("Checking to see if files that we need already"
" exist from a previous run that would allow us"
" to stop early."))
+ # no-net is written by upstart cloud-init-nonet when network failed
+ # to come up
stop_files = [
os.path.join(path_helper.get_cpath("data"), "no-net"),
- path_helper.get_ipath_cur("obj_pkl"),
]
existing_files = []
for fn in stop_files:
- try:
- c = util.load_file(fn)
- if len(c):
- existing_files.append((fn, len(c)))
- except Exception:
- pass
+ if os.path.isfile(fn):
+ existing_files.append(fn)
+
if existing_files:
- LOG.debug("Exiting early due to the existence of %s files",
- existing_files)
+ LOG.debug("[%s] Exiting. stop file %s existed",
+ mode, existing_files)
return (None, [])
else:
LOG.debug("Execution continuing, no previous run detected that"
@@ -248,34 +248,50 @@
# Stage 5
try:
init.fetch(existing=existing)
+ # if in network mode, and the datasource is local
+ # then work was done at that stage.
+ if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode:
+ LOG.debug("[%s] Exiting. datasource %s in local mode",
+ mode, init.datasource)
+ return (None, [])
except sources.DataSourceNotFoundException:
# In the case of 'cloud-init init' without '--local' it is a bit
# more likely that the user would consider it failure if nothing was
# found. When using upstart it will also mentions job failure
# in console log if exit code is != 0.
- if args.local:
+ if mode == sources.DSMODE_LOCAL:
LOG.debug("No local datasource found")
else:
util.logexc(LOG, ("No instance datasource found!"
" Likely bad things to come!"))
if not args.force:
- init.apply_network_config()
- if args.local:
+ init.apply_network_config(bring_up=not args.local)
+ LOG.debug("[%s] Exiting without datasource in local mode", mode)
+ if mode == sources.DSMODE_LOCAL:
return (None, [])
else:
return (None, ["No instance datasource found."])
-
- if args.local:
- if not init.ds_restored:
- # if local mode and the datasource was not restored from cache
- # (this is not first boot) then apply networking.
- init.apply_network_config()
else:
- LOG.debug("skipping networking config from restored datasource.")
+ LOG.debug("[%s] barreling on in force mode without datasource",
+ mode)
# Stage 6
iid = init.instancify()
- LOG.debug("%s will now be targeting instance id: %s", name, iid)
+ LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s",
+ mode, name, iid, init.is_new_instance())
+
+ init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL))
+
+ if mode == sources.DSMODE_LOCAL:
+ if init.datasource.dsmode != mode:
+ LOG.debug("[%s] Exiting. datasource %s not in local mode.",
+ mode, init.datasource)
+ return (init.datasource, [])
+ else:
+ LOG.debug("[%s] %s is in local mode, will apply init modules now.",
+ mode, init.datasource)
+
+ # update fully realizes user-data (pulling in #include if necessary)
init.update()
# Stage 7
try:
@@ -528,7 +544,7 @@
v1[mode]['errors'] = [str(e) for e in errors]
except Exception as e:
- util.logexc(LOG, "failed of stage %s", mode)
+ util.logexc(LOG, "failed stage %s", mode)
print_exc("failed run of stage %s" % mode)
v1[mode]['errors'] = [str(e)]
=== modified file 'cloudinit/distros/__init__.py'
--- cloudinit/distros/__init__.py 2016-05-12 17:56:26 +0000
+++ cloudinit/distros/__init__.py 2016-06-02 00:36:33 +0000
@@ -31,6 +31,7 @@
from cloudinit import importer
from cloudinit import log as logging
+from cloudinit import net
from cloudinit import ssh_util
from cloudinit import type_utils
from cloudinit import util
@@ -128,6 +129,8 @@
mirror_info=arch_info)
def apply_network(self, settings, bring_up=True):
+ # this applies network where 'settings' is interfaces(5) style
+ # it is obsolete compared to apply_network_config
# Write it out
dev_names = self._write_network(settings)
# Now try to bring them up
@@ -143,6 +146,9 @@
return self._bring_up_interfaces(dev_names)
return False
+ def apply_network_config_names(self, netconfig):
+ net.apply_network_config_names(netconfig)
+
@abc.abstractmethod
def apply_locale(self, locale, out_fn=None):
raise NotImplementedError()
=== modified file 'cloudinit/helpers.py'
--- cloudinit/helpers.py 2016-05-12 17:56:26 +0000
+++ cloudinit/helpers.py 2016-06-02 00:36:33 +0000
@@ -328,6 +328,7 @@
self.cfgs = path_cfgs
# Populate all the initial paths
self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud')
+ self.run_dir = path_cfgs.get('run_dir', '/run/cloud-init')
self.instance_link = os.path.join(self.cloud_dir, 'instance')
self.boot_finished = os.path.join(self.instance_link, "boot-finished")
self.upstart_conf_d = path_cfgs.get('upstart_dir')
@@ -349,26 +350,19 @@
"data": "data",
"vendordata_raw": "vendor-data.txt",
"vendordata": "vendor-data.txt.i",
+ "instance_id": ".instance-id",
}
# Set when a datasource becomes active
self.datasource = ds
# get_ipath_cur: get the current instance path for an item
def get_ipath_cur(self, name=None):
- ipath = self.instance_link
- add_on = self.lookups.get(name)
- if add_on:
- ipath = os.path.join(ipath, add_on)
- return ipath
+ return self._get_path(self.instance_link, name)
# get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
# for a name in dirmap
def get_cpath(self, name=None):
- cpath = self.cloud_dir
- add_on = self.lookups.get(name)
- if add_on:
- cpath = os.path.join(cpath, add_on)
- return cpath
+ return self._get_path(self.cloud_dir, name)
# _get_ipath : get the instance path for a name in pathmap
# (/var/lib/cloud/instances/<instance>/<name>)
@@ -397,6 +391,14 @@
else:
return ipath
+ def _get_path(self, base, name=None):
+ if name is None:
+ return base
+ return os.path.join(base, self.lookups[name])
+
+ def get_runpath(self, name=None):
+ return self._get_path(self.run_dir, name)
+
# This config parser will not throw when sections don't exist
# and you are setting values on those sections which is useful
=== modified file 'cloudinit/net/__init__.py'
--- cloudinit/net/__init__.py 2016-05-12 17:56:26 +0000
+++ cloudinit/net/__init__.py 2016-06-02 00:36:33 +0000
@@ -768,4 +768,206 @@
return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
+def convert_eni_data(eni_data):
+ # return a network config representation of what is in eni_data
+ ifaces = {}
+ parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None)
+ return _ifaces_to_net_config_data(ifaces)
+
+
+def _ifaces_to_net_config_data(ifaces):
+ """Return network config that represents the ifaces data provided.
+ ifaces = parse_deb_config("/etc/network/interfaces")
+ config = ifaces_to_net_config_data(ifaces)
+ state = parse_net_config_data(config)."""
+ devs = {}
+ for name, data in ifaces.items():
+ # devname is 'eth0' for name='eth0:1'
+ devname = name.partition(":")[0]
+ if devname not in devs:
+ devs[devname] = {'type': 'physical', 'name': devname,
+ 'subnets': []}
+ # this isnt strictly correct, but some might specify
+ # hwaddress on a nic for matching / declaring name.
+ if 'hwaddress' in data:
+ devs[devname]['mac_address'] = data['hwaddress']
+ subnet = {'_orig_eni_name': name, 'type': data['method']}
+ if data.get('auto'):
+ subnet['control'] = 'auto'
+ else:
+ subnet['control'] = 'manual'
+
+ if data.get('method') == 'static':
+ subnet['address'] = data['address']
+
+ if 'gateway' in data:
+ subnet['gateway'] = data['gateway']
+
+ if 'dns' in data:
+ for n in ('nameservers', 'search'):
+ if n in data['dns'] and data['dns'][n]:
+ subnet['dns_' + n] = data['dns'][n]
+ devs[devname]['subnets'].append(subnet)
+
+ return {'version': 1,
+ 'config': [devs[d] for d in sorted(devs)]}
+
+
+def apply_network_config_names(netcfg, strict_present=True, strict_busy=True):
+ """read the network config and rename devices accordingly.
+ if strict_present is false, then do not raise exception if no devices
+ match. if strict_busy is false, then do not raise exception if the
+ device cannot be renamed because it is currently configured."""
+ renames = []
+ for ent in netcfg.get('config', {}):
+ if ent.get('type') != 'physical':
+ continue
+ mac = ent.get('mac_address')
+ name = ent.get('name')
+ if not mac:
+ continue
+ renames.append([mac, name])
+
+ return rename_interfaces(renames)
+
+
+def _get_current_rename_info(check_downable=True):
+ """Collect information necessary for rename_interfaces."""
+ names = get_devicelist()
+ bymac = {}
+ for n in names:
+ bymac[get_interface_mac(n)] = {
+ 'name': n, 'up': is_up(n), 'downable': None}
+
+ if check_downable:
+ nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]")
+ ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent',
+ 'scope', 'global'], capture=True)
+ ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True)
+
+ nics_with_addresses = set()
+ for bytes_out in (ipv6, ipv4):
+ nics_with_addresses.update(nmatch.findall(bytes_out))
+
+ for d in bymac.values():
+ d['downable'] = (d['up'] is False or
+ d['name'] not in nics_with_addresses)
+
+ return bymac
+
+
+def rename_interfaces(renames, strict_present=True, strict_busy=True,
+ current_info=None):
+ if current_info is None:
+ current_info = _get_current_rename_info()
+
+ cur_bymac = {}
+ for mac, data in current_info.items():
+ cur = data.copy()
+ cur['mac'] = mac
+ cur_bymac[mac] = cur
+
+ def update_byname(bymac):
+ return {data['name']: data for data in bymac.values()}
+
+ def rename(cur, new):
+ util.subp(["ip", "link", "set", cur, "name", new], capture=True)
+
+ def down(name):
+ util.subp(["ip", "link", "set", name, "down"], capture=True)
+
+ def up(name):
+ util.subp(["ip", "link", "set", name, "up"], capture=True)
+
+ ops = []
+ errors = []
+ ups = []
+ cur_byname = update_byname(cur_bymac)
+ tmpname_fmt = "cirename%d"
+ tmpi = -1
+
+ for mac, new_name in renames:
+ cur = cur_bymac.get(mac, {})
+ cur_name = cur.get('name')
+ cur_ops = []
+ if cur_name == new_name:
+ # nothing to do
+ continue
+
+ if not cur_name:
+ if strict_present:
+ errors.append(
+ "[nic not present] Cannot rename mac=%s to %s"
+ ", not available." % (mac, new_name))
+ continue
+
+ if cur['up']:
+ msg = "[busy] Error renaming mac=%s from %s to %s"
+ if not cur['downable']:
+ if strict_busy:
+ errors.append(msg % (mac, cur_name, new_name))
+ continue
+ cur['up'] = False
+ cur_ops.append(("down", mac, new_name, (cur_name,)))
+ ups.append(("up", mac, new_name, (new_name,)))
+
+ if new_name in cur_byname:
+ target = cur_byname[new_name]
+ if target['up']:
+ msg = "[busy-target] Error renaming mac=%s from %s to %s."
+ if not target['downable']:
+ if strict_busy:
+ errors.append(msg % (mac, cur_name, new_name))
+ continue
+ else:
+ cur_ops.append(("down", mac, new_name, (new_name,)))
+
+ tmp_name = None
+ while tmp_name is None or tmp_name in cur_byname:
+ tmpi += 1
+ tmp_name = tmpname_fmt % tmpi
+
+ cur_ops.append(("rename", mac, new_name, (new_name, tmp_name)))
+ target['name'] = tmp_name
+ cur_byname = update_byname(cur_bymac)
+ if target['up']:
+ ups.append(("up", mac, new_name, (tmp_name,)))
+
+ cur_ops.append(("rename", mac, new_name, (cur['name'], new_name)))
+ cur['name'] = new_name
+ cur_byname = update_byname(cur_bymac)
+ ops += cur_ops
+
+ opmap = {'rename': rename, 'down': down, 'up': up}
+
+ if len(ops) + len(ups) == 0:
+ if len(errors):
+ LOG.debug("unable to do any work for renaming of %s", renames)
+ else:
+ LOG.debug("no work necessary for renaming of %s", renames)
+ else:
+ LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups)
+
+ for op, mac, new_name, params in ops + ups:
+ try:
+ opmap.get(op)(*params)
+ except Exception as e:
+ errors.append(
+ "[unknown] Error performing %s%s for %s, %s: %s" %
+ (op, params, mac, new_name, e))
+
+ if len(errors):
+ raise Exception('\n'.join(errors))
+
+
+def get_interface_mac(ifname):
+ """Returns the string value of an interface's MAC Address"""
+ return read_sys_net(ifname, "address", enoent=False)
+
+
+def get_ifname_mac_pairs():
+ """Build a list of tuples (ifname, mac)"""
+ return [(ifname, get_interface_mac(ifname)) for ifname in get_devicelist()]
+
+
# vi: ts=4 expandtab syntax=python
=== modified file 'cloudinit/sources/DataSourceCloudSigma.py'
--- cloudinit/sources/DataSourceCloudSigma.py 2016-05-12 17:56:26 +0000
+++ cloudinit/sources/DataSourceCloudSigma.py 2016-06-02 00:36:33 +0000
@@ -27,8 +27,6 @@
LOG = logging.getLogger(__name__)
-VALID_DSMODES = ("local", "net", "disabled")
-
class DataSourceCloudSigma(sources.DataSource):
"""
@@ -38,7 +36,6 @@
http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html
"""
def __init__(self, sys_cfg, distro, paths):
- self.dsmode = 'local'
self.cepko = Cepko()
self.ssh_public_key = ''
sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -84,11 +81,9 @@
LOG.debug("CloudSigma: Unable to read from serial port")
return False
- dsmode = server_meta.get('cloudinit-dsmode', self.dsmode)
- if dsmode not in VALID_DSMODES:
- LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode)
- dsmode = 'net'
- if dsmode == "disabled" or dsmode != self.dsmode:
+ self.dsmode = self._determine_dsmode(
+ [server_meta.get('cloudinit-dsmode')])
+ if dsmode == sources.DSMODE_DISABLED:
return False
base64_fields = server_meta.get('base64_fields', '').split(',')
@@ -120,17 +115,10 @@
return self.metadata['uuid']
-class DataSourceCloudSigmaNet(DataSourceCloudSigma):
- def __init__(self, sys_cfg, distro, paths):
- DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths)
- self.dsmode = 'net'
-
-
# Used to match classes to dependencies. Since this datasource uses the serial
# port network is not really required, so it's okay to load without it, too.
datasources = [
(DataSourceCloudSigma, (sources.DEP_FILESYSTEM)),
- (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
]
=== modified file 'cloudinit/sources/DataSourceConfigDrive.py'
--- cloudinit/sources/DataSourceConfigDrive.py 2016-04-29 13:04:36 +0000
+++ cloudinit/sources/DataSourceConfigDrive.py 2016-06-02 00:36:33 +0000
@@ -22,6 +22,7 @@
import os
from cloudinit import log as logging
+from cloudinit import net
from cloudinit import sources
from cloudinit import util
@@ -35,7 +36,6 @@
DEFAULT_METADATA = {
"instance-id": DEFAULT_IID,
}
-VALID_DSMODES = ("local", "net", "pass", "disabled")
FS_TYPES = ('vfat', 'iso9660')
LABEL_TYPES = ('config-2',)
POSSIBLE_MOUNTS = ('sr', 'cd')
@@ -47,12 +47,12 @@
def __init__(self, sys_cfg, distro, paths):
super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths)
self.source = None
- self.dsmode = 'local'
self.seed_dir = os.path.join(paths.seed_dir, 'config_drive')
self.version = None
self.ec2_metadata = None
self._network_config = None
self.network_json = None
+ self.network_eni = None
self.files = {}
def __str__(self):
@@ -98,38 +98,22 @@
md = results.get('metadata', {})
md = util.mergemanydict([md, DEFAULT_METADATA])
- user_dsmode = results.get('dsmode', None)
- if user_dsmode not in VALID_DSMODES + (None,):
- LOG.warn("User specified invalid mode: %s", user_dsmode)
- user_dsmode = None
-
- dsmode = get_ds_mode(cfgdrv_ver=results['version'],
- ds_cfg=self.ds_cfg.get('dsmode'),
- user=user_dsmode)
-
- if dsmode == "disabled":
- # most likely user specified
+
+ self.dsmode = self._determine_dsmode(
+ [results.get('dsmode'), self.ds_cfg.get('dsmode'),
+ sources.DSMODE_PASS if results['version'] == 1 else None])
+
+ if self.dsmode == sources.DSMODE_DISABLED:
return False
- # TODO(smoser): fix this, its dirty.
- # we want to do some things (writing files and network config)
- # only on first boot, and even then, we want to do so in the
- # local datasource (so they happen earlier) even if the configured
- # dsmode is 'net' or 'pass'. To do this, we check the previous
- # instance-id
+ # This is legacy and sneaky. If dsmode is 'pass' then write
+ # 'injected files' and apply legacy ENI network format.
prev_iid = get_previous_iid(self.paths)
cur_iid = md['instance-id']
- if prev_iid != cur_iid and self.dsmode == "local":
+ if prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS:
on_first_boot(results, distro=self.distro)
-
- # dsmode != self.dsmode here if:
- # * dsmode = "pass", pass means it should only copy files and then
- # pass to another datasource
- # * dsmode = "net" and self.dsmode = "local"
- # so that user boothooks would be applied with network, the
- # local datasource just gets out of the way, and lets the net claim
- if dsmode != self.dsmode:
- LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
+ LOG.debug("%s: not claiming datasource, dsmode=%s", self,
+ self.dsmode)
return False
self.source = found
@@ -147,12 +131,11 @@
LOG.warn("Invalid content in vendor-data: %s", e)
self.vendordata_raw = None
- try:
- self.network_json = results.get('networkdata')
- except ValueError as e:
- LOG.warn("Invalid content in network-data: %s", e)
- self.network_json = None
-
+ # network_config is an /etc/network/interfaces formated file and is
+ # obsolete compared to networkdata (from network_data.json) but both
+ # might be present.
+ self.network_eni = results.get("network_config")
+ self.network_json = results.get('networkdata')
return True
def check_instance_id(self, sys_cfg):
@@ -163,41 +146,16 @@
def network_config(self):
if self._network_config is None:
if self.network_json is not None:
+ LOG.debug("network config provided via network_json")
self._network_config = convert_network_data(self.network_json)
+ elif self.network_eni is not None:
+ self._network_config = net.convert_eni_data(self.network_eni)
+ LOG.debug("network config provided via converted eni data")
+ else:
+ LOG.debug("no network configuration available")
return self._network_config
-class DataSourceConfigDriveNet(DataSourceConfigDrive):
- def __init__(self, sys_cfg, distro, paths):
- DataSourceConfigDrive.__init__(self, sys_cfg, distro, paths)
- self.dsmode = 'net'
-
-
-def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None):
- """Determine what mode should be used.
- valid values are 'pass', 'disabled', 'local', 'net'
- """
- # user passed data trumps everything
- if user is not None:
- return user
-
- if ds_cfg is not None:
- return ds_cfg
-
- # at config-drive version 1, the default behavior was pass. That
- # meant to not use use it as primary data source, but expect a ec2 metadata
- # source. for version 2, we default to 'net', which means
- # the DataSourceConfigDriveNet, would be used.
- #
- # this could change in the future. If there was definitive metadata
- # that indicated presense of an openstack metadata service, then
- # we could change to 'pass' by default also. The motivation for that
- # would be 'cloud-init query' as the web service could be more dynamic
- if cfgdrv_ver == 1:
- return "pass"
- return "net"
-
-
def read_config_drive(source_dir):
reader = openstack.ConfigDriveReader(source_dir)
finders = [
@@ -231,9 +189,12 @@
% (type(data)))
net_conf = data.get("network_config", '')
if net_conf and distro:
- LOG.debug("Updating network interfaces from config drive")
+ LOG.warn("Updating network interfaces from config drive")
distro.apply_network(net_conf)
- files = data.get('files', {})
+ write_injected_files(data.get('files'))
+
+
+def write_injected_files(files):
if files:
LOG.debug("Writing %s injected files", len(files))
for (filename, content) in files.items():
@@ -296,7 +257,6 @@
# Used to match classes to dependencies
datasources = [
(DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )),
- (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
]
=== modified file 'cloudinit/sources/DataSourceNoCloud.py'
--- cloudinit/sources/DataSourceNoCloud.py 2016-05-12 17:56:26 +0000
+++ cloudinit/sources/DataSourceNoCloud.py 2016-06-02 00:36:33 +0000
@@ -24,6 +24,7 @@
import os
from cloudinit import log as logging
+from cloudinit import net
from cloudinit import sources
from cloudinit import util
@@ -35,7 +36,6 @@
sources.DataSource.__init__(self, sys_cfg, distro, paths)
self.dsmode = 'local'
self.seed = None
- self.cmdline_id = "ds=nocloud"
self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'),
os.path.join(paths.seed_dir, 'nocloud-net')]
self.seed_dir = None
@@ -58,7 +58,7 @@
try:
# Parse the kernel command line, getting data passed in
md = {}
- if parse_cmdline_data(self.cmdline_id, md):
+ if load_cmdline_data(md):
found.append("cmdline")
mydata = _merge_new_seed(mydata, {'meta-data': md})
except Exception:
@@ -123,12 +123,6 @@
mydata = _merge_new_seed(mydata, seeded)
- # For seed from a device, the default mode is 'net'.
- # that is more likely to be what is desired. If they want
- # dsmode of local, then they must specify that.
- if 'dsmode' not in mydata['meta-data']:
- mydata['meta-data']['dsmode'] = "net"
-
LOG.debug("Using data from %s", dev)
found.append(dev)
break
@@ -144,7 +138,6 @@
if len(found) == 0:
return False
- seeded_network = None
# The special argument "seedfrom" indicates we should
# attempt to seed the userdata / metadata from its value
# its primarily value is in allowing the user to type less
@@ -160,10 +153,6 @@
LOG.debug("Seed from %s not supported by %s", seedfrom, self)
return False
- if (mydata['meta-data'].get('network-interfaces') or
- mydata.get('network-config')):
- seeded_network = self.dsmode
-
# This could throw errors, but the user told us to do it
# so if errors are raised, let them raise
(md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
@@ -179,35 +168,21 @@
mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
defaults])
- netdata = {'format': None, 'data': None}
- if mydata['meta-data'].get('network-interfaces'):
- netdata['format'] = 'interfaces'
- netdata['data'] = mydata['meta-data']['network-interfaces']
- elif mydata.get('network-config'):
- netdata['format'] = 'network-config'
- netdata['data'] = mydata['network-config']
-
- # if this is the local datasource or 'seedfrom' was used
- # and the source of the seed was self.dsmode.
- # Then see if there is network config to apply.
- # note this is obsolete network-interfaces style seeding.
- if self.dsmode in ("local", seeded_network):
- if mydata['meta-data'].get('network-interfaces'):
- LOG.debug("Updating network interfaces from %s", self)
- self.distro.apply_network(
- mydata['meta-data']['network-interfaces'])
-
- if mydata['meta-data']['dsmode'] == self.dsmode:
- self.seed = ",".join(found)
- self.metadata = mydata['meta-data']
- self.userdata_raw = mydata['user-data']
- self.vendordata_raw = mydata['vendor-data']
- self._network_config = mydata['network-config']
- return True
-
- LOG.debug("%s: not claiming datasource, dsmode=%s", self,
- mydata['meta-data']['dsmode'])
- return False
+ self.dsmode = self._determine_dsmode(
+ [mydata['meta-data'].get('dsmode')])
+
+ if self.dsmode == sources.DSMODE_DISABLED:
+ LOG.debug("%s: not claiming datasource, dsmode=%s", self,
+ self.dsmode)
+ return False
+
+ self.seed = ",".join(found)
+ self.metadata = mydata['meta-data']
+ self.userdata_raw = mydata['user-data']
+ self.vendordata_raw = mydata['vendor-data']
+ self._network_config = mydata['network-config']
+ self._network_eni = mydata['meta-data'].get('network-interfaces')
+ return True
def check_instance_id(self, sys_cfg):
# quickly (local check only) if self.instance_id is still valid
@@ -227,6 +202,9 @@
@property
def network_config(self):
+ if self._network_config is None:
+ if self.network_eni is not None:
+ self._network_config = net.convert_eni_data(self.network_eni)
return self._network_config
@@ -254,8 +232,22 @@
return None
+def load_cmdline_data(fill, cmdline=None):
+ pairs = [("ds=nocloud", sources.DSMODE_LOCAL),
+ ("ds=nocloud-net", sources.DSMODE_NETWORK)]
+ for idstr, dsmode in pairs:
+ if parse_cmdline_data(idstr, fill, cmdline):
+ # if dsmode was explicitly in the commanad line, then
+ # prefer it to the dsmode based on the command line id
+ if 'dsmode' not in fill:
+ fill['dsmode'] = dsmode
+ return True
+ return False
+
+
# Returns true or false indicating if cmdline indicated
-# that this module should be used
+# that this module should be used. Updates dictionary 'fill'
+# with data that was found.
# Example cmdline:
# root=LABEL=uec-rootfs ro ds=nocloud
def parse_cmdline_data(ds_id, fill, cmdline=None):
@@ -319,9 +311,7 @@
class DataSourceNoCloudNet(DataSourceNoCloud):
def __init__(self, sys_cfg, distro, paths):
DataSourceNoCloud.__init__(self, sys_cfg, distro, paths)
- self.cmdline_id = "ds=nocloud-net"
self.supported_seed_starts = ("http://";, "https://";, "ftp://";)
- self.dsmode = "net"
# Used to match classes to dependencies
=== modified file 'cloudinit/sources/DataSourceOpenNebula.py'
--- cloudinit/sources/DataSourceOpenNebula.py 2016-03-04 06:45:58 +0000
+++ cloudinit/sources/DataSourceOpenNebula.py 2016-06-02 00:36:33 +0000
@@ -37,16 +37,13 @@
LOG = logging.getLogger(__name__)
DEFAULT_IID = "iid-dsopennebula"
-DEFAULT_MODE = 'net'
DEFAULT_PARSEUSER = 'nobody'
CONTEXT_DISK_FILES = ["context.sh"]
-VALID_DSMODES = ("local", "net", "disabled")
class DataSourceOpenNebula(sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
- self.dsmode = 'local'
self.seed = None
self.seed_dir = os.path.join(paths.seed_dir, 'opennebula')
@@ -93,52 +90,27 @@
md = util.mergemanydict([md, defaults])
# check for valid user specified dsmode
- user_dsmode = results['metadata'].get('DSMODE', None)
- if user_dsmode not in VALID_DSMODES + (None,):
- LOG.warn("user specified invalid mode: %s", user_dsmode)
- user_dsmode = None
-
- # decide dsmode
- if user_dsmode:
- dsmode = user_dsmode
- elif self.ds_cfg.get('dsmode'):
- dsmode = self.ds_cfg.get('dsmode')
- else:
- dsmode = DEFAULT_MODE
-
- if dsmode == "disabled":
- # most likely user specified
- return False
-
- # apply static network configuration only in 'local' dsmode
- if ('network-interfaces' in results and self.dsmode == "local"):
- LOG.debug("Updating network interfaces from %s", self)
- self.distro.apply_network(results['network-interfaces'])
-
- if dsmode != self.dsmode:
- LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
+ self.dsmode = self._determine_dsmode(
+ [results.get('DSMODE'), self.ds_cfg.get('dsmode')])
+
+ if self.dsmode == sources.DSMODE_DISABLED:
return False
self.seed = seed
+ self.network_eni = results.get("network_config")
self.metadata = md
self.userdata_raw = results.get('userdata')
return True
def get_hostname(self, fqdn=False, resolve_ip=None):
if resolve_ip is None:
- if self.dsmode == 'net':
+ if self.dsmode == sources.DSMODE_NET:
resolve_ip = True
else:
resolve_ip = False
return sources.DataSource.get_hostname(self, fqdn, resolve_ip)
-class DataSourceOpenNebulaNet(DataSourceOpenNebula):
- def __init__(self, sys_cfg, distro, paths):
- DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths)
- self.dsmode = 'net'
-
-
class NonContextDiskDir(Exception):
pass
@@ -446,7 +418,6 @@
# Used to match classes to dependencies
datasources = [
(DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )),
- (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
]
=== modified file 'cloudinit/sources/DataSourceOpenStack.py'
--- cloudinit/sources/DataSourceOpenStack.py 2016-05-16 23:08:19 +0000
+++ cloudinit/sources/DataSourceOpenStack.py 2016-06-02 00:36:33 +0000
@@ -33,13 +33,11 @@
DEFAULT_METADATA = {
"instance-id": DEFAULT_IID,
}
-VALID_DSMODES = ("net", "disabled")
class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths)
- self.dsmode = 'net'
self.metadata_address = None
self.ssl_details = util.fetch_ssl_details(self.paths)
self.version = None
@@ -125,11 +123,8 @@
self.metadata_address)
return False
- user_dsmode = results.get('dsmode', None)
- if user_dsmode not in VALID_DSMODES + (None,):
- LOG.warn("User specified invalid mode: %s", user_dsmode)
- user_dsmode = None
- if user_dsmode == 'disabled':
+ self.dsmode = self._determine_dsmode([results.get('dsmode')])
+ if self.dsmode == sources.DSMODE_DISABLED:
return False
md = results.get('metadata', {})
=== modified file 'cloudinit/sources/__init__.py'
--- cloudinit/sources/__init__.py 2016-05-12 17:56:26 +0000
+++ cloudinit/sources/__init__.py 2016-06-02 00:36:33 +0000
@@ -34,6 +34,13 @@
from cloudinit.filters import launch_index
from cloudinit.reporting import events
+DSMODE_DISABLED = "disabled"
+DSMODE_LOCAL = "local"
+DSMODE_NETWORK = "net"
+DSMODE_PASS = "pass"
+
+VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK]
+
DEP_FILESYSTEM = "FILESYSTEM"
DEP_NETWORK = "NETWORK"
DS_PREFIX = 'DataSource'
@@ -57,6 +64,7 @@
self.userdata_raw = None
self.vendordata = None
self.vendordata_raw = None
+ self.dsmode = DSMODE_NETWORK
# find the datasource config name.
# remove 'DataSource' from classname on front, and remove 'Net' on end.
@@ -223,10 +231,35 @@
# quickly (local check only) if self.instance_id is still
return False
+ @staticmethod
+ def _determine_dsmode(candidates, default=None, valid=None):
+ # return the first candidate that is non None, warn if not valid
+ if default is None:
+ default = DSMODE_NETWORK
+
+ if valid is None:
+ valid = VALID_DSMODES
+
+ for candidate in candidates:
+ if candidate is None:
+ continue
+ if candidate in valid:
+ return candidate
+ else:
+ LOG.warn("invalid dsmode '%s', using default=%s",
+ candidate, default)
+ return default
+
+ return default
+
@property
def network_config(self):
return None
+ @property
+ def first_instance_boot(self):
+ return
+
def normalize_pubkey_data(pubkey_data):
keys = []
=== modified file 'cloudinit/stages.py'
--- cloudinit/stages.py 2016-05-26 13:02:17 +0000
+++ cloudinit/stages.py 2016-06-02 00:36:33 +0000
@@ -52,6 +52,7 @@
LOG = logging.getLogger(__name__)
NULL_DATA_SOURCE = None
+NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID"
class Init(object):
@@ -67,6 +68,7 @@
# Changed only when a fetch occurs
self.datasource = NULL_DATA_SOURCE
self.ds_restored = False
+ self._previous_iid = None
if reporter is None:
reporter = events.ReportEventStack(
@@ -213,6 +215,31 @@
cfg_list = self.cfg.get('datasource_list') or []
return (cfg_list, pkg_list)
+ def _restore_from_checked_cache(self, existing):
+ if existing not in ("check", "trust"):
+ raise ValueError("Unexpected value for existing: %s" % existing)
+
+ ds = self._restore_from_cache()
+ if not ds:
+ return (None, "no cache found")
+
+ run_iid_fn = self.paths.get_runpath('instance_id')
+ if os.path.exists(run_iid_fn):
+ run_iid = util.load_file(run_iid_fn).strip()
+ else:
+ run_iid = None
+
+ if run_iid == ds.get_instance_id():
+ return (ds, "restored from cache with run check: %s" % ds)
+ elif existing == "trust":
+ return (ds, "restored from cache: %s" % ds)
+ else:
+ if (hasattr(ds, 'check_instance_id') and
+ ds.check_instance_id(self.cfg)):
+ return (ds, "restored from checked cache: %s" % ds)
+ else:
+ return (None, "cache invalid in datasource: %s" % ds)
+
def _get_data_source(self, existing):
if self.datasource is not NULL_DATA_SOURCE:
return self.datasource
@@ -221,19 +248,9 @@
name="check-cache",
description="attempting to read from cache [%s]" % existing,
parent=self.reporter) as myrep:
- ds = self._restore_from_cache()
- if ds and existing == "trust":
- myrep.description = "restored from cache: %s" % ds
- elif ds and existing == "check":
- if (hasattr(ds, 'check_instance_id') and
- ds.check_instance_id(self.cfg)):
- myrep.description = "restored from checked cache: %s" % ds
- else:
- myrep.description = "cache invalid in datasource: %s" % ds
- ds = None
- else:
- myrep.description = "no cache found"
+ ds, desc = self._restore_from_checked_cache(existing)
+ myrep.description = desc
self.ds_restored = bool(ds)
LOG.debug(myrep.description)
@@ -301,23 +318,41 @@
# What the instance id was and is...
iid = self.datasource.get_instance_id()
- previous_iid = None
iid_fn = os.path.join(dp, 'instance-id')
- try:
- previous_iid = util.load_file(iid_fn).strip()
- except Exception:
- pass
- if not previous_iid:
- previous_iid = iid
+
+ previous_iid = self.previous_iid()
util.write_file(iid_fn, "%s\n" % iid)
+ util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid)
util.write_file(os.path.join(dp, 'previous-instance-id'),
"%s\n" % (previous_iid))
+
+ self._write_to_cache()
# Ensure needed components are regenerated
# after change of instance which may cause
# change of configuration
self._reset()
return iid
+ def previous_iid(self):
+ if self._previous_iid is not None:
+ return self._previous_iid
+
+ dp = self.paths.get_cpath('data')
+ iid_fn = os.path.join(dp, 'instance-id')
+ try:
+ self._previous_iid = util.load_file(iid_fn).strip()
+ except Exception:
+ self._previous_iid = NO_PREVIOUS_INSTANCE_ID
+
+ LOG.debug("previous iid found to be %s", self._previous_iid)
+ return self._previous_iid
+
+ def is_new_instance(self):
+ previous = self.previous_iid()
+ ret = (previous == NO_PREVIOUS_INSTANCE_ID or
+ previous != self.datasource.get_instance_id())
+ return ret
+
def fetch(self, existing="check"):
return self._get_data_source(existing=existing)
@@ -332,8 +367,6 @@
reporter=self.reporter)
def update(self):
- if not self._write_to_cache():
- return
self._store_userdata()
self._store_vendordata()
@@ -593,15 +626,27 @@
return (ncfg, loc)
return (net.generate_fallback_config(), "fallback")
- def apply_network_config(self):
+ def apply_network_config(self, bring_up):
netcfg, src = self._find_networking_config()
if netcfg is None:
LOG.info("network config is disabled by %s", src)
return
- LOG.info("Applying network configuration from %s: %s", src, netcfg)
- try:
- return self.distro.apply_network_config(netcfg)
+ try:
+ LOG.debug("applying net config names for %s" % netcfg)
+ self.distro.apply_network_config_names(netcfg)
+ except Exception as e:
+ LOG.warn("Failed to rename devices: %s", e)
+
+ if (self.datasource is not NULL_DATA_SOURCE and
+ not self.is_new_instance()):
+ LOG.debug("not a new instance. network config is not applied.")
+ return
+
+ LOG.info("Applying network configuration from %s bringup=%s: %s",
+ src, bring_up, netcfg)
+ try:
+ return self.distro.apply_network_config(netcfg, bring_up=bring_up)
except NotImplementedError:
LOG.warn("distro '%s' does not implement apply_network_config. "
"networking may not be configured properly." %
=== modified file 'setup.py'
--- setup.py 2016-05-12 20:49:10 +0000
+++ setup.py 2016-06-02 00:36:33 +0000
@@ -184,7 +184,6 @@
(USR + '/share/doc/cloud-init/examples/seed',
[f for f in glob('doc/examples/seed/*') if is_f(f)]),
(LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')]),
- (LIB + '/udev', ['udev/cloud-init-wait']),
]
# Use a subclass for install that handles
# adding on the right init system configuration files
=== modified file 'systemd/cloud-init-generator'
--- systemd/cloud-init-generator 2016-03-19 00:40:54 +0000
+++ systemd/cloud-init-generator 2016-06-02 00:36:33 +0000
@@ -107,9 +107,6 @@
"ln $CLOUD_SYSTEM_TARGET $link_path"
fi
fi
- # this touches /run/cloud-init/enabled, which is read by
- # udev/cloud-init-wait. If not present, it will exit quickly.
- touch "$LOG_D/$ENABLE"
elif [ "$result" = "$DISABLE" ]; then
if [ -f "$link_path" ]; then
if rm -f "$link_path"; then
=== modified file 'tox.ini'
--- tox.ini 2016-05-24 21:05:20 +0000
+++ tox.ini 2016-06-02 00:36:33 +0000
@@ -1,6 +1,7 @@
[tox]
envlist = py27,py3,flake8
-recreate = True
+recreate = False
+skip_install = True
[testenv]
commands = python -m nose {posargs:tests}
=== removed file 'udev/79-cloud-init-net-wait.rules'
--- udev/79-cloud-init-net-wait.rules 2016-03-19 00:40:54 +0000
+++ udev/79-cloud-init-net-wait.rules 1970-01-01 00:00:00 +0000
@@ -1,10 +0,0 @@
-# cloud-init cold/hot-plug blocking mechanism
-# this file blocks further processing of network events
-# until cloud-init local has had a chance to read and apply network
-SUBSYSTEM!="net", GOTO="cloudinit_naming_end"
-ACTION!="add", GOTO="cloudinit_naming_end"
-
-IMPORT{program}="/lib/udev/cloud-init-wait"
-
-LABEL="cloudinit_naming_end"
-# vi: ts=4 expandtab syntax=udevrules
=== removed file 'udev/cloud-init-wait'
--- udev/cloud-init-wait 2016-03-29 13:11:25 +0000
+++ udev/cloud-init-wait 1970-01-01 00:00:00 +0000
@@ -1,70 +0,0 @@
-#!/bin/sh
-
-CI_NET_READY="/run/cloud-init/network-config-ready"
-LOG="/run/cloud-init/${0##*/}.log"
-LOG_INIT=0
-MAX_WAIT=60
-DEBUG=0
-
-block_until_ready() {
- local fname="$1" max="$2"
- [ -f "$fname" ] && return 0
- # udevadm settle below will exit at the first of 3 conditions
- # 1.) timeout 2.) file exists 3.) all in-flight udev events are processed
- # since this is being run from a udev event, the 3 wont happen.
- # thus, this is essentially a inotify wait or timeout on a file in /run
- # that is created by cloud-init-local.
- udevadm settle "--timeout=$max" "--exit-if-exists=$fname"
-}
-
-log() {
- [ -n "${LOG}" ] || return
- [ "${DEBUG:-0}" = "0" ] && return
-
- if [ $LOG_INIT = 0 ]; then
- if [ -d "${LOG%/*}" ] || mkdir -p "${LOG%/*}"; then
- LOG_INIT=1
- else
- echo "${0##*/}: WARN: log init to ${LOG%/*}" 1>&2
- return
- fi
- elif [ "$LOG_INIT" = "-1" ]; then
- return
- fi
- local info="$$ $INTERFACE"
- if [ "$DEBUG" -gt 1 ]; then
- local up idle
- read up idle < /proc/uptime
- info="$$ $INTERFACE $up"
- fi
- echo "[$info]" "$@" >> "$LOG"
-}
-
-main() {
- local name="" readyfile="$CI_NET_READY"
- local info="INTERFACE=${INTERFACE} ID_NET_NAME=${ID_NET_NAME}"
- info="$info ID_NET_NAME_PATH=${ID_NET_NAME_PATH}"
- info="$info MAC_ADDRESS=${MAC_ADDRESS}"
- log "$info"
-
- ## Check to see if cloud-init.target is set. If cloud-init is
- ## disabled we do not want to do anything.
- if [ ! -f "/run/cloud-init/enabled" ]; then
- log "cloud-init disabled"
- return 0
- fi
-
- if [ "${INTERFACE#lo}" != "$INTERFACE" ]; then
- return 0
- fi
-
- block_until_ready "$readyfile" "$MAX_WAIT" ||
- { log "failed waiting for ready on $INTERFACE"; return 1; }
-
- log "net config ready"
-}
-
-main "$@"
-exit
-
-# vi: ts=4 expandtab
Follow ups
