cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #01616
[Merge] ~smoser/cloud-init:feature/ds-identify-warn into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:feature/ds-identify-warn into cloud-init:master.
Requested reviews:
cloud init development team (cloud-init-dev)
Related bugs:
Bug #1661693 in cloud-init: "identify brightbox platform to enable Ec2 datasource"
https://bugs.launchpad.net/cloud-init/+bug/1661693
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/318282
--
Your team cloud init development team is requested to review the proposed merge of ~smoser/cloud-init:feature/ds-identify-warn into cloud-init:master.
diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py
index 2d00255..9debe94 100644
--- a/cloudinit/sources/DataSourceAliYun.py
+++ b/cloudinit/sources/DataSourceAliYun.py
@@ -22,6 +22,10 @@ class DataSourceAliYun(EC2.DataSourceEc2):
def get_public_ssh_keys(self):
return parse_public_keys(self.metadata.get('public-keys', {}))
+ @property
+ def cloud_platform(self):
+ return EC2.Platforms.ALIYUN
+
def parse_public_keys(public_keys):
keys = []
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index c657fd0..2fc6396 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -10,6 +10,7 @@
import os
import time
+import textwrap
from cloudinit import ec2_utils as ec2
from cloudinit import log as logging
@@ -22,6 +23,17 @@ LOG = logging.getLogger(__name__)
# Which version we are requesting of the ec2 metadata apis
DEF_MD_VERSION = '2009-04-04'
+STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
+STRICT_ID_DEFAULT = "warn"
+
+
+class Platforms(object):
+ ALIYUN = "AliYun"
+ AWS = "AWS"
+ BRIGHTBOX = "Brightbox"
+ SEEDED = "Seeded"
+ UNKNOWN = "Unknown"
+
class DataSourceEc2(sources.DataSource):
# Default metadata urls that will be used if none are provided
@@ -41,8 +53,18 @@ class DataSourceEc2(sources.DataSource):
self.userdata_raw = seed_ret['user-data']
self.metadata = seed_ret['meta-data']
LOG.debug("Using seeded ec2 data from %s", self.seed_dir)
+ self._cloud_platform = Platforms.SEEDED
return True
+ strict_mode, _sleep = read_strict_mode(
+ util.get_cfg_by_path(self.sys_cfg, STRICT_ID_PATH,
+ STRICT_ID_DEFAULT), ("warn", None))
+
+ LOG.debug("strict_mode: %s, cloud_platform=%s",
+ strict_mode, self.cloud_platform)
+ if strict_mode == "true" and self.cloud_platform == Platforms.UNKNOWN:
+ return False
+
try:
if not self.wait_for_metadata_service():
return False
@@ -51,8 +73,8 @@ class DataSourceEc2(sources.DataSource):
ec2.get_instance_userdata(self.api_ver, self.metadata_address)
self.metadata = ec2.get_instance_metadata(self.api_ver,
self.metadata_address)
- LOG.debug("Crawl of metadata service took %s seconds",
- int(time.time() - start_time))
+ LOG.debug("Crawl of metadata service took %.3f seconds",
+ time.time() - start_time)
return True
except Exception:
util.logexc(LOG, "Failed reading from metadata address %s",
@@ -190,6 +212,159 @@ class DataSourceEc2(sources.DataSource):
return az[:-1]
return None
+ @property
+ def cloud_platform(self):
+ if self._cloud_platform is None:
+ self._cloud_platform = identify_platform()
+ return self._cloud_platform
+
+ def activate(self, cfg, is_new_instance):
+ if not is_new_instance:
+ return
+ if self.cloud_platform != Platforms.UNKNOWN:
+ warn_if_necessary(
+ util.get_cfg_by_path(self.sys_cfg, STRICT_ID_PATH,
+ STRICT_ID_DEFAULT))
+
+
+def read_strict_mode(cfgval, default):
+ try:
+ return parse_strict_mode(cfgval)
+ except ValueError as e:
+ LOG.warn(e)
+ return default
+
+
+def parse_strict_mode(cfgval):
+ # given a mode like:
+ # true, false, warn,[sleep]
+ # return tuple with string mode (true|false|warn) and sleep.
+ if cfgval is True:
+ return 'true', None
+ if cfgval is False:
+ return 'false', None
+
+ if not cfgval:
+ return 'warn', 0
+
+ mode, _, sleep = cfgval.partition(",")
+ if mode not in ('true', 'false', 'warn'):
+ raise ValueError(
+ "Invalid mode '%s' in strict_id setting '%s': "
+ "Expected one of 'true', 'false', 'warn'." % (mode, cfgval))
+
+ if sleep:
+ try:
+ sleep = int(sleep)
+ except ValueError:
+ raise ValueError("Invalid sleep '%s' in strict_id setting '%s': "
+ "not an integer" % (sleep, cfgval))
+ else:
+ sleep = None
+
+ return mode, sleep
+
+
+def warn_if_necessary(cfgval):
+ try:
+ mode, sleep = parse_strict_mode(cfgval)
+ except ValueError as e:
+ LOG.warn(e)
+ return
+
+ if mode == "false":
+ return
+
+ show_warning()
+ if sleep:
+ time.sleep(sleep)
+
+
+def show_warning():
+ message = textwrap.dedent("""\
+ # **************************************************************
+ # This system is using the EC2 Metadata Service, but does not #
+ # appear to be running on Amazon EC2. In the future, #
+ # cloud-init may stop reading metadata from the EC2 Metadata #
+ # Service unless the platform can be identified locally. #
+ # #
+ # If you are seeing this message, please file a bug against #
+ # cloud-init at https://bugs.launchpad.net/cloud-init/+filebug #
+ # Make sure to include the cloud provider your instance is #
+ # running on. #
+ # #
+ # For more information see #
+ # https://bugs.launchpad.net/cloud-init/+bug/1660385 #
+ # #
+ # After you have filed a bug, you can disable this warning by #
+ # launching your instance with the cloud-config below, or #
+ # putting that content into #
+ # /etc/cloud/cloud.cfg.d/99-ec2-datasource.cfg #
+ # #
+ # #cloud-config #
+ # datasource: #
+ # Ec2: #
+ # strict_id: false #
+ # #
+ ****************************************************************""")
+ print(message)
+ LOG.warn(message)
+
+
+def identify_aws(data):
+ # data is a dictionary returned by _collect_platform_data.
+ if (data['uuid'].startswith('ec2') and
+ (data['uuid_source'] == 'hypervisor' or
+ data['uuid'] == data['serial'])):
+ return Platforms.AWS
+
+ return None
+
+
+def identify_brightbox(data):
+ if data['serial'].endswith('brightbox.com'):
+ return Platforms.BRIGHTBOX
+
+
+def identify_platform():
+ # identify the platform and return an entry in Platforms.
+ data = _collect_platform_data()
+ checks = (identify_aws, identify_brightbox, lambda x: Platforms.UNKNOWN)
+ for checker in checks:
+ try:
+ result = checker(data)
+ if result:
+ return result
+ except Exception as e:
+ LOG.warn("calling %s with %s raised exception: %s",
+ checker, data, e)
+
+
+def _collect_platform_data():
+ # returns a dictionary with all lower case values:
+ # uuid: system-uuid from dmi or /sys/hypervisor
+ # uuid_source: 'hypervisor' (/sys/hypervisor/uuid) or 'dmi'
+ # serial: dmi 'system-serial-number' (/sys/.../product_serial)
+ data = {}
+ try:
+ uuid = util.load_file("/sys/hypervisor/uuid").strip()
+ data['uuid_source'] = 'hypervisor'
+ except:
+ uuid = util.read_dmi_data('system-uuid')
+ data['uuid_source'] = 'dmi'
+
+ if uuid is None:
+ uuid = ''
+ data['uuid'] = uuid.lower()
+
+ serial = util.read_dmi_data('system-serial-number')
+ if serial is None:
+ serial = ''
+
+ data['serial'] = serial.lower()
+
+ return data
+
# Used to match classes to dependencies
datasources = [
diff --git a/tools/ds-identify b/tools/ds-identify
index 7bb6386..911d19e 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -4,12 +4,12 @@
# or on the kernel command line. It takes primarily 2 inputs:
# datasource: can specify the datasource that should be used.
# kernel command line option: ci.datasource=<dsname>
-#
+#
# policy: a string that indicates how ds-identify should operate.
# kernel command line option: ci.di.policy=<policy>
# default setting is:
# search,found=all,maybe=all,notfound=disable
-
+#
# report: write config to /run/cloud-init/cloud.cfg.report (instead of
# /run/cloud-init/cloud.cfg, which effectively makes this dry-run).
# enable: do nothing
@@ -29,17 +29,15 @@
# all: enable all DS_MAYBE
# none: ignore any DS_MAYBE
#
-# notfound: (default=disable)
-# disable: disable cloud-init
-# enable: enable cloud-init
+# notfound: (default=disabled)
+# disabled: disable cloud-init
+# enabled: enable cloud-init
#
+# ci.datasource.ec2.strict_id: (true|false|warn[,0-9])
+# if ec2 datasource does not strictly match,
+# return not_found if true
+# return maybe if false or warn*.
#
-# zesty:
-# policy: found=first,maybe=all,none=disable
-# xenial:
-# policy: found=all,maybe=all,none=enable
-# and then at a later date
-
set -u
set -f
@@ -561,10 +559,44 @@ dscheck_OpenNebula() {
return ${DS_NOT_FOUND}
}
+ovf_vmware_guest_customization() {
+ # vmware guest customization
+
+ # virt provider must be vmware
+ [ "${DI_VIRT}" = "vmware" ] || return 1
+
+ # we have to have the plugin to do vmware customization
+ local found="" pkg="" pre="/usr/lib"
+ for pkg in vmware-tools open-vm-tools; do
+ if [ -f "$pre/$pkg/plugins/vmsvc/libdeployPkgPlugin.so" ]; then
+ found="$pkg"; break;
+ fi
+ done
+ [ -n "$found" ] || return 1
+
+ # disable_vmware_customization defaults to False.
+ # any value then other than false means disabled.
+ local key="disable_vmware_customization"
+ local match="" bp="${PATH_CLOUD_CONFD}/cloud.cfg"
+ match="$bp.d/*[Oo][Vv][Ff]*.cfg"
+ if check_config "$key" "$match"; then
+ debug 2 "${_RET_fname} set $key to $_RET"
+ case "$_RET" in
+ 0|false|False) :;;
+ *) return 1;;
+ esac
+ fi
+ return 0
+}
+
dscheck_OVF() {
local p=""
check_seed_dir ovf ovf-env.xml && return "${DS_FOUND}"
+ if ovf_vmware_guest_customization; then
+ return ${DS_FOUND}
+ fi
+
has_cdrom || return ${DS_NOT_FOUND}
# FIXME: currently just return maybe if there is a cdrom
@@ -595,36 +627,118 @@ dscheck_Bigstep() {
return ${DS_NOT_FOUND}
}
-dscheck_Ec2() {
- # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
- # http://paste.ubuntu.com/23630859/
+ec2_read_strict_setting() {
+ # the 'strict_id' setting for Ec2 controls behavior when
+ # the platform does not identify itself directly as Ec2.
+ # order of precedence is:
+ # 1. builtin setting here cloud-init/ds-identify builtin
+ # 2. ds-identify config
+ # 3. system config (/etc/cloud/cloud.cfg.d/*Ec2*.cfg)
+ # 4. kernel command line (undocumented)
+ # 5. user-data or vendor-data (not available here)
+ local default="$1" key="ci.datasource.ec2.strict_id" val=""
+
+ # 4. kernel command line
+ case " ${DI_KERNEL_CMDLINE} " in
+ *\ $key=*\ )
+ val=${DI_KERNEL_CMDLINE##*$key=}
+ val=${val%% *};
+ _RET=${val:-$default}
+ return 0
+ esac
+
+ # 3. look for the key 'strict_id' (datasource/Ec2/strict_id)
+ local match="" bp="${PATH_CLOUD_CONFD}/cloud.cfg"
+ match="$bp.d/*[Ee][Cc]2*.cfg"
+ if check_config strict_id "$match"; then
+ debug 2 "${_RET_fname} set strict_id to $_RET"
+ return 0
+ fi
+
+ # 2. ds-identify config (datasource.ec2.strict)
+ local config=${PATH_DI_CONFIG}
+ if [ -f "$config" ]; then
+ if _read_config "$key" < "$config"; then
+ _RET=${_RET:-$default}
+ return 0
+ fi
+ fi
+
+ # 1. Default
+ _RET=$default
+ return 0
+}
+
+ec2_identify_platform() {
+ local default="$1"
+ local serial=${DI_DMI_PRODUCT_SERIAL}
+
+ # brightbox https://bugs.launchpad.net/cloud-init/+bug/1661693
+ case "$serial" in
+ *brightbox.com) _RET="Brightbox"; return 0;;
+ esac
+
+ # AWS http://docs.aws.amazon.com/AWSEC2/
+ # latest/UserGuide/identify_ec2_instances.html
local uuid="" hvuuid="$PATH_ROOT/sys/hypervisor/uuid"
- is_container && return ${DS_NOT_FOUND}
# if the (basically) xen specific /sys/hypervisor/uuid starts with 'ec2'
if [ -r "$hvuuid" ] && read uuid < "$hvuuid" &&
[ "${uuid#ec2}" != "$uuid" ]; then
- return ${DS_FOUND}
+ _RET="AWS"
+ return 0
fi
# product uuid and product serial start with case insensitive
- local uuid=${DI_DMI_PRODUCT_UUID} serial=${DI_DMI_PRODUCT_SERIAL}
+ local uuid=${DI_DMI_PRODUCT_UUID}
case "$uuid:$serial" in
[Ee][Cc]2*:[Ee][Cc]2)
# both start with ec2, now check for case insenstive equal
- nocase_equal "$uuid" "$serial" && return ${DS_FOUND};;
+ nocase_equal "$uuid" "$serial" &&
+ { _RET="AWS"; return 0; };;
esac
- # search through config files to check for platform
- local f="" match="${PATH_CLOUD_CONFD}/*ec2*.cfg"
- # look for the key 'platform' (datasource/ec2/look_alike/behavior)
- if check_config platform "$match"; then
- if [ "$platform" != "Unknown" ]; then
- _RET="$name"
- return "${DS_FOUND}"
- fi
+ _RET="$default"
+ return 0;
+}
+
+dscheck_Ec2() {
+ check_seed_dir "ec2" meta-data user-data && return ${DS_FOUND}
+ is_container && return ${DS_NOT_FOUND}
+
+ local unknown="Unknown" platform=""
+ if ec2_identify_platform "$unknown"; then
+ platform="$_RET"
+ else
+ warn "Failed to identify ec2 platform. Using '$unknown'."
+ platform=$unknown
+
+ debug 1 "ec2 platform is '$platform'."
+ if [ "$platform" != "$unknown" ]; then
+ return $DS_FOUND
fi
- return ${DS_NOT_FOUND}
+ local default="true"
+ if ec2_read_strict_setting "$default"; then
+ strict="$_RET"
+ else
+ debug 1 "ec2_read_strict returned non-zero: $?. using '$default'."
+ strict="$default"
+ fi
+
+ local key="datasource/Ec2/strict_id"
+ case "$strict" in
+ true|false|warn|warn,[0-9]*) :;;
+ *)
+ warn "$key was set to invalid '$strict'. using '$default'"
+ strict="$default";;
+ esac
+
+ _RET_excfg="datasource: {Ec2: {strict_id: \"$strict\"}}"
+ if [ "$strict" = "true" ]; then
+ return $DS_NOT_FOUND
+ else
+ return $DS_MAYBE
+ fi
}
dscheck_GCE() {
@@ -768,12 +882,22 @@ write_result() {
}
found() {
+ # found(ds1, [ds2 ...], [-- [extra lines]])
local list="" ds=""
# always we write the None datasource last.
- for ds in "$@" None; do
- list="${list:+${list}, }$ds"
+ while [ $# -ne 0 ]; do
+ if [ "$1" = "--" ]; then
+ shift
+ break
+ fi
+ list="${list:+${list}, }$1"
+ shift
done
- write_result "datasource_list: [ $list ]"
+ if [ $# -eq 1 ] && [ -z "$1" ]; then
+ # do not pass an empty line through.
+ shift
+ fi
+ write_result "datasource_list: [ $list ]" "$@"
return
}
@@ -794,8 +918,10 @@ unquote() {
}
_read_config() {
- # reads config from stdin, modifies _rc scoped environment vars.
- # rc_policy and _rc_dsname
+ # reads config from stdin,
+ # if no parameters are set, modifies _rc scoped environment vars.
+ # if keyname is provided, then returns found value of that key.
+ local keyname=${1:-"_unset"}
local line="" hash="#" ckey="" key="" val=""
while read line; do
line=${line%%${hash}*}
@@ -806,15 +932,28 @@ _read_config() {
trim "$key"
key=${_RET}
+ [ "$keyname" != "_unset" ] && [ "$keyname" != "$key" ] &&
+ continue
+
val="${line#*:}"
trim "$val"
unquote "${_RET}"
val=${_RET}
+
+ if [ "$keyname" = "$key" ]; then
+ _RET="$val"
+ return 0
+ fi
+
case "$key" in
datasource) _rc_dsname="$val";;
policy) _rc_policy="$val";;
esac
done
+ if [ "$keyname" = "_unset" ]; then
+ return 1
+ fi
+ return 0
}
parse_warn() {
@@ -980,7 +1119,8 @@ _main() {
return
fi
- local found="" ret="" ds="" maybe=""
+ local found="" ret="" ds="" maybe="" _RET_excfg=""
+ local exfound_cfg="" exmaybe_cfg=""
for ds in ${DI_DSLIST}; do
dscheck_fn="dscheck_${ds}"
debug 2 "Checking for datasource '$ds' via '$dscheck_fn'"
@@ -988,20 +1128,23 @@ _main() {
warn "No check method '$dscheck_fn' for datasource '$ds'"
continue
fi
+ _RET_excfg=""
$dscheck_fn
ret="$?"
case "$ret" in
$DS_FOUND)
debug 1 "check for '$ds' returned found";
+ exfound_cfg="${exfound_cfg:+${exfound_cfg}${CR}}${_RET_excfg}"
found="${found} $ds";;
$DS_MAYBE)
- debug 1 "check for $ds returned maybe";
+ debug 1 "check for '$ds' returned maybe";
+ exmaybe_cfg="${exmaybe_cfg:+${exmaybe_cfg}${CR}}${_RET_excfg}"
maybe="${maybe} $ds";;
- *) debug 2 "check for $ds returned not-found[$ret]";;
+ *) debug 2 "check for '$ds' returned not-found[$ret]";;
esac
done
- debug 2 "found=$found maybe=$maybe"
+ debug 2 "found=${found# } maybe=${maybe# }"
set -- $found
if [ $# -ne 0 ]; then
if [ $# -eq 1 ]; then
@@ -1013,14 +1156,14 @@ _main() {
set -- "$1"
fi
fi
- found "$@"
+ found "$@" -- "${exfound_cfg}"
return
fi
set -- $maybe
if [ $# -ne 0 -a "${DI_ON_MAYBE}" != "none" ]; then
debug 1 "$# datasources returned maybe: $*"
- found "$@"
+ found "$@" -- "${exmaybe_cfg}"
return
fi
Follow ups
