cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #06461
[Merge] ~chad.smith/cloud-init:feature/ec2-secondary-nics into cloud-init:master
Chad Smith has proposed merging ~chad.smith/cloud-init:feature/ec2-secondary-nics into cloud-init:master.
Commit message:
ec2: render secondary IPs on primary nic when present in metadata
Parse local-ipv4s and subnet-ipv4-cidr-block on EC2 metadata version
2018-09-24 to obtain secondary nic private IPs and network mask for
the primary nic.
In adding this feature, convert DataSourceEc2.network_config to
emit network version 2 instead of version 1.
To allow for retaining original network config behavior on earlier
distribution series, surface a datasource config option
configure_secondary_ips which defaults to True on tip.
Older/stable distribution series will set configure_secondary_ips
default to False.
Requested reviews:
Server Team CI bot (server-team-bot): continuous-integration
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/369792
--
Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:feature/ec2-secondary-nics into cloud-init:master.
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 5c017bf..f39b73f 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -54,7 +54,7 @@ class DataSourceEc2(sources.DataSource):
# Priority ordered list of additional metadata versions which will be tried
# for extended metadata content. IPv6 support comes in 2016-09-02
- extended_metadata_versions = ['2016-09-02']
+ extended_metadata_versions = ['2018-09-24', '2016-09-02']
# Setup read_url parameters per get_url_params.
url_max_wait = 120
@@ -332,8 +332,13 @@ class DataSourceEc2(sources.DataSource):
macs_to_nics = {net.get_interface_mac(iface): iface}
net_md = self.metadata.get('network')
if isinstance(net_md, dict):
+ # SRU_BLOCKER: xenial, bionic and disco should default
+ # configure_secondary_ips to False to retain original behavior on
+ # those releases.
result = convert_ec2_metadata_network_config(
- net_md, macs_to_nics=macs_to_nics, fallback_nic=iface)
+ net_md, macs_to_nics=macs_to_nics, fallback_nic=iface,
+ config_secondary_ips=util.is_true(
+ self.ds_cfg.get('configure_secondary_ips', True)))
# RELEASE_BLOCKER: xenial should drop the below if statement,
# because the issue being addressed doesn't exist pre-netplan.
@@ -373,7 +378,7 @@ class DataSourceEc2(sources.DataSource):
if not self.wait_for_metadata_service():
return {}
api_version = self.get_metadata_api_version()
- crawled_metadata = {}
+ crawled_metadata = {'_metadata_api_version': api_version}
try:
crawled_metadata['user-data'] = ec2.get_instance_userdata(
api_version, self.metadata_address)
@@ -388,7 +393,6 @@ class DataSourceEc2(sources.DataSource):
LOG, "Failed reading from metadata address %s",
self.metadata_address)
return {}
- crawled_metadata['_metadata_api_version'] = api_version
return crawled_metadata
@@ -523,8 +527,9 @@ def _collect_platform_data():
return data
-def convert_ec2_metadata_network_config(network_md, macs_to_nics=None,
- fallback_nic=None):
+def convert_ec2_metadata_network_config(
+ network_md, macs_to_nics=None, fallback_nic=None,
+ config_secondary_ips=True):
"""Convert ec2 metadata to network config version 1 data dict.
@param: network_md: 'network' portion of EC2 metadata.
@@ -535,25 +540,52 @@ def convert_ec2_metadata_network_config(network_md, macs_to_nics=None,
not provided, get_interfaces_by_mac is called to get it from the OS.
@param: fallback_nic: Optionally provide the primary nic interface name.
This nic will be guaranteed to minimally have a dhcp4 configuration.
+ @param: config_secondary_ips: Boolean set True to configure any
+ secondary IPs described by the metadata service.
- @return A dict of network config version 1 based on the metadata and macs.
+ @return A dict of network config version 2 based on the metadata and macs.
"""
- netcfg = {'version': 1, 'config': []}
+ netcfg = {'version': 2, 'ethernets': {}}
if not macs_to_nics:
macs_to_nics = net.get_interfaces_by_mac()
macs_metadata = network_md['interfaces']['macs']
for mac, nic_name in macs_to_nics.items():
+ dev_config = {}
nic_metadata = macs_metadata.get(mac)
if not nic_metadata:
continue # Not a physical nic represented in metadata
- nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []}
- nic_cfg['mac_address'] = mac
- if (nic_name == fallback_nic or nic_metadata.get('public-ipv4s') or
- nic_metadata.get('local-ipv4s')):
- nic_cfg['subnets'].append({'type': 'dhcp4'})
+ local_ipv4s = nic_metadata.get('local-ipv4s')
+ if not config_secondary_ips:
+ LOG.debug(
+ 'Skipping secondary IP config because configure_secondary_ips'
+ ' datasource config option is False')
+ elif (nic_name == fallback_nic or
+ nic_metadata.get('public-ipv4s') or
+ local_ipv4s):
+ dev_config['dhcp4'] = True
+ # In version < 2018-09-24 local_ipvs is a str with a single IP
+ if isinstance(local_ipv4s, list) and len(local_ipv4s) > 1:
+ dev_config['addresses'] = []
+ subnet_cidr = nic_metadata.get('subnet-ipv4-cidr-block')
+ if not subnet_cidr or len(subnet_cidr.split('/')) != 2:
+ LOG.warning(
+ 'Could not parse subnet-ipv4-cidr-block %s.'
+ ' Network config for Secondary IPs default to /32',
+ subnet_cidr)
+ prefix = '32'
+ else:
+ _ip, prefix = subnet_cidr.split('/')
+ # Primary nic IP is at index 0 and obtained via dhcp on Ec2
+ # Iterate over all secondary IPs in local_ipv4s at index >= 1
+ for secondary_ip in local_ipv4s[1:]:
+ dev_config['addresses'].append(
+ '{ip}/{prefix}'.format(ip=secondary_ip, prefix=prefix))
if nic_metadata.get('ipv6s'):
- nic_cfg['subnets'].append({'type': 'dhcp6'})
- netcfg['config'].append(nic_cfg)
+ dev_config['dhcp6'] = True
+ dev_config.update({
+ 'match': {'macaddress': mac.lower()},
+ 'set-name': nic_name})
+ netcfg['ethernets'][nic_name] = dev_config
return netcfg
diff --git a/doc/rtd/topics/datasources/ec2.rst b/doc/rtd/topics/datasources/ec2.rst
index 76beca9..5535edf 100644
--- a/doc/rtd/topics/datasources/ec2.rst
+++ b/doc/rtd/topics/datasources/ec2.rst
@@ -79,6 +79,9 @@ The settings that may be configured are:
* **timeout**: the timeout value provided to urlopen for each individual http
request. This is used both when selecting a metadata_url and when crawling
the metadata service. (default: 50)
+ * **configure_secondary_ips**: Boolean (default: True) to allow cloud-init
+ to configure any secondary IPs described by the metadata service.
+ On Ubuntu Xenial, Bionic and Disco, this defaults to False.
An example configuration with the default values is provided below:
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 20d59bf..8ed4c18 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -112,6 +112,96 @@ DEFAULT_METADATA = {
"services": {"domain": "amazonaws.com", "partition": "aws"},
}
+# collected from api version 2018-09-24/ with
+# python3 -c 'import json
+# from cloudinit.ec2_utils import get_instance_metadata as gm
+# print(json.dumps(gm("2018-09-24"), indent=1, sort_keys=True))'
+SECONDARY_IP_METADATA_2018_09_24 = {
+ "ami-id": "ami-0986c2ac728528ac2",
+ "ami-launch-index": "0",
+ "ami-manifest-path": "(unknown)",
+ "block-device-mapping": {
+ "ami": "/dev/sda1",
+ "root": "/dev/sda1"
+ },
+ "events": {
+ "maintenance": {
+ "history": "[]",
+ "scheduled": "[]"
+ }
+ },
+ "hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+ "identity-credentials": {
+ "ec2": {
+ "info": {
+ "AccountId": "329910648901",
+ "Code": "Success",
+ "LastUpdated": "2019-07-06T14:22:56Z"
+ }
+ }
+ },
+ "instance-action": "none",
+ "instance-id": "i-069e01e8cc43732f8",
+ "instance-type": "t2.micro",
+ "local-hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+ "local-ipv4": "172.31.44.13",
+ "mac": "0a:07:84:3d:6e:38",
+ "metrics": {
+ "vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ },
+ "network": {
+ "interfaces": {
+ "macs": {
+ "0a:07:84:3d:6e:38": {
+ "device-number": "0",
+ "interface-id": "eni-0d6335689899ce9cc",
+ "ipv4-associations": {
+ "18.218.219.181": "172.31.44.13"
+ },
+ "local-hostname": ("ip-172-31-44-13.us-east-2."
+ "compute.internal"),
+ "local-ipv4s": [
+ "172.31.44.13",
+ "172.31.45.70"
+ ],
+ "mac": "0a:07:84:3d:6e:38",
+ "owner-id": "329910648901",
+ "public-hostname": ("ec2-18-218-219-181.us-east-2."
+ "compute.amazonaws.com"),
+ "public-ipv4s": "18.218.219.181",
+ "security-group-ids": "sg-0c387755222ba8d2e",
+ "security-groups": "launch-wizard-4",
+ "subnet-id": "subnet-9d7ba0d1",
+ "subnet-ipv4-cidr-block": "172.31.32.0/20",
+ "vpc-id": "vpc-a07f62c8",
+ "vpc-ipv4-cidr-block": "172.31.0.0/16",
+ "vpc-ipv4-cidr-blocks": "172.31.0.0/16"
+ }
+ }
+ }
+ },
+ "placement": {
+ "availability-zone": "us-east-2c"
+ },
+ "profile": "default-hvm",
+ "public-hostname": (
+ "ec2-18-218-219-181.us-east-2.compute.amazonaws.com"),
+ "public-ipv4": "18.218.219.181",
+ "public-keys": {
+ "yourkeyname,e": [
+ "ssh-rsa AAAAW...DZ yourkeyname"
+ ]
+ },
+ "reservation-id": "r-09b4917135cdd33be",
+ "security-groups": "launch-wizard-4",
+ "services": {
+ "domain": "amazonaws.com",
+ "partition": "aws"
+ }
+}
+
+M_PATH_NET = 'cloudinit.sources.DataSourceEc2.net.'
+
def _register_ssh_keys(rfunc, base_url, keys_data):
"""handle ssh key inconsistencies.
@@ -261,30 +351,23 @@ class TestEc2(test_helpers.HttprettyTestCase):
register_mock_metaserver(instance_id_url, None)
return ds
- def test_network_config_property_returns_version_1_network_data(self):
- """network_config property returns network version 1 for metadata.
-
- Only one device is configured even when multiple exist in metadata.
- """
+ def test_network_config_property_returns_version_2_network_data(self):
+ """network_config property returns network version 2 for metadata"""
ds = self._setup_ds(
platform_data=self.valid_platform_data,
sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
md={'md': DEFAULT_METADATA})
- find_fallback_path = (
- 'cloudinit.sources.DataSourceEc2.net.find_fallback_nic')
+ find_fallback_path = M_PATH_NET + 'find_fallback_nic'
with mock.patch(find_fallback_path) as m_find_fallback:
m_find_fallback.return_value = 'eth9'
ds.get_data()
mac1 = '06:17:04:d7:26:09' # Defined in DEFAULT_METADATA
- expected = {'version': 1, 'config': [
- {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9',
- 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
- 'type': 'physical'}]}
- patch_path = (
- 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
- get_interface_mac_path = (
- 'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': '06:17:04:d7:26:09'}, 'set-name': 'eth9',
+ 'dhcp4': True, 'dhcp6': True}}}
+ patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+ get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
with mock.patch(patch_path) as m_get_interfaces_by_mac:
with mock.patch(find_fallback_path) as m_find_fallback:
with mock.patch(get_interface_mac_path) as m_get_mac:
@@ -302,21 +385,45 @@ class TestEc2(test_helpers.HttprettyTestCase):
platform_data=self.valid_platform_data,
sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
md={'md': DEFAULT_METADATA})
- find_fallback_path = (
- 'cloudinit.sources.DataSourceEc2.net.find_fallback_nic')
+ find_fallback_path = M_PATH_NET + 'find_fallback_nic'
with mock.patch(find_fallback_path) as m_find_fallback:
m_find_fallback.return_value = 'eth9'
ds.get_data()
mac1 = '06:17:04:d7:26:0A' # IPv4 only in DEFAULT_METADATA
- expected = {'version': 1, 'config': [
- {'mac_address': '06:17:04:d7:26:0A', 'name': 'eth9',
- 'subnets': [{'type': 'dhcp4'}],
- 'type': 'physical'}]}
- patch_path = (
- 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
- get_interface_mac_path = (
- 'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': mac1.lower()}, 'set-name': 'eth9',
+ 'dhcp4': True}}}
+ patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+ get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
+ with mock.patch(patch_path) as m_get_interfaces_by_mac:
+ with mock.patch(find_fallback_path) as m_find_fallback:
+ with mock.patch(get_interface_mac_path) as m_get_mac:
+ m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+ m_find_fallback.return_value = 'eth9'
+ m_get_mac.return_value = mac1
+ self.assertEqual(expected, ds.network_config)
+
+ def test_network_config_property_secondary_private_ips(self):
+ """network_config property configures any secondary ipv4 addresses.
+
+ Only one device is configured even when multiple exist in metadata.
+ """
+ ds = self._setup_ds(
+ platform_data=self.valid_platform_data,
+ sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+ md={'md': SECONDARY_IP_METADATA_2018_09_24})
+ find_fallback_path = M_PATH_NET + 'find_fallback_nic'
+ with mock.patch(find_fallback_path) as m_find_fallback:
+ m_find_fallback.return_value = 'eth9'
+ ds.get_data()
+
+ mac1 = '0a:07:84:3d:6e:38' # IPv4 with 1 secondary IP
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': mac1}, 'set-name': 'eth9',
+ 'addresses': ['172.31.45.70/20'], 'dhcp4': True}}}
+ patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+ get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
with mock.patch(patch_path) as m_get_interfaces_by_mac:
with mock.patch(find_fallback_path) as m_find_fallback:
with mock.patch(get_interface_mac_path) as m_get_mac:
@@ -352,8 +459,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
register_mock_metaserver(
'http://169.254.169.254/2009-04-04/meta-data/', DEFAULT_METADATA)
mac1 = '06:17:04:d7:26:09' # Defined in DEFAULT_METADATA
- get_interface_mac_path = (
- 'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+ get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
ds.fallback_nic = 'eth9'
with mock.patch(get_interface_mac_path) as m_get_interface_mac:
m_get_interface_mac.return_value = mac1
@@ -362,11 +468,9 @@ class TestEc2(test_helpers.HttprettyTestCase):
self.assertIn(
'Refreshing stale metadata from prior to upgrade',
self.logs.getvalue())
- expected = {'version': 1, 'config': [
- {'mac_address': '06:17:04:d7:26:09',
- 'name': 'eth9',
- 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
- 'type': 'physical'}]}
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': mac1}, 'set-name': 'eth9',
+ 'dhcp4': True, 'dhcp6': True}}}
self.assertEqual(expected, ds.network_config)
def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self):
@@ -546,19 +650,19 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
def setUp(self):
super(TestConvertEc2MetadataNetworkConfig, self).setUp()
- self.mac1 = '06:17:04:d7:26:09'
+ self.mac1 = '06:17:04:D7:26:09'
self.network_metadata = {
'interfaces': {'macs': {
- self.mac1: {'public-ipv4s': '172.31.2.16'}}}}
+ self.mac1: {'mac': self.mac1, 'public-ipv4s': '172.31.2.16'}}}}
def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
"""Any mac absent from metadata is skipped by network config."""
macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'}
# DE:AD:BE:EF:FF:FF represented by OS but not in metadata
- expected = {'version': 1, 'config': [
- {'mac_address': self.mac1, 'type': 'physical',
- 'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+ 'dhcp4': True}}}
self.assertEqual(
expected,
ec2.convert_ec2_metadata_network_config(
@@ -572,9 +676,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
network_metadata_ipv6['interfaces']['macs'][self.mac1])
nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
nic1_metadata.pop('public-ipv4s')
- expected = {'version': 1, 'config': [
- {'mac_address': self.mac1, 'type': 'physical',
- 'name': 'eth9', 'subnets': [{'type': 'dhcp6'}]}]}
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+ 'dhcp6': True}}}
self.assertEqual(
expected,
ec2.convert_ec2_metadata_network_config(
@@ -588,9 +692,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
network_metadata_ipv6['interfaces']['macs'][self.mac1])
nic1_metadata['local-ipv4s'] = '172.3.3.15'
nic1_metadata.pop('public-ipv4s')
- expected = {'version': 1, 'config': [
- {'mac_address': self.mac1, 'type': 'physical',
- 'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+ 'dhcp4': True}}}
self.assertEqual(
expected,
ec2.convert_ec2_metadata_network_config(
@@ -605,9 +709,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
nic1_metadata['public-ipv4s'] = ''
# When no ipv4 or ipv6 content but fallback_nic set, set dhcp4 config.
- expected = {'version': 1, 'config': [
- {'mac_address': self.mac1, 'type': 'physical',
- 'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+ 'dhcp4': True}}}
self.assertEqual(
expected,
ec2.convert_ec2_metadata_network_config(
@@ -622,10 +726,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
nic1_metadata.pop('public-ipv4s')
nic1_metadata['local-ipv4s'] = '10.0.0.42' # Local ipv4 only on vpc
- expected = {'version': 1, 'config': [
- {'mac_address': self.mac1, 'type': 'physical',
- 'name': 'eth9',
- 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+ 'dhcp4': True, 'dhcp6': True}}}
self.assertEqual(
expected,
ec2.convert_ec2_metadata_network_config(
@@ -638,10 +741,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
nic1_metadata = (
network_metadata_both['interfaces']['macs'][self.mac1])
nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
- expected = {'version': 1, 'config': [
- {'mac_address': self.mac1, 'type': 'physical',
- 'name': 'eth9',
- 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+ 'dhcp4': True, 'dhcp6': True}}}
self.assertEqual(
expected,
ec2.convert_ec2_metadata_network_config(
@@ -649,12 +751,10 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self):
"""Convert Ec2 Metadata calls get_interfaces_by_mac by default."""
- expected = {'version': 1, 'config': [
- {'mac_address': self.mac1, 'type': 'physical',
- 'name': 'eth9',
- 'subnets': [{'type': 'dhcp4'}]}]}
- patch_path = (
- 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
+ expected = {'version': 2, 'ethernets': {'eth9': {
+ 'match': {'macaddress': self.mac1.lower()},
+ 'set-name': 'eth9', 'dhcp4': True}}}
+ patch_path = M_PATH_NET + 'get_interfaces_by_mac'
with mock.patch(patch_path) as m_get_interfaces_by_mac:
m_get_interfaces_by_mac.return_value = {self.mac1: 'eth9'}
self.assertEqual(
Follow ups
