cloud-init-dev team mailing list archive

[Chad Smith <chad.smith@xxxxxxxxxxxxx>]

Chad Smith has proposed merging ~chad.smith/cloud-init:feature/ec2-secondary-nics into cloud-init:master.

Commit message:
ec2: render secondary IPs on primary nic when present in metadata

Parse local-ipv4s and subnet-ipv4-cidr-block on EC2 metadata version
2018-09-24 to obtain secondary nic private IPs and network mask for
the primary nic.


Requested reviews:
  cloud-init commiters (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/369792
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:feature/ec2-secondary-nics into cloud-init:master.

diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 5c017bf..aef5d80 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -54,7 +54,7 @@ class DataSourceEc2(sources.DataSource):
 
     # Priority ordered list of additional metadata versions which will be tried
     # for extended metadata content. IPv6 support comes in 2016-09-02
-    extended_metadata_versions = ['2016-09-02']
+    extended_metadata_versions = ['2018-09-24', '2016-09-02']
 
     # Setup read_url parameters per get_url_params.
     url_max_wait = 120
@@ -536,24 +536,35 @@ def convert_ec2_metadata_network_config(network_md, macs_to_nics=None,
     @param: fallback_nic: Optionally provide the primary nic interface name.
        This nic will be guaranteed to minimally have a dhcp4 configuration.
 
-    @return A dict of network config version 1 based on the metadata and macs.
+    @return A dict of network config version 2 based on the metadata and macs.
     """
-    netcfg = {'version': 1, 'config': []}
+    netcfg = {'version': 2, 'ethernets': {}}
     if not macs_to_nics:
         macs_to_nics = net.get_interfaces_by_mac()
     macs_metadata = network_md['interfaces']['macs']
     for mac, nic_name in macs_to_nics.items():
+        dev_config = {}
         nic_metadata = macs_metadata.get(mac)
         if not nic_metadata:
             continue  # Not a physical nic represented in metadata
-        nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []}
-        nic_cfg['mac_address'] = mac
+        local_ipv4s = nic_metadata.get('local-ipv4s')
         if (nic_name == fallback_nic or nic_metadata.get('public-ipv4s') or
-                nic_metadata.get('local-ipv4s')):
-            nic_cfg['subnets'].append({'type': 'dhcp4'})
+                local_ipv4s):
+            dev_config['dhcp4'] = True
+            if isinstance(local_ipv4s, list) and len(local_ipv4s) > 1:
+                subnet_cidr = nic_metadata.get('subnet-ipv4-cidr-block')
+                _ip, prefix = subnet_cidr.split('/')
+                for secondary_ip in local_ipv4s[1:]:
+                    if dev_config.get('addresses') is None:
+                        dev_config['addresses'] = []
+                    dev_config['addresses'].append(
+                        '{ip}/{prefix}'.format(ip=secondary_ip, prefix=prefix))
         if nic_metadata.get('ipv6s'):
-            nic_cfg['subnets'].append({'type': 'dhcp6'})
-        netcfg['config'].append(nic_cfg)
+            dev_config['dhcp6'] = True
+        dev_config.update({
+            'match': {'macaddress': nic_metadata.get('mac').lower()},
+            'set-name': nic_name})
+        netcfg['ethernets'][nic_name] = dev_config
     return netcfg
 
 
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 20d59bf..c99f58f 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -112,6 +112,94 @@ DEFAULT_METADATA = {
     "services": {"domain": "amazonaws.com", "partition": "aws"},
 }
 
+# collected from api version 2016-09-02/ with
+# python3 -c 'import json
+# from cloudinit.ec2_utils import get_instance_metadata as gm
+# print(json.dumps(gm("2018-09-24"), indent=1, sort_keys=True))'
+DEFAULT_METADATA_2018_09_24 = {
+    "ami-id": "ami-0986c2ac728528ac2",
+    "ami-launch-index": "0",
+    "ami-manifest-path": "(unknown)",
+    "block-device-mapping": {
+        "ami": "/dev/sda1",
+        "root": "/dev/sda1"
+    },
+    "events": {
+        "maintenance": {
+            "history": "[]",
+            "scheduled": "[]"
+        }
+    },
+    "hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "identity-credentials": {
+        "ec2": {
+            "info": {
+                "AccountId": "329910648901",
+                "Code": "Success",
+                "LastUpdated": "2019-07-06T14:22:56Z"
+            }
+        }
+    },
+    "instance-action": "none",
+    "instance-id": "i-069e01e8cc43732f8",
+    "instance-type": "t2.micro",
+    "local-hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "local-ipv4": "172.31.44.13",
+    "mac": "0a:07:84:3d:6e:38",
+    "metrics": {
+        "vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+    },
+    "network": {
+        "interfaces": {
+            "macs": {
+                "0a:07:84:3d:6e:38": {
+                    "device-number": "0",
+                    "interface-id": "eni-0d6335689899ce9cc",
+                    "ipv4-associations": {
+                        "18.218.219.181": "172.31.44.13"
+                    },
+                    "local-hostname": ("ip-172-31-44-13.us-east-2."
+                                       "compute.internal"),
+                    "local-ipv4s": [
+                        "172.31.44.13",
+                        "172.31.45.70"
+                    ],
+                    "mac": "0a:07:84:3d:6e:38",
+                    "owner-id": "329910648901",
+                    "public-hostname": ("ec2-18-218-219-181.us-east-2."
+                                        "compute.amazonaws.com"),
+                    "public-ipv4s": "18.218.219.181",
+                    "security-group-ids": "sg-0c387755222ba8d2e",
+                    "security-groups": "launch-wizard-4",
+                    "subnet-id": "subnet-9d7ba0d1",
+                    "subnet-ipv4-cidr-block": "172.31.32.0/20",
+                    "vpc-id": "vpc-a07f62c8",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16"
+                }
+            }
+        }
+    },
+    "placement": {
+        "availability-zone": "us-east-2c"
+    },
+    "profile": "default-hvm",
+       "public-hostname": ("ec2-18-218-219-181.us-east-2."
+                           "compute.amazonaws.com"),
+    "public-ipv4": "18.218.219.181",
+    "public-keys": {
+        "yourkeyname,e": [
+            "ssh-rsa AAAAW...DZ yourkeyname"
+        ]
+    },
+    "reservation-id": "r-09b4917135cdd33be",
+    "security-groups": "launch-wizard-4",
+    "services": {
+        "domain": "amazonaws.com",
+        "partition": "aws"
+    }
+}
+
 
 def _register_ssh_keys(rfunc, base_url, keys_data):
     """handle ssh key inconsistencies.
@@ -261,8 +349,8 @@ class TestEc2(test_helpers.HttprettyTestCase):
                         register_mock_metaserver(instance_id_url, None)
         return ds
 
-    def test_network_config_property_returns_version_1_network_data(self):
-        """network_config property returns network version 1 for metadata.
+    def test_network_config_property_returns_version_2_network_data(self):
+        """network_config property returns network version 2 for metadata.
 
         Only one device is configured even when multiple exist in metadata.
         """
@@ -277,10 +365,9 @@ class TestEc2(test_helpers.HttprettyTestCase):
             ds.get_data()
 
         mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
-        expected = {'version': 1, 'config': [
-            {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
-             'type': 'physical'}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': '06:17:04:d7:26:09'}, 'set-name': 'eth9',
+             'dhcp4': True, 'dhcp6': True}}}
         patch_path = (
             'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
         get_interface_mac_path = (
@@ -309,10 +396,40 @@ class TestEc2(test_helpers.HttprettyTestCase):
             ds.get_data()
 
         mac1 = '06:17:04:d7:26:0A'  # IPv4 only in DEFAULT_METADATA
-        expected = {'version': 1, 'config': [
-            {'mac_address': '06:17:04:d7:26:0A', 'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}],
-             'type': 'physical'}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': '06:17:04:d7:26:0a'}, 'set-name': 'eth9',
+             'dhcp4': True}}}
+        patch_path = (
+            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
+        get_interface_mac_path = (
+            'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            with mock.patch(find_fallback_path) as m_find_fallback:
+                with mock.patch(get_interface_mac_path) as m_get_mac:
+                    m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+                    m_find_fallback.return_value = 'eth9'
+                    m_get_mac.return_value = mac1
+                    self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_secondary_private_ips(self):
+        """network_config property configures any secondary ipv4 addresses.
+
+        Only one device is configured even when multiple exist in metadata.
+        """
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': DEFAULT_METADATA_2018_09_24})
+        find_fallback_path = (
+            'cloudinit.sources.DataSourceEc2.net.find_fallback_nic')
+        with mock.patch(find_fallback_path) as m_find_fallback:
+            m_find_fallback.return_value = 'eth9'
+            ds.get_data()
+
+        mac1 = '0a:07:84:3d:6e:38'  # IPv4 with 1 secondary IP
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': mac1}, 'set-name': 'eth9',
+             'addresses': ['172.31.45.70/20'], 'dhcp4': True}}}
         patch_path = (
             'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
         get_interface_mac_path = (
@@ -362,11 +479,9 @@ class TestEc2(test_helpers.HttprettyTestCase):
         self.assertIn(
             'Refreshing stale metadata from prior to upgrade',
             self.logs.getvalue())
-        expected = {'version': 1, 'config': [
-            {'mac_address': '06:17:04:d7:26:09',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
-             'type': 'physical'}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': mac1}, 'set-name': 'eth9',
+             'dhcp4': True, 'dhcp6': True}}}
         self.assertEqual(expected, ds.network_config)
 
     def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self):
@@ -546,19 +661,19 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
 
     def setUp(self):
         super(TestConvertEc2MetadataNetworkConfig, self).setUp()
-        self.mac1 = '06:17:04:d7:26:09'
+        self.mac1 = '06:17:04:D7:26:09'
         self.network_metadata = {
             'interfaces': {'macs': {
-                self.mac1: {'public-ipv4s': '172.31.2.16'}}}}
+                self.mac1: {'mac': self.mac1, 'public-ipv4s': '172.31.2.16'}}}}
 
     def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
         """Any mac absent from metadata is skipped by network config."""
         macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'}
 
         # DE:AD:BE:EF:FF:FF represented by OS but not in metadata
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+             'dhcp4': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -572,9 +687,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
             network_metadata_ipv6['interfaces']['macs'][self.mac1])
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
         nic1_metadata.pop('public-ipv4s')
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp6'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+            'dhcp6': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -588,9 +703,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
             network_metadata_ipv6['interfaces']['macs'][self.mac1])
         nic1_metadata['local-ipv4s'] = '172.3.3.15'
         nic1_metadata.pop('public-ipv4s')
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+            'dhcp4': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -605,9 +720,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         nic1_metadata['public-ipv4s'] = ''
 
         # When no ipv4 or ipv6 content but fallback_nic set, set dhcp4 config.
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+            'dhcp4': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -622,10 +737,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
         nic1_metadata.pop('public-ipv4s')
         nic1_metadata['local-ipv4s'] = '10.0.0.42'  # Local ipv4 only on vpc
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+             'dhcp4': True, 'dhcp6': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -638,10 +752,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         nic1_metadata = (
             network_metadata_both['interfaces']['macs'][self.mac1])
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+             'dhcp4': True, 'dhcp6': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -649,10 +762,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
 
     def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self):
         """Convert Ec2 Metadata calls get_interfaces_by_mac by default."""
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9':
+            {'match': {'macaddress': self.mac1.lower()}, 'set-name': 'eth9',
+             'dhcp4': True}}}
         patch_path = (
             'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
         with mock.patch(patch_path) as m_get_interfaces_by_mac: