cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #00422
[Merge] lp:~harlowja/cloud-init/shared-wait-metadata into lp:cloud-init
Joshua Harlow has proposed merging lp:~harlowja/cloud-init/shared-wait-metadata into lp:cloud-init.
Requested reviews:
cloud init development team (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~harlowja/cloud-init/shared-wait-metadata/+merge/206567
Move shared waiting function to util
The ec2 and openstack datasources use a similar
piece of code for waiting for there metadata services
to become accessible (which varies depending on cloud,
service provider...) so its much nicer if we move that
duplicated/similar code to a standard utility method and
use that instead.
--
https://code.launchpad.net/~harlowja/cloud-init/shared-wait-metadata/+merge/206567
Your team cloud init development team is requested to review the proposed merge of lp:~harlowja/cloud-init/shared-wait-metadata into lp:cloud-init.
=== modified file 'cloudinit/sources/DataSourceEc2.py'
--- cloudinit/sources/DataSourceEc2.py 2014-02-01 20:03:32 +0000
+++ cloudinit/sources/DataSourceEc2.py 2014-02-15 00:55:16 +0000
@@ -26,7 +26,7 @@
from cloudinit import ec2_utils as ec2
from cloudinit import log as logging
from cloudinit import sources
-from cloudinit import url_helper as uhelp
+from cloudinit import url_helper
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -83,65 +83,33 @@
return self.metadata['instance-id']
def _get_url_settings(self):
- mcfg = self.ds_cfg
- if not mcfg:
- mcfg = {}
+ # max_wait < 0 indicates do not wait
max_wait = 120
try:
- max_wait = int(mcfg.get("max_wait", max_wait))
+ max_wait = int(self.ds_cfg.get("max_wait", max_wait))
except Exception:
util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
timeout = 50
try:
- timeout = max(0, int(mcfg.get("timeout", timeout)))
+ timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))
except Exception:
util.logexc(LOG, "Failed to get timeout, using %s", timeout)
return (max_wait, timeout)
def wait_for_metadata_service(self):
- mcfg = self.ds_cfg
- if not mcfg:
- mcfg = {}
-
(max_wait, timeout) = self._get_url_settings()
- if max_wait <= 0:
- return False
-
- # Remove addresses from the list that wont resolve.
- mdurls = mcfg.get("metadata_urls", DEF_MD_URLS)
- filtered = [x for x in mdurls if util.is_resolvable_url(x)]
-
- if set(filtered) != set(mdurls):
- LOG.debug("Removed the following from metadata urls: %s",
- list((set(mdurls) - set(filtered))))
-
- if len(filtered):
- mdurls = filtered
- else:
- LOG.warn("Empty metadata url list! using default list")
- mdurls = DEF_MD_URLS
-
- urls = []
- url2base = {}
- for url in mdurls:
- cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
- urls.append(cur)
- url2base[cur] = url
-
- start_time = time.time()
- url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
- timeout=timeout, status_cb=LOG.warn)
-
- if url:
- LOG.debug("Using metadata source: '%s'", url2base[url])
- else:
- LOG.critical("Giving up on md from %s after %s seconds",
- urls, int(time.time() - start_time))
-
- self.metadata_address = url2base.get(url)
- return bool(url)
+ md_urls = self.ds_cfg.get("metadata_urls", [])
+ md_check_path = url_helper.combine_url(self.api_ver,
+ 'meta-data', 'instance-id')
+ md_url = util.wait_for_metadata_service(md_urls,
+ path=md_check_path,
+ max_wait=max_wait,
+ timeout=timeout,
+ fallback_urls=DEF_MD_URLS)
+ self.metadata_address = md_url
+ return bool(md_url)
def device_name_to_device(self, name):
# Consult metadata service, that has
=== modified file 'cloudinit/sources/DataSourceOpenStack.py'
--- cloudinit/sources/DataSourceOpenStack.py 2014-02-14 19:24:06 +0000
+++ cloudinit/sources/DataSourceOpenStack.py 2014-02-15 00:55:16 +0000
@@ -16,8 +16,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import time
-
from cloudinit import log as logging
from cloudinit import sources
from cloudinit import url_helper
@@ -29,6 +27,7 @@
# Various defaults/constants...
DEF_MD_URL = "http://169.254.169.254";
+DEF_MD_URLS = [DEF_MD_URL]
DEFAULT_IID = "iid-dsopenstack"
DEFAULT_METADATA = {
"instance-id": DEFAULT_IID,
@@ -54,19 +53,14 @@
return mstr
def _get_url_settings(self):
- # TODO(harlowja): this is shared with ec2 datasource, we should just
- # move it to a shared location instead...
- # Note: the defaults here are different though.
-
# max_wait < 0 indicates do not wait
max_wait = -1
- timeout = 10
-
try:
max_wait = int(self.ds_cfg.get("max_wait", max_wait))
except Exception:
util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
+ timeout = 10
try:
timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))
except Exception:
@@ -74,38 +68,18 @@
return (max_wait, timeout)
def wait_for_metadata_service(self):
- urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL])
- filtered = [x for x in urls if util.is_resolvable_url(x)]
- if set(filtered) != set(urls):
- LOG.debug("Removed the following from metadata urls: %s",
- list((set(urls) - set(filtered))))
- if len(filtered):
- urls = filtered
- else:
- LOG.warn("Empty metadata url list! using default list")
- urls = [DEF_MD_URL]
-
- md_urls = []
- url2base = {}
- for url in urls:
- md_url = url_helper.combine_url(url, 'openstack',
- openstack.OS_LATEST,
- 'meta_data.json')
- md_urls.append(md_url)
- url2base[md_url] = url
-
(max_wait, timeout) = self._get_url_settings()
- start_time = time.time()
- avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
- timeout=timeout)
- if avail_url:
- LOG.debug("Using metadata source: '%s'", url2base[avail_url])
- else:
- LOG.debug("Giving up on OpenStack md from %s after %s seconds",
- md_urls, int(time.time() - start_time))
-
- self.metadata_address = url2base.get(avail_url)
- return bool(avail_url)
+ md_urls = self.ds_cfg.get("metadata_urls", [])
+ md_check_path = url_helper.combine_url("openstack",
+ openstack.OS_LATEST,
+ 'meta_data.json')
+ md_url = util.wait_for_metadata_service(md_urls,
+ path=md_check_path,
+ max_wait=max_wait,
+ timeout=timeout,
+ fallback_urls=DEF_MD_URLS)
+ self.metadata_address = md_url
+ return bool(md_url)
def get_data(self):
try:
=== modified file 'cloudinit/util.py'
--- cloudinit/util.py 2014-02-13 11:27:22 +0000
+++ cloudinit/util.py 2014-02-15 00:55:16 +0000
@@ -1796,6 +1796,43 @@
return None
+def wait_for_metadata_service(urls, path=None, fallback_urls=None,
+ max_wait=120, timeout=50):
+ resolvable_urls = [x for x in urls if is_resolvable_url(x)]
+ if set(resolvable_urls) != set(urls):
+ LOG.debug("Removed the following non-resolveable metadata urls: %s",
+ list((set(urls) - set(resolvable_urls))))
+ if not resolvable_urls:
+ resolvable_urls = fallback_urls
+ if not resolvable_urls:
+ return None
+
+ urls = resolvable_urls
+ md_urls = []
+ url2base = {}
+ for url in urls:
+ if path:
+ md_url = url_helper.combine_url(url, path)
+ else:
+ md_url = url
+ md_urls.append(md_url)
+ url2base[md_url] = url
+
+ start_time = time.time()
+ url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
+ timeout=timeout, status_cb=LOG.warn)
+ total_time = int(time.time() - start_time)
+ if url:
+ md_url = url2base[url]
+ LOG.debug("Using metadata url: '%s' (found after %s seconds)",
+ md_url, total_time)
+ return md_url
+ else:
+ LOG.critical("Giving up on metadata from %s after %s seconds",
+ md_urls, total_time)
+ return None
+
+
def get_mount_info(path, log=LOG):
# Use /proc/$$/mountinfo to find the device where path is mounted.
# This is done because with a btrfs filesystem using os.stat(path)
Follow ups
