cloud-init-dev team mailing list archive

[Joshua Harlow <harlowja@xxxxxxxxx>]

Joshua Harlow has proposed merging lp:~harlowja/cloud-init/monotonic-compat into lp:cloud-init.

Requested reviews:
  cloud init development team (cloud-init-dev)
Related bugs:
  Bug #1273255 in cloud-init: "wait_for_url doesn't account for system clock being changed"
  https://bugs.launchpad.net/cloud-init/+bug/1273255

For more details, see:
https://code.launchpad.net/~harlowja/cloud-init/monotonic-compat/+merge/214363

Use monotonic time to avoid ntpd updates altering time.time and usage of it to measure distance between two times (for waiting, recording duration...).
-- 
https://code.launchpad.net/~harlowja/cloud-init/monotonic-compat/+merge/214363
Your team cloud init development team is requested to review the proposed merge of lp:~harlowja/cloud-init/monotonic-compat into lp:cloud-init.

=== added file 'cloudinit/compat.py'
--- cloudinit/compat.py	1970-01-01 00:00:00 +0000
+++ cloudinit/compat.py	2014-04-05 01:08:20 +0000
@@ -0,0 +1,25 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Joshua Harlow <harlowja@xxxxxxxxxxxxx>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import six
+
+if six.PY2:
+    import monotime
+    monotonic = monotime.monotonic
+else:
+    monotonic = time.monotonic

=== modified file 'cloudinit/config/cc_package_update_upgrade_install.py'
--- cloudinit/config/cc_package_update_upgrade_install.py	2014-02-05 15:36:47 +0000
+++ cloudinit/config/cc_package_update_upgrade_install.py	2014-04-05 01:08:20 +0000
@@ -19,6 +19,7 @@
 import os
 import time
 
+from cloudinit import compat
 from cloudinit import log as logging
 from cloudinit import util
 
@@ -35,15 +36,15 @@
 
 def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2):
     util.subp(REBOOT_CMD)
-    start = time.time()
+    start = compat.monotonic()
     wait_time = initial_sleep
     for _i in range(0, wait_attempts):
         time.sleep(wait_time)
         wait_time *= backoff
-        elapsed = time.time() - start
+        elapsed = compat.monotonic() - start
         log.debug("Rebooted, but still running after %s seconds", int(elapsed))
     # If we got here, not good
-    elapsed = time.time() - start
+    elapsed = compat.monotonic() - start
     raise RuntimeError(("Reboot did not happen"
                         " after %s seconds!") % (int(elapsed)))
 

=== modified file 'cloudinit/config/cc_power_state_change.py'
--- cloudinit/config/cc_power_state_change.py	2014-02-03 22:03:14 +0000
+++ cloudinit/config/cc_power_state_change.py	2014-04-05 01:08:20 +0000
@@ -16,6 +16,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from cloudinit import compat
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import util
 
@@ -136,7 +137,7 @@
     # is no longer alive.  After it is gone, or timeout has passed
     # execute func(args)
     msg = None
-    end_time = time.time() + timeout
+    end_time = compat.monotonic() + timeout
 
     def fatal(msg):
         if log:
@@ -146,7 +147,7 @@
     known_errnos = (errno.ENOENT, errno.ESRCH)
 
     while True:
-        if time.time() > end_time:
+        if compat.monotonic() > end_time:
             msg = "timeout reached before %s ended" % pid
             break
 

=== modified file 'cloudinit/sources/DataSourceCloudStack.py'
--- cloudinit/sources/DataSourceCloudStack.py	2013-06-19 06:44:00 +0000
+++ cloudinit/sources/DataSourceCloudStack.py	2014-04-05 01:08:20 +0000
@@ -25,8 +25,8 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import time
 
+from cloudinit import compat
 from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
 from cloudinit import sources
@@ -79,7 +79,7 @@
         (max_wait, timeout) = self._get_url_settings()
 
         urls = [self.metadata_address + "/latest/meta-data/instance-id"]
-        start_time = time.time()
+        start_time = compat.monotonic()
         url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
                                 timeout=timeout, status_cb=LOG.warn)
 
@@ -88,7 +88,7 @@
         else:
             LOG.critical(("Giving up on waiting for the metadata from %s"
                           " after %s seconds"),
-                          urls, int(time.time() - start_time))
+                          urls, int(compat.monotonic() - start_time))
 
         return bool(url)
 
@@ -102,13 +102,13 @@
         try:
             if not self.wait_for_metadata_service():
                 return False
-            start_time = time.time()
+            start_time = compat.monotonic()
             self.userdata_raw = ec2.get_instance_userdata(self.api_ver,
                 self.metadata_address)
             self.metadata = ec2.get_instance_metadata(self.api_ver,
                                                       self.metadata_address)
             LOG.debug("Crawl of metadata service took %s seconds",
-                      int(time.time() - start_time))
+                      int(compat.monotonic() - start_time))
             return True
         except Exception:
             util.logexc(LOG, 'Failed fetching from metadata service %s',

=== modified file 'cloudinit/sources/DataSourceEc2.py'
--- cloudinit/sources/DataSourceEc2.py	2014-02-01 20:03:32 +0000
+++ cloudinit/sources/DataSourceEc2.py	2014-04-05 01:08:20 +0000
@@ -21,8 +21,8 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import time
 
+from cloudinit import compat
 from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
 from cloudinit import sources
@@ -60,13 +60,13 @@
         try:
             if not self.wait_for_metadata_service():
                 return False
-            start_time = time.time()
+            start_time = compat.monotonic()
             self.userdata_raw = ec2.get_instance_userdata(self.api_ver,
                 self.metadata_address)
             self.metadata = ec2.get_instance_metadata(self.api_ver,
                                                       self.metadata_address)
             LOG.debug("Crawl of metadata service took %s seconds",
-                       int(time.time() - start_time))
+                       int(compat.monotonic() - start_time))
             return True
         except Exception:
             util.logexc(LOG, "Failed reading from metadata address %s",
@@ -130,7 +130,7 @@
             urls.append(cur)
             url2base[cur] = url
 
-        start_time = time.time()
+        start_time = compat.monotonic()
         url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
                                 timeout=timeout, status_cb=LOG.warn)
 
@@ -138,7 +138,7 @@
             LOG.debug("Using metadata source: '%s'", url2base[url])
         else:
             LOG.critical("Giving up on md from %s after %s seconds",
-                            urls, int(time.time() - start_time))
+                            urls, int(compat.monotonic() - start_time))
 
         self.metadata_address = url2base.get(url)
         return bool(url)

=== modified file 'cloudinit/sources/DataSourceMAAS.py'
--- cloudinit/sources/DataSourceMAAS.py	2013-04-25 15:58:38 +0000
+++ cloudinit/sources/DataSourceMAAS.py	2014-04-05 01:08:20 +0000
@@ -25,6 +25,7 @@
 import time
 import urllib2
 
+from cloudinit import compat
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper
@@ -130,7 +131,7 @@
         except Exception:
             LOG.warn("Failed to get timeout, using %s" % timeout)
 
-        starttime = time.time()
+        starttime = compat.monotonic()
         check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
         urls = [check_url]
         url = url_helper.wait_for_url(urls=urls, max_wait=max_wait,
@@ -142,7 +143,7 @@
             LOG.debug("Using metadata source: '%s'", url)
         else:
             LOG.critical("Giving up on md from %s after %i seconds",
-                         urls, int(time.time() - starttime))
+                         urls, int(compat.monotonic() - starttime))
 
         return bool(url)
 

=== modified file 'cloudinit/sources/DataSourceOpenStack.py'
--- cloudinit/sources/DataSourceOpenStack.py	2014-02-24 22:41:42 +0000
+++ cloudinit/sources/DataSourceOpenStack.py	2014-04-05 01:08:20 +0000
@@ -16,8 +16,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import time
-
+from cloudinit import compat
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper
@@ -95,14 +94,14 @@
                 url2base[md_url] = url
 
         (max_wait, timeout) = self._get_url_settings()
-        start_time = time.time()
+        start_time = compat.monotonic()
         avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
                                             timeout=timeout)
         if avail_url:
             LOG.debug("Using metadata source: '%s'", url2base[avail_url])
         else:
             LOG.debug("Giving up on OpenStack md from %s after %s seconds",
-                      md_urls, int(time.time() - start_time))
+                      md_urls, int(compat.monotonic() - start_time))
 
         self.metadata_address = url2base.get(avail_url)
         return bool(avail_url)

=== modified file 'cloudinit/url_helper.py'
--- cloudinit/url_helper.py	2014-02-13 17:13:42 +0000
+++ cloudinit/url_helper.py	2014-04-05 01:08:20 +0000
@@ -29,6 +29,7 @@
 
 from urlparse import (urlparse, urlunparse)
 
+from cloudinit import compat
 from cloudinit import log as logging
 from cloudinit import version
 
@@ -289,7 +290,7 @@
     data host (169.254.169.254) may be firewalled off Entirely for a sytem,
     meaning that the connection will block forever unless a timeout is set.
     """
-    start_time = time.time()
+    start_time = compat.monotonic()
 
     def log_status_cb(msg, exc=None):
         LOG.debug(msg)
@@ -299,13 +300,13 @@
 
     def timeup(max_wait, start_time):
         return ((max_wait <= 0 or max_wait is None) or
-                (time.time() - start_time > max_wait))
+                (compat.monotonic() - start_time > max_wait))
 
     loop_n = 0
     while True:
         sleep_time = int(loop_n / 5) + 1
         for url in urls:
-            now = time.time()
+            now = compat.monotonic()
             if loop_n != 0:
                 if timeup(max_wait, start_time):
                     break
@@ -338,7 +339,7 @@
             except Exception as e:
                 reason = "unexpected error [%s]" % e
 
-            time_taken = int(time.time() - start_time)
+            time_taken = int(compat.monotonic() - start_time)
             status_msg = "Calling '%s' failed [%s/%ss]: %s" % (url,
                                                                time_taken,
                                                                max_wait,

=== modified file 'cloudinit/util.py'
--- cloudinit/util.py	2014-02-24 22:20:12 +0000
+++ cloudinit/util.py	2014-04-05 01:08:20 +0000
@@ -51,6 +51,7 @@
 
 import yaml
 
+from cloudinit import compat
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import mergers
@@ -1862,7 +1863,7 @@
     if kwargs is None:
         kwargs = {}
 
-    start = time.time()
+    start = compat.monotonic()
 
     ustart = None
     if get_uptime:
@@ -1874,7 +1875,7 @@
     try:
         ret = func(*args, **kwargs)
     finally:
-        delta = time.time() - start
+        delta = compat.monotonic() - start
         udelta = None
         if ustart is not None:
             try:

=== modified file 'requirements.txt'
--- requirements.txt	2014-02-12 10:14:49 +0000
+++ requirements.txt	2014-04-05 01:08:20 +0000
@@ -31,3 +31,9 @@
 
 # For patching pieces of cloud-config together
 jsonpatch
+
+# For where time.monotonic isn't available (older versions of python)
+monotime
+
+# For where we need to determine if we are using py2 or py3 (and compat.)
+six