cloud-init-dev team mailing list archive

[Joshua Harlow <harlowja@xxxxxxxxx>]

Joshua Harlow has proposed merging lp:~harlowja/cloud-init/os-caching into lp:cloud-init.

Requested reviews:
  cloud init development team (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~harlowja/cloud-init/os-caching/+merge/232303

Better caching of things that exist and things that have been read.
-- 
https://code.launchpad.net/~harlowja/cloud-init/os-caching/+merge/232303
Your team cloud init development team is requested to review the proposed merge of lp:~harlowja/cloud-init/os-caching into lp:cloud-init.

=== modified file 'cloudinit/sources/helpers/openstack.py'
--- cloudinit/sources/helpers/openstack.py	2014-02-24 22:41:42 +0000
+++ cloudinit/sources/helpers/openstack.py	2014-08-26 19:42:07 +0000
@@ -304,16 +304,26 @@
 class ConfigDriveReader(BaseReader):
     def __init__(self, base_path):
         super(ConfigDriveReader, self).__init__(base_path)
+        self.read_cache = {}
+        self.exists_cache = {}
 
     def _path_join(self, base, *add_ons):
         components = [base] + list(add_ons)
         return os.path.join(*components)
 
     def _path_exists(self, path):
-        return os.path.exists(path)
+        if path in self.exists_cache:
+            return self.exists_cache[path]
+        found = os.path.exists(path)
+        self.exists_cache[path] = found
+        return found
 
     def _path_read(self, path):
-        return util.load_file(path)
+        if path in self.read_cache:
+            return self.read_cache[path]
+        blob = util.load_file(path)
+        self.read_cache[path] = blob
+        return blob
 
     def _read_ec2_metadata(self):
         path = self._path_join(self.base_path,
@@ -400,12 +410,18 @@
         self.ssl_details = ssl_details
         self.timeout = float(timeout)
         self.retries = int(retries)
+        self.read_cache = {}
+        self.exists_cache = {}
 
     def _path_read(self, path):
-        response = url_helper.readurl(path,
-                                      retries=self.retries,
-                                      ssl_details=self.ssl_details,
-                                      timeout=self.timeout)
+        if path in self.read_cache:
+            response = self.read_cache[path]
+        else:
+            response = url_helper.readurl(path,
+                                          retries=self.retries,
+                                          ssl_details=self.ssl_details,
+                                          timeout=self.timeout)
+            self.read_cache[path] = response
         return response.contents
 
     def _path_exists(self, path):
@@ -420,15 +436,20 @@
                 pass
             return True
 
-        try:
-            response = url_helper.readurl(path,
-                                          retries=self.retries,
-                                          ssl_details=self.ssl_details,
-                                          timeout=self.timeout,
-                                          exception_cb=should_retry_cb)
-            return response.ok()
-        except IOError:
-            return False
+        if path in self.exists_cache:
+            found = self.exists_cache[path]
+        else:
+            try:
+                response = url_helper.readurl(path,
+                                              retries=self.retries,
+                                              ssl_details=self.ssl_details,
+                                              timeout=self.timeout,
+                                              exception_cb=should_retry_cb)
+                found = response.ok()
+            except IOError:
+                found = False
+            self.exists_cache[path] = found
+        return found
 
     def _path_join(self, base, *add_ons):
         return url_helper.combine_url(base, *add_ons)