instant team mailing list archive

[Joachim Berdal Haga <jobh@xxxxxxxxx>]

The attached patch makes instant much faster.

For my DOLFIN simulator, the time for startup (until the time loop is
reached) is reduced from 3.1s to 1.5s with fully populated instant cache,
and from 60 seconds to 31 seconds with clean cache.

The patch just adds in-process caching of pkg-config calls, of the
swig-version call, and it collapses two calls to "python setup.py" into one.
It is a bit ugly, it was done in a hurry, but I thought the improvements
were good enough to share. Lightly tested, seems to work.

(I am actually very surprised by the clean-cache improvement, which seems
much too large. It makes me uneasy about the patch. But my simulator seems
to run the same as before...)

-j.

From fb41540f2821d1160dbba7cc60c3efd0b8b69cfd Mon Sep 17 00:00:00 2001
From: Joachim B Haga <jobh@xxxxxxxxx>
Date: Wed, 26 Jan 2011 17:07:13 +0100
Subject: [PATCH] Make stuff go faster.

Reduces startup time for my script (until time loop is reached) from
60 to 30 seconds with clean instant cache, and from 3.1 to 1.5 seconds
with fully populated instant cache.
---
 instant/build.py  |   21 +++-----------
 instant/config.py |   79 +++++++++++++++++++++++++++++++++--------------------
 2 files changed, 54 insertions(+), 46 deletions(-)

diff --git a/instant/build.py b/instant/build.py
index 05f2ea6..0eb94ce 100644
--- a/instant/build.py
+++ b/instant/build.py
@@ -5,7 +5,7 @@ from itertools import chain
 
 # TODO: Import only the official interface
 from output import *
-from config import header_and_libs_from_pkgconfig
+from config import header_and_libs_from_pkgconfig, get_swig_version
 from paths import *
 from signatures import *
 from cache import *
@@ -77,8 +77,9 @@ def recompile(modulename, module_path, setup_name, new_compilation_checksum):
             return
     
     # Verify that SWIG is on the system
-    (swig_stat, swig_out) = get_status_output("swig -version")
-    if swig_stat != 0:
+    try:
+        get_swig_version()
+    except OSError:
         instant_error("In instant.recompile: Could not find swig!"\
             " You can download swig from http://www.swig.org";)
     
@@ -87,7 +88,7 @@ def recompile(modulename, module_path, setup_name, new_compilation_checksum):
     compile_log_file = open(compile_log_filename, "w")
     try:
         # Build module
-        cmd = "python %s build_ext" % setup_name
+        cmd = "python %s build_ext install --install-platlib=." % setup_name
         instant_info("--- Instant: compiling ---")
         instant_debug("cmd = %s" % cmd)
         ret, output = get_status_output(cmd)
@@ -98,18 +99,6 @@ def recompile(modulename, module_path, setup_name, new_compilation_checksum):
                 os.remove(compilation_checksum_filename)
             instant_error("In instant.recompile: The module did not "\
                 "compile, see '%s'" % compile_log_filename)
-        
-        # 'Install' module
-        cmd = "python %s install --install-platlib=." % setup_name
-        instant_debug("cmd = %s" % cmd)
-        ret, output = get_status_output(cmd)
-        compile_log_file.write(output)
-        compile_log_file.flush()
-        if ret != 0:
-            if os.path.exists(compilation_checksum_filename):
-                os.remove(compilation_checksum_filename)
-            instant_error("In instant.recompile: Could not 'install' "\
-                "the module, see '%s'" % compile_log_filename)
     finally:
         compile_log_file.close()
     
diff --git a/instant/config.py b/instant/config.py
index 8ab66aa..4c73794 100644
--- a/instant/config.py
+++ b/instant/config.py
@@ -4,15 +4,19 @@ import os
 from output import get_status_output
 import re
 
+_swig_version = None
 def get_swig_version(): 
     """ Return the current swig version in a 'str'"""
-    # Check for swig installation
-    result, output = get_status_output("swig -version")
-    if result != 0: 
-        raise OSError("SWIG is not installed on the system.")
-    pattern = "SWIG Version (.*)"
-    r = re.search(pattern, output)
-    return r.groups(0)[0]
+    global _swig_version
+    if _swig_version is None:
+        # Check for swig installation
+        result, output = get_status_output("swig -version")
+        if result != 0: 
+            raise OSError("SWIG is not installed on the system.")
+        pattern = "SWIG Version (.*)"
+        r = re.search(pattern, output)
+        _swig_version = r.groups(0)[0]
+    return _swig_version
 
 def check_swig_version(version, same=False):
     """ Check the swig version
@@ -51,6 +55,8 @@ def check_swig_version(version, same=False):
     
     return swig_enough
 
+_pkg_config_installed = None
+_hl_cache = {}
 def header_and_libs_from_pkgconfig(*packages, **kwargs):
     """This function returns list of include files, flags, libraries and library directories obtain from a pkgconfig file.
     
@@ -59,9 +65,12 @@ def header_and_libs_from_pkgconfig(*packages, **kwargs):
     or:
         (includes, flags, libraries, libdirs, linkflags) = header_and_libs_from_pkgconfig(*list_of_packages, returnLinkFlags=True)
     """
+    global _pkg_config_installed, _hl_cache
     returnLinkFlags = kwargs.get("returnLinkFlags", False)
-    result, output = get_status_output("pkg-config --version ")
-    if result != 0: 
+    if _pkg_config_installed is None:
+        result, output = get_status_output("pkg-config --version ")
+        _pkg_config_installed = (result == 0)
+    if not _pkg_config_installed:
         raise OSError("The pkg-config package is not installed on the system.")
 
     env = os.environ.copy()
@@ -76,27 +85,37 @@ def header_and_libs_from_pkgconfig(*packages, **kwargs):
     libdirs = []
     linkflags = []
     for pack in packages:
-        result, output = get_status_output("pkg-config --exists %s " % pack, env=env)
-        if result == 0: 
-            tmp = get_status_output("pkg-config --cflags-only-I %s " % pack, env=env)[1].split()
-            includes.extend(i[2:] for i in tmp)
-            
-            tmp = get_status_output("pkg-config --cflags-only-other %s " % pack, env=env)[1].split()
-            flags.extend(tmp)
-            
-            tmp = get_status_output("pkg-config --libs-only-l  %s " % pack, env=env)[1].split()
-            libs.extend(i[2:] for i in tmp)
-            
-            tmp = get_status_output("pkg-config --libs-only-L  %s " % pack, env=env)[1].split()
-            libdirs.extend(i[2:] for i in tmp)
-
-            tmp = get_status_output("pkg-config --libs-only-other  %s " % pack, env=env)[1].split()
-            linkflags.extend(tmp)
-
-        else: 
+        if not pack in _hl_cache:
+            result, output = get_status_output("pkg-config --exists %s " % pack, env=env)
+            if result == 0: 
+                tmp = get_status_output("pkg-config --cflags-only-I %s " % pack, env=env)[1].split()
+                _includes = [i[2:] for i in tmp]
+
+                _flags = get_status_output("pkg-config --cflags-only-other %s " % pack, env=env)[1].split()
+
+                tmp = get_status_output("pkg-config --libs-only-l  %s " % pack, env=env)[1].split()
+                _libs = [i[2:] for i in tmp]
+
+                tmp = get_status_output("pkg-config --libs-only-L  %s " % pack, env=env)[1].split()
+                _libdirs = [i[2:] for i in tmp]
+
+                _linkflags = get_status_output("pkg-config --libs-only-other  %s " % pack, env=env)[1].split()
+
+                _hl_cache[pack] = (_includes, _flags, _libs, _libdirs, _linkflags)
+            else:
+                _hl_cache[pack] = None
+
+        result = _hl_cache[pack]
+        if not result:
             raise OSError("The pkg-config file %s does not exist" % pack)
 
-    if returnLinkFlags:
-        return (includes, flags, libs, libdirs, linkflags) 
-    return (includes, flags, libs, libdirs) 
+        _includes, _flags, _libs, _libdirs, _linkflags = result
+        includes.extend(_includes)
+        flags.extend(_flags)
+        libs.extend(_libs)
+        libdirs.extend(_libdirs)
+        linkflags.extend(_linkflags)
 
+    if returnLinkFlags:
+        return (includes, flags, libs, libdirs, linkflags)
+    return (includes, flags, libs, libdirs)
-- 
1.7.1