launchpad-reviewers team mailing list archive

[Benji York <benji@xxxxxxxxxxxxx>]

Benji York has proposed merging lp:~benji/launchpad/faster-wadl into lp:launchpad/devel.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)


This branch uses the stdlib multiprocess module to run WADL generation
in parallel subprocesses.  This speeds up WADL generation a fair bit:
91.9s to 70.0s on my laptop.

The change was a straight-forward refactoring to lift out the make_files
function and then use it as the target of a Process.

The script can be run by itself like so:

    LPCONFIG=development bin/py ./utilities/create-lp-wadl-and-apidoc.py --force "lib/canonical/launchpad/apidoc/wadl-development-%(version)s.xml"

-- 
https://code.launchpad.net/~benji/launchpad/faster-wadl/+merge/41315
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~benji/launchpad/faster-wadl into lp:launchpad/devel.

=== modified file 'utilities/create-lp-wadl-and-apidoc.py'
--- utilities/create-lp-wadl-and-apidoc.py	2010-10-25 13:16:10 +0000
+++ utilities/create-lp-wadl-and-apidoc.py	2010-11-19 14:39:37 +0000
@@ -12,6 +12,7 @@
 """
 import _pythonpath # Not lint, actually needed.
 
+from multiprocessing import Process
 import optparse
 import os
 import sys
@@ -32,6 +33,31 @@
     f.close()
 
 
+def make_files(path_template, directory, version, force):
+    wadl_filename = path_template % {'version': version}
+    # If the WADL file doesn't exist or we're being forced to regenerate
+    # it...
+    if (not os.path.exists(wadl_filename) or force):
+        print "Writing WADL for version %s to %s." % (
+            version, wadl_filename)
+        write(wadl_filename, generate_wadl(version))
+    else:
+        print "Skipping already present WADL file:", wadl_filename
+
+    # Now, convert the WADL into an human-readable description and
+    # put the HTML in the same directory as the WADL.
+    html_filename = os.path.join(directory, version + ".html")
+    # If the HTML file doesn't exist or we're being forced to regenerate
+    # it...
+    if (not os.path.exists(html_filename) or force):
+        print "Writing apidoc for version %s to %s" % (
+            version, html_filename)
+        write(html_filename, generate_html(wadl_filename,
+            suppress_stderr=False))
+    else:
+        print "Skipping already present HTML file:", html_filename
+
+
 def main(path_template, force=False):
     WebServiceApplication.cached_wadl = None # do not use cached file version
     execute_zcml_for_scripts()
@@ -47,29 +73,17 @@
     f = open(index_filename, 'w')
     f.write(template(config=config))
 
+    # Start a process to build each set of WADL and HTML files.
+    processes = []
     for version in config.active_versions:
-        wadl_filename = path_template % {'version': version}
-        # If the WADL file doesn't exist or we're being forced to regenerate
-        # it...
-        if (not os.path.exists(wadl_filename) or force):
-            print "Writing WADL for version %s to %s." % (
-                version, wadl_filename)
-            write(wadl_filename, generate_wadl(version))
-        else:
-            print "Skipping already present WADL file:", wadl_filename
+        p = Process(target=make_files,
+            args=(path_template, directory, version, force))
+        p.start()
+        processes.append(p)
 
-        # Now, convert the WADL into an human-readable description and
-        # put the HTML in the same directory as the WADL.
-        html_filename = os.path.join(directory, version + ".html")
-        # If the HTML file doesn't exist or we're being forced to regenerate
-        # it...
-        if (not os.path.exists(html_filename) or force):
-            print "Writing apidoc for version %s to %s" % (
-                version, html_filename)
-            write(html_filename, generate_html(wadl_filename,
-                suppress_stderr=False))
-        else:
-            print "Skipping already present HTML file:", html_filename
+    # Wait for all the subprocesses to finish.
+    for p in processes:
+        p.join()
 
     return 0