launchpad-reviewers team mailing list archive

[Aaron Bentley <aaron@xxxxxxxxxxxxx>]

Aaron Bentley has proposed merging lp:~abentley/launchpad/update-sourcecode-cache into lp:launchpad.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~abentley/launchpad/update-sourcecode-cache/+merge/62167

= Summary =
Make update-sourcecode fast.

== Proposed fix ==
Cache revision info locally as json in utilities/sourcedeps.cache.  Use that
info to skip branches whose revision info matches the cached info for that
revno.

== Pre-implementation notes ==
None

== Implementation details ==
This implementation ensures that even if the revno matches the expected value,
the revision-id must also match.  The cost is that everyone who changes
sourcedeps.conf must also commit sourcedeps.cache, which is automatically
updated.

Also, some lint fixin'.

== Tests ==
None

== Demo and Q/A ==
utilities/update-sourcecode

= Launchpad lint =

Checking for conflicts and issues in changed files.

Linting changed files:
  lib/devscripts/sourcecode.py
  utilities/sourcedeps.cache
-- 
https://code.launchpad.net/~abentley/launchpad/update-sourcecode-cache/+merge/62167
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~abentley/launchpad/update-sourcecode-cache into lp:launchpad.

=== modified file 'lib/devscripts/sourcecode.py'
--- lib/devscripts/sourcecode.py	2010-04-19 11:05:07 +0000
+++ lib/devscripts/sourcecode.py	2011-05-24 17:21:05 +0000
@@ -10,6 +10,8 @@
     'plan_update',
     ]
 
+import errno
+import json
 import optparse
 import os
 import shutil
@@ -61,6 +63,16 @@
         optional = False
     return branch_name, branch_url, revision, optional
 
+def load_cache(cache_filename):
+    try:
+        cache_file = open(cache_filename, 'rb')
+    except IOError as e:
+        if e.errno == errno.ENOENT:
+            return {}
+        else:
+            raise
+    with cache_file:
+        return json.load(cache_file)
 
 def interpret_config(config_entries, public_only):
     """Interpret a configuration stream, as parsed by 'parse_config_file'.
@@ -178,6 +190,42 @@
             possible_transports=possible_transports)
 
 
+def find_stale(updated, cache, sourcecode_directory, quiet):
+    """Find branches whose revision info doesn't match the cache."""
+    new_updated = dict(updated)
+    for project, (branch_url, revision, optional) in updated.iteritems():
+        cache_revision_info = cache.get(project)
+        if cache_revision_info is None:
+            continue
+        if cache_revision_info[0] != int(revision):
+            continue
+        destination = os.path.join(sourcecode_directory, project)
+        try:
+            branch = Branch.open(destination)
+        except BzrError:
+            continue
+        if list(branch.last_revision_info()) != cache_revision_info:
+            continue
+        if not quiet:
+            print '%s is already up to date.' % project
+        del new_updated[project]
+    return new_updated
+
+
+def update_cache(cache, cache_filename, changed, sourcecode_directory, quiet):
+    """Update the cache with the changed branches."""
+    if len(changed) == 0:
+        return
+    if not quiet:
+        print 'Cache updated.  Please commit "%s".' % cache_filename
+    for project, (branch_url, revision, optional) in changed.iteritems():
+        destination = os.path.join(sourcecode_directory, project)
+        branch = Branch.open(destination)
+        cache[project] = branch.last_revision_info()
+    with open(cache_filename, 'wb') as cache_file:
+        json.dump(cache, cache_file, indent=4)
+
+
 def update_branches(sourcecode_directory, update_branches,
                     possible_transports=None, tip=False, quiet=False):
     """Update the existing branches in sourcecode."""
@@ -246,12 +294,13 @@
             os.unlink(destination)
 
 
-def update_sourcecode(sourcecode_directory, config_filename, public_only,
-                      tip, dry_run, quiet=False):
+def update_sourcecode(sourcecode_directory, config_filename, cache_filename,
+                      public_only, tip, dry_run, quiet=False):
     """Update the sourcecode."""
     config_file = open(config_filename)
     config = interpret_config(parse_config_file(config_file), public_only)
     config_file.close()
+    cache = load_cache(cache_filename)
     branches = find_branches(sourcecode_directory)
     new, updated, removed = plan_update(branches, config)
     possible_transports = []
@@ -262,8 +311,13 @@
     else:
         get_branches(
             sourcecode_directory, new, possible_transports, tip, quiet)
+        updated = find_stale(updated, cache, sourcecode_directory, quiet)
         update_branches(
             sourcecode_directory, updated, possible_transports, tip, quiet)
+        changed = dict(updated)
+        changed.update(new)
+        update_cache(
+            cache, cache_filename, changed, sourcecode_directory, quiet)
         remove_branches(sourcecode_directory, removed, quiet)
 
 
@@ -302,6 +356,8 @@
         config_filename = args[2]
     else:
         config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf')
+    cache_filename = os.path.join(
+        root, 'utilities', 'sourcedeps.cache')
     if len(args) > 3:
         parser.error("Too many arguments.")
     if not options.quiet:
@@ -313,6 +369,6 @@
         sys.stdin, sys.stdout, sys.stderr)
     load_plugins()
     update_sourcecode(
-        sourcecode_directory, config_filename,
+        sourcecode_directory, config_filename, cache_filename,
         options.public_only, options.tip, options.dry_run, options.quiet)
     return 0

=== added file 'utilities/sourcedeps.cache'
--- utilities/sourcedeps.cache	1970-01-01 00:00:00 +0000
+++ utilities/sourcedeps.cache	2011-05-24 17:21:05 +0000
@@ -0,0 +1,70 @@
+{
+    "bzr-builder": [
+        68, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20101123183213-777lz46xgagn1deg"
+    ], 
+    "testresources": [
+        16, 
+        "robertc@xxxxxxxxxxxxxxxxx-20050911111209-ee5da49011cf936a"
+    ], 
+    "mailman": [
+        976, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20091021010617-prbs2ay6nhxx515v"
+    ], 
+    "cscvs": [
+        432, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20100414131608-cf6jatd9zk6l6wpk"
+    ], 
+    "pygpgme": [
+        49, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20100325120516-q8to5dx3gga4wlvi"
+    ], 
+    "subvertpy": [
+        2042, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20100727083620-u93rikomkjfj82dw"
+    ], 
+    "python-debian": [
+        186, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20110329053617-irncjfr14k0m00zp"
+    ], 
+    "pygettextpo": [
+        24, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20100601182722-wo7h2fh0fvyw3aaq"
+    ], 
+    "lpreview": [
+        23, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20090720061538-euyh68ifavhy0pi8"
+    ], 
+    "bzr-git": [
+        258, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20100812210320-i61pvvvo9mjo2wp7"
+    ], 
+    "bzr-loom": [
+        48, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20100806142619-4huubp7dmz6ax9j2"
+    ], 
+    "old_xmlplus": [
+        4, 
+        "sinzui-20090526164636-1swugzupwvjgomo4"
+    ], 
+    "loggerhead": [
+        445, 
+        "john@xxxxxxxxxxxxxxxxx-20110325141442-536j4be3x0c464zy"
+    ], 
+    "difftacular": [
+        6, 
+        "aaron@xxxxxxxxxxxxxxxx-20100715135013-uoi3q430urx9gwb8"
+    ], 
+    "bzr-svn": [
+        2710, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20100830004921-6c4hehto20axdkn6"
+    ], 
+    "bzr-hg": [
+        283, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20100814010415-lzqc5jufq5u1hd44"
+    ], 
+    "dulwich": [
+        424, 
+        "launchpad@xxxxxxxxxxxxxxxxx-20100812211021-x1uaubka3ael1j62"
+    ]
+}
\ No newline at end of file