launchpad-reviewers team mailing list archive

[Xaav <xaav@xxxxxxx>]

Xaav has proposed merging lp:~xaav/loggerhead/export-tarball into lp:loggerhead.

Requested reviews:
  Gavin Panella (allenap)
  Martin Albisetti (beuno)
  Vincent Ladeuil (vila)
  Launchpad code reviewers (launchpad-reviewers): code
  Robert Collins (lifeless)
Related bugs:
  Bug #240580 in loggerhead: "Ability to download a tarball for a revision"
  https://bugs.launchpad.net/loggerhead/+bug/240580

For more details, see:
https://code.launchpad.net/~xaav/loggerhead/export-tarball/+merge/66408

This branch **may** accomplish exporting the tarball using chunked transfer encoding. The code all looks to be correct, but I have not tested it, so I would like your opinion.

Thanks!
-- 
https://code.launchpad.net/~xaav/loggerhead/export-tarball/+merge/66408
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~xaav/loggerhead/export-tarball into lp:loggerhead.

=== modified file 'loggerhead/apps/branch.py'
--- loggerhead/apps/branch.py	2011-06-28 16:09:37 +0000
+++ loggerhead/apps/branch.py	2011-06-30 02:59:24 +0000
@@ -33,7 +33,7 @@
 from loggerhead.controllers.atom_ui import AtomUI
 from loggerhead.controllers.changelog_ui import ChangeLogUI
 from loggerhead.controllers.diff_ui import DiffUI
-from loggerhead.controllers.download_ui import DownloadUI
+from loggerhead.controllers.download_ui import DownloadUI, DownloadTarballUI
 from loggerhead.controllers.filediff_ui import FileDiffUI
 from loggerhead.controllers.inventory_ui import InventoryUI
 from loggerhead.controllers.revision_ui import RevisionUI
@@ -49,7 +49,7 @@
 
     def __init__(self, branch, friendly_name=None, config={},
                  graph_cache=None, branch_link=None, is_root=False,
-                 served_url=_DEFAULT, use_cdn=False):
+                 served_url=_DEFAULT, use_cdn=False, export_tarballs=True):
         self.branch = branch
         self._config = config
         self.friendly_name = friendly_name
@@ -61,6 +61,7 @@
         self.is_root = is_root
         self.served_url = served_url
         self.use_cdn = use_cdn
+        self.export_tarballs = export_tarballs
 
     def get_history(self):
         file_cache = None
@@ -126,6 +127,7 @@
         'revision': RevisionUI,
         'search': SearchUI,
         'view': ViewUI,
+        'tarball': DownloadTarballUI,
         }
 
     def last_updated(self):
@@ -176,11 +178,20 @@
         self.branch.lock_read()
         try:
             try:
+<<<<<<< TREE
                 c = self.lookup_app(environ)
                 return c(environ, start_response)
+=======
+                c = cls(self, self.get_history)
+                to_ret = c(environ, start_response)
+>>>>>>> MERGE-SOURCE
             except:
                 environ['exc_info'] = sys.exc_info()
                 environ['branch'] = self
                 raise
+            if type(to_ret) == type(httpexceptions.HTTPSeeOther('/')):
+                raise to_ret
+            else:
+                return to_ret
         finally:
             self.branch.unlock()

=== modified file 'loggerhead/config.py'
--- loggerhead/config.py	2010-05-12 14:38:05 +0000
+++ loggerhead/config.py	2011-06-30 02:59:24 +0000
@@ -36,6 +36,7 @@
         use_cdn=False,
         sql_dir=None,
         allow_writes=False,
+        export_tarballs=True,
         )
     parser.add_option("--user-dirs", action="store_true",
                       help="Serve user directories as ~user.")
@@ -75,6 +76,8 @@
                       help="The directory to place the SQL cache in")
     parser.add_option("--allow-writes", action="store_true",
                       help="Allow writing to the Bazaar server.")
+    parser.add_option("--export-tarballs", action="store_true",
+                      help="Allow exporting revisions to tarballs.")
     return parser
 
 

=== modified file 'loggerhead/controllers/__init__.py'
--- loggerhead/controllers/__init__.py	2011-06-28 16:09:37 +0000
+++ loggerhead/controllers/__init__.py	2011-06-30 02:59:24 +0000
@@ -21,7 +21,7 @@
 import simplejson
 import time
 
-from paste.httpexceptions import HTTPNotFound
+from paste.httpexceptions import HTTPNotFound, HTTPSeeOther
 from paste.request import path_info_pop, parse_querystring
 
 from loggerhead import util

=== modified file 'loggerhead/controllers/download_ui.py'
--- loggerhead/controllers/download_ui.py	2010-05-05 19:03:40 +0000
+++ loggerhead/controllers/download_ui.py	2011-06-30 02:59:24 +0000
@@ -19,46 +19,51 @@
 
 import logging
 import mimetypes
+import os
 import urllib
 
 from paste import httpexceptions
 from paste.request import path_info_pop
 
 from loggerhead.controllers import TemplatedBranchView
+from loggerhead.exporter import export_archive
 
 log = logging.getLogger("loggerhead.controllers")
 
 
 class DownloadUI (TemplatedBranchView):
 
-    def __call__(self, environ, start_response):
-        # /download/<rev_id>/<file_id>/[filename]
-
-        h = self._history
-
+    def encode_filename(self, filename):
+
+        return urllib.quote(filename.encode('utf-8'))
+
+    def get_args(self, environ):
         args = []
         while True:
             arg = path_info_pop(environ)
             if arg is None:
                 break
             args.append(arg)
+        return args
 
+    def __call__(self, environ, start_response):
+        # /download/<rev_id>/<file_id>/[filename]
+        h = self._history
+        args = self.get_args(environ)
         if len(args) < 2:
             raise httpexceptions.HTTPMovedPermanently(
                 self._branch.absolute_url('/changes'))
-
         revid = h.fix_revid(args[0])
         file_id = args[1]
         path, filename, content = h.get_file(file_id, revid)
         mime_type, encoding = mimetypes.guess_type(filename)
         if mime_type is None:
             mime_type = 'application/octet-stream'
-
         self.log.info('/download %s @ %s (%d bytes)',
                       path,
                       h.get_revno(revid),
                       len(content))
-        encoded_filename = urllib.quote(filename.encode('utf-8'))
+        encoded_filename = self.encode_filename(filename)
         headers = [
             ('Content-Type', mime_type),
             ('Content-Length', str(len(content))),
@@ -67,3 +72,29 @@
             ]
         start_response('200 OK', headers)
         return [content]
+
+
+class DownloadTarballUI(DownloadUI):
+
+    def __call__(self, environ, start_response):
+        """Stream a tarball from a bazaar branch."""
+        # Tried to re-use code from downloadui, not very successful
+        format = "tar"
+        history = self._history
+        self.args = self.get_args(environ)
+        if len(self.args):
+            revid = history.fix_revid(self.args[0])
+        else:
+            revid = self.get_revid()
+        if self._branch.export_tarballs:
+            root = 'branch'
+            encoded_filename = self.encode_filename(root + format)
+            headers = [
+                ('Content-Type', 'application/octet-stream'),
+                ('Content-Disposition',
+                 "attachment; filename*=utf-8''%s" % (encoded_filename,)),
+                ]
+            start_response('200 OK', headers)
+            return export_archive(history, root, revid, format)
+        else:
+            raise httpexceptions.HTTPSeeOther('/')

=== modified file 'loggerhead/controllers/revision_ui.py'
--- loggerhead/controllers/revision_ui.py	2011-06-27 17:11:24 +0000
+++ loggerhead/controllers/revision_ui.py	2011-06-30 02:59:24 +0000
@@ -136,9 +136,19 @@
                 self._branch.friendly_name,
                 self._branch.is_root,
                 'changes'))
+<<<<<<< TREE
 
         values.update({
             'history': self._history,
+=======
+        can_export = self._branch.export_tarballs
+        return {
+            'branch': self._branch,
+            'revid': revid,
+            'change': change,
+            'file_changes': file_changes,
+            'diff_chunks': diff_chunks,
+>>>>>>> MERGE-SOURCE
             'link_data': simplejson.dumps(link_data),
             'json_specific_path': simplejson.dumps(path),
             'path_to_id': simplejson.dumps(path_to_id),
@@ -149,6 +159,15 @@
             'filter_file_id': filter_file_id,
             'diff_chunks': diff_chunks,
             'query': query,
+<<<<<<< TREE
             'specific_path': path,
             'start_revid': start_revid,
         })
+=======
+            'remember': remember,
+            'compare_revid': compare_revid,
+            'url': self._branch.context_url,
+            'directory_breadcrumbs': directory_breadcrumbs,
+            'can_export': can_export,
+        }
+>>>>>>> MERGE-SOURCE

=== added file 'loggerhead/exporter.py'
--- loggerhead/exporter.py	1970-01-01 00:00:00 +0000
+++ loggerhead/exporter.py	2011-06-30 02:59:24 +0000
@@ -0,0 +1,46 @@
+# Copyright (C) 2011 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+"""Exports an archive from a bazaar branch"""
+
+from bzrlib.export import get_export_generator
+
+
+class ExporterFileObject(object):
+
+    def __init__(self):
+        self._buffer = []
+
+    def write(self, s):
+        self._buffer.append(s)
+
+    def get_buffer(self):
+        try:
+            return ''.join(self._buffer)
+        finally:
+            self._buffer = []
+
+
+def export_archive(history, root, revid, format=".tar.gz"):
+    """Export tree contents to an archive
+
+    :param history: Instance of history to export
+    :param root: Root location inside the archive.
+    :param revid: Revision to export
+    :param format: Format of the archive
+    """
+    fileobj = ExporterFileObject()
+    tree = history._branch.repository.revision_tree(revid)
+    for _ in get_export_generator(tree=tree, root=root, fileobj=fileobj, format=format):
+        yield fileobj.get_buffer()
+    # Might have additonal contents written
+    yield fileobj.get_buffer()

=== modified file 'loggerhead/history.py'
--- loggerhead/history.py	2011-03-25 13:09:10 +0000
+++ loggerhead/history.py	2011-06-30 02:59:24 +0000
@@ -33,6 +33,7 @@
 import re
 import textwrap
 import threading
+import tarfile
 
 from bzrlib import tag
 import bzrlib.branch
@@ -44,6 +45,7 @@
 from loggerhead import search
 from loggerhead import util
 from loggerhead.wholehistory import compute_whole_history_data
+from bzrlib.export.tar_exporter import export_tarball
 
 
 def is_branch(folder):
@@ -816,4 +818,6 @@
             renamed=sorted(reporter.renamed, key=lambda x: x.new_filename),
             removed=sorted(reporter.removed, key=lambda x: x.filename),
             modified=sorted(reporter.modified, key=lambda x: x.filename),
-            text_changes=sorted(reporter.text_changes, key=lambda x: x.filename))
+            text_changes=sorted(reporter.text_changes,
+                                key=lambda x: x.filename))
+

=== added directory 'loggerhead/static/downloads'
=== modified file 'loggerhead/templates/revision.pt'
--- loggerhead/templates/revision.pt	2011-06-28 16:51:55 +0000
+++ loggerhead/templates/revision.pt	2011-06-30 02:59:24 +0000
@@ -84,7 +84,10 @@
                tal:attributes="href python:url(['/diff', change.revno], clear=1)">download diff</a>
             <a tal:condition="python:compare_revid is not None"
                tal:attributes="href python:url(['/diff', change.revno, history.get_revno(compare_revid)], clear=1)">download diff</a>
-          </li>
+           </li>
+           <li tal:condition="python:can_export">
+            <a tal:attributes="href python:url(['/tarball', change.revno])">Download tarball</a>
+           </li>
           <li id="last"><a tal:attributes="href python:url(['/changes', change.revno]);
                                            title string:view history from revision ${change/revno}"
                            tal:content="string:view history from revision ${change/revno}"></a></li>

=== modified file 'loggerhead/tests/test_controllers.py'
--- loggerhead/tests/test_controllers.py	2011-06-28 16:06:12 +0000
+++ loggerhead/tests/test_controllers.py	2011-06-30 02:59:24 +0000
@@ -1,4 +1,14 @@
+<<<<<<< TREE
 import simplejson
+=======
+from cStringIO import StringIO
+import logging
+import tarfile
+
+from paste.httpexceptions import HTTPServerError
+
+from bzrlib import errors
+>>>>>>> MERGE-SOURCE
 
 from loggerhead.apps.branch import BranchWSGIApp
 from loggerhead.controllers.annotate_ui import AnnotateUI
@@ -135,6 +145,7 @@
             kwargs={'file_id': 'file_id'}, headers={})
         annotated = annotate_info['annotated']
         self.assertEqual(2, len(annotated))
+<<<<<<< TREE
         self.assertEqual('2', annotated[1].change.revno)
         self.assertEqual('1', annotated[2].change.revno)
 
@@ -204,3 +215,23 @@
         revlog_ui = branch_app.lookup_app(env)
         self.assertOkJsonResponse(revlog_ui, env)
 
+=======
+        self.assertEqual('2', annotated[0].change.revno)
+        self.assertEqual('1', annotated[1].change.revno)
+
+
+class TestDownloadTarballUI(BasicTests):
+
+    def setUp(self):
+        super(TestDownloadTarballUI, self).setUp()
+        self.createBranch()
+
+    def test_download_tarball(self):
+        app = self.setUpLoggerhead()
+        res = app.get('/tarball')
+        f = open('tarball', 'w')
+        f.write(res)
+        f.close()
+        self.failIf(not tarfile.is_tarfile('tarball'))
+        # Now check the content. TBC
+>>>>>>> MERGE-SOURCE