launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging lp:~cjwatson/launchpad/ptuj-via-disk into lp:launchpad.

Commit message:
Make PackageTranslationsUploadJob download librarian files to disk rather than memory.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/ptuj-via-disk/+merge/338894

This seems like a better idea for files that are potentially hundreds of megabytes (hi, LibreOffice).  I suspect that the gzip module will still buffer in memory, but this is a start.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/ptuj-via-disk into lp:launchpad.

=== modified file 'lib/lp/soyuz/model/packagetranslationsuploadjob.py'
--- lib/lp/soyuz/model/packagetranslationsuploadjob.py	2015-07-09 20:06:17 +0000
+++ lib/lp/soyuz/model/packagetranslationsuploadjob.py	2018-02-23 11:46:42 +0000
@@ -1,4 +1,4 @@
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
+# Copyright 2013-2018 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __metaclass__ = type
@@ -8,6 +8,9 @@
     'PackageTranslationsUploadJob',
     ]
 
+import os
+import tempfile
+
 from lazr.delegates import delegate_to
 import simplejson
 from zope.component import getUtility
@@ -16,8 +19,8 @@
     provider,
     )
 
+from lp.registry.interfaces.distroseries import IDistroSeriesSet
 from lp.registry.interfaces.sourcepackagename import ISourcePackageNameSet
-from lp.registry.interfaces.distroseries import IDistroSeriesSet
 from lp.services.config import config
 from lp.services.database.interfaces import IStore
 from lp.services.job.interfaces.job import JobType
@@ -27,6 +30,7 @@
     )
 from lp.services.job.runner import BaseRunnableJob
 from lp.services.librarian.interfaces import ILibraryFileAliasSet
+from lp.services.librarian.utils import filechunks
 from lp.services.mail.sendmail import format_address_for_person
 from lp.soyuz.interfaces.packagetranslationsuploadjob import (
     IPackageTranslationsUploadJob,
@@ -141,19 +145,28 @@
     def attachTranslationFiles(self, by_maintainer):
         distroseries = self.distroseries
         sourcepackagename = self.sourcepackagename
+        libraryfilealias = self.libraryfilealias
         only_templates = distroseries.getSourcePackage(
             sourcepackagename).has_sharing_translation_templates
         importer = self.requester
-        tarball = self.libraryfilealias.read()
-
-        queue = getUtility(ITranslationImportQueue)
-
-        queue.addOrUpdateEntriesFromTarball(
-            tarball, by_maintainer, importer,
-            sourcepackagename=sourcepackagename,
-            distroseries=distroseries,
-            filename_filter=_filter_ubuntu_translation_file,
-            only_templates=only_templates)
+        with tempfile.NamedTemporaryFile(
+                prefix='package-translations-upload-job-') as tarball:
+            libraryfilealias.open()
+            try:
+                for chunk in filechunks(self.libraryfilealias):
+                    tarball.write(chunk)
+            finally:
+                libraryfilealias.close()
+            tarball.seek(0, os.SEEK_SET)
+
+            queue = getUtility(ITranslationImportQueue)
+
+            queue.addOrUpdateEntriesFromTarball(
+                tarball, by_maintainer, importer,
+                sourcepackagename=sourcepackagename,
+                distroseries=distroseries,
+                filename_filter=_filter_ubuntu_translation_file,
+                only_templates=only_templates)
 
     def run(self):
         self.attachTranslationFiles(True)