launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #15738
[Merge] lp:~ursinha/launchpad/convert-translationuploads-to-job into lp:launchpad
Ursula Junque has proposed merging lp:~ursinha/launchpad/convert-translationuploads-to-job into lp:launchpad with lp:~stevenk/launchpad/packagediff-job as a prerequisite.
Commit message:
publishRosettaTranslations now fires a job that uploads and attaches the translation files, to stop slowing down the publisher
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ursinha/launchpad/convert-translationuploads-to-job/+merge/176415
This branch creates a TranslationsUploadJob that is responsible for uploading and attaching translation files to a sourcepackagerelease. Now, whenever a PackageUploadCustom of format ROSETTA_TRANSLATIONS is processed in the publisher queue, a TranslationsUploadJob is created and the publisher can move along.
--
https://code.launchpad.net/~ursinha/launchpad/convert-translationuploads-to-job/+merge/176415
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~ursinha/launchpad/convert-translationuploads-to-job into lp:launchpad.
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2013-07-23 15:53:27 +0000
+++ lib/lp/services/config/schema-lazr.conf 2013-07-23 15:53:28 +0000
@@ -1743,6 +1743,10 @@
module: lp.translations.interfaces.translationpackagingjob
dbuser: rosettaadmin
+[ITranslationsUploadJobSource]
+module: lp.soyuz.interfaces.translationsuploadjob
+dbuser: process_accepted
+
[IPersonMergeJobSource]
module: lp.registry.interfaces.persontransferjob
dbuser: person-merge-job
=== modified file 'lib/lp/services/job/interfaces/job.py'
--- lib/lp/services/job/interfaces/job.py 2013-07-23 15:53:27 +0000
+++ lib/lp/services/job/interfaces/job.py 2013-07-23 15:53:28 +0000
@@ -78,6 +78,13 @@
Job to generate the diff between two SourcePackageReleases.
""")
+ UPLOAD_TRANSLATIONS_FILES = DBItem(1, """
+ Upload Translations Files
+
+ Job to upload translations files and attach them to a
+ SourcePackageRelease.
+ """)
+
class IJob(Interface):
"""Basic attributes of a job."""
=== modified file 'lib/lp/soyuz/configure.zcml'
--- lib/lp/soyuz/configure.zcml 2013-07-23 15:53:27 +0000
+++ lib/lp/soyuz/configure.zcml 2013-07-23 15:53:28 +0000
@@ -995,6 +995,18 @@
<allow interface=".interfaces.packagediffjob.IPackageDiffJob" />
</class>
+ <!-- TranslationsUploadJobSource -->
+ <securedutility
+ component=".model.translationsuploadjob.TranslationsUploadJob"
+ provides=".interfaces.translationsuploadjob.ITranslationsUploadJobSource">
+ <allow interface=".interfaces.translationsuploadjob.ITranslationsUploadJobSource" />
+ </securedutility>
+
+ <!-- TranslationsUploadJob -->
+ <class class=".model.translationsuploadjob.TranslationsUploadJob">
+ <allow
+ interface=".interfaces.translationsuploadjob.ITranslationsUploadJob" />
+ </class>
<webservice:register module="lp.soyuz.interfaces.webservice" />
</configure>
=== modified file 'lib/lp/soyuz/doc/distroseriesqueue-translations.txt'
--- lib/lp/soyuz/doc/distroseriesqueue-translations.txt 2011-12-30 06:14:56 +0000
+++ lib/lp/soyuz/doc/distroseriesqueue-translations.txt 2013-07-23 15:53:28 +0000
@@ -31,6 +31,8 @@
>>> from lp.services.database.constants import UTC_NOW
>>> from lp.registry.interfaces.sourcepackage import SourcePackageUrgency
+ >>> from lp.soyuz.model.translationsuploadjob import TranslationsUploadJob
+
# Login as an admin.
>>> login('foo.bar@xxxxxxxxxxxxx')
@@ -157,6 +159,15 @@
... status=PackageUploadStatus.NEW)[0]
>>> queue_item.customfiles[0].publish()
+When publish() runs, it creates a TranslationsUploadJob that will effectively
+process the translation files. We need to find and run it to be able to
+verify the imported files.
+ >>> def runPendingTranslationUploadJob():
+ ... job = list(TranslationsUploadJob.iterReady())[0]
+ ... job.run()
+
+ >>> runPendingTranslationUploadJob()
+
As we can see from the translation import queue content.
>>> for entry in translation_import_queue.getAllEntries(target=ubuntu):
@@ -206,6 +217,7 @@
>>> queue_item = dapper.getPackageUploads(PackageUploadStatus.NEW)[0]
>>> queue_item.pocket = PackagePublishingPocket.UPDATES
>>> queue_item.customfiles[0].publish()
+ >>> runPendingTranslationUploadJob()
As we can see from the translation import queue content.
@@ -239,6 +251,7 @@
>>> queue_item.builds[0].build.source_package_release.override(
... component=restricted_component)
>>> queue_item.customfiles[0].publish()
+ >>> runPendingTranslationUploadJob()
As we can see from the translation import queue content.
@@ -340,154 +353,6 @@
>>> translation_import_queue.getAllEntries(target=ubuntu).count()
0
-
-Translations importer: publishRosettaTranslations
--------------------------------------------------
-
-We create mock objects for SourcePackageRelease, PackageUpload and
-PackageUploadCustom: these will emulate everything we need to document
-different interpretations of "importer" in attachTranslationFiles.
-
- >>> from zope.interface import implements
- >>> from lp.app.interfaces.launchpad import ILaunchpadCelebrities
- >>> from lp.soyuz.model.queue import PackageUploadCustom
- >>> from lp.soyuz.interfaces.archive import (
- ... IArchive, ArchivePurpose)
- >>> from lp.soyuz.interfaces.queue import (
- ... IPackageUpload, IPackageUploadCustom)
- >>> from lp.registry.interfaces.person import IPerson
- >>> from lp.soyuz.enums import PackageUploadCustomFormat
- >>> from lp.soyuz.interfaces.component import IComponentSet
- >>> from lp.soyuz.interfaces.sourcepackagerelease import (
- ... ISourcePackageRelease)
- >>> from lp.registry.interfaces.pocket import PackagePublishingPocket
-
- >>> class MockArchive:
- ... implements(IArchive)
- ... def __init__(self, purpose):
- ... self.purpose = purpose
-
- >>> class MockDistroSeries:
- ... implements(IDistroSeries)
- ... def __init__(self, version):
- ... self.version = version
-
- >>> class MockSourcePackageRelease:
- ... implements(ISourcePackageRelease)
- ... def __init__(self, component, creator, upload_distroseries):
- ... self.component = getUtility(IComponentSet)[component]
- ... self.upload_distroseries = upload_distroseries
- ... self.creator = creator
- ... self.packageupload = 1
- ...
- ... def attachTranslationFiles(self, file, imported, importer):
- ... if (importer is not None and
- ... not IPerson.providedBy(importer)):
- ... print "`importer` not a person!"
- ... print "Imported by: %s" % (
- ... getattr(importer, "name", "None"))
-
- >>> class MockPackageUpload:
- ... implements(IPackageUpload)
- ... def __init__(self, pocket, auto_sync, sourcepackagerelease,
- ... archive):
- ... self.id = 1
- ... self.pocket = pocket
- ... self.auto_sync = auto_sync
- ... self.sourcepackagerelease = sourcepackagerelease
- ... self.archive = archive
- ...
- ... def isAutoSyncUpload(self, changed_by_email=None):
- ... return self.auto_sync
-
- >>> class MockPackageUploadCustom(PackageUploadCustom):
- ... implements(IPackageUploadCustom)
- ... packageupload = None
- ...
- ... def __init__(self):
- ... self.customformat = (
- ... PackageUploadCustomFormat.ROSETTA_TRANSLATIONS)
-
-For translations from auto-synced packages we consider the importer to be
-'katie' (archive@xxxxxxxxxx).
-
- >>> katie = getUtility(ILaunchpadCelebrities).katie
- >>> release_pocket = PackagePublishingPocket.RELEASE
- >>> archive = MockArchive(ArchivePurpose.PRIMARY)
-
- >>> distro_series = MockDistroSeries(u'9.04')
- >>> katie_sourcepackagerelease = MockSourcePackageRelease(
- ... 'main', katie, distro_series)
- >>> sync_package_upload = MockPackageUpload(
- ... release_pocket, True, katie_sourcepackagerelease, archive)
- >>> sync_package_upload.isAutoSyncUpload()
- True
- >>> translations_upload = MockPackageUploadCustom()
- >>> translations_upload.packageupload = sync_package_upload
- >>> translations_upload.publishRosettaTranslations()
- Imported by: katie
-
-Non-auto-sync uploads by 'katie' still indicate 'katie' as the uploader.
-
- >>> non_sync_package_upload = MockPackageUpload(
- ... release_pocket, False, katie_sourcepackagerelease, archive)
- >>> non_sync_package_upload.isAutoSyncUpload()
- False
- >>> translations_upload.packageupload = non_sync_package_upload
- >>> translations_upload.publishRosettaTranslations()
- Imported by: katie
-
-Uploads by anyone else are treated as if importer is the packager.
-
- >>> person_set = getUtility(IPersonSet)
- >>> carlos = person_set.getByName('carlos')
- >>> carlos_sourcepackagerelease = MockSourcePackageRelease(
- ... 'main', carlos, distro_series)
- >>> carlos_package_upload = MockPackageUpload(
- ... release_pocket, False, carlos_sourcepackagerelease, archive)
- >>> carlos_package_upload.isAutoSyncUpload()
- False
- >>> translations_upload.packageupload = carlos_package_upload
- >>> translations_upload.publishRosettaTranslations()
- Imported by: carlos
-
-Uploads for distroseries before Oneiric or later may not be targeted
-to any component but 'main' and 'restricted'. The upload attempt is ignored.
-
- >>> katie_sourcepackagerelease = MockSourcePackageRelease(
- ... 'universe', katie, distro_series)
- >>> sync_package_upload = MockPackageUpload(
- ... release_pocket, True, katie_sourcepackagerelease, archive)
- >>> translations_upload = MockPackageUploadCustom()
- >>> translations_upload.packageupload = sync_package_upload
- >>> translations_upload.publishRosettaTranslations()
-
-For Oneiric the import succeeds for 'universe'.
-
- >>> distro_series = MockDistroSeries(u'11.10')
- >>> katie_sourcepackagerelease = MockSourcePackageRelease(
- ... 'universe', katie, distro_series)
- >>> sync_package_upload = MockPackageUpload(
- ... release_pocket, True, katie_sourcepackagerelease, archive)
- >>> translations_upload = MockPackageUploadCustom()
- >>> translations_upload.packageupload = sync_package_upload
- >>> translations_upload.publishRosettaTranslations()
- Imported by: katie
-
-And for the 12.04 release the import succeeds for 'universe'.
-
- >>> distro_series = MockDistroSeries(u'12.04')
- >>> katie_sourcepackagerelease = MockSourcePackageRelease(
- ... 'universe', katie, distro_series)
- >>> sync_package_upload = MockPackageUpload(
- ... release_pocket, True, katie_sourcepackagerelease, archive)
- >>> translations_upload = MockPackageUploadCustom()
- >>> translations_upload.packageupload = sync_package_upload
- >>> translations_upload.publishRosettaTranslations()
- Imported by: katie
-
-
-
Translations tarball
~~~~~~~~~~~~~~~~~~~~
=== added file 'lib/lp/soyuz/interfaces/translationsuploadjob.py'
--- lib/lp/soyuz/interfaces/translationsuploadjob.py 1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/interfaces/translationsuploadjob.py 2013-07-23 15:53:28 +0000
@@ -0,0 +1,31 @@
+# Copyright 2013 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+__all__ = [
+ "ITranslationsUploadJob",
+ "ITranslationsUploadJobSource",
+ ]
+
+from lp.services.job.interfaces.job import (
+ IJobSource,
+ IRunnableJob,
+ )
+
+
+class ITranslationsUploadJobSource(IJobSource):
+ """An interface for acquiring ITranslationsUploadJob."""
+
+ def create(sourcepackagerelease, libraryfilealias):
+ """Create new translations upload job for a source package release."""
+
+ def get(sourcepackagerelease, libraryfilealias):
+ """Retrieve the translation's upload job for a source package release.
+
+ :return: `None` or an `ITranslationsUploadJob`.
+ """
+
+
+class ITranslationsUploadJob(IRunnableJob):
+ """A `Job` that uploads and attaches files to a `ISourcePackageRelease`."""
=== modified file 'lib/lp/soyuz/model/queue.py'
--- lib/lp/soyuz/model/queue.py 2013-07-16 08:10:32 +0000
+++ lib/lp/soyuz/model/queue.py 2013-07-23 15:53:28 +0000
@@ -103,6 +103,9 @@
IPublishingSet,
name_priority_map,
)
+from lp.soyuz.interfaces.translationsuploadjob import (
+ ITranslationsUploadJobSource,
+ )
from lp.soyuz.interfaces.queue import (
IPackageUpload,
IPackageUploadBuild,
@@ -1451,19 +1454,10 @@
# Ubuntu's MOTU told us that they are not able to handle
# translations like we do in main. We are going to import only
# packages in main.
- return
-
- # Set the importer to package creator.
- importer = sourcepackagerelease.creator
-
- # Attach the translation tarball. It's always published.
- try:
- sourcepackagerelease.attachTranslationFiles(
- self.libraryfilealias, True, importer=importer)
- except DownloadFailed:
- if logger is not None:
- debug(logger, "Unable to fetch %s to import it into Rosetta" %
- self.libraryfilealias.http_url)
+ return None
+
+ getUtility(ITranslationsUploadJobSource).create(
+ sourcepackagerelease, self.libraryfilealias)
def publishStaticTranslations(self, logger=None):
"""See `IPackageUploadCustom`."""
=== added file 'lib/lp/soyuz/model/translationsuploadjob.py'
--- lib/lp/soyuz/model/translationsuploadjob.py 1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/model/translationsuploadjob.py 2013-07-23 15:53:28 +0000
@@ -0,0 +1,101 @@
+# Copyright 2013 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+__all__ = [
+ 'TranslationsUploadJob',
+ ]
+
+from lazr.delegates import delegates
+import simplejson
+from zope.component import getUtility
+from zope.interface import (
+ classProvides,
+ implements,
+ )
+
+from lp.services.config import config
+from lp.services.database.interfaces import IStore
+from lp.services.job.interfaces.job import JobType
+from lp.services.job.model.job import (
+ EnumeratedSubclass,
+ Job,
+ )
+from lp.services.job.runner import BaseRunnableJob
+from lp.services.librarian.interfaces import ILibraryFileAliasSet
+from lp.soyuz.interfaces.translationsuploadjob import (
+ ITranslationsUploadJob,
+ ITranslationsUploadJobSource,
+ )
+from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
+
+
+class TranslationsUploadJobDerived(BaseRunnableJob):
+
+ __metaclass__ = EnumeratedSubclass
+
+ delegates(ITranslationsUploadJob)
+ classProvides(ITranslationsUploadJobSource)
+ config = config.ITranslationsUploadJobSource
+
+ def __init__(self, job):
+ assert job.base_job_type == JobType.UPLOAD_TRANSLATIONS_FILES
+ self.job = job
+ self.context = self
+
+ @classmethod
+ def create(cls, sourcepackagerelease, libraryfilealias):
+ job = Job(
+ base_job_type=JobType.UPLOAD_TRANSLATIONS_FILES,
+ requester=sourcepackagerelease.creator,
+ base_json_data=simplejson.dumps(
+ {'sourcepackagerelease': sourcepackagerelease.id,
+ 'libraryfilealias': libraryfilealias.id}))
+ derived = cls(job)
+ derived.celeryRunOnCommit()
+ return derived
+
+ @classmethod
+ def get(cls, sourcepackagerelease, libraryfilealias):
+ metadata = simplejson.dumps(
+ {'sourcepackagerelease': sourcepackagerelease.id,
+ 'libraryfilealias': libraryfilealias.id})
+ return cls(IStore(Job).find(Job, Job.base_json_data == metadata).one())
+
+ @classmethod
+ def iterReady(cls):
+ jobs = IStore(Job).find(
+ Job, Job.id.is_in(Job.ready_jobs),
+ Job.base_job_type == JobType.UPLOAD_TRANSLATIONS_FILES)
+ return [cls(job) for job in jobs]
+
+
+class TranslationsUploadJob(TranslationsUploadJobDerived):
+
+ implements(ITranslationsUploadJob)
+ classProvides(ITranslationsUploadJobSource)
+
+ @property
+ def sourcepackagerelease_id(self):
+ return simplejson.loads(self.base_json_data)['sourcepackagerelease']
+
+ @property
+ def libraryfilealias_id(self):
+ return simplejson.loads(self.base_json_data)['libraryfilealias']
+
+ @property
+ def sourcepackagerelease(self):
+ return SourcePackageRelease.get(self.sourcepackagerelease_id)
+
+ @property
+ def libraryfilealias(self):
+ return getUtility(ILibraryFileAliasSet)[self.libraryfilealias_id]
+
+ def run(self):
+ sourcepackagerelease = self.sourcepackagerelease
+ if sourcepackagerelease is not None:
+ libraryfilealias = self.libraryfilealias
+ importer = sourcepackagerelease.creator
+ sourcepackagerelease.attachTranslationFiles(
+ libraryfilealias, True, importer=importer)
=== added file 'lib/lp/soyuz/tests/test_translationsuploadjob.py'
--- lib/lp/soyuz/tests/test_translationsuploadjob.py 1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/tests/test_translationsuploadjob.py 2013-07-23 15:53:28 +0000
@@ -0,0 +1,143 @@
+# Copyright 2013 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import os.path
+
+from testtools.content import text_content
+import transaction
+from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
+
+from lp.soyuz.enums import (
+ PackageDiffStatus,
+ )
+from lp.soyuz.interfaces.translationsuploadjob import (
+ ITranslationsUploadJob,
+ ITranslationsUploadJobSource,
+ )
+from lp.soyuz.model.translationsuploadjob import TranslationsUploadJob
+from lp.services.config import config
+from lp.services.features.testing import FeatureFixture
+from lp.services.job.interfaces.job import JobStatus
+from lp.testing import (
+ admin_logged_in,
+ run_script,
+ TestCaseWithFactory,
+ verifyObject,
+ )
+from lp.services.job.tests import block_on_job
+from lp.testing.fakemethod import FakeMethod
+from lp.testing.layers import (
+ CeleryJobLayer,
+ LaunchpadZopelessLayer,
+ )
+from lp.services.tarfile_helpers import LaunchpadWriteTarFile
+from lp.translations.interfaces.translationimportqueue import (
+ ITranslationImportQueue,
+ )
+
+
+class LocalTestHelper(TestCaseWithFactory):
+
+ def makeJob(self, spr_creator=None, archive=None,
+ sourcepackagerelease=None, libraryfilealias=None):
+ if spr_creator is None:
+ creator = self.factory.makePerson()
+ else:
+ creator = self.factory.makePerson(name=spr_creator)
+ if archive is None:
+ archive = self.factory.makeArchive()
+ if sourcepackagerelease is None:
+ sourcepackagerelease = self.factory.makeSourcePackageRelease(
+ archive=archive, creator=creator)
+ if libraryfilealias is None:
+ libraryfilealias = self.makeTranslationsLFA()
+ return (sourcepackagerelease,
+ getUtility(ITranslationsUploadJobSource).create(
+ sourcepackagerelease, libraryfilealias))
+
+ def makeTranslationsLFA(self):
+ """Create an LibraryFileAlias containing dummy translation data."""
+ test_tar_content = {
+ 'source/po/foo.pot': 'Foo template',
+ 'source/po/eo.po': 'Foo translation',
+ }
+ tarfile_content = LaunchpadWriteTarFile.files_to_string(
+ test_tar_content)
+ return self.factory.makeLibraryFileAlias(content=tarfile_content)
+
+
+class TestTranslationsUploadJob(LocalTestHelper):
+
+ layer = LaunchpadZopelessLayer
+
+ def test_job_implements_ITranslationsUploadJob(self):
+ _, job = self.makeJob()
+ self.assertTrue(verifyObject(ITranslationsUploadJob, job))
+
+ def test_job_source_implements_ITranslationsUploadJobSource(self):
+ job_source = getUtility(ITranslationsUploadJobSource)
+ self.assertTrue(verifyObject(ITranslationsUploadJobSource, job_source))
+
+ def test_iterReady(self):
+ _, job1 = self.makeJob()
+ removeSecurityProxy(job1).job._status = JobStatus.COMPLETED
+ _, job2 = self.makeJob()
+ jobs = list(TranslationsUploadJob.iterReady())
+ self.assertEqual(1, len(jobs))
+
+ def test_importer_is_creator(self):
+ spr, job = self.makeJob(spr_creator="foobar")
+ transaction.commit()
+ job.run()
+ translation_import_queue = getUtility(ITranslationImportQueue)
+ entries_in_queue = translation_import_queue.getAllEntries(
+ target=spr.sourcepackage)
+ self.assertEqual(entries_in_queue[0].importer.name, "foobar")
+
+ def test_run(self):
+ archive = self.factory.makeArchive()
+ foo_pkg = self.factory.makeSourcePackageRelease(archive=archive)
+ method = FakeMethod()
+ removeSecurityProxy(foo_pkg).attachTranslationFiles = method
+ spr, job = self.makeJob(archive=archive, sourcepackagerelease=foo_pkg)
+ transaction.commit()
+ job.run()
+ self.assertEqual(method.call_count, 1)
+
+ def test_smoke(self):
+ spr, job = self.makeJob()
+ transaction.commit()
+ out, err, exit_code = run_script(
+ "LP_DEBUG_SQL=1 cronscripts/process-job-source.py -vv %s" % (
+ ITranslationsUploadJobSource.getName()))
+
+ self.addDetail("stdout", text_content(out))
+ self.addDetail("stderr", text_content(err))
+
+ self.assertEqual(0, exit_code)
+ translation_import_queue = getUtility(ITranslationImportQueue)
+ entries_in_queue = translation_import_queue.getAllEntries(
+ target=spr.sourcepackage).count()
+ self.assertEqual(2, entries_in_queue)
+
+
+class TestViaCelery(LocalTestHelper):
+ """TranslationsUploadJob runs under Celery."""
+
+ layer = CeleryJobLayer
+
+ def test_run(self):
+ self.useFixture(FeatureFixture({
+ 'jobs.celery.enabled_classes': 'TranslationsUploadJob',
+ }))
+
+ spr, job = self.makeJob()
+ with block_on_job(self):
+ transaction.commit()
+ translation_import_queue = getUtility(ITranslationImportQueue)
+ entries_in_queue = translation_import_queue.getAllEntries(
+ target=spr.sourcepackage).count()
+ self.assertEqual(2, entries_in_queue)
