launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #04273
[Merge] lp:~julian-edwards/launchpad/async-copying-bug-809805 into lp:launchpad
Julian Edwards has proposed merging lp:~julian-edwards/launchpad/async-copying-bug-809805 into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~julian-edwards/launchpad/async-copying-bug-809805/+merge/67941
= Summary =
Expose a simple webservice API to copy packages asynchronously. This is
to enable derived distros API syncing of packages but it has the bonus that
you can use it to copy packages anywhere, like from PPAs to Ubuntu.
== Proposed fix ==
Add IArchive.copyPackage() and export it. It's basically a copy of
syncSource() but works asynchronously by creating a job.
== Pre-implementation notes ==
Briefly chatted with flacoste and wgrant about whether to change behaviour
of existing syncSource or not on 'devel' only. We plumped for a new method.
== Implementation details ==
* Add IArchive.copyPackage() and export it, it has the same signature as
syncSource()
* refactor some checks that syncSource does so copyPackage can use them
* copyPackage has to check upload permissions since we can't provide all the
data that the checks need via the security adapter only. The method is
exposed with launchpad.View so anyone able to see the archive can request
an upload.
* copyPackage simply creates a PackageCopyJob and defers everything else to
it.
* lots of tests!
Future work will involve exposing copyPackages (note plural) which will work
like syncSources() where you can copy lots of packages atomically and more
efficiently.
== Tests ==
bin/test -cvv test_archive TestSyncSource
bin/test -cvv test_archive_webservice TestCopyPackage
== Demo and Q/A ==
Easily QA-able on staging or dogfood.
--
https://code.launchpad.net/~julian-edwards/launchpad/async-copying-bug-809805/+merge/67941
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~julian-edwards/launchpad/async-copying-bug-809805 into lp:launchpad.
=== modified file 'lib/canonical/launchpad/interfaces/_schema_circular_imports.py'
--- lib/canonical/launchpad/interfaces/_schema_circular_imports.py 2011-07-07 04:48:45 +0000
+++ lib/canonical/launchpad/interfaces/_schema_circular_imports.py 2011-07-14 10:51:38 +0000
@@ -404,6 +404,7 @@
patch_entry_return_type(IArchive, 'newQueueAdmin', IArchivePermission)
patch_plain_parameter_type(IArchive, 'syncSources', 'from_archive', IArchive)
patch_plain_parameter_type(IArchive, 'syncSource', 'from_archive', IArchive)
+patch_plain_parameter_type(IArchive, 'copyPackage', 'from_archive', IArchive)
patch_entry_return_type(IArchive, 'newSubscription', IArchiveSubscriber)
patch_plain_parameter_type(
IArchive, 'getArchiveDependency', 'dependency', IArchive)
=== modified file 'lib/lp/soyuz/browser/tests/test_archive_webservice.py'
--- lib/lp/soyuz/browser/tests/test_archive_webservice.py 2011-06-23 16:11:42 +0000
+++ lib/lp/soyuz/browser/tests/test_archive_webservice.py 2011-07-14 10:51:38 +0000
@@ -20,6 +20,7 @@
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.soyuz.enums import ArchivePurpose
+from lp.soyuz.interfaces.packagecopyjob import IPlainPackageCopyJobSource
from lp.soyuz.interfaces.processor import IProcessorFamilySet
from lp.testing import (
celebrity_logged_in,
@@ -309,5 +310,40 @@
[ws_arm], self.service.processor_families)
+class TestCopyPackage(WebServiceTestCase):
+ """Webservice test cases for the copyPackage method"""
+
+ def test_copyPackage(self):
+ """Basic smoke test"""
+ self.ws_version = "devel"
+ uploader_dude = self.factory.makePerson()
+ source_archive = self.factory.makeArchive()
+ target_archive = self.factory.makeArchive(
+ purpose=ArchivePurpose.PRIMARY)
+ source = self.factory.makeSourcePackagePublishingHistory(
+ archive=source_archive)
+ source_name = source.source_package_name
+ version = source.source_package_version
+ to_pocket = PackagePublishingPocket.RELEASE
+ to_series = self.factory.makeDistroSeries(
+ distribution=target_archive.distribution)
+ with person_logged_in(target_archive.owner):
+ target_archive.newComponentUploader(uploader_dude, "universe")
+ transaction.commit()
+
+ ws_target_archive = self.wsObject(target_archive, user=uploader_dude)
+ ws_source_archive = self.wsObject(source_archive)
+
+ ws_target_archive.copyPackage(
+ source_name=source_name, version=version,
+ from_archive=ws_source_archive, to_pocket=to_pocket.name,
+ to_series=to_series.name, include_binaries=False)
+ transaction.commit()
+
+ job_source = getUtility(IPlainPackageCopyJobSource)
+ copy_job = job_source.getActiveJobs(target_archive).one()
+ self.assertEqual(target_archive, copy_job.target_archive)
+
+
def test_suite():
return unittest.TestLoader().loadTestsFromName(__name__)
=== modified file 'lib/lp/soyuz/interfaces/archive.py'
--- lib/lp/soyuz/interfaces/archive.py 2011-06-24 21:21:36 +0000
+++ lib/lp/soyuz/interfaces/archive.py 2011-07-14 10:51:38 +0000
@@ -1218,6 +1218,46 @@
:return: A new IArchiveAuthToken
"""
+ @call_with(person=REQUEST_USER)
+ @operation_parameters(
+ source_name=TextLine(title=_("Source package name")),
+ version=TextLine(title=_("Version")),
+ from_archive=Reference(schema=Interface),
+ # Really IArchive, see below
+ to_pocket=TextLine(title=_("Pocket name")),
+ to_series=TextLine(title=_("Distroseries name"), required=False),
+ include_binaries=Bool(
+ title=_("Include Binaries"),
+ description=_("Whether or not to copy binaries already built for"
+ " this source"),
+ required=False))
+ @export_write_operation()
+ @operation_for_version('devel')
+ def copyPackage(source_name, version, from_archive, to_pocket,
+ person, to_series=None, include_binaries=False):
+ """Copy a single named source into this archive.
+
+ Asynchronously copy a specific version of a named source to the
+ destination archive if necessary. Calls to this method will return
+ immediately if the copy passes basic security checks and the copy
+ will happen sometime later with full checking.
+
+ :param source_name: a string name of the package to copy.
+ :param version: the version of the package to copy.
+ :param from_archive: the source archive from which to copy.
+ :param to_pocket: the target pocket (as a string).
+ :param to_series: the target distroseries (as a string).
+ :param include_binaries: optional boolean, controls whether or not
+ the published binaries for each given source should also be
+ copied along with the source.
+ :param person: the `IPerson` who requests the sync.
+
+ :raises NoSuchSourcePackageName: if the source name is invalid
+ :raises PocketNotFound: if the pocket name is invalid
+ :raises NoSuchDistroSeries: if the distro series name is invalid
+ :raises CannotCopy: if there is a problem copying.
+ """
+
class IArchiveAppend(Interface):
"""Archive interface for operations restricted by append privilege."""
=== modified file 'lib/lp/soyuz/model/archive.py'
--- lib/lp/soyuz/model/archive.py 2011-06-24 21:21:36 +0000
+++ lib/lp/soyuz/model/archive.py 2011-07-14 10:51:38 +0000
@@ -115,6 +115,7 @@
ArchivePurpose,
ArchiveStatus,
ArchiveSubscriberStatus,
+ PackageCopyPolicy,
PackagePublishingStatus,
PackageUploadStatus,
)
@@ -165,6 +166,7 @@
IComponent,
IComponentSet,
)
+from lp.soyuz.interfaces.packagecopyjob import IPlainPackageCopyJobSource
from lp.soyuz.interfaces.packagecopyrequest import IPackageCopyRequestSet
from lp.soyuz.interfaces.processor import IProcessorFamilySet
from lp.soyuz.interfaces.publishing import (
@@ -200,6 +202,7 @@
)
from lp.soyuz.model.section import Section
from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
+from lp.soyuz.scripts.packagecopier import check_copy_permissions
def storm_validate_external_dependencies(archive, attr, value):
@@ -1507,20 +1510,46 @@
sources, to_pocket, to_series, include_binaries,
person=person)
- def syncSource(self, source_name, version, from_archive, to_pocket,
- to_series=None, include_binaries=False, person=None):
- """See `IArchive`."""
+ def _validateAndFindSource(self, from_archive, source_name, version):
# Check to see if the source package exists, and raise a useful error
# if it doesn't.
getUtility(ISourcePackageNameSet)[source_name]
# Find and validate the source package version required.
source = from_archive.getPublishedSources(
name=source_name, version=version, exact_match=True).first()
+ return source
+
+ def syncSource(self, source_name, version, from_archive, to_pocket,
+ to_series=None, include_binaries=False, person=None):
+ """See `IArchive`."""
+ source = self._validateAndFindSource(
+ from_archive, source_name, version)
self._copySources(
[source], to_pocket, to_series, include_binaries,
person=person)
+ def copyPackage(self, source_name, version, from_archive, to_pocket,
+ person, to_series=None, include_binaries=False):
+ """See `IArchive`."""
+ # Asynchronously copy a package using the job system.
+ pocket = self._text_to_pocket(to_pocket)
+ series = self._text_to_series(to_series)
+ # Upload permission checks, this will raise CannotCopy as
+ # necessary.
+ sourcepackagename = getUtility(ISourcePackageNameSet)[source_name]
+ check_copy_permissions(
+ person, self, series, pocket, [sourcepackagename])
+
+ self._validateAndFindSource(from_archive, source_name, version)
+ job_source = getUtility(IPlainPackageCopyJobSource)
+ job_source.create(
+ package_name=source_name, source_archive=from_archive,
+ target_archive=self, target_distroseries=series,
+ target_pocket=pocket,
+ package_version=version, include_binaries=include_binaries,
+ copy_policy=PackageCopyPolicy.INSECURE)
+
def _collectLatestPublishedSources(self, from_archive, source_names):
"""Private helper to collect the latest published sources for an
archive.
@@ -1544,6 +1573,27 @@
sources.append(first_source)
return sources
+ def _text_to_series(self, to_series):
+ if to_series is not None:
+ result = getUtility(IDistroSeriesSet).queryByName(
+ self.distribution, to_series)
+ if result is None:
+ raise NoSuchDistroSeries(to_series)
+ series = result
+ else:
+ series = None
+
+ return series
+
+ def _text_to_pocket(self, to_pocket):
+ # Convert the to_pocket string to its enum.
+ try:
+ pocket = PackagePublishingPocket.items[to_pocket.upper()]
+ except KeyError:
+ raise PocketNotFound(to_pocket.upper())
+
+ return pocket
+
def _copySources(self, sources, to_pocket, to_series=None,
include_binaries=False, person=None):
"""Private helper function to copy sources to this archive.
@@ -1553,12 +1603,8 @@
"""
# Circular imports.
from lp.soyuz.scripts.packagecopier import do_copy
- # Convert the to_pocket string to its enum.
- try:
- pocket = PackagePublishingPocket.items[to_pocket.upper()]
- except KeyError:
- raise PocketNotFound(to_pocket.upper())
+ pocket = self._text_to_pocket(to_pocket)
# Fail immediately if the destination pocket is not Release and
# this archive is a PPA.
if self.is_ppa and pocket != PackagePublishingPocket.RELEASE:
@@ -1566,14 +1612,7 @@
"Destination pocket must be 'release' for a PPA.")
# Now convert the to_series string to a real distroseries.
- if to_series is not None:
- result = getUtility(IDistroSeriesSet).queryByName(
- self.distribution, to_series)
- if result is None:
- raise NoSuchDistroSeries(to_series)
- series = result
- else:
- series = None
+ series = self._text_to_series(to_series)
# Perform the copy, may raise CannotCopy. Don't do any further
# permission checking: this method is protected by
=== modified file 'lib/lp/soyuz/tests/test_archive.py'
--- lib/lp/soyuz/tests/test_archive.py 2011-06-24 09:53:18 +0000
+++ lib/lp/soyuz/tests/test_archive.py 2011-07-14 10:51:38 +0000
@@ -49,11 +49,13 @@
ArchivePermissionType,
ArchivePurpose,
ArchiveStatus,
+ PackageCopyPolicy,
PackagePublishingStatus,
)
from lp.soyuz.interfaces.archive import (
ArchiveDependencyError,
ArchiveDisabled,
+ CannotCopy,
CannotRestrictArchitectures,
CannotUploadToPocket,
CannotUploadToPPA,
@@ -69,6 +71,7 @@
from lp.soyuz.interfaces.archivepermission import IArchivePermissionSet
from lp.soyuz.interfaces.binarypackagename import IBinaryPackageNameSet
from lp.soyuz.interfaces.component import IComponentSet
+from lp.soyuz.interfaces.packagecopyjob import IPlainPackageCopyJobSource
from lp.soyuz.interfaces.processor import IProcessorFamilySet
from lp.soyuz.model.archive import Archive
from lp.soyuz.model.archivepermission import ArchivePermission
@@ -1965,3 +1968,103 @@
1,
ubuntu.main_archive.getPublishedSources(
name=source.source_package_name).count())
+
+ def _setup_copy_data(self, target_purpose=None):
+ if target_purpose is None:
+ target_purpose = ArchivePurpose.PPA
+ source_archive = self.factory.makeArchive()
+ target_archive = self.factory.makeArchive(purpose=target_purpose)
+ source = self.factory.makeSourcePackagePublishingHistory(
+ archive=source_archive)
+ source_name = source.source_package_name
+ version = source.source_package_version
+ to_pocket = PackagePublishingPocket.RELEASE
+ to_series = self.factory.makeDistroSeries(
+ distribution=target_archive.distribution)
+ return (source, source_archive, source_name, target_archive,
+ to_pocket, to_series, version)
+
+ def test_copyPackage_creates_packagecopyjob(self):
+ # The copyPackage method should create a PCJ with the appropriate
+ # parameters.
+ (source, source_archive, source_name, target_archive, to_pocket,
+ to_series, version) = self._setup_copy_data()
+ with person_logged_in(target_archive.owner):
+ target_archive.copyPackage(
+ source_name, version, source_archive, to_pocket.name,
+ to_series=to_series.name, include_binaries=False,
+ person=target_archive.owner)
+
+ # The source should not be published yet in the target_archive.
+ published = target_archive.getPublishedSources(
+ name=source.source_package_name).any()
+ self.assertIs(None, published)
+
+ # There should be one copy job.
+ job_source = getUtility(IPlainPackageCopyJobSource)
+ copy_job = job_source.getActiveJobs(target_archive).one()
+
+ # Its data should reflect the requested copy.
+ self.assertEqual(source_name, copy_job.package_name)
+ self.assertEqual(version, copy_job.package_version)
+ self.assertEqual(target_archive, copy_job.target_archive)
+ self.assertEqual(source_archive, copy_job.source_archive)
+ self.assertEqual(to_series, copy_job.target_distroseries)
+ self.assertEqual(to_pocket, copy_job.target_pocket)
+ self.assertFalse(copy_job.include_binaries)
+ self.assertEquals(PackageCopyPolicy.INSECURE, copy_job.copy_policy)
+
+ def test_copyPackage_disallows_non_primary_archive_uploaders(self):
+ # If copying to a primary archive and you're not an uploader for
+ # the package then you can't copy.
+ (source, source_archive, source_name, target_archive, to_pocket,
+ to_series, version) = self._setup_copy_data(
+ target_purpose=ArchivePurpose.PRIMARY)
+ person = self.factory.makePerson()
+ self.assertRaises(
+ CannotCopy,
+ target_archive.copyPackage, source_name, version, source_archive,
+ to_pocket.name, to_series=to_series.name, include_binaries=False,
+ person=person)
+
+ def test_copyPackage_allows_primary_archive_uploaders(self):
+ # Copying to a primary archive if you're already an uploader is OK.
+ (source, source_archive, source_name, target_archive, to_pocket,
+ to_series, version) = self._setup_copy_data(
+ target_purpose=ArchivePurpose.PRIMARY)
+ person = self.factory.makePerson()
+ with person_logged_in(target_archive.owner):
+ target_archive.newComponentUploader(person, "universe")
+ target_archive.copyPackage(
+ source_name, version, source_archive, to_pocket.name,
+ to_series=to_series.name, include_binaries=False,
+ person=person)
+
+ # There should be one copy job.
+ job_source = getUtility(IPlainPackageCopyJobSource)
+ copy_job = job_source.getActiveJobs(target_archive).one()
+ self.assertEqual(target_archive, copy_job.target_archive)
+
+ def test_copyPackage_disallows_non_PPA_owners(self):
+ # Only people with launchpad.Append are allowed to call copyPackage.
+ (source, source_archive, source_name, target_archive, to_pocket,
+ to_series, version) = self._setup_copy_data()
+ person = self.factory.makePerson()
+ self.assertTrue(target_archive.is_ppa)
+ self.assertRaises(
+ CannotCopy,
+ target_archive.copyPackage, source_name, version, source_archive,
+ to_pocket.name, to_series=to_series.name, include_binaries=False,
+ person=person)
+
+ def test_copyPackage_disallows_non_release_target_pocket_for_PPA(self):
+ (source, source_archive, source_name, target_archive, to_pocket,
+ to_series, version) = self._setup_copy_data()
+ to_pocket = PackagePublishingPocket.UPDATES
+ self.assertTrue(target_archive.is_ppa)
+ self.assertRaises(
+ CannotCopy,
+ target_archive.copyPackage, source_name, version, source_archive,
+ to_pocket.name, to_series=to_series.name, include_binaries=False,
+ person=target_archive.owner)
+
Follow ups
