launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #21167
[Merge] lp:~cjwatson/launchpad/archive-api-dirty into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/archive-api-dirty into lp:launchpad.
Commit message:
Add Archive.markSuiteDirty to allow requesting that a given archive/suite be published.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/archive-api-dirty/+merge/310208
My proximate reason for doing this is that it will make it easier to cause index files for new series to exist in special-purpose PPAs such as ppa:snappy-dev/ubuntu/tools. However, we occasionally need to force a suite to be published for one reason or another and it would be nice for this not to require a manual publish-distro run.
As an additional demonstration of this being worthwhile, I used it to simplify the handling of staged files in publish-ftpmaster.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/archive-api-dirty into lp:launchpad.
=== modified file 'lib/lp/_schema_circular_imports.py'
--- lib/lp/_schema_circular_imports.py 2016-10-12 23:24:24 +0000
+++ lib/lp/_schema_circular_imports.py 2016-11-07 17:02:58 +0000
@@ -484,6 +484,10 @@
IArchive, '_addArchiveDependency', 'pocket', PackagePublishingPocket)
patch_entry_return_type(
IArchive, '_addArchiveDependency', IArchiveDependency)
+patch_plain_parameter_type(
+ IArchive, 'markSuiteDirty', 'distroseries', IDistroSeries)
+patch_choice_parameter_type(
+ IArchive, 'markSuiteDirty', 'pocket', PackagePublishingPocket)
# IBuildFarmJob
patch_choice_property(IBuildFarmJob, 'status', BuildStatus)
=== modified file 'lib/lp/archivepublisher/scripts/publish_ftpmaster.py'
--- lib/lp/archivepublisher/scripts/publish_ftpmaster.py 2016-01-13 17:54:05 +0000
+++ lib/lp/archivepublisher/scripts/publish_ftpmaster.py 2016-11-07 17:02:58 +0000
@@ -434,14 +434,12 @@
publish_distro.main()
def publishDistroArchive(self, distribution, archive,
- security_suites=None, updated_suites=[]):
+ security_suites=None):
"""Publish the results for an archive.
:param archive: Archive to publish.
:param security_suites: An optional list of suites to restrict
the publishing to.
- :param updated_suites: An optional list of archive/suite pairs that
- have been updated out of band and should be republished.
"""
purpose = archive.purpose
archive_config = self.configs[distribution][purpose]
@@ -456,9 +454,6 @@
arguments = ['-R', temporary_dists]
if archive.purpose == ArchivePurpose.PARTNER:
arguments.append('--partner')
- for updated_archive, updated_suite in updated_suites:
- if archive == updated_archive:
- arguments.extend(['--dirty-suite', updated_suite])
os.rename(get_backup_dists(archive_config), temporary_dists)
try:
@@ -554,15 +549,14 @@
security_suites=security_suites)
return True
- def publishDistroUploads(self, distribution, updated_suites=[]):
+ def publishDistroUploads(self, distribution):
"""Publish the distro's complete uploads."""
self.logger.debug("Full publication. This may take some time.")
for archive in get_publishable_archives(distribution):
if archive.purpose in self.configs[distribution]:
# This, for the main archive, is where the script spends
# most of its time.
- self.publishDistroArchive(
- distribution, archive, updated_suites=updated_suites)
+ self.publishDistroArchive(distribution, archive)
def updateStagedFilesForSuite(self, archive_config, suite):
"""Install all staged files for a single archive and suite.
@@ -602,7 +596,6 @@
def updateStagedFiles(self, distribution):
"""Install all staged files for a distribution's archives."""
- updated_suites = []
for archive in get_publishable_archives(distribution):
if archive.purpose not in self.configs[distribution]:
continue
@@ -613,8 +606,7 @@
if cannot_modify_suite(archive, series, pocket):
continue
if self.updateStagedFilesForSuite(archive_config, suite):
- updated_suites.append((archive, suite))
- return updated_suites
+ archive.markSuiteDirty(series, pocket)
def publish(self, distribution, security_only=False):
"""Do the main publishing work.
@@ -631,9 +623,8 @@
if security_only:
has_published = self.publishSecurityUploads(distribution)
else:
- updated_suites = self.updateStagedFiles(distribution)
- self.publishDistroUploads(
- distribution, updated_suites=updated_suites)
+ self.updateStagedFiles(distribution)
+ self.publishDistroUploads(distribution)
# Let's assume the main archive is always modified
has_published = True
=== modified file 'lib/lp/archivepublisher/scripts/publishdistro.py'
--- lib/lp/archivepublisher/scripts/publishdistro.py 2016-09-19 10:28:33 +0000
+++ lib/lp/archivepublisher/scripts/publishdistro.py 2016-11-07 17:02:58 +0000
@@ -235,6 +235,19 @@
suites.add((series.name, pocket))
return suites
+ def findDirtySuites(self, distribution, archive):
+ """Find the suites that have been explicitly marked as dirty."""
+ for suite in self.options.dirty_suites:
+ yield self.findSuite(distribution, suite)
+ if archive.dirty_suites is not None:
+ for suite in archive.dirty_suites:
+ try:
+ yield distribution.getDistroSeriesAndPocket(suite)
+ except NotFoundError:
+ self.logger.exception(
+ "Failed to parse dirty suite '%s' for archive '%s'" %
+ (suite, archive.reference))
+
def getCopyArchives(self, distribution):
"""Find copy archives for the selected distribution."""
copy_archives = list(
@@ -347,14 +360,14 @@
elif archive.can_be_published:
publisher = self.getPublisher(
distribution, archive, allowed_suites)
- for suite in self.options.dirty_suites:
- distroseries, pocket = self.findSuite(
- distribution, suite)
+ for distroseries, pocket in self.findDirtySuites(
+ distribution, archive):
if not cannot_modify_suite(
archive, distroseries, pocket):
publisher.markPocketDirty(
distroseries, pocket)
self.publishArchive(archive, publisher)
+ archive.dirty_suites = None
work_done = True
else:
work_done = False
=== modified file 'lib/lp/archivepublisher/tests/test_publish_ftpmaster.py'
--- lib/lp/archivepublisher/tests/test_publish_ftpmaster.py 2016-02-05 16:51:12 +0000
+++ lib/lp/archivepublisher/tests/test_publish_ftpmaster.py 2016-11-07 17:02:58 +0000
@@ -925,9 +925,9 @@
self.assertFalse(script.updateStagedFilesForSuite(
archive_config, distroseries.name))
- def test_updateStagedFiles_updated_suites(self):
- # updateStagedFiles returns a list of suites for which it updated
- # staged files.
+ def test_updateStagedFiles_marks_suites_dirty(self):
+ # updateStagedFiles marks the suites for which it updated staged
+ # files as dirty.
distro = self.makeDistroWithPublishDirectory()
distroseries = self.factory.makeDistroSeries(distribution=distro)
das = self.factory.makeDistroArchSeries(distroseries=distroseries)
@@ -945,9 +945,8 @@
os.makedirs(staging_suite)
write_marker_file(
[staging_suite, "%s.gz" % contents_filename], "Contents")
- self.assertEqual(
- [(distro.main_archive, distroseries.name)],
- script.updateStagedFiles(distro))
+ script.updateStagedFiles(distro)
+ self.assertEqual([distroseries.name], distro.main_archive.dirty_suites)
def test_updateStagedFiles_considers_partner_archive(self):
# updateStagedFiles considers the partner archive as well as the
=== modified file 'lib/lp/archivepublisher/tests/test_publishdistro.py'
--- lib/lp/archivepublisher/tests/test_publishdistro.py 2016-11-03 15:07:36 +0000
+++ lib/lp/archivepublisher/tests/test_publishdistro.py 2016-11-07 17:02:58 +0000
@@ -20,7 +20,9 @@
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.archivepublisher.config import getPubConfig
-from lp.archivepublisher.interfaces.archivesigningkey import IArchiveSigningKey
+from lp.archivepublisher.interfaces.archivesigningkey import (
+ IArchiveSigningKey,
+ )
from lp.archivepublisher.interfaces.publisherconfig import IPublisherConfigSet
from lp.archivepublisher.publishing import Publisher
from lp.archivepublisher.scripts.publishdistro import PublishDistro
@@ -436,6 +438,24 @@
# breezy-autotest has its Release files rewritten.
self.assertThat(breezy_inrelease_path, PathExists())
+ def testDirtySuites(self):
+ """publish-distro can be told to publish specific suites."""
+ archive = self.factory.makeArchive(distribution=self.ubuntutest)
+ self.layer.txn.commit()
+
+ # publish-distro has nothing to publish.
+ self.runPublishDistro(['--ppa'])
+ breezy_release_path = os.path.join(
+ getPubConfig(archive).distsroot, 'breezy-autotest', 'Release')
+ self.assertThat(breezy_release_path, Not(PathExists()))
+
+ # ... but it will publish a suite anyway if it is marked as dirty.
+ archive.markSuiteDirty(
+ archive.distribution.getSeries('breezy-autotest'),
+ PackagePublishingPocket.RELEASE)
+ self.runPublishDistro(['--ppa'])
+ self.assertThat(breezy_release_path, PathExists())
+
class FakeArchive:
"""A very simple fake `Archive`."""
@@ -444,6 +464,7 @@
self.can_be_published = True
self.purpose = purpose
self.status = ArchiveStatus.ACTIVE
+ self.dirty_suites = []
class FakePublisher:
=== modified file 'lib/lp/archivepublisher/tests/test_publisher.py'
--- lib/lp/archivepublisher/tests/test_publisher.py 2016-10-03 13:10:11 +0000
+++ lib/lp/archivepublisher/tests/test_publisher.py 2016-11-07 17:02:58 +0000
@@ -1318,6 +1318,16 @@
now - timedelta(hours=12))
self.assertIn(archive, ubuntu.getPendingPublicationPPAs())
+ def testDirtySuitesArchive(self):
+ # getPendingPublicationPPAs returns archives that have dirty_suites
+ # set.
+ ubuntu = getUtility(IDistributionSet)['ubuntu']
+ archive = self.factory.makeArchive()
+ self.assertNotIn(archive, ubuntu.getPendingPublicationPPAs())
+ archive.markSuiteDirty(
+ ubuntu.currentseries, PackagePublishingPocket.RELEASE)
+ self.assertIn(archive, ubuntu.getPendingPublicationPPAs())
+
def _checkCompressedFiles(self, archive_publisher, base_file_path,
suffixes):
"""Assert that the various compressed versions of a file are equal.
=== modified file 'lib/lp/registry/model/distribution.py'
--- lib/lp/registry/model/distribution.py 2016-10-03 13:10:11 +0000
+++ lib/lp/registry/model/distribution.py 2016-11-07 17:02:58 +0000
@@ -1296,13 +1296,23 @@
reapable_af_query, clauseTables=['ArchiveFile'],
orderBy=['archive.id'], distinct=True)
+ dirty_suites_query = """
+ Archive.purpose = %s AND
+ Archive.distribution = %s AND
+ Archive.dirty_suites IS NOT NULL
+ """ % sqlvalues(ArchivePurpose.PPA, self)
+
+ dirty_suites_archives = Archive.select(
+ dirty_suites_query, orderBy=['archive.id'], distinct=True)
+
deleting_archives = Archive.selectBy(
distribution=self,
purpose=ArchivePurpose.PPA,
status=ArchiveStatus.DELETING).orderBy(['archive.id'])
return src_archives.union(bin_archives).union(
- reapable_af_archives).union(deleting_archives)
+ reapable_af_archives).union(dirty_suites_archives).union(
+ deleting_archives)
def getArchiveByComponent(self, component_name):
"""See `IDistribution`."""
=== modified file 'lib/lp/soyuz/configure.zcml'
--- lib/lp/soyuz/configure.zcml 2016-09-24 06:22:43 +0000
+++ lib/lp/soyuz/configure.zcml 2016-11-07 17:02:58 +0000
@@ -327,8 +327,8 @@
<require
permission="launchpad.Edit"
interface="lp.soyuz.interfaces.archive.IArchiveEdit"
- set_attributes="build_debug_symbols description displayname
- publish publish_debug_symbols status
+ set_attributes="build_debug_symbols description dirty_suites
+ displayname publish publish_debug_symbols status
suppress_subscription_notifications"/>
<!--
NOTE: The 'private' permission controls who can turn a public
=== modified file 'lib/lp/soyuz/interfaces/archive.py'
--- lib/lp/soyuz/interfaces/archive.py 2016-10-20 11:49:45 +0000
+++ lib/lp/soyuz/interfaces/archive.py 2016-11-07 17:02:58 +0000
@@ -671,6 +671,10 @@
"context build.\n"
"NOTE: This is for migration of OEM PPAs only!")))
+ dirty_suites = Attribute(
+ "Suites that the next publisher run should publish regardless of "
+ "pending publications.")
+
processors = exported(
CollectionField(
title=_("Processors"),
@@ -2203,6 +2207,29 @@
:param names: A list of token names.
"""
+ @operation_parameters(
+ distroseries=Reference(
+ # Really IDistroSeries.
+ Interface,
+ title=_("Distro series"), required=True),
+ pocket=Choice(
+ title=_("Pocket"),
+ # Really PackagePublishingPocket.
+ vocabulary=DBEnumeratedType,
+ required=True),
+ )
+ @export_write_operation()
+ @operation_for_version("devel")
+ def markSuiteDirty(distroseries, pocket):
+ """Mark a suite as dirty in this archive.
+
+ The next publisher run will publish this suite regardless of whether
+ it has any pending publications.
+
+ :param distroseries: An `IDistroSeries`.
+ :param pocket: A `PackagePublishingPocket`.
+ """
+
class IArchiveAdmin(Interface):
"""Archive interface for operations restricted by commercial."""
=== modified file 'lib/lp/soyuz/model/archive.py'
--- lib/lp/soyuz/model/archive.py 2016-10-20 11:49:45 +0000
+++ lib/lp/soyuz/model/archive.py 2016-11-07 17:02:58 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Database class for table Archive."""
@@ -36,6 +36,7 @@
)
from storm.properties import (
Int,
+ JSON,
Unicode,
)
from storm.references import Reference
@@ -362,6 +363,8 @@
dbName='suppress_subscription_notifications',
notNull=True, default=False)
+ dirty_suites = JSON(name='dirty_suites', allow_none=True)
+
def _init(self, *args, **kw):
"""Provide the right interface for URL traversal."""
SQLBase._init(self, *args, **kw)
@@ -2414,6 +2417,14 @@
Store.of(pcj.context).remove(pcj.context)
job.destroySelf()
+ def markSuiteDirty(self, distroseries, pocket):
+ """See `IArchive`."""
+ suite = distroseries.getSuite(pocket)
+ if self.dirty_suites is None:
+ self.dirty_suites = [suite]
+ elif suite not in self.dirty_suites:
+ self.dirty_suites.append(suite)
+
def validate_ppa(owner, distribution, proposed_name, private=False):
"""Can 'person' create a PPA called 'proposed_name'?
=== modified file 'lib/lp/soyuz/tests/test_archive.py'
--- lib/lp/soyuz/tests/test_archive.py 2016-07-14 16:06:01 +0000
+++ lib/lp/soyuz/tests/test_archive.py 2016-11-07 17:02:58 +0000
@@ -113,8 +113,8 @@
login,
login_person,
person_logged_in,
+ RequestTimelineCollector,
StormStatementRecorder,
- RequestTimelineCollector,
TestCaseWithFactory,
)
from lp.testing.layers import (
@@ -4136,3 +4136,51 @@
(existing_bpn, 'i386'): BinaryOverride(component=self.main),
(other_bpn, 'amd64'): BinaryOverride(component=self.main),
}))
+
+
+class TestMarkSuiteDirty(TestCaseWithFactory):
+
+ layer = DatabaseFunctionalLayer
+
+ def test_default_is_none(self):
+ archive = self.factory.makeArchive()
+ self.assertIsNone(archive.dirty_suites)
+
+ def test_requires_owner(self):
+ archive = self.factory.makeArchive()
+ self.assertRaises(Unauthorized, getattr, archive, "markSuiteDirty")
+
+ def test_first_suite(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ with person_logged_in(archive.owner):
+ archive.markSuiteDirty(
+ distroseries, PackagePublishingPocket.UPDATES)
+ self.assertEqual(
+ ["%s-updates" % distroseries.name], archive.dirty_suites)
+
+ def test_already_dirty(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ with person_logged_in(archive.owner):
+ archive.markSuiteDirty(
+ distroseries, PackagePublishingPocket.UPDATES)
+ archive.markSuiteDirty(
+ distroseries, PackagePublishingPocket.UPDATES)
+ self.assertEqual(
+ ["%s-updates" % distroseries.name], archive.dirty_suites)
+
+ def test_second_suite(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ with person_logged_in(archive.owner):
+ archive.markSuiteDirty(
+ distroseries, PackagePublishingPocket.UPDATES)
+ archive.markSuiteDirty(
+ distroseries, PackagePublishingPocket.RELEASE)
+ self.assertContentEqual(
+ ["%s-updates" % distroseries.name, distroseries.name],
+ archive.dirty_suites)
Follow ups
