launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #26845
[Merge] ~cjwatson/launchpad:avoid-pristine-ppas into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:avoid-pristine-ppas into launchpad:master.
Commit message:
Make process-death-row and update-pkgcache skip pristine PPAs
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/400735
These two scripts will never have anything to do on "pristine" PPAs (that is, those that have never had anything published to them; thanks to Jacob Nevins for the naming suggestion), so filter those out of the relevant query. They can otherwise spend a long time on unnecessary iteration.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:avoid-pristine-ppas into launchpad:master.
diff --git a/cronscripts/update-pkgcache.py b/cronscripts/update-pkgcache.py
index 0b01ed3..12bd926 100755
--- a/cronscripts/update-pkgcache.py
+++ b/cronscripts/update-pkgcache.py
@@ -1,6 +1,6 @@
#!/usr/bin/python2 -S
#
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
# This script updates the cached source package information in the system.
@@ -13,6 +13,8 @@ from zope.component import getUtility
from lp.registry.interfaces.distribution import IDistributionSet
from lp.services.scripts.base import LaunchpadCronScript
+from lp.soyuz.enums import ArchivePurpose
+from lp.soyuz.interfaces.archive import IArchiveSet
from lp.soyuz.model.distributionsourcepackagecache import (
DistributionSourcePackageCache,
)
@@ -97,7 +99,10 @@ class PackageCacheUpdater(LaunchpadCronScript):
self.logger.info(
'Updating %s PPAs' % distribution.name)
- for archive in distribution.getAllPPAs():
+ archives = getUtility(IArchiveSet).getArchivesForDistribution(
+ distribution, purposes=[ArchivePurpose.PPA],
+ exclude_pristine=True)
+ for archive in archives:
self.updateDistributionCache(distribution, archive)
archive.updateArchiveCache()
diff --git a/lib/lp/archivepublisher/scripts/processdeathrow.py b/lib/lp/archivepublisher/scripts/processdeathrow.py
index a7f230e..f95535d 100644
--- a/lib/lp/archivepublisher/scripts/processdeathrow.py
+++ b/lib/lp/archivepublisher/scripts/processdeathrow.py
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Death row processor base script class
@@ -14,6 +14,7 @@ __all__ = [
'DeathRowProcessor',
]
+from zope.component import getUtility
from lp.archivepublisher.deathrow import getDeathRow
from lp.archivepublisher.scripts.base import PublisherScript
@@ -22,6 +23,8 @@ from lp.services.webapp.adapter import (
clear_request_started,
set_request_started,
)
+from lp.soyuz.enums import ArchivePurpose
+from lp.soyuz.interfaces.archive import IArchiveSet
class DeathRowProcessor(PublisherScript):
@@ -41,14 +44,18 @@ class DeathRowProcessor(PublisherScript):
"--ppa", action="store_true", default=False,
help="Run only over PPA archives.")
+ def getTargetArchives(self, distribution):
+ """Find archives to target based on given options."""
+ if self.options.ppa:
+ return getUtility(IArchiveSet).getArchivesForDistribution(
+ distribution, purposes=[ArchivePurpose.PPA],
+ exclude_pristine=True)
+ else:
+ return distribution.all_distro_archives
+
def main(self):
for distribution in self.findDistros():
- if self.options.ppa:
- archives = distribution.getAllPPAs()
- else:
- archives = distribution.all_distro_archives
-
- for archive in archives:
+ for archive in self.getTargetArchives(distribution):
self.logger.info("Processing %s" % archive.archive_url)
self.processDeathRow(archive)
diff --git a/lib/lp/archivepublisher/tests/test_processdeathrow.py b/lib/lp/archivepublisher/tests/test_processdeathrow.py
index e333727..21160fe 100644
--- a/lib/lp/archivepublisher/tests/test_processdeathrow.py
+++ b/lib/lp/archivepublisher/tests/test_processdeathrow.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Functional tests for process-death-row.py script.
@@ -23,6 +23,7 @@ import pytz
from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy
+from lp.archivepublisher.scripts.processdeathrow import DeathRowProcessor
from lp.registry.interfaces.distribution import IDistributionSet
from lp.registry.interfaces.person import IPersonSet
from lp.services.config import config
@@ -175,6 +176,25 @@ class TestProcessDeathRow(TestCaseWithFactory):
spph.dateremoved is None,
"ID %s -> removed" % (spph.id))
+ def test_getTargetArchives_ppa(self):
+ """With --ppa, getTargetArchives returns all non-empty PPAs."""
+ ubuntutest = getUtility(IDistributionSet)["ubuntutest"]
+ cprov_archive = getUtility(IPersonSet).getByName("cprov").archive
+ mark_archive = getUtility(IPersonSet).getByName("mark").archive
+ self.factory.makeArchive(distribution=ubuntutest)
+ script = DeathRowProcessor(test_args=["-d", "ubuntutest", "--ppa"])
+ self.assertContentEqual(
+ [cprov_archive, mark_archive],
+ script.getTargetArchives(ubuntutest))
+
+ def test_getTargetArchives_main(self):
+ """Without --ppa, getTargetArchives returns main archives."""
+ ubuntutest = getUtility(IDistributionSet)["ubuntutest"]
+ script = DeathRowProcessor(test_args=["-d", "ubuntutest"])
+ self.assertContentEqual(
+ ubuntutest.all_distro_archives,
+ script.getTargetArchives(ubuntutest))
+
def testDryRun(self):
"""Test we don't delete the file or change the db in dry run mode."""
self.runDeathRow(["-d", "ubuntutest", "-n"])
diff --git a/lib/lp/soyuz/doc/archive.txt b/lib/lp/soyuz/doc/archive.txt
index 4e7d2bd..db8b20f 100644
--- a/lib/lp/soyuz/doc/archive.txt
+++ b/lib/lp/soyuz/doc/archive.txt
@@ -1443,11 +1443,11 @@ First we create four copy archives for ubuntu:
>>> copy_owner1 = factory.makePerson(name="copy-owner1")
>>> copy_owner2 = factory.makePerson(name="copy-owner2")
- >>> copy = factory.makeCopyArchiveLocation(
+ >>> ultimate_copy = factory.makeCopyArchiveLocation(
... distribution=ubuntu, name="ultimate-copy", owner=copy_owner1)
- >>> copy = factory.makeCopyArchiveLocation(
+ >>> fine_copy = factory.makeCopyArchiveLocation(
... distribution=ubuntu, name="fine-copy", owner=copy_owner2)
- >>> copy = factory.makeCopyArchiveLocation(
+ >>> true_copy = factory.makeCopyArchiveLocation(
... distribution=ubuntu, name="true-copy", owner=copy_owner2,
... enabled=False)
@@ -1577,6 +1577,27 @@ of juergen that is disabled.
true-copy copy-owner2 False False
ultimate-copy copy-owner1 False True
+A separate argument allows excluding archives that have never had any
+publications, allowing jobs to skip over trivial cases.
+
+ >>> ubuntu_copy_archives = archive_set.getArchivesForDistribution(
+ ... ubuntu, purposes=[ArchivePurpose.COPY], user=copy_owner2,
+ ... exclude_pristine=True)
+ >>> print_archive_names(ubuntu_copy_archives)
+ Name Owner Private Enabled
+
+ >>> _ = factory.makeSourcePackagePublishingHistory(
+ ... archive=removeSecurityProxy(fine_copy).archive)
+ >>> _ = factory.makeSourcePackagePublishingHistory(
+ ... archive=removeSecurityProxy(ultimate_copy).archive)
+ >>> ubuntu_copy_archives = archive_set.getArchivesForDistribution(
+ ... ubuntu, purposes=[ArchivePurpose.COPY], user=copy_owner2,
+ ... exclude_pristine=True)
+ >>> print_archive_names(ubuntu_copy_archives)
+ Name Owner Private Enabled
+ fine-copy copy-owner2 False True
+ ultimate-copy copy-owner1 False True
+
Archive Permission Checking
---------------------------
diff --git a/lib/lp/soyuz/interfaces/archive.py b/lib/lp/soyuz/interfaces/archive.py
index b6cc663..e84542e 100644
--- a/lib/lp/soyuz/interfaces/archive.py
+++ b/lib/lp/soyuz/interfaces/archive.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Archive interfaces."""
@@ -2445,7 +2445,7 @@ class IArchiveSet(Interface):
"""
def getArchivesForDistribution(distribution, name=None, purposes=None,
- user=None, exclude_disabled=True):
+ user=None, exclude_disabled=True, exclude_pristine=False):
"""Return a list of all the archives for a distribution.
This will return all the archives for the given distribution, with
@@ -2461,6 +2461,8 @@ class IArchiveSet(Interface):
has permission. If it is not supplied, only public archives
will be returned.
:param exclude_disabled: Whether to exclude disabled archives.
+ :param exclude_pristine: Whether to exclude archives that have never
+ had any publications.
:return: A queryset of all the archives for the given
distribution matching the given params.
diff --git a/lib/lp/soyuz/model/archive.py b/lib/lp/soyuz/model/archive.py
index e51898b..c3cbaa1 100644
--- a/lib/lp/soyuz/model/archive.py
+++ b/lib/lp/soyuz/model/archive.py
@@ -2895,7 +2895,8 @@ class ArchiveSet:
def getArchivesForDistribution(self, distribution, name=None,
purposes=None, user=None,
- exclude_disabled=True):
+ exclude_disabled=True,
+ exclude_pristine=False):
"""See `IArchiveSet`."""
extra_exprs = []
@@ -2948,10 +2949,14 @@ class ArchiveSet:
if exclude_disabled:
extra_exprs.append(Archive._enabled == True)
+ if exclude_pristine:
+ extra_exprs.append(
+ SourcePackagePublishingHistory.archive == Archive.id)
+
query = Store.of(distribution).find(
Archive,
Archive.distribution == distribution,
- *extra_exprs)
+ *extra_exprs).config(distinct=True)
return query.order_by(Archive.name)
