launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #05259
[Merge] lp:~wgrant/launchpad/bug-876171 into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/bug-876171 into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #876171 in Launchpad itself: "obsolete-distroseries.py should condemn all remaining publications"
https://bugs.launchpad.net/launchpad/+bug/876171
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/bug-876171/+merge/79639
This branch extends obsolete-distroseries.py to also schedule deletion of any publications that are not yet condemned. This fixes bug #876171 -- see the bug for a slightly more detailed explanation.
Since it's now a two-stage process, it's awkward to continue to0 error if there's nothing to do. It's also pretty pointless to error in that case. So I removed the no-op error and its test.
Tested against some existing obsolete series on dogfood, and it indeed condemns sources that have FTBFS/NBS binaries:
2011-10-18 00:50:56 INFO Obsoleting all packages for distroseries dapper in the ubuntu distribution.
2011-10-18 00:51:10 INFO Obsoleting published packages (0 sources, 0 binaries).
2011-10-18 00:51:26 INFO Scheduling deletion of other packages (165 sources, 0 binaries).
--
https://code.launchpad.net/~wgrant/launchpad/bug-876171/+merge/79639
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/bug-876171 into lp:launchpad.
=== modified file 'lib/lp/registry/interfaces/distroseries.py'
--- lib/lp/registry/interfaces/distroseries.py 2011-09-10 00:16:56 +0000
+++ lib/lp/registry/interfaces/distroseries.py 2011-10-18 00:53:25 +0000
@@ -628,12 +628,28 @@
Return publications in the main archives only.
"""
+ def getAllUncondemnedSources():
+ """Return all uncondemned sources for the distroseries.
+
+ An uncondemned publication is one without scheduleddeletiondate set.
+
+ Return publications in the main archives only.
+ """
+
def getAllPublishedBinaries():
"""Return all currently published binaries for the distroseries.
Return publications in the main archives only.
"""
+ def getAllUncondemnedBinaries():
+ """Return all uncondemned binaries for the distroseries.
+
+ An uncondemned publication is one without scheduleddeletiondate set.
+
+ Return publications in the main archives only.
+ """
+
def getSourcesPublishedForAllArchives():
"""Return all sourcepackages published across all the archives.
=== modified file 'lib/lp/registry/model/distroseries.py'
--- lib/lp/registry/model/distroseries.py 2011-09-28 03:28:50 +0000
+++ lib/lp/registry/model/distroseries.py 2011-10-18 00:53:25 +0000
@@ -1141,35 +1141,47 @@
SeriesStatus.EXPERIMENTAL,
]
+ def _getAllSources(self):
+ """Get all sources ever published in this series' main archives."""
+ return IStore(SourcePackagePublishingHistory).find(
+ SourcePackagePublishingHistory,
+ SourcePackagePublishingHistory.distroseriesID == self.id,
+ SourcePackagePublishingHistory.archiveID.is_in(
+ self.distribution.all_distro_archive_ids),
+ ).order_by(SourcePackagePublishingHistory.id)
+
+ def _getAllBinaries(self):
+ """Get all binaries ever published in this series' main archives."""
+ return IStore(BinaryPackagePublishingHistory).find(
+ BinaryPackagePublishingHistory,
+ DistroArchSeries.distroseriesID == self.id,
+ BinaryPackagePublishingHistory.distroarchseriesID
+ == DistroArchSeries.id,
+ BinaryPackagePublishingHistory.archiveID.is_in(
+ self.distribution.all_distro_archive_ids),
+ ).order_by(BinaryPackagePublishingHistory.id)
+
def getAllPublishedSources(self):
"""See `IDistroSeries`."""
# Consider main archives only, and return all sources in
# the PUBLISHED state.
- archives = self.distribution.getArchiveIDList()
- return SourcePackagePublishingHistory.select("""
- distroseries = %s AND
- status = %s AND
- archive in %s
- """ % sqlvalues(self, PackagePublishingStatus.PUBLISHED,
- archives),
- orderBy="id")
+ return self._getAllSources().find(
+ status=PackagePublishingStatus.PUBLISHED)
def getAllPublishedBinaries(self):
"""See `IDistroSeries`."""
# Consider main archives only, and return all binaries in
# the PUBLISHED state.
- archives = self.distribution.getArchiveIDList()
- return BinaryPackagePublishingHistory.select("""
- BinaryPackagePublishingHistory.distroarchseries =
- DistroArchSeries.id AND
- DistroArchSeries.distroseries = DistroSeries.id AND
- DistroSeries.id = %s AND
- BinaryPackagePublishingHistory.status = %s AND
- BinaryPackagePublishingHistory.archive in %s
- """ % sqlvalues(self, PackagePublishingStatus.PUBLISHED,
- archives),
- clauseTables=["DistroArchSeries", "DistroSeries"],
- orderBy="BinaryPackagePublishingHistory.id")
+ return self._getAllBinaries().find(
+ status=PackagePublishingStatus.PUBLISHED)
+
+ def getAllUncondemnedSources(self):
+ """See `IDistroSeries`."""
+ return self._getAllSources().find(scheduleddeletiondate=None)
+
+ def getAllUncondemnedBinaries(self):
+ """See `IDistroSeries`."""
+ return self._getAllBinaries().find(scheduleddeletiondate=None)
def getSourcesPublishedForAllArchives(self):
"""See `IDistroSeries`."""
=== modified file 'lib/lp/soyuz/scripts/ftpmaster.py'
--- lib/lp/soyuz/scripts/ftpmaster.py 2011-09-30 16:05:20 +0000
+++ lib/lp/soyuz/scripts/ftpmaster.py 2011-10-18 00:53:25 +0000
@@ -18,6 +18,7 @@
]
import hashlib
+from itertools import chain
import os
import stat
import sys
@@ -26,6 +27,7 @@
from debian.deb822 import Changes
from zope.component import getUtility
+from canonical.database.constants import UTC_NOW
from canonical.launchpad.helpers import filenameToContentType
from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
from canonical.librarian.interfaces import (
@@ -902,29 +904,33 @@
distroseries.name,
distroseries.distribution.name))
+ # First, mark all Published sources as Obsolete.
sources = distroseries.getAllPublishedSources()
binaries = distroseries.getAllPublishedBinaries()
- num_sources = sources.count()
- num_binaries = binaries.count()
- self.logger.info("There are %d sources and %d binaries." % (
- num_sources, num_binaries))
-
- if num_sources == 0 and num_binaries == 0:
- raise SoyuzScriptError("Nothing to do, no published packages.")
-
- self.logger.info("Obsoleting sources...")
- for package in sources:
- self.logger.debug("Obsoleting %s" % package.displayname)
- package.requestObsolescence()
-
- self.logger.info("Obsoleting binaries...")
- for package in binaries:
- self.logger.debug("Obsoleting %s" % package.displayname)
- package.requestObsolescence()
-
- # The obsoleted packages will be caught by death row processing
- # the next time it runs. We skip the domination phase in the
- # publisher because it won't consider stable distroseries.
+ self.logger.info(
+ "Obsoleting published packages (%d sources, %d binaries)."
+ % (sources.count(), binaries.count()))
+ for package in chain(sources, binaries):
+ self.logger.debug("Obsoleting %s" % package.displayname)
+ package.requestObsolescence()
+
+ # Next, ensure that everything is scheduled for deletion. The
+ # dominator will normally leave some superseded publications
+ # uncondemned, for example sources that built NBSed binaries.
+ sources = distroseries.getAllUncondemnedSources()
+ binaries = distroseries.getAllUncondemnedBinaries()
+ self.logger.info(
+ "Scheduling deletion of other packages (%d sources, %d binaries)."
+ % (sources.count(), binaries.count()))
+ for package in chain(sources, binaries):
+ self.logger.debug(
+ "Scheduling deletion of %s" % package.displayname)
+ package.scheduleddeletiondate = UTC_NOW
+
+ # The packages from both phases will be caught by death row
+ # processing the next time it runs. We skip the domination
+ # phase in the publisher because it won't consider stable
+ # distroseries.
def _checkParameters(self, distroseries):
"""Sanity check the supplied script parameters."""
=== modified file 'lib/lp/soyuz/scripts/tests/test_obsoletedistroseries.py'
--- lib/lp/soyuz/scripts/tests/test_obsoletedistroseries.py 2010-12-20 03:21:03 +0000
+++ lib/lp/soyuz/scripts/tests/test_obsoletedistroseries.py 2011-10-18 00:53:25 +0000
@@ -6,7 +6,6 @@
import os
import subprocess
import sys
-import unittest
from zope.component import getUtility
@@ -25,9 +24,13 @@
ObsoleteDistroseries,
SoyuzScriptError,
)
-
-
-class TestObsoleteDistroseriesScript(unittest.TestCase):
+from lp.testing import (
+ TestCase,
+ TestCaseWithFactory,
+ )
+
+
+class TestObsoleteDistroseriesScript(TestCase):
"""Test the obsolete-distroseries.py script."""
layer = LaunchpadZopelessLayer
@@ -65,12 +68,13 @@
"Expected %s, got %s" % (expected, err))
-class TestObsoleteDistroseries(unittest.TestCase):
+class TestObsoleteDistroseries(TestCaseWithFactory):
"""Test the ObsoleteDistroseries class."""
layer = LaunchpadZopelessLayer
def setUp(self):
"""Set up test data common to all test cases."""
+ super(TestObsoleteDistroseries, self).setUp()
self.warty = getUtility(IDistributionSet)['ubuntu']['warty']
# Re-process the returned list otherwise it ends up being a list
@@ -129,25 +133,6 @@
obsoleter = self.getObsoleter(suite='warty')
self.assertRaises(SoyuzScriptError, obsoleter.mainTask)
- def testNothingToDoCase(self):
- """When there is nothing to do, we expect an exception."""
- obsoleter = self.getObsoleter()
- self.warty.status = SeriesStatus.OBSOLETE
-
- # Get all the published sources in warty.
- published_sources, published_binaries = (
- self.getPublicationsForDistroseries())
-
- # Reset their status to OBSOLETE.
- for package in published_sources:
- package.status = PackagePublishingStatus.OBSOLETE
- for package in published_binaries:
- package.status = PackagePublishingStatus.OBSOLETE
-
- # Call the script and ensure it does nothing.
- self.layer.txn.commit()
- self.assertRaises(SoyuzScriptError, obsoleter.mainTask)
-
def testObsoleteDistroseriesWorks(self):
"""Make sure the required publications are obsoleted."""
obsoleter = self.getObsoleter()
@@ -210,3 +195,40 @@
binary = BinaryPackagePublishingHistory.get(id)
self.assertTrue(
binary.status != PackagePublishingStatus.OBSOLETE)
+
+ def test_schedules_deletion_of_uncondemned_pubs(self):
+ # Any publications that were no longer Published but never
+ # condemned by the dominator get condemned now.
+ # eg. superseded sources that released with published NBS
+ # binaries.
+
+ obsolete_series = self.factory.makeDistroSeries(
+ status=SeriesStatus.OBSOLETE)
+ other_series = self.factory.makeDistroSeries(
+ distribution=obsolete_series.distribution,
+ status=SeriesStatus.CURRENT)
+ obsoleter = self.getObsoleter(
+ distribution=obsolete_series.distribution.name,
+ suite=obsolete_series.name)
+
+ pubs = dict()
+ for series in (obsolete_series, other_series):
+ arch = self.factory.makeDistroArchSeries(distroseries=series)
+ pubs[series] = [
+ self.factory.makeSourcePackagePublishingHistory(
+ distroseries=series,
+ status=PackagePublishingStatus.SUPERSEDED),
+ self.factory.makeBinaryPackagePublishingHistory(
+ distroarchseries=arch,
+ status=PackagePublishingStatus.SUPERSEDED),
+ ]
+
+ for pub in pubs[obsolete_series] + pubs[other_series]:
+ self.assertIs(None, pub.scheduleddeletiondate)
+
+ obsoleter.mainTask()
+
+ for pub in pubs[obsolete_series]:
+ self.assertIsNot(None, pub.scheduleddeletiondate)
+ for pub in pubs[other_series]:
+ self.assertIs(None, pub.scheduleddeletiondate)
