launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #05127
[Merge] lp:~cjwatson/launchpad/remove-archive-cruft-check into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/remove-archive-cruft-check into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/remove-archive-cruft-check/+merge/77524
Remove archive-cruft-check. The Ubuntu archive admins now use an API script based on this instead.
I believe this is qa-untestable. Robert and William expressed general approbation on IRC.
--
https://code.launchpad.net/~cjwatson/launchpad/remove-archive-cruft-check/+merge/77524
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/remove-archive-cruft-check into lp:launchpad.
=== modified file 'lib/lp/soyuz/doc/soyuz-upload.txt'
--- lib/lp/soyuz/doc/soyuz-upload.txt 2011-09-29 09:37:58 +0000
+++ lib/lp/soyuz/doc/soyuz-upload.txt 2011-09-29 13:11:28 +0000
@@ -615,36 +615,6 @@
END
-Testing archive-cruft-check-ng behaviour:
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Defining path to the script:
-
- >>> script = os.path.join(config.root, "scripts", "ftpmaster-tools",
- ... "archive-cruft-check.py")
- >>> process = subprocess.Popen([sys.executable, script, "-vn",
- ... "-d", "ubuntutest",
- ... "-s", "breezy-autotest",
- ... "/var/tmp/archive"],
- ... stdout=subprocess.PIPE,
- ... stderr=subprocess.PIPE,)
- >>> stdout, stderr = process.communicate()
- >>> process.returncode
- 0
- >>> print stderr
- INFO Creating lockfile: ...
- DEBUG Considering Sources:
- DEBUG Processing /var/tmp/archive/ubuntutest/dists/breezy-autotest/restricted/source/Sources.gz
- DEBUG Processing /var/tmp/archive/ubuntutest/dists/breezy-autotest/main/source/Sources.gz
- DEBUG Processing /var/tmp/archive/ubuntutest/dists/breezy-autotest/multiverse/source/Sources.gz
- DEBUG Processing /var/tmp/archive/ubuntutest/dists/breezy-autotest/universe/source/Sources.gz
- DEBUG Building not build from source list (NBS):
- DEBUG Building all superseded by any list (ASBA):
- DEBUG No NBS found
- DEBUG Removing lock file: ...
- <BLANKLINE>
-
-
Nice! That's enough for now.. let's kill the process and clean
everything up.
=== modified file 'lib/lp/soyuz/scripts/ftpmaster.py'
--- lib/lp/soyuz/scripts/ftpmaster.py 2011-09-05 03:12:47 +0000
+++ lib/lp/soyuz/scripts/ftpmaster.py 2011-09-29 13:11:28 +0000
@@ -6,8 +6,6 @@
__metaclass__ = type
__all__ = [
- 'ArchiveCruftChecker',
- 'ArchiveCruftCheckerError',
'ChrootManager',
'ChrootManagerError',
'LpQueryDistro',
@@ -69,408 +67,6 @@
)
-class ArchiveCruftCheckerError(Exception):
- """ArchiveCruftChecker specific exception.
-
- Mostly used to describe errors in the initialization of this object.
- """
-
-
-class TagFileNotFound(Exception):
- """Raised when an archive tag file could not be found."""
-
-
-class ArchiveCruftChecker:
- """Perform overall checks to identify and remove obsolete records.
-
- Use initialize() method to validate passed parameters and build the
- infrastructure variables. It will raise ArchiveCruftCheckerError if
- something goes wrong.
- """
-
- # XXX cprov 2006-05-15: the default archive path should come
- # from the config.
- def __init__(self, logger, distribution_name='ubuntu', suite=None,
- archive_path='/srv/launchpad.net/ubuntu-archive'):
- """Store passed arguments.
-
- Also Initialize empty variables for storing preliminar results.
- """
- self.distribution_name = distribution_name
- self.suite = suite
- self.archive_path = archive_path
- self.logger = logger
- # initialize a group of variables to store temporary results
- # available versions of published sources
- self.source_versions = {}
- # available binaries produced by published sources
- self.source_binaries = {}
- # 'Not Build From Source' binaries
- self.nbs = {}
- # 'All superseded by Any' binaries
- self.asba = {}
- # published binary package names
- self.bin_pkgs = {}
- # Architecture specific binary packages
- self.arch_any = {}
- # proposed NBS (before clean up)
- self.dubious_nbs = {}
- # NBS after clean up
- self.real_nbs = {}
- # definitive NBS organized for clean up
- self.nbs_to_remove = []
-
- @property
- def architectures(self):
- return dict([(a.architecturetag, a)
- for a in self.distroseries.architectures])
-
- @property
- def components(self):
- return dict([(c.name, c) for c in self.distroseries.components])
-
- @property
- def components_and_di(self):
- components_and_di = []
- for component in self.components:
- components_and_di.append(component)
- components_and_di.append('%s/debian-installer' % (component))
- return components_and_di
-
- @property
- def dist_archive(self):
- return os.path.join(
- self.archive_path, self.distro.name, 'dists',
- self.distroseries.name + pocketsuffix[self.pocket])
-
- def gunzipTagFileContent(self, filename):
- """Gunzip the contents of passed filename.
-
- Check filename presence, if not present in the filesystem,
- raises ArchiveCruftCheckerError. Use an tempfile.mkstemp()
- to store the uncompressed content. Invoke system available
- gunzip`, raises ArchiveCruftCheckError if it fails.
-
- This method doesn't close the file descriptor used and does not
- remove the temporary file from the filesystem, those actions
- are required in the callsite. (apt_pkg.ParseTagFile is lazy)
-
- Return a tuple containing:
- * temp file descriptor
- * temp filename
- * the contents parsed by apt_pkg.ParseTagFile()
- """
- if not os.path.exists(filename):
- raise TagFileNotFound("File does not exist: %s" % filename)
-
- temp_fd, temp_filename = tempfile.mkstemp()
- (result, output) = commands.getstatusoutput(
- "gunzip -c %s > %s" % (filename, temp_filename))
- if result != 0:
- raise ArchiveCruftCheckerError(
- "Gunzip invocation failed!\n%s" % output)
-
- temp_file = os.fdopen(temp_fd)
- # XXX cprov 2006-05-15: maybe we need some sort of data integrity
- # check at this point, and maybe keep the uncrompressed file
- # for debug purposes, let's see how it behaves in real conditions.
- parsed_contents = apt_pkg.ParseTagFile(temp_file)
-
- return temp_file, temp_filename, parsed_contents
-
- def processSources(self):
- """Process archive sources index.
-
- Build source_binaries, source_versions and bin_pkgs lists.
- """
- self.logger.debug("Considering Sources:")
- for component in self.components:
- filename = os.path.join(
- self.dist_archive, "%s/source/Sources.gz" % component)
-
- self.logger.debug("Processing %s" % filename)
- try:
- temp_fd, temp_filename, parsed_sources = (
- self.gunzipTagFileContent(filename))
- except TagFileNotFound, warning:
- self.logger.warn(warning)
- return
- try:
- while parsed_sources.Step():
- source = parsed_sources.Section.Find("Package")
- source_version = parsed_sources.Section.Find("Version")
- binaries = parsed_sources.Section.Find("Binary")
- for binary in [
- item.strip() for item in binaries.split(',')]:
- self.bin_pkgs.setdefault(binary, [])
- self.bin_pkgs[binary].append(source)
-
- self.source_binaries[source] = binaries
- self.source_versions[source] = source_version
- finally:
- # close fd and remove temporary file used to store
- # uncompressed tag file content from the filesystem.
- temp_fd.close()
- os.unlink(temp_filename)
-
- def buildNBS(self):
- """Build the group of 'not build from source' binaries"""
- # Checks based on the Packages files
- self.logger.debug("Building not build from source list (NBS):")
- for component in self.components_and_di:
- for architecture in self.architectures:
- self.buildArchNBS(component, architecture)
-
- def buildArchNBS(self, component, architecture):
- """Build NBS per architecture.
-
- Store results in self.nbs, also build architecture specific
- binaries group (stored in self.arch_any)
- """
- filename = os.path.join(
- self.dist_archive,
- "%s/binary-%s/Packages.gz" % (component, architecture))
-
- self.logger.debug("Processing %s" % filename)
- try:
- temp_fd, temp_filename, parsed_packages = (
- self.gunzipTagFileContent(filename))
- except TagFileNotFound, warning:
- self.logger.warn(warning)
- return
-
- try:
- while parsed_packages.Step():
- package = parsed_packages.Section.Find('Package')
- source = parsed_packages.Section.Find('Source', "")
- version = parsed_packages.Section.Find('Version')
- architecture = parsed_packages.Section.Find('Architecture')
-
- if source == "":
- source = package
-
- if source.find("(") != -1:
- m = re_extract_src_version.match(source)
- source = m.group(1)
- version = m.group(2)
-
- if package not in self.bin_pkgs:
- self.nbs.setdefault(source, {})
- self.nbs[source].setdefault(package, {})
- self.nbs[source][package][version] = ""
-
- if architecture != "all":
- self.arch_any.setdefault(package, "0")
- if apt_pkg.VersionCompare(
- version, self.arch_any[package]) < 1:
- self.arch_any[package] = version
- finally:
- # close fd and remove temporary file used to store uncompressed
- # tag file content from the filesystem.
- temp_fd.close()
- os.unlink(temp_filename)
-
- def buildASBA(self):
- """Build the group of 'all superseded by any' binaries."""
- self.logger.debug("Building all superseded by any list (ASBA):")
- for component in self.components_and_di:
- for architecture in self.architectures:
- self.buildArchASBA(component, architecture)
-
- def buildArchASBA(self, component, architecture):
- """Build ASBA per architecture.
-
- Store the result in self.asba, require self.arch_any to be built
- previously.
- """
- filename = os.path.join(
- self.dist_archive,
- "%s/binary-%s/Packages.gz" % (component, architecture))
-
- try:
- temp_fd, temp_filename, parsed_packages = (
- self.gunzipTagFileContent(filename))
- except TagFileNotFound, warning:
- self.logger.warn(warning)
- return
-
- try:
- while parsed_packages.Step():
- package = parsed_packages.Section.Find('Package')
- source = parsed_packages.Section.Find('Source', "")
- version = parsed_packages.Section.Find('Version')
- architecture = parsed_packages.Section.Find('Architecture')
-
- if source == "":
- source = package
-
- if source.find("(") != -1:
- m = re_extract_src_version.match(source)
- source = m.group(1)
- version = m.group(2)
-
- if architecture == "all":
- if (package in self.arch_any and
- apt_pkg.VersionCompare(
- version, self.arch_any[package]) > -1):
- self.asba.setdefault(source, {})
- self.asba[source].setdefault(package, {})
- self.asba[source][package].setdefault(version, {})
- self.asba[source][package][version][architecture] = ""
- finally:
- # close fd and remove temporary file used to store uncompressed
- # tag file content from the filesystem.
- temp_fd.close()
- os.unlink(temp_filename)
-
- def addNBS(self, nbs_d, source, version, package):
- """Add a new entry in given organized nbs_d list
-
- Ensure the package is still published in the suite before add.
- """
- result = self.distroseries.getBinaryPackagePublishing(name=package)
-
- if len(list(result)) == 0:
- return
-
- nbs_d.setdefault(source, {})
- nbs_d[source].setdefault(version, {})
- nbs_d[source][version][package] = ""
-
- def refineNBS(self):
- """ Distinguish dubious from real NBS.
-
- They are 'dubious' if the version numbers match and 'real'
- if the versions don't match.
- It stores results in self.dubious_nbs and self.real_nbs.
- """
- for source in self.nbs.keys():
- for package in self.nbs[source].keys():
- versions = self.nbs[source][package].keys()
- versions.sort(apt_pkg.VersionCompare)
- latest_version = versions.pop()
-
- source_version = self.source_versions.get(source, "0")
-
- if apt_pkg.VersionCompare(latest_version,
- source_version) == 0:
- self.addNBS(self.dubious_nbs, source, latest_version,
- package)
- else:
- self.addNBS(self.real_nbs, source, latest_version,
- package)
-
- def outputNBS(self):
- """Properly display built NBS entries.
-
- Also organize the 'real' NBSs for removal in self.nbs_to_remove
- attribute.
- """
- output = "Not Built from Source\n"
- output += "---------------------\n\n"
-
- nbs_keys = self.real_nbs.keys()
- nbs_keys.sort()
-
- for source in nbs_keys:
- proposed_bin = self.source_binaries.get(
- source, "(source does not exist)")
- porposed_version = self.source_versions.get(source, "??")
- output += (" * %s_%s builds: %s\n"
- % (source, porposed_version, proposed_bin))
- output += "\tbut no longer builds:\n"
- versions = self.real_nbs[source].keys()
- versions.sort(apt_pkg.VersionCompare)
-
- for version in versions:
- packages = self.real_nbs[source][version].keys()
- packages.sort()
-
- for pkg in packages:
- self.nbs_to_remove.append(pkg)
-
- output += " o %s: %s\n" % (
- version, ", ".join(packages))
-
- output += "\n"
-
- if self.nbs_to_remove:
- self.logger.info(output)
- else:
- self.logger.debug("No NBS found")
-
- def initialize(self):
- """Initialize and build required lists of obsolete entries in archive.
-
- Check integrity of passed parameters and store organised data.
- The result list is the self.nbs_to_remove which should contain
- obsolete packages not currently able to be built from again.
- Another preliminary lists can be inspected in order to have better
- idea of what was computed.
- If anything goes wrong mid-process, it raises ArchiveCruftCheckError,
- otherwise a list of packages to be removes is printed.
- """
- if self.distribution_name is None:
- self.distro = getUtility(ILaunchpadCelebrities).ubuntu
- else:
- try:
- self.distro = getUtility(IDistributionSet)[
- self.distribution_name]
- except NotFoundError:
- raise ArchiveCruftCheckerError(
- "Invalid distribution: '%s'" % self.distribution_name)
-
- if not self.suite:
- self.distroseries = self.distro.currentseries
- self.pocket = PackagePublishingPocket.RELEASE
- else:
- try:
- self.distroseries, self.pocket = (
- self.distro.getDistroSeriesAndPocket(self.suite))
- except NotFoundError:
- raise ArchiveCruftCheckerError(
- "Invalid suite: '%s'" % self.suite)
-
- if not os.path.exists(self.dist_archive):
- raise ArchiveCruftCheckerError(
- "Invalid archive path: '%s'" % self.dist_archive)
-
- apt_pkg.init()
- self.processSources()
- self.buildNBS()
- self.buildASBA()
- self.refineNBS()
- self.outputNBS()
-
- def doRemovals(self):
- """Perform the removal of the obsolete packages found.
-
- It iterates over the previously build list (self.nbs_to_remove)
- and mark them as 'superseded' in the archive DB model. They will
- get removed later by the archive sanity check run each cycle
- of the cron.daily.
- """
- for package in self.nbs_to_remove:
-
- for distroarchseries in self.distroseries.architectures:
- binarypackagename = getUtility(IBinaryPackageNameSet)[package]
- dasbp = distroarchseries.getBinaryPackage(binarypackagename)
- dasbpr = dasbp.currentrelease
- try:
- bpph = dasbpr.current_publishing_record
- bpph.supersede()
- # We're blindly removing for all arches, if it's not there
- # for some, that's fine ...
- except NotFoundError:
- pass
- else:
- version = bpph.binarypackagerelease.version
- self.logger.info("Removed %s_%s from %s/%s ... "
- % (package, version,
- self.distroseries.name,
- distroarchseries.architecturetag))
-
-
class PubBinaryContent:
"""Binary publication container.
=== removed file 'lib/lp/soyuz/scripts/tests/test_archivecruftchecker.py'
--- lib/lp/soyuz/scripts/tests/test_archivecruftchecker.py 2011-07-20 14:46:38 +0000
+++ lib/lp/soyuz/scripts/tests/test_archivecruftchecker.py 1970-01-01 00:00:00 +0000
@@ -1,147 +0,0 @@
-# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""ArchiveCruftChecker tests.
-
-Check how scripts/ftpmaster-tools/archive-cruft-check.py works on a
-just-published 'ubuntutest' archive.
-"""
-
-__metaclass__ = type
-
-import shutil
-import transaction
-import unittest
-
-from zope.component import getUtility
-
-from canonical.config import config
-from canonical.testing.layers import LaunchpadZopelessLayer
-from lp.registry.interfaces.distribution import IDistributionSet
-from lp.registry.interfaces.pocket import PackagePublishingPocket
-from lp.services.log.logger import BufferLogger
-from lp.soyuz.scripts.ftpmaster import (
- ArchiveCruftChecker,
- ArchiveCruftCheckerError,
- )
-from lp.soyuz.scripts.publishdistro import PublishDistro
-
-
-# XXX cprov 2006-05-15: {create, remove}TestArchive functions should be
-# moved to the publisher test domain as soon as we have it.
-def createTestArchive():
- """Creates a fresh test archive based on sampledata."""
- script = PublishDistro(test_args=["-C", "-q", "-d", "ubuntutest"])
- script.txn = transaction
- script.main()
-
-
-def removeTestArchive():
- # XXX JeroenVermeulen 2011-07-20 bug=813538: Use a temporary
- # directory so we don't have to commit this horror.
- """Remove the entire test archive directory from the filesystem."""
- shutil.rmtree("/var/tmp/archive/")
-
-
-class TestArchiveCruftChecker(unittest.TestCase):
- layer = LaunchpadZopelessLayer
-
- def setUp(self):
- """Setup the test environment."""
- self.layer.switchDbUser(config.archivepublisher.dbuser)
- self.log = BufferLogger()
- self.ubuntutest = getUtility(IDistributionSet)['ubuntutest']
- self.breezy_autotest = self.ubuntutest['breezy-autotest']
- self.archive_path = "/var/tmp/archive"
- createTestArchive()
-
- def tearDown(self):
- """Clean up test environment and remove the test archive."""
- removeTestArchive()
-
- def testInitializeSuccess(self):
- """Test ArchiveCruftChecker initialization process.
-
- Check if the correct attributes are built after initialization.
- """
- checker = ArchiveCruftChecker(
- self.log, distribution_name='ubuntutest', suite='breezy-autotest',
- archive_path=self.archive_path)
- checker.initialize()
-
- self.assertEqual(self.ubuntutest, checker.distro)
- self.assertEqual(self.breezy_autotest, checker.distroseries)
- self.assertEqual(PackagePublishingPocket.RELEASE, checker.pocket)
- self.assertEqual(0, len(checker.nbs_to_remove))
- self.assertEqual(0, len(checker.real_nbs))
- self.assertEqual(0, len(checker.dubious_nbs))
- self.assertEqual(0, len(checker.bin_pkgs))
- self.assertEqual(0, len(checker.arch_any))
- self.assertEqual(0, len(checker.source_versions))
- self.assertEqual(0, len(checker.source_binaries))
-
- # The 'dist_archive' is an absolute path to the 'dists' section
- # based on the given 'archive_path'.
- self.assertEqual(
- checker.dist_archive,
- '/var/tmp/archive/ubuntutest/dists/breezy-autotest')
-
- # The 'components' dictionary contains all components selected
- # for the given distroseries organized as:
- # {$component_name: IComponent, ...}
- for component_name, component in checker.components.iteritems():
- self.assertEqual(component_name, component.name)
- checker_components = sorted(
- [component_name for component_name in checker.components.keys()])
- self.assertEqual(
- checker_components,
- ['main', 'multiverse', 'restricted', 'universe'])
-
- # The 'components_and_di' lists the relative 'dists' paths
- # for all components subsections of the archive which contain
- # indexes.
- expected = [
- 'main',
- 'main/debian-installer',
- 'multiverse',
- 'multiverse/debian-installer',
- 'restricted',
- 'restricted/debian-installer',
- 'universe',
- 'universe/debian-installer',
- ]
- self.assertEqual(sorted(checker.components_and_di), expected)
-
- def testSuiteDistArchive(self):
- """Check if 'dist_archive' path considers pocket correctly."""
- checker = ArchiveCruftChecker(
- self.log, distribution_name='ubuntutest',
- suite='breezy-autotest-security',
- archive_path=self.archive_path)
- checker.initialize()
-
- self.assertEqual(
- checker.dist_archive,
- '/var/tmp/archive/ubuntutest/dists/breezy-autotest-security')
-
- def testInitializeFailure(self):
- """ArchiveCruftCheck initialization failures.
-
- * An unknown suite;
- * An unknown distribution;
- * The absence of the distribution in the given archive path.
- """
- checker = ArchiveCruftChecker(
- self.log, distribution_name='ubuntu', suite='miserable',
- archive_path=self.archive_path)
- self.assertRaises(ArchiveCruftCheckerError, checker.initialize)
-
- checker = ArchiveCruftChecker(
- self.log, distribution_name='foobuntu', suite='breezy-autotest',
- archive_path=self.archive_path)
- self.assertRaises(ArchiveCruftCheckerError, checker.initialize)
-
- checker = ArchiveCruftChecker(
- self.log, distribution_name='ubuntu', suite='breezy-autotest',
- archive_path=self.archive_path)
- self.assertRaises(ArchiveCruftCheckerError, checker.initialize)
=== removed file 'scripts/ftpmaster-tools/archive-cruft-check.py'
--- scripts/ftpmaster-tools/archive-cruft-check.py 2011-09-18 05:45:56 +0000
+++ scripts/ftpmaster-tools/archive-cruft-check.py 1970-01-01 00:00:00 +0000
@@ -1,56 +0,0 @@
-#!/usr/bin/python -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# pylint: disable-msg=W0403
-
-"""Archive Cruft checker.
-
-A kind of archive garbage collector, supersede NBS binaries (not build
-from source).
-"""
-
-import _pythonpath
-
-from canonical.config import config
-from lp.services.scripts.base import LaunchpadScript, LaunchpadScriptFailure
-from lp.soyuz.scripts.ftpmaster import (
- ArchiveCruftChecker, ArchiveCruftCheckerError)
-
-
-class ArchiveCruftCheckerScript(LaunchpadScript):
-
- usage = "Usage: archive-cruft-check.py [options] <ARCHIVE_PATH>"
-
- def add_my_options(self):
- self.parser.add_option(
- "-d", "--distro", dest="distro", help="remove from DISTRO")
- self.parser.add_option(
- "-n", "--no-action", dest="action", default=True,
- action="store_false", help="don't do anything")
- self.parser.add_option(
- "-s", "--suite", dest="suite", help="only act on SUITE")
-
- def main(self):
- if len(self.args) != 1:
- self.parser.error('ARCHIVEPATH is require')
- archive_path = self.args[0]
-
- checker = ArchiveCruftChecker(
- self.logger, distribution_name=self.options.distro,
- suite=self.options.suite, archive_path=archive_path)
-
- try:
- checker.initialize()
- except ArchiveCruftCheckerError, info:
- raise LaunchpadScriptFailure(info)
-
- # XXX cprov 2007-06-26 bug=121784: Disabling by distro-team request.
- # if checker.nbs_to_remove and options.action:
- # checker.doRemovals()
- # ztm.commit()
-
-if __name__ == '__main__':
- ArchiveCruftCheckerScript(
- 'archive-cruft-check', config.archivepublisher.dbuser).lock_and_run()
