launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #06900
[Merge] lp:~cjwatson/launchpad/remove-archive-override-check into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/remove-archive-override-check into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/remove-archive-override-check/+merge/99514
archive-override-check was written at our request:
https://launchpad.net/launchpad/+spec/overrides-consistency-check
However, we have in fact been using a different tool for some time, which doesn't talk to the Launchpad database - it's sufficient to do Sources/Packages file analysis for this - and so doesn't need to run on ftpmaster, and which is orders of magnitude faster to run (output at http://people.canonical.com/~ubuntu-archive/architecture-mismatches.txt, currently empty). Given that we want to stop using tools that only work on ftpmaster, we aren't going to use this again, and so we should remove it.
--
https://code.launchpad.net/~cjwatson/launchpad/remove-archive-override-check/+merge/99514
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/remove-archive-override-check into lp:launchpad.
=== removed file 'lib/lp/soyuz/doc/archive-override-check.txt'
--- lib/lp/soyuz/doc/archive-override-check.txt 2011-12-29 05:29:36 +0000
+++ lib/lp/soyuz/doc/archive-override-check.txt 1970-01-01 00:00:00 +0000
@@ -1,44 +0,0 @@
-Check for discrepancies in overrides between architectures
-==========================================================
-
-This script looks for out-of-sync overrides, checking for
-discrepancies in overrides between architectures.
-
-It should run daily via cronjob, sending the STDOUT result report via
-email to distribution maintainers to ensure that only necessary
-discrepancies are present during the development cycle.
-
-See also:
- https://launchpad.net/products/soyuz/+spec/overrides-consistency-check
- https://wiki.ubuntu.com/MilestoneRhythm
-
-XXX cprov 20060714: we need better populate archive/"publishing
-history" in the sampledata to test those tools properly.
-
- >>> import os
- >>> import subprocess
- >>> import sys
- >>> from lp.services.config import config
-
- >>> script = os.path.join(
- ... config.root, "scripts", "ftpmaster-tools",
- ... "archive-override-check.py")
-
- >>> process = subprocess.Popen(
- ... [sys.executable, script, "-v", "-d", "ubuntu", "-s", "warty"],
- ... stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- >>> stdout, stderr = process.communicate()
- >>> process.returncode
- 0
- >>> print stderr
- INFO Creating lockfile: ...
- DEBUG Considering: ubuntu/warty/RELEASE/CURRENT.
- DEBUG ... published sources
- DEBUG Rolling back any remaining transactions.
- DEBUG Removing lock file: ...
- <BLANKLINE>
-
-
-Since its data is sane, empty STDOUT is okay.
-
- >>> print stdout
=== removed file 'lib/lp/soyuz/scripts/pubsourcechecker.py'
--- lib/lp/soyuz/scripts/pubsourcechecker.py 2012-01-19 03:06:04 +0000
+++ lib/lp/soyuz/scripts/pubsourcechecker.py 1970-01-01 00:00:00 +0000
@@ -1,214 +0,0 @@
-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""FTPMaster utilities."""
-
-__metaclass__ = type
-
-__all__ = ['PubSourceChecker']
-
-
-class PubBinaryContent:
- """Binary publication container.
-
- Currently used for auxiliary storage in PubSourceChecker.
- """
-
- def __init__(self, name, version, arch, component, section, priority):
- self.name = name
- self.version = version
- self.arch = arch
- self.component = component
- self.section = section
- self.priority = priority
- self.messages = []
-
- def warn(self, message):
- """Append a warning in the message list."""
- self.messages.append('W: %s' % message)
-
- def error(self, message):
- """Append a error in the message list."""
- self.messages.append('E: %s' % message)
-
- def renderReport(self):
- """Render a report with the appended messages (self.messages).
-
- Return None if no message was found, otherwise return
- a properly formatted string, including
-
- <TAB>BinaryName_Version Arch Component/Section/Priority
- <TAB><TAB>MESSAGE
- """
- if not len(self.messages):
- return
-
- report = [('\t%s_%s %s %s/%s/%s'
- % (self.name, self.version, self.arch,
- self.component, self.section, self.priority))]
-
- for message in self.messages:
- report.append('\t\t%s' % message)
-
- return "\n".join(report)
-
-
-class PubBinaryDetails:
- """Store the component, section and priority of binary packages and, for
- each binary package the most frequent component, section and priority.
-
- These are stored in the following attributes:
-
- - components: A dictionary mapping binary package names to other
- dictionaries mapping component names to binary packages published
- in this component.
- - sections: The same as components, but for sections.
- - priorities: The same as components, but for priorities.
- - correct_components: a dictionary mapping binary package name
- to the most frequent (considered the correct) component name.
- - correct_sections: same as correct_components, but for sections
- - correct_priorities: same as correct_components, but for priorities
- """
-
- def __init__(self):
- self.components = {}
- self.sections = {}
- self.priorities = {}
- self.correct_components = {}
- self.correct_sections = {}
- self.correct_priorities = {}
-
- def addBinaryDetails(self, bin):
- """Include a binary publication and update internal registers."""
- name_components = self.components.setdefault(bin.name, {})
- bin_component = name_components.setdefault(bin.component, [])
- bin_component.append(bin)
-
- name_sections = self.sections.setdefault(bin.name, {})
- bin_section = name_sections.setdefault(bin.section, [])
- bin_section.append(bin)
-
- name_priorities = self.priorities.setdefault(bin.name, {})
- bin_priority = name_priorities.setdefault(bin.priority, [])
- bin_priority.append(bin)
-
- def _getMostFrequentValue(self, data):
- """Return a dict of name and the most frequent value.
-
- Used for self.{components, sections, priorities}
- """
- results = {}
-
- for name, items in data.iteritems():
- highest = 0
- for item, occurrences in items.iteritems():
- if len(occurrences) > highest:
- highest = len(occurrences)
- results[name] = item
-
- return results
-
- def setCorrectValues(self):
- """Find out the correct values for the same binary name
-
- Consider correct the most frequent.
- """
- self.correct_components = self._getMostFrequentValue(self.components)
- self.correct_sections = self._getMostFrequentValue(self.sections)
- self.correct_priorities = self._getMostFrequentValue(self.priorities)
-
-
-class PubSourceChecker:
- """Map and probe a Source/Binaries publication couple.
-
- Receive the source publication data and its binaries and perform
- a group of heuristic consistency checks.
- """
-
- def __init__(self, name, version, component, section, urgency):
- self.name = name
- self.version = version
- self.component = component
- self.section = section
- self.urgency = urgency
- self.binaries = []
- self.binaries_details = PubBinaryDetails()
-
- def addBinary(self, name, version, architecture, component, section,
- priority):
- """Append the binary data to the current publication list."""
- bin = PubBinaryContent(
- name, version, architecture, component, section, priority)
-
- self.binaries.append(bin)
-
- self.binaries_details.addBinaryDetails(bin)
-
- def check(self):
- """Setup check environment and perform the required checks."""
- self.binaries_details.setCorrectValues()
-
- for bin in self.binaries:
- self._checkComponent(bin)
- self._checkSection(bin)
- self._checkPriority(bin)
-
- def _checkComponent(self, bin):
- """Check if the binary component matches the correct component.
-
- 'correct' is the most frequent component in this binary package
- group
- """
- correct_component = self.binaries_details.correct_components[bin.name]
- if bin.component != correct_component:
- bin.warn('Component mismatch: %s != %s'
- % (bin.component, correct_component))
-
- def _checkSection(self, bin):
- """Check if the binary section matches the correct section.
-
- 'correct' is the most frequent section in this binary package
- group
- """
- correct_section = self.binaries_details.correct_sections[bin.name]
- if bin.section != correct_section:
- bin.warn('Section mismatch: %s != %s'
- % (bin.section, correct_section))
-
- def _checkPriority(self, bin):
- """Check if the binary priority matches the correct priority.
-
- 'correct' is the most frequent priority in this binary package
- group
- """
- correct_priority = self.binaries_details.correct_priorities[bin.name]
- if bin.priority != correct_priority:
- bin.warn('Priority mismatch: %s != %s'
- % (bin.priority, correct_priority))
-
- def renderReport(self):
- """Render a formatted report for the publication group.
-
- Return None if no issue was annotated or an formatted string
- including:
-
- SourceName_Version Component/Section/Urgency | # bin
- <BINREPORTS>
- """
- report = []
-
- for bin in self.binaries:
- bin_report = bin.renderReport()
- if bin_report:
- report.append(bin_report)
-
- if not len(report):
- return
-
- result = [('%s_%s %s/%s/%s | %s bin'
- % (self.name, self.version, self.component,
- self.section, self.urgency, len(self.binaries)))]
-
- result.extend(report)
-
- return "\n".join(result)
=== removed file 'lib/lp/soyuz/scripts/tests/test_overrides_checker.py'
--- lib/lp/soyuz/scripts/tests/test_overrides_checker.py 2012-01-19 03:09:38 +0000
+++ lib/lp/soyuz/scripts/tests/test_overrides_checker.py 1970-01-01 00:00:00 +0000
@@ -1,174 +0,0 @@
-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""archive-override-check tool base class tests."""
-
-__metaclass__ = type
-
-from unittest import TestCase
-
-from lp.soyuz.scripts.pubsourcechecker import (
- PubBinaryContent,
- PubBinaryDetails,
- PubSourceChecker,
- )
-
-
-class TestPubBinaryDetails(TestCase):
-
- def setUp(self):
- self.binary_details = PubBinaryDetails()
-
- def test_single_binary(self):
- """Single binary inclusion."""
- bin = PubBinaryContent('foo-dev', '1.0', 'i386', 'main',
- 'base', 'REQUIRED')
-
- self.binary_details.addBinaryDetails(bin)
-
- # components/sections/priorities have symetric behaviour
-
- # priorities[name] -> list of added priorities
- self.assertEqual(
- 1, len(self.binary_details.priorities['foo-dev']))
- # not correct value was set yet
- self.assertEqual(
- False, 'foo-dev' in self.binary_details.correct_priorities)
- # set correct values
- self.binary_details.setCorrectValues()
- # now we have the correct value in place
- self.assertEqual(
- 'REQUIRED', self.binary_details.correct_priorities['foo-dev'])
-
- def test_multi_binaries(self):
- """Multiple binaries inclusion."""
- values_map = [
- ('i386', 'REQUIRED'),
- ('amd64', 'REQUIRED'),
- ('powerpc', 'REQUIRED'),
- ('sparc', 'REQUIRED'),
- ('hppa', 'IMPORTANT'),
- ('ia64', 'IMPORTANT'),
- ]
- # add multiple binaries systematically according values_map
- for arch, priority in values_map:
- bin = PubBinaryContent('foo-dev', '1.0', arch, 'main',
- 'base', priority)
-
- self.binary_details.addBinaryDetails(bin)
-
- # expects 2 distinct priorities
- self.assertEqual(
- 2, len(self.binary_details.priorities['foo-dev']))
- # set correct values
- self.binary_details.setCorrectValues()
- # 'REQUIRED' is the most frequent priority in this group of binary
- self.assertEqual(
- 'REQUIRED', self.binary_details.correct_priorities['foo-dev'])
-
-
-class TestPubSourceChecker(TestCase):
-
- def setUp(self):
- """Initialize useful constant values."""
- self.name = 'foo'
- self.version = '1.0'
- self.component = 'main'
- self.section = 'python'
- self.urgency = 'URGENT'
- self.default_checker = PubSourceChecker(
- self.name, self.version, self.component,
- self.section, self.urgency)
-
- def test_initialization(self):
- """Check PubSourceChecker class initialization."""
- checker = self.default_checker
- self.assertEqual(self.name, checker.name)
- self.assertEqual(self.version, checker.version)
- self.assertEqual(self.component, checker.component)
- self.assertEqual(self.section, checker.section)
- self.assertEqual(self.urgency, checker.urgency)
- self.assertEqual(0, len(checker.binaries))
-
- def test_single_binary_ok(self):
- """Probe single correct binary addition."""
- checker = self.default_checker
-
- checker.addBinary('foo-dev', self.version, 'i386', self.component,
- self.section, 'REQUIRED')
-
- checker.check()
-
- self.assertEqual(None, checker.renderReport())
-
- # inspect PubBinaryDetails attributesm check if they are populated
- # correctly see TestPubBinaryDetails above.
- self.assertEqual(
- 1, len(checker.binaries_details.components['foo-dev']))
- self.assertEqual(
- 1, len(checker.binaries_details.sections['foo-dev']))
- self.assertEqual(
- 1, len(checker.binaries_details.priorities['foo-dev']))
- self.assertEqual(
- self.component,
- checker.binaries_details.correct_components['foo-dev'])
- self.assertEqual(
- self.section,
- checker.binaries_details.correct_sections['foo-dev'])
- self.assertEqual(
- 'REQUIRED',
- checker.binaries_details.correct_priorities['foo-dev'])
-
- def test_multi_binary_component_failure(self):
- """Probe multi binary with wrong component."""
- checker = self.default_checker
-
- checker.addBinary('foo-dev', self.version, 'i386', 'universe',
- self.section, 'REQUIRED')
- checker.addBinary('foo-dev', self.version, 'amd64', 'multiverse',
- self.section, 'REQUIRED')
-
- checker.check()
-
- self.assertEqual(
- "foo_1.0 main/python/URGENT | 2 bin\n\t"
- "foo-dev_1.0 amd64 multiverse/python/REQUIRED\n\t\t"
- "W: Component mismatch: multiverse != universe",
- checker.renderReport())
-
- def test_multi_binary_priority_failure(self):
- """Probe multiple binaries with priority conflict."""
- checker = self.default_checker
-
- checker.addBinary('foo-dev', self.version, 'i386', self.component,
- self.section, 'REQUIRED')
- checker.addBinary('foo-dbg', self.version, 'i386', self.component,
- self.section, 'EXTRA')
- checker.addBinary('foo-dev', self.version, 'amd64', self.component,
- self.section, 'EXTRA')
-
- checker.check()
-
- self.assertEqual(
- "foo_1.0 main/python/URGENT | 3 bin\n"
- "\tfoo-dev_1.0 amd64 main/python/EXTRA\n"
- "\t\tW: Priority mismatch: EXTRA != REQUIRED",
- checker.renderReport())
-
- def test_multi_binary_priority_success(self):
- """Probe multiple binaries with correct priorities.
-
- Following UNIX approach, no output is produce for correct input.
- """
- checker = self.default_checker
-
- checker.addBinary('foo-dev', self.version, 'i386', self.component,
- self.section, 'EXTRA')
- checker.addBinary('foo-dbg', self.version, 'i386', self.component,
- self.section, 'EXTRA')
- checker.addBinary('foo-dev', self.version, 'amd64', self.component,
- self.section, 'EXTRA')
-
- checker.check()
-
- self.assertEqual(None, checker.renderReport())
=== removed file 'scripts/ftpmaster-tools/archive-override-check.py'
--- scripts/ftpmaster-tools/archive-override-check.py 2012-01-19 03:09:38 +0000
+++ scripts/ftpmaster-tools/archive-override-check.py 1970-01-01 00:00:00 +0000
@@ -1,99 +0,0 @@
-#!/usr/bin/python -S
-#
-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Archive Override Check
-
-Given a distribution to run on, report any override inconsistence found.
-It basically check if all published source and binaries are coherent.
-"""
-
-import _pythonpath
-
-import transaction
-from zope.component import getUtility
-
-from lp.app.errors import NotFoundError
-from lp.registry.interfaces.distribution import IDistributionSet
-from lp.registry.interfaces.pocket import PackagePublishingPocket
-from lp.services.config import config
-from lp.services.scripts.base import LaunchpadScript
-from lp.soyuz.enums import PackagePublishingStatus
-from lp.soyuz.scripts.pubsourcechecker import PubSourceChecker
-
-
-class ArchiveOverrideCheckScript(LaunchpadScript):
-
- def add_my_options(self):
- self.parser.add_option(
- "-d", "--distribution", action="store",
- dest="distribution", metavar="DISTRO", default="ubuntu",
- help="Distribution to consider")
- self.parser.add_option(
- "-s", "--suite", action="store",
- dest="suite", metavar="SUITE", default=None,
- help=("Suite to consider, if not passed consider the "
- "currentseries and the RELEASE pocket"))
-
- def main(self):
- try:
- try:
- distribution = getUtility(IDistributionSet)[
- self.options.distribution]
- if self.options.suite is None:
- distroseries = distribution.currentseries
- pocket = PackagePublishingPocket.RELEASE
- else:
- distroseries, pocket = (
- distribution.getDistroSeriesAndPocket(
- self.options.suite))
-
- self.logger.debug(
- "Considering: %s/%s/%s/%s."
- % (distribution.name, distroseries.name, pocket.name,
- distroseries.status.name))
-
- checkOverrides(distroseries, pocket, self.logger)
- except NotFoundError, info:
- self.logger.error('Not found: %s' % info)
- finally:
- self.logger.debug("Rolling back any remaining transactions.")
- transaction.abort()
-
-
-def checkOverrides(distroseries, pocket, log):
- """Initialize and handle PubSourceChecker.
-
- Iterate over PUBLISHED sources and perform PubSourceChecker.check()
- on each published Source/Binaries couple.
- """
- spps = distroseries.getSourcePackagePublishing(
- status=PackagePublishingStatus.PUBLISHED,
- pocket=pocket)
-
- log.debug('%s published sources' % spps.count())
-
- for spp in spps:
- spr = spp.sourcepackagerelease
- checker = PubSourceChecker(
- spr.name, spr.version, spp.component.name, spp.section.name,
- spr.urgency.name)
-
- for bpp in spp.getPublishedBinaries():
- bpr = bpp.binarypackagerelease
- checker.addBinary(
- bpr.name, bpr.version, bpp.distroarchseries.architecturetag,
- bpp.component.name, bpp.section.name, bpr.priority.name)
-
- checker.check()
-
- report = checker.renderReport()
-
- if report:
- print report
-
-if __name__ == '__main__':
- script = ArchiveOverrideCheckScript(
- 'archive-override-check', config.archivepublisher.dbuser)
- script.lock_and_run()
