launchpad-reviewers team mailing list archive

[William Grant <me@xxxxxxxxxxxxxxxxxx>]

William Grant has proposed merging lp:~wgrant/launchpad/gina-skippage into lp:launchpad.

Commit message:
Add a soyuz.gina.skip_source_versions feature flag to ask gina to skip certain source versions, eg. if they're crashing it.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~wgrant/launchpad/gina-skippage/+merge/165048

Add a soyuz.gina.skip_source_versions feature flag to ask gina to skip particular source versions by distribution. This is mostly to let us skip packages that currently crash gina, eg. gdb 7.6-1.

For code simplicity and safety the key is LP_DISTRIBUTION/SOURCE_NAME/SOURCE_VERSION.
-- 
https://code.launchpad.net/~wgrant/launchpad/gina-skippage/+merge/165048
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/gina-skippage into lp:launchpad.

=== modified file 'lib/lp/services/features/flags.py'
--- lib/lp/services/features/flags.py	2013-04-08 04:22:35 +0000
+++ lib/lp/services/features/flags.py	2013-05-22 09:59:41 +0000
@@ -190,6 +190,13 @@
      '',
      '',
      ''),
+    ('soyuz.gina.skip_source_versions',
+     'space delimited',
+     ('List of source versions for gina to skip when importing into a '
+      'distribution, formatted as distro/package/version.'),
+     '',
+     '',
+     ''),
     ('auditor.enabled',
      'boolean',
      'If true, send audit data to an auditor instance.',

=== modified file 'lib/lp/soyuz/scripts/gina/runner.py'
--- lib/lp/soyuz/scripts/gina/runner.py	2013-05-22 09:51:08 +0000
+++ lib/lp/soyuz/scripts/gina/runner.py	2013-05-22 09:59:41 +0000
@@ -13,6 +13,7 @@
 from zope.component import getUtility
 
 from lp.services.config import config
+from lp.services.features import getFeatureFlag
 from lp.services.scripts import log
 from lp.soyuz.interfaces.component import IComponentSet
 from lp.soyuz.scripts.gina import ExecutionError
@@ -37,9 +38,6 @@
     SourcePackageData,
     )
 
-# Set to non-zero if you'd like to be warned every so often
-COUNTDOWN = 0
-
 
 def run_gina(options, ztm, target_section):
     # Avoid circular imports.
@@ -167,12 +165,17 @@
             count += 1
             attempt_source_package_import(
                 distro, source, package_root, importer_handler)
-            if COUNTDOWN and (count % COUNTDOWN == 0):
-                log.warn('%i/%i sourcepackages processed', count, npacks)
 
 
 def do_one_sourcepackage(distro, source, package_root, importer_handler):
     source_data = SourcePackageData(**source)
+    skip_key = u'%s/%s/%s' % (distro, source_data.package, source_data.version)
+    skip_list = getFeatureFlag('soyuz.gina.skip_source_versions')
+    if skip_list is not None and skip_key in skip_list.split():
+        log.info(
+            "Skipping %s %s as requested by feature flag.",
+            source_data.package, source_data.version)
+        return
     if importer_handler.preimport_sourcecheck(source_data):
         # Don't bother reading package information if the source package
         # already exists in the database
@@ -237,10 +240,6 @@
                 nosource.append(binary)
                 continue
 
-            if COUNTDOWN and count % COUNTDOWN == 0:
-                # XXX kiko 2005-10-23: untested
-                log.warn('%i/%i binary packages processed', count, npacks)
-
         if nosource:
             # XXX kiko 2005-10-23: untested
             log.warn('%i source packages not found', len(nosource))

=== modified file 'lib/lp/soyuz/scripts/tests/test_gina.py'
--- lib/lp/soyuz/scripts/tests/test_gina.py	2012-06-20 02:04:01 +0000
+++ lib/lp/soyuz/scripts/tests/test_gina.py	2013-05-22 09:59:41 +0000
@@ -7,25 +7,37 @@
 from textwrap import dedent
 from unittest import TestLoader
 
+import transaction
+
 from lp.archiveuploader.tagfiles import parse_tagfile
 from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.services.features.testing import FeatureFixture
 from lp.services.log.logger import DevNullLogger
 from lp.services.tarfile_helpers import LaunchpadWriteTarFile
 from lp.soyuz.enums import PackagePublishingStatus
 from lp.soyuz.scripts.gina import ExecutionError
 from lp.soyuz.scripts.gina.dominate import dominate_imported_source_packages
+from lp.soyuz.scripts.gina.archive import (
+    ArchiveComponentItems,
+    PackagesMap,
+    )
 import lp.soyuz.scripts.gina.handlers
 from lp.soyuz.scripts.gina.handlers import (
     BinaryPackagePublisher,
+    ImporterHandler,
     SourcePackagePublisher,
     )
 from lp.soyuz.scripts.gina.packages import (
     BinaryPackageData,
     SourcePackageData,
     )
+from lp.soyuz.scripts.gina.runner import import_sourcepackages
 from lp.testing import TestCaseWithFactory
 from lp.testing.faketransaction import FakeTransaction
-from lp.testing.layers import ZopelessDatabaseLayer
+from lp.testing.layers import (
+    LaunchpadZopelessLayer,
+    ZopelessDatabaseLayer,
+    )
 
 
 class FakePackagesMap:
@@ -231,6 +243,50 @@
         self.assertEqual(PackagePublishingStatus.PUBLISHED, bpph.status)
 
 
+class TestRunner(TestCaseWithFactory):
+
+    layer = LaunchpadZopelessLayer
+
+    def test_import_sourcepackages_skip(self):
+        # gina can be told to skip particular source versions by setting
+        # soyuz.gina.skip_source_versions to a space-separated list of
+        # $DISTRO/$NAME/$VERSION.
+        series = self.factory.makeDistroSeries()
+
+        archive_root = os.path.join(
+            os.path.dirname(__file__), 'gina_test_archive')
+        arch_component_items = ArchiveComponentItems(
+            archive_root, 'lenny', ['main'], [], True)
+        packages_map = PackagesMap(arch_component_items)
+        importer_handler = ImporterHandler(
+            transaction, series.distribution.name, series.name, archive_root,
+            PackagePublishingPocket.RELEASE, None)
+
+        # Our test archive has archive-copier 0.1.5 and 0.3.6 With
+        # soyuz.gina.skip_source_versions set to
+        # '$distro/archive-copier/0.1.5', an import will grab only
+        # 0.3.6.
+
+        skiplist = '%s/archive-copier/0.1.5' % series.distribution.name
+        with FeatureFixture({'soyuz.gina.skip_source_versions': skiplist}):
+            import_sourcepackages(
+                series.distribution.name, packages_map, archive_root,
+                importer_handler)
+            self.assertContentEqual(
+                ['0.3.6'],
+                [p.source_package_version
+                for p in series.getPublishedSources('archive-copier')])
+
+        # Importing again with the feature flag removed grabs both.
+        import_sourcepackages(
+            series.distribution.name, packages_map, archive_root,
+            importer_handler)
+        self.assertContentEqual(
+            ['0.1.5', '0.3.6'],
+            [p.source_package_version
+             for p in series.getPublishedSources('archive-copier')])
+
+
 def test_suite():
     suite = TestLoader().loadTestsFromName(__name__)
     suite.addTest(DocTestSuite(lp.soyuz.scripts.gina.handlers))