launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging lp:~cjwatson/launchpad/configurable-germinate-base into lp:launchpad.

Commit message:
Make germinate's seed base location configurable.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/configurable-germinate-base/+merge/194340

Allow us to configure the seed base URL used by germinate in generate-extra-overrides (by default "http://people.canonical.com/~ubuntu-archive/seeds/";) so that we can have it point to http://archive-team.internal/seeds/ instead in production.  This will avoid the publisher being reliant on two other machines rather than one for proper functioning, since http://people.canonical.com/~ubuntu-archive/... is now handled by a proxy.
-- 
https://code.launchpad.net/~cjwatson/launchpad/configurable-germinate-base/+merge/194340
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/configurable-germinate-base into lp:launchpad.

=== modified file 'configs/development/launchpad-lazr.conf'
--- configs/development/launchpad-lazr.conf	2013-04-09 11:51:24 +0000
+++ configs/development/launchpad-lazr.conf	2013-11-07 13:43:38 +0000
@@ -7,6 +7,7 @@
 
 [archivepublisher]
 run_parts_location: none
+germinate_seed_base: none
 
 [builddmaster]
 root: /var/tmp/builddmaster/

=== modified file 'lib/lp/archivepublisher/scripts/generate_extra_overrides.py'
--- lib/lp/archivepublisher/scripts/generate_extra_overrides.py	2012-11-10 02:21:31 +0000
+++ lib/lp/archivepublisher/scripts/generate_extra_overrides.py	2013-11-07 13:43:38 +0000
@@ -1,4 +1,4 @@
-# Copyright 2011-2012 Canonical Ltd.  This software is licensed under the
+# Copyright 2011-2013 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Generate extra overrides using Germinate."""
@@ -27,6 +27,7 @@
 from lp.archivepublisher.config import getPubConfig
 from lp.registry.interfaces.distribution import IDistributionSet
 from lp.registry.interfaces.series import SeriesStatus
+from lp.services.config import config
 from lp.services.database.policy import (
     DatabaseBlockedPolicy,
     SlaveOnlyDatabasePolicy,
@@ -155,7 +156,9 @@
                 for component in series.component_names
                 if component != "partner"]
 
-    def makeSeedStructures(self, series_name, flavours, seed_bases=None):
+    def makeSeedStructures(self, series_name, flavours):
+        seed_base = config.archivepublisher.germinate_seed_base
+        seed_bases = None if seed_base is None else [seed_base]
         structures = {}
         for flavour in flavours:
             try:
@@ -343,9 +346,8 @@
                 os.remove(output)
 
     def generateExtraOverrides(self, series_name, components, architectures,
-                               flavours, seed_bases=None):
-        structures = self.makeSeedStructures(
-            series_name, flavours, seed_bases=seed_bases)
+                               flavours):
+        structures = self.makeSeedStructures(series_name, flavours)
 
         if structures:
             seed_outputs = set()
@@ -359,7 +361,7 @@
                         flavours, structures, seed_outputs=seed_outputs)
             self.removeStaleOutputs(series_name, seed_outputs)
 
-    def process(self, seed_bases=None):
+    def process(self):
         """Do the bulk of the work."""
         self.setUp()
 
@@ -374,8 +376,7 @@
             self.txn.commit()
             with DatabaseBlockedPolicy():
                 self.generateExtraOverrides(
-                    series_name, components, architectures, self.args,
-                    seed_bases=seed_bases)
+                    series_name, components, architectures, self.args)
 
     def main(self):
         """See `LaunchpadScript`."""

=== modified file 'lib/lp/archivepublisher/tests/test_generate_extra_overrides.py'
--- lib/lp/archivepublisher/tests/test_generate_extra_overrides.py	2013-09-24 05:45:06 +0000
+++ lib/lp/archivepublisher/tests/test_generate_extra_overrides.py	2013-11-07 13:43:38 +0000
@@ -1,4 +1,4 @@
-# Copyright 2011-2012 Canonical Ltd.  This software is licensed under the
+# Copyright 2011-2013 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Test for the `generate-extra-overrides` script."""
@@ -10,6 +10,7 @@
 from optparse import OptionValueError
 import os
 import tempfile
+from textwrap import dedent
 
 from germinate import (
     archive,
@@ -29,6 +30,7 @@
 from lp.archivepublisher.utils import RepositoryIndexFile
 from lp.registry.interfaces.pocket import PackagePublishingPocket
 from lp.registry.interfaces.series import SeriesStatus
+from lp.services.config import config
 from lp.services.log.logger import DevNullLogger
 from lp.services.osutils import (
     ensure_directory_exists,
@@ -85,6 +87,10 @@
     def setUp(self):
         super(TestGenerateExtraOverrides, self).setUp()
         self.seeddir = self.makeTemporaryDirectory()
+        config.push("seed-base", dedent("""\
+            [archivepublisher]
+            germinate_seed_base: file://%s
+            """ % self.seeddir))
         # XXX cjwatson 2011-12-06 bug=694140: Make sure germinate doesn't
         # lose its loggers between tests, due to Launchpad's messing with
         # global log state.
@@ -322,8 +328,7 @@
         series_name = self.distroseries[0].name
         flavour = self.factory.getUniqueString()
 
-        structures = self.script.makeSeedStructures(
-            series_name, [flavour], seed_bases=["file://%s" % self.seeddir])
+        structures = self.script.makeSeedStructures(series_name, [flavour])
         self.assertEqual({}, structures)
 
     def test_make_seed_structures_empty_seed_structure(self):
@@ -333,8 +338,7 @@
         flavour = self.factory.getUniqueString()
         self.makeSeedStructure(flavour, series_name, [])
 
-        structures = self.script.makeSeedStructures(
-            series_name, [flavour], seed_bases=["file://%s" % self.seeddir])
+        structures = self.script.makeSeedStructures(series_name, [flavour])
         self.assertEqual({}, structures)
 
     def test_make_seed_structures_valid_seeds(self):
@@ -346,15 +350,12 @@
         self.makeSeedStructure(flavour, series_name, [seed])
         self.makeSeed(flavour, series_name, seed, [])
 
-        structures = self.script.makeSeedStructures(
-            series_name, [flavour], seed_bases=["file://%s" % self.seeddir])
+        structures = self.script.makeSeedStructures(series_name, [flavour])
         self.assertIn(flavour, structures)
 
     def fetchGerminatedOverrides(self, script, distroseries, arch, flavours):
         """Helper to call script.germinateArch and return overrides."""
-        structures = script.makeSeedStructures(
-            distroseries.name, flavours,
-            seed_bases=["file://%s" % self.seeddir])
+        structures = script.makeSeedStructures(distroseries.name, flavours)
 
         override_fd, override_path = tempfile.mkstemp()
         with os.fdopen(override_fd, "w") as override_file:
@@ -579,7 +580,7 @@
         self.makeSeedStructure(flavour, self.distroseries[1].name, [seed])
         self.makeSeed(flavour, self.distroseries[1].name, seed, [])
 
-        self.script.process(seed_bases=["file://%s" % self.seeddir])
+        self.script.process()
         self.assertFalse(os.path.exists(os.path.join(
             self.script.config.miscroot,
             "more-extra.override.%s.main" % self.distroseries[0].name)))
@@ -602,13 +603,13 @@
         seed_new = self.factory.getUniqueString()
         self.makeSeedStructure(flavour, series_name, [seed_old])
         self.makeSeed(flavour, series_name, seed_old, [])
-        self.script.process(seed_bases=["file://%s" % self.seeddir])
+        self.script.process()
         output = partial(
             self.script.composeOutputPath, flavour, series_name, arch)
         self.assertTrue(os.path.exists(output(seed_old)))
         self.makeSeedStructure(flavour, series_name, [seed_new])
         self.makeSeed(flavour, series_name, seed_new, [])
-        self.script.process(seed_bases=["file://%s" % self.seeddir])
+        self.script.process()
         self.assertTrue(os.path.exists(os.path.join(self.script.log_file)))
         self.assertTrue(os.path.exists(output("structure")))
         self.assertTrue(os.path.exists(output("all")))
@@ -652,7 +653,7 @@
             flavour, series_name, seed, [package.name],
             headers=["Task-Description: task"])
 
-        self.script.process(seed_bases=["file://%s" % self.seeddir])
+        self.script.process()
         override_path = os.path.join(
             self.script.config.miscroot,
             "more-extra.override.%s.main" % series_name)

=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf	2013-08-27 01:30:47 +0000
+++ lib/lp/services/config/schema-lazr.conf	2013-11-07 13:43:38 +0000
@@ -32,6 +32,10 @@
 # datatype: string
 run_parts_location: none
 
+# Base URL from which generate-extra-overrides fetches seeds using
+# germinate.  If "none", use germinate's built-in default.
+germinate_seed_base: none
+
 
 [auditor]
 host: localhost