launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #04328
[Merge] lp:~jtv/launchpad/clean-up-generate-contents into lp:launchpad
Jeroen T. Vermeulen has proposed merging lp:~jtv/launchpad/clean-up-generate-contents into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #812165 in Launchpad itself: "Retire archivepublisher.content_archive_root config item"
https://bugs.launchpad.net/launchpad/+bug/812165
For more details, see:
https://code.launchpad.net/~jtv/launchpad/clean-up-generate-contents/+merge/68689
We just started using the new, python-based generate-contents-files script. Once this has run everywhere it needs to, we can retire the migration code for legacy contents directories.
To test:
{{{
./bin/test -vvc lp.archivepublisher.tests.test_generate_contents_files
}}}
For Q/A, the generate-contents-files.py script should still run successfully.
The only lint is some over-long lines in schema-lazr.conf that I can't do anything about. I'm not sure it's worth checking at all, really.
Jeroen
--
https://code.launchpad.net/~jtv/launchpad/clean-up-generate-contents/+merge/68689
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~jtv/launchpad/clean-up-generate-contents into lp:launchpad.
=== removed file 'cronscripts/publishing/gen-contents/generate-contents'
--- cronscripts/publishing/gen-contents/generate-contents 2009-06-24 20:52:01 +0000
+++ cronscripts/publishing/gen-contents/generate-contents 1970-01-01 00:00:00 +0000
@@ -1,144 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# Archive Contents files generator.
-#
-# It uses scripts/ftpmaster-tools/lp-query-distro.py script to
-# collect up-to-date information about the Ubuntu distribution from
-# Launchpad, then sets up an auxiliary archive and a set of
-# configuration files to run apt-ftparchive generating Contents
-# files for all supported suites.
-#
-
-set -x
-set -e
-set -u
-
-# share lockfile with cron.daily
-LOCKFILE=/srv/launchpad.net/ubuntu-archive/cron.daily.lock
-
-# main archive path (target)
-ARCHIVE='/srv/launchpad.net/ubuntu-archive'
-
-# auxiliary archive where the contents will be generated (source)
-CONTENTARCHIVE='/srv/launchpad.net/ubuntu-contents'
-
-LPPATH='/srv/launchpad.net/codelines/current'
-LPCONTENT="$LPPATH/cronscripts/publishing/gen-contents"
-
-# apt-ftparchive configuration file destination
-APTCONF_HEADER_TEMPLATE="apt_conf_header.template"
-APTCONF_DIST_TEMPLATE="apt_conf_dist.template"
-APTCONF="apt-contents.conf"
-CONTENTHEADER="Contents.top"
-
-# target pocket suffixes
-pocket_suffixes=`$LPPATH/scripts/ftpmaster-tools/lp-query-distro.py pocket_suffixes`
-
-# target suites ()
-suites=`$LPPATH/scripts/ftpmaster-tools/lp-query-distro.py supported`
-
-# Combine all supported series with all pocket suffixes to build the list
-# of suites for which we will generate/update Contents.gz.
-new_suites=
-for suite in $suites; do
- new_suites="${new_suites:+$new_suites }$suite"
- for pocket_suffix in $pocket_suffixes; do
- if [ -d "$ARCHIVE/ubuntu/dists/$suite$pocket_suffix" ]; then
- new_suites="$new_suites $suite$pocket_suffix"
- fi
- done
-done
-suites="$new_suites"
-
-# target architectures for the development series
-devel=`$LPPATH/scripts/ftpmaster-tools/lp-query-distro.py development`
-archs=`$LPPATH/scripts/ftpmaster-tools/lp-query-distro.py -s $devel archs`
-
-# target components
-components='main restricted universe multiverse'
-
-# directories to be pre-created when setting up a fresh tree
-dirs='source debian-installer'
-for i in $archs; do dirs="$dirs binary-$i"; done
-
-GENERATEFILES="yes"
-COMPAREFILES="yes"
-MOVEFILES="yes"
-
-set -e
-
-# claim the lock
-if ! lockfile -r1 $LOCKFILE; then
- echo "Could not claim lock file."
- exit 1
-fi
-
-cleanup () {
- echo "Cleaning up lockfile."
- rm -f $LOCKFILE
-}
-
-trap cleanup EXIT
-
-echo 'Ensure we have a private tree in place ...'
-mkdir -p $CONTENTARCHIVE/ubuntu-misc
-mkdir -p $CONTENTARCHIVE/ubuntu-cache
-
-cp $LPCONTENT/$APTCONF_HEADER_TEMPLATE $CONTENTARCHIVE/ubuntu-misc/$APTCONF
-for suite in $suites; do
- sed -e "s/\$SUITE/$suite/g" -e "s/\$ARCHS/$archs/g" \
- $LPCONTENT/$APTCONF_DIST_TEMPLATE >> $CONTENTARCHIVE/ubuntu-misc/$APTCONF
- for component in $components; do
- for dir in $dirs; do
- path="$CONTENTARCHIVE/ubuntu/dists/$suite/$component/$dir"
- if test ! -d $path; then
- echo " $path"
- mkdir -p $path
- fi
- done
- done
-done
-
-if test $GENERATEFILES = "yes" ; then
- echo 'Running apt in your private tree to generate new contents ...'
- cp -a $ARCHIVE/ubuntu-overrides $CONTENTARCHIVE/
- cp $LPCONTENT/$CONTENTHEADER $CONTENTARCHIVE/ubuntu-misc/
- apt-ftparchive generate $CONTENTARCHIVE/ubuntu-misc/$APTCONF
-else
- echo "Skipping apt-ftparchive run"
-fi
-
-
-if test $COMPAREFILES = "yes" ; then
- echo 'Comparing contents files with public tree ...'
- for suite in $suites; do
- for arch in $archs; do
- last_contents="$CONTENTARCHIVE/ubuntu/dists/$suite/.Contents-$arch"
- current_contents="$CONTENTARCHIVE/ubuntu/dists/$suite/Contents-$arch"
- new_contents="$CONTENTARCHIVE/ubuntu/dists/$suite/Contents-$arch.gz"
- contents_dest="$ARCHIVE/ubuntu/dists/$suite/Contents-$arch.gz"
- # we don't want to overwrite existing files because they are
- # precious and mirrors would need to refetch them.
- if ! cmp -s $current_contents $last_contents; then
- if test $MOVEFILES = "yes"; then
- echo " installing new Contents file for $suite/$arch"
- # update cached plain content
- mv "$current_contents" "$last_contents"
- # update real gzipped Content
- mv "$new_contents" "$contents_dest"
- chmod ug=rw,o=r "$contents_dest"
- else
- echo " not Installing updated Contents file for $suite/$arch"
- fi
- else
- echo " skipping unmodified Contents file for $suite/$arch"
- fi
- done
- done
-else
- echo 'Skipping update of contents files in public tree ...'
-fi
-
=== modified file 'lib/canonical/config/schema-lazr.conf'
--- lib/canonical/config/schema-lazr.conf 2011-07-15 15:46:51 +0000
+++ lib/canonical/config/schema-lazr.conf 2011-07-21 13:17:58 +0000
@@ -24,17 +24,6 @@
# datatype: string
dbuser: archivepublisher
-# Base directory for auxiliary archive where Contents files will be
-# generated.
-#
-# Subdirectories inside this directory will be named after the
-# distribution. For example, if content_archive_root is set to
-# /srv/launchpad.net then Ubuntu's Contents files will be created in
-# /srv/launchpad.net/ubuntu-contents
-#
-# datatype: string
-content_archive_root: /var/tmp/archive
-
# Location where the run-parts directories for publish-ftpmaster
# customization are to be found. Absolute path, or path relative to the
# Launchpad source tree, or "none" to skip execution of run-parts.
=== modified file 'lib/lp/archivepublisher/scripts/generate_contents_files.py'
--- lib/lp/archivepublisher/scripts/generate_contents_files.py 2011-07-12 16:50:53 +0000
+++ lib/lp/archivepublisher/scripts/generate_contents_files.py 2011-07-21 13:17:58 +0000
@@ -292,40 +292,6 @@
for arch in archs:
self.updateContentsFile(suite, arch)
- def updateLegacyContentArchiveRoot(self):
- """Replace content archive root with new contents location.
-
- If there is a legacy content_archive_root, move it to the new
- location.
-
- This is a temporary migration tool for the fix to bug 809211.
- After this has run at least once on each system that needs it, for
- each distribution archive, this method can go away.
- """
- content_archive_root = getattr(
- config.archivepublisher, 'content_archive_root', None)
- if content_archive_root is None:
- # No legacy configuration; nothing to do.
- return
- old_content_dir = os.path.join(
- content_archive_root, self.distribution.name + "-contents")
- if not file_exists(old_content_dir):
- # The data has already been moved (or didn't exist at all).
- return
- if file_exists(self.content_archive):
- # The new contents directory has already been created.
- # Confusing!
- self.logger.warn(
- "New contents directory %s has been created, "
- "but legacy directory %s still exists. Ignoring the latter.",
- self.content_archive, old_content_dir)
- return
-
- self.logger.info(
- "Moving legacy contents directory %s into %s.",
- old_content_dir, self.content_archive)
- os.rename(old_content_dir, self.content_archive)
-
def setUp(self):
"""Prepare configuration and filesystem state for the script's work.
@@ -337,7 +303,6 @@
self.config = getPubConfig(self.distribution.main_archive)
self.content_archive = os.path.join(
self.config.distroroot, "contents-generation")
- self.updateLegacyContentArchiveRoot()
self.setUpContentArchive()
def main(self):
=== modified file 'lib/lp/archivepublisher/tests/test_generate_contents_files.py'
--- lib/lp/archivepublisher/tests/test_generate_contents_files.py 2011-07-13 16:33:31 +0000
+++ lib/lp/archivepublisher/tests/test_generate_contents_files.py 2011-07-21 13:17:58 +0000
@@ -8,9 +8,7 @@
from optparse import OptionValueError
import os
from testtools.matchers import StartsWith
-from textwrap import dedent
-from canonical.config import config
from canonical.testing.layers import (
LaunchpadZopelessLayer,
ZopelessDatabaseLayer,
@@ -128,16 +126,6 @@
layer = LaunchpadZopelessLayer
- def makeLegacyContentArchive(self):
- """Prepare a legacy "content archive" directory."""
- content_archive = self.makeTemporaryDirectory()
- config.push("content-archive", dedent("""\
- [archivepublisher]
- content_archive_root: %s
- """ % content_archive))
- self.addCleanup(config.pop, "content-archive")
- return content_archive
-
def makeDistro(self):
"""Create a distribution for testing.
@@ -298,59 +286,6 @@
self.assertIn("This file maps", contents_top)
self.assertIn(distro.title, contents_top)
- def test_updateLegacyContentArchiveRoot_moves_legacy_contents(self):
- # If updateLegacyContentArchiveRoot finds entries in the legacy
- # content-archive directory, it moves them into their new
- # location inside distroroot.
- distro = self.makeDistro()
- script = self.makeScript(distro, run_setup=False)
- content_archive = self.makeLegacyContentArchive()
- script.content_archive = os.path.join(
- self.makeTemporaryDirectory(), "contents-generation")
- old_contents_dir = os.path.join(
- content_archive, "%s-contents" % distro.name)
-
- marker_contents = self.writeMarkerFile(os.path.join(
- old_contents_dir, "%s-misc" % distro.name, "Contents.top"))
-
- script.updateLegacyContentArchiveRoot()
-
- self.assertFalse(file_exists(old_contents_dir))
- new_path = os.path.join(
- script.content_archive, "%s-misc" % distro.name, "Contents.top")
- self.assertEqual(marker_contents, file(new_path).read())
-
- def test_updateLegacyContentArchiveRoot_is_harmless_without_config(self):
- # If no legacy content-archive root directory is configured,
- # that's fine. It means that updateLegacyContentArchiveRoot has
- # nothing to do.
- script = self.makeScript()
- script.updateLegacyContentArchiveRoot()
- self.assertTrue(file_exists(script.content_archive))
- self.assertThat(
- script.content_archive, StartsWith(script.config.distroroot))
-
- def test_updateLegacyContentArchiveRoot_is_harmless_without_legacy(self):
- # If a legacy content-archive root directory is configured but
- # it does not actually exist, updateLegacyContentArchiveRoot
- # does nothing.
- self.makeLegacyContentArchive()
- script = self.makeScript()
- script.updateLegacyContentArchiveRoot()
- self.assertTrue(file_exists(script.content_archive))
-
- def test_setUp_moves_legacy_content_archive(self):
- # setUp calls updateLegacyContentArchiveRoot, which moves any
- # legacy content-archive contents to their new location.
- content_archive = self.makeLegacyContentArchive()
- distro = self.makeDistro()
- marker_dir = os.path.join(
- content_archive, "%s-contents" % distro.name,
- "%s-misc" % distro.name)
- self.writeMarkerFile(os.path.join(marker_dir, "Contents.top"))
- self.makeScript(distro, run_setup=True)
- self.assertFalse(file_exists(marker_dir))
-
def test_setUp_places_content_archive_in_distroroot(self):
# The contents files are kept in subdirectories of distroroot.
script = self.makeScript()
