launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #16859
[Merge] lp:~cjwatson/launchpad/remove-germinate-scripts into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/remove-germinate-scripts into lp:launchpad.
Commit message:
Remove all germinate-related code, now moved into lp:ubuntu-archive-publishing.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/remove-germinate-scripts/+merge/222478
https://code.launchpad.net/~cjwatson/ubuntu-archive-publishing/add-germinate-scripts/+merge/222477 adds the germinate-related code from Launchpad to ubuntu-archive-publishing. Once this has been deployed, there is no further reason to keep that code in Launchpad, and so we can remove it.
The only non-obvious thing to note here is that cron.germinate is the last remaining user of lp-query-distro.py. It can probably be converted to get the same information from launchpadlib without too much difficulty, but I didn't do that in the initial port. We need to keep lp-query-distro.py (another 386 LoC) in Launchpad until that's been done.
--
https://code.launchpad.net/~cjwatson/launchpad/remove-germinate-scripts/+merge/222478
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/remove-germinate-scripts into lp:launchpad.
=== removed file 'cronscripts/generate-extra-overrides.py'
--- cronscripts/generate-extra-overrides.py 2011-12-04 15:14:23 +0000
+++ cronscripts/generate-extra-overrides.py 1970-01-01 00:00:00 +0000
@@ -1,18 +0,0 @@
-#!/usr/bin/python -S
-#
-# Copyright 2011 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Generate extra overrides using Germinate."""
-
-import _pythonpath
-
-from lp.archivepublisher.scripts.generate_extra_overrides import (
- GenerateExtraOverrides,
- )
-
-
-if __name__ == '__main__':
- script = GenerateExtraOverrides(
- "generate-extra-overrides", dbuser='generate_extra_overrides')
- script.lock_and_run()
=== removed file 'cronscripts/publishing/cron.germinate'
--- cronscripts/publishing/cron.germinate 2014-05-20 11:59:54 +0000
+++ cronscripts/publishing/cron.germinate 1970-01-01 00:00:00 +0000
@@ -1,61 +0,0 @@
-#! /bin/sh
-#
-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-set -e
-set -u
-
-ARCHIVEROOT=${TEST_ARCHIVEROOT:-/srv/launchpad.net/ubuntu-archive/ubuntu}
-MISCROOT=$ARCHIVEROOT/../ubuntu-misc
-LOCKROOT=$ARCHIVEROOT/..
-GERMINATEROOT=$ARCHIVEROOT/../ubuntu-germinate
-
-LAUNCHPADROOT=${TEST_LAUNCHPADROOT:-`dirname $0`/../..}
-GENERATE=$LAUNCHPADROOT/cronscripts/generate-extra-overrides.py
-MAINTAINCE_CHECK=$LAUNCHPADROOT/cronscripts/publishing/maintenance-check.py
-
-FLAVOURS="ubuntu kubuntu kubuntu-active edubuntu xubuntu mythbuntu lubuntu"
-FLAVOURS="$FLAVOURS ubuntustudio ubuntu-gnome ubuntu-touch"
-
-## Check to see if another germinate run is in progress
-
-LOCKFILE=$LOCKROOT/cron.germinate.lock
-if lockfile -! -l 43200 -r 0 "${LOCKFILE}"; then
- echo Another cron.germinate appears to be running
- exit 1
-fi
-
-cleanup () {
- rm -f "$LOCKFILE"
-}
-
-trap cleanup EXIT
-
-cd $GERMINATEROOT
-
-$GENERATE -d ubuntu $FLAVOURS
-
-# Now generate the Supported extra overrides for all supported distros.
-SUITES=`$LAUNCHPADROOT/scripts/ftpmaster-tools/lp-query-distro.py supported`
-for supported_suite in $SUITES; do
- echo -n "Running maintenance-check for $supported_suite... "
- # The support timeframe information is stored here
- SUPPORTED="$MISCROOT/more-extra.override.$supported_suite.main.supported"
- # This is the target override file that contains germinate plus
- # support info.
- TARGET="$MISCROOT/more-extra.override.$supported_suite.main"
- # Debug/Log information
- LOG="_maintenance-check.$supported_suite.stderr"
- if $MAINTAINCE_CHECK $supported_suite > $SUPPORTED 2> $LOG; then
- # The target file may be missing on the server and the script should
- # not fail if that is the case.
- touch $TARGET
- # Remove old "Supported" info from extra-overrides as it may be
- # stale now and we replace it with fresh content below.
- sed /"^.* Supported"/d $TARGET > ${TARGET}.new
- cat $SUPPORTED >> ${TARGET}.new
- mv ${TARGET}.new $TARGET
- fi
- echo " done"
-done
=== removed file 'cronscripts/publishing/maintenance-check.py'
--- cronscripts/publishing/maintenance-check.py 2014-03-18 17:30:51 +0000
+++ cronscripts/publishing/maintenance-check.py 1970-01-01 00:00:00 +0000
@@ -1,529 +0,0 @@
-#!/usr/bin/python
-#
-# python port of the nice maintainace-check script by Nick Barcet
-#
-
-import logging
-from optparse import OptionParser
-import os
-import sys
-import urllib2
-import urlparse
-
-import apt
-import apt_pkg
-
-
-class UbuntuMaintenance(object):
- """ Represents the support timeframe for a regular ubuntu release """
-
- # architectures that are full supported (including LTS time)
- PRIMARY_ARCHES = [
- "i386",
- "amd64",
- ]
-
- # architectures we support (but not for LTS time)
- SUPPORTED_ARCHES = PRIMARY_ARCHES + [
- "armel",
- "armhf",
- "arm64",
- "ppc64el",
- ]
-
- # what defines the seeds is documented in wiki.ubuntu.com/SeedManagement
- SERVER_SEEDS = [
- "server-ship",
- "supported-server",
- ]
- DESKTOP_SEEDS = [
- "ship",
- "supported-desktop",
- "supported-desktop-extra",
- ]
- SUPPORTED_SEEDS = ["all"]
-
- # normal support timeframe
- # time, seeds
- SUPPORT_TIMEFRAME = [
- ("9m", SUPPORTED_SEEDS),
- ]
-
- # shorter than normal support timeframe
- # time, seed
- SUPPORT_TIMEFRAME_SHORT = [
- ]
-
- # distro names that we check the seeds for
- DISTRO_NAMES = [
- "ubuntu",
- ]
-
- # distro names for shorter than default support cycles
- DISTRO_NAMES_SHORT = [
- ]
-
-
-# This is fun! We have a bunch of cases for 10.04 LTS
-#
-# - distro "ubuntu" follows SUPPORT_TIMEFRAME_LTS but only for
-# amd64/i386
-# - distros "kubuntu", "edubuntu" and "netbook" need to be
-# considered *but* only follow SUPPORT_TIMEFRAME
-# - anything that is in armel follows SUPPORT_TIMEFRAME
-#
-class LucidUbuntuMaintenance(UbuntuMaintenance):
- """ Represents the support timeframe for a 10.04 (lucid) LTS release,
- the exact rules differ from LTS release to LTS release
- """
-
- # lts support timeframe, order is important, least supported must be last
- # time, seeds
- SUPPORT_TIMEFRAME = [
- ("5y", UbuntuMaintenance.SERVER_SEEDS),
- ("3y", UbuntuMaintenance.DESKTOP_SEEDS),
- ("18m", UbuntuMaintenance.SUPPORTED_SEEDS),
- ]
-
- # on a LTS this is significant, it defines what names get LTS support
- DISTRO_NAMES = [
- "ubuntu",
- "kubuntu",
- ]
-
-
-class PreciseUbuntuMaintenance(UbuntuMaintenance):
- """ The support timeframe for the 12.04 (precise) LTS release.
- This changes the timeframe for desktop packages from 3y to 5y
- """
-
- # lts support timeframe, order is important, least supported must be last
- # time, seeds
- SUPPORT_TIMEFRAME = [
- ("5y", UbuntuMaintenance.SERVER_SEEDS),
- ("5y", UbuntuMaintenance.DESKTOP_SEEDS),
- ("18m", UbuntuMaintenance.SUPPORTED_SEEDS),
- ]
-
- # on a LTS this is significant, it defines what names get LTS support
- DISTRO_NAMES = [
- "ubuntu",
- "kubuntu",
- ]
-
-
-class QuantalUbuntuMaintenance(UbuntuMaintenance):
-
- SUPPORT_TIMEFRAME = [
- ("18m", UbuntuMaintenance.SUPPORTED_SEEDS),
- ]
-
-
-OneiricUbuntuMaintenance = QuantalUbuntuMaintenance
-
-
-class TrustyUbuntuMaintenance(UbuntuMaintenance):
- """ The support timeframe for the 14.04 (trusty) LTS release.
- This changes the timeframe for desktop packages from 3y to 5y
- """
-
- # lts support timeframe, order is important, least supported must be last
- # time, seeds
- SUPPORT_TIMEFRAME = [
- ("5y", UbuntuMaintenance.SERVER_SEEDS),
- ("5y", UbuntuMaintenance.DESKTOP_SEEDS),
- ("9m", UbuntuMaintenance.SUPPORTED_SEEDS),
- ]
-
- SUPPORT_TIMEFRAME_SHORT = [
- ("3y", UbuntuMaintenance.SERVER_SEEDS),
- ("3y", UbuntuMaintenance.DESKTOP_SEEDS),
- ("9m", UbuntuMaintenance.SUPPORTED_SEEDS),
- ]
-
- # on a LTS this is significant, it defines what names get LTS support
- #
- # Kylin is not in this list, becuase it's not seed managed =/,
- # it is LTS however as per 2014-03-17 Ubuntu TechBoard meeting
- DISTRO_NAMES = [
- "ubuntu",
- "kubuntu",
- "edubuntu",
- ]
-
- DISTRO_NAMES_SHORT = [
- "ubuntu-gnome",
- "xubuntu",
- "mythbuntu",
- "ubuntustudio",
- "lubuntu",
- ]
-
-
-# Names of the distribution releases that are not supported by this
-# tool. All later versions are supported.
-UNSUPPORTED_DISTRO_RELEASED = [
- "dapper",
- "edgy",
- "feisty",
- "gutsy",
- "hardy",
- "intrepid",
- "jaunty",
- "karmic",
- ]
-
-
-# germinate output base directory
-BASE_URL = os.environ.get(
- "MAINTENANCE_CHECK_BASE_URL",
- "http://people.canonical.com/~ubuntu-archive/germinate-output/";)
-
-# hints dir url, hints file is "$distro.hints" by default
-# (e.g. lucid.hints)
-HINTS_DIR_URL = os.environ.get(
- "MAINTENANCE_CHECK_HINTS_DIR_URL",
- "http://people.canonical.com/";
- "~ubuntu-archive/seeds/platform.%s/SUPPORTED_HINTS")
-
-# we need the archive root to parse the Sources file to support
-# by-source hints
-ARCHIVE_ROOT = os.environ.get(
- "MAINTENANCE_CHECK_ARCHIVE_ROOT", "http://archive.ubuntu.com/ubuntu";)
-
-# support timeframe tag used in the Packages file
-SUPPORT_TAG = "Supported"
-
-
-def get_binaries_for_source_pkg(srcname):
- """ Return all binary package names for the given source package name.
-
- :param srcname: The source package name.
- :return: A list of binary package names.
- """
- pkgnames = set()
- recs = apt_pkg.SourceRecords()
- while recs.lookup(srcname):
- for binary in recs.binaries:
- pkgnames.add(binary)
- return pkgnames
-
-
-def expand_src_pkgname(pkgname):
- """ Expand a package name if it is prefixed with src.
-
- If the package name is prefixed with src it will be expanded
- to a list of binary package names. Otherwise the original
- package name will be returned.
-
- :param pkgname: The package name (that may include src:prefix).
- :return: A list of binary package names (the list may be one element
- long).
- """
- if not pkgname.startswith("src:"):
- return [pkgname]
- return get_binaries_for_source_pkg(pkgname.split("src:")[1])
-
-
-def create_and_update_deb_src_source_list(distroseries):
- """ Create sources.list and update cache.
-
- This creates a sources.list file with deb-src entries for a given
- distroseries and apt.Cache.update() to make sure the data is up-to-date.
- :param distro: The code name of the distribution series (e.g. lucid).
- :return: None
- :raises: IOError: When cache update fails.
- """
- # apt root dir
- rootdir = "./aptroot.%s" % distroseries
- sources_list_dir = os.path.join(rootdir, "etc", "apt")
- if not os.path.exists(sources_list_dir):
- os.makedirs(sources_list_dir)
- sources_list = open(os.path.join(sources_list_dir, "sources.list"), "w")
- for pocket in [
- "%s" % distroseries,
- "%s-updates" % distroseries,
- "%s-security" % distroseries]:
- sources_list.write(
- "deb-src %s %s main restricted\n" % (
- ARCHIVE_ROOT, pocket))
- sources_list.write(
- "deb %s %s main restricted\n" % (
- ARCHIVE_ROOT, pocket))
- sources_list.close()
- # create required dirs/files for apt.Cache(rootdir) to work on older
- # versions of python-apt. once lucid is used it can be removed
- for d in ["var/lib/dpkg",
- "var/cache/apt/archives/partial",
- "var/lib/apt/lists/partial"]:
- if not os.path.exists(os.path.join(rootdir, d)):
- os.makedirs(os.path.join(rootdir, d))
- if not os.path.exists(os.path.join(rootdir, "var/lib/dpkg/status")):
- open(os.path.join(rootdir, "var/lib/dpkg/status"), "w")
- # open cache with our just prepared rootdir
- cache = apt.Cache(rootdir=rootdir)
- try:
- cache.update()
- except SystemError:
- logging.exception("cache.update() failed")
-
-
-def get_structure(distroname, version):
- """ Get structure file conent for named distro and distro version.
-
- :param name: Name of the distribution (e.g. kubuntu, ubuntu, xubuntu).
- :param version: Code name of the distribution version (e.g. lucid).
- :return: List of strings with the structure file content
- """
- f = urllib2.urlopen("%s/%s.%s/structure" % (
- BASE_URL, distroname, version))
- structure = f.readlines()
- f.close()
- return structure
-
-
-def expand_seeds(structure, seedname):
- """Expand seed by its dependencies using the strucure file.
-
- :param structure: The content of the STRUCTURE file as string list.
- :param seedname: The name of the seed as string that needs to be expanded.
- :return: A set() for the seed dependencies (excluding the original
- seedname).
- """
- seeds = []
- for line in structure:
- if line.startswith("%s:" % seedname):
- seeds += line.split(":")[1].split()
- for seed in seeds:
- seeds += expand_seeds(structure, seed)
- return set(seeds)
-
-
-def get_packages_for_seeds(name, distro, seeds):
- """
- Get packages for the given name (e.g. ubuntu) and distro release
- (e.g. lucid) that are in the given list of seeds
- returns a set() of package names.
- """
- pkgs_in_seeds = {}
- for seed in seeds:
- pkgs_in_seeds[seed] = set()
- seedurl = "%s/%s.%s/%s" % (BASE_URL, name, distro, seed)
- logging.debug("looking for '%s'", seedurl)
- try:
- f = urllib2.urlopen(seedurl)
- for line in f:
- # Ignore lines that are not package names (headers etc).
- if line[0] < 'a' or line[0] > 'z':
- continue
- # Each line contains these fields:
- # (package, source, why, maintainer, size, inst-size)
- if options.source_packages:
- pkgname = line.split("|")[1]
- else:
- pkgname = line.split("|")[0]
- pkgs_in_seeds[seed].add(pkgname.strip())
- f.close()
- except Exception as e:
- logging.error("seed %s failed (%s)" % (seedurl, e))
- return pkgs_in_seeds
-
-
-def what_seeds(pkgname, seeds):
- in_seeds = set()
- for s in seeds:
- if pkgname in seeds[s]:
- in_seeds.add(s)
- return in_seeds
-
-
-def compare_support_level(x, y):
- """
- compare two support level strings of the form 18m, 3y etc
- :parm x: the first support level
- :parm y: the second support level
- :return: negative if x < y, zero if x==y, positive if x > y
- """
-
- def support_to_int(support_time):
- """
- helper that takes a support time string and converts it to
- a integer for cmp()
- """
- # allow strings like "5y (kubuntu-common)
- x = support_time.split()[0]
- if x.endswith("y"):
- return 12 * int(x[0:-1])
- elif x.endswith("m"):
- return int(x[0:-1])
- else:
- raise ValueError("support time '%s' has to end with y or m" % x)
- return cmp(support_to_int(x), support_to_int(y))
-
-
-def get_packages_support_time(structure, name, pkg_support_time,
- support_timeframe_list):
- """
- input a structure file and a list of pair<timeframe, seedlist>
- return a dict of pkgnames -> support timeframe string
- """
- for (timeframe, seedlist) in support_timeframe_list:
- expanded = set()
- for s in seedlist:
- expanded.add(s)
- expanded |= expand_seeds(structure, s)
- pkgs_in_seeds = get_packages_for_seeds(name, distro, expanded)
- for seed in pkgs_in_seeds:
- for pkg in pkgs_in_seeds[seed]:
- if not pkg in pkg_support_time:
- pkg_support_time[pkg] = timeframe
- else:
- old_timeframe = pkg_support_time[pkg]
- if compare_support_level(old_timeframe, timeframe) < 0:
- logging.debug("overwriting %s from %s to %s" % (
- pkg, old_timeframe, timeframe))
- pkg_support_time[pkg] = timeframe
- if options.with_seeds:
- pkg_support_time[pkg] += " (%s)" % ", ".join(
- what_seeds(pkg, pkgs_in_seeds))
-
- return pkg_support_time
-
-
-if __name__ == "__main__":
- parser = OptionParser()
- parser.add_option("--with-seeds", "", default=False,
- action="store_true",
- help="add seed(s) of the package that are responsible "
- "for the maintaince time")
- parser.add_option("--source-packages", "", default=False,
- action="store_true",
- help="show as source pkgs")
- parser.add_option("--hints-file", "", default=None,
- help="use diffenrt use hints file location")
- (options, args) = parser.parse_args()
-
- # init
- if len(args) > 0:
- distro = args[0]
- if distro in UNSUPPORTED_DISTRO_RELEASED:
- logging.error("only lucid or later is supported")
- sys.exit(1)
- else:
- distro = "lucid"
-
- # maintenance class to use
- klass = globals().get("%sUbuntuMaintenance" % distro.capitalize())
- if klass is None:
- klass = UbuntuMaintenance
- ubuntu_maintenance = klass()
-
- # make sure our deb-src information is up-to-date
- create_and_update_deb_src_source_list(distro)
-
- if options.hints_file:
- hints_file = options.hints_file
- (schema, netloc, path, query, fragment) = urlparse.urlsplit(
- hints_file)
- if not schema:
- hints_file = "file:%s" % path
- else:
- hints_file = HINTS_DIR_URL % distro
-
- # go over the distros we need to check
- pkg_support_time = {}
- for names, support_timeframe in (
- (ubuntu_maintenance.DISTRO_NAMES_SHORT, ubuntu_maintenance.SUPPORT_TIMEFRAME_SHORT),
- (ubuntu_maintenance.DISTRO_NAMES, ubuntu_maintenance.SUPPORT_TIMEFRAME),
- ):
- for name in names:
- # get basic structure file
- try:
- structure = get_structure(name, distro)
- except urllib2.HTTPError:
- logging.error("Can not get structure for '%s'." % name)
- continue
-
- # get dicts of pkgname -> support timeframe string
- get_packages_support_time(
- structure, name, pkg_support_time, support_timeframe)
-
- # now go over the bits in main that we have not seen (because
- # they are not in any seed and got added manually into "main"
- for arch in ubuntu_maintenance.PRIMARY_ARCHES:
- rootdir = "./aptroot.%s" % distro
- apt_pkg.config.set("APT::Architecture", arch)
- cache = apt.Cache(rootdir=rootdir)
- try:
- cache.update()
- except SystemError:
- logging.exception("cache.update() failed")
- cache.open()
- for pkg in cache:
- # ignore multiarch package names
- if ":" in pkg.name:
- continue
- if not pkg.name in pkg_support_time:
- pkg_support_time[pkg.name] = support_timeframe[-1][0]
- logging.warn(
- "add package in main but not in seeds %s with %s" % (
- pkg.name, pkg_support_time[pkg.name]))
-
- # now check the hints file that is used to overwrite
- # the default seeds
- try:
- for line in urllib2.urlopen(hints_file):
- line = line.strip()
- if not line or line.startswith("#"):
- continue
- try:
- (raw_pkgname, support_time) = line.split()
- for pkgname in expand_src_pkgname(raw_pkgname):
- if support_time == 'unsupported':
- try:
- del pkg_support_time[pkgname]
- sys.stderr.write("hints-file: marking %s "
- "unsupported\n" % pkgname)
- except KeyError:
- pass
- else:
- if pkg_support_time.get(pkgname) != support_time:
- sys.stderr.write(
- "hints-file: changing %s from %s to %s\n" % (
- pkgname, pkg_support_time.get(pkgname),
- support_time))
- pkg_support_time[pkgname] = support_time
- except:
- logging.exception("can not parse line '%s'" % line)
- except urllib2.HTTPError as e:
- if e.code != 404:
- raise
- sys.stderr.write("hints-file: %s gave 404 error\n" % hints_file)
-
- # output suitable for the extra-override file
- for pkgname in sorted(pkg_support_time.keys()):
- # special case, the hints file may contain overrides that
- # are arch-specific (like zsh-doc/armel)
- if "/" in pkgname:
- print "%s %s %s" % (
- pkgname, SUPPORT_TAG, pkg_support_time[pkgname])
- else:
- # go over the supported arches, they are divided in
- # first-class (PRIMARY) and second-class with different
- # support levels
- for arch in ubuntu_maintenance.SUPPORTED_ARCHES:
- # ensure we do not overwrite arch-specific overwrites
- pkgname_and_arch = "%s/%s" % (pkgname, arch)
- if pkgname_and_arch in pkg_support_time:
- break
- if arch in ubuntu_maintenance.PRIMARY_ARCHES:
- # arch with full LTS support
- print "%s %s %s" % (
- pkgname_and_arch, SUPPORT_TAG,
- pkg_support_time[pkgname])
- else:
- # not a LTS supported architecture, gets only regular
- # support_timeframe
- print "%s %s %s" % (
- pkgname_and_arch, SUPPORT_TAG,
- ubuntu_maintenance.SUPPORT_TIMEFRAME[-1][0])
=== modified file 'lib/lp/archivepublisher/config.py'
--- lib/lp/archivepublisher/config.py 2012-09-20 12:00:22 +0000
+++ lib/lp/archivepublisher/config.py 2014-06-09 09:51:14 +0000
@@ -74,12 +74,10 @@
pubconf.overrideroot = pubconf.archiveroot + '-overrides'
pubconf.cacheroot = pubconf.archiveroot + '-cache'
pubconf.miscroot = pubconf.archiveroot + '-misc'
- pubconf.germinateroot = pubconf.archiveroot + '-germinate'
else:
pubconf.overrideroot = None
pubconf.cacheroot = None
pubconf.miscroot = None
- pubconf.germinateroot = None
if archive.is_main:
pubconf.uefiroot = pubconf.archiveroot + '-uefi'
@@ -117,7 +115,6 @@
self.cacheroot,
self.overrideroot,
self.miscroot,
- self.germinateroot,
self.temproot,
]
=== removed file 'lib/lp/archivepublisher/scripts/generate_extra_overrides.py'
--- lib/lp/archivepublisher/scripts/generate_extra_overrides.py 2012-11-10 02:21:31 +0000
+++ lib/lp/archivepublisher/scripts/generate_extra_overrides.py 1970-01-01 00:00:00 +0000
@@ -1,384 +0,0 @@
-# Copyright 2011-2012 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Generate extra overrides using Germinate."""
-
-__metaclass__ = type
-__all__ = [
- 'GenerateExtraOverrides',
- ]
-
-from functools import partial
-import glob
-import logging
-from optparse import OptionValueError
-import os
-import re
-
-from germinate.archive import TagFile
-from germinate.germinator import Germinator
-from germinate.log import GerminateFormatter
-from germinate.seeds import (
- SeedError,
- SeedStructure,
- )
-from zope.component import getUtility
-
-from lp.archivepublisher.config import getPubConfig
-from lp.registry.interfaces.distribution import IDistributionSet
-from lp.registry.interfaces.series import SeriesStatus
-from lp.services.database.policy import (
- DatabaseBlockedPolicy,
- SlaveOnlyDatabasePolicy,
- )
-from lp.services.scripts.base import (
- LaunchpadScript,
- LaunchpadScriptFailure,
- )
-from lp.services.utils import file_exists
-
-
-class AtomicFile:
- """Facilitate atomic writing of files."""
-
- def __init__(self, filename):
- self.filename = filename
- self.fd = open("%s.new" % self.filename, "w")
-
- def __enter__(self):
- return self.fd
-
- def __exit__(self, exc_type, exc_value, exc_tb):
- self.fd.close()
- if exc_type is None:
- os.rename("%s.new" % self.filename, self.filename)
-
-
-def find_operable_series(distribution):
- """Find all the series we can operate on in this distribution.
-
- We are allowed to modify DEVELOPMENT or FROZEN series, but should leave
- series with any other status alone.
- """
- return [
- series for series in distribution.series
- if series.status in (SeriesStatus.DEVELOPMENT, SeriesStatus.FROZEN)]
-
-
-class GenerateExtraOverrides(LaunchpadScript):
- """Main class for scripts/ftpmaster-tools/generate-task-overrides.py."""
-
- def __init__(self, *args, **kwargs):
- super(GenerateExtraOverrides, self).__init__(*args, **kwargs)
- self.germinate_logger = None
-
- def add_my_options(self):
- """Add a 'distribution' context option."""
- self.parser.add_option(
- "-d", "--distribution", dest="distribution",
- help="Context distribution name.")
-
- @property
- def name(self):
- """See `LaunchpadScript`."""
- # Include distribution name. Clearer to admins, but also
- # puts runs for different distributions under separate
- # locks so that they can run simultaneously.
- return "%s-%s" % (self._name, self.options.distribution)
-
- def processOptions(self):
- """Handle command-line options."""
- if self.options.distribution is None:
- raise OptionValueError("Specify a distribution.")
-
- self.distribution = getUtility(IDistributionSet).getByName(
- self.options.distribution)
- if self.distribution is None:
- raise OptionValueError(
- "Distribution '%s' not found." % self.options.distribution)
-
- self.series = find_operable_series(self.distribution)
- if not self.series:
- raise LaunchpadScriptFailure(
- "There is no DEVELOPMENT or FROZEN distroseries for %s." %
- self.options.distribution)
-
- def getConfig(self):
- """Set up a configuration object for this archive."""
- archive = self.distribution.main_archive
- if archive:
- return getPubConfig(archive)
- else:
- raise LaunchpadScriptFailure(
- "There is no PRIMARY archive for %s." %
- self.options.distribution)
-
- def setUpDirs(self):
- """Create output directories if they did not already exist."""
- germinateroot = self.config.germinateroot
- if not file_exists(germinateroot):
- self.logger.debug("Creating germinate root %s.", germinateroot)
- os.makedirs(germinateroot)
- miscroot = self.config.miscroot
- if not file_exists(miscroot):
- self.logger.debug("Creating misc root %s.", miscroot)
- os.makedirs(miscroot)
-
- def addLogHandler(self):
- """Send germinate's log output to a separate file."""
- if self.germinate_logger is not None:
- return
-
- self.germinate_logger = logging.getLogger("germinate")
- self.germinate_logger.setLevel(logging.INFO)
- self.log_file = os.path.join(
- self.config.germinateroot, "germinate.output")
- handler = logging.FileHandler(self.log_file, mode="w")
- handler.setFormatter(GerminateFormatter())
- self.germinate_logger.addHandler(handler)
- self.germinate_logger.propagate = False
-
- def setUp(self):
- """Process options, and set up internal state."""
- self.processOptions()
- self.config = self.getConfig()
- self.setUpDirs()
- self.addLogHandler()
-
- def getComponents(self, series):
- """Get the list of components to process for a given distroseries.
-
- Even if DistroSeries.component_names starts including partner,
- we don't want it; this applies to the primary archive only.
- """
- return [component
- for component in series.component_names
- if component != "partner"]
-
- def makeSeedStructures(self, series_name, flavours, seed_bases=None):
- structures = {}
- for flavour in flavours:
- try:
- structure = SeedStructure(
- "%s.%s" % (flavour, series_name), seed_bases=seed_bases)
- if len(structure):
- structures[flavour] = structure
- else:
- self.logger.warning(
- "Skipping empty seed structure for %s.%s",
- flavour, series_name)
- except SeedError as e:
- self.logger.warning(
- "Failed to fetch seeds for %s.%s: %s",
- flavour, series_name, e)
- return structures
-
- def logGerminateProgress(self, *args):
- """Log a "progress" entry to the germinate log file.
-
- Germinate logs quite a bit of detailed information. To make it
- easier to see the structure of its operation, GerminateFormatter
- allows tagging some log entries as "progress" entries, which are
- printed without a prefix.
- """
- self.germinate_logger.info(*args, extra={"progress": True})
-
- def composeOutputPath(self, flavour, series_name, arch, base):
- return os.path.join(
- self.config.germinateroot,
- "%s_%s_%s_%s" % (base, flavour, series_name, arch))
-
- def recordOutput(self, path, seed_outputs):
- if seed_outputs is not None:
- seed_outputs.add(os.path.basename(path))
-
- def writeGerminateOutput(self, germinator, structure, flavour,
- series_name, arch, seed_outputs=None):
- """Write dependency-expanded output files.
-
- These files are a reduced subset of those written by the germinate
- command-line program.
- """
- path = partial(self.composeOutputPath, flavour, series_name, arch)
-
- # The structure file makes it possible to figure out how the other
- # output files relate to each other.
- structure.write(path("structure"))
- self.recordOutput(path("structure"), seed_outputs)
-
- # "all" and "all.sources" list the full set of binary and source
- # packages respectively for a given flavour/suite/architecture
- # combination.
- germinator.write_all_list(structure, path("all"))
- self.recordOutput(path("all"), seed_outputs)
- germinator.write_all_source_list(structure, path("all.sources"))
- self.recordOutput(path("all.sources"), seed_outputs)
-
- # Write the dependency-expanded output for each seed. Several of
- # these are used by archive administration tools, and others are
- # useful for debugging, so it's best to just write them all.
- for seedname in structure.names:
- germinator.write_full_list(structure, path(seedname), seedname)
- self.recordOutput(path(seedname), seed_outputs)
-
- def parseTaskHeaders(self, seedtext):
- """Parse a seed for Task headers.
-
- seedtext is a file-like object. Return a dictionary of Task headers,
- with keys canonicalised to lower-case.
- """
- task_headers = {}
- task_header_regex = re.compile(
- r"task-(.*?):(.*)", flags=re.IGNORECASE)
- for line in seedtext:
- match = task_header_regex.match(line)
- if match is not None:
- key, value = match.groups()
- task_headers[key.lower()] = value.strip()
- return task_headers
-
- def getTaskName(self, task_headers, flavour, seedname, primary_flavour):
- """Work out the name of the Task to be generated from this seed.
-
- If there is a Task-Name header, it wins; otherwise, seeds with a
- Task-Per-Derivative header are honoured for all flavours and put in
- an appropriate namespace, while other seeds are only honoured for
- the first flavour and have archive-global names.
- """
- if "name" in task_headers:
- return task_headers["name"]
- elif "per-derivative" in task_headers:
- return "%s-%s" % (flavour, seedname)
- elif primary_flavour:
- return seedname
- else:
- return None
-
- def getTaskSeeds(self, task_headers, seedname):
- """Return the list of seeds used to generate a task from this seed.
-
- The list of packages in this task comes from this seed plus any
- other seeds listed in a Task-Seeds header.
- """
- scan_seeds = set([seedname])
- if "seeds" in task_headers:
- scan_seeds.update(task_headers["seeds"].split())
- return sorted(scan_seeds)
-
- def writeOverrides(self, override_file, germinator, structure, arch,
- seedname, key, value):
- packages = germinator.get_full(structure, seedname)
- for package in sorted(packages):
- print >>override_file, "%s/%s %s %s" % (
- package, arch, key, value)
-
- def germinateArchFlavour(self, override_file, germinator, series_name,
- arch, flavour, structure, primary_flavour,
- seed_outputs=None):
- """Germinate seeds on a single flavour for a single architecture."""
- # Expand dependencies.
- germinator.plant_seeds(structure)
- germinator.grow(structure)
- germinator.add_extras(structure)
-
- self.writeGerminateOutput(
- germinator, structure, flavour, series_name, arch,
- seed_outputs=seed_outputs)
-
- write_overrides = partial(
- self.writeOverrides, override_file, germinator, structure, arch)
-
- # Generate apt-ftparchive "extra overrides" for Task fields.
- seednames = [name for name in structure.names if name != "extra"]
- for seedname in seednames:
- with structure[seedname] as seedtext:
- task_headers = self.parseTaskHeaders(seedtext)
- if task_headers:
- task = self.getTaskName(
- task_headers, flavour, seedname, primary_flavour)
- if task is not None:
- scan_seeds = self.getTaskSeeds(task_headers, seedname)
- for scan_seed in scan_seeds:
- write_overrides(scan_seed, "Task", task)
-
- # Generate apt-ftparchive "extra overrides" for Build-Essential
- # fields.
- if "build-essential" in structure.names and primary_flavour:
- write_overrides("build-essential", "Build-Essential", "yes")
-
- def germinateArch(self, override_file, series_name, components, arch,
- flavours, structures, seed_outputs=None):
- """Germinate seeds on all flavours for a single architecture."""
- germinator = Germinator(arch)
-
- # Read archive metadata.
- archive = TagFile(
- series_name, components, arch,
- "file://%s" % self.config.archiveroot, cleanup=True)
- germinator.parse_archive(archive)
-
- for flavour in flavours:
- self.logger.info(
- "Germinating for %s/%s/%s", flavour, series_name, arch)
- # Add this to the germinate log as well so that that can be
- # debugged more easily. Log a separator line first.
- self.logGerminateProgress("")
- self.logGerminateProgress(
- "Germinating for %s/%s/%s", flavour, series_name, arch)
-
- self.germinateArchFlavour(
- override_file, germinator, series_name, arch, flavour,
- structures[flavour], flavour == flavours[0],
- seed_outputs=seed_outputs)
-
- def removeStaleOutputs(self, series_name, seed_outputs):
- """Remove stale outputs for a series.
-
- Any per-seed outputs not in seed_outputs are considered stale.
- """
- all_outputs = glob.glob(
- os.path.join(self.config.germinateroot, "*_*_%s_*" % series_name))
- for output in all_outputs:
- if os.path.basename(output) not in seed_outputs:
- os.remove(output)
-
- def generateExtraOverrides(self, series_name, components, architectures,
- flavours, seed_bases=None):
- structures = self.makeSeedStructures(
- series_name, flavours, seed_bases=seed_bases)
-
- if structures:
- seed_outputs = set()
- override_path = os.path.join(
- self.config.miscroot,
- "more-extra.override.%s.main" % series_name)
- with AtomicFile(override_path) as override_file:
- for arch in architectures:
- self.germinateArch(
- override_file, series_name, components, arch,
- flavours, structures, seed_outputs=seed_outputs)
- self.removeStaleOutputs(series_name, seed_outputs)
-
- def process(self, seed_bases=None):
- """Do the bulk of the work."""
- self.setUp()
-
- for series in self.series:
- series_name = series.name
- components = self.getComponents(series)
- architectures = sorted(
- arch.architecturetag for arch in series.enabled_architectures)
-
- # This takes a while. Ensure that we do it without keeping a
- # database transaction open.
- self.txn.commit()
- with DatabaseBlockedPolicy():
- self.generateExtraOverrides(
- series_name, components, architectures, self.args,
- seed_bases=seed_bases)
-
- def main(self):
- """See `LaunchpadScript`."""
- # This code has no need to alter the database.
- with SlaveOnlyDatabasePolicy():
- self.process()
=== modified file 'lib/lp/archivepublisher/tests/test_config.py'
--- lib/lp/archivepublisher/tests/test_config.py 2013-05-01 18:39:38 +0000
+++ lib/lp/archivepublisher/tests/test_config.py 2014-06-09 09:51:14 +0000
@@ -45,8 +45,6 @@
self.assertEqual(archiveroot + "-cache", primary_config.cacheroot)
self.assertEqual(archiveroot + "-misc", primary_config.miscroot)
self.assertEqual(
- archiveroot + "-germinate", primary_config.germinateroot)
- self.assertEqual(
self.root + "/ubuntutest-temp", primary_config.temproot)
self.assertEqual(archiveroot + "-uefi", primary_config.uefiroot)
@@ -67,7 +65,6 @@
self.assertIsNone(partner_config.overrideroot)
self.assertIsNone(partner_config.cacheroot)
self.assertIsNone(partner_config.miscroot)
- self.assertIsNone(partner_config.germinateroot)
self.assertEqual(
self.root + "/ubuntutest-temp", partner_config.temproot)
self.assertEqual(archiveroot + "-uefi", partner_config.uefiroot)
@@ -89,8 +86,6 @@
archiveroot + "-overrides", copy_config.overrideroot)
self.assertEqual(archiveroot + "-cache", copy_config.cacheroot)
self.assertEqual(archiveroot + "-misc", copy_config.miscroot)
- self.assertEqual(
- archiveroot + "-germinate", copy_config.germinateroot)
self.assertEqual(archiveroot + "-temp", copy_config.temproot)
self.assertIsNone(copy_config.uefiroot)
@@ -125,7 +120,6 @@
self.assertIsNone(self.ppa_config.overrideroot)
self.assertIsNone(self.ppa_config.cacheroot)
self.assertIsNone(self.ppa_config.miscroot)
- self.assertIsNone(self.ppa_config.germinateroot)
self.assertEqual(
"/var/tmp/archive/ubuntutest-temp", self.ppa_config.temproot)
uefiroot = "/var/tmp/ppa-signing-keys.test/uefi/%s/%s" % (
@@ -156,7 +150,6 @@
self.assertIsNone(p3a_config.overrideroot)
self.assertIsNone(p3a_config.cacheroot)
self.assertIsNone(p3a_config.miscroot)
- self.assertIsNone(p3a_config.germinateroot)
self.assertEqual(
"/var/tmp/archive/ubuntutest-temp", p3a_config.temproot)
# It's OK for the signing keys to be in the same location as for
=== removed file 'lib/lp/archivepublisher/tests/test_generate_extra_overrides.py'
--- lib/lp/archivepublisher/tests/test_generate_extra_overrides.py 2013-09-24 05:45:06 +0000
+++ lib/lp/archivepublisher/tests/test_generate_extra_overrides.py 1970-01-01 00:00:00 +0000
@@ -1,678 +0,0 @@
-# Copyright 2011-2012 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Test for the `generate-extra-overrides` script."""
-
-__metaclass__ = type
-
-from functools import partial
-import logging
-from optparse import OptionValueError
-import os
-import tempfile
-
-from germinate import (
- archive,
- germinator,
- seeds,
- )
-import transaction
-
-from lp.archivepublisher.publishing import (
- get_packages_path,
- get_sources_path,
- )
-from lp.archivepublisher.scripts.generate_extra_overrides import (
- AtomicFile,
- GenerateExtraOverrides,
- )
-from lp.archivepublisher.utils import RepositoryIndexFile
-from lp.registry.interfaces.pocket import PackagePublishingPocket
-from lp.registry.interfaces.series import SeriesStatus
-from lp.services.log.logger import DevNullLogger
-from lp.services.osutils import (
- ensure_directory_exists,
- open_for_writing,
- write_file,
- )
-from lp.services.scripts.base import LaunchpadScriptFailure
-from lp.services.scripts.tests import run_script
-from lp.services.utils import file_exists
-from lp.soyuz.enums import PackagePublishingStatus
-from lp.testing import TestCaseWithFactory
-from lp.testing.fakemethod import FakeMethod
-from lp.testing.faketransaction import FakeTransaction
-from lp.testing.layers import (
- LaunchpadZopelessLayer,
- ZopelessDatabaseLayer,
- )
-
-
-def file_contents(path):
- """Return the contents of the file at path."""
- with open(path) as handle:
- return handle.read()
-
-
-class TestAtomicFile(TestCaseWithFactory):
- """Tests for the AtomicFile helper class."""
-
- layer = ZopelessDatabaseLayer
-
- def test_atomic_file_creates_file(self):
- # AtomicFile creates the named file with the requested contents.
- self.useTempDir()
- filename = self.factory.getUniqueString()
- text = self.factory.getUniqueString()
- with AtomicFile(filename) as test:
- test.write(text)
- self.assertEqual(text, file_contents(filename))
-
- def test_atomic_file_removes_dot_new(self):
- # AtomicFile does not leave .new files lying around.
- self.useTempDir()
- filename = self.factory.getUniqueString()
- with AtomicFile(filename):
- pass
- self.assertFalse(file_exists("%s.new" % filename))
-
-
-class TestGenerateExtraOverrides(TestCaseWithFactory):
- """Tests for the actual `GenerateExtraOverrides` script."""
-
- layer = LaunchpadZopelessLayer
-
- def setUp(self):
- super(TestGenerateExtraOverrides, self).setUp()
- self.seeddir = self.makeTemporaryDirectory()
- # XXX cjwatson 2011-12-06 bug=694140: Make sure germinate doesn't
- # lose its loggers between tests, due to Launchpad's messing with
- # global log state.
- archive._logger = logging.getLogger("germinate.archive")
- germinator._logger = logging.getLogger("germinate.germinator")
- seeds._logger = logging.getLogger("germinate.seeds")
-
- def assertFilesEqual(self, expected_path, observed_path):
- self.assertEqual(
- file_contents(expected_path), file_contents(observed_path))
-
- def makeDistro(self):
- """Create a distribution for testing.
-
- The distribution will have a root directory set up, which will
- be cleaned up after the test. It will have an attached archive.
- """
- return self.factory.makeDistribution(
- publish_root_dir=unicode(self.makeTemporaryDirectory()))
-
- def makeScript(self, distribution, run_setup=True, extra_args=None):
- """Create a script for testing."""
- test_args = []
- if distribution is not None:
- test_args.extend(["-d", distribution.name])
- if extra_args is not None:
- test_args.extend(extra_args)
- script = GenerateExtraOverrides(test_args=test_args)
- script.logger = DevNullLogger()
- script.txn = FakeTransaction()
- if distribution is not None and run_setup:
- script.setUp()
- else:
- script.distribution = distribution
- return script
-
- def setUpDistroAndScript(self, series_statuses=["DEVELOPMENT"], **kwargs):
- """Helper wrapping distro and script setup."""
- self.distro = self.makeDistro()
- self.distroseries = [
- self.factory.makeDistroSeries(
- distribution=self.distro, status=SeriesStatus.items[status])
- for status in series_statuses]
- self.script = self.makeScript(self.distro, **kwargs)
-
- def setUpComponent(self, component=None):
- """Create a component and attach it to all distroseries."""
- if component is None:
- component = self.factory.makeComponent()
- for distroseries in self.distroseries:
- self.factory.makeComponentSelection(
- distroseries=distroseries, component=component)
- return component
-
- def makePackage(self, component, dases, **kwargs):
- """Create a published source and binary package for testing."""
- package = self.factory.makeDistributionSourcePackage(
- distribution=dases[0].distroseries.distribution)
- spph = self.factory.makeSourcePackagePublishingHistory(
- distroseries=dases[0].distroseries,
- pocket=PackagePublishingPocket.RELEASE,
- status=PackagePublishingStatus.PUBLISHED,
- sourcepackagename=package.name, component=component)
- for das in dases:
- build = self.factory.makeBinaryPackageBuild(
- source_package_release=spph.sourcepackagerelease,
- distroarchseries=das, processor=das.processor)
- bpr = self.factory.makeBinaryPackageRelease(
- binarypackagename=package.name, build=build,
- component=component, architecturespecific=True,
- **kwargs)
- lfa = self.factory.makeLibraryFileAlias(
- filename="%s.deb" % package.name)
- transaction.commit()
- bpr.addFile(lfa)
- self.factory.makeBinaryPackagePublishingHistory(
- binarypackagerelease=bpr, distroarchseries=das,
- pocket=PackagePublishingPocket.RELEASE,
- status=PackagePublishingStatus.PUBLISHED)
- return package
-
- def makeIndexFiles(self, script, distroseries):
- """Create a limited subset of index files for testing."""
- ensure_directory_exists(script.config.temproot)
-
- for component in distroseries.components:
- source_index = RepositoryIndexFile(
- get_sources_path(script.config, distroseries.name, component),
- script.config.temproot)
- for spp in distroseries.getSourcePackagePublishing(
- PackagePublishingPocket.RELEASE, component,
- distroseries.main_archive):
- stanza = spp.getIndexStanza().encode("utf-8") + "\n\n"
- source_index.write(stanza)
- source_index.close()
-
- for arch in distroseries.architectures:
- package_index = RepositoryIndexFile(
- get_packages_path(
- script.config, distroseries.name, component, arch),
- script.config.temproot)
- for bpp in distroseries.getBinaryPackagePublishing(
- arch.architecturetag, PackagePublishingPocket.RELEASE,
- component, distroseries.main_archive):
- stanza = bpp.getIndexStanza().encode("utf-8") + "\n\n"
- package_index.write(stanza)
- package_index.close()
-
- def composeSeedPath(self, flavour, series_name, seed_name):
- return os.path.join(
- self.seeddir, "%s.%s" % (flavour, series_name), seed_name)
-
- def makeSeedStructure(self, flavour, series_name, seed_names,
- seed_inherit={}):
- """Create a simple seed structure file."""
- structure_path = self.composeSeedPath(
- flavour, series_name, "STRUCTURE")
- with open_for_writing(structure_path, "w") as structure:
- for seed_name in seed_names:
- inherit = seed_inherit.get(seed_name, [])
- line = "%s: %s" % (seed_name, " ".join(inherit))
- print >>structure, line.strip()
-
- def makeSeed(self, flavour, series_name, seed_name, entries,
- headers=None):
- """Create a simple seed file."""
- seed_path = self.composeSeedPath(flavour, series_name, seed_name)
- with open_for_writing(seed_path, "w") as seed:
- if headers is not None:
- for header in headers:
- print >>seed, header
- print >>seed
- for entry in entries:
- print >>seed, " * %s" % entry
-
- def getTaskNameFromSeed(self, script, flavour, series_name, seed,
- primary_flavour):
- """Use script to parse a seed and return its task name."""
- seed_path = self.composeSeedPath(flavour, series_name, seed)
- with open(seed_path) as seed_text:
- task_headers = script.parseTaskHeaders(seed_text)
- return script.getTaskName(
- task_headers, flavour, seed, primary_flavour)
-
- def getTaskSeedsFromSeed(self, script, flavour, series_name, seed):
- """Use script to parse a seed and return its task seed list."""
- seed_path = self.composeSeedPath(flavour, series_name, seed)
- with open(seed_path) as seed_text:
- task_headers = script.parseTaskHeaders(seed_text)
- return script.getTaskSeeds(task_headers, seed)
-
- def test_name_is_consistent(self):
- # Script instances for the same distro get the same name.
- distro = self.factory.makeDistribution()
- self.assertEqual(
- GenerateExtraOverrides(test_args=["-d", distro.name]).name,
- GenerateExtraOverrides(test_args=["-d", distro.name]).name)
-
- def test_name_is_unique_for_each_distro(self):
- # Script instances for different distros get different names.
- self.assertNotEqual(
- GenerateExtraOverrides(
- test_args=["-d", self.factory.makeDistribution().name]).name,
- GenerateExtraOverrides(
- test_args=["-d", self.factory.makeDistribution().name]).name)
-
- def test_requires_distro(self):
- # The --distribution or -d argument is mandatory.
- script = self.makeScript(None)
- self.assertRaises(OptionValueError, script.processOptions)
-
- def test_requires_real_distro(self):
- # An incorrect distribution name is flagged as an invalid option
- # value.
- script = self.makeScript(
- None, extra_args=["-d", self.factory.getUniqueString()])
- self.assertRaises(OptionValueError, script.processOptions)
-
- def test_looks_up_distro(self):
- # The script looks up and keeps the distribution named on the
- # command line.
- self.setUpDistroAndScript()
- self.assertEqual(self.distro, self.script.distribution)
-
- def test_prefers_development_distro_series(self):
- # The script prefers a DEVELOPMENT series for the named
- # distribution over CURRENT and SUPPORTED series.
- self.setUpDistroAndScript(["SUPPORTED", "CURRENT", "DEVELOPMENT"])
- self.assertEqual([self.distroseries[2]], self.script.series)
-
- def test_permits_frozen_distro_series(self):
- # If there is no DEVELOPMENT series, a FROZEN one will do.
- self.setUpDistroAndScript(["SUPPORTED", "CURRENT", "FROZEN"])
- self.assertEqual([self.distroseries[2]], self.script.series)
-
- def test_requires_development_frozen_distro_series(self):
- # If there is no DEVELOPMENT or FROZEN series, the script fails.
- self.setUpDistroAndScript(["SUPPORTED", "CURRENT"], run_setup=False)
- self.assertRaises(LaunchpadScriptFailure, self.script.processOptions)
-
- def test_multiple_development_frozen_distro_series(self):
- # If there are multiple DEVELOPMENT or FROZEN series, they are all
- # used.
- self.setUpDistroAndScript(
- ["DEVELOPMENT", "DEVELOPMENT", "FROZEN", "FROZEN"])
- self.assertContentEqual(self.distroseries, self.script.series)
-
- def test_components_exclude_partner(self):
- # If a 'partner' component exists, it is excluded.
- self.setUpDistroAndScript()
- self.setUpComponent(component="main")
- self.setUpComponent(component="partner")
- self.assertEqual(1, len(self.script.series))
- self.assertEqual(
- ["main"], self.script.getComponents(self.script.series[0]))
-
- def test_compose_output_path_in_germinateroot(self):
- # Output files are written to the correct locations under
- # germinateroot.
- self.setUpDistroAndScript()
- flavour = self.factory.getUniqueString()
- arch = self.factory.getUniqueString()
- base = self.factory.getUniqueString()
- output = self.script.composeOutputPath(
- flavour, self.distroseries[0].name, arch, base)
- self.assertEqual(
- "%s/%s_%s_%s_%s" % (
- self.script.config.germinateroot, base, flavour,
- self.distroseries[0].name, arch),
- output)
-
- def test_make_seed_structures_missing_seeds(self):
- # makeSeedStructures ignores missing seeds.
- self.setUpDistroAndScript()
- series_name = self.distroseries[0].name
- flavour = self.factory.getUniqueString()
-
- structures = self.script.makeSeedStructures(
- series_name, [flavour], seed_bases=["file://%s" % self.seeddir])
- self.assertEqual({}, structures)
-
- def test_make_seed_structures_empty_seed_structure(self):
- # makeSeedStructures ignores an empty seed structure.
- self.setUpDistroAndScript()
- series_name = self.distroseries[0].name
- flavour = self.factory.getUniqueString()
- self.makeSeedStructure(flavour, series_name, [])
-
- structures = self.script.makeSeedStructures(
- series_name, [flavour], seed_bases=["file://%s" % self.seeddir])
- self.assertEqual({}, structures)
-
- def test_make_seed_structures_valid_seeds(self):
- # makeSeedStructures reads valid seeds successfully.
- self.setUpDistroAndScript()
- series_name = self.distroseries[0].name
- flavour = self.factory.getUniqueString()
- seed = self.factory.getUniqueString()
- self.makeSeedStructure(flavour, series_name, [seed])
- self.makeSeed(flavour, series_name, seed, [])
-
- structures = self.script.makeSeedStructures(
- series_name, [flavour], seed_bases=["file://%s" % self.seeddir])
- self.assertIn(flavour, structures)
-
- def fetchGerminatedOverrides(self, script, distroseries, arch, flavours):
- """Helper to call script.germinateArch and return overrides."""
- structures = script.makeSeedStructures(
- distroseries.name, flavours,
- seed_bases=["file://%s" % self.seeddir])
-
- override_fd, override_path = tempfile.mkstemp()
- with os.fdopen(override_fd, "w") as override_file:
- script.germinateArch(
- override_file, distroseries.name,
- script.getComponents(distroseries), arch, flavours,
- structures)
- return file_contents(override_path).splitlines()
-
- def test_germinate_output(self):
- # A single call to germinateArch produces output for all flavours on
- # one architecture.
- self.setUpDistroAndScript()
- series_name = self.distroseries[0].name
- component = self.setUpComponent()
- das = self.factory.makeDistroArchSeries(
- distroseries=self.distroseries[0])
- arch = das.architecturetag
- one = self.makePackage(component, [das])
- two = self.makePackage(component, [das])
- self.makeIndexFiles(self.script, self.distroseries[0])
-
- flavour_one = self.factory.getUniqueString()
- flavour_two = self.factory.getUniqueString()
- seed = self.factory.getUniqueString()
- self.makeSeedStructure(flavour_one, series_name, [seed])
- self.makeSeed(flavour_one, series_name, seed, [one.name])
- self.makeSeedStructure(flavour_two, series_name, [seed])
- self.makeSeed(flavour_two, series_name, seed, [two.name])
-
- overrides = self.fetchGerminatedOverrides(
- self.script, self.distroseries[0], arch,
- [flavour_one, flavour_two])
- self.assertEqual([], overrides)
-
- seed_dir_one = os.path.join(
- self.seeddir, "%s.%s" % (flavour_one, series_name))
- self.assertFilesEqual(
- os.path.join(seed_dir_one, "STRUCTURE"),
- self.script.composeOutputPath(
- flavour_one, series_name, arch, "structure"))
- self.assertTrue(file_exists(self.script.composeOutputPath(
- flavour_one, series_name, arch, "all")))
- self.assertTrue(file_exists(self.script.composeOutputPath(
- flavour_one, series_name, arch, "all.sources")))
- self.assertTrue(file_exists(self.script.composeOutputPath(
- flavour_one, series_name, arch, seed)))
-
- seed_dir_two = os.path.join(
- self.seeddir, "%s.%s" % (flavour_two, series_name))
- self.assertFilesEqual(
- os.path.join(seed_dir_two, "STRUCTURE"),
- self.script.composeOutputPath(
- flavour_two, series_name, arch, "structure"))
- self.assertTrue(file_exists(self.script.composeOutputPath(
- flavour_two, series_name, arch, "all")))
- self.assertTrue(file_exists(self.script.composeOutputPath(
- flavour_two, series_name, arch, "all.sources")))
- self.assertTrue(file_exists(self.script.composeOutputPath(
- flavour_two, series_name, arch, seed)))
-
- def test_germinate_output_task(self):
- # germinateArch produces Task extra overrides.
- self.setUpDistroAndScript()
- series_name = self.distroseries[0].name
- component = self.setUpComponent()
- das = self.factory.makeDistroArchSeries(
- distroseries=self.distroseries[0])
- arch = das.architecturetag
- one = self.makePackage(component, [das])
- two = self.makePackage(component, [das], depends=one.name)
- three = self.makePackage(component, [das])
- self.makePackage(component, [das])
- self.makeIndexFiles(self.script, self.distroseries[0])
-
- flavour = self.factory.getUniqueString()
- seed_one = self.factory.getUniqueString()
- seed_two = self.factory.getUniqueString()
- self.makeSeedStructure(flavour, series_name, [seed_one, seed_two])
- self.makeSeed(
- flavour, series_name, seed_one, [two.name],
- headers=["Task-Description: one"])
- self.makeSeed(
- flavour, series_name, seed_two, [three.name],
- headers=["Task-Description: two"])
-
- overrides = self.fetchGerminatedOverrides(
- self.script, self.distroseries[0], arch, [flavour])
- expected_overrides = [
- "%s/%s Task %s" % (one.name, arch, seed_one),
- "%s/%s Task %s" % (two.name, arch, seed_one),
- "%s/%s Task %s" % (three.name, arch, seed_two),
- ]
- self.assertContentEqual(expected_overrides, overrides)
-
- def test_task_name(self):
- # The Task-Name field is honoured.
- series_name = self.factory.getUniqueString()
- package = self.factory.getUniqueString()
- script = self.makeScript(None)
-
- flavour = self.factory.getUniqueString()
- seed = self.factory.getUniqueString()
- task = self.factory.getUniqueString()
- self.makeSeed(
- flavour, series_name, seed, [package],
- headers=["Task-Name: %s" % task])
-
- observed_task = self.getTaskNameFromSeed(
- script, flavour, series_name, seed, True)
- self.assertEqual(task, observed_task)
-
- def test_task_per_derivative(self):
- # The Task-Per-Derivative field is honoured.
- series_name = self.factory.getUniqueString()
- package = self.factory.getUniqueString()
- script = self.makeScript(None)
-
- flavour_one = self.factory.getUniqueString()
- flavour_two = self.factory.getUniqueString()
- seed_one = self.factory.getUniqueString()
- seed_two = self.factory.getUniqueString()
- self.makeSeed(
- flavour_one, series_name, seed_one, [package],
- headers=["Task-Description: one"])
- self.makeSeed(
- flavour_one, series_name, seed_two, [package],
- headers=["Task-Per-Derivative: 1"])
- self.makeSeed(
- flavour_two, series_name, seed_one, [package],
- headers=["Task-Description: one"])
- self.makeSeed(
- flavour_two, series_name, seed_two, [package],
- headers=["Task-Per-Derivative: 1"])
-
- observed_task_one_one = self.getTaskNameFromSeed(
- script, flavour_one, series_name, seed_one, True)
- observed_task_one_two = self.getTaskNameFromSeed(
- script, flavour_one, series_name, seed_two, True)
- observed_task_two_one = self.getTaskNameFromSeed(
- script, flavour_two, series_name, seed_one, False)
- observed_task_two_two = self.getTaskNameFromSeed(
- script, flavour_two, series_name, seed_two, False)
-
- # seed_one is not per-derivative, so it is honoured only for
- # flavour_one and has a global name.
- self.assertEqual(seed_one, observed_task_one_one)
- self.assertIsNone(observed_task_two_one)
-
- # seed_two is per-derivative, so it is honoured for both flavours
- # and has the flavour name prefixed.
- self.assertEqual(
- "%s-%s" % (flavour_one, seed_two), observed_task_one_two)
- self.assertEqual(
- "%s-%s" % (flavour_two, seed_two), observed_task_two_two)
-
- def test_task_seeds(self):
- # The Task-Seeds field is honoured.
- series_name = self.factory.getUniqueString()
- one = self.getUniqueString()
- two = self.getUniqueString()
- script = self.makeScript(None)
-
- flavour = self.factory.getUniqueString()
- seed_one = self.factory.getUniqueString()
- seed_two = self.factory.getUniqueString()
- self.makeSeed(flavour, series_name, seed_one, [one])
- self.makeSeed(
- flavour, series_name, seed_two, [two],
- headers=["Task-Seeds: %s" % seed_one])
-
- task_seeds = self.getTaskSeedsFromSeed(
- script, flavour, series_name, seed_two)
- self.assertContentEqual([seed_one, seed_two], task_seeds)
-
- def test_germinate_output_build_essential(self):
- # germinateArch produces Build-Essential extra overrides.
- self.setUpDistroAndScript()
- series_name = self.distroseries[0].name
- component = self.setUpComponent()
- das = self.factory.makeDistroArchSeries(
- distroseries=self.distroseries[0])
- arch = das.architecturetag
- package = self.makePackage(component, [das])
- self.makeIndexFiles(self.script, self.distroseries[0])
-
- flavour = self.factory.getUniqueString()
- seed = "build-essential"
- self.makeSeedStructure(flavour, series_name, [seed])
- self.makeSeed(flavour, series_name, seed, [package.name])
-
- overrides = self.fetchGerminatedOverrides(
- self.script, self.distroseries[0], arch, [flavour])
- self.assertContentEqual(
- ["%s/%s Build-Essential yes" % (package.name, arch)], overrides)
-
- def test_removes_only_stale_files(self):
- # removeStaleOutputs removes only stale germinate output files.
- self.setUpDistroAndScript()
- series_name = self.distroseries[0].name
- seed_old_file = "old_flavour_%s_i386" % series_name
- seed_new_file = "new_flavour_%s_i386" % series_name
- other_file = "other-file"
- output = partial(os.path.join, self.script.config.germinateroot)
- for base in (seed_old_file, seed_new_file, other_file):
- write_file(output(base), "")
- self.script.removeStaleOutputs(series_name, set([seed_new_file]))
- self.assertFalse(os.path.exists(output(seed_old_file)))
- self.assertTrue(os.path.exists(output(seed_new_file)))
- self.assertTrue(os.path.exists(output(other_file)))
-
- def test_process_missing_seeds(self):
- # The script ignores series with no seed structures.
- flavour = self.factory.getUniqueString()
- self.setUpDistroAndScript(
- ["DEVELOPMENT", "DEVELOPMENT"], extra_args=[flavour])
- self.setUpComponent()
- self.factory.makeDistroArchSeries(distroseries=self.distroseries[0])
- self.factory.makeDistroArchSeries(distroseries=self.distroseries[1])
- self.makeIndexFiles(self.script, self.distroseries[1])
- seed = self.factory.getUniqueString()
- self.makeSeedStructure(flavour, self.distroseries[1].name, [seed])
- self.makeSeed(flavour, self.distroseries[1].name, seed, [])
-
- self.script.process(seed_bases=["file://%s" % self.seeddir])
- self.assertFalse(os.path.exists(os.path.join(
- self.script.config.miscroot,
- "more-extra.override.%s.main" % self.distroseries[0].name)))
- self.assertTrue(os.path.exists(os.path.join(
- self.script.config.miscroot,
- "more-extra.override.%s.main" % self.distroseries[1].name)))
-
- def test_process_removes_only_stale_files(self):
- # The script removes only stale germinate output files.
- flavour = self.factory.getUniqueString()
- self.setUpDistroAndScript(extra_args=[flavour])
- series_name = self.distroseries[0].name
- self.setUpComponent()
- das = self.factory.makeDistroArchSeries(
- distroseries=self.distroseries[0])
- arch = das.architecturetag
- self.makeIndexFiles(self.script, self.distroseries[0])
-
- seed_old = self.factory.getUniqueString()
- seed_new = self.factory.getUniqueString()
- self.makeSeedStructure(flavour, series_name, [seed_old])
- self.makeSeed(flavour, series_name, seed_old, [])
- self.script.process(seed_bases=["file://%s" % self.seeddir])
- output = partial(
- self.script.composeOutputPath, flavour, series_name, arch)
- self.assertTrue(os.path.exists(output(seed_old)))
- self.makeSeedStructure(flavour, series_name, [seed_new])
- self.makeSeed(flavour, series_name, seed_new, [])
- self.script.process(seed_bases=["file://%s" % self.seeddir])
- self.assertTrue(os.path.exists(os.path.join(self.script.log_file)))
- self.assertTrue(os.path.exists(output("structure")))
- self.assertTrue(os.path.exists(output("all")))
- self.assertTrue(os.path.exists(output("all.sources")))
- self.assertTrue(os.path.exists(output(seed_new)))
- self.assertFalse(os.path.exists(output(seed_old)))
-
- def test_process_skips_disabled_distroarchseries(self):
- # The script does not generate overrides for disabled DistroArchSeries.
- flavour = self.factory.getUniqueString()
- self.setUpDistroAndScript(extra_args=[flavour])
- das = self.factory.makeDistroArchSeries(
- distroseries=self.distroseries[0])
- self.factory.makeDistroArchSeries(
- distroseries=self.distroseries[0], enabled=False)
- self.script.generateExtraOverrides = FakeMethod()
- self.script.process()
- self.assertEqual(1, self.script.generateExtraOverrides.call_count)
- self.assertEqual(
- [das.architecturetag],
- self.script.generateExtraOverrides.calls[0][0][2])
-
- def test_main(self):
- # If run end-to-end, the script generates override files containing
- # output for all architectures, and sends germinate's log output to
- # a file.
- flavour = self.factory.getUniqueString()
- self.setUpDistroAndScript(extra_args=[flavour])
- series_name = self.distroseries[0].name
- component = self.setUpComponent()
- das_one = self.factory.makeDistroArchSeries(
- distroseries=self.distroseries[0])
- das_two = self.factory.makeDistroArchSeries(
- distroseries=self.distroseries[0])
- package = self.makePackage(component, [das_one, das_two])
- self.makeIndexFiles(self.script, self.distroseries[0])
-
- seed = self.factory.getUniqueString()
- self.makeSeedStructure(flavour, series_name, [seed])
- self.makeSeed(
- flavour, series_name, seed, [package.name],
- headers=["Task-Description: task"])
-
- self.script.process(seed_bases=["file://%s" % self.seeddir])
- override_path = os.path.join(
- self.script.config.miscroot,
- "more-extra.override.%s.main" % series_name)
- expected_overrides = [
- "%s/%s Task %s" % (package.name, das_one.architecturetag, seed),
- "%s/%s Task %s" % (package.name, das_two.architecturetag, seed),
- ]
- self.assertContentEqual(
- expected_overrides, file_contents(override_path).splitlines())
-
- log_file = os.path.join(
- self.script.config.germinateroot, "germinate.output")
- self.assertIn("Downloading file://", file_contents(log_file))
-
- def test_run_script(self):
- # The script will run stand-alone.
- distro = self.makeDistro()
- self.factory.makeDistroSeries(distro)
- transaction.commit()
- retval, out, err = run_script(
- "cronscripts/generate-extra-overrides.py",
- ["-d", distro.name, "-q"])
- self.assertEqual(0, retval)
=== removed directory 'lib/lp/soyuz/scripts/tests/germinate-test-data'
=== removed directory 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-bin'
=== removed file 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-bin/lockfile'
--- lib/lp/soyuz/scripts/tests/germinate-test-data/mock-bin/lockfile 2010-11-11 18:22:00 +0000
+++ lib/lp/soyuz/scripts/tests/germinate-test-data/mock-bin/lockfile 1970-01-01 00:00:00 +0000
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# This is a mock "lockfile" command that is used during the test
-# There is no need to have the real lockfile installed (its part
-# of procmail) because there can be no multiple germinate instances
-# while we run the tests
-
-exit 1
=== removed directory 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root'
=== removed directory 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/cronscripts'
=== removed file 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/cronscripts/generate-extra-overrides.py'
--- lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/cronscripts/generate-extra-overrides.py 2011-12-14 15:21:50 +0000
+++ lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/cronscripts/generate-extra-overrides.py 1970-01-01 00:00:00 +0000
@@ -1,5 +0,0 @@
-#! /usr/bin/python
-#
-# We don't need to run generate-extra-overrides.py when testing
-# maintenance-check.py. We could, but it would slow down the tests and its
-# output is not interesting here.
=== removed directory 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/cronscripts/publishing'
=== removed symlink 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/cronscripts/publishing/maintenance-check.py'
=== target was u'../../../../../../../../../cronscripts/publishing/maintenance-check.py'
=== removed directory 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/scripts'
=== removed directory 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/scripts/ftpmaster-tools'
=== removed file 'lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/scripts/ftpmaster-tools/lp-query-distro.py'
--- lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/scripts/ftpmaster-tools/lp-query-distro.py 2012-04-16 13:56:45 +0000
+++ lib/lp/soyuz/scripts/tests/germinate-test-data/mock-lp-root/scripts/ftpmaster-tools/lp-query-distro.py 1970-01-01 00:00:00 +0000
@@ -1,27 +0,0 @@
-#!/usr/bin/python
-#
-# This is a mock version of the lp-query-distro.py script that
-# returns static data so that the cron.germinate shell script
-# can be tested.
-
-import sys
-
-
-def error_and_exit():
- sys.stderr.write("ERROR: I'm a mock, I only support 'supported' as "
- "argument\n")
- sys.exit(1)
-
-
-def main(args):
- # There is only a very limited subset of arguments that we support,
- # test for it and error if it looks wrong
- if len(args) == 2:
- distro = args[1]
- if distro == "supported":
- return "hardy jaunty karmic lucid maverick"
- error_and_exit()
-
-
-if __name__ == "__main__":
- print main(sys.argv)
=== removed file 'lib/lp/soyuz/scripts/tests/test_cron_germinate.py'
--- lib/lp/soyuz/scripts/tests/test_cron_germinate.py 2011-12-14 15:21:50 +0000
+++ lib/lp/soyuz/scripts/tests/test_cron_germinate.py 1970-01-01 00:00:00 +0000
@@ -1,226 +0,0 @@
-#!/usr/bin/python
-# Copyright 2010 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""This is a test for the soyuz cron.germinate script."""
-
-__metaclass__ = type
-
-import copy
-import gzip
-import os
-import subprocess
-
-from lp.testing import TestCase
-
-
-class TestCronGerminate(TestCase):
-
- DISTRO_NAMES = ["platform", "ubuntu", "kubuntu", "netbook"]
- DISTS = ["hardy", "lucid", "maverick"]
- DEVELOPMENT_DIST = "natty"
- COMPONENTS = ["main", "restricted", "universe", "multiverse"]
- ARCHES = ["i386", "amd64", "armel", "powerpc"]
- BASEPATH = os.path.abspath(os.path.dirname(__file__))
- source_root = os.path.normpath(
- os.path.join(BASEPATH, "..", "..", "..", "..", ".."))
-
- def setUp(self):
- super(TestCronGerminate, self).setUp()
-
- # Setup a temp archive directory and populate it with the right
- # sub-directories.
- self.tmpdir = self.makeTemporaryDirectory()
- self.archive_dir = self.setup_mock_archive_environment()
- self.ubuntu_misc_dir = os.path.join(self.archive_dir, "ubuntu-misc")
- self.ubuntu_germinate_dir = os.path.join(
- self.archive_dir, "ubuntu-germinate")
- # Create a mock archive environment for all the distros we support and
- # also include "updates" and "security".
- for dist in self.DISTS + [self.DEVELOPMENT_DIST]:
- self.populate_mock_archive_environment(
- self.archive_dir, self.COMPONENTS, self.ARCHES, dist)
- for component in ["security", "updates"]:
- self.populate_mock_archive_environment(
- self.archive_dir, self.COMPONENTS, self.ARCHES,
- "%s-%s" % (dist, component))
- # Generate test dummies for maintenance-time.py, if this is set to
- # "None" instead it will use the network to test against the real
- # data.
- self.germinate_output_dir = self.setup_mock_germinate_output()
-
- def create_directory_if_missing(self, directory):
- """Create the given directory if it does not exist."""
- if not os.path.exists(directory):
- os.makedirs(directory)
-
- def create_directory_list_if_missing(self, directory_list):
- """Create the given directories from the list if they don't exist."""
- for directory in directory_list:
- self.create_directory_if_missing(directory)
-
- def create_gzip_file(self, filepath, content=""):
- """Create a gziped file in the given path with the given content.
-
- If no content is given a empty file is created.
- """
- gz = gzip.GzipFile(filepath, "w")
- gz.write(content)
- gz.close()
-
- def create_file(self, filepath, content=""):
- """Create a file in the given path with the given content.
-
- If no content is given a empty file is created.
- """
- f = open(filepath, "w")
- f.write(content)
- f.close()
-
- def setup_mock_germinate_output(self):
- # empty structure files
- germinate_output_dir = os.path.join(
- self.tmpdir, "germinate-test-data", "germinate-output")
- dirs = []
- for distro_name in self.DISTRO_NAMES:
- for distro_series in self.DISTS:
- dirs.append(
- os.path.join(
- germinate_output_dir,
- "%s.%s" % (distro_name, distro_series)))
- self.create_directory_list_if_missing(dirs)
- for dir in dirs:
- self.create_file(os.path.join(dir, "structure"))
- return germinate_output_dir
-
- def setup_mock_archive_environment(self):
- """
- Creates a mock archive environment and populate it with the
- subdirectories that germinate will expect.
- """
- archive_dir = os.path.join(
- self.tmpdir, "germinate-test-data", "ubuntu-archive")
- ubuntu_misc_dir = os.path.join(archive_dir, "ubuntu-misc")
- ubuntu_germinate_dir = os.path.join(archive_dir, "ubuntu-germinate")
- ubuntu_dists_dir = os.path.join(archive_dir, "ubuntu", "dists")
- self.create_directory_list_if_missing([
- archive_dir,
- ubuntu_misc_dir,
- ubuntu_germinate_dir,
- ubuntu_dists_dir])
- return archive_dir
-
- def populate_mock_archive_environment(self, archive_dir, components_list,
- arches_list, current_devel_distro):
- """
- Populates a mock archive environment with empty source packages and
- empty binary packages.
- """
- for component in components_list:
- # Create the environment for the source packages.
- targetdir = os.path.join(
- archive_dir,
- "ubuntu/dists/%s/%s/source" % (
- current_devel_distro, component))
- self.create_directory_if_missing(targetdir)
- self.create_gzip_file(os.path.join(targetdir, "Sources.gz"))
-
- # Create the environment for the binary packages.
- for arch in arches_list:
- for subpath in ["", "debian-installer"]:
- targetdir = os.path.join(
- self.archive_dir,
- "ubuntu/dists/%s/%s/%s/binary-%s" % (
- current_devel_distro, component, subpath, arch))
- self.create_directory_if_missing(targetdir)
- self.create_gzip_file(os.path.join(
- targetdir, "Packages.gz"))
-
- def create_fake_environment(self, basepath, archive_dir,
- germinate_output_dir):
- """
- Create a fake process envirionment based on os.environ that sets
- TEST_ARCHIVEROOT, TEST_LAUNCHPADROOT and modifies PATH to point to the
- mock lp-bin directory.
- """
- fake_environ = copy.copy(os.environ)
- fake_environ["TEST_ARCHIVEROOT"] = os.path.abspath(
- os.path.join(archive_dir, "ubuntu"))
- fake_environ["TEST_LAUNCHPADROOT"] = os.path.abspath(
- os.path.join(basepath, "germinate-test-data/mock-lp-root"))
- # Set the PATH in the fake environment so that our mock lockfile is
- # used.
- fake_environ["PATH"] = "%s:%s" % (
- os.path.abspath(os.path.join(
- basepath, "germinate-test-data/mock-bin")),
- os.environ["PATH"])
- # test dummies for get-support-timeframe.py, they need to be
- # in URI format
- if germinate_output_dir:
- # redirect base url to the mock environment
- fake_environ["MAINTENANCE_CHECK_BASE_URL"] = "file://%s" % \
- germinate_output_dir
- # point to mock archive root
- archive_root_url = "file://%s" % os.path.abspath(
- os.path.join(archive_dir, "ubuntu"))
- fake_environ["MAINTENANCE_CHECK_ARCHIVE_ROOT"] = archive_root_url
- # maintenance-check.py expects a format string
- hints_file_url = (
- germinate_output_dir + "/platform.%s/SUPPORTED_HINTS")
- for distro in self.DISTS:
- open(hints_file_url % distro, "w")
- fake_environ["MAINTENANCE_CHECK_HINTS_DIR_URL"] = "file://%s" % \
- os.path.abspath(hints_file_url)
- # add hints override to test that feature
- f=open(hints_file_url % "lucid", "a")
- f.write("linux-image-2.6.32-25-server 5y\n")
- f.close()
- return fake_environ
-
- def test_maintenance_update(self):
- """
- Test the maintenance-check.py porition of the soyuz cron.germinate
- shell script by running it inside a fake environment and ensure that
- it did update the "Support" override information for apt-ftparchive
- without destroying/modifying the information that the "germinate"
- script added to it earlier.
- """
- # Write into more-extras.overrides to ensure it is alive after we
- # mucked around.
- canary = "abrowser Task mock\n"
- # Build fake environment based on the real one.
- fake_environ = self.create_fake_environment(
- self.BASEPATH, self.archive_dir, self.germinate_output_dir)
- # Create mock override data files that include the canary string
- # so that we can test later if it is still there.
- for dist in self.DISTS:
- self.create_file(
- os.path.join(self.ubuntu_misc_dir,
- "more-extra.override.%s.main" % dist),
- canary)
-
- # Run cron.germinate in the fake environment.
- cron_germinate_path = os.path.join(
- self.source_root, "cronscripts", "publishing", "cron.germinate")
- subprocess.call(
- [cron_germinate_path], env=fake_environ, cwd=self.BASEPATH)
-
- # And check the output it generated for correctness.
- for dist in self.DISTS:
- supported_override_file = os.path.join(
- self.ubuntu_misc_dir,
- "more-extra.override.%s.main.supported" % dist)
- self.assertTrue(os.path.exists(supported_override_file),
- "no override file created for '%s'" % dist)
- main_override_file = os.path.join(
- self.ubuntu_misc_dir,
- "more-extra.override.%s.main" % dist)
- self.assertIn(canary, open(main_override_file).read())
-
- # Check here if we got the data from maintenance-check.py that
- # we expected. This is a kernel name from lucid-updates and it
- # will be valid for 5 years.
- needle = "linux-image-2.6.32-25-server/i386 Supported 5y"
- lucid_supported_override_file = os.path.join(
- self.ubuntu_misc_dir, "more-extra.override.lucid.main")
- self.assertIn(needle, open(lucid_supported_override_file).read())
Follow ups
