launchpad-reviewers team mailing list archive

["Jeroen T. Vermeulen" <jtv@xxxxxxxxxxxxx>]

Jeroen T. Vermeulen has proposed merging lp:~jtv/launchpad/retire-old-cron-publish-ftpmaster into lp:launchpad.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~jtv/launchpad/retire-old-cron-publish-ftpmaster/+merge/70742

= Summary =

We have started to use the new, python-based publish-ftpmaster script to replace the old cron.publish-ftpmaster shell script.

Once we're quite, quite sure that things are alright now, we can finally ditch the old shell script.


== Pre-implementation notes ==

Julian points out we want to do this quickly, to avoid confusion with people accidentally invoking the old script in manual runs.


(No files to lint)
-- 
https://code.launchpad.net/~jtv/launchpad/retire-old-cron-publish-ftpmaster/+merge/70742
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~jtv/launchpad/retire-old-cron-publish-ftpmaster into lp:launchpad.

=== removed file 'cronscripts/publishing/cron.publish-ftpmaster'
--- cronscripts/publishing/cron.publish-ftpmaster	2011-04-05 06:59:48 +0000
+++ cronscripts/publishing/cron.publish-ftpmaster	1970-01-01 00:00:00 +0000
@@ -1,266 +0,0 @@
-#!/bin/sh
-#
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# LPCONFIG will come from the environment so this script can run unaltered
-# on dogfood.
-if [ -z $LPCONFIG ]; then
-    echo LPCONFIG must be set to run this script.
-    exit 1
-fi
-SECURITY_PUBLISHER="no"
-if [ "$1" = "security" ]; then
-    # We are running a security publisher run, which skips some steps.
-    SECURITY_PUBLISHER="yes"
-fi
-
-set -x
-set -e
-set -u
-
-# Launchpad cron.daily (currently just for Ubuntu).
-
-# Informational -- this *MUST* match the database.
-ARCHIVEROOT=/srv/launchpad.net/ubuntu-archive/ubuntu
-DISTSROOT=$ARCHIVEROOT/dists
-OVERRIDEROOT=$ARCHIVEROOT/../ubuntu-overrides
-CACHEROOT=$ARCHIVEROOT/../ubuntu-cache
-DISTSCOPYROOT=$ARCHIVEROOT/../ubuntu-distscopy
-INDICES=$ARCHIVEROOT/indices
-PRODUCTION_CONFIG=ftpmaster-publish
-
-if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
-    ARCHIVEROOT_PARTNER=/srv/launchpad.net/ubuntu-archive/ubuntu-partner
-    GNUPGHOME=/srv/launchpad.net/ubuntu-archive/gnupg-home
-else
-    # GNUPGHOME does not need to be set, keys can come from ~/.gnupg.
-    ARCHIVEROOT_PARTNER=/srv/launchpad.net/ppa/ubuntu-partner
-fi
-DISTSROOT_PARTNER=$ARCHIVEROOT_PARTNER/dists
-DISTSCOPYROOT_PARTNER=$ARCHIVEROOT_PARTNER/../ubuntu-partner-distscopy
-
-# Configuration options.
-LAUNCHPADROOT=/srv/launchpad.net/codelines/current
-LOCKFILE=/srv/launchpad.net/ubuntu-archive/cron.daily.lock
-DISTRONAME=ubuntu
-TRACEFILE=$ARCHIVEROOT/project/trace/$(hostname --fqdn)
-DSYNCLIST=$CACHEROOT/dsync.list
-MD5LIST=$INDICES/md5sums.gz
-
-# Mirrors to push at the end of the publishing run.
-MASTERMIRRORS="syowa frei wahoo scandium"
-
-# Manipulate the environment.
-export GNUPGHOME
-PATH=$PATH:$LAUNCHPADROOT/scripts:$LAUNCHPADROOT/cronscripts:$LAUNCHPADROOT/cronscripts/publishing:$LAUNCHPADROOT/scripts/ftpmaster-tools
-
-# Claim the lock.
-if ! lockfile -r1 $LOCKFILE; then
-  echo "Could not claim lock file."
-  exit 1
-fi
-
-echo "$(date -R): Initiating archive publishing operations..."
-
-DONEPUB=no
-
-cleanup () {
-  echo "$(date -R): Cleaning up lockfile."
-  rm -f $LOCKFILE
-  echo "$(date -R): Moving dists backup to safe keeping for next time."
-  if [ "x$DONEPUB" = "xyes" ]; then
-    if [ -d ${DISTSROOT}.old ]; then
-      mv ${DISTSROOT}.old ${DISTSCOPYROOT}/dists
-    fi
-    if [ -d ${DISTSROOT_PARTNER}.old ]; then
-      mv ${DISTSROOT_PARTNER}.old ${DISTSCOPYROOT_PARTNER}/dists
-    fi
-  else
-    if [ -d ${DISTSROOT}.new ]; then
-      mv ${DISTSROOT}.new ${DISTSCOPYROOT}/dists
-    fi
-    if [ -d ${DISTSROOT_PARTNER}.new ]; then
-      mv ${DISTSROOT_PARTNER}.new ${DISTSCOPYROOT_PARTNER}/dists
-    fi
-  fi
-  if [ "$SECURITY_PUBLISHER" = "yes" ]; then
-    exec $0
-  fi
-}
-
-trap cleanup EXIT
-
-# Lock claimed.
-
-# Process the accepted queue into the publishing records.
-echo "$(date -R): Processing the accepted queue into the publishing records..."
-process-accepted.py -v -v -v $DISTRONAME
-
-# If doing a security run, find out which suites are pending publication.
-SUITEOPTS=""
-if [ "$SECURITY_PUBLISHER" = "yes" ]; then
-    echo "$(date -R): Querying which suites are pending publication..."
-    SUITES=$(lp-query-distro.py pending_suites)
-    SECURITY_SUITES="no"
-    if [ -n "$SUITES" ]; then
-        for SUITE in $SUITES; do
-            case $SUITE in
-                *-security)
-                    SECURITY_SUITES="yes"
-                    SUITEOPTS="$SUITEOPTS -s $SUITE"
-                    ;;
-                *)
-                    ;;
-            esac
-        done
-    fi
-    if [ "$SECURITY_SUITES" != "yes" ]; then
-        echo "$(date -R): Nothing to do for security publisher; exiting."
-        exit 0
-    fi
-fi
-
-# Copy the dists tree ready for publishing into so that we don't get
-# an inconsistent dists tree at any point during the publishing cycle.
-# Because this would cause buildds to explode.
-# This is now done through maintaining a persistent backup copy of the
-# dists folder, which we move into place and bring up-to-date with rsync.
-# This should achieve the same as copying, only faster.
-
-# Create backup dists folder, if this is the first time.
-echo "$(date -R): Creating backup dists directories..."
-mkdir -p ${DISTSCOPYROOT}/dists
-mkdir -p ${DISTSCOPYROOT_PARTNER}/dists
-
-# Move the backup dists folder into place.
-echo "$(date -R): Moving backup dists into place..."
-mv ${DISTSCOPYROOT}/dists ${ARCHIVEROOT}/dists.new
-mv ${DISTSCOPYROOT_PARTNER}/dists ${ARCHIVEROOT_PARTNER}/dists.new
-
-# Bring it up-to-date efficiently with rsync. --delete is required to
-# ensure we don't ressurect things previously deleted, bug 58835.
-echo "$(date -R): Updating dists directories..."
-rsync -aH --delete ${DISTSROOT}/ ${ARCHIVEROOT}/dists.new
-rsync -aH --delete ${DISTSROOT_PARTNER}/ ${ARCHIVEROOT_PARTNER}/dists.new
-
-# Publish the results for all archives (except PPA).
-# The -R only affects the primary and the partner archive.
-echo "$(date -R): Publishing the $DISTRONAME partner archive..."
-publish-distro.py -v -v --partner -d $DISTRONAME -R ${DISTSROOT_PARTNER}.new
-
-echo "$(date -R): Publishing the $DISTRONAME archive..."
-publish-distro.py -v -v -d $DISTRONAME $SUITEOPTS -R ${DISTSROOT}.new
-
-set +x
-# Find all the Release files for which the Release.GPG is missing/too-old
-# We use -maxdepth 2 to only sign Release files for distroreleases,
-# not distroarchreleases/distrosourcereleases.
-# Also we sign the dist-upgrader tarballs because they're handy too.
-for CANDIDATE in $(find ${DISTSROOT}.new -maxdepth 2 -name Release) \
-       $(find ${DISTSROOT}.new/*/*/dist-upgrader* -name "*.tar.gz"); do
-  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ]
-  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
-    echo "$(date -R): (re-)signing $CANDIDATE"
-    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
-  else
-    echo "$(date -R): Not re-signing $CANDIDATE"
-  fi
-done
-SIGNLIST_PARTNER=$(find ${DISTSROOT_PARTNER}.new -maxdepth 2 -name Release)
-for CANDIDATE in $SIGNLIST_PARTNER; do
-  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ].
-  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
-    echo "$(date -R): (re-)signing $CANDIDATE"
-    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
-  else
-    echo "$(date -R): Not re-signing $CANDIDATE"
-  fi
-done
-
-# The Packages and Sources files are very large and would cripple our
-# mirrors, so we remove them now that the uncompressed MD5SUMS are in the
-# Release files.
-echo "$(date -R): Removing uncompressed Packages and Sources files"
-find ${DISTSROOT}.new \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
-find ${DISTSROOT_PARTNER} \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
-
-# Copy in the indices.
-if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
-    echo "$(date -R): Copying the indices into place."
-    rm -f $INDICES/override.*
-    cp $OVERRIDEROOT/override.* $INDICES
-fi
-
-# As close to atomically as possible, put the new dists into place for the
-# primary and partner archives.
-echo "$(date -R): Placing the new dists into place..."
-mv $DISTSROOT ${DISTSROOT}.old
-mv ${DISTSROOT}.new $DISTSROOT
-mv $DISTSROOT_PARTNER ${DISTSROOT_PARTNER}.old
-mv ${DISTSROOT_PARTNER}.new $DISTSROOT_PARTNER
-DONEPUB=yes
-mv ${DISTSROOT}.old ${DISTSCOPYROOT}/dists
-mv ${DISTSROOT_PARTNER}.old ${DISTSCOPYROOT_PARTNER}/dists
-
-# Generate the -commercial pocket for backwards compatibility with
-# dapper, edgy and feisty releases.  Don't fail the whole script if it
-# fails.
-echo "$(date -R): Generating -commerical pocket..."
-commercial-compat.sh || true
-
-# Timestamp our trace file to track when the last archive publisher run took
-# place.
-if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
-    echo "$(date -R): Timestamping trace file..."
-    date -u > "$TRACEFILE"
-fi
-
-# Skip long-running processes when doing a quick security run.
-if [ "$SECURITY_PUBLISHER" != "yes" ]; then
-
-    # Make the lslr because we all love those.
-    echo "$(date -R): Creating ls-lR.gz..."
-    LSLR=ls-lR.gz
-    ( cd $ARCHIVEROOT ; \
-      rm -f .$LSLR.new ; \
-      TZ=UTC ls -lR  | gzip -9n > .$LSLR.new ; \
-      mv -f .$LSLR.new $LSLR )
-    ( cd $ARCHIVEROOT_PARTNER ; \
-      rm -f .$LSLR.new ; \
-      TZ=UTC ls -lR  | gzip -9n > .$LSLR.new ; \
-      mv -f .$LSLR.new $LSLR )
-
-    # Run dsync over primary archive only.
-    echo "$(date -R): Running dsync over primary archive..."
-    ( cd $ARCHIVEROOT ; \
-      dsync-flist -q generate $DSYNCLIST -e 'Packages*' -e 'Sources*' -e 'Release*' --md5 ; \
-      (dsync-flist -q md5sums $DSYNCLIST; find dists '(' -name 'Packages*' -o -name 'Sources*' -o -name 'Release*' ')' -print | xargs -r md5sum) | gzip -9n > ${MD5LIST} ; \
-      dsync-flist -q link-dups $DSYNCLIST || true )
-
-    # Clear out empty and thus redundant dirs.
-    echo "$(date -R): Clearing out empty directories..."
-    find $ARCHIVEROOT -type d -empty | xargs -r rmdir
-    find $ARCHIVEROOT_PARTNER -type d -empty | xargs -r rmdir
-
-    # Invoke cron.germinate to fill ubuntu tasks and ignore failures,
-    # the mirrors should be always triggered.
-    echo "$(date -R): Running cron.germinate..."
-    cron.germinate || echo "$(date -R): cron.germinate failed with exit code $?"
-
-# End of block skipped by security publishing.
-fi
-
-# Trigger master mirrors.
-if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
-    echo "$(date -R): Triggering master mirrors..."
-
-    for HOST in $MASTERMIRRORS; do
-        echo "$(date -R): Triggering $HOST:"
-        ssh archvsync@$HOST
-    done
-
-    echo "$(date -R): Master mirror triggers completed."
-fi
-
-echo "$(date -R): Archive publishing operations completed."