launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #17301
[Merge] lp:~wgrant/launchpad/non-ubuntu-ppa-publish into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/non-ubuntu-ppa-publish into lp:launchpad.
Commit message:
cron.publish-ppa now publishes derived distros after Ubuntu, and process-death-row and process-accepted take them same distro arguments as publish-distro.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/non-ubuntu-ppa-publish/+merge/229740
cron.publish-ppa now publishes derived distros after Ubuntu.
I also fixed process-death-row to take --all-derived, and process-accepted uses the same -d/--all-derived syntax as the other publishing scripts (it previously used a positional argument or --derived).
--
https://code.launchpad.net/~wgrant/launchpad/non-ubuntu-ppa-publish/+merge/229740
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/non-ubuntu-ppa-publish into lp:launchpad.
=== modified file 'cronscripts/publishing/cron.publish-copy-archives'
--- cronscripts/publishing/cron.publish-copy-archives 2010-11-18 15:19:54 +0000
+++ cronscripts/publishing/cron.publish-copy-archives 2014-08-06 07:18:09 +0000
@@ -57,10 +57,10 @@
trap cleanup EXIT
# Process the accepted queue into the publishing records.
-process-accepted.py --copy-archive -v -v -v $DISTRONAME
+process-accepted.py --copy-archive -vvv -d $DISTRONAME
# Publish the packages to disk.
-publish-distro.py -v -v --copy-archive -d $DISTRONAME
+publish-distro.py -vv --copy-archive -d $DISTRONAME
echo Removing uncompressed Packages and Sources files
find ${DISTSROOT} \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
=== modified file 'cronscripts/publishing/cron.publish-ppa'
--- cronscripts/publishing/cron.publish-ppa 2012-10-18 00:02:35 +0000
+++ cronscripts/publishing/cron.publish-ppa 2014-08-06 07:18:09 +0000
@@ -10,12 +10,12 @@
LPCURRENT=`dirname $0`/../..
-# Accept PPA binary uploads
-$LPCURRENT/scripts/process-accepted.py -v --ppa ubuntu
-
-# Publish source and binary PPAs into disk.
+# Publish Ubuntu PPAs.
+$LPCURRENT/scripts/process-accepted.py -v --ppa -d ubuntu
$LPCURRENT/scripts/publish-distro.py -v --ppa -d ubuntu
-
-# Publish private PPAs.
$LPCURRENT/scripts/publish-distro.py -v --private-ppa -d ubuntu
+# Publish PPAs for other distros.
+$LPCURRENT/scripts/process-accepted.py -v --ppa --all-derived
+$LPCURRENT/scripts/publish-distro.py -v --ppa --all-derived
+$LPCURRENT/scripts/publish-distro.py -v --private-ppa --all-derived
=== added file 'lib/lp/archivepublisher/scripts/base.py'
--- lib/lp/archivepublisher/scripts/base.py 1970-01-01 00:00:00 +0000
+++ lib/lp/archivepublisher/scripts/base.py 2014-08-06 07:18:09 +0000
@@ -0,0 +1,56 @@
+# Copyright 2009-2013 Canonical Ltd. This software is licensed under
+# the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Publisher script class."""
+
+__metaclass__ = type
+__all__ = [
+ 'PublisherScript',
+ ]
+
+from optparse import OptionValueError
+
+from zope.component import getUtility
+
+from lp.registry.interfaces.distribution import IDistributionSet
+from lp.services.scripts.base import LaunchpadCronScript
+
+
+class PublisherScript(LaunchpadCronScript):
+
+ def addDistroOptions(self):
+ self.parser.add_option(
+ "-d", "--distribution", dest="distribution", metavar="DISTRO",
+ default=None, help="The distribution to publish.")
+
+ self.parser.add_option(
+ "-a", "--all-derived", action="store_true", dest="all_derived",
+ default=False, help="Publish all Ubuntu-derived distributions.")
+
+ def findSelectedDistro(self):
+ """Find the `Distribution` named by the --distribution option.
+
+ Defaults to Ubuntu if no name was given.
+ """
+ self.logger.debug("Finding distribution object.")
+ name = self.options.distribution
+ if name is None:
+ # Default to publishing Ubuntu.
+ name = "ubuntu"
+ distro = getUtility(IDistributionSet).getByName(name)
+ if distro is None:
+ raise OptionValueError("Distribution '%s' not found." % name)
+ return distro
+
+ def findDerivedDistros(self):
+ """Find all Ubuntu-derived distributions."""
+ self.logger.debug("Finding derived distributions.")
+ return getUtility(IDistributionSet).getDerivedDistributions()
+
+ def findDistros(self):
+ """Find the selected distribution(s)."""
+ if self.options.all_derived:
+ return self.findDerivedDistros()
+ else:
+ return [self.findSelectedDistro()]
=== modified file 'lib/lp/archivepublisher/scripts/publish_ftpmaster.py'
--- lib/lp/archivepublisher/scripts/publish_ftpmaster.py 2014-04-15 16:32:03 +0000
+++ lib/lp/archivepublisher/scripts/publish_ftpmaster.py 2014-08-06 07:18:09 +0000
@@ -315,7 +315,7 @@
self.logger.debug(
"Processing the accepted queue into the publishing records...")
script = ProcessAccepted(
- test_args=[distribution.name], logger=self.logger)
+ test_args=["-d", distribution.name], logger=self.logger)
script.txn = self.txn
script.main()
=== modified file 'lib/lp/soyuz/doc/soyuz-upload.txt'
--- lib/lp/soyuz/doc/soyuz-upload.txt 2012-12-26 23:22:06 +0000
+++ lib/lp/soyuz/doc/soyuz-upload.txt 2014-08-06 07:18:09 +0000
@@ -418,7 +418,7 @@
>>> transaction.commit()
>>> script = os.path.join(config.root, "scripts", "process-accepted.py")
>>> process = subprocess.Popen(
- ... [sys.executable, script, "ubuntutest", "-q"])
+ ... [sys.executable, script, "-d", "ubuntutest", "-q"])
>>> process.wait()
0
=== modified file 'lib/lp/soyuz/scripts/processaccepted.py'
--- lib/lp/soyuz/scripts/processaccepted.py 2013-09-27 17:24:29 +0000
+++ lib/lp/soyuz/scripts/processaccepted.py 2014-08-06 07:18:09 +0000
@@ -20,13 +20,9 @@
from zope.security.management import getSecurityPolicy
from lp.archivepublisher.publishing import GLOBAL_PUBLISHER_LOCK
+from lp.archivepublisher.scripts.base import PublisherScript
from lp.archiveuploader.tagfiles import parse_tagfile_content
-from lp.registry.interfaces.distribution import IDistributionSet
from lp.registry.interfaces.pocket import PackagePublishingPocket
-from lp.services.scripts.base import (
- LaunchpadCronScript,
- LaunchpadScriptFailure,
- )
from lp.services.webapp.authorization import LaunchpadPermissiveSecurityPolicy
from lp.services.webapp.errorlog import (
ErrorReportingUtility,
@@ -251,7 +247,7 @@
close_bugs_for_queue_item(queue_item)
-class ProcessAccepted(LaunchpadCronScript):
+class ProcessAccepted(PublisherScript):
"""Queue/Accepted processor.
Given a distribution to run on, obtains all the queue items for the
@@ -266,9 +262,7 @@
def add_my_options(self):
"""Command line options for this script."""
- self.parser.add_option(
- '-D', '--derived', action="store_true", dest="derived",
- default=False, help="Process all Ubuntu-derived distributions.")
+ self.addDistroOptions()
self.parser.add_option(
"--ppa", action="store_true", dest="ppa", default=False,
@@ -278,36 +272,14 @@
"--copy-archives", action="store_true", dest="copy_archives",
default=False, help="Run only over COPY archives.")
- def findNamedDistro(self, distro_name):
- """Find the `Distribution` called `distro_name`."""
- self.logger.debug("Finding distribution %s.", distro_name)
- distro = getUtility(IDistributionSet).getByName(distro_name)
- if distro is None:
- raise LaunchpadScriptFailure(
- "Distribution '%s' not found." % distro_name)
- return distro
-
- def findTargetDistros(self):
- """Find the distribution(s) to process, based on arguments."""
- if self.options.derived:
- return getUtility(IDistributionSet).getDerivedDistributions()
- else:
- return [self.findNamedDistro(self.args[0])]
-
def validateArguments(self):
"""Validate command-line arguments."""
if self.options.ppa and self.options.copy_archives:
raise OptionValueError(
"Specify only one of copy archives or ppa archives.")
- if self.options.derived:
- if len(self.args) != 0:
- raise OptionValueError(
- "Can't combine --derived with a distribution name.")
- else:
- if len(self.args) != 1:
- raise OptionValueError(
- "Need to be given exactly one non-option argument. "
- "Namely the distribution to process.")
+ if self.options.all_derived and self.options.distribution:
+ raise OptionValueError(
+ "Can't combine --derived with a distribution name.")
def makeTargetPolicy(self):
"""Pick and instantiate a `TargetPolicy` based on given options."""
@@ -376,7 +348,7 @@
self.validateArguments()
target_policy = self.makeTargetPolicy()
try:
- for distro in self.findTargetDistros():
+ for distro in self.findDistros():
queue_ids = self.processForDistro(distro, target_policy)
self.txn.commit()
target_policy.postprocessSuccesses(queue_ids)
=== modified file 'lib/lp/soyuz/scripts/processdeathrow.py'
--- lib/lp/soyuz/scripts/processdeathrow.py 2013-01-07 02:40:55 +0000
+++ lib/lp/soyuz/scripts/processdeathrow.py 2014-08-06 07:18:09 +0000
@@ -15,23 +15,18 @@
]
-from zope.component import getUtility
-
from lp.archivepublisher.deathrow import getDeathRow
-from lp.registry.interfaces.distribution import IDistributionSet
-from lp.services.scripts.base import LaunchpadCronScript
-
-
-class DeathRowProcessor(LaunchpadCronScript):
+from lp.archivepublisher.scripts.base import PublisherScript
+
+
+class DeathRowProcessor(PublisherScript):
def add_my_options(self):
self.parser.add_option(
"-n", "--dry-run", action="store_true", default=False,
help="Dry run: goes through the motions but commits to nothing.")
- self.parser.add_option(
- "-d", "--distribution", metavar="DISTRO", default='ubuntu',
- help="Specified the distribution name.")
+ self.addDistroOptions()
self.parser.add_option(
"-p", "--pool-root", metavar="PATH",
@@ -42,17 +37,15 @@
help="Run only over PPA archives.")
def main(self):
- distribution = getUtility(IDistributionSet).getByName(
- self.options.distribution)
-
- if self.options.ppa:
- archives = distribution.getAllPPAs()
- else:
- archives = distribution.all_distro_archives
-
- for archive in archives:
- self.logger.info("Processing %s" % archive.archive_url)
- self.processDeathRow(archive)
+ for distribution in self.findDistros():
+ if self.options.ppa:
+ archives = distribution.getAllPPAs()
+ else:
+ archives = distribution.all_distro_archives
+
+ for archive in archives:
+ self.logger.info("Processing %s" % archive.archive_url)
+ self.processDeathRow(archive)
def processDeathRow(self, archive):
"""Process death-row for the given archive.
@@ -78,4 +71,3 @@
else:
self.logger.debug("Committing")
self.txn.commit()
-
=== modified file 'lib/lp/soyuz/scripts/publishdistro.py'
--- lib/lp/soyuz/scripts/publishdistro.py 2013-08-01 14:09:45 +0000
+++ lib/lp/soyuz/scripts/publishdistro.py 2014-08-06 07:18:09 +0000
@@ -16,11 +16,8 @@
getPublisher,
GLOBAL_PUBLISHER_LOCK,
)
-from lp.registry.interfaces.distribution import IDistributionSet
-from lp.services.scripts.base import (
- LaunchpadCronScript,
- LaunchpadScriptFailure,
- )
+from lp.archivepublisher.scripts.base import PublisherScript
+from lp.services.scripts.base import LaunchpadScriptFailure
from lp.soyuz.enums import (
ArchivePurpose,
ArchiveStatus,
@@ -41,12 +38,14 @@
return not ppa.private
-class PublishDistro(LaunchpadCronScript):
+class PublishDistro(PublisherScript):
"""Distro publisher."""
lockfilename = GLOBAL_PUBLISHER_LOCK
def add_my_options(self):
+ self.addDistroOptions()
+
self.parser.add_option(
"-C", "--careful", action="store_true", dest="careful",
default=False, help="Turns on all the below careful options.")
@@ -67,14 +66,6 @@
help="Make index generation (e.g. apt-ftparchive) careful.")
self.parser.add_option(
- "-d", "--distribution", dest="distribution", metavar="DISTRO",
- default=None, help="The distribution to publish.")
-
- self.parser.add_option(
- "-a", "--all-derived", action="store_true", dest="all_derived",
- default=False, help="Publish all Ubuntu-derived distributions.")
-
- self.parser.add_option(
'-s', '--suite', metavar='SUITE', dest='suite', action='append',
type='string', default=[], help='The suite to publish')
@@ -179,33 +170,6 @@
raise OptionValueError(
"We should not define 'distsroot' in PPA mode!", )
- def findSelectedDistro(self):
- """Find the `Distribution` named by the --distribution option.
-
- Defaults to Ubuntu if no name was given.
- """
- self.logger.debug("Finding distribution object.")
- name = self.options.distribution
- if name is None:
- # Default to publishing Ubuntu.
- name = "ubuntu"
- distro = getUtility(IDistributionSet).getByName(name)
- if distro is None:
- raise OptionValueError("Distribution '%s' not found." % name)
- return distro
-
- def findDerivedDistros(self):
- """Find all Ubuntu-derived distributions."""
- self.logger.debug("Finding derived distributions.")
- return getUtility(IDistributionSet).getDerivedDistributions()
-
- def findDistros(self):
- """Find the selected distribution(s)."""
- if self.options.all_derived:
- return self.findDerivedDistros()
- else:
- return [self.findSelectedDistro()]
-
def findSuite(self, distribution, suite):
"""Find the named `suite` in the selected `Distribution`.
=== modified file 'lib/lp/soyuz/scripts/tests/test_processdeathrow.py'
--- lib/lp/soyuz/scripts/tests/test_processdeathrow.py 2012-09-27 02:53:00 +0000
+++ lib/lp/soyuz/scripts/tests/test_processdeathrow.py 2014-08-06 07:18:09 +0000
@@ -16,7 +16,6 @@
import subprocess
import sys
from tempfile import mkdtemp
-from unittest import TestCase
import pytz
from zope.component import getUtility
@@ -27,10 +26,11 @@
from lp.services.config import config
from lp.soyuz.enums import PackagePublishingStatus
from lp.soyuz.model.publishing import SourcePackagePublishingHistory
+from lp.testing import TestCaseWithFactory
from lp.testing.layers import LaunchpadZopelessLayer
-class TestProcessDeathRow(TestCase):
+class TestProcessDeathRow(TestCaseWithFactory):
"""Test the process-death-row.py script works properly."""
layer = LaunchpadZopelessLayer
@@ -38,8 +38,7 @@
def runDeathRow(self, extra_args, distribution="ubuntutest"):
"""Run process-death-row.py, returning the result and output."""
script = os.path.join(config.root, "scripts", "process-death-row.py")
- args = [sys.executable, script, "-v", "-d", distribution,
- "-p", self.primary_test_folder]
+ args = [sys.executable, script, "-v", "-p", self.primary_test_folder]
args.extend(extra_args)
process = subprocess.Popen(
args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -52,6 +51,7 @@
def setUp(self):
"""Set up for a test death row run."""
+ super(TestProcessDeathRow, self).setUp()
self.setupPrimaryArchive()
self.setupPPA()
@@ -62,6 +62,7 @@
"""Clean up after ourselves."""
self.tearDownPrimaryArchive()
self.tearDownPPA()
+ super(TestProcessDeathRow, self).tearDown()
def setupPrimaryArchive(self):
"""Create pending removal publications in ubuntutest PRIMARY archive.
@@ -70,6 +71,8 @@
and verified.
"""
ubuntutest = getUtility(IDistributionSet)["ubuntutest"]
+ self.factory.makeDistroSeriesParent(
+ derived_series=ubuntutest.currentseries)
ut_alsautils = ubuntutest.getSourcePackage("alsa-utils")
ut_alsautils_109a4 = ut_alsautils.getVersion("1.0.9a-4")
primary_pubrecs = ut_alsautils_109a4.publishing_history
@@ -168,7 +171,7 @@
def testDryRun(self):
"""Test we don't delete the file or change the db in dry run mode."""
- self.runDeathRow(["-n"])
+ self.runDeathRow(["-d", "ubuntutest", "-n"])
self.assertTrue(os.path.exists(self.primary_package_path))
self.assertTrue(os.path.exists(self.ppa_package_path))
@@ -181,7 +184,7 @@
def testWetRun(self):
"""Test we do delete the file and change the db in wet run mode."""
- self.runDeathRow([])
+ self.runDeathRow(["-d", "ubuntutest"])
self.assertFalse(os.path.exists(self.primary_package_path))
self.assertTrue(os.path.exists(self.ppa_package_path))
@@ -194,7 +197,7 @@
def testPPARun(self):
"""Test we only work upon PPA."""
- self.runDeathRow(["--ppa"])
+ self.runDeathRow(["-d", "ubuntutest", "--ppa"])
self.assertTrue(os.path.exists(self.primary_package_path))
self.assertFalse(os.path.exists(self.ppa_package_path))
@@ -205,3 +208,14 @@
self.probePublishingStatus(
self.ppa_pubrec_ids, PackagePublishingStatus.SUPERSEDED)
self.probeRemoved(self.ppa_pubrec_ids)
+
+ def testDerivedRun(self):
+ self.runDeathRow([])
+ self.assertTrue(os.path.exists(self.primary_package_path))
+ self.assertTrue(os.path.exists(self.ppa_package_path))
+ self.runDeathRow(['--all-derived'])
+ self.assertFalse(os.path.exists(self.primary_package_path))
+ self.assertTrue(os.path.exists(self.ppa_package_path))
+ self.runDeathRow(['--all-derived', '--ppa'])
+ self.assertFalse(os.path.exists(self.primary_package_path))
+ self.assertFalse(os.path.exists(self.ppa_package_path))
=== modified file 'lib/lp/soyuz/tests/test_processaccepted.py'
--- lib/lp/soyuz/tests/test_processaccepted.py 2013-09-27 17:24:29 +0000
+++ lib/lp/soyuz/tests/test_processaccepted.py 2014-08-06 07:18:09 +0000
@@ -14,7 +14,6 @@
from lp.services.config import config
from lp.services.database.interfaces import IStore
from lp.services.log.logger import BufferLogger
-from lp.services.scripts.base import LaunchpadScriptFailure
from lp.soyuz.enums import (
ArchivePurpose,
PackagePublishingStatus,
@@ -48,7 +47,7 @@
"""Return a ProcessAccepted instance."""
if test_args is None:
test_args = []
- test_args.append(self.distro.name)
+ test_args.extend(['-d', self.distro.name])
script = ProcessAccepted("process accepted", test_args=test_args)
script.logger = BufferLogger()
script.txn = self.layer.txn
@@ -179,38 +178,17 @@
self.assertEqual(
upload, IStore(PackageUpload).get(PackageUpload, upload_id))
- def test_validateArguments_requires_distro_by_default(self):
- self.assertRaises(
- OptionValueError, ProcessAccepted(test_args=[]).validateArguments)
-
def test_validateArguments_requires_no_distro_for_derived_run(self):
- ProcessAccepted(test_args=['--derived']).validateArguments()
+ ProcessAccepted(test_args=['--all-derived']).validateArguments()
# The test is that this does not raise an exception.
pass
def test_validateArguments_does_not_accept_distro_for_derived_run(self):
distro = self.factory.makeDistribution()
- script = ProcessAccepted(test_args=['--derived', distro.name])
+ script = ProcessAccepted(
+ test_args=['--all-derived', '-d', distro.name])
self.assertRaises(OptionValueError, script.validateArguments)
- def test_findTargetDistros_finds_named_distro(self):
- distro = self.factory.makeDistribution()
- script = ProcessAccepted(test_args=[distro.name])
- self.assertContentEqual([distro], script.findTargetDistros())
-
- def test_findNamedDistro_raises_error_if_not_found(self):
- nonexistent_distro = self.factory.getUniqueString()
- script = ProcessAccepted(test_args=[nonexistent_distro])
- self.assertRaises(
- LaunchpadScriptFailure,
- script.findNamedDistro, nonexistent_distro)
-
- def test_findTargetDistros_for_derived_finds_derived_distro(self):
- dsp = self.factory.makeDistroSeriesParent()
- script = ProcessAccepted(test_args=['--derived'])
- self.assertIn(
- dsp.derived_series.distribution, script.findTargetDistros())
-
class TestBugIDsFromChangesFile(TestCaseWithFactory):
"""Test get_bug_ids_from_changes_file."""
Follow ups
