launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #15717
[Merge] lp:~stevenk/launchpad/destroy-pppd into lp:launchpad
Steve Kowalik has proposed merging lp:~stevenk/launchpad/destroy-pppd into lp:launchpad with lp:~stevenk/launchpad/packagediff-job as a prerequisite.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~stevenk/launchpad/destroy-pppd/+merge/172945
Destroy the process-pending-packagediffs cronjob and friends, now it is no longer needed due to the switch to the job system.
--
https://code.launchpad.net/~stevenk/launchpad/destroy-pppd/+merge/172945
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~stevenk/launchpad/destroy-pppd into lp:launchpad.
=== removed file 'cronscripts/process-pending-packagediffs.py'
--- cronscripts/process-pending-packagediffs.py 2013-01-07 02:40:55 +0000
+++ cronscripts/process-pending-packagediffs.py 1970-01-01 00:00:00 +0000
@@ -1,35 +0,0 @@
-#!/usr/bin/python -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Process pending PackageDiffs.
-
-Process a optionally limited set of pending PackageDiffs.
-
-By default it process up to 50 diffs each run, which is enough to catch
-up on 1 hour of uploads (on average).
-
-However users might benefit for more frequently runs since the diff ETA
-relative to the upload will be shorter.
-
-The cycle time needs to be balanced with the run time to produce the shortest
-diff ETA and to not overlap much, for instance, if it has to diff monster
-sources like openoffice or firefox.
-
-Experiments with the cycle time will be safe enough and won't sink the host
-performance, since the lock file is exclusive.
-"""
-
-__metaclass__ = type
-
-import _pythonpath
-
-from lp.services.config import config
-from lp.soyuz.scripts.packagediff import ProcessPendingPackageDiffs
-
-
-if __name__ == '__main__':
- script = ProcessPendingPackageDiffs(
- 'process-pending-packagediffs', dbuser=config.uploader.dbuser)
- script.lock_and_run()
=== modified file 'lib/lp/soyuz/doc/package-diff.txt'
--- lib/lp/soyuz/doc/package-diff.txt 2013-01-08 07:54:50 +0000
+++ lib/lp/soyuz/doc/package-diff.txt 2013-07-04 02:43:27 +0000
@@ -451,14 +451,9 @@
database ID, i.e. newest first, and getPendingDiffs() results are
ordered by ascending database IDs, oldest first.
-getPendingDiffs() results can optionally be limited.
-
>>> packagediff_set.getPendingDiffs().count()
7
- >>> packagediff_set.getPendingDiffs(limit=2).count()
- 2
-
All package diffs targeting a set of source package releases can also
be requested. The results are ordered by the source package release
ID:
=== modified file 'lib/lp/soyuz/interfaces/packagediff.py'
--- lib/lp/soyuz/interfaces/packagediff.py 2013-01-07 02:40:55 +0000
+++ lib/lp/soyuz/interfaces/packagediff.py 2013-07-04 02:43:27 +0000
@@ -92,12 +92,10 @@
def get(diff_id):
"""Retrieve a `PackageDiff` for the given id."""
- def getPendingDiffs(limit=None):
+ def getPendingDiffs():
"""Return all pending `PackageDiff` records.
- :param limit: optional results limitation.
-
- :return a `SelectResult` ordered by id respecting the given limit.
+ :return a `ResultSet` ordered by id respecting the given limit.
"""
def getDiffsToReleases(sprs, preload_for_display=False):
=== modified file 'lib/lp/soyuz/model/packagediff.py'
--- lib/lp/soyuz/model/packagediff.py 2013-06-20 05:50:00 +0000
+++ lib/lp/soyuz/model/packagediff.py 2013-07-04 02:43:27 +0000
@@ -264,11 +264,11 @@
"""See `IPackageDiffSet`."""
return PackageDiff.get(diff_id)
- def getPendingDiffs(self, limit=None):
+ def getPendingDiffs(self):
return IStore(PackageDiff).find(
PackageDiff,
PackageDiff.status == PackageDiffStatus.PENDING).order_by(
- PackageDiff.id).config(limit=limit)
+ PackageDiff.id)
def getDiffsToReleases(self, sprs, preload_for_display=False):
"""See `IPackageDiffSet`."""
=== removed file 'lib/lp/soyuz/scripts/packagediff.py'
--- lib/lp/soyuz/scripts/packagediff.py 2010-08-20 20:31:18 +0000
+++ lib/lp/soyuz/scripts/packagediff.py 1970-01-01 00:00:00 +0000
@@ -1,62 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""PackageDiff cronscript class."""
-
-__metaclass__ = type
-
-__all__ = [
- 'ProcessPendingPackageDiffs',
- ]
-
-from zope.component import getUtility
-
-from lp.services.scripts.base import (
- LaunchpadCronScript,
- LaunchpadScriptFailure,
- )
-from lp.soyuz.interfaces.packagediff import IPackageDiffSet
-
-
-class ProcessPendingPackageDiffs(LaunchpadCronScript):
-
- def add_my_options(self):
- # 50 diffs seems to be more them enough to process all uploaded
- # source packages for 1 hour (average upload rate) for ubuntu
- # primary archive, security and PPAs in general.
- self.parser.add_option(
- "-l", "--limit", type="int", default=50,
- help="Maximum number of requests to be processed in this run.")
-
- self.parser.add_option(
- "-n", "--dry-run",
- dest="dryrun", action="store_true", default=False,
- help="Whether or not to commit the transaction.")
-
- def main(self):
- """Process pending `PackageDiff` records.
-
- Collect up to the maximum number of pending `PackageDiff` records
- available and process them.
-
- Processed diffs results are commited individually.
- """
- if self.args:
- raise LaunchpadScriptFailure("Unhandled arguments %r" % self.args)
-
- packagediff_set = getUtility(IPackageDiffSet)
-
- pending_diffs = packagediff_set.getPendingDiffs(
- limit=self.options.limit)
- self.logger.debug(
- 'Considering %s diff requests' % pending_diffs.count())
-
- # Iterate over all pending packagediffs.
- for packagediff in pending_diffs:
- self.logger.debug(
- 'Performing package diff for %s from %s' % (
- packagediff.from_source.name, packagediff.title))
- packagediff.performDiff()
- if not self.options.dryrun:
- self.logger.debug('Commiting the transaction.')
- self.txn.commit()
=== removed file 'lib/lp/soyuz/scripts/tests/test_processpendingpackagediffs.py'
--- lib/lp/soyuz/scripts/tests/test_processpendingpackagediffs.py 2012-01-01 02:58:52 +0000
+++ lib/lp/soyuz/scripts/tests/test_processpendingpackagediffs.py 1970-01-01 00:00:00 +0000
@@ -1,106 +0,0 @@
-# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-__metaclass__ = type
-
-import os
-import subprocess
-import sys
-
-from lp.services.config import config
-from lp.services.log.logger import BufferLogger
-from lp.soyuz.scripts.packagediff import ProcessPendingPackageDiffs
-from lp.soyuz.tests.soyuz import TestPackageDiffsBase
-from lp.testing.layers import LaunchpadZopelessLayer
-
-
-class TestProcessPendingPackageDiffsScript(TestPackageDiffsBase):
- """Test the process-pending-packagediffs.py script."""
- layer = LaunchpadZopelessLayer
- dbuser = config.uploader.dbuser
-
- def runProcessPendingPackageDiffs(self, extra_args=None):
- """Run process-pending-packagediffs.py.
-
- Returns a tuple of the process's return code, stdout output and
- stderr output."""
- if extra_args is None:
- extra_args = []
- script = os.path.join(
- config.root, "cronscripts", "process-pending-packagediffs.py")
- args = [sys.executable, script]
- args.extend(extra_args)
- process = subprocess.Popen(
- args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = process.communicate()
- return (process.returncode, stdout, stderr)
-
- def testSimpleScriptRun(self):
- """Try a simple processing-pending-packagediffs.py run."""
- returncode, out, err = self.runProcessPendingPackageDiffs()
- if returncode != 0:
- print "\nStdout:\n%s\nStderr\n%s\n" % (out, err)
- self.assertEqual(0, returncode)
-
- self.layer.txn.abort()
-
- # The pending PackageDiff request was processed.
- self.assertEqual(self.getPendingDiffs().count(), 0)
-
- def getDiffProcessor(self, limit=None):
- """Return a `ProcessPendingPackageDiffs` instance.
-
- :param limit: if passed, it will be used as the 'limit' script
- argument.
-
- :return the initialized script object using `BufferLogger` and
- the given parameters.
- """
- test_args = []
- if limit is not None:
- test_args.append('-l %s' % limit)
-
- diff_processor = ProcessPendingPackageDiffs(
- name='process-pending-packagediffs', test_args=test_args)
- diff_processor.logger = BufferLogger()
- diff_processor.txn = self.layer.txn
- return diff_processor
-
- def testSimpleRun(self):
- """Simple run of the script class.
-
- The only diff available is processed after its run.
- """
- # Setup a DiffProcessor.
- diff_processor = self.getDiffProcessor()
- diff_processor.main()
-
- # The pending PackageDiff request was processed.
- # See doc/package-diff.txt for more information.
- pending_diffs = self.getPendingDiffs()
- self.assertEqual(pending_diffs.count(), 0)
-
- def testLimitedRun(self):
- """Run the script with a limited scope.
-
- Check if a limited run of the script only processes up to 'limit'
- pending diff records and exits.
- """
- # Setup a DiffProcessor limited to one request per run.
- diff_processor = self.getDiffProcessor(limit=1)
-
- # Upload a new source version, so we have two pending PackageDiff
- # records to process.
- self.packager.buildVersion('1.0-3', changelog_text="biscuits")
- self.packager.buildSource(include_orig=False)
- self.packager.uploadSourceVersion('1.0-3', suite="warty-updates")
- self.assertEqual(self.getPendingDiffs().count(), 2)
-
- # The first processor run will process only one PackageDiff,
- # the other will remain.
- diff_processor.main()
- self.assertEqual(self.getPendingDiffs().count(), 1)
-
- # The next run process the remaining one.
- diff_processor.main()
- self.assertEqual(self.getPendingDiffs().count(), 0)
