← Back to team overview

launchpad-reviewers team mailing list archive

[Merge] lp:~wgrant/launchpad/reretrydepwait into lp:launchpad

 

William Grant has proposed merging lp:~wgrant/launchpad/reretrydepwait into lp:launchpad.

Commit message:
Rewrite buildd-retry-depwait as a looptuner.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)
Related bugs:
  Bug #739067 in Launchpad itself: "BinaryPackageBuild:+retry timeouts"
  https://bugs.launchpad.net/launchpad/+bug/739067

For more details, see:
https://code.launchpad.net/~wgrant/launchpad/reretrydepwait/+merge/128688

This branch rewrites retry-depwait as a looptuner, fixing bug #739067 by using more sensible transaction lengths. While the old script was DAS-driven, this one just goes through all depwait BFJs in order and filters them afterwards. When we merge BFJ into BPB it will be more efficient to filter out obsolete series in the candidate selection query, but right now it's more important that it use the BFJ.status index.

Performance is pretty similar as long as there's an index on buildfarmjob(status, id), which I'll add separately.
-- 
https://code.launchpad.net/~wgrant/launchpad/reretrydepwait/+merge/128688
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/reretrydepwait into lp:launchpad.
=== modified file 'cronscripts/buildd-retry-depwait.py'
--- cronscripts/buildd-retry-depwait.py	2012-01-01 03:14:54 +0000
+++ cronscripts/buildd-retry-depwait.py	2012-10-09 11:35:18 +0000
@@ -1,36 +1,26 @@
 #!/usr/bin/python -S
 #
-# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
-# pylint: disable-msg=C0103,W0403
-"""Retrying build in MANUALDEPWAIT state.
-
-This procedure aims to retry builds in all supported series and architectures
-in the given distribution which have failed due to unsatisfied
-build-dependencies.
-
-It checks every build in this state, including PPA and PARTNER ones, and
-retries the ones which got their dependencies published after they were tried.
-
-Unlike the other buildd-cronscripts, this one it targeted to run via cron
-in parallel with other tasks happening in build farm.
-
-As an optimization, distroseries from the selected distribution which are not
-supported anymore (OBSOLETE status) are skipped.
-"""
-
-__metaclass__ = type
-
 import _pythonpath
 
-from lp.soyuz.scripts.buildd import RetryDepwait
-
+from lp.services.scripts.base import LaunchpadScript
+from lp.soyuz.scripts.retrydepwait import RetryDepwaitTunableLoop
+
+
+class RetryDepwait(LaunchpadScript):
+
+    def add_my_options(self):
+        self.parser.add_option(
+            "-n", "--dry-run", action="store_true",
+            dest="dry_run", default=False,
+            help="Don't commit changes to the DB.")
+
+    def main(self):
+        updater = RetryDepwaitTunableLoop(self.logger, self.options.dry_run)
+        updater.run()
 
 if __name__ == '__main__':
     script = RetryDepwait('retry-depwait', dbuser='retry_depwait')
-    script.lock_or_quit()
-    try:
-        script.run()
-    finally:
-        script.unlock()
+    script.lock_and_run()

=== modified file 'lib/lp/soyuz/interfaces/binarypackagebuild.py'
--- lib/lp/soyuz/interfaces/binarypackagebuild.py	2012-10-03 09:15:55 +0000
+++ lib/lp/soyuz/interfaces/binarypackagebuild.py	2012-10-09 11:35:18 +0000
@@ -354,14 +354,6 @@
         sourcepackagename matches (SQL LIKE).
         """
 
-    def retryDepWaiting(distroarchseries):
-        """Re-process all MANUALDEPWAIT builds for a given IDistroArchSeries.
-
-        This method will update all the dependency lines of all MANUALDEPWAIT
-        records in the given architecture and those with all dependencies
-        satisfied at this point will be automatically retried and re-scored.
-        """
-
     def getBuildsBySourcePackageRelease(sourcepackagerelease_ids,
                                         buildstate=None):
         """Return all builds related with the given list of source releases.

=== modified file 'lib/lp/soyuz/model/binarypackagebuild.py'
--- lib/lp/soyuz/model/binarypackagebuild.py	2012-10-03 09:15:55 +0000
+++ lib/lp/soyuz/model/binarypackagebuild.py	2012-10-09 11:35:18 +0000
@@ -1102,53 +1102,6 @@
             result_set, pre_iter_hook=self._prefetchBuildData)
         return decorated_results
 
-    def retryDepWaiting(self, distroarchseries):
-        """See `IBinaryPackageBuildSet`. """
-        # XXX cprov 20071122: use the root logger once bug 164203 is fixed.
-        logger = logging.getLogger('retry-depwait')
-
-        # Get the MANUALDEPWAIT records for all archives.
-        store = ISlaveStore(BinaryPackageBuild)
-
-        candidates = store.find(
-            BinaryPackageBuild,
-            BinaryPackageBuild.distro_arch_series == distroarchseries,
-            BinaryPackageBuild.package_build == PackageBuild.id,
-            PackageBuild.build_farm_job == BuildFarmJob.id,
-            BuildFarmJob.status == BuildStatus.MANUALDEPWAIT)
-
-        # Materialise the results right away to avoid hitting the
-        # database multiple times.
-        candidates = list(candidates)
-
-        candidates_count = len(candidates)
-        if candidates_count == 0:
-            logger.info("No MANUALDEPWAIT record found.")
-            return
-
-        logger.info(
-            "Found %d builds in MANUALDEPWAIT state." % candidates_count)
-
-        for build in candidates:
-            if not build.can_be_retried:
-                continue
-            # We're changing 'build' so make sure we have an object from
-            # the master store.
-            build = IMasterObject(build)
-            try:
-                build.updateDependencies()
-            except UnparsableDependencies as e:
-                logger.error(e)
-                continue
-
-            if build.dependencies:
-                logger.debug(
-                    "Skipping %s: %s" % (build.title, build.dependencies))
-                continue
-            logger.info("Retrying %s" % build.title)
-            build.retry()
-            build.buildqueue_record.score()
-
     def getBuildsBySourcePackageRelease(self, sourcepackagerelease_ids,
                                         buildstate=None):
         """See `IBinaryPackageBuildSet`."""

=== removed file 'lib/lp/soyuz/scripts/buildd.py'
--- lib/lp/soyuz/scripts/buildd.py	2012-10-03 08:41:06 +0000
+++ lib/lp/soyuz/scripts/buildd.py	1970-01-01 00:00:00 +0000
@@ -1,73 +0,0 @@
-# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Buildd cronscript classes """
-
-__metaclass__ = type
-
-__all__ = [
-    'RetryDepwait',
-    ]
-
-from zope.component import getUtility
-
-from lp.app.errors import NotFoundError
-from lp.registry.interfaces.distribution import IDistributionSet
-from lp.registry.interfaces.series import SeriesStatus
-from lp.services.scripts.base import (
-    LaunchpadCronScript,
-    LaunchpadScriptFailure,
-    )
-from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
-
-
-class RetryDepwait(LaunchpadCronScript):
-
-    def add_my_options(self):
-        self.parser.add_option(
-            "-d", "--distribution", default="ubuntu",
-            help="Context distribution.")
-
-        self.parser.add_option(
-            "-n", "--dry-run",
-            dest="dryrun", action="store_true", default=False,
-            help="Whether or not to commit the transaction.")
-
-    def main(self):
-        """Retry all builds that do not fit in MANUALDEPWAIT.
-
-        Iterate over all supported series in the given distribution and
-        their architectures with existent chroots and update all builds
-        found in MANUALDEPWAIT status.
-        """
-        if self.args:
-            raise LaunchpadScriptFailure("Unhandled arguments %r" % self.args)
-
-        distribution_set = getUtility(IDistributionSet)
-        try:
-            distribution = distribution_set[self.options.distribution]
-        except NotFoundError:
-            raise LaunchpadScriptFailure(
-                "Could not find distribution: %s" % self.options.distribution)
-
-        # Iterate over all supported distroarchseries with available chroot.
-        build_set = getUtility(IBinaryPackageBuildSet)
-        for distroseries in distribution:
-            if distroseries.status == SeriesStatus.OBSOLETE:
-                self.logger.debug(
-                    "Skipping obsolete distroseries: %s" % distroseries.title)
-                continue
-            for distroarchseries in distroseries.architectures:
-                self.logger.info("Processing %s" % distroarchseries.title)
-                if not distroarchseries.getChroot:
-                    self.logger.debug("Chroot not found")
-                    continue
-                build_set.retryDepWaiting(distroarchseries)
-
-        # XXX cprov 20071122:  LaunchpadScript should provide some
-        # infrastructure for dry-run operations and not simply rely
-        # on the transaction being discarded by the garbage-collector.
-        # See further information in bug #165200.
-        if not self.options.dryrun:
-            self.logger.info('Commiting the transaction.')
-            self.txn.commit()

=== added file 'lib/lp/soyuz/scripts/retrydepwait.py'
--- lib/lp/soyuz/scripts/retrydepwait.py	1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/scripts/retrydepwait.py	2012-10-09 11:35:18 +0000
@@ -0,0 +1,86 @@
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+__all__ = [
+    'RetryDepwaitTunableLoop',
+    ]
+
+
+import transaction
+from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
+
+from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.model.buildfarmjob import BuildFarmJob
+from lp.registry.interfaces.series import SeriesStatus
+from lp.registry.model.sourcepackagename import SourcePackageName
+from lp.services.database.bulk import load_related
+from lp.services.database.lpstorm import IStore
+from lp.services.looptuner import TunableLoop
+from lp.soyuz.interfaces.binarypackagebuild import (
+    IBinaryPackageBuildSet,
+    UnparsableDependencies,
+    )
+from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild
+from lp.soyuz.model.distroarchseries import PocketChroot
+from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
+
+
+class RetryDepwaitTunableLoop(TunableLoop):
+
+    maximum_chunk_size = 5000
+
+    def __init__(self, log, dry_run, abort_time=None):
+        super(RetryDepwaitTunableLoop, self).__init__(log, abort_time)
+        self.dry_run = dry_run
+        self.start_at = 1
+        self.store = IStore(BinaryPackageBuild)
+
+    def findBuildFarmJobs(self):
+        return self.store.find(
+            BuildFarmJob,
+            BuildFarmJob.id >= self.start_at,
+            BuildFarmJob.status == BuildStatus.MANUALDEPWAIT,
+            ).order_by(BuildFarmJob.id)
+
+    def isDone(self):
+        return self.findBuildFarmJobs().is_empty()
+
+    def __call__(self, chunk_size):
+        chunk = [
+            removeSecurityProxy(build) for build in
+            getUtility(IBinaryPackageBuildSet).getByBuildFarmJobs(
+                list(self.findBuildFarmJobs()[:chunk_size]))]
+        sprs = load_related(
+            SourcePackageRelease, chunk, ['source_package_release_id'])
+        load_related(SourcePackageName, sprs, ['sourcepackagenameID'])
+        chroots = IStore(PocketChroot).find(
+            PocketChroot,
+            PocketChroot.distroarchseriesID.is_in(
+                b.distro_arch_series_id for b in chunk),
+            PocketChroot.chroot != None)
+        chroot_series = set(chroot.distroarchseriesID for chroot in chroots)
+        for build in chunk:
+            if (build.distro_arch_series.distroseries.status ==
+                    SeriesStatus.OBSOLETE
+                or not build.can_be_retried
+                or build.distro_arch_series_id not in chroot_series):
+                continue
+            try:
+                build.updateDependencies()
+            except UnparsableDependencies as e:
+                self.log.error(e)
+                continue
+
+            if not build.dependencies:
+                self.log.debug('Retrying %s', build.title)
+                build.retry()
+                build.buildqueue_record.score()
+
+        self.start_at = chunk[-1].package_build.build_farm_job_id + 1
+
+        if not self.dry_run:
+            transaction.commit()
+        else:
+            transaction.abort()

=== removed file 'lib/lp/soyuz/scripts/tests/test_buildd_cronscripts.py'
--- lib/lp/soyuz/scripts/tests/test_buildd_cronscripts.py	2012-10-03 07:47:08 +0000
+++ lib/lp/soyuz/scripts/tests/test_buildd_cronscripts.py	1970-01-01 00:00:00 +0000
@@ -1,147 +0,0 @@
-# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""cronscripts/buildd-* tests."""
-
-__metaclass__ = type
-
-import logging
-import os
-import subprocess
-import sys
-from unittest import TestCase
-
-from zope.component import getUtility
-
-from lp.buildmaster.enums import BuildStatus
-from lp.buildmaster.model.buildfarmjob import BuildFarmJob
-from lp.buildmaster.model.packagebuild import PackageBuild
-from lp.services.config import config
-from lp.services.database.lpstorm import IStore
-from lp.services.log.logger import BufferLogger
-from lp.services.scripts.base import LaunchpadScriptFailure
-from lp.soyuz.interfaces.component import IComponentSet
-from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild
-from lp.soyuz.scripts.buildd import RetryDepwait
-from lp.testing.layers import (
-    DatabaseLayer,
-    LaunchpadZopelessLayer,
-    )
-
-
-class TestCronscriptBase(TestCase):
-    """Buildd cronscripts test classes."""
-
-    def runCronscript(self, name, extra_args):
-        """Run given cronscript, returning the result and output.
-
-        Always set verbosity level.
-        """
-        # Scripts will write to the database.  The test runner won't see
-        # this and not know that the database needs restoring.
-        DatabaseLayer.force_dirty_database()
-
-        script = os.path.join(config.root, "cronscripts", name)
-        args = [sys.executable, script, "-v"]
-        args.extend(extra_args)
-        process = subprocess.Popen(
-            args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = process.communicate()
-        return (process.returncode, stdout, stderr)
-
-    def runBuilddRetryDepwait(self, extra_args=None):
-        if extra_args is None:
-            extra_args = []
-        return self.runCronscript("buildd-retry-depwait.py", extra_args)
-
-    def assertRuns(self, runner, *args):
-        """Invokes given runner with given arguments.
-
-        Asserts the result code is 0 (zero) and returns a triple containing:
-        (result_code, standart_output, error_output).
-        """
-        rc, out, err = runner()
-        self.assertEqual(0, rc, "Err:\n%s" % err)
-
-        return rc, out, err
-
-
-class TestRetryDepwait(TestCronscriptBase):
-    """Test RetryDepwait buildd script class."""
-    layer = LaunchpadZopelessLayer
-
-    def setUp(self):
-        """Store the number of pending builds present before run the tests."""
-        self.number_of_pending_builds = self.getPendingBuilds().count()
-
-    def getPendingBuilds(self):
-        pending_builds = IStore(BinaryPackageBuild).find(
-            BinaryPackageBuild,
-            BinaryPackageBuild.package_build == PackageBuild.id,
-            PackageBuild.build_farm_job == BuildFarmJob.id,
-            BuildFarmJob.status == BuildStatus.NEEDSBUILD)
-        return pending_builds
-
-    def getRetryDepwait(self, distribution=None):
-        test_args = ['-n']
-        if distribution is not None:
-            test_args.extend(['-d', distribution])
-
-        retry_depwait = RetryDepwait(
-            name='retry-depwait', test_args=test_args)
-        retry_depwait.logger = BufferLogger()
-
-        # `IBuildSet.retryDepwait` retrieve a specific logger instance
-        # from the global registry, we have to silence that too.
-        root_logger = logging.getLogger('retry-depwait')
-        root_logger.setLevel(logging.CRITICAL)
-
-        return retry_depwait
-
-    def testUnknownDistribution(self):
-        """A error is raised on unknown distributions."""
-        retry_depwait = self.getRetryDepwait(distribution='foobar')
-        self.assertRaises(LaunchpadScriptFailure, retry_depwait.main)
-
-    def testRunRetryDepwait(self):
-        """Check if actual buildd-retry-depwait script runs without errors."""
-        self.assertRuns(runner=self.runBuilddRetryDepwait)
-
-    def testEmptyRun(self):
-        """Check the results of a run against pristine sampledata.
-
-        Since the only record in MANUALDEPWAIT in sampledata can't be
-        satisfied we expect the number of pending builds to be constant.
-        """
-        retry_depwait = self.getRetryDepwait()
-        retry_depwait.main()
-        self.assertEqual(
-            self.number_of_pending_builds, self.getPendingBuilds().count())
-
-    def testWorkingRun(self):
-        """Modify sampledata and expects a new pending build to be created."""
-        depwait_build = BinaryPackageBuild.get(12)
-
-        # Moving the target source to universe, so it can reach the only
-        # published binary we have in sampledata.
-        source_release = depwait_build.distributionsourcepackagerelease
-        pub = source_release.publishing_history[0]
-        pub.component = getUtility(IComponentSet)['universe']
-
-        # Make it dependend on the only binary that can be satisfied in
-        # the sampledata.
-        depwait_build.dependencies = u'pmount'
-
-        self.layer.commit()
-
-        retry_depwait = self.getRetryDepwait()
-        retry_depwait.main()
-        self.layer.commit()
-
-        # Reload the build record after the multiple commits.
-        depwait_build = BinaryPackageBuild.get(12)
-        self.assertEqual(
-            self.number_of_pending_builds + 1,
-            self.getPendingBuilds().count())
-        self.assertEqual(depwait_build.status.name, 'NEEDSBUILD')
-        self.assertEqual(depwait_build.buildqueue_record.lastscore, 1755)

=== added file 'lib/lp/soyuz/scripts/tests/test_retrydepwait.py'
--- lib/lp/soyuz/scripts/tests/test_retrydepwait.py	1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/scripts/tests/test_retrydepwait.py	2012-10-09 11:35:18 +0000
@@ -0,0 +1,112 @@
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import transaction
+from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
+
+from lp.buildmaster.enums import BuildStatus
+from lp.registry.interfaces.series import SeriesStatus
+from lp.services.librarian.interfaces import ILibraryFileAliasSet
+from lp.services.log.logger import DevNullLogger
+from lp.soyuz.interfaces.publishing import PackagePublishingStatus
+from lp.soyuz.scripts.retrydepwait import RetryDepwaitTunableLoop
+from lp.testing import TestCaseWithFactory
+from lp.testing.dbuser import dbuser
+from lp.testing.fakemethod import FakeMethod
+from lp.testing.layers import ZopelessDatabaseLayer
+from lp.testing.script import run_script
+
+
+class TestRetryDepwait(TestCaseWithFactory):
+
+    layer = ZopelessDatabaseLayer
+
+    def setUp(self):
+        super(TestRetryDepwait, self).setUp()
+        self.chroot = getUtility(ILibraryFileAliasSet)[1]
+        self.build = removeSecurityProxy(
+            self.factory.makeBinaryPackageBuild(
+                status=BuildStatus.MANUALDEPWAIT))
+
+        # Most tests want a no-op updateDependencies and a chroot.
+        self.build.updateDependencies = FakeMethod()
+        self.setChroot()
+
+    def setChroot(self):
+        self.build.distro_arch_series.addOrUpdateChroot(self.chroot)
+
+    def unsetChroot(self):
+        self.build.distro_arch_series.addOrUpdateChroot(None)
+
+    def assertStatusAfterLoop(self, status, dry_run=False):
+        with dbuser('retry_depwait'):
+            RetryDepwaitTunableLoop(DevNullLogger(), dry_run).run()
+        self.assertEqual(status, self.build.status)
+
+    def test_ignores_when_dependencies_unsatisfied(self):
+        # Builds with unsatisfied dependencies are not retried.
+        self.build.dependencies = u'something'
+        self.assertStatusAfterLoop(BuildStatus.MANUALDEPWAIT)
+        self.assertEqual(1, self.build.updateDependencies.call_count)
+
+        self.build.dependencies = None
+        self.assertStatusAfterLoop(BuildStatus.NEEDSBUILD)
+        self.assertEqual(2, self.build.updateDependencies.call_count)
+
+    def test_ignores_when_series_is_obsolete(self):
+        # Builds for an obsolete series are not retried.
+        self.build.distro_arch_series.distroseries.status = (
+            SeriesStatus.OBSOLETE)
+        self.assertStatusAfterLoop(BuildStatus.MANUALDEPWAIT)
+
+        self.build.distro_arch_series.distroseries.status = (
+            SeriesStatus.DEVELOPMENT)
+        self.assertStatusAfterLoop(BuildStatus.NEEDSBUILD)
+
+    def test_ignores_when_chroot_is_missing(self):
+        # Builds without a chroot are not retried.
+        self.unsetChroot()
+        self.assertStatusAfterLoop(BuildStatus.MANUALDEPWAIT)
+
+        self.setChroot()
+        self.assertStatusAfterLoop(BuildStatus.NEEDSBUILD)
+
+    def test_dry_run_aborts(self):
+        # Changes are thrown away when in dry run mode.
+        self.assertStatusAfterLoop(BuildStatus.MANUALDEPWAIT, dry_run=True)
+        self.assertStatusAfterLoop(BuildStatus.NEEDSBUILD, dry_run=False)
+
+    def test_only_retries_depwait(self):
+        # Builds in non-depwait statuses aren't retried.
+        self.build.status = BuildStatus.FAILEDTOBUILD
+        self.assertStatusAfterLoop(BuildStatus.FAILEDTOBUILD)
+
+        self.build.status = BuildStatus.MANUALDEPWAIT
+        self.assertStatusAfterLoop(BuildStatus.NEEDSBUILD)
+
+    def runScript(self):
+        transaction.commit()
+        (ret, out, err) = run_script('cronscripts/buildd-retry-depwait.py')
+        self.assertEqual(0, ret)
+        transaction.commit()
+
+    def test_script(self):
+        # Setting up a real depwait scenario and running the script
+        # works.
+        self.assertEqual(BuildStatus.MANUALDEPWAIT, self.build.status)
+        self.build.dependencies = bpn = self.factory.getUniqueUnicode()
+
+        # With no binary to satisfy the dependency, running the script
+        # does nothing.
+        self.runScript()
+        self.assertEqual(BuildStatus.MANUALDEPWAIT, self.build.status)
+
+        # If we create a matching binary and rerun, the script retries
+        # the build.
+        self.factory.makeBinaryPackagePublishingHistory(
+            archive=self.build.archive, pocket=self.build.pocket,
+            distroarchseries=self.build.distro_arch_series,
+            status=PackagePublishingStatus.PUBLISHED, binarypackagename=bpn)
+        self.runScript()
+        self.assertEqual(BuildStatus.NEEDSBUILD, self.build.status)

=== modified file 'lib/lp/soyuz/tests/test_build_depwait.py'
--- lib/lp/soyuz/tests/test_build_depwait.py	2012-01-01 02:58:52 +0000
+++ lib/lp/soyuz/tests/test_build_depwait.py	2012-10-09 11:35:18 +0000
@@ -12,7 +12,6 @@
     ArchivePurpose,
     PackagePublishingStatus,
     )
-from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
 from lp.soyuz.interfaces.component import IComponentSet
 from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
 from lp.testing import (
@@ -99,22 +98,3 @@
         # Now that we have moved it main, we can see it.
         build.updateDependencies()
         self.assertEquals(u'', build.dependencies)
-
-    def test_retry_dep_waiting(self):
-        # Builds in MANUALDEPWAIT can be automatically retried.
-        spph = self.publisher.getPubSource(
-            sourcename=self.factory.getUniqueString(),
-            version="%s.1" % self.factory.getUniqueInteger(),
-            distroseries=self.distroseries, archive=self.archive)
-        [build] = spph.createMissingBuilds()
-        with person_logged_in(self.admin):
-            build.status = BuildStatus.MANUALDEPWAIT
-            # .createMissingBuilds() queues the build for us, and we need to
-            # undo that if we're about to retry it.
-            build.buildqueue_record.destroySelf()
-            build.dependencies = u''
-            # Commit to make sure stuff hits the database.
-            transaction.commit()
-        getUtility(IBinaryPackageBuildSet).retryDepWaiting(self.das)
-        self.assertEquals(BuildStatus.NEEDSBUILD, build.status)
-        self.assertTrue(build.buildqueue_record.lastscore > 0)


Follow ups