launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24739
[Merge] ~cjwatson/launchpad:dispatch-builders-by-group into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:dispatch-builders-by-group into launchpad:master with ~cjwatson/launchpad:bulk-logtail-update as a prerequisite.
Commit message:
Batch build candidate selection queries
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1866868 in Launchpad itself: "buildd-manager frequently gets stuck and stops gathering files from builders"
https://bugs.launchpad.net/launchpad/+bug/1866868
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/384139
We now select a batch of build candidates once per processor/virtualization combination per scan cycle, rather than once per idle builder per scan cycle. This should allow buildd-manager to scale much better to large numbers of builders.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:dispatch-builders-by-group into launchpad:master.
diff --git a/lib/lp/buildmaster/interactor.py b/lib/lp/buildmaster/interactor.py
index d02249b..07dedda 100644
--- a/lib/lp/buildmaster/interactor.py
+++ b/lib/lp/buildmaster/interactor.py
@@ -337,8 +337,9 @@ class BuilderSlave(object):
BuilderVitals = namedtuple(
'BuilderVitals',
- ('name', 'url', 'virtualized', 'vm_host', 'vm_reset_protocol',
- 'builderok', 'manual', 'build_queue', 'version', 'clean_status'))
+ ('name', 'url', 'processors', 'virtualized', 'vm_host',
+ 'vm_reset_protocol', 'builderok', 'manual', 'build_queue', 'version',
+ 'clean_status'))
_BQ_UNSPECIFIED = object()
@@ -347,9 +348,10 @@ def extract_vitals_from_db(builder, build_queue=_BQ_UNSPECIFIED):
if build_queue == _BQ_UNSPECIFIED:
build_queue = builder.currentjob
return BuilderVitals(
- builder.name, builder.url, builder.virtualized, builder.vm_host,
- builder.vm_reset_protocol, builder.builderok, builder.manual,
- build_queue, builder.version, builder.clean_status)
+ builder.name, builder.url, removeSecurityProxy(builder.processors),
+ builder.virtualized, builder.vm_host, builder.vm_reset_protocol,
+ builder.builderok, builder.manual, build_queue, builder.version,
+ builder.clean_status)
class BuilderInteractor(object):
@@ -500,20 +502,31 @@ class BuilderInteractor(object):
@classmethod
@defer.inlineCallbacks
- def findAndStartJob(cls, vitals, builder, slave):
+ def findAndStartJob(cls, vitals, builder, slave, builder_factory):
"""Find a job to run and send it to the buildd slave.
:return: A Deferred whose value is the `IBuildQueue` instance
found or None if no job was found.
"""
logger = cls._getSlaveScannerLogger()
- # XXX This method should be removed in favour of two separately
- # called methods that find and dispatch the job. It will
- # require a lot of test fixing.
- candidate = builder.acquireBuildCandidate()
+
+ # Find a build candidate. If we succeed, mark it as building
+ # immediately so that it is not dispatched by another builder in the
+ # build manager.
+ #
+ # We can consider this to be atomic, because although the build
+ # manager is a Twisted app and gives the appearance of doing lots of
+ # things at once, it's still single-threaded so no more than one
+ # builder scan can be in this code at the same time.
+ #
+ # If there's ever more than one build manager running at once, then
+ # this code will need some sort of mutex.
+ candidate = builder_factory.findBuildCandidate(vitals)
if candidate is None:
logger.debug("No build candidates available for builder.")
defer.returnValue(None)
+ candidate.markAsBuilding(builder)
+ transaction.commit()
new_behaviour = cls.getBuildBehaviour(candidate, builder, slave)
needed_bfjb = type(removeSecurityProxy(
diff --git a/lib/lp/buildmaster/interfaces/builder.py b/lib/lp/buildmaster/interfaces/builder.py
index aefaf26..ac798a5 100644
--- a/lib/lp/buildmaster/interfaces/builder.py
+++ b/lib/lp/buildmaster/interfaces/builder.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2014 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Builder interfaces."""
@@ -208,22 +208,6 @@ class IBuilderView(IHasBuildRecords, IHasOwner):
def failBuilder(reason):
"""Mark builder as failed for a given reason."""
- def acquireBuildCandidate():
- """Acquire a build candidate in an atomic fashion.
-
- When retrieiving a candidate we need to mark it as building
- immediately so that it is not dispatched by another builder in the
- build manager.
-
- We can consider this to be atomic because although the build manager
- is a Twisted app and gives the appearance of doing lots of things at
- once, it's still single-threaded so no more than one builder scan
- can be in this code at the same time.
-
- If there's ever more than one build manager running at once, then
- this code will need some sort of mutex.
- """
-
class IBuilderEdit(Interface):
@@ -298,6 +282,9 @@ class IBuilderSet(IBuilderSetAdmin):
def get(builder_id):
"""Return the IBuilder with the given builderid."""
+ def preloadProcessors(builders):
+ """Preload processors for a collection of `IBuilder`s."""
+
@collection_default_content()
def getBuilders():
"""Return all active configured builders."""
diff --git a/lib/lp/buildmaster/interfaces/buildfarmjob.py b/lib/lp/buildmaster/interfaces/buildfarmjob.py
index fef487d..79a948f 100644
--- a/lib/lp/buildmaster/interfaces/buildfarmjob.py
+++ b/lib/lp/buildmaster/interfaces/buildfarmjob.py
@@ -260,7 +260,7 @@ class ISpecificBuildFarmJobSource(Interface):
job.
"""
- def addCandidateSelectionCriteria(processor, virtualized):
+ def addCandidateSelectionCriteria():
"""Provide a sub-query to refine the candidate job selection.
Return a sub-query to narrow down the list of candidate jobs.
@@ -268,10 +268,6 @@ class ISpecificBuildFarmJobSource(Interface):
refer to the `BuildQueue` and `BuildFarmJob` tables already utilized
in the latter.
- :param processor: the type of processor that the candidate jobs are
- expected to run on.
- :param virtualized: whether the candidate jobs are expected to run on
- the `processor` natively or inside a virtual machine.
:return: a string containing a sub-query that narrows down the list of
candidate jobs.
"""
diff --git a/lib/lp/buildmaster/interfaces/buildqueue.py b/lib/lp/buildmaster/interfaces/buildqueue.py
index 9bcf346..c067428 100644
--- a/lib/lp/buildmaster/interfaces/buildqueue.py
+++ b/lib/lp/buildmaster/interfaces/buildqueue.py
@@ -144,3 +144,11 @@ class IBuildQueueSet(Interface):
def preloadForBuildFarmJobs(builds):
"""Preload buildqueue_record for the given IBuildFarmJobs."""
+
+ def findBuildCandidates(processor, virtualized, limit):
+ """Find candidate jobs for dispatch to idle builders.
+
+ :return: A sequence of up to `limit` `IBuildQueue` items with the
+ highest score that are for the given `processor` and that match
+ the given value of `virtualized`.
+ """
diff --git a/lib/lp/buildmaster/manager.py b/lib/lp/buildmaster/manager.py
index 8d9ffca..e4fb9f5 100644
--- a/lib/lp/buildmaster/manager.py
+++ b/lib/lp/buildmaster/manager.py
@@ -11,8 +11,10 @@ __all__ = [
'SlaveScanner',
]
+from collections import defaultdict
import datetime
import functools
+from itertools import chain
import logging
import six
@@ -49,6 +51,7 @@ from lp.buildmaster.interfaces.builder import (
CannotResumeHost,
IBuilderSet,
)
+from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet
from lp.buildmaster.model.builder import Builder
from lp.buildmaster.model.buildqueue import BuildQueue
from lp.services.database.bulk import dbify_value
@@ -72,6 +75,13 @@ JOB_RESET_THRESHOLD = 3
BUILDER_FAILURE_THRESHOLD = 5
+def sort_build_candidates(candidates):
+ # Re-sort a combined list of candidates. This must match the ordering
+ # used in BuildQueueSet.findBuildCandidates.
+ return sorted(
+ candidates, key=lambda candidate: (-candidate.lastscore, candidate.id))
+
+
class BuilderFactory:
"""A dumb builder factory that just talks to the DB."""
@@ -109,6 +119,14 @@ class BuilderFactory:
extract_vitals_from_db(b)
for b in getUtility(IBuilderSet).__iter__())
+ def findBuildCandidate(self, vitals):
+ """Find the next build candidate for this `BuilderVitals`, or None."""
+ bq_set = getUtility(IBuildQueueSet)
+ candidates = sort_build_candidates(chain.from_iterable(
+ bq_set.findBuildCandidates(processor, vitals.virtualized, 1)
+ for processor in vitals.processors + [None]))
+ return candidates[0] if candidates else None
+
class PrefetchedBuilderFactory:
"""A smart builder factory that does efficient bulk queries.
@@ -119,15 +137,28 @@ class PrefetchedBuilderFactory:
date_updated = None
+ @staticmethod
+ def _getBuilderGroupKeys(vitals):
+ return [
+ (processor, vitals.virtualized)
+ for processor in vitals.processors + [None]]
+
def update(self):
"""See `BuilderFactory`."""
transaction.abort()
- builders_and_bqs = IStore(Builder).using(
+ builders_and_bqs = list(IStore(Builder).using(
Builder, LeftJoin(BuildQueue, BuildQueue.builderID == Builder.id)
- ).find((Builder, BuildQueue))
+ ).find((Builder, BuildQueue)))
+ getUtility(IBuilderSet).preloadProcessors(
+ [b for b, _ in builders_and_bqs])
self.vitals_map = dict(
(b.name, extract_vitals_from_db(b, bq))
for b, bq in builders_and_bqs)
+ self.builder_groups = defaultdict(list)
+ for vitals in self.vitals_map.values():
+ for builder_group_key in self._getBuilderGroupKeys(vitals):
+ self.builder_groups[builder_group_key].append(vitals)
+ self.candidates_map = {}
transaction.abort()
self.date_updated = datetime.datetime.utcnow()
@@ -151,6 +182,22 @@ class PrefetchedBuilderFactory:
"""See `BuilderFactory`."""
return (b for n, b in sorted(six.iteritems(self.vitals_map)))
+ def findBuildCandidate(self, vitals):
+ """See `BuilderFactory`."""
+ bq_set = getUtility(IBuildQueueSet)
+ builder_group_keys = self._getBuilderGroupKeys(vitals)
+ for builder_group_key in builder_group_keys:
+ if builder_group_key not in self.candidates_map:
+ processor, virtualized = builder_group_key
+ self.candidates_map[builder_group_key] = (
+ bq_set.findBuildCandidates(
+ processor, virtualized,
+ len(self.builder_groups[builder_group_key])))
+ candidates = sorted(chain.from_iterable(
+ self.candidates_map[builder_group_key]
+ for builder_group_key in builder_group_keys))
+ return candidates.pop(0) if candidates else None
+
def judge_failure(builder_count, job_count, exc, retry=True):
"""Judge how to recover from a scan failure.
@@ -517,7 +564,8 @@ class SlaveScanner:
# attempt to just retry the scan; we need to reset
# the job so the dispatch will be reattempted.
builder = self.builder_factory[self.builder_name]
- d = interactor.findAndStartJob(vitals, builder, slave)
+ d = interactor.findAndStartJob(
+ vitals, builder, slave, self.builder_factory)
d.addErrback(functools.partial(self._scanFailed, False))
yield d
if builder.currentjob is not None:
diff --git a/lib/lp/buildmaster/model/builder.py b/lib/lp/buildmaster/model/builder.py
index a44e7fc..473b923 100644
--- a/lib/lp/buildmaster/model/builder.py
+++ b/lib/lp/buildmaster/model/builder.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -9,9 +9,6 @@ __all__ = [
'BuilderSet',
]
-import logging
-
-import six
from sqlobject import (
BoolCol,
ForeignKey,
@@ -20,23 +17,15 @@ from sqlobject import (
StringCol,
)
from storm.expr import (
- And,
Coalesce,
Count,
- Desc,
- Exists,
- Or,
- Select,
- SQL,
Sum,
)
from storm.properties import Int
from storm.references import Reference
from storm.store import Store
-import transaction
from zope.component import getUtility
from zope.interface import implementer
-from zope.security.proxy import removeSecurityProxy
from lp.app.errors import (
IncompatibleArguments,
@@ -53,11 +42,7 @@ from lp.buildmaster.interfaces.builder import (
)
from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSet
from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet
-from lp.buildmaster.model.buildfarmjob import BuildFarmJob
-from lp.buildmaster.model.buildqueue import (
- BuildQueue,
- specific_build_farm_job_sources,
- )
+from lp.buildmaster.model.buildqueue import BuildQueue
from lp.buildmaster.model.processor import Processor
from lp.registry.interfaces.person import validate_public_person
from lp.services.database.bulk import (
@@ -74,7 +59,6 @@ from lp.services.database.interfaces import (
)
from lp.services.database.sqlbase import SQLBase
from lp.services.database.stormbase import StormBase
-from lp.services.features import getFeatureFlag
from lp.services.propertycache import (
cachedproperty,
get_property_cache,
@@ -215,97 +199,6 @@ class Builder(SQLBase):
return getUtility(IBuildFarmJobSet).getBuildsForBuilder(
self, status=build_state, user=user)
- def _getSlaveScannerLogger(self):
- """Return the logger instance from buildd-slave-scanner.py."""
- # XXX cprov 20071120: Ideally the Launchpad logging system
- # should be able to configure the root-logger instead of creating
- # a new object, then the logger lookups won't require the specific
- # name argument anymore. See bug 164203.
- logger = logging.getLogger('slave-scanner')
- return logger
-
- def acquireBuildCandidate(self):
- """See `IBuilder`."""
- candidate = self._findBuildCandidate()
- if candidate is not None:
- candidate.markAsBuilding(self)
- transaction.commit()
- return candidate
-
- def _findBuildCandidate(self):
- """Find a candidate job for dispatch to an idle buildd slave.
-
- The pending BuildQueue item with the highest score for this builder
- or None if no candidate is available.
-
- :return: A candidate job.
- """
- logger = self._getSlaveScannerLogger()
-
- job_type_conditions = []
- job_sources = specific_build_farm_job_sources()
- for job_type, job_source in six.iteritems(job_sources):
- query = job_source.addCandidateSelectionCriteria(
- self.processor, self.virtualized)
- if query:
- job_type_conditions.append(
- Or(
- BuildFarmJob.job_type != job_type,
- Exists(SQL(query))))
-
- def get_int_feature_flag(flag):
- value_str = getFeatureFlag(flag)
- if value_str is not None:
- try:
- return int(value_str)
- except ValueError:
- logger.error('invalid %s %r', flag, value_str)
-
- score_conditions = []
- minimum_scores = set()
- for processor in self.processors:
- minimum_scores.add(get_int_feature_flag(
- 'buildmaster.minimum_score.%s' % processor.name))
- minimum_scores.add(get_int_feature_flag('buildmaster.minimum_score'))
- minimum_scores.discard(None)
- # If there are minimum scores set for any of the processors
- # supported by this builder, use the highest of them. This is a bit
- # weird and not completely ideal, but it's a safe conservative
- # option and avoids substantially complicating the candidate query.
- if minimum_scores:
- score_conditions.append(
- BuildQueue.lastscore >= max(minimum_scores))
-
- store = IStore(self.__class__)
- candidate_jobs = store.using(BuildQueue, BuildFarmJob).find(
- (BuildQueue.id,),
- BuildFarmJob.id == BuildQueue._build_farm_job_id,
- BuildQueue.status == BuildQueueStatus.WAITING,
- Or(
- BuildQueue.processorID.is_in(Select(
- BuilderProcessor.processor_id, tables=[BuilderProcessor],
- where=BuilderProcessor.builder == self)),
- BuildQueue.processor == None),
- BuildQueue.virtualized == self.virtualized,
- BuildQueue.builder == None,
- And(*(job_type_conditions + score_conditions))
- ).order_by(Desc(BuildQueue.lastscore), BuildQueue.id)
-
- # Only try the first handful of jobs. It's much easier on the
- # database, the chance of a large prefix of the queue being
- # bad candidates is negligible, and we want reasonably bounded
- # per-cycle performance even if the prefix is large.
- for (candidate_id,) in candidate_jobs[:10]:
- candidate = getUtility(IBuildQueueSet).get(candidate_id)
- job_source = job_sources[
- removeSecurityProxy(candidate)._build_farm_job.job_type]
- candidate_approved = job_source.postprocessCandidate(
- candidate, logger)
- if candidate_approved:
- return candidate
-
- return None
-
class BuilderProcessor(StormBase):
__storm_table__ = 'BuilderProcessor'
@@ -354,18 +247,20 @@ class BuilderSet(object):
"""See IBuilderSet."""
return Builder.select().count()
- def _preloadProcessors(self, rows):
+ def preloadProcessors(self, builders):
+ """See `IBuilderSet`."""
# Grab (Builder.id, Processor.id) pairs and stuff them into the
# Builders' processor caches.
store = IStore(BuilderProcessor)
+ builder_ids = [b.id for b in builders]
pairs = list(store.using(BuilderProcessor, Processor).find(
(BuilderProcessor.builder_id, BuilderProcessor.processor_id),
BuilderProcessor.processor_id == Processor.id,
- BuilderProcessor.builder_id.is_in([b.id for b in rows])).order_by(
+ BuilderProcessor.builder_id.is_in(builder_ids)).order_by(
BuilderProcessor.builder_id, Processor.name))
load(Processor, [pid for bid, pid in pairs])
- for row in rows:
- get_property_cache(row)._processors_cache = []
+ for builder in builders:
+ get_property_cache(builder)._processors_cache = []
for bid, pid in pairs:
cache = get_property_cache(store.get(Builder, bid))
cache._processors_cache.append(store.get(Processor, pid))
@@ -378,7 +273,7 @@ class BuilderSet(object):
Builder.virtualized, Builder.name)
def preload(rows):
- self._preloadProcessors(rows)
+ self.preloadProcessors(rows)
load_related(Person, rows, ['ownerID'])
bqs = getUtility(IBuildQueueSet).preloadForBuilders(rows)
BuildQueue.preloadSpecificBuild(bqs)
diff --git a/lib/lp/buildmaster/model/buildfarmjob.py b/lib/lp/buildmaster/model/buildfarmjob.py
index cd53136..e490204 100644
--- a/lib/lp/buildmaster/model/buildfarmjob.py
+++ b/lib/lp/buildmaster/model/buildfarmjob.py
@@ -251,7 +251,7 @@ class BuildFarmJobMixin:
class SpecificBuildFarmJobSourceMixin:
@staticmethod
- def addCandidateSelectionCriteria(processor, virtualized):
+ def addCandidateSelectionCriteria():
"""See `ISpecificBuildFarmJobSource`."""
return ('')
diff --git a/lib/lp/buildmaster/model/buildqueue.py b/lib/lp/buildmaster/model/buildqueue.py
index 6173a48..1e6bae3 100644
--- a/lib/lp/buildmaster/model/buildqueue.py
+++ b/lib/lp/buildmaster/model/buildqueue.py
@@ -11,9 +11,11 @@ __all__ = [
from datetime import datetime
from itertools import groupby
+import logging
from operator import attrgetter
import pytz
+import six
from sqlobject import (
BoolCol,
ForeignKey,
@@ -21,6 +23,13 @@ from sqlobject import (
IntervalCol,
StringCol,
)
+from storm.expr import (
+ And,
+ Desc,
+ Exists,
+ Or,
+ SQL,
+ )
from storm.properties import (
DateTime,
Int,
@@ -55,6 +64,7 @@ from lp.services.database.constants import (
from lp.services.database.enumcol import EnumCol
from lp.services.database.interfaces import IStore
from lp.services.database.sqlbase import SQLBase
+from lp.services.features import getFeatureFlag
from lp.services.propertycache import (
cachedproperty,
get_property_cache,
@@ -273,3 +283,81 @@ class BuildQueueSet(object):
removeSecurityProxy(build).build_farm_job_id)
get_property_cache(build).buildqueue_record = bq
return bqs
+
+ def _getSlaveScannerLogger(self):
+ """Return the logger instance from buildd-slave-scanner.py."""
+ # XXX cprov 20071120: Ideally the Launchpad logging system
+ # should be able to configure the root-logger instead of creating
+ # a new object, then the logger lookups won't require the specific
+ # name argument anymore. See bug 164203.
+ logger = logging.getLogger('slave-scanner')
+ return logger
+
+ def findBuildCandidates(self, processor, virtualized, limit):
+ """See `IBuildQueueSet`."""
+ # Circular import.
+ from lp.buildmaster.model.buildfarmjob import BuildFarmJob
+
+ logger = self._getSlaveScannerLogger()
+
+ job_type_conditions = []
+ job_sources = specific_build_farm_job_sources()
+ for job_type, job_source in six.iteritems(job_sources):
+ query = job_source.addCandidateSelectionCriteria()
+ if query:
+ job_type_conditions.append(
+ Or(
+ BuildFarmJob.job_type != job_type,
+ Exists(SQL(query))))
+
+ def get_int_feature_flag(flag):
+ value_str = getFeatureFlag(flag)
+ if value_str is not None:
+ try:
+ return int(value_str)
+ except ValueError:
+ logger.error('invalid %s %r', flag, value_str)
+
+ score_conditions = []
+ minimum_scores = set()
+ if processor is not None:
+ minimum_scores.add(get_int_feature_flag(
+ 'buildmaster.minimum_score.%s' % processor.name))
+ minimum_scores.add(get_int_feature_flag('buildmaster.minimum_score'))
+ minimum_scores.discard(None)
+ # If there are minimum scores set for any of the processors
+ # supported by this builder, use the highest of them. This is a bit
+ # weird and not completely ideal, but it's a safe conservative
+ # option and avoids substantially complicating the candidate query.
+ if minimum_scores:
+ score_conditions.append(
+ BuildQueue.lastscore >= max(minimum_scores))
+
+ store = IStore(BuildQueue)
+ candidate_jobs = store.using(BuildQueue, BuildFarmJob).find(
+ (BuildQueue.id,),
+ BuildFarmJob.id == BuildQueue._build_farm_job_id,
+ BuildQueue.status == BuildQueueStatus.WAITING,
+ BuildQueue.processor == processor,
+ BuildQueue.virtualized == virtualized,
+ BuildQueue.builder == None,
+ And(*(job_type_conditions + score_conditions))
+ ).order_by(Desc(BuildQueue.lastscore), BuildQueue.id)
+
+ # Only try a limited number of jobs. It's much easier on the
+ # database, the chance of a large prefix of the queue being
+ # bad candidates is negligible, and we want reasonably bounded
+ # per-cycle performance even if the prefix is large.
+ candidates = []
+ for (candidate_id,) in candidate_jobs[:max(limit * 2, 10)]:
+ candidate = getUtility(IBuildQueueSet).get(candidate_id)
+ job_source = job_sources[
+ removeSecurityProxy(candidate)._build_farm_job.job_type]
+ candidate_approved = job_source.postprocessCandidate(
+ candidate, logger)
+ if candidate_approved:
+ candidates.append(candidate)
+ if len(candidates) >= limit:
+ break
+
+ return candidates
diff --git a/lib/lp/buildmaster/tests/mock_slaves.py b/lib/lp/buildmaster/tests/mock_slaves.py
index 32633ea..3a42f8c 100644
--- a/lib/lp/buildmaster/tests/mock_slaves.py
+++ b/lib/lp/buildmaster/tests/mock_slaves.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Mock Build objects for tests soyuz buildd-system."""
@@ -56,14 +56,16 @@ class MockBuilder:
"""Emulates a IBuilder class."""
def __init__(self, name='mock-builder', builderok=True, manual=False,
- virtualized=True, vm_host=None, url='http://fake:0000',
- version=None, clean_status=BuilderCleanStatus.DIRTY,
+ processors=None, virtualized=True, vm_host=None,
+ url='http://fake:0000', version=None,
+ clean_status=BuilderCleanStatus.DIRTY,
vm_reset_protocol=BuilderResetProtocol.PROTO_1_1):
self.currentjob = None
self.builderok = builderok
self.manual = manual
self.url = url
self.name = name
+ self.processors = processors or []
self.virtualized = virtualized
self.vm_host = vm_host
self.vm_reset_protocol = vm_reset_protocol
diff --git a/lib/lp/buildmaster/tests/test_builder.py b/lib/lp/buildmaster/tests/test_builder.py
index a97d3c9..4200c7a 100644
--- a/lib/lp/buildmaster/tests/test_builder.py
+++ b/lib/lp/buildmaster/tests/test_builder.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Test Builder features."""
@@ -11,13 +11,13 @@ from zope.security.proxy import removeSecurityProxy
from lp.buildmaster.enums import (
BuilderCleanStatus,
- BuildQueueStatus,
BuildStatus,
)
from lp.buildmaster.interfaces.builder import (
IBuilder,
IBuilderSet,
)
+from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet
from lp.buildmaster.interfaces.processor import IProcessorSet
from lp.buildmaster.model.buildqueue import BuildQueue
from lp.buildmaster.tests.mock_slaves import make_publisher
@@ -98,16 +98,21 @@ class TestBuilder(TestCaseWithFactory):
self.assertEqual([proc], builder.processors)
-class TestFindBuildCandidateBase(TestCaseWithFactory):
+# XXX cjwatson 2020-05-18: All these tests would now make more sense in
+# lp.buildmaster.tests.test_buildqueue, and should be moved there when
+# convenient.
+class TestFindBuildCandidatesBase(TestCaseWithFactory):
"""Setup the test publisher and some builders."""
layer = LaunchpadZopelessLayer
def setUp(self):
- super(TestFindBuildCandidateBase, self).setUp()
+ super(TestFindBuildCandidatesBase, self).setUp()
self.publisher = make_publisher()
self.publisher.prepareBreezyAutotest()
+ self.proc_386 = getUtility(IProcessorSet).getByName('386')
+
# Create some i386 builders ready to build PPA builds. Two
# already exist in sampledata so we'll use those first.
self.builder1 = getUtility(IBuilderSet)['bob']
@@ -128,182 +133,167 @@ class TestFindBuildCandidateBase(TestCaseWithFactory):
builder.builderok = True
builder.manual = False
+ self.bq_set = getUtility(IBuildQueueSet)
+
-class TestFindBuildCandidateGeneralCases(TestFindBuildCandidateBase):
- # Test usage of findBuildCandidate not specific to any archive type.
+class TestFindBuildCandidatesGeneralCases(TestFindBuildCandidatesBase):
+ # Test usage of findBuildCandidates not specific to any archive type.
- def test_findBuildCandidate_matches_processor(self):
- # Builder._findBuildCandidate returns the highest scored build
- # for any of the builder's architectures.
+ def test_findBuildCandidates_matches_processor(self):
+ # BuildQueueSet.findBuildCandidates returns the highest scored build
+ # for the given processor and the given virtualization setting.
bq1 = self.factory.makeBinaryPackageBuild().queueBuild()
bq2 = self.factory.makeBinaryPackageBuild().queueBuild()
+ bq3 = self.factory.makeBinaryPackageBuild(
+ processor=bq2.processor).queueBuild()
- # With no job for the builder's processor, no job is returned.
+ # No job is returned for a fresh processor.
proc = self.factory.makeProcessor()
- builder = removeSecurityProxy(
- self.factory.makeBuilder(processors=[proc], virtualized=True))
- self.assertIs(None, builder._findBuildCandidate())
+ self.assertEqual([], self.bq_set.findBuildCandidates(proc, True, 3))
- # Once bq1's processor is added to the mix, it's the best
- # candidate.
- builder.processors = [proc, bq1.processor]
- self.assertEqual(bq1, builder._findBuildCandidate())
+ # bq1 is the best candidate for its processor.
+ self.assertEqual(
+ [bq1], self.bq_set.findBuildCandidates(bq1.processor, True, 3))
- # bq2's score doesn't matter, as its processor isn't suitable
- # for our builder.
+ # bq2's score doesn't matter when finding candidates for bq1's
+ # processor.
bq2.manualScore(3000)
- self.assertEqual(bq1, builder._findBuildCandidate())
-
- # But once we add bq2's processor, its higher score makes it win.
- builder.processors = [bq1.processor, bq2.processor]
- self.assertEqual(bq2, builder._findBuildCandidate())
-
- def test_findBuildCandidate_supersedes_builds(self):
- # IBuilder._findBuildCandidate identifies if there are builds
- # for superseded source package releases in the queue and marks
- # the corresponding build record as SUPERSEDED.
+ self.assertEqual([], self.bq_set.findBuildCandidates(proc, True, 3))
+ self.assertEqual(
+ [bq1], self.bq_set.findBuildCandidates(bq1.processor, True, 3))
+
+ # When looking at bq2's processor, the build with the higher score
+ # wins.
+ self.assertEqual(
+ [bq2, bq3],
+ self.bq_set.findBuildCandidates(bq2.processor, True, 3))
+ bq3.manualScore(4000)
+ self.assertEqual(
+ [bq3, bq2],
+ self.bq_set.findBuildCandidates(bq2.processor, True, 3))
+
+ def test_findBuildCandidates_supersedes_builds(self):
+ # BuildQueueSet.findBuildCandidates identifies if there are builds
+ # for superseded source package releases in the queue and marks the
+ # corresponding build record as SUPERSEDED.
archive = self.factory.makeArchive()
self.publisher.getPubSource(
sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,
archive=archive).createMissingBuilds()
- old_candidate = removeSecurityProxy(
- self.frog_builder)._findBuildCandidate()
+ old_candidates = self.bq_set.findBuildCandidates(
+ self.proc_386, True, 2)
# The candidate starts off as NEEDSBUILD:
build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(
- old_candidate)
+ old_candidates[0])
self.assertEqual(BuildStatus.NEEDSBUILD, build.status)
# Now supersede the source package:
publication = build.current_source_publication
publication.status = PackagePublishingStatus.SUPERSEDED
- # The candidate returned is now a different one:
- new_candidate = removeSecurityProxy(
- self.frog_builder)._findBuildCandidate()
- self.assertNotEqual(new_candidate, old_candidate)
+ # The list of candidates returned is now different:
+ new_candidates = self.bq_set.findBuildCandidates(
+ self.proc_386, True, 2)
+ self.assertNotEqual(new_candidates, old_candidates)
# And the old_candidate is superseded:
self.assertEqual(BuildStatus.SUPERSEDED, build.status)
- def test_findBuildCandidate_honours_minimum_score(self):
+ def test_findBuildCandidates_honours_limit(self):
+ # BuildQueueSet.findBuildCandidates returns no more than the number
+ # of candidates requested.
+ processor = self.factory.makeProcessor()
+ bqs = [
+ self.factory.makeBinaryPackageBuild(
+ processor=processor).queueBuild()
+ for _ in range(10)]
+
+ self.assertEqual(
+ bqs[:5], self.bq_set.findBuildCandidates(processor, True, 5))
+ self.assertEqual(
+ bqs, self.bq_set.findBuildCandidates(processor, True, 10))
+ self.assertEqual(
+ bqs, self.bq_set.findBuildCandidates(processor, True, 11))
+
+ build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(bqs[0])
+ build.current_source_publication.status = (
+ PackagePublishingStatus.SUPERSEDED)
+
+ self.assertEqual(
+ bqs[1:6], self.bq_set.findBuildCandidates(processor, True, 5))
+
+ def test_findBuildCandidates_honours_minimum_score(self):
# Sometimes there's an emergency that requires us to lock down the
# build farm except for certain whitelisted builds. We do this by
# way of a feature flag to set a minimum score; if this is set,
- # Builder._findBuildCandidate will ignore any build with a lower
- # score.
- bq1 = self.factory.makeBinaryPackageBuild().queueBuild()
- bq1.manualScore(100000)
- bq2 = self.factory.makeBinaryPackageBuild().queueBuild()
- bq2.manualScore(99999)
- builder1 = removeSecurityProxy(
- self.factory.makeBuilder(
- processors=[bq1.processor, self.factory.makeProcessor()],
- virtualized=True))
- builder2 = removeSecurityProxy(
- self.factory.makeBuilder(
- processors=[bq2.processor, self.factory.makeProcessor()],
- virtualized=True))
-
- # By default, each builder has the appropriate one of the two builds
- # we just created as a candidate.
- self.assertEqual(bq1, builder1._findBuildCandidate())
- self.assertEqual(bq2, builder2._findBuildCandidate())
+ # BuildQueueSet.findBuildCandidates will ignore any build with a
+ # lower score.
+ processors = []
+ bqs = []
+ for _ in range(2):
+ processors.append(self.factory.makeProcessor())
+ bqs.append([])
+ for score in (100000, 99999):
+ bq = self.factory.makeBinaryPackageBuild(
+ processor=processors[-1]).queueBuild()
+ bq.manualScore(score)
+ bqs[-1].append(bq)
+ processors.append(self.factory.makeProcessor())
+
+ # By default, each processor has the two builds we just created for
+ # it as candidates, with the highest score first.
+ self.assertEqual(
+ bqs[0], self.bq_set.findBuildCandidates(processors[0], True, 3))
+ self.assertEqual(
+ bqs[1], self.bq_set.findBuildCandidates(processors[1], True, 3))
# If we set a minimum score, then only builds above that threshold
# are candidates.
with FeatureFixture({'buildmaster.minimum_score': '100000'}):
- self.assertEqual(bq1, builder1._findBuildCandidate())
- self.assertIsNone(builder2._findBuildCandidate())
+ self.assertEqual(
+ [bqs[0][0]],
+ self.bq_set.findBuildCandidates(processors[0], True, 3))
+ self.assertEqual(
+ [bqs[1][0]],
+ self.bq_set.findBuildCandidates(processors[1], True, 3))
# We can similarly set a minimum score for individual processors.
- # The maximum of these for any processor supported by the builder is
- # used.
cases = [
- ({0: '99999'}, bq2),
- ({1: '99999'}, bq2),
- ({0: '100000'}, None),
- ({1: '100000'}, None),
- ({0: '99999', 1: '99999'}, bq2),
- ({0: '99999', 1: '100000'}, None),
- ({0: '100000', 1: '99999'}, None),
+ ({0: '99999'}, [bqs[0], bqs[1], []]),
+ ({1: '99999'}, [bqs[0], bqs[1], []]),
+ ({2: '99999'}, [bqs[0], bqs[1], []]),
+ ({0: '100000'}, [[bqs[0][0]], bqs[1], []]),
+ ({1: '100000'}, [bqs[0], [bqs[1][0]], []]),
+ ({2: '100000'}, [bqs[0], bqs[1], []]),
]
- for feature_spec, expected_bq in cases:
+ for feature_spec, expected_bqs in cases:
features = {
- 'buildmaster.minimum_score.%s' % builder2.processors[i].name:
- score
+ 'buildmaster.minimum_score.%s' % processors[i].name: score
for i, score in feature_spec.items()}
with FeatureFixture(features):
- self.assertEqual(expected_bq, builder2._findBuildCandidate())
+ for i, processor in enumerate(processors):
+ self.assertEqual(
+ expected_bqs[i],
+ self.bq_set.findBuildCandidates(processor, True, 3))
# If we set an invalid minimum score, buildd-manager doesn't
# explode.
with FakeLogger() as logger:
with FeatureFixture({'buildmaster.minimum_score': 'nonsense'}):
- self.assertEqual(bq1, builder1._findBuildCandidate())
- self.assertEqual(bq2, builder2._findBuildCandidate())
+ self.assertEqual(
+ bqs[0],
+ self.bq_set.findBuildCandidates(processors[0], True, 3))
+ self.assertEqual(
+ bqs[1],
+ self.bq_set.findBuildCandidates(processors[1], True, 3))
self.assertEqual(
"invalid buildmaster.minimum_score u'nonsense'\n"
"invalid buildmaster.minimum_score u'nonsense'\n",
logger.output)
- def test_acquireBuildCandidate_marks_building(self):
- # acquireBuildCandidate() should call _findBuildCandidate and
- # mark the build as building.
- archive = self.factory.makeArchive()
- self.publisher.getPubSource(
- sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,
- archive=archive).createMissingBuilds()
- candidate = removeSecurityProxy(
- self.frog_builder).acquireBuildCandidate()
- self.assertEqual(BuildQueueStatus.RUNNING, candidate.status)
-
-
-class TestFindBuildCandidatePPAWithSingleBuilder(TestCaseWithFactory):
-
- layer = LaunchpadZopelessLayer
-
- def setUp(self):
- super(TestFindBuildCandidatePPAWithSingleBuilder, self).setUp()
- self.publisher = make_publisher()
- self.publisher.prepareBreezyAutotest()
-
- self.bob_builder = getUtility(IBuilderSet)['bob']
- self.frog_builder = getUtility(IBuilderSet)['frog']
- # Disable bob so only frog is available.
- self.bob_builder.manual = True
- self.bob_builder.builderok = True
- self.frog_builder.manual = False
- self.frog_builder.builderok = True
-
- # Make a new PPA and give it some builds.
- self.ppa_joe = self.factory.makeArchive(name="joesppa")
- self.publisher.getPubSource(
- sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,
- archive=self.ppa_joe).createMissingBuilds()
-
- def test_findBuildCandidate_first_build_started(self):
- # The allocation rule for PPA dispatching doesn't apply when
- # there's only one builder available.
-
- # Asking frog to find a candidate should give us the joesppa build.
- next_job = removeSecurityProxy(
- self.frog_builder)._findBuildCandidate()
- build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)
- self.assertEqual('joesppa', build.archive.name)
-
- # If bob is in a failed state the joesppa build is still
- # returned.
- self.bob_builder.builderok = False
- self.bob_builder.manual = False
- next_job = removeSecurityProxy(
- self.frog_builder)._findBuildCandidate()
- build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)
- self.assertEqual('joesppa', build.archive.name)
-
-
-class TestFindBuildCandidatePPABase(TestFindBuildCandidateBase):
+class TestFindBuildCandidatesPPABase(TestFindBuildCandidatesBase):
ppa_joe_private = False
ppa_jim_private = False
@@ -324,7 +314,7 @@ class TestFindBuildCandidatePPABase(TestFindBuildCandidateBase):
def setUp(self):
"""Publish some builds for the test archive."""
- super(TestFindBuildCandidatePPABase, self).setUp()
+ super(TestFindBuildCandidatesPPABase, self).setUp()
# Create two PPAs and add some builds to each.
self.ppa_joe = self.factory.makeArchive(
@@ -374,32 +364,33 @@ class TestFindBuildCandidatePPABase(TestFindBuildCandidateBase):
self.assertEqual(num_free_builders, 2)
-class TestFindBuildCandidatePPA(TestFindBuildCandidatePPABase):
+class TestFindBuildCandidatesPPA(TestFindBuildCandidatesPPABase):
def test_findBuildCandidate(self):
# joe's fourth i386 build will be the next build candidate.
- next_job = removeSecurityProxy(self.builder4)._findBuildCandidate()
+ [next_job] = self.bq_set.findBuildCandidates(self.proc_386, True, 1)
build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)
self.assertEqual('joesppa', build.archive.name)
def test_findBuildCandidate_with_disabled_archive(self):
# Disabled archives should not be considered for dispatching
# builds.
- disabled_job = removeSecurityProxy(self.builder4)._findBuildCandidate()
+ [disabled_job] = self.bq_set.findBuildCandidates(
+ self.proc_386, True, 1)
build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(
disabled_job)
build.archive.disable()
- next_job = removeSecurityProxy(self.builder4)._findBuildCandidate()
+ [next_job] = self.bq_set.findBuildCandidates(self.proc_386, True, 1)
self.assertNotEqual(disabled_job, next_job)
-class TestFindBuildCandidatePrivatePPA(TestFindBuildCandidatePPABase):
+class TestFindBuildCandidatesPrivatePPA(TestFindBuildCandidatesPPABase):
ppa_joe_private = True
def test_findBuildCandidate_for_private_ppa(self):
# joe's fourth i386 build will be the next build candidate.
- next_job = removeSecurityProxy(self.builder4)._findBuildCandidate()
+ [next_job] = self.bq_set.findBuildCandidates(self.proc_386, True, 1)
build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)
self.assertEqual('joesppa', build.archive.name)
@@ -408,15 +399,15 @@ class TestFindBuildCandidatePrivatePPA(TestFindBuildCandidatePPABase):
# from the (password protected) repo area, not the librarian.
pub = build.current_source_publication
pub.status = PackagePublishingStatus.PENDING
- candidate = removeSecurityProxy(self.builder4)._findBuildCandidate()
+ [candidate] = self.bq_set.findBuildCandidates(self.proc_386, True, 1)
self.assertNotEqual(next_job.id, candidate.id)
-class TestFindBuildCandidateDistroArchive(TestFindBuildCandidateBase):
+class TestFindBuildCandidatesDistroArchive(TestFindBuildCandidatesBase):
def setUp(self):
"""Publish some builds for the test archive."""
- super(TestFindBuildCandidateDistroArchive, self).setUp()
+ super(TestFindBuildCandidatesDistroArchive, self).setUp()
# Create a primary archive and publish some builds for the
# queue.
self.non_ppa = self.factory.makeArchive(
@@ -432,29 +423,22 @@ class TestFindBuildCandidateDistroArchive(TestFindBuildCandidateBase):
def test_findBuildCandidate_for_non_ppa(self):
# Normal archives are not restricted to serial builds per
# arch.
-
- next_job = removeSecurityProxy(
- self.frog_builder)._findBuildCandidate()
- build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)
- self.assertEqual('primary', build.archive.name)
- self.assertEqual('gedit', build.source_package_release.name)
+ self.assertEqual(
+ [self.gedit_build.buildqueue_record,
+ self.firefox_build.buildqueue_record],
+ self.bq_set.findBuildCandidates(self.proc_386, True, 3))
# Now even if we set the build building, we'll still get the
# second non-ppa build for the same archive as the next candidate.
- build.updateStatus(BuildStatus.BUILDING, builder=self.frog_builder)
- next_job = removeSecurityProxy(
- self.frog_builder)._findBuildCandidate()
- build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job)
- self.assertEqual('primary', build.archive.name)
- self.assertEqual('firefox', build.source_package_release.name)
+ self.gedit_build.updateStatus(
+ BuildStatus.BUILDING, builder=self.frog_builder)
+ self.assertEqual(
+ [self.firefox_build.buildqueue_record],
+ self.bq_set.findBuildCandidates(self.proc_386, True, 3))
def test_findBuildCandidate_for_recipe_build(self):
# Recipe builds with a higher score are selected first.
# This test is run in a context with mixed recipe and binary builds.
-
- self.assertIsNot(self.frog_builder.processor, None)
- self.assertEqual(self.frog_builder.virtualized, True)
-
self.assertEqual(self.gedit_build.buildqueue_record.lastscore, 2505)
self.assertEqual(self.firefox_build.buildqueue_record.lastscore, 2505)
@@ -467,13 +451,14 @@ class TestFindBuildCandidateDistroArchive(TestFindBuildCandidateBase):
self.assertEqual(recipe_build_job.lastscore, 9999)
- next_job = removeSecurityProxy(
- self.frog_builder)._findBuildCandidate()
-
- self.assertEqual(recipe_build_job, next_job)
+ self.assertEqual(
+ [recipe_build_job,
+ self.gedit_build.buildqueue_record,
+ self.firefox_build.buildqueue_record],
+ self.bq_set.findBuildCandidates(self.proc_386, True, 3))
-class TestFindRecipeBuildCandidates(TestFindBuildCandidateBase):
+class TestFindRecipeBuildCandidates(TestFindBuildCandidatesBase):
# These tests operate in a "recipe builds only" setting.
# Please see also bug #507782.
@@ -504,11 +489,6 @@ class TestFindRecipeBuildCandidates(TestFindBuildCandidateBase):
def test_findBuildCandidate_with_highest_score(self):
# The recipe build with the highest score is selected first.
# This test is run in a "recipe builds only" context.
-
- self.assertIsNot(self.frog_builder.processor, None)
- self.assertEqual(self.frog_builder.virtualized, True)
-
- next_job = removeSecurityProxy(
- self.frog_builder)._findBuildCandidate()
-
- self.assertEqual(self.bq2, next_job)
+ self.assertEqual(
+ [self.bq2, self.bq1],
+ self.bq_set.findBuildCandidates(self.proc_386, True, 2))
diff --git a/lib/lp/buildmaster/tests/test_interactor.py b/lib/lp/buildmaster/tests/test_interactor.py
index 8514bb5..fa21e9e 100644
--- a/lib/lp/buildmaster/tests/test_interactor.py
+++ b/lib/lp/buildmaster/tests/test_interactor.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Test BuilderInteractor features."""
@@ -37,7 +37,6 @@ from twisted.internet import (
)
from twisted.internet.task import Clock
from twisted.python.failure import Failure
-from zope.security.proxy import removeSecurityProxy
from lp.buildmaster.enums import (
BuilderCleanStatus,
@@ -408,28 +407,30 @@ class TestBuilderInteractorDB(TestCaseWithFactory):
return builder, build
def test_findAndStartJob_returns_candidate(self):
- # findAndStartJob finds the next queued job using _findBuildCandidate.
+ # findAndStartJob finds the next queued job using findBuildCandidate.
# We don't care about the type of build at all.
builder, build = self._setupRecipeBuildAndBuilder()
candidate = build.queueBuild()
- # _findBuildCandidate is tested elsewhere, we just make sure that
+ builder_factory = MockBuilderFactory(builder, candidate)
+ # findBuildCandidate is tested elsewhere, we just make sure that
# findAndStartJob delegates to it.
- removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
- result=candidate)
+ builder_factory.findBuildCandidate = FakeMethod(result=candidate)
vitals = extract_vitals_from_db(builder)
- d = BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
+ d = BuilderInteractor.findAndStartJob(
+ vitals, builder, OkSlave(), builder_factory)
return d.addCallback(self.assertEqual, candidate)
def test_findAndStartJob_starts_job(self):
- # findAndStartJob finds the next queued job using _findBuildCandidate
+ # findAndStartJob finds the next queued job using findBuildCandidate
# and then starts it.
# We don't care about the type of build at all.
builder, build = self._setupRecipeBuildAndBuilder()
candidate = build.queueBuild()
- removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
- result=candidate)
+ builder_factory = MockBuilderFactory(builder, candidate)
+ builder_factory.findBuildCandidate = FakeMethod(result=candidate)
vitals = extract_vitals_from_db(builder)
- d = BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
+ d = BuilderInteractor.findAndStartJob(
+ vitals, builder, OkSlave(), builder_factory)
def check_build_started(candidate):
self.assertEqual(candidate.builder, builder)
@@ -443,23 +444,25 @@ class TestBuilderInteractorDB(TestCaseWithFactory):
builder, build = self._setupBinaryBuildAndBuilder()
builder.setCleanStatus(BuilderCleanStatus.DIRTY)
candidate = build.queueBuild()
- removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
- result=candidate)
+ builder_factory = MockBuilderFactory(builder, candidate)
+ builder_factory.findBuildCandidate = FakeMethod(result=candidate)
vitals = extract_vitals_from_db(builder)
with ExpectedException(
BuildDaemonIsolationError,
"Attempted to start build on a dirty slave."):
- yield BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
+ yield BuilderInteractor.findAndStartJob(
+ vitals, builder, OkSlave(), builder_factory)
@defer.inlineCallbacks
def test_findAndStartJob_dirties_slave(self):
# findAndStartJob marks its builder DIRTY before dispatching.
builder, build = self._setupBinaryBuildAndBuilder()
candidate = build.queueBuild()
- removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
- result=candidate)
+ builder_factory = MockBuilderFactory(builder, candidate)
+ builder_factory.findBuildCandidate = FakeMethod(result=candidate)
vitals = extract_vitals_from_db(builder)
- yield BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
+ yield BuilderInteractor.findAndStartJob(
+ vitals, builder, OkSlave(), builder_factory)
self.assertEqual(BuilderCleanStatus.DIRTY, builder.clean_status)
diff --git a/lib/lp/buildmaster/tests/test_manager.py b/lib/lp/buildmaster/tests/test_manager.py
index 9257ac8..6d16055 100644
--- a/lib/lp/buildmaster/tests/test_manager.py
+++ b/lib/lp/buildmaster/tests/test_manager.py
@@ -763,7 +763,8 @@ class TestPrefetchedBuilderFactory(TestCaseWithFactory):
def test_update(self):
# update grabs all of the Builders and their BuildQueues in a
- # single query.
+ # single query, plus an additional two queries to grab all the
+ # associated Processors.
builders = [self.factory.makeBuilder() for i in range(5)]
for i in range(3):
bq = self.factory.makeBinaryPackageBuild().queueBuild()
@@ -773,7 +774,7 @@ class TestPrefetchedBuilderFactory(TestCaseWithFactory):
pbf.update()
with StormStatementRecorder() as recorder:
pbf.update()
- self.assertThat(recorder, HasQueryCount(Equals(1)))
+ self.assertThat(recorder, HasQueryCount(Equals(3)))
def test_getVitals(self):
# PrefetchedBuilderFactory.getVitals looks up the BuilderVitals
diff --git a/lib/lp/soyuz/model/binarypackagebuild.py b/lib/lp/soyuz/model/binarypackagebuild.py
index 0551b82..998682a 100644
--- a/lib/lp/soyuz/model/binarypackagebuild.py
+++ b/lib/lp/soyuz/model/binarypackagebuild.py
@@ -1231,7 +1231,7 @@ class BinaryPackageBuildSet(SpecificBuildFarmJobSourceMixin):
BinaryPackageBuild, build_farm_job_id=bfj_id).one()
@staticmethod
- def addCandidateSelectionCriteria(processor, virtualized):
+ def addCandidateSelectionCriteria():
"""See `ISpecificBuildFarmJobSource`."""
private_statuses = (
PackagePublishingStatus.PUBLISHED,
