launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #16223
[Merge] lp:~wgrant/launchpad/antibfjo-7-no-set-pls into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/antibfjo-7-no-set-pls into lp:launchpad with lp:~wgrant/launchpad/antibfjo-6-db-prekill as a prerequisite.
Commit message:
Stop setting the old Job/BuildFarmJobOld columns; everything uses BuildQueue directly now.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/antibfjo-7-no-set-pls/+merge/195894
IBuildFarmJobOld and its implementations (BuildPackageJob, SourcePackageRecipeBuildJob, TranslationTemplatesBuildJob) are finally being replaced with an FK from BuildQueue to BuildFarmJob. Additionally, the two fields of Job that are used by BuildQueue are being inlined, and the FK to Job dropped, for performance reasons.
This branch stops creating/setting the old rows and columns. Basic code is retained to delete the old Job and BuildFarmJobOld if present, until the old data is purged in bulk.
--
https://code.launchpad.net/~wgrant/launchpad/antibfjo-7-no-set-pls/+merge/195894
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/antibfjo-7-no-set-pls into lp:launchpad.
=== modified file 'lib/lp/buildmaster/doc/buildqueue.txt'
--- lib/lp/buildmaster/doc/buildqueue.txt 2013-11-20 00:24:11 +0000
+++ lib/lp/buildmaster/doc/buildqueue.txt 2013-11-20 00:24:12 +0000
@@ -39,13 +39,9 @@
>>> build.status.name
'BUILDING'
-The static timestamps, representing when the record was initialized
-(inserted) and when the job was dispatched are provided as datetime
+The timestamp of when the job was dispatched is provided as datetime
instances:
- >>> bq.job.date_created
- datetime.datetime(2005, 6, 15, 9, 14, 12, 820778, tzinfo=<UTC>)
-
>>> bq.date_started
datetime.datetime(2005, 6, 15, 9, 20, 12, 820778, tzinfo=<UTC>)
=== modified file 'lib/lp/buildmaster/interfaces/buildfarmjob.py'
--- lib/lp/buildmaster/interfaces/buildfarmjob.py 2013-11-20 00:24:11 +0000
+++ lib/lp/buildmaster/interfaces/buildfarmjob.py 2013-11-20 00:24:12 +0000
@@ -203,15 +203,6 @@
default=0,
description=_("Number of consecutive failures for this job."))
- def makeJob():
- """Create the specific job relating this with an lp.services.job.
-
- XXX 2010-04-26 michael.nelson bug=567922
- Once all *Build classes are using BuildFarmJob we can lose the
- 'specific_job' attributes and simply have a reference to the
- services job directly on the BuildFarmJob.
- """
-
def setLog(log):
"""Set the `LibraryFileAlias` that contains the job log."""
=== modified file 'lib/lp/buildmaster/interfaces/buildqueue.py'
--- lib/lp/buildmaster/interfaces/buildqueue.py 2013-11-20 00:24:11 +0000
+++ lib/lp/buildmaster/interfaces/buildqueue.py 2013-11-20 00:24:12 +0000
@@ -25,16 +25,12 @@
)
from lp import _
-from lp.buildmaster.enums import (
- BuildFarmJobType,
- BuildQueueStatus,
- )
+from lp.buildmaster.enums import BuildQueueStatus
from lp.buildmaster.interfaces.builder import IBuilder
from lp.buildmaster.interfaces.buildfarmjob import (
IBuildFarmJob,
IBuildFarmJobOld,
)
-from lp.services.job.interfaces.job import IJob
from lp.soyuz.interfaces.processor import IProcessor
@@ -73,14 +69,6 @@
title=_("Status"), vocabulary=BuildQueueStatus, readonly=True,
description=_("The status of this build queue item."))
- job = Reference(
- IJob, title=_("Job"), required=True, readonly=True,
- description=_("Data common to all job types."))
-
- job_type = Choice(
- title=_('Job type'), required=True, vocabulary=BuildFarmJobType,
- description=_("The type of this job."))
-
estimated_duration = Timedelta(
title=_("Estimated Job Duration"), required=True,
description=_("Estimated job duration interval."))
=== modified file 'lib/lp/buildmaster/model/buildfarmjob.py'
--- lib/lp/buildmaster/model/buildfarmjob.py 2013-11-20 00:24:11 +0000
+++ lib/lp/buildmaster/model/buildfarmjob.py 2013-11-20 00:24:12 +0000
@@ -135,10 +135,6 @@
return None
return self.date_finished - self.date_started
- def makeJob(self):
- """See `IBuildFarmJobOld`."""
- raise NotImplementedError
-
@property
def buildqueue_record(self):
"""See `IBuildFarmJob`."""
@@ -223,16 +219,11 @@
def queueBuild(self, suspended=False):
"""See `IBuildFarmJob`."""
- specific_job = self.makeJob()
-
duration_estimate = self.estimateDuration()
- job = specific_job.job
queue_entry = BuildQueue(
estimated_duration=duration_estimate,
build_farm_job=self.build_farm_job,
- job_type=self.job_type,
- job=job, processor=self.processor,
- virtualized=self.virtualized)
+ processor=self.processor, virtualized=self.virtualized)
# This build queue job is to be created in a suspended state.
if suspended:
=== modified file 'lib/lp/buildmaster/model/buildqueue.py'
--- lib/lp/buildmaster/model/buildqueue.py 2013-11-20 00:24:11 +0000
+++ lib/lp/buildmaster/model/buildqueue.py 2013-11-20 00:24:12 +0000
@@ -55,7 +55,6 @@
)
from lp.services.database.enumcol import EnumCol
from lp.services.database.sqlbase import SQLBase
-from lp.services.job.interfaces.job import JobStatus
from lp.services.propertycache import (
cachedproperty,
get_property_cache,
@@ -103,11 +102,10 @@
_table = "BuildQueue"
_defaultOrder = "id"
- def __init__(self, build_farm_job, job, job_type=DEFAULT,
- estimated_duration=DEFAULT, virtualized=DEFAULT,
- processor=DEFAULT, lastscore=None):
- super(BuildQueue, self).__init__(_build_farm_job=build_farm_job,
- job_type=job_type, job=job, virtualized=virtualized,
+ def __init__(self, build_farm_job, estimated_duration=DEFAULT,
+ virtualized=DEFAULT, processor=DEFAULT, lastscore=None):
+ super(BuildQueue, self).__init__(
+ _build_farm_job=build_farm_job, virtualized=virtualized,
processor=processor, estimated_duration=estimated_duration,
lastscore=lastscore)
if lastscore is None and self.specific_build is not None:
@@ -118,8 +116,8 @@
status = EnumCol(enum=BuildQueueStatus, default=BuildQueueStatus.WAITING)
date_started = DateTime(tzinfo=pytz.UTC)
- job = ForeignKey(dbName='job', foreignKey='Job')
- job_type = EnumCol(
+ _job = ForeignKey(dbName='job', foreignKey='Job')
+ _job_type = EnumCol(
enum=BuildFarmJobType, notNull=True,
default=BuildFarmJobType.PACKAGEBUILD, dbName='job_type')
builder = ForeignKey(dbName='builder', foreignKey='Builder', default=None)
@@ -140,25 +138,21 @@
def _clear_specific_build_cache(self):
del get_property_cache(self).specific_build
- @cachedproperty
+ @property
def specific_old_job(self):
"""See `IBuildQueue`."""
- if self.job is None:
+ if self._job is None:
return None
- specific_class = specific_job_classes()[self.job_type]
- return specific_class.getByJob(self.job)
-
- def _clear_specific_old_job_cache(self):
- del get_property_cache(self).specific_old_job
+ specific_class = specific_job_classes()[self._job_type]
+ return specific_class.getByJob(self._job)
@staticmethod
def preloadSpecificBuild(queues):
from lp.buildmaster.model.buildfarmjob import BuildFarmJob
+ queues = [removeSecurityProxy(bq) for bq in queues]
load_related(BuildFarmJob, queues, ['_build_farm_job_id'])
- bfj_to_bq = dict(
- (removeSecurityProxy(bq)._build_farm_job, bq)
- for bq in queues)
- key = attrgetter('job_type')
+ bfj_to_bq = dict((bq._build_farm_job, bq) for bq in queues)
+ key = attrgetter('_build_farm_job.job_type')
for job_type, grouped_queues in groupby(queues, key=key):
source = getUtility(ISpecificBuildFarmJobSource, job_type.name)
builds = source.getByBuildFarmJobs(
@@ -177,8 +171,8 @@
return self._now() - date_started
def destroySelf(self):
- """Remove this record and associated job/specific_old_job."""
- job = self.job
+ """Remove this record."""
+ job = self._job
specific_old_job = self.specific_old_job
builder = self.builder
Store.of(self).remove(self)
@@ -189,7 +183,6 @@
job.destroySelf()
if builder is not None:
del get_property_cache(builder).currentjob
- self._clear_specific_old_job_cache()
self._clear_specific_build_cache()
def manualScore(self, value):
@@ -208,8 +201,6 @@
def markAsBuilding(self, builder):
"""See `IBuildQueue`."""
self.builder = builder
- if self.job is not None and self.job.status != JobStatus.RUNNING:
- self.job.start()
self.status = BuildQueueStatus.RUNNING
self.date_started = UTC_NOW
self.specific_build.updateStatus(BuildStatus.BUILDING)
@@ -220,29 +211,20 @@
"""See `IBuildQueue`."""
if self.status != BuildQueueStatus.WAITING:
raise AssertionError("Only waiting jobs can be suspended.")
- if self.job is not None:
- self.job.suspend()
self.status = BuildQueueStatus.SUSPENDED
def resume(self):
"""See `IBuildQueue`."""
if self.status != BuildQueueStatus.SUSPENDED:
raise AssertionError("Only suspended jobs can be resumed.")
- if self.job is not None:
- self.job.resume()
self.status = BuildQueueStatus.WAITING
def reset(self):
"""See `IBuildQueue`."""
builder = self.builder
self.builder = None
- if self.job is not None and self.job.status != JobStatus.WAITING:
- self.job.queue()
self.status = BuildQueueStatus.WAITING
self.date_started = None
- if self.job is not None:
- self.job.date_started = None
- self.job.date_finished = None
self.logtail = None
self.specific_build.updateStatus(BuildStatus.NEEDSBUILD)
if builder is not None:
=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
--- lib/lp/buildmaster/tests/test_builder.py 2013-11-20 00:24:11 +0000
+++ lib/lp/buildmaster/tests/test_builder.py 2013-11-20 00:24:12 +0000
@@ -18,7 +18,6 @@
from lp.buildmaster.tests.mock_slaves import make_publisher
from lp.services.database.interfaces import IStore
from lp.services.database.sqlbase import flush_database_updates
-from lp.services.job.interfaces.job import JobStatus
from lp.services.log.logger import BufferLogger
from lp.soyuz.enums import (
ArchivePurpose,
@@ -136,7 +135,6 @@
archive=archive).createMissingBuilds()
candidate = removeSecurityProxy(
self.frog_builder).acquireBuildCandidate()
- self.assertEqual(JobStatus.RUNNING, candidate.job.status)
self.assertEqual(BuildQueueStatus.RUNNING, candidate.status)
=== modified file 'lib/lp/buildmaster/tests/test_buildqueue.py'
--- lib/lp/buildmaster/tests/test_buildqueue.py 2013-11-20 00:24:11 +0000
+++ lib/lp/buildmaster/tests/test_buildqueue.py 2013-11-20 00:24:12 +0000
@@ -5,26 +5,22 @@
from datetime import timedelta
from storm.sqlobject import SQLObjectNotFound
-from zope import component
-from zope.component import getGlobalSiteManager
+from storm.store import Store
from zope.security.proxy import removeSecurityProxy
from lp.buildmaster.enums import (
BuildFarmJobType,
BuildStatus,
)
-from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJob
-from lp.buildmaster.model.buildfarmjob import BuildFarmJobMixin
-from lp.buildmaster.model.buildqueue import (
- BuildQueue,
- specific_job_classes,
- )
+from lp.buildmaster.model.buildqueue import BuildQueue
from lp.services.database.interfaces import IStore
+from lp.services.job.model.job import Job
from lp.soyuz.enums import (
ArchivePurpose,
PackagePublishingStatus,
)
from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild
+from lp.soyuz.model.buildpackagejob import BuildPackageJob
from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
from lp.testing import TestCaseWithFactory
from lp.testing.fakemethod import FakeMethod
@@ -78,6 +74,49 @@
queue_entry.lastscore)
+class TestBuildQueueOldJobDestruction(TestCaseWithFactory):
+
+ layer = ZopelessDatabaseLayer
+
+ def test_destroy_without_job(self):
+ # Newly created BuildQueues won't have an associated Job.
+ build = self.factory.makeBinaryPackageBuild()
+ bq = removeSecurityProxy(build.queueBuild())
+ self.assertIs(None, bq._job)
+ self.assertIs(
+ None, Store.of(build).find(BuildPackageJob, build=build).one())
+ bq.destroySelf()
+ self.assertIs(None, Store.of(build).find(BuildQueue, id=bq.id).one())
+
+ def test_destroy_with_job(self):
+ # Old BuildQueues will have a Job and an IBuildFarmJobOld during
+ # the migration. They're all destroyed.
+ build = self.factory.makeBinaryPackageBuild()
+ bq = removeSecurityProxy(build.queueBuild())
+ bfjo = removeSecurityProxy(build).makeJob()
+ job = bfjo.job
+ bq._job = job
+ bq._job_type = BuildFarmJobType.PACKAGEBUILD
+ self.assertIsNot(None, bq.specific_old_job)
+ self.assertIsNot(None, bq._job)
+ bq.destroySelf()
+ self.assertIs(None, Store.of(build).find(BuildQueue, id=bq.id).one())
+ self.assertIs(None, Store.of(build).find(Job, id=job.id).one())
+ self.assertIs(
+ None, Store.of(build).find(BuildPackageJob, id=bfjo.id).one())
+
+ def test_destroy_with_dangling_job(self):
+ # Old BuildQueues may even have a dangling Job FK between data
+ # cleaning and schema dropping. We ignore it and just kill the
+ # remaining BuildQueue.
+ build = self.factory.makeBinaryPackageBuild()
+ bq = removeSecurityProxy(build.queueBuild())
+ bq._jobID = 123456
+ bq._job_type = BuildFarmJobType.PACKAGEBUILD
+ bq.destroySelf()
+ self.assertIs(None, Store.of(build).find(BuildQueue, id=bq.id).one())
+
+
class TestBuildCancellation(TestCaseWithFactory):
"""Test cases for cancelling builds."""
@@ -89,7 +128,6 @@
def assertCancelled(self, build, bq):
self.assertEqual(BuildStatus.CANCELLED, build.status)
- self.assertIs(None, bq.specific_old_job)
self.assertRaises(SQLObjectNotFound, BuildQueue.get, bq.id)
def test_binarypackagebuild_cancel(self):
@@ -128,90 +166,6 @@
self.assertEqual(age, buildqueue.current_build_duration)
-class TestJobClasses(TestCaseWithFactory):
- """Tests covering build farm job type classes."""
- layer = LaunchpadZopelessLayer
-
- def setUp(self):
- """Set up a native x86 build for the test archive."""
- super(TestJobClasses, self).setUp()
-
- self.publisher = SoyuzTestPublisher()
- self.publisher.prepareBreezyAutotest()
-
- # First mark all builds in the sample data as already built.
- sample_data = IStore(BinaryPackageBuild).find(BinaryPackageBuild)
- for build in sample_data:
- build.buildstate = BuildStatus.FULLYBUILT
- IStore(BinaryPackageBuild).flush()
-
- # We test builds that target a primary archive.
- self.non_ppa = self.factory.makeArchive(
- name="primary", purpose=ArchivePurpose.PRIMARY)
- self.non_ppa.require_virtualized = False
-
- self.builds = []
- self.builds.extend(
- self.publisher.getPubSource(
- sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,
- archive=self.non_ppa).createMissingBuilds())
-
- def test_BuildPackageJob(self):
- """`BuildPackageJob` is one of the job type classes."""
- from lp.soyuz.model.buildpackagejob import BuildPackageJob
- _build, bq = find_job(self, 'gedit')
-
- # This is a binary package build.
- self.assertEqual(
- BuildFarmJobType.PACKAGEBUILD, bq.job_type,
- "This is a binary package build")
-
- # The class registered for 'PACKAGEBUILD' is `BuildPackageJob`.
- self.assertEqual(
- BuildPackageJob,
- specific_job_classes()[BuildFarmJobType.PACKAGEBUILD],
- "The class registered for 'PACKAGEBUILD' is `BuildPackageJob`")
-
- # The 'specific_job' object associated with this `BuildQueue`
- # instance is of type `BuildPackageJob`.
- self.assertTrue(bq.specific_old_job is not None)
- self.assertEqual(
- BuildPackageJob, bq.specific_old_job.__class__,
- "The 'specific_old_job' object associated with this `BuildQueue` "
- "instance is of type `BuildPackageJob`")
-
- def test_OtherTypeClasses(self):
- """Other job type classes are picked up as well."""
-
- class FakeBranchBuild(BuildFarmJobMixin):
- pass
-
- _build, bq = find_job(self, 'gedit')
- # First make sure that we don't have a job type class registered for
- # 'BRANCHBUILD' yet.
- self.assertTrue(
- specific_job_classes().get(BuildFarmJobType.BRANCHBUILD) is None)
-
- try:
- # Pretend that our `FakeBranchBuild` class implements the
- # `IBuildFarmJob` interface.
- component.provideUtility(
- FakeBranchBuild, IBuildFarmJob, 'BRANCHBUILD')
-
- # Now we should see the `FakeBranchBuild` class "registered"
- # in the `specific_job_classes` dictionary under the
- # 'BRANCHBUILD' key.
- self.assertEqual(
- specific_job_classes()[BuildFarmJobType.BRANCHBUILD],
- FakeBranchBuild)
- finally:
- # Just de-register the utility so we don't affect other
- # tests.
- site_manager = getGlobalSiteManager()
- site_manager.unregisterUtility(
- FakeBranchBuild, IBuildFarmJob, 'BRANCHBUILD')
-
-
class TestPlatformData(TestCaseWithFactory):
"""Tests covering the processor/virtualized properties."""
=== modified file 'lib/lp/buildmaster/tests/test_manager.py'
--- lib/lp/buildmaster/tests/test_manager.py 2013-11-20 00:24:11 +0000
+++ lib/lp/buildmaster/tests/test_manager.py 2013-11-20 00:24:12 +0000
@@ -59,7 +59,6 @@
)
from lp.registry.interfaces.distribution import IDistributionSet
from lp.services.config import config
-from lp.services.job.interfaces.job import JobStatus
from lp.services.log.logger import BufferLogger
from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
from lp.testing import (
@@ -122,7 +121,6 @@
self.assertEqual(job.builder, builder)
self.assertTrue(job.date_started is not None)
self.assertEqual(job.status, BuildQueueStatus.RUNNING)
- self.assertEqual(job.job.status, JobStatus.RUNNING)
build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(job)
self.assertEqual(build.status, BuildStatus.BUILDING)
self.assertEqual(job.logtail, logtail)
=== modified file 'lib/lp/buildmaster/tests/test_queuedepth.py'
--- lib/lp/buildmaster/tests/test_queuedepth.py 2013-11-20 00:24:11 +0000
+++ lib/lp/buildmaster/tests/test_queuedepth.py 2013-11-20 00:24:12 +0000
@@ -11,10 +11,7 @@
from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy
-from lp.buildmaster.enums import (
- BuildFarmJobType,
- BuildStatus,
- )
+from lp.buildmaster.enums import BuildStatus
from lp.buildmaster.interfaces.builder import IBuilderSet
from lp.buildmaster.model.buildqueue import BuildQueue
from lp.buildmaster.queuedepth import (
@@ -235,11 +232,8 @@
if recipe_build is None:
recipe_build = self.factory.makeSourcePackageRecipeBuild(
sourcename=sourcename)
- recipe_build_job = recipe_build.makeJob()
bq = BuildQueue(
- build_farm_job=recipe_build.build_farm_job,
- job=recipe_build_job.job, lastscore=score,
- job_type=BuildFarmJobType.RECIPEBRANCHBUILD,
+ build_farm_job=recipe_build.build_farm_job, lastscore=score,
estimated_duration=timedelta(seconds=estimated_duration),
virtualized=virtualized)
IStore(BuildQueue).add(bq)
=== modified file 'lib/lp/code/model/tests/test_sourcepackagerecipe.py'
--- lib/lp/code/model/tests/test_sourcepackagerecipe.py 2013-11-20 00:24:11 +0000
+++ lib/lp/code/model/tests/test_sourcepackagerecipe.py 2013-11-20 00:24:12 +0000
@@ -23,7 +23,10 @@
from zope.security.proxy import removeSecurityProxy
from lp.app.enums import InformationType
-from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.enums import (
+ BuildQueueStatus,
+ BuildStatus,
+ )
from lp.buildmaster.interfaces.buildqueue import IBuildQueue
from lp.buildmaster.model.buildqueue import BuildQueue
from lp.code.errors import (
@@ -40,26 +43,18 @@
)
from lp.code.interfaces.sourcepackagerecipebuild import (
ISourcePackageRecipeBuild,
- ISourcePackageRecipeBuildJob,
)
from lp.code.model.sourcepackagerecipe import (
NonPPABuildRequest,
SourcePackageRecipe,
)
-from lp.code.model.sourcepackagerecipebuild import (
- SourcePackageRecipeBuild,
- SourcePackageRecipeBuildJob,
- )
+from lp.code.model.sourcepackagerecipebuild import SourcePackageRecipeBuild
from lp.code.model.sourcepackagerecipedata import SourcePackageRecipeData
from lp.code.tests.helpers import recipe_parser_newest_version
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.interfaces.series import SeriesStatus
from lp.services.database.bulk import load_referencing
from lp.services.database.constants import UTC_NOW
-from lp.services.job.interfaces.job import (
- IJob,
- JobStatus,
- )
from lp.services.propertycache import clear_property_cache
from lp.services.webapp.authorization import check_permission
from lp.services.webapp.publisher import canonical_url
@@ -340,18 +335,13 @@
self.assertTrue(build.virtualized)
store = Store.of(build)
store.flush()
- build_job = store.find(SourcePackageRecipeBuildJob,
- SourcePackageRecipeBuildJob.build_id == build.id).one()
- self.assertProvides(build_job, ISourcePackageRecipeBuildJob)
- job = build_job.job
- self.assertProvides(job, IJob)
- self.assertEquals(job.status, JobStatus.WAITING)
build_queue = store.find(
BuildQueue,
BuildQueue._build_farm_job_id ==
removeSecurityProxy(build).build_farm_job_id).one()
self.assertProvides(build_queue, IBuildQueue)
self.assertTrue(build_queue.virtualized)
+ self.assertEquals(build_queue.status, BuildQueueStatus.WAITING)
def test_requestBuildRejectsNotPPA(self):
recipe = self.factory.makeSourcePackageRecipe()
=== modified file 'lib/lp/code/model/tests/test_sourcepackagerecipebuild.py'
--- lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2013-11-20 00:24:11 +0000
+++ lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2013-11-20 00:24:12 +0000
@@ -95,7 +95,7 @@
def test_makeJob(self):
# A build farm job can be obtained from a SourcePackageRecipeBuild
spb = self.makeSourcePackageRecipeBuild()
- job = spb.makeJob()
+ job = removeSecurityProxy(spb).makeJob()
self.assertProvides(job, ISourcePackageRecipeBuildJob)
def test_queueBuild(self):
@@ -106,7 +106,6 @@
self.assertEqual(
spb.build_farm_job, removeSecurityProxy(bq)._build_farm_job)
self.assertEqual(spb, bq.specific_build)
- self.assertProvides(bq.specific_old_job, ISourcePackageRecipeBuildJob)
self.assertEqual(True, bq.virtualized)
# The processor for SourcePackageRecipeBuilds should not be None.
@@ -151,17 +150,13 @@
with person_logged_in(owner):
recipe = self.factory.makeSourcePackageRecipe(branches=[branch])
build = self.factory.makeSourcePackageRecipeBuild(recipe=recipe)
- job = build.makeJob()
self.assertTrue(check_permission('launchpad.View', build))
- self.assertTrue(check_permission('launchpad.View', job))
removeSecurityProxy(branch).information_type = (
InformationType.USERDATA)
with person_logged_in(self.factory.makePerson()):
self.assertFalse(check_permission('launchpad.View', build))
- self.assertFalse(check_permission('launchpad.View', job))
login(ANONYMOUS)
self.assertFalse(check_permission('launchpad.View', build))
- self.assertFalse(check_permission('launchpad.View', job))
def test_view_private_archive(self):
"""Recipebuilds with private branches are restricted."""
@@ -169,15 +164,11 @@
archive = self.factory.makeArchive(owner=owner, private=True)
with person_logged_in(owner):
build = self.factory.makeSourcePackageRecipeBuild(archive=archive)
- job = build.makeJob()
self.assertTrue(check_permission('launchpad.View', build))
- self.assertTrue(check_permission('launchpad.View', job))
with person_logged_in(self.factory.makePerson()):
self.assertFalse(check_permission('launchpad.View', build))
- self.assertFalse(check_permission('launchpad.View', job))
login(ANONYMOUS)
self.assertFalse(check_permission('launchpad.View', build))
- self.assertFalse(check_permission('launchpad.View', job))
def test_estimateDuration(self):
# If there are no successful builds, estimate 10 minutes.
=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuild.py'
--- lib/lp/soyuz/tests/test_binarypackagebuild.py 2013-11-20 00:24:11 +0000
+++ lib/lp/soyuz/tests/test_binarypackagebuild.py 2013-11-20 00:24:12 +0000
@@ -10,7 +10,6 @@
import pytz
from simplejson import dumps
-from storm.store import Store
from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy
@@ -18,11 +17,9 @@
from lp.buildmaster.enums import BuildStatus
from lp.buildmaster.interfaces.buildqueue import IBuildQueue
from lp.buildmaster.interfaces.packagebuild import IPackageBuild
-from lp.buildmaster.model.buildqueue import BuildQueue
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.interfaces.series import SeriesStatus
from lp.registry.interfaces.sourcepackage import SourcePackageUrgency
-from lp.services.job.model.job import Job
from lp.services.log.logger import DevNullLogger
from lp.services.webapp.interaction import ANONYMOUS
from lp.services.webapp.interfaces import OAuthPermission
@@ -35,10 +32,8 @@
IBinaryPackageBuildSet,
UnparsableDependencies,
)
-from lp.soyuz.interfaces.buildpackagejob import IBuildPackageJob
from lp.soyuz.interfaces.component import IComponentSet
from lp.soyuz.model.binarypackagebuild import (
- BinaryPackageBuild,
BinaryPackageBuildSet,
COPY_ARCHIVE_SCORE_PENALTY,
PRIVATE_ARCHIVE_SCORE_BONUS,
@@ -46,7 +41,6 @@
SCORE_BY_POCKET,
SCORE_BY_URGENCY,
)
-from lp.soyuz.model.buildpackagejob import BuildPackageJob
from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
from lp.testing import (
anonymous_logged_in,
@@ -84,7 +78,6 @@
self.assertEqual(
self.build.build_farm_job, removeSecurityProxy(bq)._build_farm_job)
self.assertEqual(self.build, bq.specific_build)
- self.assertProvides(bq.specific_old_job, IBuildPackageJob)
self.assertEqual(self.build.is_virtualized, bq.virtualized)
self.assertIsNotNone(bq.processor)
self.assertEqual(bq, self.build.buildqueue_record)
@@ -217,42 +210,6 @@
slave_status={'dependencies': u'dep-bin'})
return depwait_build
- def testBuildqueueRemoval(self):
- """Test removing buildqueue items.
-
- Removing a Buildqueue row should also remove its associated
- BuildPackageJob and Job rows.
- """
- # Create a build in depwait.
- depwait_build = self._setupSimpleDepwaitContext()
- depwait_build_id = depwait_build.id
-
- # Grab the relevant DB IDs for later queries.
- store = Store.of(depwait_build)
- build_queue_id = depwait_build.buildqueue_record.id
- job_id = removeSecurityProxy(depwait_build.buildqueue_record.job).id
-
- depwait_build.buildqueue_record.destroySelf()
-
- # Test that the records above no longer exist in the db.
- self.assertEqual(
- store.find(
- BuildPackageJob,
- BuildPackageJob.build == depwait_build_id).count(),
- 0)
- self.assertEqual(
- store.find(Job, Job.id == job_id).count(),
- 0)
- self.assertEqual(
- store.find(BuildQueue, BuildQueue.id == build_queue_id).count(),
- 0)
- # But the build itself still exists.
- self.assertEqual(
- store.find(
- BinaryPackageBuild,
- BinaryPackageBuild.id == depwait_build_id).count(),
- 1)
-
def testUpdateDependenciesWorks(self):
# Calling `IBinaryPackageBuild.updateDependencies` makes the build
# record ready for dispatch.
=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py'
--- lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py 2013-11-20 00:24:11 +0000
+++ lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py 2013-11-20 00:24:12 +0000
@@ -47,7 +47,6 @@
)
from lp.registry.interfaces.series import SeriesStatus
from lp.services.config import config
-from lp.services.job.interfaces.job import JobStatus
from lp.services.librarian.interfaces import ILibraryFileAliasSet
from lp.services.log.logger import BufferLogger
from lp.soyuz.adapters.archivedependencies import (
@@ -395,7 +394,6 @@
self.assertIs(None, self.candidate.builder)
self.assertEqual(BuildStatus.NEEDSBUILD, self.build.status)
self.assertEqual(BuildQueueStatus.WAITING, self.candidate.status)
- self.assertEqual(JobStatus.WAITING, self.candidate.job.status)
d = self.updateBuild(
self.candidate, WaitingSlave('BuildStatus.BUILDERFAIL'))
@@ -455,7 +453,6 @@
self.assertEqual(score, self.candidate.lastscore)
self.assertEqual(BuildStatus.NEEDSBUILD, self.build.status)
self.assertEqual(BuildQueueStatus.WAITING, self.candidate.status)
- self.assertEqual(JobStatus.WAITING, self.candidate.job.status)
d = self.updateBuild(
self.candidate, WaitingSlave('BuildStatus.GIVENBACK'))
=== removed file 'lib/lp/soyuz/tests/test_buildpackagejob.py'
--- lib/lp/soyuz/tests/test_buildpackagejob.py 2013-11-20 00:24:11 +0000
+++ lib/lp/soyuz/tests/test_buildpackagejob.py 1970-01-01 00:00:00 +0000
@@ -1,178 +0,0 @@
-# Copyright 2009-2013 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Test BuildQueue features."""
-
-from datetime import timedelta
-
-from zope.component import getUtility
-from zope.security.proxy import removeSecurityProxy
-
-from lp.buildmaster.enums import BuildStatus
-from lp.buildmaster.interfaces.builder import IBuilderSet
-from lp.services.database.interfaces import IStore
-from lp.soyuz.enums import (
- ArchivePurpose,
- PackagePublishingStatus,
- )
-from lp.soyuz.interfaces.buildpackagejob import IBuildPackageJob
-from lp.soyuz.interfaces.processor import IProcessorSet
-from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild
-from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
-from lp.testing import TestCaseWithFactory
-from lp.testing.layers import LaunchpadZopelessLayer
-
-
-def find_job(test, name, processor='386'):
- """Find build and queue instance for the given source and processor."""
- for build in test.builds:
- if (build.source_package_release.name == name
- and build.processor.name == processor):
- return (build, build.buildqueue_record)
- return (None, None)
-
-
-def builder_key(build):
- """Return processor and virtualization for the given build."""
- return (build.processor.id, build.is_virtualized)
-
-
-def assign_to_builder(test, job_name, builder_number, processor='386'):
- """Simulate assigning a build to a builder."""
- def nth_builder(test, build, n):
- """Get builder #n for the given build processor and virtualization."""
- builder = None
- builders = test.builders.get(builder_key(build), [])
- try:
- builder = builders[n - 1]
- except IndexError:
- pass
- return builder
-
- build, bq = find_job(test, job_name, processor)
- builder = nth_builder(test, build, builder_number)
- bq.markAsBuilding(builder)
-
-
-class TestBuildJobBase(TestCaseWithFactory):
- """Setup the test publisher and some builders."""
-
- layer = LaunchpadZopelessLayer
-
- def setUp(self):
- super(TestBuildJobBase, self).setUp()
- self.publisher = SoyuzTestPublisher()
- self.publisher.prepareBreezyAutotest()
-
- self.i8 = self.factory.makeBuilder(name='i386-n-8', virtualized=False)
- self.i9 = self.factory.makeBuilder(name='i386-n-9', virtualized=False)
-
- processor = getUtility(IProcessorSet).getByName('hppa')
- self.h6 = self.factory.makeBuilder(
- name='hppa-n-6', processor=processor, virtualized=False)
- self.h7 = self.factory.makeBuilder(
- name='hppa-n-7', processor=processor, virtualized=False)
-
- self.builders = dict()
- # x86 native
- self.builders[(1, False)] = [self.i8, self.i9]
-
- # hppa native
- self.builders[(3, True)] = [self.h6, self.h7]
-
- # Ensure all builders are operational.
- for builders in self.builders.values():
- for builder in builders:
- builder.builderok = True
- builder.manual = False
-
- # Disable the sample data builders.
- getUtility(IBuilderSet)['bob'].builderok = False
- getUtility(IBuilderSet)['frog'].builderok = False
-
-
-class TestBuildPackageJob(TestBuildJobBase):
- """Test dispatch time estimates for binary builds (i.e. single build
- farm job type) targetting a single processor architecture and the primary
- archive.
- """
-
- def setUp(self):
- """Set up some native x86 builds for the test archive."""
- super(TestBuildPackageJob, self).setUp()
- # The builds will be set up as follows:
- #
- # j: 3 gedit p: hppa v:False e:0:01:00 *** s: 1001
- # j: 4 gedit p: 386 v:False e:0:02:00 *** s: 1002
- # j: 5 firefox p: hppa v:False e:0:03:00 *** s: 1003
- # j: 6 firefox p: 386 v:False e:0:04:00 *** s: 1004
- # j: 7 cobblers p: hppa v:False e:0:05:00 *** s: 1005
- # j: 8 cobblers p: 386 v:False e:0:06:00 *** s: 1006
- # j: 9 thunderpants p: hppa v:False e:0:07:00 *** s: 1007
- # j:10 thunderpants p: 386 v:False e:0:08:00 *** s: 1008
- # j:11 apg p: hppa v:False e:0:09:00 *** s: 1009
- # j:12 apg p: 386 v:False e:0:10:00 *** s: 1010
- # j:13 vim p: hppa v:False e:0:11:00 *** s: 1011
- # j:14 vim p: 386 v:False e:0:12:00 *** s: 1012
- # j:15 gcc p: hppa v:False e:0:13:00 *** s: 1013
- # j:16 gcc p: 386 v:False e:0:14:00 *** s: 1014
- # j:17 bison p: hppa v:False e:0:15:00 *** s: 1015
- # j:18 bison p: 386 v:False e:0:16:00 *** s: 1016
- # j:19 flex p: hppa v:False e:0:17:00 *** s: 1017
- # j:20 flex p: 386 v:False e:0:18:00 *** s: 1018
- # j:21 postgres p: hppa v:False e:0:19:00 *** s: 1019
- # j:22 postgres p: 386 v:False e:0:20:00 *** s: 1020
- #
- # j=job, p=processor, v=virtualized, e=estimated_duration, s=score
-
- # First mark all builds in the sample data as already built.
- store = IStore(BinaryPackageBuild)
- sample_data = store.find(BinaryPackageBuild)
- for build in sample_data:
- build.buildstate = BuildStatus.FULLYBUILT
- store.flush()
-
- # We test builds that target a primary archive.
- self.non_ppa = self.factory.makeArchive(
- name="primary", purpose=ArchivePurpose.PRIMARY)
- self.non_ppa.require_virtualized = False
-
- self.builds = []
- sourcenames = [
- "gedit",
- "firefox",
- "cobblers",
- "thunderpants",
- "apg",
- "vim",
- "gcc",
- "bison",
- "flex",
- "postgres",
- ]
- for sourcename in sourcenames:
- self.builds.extend(
- self.publisher.getPubSource(
- sourcename=sourcename,
- status=PackagePublishingStatus.PUBLISHED,
- archive=self.non_ppa,
- architecturehintlist='any').createMissingBuilds())
-
- # We want the builds to have a lot of variety when it comes to score
- # and estimated duration etc. so that the queries under test get
- # exercised properly.
- score = 1000
- duration = 0
- for build in self.builds:
- score += 1
- duration += 60
- bq = build.buildqueue_record
- bq.lastscore = score
- removeSecurityProxy(bq).estimated_duration = timedelta(
- seconds=duration)
-
- def test_providesInterfaces(self):
- # Ensure that a BuildPackageJob generates an appropriate cookie.
- build, bq = find_job(self, 'gcc', '386')
- build_farm_job = bq.specific_old_job
- self.assertProvides(build_farm_job, IBuildPackageJob)
=== modified file 'lib/lp/translations/tests/test_translationtemplatesbuild.py'
--- lib/lp/translations/tests/test_translationtemplatesbuild.py 2013-11-20 00:24:11 +0000
+++ lib/lp/translations/tests/test_translationtemplatesbuild.py 2013-11-20 00:24:12 +0000
@@ -14,10 +14,8 @@
from lp.app.enums import InformationType
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJob
-from lp.code.model.branchjob import BranchJob
from lp.code.model.directbranchcommit import DirectBranchCommit
from lp.codehosting.scanner import events
-from lp.services.database.interfaces import IStore
from lp.testing import TestCaseWithFactory
from lp.testing.dbuser import switch_dbuser
from lp.testing.layers import LaunchpadZopelessLayer
@@ -31,9 +29,6 @@
from lp.translations.model.translationtemplatesbuild import (
TranslationTemplatesBuild,
)
-from lp.translations.model.translationtemplatesbuildjob import (
- TranslationTemplatesBuildJob,
- )
class FakeTranslationTemplatesSource(TranslationTemplatesBuild):
@@ -123,7 +118,6 @@
def test_queueBuild(self):
build = self.factory.makeTranslationTemplatesBuild()
bq = build.queueBuild()
- self.assertEqual(build, bq.specific_old_job.build)
self.assertEqual(build, bq.specific_build)
self.assertEqual(
build.build_farm_job, removeSecurityProxy(bq)._build_farm_job)
@@ -131,15 +125,6 @@
self.assertEquals(
ubuntu.currentseries.nominatedarchindep.processor, bq.processor)
- # A job is created with the branch URL in its metadata.
- metadata = removeSecurityProxy(bq.specific_old_job).metadata
- self.assertIn('branch_url', metadata)
- url = metadata['branch_url']
- head = 'http://'
- self.assertEqual(head, url[:len(head)])
- tail = build.branch.name
- self.assertEqual(tail, url[-len(tail):])
-
def test_score(self):
# For now, these jobs always score themselves at 2510. In the
# future however the scoring system is to be revisited.
@@ -195,22 +180,17 @@
commit.writeFile('POTFILES.in', 'foo')
commit.commit('message')
notify(events.TipChanged(branch, None, False))
- branchjobs = list(TranslationTemplatesBuildJob.iterReady())
- self.assertEqual(1, len(branchjobs))
- self.assertEqual(branch, branchjobs[0].branch)
+ self.assertEqual(
+ 1, TranslationTemplatesBuild.findByBranch(branch).count())
def test_scheduleTranslationTemplatesBuild(self):
# If the feature is enabled, scheduleTranslationTemplatesBuild
# will schedule a templates build whenever a change is pushed to
# a branch that generates templates.
branch = self._makeTranslationBranch(fake_pottery_compatible=True)
-
self.jobsource.scheduleTranslationTemplatesBuild(branch)
-
- store = IStore(BranchJob)
- branchjobs = list(store.find(BranchJob, BranchJob.branch == branch))
- self.assertEqual(1, len(branchjobs))
- self.assertEqual(branch, branchjobs[0].branch)
+ self.assertEqual(
+ 1, TranslationTemplatesBuild.findByBranch(branch).count())
def test_findByBranch(self):
source = getUtility(ITranslationTemplatesBuildSource)
=== modified file 'lib/lp/translations/tests/test_translationtemplatesbuildjob.py'
--- lib/lp/translations/tests/test_translationtemplatesbuildjob.py 2013-11-14 09:37:03 +0000
+++ lib/lp/translations/tests/test_translationtemplatesbuildjob.py 2013-11-20 00:24:12 +0000
@@ -7,11 +7,11 @@
from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy
+from lp.buildmaster.enums import BuildFarmJobType
from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobOld
from lp.buildmaster.model.buildqueue import BuildQueue
from lp.code.interfaces.branch import IBranchSet
from lp.code.interfaces.branchjob import IBranchJob
-from lp.services.database.interfaces import IStore
from lp.services.job.model.job import Job
from lp.testing import (
TestCaseWithFactory,
@@ -43,7 +43,10 @@
super(TestTranslationTemplatesBuildJob, self).setUp()
self.jobset = getUtility(ITranslationTemplatesBuildJobSource)
self.build = self.factory.makeTranslationTemplatesBuild()
- self.build.queueBuild()
+ self.buildqueue = removeSecurityProxy(self.build.queueBuild())
+ bfjo = removeSecurityProxy(self.build).makeJob()
+ self.buildqueue._job = bfjo.job
+ self.buildqueue._job_type = BuildFarmJobType.TRANSLATIONTEMPLATESBUILD
self.specific_job = self.jobset.getByBranch(self.build.branch)
def test_new_TranslationTemplatesBuildJob(self):
@@ -70,17 +73,16 @@
# TranslationTemplatesBuildJob has its own customized cleanup
# behaviour, since it's actually a BranchJob.
job = removeSecurityProxy(self.specific_job.job)
- buildqueue = IStore(BuildQueue).find(BuildQueue, job=job).one()
job_id = job.id
store = Store.of(job)
branch_name = self.build.branch.unique_name
- buildqueue.destroySelf()
+ self.buildqueue.destroySelf()
# BuildQueue is gone.
self.assertIs(
- None, store.find(BuildQueue, BuildQueue.job == job_id).one())
+ None, store.find(BuildQueue, id=self.buildqueue.id).one())
# Job is gone.
self.assertIs(None, store.find(Job, Job.id == job_id).one())
# TranslationTemplatesBuildJob is gone.
@@ -99,4 +101,3 @@
def test_baseline(self):
utility = getUtility(ITranslationTemplatesBuildJobSource)
verifyObject(ITranslationTemplatesBuildJobSource, utility)
-
Follow ups
