launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #16213
[Merge] lp:~wgrant/launchpad/antibfjo-1-set into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/antibfjo-1-set into lp:launchpad with lp:~wgrant/launchpad/antibfjo-0-db as a prerequisite.
Commit message:
Set the new BuildQueue columns.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/antibfjo-1-set/+merge/195886
IBuildFarmJobOld and its implementations (BuildPackageJob, SourcePackageRecipeBuildJob, TranslationTemplatesBuildJob) are finally being replaced with an FK from BuildQueue to BuildFarmJob. Additionally, the two fields of Job that are used by BuildQueue are being inlined, and the FK to Job dropped, for performance reasons.
This branch adds the new columns to the model and sets them alongside the old columns. Existing rows will be backfilled by garbo.
--
https://code.launchpad.net/~wgrant/launchpad/antibfjo-1-set/+merge/195886
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/antibfjo-1-set into lp:launchpad.
=== modified file 'lib/lp/buildmaster/doc/buildqueue.txt'
--- lib/lp/buildmaster/doc/buildqueue.txt 2013-09-02 08:11:21 +0000
+++ lib/lp/buildmaster/doc/buildqueue.txt 2013-11-20 00:21:08 +0000
@@ -125,6 +125,8 @@
>>> print bob.currentjob
None
+ >>> print job.status.name
+ WAITING
>>> print job.builder
None
>>> print job.date_started
=== modified file 'lib/lp/buildmaster/enums.py'
--- lib/lp/buildmaster/enums.py 2013-01-07 02:40:55 +0000
+++ lib/lp/buildmaster/enums.py 2013-11-20 00:21:08 +0000
@@ -7,6 +7,7 @@
__all__ = [
'BuildStatus',
+ 'BuildQueueStatus',
'BuildFarmJobType',
]
@@ -147,3 +148,38 @@
Generate translation templates from a bazaar branch.
""")
+
+
+class BuildQueueStatus(DBEnumeratedType):
+ """Build queue status.
+
+ The status of a job in the build farm queue. The queue record only
+ exists while the job is running or waiting to run.
+
+ Not to be confused with BuildStatus, which is persistent and
+ includes values to represent the result of a completed job.
+ """
+
+ WAITING = DBItem(0, """
+ Waiting
+
+ The job is waiting to be run.
+ """)
+
+ RUNNING = DBItem(1, """
+ Running
+
+ The job is currently running.
+ """)
+
+ CANCELLING = DBItem(2, """
+ Cancelling
+
+ The job has been cancelled, so should be terminated.
+ """)
+
+ SUSPENDED = DBItem(3, """
+ Suspended
+
+ The job is suspended, so should not be run.
+ """)
=== modified file 'lib/lp/buildmaster/interfaces/buildqueue.py'
--- lib/lp/buildmaster/interfaces/buildqueue.py 2013-11-12 09:07:01 +0000
+++ lib/lp/buildmaster/interfaces/buildqueue.py 2013-11-20 00:21:08 +0000
@@ -25,7 +25,10 @@
)
from lp import _
-from lp.buildmaster.enums import BuildFarmJobType
+from lp.buildmaster.enums import (
+ BuildFarmJobType,
+ BuildQueueStatus,
+ )
from lp.buildmaster.interfaces.builder import IBuilder
from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJob
from lp.services.job.interfaces.job import IJob
@@ -63,6 +66,10 @@
description=_(
"The virtualization setting required by this build farm job."))
+ status = Choice(
+ title=_("Status"), vocabulary=BuildQueueStatus, readonly=True,
+ description=_("The status of this build queue item."))
+
job = Reference(
IJob, title=_("Job"), required=True, readonly=True,
description=_("Data common to all job types."))
@@ -91,6 +98,9 @@
def markAsBuilding(builder):
"""Set this queue item to a 'building' state."""
+ def suspend():
+ """Suspend this job, removing it from the active queue."""
+
def reset():
"""Reset this job, so it can be re-dispatched."""
=== modified file 'lib/lp/buildmaster/model/buildfarmjob.py'
--- lib/lp/buildmaster/model/buildfarmjob.py 2013-11-14 10:09:46 +0000
+++ lib/lp/buildmaster/model/buildfarmjob.py 2013-11-20 00:21:08 +0000
@@ -232,17 +232,19 @@
"""See `IBuildFarmJob`."""
specific_job = self.makeJob()
- # This build queue job is to be created in a suspended state.
- if suspended:
- specific_job.job.suspend()
-
duration_estimate = self.estimateDuration()
job = specific_job.job
queue_entry = BuildQueue(
estimated_duration=duration_estimate,
+ build_farm_job=self.build_farm_job,
job_type=self.job_type,
job=job, processor=self.processor,
virtualized=self.virtualized)
+
+ # This build queue job is to be created in a suspended state.
+ if suspended:
+ queue_entry.suspend()
+
Store.of(self).add(queue_entry)
return queue_entry
=== modified file 'lib/lp/buildmaster/model/buildqueue.py'
--- lib/lp/buildmaster/model/buildqueue.py 2013-11-12 09:07:01 +0000
+++ lib/lp/buildmaster/model/buildqueue.py 2013-11-20 00:21:08 +0000
@@ -22,12 +22,18 @@
IntervalCol,
StringCol,
)
+from storm.properties import (
+ DateTime,
+ Int,
+ )
+from storm.references import Reference
from storm.store import Store
from zope.component import getSiteManager
from zope.interface import implements
from lp.buildmaster.enums import (
BuildFarmJobType,
+ BuildQueueStatus,
BuildStatus,
)
from lp.buildmaster.interfaces.buildfarmjob import (
@@ -39,7 +45,10 @@
IBuildQueueSet,
)
from lp.services.database.bulk import load_related
-from lp.services.database.constants import DEFAULT
+from lp.services.database.constants import (
+ DEFAULT,
+ UTC_NOW,
+ )
from lp.services.database.enumcol import EnumCol
from lp.services.database.sqlbase import SQLBase
from lp.services.job.interfaces.job import JobStatus
@@ -91,14 +100,21 @@
_table = "BuildQueue"
_defaultOrder = "id"
- def __init__(self, job, job_type=DEFAULT, estimated_duration=DEFAULT,
- virtualized=DEFAULT, processor=DEFAULT, lastscore=None):
- super(BuildQueue, self).__init__(job_type=job_type, job=job,
- virtualized=virtualized, processor=processor,
- estimated_duration=estimated_duration, lastscore=lastscore)
+ def __init__(self, build_farm_job, job, job_type=DEFAULT,
+ estimated_duration=DEFAULT, virtualized=DEFAULT,
+ processor=DEFAULT, lastscore=None):
+ super(BuildQueue, self).__init__(_build_farm_job=build_farm_job,
+ job_type=job_type, job=job, virtualized=virtualized,
+ processor=processor, estimated_duration=estimated_duration,
+ lastscore=lastscore)
if lastscore is None and self.specific_job is not None:
self.score()
+ _build_farm_job_id = Int(name='build_farm_job')
+ _build_farm_job = Reference(_build_farm_job_id, 'BuildFarmJob.id')
+ status = EnumCol(enum=BuildQueueStatus, default=BuildQueueStatus.WAITING)
+ _date_started = DateTime(tzinfo=pytz.UTC, name='date_started')
+
job = ForeignKey(dbName='job', foreignKey='Job', notNull=True)
job_type = EnumCol(
enum=BuildFarmJobType, notNull=True,
@@ -191,16 +207,27 @@
self.builder = builder
if self.job.status != JobStatus.RUNNING:
self.job.start()
+ self.status = BuildQueueStatus.RUNNING
+ self._date_started = UTC_NOW
self.specific_build.updateStatus(BuildStatus.BUILDING)
if builder is not None:
del get_property_cache(builder).currentjob
+ def suspend(self):
+ """See `IBuildQueue`."""
+ if self.job.status != JobStatus.WAITING:
+ raise AssertionError("Only waiting jobs can be suspended.")
+ self.job.suspend()
+ self.status = BuildQueueStatus.SUSPENDED
+
def reset(self):
"""See `IBuildQueue`."""
builder = self.builder
self.builder = None
if self.job.status != JobStatus.WAITING:
self.job.queue()
+ self.status = BuildQueueStatus.WAITING
+ self._date_started = None
self.job.date_started = None
self.job.date_finished = None
self.logtail = None
=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
--- lib/lp/buildmaster/tests/test_builder.py 2013-11-14 07:36:26 +0000
+++ lib/lp/buildmaster/tests/test_builder.py 2013-11-20 00:21:08 +0000
@@ -6,7 +6,10 @@
from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy
-from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.enums import (
+ BuildQueueStatus,
+ BuildStatus,
+ )
from lp.buildmaster.interfaces.builder import (
IBuilder,
IBuilderSet,
@@ -134,6 +137,7 @@
candidate = removeSecurityProxy(
self.frog_builder).acquireBuildCandidate()
self.assertEqual(JobStatus.RUNNING, candidate.job.status)
+ self.assertEqual(BuildQueueStatus.RUNNING, candidate.status)
class TestFindBuildCandidatePPAWithSingleBuilder(TestCaseWithFactory):
=== modified file 'lib/lp/buildmaster/tests/test_manager.py'
--- lib/lp/buildmaster/tests/test_manager.py 2013-11-12 09:07:01 +0000
+++ lib/lp/buildmaster/tests/test_manager.py 2013-11-20 00:21:08 +0000
@@ -24,7 +24,10 @@
from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy
-from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.enums import (
+ BuildQueueStatus,
+ BuildStatus,
+ )
from lp.buildmaster.interactor import (
BuilderInteractor,
BuilderSlave,
@@ -56,6 +59,7 @@
)
from lp.registry.interfaces.distribution import IDistributionSet
from lp.services.config import config
+from lp.services.job.interfaces.job import JobStatus
from lp.services.log.logger import BufferLogger
from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
from lp.testing import (
@@ -111,7 +115,6 @@
def assertBuildingJob(self, job, builder, logtail=None):
"""Assert the given job is building on the given builder."""
- from lp.services.job.interfaces.job import JobStatus
if logtail is None:
logtail = 'Dummy sampledata entry, not processing'
@@ -119,6 +122,9 @@
self.assertEqual(job.builder, builder)
self.assertTrue(job.date_started is not None)
self.assertEqual(job.job.status, JobStatus.RUNNING)
+ # XXX: Only unconditional until sampledata is migrated.
+ if job.status is not None:
+ self.assertEqual(job.status, BuildQueueStatus.RUNNING)
build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(job)
self.assertEqual(build.status, BuildStatus.BUILDING)
self.assertEqual(job.logtail, logtail)
=== modified file 'lib/lp/buildmaster/tests/test_queuedepth.py'
--- lib/lp/buildmaster/tests/test_queuedepth.py 2013-11-14 07:36:26 +0000
+++ lib/lp/buildmaster/tests/test_queuedepth.py 2013-11-20 00:21:08 +0000
@@ -237,6 +237,7 @@
sourcename=sourcename)
recipe_build_job = recipe_build.makeJob()
bq = BuildQueue(
+ build_farm_job=recipe_build.build_farm_job,
job=recipe_build_job.job, lastscore=score,
job_type=BuildFarmJobType.RECIPEBRANCHBUILD,
estimated_duration=timedelta(seconds=estimated_duration),
=== modified file 'lib/lp/code/model/tests/test_sourcepackagerecipebuild.py'
--- lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2013-11-11 11:26:02 +0000
+++ lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2013-11-20 00:21:08 +0000
@@ -103,6 +103,8 @@
bq = spb.queueBuild(spb)
self.assertProvides(bq, IBuildQueue)
+ self.assertEqual(
+ spb.build_farm_job, removeSecurityProxy(bq)._build_farm_job)
self.assertProvides(bq.specific_job, ISourcePackageRecipeBuildJob)
self.assertEqual(True, bq.virtualized)
=== modified file 'lib/lp/soyuz/model/archive.py'
--- lib/lp/soyuz/model/archive.py 2013-09-13 07:09:55 +0000
+++ lib/lp/soyuz/model/archive.py 2013-11-20 00:21:08 +0000
@@ -52,7 +52,10 @@
re_isadeb,
re_issource,
)
-from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.enums import (
+ BuildQueueStatus,
+ BuildStatus,
+ )
from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSet
from lp.registry.enums import (
INCLUSIVE_TEAM_POLICY,
@@ -1933,18 +1936,34 @@
store = Store.of(self)
store.execute(query)
+ def _setBuildQueueStatuses(self, status):
+ """Update the pending BuildQueues' statuses for this archive."""
+ Store.of(self).execute("""
+ UPDATE BuildQueue SET status = %s
+ FROM BinaryPackageBuild, BuildPackageJob
+ WHERE
+ -- insert self.id here
+ BinaryPackageBuild.archive = %s
+ AND BuildPackageJob.build = BinaryPackageBuild.id
+ AND BuildPackageJob.job = BuildQueue.job
+ -- Build is in state BuildStatus.NEEDSBUILD (0)
+ AND BinaryPackageBuild.status = %s;
+ """, params=(status.value, self.id, BuildStatus.NEEDSBUILD.value))
+
def enable(self):
"""See `IArchive`."""
assert self._enabled == False, "This archive is already enabled."
assert self.is_active, "Deleted archives can't be enabled."
self._enabled = True
self._setBuildStatuses(JobStatus.WAITING)
+ self._setBuildQueueStatuses(BuildQueueStatus.WAITING)
def disable(self):
"""See `IArchive`."""
assert self._enabled == True, "This archive is already disabled."
self._enabled = False
self._setBuildStatuses(JobStatus.SUSPENDED)
+ self._setBuildQueueStatuses(BuildQueueStatus.SUSPENDED)
def delete(self, deleted_by):
"""See `IArchive`."""
=== modified file 'lib/lp/soyuz/tests/test_archive.py'
--- lib/lp/soyuz/tests/test_archive.py 2013-09-27 05:41:56 +0000
+++ lib/lp/soyuz/tests/test_archive.py 2013-11-20 00:21:08 +0000
@@ -24,7 +24,10 @@
from lp.app.errors import NotFoundError
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
-from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.enums import (
+ BuildQueueStatus,
+ BuildStatus,
+ )
from lp.registry.enums import (
PersonVisibility,
TeamMembershipPolicy,
@@ -316,6 +319,31 @@
# the specified status.
self.assertEqual(self._getBuildJobsByStatus(archive, status), count)
+ def _getBuildQueuesByStatus(self, archive, status):
+ # Return the count for archive build jobs with the given status.
+ query = """
+ SELECT COUNT(BuildQueue.id)
+ FROM BinaryPackageBuild, BuildPackageJob, BuildQueue
+ WHERE
+ BuildPackageJob.build = BinaryPackageBuild.id
+ AND BuildPackageJob.job = BuildQueue.job
+ AND BinaryPackageBuild.archive = %s
+ AND BinaryPackageBuild.status = %s
+ AND BuildQueue.status = %s;
+ """ % sqlvalues(archive, BuildStatus.NEEDSBUILD, status)
+
+ return IStore(Archive).execute(query).get_one()[0]
+
+ def assertNoBuildQueuesHaveStatus(self, archive, status):
+ # Check that that the jobs attached to this archive do not have this
+ # status.
+ self.assertEqual(self._getBuildQueuesByStatus(archive, status), 0)
+
+ def assertHasBuildQueuesWithStatus(self, archive, status, count):
+ # Check that that there are jobs attached to this archive that have
+ # the specified status.
+ self.assertEqual(self._getBuildQueuesByStatus(archive, status), count)
+
def test_enableArchive(self):
# Enabling an archive should set all the Archive's suspended builds to
# WAITING.
@@ -326,8 +354,11 @@
# disable the archive, as it is currently enabled
removeSecurityProxy(archive).disable()
self.assertHasBuildJobsWithStatus(archive, JobStatus.SUSPENDED, 1)
+ self.assertHasBuildQueuesWithStatus(
+ archive, BuildQueueStatus.SUSPENDED, 1)
removeSecurityProxy(archive).enable()
self.assertNoBuildJobsHaveStatus(archive, JobStatus.SUSPENDED)
+ self.assertNoBuildQueuesHaveStatus(archive, BuildQueueStatus.SUSPENDED)
self.assertTrue(archive.enabled)
def test_enableArchiveAlreadyEnabled(self):
@@ -343,8 +374,11 @@
archive=archive, status=BuildStatus.NEEDSBUILD)
build.queueBuild()
self.assertHasBuildJobsWithStatus(archive, JobStatus.WAITING, 1)
+ self.assertHasBuildQueuesWithStatus(
+ archive, BuildQueueStatus.WAITING, 1)
removeSecurityProxy(archive).disable()
self.assertNoBuildJobsHaveStatus(archive, JobStatus.WAITING)
+ self.assertNoBuildQueuesHaveStatus(archive, BuildQueueStatus.WAITING)
self.assertFalse(archive.enabled)
def test_disableArchiveAlreadyDisabled(self):
=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuild.py'
--- lib/lp/soyuz/tests/test_binarypackagebuild.py 2013-11-14 08:01:55 +0000
+++ lib/lp/soyuz/tests/test_binarypackagebuild.py 2013-11-20 00:21:08 +0000
@@ -81,6 +81,8 @@
# BinaryPackageBuild can create the queue entry for itself.
bq = self.build.queueBuild()
self.assertProvides(bq, IBuildQueue)
+ self.assertEqual(
+ self.build.build_farm_job, removeSecurityProxy(bq)._build_farm_job)
self.assertProvides(bq.specific_job, IBuildPackageJob)
self.assertEqual(self.build.is_virtualized, bq.virtualized)
self.assertIsNotNone(bq.processor)
=== modified file 'lib/lp/translations/tests/test_translationtemplatesbuild.py'
--- lib/lp/translations/tests/test_translationtemplatesbuild.py 2013-11-15 09:31:08 +0000
+++ lib/lp/translations/tests/test_translationtemplatesbuild.py 2013-11-20 00:21:08 +0000
@@ -125,6 +125,8 @@
bq = build.queueBuild()
self.assertEqual(build, bq.specific_job.build)
self.assertEqual(build, bq.specific_build)
+ self.assertEqual(
+ build.build_farm_job, removeSecurityProxy(bq)._build_farm_job)
ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
self.assertEquals(
ubuntu.currentseries.nominatedarchindep.processor, bq.processor)
Follow ups
