launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #15067
[Merge] lp:~wgrant/launchpad/flatten-bfj-4-app-eliminate into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/flatten-bfj-4-app-eliminate into lp:launchpad with lp:~wgrant/launchpad/flatten-bfj-3.5-more-query as a prerequisite.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #758258 in Launchpad itself: "buildfarmjob schema is inefficient for reporting"
https://bugs.launchpad.net/launchpad/+bug/758258
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/flatten-bfj-4-app-eliminate/+merge/145820
The build farm job schema is being reworked to improve performance. Columns from PackageBuild and BuildFarmJob are being merged into tables that previously delegated to them. The PackageBuild table will end up dying entirely, but BuildFarmJob will remain, a shadow of its former self, to answer questions about Archive:+builds and Builder:+history. Additionally, BinaryPackageBuild is growing new distribution, distroseries, sourcepackagename and is_distro_archive columns to make searches even faster.
This branch removes the writes to deprecated columns. PackageBuild records are no longer created for BPB/SPRBs, and most of BuildFarmJob's columns have been removed.
--
https://code.launchpad.net/~wgrant/launchpad/flatten-bfj-4-app-eliminate/+merge/145820
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/flatten-bfj-4-app-eliminate into lp:launchpad.
=== modified file 'lib/lp/buildmaster/model/buildfarmjob.py'
--- lib/lp/buildmaster/model/buildfarmjob.py 2013-01-31 23:46:24 +0000
+++ lib/lp/buildmaster/model/buildfarmjob.py 2013-01-31 23:46:24 +0000
@@ -18,7 +18,6 @@
Or,
)
from storm.locals import (
- Bool,
DateTime,
Int,
Reference,
@@ -166,56 +165,37 @@
id = Int(primary=True)
- processor_id = Int(name='processor', allow_none=True)
- processor = Reference(processor_id, 'Processor.id')
-
- virtualized = Bool()
-
date_created = DateTime(
name='date_created', allow_none=False, tzinfo=pytz.UTC)
- date_started = DateTime(
- name='date_started', allow_none=True, tzinfo=pytz.UTC)
-
date_finished = DateTime(
name='date_finished', allow_none=True, tzinfo=pytz.UTC)
- date_first_dispatched = DateTime(
- name='date_first_dispatched', allow_none=True, tzinfo=pytz.UTC)
-
builder_id = Int(name='builder', allow_none=True)
builder = Reference(builder_id, 'Builder.id')
status = DBEnum(name='status', allow_none=False, enum=BuildStatus)
- log_id = Int(name='log', allow_none=True)
- log = Reference(log_id, 'LibraryFileAlias.id')
-
job_type = DBEnum(
name='job_type', allow_none=False, enum=BuildFarmJobType)
- failure_count = Int(name='failure_count', allow_none=False)
-
archive_id = Int(name='archive')
archive = Reference(archive_id, 'Archive.id')
def __init__(self, job_type, status=BuildStatus.NEEDSBUILD,
- processor=None, virtualized=None, date_created=None,
- builder=None, archive=None):
+ date_created=None, builder=None, archive=None):
super(BuildFarmJob, self).__init__()
- (self.job_type, self.status, self.processor, self.virtualized,
- self.builder, self.archive) = (
- job_type, status, processor, virtualized, builder, archive)
+ (self.job_type, self.status, self.builder, self.archive) = (
+ job_type, status, builder, archive)
if date_created is not None:
self.date_created = date_created
@classmethod
- def new(cls, job_type, status=BuildStatus.NEEDSBUILD, processor=None,
- virtualized=None, date_created=None, builder=None, archive=None):
+ def new(cls, job_type, status=BuildStatus.NEEDSBUILD, date_created=None,
+ builder=None, archive=None):
"""See `IBuildFarmJobSource`."""
build_farm_job = BuildFarmJob(
- job_type, status, processor, virtualized, date_created, builder,
- archive)
+ job_type, status, date_created, builder, archive)
store = IMasterStore(BuildFarmJob)
store.add(build_farm_job)
return build_farm_job
@@ -318,7 +298,7 @@
def setLog(self, log):
"""See `IBuildFarmJob`."""
- self.build_farm_job.log = self._new_log = log
+ self._new_log = log
def updateStatus(self, status, builder=None, slave_status=None,
date_started=None, date_finished=None):
@@ -337,10 +317,9 @@
# If we're starting to build, set date_started and
# date_first_dispatched if required.
if self.date_started is None and status == BuildStatus.BUILDING:
- self.build_farm_job.date_started = self._new_date_started = (
+ self._new_date_started = (
date_started or datetime.datetime.now(pytz.UTC))
if self.date_first_dispatched is None:
- self.build_farm_job.date_first_dispatched = self.date_started
self._new_date_first_dispatched = self.date_started
# If we're in a final build state (or UPLOADING, which sort of
@@ -357,7 +336,6 @@
def gotFailure(self):
"""See `IBuildFarmJob`."""
- self.build_farm_job.failure_count += 1
self._new_failure_count += 1
=== modified file 'lib/lp/buildmaster/model/packagebuild.py'
--- lib/lp/buildmaster/model/packagebuild.py 2013-01-31 23:46:24 +0000
+++ lib/lp/buildmaster/model/packagebuild.py 2013-01-31 23:46:24 +0000
@@ -158,10 +158,9 @@
if (status == BuildStatus.MANUALDEPWAIT and slave_status is not None
and slave_status.get('dependencies') is not None):
- self.package_build.dependencies = self._new_dependencies = (
- unicode(slave_status.get('dependencies')))
+ self._new_dependencies = unicode(slave_status.get('dependencies'))
else:
- self.package_build.dependencies = self._new_dependencies = None
+ self._new_dependencies = None
def verifySuccessfulUpload(self):
"""See `IPackageBuild`."""
@@ -197,7 +196,7 @@
"""See `IPackageBuild`."""
filename = "upload_%s_log.txt" % self.id
library_file = self.createUploadLog(content, filename=filename)
- self.package_build.upload_log = self._new_upload_log = library_file
+ self._new_upload_log = library_file
def notify(self, extra_info=None):
"""See `IPackageBuild`."""
=== modified file 'lib/lp/buildmaster/tests/test_buildfarmjob.py'
--- lib/lp/buildmaster/tests/test_buildfarmjob.py 2013-01-31 23:46:24 +0000
+++ lib/lp/buildmaster/tests/test_buildfarmjob.py 2013-01-31 23:46:24 +0000
@@ -92,16 +92,9 @@
# The job type is required to create a build farm job.
self.assertEqual(
BuildFarmJobType.PACKAGEBUILD, bfj.job_type)
- # Failure count defaults to zero.
- self.assertEqual(0, bfj.failure_count)
# Other attributes are unset by default.
- self.assertEqual(None, bfj.processor)
- self.assertEqual(None, bfj.virtualized)
- self.assertEqual(None, bfj.date_started)
self.assertEqual(None, bfj.date_finished)
- self.assertEqual(None, bfj.date_first_dispatched)
self.assertEqual(None, bfj.builder)
- self.assertEqual(None, bfj.log)
def test_date_created(self):
# date_created can be passed optionally when creating a
=== modified file 'lib/lp/buildmaster/tests/test_manager.py'
--- lib/lp/buildmaster/tests/test_manager.py 2013-01-30 07:49:52 +0000
+++ lib/lp/buildmaster/tests/test_manager.py 2013-01-31 23:46:24 +0000
@@ -322,7 +322,6 @@
builder.failure_count = builder_count
naked_job = removeSecurityProxy(builder.currentjob.specific_job)
- naked_job.build.build_farm_job.failure_count = job_count
naked_job.build._new_failure_count = job_count
# The _scanFailed() calls abort, so make sure our existing
# failure counts are persisted.
=== modified file 'lib/lp/buildmaster/tests/test_packagebuild.py'
--- lib/lp/buildmaster/tests/test_packagebuild.py 2013-01-31 23:46:24 +0000
+++ lib/lp/buildmaster/tests/test_packagebuild.py 2013-01-31 23:46:24 +0000
@@ -46,8 +46,7 @@
if archive is None:
archive = self.factory.makeArchive()
- bfj = getUtility(IBuildFarmJobSource).new(
- job_type, virtualized=True, status=status)
+ bfj = getUtility(IBuildFarmJobSource).new(job_type, status=status)
return getUtility(IPackageBuildSource).new(bfj, archive, pocket)
=== modified file 'lib/lp/code/model/sourcepackagerecipebuild.py'
--- lib/lp/code/model/sourcepackagerecipebuild.py 2013-01-31 23:46:24 +0000
+++ lib/lp/code/model/sourcepackagerecipebuild.py 2013-01-31 23:46:24 +0000
@@ -40,16 +40,12 @@
BuildStatus,
)
from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSource
-from lp.buildmaster.interfaces.packagebuild import IPackageBuildSource
from lp.buildmaster.model.buildfarmjob import (
BuildFarmJob,
BuildFarmJobOld,
)
from lp.buildmaster.model.buildqueue import BuildQueue
-from lp.buildmaster.model.packagebuild import (
- PackageBuild,
- PackageBuildMixin,
- )
+from lp.buildmaster.model.packagebuild import PackageBuildMixin
from lp.code.errors import (
BuildAlreadyPending,
BuildNotAllowedForDistro,
@@ -89,9 +85,6 @@
implements(ISourcePackageRecipeBuild)
classProvides(ISourcePackageRecipeBuildSource)
- package_build_id = Int(name='package_build', allow_none=False)
- package_build = Reference(package_build_id, 'PackageBuild.id')
-
build_farm_job_type = BuildFarmJobType.RECIPEBRANCHBUILD
job_type = build_farm_job_type
@@ -213,12 +206,11 @@
branch_name = self.recipe.base_branch.unique_name
return '%s recipe build' % branch_name
- def __init__(self, build_farm_job, package_build, distroseries, recipe,
- requester, archive, pocket, date_created):
+ def __init__(self, build_farm_job, distroseries, recipe, requester,
+ archive, pocket, date_created):
"""Construct a SourcePackageRecipeBuild."""
super(SourcePackageRecipeBuild, self).__init__()
self.build_farm_job = build_farm_job
- self.package_build = package_build
self.distroseries = distroseries
self.recipe = recipe
self.requester = requester
@@ -239,13 +231,11 @@
if date_created is None:
date_created = UTC_NOW
build_farm_job = getUtility(IBuildFarmJobSource).new(
- cls.build_farm_job_type, BuildStatus.NEEDSBUILD, None, True,
- date_created, None, archive)
- packagebuild = getUtility(IPackageBuildSource).new(
- build_farm_job, archive, pocket)
+ cls.build_farm_job_type, BuildStatus.NEEDSBUILD, date_created,
+ None, archive)
spbuild = cls(
- build_farm_job, packagebuild, distroseries, recipe, requester,
- archive, pocket, date_created)
+ build_farm_job, distroseries, recipe, requester, archive, pocket,
+ date_created)
store.add(spbuild)
return spbuild
@@ -315,9 +305,8 @@
SourcePackageRelease.source_package_recipe_build == self.id)
for release in releases:
release.source_package_recipe_build = None
- package_build = self.package_build
store.remove(self)
- package_build.destroySelf()
+ store.remove(self.build_farm_job)
@classmethod
def getByID(cls, build_id):
=== modified file 'lib/lp/code/model/tests/test_sourcepackagerecipebuild.py'
--- lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2013-01-23 10:16:18 +0000
+++ lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2013-01-31 23:46:24 +0000
@@ -22,7 +22,6 @@
from lp.buildmaster.enums import BuildStatus
from lp.buildmaster.interfaces.buildqueue import IBuildQueue
from lp.buildmaster.model.buildfarmjob import BuildFarmJob
-from lp.buildmaster.model.packagebuild import PackageBuild
from lp.code.interfaces.sourcepackagerecipebuild import (
ISourcePackageRecipeBuild,
ISourcePackageRecipeBuildJob,
@@ -479,14 +478,9 @@
naked_build = removeSecurityProxy(build)
# Ensure database ids are set.
store.flush()
- package_build_id = naked_build.package_build_id
- build_farm_job_id = naked_build.package_build.build_farm_job_id
+ build_farm_job_id = naked_build.build_farm_job_id
build.destroySelf()
- result = store.find(PackageBuild, PackageBuild.id == package_build_id)
- self.assertIs(None, result.one())
- result = store.find(
- BuildFarmJob, BuildFarmJob.id == build_farm_job_id)
- self.assertIs(None, result.one())
+ self.assertIs(None, store.get(BuildFarmJob, build_farm_job_id))
def test_cancelBuild(self):
# ISourcePackageRecipeBuild should make sure to remove jobs and build
=== modified file 'lib/lp/soyuz/model/binarypackagebuild.py'
--- lib/lp/soyuz/model/binarypackagebuild.py 2013-01-31 23:46:24 +0000
+++ lib/lp/soyuz/model/binarypackagebuild.py 2013-01-31 23:46:24 +0000
@@ -43,7 +43,6 @@
BuildStatus,
)
from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSource
-from lp.buildmaster.interfaces.packagebuild import IPackageBuildSource
from lp.buildmaster.model.builder import Builder
from lp.buildmaster.model.buildfarmjob import BuildFarmJob
from lp.buildmaster.model.buildqueue import BuildQueue
@@ -104,9 +103,6 @@
build_farm_job_type = BuildFarmJobType.PACKAGEBUILD
job_type = build_farm_job_type
- package_build_id = Int(name='package_build', allow_none=False)
- package_build = Reference(package_build_id, 'PackageBuild.id')
-
distro_arch_series_id = Int(name='distro_arch_series', allow_none=False)
distro_arch_series = Reference(
distro_arch_series_id, 'DistroArchSeries.id')
@@ -403,11 +399,11 @@
assert self.can_be_retried, "Build %s cannot be retried" % self.id
self.build_farm_job.status = self._new_status = BuildStatus.NEEDSBUILD
self.build_farm_job.date_finished = self._new_date_finished = None
- self.build_farm_job.date_started = self._new_date_started = None
+ self._new_date_started = None
self.build_farm_job.builder = self._new_builder = None
- self.build_farm_job.log = self._new_log = None
- self.package_build.upload_log = self._new_upload_log = None
- self.package_build.dependencies = self._new_upload_log = None
+ self._new_log = None
+ self._new_upload_log = None
+ self._new_dependencies = None
self.queueBuild()
def rescore(self, score):
@@ -557,8 +553,7 @@
if not self._isDependencySatisfied(token)]
# Update dependencies line
- self.package_build.dependencies = self._new_dependencies = (
- u", ".join(remaining_deps))
+ self._new_dependencies = u", ".join(remaining_deps)
def __getitem__(self, name):
return self.getBinaryPackageRelease(name)
@@ -854,17 +849,12 @@
archive, pocket, status=BuildStatus.NEEDSBUILD,
date_created=None, builder=None):
"""See `IBinaryPackageBuildSet`."""
- # Create the PackageBuild to which the new BinaryPackageBuild
- # will delegate.
+ # Create the BuildFarmJob for the new BinaryPackageBuild.
build_farm_job = getUtility(IBuildFarmJobSource).new(
- BinaryPackageBuild.build_farm_job_type, status, processor,
- archive.require_virtualized, date_created, builder, archive)
- package_build = getUtility(IPackageBuildSource).new(
- build_farm_job, archive, pocket)
-
+ BinaryPackageBuild.build_farm_job_type, status, date_created,
+ builder, archive)
binary_package_build = BinaryPackageBuild(
build_farm_job=build_farm_job,
- package_build=package_build,
distro_arch_series=distro_arch_series,
source_package_release=source_package_release,
_new_archive=archive, _new_pocket=pocket,
=== modified file 'lib/lp/translations/model/translationtemplatesbuild.py'
--- lib/lp/translations/model/translationtemplatesbuild.py 2013-01-31 23:46:24 +0000
+++ lib/lp/translations/model/translationtemplatesbuild.py 2013-01-31 23:46:24 +0000
@@ -138,7 +138,7 @@
"""See `ITranslationTemplatesBuildSource`."""
processor = cls._getBuildArch()
build_farm_job = getUtility(IBuildFarmJobSource).new(
- BuildFarmJobType.TRANSLATIONTEMPLATESBUILD, processor=processor)
+ BuildFarmJobType.TRANSLATIONTEMPLATESBUILD)
build = TranslationTemplatesBuild(build_farm_job, branch, processor)
store = cls._getStore()
store.add(build)
