launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24151
[Merge] ~twom/launchpad:oci-ocirecipebuild into launchpad:master
Tom Wardill has proposed merging ~twom/launchpad:oci-ocirecipebuild into launchpad:master with ~twom/launchpad:concrete-oci-projects as a prerequisite.
Commit message:
Implement more of OCIRecipeBuild
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~twom/launchpad/+git/launchpad/+merge/376431
Using SnapBuild as a model, implement more of the required methods to meet the interfaces of IOCIRecipeBuild and IPackageBuild.
Ensure queueBuild works, for further work in to OCIRecipeBuildJob and BuildBehaviour
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~twom/launchpad:oci-ocirecipebuild into launchpad:master.
diff --git a/lib/lp/_schema_circular_imports.py b/lib/lp/_schema_circular_imports.py
index 7743294..eaeff51 100644
--- a/lib/lp/_schema_circular_imports.py
+++ b/lib/lp/_schema_circular_imports.py
@@ -97,7 +97,10 @@ from lp.hardwaredb.interfaces.hwdb import (
IHWVendorID,
)
from lp.oci.interfaces.ocirecipe import IOCIRecipe
-from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild
+from lp.oci.interfaces.ocirecipebuild import (
+ IOCIFile,
+ IOCIRecipeBuild,
+ )
from lp.registry.interfaces.commercialsubscription import (
ICommercialSubscription,
)
@@ -1094,3 +1097,6 @@ patch_entry_explicit_version(IWikiName, 'beta')
patch_collection_property(IOCIRecipe, 'builds', IOCIRecipeBuild)
patch_collection_property(IOCIRecipe, 'completed_builds', IOCIRecipeBuild)
patch_collection_property(IOCIRecipe, 'pending_builds', IOCIRecipeBuild)
+
+# IOCIRecipeBuild
+patch_reference_property(IOCIFile, 'layer_file_digest', IOCIRecipeBuild)
diff --git a/lib/lp/oci/interfaces/ocirecipebuild.py b/lib/lp/oci/interfaces/ocirecipebuild.py
index dff4d57..7a6827f 100644
--- a/lib/lp/oci/interfaces/ocirecipebuild.py
+++ b/lib/lp/oci/interfaces/ocirecipebuild.py
@@ -7,13 +7,18 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
+ 'IOCIFile',
'IOCIRecipeBuild',
- 'IOCIRecipeBuildSet'
+ 'IOCIRecipeBuildSet',
]
from lazr.restful.fields import Reference
from zope.interface import Interface
-from zope.schema import Text
+from zope.schema import (
+ Object,
+ Text,
+ TextLine,
+ )
from lp import _
from lp.buildmaster.interfaces.buildfarmjob import ISpecificBuildFarmJobSource
@@ -21,10 +26,18 @@ from lp.buildmaster.interfaces.packagebuild import IPackageBuild
from lp.oci.interfaces.ocirecipe import IOCIRecipe
from lp.services.database.constants import DEFAULT
from lp.services.fields import PublicPersonChoice
+from lp.services.librarian.interfaces import ILibraryFileAlias
class IOCIRecipeBuildEdit(Interface):
- pass
+
+ def addFile(lfa, layer_file_digest):
+ """Add an OCI file to this build.
+
+ :param lfa: An `ILibraryFileAlias`.
+ :param layer_file_digest: Digest for this file, used for image layers.
+ :return: An `IOCILayerFile`.
+ """
class IOCIRecipeBuildView(IPackageBuild):
@@ -44,6 +57,32 @@ class IOCIRecipeBuildView(IPackageBuild):
title=_("The name of the OCI recipe channel to build."),
required=True, readonly=True)
+ upload_log = Object(
+ schema=ILibraryFileAlias, required=False,
+ title=_('The LibraryFileAlias containing the upload log for a'
+ 'build resulting in an upload that could not be processed '
+ 'successfully. Otherwise it will be None.'))
+
+ upload_log_url = TextLine(
+ title=_("Upload Log URL"), required=False,
+ description=_("A URL for failed upload logs."
+ "Will be None if there was no failure."))
+
+ def getFileByFileName():
+ """Retrieve a file by filename
+
+ :return: A result set of (`IOCIFile`, `ILibraryFileAlias`,
+ `ILibraryFileContent`).
+ """
+
+ def getLayerFileByDigest(layer_file_digest):
+ """Retrieve a layer file by the digest.
+
+ :param layer_file_digest: The digest to look up.
+ :raises NotFoundError: if no file exists with the given digest.
+ :return: The corresponding `ILibraryFileAlias`.
+ """
+
class IOCIRecipeBuildAdmin(Interface):
pass
@@ -61,5 +100,24 @@ class IOCIRecipeBuildSet(ISpecificBuildFarmJobSource):
date_created=DEFAULT):
"""Create an `IOCIRecipeBuild`."""
- def preloadBuildsData(self, builds):
+ def preloadBuildsData(builds):
"""Load the data related to a list of OCI recipe builds."""
+
+
+class IOCIFile(Interface):
+ """A link between an OCI recipe build and a file in the librarian."""
+
+ build = Reference(
+ # Really IOCIBuild, patched in _schema_circular_imports.py.
+ Interface,
+ title=_("The OCI recipe build producing this file."),
+ required=True, readonly=True)
+
+ library_file = Reference(
+ ILibraryFileAlias, title=_("A file in the librarian."),
+ required=True, readonly=True)
+
+ layer_file_digest = TextLine(
+ title=_("Content-addressable hash of the file''s contents, "
+ "used for image layers."),
+ required=False, readonly=True)
diff --git a/lib/lp/oci/model/ocirecipebuild.py b/lib/lp/oci/model/ocirecipebuild.py
index 27fc29a..608fc2b 100644
--- a/lib/lp/oci/model/ocirecipebuild.py
+++ b/lib/lp/oci/model/ocirecipebuild.py
@@ -7,23 +7,30 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
+ 'OCIFile',
'OCIRecipeBuild',
'OCIRecipeBuildSet'
]
+from datetime import timedelta
+
import pytz
from storm.locals import (
Bool,
DateTime,
+ Desc,
Int,
Reference,
+ Store,
Storm,
Unicode,
)
+from storm.store import EmptyResultSet
from zope.component import getUtility
from zope.interface import implementer
+from lp.app.errors import NotFoundError
from lp.buildmaster.enums import (
BuildFarmJobType,
BuildStatus,
@@ -32,12 +39,47 @@ from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSource
from lp.buildmaster.model.buildfarmjob import SpecificBuildFarmJobSourceMixin
from lp.buildmaster.model.packagebuild import PackageBuildMixin
from lp.oci.interfaces.ocirecipebuild import (
+ IOCIFile,
IOCIRecipeBuild,
IOCIRecipeBuildSet,
)
+from lp.registry.model.person import Person
+from lp.services.database.bulk import load_related
from lp.services.database.constants import DEFAULT
+from lp.services.database.decoratedresultset import DecoratedResultSet
from lp.services.database.enumcol import DBEnum
-from lp.services.database.interfaces import IMasterStore
+from lp.services.database.interfaces import (
+ IMasterStore,
+ IStore,
+ )
+from lp.services.librarian.browser import ProxiedLibraryFileAlias
+from lp.services.librarian.model import (
+ LibraryFileAlias,
+ LibraryFileContent,
+ )
+
+
+@implementer(IOCIFile)
+class OCIFile(Storm):
+
+ __storm_table__ = 'OCIFile'
+
+ id = Int(name='id', primary=True)
+
+ build_id = Int(name='build', allow_none=False)
+ build = Reference(build_id, 'OCIRecipeBuild.id')
+
+ libraryfile_id = Int(name='library_file', allow_none=False)
+ library_file = Reference(libraryfile_id, 'LibraryFileAlias.id')
+
+ layer_file_digest = Unicode(name='layer_file_digest', allow_none=True)
+
+ def __init__(self, build, library_file, layer_file_digest=None):
+ """Construct a `OCIFile`."""
+ super(OCIFile, self).__init__()
+ self.build = build
+ self.library_file = library_file
+ self.layer_file_digest = layer_file_digest
@implementer(IOCIRecipeBuild)
@@ -86,6 +128,11 @@ class OCIRecipeBuild(PackageBuildMixin, Storm):
failure_count = Int(name='failure_count', allow_none=False)
+ # Stub attributes to match the IPackageBuild interface that we
+ # will not use in this implementation.
+ pocket = None
+ distro_series = None
+
def __init__(self, build_farm_job, requester, recipe, channel_name,
processor, virtualized, date_created):
@@ -98,10 +145,75 @@ class OCIRecipeBuild(PackageBuildMixin, Storm):
self.date_created = date_created
self.status = BuildStatus.NEEDSBUILD
- def queueBuild(self):
- """See `IPackageBuild`."""
- # XXX twom 2019-11-28 Currently a no-op skeleton, to be filled in
- return
+ @property
+ def upload_log_url(self):
+ """See `IOCIRecipeBuild`."""
+ if self.upload_log is None:
+ return None
+ return ProxiedLibraryFileAlias(self.upload_log, self).http_url
+
+ def getFileByFileName(self, filename):
+ result = Store.of(self).find(
+ (OCIFile, LibraryFileAlias, LibraryFileContent),
+ OCIFile.build == self.id,
+ LibraryFileAlias.id == OCIFile.libraryfile_id,
+ LibraryFileContent.id == LibraryFileAlias.contentID,
+ LibraryFileAlias.filename == filename)
+ return result.one()
+
+ def getLayerFileByDigest(self, layer_file_digest):
+ file_object = Store.of(self).find(
+ (OCIFile, LibraryFileAlias, LibraryFileContent),
+ OCIFile.build == self.id,
+ LibraryFileAlias.id == OCIFile.libraryfile_id,
+ LibraryFileContent.id == LibraryFileAlias.contentID,
+ OCIFile.layer_file_digest == layer_file_digest).one()
+ if file_object is not None:
+ return file_object
+ raise NotFoundError(layer_file_digest)
+
+ def addFile(self, lfa, layer_file_digest=None):
+ oci_file = OCIFile(
+ build=self, library_file=lfa, layer_file_digest=layer_file_digest)
+ IMasterStore(OCIFile).add(oci_file)
+ return oci_file
+
+ @property
+ def archive(self):
+ # XXX twom 2019-12-05 This may need to change when an OCIProject
+ # pillar isn't just a distribution
+ return self.recipe.ociproject.distribution.main_archive
+
+ @property
+ def distribution(self):
+ # XXX twom 2019-12-05 This may need to change when an OCIProject
+ # pillar isn't just a distribution
+ return self.recipe.ociproject.distribution
+
+ def getMedianBuildDuration(self):
+ """Return the median duration of our successful builds."""
+ store = IStore(self)
+ result = store.find(
+ (OCIRecipeBuild.date_started, OCIRecipeBuild.date_finished),
+ OCIRecipeBuild.recipe == self.recipe,
+ OCIRecipeBuild.status == BuildStatus.FULLYBUILT)
+ result.order_by(Desc(OCIRecipeBuild.date_finished))
+ durations = [row[1] - row[0] for row in result[:9]]
+ if len(durations) == 0:
+ return None
+ durations.sort()
+ return durations[len(durations) // 2]
+
+ def estimateDuration(self):
+ """See `IBuildFarmJob`."""
+ median = self.getMedianBuildDuration()
+ if median is not None:
+ return median
+ return timedelta(minutes=30)
+
+ def calculateScore(self):
+ """See `IBuildFarmJob`."""
+ return 2510 + self.archive.relative_build_score
@implementer(IOCIRecipeBuildSet)
@@ -122,10 +234,31 @@ class OCIRecipeBuildSet(SpecificBuildFarmJobSourceMixin):
def preloadBuildsData(self, builds):
"""See `IOCIRecipeBuildSet`."""
- # XXX twom 2019-12-02 Currently a no-op skeleton, to be filled in
+ # Circular import.
+ from lp.oci.model.ocirecipe import OCIRecipe
+ load_related(Person, builds, ["requester_id"])
+ lfas = load_related(LibraryFileAlias, builds, ["log_id"])
+ load_related(LibraryFileContent, lfas, ["contentID"])
+ load_related(OCIRecipe, builds, ["recipe_id"])
+ # XXX twom 2019-12-05 This needs to be extended to include
+ # OCIRecipeBuildJob when that exists.
return
def getByID(self, build_id):
"""See `ISpecificBuildFarmJobSource`."""
store = IMasterStore(OCIRecipeBuild)
return store.get(OCIRecipeBuild, build_id)
+
+ def getByBuildFarmJob(self, build_farm_job):
+ """See `ISpecificBuildFarmJobSource`."""
+ return Store.of(build_farm_job).find(
+ OCIRecipeBuild, build_farm_job_id=build_farm_job.id).one()
+
+ def getByBuildFarmJobs(self, build_farm_jobs):
+ """See `ISpecificBuildFarmJobSource`."""
+ if len(build_farm_jobs) == 0:
+ return EmptyResultSet()
+ rows = Store.of(build_farm_jobs[0]).find(
+ OCIRecipeBuild, OCIRecipeBuild.build_farm_job_id.is_in(
+ bfj.id for bfj in build_farm_jobs))
+ return DecoratedResultSet(rows, pre_iter_hook=self.preloadBuildsData)
diff --git a/lib/lp/oci/tests/test_ocirecipebuild.py b/lib/lp/oci/tests/test_ocirecipebuild.py
new file mode 100644
index 0000000..502cf06
--- /dev/null
+++ b/lib/lp/oci/tests/test_ocirecipebuild.py
@@ -0,0 +1,140 @@
+# Copyright 2019 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for OCI image building recipe functionality."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from datetime import timedelta
+
+from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
+
+from lp.app.errors import NotFoundError
+from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.interfaces.buildqueue import IBuildQueue
+from lp.buildmaster.interfaces.packagebuild import IPackageBuild
+from lp.oci.interfaces.ocirecipebuild import (
+ IOCIRecipeBuild,
+ IOCIRecipeBuildSet,
+ )
+from lp.oci.model.ocirecipebuild import OCIRecipeBuildSet
+from lp.testing import (
+ admin_logged_in,
+ TestCaseWithFactory,
+ )
+from lp.testing.layers import (
+ DatabaseFunctionalLayer,
+ LaunchpadZopelessLayer,
+ )
+
+
+class TestOCIRecipeBuild(TestCaseWithFactory):
+
+ layer = LaunchpadZopelessLayer
+
+ def setUp(self):
+ super(TestOCIRecipeBuild, self).setUp()
+ self.build = self.factory.makeOCIRecipeBuild()
+
+ def test_implements_interface(self):
+ with admin_logged_in():
+ self.assertProvides(self.build, IOCIRecipeBuild)
+ self.assertProvides(self.build, IPackageBuild)
+
+ def test_addFile(self):
+ lfa = self.factory.makeLibraryFileAlias()
+ self.build.addFile(lfa)
+ _, result_lfa, _ = self.build.getFileByFileName(lfa.filename)
+ self.assertEqual(result_lfa, lfa)
+
+ def test_getFileByFileName(self):
+ files = [self.factory.makeOCIFile(build=self.build) for x in range(3)]
+ result, _, _ = self.build.getFileByFileName(
+ files[0].library_file.filename)
+ self.assertEqual(result, files[0])
+
+ def test_getLayerFileByDigest(self):
+ files = [self.factory.makeOCILayerFile(build=self.build)
+ for x in range(3)]
+ result, _, _ = self.build.getLayerFileByDigest(
+ files[0].layer_file_digest)
+ self.assertEqual(result, files[0])
+
+ def test_getLayerFileByDigest_missing(self):
+ [self.factory.makeOCILayerFile(build=self.build)
+ for x in range(3)]
+ self.assertRaises(
+ NotFoundError,
+ self.build.getLayerFileByDigest,
+ 'missing')
+
+ def test_estimateDuration(self):
+ # Without previous builds, the default time estimate is 30m.
+ self.assertEqual(1800, self.build.estimateDuration().seconds)
+
+ def test_estimateDuration_with_history(self):
+ # Previous successful builds of the same OCI recipe are used for
+ # estimates.
+ oci_build = self.factory.makeOCIRecipeBuild(
+ status=BuildStatus.FULLYBUILT, duration=timedelta(seconds=335))
+ for i in range(3):
+ self.factory.makeOCIRecipeBuild(
+ requester=oci_build.requester, recipe=oci_build.recipe,
+ status=BuildStatus.FAILEDTOBUILD,
+ duration=timedelta(seconds=20))
+ self.assertEqual(335, oci_build.estimateDuration().seconds)
+
+ def test_queueBuild(self):
+ # OCIRecipeBuild can create the queue entry for itself.
+ bq = self.build.queueBuild()
+ self.assertProvides(bq, IBuildQueue)
+ self.assertEqual(
+ self.build.build_farm_job, removeSecurityProxy(bq)._build_farm_job)
+ self.assertEqual(self.build, bq.specific_build)
+ self.assertEqual(self.build.virtualized, bq.virtualized)
+ self.assertIsNotNone(bq.processor)
+ self.assertEqual(bq, self.build.buildqueue_record)
+
+
+class TestOCIRecipeBuildSet(TestCaseWithFactory):
+
+ layer = DatabaseFunctionalLayer
+
+ def test_implements_interface(self):
+ target = OCIRecipeBuildSet()
+ with admin_logged_in():
+ self.assertProvides(target, IOCIRecipeBuildSet)
+
+ def test_new(self):
+ requester = self.factory.makePerson()
+ recipe = self.factory.makeOCIRecipe()
+ channel_name = 'test'
+ processor = self.factory.makeProcessor()
+ virtualized = False
+ target = getUtility(IOCIRecipeBuildSet).new(
+ requester, recipe, channel_name, processor, virtualized)
+ with admin_logged_in():
+ self.assertProvides(target, IOCIRecipeBuild)
+
+ def test_getById(self):
+ builds = [self.factory.makeOCIRecipeBuild() for x in range(3)]
+ result = getUtility(IOCIRecipeBuildSet).getByID(builds[1].id)
+ self.assertEqual(result, builds[1])
+
+ def test_getByBuildFarmJob(self):
+ builds = [self.factory.makeOCIRecipeBuild() for x in range(3)]
+ result = getUtility(IOCIRecipeBuildSet).getByBuildFarmJob(
+ builds[1].build_farm_job)
+ self.assertEqual(result, builds[1])
+
+ def test_getByBuildFarmJobs(self):
+ builds = [self.factory.makeOCIRecipeBuild() for x in range(3)]
+ self.assertContentEqual(
+ builds,
+ getUtility(IOCIRecipeBuildSet).getByBuildFarmJobs(
+ [build.build_farm_job for build in builds]))
+
+ def test_getByBuildFarmJobs_empty(self):
+ self.assertContentEqual(
+ [], getUtility(IOCIRecipeBuildSet).getByBuildFarmJobs([]))
diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
index e519ed0..79d9e5e 100644
--- a/lib/lp/testing/factory.py
+++ b/lib/lp/testing/factory.py
@@ -163,6 +163,7 @@ from lp.oci.model.ocirecipe import (
OCIRecipe,
OCIRecipeArch,
)
+from lp.oci.model.ocirecipebuild import OCIFile
from lp.oci.model.ocirecipechannel import OCIRecipeChannel
from lp.registry.enums import (
BranchSharingPolicy,
@@ -4982,18 +4983,55 @@ class BareLaunchpadObjectFactory(ObjectFactory):
def makeOCIRecipeBuild(self, requester=None, recipe=None,
channel_name=None, processor=None,
- virtualized=False, date_created=DEFAULT):
+ virtualized=False, date_created=DEFAULT,
+ status=BuildStatus.NEEDSBUILD, builder=None,
+ duration=None):
"""Make a new OCIRecipeBuild."""
if requester is None:
requester = self.makePerson()
+ if recipe is None:
+ recipe = self.makeOCIRecipe()
if channel_name is None:
channel_name = self.getUniqueString(u"oci-recipe-channel-name")
if processor is None:
processor = self.makeProcessor()
-
- return getUtility(IOCIRecipeBuildSet).new(
+ oci_build = getUtility(IOCIRecipeBuildSet).new(
requester, recipe, channel_name, processor, virtualized,
date_created)
+ if duration is not None:
+ removeSecurityProxy(oci_build).updateStatus(
+ BuildStatus.BUILDING, builder=builder,
+ date_started=oci_build.date_created)
+ removeSecurityProxy(oci_build).updateStatus(
+ status, builder=builder,
+ date_finished=oci_build.date_started + duration)
+ else:
+ removeSecurityProxy(oci_build).updateStatus(
+ status, builder=builder)
+ return oci_build
+
+ def makeOCIFile(self, build=None, library_file=None,
+ layer_file_digest=None):
+ """Make a new OCIFile."""
+ if build is None:
+ build = self.makeOCIRecipeBuild()
+ if library_file is None:
+ library_file = self.makeLibraryFileAlias()
+ # layer_file_digest can be None
+ return OCIFile(build=build, library_file=library_file,
+ layer_file_digest=layer_file_digest)
+
+ def makeOCILayerFile(self, build=None, library_file=None,
+ layer_file_digest=None):
+ """Make a new OCIFile, but with a sample digest."""
+ if build is None:
+ build = self.makeOCIRecipeBuild()
+ if library_file is None:
+ library_file = self.makeLibraryFileAlias()
+ if layer_file_digest is None:
+ layer_file_digest = self.getUniqueString(u"layer-file-digest")
+ return OCIFile(build=build, library_file=library_file,
+ layer_file_digest=layer_file_digest)
# Some factory methods return simple Python types. We don't add
Follow ups
