launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27483
[Merge] ~cjwatson/launchpad:charmhub-schedule-upload-job into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:charmhub-schedule-upload-job into launchpad:master with ~cjwatson/launchpad:charmhub-upload-job as a prerequisite.
Commit message:
Schedule Charmhub upload jobs when builds complete
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/407979
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:charmhub-schedule-upload-job into launchpad:master.
diff --git a/lib/lp/charms/interfaces/charmrecipebuild.py b/lib/lp/charms/interfaces/charmrecipebuild.py
index 6f162cc..cefae7f 100644
--- a/lib/lp/charms/interfaces/charmrecipebuild.py
+++ b/lib/lp/charms/interfaces/charmrecipebuild.py
@@ -5,11 +5,20 @@
__metaclass__ = type
__all__ = [
+ "CannotScheduleStoreUpload",
+ "CharmRecipeBuildStoreUploadStatus",
"ICharmFile",
"ICharmRecipeBuild",
"ICharmRecipeBuildSet",
]
+import http.client
+
+from lazr.enum import (
+ EnumeratedType,
+ Item,
+ )
+from lazr.restful.declarations import error_status
from lazr.restful.fields import (
CollectionField,
Reference,
@@ -20,6 +29,7 @@ from zope.interface import (
)
from zope.schema import (
Bool,
+ Choice,
Datetime,
Dict,
Int,
@@ -39,6 +49,51 @@ from lp.services.librarian.interfaces import ILibraryFileAlias
from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries
+@error_status(http.client.BAD_REQUEST)
+class CannotScheduleStoreUpload(Exception):
+ """This build cannot be uploaded to the store."""
+
+
+class CharmRecipeBuildStoreUploadStatus(EnumeratedType):
+ """Charm recipe build store upload status type
+
+ Charm recipe builds may be uploaded to Charmhub. This represents the
+ state of that process.
+ """
+
+ UNSCHEDULED = Item("""
+ Unscheduled
+
+ No upload of this charm recipe build to Charmhub is scheduled.
+ """)
+
+ PENDING = Item("""
+ Pending
+
+ This charm recipe build is queued for upload to Charmhub.
+ """)
+
+ FAILEDTOUPLOAD = Item("""
+ Failed to upload
+
+ The last attempt to upload this charm recipe build to Charmhub
+ failed.
+ """)
+
+ FAILEDTORELEASE = Item("""
+ Failed to release to channels
+
+ The last attempt to release this charm recipe build to its intended
+ set of channels failed.
+ """)
+
+ UPLOADED = Item("""
+ Uploaded
+
+ This charm recipe build was successfully uploaded to Charmhub.
+ """)
+
+
class ICharmRecipeBuildView(IPackageBuild):
"""`ICharmRecipeBuild` attributes that require launchpad.View."""
@@ -120,6 +175,24 @@ class ICharmRecipeBuildView(IPackageBuild):
last_store_upload_job = Reference(
title=_("Last store upload job for this build."), schema=Interface)
+ store_upload_status = Choice(
+ title=_("Store upload status"),
+ vocabulary=CharmRecipeBuildStoreUploadStatus,
+ required=True, readonly=False)
+
+ store_upload_revision = Int(
+ title=_("Store revision"),
+ description=_(
+ "The revision assigned to this charm recipe build by Charmhub."),
+ required=False, readonly=True)
+
+ store_upload_error_message = TextLine(
+ title=_("Store upload error message"),
+ description=_(
+ "The error message, if any, from the last attempt to upload "
+ "this charm recipe build to Charmhub."),
+ required=False, readonly=True)
+
store_upload_metadata = Attribute(
_("A dict of data about store upload progress."))
@@ -157,6 +230,13 @@ class ICharmRecipeBuildEdit(Interface):
:return: An `ICharmFile`.
"""
+ def scheduleStoreUpload():
+ """Schedule an upload of this build to the store.
+
+ :raises CannotScheduleStoreUpload: if the build is not in a state
+ where an upload can be scheduled.
+ """
+
def retry():
"""Restore the build record to its initial state.
diff --git a/lib/lp/charms/model/charmrecipebuild.py b/lib/lp/charms/model/charmrecipebuild.py
index c6b10ec..531d8b2 100644
--- a/lib/lp/charms/model/charmrecipebuild.py
+++ b/lib/lp/charms/model/charmrecipebuild.py
@@ -38,10 +38,13 @@ from lp.buildmaster.model.buildfarmjob import SpecificBuildFarmJobSourceMixin
from lp.buildmaster.model.packagebuild import PackageBuildMixin
from lp.charms.interfaces.charmrecipe import ICharmRecipeSet
from lp.charms.interfaces.charmrecipebuild import (
+ CannotScheduleStoreUpload,
+ CharmRecipeBuildStoreUploadStatus,
ICharmFile,
ICharmRecipeBuild,
ICharmRecipeBuildSet,
)
+from lp.charms.interfaces.charmrecipebuildjob import ICharmhubUploadJobSource
from lp.charms.mail.charmrecipebuild import CharmRecipeBuildMailer
from lp.charms.model.charmrecipebuildjob import (
CharmRecipeBuildJob,
@@ -62,6 +65,7 @@ from lp.services.database.interfaces import (
IStore,
)
from lp.services.database.stormbase import StormBase
+from lp.services.job.interfaces.job import JobStatus
from lp.services.job.model.job import Job
from lp.services.librarian.model import (
LibraryFileAlias,
@@ -348,6 +352,51 @@ class CharmRecipeBuild(PackageBuildMixin, StormBase):
def last_store_upload_job(self):
return self.store_upload_jobs.first()
+ @property
+ def store_upload_status(self):
+ job = self.last_store_upload_job
+ if job is None or job.job.status == JobStatus.SUSPENDED:
+ return CharmRecipeBuildStoreUploadStatus.UNSCHEDULED
+ elif job.job.status in (JobStatus.WAITING, JobStatus.RUNNING):
+ return CharmRecipeBuildStoreUploadStatus.PENDING
+ elif job.job.status == JobStatus.COMPLETED:
+ return CharmRecipeBuildStoreUploadStatus.UPLOADED
+ else:
+ if job.store_revision:
+ return CharmRecipeBuildStoreUploadStatus.FAILEDTORELEASE
+ else:
+ return CharmRecipeBuildStoreUploadStatus.FAILEDTOUPLOAD
+
+ @property
+ def store_upload_revision(self):
+ job = self.last_store_upload_job
+ return job and job.store_revision
+
+ @property
+ def store_upload_error_message(self):
+ job = self.last_store_upload_job
+ return job and job.error_message
+
+ def scheduleStoreUpload(self):
+ """See `ICharmRecipeBuild`."""
+ if not self.recipe.can_upload_to_store:
+ raise CannotScheduleStoreUpload(
+ "Cannot upload this charm to Charmhub because it is not "
+ "properly configured.")
+ if not self.was_built or self.getFiles().is_empty():
+ raise CannotScheduleStoreUpload(
+ "Cannot upload this charm because it has no files.")
+ if (self.store_upload_status ==
+ CharmRecipeBuildStoreUploadStatus.PENDING):
+ raise CannotScheduleStoreUpload(
+ "An upload of this charm is already in progress.")
+ elif (self.store_upload_status ==
+ CharmRecipeBuildStoreUploadStatus.UPLOADED):
+ raise CannotScheduleStoreUpload(
+ "Cannot upload this charm because it has already been "
+ "uploaded.")
+ getUtility(ICharmhubUploadJobSource).create(self)
+
def getFiles(self):
"""See `ICharmRecipeBuild`."""
result = Store.of(self).find(
diff --git a/lib/lp/charms/model/charmrecipebuildjob.py b/lib/lp/charms/model/charmrecipebuildjob.py
index bc810eb..54b6dbb 100644
--- a/lib/lp/charms/model/charmrecipebuildjob.py
+++ b/lib/lp/charms/model/charmrecipebuildjob.py
@@ -58,6 +58,7 @@ from lp.services.job.model.job import (
)
from lp.services.job.runner import BaseRunnableJob
from lp.services.propertycache import get_property_cache
+from lp.services.webapp.snapshot import notify_modified
class CharmRecipeBuildJobType(DBEnumeratedType):
@@ -188,11 +189,16 @@ class CharmhubUploadJob(CharmRecipeBuildJobDerived):
@classmethod
def create(cls, build):
"""See `ICharmhubUploadJobSource`."""
- charm_recipe_build_job = CharmRecipeBuildJob(
- build, cls.class_job_type, {})
- job = cls(charm_recipe_build_job)
- job.celeryRunOnCommit()
- del get_property_cache(build).last_store_upload_job
+ edited_fields = set()
+ with notify_modified(build, edited_fields) as before_modification:
+ charm_recipe_build_job = CharmRecipeBuildJob(
+ build, cls.class_job_type, {})
+ job = cls(charm_recipe_build_job)
+ job.celeryRunOnCommit()
+ del get_property_cache(build).last_store_upload_job
+ upload_status = build.store_upload_status
+ if upload_status != before_modification.store_upload_status:
+ edited_fields.add("store_upload_status")
return job
@property
@@ -246,6 +252,39 @@ class CharmhubUploadJob(CharmRecipeBuildJobDerived):
self.build.store_upload_metadata = {}
self.build.store_upload_metadata["status_url"] = url
+ # Ideally we'd just override Job._set_status or similar, but
+ # lazr.delegates makes that difficult, so we use this to override all
+ # the individual Job lifecycle methods instead.
+ def _do_lifecycle(self, method_name, manage_transaction=False,
+ *args, **kwargs):
+ edited_fields = set()
+ with notify_modified(self.build, edited_fields) as before_modification:
+ getattr(super(CharmhubUploadJob, self), method_name)(
+ *args, manage_transaction=manage_transaction, **kwargs)
+ upload_status = self.build.store_upload_status
+ if upload_status != before_modification.store_upload_status:
+ edited_fields.add("store_upload_status")
+ if edited_fields and manage_transaction:
+ transaction.commit()
+
+ def start(self, *args, **kwargs):
+ self._do_lifecycle("start", *args, **kwargs)
+
+ def complete(self, *args, **kwargs):
+ self._do_lifecycle("complete", *args, **kwargs)
+
+ def fail(self, *args, **kwargs):
+ self._do_lifecycle("fail", *args, **kwargs)
+
+ def queue(self, *args, **kwargs):
+ self._do_lifecycle("queue", *args, **kwargs)
+
+ def suspend(self, *args, **kwargs):
+ self._do_lifecycle("suspend", *args, **kwargs)
+
+ def resume(self, *args, **kwargs):
+ self._do_lifecycle("resume", *args, **kwargs)
+
def getOopsVars(self):
"""See `IRunnableJob`."""
oops_vars = super(CharmhubUploadJob, self).getOopsVars()
diff --git a/lib/lp/charms/subscribers/charmrecipebuild.py b/lib/lp/charms/subscribers/charmrecipebuild.py
index ebfa796..2eb5534 100644
--- a/lib/lp/charms/subscribers/charmrecipebuild.py
+++ b/lib/lp/charms/subscribers/charmrecipebuild.py
@@ -7,11 +7,14 @@ __metaclass__ = type
from zope.component import getUtility
+from lp.buildmaster.enums import BuildStatus
from lp.charms.interfaces.charmrecipe import (
CHARM_RECIPE_WEBHOOKS_FEATURE_FLAG,
)
from lp.charms.interfaces.charmrecipebuild import ICharmRecipeBuild
+from lp.charms.interfaces.charmrecipebuildjob import ICharmhubUploadJobSource
from lp.services.features import getFeatureFlag
+from lp.services.scripts import log
from lp.services.webapp.publisher import canonical_url
from lp.services.webhooks.interfaces import IWebhookSet
from lp.services.webhooks.payload import compose_webhook_payload
@@ -25,7 +28,7 @@ def _trigger_charm_recipe_build_webhook(build, action):
}
payload.update(compose_webhook_payload(
ICharmRecipeBuild, build,
- ["recipe", "build_request", "status"]))
+ ["recipe", "build_request", "status", "store_upload_status"]))
getUtility(IWebhookSet).trigger(
build.recipe, "charm-recipe:build:0.1", payload)
@@ -38,5 +41,18 @@ def charm_recipe_build_created(build, event):
def charm_recipe_build_modified(build, event):
"""Trigger events when a charm recipe build is modified."""
if event.edited_fields is not None:
- if "status" in event.edited_fields:
+ status_changed = "status" in event.edited_fields
+ store_upload_status_changed = (
+ "store_upload_status" in event.edited_fields)
+ if status_changed or store_upload_status_changed:
_trigger_charm_recipe_build_webhook(build, "status-changed")
+ if status_changed:
+ if build.status == BuildStatus.FULLYBUILT:
+ if (build.recipe.can_upload_to_store and
+ build.recipe.store_upload):
+ log.info("Scheduling upload of %r to the store." % build)
+ getUtility(ICharmhubUploadJobSource).create(build)
+ else:
+ log.info(
+ "%r is not configured for upload to the store." %
+ build.recipe)
diff --git a/lib/lp/charms/tests/test_charmrecipe.py b/lib/lp/charms/tests/test_charmrecipe.py
index f1e2443..7599d27 100644
--- a/lib/lp/charms/tests/test_charmrecipe.py
+++ b/lib/lp/charms/tests/test_charmrecipe.py
@@ -381,6 +381,7 @@ class TestCharmRecipe(TestCaseWithFactory):
"build_request": Equals(
canonical_url(build_request, force_local_path=True)),
"status": Equals("Needs building"),
+ "store_upload_status": Equals("Unscheduled"),
}
with person_logged_in(recipe.owner):
delivery = hook.deliveries.one()
@@ -625,6 +626,7 @@ class TestCharmRecipe(TestCaseWithFactory):
"build_request": Equals(canonical_url(
job.build_request, force_local_path=True)),
"status": Equals("Needs building"),
+ "store_upload_status": Equals("Unscheduled"),
})
for build in builds]
self.assertThat(hook.deliveries, MatchesSetwise(*(
diff --git a/lib/lp/charms/tests/test_charmrecipebuild.py b/lib/lp/charms/tests/test_charmrecipebuild.py
index e81bd0d..f4adff6 100644
--- a/lib/lp/charms/tests/test_charmrecipebuild.py
+++ b/lib/lp/charms/tests/test_charmrecipebuild.py
@@ -5,12 +5,15 @@
__metaclass__ = type
+import base64
from datetime import (
datetime,
timedelta,
)
from fixtures import FakeLogger
+from nacl.public import PrivateKey
+from pymacaroons import Macaroon
import pytz
import six
from testtools.matchers import (
@@ -34,16 +37,21 @@ from lp.charms.interfaces.charmrecipe import (
CHARM_RECIPE_WEBHOOKS_FEATURE_FLAG,
)
from lp.charms.interfaces.charmrecipebuild import (
+ CannotScheduleStoreUpload,
+ CharmRecipeBuildStoreUploadStatus,
ICharmRecipeBuild,
ICharmRecipeBuildSet,
)
+from lp.charms.interfaces.charmrecipebuildjob import ICharmhubUploadJobSource
from lp.registry.enums import (
PersonVisibility,
TeamMembershipPolicy,
)
from lp.registry.interfaces.series import SeriesStatus
from lp.services.config import config
+from lp.services.crypto.interfaces import IEncryptedContainer
from lp.services.features.testing import FeatureFixture
+from lp.services.job.interfaces.job import JobStatus
from lp.services.propertycache import clear_property_cache
from lp.services.webapp.publisher import canonical_url
from lp.services.webhooks.testing import LogsScheduledWebhooks
@@ -284,6 +292,7 @@ class TestCharmRecipeBuild(TestCaseWithFactory):
"build_request": Equals(canonical_url(
self.build.build_request, force_local_path=True)),
"status": Equals("Successfully built"),
+ "store_upload_status": Equals("Unscheduled"),
}
delivery = hook.deliveries.one()
self.assertThat(
@@ -334,6 +343,55 @@ class TestCharmRecipeBuild(TestCaseWithFactory):
self.assertEqual(2, hook.deliveries.count())
self.assertThat(logger.output, LogsScheduledWebhooks(expected_logs))
+ def test_updateStatus_failure_does_not_trigger_store_uploads(self):
+ # A failed build does not trigger store uploads.
+ self.pushConfig("charms", charmhub_url="http://charmhub.example/";)
+ self.build.recipe.store_name = self.factory.getUniqueUnicode()
+ self.build.recipe.store_upload = True
+ # CharmRecipe.can_upload_to_store only checks whether
+ # "exchanged_encrypted" is present, so don't bother setting up
+ # encryption keys here.
+ self.build.recipe.store_secrets = {
+ "exchanged_encrypted": Macaroon().serialize()}
+ with dbuser(config.builddmaster.dbuser):
+ self.build.updateStatus(BuildStatus.FAILEDTOBUILD)
+ self.assertContentEqual([], self.build.store_upload_jobs)
+
+ def test_updateStatus_fullybuilt_not_configured(self):
+ # A completed build does not trigger store uploads if the recipe is
+ # not properly configured for that.
+ logger = self.useFixture(FakeLogger())
+ with dbuser(config.builddmaster.dbuser):
+ self.build.updateStatus(BuildStatus.FULLYBUILT)
+ self.assertEqual(0, len(list(self.build.store_upload_jobs)))
+ self.assertIn(
+ "<CharmRecipe ~%s/%s/+charm/%s> is not configured for upload to "
+ "the store." % (
+ self.build.recipe.owner.name, self.build.recipe.project.name,
+ self.build.recipe.name),
+ logger.output.splitlines())
+
+ def test_updateStatus_fullybuilt_triggers_store_uploads(self):
+ # A completed build triggers store uploads.
+ self.pushConfig("charms", charmhub_url="http://charmhub.example/";)
+ logger = self.useFixture(FakeLogger())
+ self.build.recipe.store_name = self.factory.getUniqueUnicode()
+ self.build.recipe.store_upload = True
+ # CharmRecipe.can_upload_to_store only checks whether
+ # "exchanged_encrypted" is present, so don't bother setting up
+ # encryption keys here.
+ self.build.recipe.store_secrets = {
+ "exchanged_encrypted": Macaroon().serialize()}
+ with dbuser(config.builddmaster.dbuser):
+ self.build.updateStatus(BuildStatus.FULLYBUILT)
+ self.assertEqual(1, len(list(self.build.store_upload_jobs)))
+ self.assertIn(
+ "Scheduling upload of <CharmRecipeBuild "
+ "~%s/%s/+charm/%s/+build/%d> to the store." % (
+ self.build.recipe.owner.name, self.build.recipe.project.name,
+ self.build.recipe.name, self.build.id),
+ logger.output.splitlines())
+
def test_notify_fullybuilt(self):
# notify does not send mail when a recipe build completes normally.
build = self.factory.makeCharmRecipeBuild(
@@ -424,6 +482,179 @@ class TestCharmRecipeBuild(TestCaseWithFactory):
clear_property_cache(self.build)
self.assertFalse(self.build.estimate)
+ def setUpStoreUpload(self):
+ self.private_key = PrivateKey.generate()
+ self.pushConfig(
+ "charms",
+ charmhub_url="http://charmhub.example/";,
+ charmhub_secrets_public_key=base64.b64encode(
+ bytes(self.private_key.public_key)).decode())
+ self.build.recipe.store_name = self.factory.getUniqueUnicode()
+ container = getUtility(IEncryptedContainer, "charmhub-secrets")
+ self.build.recipe.store_secrets = {
+ "exchanged_encrypted": removeSecurityProxy(
+ container.encrypt(Macaroon().serialize().encode())),
+ }
+
+ def test_store_upload_status_unscheduled(self):
+ build = self.factory.makeCharmRecipeBuild(
+ status=BuildStatus.FULLYBUILT)
+ self.assertEqual(
+ CharmRecipeBuildStoreUploadStatus.UNSCHEDULED,
+ build.store_upload_status)
+
+ def test_store_upload_status_pending(self):
+ build = self.factory.makeCharmRecipeBuild(
+ status=BuildStatus.FULLYBUILT)
+ getUtility(ICharmhubUploadJobSource).create(build)
+ self.assertEqual(
+ CharmRecipeBuildStoreUploadStatus.PENDING,
+ build.store_upload_status)
+
+ def test_store_upload_status_uploaded(self):
+ build = self.factory.makeCharmRecipeBuild(
+ status=BuildStatus.FULLYBUILT)
+ job = getUtility(ICharmhubUploadJobSource).create(build)
+ naked_job = removeSecurityProxy(job)
+ naked_job.job._status = JobStatus.COMPLETED
+ self.assertEqual(
+ CharmRecipeBuildStoreUploadStatus.UPLOADED,
+ build.store_upload_status)
+
+ def test_store_upload_status_failed_to_upload(self):
+ build = self.factory.makeCharmRecipeBuild(
+ status=BuildStatus.FULLYBUILT)
+ job = getUtility(ICharmhubUploadJobSource).create(build)
+ naked_job = removeSecurityProxy(job)
+ naked_job.job._status = JobStatus.FAILED
+ self.assertEqual(
+ CharmRecipeBuildStoreUploadStatus.FAILEDTOUPLOAD,
+ build.store_upload_status)
+
+ def test_store_upload_status_failed_to_release(self):
+ build = self.factory.makeCharmRecipeBuild(
+ status=BuildStatus.FULLYBUILT)
+ job = getUtility(ICharmhubUploadJobSource).create(build)
+ naked_job = removeSecurityProxy(job)
+ naked_job.job._status = JobStatus.FAILED
+ naked_job.store_revision = 1
+ self.assertEqual(
+ CharmRecipeBuildStoreUploadStatus.FAILEDTORELEASE,
+ build.store_upload_status)
+
+ def test_scheduleStoreUpload(self):
+ # A build not previously uploaded to the store can be uploaded
+ # manually.
+ self.setUpStoreUpload()
+ self.build.updateStatus(BuildStatus.FULLYBUILT)
+ self.factory.makeCharmFile(
+ build=self.build,
+ library_file=self.factory.makeLibraryFileAlias(db_only=True))
+ self.build.scheduleStoreUpload()
+ [job] = getUtility(ICharmhubUploadJobSource).iterReady()
+ self.assertEqual(JobStatus.WAITING, job.job.status)
+ self.assertEqual(self.build, job.build)
+
+ def test_scheduleStoreUpload_not_configured(self):
+ # A build that is not properly configured cannot be uploaded to the
+ # store.
+ self.setUpStoreUpload()
+ self.build.updateStatus(BuildStatus.FULLYBUILT)
+ self.build.recipe.store_name = None
+ self.assertRaisesWithContent(
+ CannotScheduleStoreUpload,
+ "Cannot upload this charm to Charmhub because it is not properly "
+ "configured.",
+ self.build.scheduleStoreUpload)
+ self.assertEqual(
+ [], list(getUtility(ICharmhubUploadJobSource).iterReady()))
+
+ def test_scheduleStoreUpload_no_files(self):
+ # A build with no files cannot be uploaded to the store.
+ self.setUpStoreUpload()
+ self.build.updateStatus(BuildStatus.FULLYBUILT)
+ self.assertRaisesWithContent(
+ CannotScheduleStoreUpload,
+ "Cannot upload this charm because it has no files.",
+ self.build.scheduleStoreUpload)
+ self.assertEqual(
+ [], list(getUtility(ICharmhubUploadJobSource).iterReady()))
+
+ def test_scheduleStoreUpload_already_in_progress(self):
+ # A build with an upload already in progress will not have another
+ # one created.
+ self.setUpStoreUpload()
+ self.build.updateStatus(BuildStatus.FULLYBUILT)
+ self.factory.makeCharmFile(
+ build=self.build,
+ library_file=self.factory.makeLibraryFileAlias(db_only=True))
+ old_job = getUtility(ICharmhubUploadJobSource).create(self.build)
+ self.assertRaisesWithContent(
+ CannotScheduleStoreUpload,
+ "An upload of this charm is already in progress.",
+ self.build.scheduleStoreUpload)
+ self.assertEqual(
+ [old_job], list(getUtility(ICharmhubUploadJobSource).iterReady()))
+
+ def test_scheduleStoreUpload_already_uploaded(self):
+ # A build with an upload that has already completed will not have
+ # another one created.
+ self.setUpStoreUpload()
+ self.build.updateStatus(BuildStatus.FULLYBUILT)
+ self.factory.makeCharmFile(
+ build=self.build,
+ library_file=self.factory.makeLibraryFileAlias(db_only=True))
+ old_job = getUtility(ICharmhubUploadJobSource).create(self.build)
+ removeSecurityProxy(old_job).job._status = JobStatus.COMPLETED
+ self.assertRaisesWithContent(
+ CannotScheduleStoreUpload,
+ "Cannot upload this charm because it has already been uploaded.",
+ self.build.scheduleStoreUpload)
+ self.assertEqual(
+ [], list(getUtility(ICharmhubUploadJobSource).iterReady()))
+
+ def test_scheduleStoreUpload_triggers_webhooks(self):
+ # Scheduling a store upload triggers webhooks on the corresponding
+ # recipe.
+ self.useFixture(FeatureFixture({
+ CHARM_RECIPE_ALLOW_CREATE: "on",
+ CHARM_RECIPE_WEBHOOKS_FEATURE_FLAG: "on",
+ }))
+ logger = self.useFixture(FakeLogger())
+ self.setUpStoreUpload()
+ self.build.updateStatus(BuildStatus.FULLYBUILT)
+ self.factory.makeCharmFile(
+ build=self.build,
+ library_file=self.factory.makeLibraryFileAlias(db_only=True))
+ hook = self.factory.makeWebhook(
+ target=self.build.recipe, event_types=["charm-recipe:build:0.1"])
+ self.build.scheduleStoreUpload()
+ expected_payload = {
+ "recipe_build": Equals(canonical_url(
+ self.build, force_local_path=True)),
+ "action": Equals("status-changed"),
+ "recipe": Equals(canonical_url(
+ self.build.recipe, force_local_path=True)),
+ "build_request": Equals(canonical_url(
+ self.build.build_request, force_local_path=True)),
+ "status": Equals("Successfully built"),
+ "store_upload_status": Equals("Pending"),
+ }
+ delivery = hook.deliveries.one()
+ self.assertThat(
+ delivery, MatchesStructure(
+ event_type=Equals("charm-recipe:build:0.1"),
+ payload=MatchesDict(expected_payload)))
+ with dbuser(config.IWebhookDeliveryJobSource.dbuser):
+ self.assertEqual(
+ "<WebhookDeliveryJob for webhook %d on %r>" % (
+ hook.id, hook.target),
+ repr(delivery))
+ self.assertThat(
+ logger.output, LogsScheduledWebhooks([
+ (hook, "charm-recipe:build:0.1",
+ MatchesDict(expected_payload))]))
+
class TestCharmRecipeBuildSet(TestCaseWithFactory):
diff --git a/lib/lp/charms/tests/test_charmrecipebuildjob.py b/lib/lp/charms/tests/test_charmrecipebuildjob.py
index 97d2f21..33dc021 100644
--- a/lib/lp/charms/tests/test_charmrecipebuildjob.py
+++ b/lib/lp/charms/tests/test_charmrecipebuildjob.py
@@ -8,6 +8,12 @@ __metaclass__ = type
from datetime import timedelta
from fixtures import FakeLogger
+from testtools.matchers import (
+ Equals,
+ MatchesDict,
+ MatchesListwise,
+ MatchesStructure,
+ )
import transaction
from zope.interface import implementer
from zope.security.proxy import removeSecurityProxy
@@ -40,6 +46,8 @@ from lp.services.database.interfaces import IStore
from lp.services.features.testing import FeatureFixture
from lp.services.job.interfaces.job import JobStatus
from lp.services.job.runner import JobRunner
+from lp.services.webapp.publisher import canonical_url
+from lp.services.webhooks.testing import LogsScheduledWebhooks
from lp.testing import TestCaseWithFactory
from lp.testing.dbuser import dbuser
from lp.testing.fakemethod import FakeMethod
@@ -127,9 +135,44 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
target=build.recipe, event_types=["charm-recipe:build:0.1"])
return build
+ def assertWebhookDeliveries(self, build,
+ expected_store_upload_statuses, logger):
+ hook = build.recipe.webhooks.one()
+ deliveries = list(hook.deliveries)
+ deliveries.reverse()
+ expected_payloads = [{
+ "recipe_build": Equals(
+ canonical_url(build, force_local_path=True)),
+ "action": Equals("status-changed"),
+ "recipe": Equals(
+ canonical_url(build.recipe, force_local_path=True)),
+ "build_request": Equals(
+ canonical_url(build.build_request, force_local_path=True)),
+ "status": Equals("Successfully built"),
+ "store_upload_status": Equals(expected),
+ } for expected in expected_store_upload_statuses]
+ matchers = [
+ MatchesStructure(
+ event_type=Equals("charm-recipe:build:0.1"),
+ payload=MatchesDict(expected_payload))
+ for expected_payload in expected_payloads]
+ self.assertThat(deliveries, MatchesListwise(matchers))
+ with dbuser(config.IWebhookDeliveryJobSource.dbuser):
+ for delivery in deliveries:
+ self.assertEqual(
+ "<WebhookDeliveryJob for webhook %d on %r>" % (
+ hook.id, hook.target),
+ repr(delivery))
+ self.assertThat(
+ logger.output, LogsScheduledWebhooks([
+ (hook, "charm-recipe:build:0.1",
+ MatchesDict(expected_payload))
+ for expected_payload in expected_payloads]))
+
def test_run(self):
# The job uploads the build to Charmhub and records the Charmhub
# revision.
+ logger = self.useFixture(FakeLogger())
build = self.makeCharmRecipeBuild()
self.assertContentEqual([], build.store_upload_jobs)
job = CharmhubUploadJob.create(build)
@@ -146,9 +189,11 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
+ self.assertWebhookDeliveries(build, ["Pending", "Uploaded"], logger)
def test_run_failed(self):
# A failed run sets the store upload status to FAILED.
+ logger = self.useFixture(FakeLogger())
build = self.makeCharmRecipeBuild()
self.assertContentEqual([], build.store_upload_jobs)
job = CharmhubUploadJob.create(build)
@@ -164,9 +209,12 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
self.assertIsNone(job.store_revision)
self.assertEqual("An upload failure", job.error_message)
self.assertEqual([], pop_notifications())
+ self.assertWebhookDeliveries(
+ build, ["Pending", "Failed to upload"], logger)
def test_run_unauthorized_notifies(self):
# A run that gets 401 from Charmhub sends mail.
+ logger = self.useFixture(FakeLogger())
requester = self.factory.makePerson(name="requester")
requester_team = self.factory.makeTeam(
owner=requester, name="requester-team", members=[requester])
@@ -216,10 +264,13 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
"test-charm/+build/%d\n"
"Your team Requester Team is the requester of the build.\n" %
build.id, footer)
+ self.assertWebhookDeliveries(
+ build, ["Pending", "Failed to upload"], logger)
def test_run_502_retries(self):
# A run that gets a 502 error from Charmhub schedules itself to be
# retried.
+ logger = self.useFixture(FakeLogger())
build = self.makeCharmRecipeBuild()
self.assertContentEqual([], build.store_upload_jobs)
job = CharmhubUploadJob.create(build)
@@ -237,6 +288,7 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.WAITING, job.job.status)
+ self.assertWebhookDeliveries(build, ["Pending"], logger)
# Try again. The upload part of the job is retried, and this time
# it succeeds.
job.scheduled_start = None
@@ -254,10 +306,12 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
+ self.assertWebhookDeliveries(build, ["Pending", "Uploaded"], logger)
def test_run_upload_failure_notifies(self):
# A run that gets some other upload failure from Charmhub sends
# mail.
+ logger = self.useFixture(FakeLogger())
requester = self.factory.makePerson(name="requester")
requester_team = self.factory.makeTeam(
owner=requester, name="requester-team", members=[requester])
@@ -306,12 +360,15 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
self.assertEqual(
"%s\nYour team Requester Team is the requester of the build.\n" %
build_url, footer)
+ self.assertWebhookDeliveries(
+ build, ["Pending", "Failed to upload"], logger)
self.assertIn(
("error_detail", "The proxy exploded.\n"), job.getOopsVars())
def test_run_review_pending_retries(self):
# A run that finds that Charmhub has not yet finished reviewing the
# charm schedules itself to be retried.
+ logger = self.useFixture(FakeLogger())
build = self.makeCharmRecipeBuild()
self.assertContentEqual([], build.store_upload_jobs)
job = CharmhubUploadJob.create(build)
@@ -329,6 +386,7 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.WAITING, job.job.status)
+ self.assertWebhookDeliveries(build, ["Pending"], logger)
# Try again. The upload part of the job is not retried, and this
# time the review completes.
job.scheduled_start = None
@@ -346,9 +404,11 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
+ self.assertWebhookDeliveries(build, ["Pending", "Uploaded"], logger)
def test_run_review_failure_notifies(self):
# A run that gets a review failure from Charmhub sends mail.
+ logger = self.useFixture(FakeLogger())
requester = self.factory.makePerson(name="requester")
requester_team = self.factory.makeTeam(
owner=requester, name="requester-team", members=[requester])
@@ -397,10 +457,13 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
"test-charm/+build/%d\n"
"Your team Requester Team is the requester of the build.\n" %
build.id, footer)
+ self.assertWebhookDeliveries(
+ build, ["Pending", "Failed to upload"], logger)
def test_run_release(self):
# A run configured to automatically release the charm to certain
# channels does so.
+ logger = self.useFixture(FakeLogger())
build = self.makeCharmRecipeBuild(store_channels=["stable", "edge"])
self.assertContentEqual([], build.store_upload_jobs)
job = CharmhubUploadJob.create(build)
@@ -417,10 +480,12 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
+ self.assertWebhookDeliveries(build, ["Pending", "Uploaded"], logger)
def test_run_release_failure_notifies(self):
# A run configured to automatically release the charm to certain
# channels but that fails to do so sends mail.
+ logger = self.useFixture(FakeLogger())
requester = self.factory.makePerson(name="requester")
requester_team = self.factory.makeTeam(
owner=requester, name="requester-team", members=[requester])
@@ -467,6 +532,8 @@ class TestCharmhubUploadJob(TestCaseWithFactory):
"test-charm/+build/%d\n"
"Your team Requester Team is the requester of the build.\n" %
build.id, footer)
+ self.assertWebhookDeliveries(
+ build, ["Pending", "Failed to release to channels"], logger)
def test_retry_delay(self):
# The job is retried every minute, unless it just made one of its
