launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #18950
[Merge] lp:~wgrant/launchpad/webhook-model into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/webhook-model into lp:launchpad with lp:~wgrant/launchpad/webhook-db as a prerequisite.
Commit message:
Add basic webhook model for Git repositories.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/webhook-model/+merge/264004
This branch adds the initial model for webhooks.
The owner of an IWebhookTarget (currently IGitRepository) can create IWebhooks to receive events related to that target. Each event creates an IWebhookDeliveryJob (a "delivery") which is stored as a WebhookJob. A delivery may be retried many times, automatically via the normal Job retry machinery, or triggered manually by the user.
This is a bit of a skeleton. Retries and pruning will have their own branches later, and the actual delivery method is the minimum needed for testing.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/webhook-model into lp:launchpad.
=== modified file 'configs/testrunner/launchpad-lazr.conf'
--- configs/testrunner/launchpad-lazr.conf 2014-01-15 22:24:31 +0000
+++ configs/testrunner/launchpad-lazr.conf 2015-07-07 08:01:42 +0000
@@ -180,3 +180,6 @@
[testkeyserver]
root: /var/tmp/testkeyserver.test
+
+[webhooks]
+http_proxy: http://example.com:3128/
=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg 2015-06-08 05:02:31 +0000
+++ database/schema/security.cfg 2015-07-07 08:01:42 +0000
@@ -321,6 +321,8 @@
public.validpersonorteamcache = SELECT
public.vote = SELECT, INSERT, UPDATE
public.votecast = SELECT, INSERT
+public.webhook = SELECT, INSERT, UPDATE, DELETE
+public.webhookjob = SELECT, INSERT, UPDATE, DELETE
public.wikiname = SELECT, INSERT, UPDATE, DELETE
type=user
@@ -2448,3 +2450,10 @@
public.product = SELECT
public.productseries = SELECT
public.sourcepackagename = SELECT
+
+[webhookrunner]
+type=user
+groups=script
+public.job = SELECT, UPDATE
+public.webhook = SELECT
+public.webhookjob = SELECT, UPDATE
=== modified file 'lib/lp/security.py'
--- lib/lp/security.py 2015-06-26 06:30:46 +0000
+++ lib/lp/security.py 2015-07-07 08:01:42 +0000
@@ -183,6 +183,7 @@
)
from lp.services.openid.interfaces.openididentifier import IOpenIdIdentifier
from lp.services.webapp.interfaces import ILaunchpadRoot
+from lp.services.webhooks.interfaces import IWebhook
from lp.services.worlddata.interfaces.country import ICountry
from lp.services.worlddata.interfaces.language import (
ILanguage,
@@ -3053,3 +3054,16 @@
class AdminLiveFSBuild(AdminByBuilddAdmin):
usedfor = ILiveFSBuild
+
+
+class ViewWebhook(AuthorizationBase):
+ """Webhooks can be viewed and edited by someone who can edit the target."""
+ permission = 'launchpad.View'
+ usedfor = IWebhook
+
+ def checkUnauthenticated(self):
+ return False
+
+ def checkAuthenticated(self, user):
+ return self.forwardCheckAuthenticated(
+ user, self.obj.target, 'launchpad.Edit')
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2015-05-27 11:04:43 +0000
+++ lib/lp/services/config/schema-lazr.conf 2015-07-07 08:01:42 +0000
@@ -1715,6 +1715,13 @@
# datatype: boolean
send_email: true
+[webhooks]
+# Outbound webhook request proxy. Users can use webhooks to trigger requests to
+# arbitrary URLs with somewhat user-controlled content, and services
+# like xmlrpc-private are restricted only by firewalls rather than
+# credentials, so the proxy needs to be very carefully secured.
+http_proxy: none
+
[process-job-source-groups]
# This section is used by cronscripts/process-job-source-groups.py.
dbuser: process-job-source-groups
@@ -1851,6 +1858,10 @@
dbuser: product-job
crontab_group: MAIN
+[IWebhookDeliveryJobSource]
+module: lp.services.webhooks.interfaces
+dbuser: webhookrunner
+
[job_runner_queues]
# The names of all queues.
queues: launchpad_job launchpad_job_slow bzrsyncd_job bzrsyncd_job_slow branch_write_job branch_write_job_slow celerybeat
=== modified file 'lib/lp/services/configure.zcml'
--- lib/lp/services/configure.zcml 2011-12-24 18:33:21 +0000
+++ lib/lp/services/configure.zcml 2015-07-07 08:01:42 +0000
@@ -31,6 +31,7 @@
<include package=".statistics" />
<include package=".temporaryblobstorage" />
<include package=".verification" />
+ <include package=".webhooks" />
<include package=".webservice" />
<include package=".worlddata" />
</configure>
=== added directory 'lib/lp/services/webhooks'
=== added file 'lib/lp/services/webhooks/__init__.py'
=== added file 'lib/lp/services/webhooks/client.py'
--- lib/lp/services/webhooks/client.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/webhooks/client.py 2015-07-07 08:01:42 +0000
@@ -0,0 +1,54 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Communication with the Git hosting service."""
+
+__metaclass__ = type
+__all__ = [
+ 'WebhookClient',
+ ]
+
+import requests
+from zope.interface import implements
+
+from lp.services.webhooks.interfaces import IWebhookClient
+
+
+class WebhookClient:
+ implements(IWebhookClient)
+
+ def deliver(self, url, proxy, payload):
+ """See `IWebhookClient`."""
+ # We never want to execute a job if there's no proxy configured, as
+ # we'd then be sending near-arbitrary requests from a trusted
+ # machine.
+ if proxy is None:
+ raise Exception("No webhook proxy configured.")
+ proxies = {'http': proxy, 'https': proxy}
+ if not any(
+ url.startswith("%s://" % scheme)
+ for scheme in proxies.keys()):
+ raise Exception("Unproxied scheme!")
+ session = requests.Session()
+ session.trust_env = False
+ session.headers = {}
+ preq = session.prepare_request(
+ requests.Request('POST', url, json=payload))
+ result = {
+ 'request': {
+ 'url': url,
+ 'method': 'POST',
+ 'headers': dict(preq.headers),
+ 'body': preq.body,
+ },
+ }
+ try:
+ resp = session.send(preq, proxies=proxies)
+ result['response'] = {
+ 'status_code': resp.status_code,
+ 'headers': dict(resp.headers),
+ 'body': resp.content,
+ }
+ except requests.ConnectionError as e:
+ result['connection_error'] = str(e)
+ return result
=== added file 'lib/lp/services/webhooks/configure.zcml'
--- lib/lp/services/webhooks/configure.zcml 1970-01-01 00:00:00 +0000
+++ lib/lp/services/webhooks/configure.zcml 2015-07-07 08:01:42 +0000
@@ -0,0 +1,28 @@
+<!-- Copyright 2010 Canonical Ltd. This software is licensed under the
+ GNU Affero General Public License version 3 (see the file LICENSE).
+-->
+
+<configure xmlns="http://namespaces.zope.org/zope";>
+
+ <class class="lp.services.webhooks.model.Webhook">
+ <require
+ permission="launchpad.View"
+ interface="lp.services.webhooks.interfaces.IWebhook"
+ set_schema="lp.services.webhooks.interfaces.IWebhook"/>
+ </class>
+ <subscriber
+ for="lp.services.webhooks.interfaces.IWebhook zope.lifecycleevent.interfaces.IObjectModifiedEvent"
+ handler="lp.services.webhooks.model.webhook_modified"/>
+
+ <securedutility
+ class="lp.services.webhooks.model.WebhookSource"
+ provides="lp.services.webhooks.interfaces.IWebhookSource">
+ <allow interface="lp.services.webhooks.interfaces.IWebhookSource"/>
+ </securedutility>
+
+ <utility
+ provides="lp.services.webhooks.interfaces.IWebhookClient"
+ factory="lp.services.webhooks.client.WebhookClient"
+ permission="zope.Public"/>
+
+</configure>
=== added file 'lib/lp/services/webhooks/interfaces.py'
--- lib/lp/services/webhooks/interfaces.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/webhooks/interfaces.py 2015-07-07 08:01:42 +0000
@@ -0,0 +1,153 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Webhook interfaces."""
+
+__metaclass__ = type
+
+__all__ = [
+ 'IWebhook',
+ 'IWebhookClient',
+ 'IWebhookDeliveryJob',
+ 'IWebhookDeliveryJobSource',
+ 'IWebhookJob',
+ 'IWebhookSource',
+ ]
+
+from lazr.restful.declarations import exported
+from lazr.restful.fields import Reference
+from zope.interface import (
+ Attribute,
+ Interface,
+ )
+from zope.schema import (
+ Bool,
+ Datetime,
+ Dict,
+ Int,
+ List,
+ TextLine,
+ )
+
+from lp import _
+from lp.registry.interfaces.person import IPerson
+from lp.services.job.interfaces.job import (
+ IJob,
+ IJobSource,
+ IRunnableJob,
+ )
+
+
+class IWebhook(Interface):
+
+ id = Int(title=_("ID"), readonly=True, required=True)
+
+ target = exported(Reference(
+ title=_("Target"), schema=IPerson, required=True, readonly=True,
+ description=_("The object for which this webhook receives events.")))
+ event_types = exported(List(
+ TextLine(), title=_("Event types"),
+ description=_(
+ "The event types for which this webhook receives events."),
+ required=True, readonly=False))
+ registrant = exported(Reference(
+ title=_("Registrant"), schema=IPerson, required=True, readonly=True,
+ description=_("The person who created this webhook.")))
+ date_created = exported(Datetime(
+ title=_("Date created"), required=True, readonly=True))
+ date_last_modified = exported(Datetime(
+ title=_("Date last modified"), required=True, readonly=True))
+
+ delivery_url = exported(Bool(
+ title=_("URL"), required=True, readonly=False))
+ active = exported(Bool(
+ title=_("Active"), required=True, readonly=False))
+ secret = TextLine(
+ title=_("Unique name"), required=False, readonly=True)
+
+
+class IWebhookSource(Interface):
+
+ def new(target, registrant, delivery_url, event_types, active, secret):
+ """Create a new webhook."""
+
+ def delete(hooks):
+ """Delete a collection of webhooks."""
+
+ def getByID(id):
+ """Get a webhook by its ID."""
+
+ def findByTarget(target):
+ """Find all webhooks for the given target."""
+
+
+class IWebhookJob(Interface):
+ """A job related to a webhook."""
+
+ job = Reference(
+ title=_("The common Job attributes."), schema=IJob,
+ required=True, readonly=True)
+
+ webhook = Reference(
+ title=_("The webhook that this job is for."),
+ schema=IWebhook, required=True, readonly=True)
+
+ json_data = Attribute(_("A dict of data about the job."))
+
+
+class IWebhookDeliveryJob(IRunnableJob):
+ """A Job that delivers an event to a webhook consumer."""
+
+ webhook = exported(Reference(
+ title=_("Webhook"),
+ description=_("The webhook that this delivery is for."),
+ schema=IWebhook, required=True, readonly=True))
+
+ pending = exported(Bool(
+ title=_("Pending"),
+ description=_("Whether a delivery attempt is in progress."),
+ required=True, readonly=True))
+
+ successful = exported(Bool(
+ title=_("Successful"),
+ description=_(
+ "Whether the most recent delivery attempt succeeded, or null if "
+ "no attempts have been made yet."),
+ required=False, readonly=True))
+
+ date_created = exported(Datetime(
+ title=_("Date created"), required=True, readonly=True))
+
+ date_sent = exported(Datetime(
+ title=_("Date sent"),
+ description=_("Timestamp of the last delivery attempt."),
+ required=False, readonly=True))
+
+ payload = exported(Dict(
+ title=_('Event payload'),
+ key_type=TextLine(), required=True, readonly=True))
+
+
+class IWebhookDeliveryJobSource(IJobSource):
+
+ def create(webhook):
+ """Deliver an event to a webhook consumer.
+
+ :param webhook: The webhook to deliver to.
+ """
+
+
+class IWebhookClient(Interface):
+
+ def deliver(self, url, proxy, payload):
+ """Deliver a payload to a webhook endpoint.
+
+ Returns a dict of request and response details. The 'request' key
+ and one of either 'response' or 'connection_error' are always
+ present.
+
+ An exception will be raised if an internal error has occurred that
+ cannot be the fault of the remote endpoint. For example, a 404 will
+ return a response, and a DNS error returns a connection_error, but
+ the proxy being offline will raise an exception.
+ """
=== added file 'lib/lp/services/webhooks/model.py'
--- lib/lp/services/webhooks/model.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/webhooks/model.py 2015-07-07 08:01:42 +0000
@@ -0,0 +1,257 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+__all__ = [
+ 'Webhook',
+ 'WebhookJob',
+ 'WebhookJobType',
+ ]
+
+import datetime
+
+import iso8601
+from lazr.delegates import delegates
+from lazr.enum import (
+ DBEnumeratedType,
+ DBItem,
+ )
+import pytz
+from storm.properties import (
+ Bool,
+ DateTime,
+ Int,
+ JSON,
+ Unicode,
+ )
+from storm.references import Reference
+from storm.store import Store
+from zope.component import getUtility
+from zope.interface import (
+ classProvides,
+ implements,
+ )
+from zope.security.proxy import removeSecurityProxy
+
+from lp.services.config import config
+from lp.services.database.constants import UTC_NOW
+from lp.services.database.enumcol import EnumCol
+from lp.services.database.interfaces import IStore
+from lp.services.database.stormbase import StormBase
+from lp.services.job.model.job import (
+ EnumeratedSubclass,
+ Job,
+ )
+from lp.services.job.runner import BaseRunnableJob
+from lp.services.webhooks.interfaces import (
+ IWebhook,
+ IWebhookClient,
+ IWebhookDeliveryJob,
+ IWebhookDeliveryJobSource,
+ IWebhookJob,
+ )
+
+
+def webhook_modified(webhook, event):
+ """Update the date_last_modified property when a Webhook is modified.
+
+ This method is registered as a subscriber to `IObjectModifiedEvent`
+ events on Webhooks.
+ """
+ if event.edited_fields:
+ removeSecurityProxy(webhook).date_last_modified = UTC_NOW
+
+
+class Webhook(StormBase):
+ """See `IWebhook`."""
+
+ implements(IWebhook)
+
+ __storm_table__ = 'Webhook'
+
+ id = Int(primary=True)
+
+ git_repository_id = Int(name='git_repository')
+ git_repository = Reference(git_repository_id, 'GitRepository.id')
+
+ registrant_id = Int(name='registrant')
+ registrant = Reference(registrant_id, 'Person.id')
+ date_created = DateTime(tzinfo=pytz.UTC, allow_none=False)
+ date_last_modified = DateTime(tzinfo=pytz.UTC, allow_none=False)
+
+ delivery_url = Unicode(allow_none=False)
+ active = Bool(default=True, allow_none=False)
+ secret = Unicode(allow_none=False)
+
+ json_data = JSON(name='json_data')
+
+ @property
+ def target(self):
+ if self.git_repository is not None:
+ return self.git_repository
+ else:
+ raise AssertionError("No target.")
+
+ @property
+ def event_types(self):
+ return (self.json_data or {}).get('event_types', [])
+
+ @event_types.setter
+ def event_types(self, event_types):
+ updated_data = self.json_data or {}
+ assert isinstance(event_types, (list, tuple))
+ assert all(isinstance(v, basestring) for v in event_types)
+ updated_data['event_types'] = event_types
+ self.json_data = updated_data
+
+
+class WebhookSource:
+ """See `IWebhookSource`."""
+
+ def new(self, target, registrant, delivery_url, event_types, active,
+ secret):
+ from lp.code.interfaces.gitrepository import IGitRepository
+ hook = Webhook()
+ if IGitRepository.providedBy(target):
+ hook.git_repository = target
+ else:
+ raise AssertionError("Unsupported target: %r" % (target,))
+ hook.registrant = registrant
+ hook.delivery_url = delivery_url
+ hook.active = active
+ hook.secret = secret
+ hook.event_types = event_types
+ IStore(Webhook).add(hook)
+ return hook
+
+ def delete(self, hooks):
+ for hook in hooks:
+ Store.of(hook).remove(hook)
+
+ def getByID(self, id):
+ return IStore(Webhook).get(Webhook, id)
+
+ def findByTarget(self, target):
+ from lp.code.interfaces.gitrepository import IGitRepository
+ if IGitRepository.providedBy(target):
+ target_filter = Webhook.git_repository == target
+ else:
+ raise AssertionError("Unsupported target: %r" % (target,))
+ return IStore(Webhook).find(Webhook, target_filter)
+
+
+class WebhookJobType(DBEnumeratedType):
+ """Values that `IWebhookJob.job_type` can take."""
+
+ DELIVERY = DBItem(0, """
+ DELIVERY
+
+ This job delivers an event to a webhook's endpoint.
+ """)
+
+
+class WebhookJob(StormBase):
+ """See `IWebhookJob`."""
+
+ __storm_table__ = 'WebhookJob'
+
+ implements(IWebhookJob)
+
+ job_id = Int(name='job', primary=True)
+ job = Reference(job_id, 'Job.id')
+
+ webhook_id = Int(name='webhook', allow_none=False)
+ webhook = Reference(webhook_id, 'Webhook.id')
+
+ job_type = EnumCol(enum=WebhookJobType, notNull=True)
+
+ json_data = JSON('json_data')
+
+ def __init__(self, webhook, job_type, json_data, **job_args):
+ """Constructor.
+
+ Extra keyword arguments are used to construct the underlying Job
+ object.
+
+ :param webhook: The `IWebhook` this job relates to.
+ :param job_type: The `WebhookJobType` of this job.
+ :param json_data: The type-specific variables, as a JSON-compatible
+ dict.
+ """
+ super(WebhookJob, self).__init__()
+ self.job = Job(**job_args)
+ self.webhook = webhook
+ self.job_type = job_type
+ self.json_data = json_data
+
+ def makeDerived(self):
+ return WebhookJobDerived.makeSubclass(self)
+
+
+class WebhookJobDerived(BaseRunnableJob):
+
+ __metaclass__ = EnumeratedSubclass
+
+ delegates(IWebhookJob)
+
+ def __init__(self, webhook_job):
+ self.context = webhook_job
+
+
+class WebhookDeliveryJob(WebhookJobDerived):
+ """A job that delivers an event to a webhook endpoint."""
+
+ implements(IWebhookDeliveryJob)
+
+ classProvides(IWebhookDeliveryJobSource)
+ class_job_type = WebhookJobType.DELIVERY
+
+ config = config.IWebhookDeliveryJobSource
+
+ @classmethod
+ def create(cls, webhook, payload):
+ webhook_job = WebhookJob(
+ webhook, cls.class_job_type, {"payload": payload})
+ job = cls(webhook_job)
+ job.celeryRunOnCommit()
+ return job
+
+ @property
+ def pending(self):
+ return self.job.is_pending
+
+ @property
+ def successful(self):
+ if 'result' not in self.json_data:
+ return None
+ if 'connection_error' in self.json_data['result']:
+ return False
+ status_code = self.json_data['result']['response']['status_code']
+ return 200 <= status_code <= 299
+
+ @property
+ def date_sent(self):
+ if 'date_sent' not in self.json_data:
+ return None
+ return iso8601.parse_date(self.json_data['date_sent'])
+
+ @property
+ def payload(self):
+ return self.json_data['payload']
+
+ def run(self):
+ result = getUtility(IWebhookClient).deliver(
+ self.webhook.delivery_url, config.webhooks.http_proxy,
+ self.payload)
+ # Request and response headers and body may be large, so don't
+ # store them in the frequently-used JSON. We could store them in
+ # the librarian if we wanted them in future.
+ for direction in ('request', 'response'):
+ for attr in ('headers', 'body'):
+ if direction in result and attr in result[direction]:
+ del result[direction][attr]
+ updated_data = self.json_data
+ updated_data['result'] = result
+ updated_data['date_sent'] = datetime.datetime.now(pytz.UTC).isoformat()
+ self.json_data = updated_data
=== added directory 'lib/lp/services/webhooks/tests'
=== added file 'lib/lp/services/webhooks/tests/__init__.py'
=== added file 'lib/lp/services/webhooks/tests/test_webhook.py'
--- lib/lp/services/webhooks/tests/test_webhook.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/webhooks/tests/test_webhook.py 2015-07-07 08:01:42 +0000
@@ -0,0 +1,153 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from lazr.lifecycle.event import ObjectModifiedEvent
+from storm.store import Store
+from testtools.matchers import GreaterThan
+import transaction
+from zope.component import getUtility
+from zope.event import notify
+from zope.security.checker import getChecker
+
+from lp.services.webapp.authorization import check_permission
+from lp.services.webhooks.interfaces import (
+ IWebhook,
+ IWebhookSource,
+ )
+from lp.testing import (
+ admin_logged_in,
+ anonymous_logged_in,
+ login_person,
+ person_logged_in,
+ TestCaseWithFactory,
+ )
+from lp.testing.layers import DatabaseFunctionalLayer
+
+
+class TestWebhook(TestCaseWithFactory):
+
+ layer = DatabaseFunctionalLayer
+
+ def test_modifiedevent_sets_date_last_modified(self):
+ # When a Webhook receives an object modified event, the last modified
+ # date is set to UTC_NOW.
+ webhook = self.factory.makeWebhook()
+ transaction.commit()
+ with admin_logged_in():
+ old_mtime = webhook.date_last_modified
+ notify(ObjectModifiedEvent(
+ webhook, webhook, [IWebhook["delivery_url"]]))
+ with admin_logged_in():
+ self.assertThat(
+ webhook.date_last_modified,
+ GreaterThan(old_mtime))
+
+
+class TestWebhookPermissions(TestCaseWithFactory):
+
+ layer = DatabaseFunctionalLayer
+
+ def test_target_owner_can_view(self):
+ target = self.factory.makeGitRepository()
+ webhook = self.factory.makeWebhook(target=target)
+ with person_logged_in(target.owner):
+ self.assertTrue(check_permission('launchpad.View', webhook))
+
+ def test_random_cannot_view(self):
+ webhook = self.factory.makeWebhook()
+ with person_logged_in(self.factory.makePerson()):
+ self.assertFalse(check_permission('launchpad.View', webhook))
+
+ def test_anonymous_cannot_view(self):
+ webhook = self.factory.makeWebhook()
+ with anonymous_logged_in():
+ self.assertFalse(check_permission('launchpad.View', webhook))
+
+ def test_get_permissions(self):
+ expected_get_permissions = {
+ 'launchpad.View': set((
+ 'active', 'date_created', 'date_last_modified', 'delivery_url',
+ 'event_types', 'id', 'registrant', 'secret', 'target')),
+ }
+ webhook = self.factory.makeWebhook()
+ checker = getChecker(webhook)
+ self.checkPermissions(
+ expected_get_permissions, checker.get_permissions, 'get')
+
+ def test_set_permissions(self):
+ expected_set_permissions = {
+ 'launchpad.View': set(('active', 'delivery_url', 'event_types')),
+ }
+ webhook = self.factory.makeWebhook()
+ checker = getChecker(webhook)
+ self.checkPermissions(
+ expected_set_permissions, checker.set_permissions, 'set')
+
+
+class TestWebhookSource(TestCaseWithFactory):
+
+ layer = DatabaseFunctionalLayer
+
+ def test_new(self):
+ target = self.factory.makeGitRepository()
+ login_person(target.owner)
+ person = self.factory.makePerson()
+ hook = getUtility(IWebhookSource).new(
+ target, person, u'http://path/to/something', ['git:push'], True,
+ u'sekrit')
+ Store.of(hook).flush()
+ self.assertEqual(target, hook.target)
+ self.assertEqual(person, hook.registrant)
+ self.assertIsNot(None, hook.date_created)
+ self.assertEqual(hook.date_created, hook.date_last_modified)
+ self.assertEqual(u'http://path/to/something', hook.delivery_url)
+ self.assertEqual(True, hook.active)
+ self.assertEqual(u'sekrit', hook.secret)
+ self.assertEqual(['git:push'], hook.event_types)
+
+ def test_getByID(self):
+ hook1 = self.factory.makeWebhook()
+ hook2 = self.factory.makeWebhook()
+ with admin_logged_in():
+ self.assertEqual(
+ hook1, getUtility(IWebhookSource).getByID(hook1.id))
+ self.assertEqual(
+ hook2, getUtility(IWebhookSource).getByID(hook2.id))
+ self.assertIs(
+ None, getUtility(IWebhookSource).getByID(1234))
+
+ def test_findByTarget(self):
+ target1 = self.factory.makeGitRepository()
+ target2 = self.factory.makeGitRepository()
+ for target, name in ((target1, 'one'), (target2, 'two')):
+ for i in range(3):
+ self.factory.makeWebhook(
+ target, u'http://path/%s/%d' % (name, i))
+ with person_logged_in(target1.owner):
+ self.assertContentEqual(
+ [u'http://path/one/0', u'http://path/one/1',
+ u'http://path/one/2'],
+ [hook.delivery_url for hook in
+ getUtility(IWebhookSource).findByTarget(target1)])
+ with person_logged_in(target2.owner):
+ self.assertContentEqual(
+ [u'http://path/two/0', u'http://path/two/1',
+ u'http://path/two/2'],
+ [hook.delivery_url for hook in
+ getUtility(IWebhookSource).findByTarget(target2)])
+
+ def test_delete(self):
+ target = self.factory.makeGitRepository()
+ login_person(target.owner)
+ hooks = [
+ self.factory.makeWebhook(target, u'http://path/to/%d' % i)
+ for i in range(3)]
+ self.assertContentEqual(
+ [u'http://path/to/0', u'http://path/to/1', u'http://path/to/2'],
+ [hook.delivery_url for hook in
+ getUtility(IWebhookSource).findByTarget(target)])
+ getUtility(IWebhookSource).delete(hooks[:2])
+ self.assertContentEqual(
+ [u'http://path/to/2'],
+ [hook.delivery_url for hook in
+ getUtility(IWebhookSource).findByTarget(target)])
=== added file 'lib/lp/services/webhooks/tests/test_webhookjob.py'
--- lib/lp/services/webhooks/tests/test_webhookjob.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/webhooks/tests/test_webhookjob.py 2015-07-07 08:01:42 +0000
@@ -0,0 +1,253 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for `WebhookJob`s."""
+
+__metaclass__ = type
+
+from httmock import (
+ HTTMock,
+ urlmatch,
+ )
+import requests
+from testtools import TestCase
+from testtools.matchers import (
+ Contains,
+ ContainsDict,
+ Equals,
+ Is,
+ KeysEqual,
+ MatchesAll,
+ MatchesStructure,
+ Not,
+ )
+
+from lp.services.job.interfaces.job import JobStatus
+from lp.services.job.runner import JobRunner
+from lp.services.webhooks.client import WebhookClient
+from lp.services.webhooks.interfaces import (
+ IWebhookClient,
+ IWebhookDeliveryJob,
+ IWebhookJob,
+ )
+from lp.services.webhooks.model import (
+ WebhookDeliveryJob,
+ WebhookJob,
+ WebhookJobDerived,
+ WebhookJobType,
+ )
+from lp.testing import TestCaseWithFactory
+from lp.testing.dbuser import dbuser
+from lp.testing.fixture import (
+ CaptureOops,
+ ZopeUtilityFixture,
+ )
+from lp.testing.layers import (
+ DatabaseFunctionalLayer,
+ LaunchpadZopelessLayer,
+ )
+
+
+class TestWebhookJob(TestCaseWithFactory):
+ """Tests for `WebhookJob`."""
+
+ layer = DatabaseFunctionalLayer
+
+ def test_provides_interface(self):
+ # `WebhookJob` objects provide `IWebhookJob`.
+ hook = self.factory.makeWebhook()
+ self.assertProvides(
+ WebhookJob(hook, WebhookJobType.DELIVERY, {}), IWebhookJob)
+
+
+class TestWebhookJobDerived(TestCaseWithFactory):
+ """Tests for `WebhookJobDerived`."""
+
+ layer = LaunchpadZopelessLayer
+
+ def test_getOopsMailController(self):
+ """By default, no mail is sent about failed WebhookJobs."""
+ hook = self.factory.makeWebhook()
+ job = WebhookJob(hook, WebhookJobType.DELIVERY, {})
+ derived = WebhookJobDerived(job)
+ self.assertIsNone(derived.getOopsMailController("x"))
+
+
+class TestWebhookClient(TestCase):
+ """Tests for `WebhookClient`."""
+
+ def sendToWebhook(self, response_status=200, raises=None):
+ reqs = []
+
+ @urlmatch(netloc='hookep.com')
+ def endpoint_mock(url, request):
+ if raises:
+ raise raises
+ reqs.append(request)
+ return {'status_code': response_status, 'content': 'Content'}
+
+ with HTTMock(endpoint_mock):
+ result = WebhookClient().deliver(
+ 'http://hookep.com/foo',
+ {'http': 'http://squid.example.com:3128'},
+ {'foo': 'bar'})
+
+ return reqs, result
+
+ def test_sends_request(self):
+ [request], result = self.sendToWebhook()
+ self.assertEqual(
+ {'Content-Type': 'application/json', 'Content-Length': '14'},
+ result['request']['headers'])
+ self.assertEqual('{"foo": "bar"}', result['request']['body'])
+ self.assertEqual(200, result['response']['status_code'])
+ self.assertEqual({}, result['response']['headers'])
+ self.assertEqual('Content', result['response']['body'])
+
+ def test_accepts_404(self):
+ [request], result = self.sendToWebhook(response_status=404)
+ self.assertEqual(
+ {'Content-Type': 'application/json', 'Content-Length': '14'},
+ result['request']['headers'])
+ self.assertEqual('{"foo": "bar"}', result['request']['body'])
+ self.assertEqual(404, result['response']['status_code'])
+ self.assertEqual({}, result['response']['headers'])
+ self.assertEqual('Content', result['response']['body'])
+
+ def test_connection_error(self):
+ # Attempts that fail to connect have a connection_error rather
+ # than a response.
+ reqs, result = self.sendToWebhook(
+ raises=requests.ConnectionError('Connection refused'))
+ self.assertNotIn('response', result)
+ self.assertEqual(
+ 'Connection refused', result['connection_error'])
+ self.assertEqual([], reqs)
+
+
+class MockWebhookClient:
+
+ def __init__(self, response_status=200, raises=None):
+ self.response_status = response_status
+ self.raises = raises
+ self.requests = []
+
+ def deliver(self, url, proxy, payload):
+ result = {'request': {}}
+ if isinstance(self.raises, requests.ConnectionError):
+ result['connection_error'] = str(self.raises)
+ elif self.raises is not None:
+ raise self.raises
+ else:
+ self.requests.append(('POST', url))
+ result['response'] = {'status_code': self.response_status}
+ return result
+
+
+class TestWebhookDeliveryJob(TestCaseWithFactory):
+ """Tests for `WebhookDeliveryJob`."""
+
+ layer = LaunchpadZopelessLayer
+
+ def makeAndRunJob(self, response_status=200, raises=None, mock=True):
+ hook = self.factory.makeWebhook(delivery_url=u'http://hookep.com/foo')
+ job = WebhookDeliveryJob.create(hook, payload={'foo': 'bar'})
+
+ client = MockWebhookClient(
+ response_status=response_status, raises=raises)
+ if mock:
+ self.useFixture(ZopeUtilityFixture(client, IWebhookClient))
+ with dbuser("webhookrunner"):
+ JobRunner([job]).runAll()
+ return job, client.requests
+
+ def test_provides_interface(self):
+ # `WebhookDeliveryJob` objects provide `IWebhookDeliveryJob`.
+ hook = self.factory.makeWebhook()
+ self.assertProvides(
+ WebhookDeliveryJob.create(hook, payload={}), IWebhookDeliveryJob)
+
+ def test_run_200(self):
+ # A request that returns 200 is a success.
+ with CaptureOops() as oopses:
+ job, reqs = self.makeAndRunJob(response_status=200)
+ self.assertThat(
+ job,
+ MatchesStructure(
+ status=Equals(JobStatus.COMPLETED),
+ pending=Equals(False),
+ successful=Equals(True),
+ date_sent=Not(Is(None)),
+ json_data=ContainsDict(
+ {'result': MatchesAll(
+ KeysEqual('request', 'response'),
+ ContainsDict(
+ {'response': ContainsDict(
+ {'status_code': Equals(200)})}))})))
+ self.assertEqual(1, len(reqs))
+ self.assertEqual([('POST', 'http://hookep.com/foo')], reqs)
+ self.assertEqual([], oopses.oopses)
+
+ def test_run_404(self):
+ # The job succeeds even if the response is an error. A job only
+ # fails if it was definitely a problem on our end.
+ with CaptureOops() as oopses:
+ job, reqs = self.makeAndRunJob(response_status=404)
+ self.assertThat(
+ job,
+ MatchesStructure(
+ status=Equals(JobStatus.COMPLETED),
+ pending=Equals(False),
+ successful=Equals(False),
+ date_sent=Not(Is(None)),
+ json_data=ContainsDict(
+ {'result': MatchesAll(
+ KeysEqual('request', 'response'),
+ ContainsDict(
+ {'response': ContainsDict(
+ {'status_code': Equals(404)})}))})))
+ self.assertEqual(1, len(reqs))
+ self.assertEqual([], oopses.oopses)
+
+ def test_run_connection_error(self):
+ # Jobs that fail to connecthave a connection_error rather than a
+ # response.
+ with CaptureOops() as oopses:
+ job, reqs = self.makeAndRunJob(
+ raises=requests.ConnectionError('Connection refused'))
+ self.assertThat(
+ job,
+ MatchesStructure(
+ status=Equals(JobStatus.COMPLETED),
+ pending=Equals(False),
+ successful=Equals(False),
+ date_sent=Not(Is(None)),
+ json_data=ContainsDict(
+ {'result': MatchesAll(
+ KeysEqual('request', 'connection_error'),
+ ContainsDict(
+ {'connection_error': Equals('Connection refused')})
+ )})))
+ self.assertEqual([], reqs)
+ self.assertEqual([], oopses.oopses)
+
+ def test_run_no_proxy(self):
+ # Since users can cause the webhook runner to make somewhat
+ # controlled POST requests to arbitrary URLs, they're forced to
+ # go through a locked-down HTTP proxy. If none is configured,
+ # the job crashes.
+ self.pushConfig('webhooks', http_proxy=None)
+ with CaptureOops() as oopses:
+ job, reqs = self.makeAndRunJob(response_status=200, mock=False)
+ self.assertThat(
+ job,
+ MatchesStructure(
+ status=Equals(JobStatus.FAILED),
+ pending=Equals(False),
+ successful=Is(None),
+ date_sent=Is(None),
+ json_data=Not(Contains('result'))))
+ self.assertEqual([], reqs)
+ self.assertEqual(1, len(oopses.oopses))
+ self.assertEqual(
+ 'No webhook proxy configured.', oopses.oopses[0]['value'])
=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py 2015-06-25 04:42:48 +0000
+++ lib/lp/testing/factory.py 2015-07-07 08:01:42 +0000
@@ -261,6 +261,7 @@
from lp.services.utils import AutoDecorate
from lp.services.webapp.interfaces import OAuthPermission
from lp.services.webapp.sorting import sorted_version_numbers
+from lp.services.webhooks.interfaces import IWebhookSource
from lp.services.worlddata.interfaces.country import ICountrySet
from lp.services.worlddata.interfaces.language import ILanguageSet
from lp.soyuz.adapters.overrides import SourceOverride
@@ -4521,6 +4522,15 @@
return ProxyFactory(
LiveFSFile(livefsbuild=livefsbuild, libraryfile=libraryfile))
+ def makeWebhook(self, target=None, delivery_url=None):
+ if target is None:
+ target = self.makeGitRepository()
+ if delivery_url is None:
+ delivery_url = self.getUniqueURL().decode('utf-8')
+ return getUtility(IWebhookSource).new(
+ target, self.makePerson(), delivery_url, [], True,
+ self.getUniqueUnicode())
+
# Some factory methods return simple Python types. We don't add
# security wrappers for them, as well as for objects created by
=== modified file 'setup.py'
--- setup.py 2015-03-03 01:36:19 +0000
+++ setup.py 2015-07-07 08:01:42 +0000
@@ -41,6 +41,7 @@
'feedvalidator',
'funkload',
'html5browser',
+ 'httmock',
'pygpgme',
'python-debian',
'python-keystoneclient',
=== modified file 'versions.cfg'
--- versions.cfg 2015-07-02 11:58:33 +0000
+++ versions.cfg 2015-07-07 08:01:42 +0000
@@ -36,6 +36,7 @@
funkload = 1.16.1
grokcore.component = 1.6
html5browser = 0.0.9
+httmock = 1.2.3
httplib2 = 0.8
importlib = 1.0.2
ipython = 0.13.2
@@ -107,7 +108,7 @@
python-swiftclient = 1.5.0
PyYAML = 3.10
rabbitfixture = 0.3.6
-requests = 2.5.1
+requests = 2.7.0
s4 = 0.1.2
setproctitle = 1.1.7
setuptools-git = 1.0
Follow ups
