launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging lp:~cjwatson/launchpad/celery-disable-pickle into lp:launchpad.

Commit message:
Use JSON to serialise Celery tasks and results.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/celery-disable-pickle/+merge/364045

This will help with migrating to Celery 4.x.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/celery-disable-pickle into lp:launchpad.

=== modified file 'lib/lp/services/job/celeryconfig.py'
--- lib/lp/services/job/celeryconfig.py	2015-10-12 13:16:54 +0000
+++ lib/lp/services/job/celeryconfig.py	2019-03-06 16:28:19 +0000
@@ -1,4 +1,4 @@
-# Copyright 2012-2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2019 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 from datetime import timedelta
@@ -84,6 +84,8 @@
     # XXX wgrant 2015-08-03: Celery 3.2 won't read pickles by default,
     # and Celery 3.1 can send only pickles for some things. Let's accept
     # both until they sort things out.
+    # XXX cjwatson 2019-03-06: Remove this once production is using json as
+    # its task/result serialiser.
     result['CELERY_ACCEPT_CONTENT'] = ['pickle', 'json']
     result['CELERY_CREATE_MISSING_QUEUES'] = False
     result['CELERY_DEFAULT_EXCHANGE'] = 'job'
@@ -92,6 +94,8 @@
     result['CELERY_IMPORTS'] = ("lp.services.job.celeryjob", )
     result['CELERY_QUEUES'] = celery_queues
     result['CELERY_RESULT_BACKEND'] = 'amqp'
+    result['CELERY_RESULT_SERIALIZER'] = 'json'
+    result['CELERY_TASK_SERIALIZER'] = 'json'
     result['CELERYBEAT_SCHEDULE'] = {
         'schedule-missing': {
             'task': 'lp.services.job.celeryjob.run_missing_ready',

=== modified file 'lib/lp/services/job/tests/__init__.py'
--- lib/lp/services/job/tests/__init__.py	2016-10-06 12:03:47 +0000
+++ lib/lp/services/job/tests/__init__.py	2019-03-06 16:28:19 +0000
@@ -1,4 +1,4 @@
-# Copyright 2012-2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2019 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __metaclass__ = type
@@ -12,6 +12,7 @@
 
 
 from contextlib import contextmanager
+import email
 import subprocess
 
 from testtools.content import text_content
@@ -94,4 +95,6 @@
 def pop_remote_notifications():
     """Pop the notifications from a celery worker."""
     from lp.services.job.tests.celery_helpers import pop_notifications
-    return pop_notifications.delay().get(30)
+    return [
+        email.message_from_string(message)
+        for message in pop_notifications.delay().get(30)]

=== modified file 'lib/lp/services/job/tests/celery_helpers.py'
--- lib/lp/services/job/tests/celery_helpers.py	2012-07-26 14:31:45 +0000
+++ lib/lp/services/job/tests/celery_helpers.py	2019-03-06 16:28:19 +0000
@@ -1,4 +1,4 @@
-# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2019 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __metaclass__ = type
@@ -20,7 +20,7 @@
 @task
 def pop_notifications():
     from lp.testing.mail_helpers import pop_notifications
-    return pop_notifications()
+    return [message.as_string() for message in pop_notifications()]
 
 
 @task