launchpad-reviewers team mailing list archive

[Ioana Lasc <ioana.lasc@xxxxxxxxxxxxx>]

Ioana Lasc has proposed merging ~ilasc/turnip:celery-queue-for-repacks into turnip:master.

Commit message:
Move repacks to their own celery queue

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~ilasc/turnip/+git/turnip/+merge/402485

This moves repacks to their own Celery queue "repacks".
Everything else comes in on the default queue "celery".
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~ilasc/turnip:celery-queue-for-repacks into turnip:master.

diff --git a/Makefile b/Makefile
index cb4204c..1ecfd8e 100644
--- a/Makefile
+++ b/Makefile
@@ -113,7 +113,8 @@ run-worker: $(ENV)
 	$(CELERY) -A turnip.tasks worker \
 		--loglevel=debug \
 		--concurrency=20 \
-		--pool=gevent
+		--pool=gevent \
+		--queue=repacks,celery
 
 run:
 	make run-api &\
diff --git a/charm/turnip-celery/config.yaml b/charm/turnip-celery/config.yaml
index 3b2cad9..2d0b901 100644
--- a/charm/turnip-celery/config.yaml
+++ b/charm/turnip-celery/config.yaml
@@ -15,3 +15,7 @@ options:
     type: int
     default: 1
     description: Celery prefetch multiplier for each worker.
+  celery_queues:
+    type: string
+    default: repacks,celery
+    description: Celery queues
diff --git a/charm/turnip-celery/templates/turnip-celery.service.j2 b/charm/turnip-celery/templates/turnip-celery.service.j2
index a631b86..b612efa 100644
--- a/charm/turnip-celery/templates/turnip-celery.service.j2
+++ b/charm/turnip-celery/templates/turnip-celery.service.j2
@@ -16,7 +16,7 @@ Environment=TURNIP_LOG_DIR={{ logs_dir }}
 Environment=CELERY_BROKER={{ celery_broker }}
 Environment=VIRTINFO_ENDPOINT={{ virtinfo_endpoint }}
 Environment=VIRTINFO_TIMEOUT={{ virtinfo_timeout }}
-ExecStart={{ venv_dir }}/bin/celery -A turnip.tasks worker --logfile={{ logs_dir }}/turnip-celery.log --loglevel=DEBUG --pool=gevent --prefetch-multiplier={{ prefetch_multiplier }}
+ExecStart={{ venv_dir }}/bin/celery -A turnip.tasks worker --logfile={{ logs_dir }}/turnip-celery.log --loglevel=DEBUG --pool=gevent --prefetch-multiplier={{ prefetch_multiplier }} --queue={{ celery_queues }}
 ExecReload=/bin/kill -s HUP $MAINPID
 ExecStop=/bin/kill -s TERM $MAINPID
 LimitNOFILE=1048576
diff --git a/turnip/api/store.py b/turnip/api/store.py
index f1e4664..c7d9c69 100644
--- a/turnip/api/store.py
+++ b/turnip/api/store.py
@@ -472,7 +472,7 @@ def delete_repo(repo_path):
     shutil.rmtree(repo_path)
 
 
-@app.task
+@app.task(queue='repacks')
 def repack(repo_path):
     """Repack a repository with git-repack."""
     logger = tasks_logger
diff --git a/turnip/api/tests/test_store.py b/turnip/api/tests/test_store.py
index e54c127..816887c 100644
--- a/turnip/api/tests/test_store.py
+++ b/turnip/api/tests/test_store.py
@@ -470,7 +470,8 @@ class InitTestCase(TestCase):
         self.assertFalse(self.hasZeroLooseObjects(orig_path))
 
         # Trigger the repack job
-        store.repack.apply_async((orig_path, ))
+        store.repack.apply_async(queue='repacks',
+                                 kwargs={'repo_path': orig_path})
 
         # Assert we have 0 loose objects after repack job ran
         celery_fixture.waitUntil(
diff --git a/turnip/api/views.py b/turnip/api/views.py
index 5bb8d8f..19bd815 100644
--- a/turnip/api/views.py
+++ b/turnip/api/views.py
@@ -148,7 +148,7 @@ class RepackAPI(BaseAPI):
                 'body', 'name', 'repository does not exist')
             raise exc.HTTPNotFound()
         kwargs = dict(repo_path=repo_path)
-        store.repack.apply_async(kwargs=kwargs)
+        store.repack.apply_async(queue='repacks', kwargs=kwargs)
         return Response(status=200)
 
 
diff --git a/turnip/tests/tasks.py b/turnip/tests/tasks.py
index 01845ec..791d522 100644
--- a/turnip/tests/tasks.py
+++ b/turnip/tests/tasks.py
@@ -60,7 +60,8 @@ class CeleryWorkerFixture(fixtures.Fixture):
             '--pool=gevent',
             '--concurrency=20',
             '--broker=%s' % BROKER_URL,
-            '--loglevel=%s' % self.loglevel]
+            '--loglevel=%s' % self.loglevel,
+            '--queue=repacks,celery']
         if self.logfile:
             cmd += ["--logfile=%s" % self.logfile]