launchpad-reviewers team mailing list archive

[Ioana Lasc <ioana.lasc@xxxxxxxxxxxxx>]

Ioana Lasc has proposed merging ~ilasc/turnip:one-repack-task-at-a-time into turnip:master.

Commit message:
Run a dedicated repack celery worker

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~ilasc/turnip/+git/turnip/+merge/402770

This proposes we run a dedicated repack worker and limit its concurrency to 1. 
I've hardcoded the worker name (repack-worker), dedicated queue name (repacks), concurrency (1) and prefetch-multiplier (1) for it.
For the default worker I've specified the default queue (celery) everything else comes on this queue.
All of the above can be of course turned into configurable items.
Local end to end test results with the 2 workers and queues documented here: https://docs.google.com/document/d/1nhwft8gOB24_c34is3UOqWm7r-v0uKwazPvQRo7WgU0/edit?usp=sharing


-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~ilasc/turnip:one-repack-task-at-a-time into turnip:master.

diff --git a/Makefile b/Makefile
index 1ecfd8e..dc0f1e8 100644
--- a/Makefile
+++ b/Makefile
@@ -114,11 +114,21 @@ run-worker: $(ENV)
 		--loglevel=debug \
 		--concurrency=20 \
 		--pool=gevent \
-		--queue=repacks,celery
+		--prefetch-multiplier=1 \
+		--queue=celery
+
+run-repack-worker: $(ENV)
+	$(CELERY) -A turnip.tasks worker -n repack-worker \
+		--loglevel=debug \
+		--concurrency=1 \
+		--pool=gevent \
+		--prefetch-multiplier=1 \
+		--queue=repacks
 
 run:
 	make run-api &\
 	make run-pack &\
+	make run-repack-worker&\
 	make run-worker&\
 	wait;
 
@@ -126,7 +136,9 @@ stop:
 	-pkill -f 'make run-api'
 	-pkill -f 'make run-pack'
 	-pkill -f 'make run-worker'
+	-pkill -f 'make run-repack-worker'	
 	-pkill -f '$(CELERY) -A tasks worker'
+	-pkill -f '$(CELERY) -A tasks repack-worker'	
 
 $(PIP_CACHE): $(ENV)
 	mkdir -p $(PIP_CACHE)
diff --git a/charm/turnip-celery/config.yaml b/charm/turnip-celery/config.yaml
index 5d0773a..6cc7929 100644
--- a/charm/turnip-celery/config.yaml
+++ b/charm/turnip-celery/config.yaml
@@ -15,7 +15,4 @@ options:
     type: int
     default: 1
     description: Celery prefetch multiplier for each worker.
-  celery_queues:
-    type: string
-    default: repacks,celery
-    description: Celery queues, comma-separated and enumerated without spaces.
+
diff --git a/charm/turnip-celery/templates/turnip-celery.service.j2 b/charm/turnip-celery/templates/turnip-celery.service.j2
index b612efa..7e0032b 100644
--- a/charm/turnip-celery/templates/turnip-celery.service.j2
+++ b/charm/turnip-celery/templates/turnip-celery.service.j2
@@ -16,7 +16,8 @@ Environment=TURNIP_LOG_DIR={{ logs_dir }}
 Environment=CELERY_BROKER={{ celery_broker }}
 Environment=VIRTINFO_ENDPOINT={{ virtinfo_endpoint }}
 Environment=VIRTINFO_TIMEOUT={{ virtinfo_timeout }}
-ExecStart={{ venv_dir }}/bin/celery -A turnip.tasks worker --logfile={{ logs_dir }}/turnip-celery.log --loglevel=DEBUG --pool=gevent --prefetch-multiplier={{ prefetch_multiplier }} --queue={{ celery_queues }}
+ExecStart={{ venv_dir }}/bin/celery -A turnip.tasks worker --logfile={{ logs_dir }}/turnip-celery.log --loglevel=DEBUG --pool=gevent --prefetch-multiplier={{ prefetch_multiplier }} --queue=celery
+ExecStart={{ venv_dir }}/bin/celery -A turnip.tasks worker -n repack-worker --logfile={{ logs_dir }}/turnip-celery.log --loglevel=DEBUG --pool=gevent --prefetch-multiplier=1 --queue=repacks
 ExecReload=/bin/kill -s HUP $MAINPID
 ExecStop=/bin/kill -s TERM $MAINPID
 LimitNOFILE=1048576