launchpad-reviewers team mailing list archive

Thread
Date

[Merge] ~pappacena/turnip:celery-worker-charm into turnip:master

To: mp+387362@xxxxxxxxxxxxxxxxxx
From: "Thiago F. Pappacena" <pappacena@xxxxxxxxx>
Date: Fri, 17 Jul 2020 22:22:22 -0000
Reply-to: mp+387362@xxxxxxxxxxxxxxxxxx
Sender: bounces@xxxxxxxxxxxxx

Thiago F. Pappacena has proposed merging ~pappacena/turnip:celery-worker-charm into turnip:master with ~pappacena/turnip:celery-repo-creation as a prerequisite.

Commit message:
WIP: Adding juju setup for celery and rabbitmq

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~pappacena/turnip/+git/turnip/+merge/387362
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~pappacena/turnip:celery-worker-charm into turnip:master.

diff --git a/charm/Makefile b/charm/Makefile
index 2b01ef9..147eeb2 100644
--- a/charm/Makefile
+++ b/charm/Makefile
@@ -19,7 +19,8 @@ CHARMS := \
 	turnip-pack-frontend-git \
 	turnip-pack-frontend-ssh \
 	turnip-pack-frontend-http \
-	turnip-api
+	turnip-api \
+	turnip-celery
 
 PUBLISH_REPO_PREFIX := lp:~canonical-launchpad-branches/turnip/+git/charm-build-
 PUBLISHDIR := $(BUILDDIR)/publish
@@ -49,6 +50,7 @@ build-turnip-pack-frontend-git: dist/.built-turnip-pack-frontend-git
 build-turnip-pack-frontend-ssh: dist/.built-turnip-pack-frontend-ssh
 build-turnip-pack-frontend-http: dist/.built-turnip-pack-frontend-http
 build-turnip-api: dist/.built-turnip-api
+build-turnip-celery: dist/.built-turnip-celery
 
 dist/.built-%: $(CHARM_DEPS) | $(BUILDDIR)
 	@echo "Building $*..."
diff --git a/charm/bundle.yaml.in b/charm/bundle.yaml.in
index 8078e4b..f098894 100644
--- a/charm/bundle.yaml.in
+++ b/charm/bundle.yaml.in
@@ -25,6 +25,9 @@ applications:
           service_port: 9419
       ssl_cert: "%SSL_CERT%"
       ssl_key: "%SSL_KEY%"
+  rabbitmq-server:
+    charm: cs:rabbitmq-server
+    num_units: 1
   turnip-pack-backend:
     charm: ./dist/builds/turnip-pack-backend
     num_units: 1
@@ -74,6 +77,14 @@ applications:
       build_label: "%BUILD_LABEL%"
     resources:
       turnip: "../build/%BUILD_LABEL%/turnip.tar.gz"
+  turnip-celery:
+    charm: ./dist/builds/turnip-celery
+    num_units: 1
+    to: [turnip-pack-backend]
+    options:
+      build_label: "%BUILD_LABEL%"
+    resources:
+      turnip: "../build/%BUILD_LABEL%/turnip.tar.gz"
 relations:
   - ["haproxy", "turnip-pack-backend"]
   - ["haproxy", "turnip-pack-virt:turnip-pack-backend"]
@@ -85,3 +96,5 @@ relations:
   - ["haproxy", "turnip-pack-frontend-http:turnip-pack-virt"]
   - ["haproxy", "turnip-pack-frontend-http:turnip-pack-frontend-http"]
   - ["haproxy", "turnip-api"]
+  - ['rabbitmq-server:amqp', "turnip-api:amqp"]
+  - ['rabbitmq-server:amqp', "turnip-celery:amqp"]
diff --git a/charm/dependencies.txt b/charm/dependencies.txt
index dc73e4c..b0ee2a5 100644
--- a/charm/dependencies.txt
+++ b/charm/dependencies.txt
@@ -1,4 +1,5 @@
 interface			@
+interface/rabbitmq		git+https://github.com/openstack/charm-interface-rabbitmq;revno=571f486
 interface/http			git+https://github.com/juju-solutions/interface-http;revno=4a232c69
 interface/mount			git+https://github.com/juju-solutions/interface-mount;revno=d5a2526f
 interface/nrpe-external-master	git+https://github.com/cmars/nrpe-external-master-interface;revno=20b2b9fb
diff --git a/charm/layer/turnip-base/lib/charms/turnip/base.py b/charm/layer/turnip-base/lib/charms/turnip/base.py
index 7988fd4..dbeada4 100644
--- a/charm/layer/turnip-base/lib/charms/turnip/base.py
+++ b/charm/layer/turnip-base/lib/charms/turnip/base.py
@@ -16,6 +16,7 @@ from charmhelpers.core import (
 from charmhelpers.fetch import apt_install
 from charmhelpers.payload import archive
 from charms.layer import status
+from charms.reactive import endpoint_from_name
 import six
 import yaml
 
@@ -288,7 +289,9 @@ def configure_service(service_name=None):
             host.service_stop(service_name)
         if not host.service_resume(service_name):
             raise RuntimeError('Failed to start {}'.format(service_name))
-    hookenv.open_port(config['port'])
+    port = config.get('port')
+    if port is not None:
+        hookenv.open_port(port)
     configure_logrotate(config)
 
 
@@ -360,3 +363,20 @@ def publish_website(website, name, port):
             'port': str(port),
             'services': haproxy_services_yaml,
             })
+
+
+def get_rabbitmq_url():
+    rabbitmq = endpoint_from_name('amqp')
+
+    vhost = rabbitmq.vhost() if rabbitmq.vhost() else "/"
+    if not rabbitmq.username():
+        try:
+            rabbitmq.request_access(username="turnip", vhost=vhost)
+        except Exception:
+            pass
+
+    if not rabbitmq.username() or not rabbitmq.password():
+        return None
+
+    user = "%s:%s" % (rabbitmq.username(), rabbitmq.password())
+    return "pyamqp://%s@%s/%s" % (user, rabbitmq.private_address(), vhost)
diff --git a/charm/turnip-api/config.yaml b/charm/turnip-api/config.yaml
index e773d0f..cff36b9 100644
--- a/charm/turnip-api/config.yaml
+++ b/charm/turnip-api/config.yaml
@@ -45,3 +45,7 @@ options:
       - option httplog
       - option httpchk /repo
       - balance leastconn
+  celery_broker:
+    type: string
+    default: pyamqp://guest@localhost//
+    description: Celery broker URL
diff --git a/charm/turnip-api/layer.yaml b/charm/turnip-api/layer.yaml
index 80010c8..3f7b2ef 100644
--- a/charm/turnip-api/layer.yaml
+++ b/charm/turnip-api/layer.yaml
@@ -2,4 +2,5 @@ includes:
     - layer:status
     - layer:turnip-base
     - layer:turnip-storage
+    - interface:rabbitmq
 repo: https://git.launchpad.net/turnip
diff --git a/charm/turnip-api/lib/charms/turnip/api.py b/charm/turnip-api/lib/charms/turnip/api.py
index f55315a..9c3abfc 100644
--- a/charm/turnip-api/lib/charms/turnip/api.py
+++ b/charm/turnip-api/lib/charms/turnip/api.py
@@ -12,10 +12,12 @@ from charmhelpers.core import (
     templating,
     )
 
+from charms.layer import status
 from charms.turnip.base import (
     code_dir,
     data_dir,
     data_mount_unit,
+    get_rabbitmq_url,
     logs_dir,
     reload_systemd,
     venv_dir,
@@ -23,6 +25,11 @@ from charms.turnip.base import (
 
 
 def configure_wsgi():
+    celery_broker = get_rabbitmq_url()
+    if celery_broker is None:
+        if not host.service_running('turnip-api'):
+            status.blocked('Waiting for rabbitmq username / password')
+        return
     config = hookenv.config()
     context = dict(config)
     context.update({
@@ -32,6 +39,7 @@ def configure_wsgi():
         'data_mount_unit': data_mount_unit(),
         'logs_dir': logs_dir(),
         'venv_dir': venv_dir(),
+        'celery_broker': celery_broker,
         })
     if context['wsgi_workers'] == 0:
         context['wsgi_workers'] = cpu_count() * 2 + 1
diff --git a/charm/turnip-api/metadata.yaml b/charm/turnip-api/metadata.yaml
index 7a10773..f506fcb 100644
--- a/charm/turnip-api/metadata.yaml
+++ b/charm/turnip-api/metadata.yaml
@@ -19,3 +19,7 @@ provides:
   nrpe-external-master:
     interface: nrpe-external-master
     scope: container
+requires:
+  amqp:
+    interface: rabbitmq
+    optional: true
diff --git a/charm/turnip-api/reactive/turnip-api.py b/charm/turnip-api/reactive/turnip-api.py
index 565ba92..7d4fc50 100644
--- a/charm/turnip-api/reactive/turnip-api.py
+++ b/charm/turnip-api/reactive/turnip-api.py
@@ -41,6 +41,12 @@ def deconfigure_turnip():
     status.blocked('Waiting for storage to be available')
 
 
+@when('amqp.connected')
+def rabbitmq_available():
+    configure_wsgi()
+    status.active('Ready')
+
+
 @when('nrpe-external-master.available', 'turnip.configured')
 @when_not('turnip.nrpe-external-master.published')
 def nrpe_available():
diff --git a/charm/turnip-api/templates/turnip-api.service.j2 b/charm/turnip-api/templates/turnip-api.service.j2
index 3bb4336..2cecf01 100644
--- a/charm/turnip-api/templates/turnip-api.service.j2
+++ b/charm/turnip-api/templates/turnip-api.service.j2
@@ -13,6 +13,7 @@ WorkingDirectory={{ code_dir }}
 Environment=PATH={{ venv_dir }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 Environment=REPO_STORE={{ data_dir }}/repos
 Environment=TURNIP_LOG_DIR={{ logs_dir }}
+Environment=CELERY_BROKER={{ celery_broker }}
 ExecStart={{ venv_dir }}/bin/gunicorn --config {{ config_file }} --paste api.ini
 ExecReload=/bin/kill -s HUP $MAINPID
 ExecStop=/bin/kill -s TERM $MAINPID
diff --git a/charm/turnip-celery/config.yaml b/charm/turnip-celery/config.yaml
new file mode 100644
index 0000000..9cf765d
--- /dev/null
+++ b/charm/turnip-celery/config.yaml
@@ -0,0 +1,5 @@
+options:
+  celery_broker:
+    type: string
+    default: pyamqp://guest@localhost//
+    description: Celery broker URL
diff --git a/charm/turnip-celery/layer.yaml b/charm/turnip-celery/layer.yaml
new file mode 100644
index 0000000..3f7b2ef
--- /dev/null
+++ b/charm/turnip-celery/layer.yaml
@@ -0,0 +1,6 @@
+includes:
+    - layer:status
+    - layer:turnip-base
+    - layer:turnip-storage
+    - interface:rabbitmq
+repo: https://git.launchpad.net/turnip
diff --git a/charm/turnip-celery/lib/charms/turnip/celery.py b/charm/turnip-celery/lib/charms/turnip/celery.py
new file mode 100644
index 0000000..0fb1ff3
--- /dev/null
+++ b/charm/turnip-celery/lib/charms/turnip/celery.py
@@ -0,0 +1,49 @@
+# Copyright 2018 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+
+from charmhelpers.core import (
+    hookenv,
+    host,
+    templating,
+    )
+
+from charms.layer import status
+from charms.turnip.base import (
+    code_dir,
+    data_dir,
+    data_mount_unit,
+    get_rabbitmq_url,
+    logs_dir,
+    reload_systemd,
+    venv_dir,
+    )
+
+
+def configure_celery():
+    celery_broker = get_rabbitmq_url()
+    if celery_broker is None:
+        if not host.service_running('turnip-celery'):
+            status.blocked('Waiting for rabbitmq username / password')
+        return
+    config = hookenv.config()
+    context = dict(config)
+    context.update({
+        'code_dir': code_dir(),
+        'data_dir': data_dir(),
+        'data_mount_unit': data_mount_unit(),
+        'logs_dir': logs_dir(),
+        'venv_dir': venv_dir(),
+        'celery_broker': celery_broker,
+        })
+    templating.render(
+        'turnip-celery.service.j2',
+        '/lib/systemd/system/turnip-celery.service',
+        context, perms=0o644)
+    reload_systemd()
+    if host.service_running('turnip-celery'):
+        host.service_stop('turnip-celery')
+    if not host.service_resume('turnip-celery'):
+        raise RuntimeError('Failed to start turnip-celery')
diff --git a/charm/turnip-celery/metadata.yaml b/charm/turnip-celery/metadata.yaml
new file mode 100644
index 0000000..5105848
--- /dev/null
+++ b/charm/turnip-celery/metadata.yaml
@@ -0,0 +1,23 @@
+name: turnip-celery
+display-name: turnip-celery
+summary: Turnip celery worker
+maintainer: Colin Watson <cjwatson@xxxxxxxxxxxxx>
+description: >
+  Turnip is a flexible and scalable Git server suite written in Python
+  using Twisted.  This component provides asynchronous processing workers.
+tags:
+  # https://docs.jujucharms.com/devel/en/authors-charm-metadata#charm-store-fields
+  - network
+  - web_server
+series:
+  - bionic
+  - xenial
+subordinate: false
+provides:
+  nrpe-external-master:
+    interface: nrpe-external-master
+    scope: container
+requires:
+  amqp:
+    interface: rabbitmq
+    optional: true
diff --git a/charm/turnip-celery/reactive/turnip-celery.py b/charm/turnip-celery/reactive/turnip-celery.py
new file mode 100644
index 0000000..0573f7d
--- /dev/null
+++ b/charm/turnip-celery/reactive/turnip-celery.py
@@ -0,0 +1,65 @@
+# Copyright 2018 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from charmhelpers.core import hookenv
+from charms.layer import status
+from charms.reactive import (
+    clear_flag,
+    endpoint_from_flag,
+    set_flag,
+    when,
+    when_not,
+    )
+
+from charms.turnip.celery import configure_celery
+from charms.turnip.base import (
+    configure_service,
+    deconfigure_service,
+    )
+
+
+@when('turnip.installed', 'turnip.storage.available')
+@when_not('turnip.configured')
+def configure_turnip():
+    configure_service()
+    set_flag('turnip.configured')
+    clear_flag('turnip.storage.nrpe-external-master.published')
+    clear_flag('turnip.nrpe-external-master.published')
+    clear_flag('turnip.turnip-celery.published')
+    status.active('Ready')
+
+
+@when('turnip.configured')
+@when_not('turnip.storage.available')
+def deconfigure_turnip():
+    deconfigure_service('turnip-celery')
+    clear_flag('turnip.configured')
+    status.blocked('Waiting for storage to be available')
+
+
+@when('amqp.connected')
+def rabbitmq_available():
+    configure_celery()
+    status.active('Ready')
+
+
+@when('nrpe-external-master.available', 'turnip.configured')
+@when_not('turnip.nrpe-external-master.published')
+def nrpe_available():
+    nagios = endpoint_from_flag('nrpe-external-master.available')
+    config = hookenv.config()
+    nagios.add_check(
+        ['/usr/lib/nagios/plugins/check_http', '-H', 'localhost',
+         '-p', str(config['port']), '-j', 'OPTIONS', '-u', '/repo'],
+        name='check_api',
+        description='Git API check',
+        context=config['nagios_context'])
+    set_flag('turnip.nrpe-external-master.published')
+
+
+@when('turnip.nrpe-external-master.published')
+@when_not('nrpe-external-master.available')
+def nrpe_unavailable():
+    clear_flag('turnip.nrpe-external-master.published')
diff --git a/charm/turnip-celery/templates/logrotate.j2 b/charm/turnip-celery/templates/logrotate.j2
new file mode 100644
index 0000000..5087f4f
--- /dev/null
+++ b/charm/turnip-celery/templates/logrotate.j2
@@ -0,0 +1,13 @@
+{{ base_dir }}/logs/turnip-celery.log {
+    rotate 90
+    daily
+    dateext
+    delaycompress
+    compress
+    missingok
+    create 0644 {{ user }} {{ group }}
+    postrotate
+        service turnip-celery reload
+    endscript
+}
+
diff --git a/charm/turnip-celery/templates/turnip-celery.service.j2 b/charm/turnip-celery/templates/turnip-celery.service.j2
new file mode 100644
index 0000000..6d3c839
--- /dev/null
+++ b/charm/turnip-celery/templates/turnip-celery.service.j2
@@ -0,0 +1,29 @@
+[Unit]
+Description=Turnip celery server
+After=network.target
+{%- if nfs %}
+BindsTo={{ data_mount_unit }}
+After={{ data_mount_unit }}
+{%- endif %}
+
+[Service]
+User={{ user }}
+Group={{ group }}
+WorkingDirectory={{ code_dir }}
+Environment=PATH={{ venv_dir }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+Environment=REPO_STORE={{ data_dir }}/repos
+Environment=TURNIP_LOG_DIR={{ logs_dir }}
+Environment=CELERY_BROKER={{ celery_broker }}
+Environment=PYTHONPATH=turnip
+ExecStart={{ venv_dir }}/bin/celery -A tasks worker --logfile={{ logs_dir }}/turnip-celery.log
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+LimitNOFILE=1048576
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target
+{%- if nfs %}
+WantedBy={{ data_mount_unit }}
+{%- endif %}
+

Follow ups

Re: [Merge] ~pappacena/turnip:celery-worker-charm into turnip:master
From: Thiago F. Pappacena, 2020-07-20
[Merge] ~pappacena/turnip:celery-worker-charm into turnip:master
From: Thiago F. Pappacena, 2020-07-20