launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29824
[Merge] ~cjwatson/launchpad:charm-librarian into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:charm-librarian into launchpad:master.
Commit message:
charm: Add a launchpad-librarian charm
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/439924
This also needs https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/439909 in order for the `librarian-gc` cron job to work, although the merge proposals can land in either order.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:charm-librarian into launchpad:master.
diff --git a/charm/launchpad-librarian/README.md b/charm/launchpad-librarian/README.md
new file mode 100644
index 0000000..4a7f5c0
--- /dev/null
+++ b/charm/launchpad-librarian/README.md
@@ -0,0 +1,25 @@
+# Launchpad librarian
+
+This charm runs a Launchpad librarian.
+
+You will need the following relations:
+
+ juju relate launchpad-librarian:db postgresql:db
+ juju relate launchpad-librarian:session-db postgresql:db
+ juju relate launchpad-librarian rabbitmq-server
+
+The librarian listens on four ports. By default, these are:
+
+- Public download: 8000
+- Public upload: 9090
+- Restricted download: 8005
+- Restricted upload: 9095
+
+The restricted ports allow access to restricted files without
+authentication; firewall rules should ensure that they are only accessible
+by other parts of Launchpad.
+
+You will normally want to mount a persistent volume on
+`/srv/launchpad/librarian/`. (Even when writing uploads to Swift, this is
+currently used as a temporary spool; it is therefore not currently valid to
+deploy more than one unit of this charm.)
diff --git a/charm/launchpad-librarian/charmcraft.yaml b/charm/launchpad-librarian/charmcraft.yaml
new file mode 100644
index 0000000..0556d43
--- /dev/null
+++ b/charm/launchpad-librarian/charmcraft.yaml
@@ -0,0 +1,60 @@
+type: charm
+bases:
+ - build-on:
+ - name: ubuntu
+ channel: "20.04"
+ architectures: [amd64]
+ run-on:
+ - name: ubuntu
+ channel: "20.04"
+ architectures: [amd64]
+parts:
+ charm-wheels:
+ source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels
+ source-commit: "59b32ae07f98051385c96d6d8e7e02ca4f197fe5"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ "*": charm-wheels/
+ prime:
+ - "-charm-wheels"
+ ols-layers:
+ source: https://git.launchpad.net/ols-charm-deps
+ source-commit: "56d219f60a293a6c73759b8439ef5fdb11e19d1f"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ "*": layers/
+ stage:
+ - layers
+ prime:
+ - "-layers"
+ launchpad-layers:
+ after:
+ - ols-layers
+ source: https://git.launchpad.net/launchpad-layers
+ source-commit: "1920a6f823d8d882a99662bdda55f67f37359850"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ launchpad-base: layers/layer/launchpad-base
+ stage:
+ - layers
+ prime:
+ - "-layers"
+ launchpad-librarian:
+ after:
+ - charm-wheels
+ - launchpad-layers
+ source: .
+ plugin: reactive
+ build-snaps: [charm/2.x/stable]
+ build-packages: [libpq-dev]
+ build-environment:
+ - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer
+ - CHARM_INTERFACES_DIR: $CRAFT_STAGE/layers/interface
+ - PIP_NO_INDEX: "true"
+ - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels
diff --git a/charm/launchpad-librarian/config.yaml b/charm/launchpad-librarian/config.yaml
new file mode 100644
index 0000000..36755aa
--- /dev/null
+++ b/charm/launchpad-librarian/config.yaml
@@ -0,0 +1,106 @@
+options:
+ active:
+ type: boolean
+ default: true
+ description: If true, enable jobs that may change the database.
+ nagios_path:
+ type: string
+ default: /
+ description: Path to a file on this librarian to use for Nagios checks.
+ nagios_expected_regex:
+ type: string
+ default: "Launchpad Librarian"
+ description: >
+ A regular expression that the response to `nagios_path` is expected to
+ match.
+ old_os_auth_url:
+ type: string
+ description: >
+ OpenStack authentication URL for a previous Swift instance that we're
+ migrating away from, but should still read from if necessary.
+ default:
+ old_os_auth_version:
+ type: string
+ description: >
+ OpenStack authentication protocol version for a previous Swift
+ instance that we're migrating away from, but should still read from if
+ necessary.
+ default: "2.0"
+ old_os_password:
+ type: string
+ description: >
+ OpenStack password for a previous Swift instance that we're migrating
+ away from, but should still read from if necessary.
+ default:
+ old_os_tenant_name:
+ type: string
+ description: >
+ OpenStack tenant name for a previous Swift instance that we're
+ migrating away from, but should still read from if necessary.
+ default:
+ old_os_username:
+ type: string
+ description: >
+ OpenStack username for a previous Swift instance that we're migrating
+ away from, but should still read from if necessary.
+ default:
+ os_auth_url:
+ type: string
+ description: OpenStack authentication URL.
+ default:
+ os_auth_version:
+ type: string
+ description: OpenStack authentication protocol version.
+ default: "2.0"
+ os_password:
+ type: string
+ description: OpenStack password.
+ default:
+ os_tenant_name:
+ type: string
+ description: OpenStack tenant name.
+ default:
+ os_username:
+ type: string
+ description: OpenStack username.
+ default:
+ port_download_base:
+ type: int
+ description: Base port number for public download workers.
+ default: 8000
+ port_restricted_download_base:
+ type: int
+ description: Base port number for restricted download workers.
+ default: 8005
+ port_restricted_upload_base:
+ type: int
+ description: Base port number for restricted upload workers.
+ default: 9095
+ port_upload_base:
+ type: int
+ description: Base port number for public upload workers.
+ default: 9090
+ swift_feed_workers:
+ type: int
+ description: Number of librarian-feed-swift workers to run in parallel.
+ default: 1
+ swift_timeout:
+ type: int
+ description: Time in seconds to wait for a response from Swift.
+ default: 15
+ upstream_host:
+ type: string
+ description: Host name for the upstream librarian, if any.
+ default:
+ upstream_port:
+ type: int
+ description: Port for the upstream librarian, if any.
+ default: 80
+ workers:
+ type: int
+ description: >
+ Number of librarian worker processes. If set, each worker will listen
+ on consecutive ports starting from each of `port_download_base`,
+ `port_restricted_download_base`, `port_restricted_upload_base`, and
+ `port_upload_base`, so make sure there is enough space between those.
+ default: 1
diff --git a/charm/launchpad-librarian/layer.yaml b/charm/launchpad-librarian/layer.yaml
new file mode 100644
index 0000000..12f6b73
--- /dev/null
+++ b/charm/launchpad-librarian/layer.yaml
@@ -0,0 +1,19 @@
+includes:
+ - layer:launchpad-base
+repo: https://git.launchpad.net/launchpad
+options:
+ ols-pg:
+ apt:
+ packages:
+ - run-one
+ databases:
+ db:
+ name: launchpad_dev
+ roles:
+ - binaryfile-expire
+ - librarian
+ - librarianfeedswift
+ - librariangc
+ session-db:
+ name: session_dev
+ roles: session
diff --git a/charm/launchpad-librarian/metadata.yaml b/charm/launchpad-librarian/metadata.yaml
new file mode 100644
index 0000000..da12af8
--- /dev/null
+++ b/charm/launchpad-librarian/metadata.yaml
@@ -0,0 +1,18 @@
+name: launchpad-librarian
+display-name: launchpad-librarian
+summary: Launchpad librarian
+maintainer: Colin Watson <cjwatson@xxxxxxxxxxxxx>
+description: |
+ Launchpad is an open source suite of tools that help people and teams
+ to work together on software projects.
+
+ This charm runs a Launchpad librarian.
+tags:
+ # https://juju.is/docs/charm-metadata#heading--charm-store-fields
+ - network
+series:
+ - focal
+subordinate: false
+requires:
+ session-db:
+ interface: pgsql
diff --git a/charm/launchpad-librarian/reactive/launchpad-librarian.py b/charm/launchpad-librarian/reactive/launchpad-librarian.py
new file mode 100644
index 0000000..e8d5adc
--- /dev/null
+++ b/charm/launchpad-librarian/reactive/launchpad-librarian.py
@@ -0,0 +1,194 @@
+# Copyright 2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import os.path
+import shlex
+import subprocess
+
+from charmhelpers.core import hookenv, host, templating
+from charms.launchpad.base import (
+ config_file_path,
+ configure_cron,
+ configure_lazr,
+ get_service_config,
+ lazr_config_files,
+ strip_dsn_authentication,
+ update_pgpass,
+)
+from charms.reactive import (
+ clear_flag,
+ endpoint_from_flag,
+ helpers,
+ hook,
+ set_flag,
+ set_state,
+ when,
+ when_any,
+ when_not,
+)
+from ols import base, postgres
+from psycopg2.extensions import parse_dsn
+
+
+def configure_systemd(config):
+ hookenv.log("Writing systemd service.")
+ templating.render(
+ "launchpad-librarian.service.j2",
+ "/lib/systemd/system/launchpad-librarian.service",
+ config,
+ )
+ templating.render(
+ "launchpad-librarian@.service.j2",
+ "/lib/systemd/system/launchpad-librarian@.service",
+ config,
+ )
+ subprocess.run(["systemctl", "daemon-reload"])
+
+
+def configure_logrotate(config):
+ hookenv.log("Writing logrotate configuration.")
+ templating.render(
+ "logrotate.conf.j2",
+ "/etc/logrotate.d/launchpad-librarian",
+ config,
+ perms=0o644,
+ )
+
+
+def config_files():
+ files = []
+ files.extend(lazr_config_files())
+ files.append(config_file_path("launchpad-librarian/launchpad-lazr.conf"))
+ files.append(
+ config_file_path("launchpad-librarian-secrets-lazr.conf", secret=True)
+ )
+ return files
+
+
+@when(
+ "config.set.port_download_base",
+ "config.set.port_restricted_download_base",
+ "config.set.port_restricted_upload_base",
+ "config.set.port_upload_base",
+ "launchpad.base.configured",
+ "session-db.master.available",
+)
+@when_not("service.configured")
+def configure():
+ session_db = endpoint_from_flag("session-db.master.available")
+ config = get_service_config()
+ session_db_primary, _ = postgres.get_db_uris(session_db)
+ # XXX cjwatson 2022-09-23: Mangle the connection string into a form
+ # Launchpad understands. In the long term it would be better to have
+ # Launchpad be able to consume unmodified connection strings.
+ update_pgpass(session_db_primary)
+ config["db_session"] = strip_dsn_authentication(session_db_primary)
+ config["db_session_user"] = parse_dsn(session_db_primary)["user"]
+ config["librarian_dir"] = os.path.join(base.base_dir(), "librarian")
+ host.mkdir(
+ config["librarian_dir"],
+ owner=base.user(),
+ group=base.user(),
+ perms=0o700,
+ )
+ for i in range(config["workers"]):
+ config["logfile"] = os.path.join(
+ base.logs_dir(), f"librarian{i + 1}.log"
+ )
+ config["worker_download_port"] = config["port_download_base"] + i
+ config["worker_restricted_download_port"] = (
+ config["port_restricted_download_base"] + i
+ )
+ config["worker_restricted_upload_port"] = (
+ config["port_restricted_upload_base"] + i
+ )
+ config["worker_upload_port"] = config["port_upload_base"] + i
+ configure_lazr(
+ config,
+ "launchpad-librarian-lazr.conf",
+ f"launchpad-librarian{i + 1}/launchpad-lazr.conf",
+ )
+ configure_lazr(
+ config,
+ "launchpad-librarian-secrets-lazr.conf",
+ "launchpad-librarian-secrets-lazr.conf",
+ secret=True,
+ )
+ configure_systemd(config)
+ configure_logrotate(config)
+ configure_cron(config, "crontab.j2")
+
+ if helpers.any_file_changed(
+ [
+ base.version_info_path(),
+ "/lib/systemd/system/launchpad-librarian.service",
+ "/lib/systemd/system/launchpad-librarian@.service",
+ ]
+ + config_files()
+ ):
+ hookenv.log("Config files changed; restarting")
+ # Be careful to restart instances one at a time to minimize downtime.
+ # XXX cjwatson 2023-03-28: This doesn't deal with stopping instances
+ # when the worker count is reduced.
+ for i in range(config["workers"]):
+ host.service_restart(f"launchpad-librarian@{i + 1}")
+ else:
+ hookenv.log("Not restarting, since no config files were changed")
+ host.service_resume("launchpad-librarian.service")
+
+ set_state("service.configured")
+
+
+@when("service.configured")
+def check_is_running():
+ hookenv.status_set("active", "Ready")
+
+
+@when("nrpe-external-master.available", "service.configured")
+@when_not("launchpad.librarian.nrpe-external-master.published")
+def nrpe_available():
+ nrpe = endpoint_from_flag("nrpe-external-master.available")
+ config = hookenv.config()
+ # XXX cjwatson 2023-03-28: This doesn't deal with removing checks when
+ # the worker count is reduced.
+ for i in range(config["workers"]):
+ nrpe.add_check(
+ [
+ "/usr/lib/nagios/plugins/check_http",
+ "-H",
+ "localhost",
+ "-p",
+ str(config["port_download_base"] + i),
+ "-u",
+ config["nagios_path"],
+ "-s",
+ shlex.quote(config["nagios_expected_regex"]),
+ "-f",
+ "critical",
+ ],
+ name=f"check_librarian{i + 1}",
+ description=f"Launchpad librarian{i + 1}",
+ context=config["nagios_context"],
+ )
+ set_flag("launchpad.librarian.nrpe-external-master.published")
+
+
+@when("launchpad.librarian.nrpe-external-master.published")
+@when_not("nrpe-external-master.available")
+def nrpe_unavailable():
+ clear_flag("launchpad.librarian.nrpe-external-master.published")
+
+
+@when_any(
+ "config.changed.nagios_expected_regex",
+ "config.changed.nagios_path",
+ "config.changed.workers",
+)
+def nagios_options_changed():
+ clear_flag("launchpad.librarian.nrpe-external-master.published")
+
+
+@hook("upgrade-charm")
+def upgrade_charm():
+ # The ols and launchpad-base layer take care of clearing other flags.
+ clear_flag("launchpad.librarian.nrpe-external-master.published")
diff --git a/charm/launchpad-librarian/templates/crontab.j2 b/charm/launchpad-librarian/templates/crontab.j2
new file mode 100644
index 0000000..a545a63
--- /dev/null
+++ b/charm/launchpad-librarian/templates/crontab.j2
@@ -0,0 +1,25 @@
+TZ=UTC
+MAILTO={{ cron_mailto }}
+LPCONFIG=launchpad-librarian1
+
+{% if active -%}
+45 17 * * * {{ code_dir }}/cronscripts/expire-archive-files.py --expire-after=7 >> {{ logs_dir }}/expire-archive-files.log 2>&1
+
+# Garbage collector. Ensure it doesn't run during a backup, or clash with a
+# fastdowntime.
+15 10 * * * {{ code_dir }}/cronscripts/librarian-gc.py -q --log-file=INFO:{{ logs_dir }}/librarian-gc.log
+
+{% endif -%}
+{% if os_password and not upstream_host -%}
+# Feed locally-spooled uploads into Swift.
+{% for i in range(swift_feed_workers) -%}
+*/10 * * * * run-one {{ code_dir }}/cronscripts/librarian-feed-swift.py --remove -q --log-file=INFO:{{ logs_dir }}/librarian-feed-swift-{{ i }}.log --num-instances={{ swift_feed_workers }} --instance-id={{ i }}
+{% endfor %}
+{% endif -%}
+# Delete old logs
+15 0 * * * find {{ logs_dir }} -maxdepth 1 -type f -mtime +90 -name 'librarian.log.*' -delete
+
+# Catch up with publishing OOPSes that were temporarily spooled to disk due
+# to RabbitMQ being unavailable.
+*/15 * * * * {{ code_dir }}/bin/datedir2amqp --exchange oopses --host {{ rabbitmq_host }} --username {{ rabbitmq_username }} --password {{ rabbitmq_password }} --vhost {{ rabbitmq_vhost }} --repo {{ oopses_dir }} --key ""
+
diff --git a/charm/launchpad-librarian/templates/gunicorn.conf.py.j2 b/charm/launchpad-librarian/templates/gunicorn.conf.py.j2
new file mode 100644
index 0000000..efd8a1e
--- /dev/null
+++ b/charm/launchpad-librarian/templates/gunicorn.conf.py.j2
@@ -0,0 +1,9 @@
+bind = [":{{ port_main }}", ":{{ port_xmlrpc }}"]
+workers = {{ wsgi_workers }}
+# This is set relatively low to work around memory leaks on Python 3.
+max_requests = 2000
+log_level = "DEBUG"
+# Must be higher than the highest hard_timeout feature rule.
+timeout = 65
+graceful_timeout = 120
+
diff --git a/charm/launchpad-librarian/templates/launchpad-librarian-generator.j2 b/charm/launchpad-librarian/templates/launchpad-librarian-generator.j2
new file mode 100755
index 0000000..4929624
--- /dev/null
+++ b/charm/launchpad-librarian/templates/launchpad-librarian-generator.j2
@@ -0,0 +1,18 @@
+#! /bin/sh
+# Copyright 2022 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+# Part of the launchpad-librarian Juju charm.
+
+set -e
+
+wantdir="$1/launchpad-librarian.service.wants"
+template=/lib/systemd/system/launchpad-librarian@.service
+
+# Generate systemd unit dependency symlinks for all configured
+# launchpad-librarian instances.
+mkdir -p "$wantdir"
+for i in $(seq {{ workers }}); do
+ ln -s "$template" "$wantdir/launchpad-librarian@$i.service"
+done
+
diff --git a/charm/launchpad-librarian/templates/launchpad-librarian-lazr.conf b/charm/launchpad-librarian/templates/launchpad-librarian-lazr.conf
new file mode 100644
index 0000000..5cc7c8b
--- /dev/null
+++ b/charm/launchpad-librarian/templates/launchpad-librarian-lazr.conf
@@ -0,0 +1,41 @@
+# Public configuration data. The contents of this file may be freely shared
+# with developers if needed for debugging.
+
+# A schema's sections, keys, and values are automatically inherited, except
+# for '.optional' sections. Update this config to override key values.
+# Values are strings, except for numbers that look like ints. The tokens
+# true, false, and none are treated as True, False, and None.
+
+{% from "macros.j2" import opt -%}
+
+[meta]
+extends: ../launchpad-base-lazr.conf
+
+[launchpad_session]
+database: {{ db_session }}
+dbuser: {{ db_session_user }}
+
+[librarian]
+download_port: {{ worker_download_port }}
+restricted_download_port: {{ worker_restricted_download_port }}
+restricted_upload_port: {{ worker_restricted_upload_port }}
+upload_port: {{ worker_upload_port }}
+
+[librarian_server]
+launch: true
+logfile: {{ logfile }}
+{{- opt("old_os_auth_url", old_os_auth_url) }}
+{{- opt("old_os_auth_version", old_os_auth_version) }}
+{{- opt("old_os_tenant_name", old_os_tenant_name) }}
+{{- opt("old_os_username", old_os_username) }}
+{{- opt("os_auth_url", os_auth_url) }}
+{{- opt("os_auth_version", os_auth_version) }}
+{{- opt("os_tenant_name", os_tenant_name) }}
+{{- opt("os_username", os_username) }}
+root: {{ librarian_dir }}
+swift_timeout: {{ swift_timeout }}
+{%- if upstream_host %}
+upstream_host: {{ upstream_host }}
+upstream_port: {{ upstream_port }}
+{%- endif %}
+
diff --git a/charm/launchpad-librarian/templates/launchpad-librarian-secrets-lazr.conf b/charm/launchpad-librarian/templates/launchpad-librarian-secrets-lazr.conf
new file mode 100644
index 0000000..d030f1d
--- /dev/null
+++ b/charm/launchpad-librarian/templates/launchpad-librarian-secrets-lazr.conf
@@ -0,0 +1,16 @@
+# Secret configuration data. This is stored in an overlay directory, mainly
+# to avoid accidental information leaks from the public configuration file.
+# Entries in this file should not be shared with developers, although the
+# structure of the file is not secret, only configuration values.
+
+# A schema's sections, keys, and values are automatically inherited, except
+# for '.optional' sections. Update this config to override key values.
+# Values are strings, except for numbers that look like ints. The tokens
+# true, false, and none are treated as True, False, and None.
+
+{% from "macros.j2" import opt -%}
+
+[librarian_server]
+{{- opt("old_os_password", old_os_password) }}
+{{- opt("os_password", os_password) }}
+
diff --git a/charm/launchpad-librarian/templates/launchpad-librarian.service.j2 b/charm/launchpad-librarian/templates/launchpad-librarian.service.j2
new file mode 100644
index 0000000..fc7469d
--- /dev/null
+++ b/charm/launchpad-librarian/templates/launchpad-librarian.service.j2
@@ -0,0 +1,16 @@
+# This service is really a systemd target, but we use a service since
+# targets cannot be reloaded. See launchpad-librarian@.service for instance
+# configuration.
+
+[Unit]
+Description=Launchpad librarian
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/true
+ExecReload=/bin/true
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/charm/launchpad-librarian/templates/launchpad-librarian@.service.j2 b/charm/launchpad-librarian/templates/launchpad-librarian@.service.j2
new file mode 100644
index 0000000..846016e
--- /dev/null
+++ b/charm/launchpad-librarian/templates/launchpad-librarian@.service.j2
@@ -0,0 +1,25 @@
+[Unit]
+Description=Launchpad librarian (%i)
+PartOf=launchpad-librarian.service
+Before=launchpad-librarian.service
+ReloadPropagatedFrom=launchpad-librarian.service
+After=network.target
+ConditionPathExists=!{{ code_dir }}/maintenance.txt
+
+[Service]
+User=launchpad
+Group=launchpad
+WorkingDirectory={{ code_dir }}
+# https://portal.admin.canonical.com/C44221
+MemoryMax=4G
+Environment=LPCONFIG=launchpad-librarian%i
+SyslogIdentifier=librarian
+ExecStart={{ code_dir }}/bin/twistd --python daemons/librarian.tac --pidfile {{ var_dir }}/librarian%i.pid --prefix librarian --logfile {{ logfile }} --nodaemon
+ExecReload=/bin/kill -USR1 $MAINPID
+KillMode=mixed
+Restart=on-failure
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/charm/launchpad-librarian/templates/logrotate.conf.j2 b/charm/launchpad-librarian/templates/logrotate.conf.j2
new file mode 100644
index 0000000..66e2b12
--- /dev/null
+++ b/charm/launchpad-librarian/templates/logrotate.conf.j2
@@ -0,0 +1,28 @@
+{{ logs_dir }}/librarian[0-9]*.log
+{
+ rotate 21
+ daily
+ dateext
+ delaycompress
+ compress
+ notifempty
+ missingok
+ create 0644 syslog adm
+ sharedscripts
+ postrotate
+ systemctl reload launchpad-librarian.service
+ endscript
+}
+
+{{ logs_dir }}/expire-archive-files.log {{ logs_dir }}/librarian-gc.log {{ logs_dir }}/librarian-feed-swift-*.log
+{
+ rotate 21
+ daily
+ dateext
+ delaycompress
+ compress
+ notifempty
+ missingok
+ create 0644 syslog adm
+}
+
