launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29977
[Merge] ~cjwatson/launchpad:charm-debian-importer into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:charm-debian-importer into launchpad:master.
Commit message:
charm: Add a launchpad-debian-importer charm
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/442049
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:charm-debian-importer into launchpad:master.
diff --git a/charm/Makefile b/charm/Makefile
index 6c4f214..a36139e 100644
--- a/charm/Makefile
+++ b/charm/Makefile
@@ -16,6 +16,7 @@ ASSET = ../build/$(BUILD_LABEL)/$(TARBALL)
CHARMS := \
launchpad-admin \
launchpad-appserver \
+ launchpad-debian-importer \
launchpad-librarian
all: ## alias to build
diff --git a/charm/launchpad-debian-importer/README.md b/charm/launchpad-debian-importer/README.md
new file mode 100644
index 0000000..6ae99ca
--- /dev/null
+++ b/charm/launchpad-debian-importer/README.md
@@ -0,0 +1,13 @@
+# Launchpad Debian importer
+
+This charm runs a job that imports Debian packages into Launchpad.
+
+You will need the following relations:
+
+ juju relate launchpad-debian-importer:db postgresql:db
+ juju relate launchpad-debian-importer rabbitmq-server
+
+You will also need to set the `librarian_upload_host` and
+`librarian_upload_port` options to point to a local
+[launchpad-librarian](https://charmhub.io/launchpad-librarian) instance;
+this is not currently handled using relations.
diff --git a/charm/launchpad-debian-importer/charmcraft.yaml b/charm/launchpad-debian-importer/charmcraft.yaml
new file mode 100644
index 0000000..1634d67
--- /dev/null
+++ b/charm/launchpad-debian-importer/charmcraft.yaml
@@ -0,0 +1,60 @@
+type: charm
+bases:
+ - build-on:
+ - name: ubuntu
+ channel: "20.04"
+ architectures: [amd64]
+ run-on:
+ - name: ubuntu
+ channel: "20.04"
+ architectures: [amd64]
+parts:
+ charm-wheels:
+ source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels
+ source-commit: "59b32ae07f98051385c96d6d8e7e02ca4f197fe5"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ "*": charm-wheels/
+ prime:
+ - "-charm-wheels"
+ ols-layers:
+ source: https://git.launchpad.net/ols-charm-deps
+ source-commit: "56d219f60a293a6c73759b8439ef5fdb11e19d1f"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ "*": layers/
+ stage:
+ - layers
+ prime:
+ - "-layers"
+ launchpad-layers:
+ after:
+ - ols-layers
+ source: https://git.launchpad.net/launchpad-layers
+ source-commit: "6a50917f5f6163069ae1661e3320abb5b48173a3"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ launchpad-base: layers/layer/launchpad-base
+ stage:
+ - layers
+ prime:
+ - "-layers"
+ launchpad-debian-importer:
+ after:
+ - charm-wheels
+ - launchpad-layers
+ source: .
+ plugin: reactive
+ build-snaps: [charm/2.x/stable]
+ build-packages: [libpq-dev]
+ build-environment:
+ - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer
+ - CHARM_INTERFACES_DIR: $CRAFT_STAGE/layers/interface
+ - PIP_NO_INDEX: "true"
+ - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels
diff --git a/charm/launchpad-debian-importer/config.yaml b/charm/launchpad-debian-importer/config.yaml
new file mode 100644
index 0000000..6b0c325
--- /dev/null
+++ b/charm/launchpad-debian-importer/config.yaml
@@ -0,0 +1,14 @@
+options:
+ active:
+ type: boolean
+ description: If true, enable jobs that may change the database.
+ default: true
+ debian_mirror_host:
+ type: string
+ description: Hostname of the Debian mirror to use.
+ default: "ftp.debian.org"
+ debian_suites:
+ type: string
+ description: YAML-encoded dict mapping Debian suites to components.
+ default: |
+ sid: ["main", "contrib", "non-free"]
diff --git a/charm/launchpad-debian-importer/layer.yaml b/charm/launchpad-debian-importer/layer.yaml
new file mode 100644
index 0000000..88e2ce1
--- /dev/null
+++ b/charm/launchpad-debian-importer/layer.yaml
@@ -0,0 +1,13 @@
+includes:
+ - layer:launchpad-base
+repo: https://git.launchpad.net/launchpad
+options:
+ apt:
+ packages:
+ - debian-archive-keyring
+ - debmirror
+ ols-pg:
+ databases:
+ db:
+ name: launchpad_dev
+ roles: gina
diff --git a/charm/launchpad-debian-importer/metadata.yaml b/charm/launchpad-debian-importer/metadata.yaml
new file mode 100644
index 0000000..ff28cae
--- /dev/null
+++ b/charm/launchpad-debian-importer/metadata.yaml
@@ -0,0 +1,15 @@
+name: launchpad-debian-importer
+display-name: launchpad-debian-importer
+summary: Launchpad Debian importer
+maintainer: Colin Watson <cjwatson@xxxxxxxxxxxxx>
+description: |
+ Launchpad is an open source suite of tools that help people and teams
+ to work together on software projects.
+
+ This charm runs a job that imports Debian packages into Launchpad.
+tags:
+ # https://juju.is/docs/charm-metadata#heading--charm-store-fields
+ - network
+series:
+ - focal
+subordinate: false
diff --git a/charm/launchpad-debian-importer/reactive/launchpad-debian-importer.py b/charm/launchpad-debian-importer/reactive/launchpad-debian-importer.py
new file mode 100644
index 0000000..f87e95b
--- /dev/null
+++ b/charm/launchpad-debian-importer/reactive/launchpad-debian-importer.py
@@ -0,0 +1,62 @@
+# Copyright 2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import os.path
+
+import yaml
+from charmhelpers.core import hookenv, host, templating
+from charms.launchpad.base import (
+ configure_cron,
+ configure_lazr,
+ get_service_config,
+)
+from charms.reactive import set_state, when, when_not
+from ols import base
+
+
+def configure_logrotate(config):
+ hookenv.log("Writing logrotate configuration.")
+ templating.render(
+ "logrotate.conf.j2",
+ "/etc/logrotate.d/launchpad-debian-importer",
+ config,
+ perms=0o644,
+ )
+
+
+@when("launchpad.base.configured")
+@when_not("service.configured")
+def configure():
+ config = get_service_config()
+ config["debian_suites"] = yaml.safe_load(config["debian_suites"])
+ config["debian_components"] = []
+ for suite, components in config["debian_suites"].items():
+ for component in components:
+ if component not in config["debian_components"]:
+ config["debian_components"].append(component)
+ config["mirror_dir"] = os.path.join(base.base_dir(), "mirror")
+ host.mkdir(
+ config["mirror_dir"], owner=base.user(), group=base.user(), perms=0o755
+ )
+ config["scripts_dir"] = os.path.join(base.base_dir(), "scripts")
+ host.mkdir(config["scripts_dir"], perms=0o755)
+ templating.render(
+ "mirror-update.sh.j2",
+ os.path.join(config["scripts_dir"], "mirror-update.sh"),
+ config,
+ perms=0o755,
+ )
+ configure_lazr(
+ config,
+ "launchpad-debian-importer-lazr.conf",
+ "launchpad-debian-importer/launchpad-lazr.conf",
+ )
+ configure_logrotate(config)
+ configure_cron(config, "crontab.j2")
+
+ set_state("service.configured")
+
+
+@when("service.configured")
+def check_is_running():
+ hookenv.status_set("active", "Ready")
diff --git a/charm/launchpad-debian-importer/templates/crontab.j2 b/charm/launchpad-debian-importer/templates/crontab.j2
new file mode 100644
index 0000000..2f9d7e2
--- /dev/null
+++ b/charm/launchpad-debian-importer/templates/crontab.j2
@@ -0,0 +1,15 @@
+TZ=UTC
+MAILFROM={{ bounce_address }}
+MAILTO={{ cron_mailto }}
+LPCONFIG=launchpad-debian-importer
+
+{% if active -%}
+# Mirror from Debian and then run the importer. Debian publishes at "52
+# 1,7,13,19", but we also need to allow time for the above mirror to update.
+25 4,10,16,22 * * * {{ scripts_dir }}/mirror-update.sh >> {{ logs_dir }}/mirror-update.log 2>&1 && {{ code_dir }}/scripts/gina.py {{ debian_suites.keys()|reverse|join(" ") }} -q --log-file=INFO:{{ logs_dir }}/gina.log
+
+{% endif -%}
+# Catch up with publishing OOPSes that were temporarily spooled to disk due
+# to RabbitMQ being unavailable.
+*/15 * * * * {{ code_dir }}/bin/datedir2amqp --exchange oopses --host {{ rabbitmq_host }} --username {{ rabbitmq_username }} --password {{ rabbitmq_password }} --vhost {{ rabbitmq_vhost }} --repo {{ oopses_dir }} --key ""
+
diff --git a/charm/launchpad-debian-importer/templates/launchpad-debian-importer-lazr.conf b/charm/launchpad-debian-importer/templates/launchpad-debian-importer-lazr.conf
new file mode 100644
index 0000000..2152ea5
--- /dev/null
+++ b/charm/launchpad-debian-importer/templates/launchpad-debian-importer-lazr.conf
@@ -0,0 +1,20 @@
+# Public configuration data. The contents of this file may be freely shared
+# with developers if needed for debugging.
+
+# A schema's sections, keys, and values are automatically inherited, except
+# for '.optional' sections. Update this config to override key values.
+# Values are strings, except for numbers that look like ints. The tokens
+# true, false, and none are treated as True, False, and None.
+
+{% from "macros.j2" import opt -%}
+
+[meta]
+extends: ../launchpad-base-lazr.conf
+
+{% for suite, components in debian_suites.items() -%}
+[gina_target.{{ suite }}]
+components: {{ components|join(",") }}
+source_only: True
+root: {{ mirror_dir }}/
+
+{% endfor -%}
diff --git a/charm/launchpad-debian-importer/templates/logrotate.conf.j2 b/charm/launchpad-debian-importer/templates/logrotate.conf.j2
new file mode 100644
index 0000000..51d1421
--- /dev/null
+++ b/charm/launchpad-debian-importer/templates/logrotate.conf.j2
@@ -0,0 +1,11 @@
+{{ logs_dir }}/*.log
+{
+ rotate 21
+ daily
+ dateext
+ delaycompress
+ compress
+ notifempty
+ missingok
+}
+
diff --git a/charm/launchpad-debian-importer/templates/mirror-update.sh.j2 b/charm/launchpad-debian-importer/templates/mirror-update.sh.j2
new file mode 100644
index 0000000..446c3af
--- /dev/null
+++ b/charm/launchpad-debian-importer/templates/mirror-update.sh.j2
@@ -0,0 +1,24 @@
+#! /bin/sh
+# Copyright 2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+# Part of the launchpad-debian-importer Juju charm.
+
+set -e
+
+DEST="{{ mirror_dir }}/"
+HOST="{{ debian_mirror_host }}"
+DISTS="{{ debian_suites.keys()|join(',') }}"
+SECTIONS="{{ debian_components|join(',') }}"
+
+{% if http_proxy -%}
+http_proxy="{{ http_proxy }}"
+ftp_proxy="{{ http_proxy }}"
+export http_proxy ftp_proxy
+
+{% endif -%}
+echo "Starting Mirror: $(date)"
+rm -rf "${DEST}/.temp"
+debmirror --keyring /usr/share/keyrings/debian-archive-keyring.gpg --root=debian --host="$HOST" --method=http --dist="$DISTS" --section="$SECTIONS" --arch=none --diff=none --progress --rsync-extra=none --allow-dist-rename "$DEST"
+echo "Finished Mirror: $(date)"
+