launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29783
[Merge] ~cjwatson/lp-codeimport:charm-codeimport-storage into lp-codeimport:master
Colin Watson has proposed merging ~cjwatson/lp-codeimport:charm-codeimport-storage into lp-codeimport:master with ~cjwatson/lp-codeimport:charmcraft as a prerequisite.
Commit message:
charm: Add an lp-codeimport-storage charm
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/lp-codeimport/+git/lp-codeimport/+merge/439271
This can be used to provide import data storage instead of the older approach of setting `bazaar_branch_store` and `foreign_tree_store` on `lp-codeimport`. It isn't much more than a vanilla system with SSH key authorization and a couple of pre-created directories, but the new `codeimport-storage` relation makes it a little easier to set things up in a Mojo spec.
Data migration from any previous storage systems needs to be handled manually.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-codeimport:charm-codeimport-storage into lp-codeimport:master.
diff --git a/charm/lp-codeimport-storage/README.md b/charm/lp-codeimport-storage/README.md
new file mode 100644
index 0000000..cad1a3b
--- /dev/null
+++ b/charm/lp-codeimport-storage/README.md
@@ -0,0 +1,3 @@
+# Launchpad code import storage
+
+This charm provides storage for the `lp-codeimport` charm.
diff --git a/charm/lp-codeimport-storage/charmcraft.yaml b/charm/lp-codeimport-storage/charmcraft.yaml
new file mode 100644
index 0000000..ac76470
--- /dev/null
+++ b/charm/lp-codeimport-storage/charmcraft.yaml
@@ -0,0 +1,44 @@
+type: charm
+bases:
+ - build-on:
+ - name: ubuntu
+ channel: "22.04"
+ architectures: [amd64]
+ run-on:
+ - name: ubuntu
+ channel: "22.04"
+ architectures: [amd64]
+parts:
+ charm-wheels:
+ source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels
+ source-commit: "59b32ae07f98051385c96d6d8e7e02ca4f197fe5"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ "*": charm-wheels/
+ prime:
+ - "-charm-wheels"
+ ols-layers:
+ source: https://git.launchpad.net/ols-charm-deps
+ source-commit: "1ca8acbef7eb49b8a2cc81e5e13479b4f226a48b"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ "*": layers/
+ stage:
+ - layers
+ prime:
+ - "-layers"
+ lp-codeimport-storage:
+ after:
+ - charm-wheels
+ - ols-layers
+ source: .
+ plugin: reactive
+ build-snaps: [charm/2.x/stable]
+ build-environment:
+ - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer
+ - PIP_NO_INDEX: "true"
+ - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels
diff --git a/charm/lp-codeimport-storage/config.yaml b/charm/lp-codeimport-storage/config.yaml
new file mode 100644
index 0000000..4f9deac
--- /dev/null
+++ b/charm/lp-codeimport-storage/config.yaml
@@ -0,0 +1,5 @@
+options:
+ public_ssh_key:
+ type: string
+ default: ""
+ description: Base64-encoded public SSH key of the code import workers.
diff --git a/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/__init__.py b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/__init__.py
diff --git a/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/provides.py b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/provides.py
new file mode 100644
index 0000000..c6b06a0
--- /dev/null
+++ b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/provides.py
@@ -0,0 +1,39 @@
+# Copyright 2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from charmhelpers.core import hookenv
+from charms.reactive import (
+ Endpoint,
+ remove_state,
+ set_state,
+ when,
+ when_not,
+ )
+
+
+class CodeImportStorageProvides(Endpoint):
+ @when("endpoint.{endpoint_name}.joined")
+ @when_not("endpoint.{endpoint_name}.available")
+ def joined(self):
+ for relation in self.relations:
+ ingress_address = hookenv.network_get(
+ relation.endpoint_name, relation_id=relation.relation_id
+ )["ingress-addresses"][0]
+ relation.to_publish[
+ "bazaar_branch_store"
+ ] = f"sftp://importd@{ingress_address}/srv/importd/www/";
+ relation.to_publish[
+ "foreign_tree_store"
+ ] = f"sftp://importd@{ingress_address}/srv/importd/sources/";
+ set_state(self.expand_name("endpoint.{endpoint_name}.available"))
+
+ @when("endpoint.{endpoint_name}.available")
+ @when_not("endpoint.{endpoint_name}.joined")
+ def departed(self):
+ remove_state(self.expand_name("endpoint.{endpoint_name}.available"))
+
+ def get_codeimport_subnets(self):
+ subnets = set()
+ for unit in self.all_joined_units:
+ subnets.update(unit.received_raw["egress-subnets"].split(","))
+ return sorted(subnets)
diff --git a/charm/lp-codeimport-storage/layer.yaml b/charm/lp-codeimport-storage/layer.yaml
new file mode 100644
index 0000000..590c59f
--- /dev/null
+++ b/charm/lp-codeimport-storage/layer.yaml
@@ -0,0 +1,2 @@
+includes:
+ - layer:basic
diff --git a/charm/lp-codeimport-storage/metadata.yaml b/charm/lp-codeimport-storage/metadata.yaml
new file mode 100644
index 0000000..95de7f8
--- /dev/null
+++ b/charm/lp-codeimport-storage/metadata.yaml
@@ -0,0 +1,14 @@
+name: lp-codeimport-storage
+display-name: lp-codeimport-storage
+summary: Launchpad code import storage
+maintainer: Colin Watson <cjwatson@xxxxxxxxxxxxx>
+description: Storage for use with lp-codeimport workers.
+tags:
+ # https://juju.is/docs/charm-metadata#heading--charm-store-fields
+ - network
+series:
+ - jammy
+subordinate: false
+provides:
+ codeimport-storage:
+ interface: codeimport-storage
diff --git a/charm/lp-codeimport-storage/reactive/lp-codeimport-storage.py b/charm/lp-codeimport-storage/reactive/lp-codeimport-storage.py
new file mode 100644
index 0000000..5be4349
--- /dev/null
+++ b/charm/lp-codeimport-storage/reactive/lp-codeimport-storage.py
@@ -0,0 +1,74 @@
+# Copyright 2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import base64
+import os.path
+
+from charmhelpers.core import (
+ hookenv,
+ host,
+ templating,
+ )
+from charms.reactive import (
+ endpoint_from_flag,
+ hook,
+ remove_state,
+ set_state,
+ when,
+ when_not,
+ )
+
+
+@hook("upgrade-charm")
+def upgrade_charm():
+ remove_state("service.configured")
+
+
+@hook("config.changed")
+def config_changed():
+ remove_state("service.configured")
+
+
+@hook(
+ "{provides:codeimport-storage}-relation-{joined,changed,broken,departed}"
+)
+def codeimport_storage_changed(*args):
+ remove_state("service.configured")
+
+
+@when("config.set.public_ssh_key", "endpoint.codeimport-storage.available")
+@when_not("service.configured")
+def configure():
+ codeimport_storage = endpoint_from_flag(
+ "endpoint.codeimport-storage.available"
+ )
+ hookenv.log("Creating service user")
+ host.add_group("importd")
+ host.adduser("importd", primary_group="importd")
+ for directory in ("/srv/importd/sources", "/srv/importd/www"):
+ if not os.path.exists(directory):
+ host.mkdir(
+ directory, owner="importd", group="importd", perms=0o755
+ )
+ ssh_dir = "/home/importd/.ssh"
+ if not os.path.exists(ssh_dir):
+ host.mkdir(ssh_dir, owner="importd", group="importd", perms=0o700)
+ config = dict(hookenv.config())
+ config["codeimport_subnets"] = codeimport_storage.get_codeimport_subnets()
+ config["public_ssh_key"] = base64.b64decode(
+ config["public_ssh_key"].encode("ASCII")
+ ).decode("ASCII")
+ templating.render(
+ "authorized_keys.j2",
+ os.path.join(ssh_dir, "authorized_keys"),
+ config,
+ owner="importd",
+ group="importd",
+ perms=0o600,
+ )
+ set_state("service.configured")
+
+
+@when("service.configured")
+def check_is_running():
+ hookenv.status_set("active", "Ready")
diff --git a/charm/lp-codeimport-storage/templates/authorized_keys.j2 b/charm/lp-codeimport-storage/templates/authorized_keys.j2
new file mode 100644
index 0000000..77d398e
--- /dev/null
+++ b/charm/lp-codeimport-storage/templates/authorized_keys.j2
@@ -0,0 +1,2 @@
+restrict,from="{{ codeimport_subnets|join(",") }}" {{ public_ssh_key }}
+
diff --git a/charm/lp-codeimport/config.yaml b/charm/lp-codeimport/config.yaml
index cac88ce..37f990c 100644
--- a/charm/lp-codeimport/config.yaml
+++ b/charm/lp-codeimport/config.yaml
@@ -11,13 +11,16 @@ options:
default is to use the global CA infrastructure.
bazaar_branch_store:
type: string
- default: sftp://hoover@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/srv/importd/www/
- description: Where the Bazaar imports are stored.
+ default: ""
+ description: >
+ Where the Bazaar imports are stored. If unset, rely on the
+ codeimport-storage relation instead.
foreign_tree_store:
type: string
- default: sftp://hoover@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/srv/importd/sources/
+ default: ""
description: >
- Where the tarballs of foreign branches are uploaded for storage.
+ Where the tarballs of foreign branches are uploaded for storage. If
+ unset, rely on the codeimport-storage relation instead.
private_ssh_key:
type: string
default: ""
diff --git a/charm/lp-codeimport/hooks/relations/codeimport-storage/__init__.py b/charm/lp-codeimport/hooks/relations/codeimport-storage/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/charm/lp-codeimport/hooks/relations/codeimport-storage/__init__.py
diff --git a/charm/lp-codeimport/hooks/relations/codeimport-storage/requires.py b/charm/lp-codeimport/hooks/relations/codeimport-storage/requires.py
new file mode 100644
index 0000000..16e31d4
--- /dev/null
+++ b/charm/lp-codeimport/hooks/relations/codeimport-storage/requires.py
@@ -0,0 +1,32 @@
+# Copyright 2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from charms.reactive import (
+ clear_flag,
+ Endpoint,
+ toggle_flag,
+ when,
+ when_not,
+ )
+
+
+class CodeImportStorageRequires(Endpoint):
+ @when("endpoint.{endpoint_name}.changed")
+ def joined(self):
+ toggle_flag(
+ self.expand_name("endpoint.{endpoint_name}.available"),
+ bool(self.bazaar_branch_store) and bool(self.foreign_tree_store),
+ )
+
+ @when("endpoint.{endpoint_name}.available")
+ @when_not("endpoint.{endpoint_name}.joined")
+ def departed(self):
+ clear_flag(self.expand_name("endpoint.{endpoint_name}.available"))
+
+ @property
+ def bazaar_branch_store(self):
+ return self.all_joined_units.received["bazaar_branch_store"]
+
+ @property
+ def foreign_tree_store(self):
+ return self.all_joined_units.received["foreign_tree_store"]
diff --git a/charm/lp-codeimport/metadata.yaml b/charm/lp-codeimport/metadata.yaml
index 6edc5b9..9df966d 100644
--- a/charm/lp-codeimport/metadata.yaml
+++ b/charm/lp-codeimport/metadata.yaml
@@ -9,6 +9,9 @@ tags:
series:
- bionic
subordinate: false
+requires:
+ codeimport-storage:
+ interface: codeimport-storage
resources:
lp-codeimport:
type: file
diff --git a/charm/lp-codeimport/reactive/lp-codeimport.py b/charm/lp-codeimport/reactive/lp-codeimport.py
index 04df2a6..9c7c9e7 100644
--- a/charm/lp-codeimport/reactive/lp-codeimport.py
+++ b/charm/lp-codeimport/reactive/lp-codeimport.py
@@ -19,9 +19,12 @@ from charmhelpers.core import (
templating,
)
from charms.reactive import (
+ endpoint_from_flag,
remove_state,
set_state,
when,
+ when_any,
+ when_none,
when_not,
)
from ols import base
@@ -166,9 +169,51 @@ def configure_rsync(config):
raise RuntimeError('Failed to restart rsync')
-@when('ols.configured')
+@when_any(
+ "endpoint.codeimport-storage.available",
+ "config.set.bazaar_branch_store",
+)
+def bazaar_branch_store_available():
+ set_state("service.bazaar_branch_store.available")
+
+
+@when_none(
+ "endpoint.codeimport-storage.available",
+ "config.set.bazaar_branch_store",
+)
+def bazaar_branch_store_unavailable():
+ remove_state("service.bazaar_branch_store.available")
+ remove_state("service.configured")
+
+
+@when_any(
+ "endpoint.codeimport-storage.available",
+ "config.set.foreign_tree_store",
+)
+def foreign_tree_store_available():
+ set_state("service.foreign_tree_store.available")
+
+
+@when_none(
+ "endpoint.codeimport-storage.available",
+ "config.set.foreign_tree_store",
+)
+def foreign_tree_store_unavailable():
+ remove_state("service.foreign_tree_store.available")
+ remove_state("service.configured")
+
+
+@when(
+ "ols.configured",
+ "service.bazaar_branch_store.available",
+ "service.foreign_tree_store.available",
+)
@when_not('service.configured')
def configure():
+ codeimport_storage = endpoint_from_flag(
+ "endpoint.codeimport-storage.available"
+ )
+
ensure_lp_directories()
system_packages = os.path.join(base.code_dir(), 'system-packages.txt')
@@ -188,6 +233,14 @@ def configure():
config_path = os.path.join(
base.code_dir(), 'production-configs', 'charm', 'codeimport-lazr.conf')
svc_config = dict(config)
+ if not svc_config["bazaar_branch_store"]:
+ svc_config["bazaar_branch_store"] = (
+ codeimport_storage.bazaar_branch_store
+ )
+ if not svc_config["foreign_tree_store"]:
+ svc_config["foreign_tree_store"] = (
+ codeimport_storage.foreign_tree_store
+ )
svc_config.update({
'base_dir': base.base_dir(),
'code_dir': base.code_dir(),
@@ -199,7 +252,7 @@ def configure():
'home_dir': home_dir(),
'user': base.user(),
'code_import_storage_host': (
- urlparse(config['bazaar_branch_store']).hostname),
+ urlparse(svc_config['bazaar_branch_store']).hostname),
# Chosen to allow distributing dispatch start time over a 30-second
# interval.
'dispatch_offset': host.modulo_distribution(modulo=6, wait=5),
