launchpad-reviewers team mailing list archive

[Colin Watson <mp+455583@xxxxxxxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/launchpad:charm-codehosting-rsync-mirrors into launchpad:master.

Commit message:
charm: Add bzr_repositories_rsync_hosts_allow option to codehosting

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/455583

Staging codehosting rsyncs a small set of Bazaar branches from production codehosting.  To make this work, the charm that we'll soon be deploying to replace production codehosting needs to support that.

I was able to tidy up some minor duplication in the process.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:charm-codehosting-rsync-mirrors into launchpad:master.

diff --git a/charm/launchpad-codehosting/config.yaml b/charm/launchpad-codehosting/config.yaml
index 152b0be..c41fd80 100644
--- a/charm/launchpad-codehosting/config.yaml
+++ b/charm/launchpad-codehosting/config.yaml
@@ -3,6 +3,12 @@ options:
     type: boolean
     description: If true, enable jobs that may change the database.
     default: true
+  bzr_repositories_rsync_hosts_allow:
+    type: string
+    description: >
+      Space-separated list of hosts that should be allowed to rsync Bazaar
+      repositories (both public and private).
+    default: ""
   codehosting_private_ssh_key:
     type: string
     description: >
diff --git a/charm/launchpad-codehosting/reactive/launchpad-codehosting.py b/charm/launchpad-codehosting/reactive/launchpad-codehosting.py
index 537cccb..b8eb323 100644
--- a/charm/launchpad-codehosting/reactive/launchpad-codehosting.py
+++ b/charm/launchpad-codehosting/reactive/launchpad-codehosting.py
@@ -31,6 +31,16 @@ def base64_decode(value):
     return base64.b64decode(value.encode("ASCII"))
 
 
+def get_data_dir():
+    return os.path.join(base.base_dir(), "data")
+
+
+def get_codehosting_service_config():
+    config = get_service_config()
+    config["bzr_repositories_root"] = f"{get_data_dir()}/mirrors"
+    return config
+
+
 def configure_logrotate(config):
     hookenv.log("Writing logrotate configuration.")
     templating.render(
@@ -65,7 +75,7 @@ def configure_systemd(config):
 def config_files():
     files = []
     files.extend(lazr_config_files())
-    config = get_service_config()
+    config = get_codehosting_service_config()
     files.append(config_file_path("launchpad-codehosting/launchpad-lazr.conf"))
     for i in range(config["workers"]):
         files.append(
@@ -155,11 +165,6 @@ def configure_ssh_keys(config):
 
 def configure_codehosting_lazr_config(config):
     hookenv.log("Writing lazr configuration.")
-    # XXX lgp171188: 2023-10-26: This template recreates the value of
-    # config["bzr_repositories_root"] since it can't use it directly
-    # due to it being in a separate handler that is executed much later.
-    # Fix this to unify the definition and usage of this configuration
-    # value.
     configure_lazr(
         config,
         "launchpad-codehosting-lazr-common.conf.j2",
@@ -177,10 +182,21 @@ def configure_codehosting_lazr_config(config):
         )
 
 
+def configure_rsync(config):
+    rsync_path = "/etc/rsync-juju.d/020-mirrors.conf"
+    if config["bzr_repositories_rsync_hosts_allow"]:
+        hookenv.log("Writing rsync configuration for Bazaar repositories.")
+        templating.render(
+            "mirrors-rsync.conf.j2", rsync_path, config, perms=0o644
+        )
+    elif os.path.exists(rsync_path):
+        os.unlink(rsync_path)
+
+
 @when("launchpad.db.configured")
 @when_not("service.configured")
 def configure():
-    config = get_service_config()
+    config = get_codehosting_service_config()
     configure_codehosting_lazr_config(config)
     configure_email(config, "launchpad-codehosting")
     configure_logrotate(config)
@@ -188,6 +204,7 @@ def configure():
     configure_scripts(config)
     configure_ssh_keys(config)
     configure_systemd(config)
+    configure_rsync(config)
     if config["active"]:
         if helpers.any_file_changed(
             [
@@ -238,10 +255,9 @@ def configure_document_root(config):
         perms=0o755,
         force=True,
     )
-    data_dir = f"{config['base_dir']}/data"
+    data_dir = get_data_dir()
     hookenv.log(f"Creating the data directory {data_dir}")
     host.mkdir(data_dir, owner=user, group=user, perms=0o755, force=True)
-    config["bzr_repositories_root"] = f"{data_dir}/mirrors"
     host.mkdir(
         config["bzr_repositories_root"],
         owner=user,
@@ -276,7 +292,7 @@ def configure_document_root(config):
 @when_not("service.apache-website.configured")
 def configure_apache_website():
     apache_website = endpoint_from_flag("apache-website.available")
-    config = get_service_config()
+    config = get_codehosting_service_config()
     configure_document_root(config)
     apache_website.set_remote(
         domain=config["domain_bzr"],
diff --git a/charm/launchpad-codehosting/templates/launchpad-codehosting-lazr-common.conf.j2 b/charm/launchpad-codehosting/templates/launchpad-codehosting-lazr-common.conf.j2
index de8d3aa..3a1a521 100644
--- a/charm/launchpad-codehosting/templates/launchpad-codehosting-lazr-common.conf.j2
+++ b/charm/launchpad-codehosting/templates/launchpad-codehosting-lazr-common.conf.j2
@@ -19,7 +19,7 @@ blacklisted_hostnames: localhost,127.0.0.1
 host_key_pair_path: {{ base_dir }}/keys/
 internal_branch_by_id_root: {{ internal_branch_by_id_root }}
 internal_codebrowse_root: {{ internal_codebrowse_root }}
-mirrored_branches_root: {{ base_dir }}/data/mirrors
+mirrored_branches_root: {{ bzr_repositories_root }}
 rewrite_script_log_file: {{ logs_dir }}/rewrite.log
 
 [error_reports]
diff --git a/charm/launchpad-codehosting/templates/mirrors-rsync.conf.j2 b/charm/launchpad-codehosting/templates/mirrors-rsync.conf.j2
new file mode 100644
index 0000000..ab23ae0
--- /dev/null
+++ b/charm/launchpad-codehosting/templates/mirrors-rsync.conf.j2
@@ -0,0 +1,8 @@
+
+[mirrors]
+  path = {{ bzr_repositories_root }}
+  comment = Mirrored Branches
+  list = false
+  read only = true
+  hosts allow = {{ bzr_repositories_rsync_hosts_allow }}
+