launchpad-reviewers team mailing list archive

[Tushar Gupta <mp+482174@xxxxxxxxxxxxxxxxxx>]

Tushar Gupta has proposed merging ~tushar5526/launchpad:refactoring-publisher-run-cmd-into-bash-script into launchpad:master.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~tushar5526/launchpad/+git/launchpad/+merge/482174
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~tushar5526/launchpad:refactoring-publisher-run-cmd-into-bash-script into launchpad:master.

diff --git a/charm/launchpad-ppa-publisher/reactive/launchpad-ppa-publisher.py b/charm/launchpad-ppa-publisher/reactive/launchpad-ppa-publisher.py
index f6a8e3e..da88c13 100644
--- a/charm/launchpad-ppa-publisher/reactive/launchpad-ppa-publisher.py
+++ b/charm/launchpad-ppa-publisher/reactive/launchpad-ppa-publisher.py
@@ -6,7 +6,7 @@ import os
 import yaml
 from charmhelpers.core import hookenv, host, templating
 from charms.launchpad.base import configure_email, get_service_config
-from charms.launchpad.payload import configure_cron, configure_lazr
+from charms.launchpad.payload import configure_cron, configure_lazr, home_dir
 from charms.reactive import (
     clear_flag,
     endpoint_from_flag,
@@ -76,8 +76,9 @@ def validate_parallel_publisher_config(parallel_publisher_config):
             "are configured but there are no PPAs excluded."
         )
         return False
-
-    run_ids = set()
+    # exclude "main-publisher-run", as its used by the gloabl publisher
+    # run bash script
+    run_ids = {"main-publisher-run"}
     for run in runs:
         distro = run.get("distro")
         run_id = run.get("id")
@@ -123,7 +124,7 @@ def validate_parallel_publisher_config(parallel_publisher_config):
     return True
 
 
-def generate_parallel_publisher_cron_configs(parallel_publisher_config):
+def generate_parallel_publisher_configs(parallel_publisher_config):
     if not parallel_publisher_config:
         return []
     runs = parallel_publisher_config.get("runs")
@@ -155,6 +156,7 @@ def generate_parallel_publisher_cron_configs(parallel_publisher_config):
             continue
 
         config = {}
+        config["run_id"] = run_id
         config["distro_option"] = (
             "--all-derived" if distro == "all_derived" else f"-d {distro}"
         )
@@ -169,6 +171,44 @@ def generate_parallel_publisher_cron_configs(parallel_publisher_config):
     return cron_configs
 
 
+def configure_main_publisher_run_script(config):
+    hookenv.log("Creating the main publisher script.")
+    host.mkdir(
+        config["publisher_scripts_dir"],
+        owner=base.user(),
+        group=base.user(),
+        perms=0o755,
+        force=True,
+    )
+    templating.render(
+        "main-publisher-run.j2",
+        f"{config['publisher_scripts_dir']}/main-publisher-run",
+        config,
+        owner=base.user(),
+        group=base.user(),
+        perms=0o755,
+    )
+
+
+def configure_parallel_publisher_run_script(config):
+    for parallel_run_config in config["parallel_publisher_configs"]:
+        run_id = parallel_run_config["run_id"]
+        hookenv.log(f'Creating the parallel publisher scripts for "{run_id}".')
+        template_config = {
+            "code_dir": config["code_dir"],
+            "logs_dir": config["logs_dir"],
+            **parallel_run_config,
+        }
+        templating.render(
+            "parallel-publisher-run.j2",
+            f"{config['publisher_scripts_dir']}/{run_id}",
+            template_config,
+            owner=base.user(),
+            group=base.user(),
+            perms=0o755,
+        )
+
+
 @when(
     "launchpad.db.configured",
     "memcache.available",
@@ -195,8 +235,8 @@ def configure():
         parallel_publisher_config
     )
 
-    config["parallel_publisher_cron_configs"] = (
-        generate_parallel_publisher_cron_configs(parallel_publisher_config)
+    config["parallel_publisher_configs"] = generate_parallel_publisher_configs(
+        parallel_publisher_config
     )
 
     host.mkdir(data_dir, owner=base.user(), group=base.user(), perms=0o775)
@@ -246,6 +286,9 @@ def configure():
     )
     configure_email(config, "launchpad-ppa-publisher")
     configure_logrotate(config)
+    config["publisher_scripts_dir"] = f"{home_dir()}/publisher-scripts"
+    configure_parallel_publisher_run_script(config)
+    configure_main_publisher_run_script(config)
     configure_cron(config, "crontab.j2")
     set_flag("service.configured")
 
diff --git a/charm/launchpad-ppa-publisher/templates/crontab.j2 b/charm/launchpad-ppa-publisher/templates/crontab.j2
index 8b10e67..11c212d 100644
--- a/charm/launchpad-ppa-publisher/templates/crontab.j2
+++ b/charm/launchpad-ppa-publisher/templates/crontab.j2
@@ -9,10 +9,10 @@ PPAROOT={{ ppa_archive_root }}
 P3AROOT={{ ppa_archive_private_root }}
 
 {% if active -%}
-* * * * * umask 022; nice -n 5 ionice -c 2 -n 7 {{ code_dir }}/cronscripts/publishing/cron.publish-ppa "-d ubuntu{{ excluded_ppas_options }}" "--all-derived{{ excluded_ppas_options }}" "-d debian{{ excluded_ppas_options }}" >> {{ logs_dir }}/cron.ppa.log 2>&1
+* * * * * umask 022; nice -n 5 ionice -c 2 -n 7 {{ publisher_scripts_dir }}/main-publisher-run
 
-{% for config in parallel_publisher_cron_configs %}
-* * * * * umask 022; nice -n 5 ionice -c 2 -n 7 {{ code_dir }}/cronscripts/publishing/cron.publish-parallel-ppa "{{ config.distro_option }} {{ config.archive_options }} {{ config.lockfilename_option }}" >> {{ logs_dir }}/{{ config.logfilename }} 2>&1
+{% for config in parallel_publisher_configs %}
+* * * * * umask 022; nice -n 5 ionice -c 2 -n 7 {{ publisher_scripts_dir  }}/{{ config.run_id }}
 {% endfor %}
 
 17,47 * * * * nice -n 15 {{ code_dir }}/cronscripts/parse-ppa-apache-access-logs.py -q --log-file=INFO:{{ logs_dir }}/parse-ppa-apache-access-logs.log
diff --git a/charm/launchpad-ppa-publisher/templates/main-publisher-run.j2 b/charm/launchpad-ppa-publisher/templates/main-publisher-run.j2
new file mode 100644
index 0000000..31ddee9
--- /dev/null
+++ b/charm/launchpad-ppa-publisher/templates/main-publisher-run.j2
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+{{ code_dir }}/cronscripts/publishing/cron.publish-ppa "-d ubuntu{{ excluded_ppas_options }}" "--all-derived{{ excluded_ppas_options }}" "-d debian{{ excluded_ppas_options }}" >> {{ logs_dir }}/cron.ppa.log 2>&1
+
diff --git a/charm/launchpad-ppa-publisher/templates/parallel-publisher-run.j2 b/charm/launchpad-ppa-publisher/templates/parallel-publisher-run.j2
new file mode 100644
index 0000000..262dbe0
--- /dev/null
+++ b/charm/launchpad-ppa-publisher/templates/parallel-publisher-run.j2
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+{{ code_dir }}/cronscripts/publishing/cron.publish-parallel-ppa "{{ distro_option }} {{ archive_options }} {{ lockfilename_option }}" >> {{ logs_dir }}/{{ logfilename }} 2>&1
+