canonical-ubuntu-qa team mailing list archive

[Skia <mp+471159@xxxxxxxxxxxxxxxxxx>]

Skia has proposed merging ~hyask/autopkgtest-cloud:skia/git_clone_in_charms into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~hyask/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/471159

Change the way the code is run, to have the `git` repo available and ease the way we handle cowboys.
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~hyask/autopkgtest-cloud:skia/git_clone_in_charms into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/actions/update-sources b/charms/focal/autopkgtest-cloud-worker/actions/update-sources
index 7bd2f9d..449d462 100755
--- a/charms/focal/autopkgtest-cloud-worker/actions/update-sources
+++ b/charms/focal/autopkgtest-cloud-worker/actions/update-sources
@@ -1,10 +1,12 @@
 #!/usr/local/sbin/charm-env python3
 
-import sys
-
 from reactive.autopkgtest_cloud_worker import (
     AUTOPKGTEST_LOCATION,
+    AUTOPKGTEST_CLONE_BRANCH,
     AUTODEP8_LOCATION,
+    AUTODEP8_CLONE_BRANCH,
+    AUTOPKGTEST_CLOUD_GIT_LOCATION,
+    AUTOPKGTEST_CLOUD_CLONE_BRANCH,
 )
 
 import pygit2
@@ -12,8 +14,12 @@ from utils import pull, install_autodep8, UnixUser
 
 
 if __name__ == "__main__":
-    for location in [AUTOPKGTEST_LOCATION, AUTODEP8_LOCATION]:
+    for location, branch in [
+        (AUTOPKGTEST_LOCATION, AUTOPKGTEST_CLONE_BRANCH),
+        (AUTODEP8_LOCATION, AUTODEP8_CLONE_BRANCH),
+        (AUTOPKGTEST_CLOUD_GIT_LOCATION, AUTOPKGTEST_CLOUD_CLONE_BRANCH),
+    ]:
         repo = pygit2.Repository(location)
         with UnixUser("ubuntu"):
-            pull(repo)
+            pull(repo, branch)
     install_autodep8(AUTODEP8_LOCATION)
diff --git a/charms/focal/autopkgtest-cloud-worker/config.yaml b/charms/focal/autopkgtest-cloud-worker/config.yaml
index 17dc182..6c61b2d 100644
--- a/charms/focal/autopkgtest-cloud-worker/config.yaml
+++ b/charms/focal/autopkgtest-cloud-worker/config.yaml
@@ -1,4 +1,8 @@
 options:
+  package_status:
+    type: string
+    description: "APT layer package_status"
+    default: install
   nova-rcs:
     type: string
     description: "base64 encoded tarball of nova OpenStack credential .rc files"
diff --git a/charms/focal/autopkgtest-cloud-worker/lib/utils.py b/charms/focal/autopkgtest-cloud-worker/lib/utils.py
index 29ae3f4..722aa37 100644
--- a/charms/focal/autopkgtest-cloud-worker/lib/utils.py
+++ b/charms/focal/autopkgtest-cloud-worker/lib/utils.py
@@ -46,19 +46,23 @@ def install_autodep8(location):
     subprocess.check_call(["make", "install"])
 
 
-def pull(repository):
-    """This will do a sort of git fetch origin && git reset --hard origin/master"""
+def pull(repository, branch):
+    """
+    This will do a sort of git fetch origin && git reset --hard origin/master
+    This raises pygit2.GitError whenever the tree is not clean, preventing to loose cowboys.
+    """
     origin = [
         remote for remote in repository.remotes if remote.name == "origin"
     ][0]
     origin.fetch()
     remote_master_id = repository.lookup_reference(
-        "refs/remotes/origin/master"
+        f"refs/remotes/origin/{branch}"
     ).target
     repository.checkout_tree(repository.get(remote_master_id))
-    master_ref = repository.lookup_reference("refs/heads/master")
+    master_ref = repository.lookup_reference(f"refs/heads/{branch}")
     master_ref.set_target(remote_master_id)
     repository.head.set_target(remote_master_id)
+    log(f"Updated repository {repository} to {remote_master_id}", "INFO")
 
 
 @total_ordering
diff --git a/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py b/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py
index f2a8ab7..bd6e4e4 100644
--- a/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py
+++ b/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py
@@ -9,13 +9,7 @@ from textwrap import dedent
 
 import pygit2
 import yaml
-from charmhelpers.core.hookenv import (
-    charm_dir,
-    config,
-    log,
-    storage_get,
-    storage_list,
-)
+from charmhelpers.core.hookenv import config, log, storage_get, storage_list
 from charms.layer import status
 from charms.reactive import (
     clear_flag,
@@ -30,27 +24,35 @@ from charms.reactive import (
 )
 from utils import UnixUser, install_autodep8
 
-AUTOPKGTEST_LOCATION = os.path.expanduser("~ubuntu/autopkgtest")
 AUTOPKGTEST_CLOUD_LOCATION = os.path.expanduser("~ubuntu/autopkgtest-cloud")
-AUTODEP8_LOCATION = os.path.expanduser("~ubuntu/autodep8")
-AUTOPKGTEST_PER_PACKAGE_LOCATION = os.path.expanduser(
-    "~ubuntu/autopkgtest-package-configs/"
+AUTOPKGTEST_CLOUD_GIT_LOCATION = os.path.expanduser(
+    "~ubuntu/autopkgtest-cloud.git"
+)
+AUTOPKGTEST_CLOUD_CLONE_LOCATION = (
+    "https://git.launchpad.net/autopkgtest-cloud";
 )
+AUTOPKGTEST_CLOUD_CLONE_BRANCH = "master"
 
+AUTOPKGTEST_LOCATION = os.path.expanduser("~ubuntu/autopkgtest")
 AUTOPKGTEST_CLONE_LOCATION = (
     "https://salsa.debian.org/ubuntu-ci-team/autopkgtest.git";
 )
-
 AUTOPKGTEST_CLONE_BRANCH = "ubuntu/production"
 
+AUTODEP8_LOCATION = os.path.expanduser("~ubuntu/autodep8")
 AUTODEP8_CLONE_LOCATION = (
     "https://git.launchpad.net/~ubuntu-release/+git/autodep8";
 )
+AUTODEP8_CLONE_BRANCH = "master"
 
+AUTOPKGTEST_PER_PACKAGE_LOCATION = os.path.expanduser(
+    "~ubuntu/autopkgtest-package-configs/"
+)
 AUTOPKGTEST_PER_PACKAGE_CLONE_LOCATION = (
     "https://git.launchpad.net/~ubuntu-release/autopkgtest";
     + "-cloud/+git/autopkgtest-package-configs"
 )
+AUTOPKGTEST_PER_PACKAGE_CLONE_BRANCH = "main"
 
 RABBITMQ_CRED_PATH = os.path.expanduser("~ubuntu/rabbitmq.cred")
 
@@ -72,18 +74,23 @@ match ExtraWhitespace /\\s\\+$\\| \\+\\ze\\t/                         " define t
     )
 
 
+@when("autopkgtest.autopkgtest-cloud_cloned")
 @when_not("autopkgtest.autopkgtest_cloud_symlinked")
 def symlink_autopkgtest_cloud():
     status.maintenance("Creating symlink to charmed autopkgtest-cloud code")
     with UnixUser("ubuntu"):
         try:
-            autopkgtest_cloud = os.path.join(charm_dir(), "autopkgtest-cloud")
+            autopkgtest_cloud = os.path.join(
+                AUTOPKGTEST_CLOUD_GIT_LOCATION,
+                "charms",
+                "focal",
+                "autopkgtest-cloud-worker",
+                "autopkgtest-cloud",
+            )
             os.symlink(autopkgtest_cloud, AUTOPKGTEST_CLOUD_LOCATION)
         except FileExistsError:
             pass
-    status.maintenance(
-        "Done creating symlink to charmed autopkgtest-cloud code"
-    )
+    status.maintenance("Done creating symlink to autopkgtest-cloud code")
     set_flag("autopkgtest.autopkgtest_cloud_symlinked")
 
 
@@ -94,7 +101,11 @@ def clone_autodep8():
     log("Cloning autodep8", "INFO")
     with UnixUser("ubuntu"):
         try:
-            pygit2.clone_repository(AUTODEP8_CLONE_LOCATION, AUTODEP8_LOCATION)
+            pygit2.clone_repository(
+                AUTODEP8_CLONE_LOCATION,
+                AUTODEP8_LOCATION,
+                checkout_branch=AUTODEP8_CLONE_BRANCH,
+            )
             log("autodep8 cloned", "INFO")
             status.maintenance("autodep8 cloned")
             set_flag("autopkgtest.autodep8_cloned")
@@ -112,6 +123,7 @@ def clone_autopkgtest_package_configs():
             pygit2.clone_repository(
                 AUTOPKGTEST_PER_PACKAGE_CLONE_LOCATION,
                 AUTOPKGTEST_PER_PACKAGE_LOCATION,
+                checkout_branch=AUTOPKGTEST_PER_PACKAGE_CLONE_BRANCH,
             )
             log("autopkgtest-package-configs cloned", "INFO")
             status.maintenance("autopkgtest-package-configs cloned")
@@ -149,6 +161,26 @@ def clone_autopkgtest():
             log("autopkgtest already cloned", "INFO")
 
 
+@when("apt.installed.git")
+@when_not("autopkgtest.autopkgtest-cloud_cloned")
+def clone_autopkgtest_cloud():
+    status.maintenance("Cloning autopkgtest-cloud")
+    log("Cloning autopkgtest-cloud", "INFO")
+    with UnixUser("ubuntu"):
+        try:
+            pygit2.clone_repository(
+                AUTOPKGTEST_CLOUD_CLONE_LOCATION,
+                AUTOPKGTEST_CLOUD_GIT_LOCATION,
+                checkout_branch=AUTOPKGTEST_CLOUD_CLONE_BRANCH,
+            )
+            status.maintenance("autopkgtest-cloud cloned")
+            log("autopkgtest-cloud cloned", "INFO")
+            set_flag("autopkgtest.autopkgtest-cloud_cloned")
+        except ValueError:
+            # exists
+            log("autopkgtest-cloud already cloned", "INFO")
+
+
 @when_all(
     "autopkgtest.autopkgtest_cloud_symlinked",
     "autopkgtest.rabbitmq-configured",
@@ -156,7 +188,16 @@ def clone_autopkgtest():
 )
 def set_up_systemd_units():
     status.maintenance("Installing systemd units")
-    for unit in glob.glob(os.path.join(charm_dir(), "units", "*")):
+    for unit in glob.glob(
+        os.path.join(
+            AUTOPKGTEST_CLOUD_GIT_LOCATION,
+            "charms",
+            "focal",
+            "autopkgtest-cloud-worker",
+            "units",
+            "*",
+        )
+    ):
         base = os.path.basename(unit)
         dest = os.path.join(os.path.sep, "etc", "systemd", "system", base)
 
diff --git a/charms/focal/autopkgtest-web/actions.yaml b/charms/focal/autopkgtest-web/actions.yaml
new file mode 100644
index 0000000..efe984d
--- /dev/null
+++ b/charms/focal/autopkgtest-web/actions.yaml
@@ -0,0 +1,2 @@
+update-sources:
+    description: Update (git pull) cloned repositories
diff --git a/charms/focal/autopkgtest-web/actions/update-sources b/charms/focal/autopkgtest-web/actions/update-sources
new file mode 100755
index 0000000..46a562e
--- /dev/null
+++ b/charms/focal/autopkgtest-web/actions/update-sources
@@ -0,0 +1,18 @@
+#!/usr/local/sbin/charm-env python3
+
+from reactive.autopkgtest_web import (
+    AUTOPKGTEST_CLOUD_GIT_LOCATION,
+    AUTOPKGTEST_CLOUD_CLONE_BRANCH,
+)
+
+import pygit2
+from utils import pull, UnixUser
+
+
+if __name__ == "__main__":
+    for location, branch in [
+        (AUTOPKGTEST_CLOUD_GIT_LOCATION, AUTOPKGTEST_CLOUD_CLONE_BRANCH),
+    ]:
+        repo = pygit2.Repository(location)
+        with UnixUser("ubuntu"):
+            pull(repo, branch)
diff --git a/charms/focal/autopkgtest-web/config.yaml b/charms/focal/autopkgtest-web/config.yaml
index 51df278..1cab2a2 100644
--- a/charms/focal/autopkgtest-web/config.yaml
+++ b/charms/focal/autopkgtest-web/config.yaml
@@ -1,4 +1,8 @@
 options:
+  package_status:
+    type: string
+    description: "APT layer package_status"
+    default: install
   storage-url-internal:
     type: string
     default:
diff --git a/charms/focal/autopkgtest-web/layer.yaml b/charms/focal/autopkgtest-web/layer.yaml
index d003a4a..d49c75e 100644
--- a/charms/focal/autopkgtest-web/layer.yaml
+++ b/charms/focal/autopkgtest-web/layer.yaml
@@ -20,6 +20,7 @@ options:
       - python3-flask
       - python3-flask-openid
       - python3-influxdb
+      - python3-pygit2
       - python3-swiftclient
       - python3-werkzeug
       - vim
diff --git a/charms/focal/autopkgtest-web/lib/utils.py b/charms/focal/autopkgtest-web/lib/utils.py
new file mode 100644
index 0000000..4f1316c
--- /dev/null
+++ b/charms/focal/autopkgtest-web/lib/utils.py
@@ -0,0 +1,55 @@
+# pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring
+import os
+import pwd
+
+from charmhelpers.core.hookenv import log
+
+
+# class UnixUser(object):
+class UnixUser:
+    def __init__(self, username):
+        self.username = username
+        pwnam = pwd.getpwnam(username)
+        self.uid = pwnam.pw_uid
+        self.gid = pwnam.pw_gid
+
+    def __enter__(self):
+        log(
+            "Raising UID to {uid} and GID to {gid}".format(
+                uid=self.uid, gid=self.gid
+            ),
+            "INFO",
+        )
+        os.setresgid(self.gid, self.gid, os.getgid())
+        os.setresuid(self.uid, self.uid, os.getuid())
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        _, _, suid = os.getresuid()
+        _, _, sgid = os.getresgid()
+        log(
+            "Restoring UID to {suid} and GID to {sgid}".format(
+                suid=suid, sgid=sgid
+            ),
+            "INFO",
+        )
+        os.setresuid(suid, suid, suid)
+        os.setresgid(sgid, sgid, sgid)
+
+
+def pull(repository, branch):
+    """
+    This will do a sort of git fetch origin && git reset --hard origin/master
+    This raises pygit2.GitError whenever the tree is not clean, preventing to loose cowboys.
+    """
+    origin = [
+        remote for remote in repository.remotes if remote.name == "origin"
+    ][0]
+    origin.fetch()
+    remote_master_id = repository.lookup_reference(
+        f"refs/remotes/origin/{branch}"
+    ).target
+    repository.checkout_tree(repository.get(remote_master_id))
+    master_ref = repository.lookup_reference(f"refs/heads/{branch}")
+    master_ref.set_target(remote_master_id)
+    repository.head.set_target(remote_master_id)
+    log(f"Updated repository {repository} to {remote_master_id}", "INFO")
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index ba091b7..df18d70 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -6,7 +6,8 @@ import shutil
 import subprocess
 from textwrap import dedent
 
-from charmhelpers.core.hookenv import charm_dir, config
+import pygit2
+from charmhelpers.core.hookenv import config, log
 from charms.layer import status
 from charms.reactive import (
     clear_flag,
@@ -18,8 +19,18 @@ from charms.reactive import (
     when_not,
     when_not_all,
 )
+from utils import UnixUser
 
 AUTOPKGTEST_CLOUD_CONF = os.path.expanduser("~ubuntu/autopkgtest-cloud.conf")
+
+AUTOPKGTEST_CLOUD_GIT_LOCATION = os.path.expanduser(
+    "~ubuntu/autopkgtest-cloud.git"
+)
+AUTOPKGTEST_CLOUD_CLONE_LOCATION = (
+    "https://git.launchpad.net/autopkgtest-cloud";
+)
+AUTOPKGTEST_CLOUD_CLONE_BRANCH = "master"
+
 GITHUB_SECRETS_PATH = os.path.expanduser("~ubuntu/github-secrets.json")
 GITHUB_STATUS_CREDENTIALS_PATH = os.path.expanduser(
     "~ubuntu/github-status-credentials.txt"
@@ -85,15 +96,22 @@ match ExtraWhitespace /\\s\\+$\\| \\+\\ze\\t/                         " define t
     )
 
 
+@when("autopkgtest.autopkgtest-cloud_cloned")
 @when_not("autopkgtest-web.autopkgtest_web_symlinked")
 def symlink_autopkgtest_cloud():
     status.maintenance("Creating symlink to charmed autopkgtest-web code")
     try:
-        autopkgtest_cloud = os.path.join(charm_dir(), "webcontrol")
+        autopkgtest_cloud = os.path.join(
+            AUTOPKGTEST_CLOUD_GIT_LOCATION,
+            "charms",
+            "focal",
+            "autopkgtest-web",
+            "webcontrol",
+        )
         os.symlink(autopkgtest_cloud, os.path.expanduser("~ubuntu/webcontrol"))
     except FileExistsError:
         pass
-    status.maintenance("Done creating symlink to charmed autopkgtest-web code")
+    status.maintenance("Done creating symlink to autopkgtest-web code")
     set_flag("autopkgtest-web.autopkgtest_web_symlinked")
 
 
@@ -144,6 +162,26 @@ def write_autopkgtest_cloud_conf(rabbitmq):
     set_flag("autopkgtest-web.config-written")
 
 
+@when("apt.installed.git")
+@when_not("autopkgtest.autopkgtest-cloud_cloned")
+def clone_autopkgtest_cloud():
+    status.maintenance("Cloning autopkgtest-cloud")
+    log("Cloning autopkgtest-cloud", "INFO")
+    with UnixUser("ubuntu"):
+        try:
+            pygit2.clone_repository(
+                AUTOPKGTEST_CLOUD_CLONE_LOCATION,
+                AUTOPKGTEST_CLOUD_GIT_LOCATION,
+                checkout_branch=AUTOPKGTEST_CLOUD_CLONE_BRANCH,
+            )
+            status.maintenance("autopkgtest-cloud cloned")
+            log("autopkgtest-cloud cloned", "INFO")
+            set_flag("autopkgtest.autopkgtest-cloud_cloned")
+        except ValueError:
+            # exists
+            log("autopkgtest-cloud already cloned", "INFO")
+
+
 @when_all(
     "autopkgtest-web.autopkgtest_web_symlinked",
     "autopkgtest-web.config-written",
@@ -151,7 +189,16 @@ def write_autopkgtest_cloud_conf(rabbitmq):
 def set_up_systemd_units():
     status.maintenance("Setting up systemd units")
     any_changed = False
-    for unit in glob.glob(os.path.join(charm_dir(), "units", "*")):
+    for unit in glob.glob(
+        os.path.join(
+            AUTOPKGTEST_CLOUD_GIT_LOCATION,
+            "charms",
+            "focal",
+            "autopkgtest-web",
+            "units",
+            "*",
+        )
+    ):
         base = os.path.basename(unit)
         try:
             os.symlink(
@@ -202,7 +249,7 @@ def initially_configure_website(website):
 )
 def set_up_web_config(apache):
     status.maintenance("Setting up web config (apache)")
-    webcontrol_dir = os.path.join(charm_dir(), "webcontrol")
+    webcontrol_dir = os.path.expanduser("~ubuntu/webcontrol")
     sn = config().get("hostname")
     https_proxy = config().get("https-proxy")
     no_proxy = config().get("no-proxy")
@@ -485,6 +532,7 @@ def make_runtime_tmpfiles():
     set_flag("autopkgtest-web.runtime-dir-created")
 
 
+@when("autopkgtest-web.autopkgtest_web_symlinked")
 @when_not("autopkgtest-web.running-json-symlinked")
 def symlink_running():
     status.maintenance("Creating symlink to running.json")
@@ -493,7 +541,12 @@ def symlink_running():
             os.path.join(
                 os.path.sep, "run", "amqp-status-collector", "running.json"
             ),
-            os.path.join(charm_dir(), "webcontrol", "static", "running.json"),
+            os.path.join(
+                os.path.expanduser("~ubuntu"),
+                "webcontrol",
+                "static",
+                "running.json",
+            ),
         )
         status.maintenance("Done creating symlink to running.json")
         set_flag("autopkgtest-web.running-json-symlinked")
@@ -501,6 +554,7 @@ def symlink_running():
         pass
 
 
+@when("autopkgtest-web.autopkgtest_web_symlinked")
 @when_not_all(
     "autopkgtest-web.public-db-symlinked",
     "autopkgtest-web.public-db-sha256-symlinked",
@@ -523,7 +577,10 @@ def symlink_public_db():
             os.symlink(
                 os.path.join(publicdir, symlink_file),
                 os.path.join(
-                    charm_dir(), "webcontrol", "static", symlink_file
+                    os.path.expanduser("~ubuntu"),
+                    "webcontrol",
+                    "static",
+                    symlink_file,
                 ),
             )
             set_flag(symlink_flag)