canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+457239@xxxxxxxxxxxxxxxxxx>]

Diff comments:

> diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/check-config-files b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/check-config-files
> new file mode 100755
> index 0000000..d050040
> --- /dev/null
> +++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/check-config-files
> @@ -0,0 +1,244 @@
> +#!/usr/bin/python3
> +import json
> +import os
> +import socket
> +import subprocess
> +import urllib.parse
> +import urllib.request
> +
> +import swiftclient
> +from influxdb import InfluxDBClient
> +
> +INTEGRATION_KEY_FP = "/home/ubuntu/integration-key"
> +
> +bos_arches = ["arm64", "ppc64el", "s390x"]
> +
> +centres = {
> +    "bos01": bos_arches,
> +    "bos02": bos_arches,
> +    "lcy02": None,
> +}
> +
> +openstack_cmd = "openstack project list"
> +
> +openstack_commands = [
> +    "openstack project list",
> +    "openstack server list",
> +    "openstack network list",
> +    "openstack image list",
> +]
> +
> +openstack_vars = [
> +    "OS_USERNAME",
> +    "OS_TENANT_NAME",
> +    "OS_PASSWORD",
> +    "OS_AUTH_URL",
> +    "OS_REGION_NAME",
> +]
> +
> +influx_vars = [
> +    "INFLUXDB_HOSTNAME",
> +    "INFLUXDB_PORT",
> +    "INFLUXDB_USERNAME",
> +    "INFLUXDB_PASSWORD",
> +    "INFLUXDB_DATABASE",
> +    "INFLUXDB_CONTEXT",
> +]
> +
> +rabbit_vars = [
> +    "RABBIT_HOST",
> +    "RABBIT_USER",
> +    "RABBIT_PASSWORD",
> +]
> +
> +swift_vars = [
> +    "SWIFT_AUTH_URL",
> +    "SWIFT_AUTH_VERSION",
> +    "SWIFT_PASSWORD",
> +    "SWIFT_PROJECT_DOMAIN_NAME",
> +    "SWIFT_PROJECT_NAME",
> +    "SWIFT_REGION",
> +    "SWIFT_TENANT",
> +    "SWIFT_USER_DOMAIN_NAME",
> +    "SWIFT_USERNAME",
> +]
> +
> +worker_args = [
> +    "[autopkgtest]",
> +    "[virt]",
> +    "checkout_dir",
> +    "releases",
> +    "setup_command",
> +    "setup_command2",
> +    "per_package_config_dir",
> +    "architectures",
> +    "package_size_default",
> +    "package_size_big",
> +    "args",
> +]
> +
> +
> +def check_cloudrcs(args):
> +    centres, openstack_commands, openstack_vars = args
> +    for centre, arches in centres.items():
> +        for arch in arches:
> +            if arch is None:
> +                this_path = "~/cloudrcs/" + centre + ".rc"
> +            else:
> +                this_path = "~/cloudrcs/" + centre + "-" + arch + ".rc"
> +            if not os.path.isfile(this_path):
> +                return False
> +            with open(this_path, "r") as f:
> +                rc_file = f.read()
> +            vars = []
> +            for line in rc_file.splitlines():
> +                if "export" in line:
> +                    this_line = line.split(" ")
> +                    var, value = this_line.split("=")
> +                    vars.append(var)
> +                    os.environ[var] = value
> +            for var in openstack_vars:
> +                if var not in rc_file:
> +                    return False
> +            for command in openstack_commands:
> +                _ = subprocess.run(command.split(" "), check=True)
> +    return True
> +
> +
> +def check_influx_creds(influx_file, influx_keys):
> +    creds = check_env_file_and_keys(influx_file, influx_keys)
> +    for cred in creds.splitlines():
> +        if "=" in cred:
> +            var, value = cred.split("=")
> +            os.environ[var] = value
> +    influx_client = InfluxDBClient(
> +        os.environ["INFLUXDB_HOSTNAME"],
> +        os.environ["INFLUXDB_PORT"],
> +        os.environ["INFLUXDB_USERNAME"],
> +        os.environ["INFLUXDB_PASSWORD"],
> +        os.environ["INFLUXDB_DATABASE"],
> +    )
> +    influx_client.ping()
> +    return True
> +
> +
> +def check_env_file_and_keys(file, file_keys):
> +    mypath = os.path.expanduser("~/" + file)
> +    if not os.path.isfile(mypath):
> +        raise FileNotFoundError("file %s doesn't exist" % file)
> +    with open(mypath, "r") as f:
> +        myf = f.read()
> +    for k in file_keys:
> +        if k not in myf:
> +            raise KeyError("key %s not found in %s" % (k, file))
> +    return myf
> +
> +
> +def check_mirror_rc(mirror_file, mirror_vars):
> +    _ = check_env_file_and_keys(mirror_file, mirror_vars)
> +
> +
> +def check_rabbitmq_creds(rabbit_file, rabbit_vars):
> +    _ = check_env_file_and_keys(rabbit_file, rabbit_vars)
> +
> +
> +def check_net_name(net_file, net_vars):
> +    _ = check_env_file_and_keys(net_file, net_vars)
> +
> +
> +def check_swift_creds(swift_file, swift_vars):
> +    creds = check_env_file_and_keys(swift_file, swift_vars)
> +    for line in creds.splitlines():
> +        var, value = line.split("=")
> +        os.environ[var] = value.replace('"', "")
> +    swift_creds = {
> +        "authurl": os.environ["SWIFT_AUTH_URL"],
> +        "user": os.environ["SWIFT_USERNAME"],
> +        "key": os.environ["SWIFT_PASSWORD"],
> +        "os_options": {
> +            "region_name": os.environ["SWIFT_REGION"],
> +            "project_domain_name": os.environ["SWIFT_PROJECT_DOMAIN_NAME"],
> +            "project_name": os.environ["SWIFT_PROJECT_NAME"],
> +            "user_domain_name": os.environ["SWIFT_USER_DOMAIN_NAME"],
> +        },
> +        "auth_version": 3,
> +    }
> +    swift_conn = swiftclient.Connection(**swift_creds)
> +    _ = swift_conn.get_account()
> +    swift_conn.close()
> +    return True
> +
> +
> +def check_worker_conf_files(args):
> +    worker_args, centres = args
> +    for centre, arches in centres.items():
> +        if arches is not None:
> +            for a in arches:
> +                workerfile = "-".join(["worker", centre, a]) + ".conf"
> +                _ = check_env_file_and_keys(workerfile, worker_args)
> +
> +
> +files_and_keys = {

rename this

> +    "influx.cred": {
> +        "vars": influx_vars,

rename to callback_args

> +        "func": check_influx_creds,

rename to callback_function

> +    },
> +    "rabbitmq.cred": {
> +        "vars": rabbit_vars,
> +        "func": check_rabbitmq_creds,
> +    },
> +    "swift-password.cred": {
> +        "vars": swift_vars,
> +        "func": check_swift_creds,
> +    },
> +    "mirror.rc": {
> +        "vars": ["MIRROR"],
> +        "func": check_mirror_rc,
> +    },
> +    "net-name.rc": {
> +        "vars": ["NET_NAME"],
> +        "func": check_net_name,
> +    },
> +    "worker-configs": {
> +        "vars": (worker_args, centres),
> +        "func": check_worker_conf_files,
> +    },
> +    "cloudrcs": {
> +        "vars": (centres, openstack_commands, openstack_vars),
> +        "func": check_cloudrcs,
> +    },
> +}
> +
> +RESULTS = {}
> +
> +if __name__ == "__main__":
> +    for file, item in files_and_keys.items():
> +        try:
> +            if "." in file:

change this, don't differentiate via .
pass to functions as _

> +                item["func"](file, item["vars"])
> +            else:
> +                item["func"](item["vars"])
> +            RESULTS[file] = True
> +        except Exception as _:

change

> +            RESULTS[file] = False
> +    with open("/home/ubuntu/check-config-files-results.json", "w") as f:
> +        f.write(json.dumps(RESULTS, indent=2))
> +    if os.path.isfile("/home/ubuntu/autopkgtest-url"):

use pathlib.Path instead of os.path

> +        with open("/home/ubuntu/autopkgtest-url", "r") as f:
> +            webpage = f.read().rstrip()
> +        keypass = ""
> +        with open(INTEGRATION_KEY_FP, "r") as f:
> +            keypass = f.read().rstrip()
> +        post_me = {
> +            "type": "cloud",
> +            "source": socket.gethostname(),
> +            "pass": keypass,
> +            "test": __file__,
> +            "results": RESULTS,
> +        }
> +        results_url = webpage + "/post-integration-results"
> +        req = urllib.request.Request(results_url)
> +        req.add_header("Content-Type", "application/json; charset=utf-8")
> +        jsondata = json.dumps(post_me).encode("utf-8")
> +        req.add_header("Content-Length", len(jsondata))
> +        response = urllib.request.urlopen(req, jsondata)
> diff --git a/charms/focal/autopkgtest-web/webcontrol/check-config-files b/charms/focal/autopkgtest-web/webcontrol/check-config-files
> new file mode 100755
> index 0000000..a28bcf7
> --- /dev/null
> +++ b/charms/focal/autopkgtest-web/webcontrol/check-config-files
> @@ -0,0 +1,179 @@
> +#!/usr/bin/python3
> +import configparser
> +import json
> +import os
> +import socket
> +import sys
> +import urllib.parse
> +import urllib.request
> +
> +import requests
> +from distro_info import UbuntuDistroInfo
> +
> +# openstack-creds <- todo: part of the update-github-jobs-swiftclient-refactor mp
> +UDI = UbuntuDistroInfo()
> +
> +INTEGRATION_KEY_FP = "/home/ubuntu/integration-key"
> +
> +
> +def check_env_file_and_keys(file, file_keys):
> +    mypath = os.path.expanduser("~/" + file)
> +    if not os.path.isfile(mypath):
> +        raise FileNotFoundError("file %s doesn't exist" % file)
> +    with open(mypath, "r") as f:
> +        myf = f.read()
> +    for k in file_keys:
> +        if k not in myf:
> +            raise KeyError("key %s not found in %s" % (k, file))
> +    return myf
> +
> +
> +def check_autopkgtest_cloud_conf(conf_file, conf_keys):
> +    myf = check_env_file_and_keys(conf_file, conf_keys)
> +    sw_url = ""
> +    for line in myf.splitlines():
> +        if "SwiftURL" in line:
> +            sw_url = line.split("=")[1]
> +    for supported in UDI.supported():
> +        req = requests.get(sw_url + "/autopkgtest-" + supported)
> +        if req.status_code != 200:
> +            raise requests.ConnectionError(
> +                "Container autopkgtest-%s is unreachable - something wrong with the SwiftURL: %s"
> +                % (supported, sw_url)
> +            )
> +    cp = configparser.ConfigParser()
> +    cp.read(os.path.expanduser("~/" + conf_file))
> +
> +
> +def check_github_secrets(secrets_file, vars):
> +    _ = check_env_file_and_keys(secrets_file, vars)
> +
> +
> +def check_github_status_creds(creds_file, vars):
> +    _ = check_env_file_and_keys(creds_file, vars)
> +
> +
> +def check_swift_web_creds(creds_file, vars):
> +    _ = check_env_file_and_keys(creds_file, vars)
> +    cp = configparser.ConfigParser()
> +    cp.read(os.path.expanduser("~/" + creds_file))
> +    for var in vars[1:]:
> +        _ = cp.get("swift", var)
> +
> +
> +def check_openstack_creds(creds_file, vars):
> +    _ = check_env_file_and_keys(creds_file, vars)
> +
> +
> +autopkgtest_cloud_vars = [
> +    "[web]",
> +    "database",
> +    "database_ro",
> +    "SwiftURL",
> +    "ExternalURL",
> +    "cookies",
> +    "[amqp]",
> +    "uri",
> +]
> +
> +github_secrets_keys = [
> +    "systemd-upstream",
> +    "ovs-upstream",
> +    "ubuntu-image-autopkgtest",
> +    "snapcraft",
> +    "snapd",
> +]
> +
> +github_status_credentials = [
> +    "systemd-upstream",
> +    "ubuntu-image-autopkgtest",
> +    "snapcraft",
> +    "snapd",
> +]
> +
> +swift_web_creds = [
> +    "[swift]",
> +    "auth_url",
> +    "username",
> +    "password",
> +    "tenant",
> +    "region_name",
> +    "project_name",
> +    "project_domain_name",
> +    "user_domain_name",
> +    "obj_storage_url",
> +]
> +
> +openstack_creds = [
> +    "OS_REGION_NAME",
> +    "OS_INTERFACE",
> +    "OS_AUTH_URL",
> +    "OS_PROJECT_DOMAIN_NAME",
> +    "OS_USERNAME",
> +    "OS_USER_DOMAIN_NAME",
> +    "OS_PROJECT_NAME",
> +    "OS_PASSWORD",
> +    "OS_IDENTITY_API_VERSION",
> +]
> +
> +
> +files_and_keys = {

rename this (list_of_checks)

> +    "autopkgtest-cloud.conf": {
> +        "vars": autopkgtest_cloud_vars,
> +        "func": check_autopkgtest_cloud_conf,
> +    },
> +    "github-secrets.json": {
> +        "vars": github_secrets_keys,
> +        "func": check_github_secrets,
> +    },
> +    "github-status-credentials.txt": {
> +        "vars": github_status_credentials,
> +        "func": check_github_status_creds,
> +    },
> +    "swift-web-credentials.conf": {
> +        "vars": swift_web_creds,
> +        "func": check_swift_web_creds,
> +    },
> +    "openstack-creds": {
> +        "vars": openstack_creds,
> +        "func": check_openstack_creds,
> +    },
> +}
> +
> +RESULTS = {}
> +
> +if __name__ == "__main__":
> +    for file, item in files_and_keys.items():
> +        try:
> +            item["func"](file, item["vars"])
> +            RESULTS[file] = True
> +        except Exception as _:

also change

> +            RESULTS[file] = False
> +
> +    cp = configparser.ConfigParser()
> +    cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
> +    try:
> +        webpage = cp["web"]["ExternalURL"].replace("/results", "")
> +    except KeyError:
> +        # change to logging maybe ?
> +        print("No external url found!")
> +        sys.exit(1)
> +    keypass = ""
> +    with open(INTEGRATION_KEY_FP, "r") as f:
> +        keypass = f.read().rstrip()
> +    post_me = {
> +        "type": "web",
> +        "source": socket.gethostname(),
> +        "pass": keypass,
> +        "test": __file__,
> +        "results": RESULTS,
> +    }
> +    results_url = webpage + "/post-integration-results"
> +    req = urllib.request.Request(results_url)
> +    req.add_header("Content-Type", "application/json; charset=utf-8")
> +    jsondata = json.dumps(post_me).encode("utf-8")
> +    req.add_header("Content-Length", len(jsondata))
> +    response = urllib.request.urlopen(req, jsondata)
> +
> +    with open("/home/ubuntu/check-config-files-results.json", "w") as f:
> +        f.write(json.dumps(RESULTS, indent=2))


-- 
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/457239
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:integration-tests into autopkgtest-cloud:master.