canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+462054@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:configparser-read-refactor into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/462054
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:configparser-read-refactor into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/copy-swift-results b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/copy-swift-results
index ba8a40e..9484539 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/copy-swift-results
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/copy-swift-results
@@ -48,7 +48,8 @@ releases = "noble mantic jammy focal lunar"
 
 
 conf = configparser.ConfigParser()
-conf.read(creds_file)
+with open(creds_file, "r") as f:
+    conf.read_file(f)
 
 swift_creds = {
     "old": {
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
index 9ff2ff1..f44ad1d 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
@@ -1498,7 +1498,8 @@ def main():
         },
         allow_no_value=True,
     )
-    cfg.read(args.config)
+    with open(args.config, "r") as f:
+        cfg.read_file(f)
 
     logging.basicConfig(
         level=(args.debug and logging.DEBUG or logging.INFO),
diff --git a/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector b/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector
index a61a949..1169f2e 100755
--- a/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector
+++ b/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector
@@ -2,7 +2,6 @@
 # Pick up running tests, their status and logtail from the "teststatus" fanout
 # queue, and regularly write it into /run/amqp-status-collector/running.json
 
-import configparser
 import json
 import logging
 import os
@@ -11,6 +10,7 @@ import time
 import urllib.parse
 
 import amqplib.client_0_8 as amqp
+from helpers.utils import get_autopkgtest_cloud_conf
 
 exchange_name = "teststatus.fanout"
 running_name = os.path.join(
@@ -26,8 +26,7 @@ last_update = 0
 def amqp_connect():
     """Connect to AMQP server"""
 
-    cp = configparser.ConfigParser()
-    cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    cp = get_autopkgtest_cloud_conf()
     amqp_uri = cp["amqp"]["uri"]
     parts = urllib.parse.urlsplit(amqp_uri, allow_fragments=False)
     amqp_con = amqp.Connection(
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index f71b406..12a8af5 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -2,7 +2,6 @@
 
 """Browse autopkgtest results"""
 
-import configparser
 import json
 import os
 import re
@@ -13,7 +12,11 @@ from wsgiref.handlers import CGIHandler
 import flask
 from helpers.admin import select_abnormally_long_jobs
 from helpers.exceptions import RunningJSONNotFound
-from helpers.utils import get_all_releases, get_supported_releases
+from helpers.utils import (
+    get_all_releases,
+    get_autopkgtest_cloud_conf,
+    get_supported_releases,
+)
 from werkzeug.middleware.proxy_fix import ProxyFix
 
 app = flask.Flask("browse")
@@ -31,8 +34,7 @@ RUNNING_CACHE = "/run/amqp-status-collector/running.json"
 def init_config():
     global db_con, swift_container_url, INDEXED_PACKAGES_FP
 
-    cp = configparser.ConfigParser()
-    cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    cp = get_autopkgtest_cloud_conf()
 
     db_con = sqlite3.connect(
         "file:%s?mode=ro" % cp["web"]["database_ro"],
diff --git a/charms/focal/autopkgtest-web/webcontrol/cache-amqp b/charms/focal/autopkgtest-web/webcontrol/cache-amqp
index 6ebcfc5..a03550e 100755
--- a/charms/focal/autopkgtest-web/webcontrol/cache-amqp
+++ b/charms/focal/autopkgtest-web/webcontrol/cache-amqp
@@ -1,7 +1,6 @@
 #!/usr/bin/python3
 
 import argparse
-import configparser
 import json
 import logging
 import os
@@ -14,6 +13,7 @@ import urllib.request
 
 import amqplib.client_0_8 as amqp
 from amqplib.client_0_8.exceptions import AMQPChannelException
+from helpers.utils import get_autopkgtest_cloud_conf
 
 AMQP_CONTEXTS = ["ubuntu", "huge", "ppa", "upstream"]
 
@@ -233,8 +233,7 @@ if __name__ == "__main__":
 
     args = parser.parse_args()
 
-    cp = configparser.ConfigParser()
-    cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    cp = get_autopkgtest_cloud_conf()
 
     logger = logging.getLogger()
     formatter = logging.Formatter(
diff --git a/charms/focal/autopkgtest-web/webcontrol/db-backup b/charms/focal/autopkgtest-web/webcontrol/db-backup
index fc5190a..418c179 100755
--- a/charms/focal/autopkgtest-web/webcontrol/db-backup
+++ b/charms/focal/autopkgtest-web/webcontrol/db-backup
@@ -5,7 +5,6 @@ and clears up old backups
 """
 
 import atexit
-import configparser
 import datetime
 import gzip
 import logging
@@ -16,7 +15,7 @@ import sys
 from pathlib import Path
 
 import swiftclient
-from helpers.utils import init_db
+from helpers.utils import get_autopkgtest_cloud_conf, init_db
 
 DB_PATH = ""
 DB_NAME = ""
@@ -35,8 +34,7 @@ def db_connect() -> sqlite3.Connection:
     global DB_NAME
     global DB_BACKUP_NAME
     global DB_BACKUP_PATH
-    cp = configparser.ConfigParser()
-    cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    cp = get_autopkgtest_cloud_conf()
     DB_PATH = Path(cp["web"]["database"])
     DB_NAME = DB_PATH.name
     DB_BACKUP_NAME = "%s.bak" % DB_NAME
diff --git a/charms/focal/autopkgtest-web/webcontrol/download-all-results b/charms/focal/autopkgtest-web/webcontrol/download-all-results
index e0a11b0..3396403 100755
--- a/charms/focal/autopkgtest-web/webcontrol/download-all-results
+++ b/charms/focal/autopkgtest-web/webcontrol/download-all-results
@@ -10,7 +10,6 @@
 # notification of completed jobs, in case of bugs or network outages etc, this
 # script can be used to find any results which were missed and insert them.
 
-import configparser
 import http
 import io
 import json
@@ -25,7 +24,7 @@ import urllib.parse
 from urllib.request import urlopen
 
 from distro_info import UbuntuDistroInfo
-from helpers.utils import get_test_id, init_db
+from helpers.utils import get_autopkgtest_cloud_conf, get_test_id, init_db
 
 LOGGER = logging.getLogger(__name__)
 
@@ -257,8 +256,7 @@ if __name__ == "__main__":
     )
     releases.sort(key=UbuntuDistroInfo().all.index, reverse=True)
 
-    config = configparser.ConfigParser()
-    config.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    config = get_autopkgtest_cloud_conf()
 
     try:
         for release in releases:
diff --git a/charms/focal/autopkgtest-web/webcontrol/download-results b/charms/focal/autopkgtest-web/webcontrol/download-results
index e71d4a0..e4b6e6f 100755
--- a/charms/focal/autopkgtest-web/webcontrol/download-results
+++ b/charms/focal/autopkgtest-web/webcontrol/download-results
@@ -1,6 +1,5 @@
 #!/usr/bin/python3
 
-import configparser
 import json
 import logging
 import os
@@ -11,7 +10,7 @@ import time
 import urllib.parse
 
 import amqplib.client_0_8 as amqp
-from helpers.utils import get_test_id, init_db
+from helpers.utils import get_autopkgtest_cloud_conf, get_test_id, init_db
 
 EXCHANGE_NAME = "testcomplete.fanout"
 
@@ -19,8 +18,7 @@ EXCHANGE_NAME = "testcomplete.fanout"
 def amqp_connect():
     """Connect to AMQP server"""
 
-    cp = configparser.ConfigParser()
-    cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    cp = get_autopkgtest_cloud_conf()
     amqp_uri = cp["amqp"]["uri"]
     parts = urllib.parse.urlsplit(amqp_uri, allow_fragments=False)
     amqp_con = amqp.Connection(
@@ -35,8 +33,7 @@ def amqp_connect():
 
 def db_connect():
     """Connect to SQLite DB"""
-    cp = configparser.ConfigParser()
-    cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    cp = get_autopkgtest_cloud_conf()
 
     db_con = init_db(cp["web"]["database"])
 
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
index cf9ca27..ea1f758 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
@@ -1,6 +1,8 @@
 """
 utilities for autopkgtest-web webcontrol
 """
+import configparser
+
 # pylint: disable=protected-access
 import logging
 import os
@@ -11,6 +13,27 @@ import time
 import distro_info
 
 
+def read_config_file(
+    filepath: str, env_file: bool = False, cfg_key: str = None
+):
+    if env_file is True and cfg_key is None:
+        raise ValueError(
+            "Specifying envfile as True requires cfg_key to be passed"
+        )
+    config = configparser.ConfigParser()
+    if not env_file:
+        with open(filepath, "r") as f:
+            config.read_file(f)
+        return config
+    with open(filepath, "r") as fp:
+        config.read_string(("[%s]\n" % cfg_key) + fp.read())
+    return config
+
+
+def get_autopkgtest_cloud_conf():
+    return read_config_file("/home/ubuntu/autopkgtest-cloud.conf")
+
+
 def get_all_releases():
     udi = distro_info.UbuntuDistroInfo()
     return udi.all
diff --git a/charms/focal/autopkgtest-web/webcontrol/indexed-packages b/charms/focal/autopkgtest-web/webcontrol/indexed-packages
index 82502b1..1122bd2 100755
--- a/charms/focal/autopkgtest-web/webcontrol/indexed-packages
+++ b/charms/focal/autopkgtest-web/webcontrol/indexed-packages
@@ -1,10 +1,10 @@
 #!/usr/bin/python3
 
-import configparser
 import json
-import os
 import sqlite3
 
+from helpers.utils import get_autopkgtest_cloud_conf
+
 
 def srchash(src):
     if src.startswith("lib"):
@@ -14,8 +14,7 @@ def srchash(src):
 
 
 if __name__ == "__main__":
-    cp = configparser.ConfigParser()
-    cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    cp = get_autopkgtest_cloud_conf()
     indexed_packages_fp = cp["web"]["indexed_packages_fp"]
 
     db_con = sqlite3.connect(
diff --git a/charms/focal/autopkgtest-web/webcontrol/private_results/app.py b/charms/focal/autopkgtest-web/webcontrol/private_results/app.py
index 0a793bc..36b1fd0 100644
--- a/charms/focal/autopkgtest-web/webcontrol/private_results/app.py
+++ b/charms/focal/autopkgtest-web/webcontrol/private_results/app.py
@@ -1,5 +1,4 @@
 """Test Result Fetcher Flask App"""
-import configparser
 import logging
 import os
 import sys
@@ -15,7 +14,11 @@ from flask import (
     session,
 )
 from flask_openid import OpenID
-from helpers.utils import setup_key
+from helpers.utils import (
+    get_autopkgtest_cloud_conf,
+    read_config_file,
+    setup_key,
+)
 from request.submit import Submit
 from werkzeug.middleware.proxy_fix import ProxyFix
 
@@ -94,11 +97,11 @@ secret_path = os.path.join(PATH, "secret_key")
 setup_key(app, secret_path)
 oid = OpenID(app, os.path.join(PATH, "openid"), safe_roots=[])
 # Load configuration
-cfg = configparser.ConfigParser()
-cfg.read(os.path.expanduser("~ubuntu/swift-web-credentials.conf"))
+cfg = read_config_file(
+    os.path.expanduser("~ubuntu/swift-web-credentials.conf")
+)
 # The web configuration as well
-cfg_web = configparser.ConfigParser()
-cfg_web.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+cfg_web = get_autopkgtest_cloud_conf()
 # Build swift credentials
 auth_url = cfg.get("swift", "auth_url")
 if "/v2.0" in auth_url:
diff --git a/charms/focal/autopkgtest-web/webcontrol/publish-db b/charms/focal/autopkgtest-web/webcontrol/publish-db
index 76883bd..4b4a9da 100755
--- a/charms/focal/autopkgtest-web/webcontrol/publish-db
+++ b/charms/focal/autopkgtest-web/webcontrol/publish-db
@@ -3,7 +3,6 @@
 # a copy of autopkgtest.db, which is then published to the public location.
 # This is being used for statistics.
 
-import configparser
 import gzip
 import logging
 import os
@@ -12,6 +11,7 @@ import tempfile
 import urllib.request
 
 import apt_pkg
+from helpers.utils import get_autopkgtest_cloud_conf
 
 config = None
 db_con = None
@@ -182,8 +182,7 @@ if __name__ == "__main__":
     if "DEBUG" in os.environ:
         logging.basicConfig(level=logging.DEBUG)
 
-    config = configparser.ConfigParser()
-    config.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    config = get_autopkgtest_cloud_conf()
 
     target = config["web"]["database_ro"]
     target_new = "{}.new".format(target)
diff --git a/charms/focal/autopkgtest-web/webcontrol/request/submit.py b/charms/focal/autopkgtest-web/webcontrol/request/submit.py
index 0133d44..7cad9ac 100644
--- a/charms/focal/autopkgtest-web/webcontrol/request/submit.py
+++ b/charms/focal/autopkgtest-web/webcontrol/request/submit.py
@@ -4,7 +4,6 @@ Author: Martin Pitt <martin.pitt@xxxxxxxxxx>
 """
 
 import base64
-import configparser
 import json
 import logging
 import os
@@ -26,6 +25,7 @@ from helpers.exceptions import (
     RequestInQueue,
     RequestRunning,
 )
+from helpers.utils import get_autopkgtest_cloud_conf
 
 # Launchpad REST API base
 LP = "https://api.launchpad.net/1.0/";
@@ -56,8 +56,7 @@ RUNNING_FP = "/run/amqp-status-collector/running.json"
 
 class Submit:
     def __init__(self):
-        cp = configparser.ConfigParser()
-        cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+        cp = get_autopkgtest_cloud_conf()
 
         # read valid releases and architectures from DB
         self.db_con = sqlite3.connect(
diff --git a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
index 9aeb8e7..2757eb5 100755
--- a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
+++ b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
@@ -1,6 +1,5 @@
 #!/usr/bin/python3
 
-import configparser
 import datetime
 import io
 import json
@@ -12,6 +11,7 @@ import tarfile
 from datetime import datetime, timedelta
 
 import swiftclient
+from helpers.utils import get_autopkgtest_cloud_conf, read_config_file
 from request.submit import Submit
 
 PENDING_DIR = "/run/autopkgtest_webcontrol/github-pending"
@@ -216,13 +216,11 @@ if __name__ == "__main__":
         logging.info("%s does not exist, nothing to do", PENDING_DIR)
         sys.exit(0)
 
-    config = configparser.ConfigParser()
-    config.read("/home/ubuntu/autopkgtest-cloud.conf")
+    config = get_autopkgtest_cloud_conf()
     external_url = config["web"]["ExternalURL"]
-    swift_cfg = configparser.ConfigParser()
-    # Nice way of passing an environment-like file to configparser
-    with open(SWIFT_CREDS_FILE, "r") as fp:
-        swift_cfg.read_string("[swift]\n" + fp.read())
+    swift_cfg = read_config_file(
+        filepath=SWIFT_CREDS_FILE, env_file=True, cfg_key="swift"
+    )
 
     swift_creds = {
         "authurl": swift_cfg["swift"]["OS_AUTH_URL"],