canonical-ubuntu-qa team mailing list archive
-
canonical-ubuntu-qa team
-
Mailing list archive
-
Message #05719
[Merge] ~hyask/autopkgtest-cloud:skia/switch_web_to_amqp into autopkgtest-cloud:master
Skia has proposed merging ~hyask/autopkgtest-cloud:skia/switch_web_to_amqp into autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~hyask/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/476752
Big changes in the autopkgtest-cloud web part:
- python3-amqplib -> python3-amqp
- better local development experience
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~hyask/autopkgtest-cloud:skia/switch_web_to_amqp into autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/filter-amqp b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/filter-amqp
index a6f7f09..d70dd69 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/filter-amqp
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/filter-amqp
@@ -1,35 +1,54 @@
#!/usr/bin/python3
# Filter out AMQP requests that match a given regex
+import argparse
import configparser
import logging
-import optparse # pylint: disable=deprecated-module
+import os
import re
import sys
import time
-import urllib.parse
-import amqplib.client_0_8 as amqp
+import amqp
import distro_info
-def filter_amqp(options, host, queue_name, regex):
- url_parts = urllib.parse.urlsplit(host, allow_fragments=False)
- filter_re = re.compile(regex.encode("UTF-8"), re.DOTALL)
- amqp_con = amqp.Connection(
- url_parts.hostname,
- userid=url_parts.username,
- password=url_parts.password,
- )
- ch = amqp_con.channel()
+def get_amqp_channel():
+ try:
+ cp = configparser.ConfigParser()
+ with open("/home/ubuntu/rabbitmq.cred", "r") as f:
+ cp.read_string("[rabbit]\n" + f.read().replace('"', ""))
+ amqp_con = amqp.Connection(
+ cp["rabbit"]["RABBIT_HOST"],
+ cp["rabbit"]["RABBIT_USER"],
+ cp["rabbit"]["RABBIT_PASSWORD"],
+ )
+ except FileNotFoundError:
+ amqp_con = amqp.Connection(
+ os.environ["RABBIT_HOST"],
+ userid=os.environ["RABBIT_USER"],
+ password=os.environ["RABBIT_PASSWORD"],
+ )
+ amqp_con.connect()
+ return amqp_con.channel()
+
+
+def filter_amqp(options, queue_name, regex):
num_items_deleted = 0
+ filter_re = re.compile(regex.encode("UTF-8"), re.DOTALL)
+
+ channel = get_amqp_channel()
while True:
- r = ch.basic_get(queue_name)
+ try:
+ r = channel.basic_get(queue_name)
+ except amqp.NotFound:
+ logging.warning(f"Queue {queue_name} not found")
+ return None
if r is None:
- logging.debug("r is none, exiting")
- ch.close()
- amqp_con.close()
+ logging.info(
+ "Message empty, we probably reached the end of the queue"
+ )
break
if isinstance(r.body, str):
body = r.body.encode("UTF-8")
@@ -45,7 +64,7 @@ def filter_amqp(options, host, queue_name, regex):
logging.info("queue item: %s (would delete)", body)
else:
logging.info("queue item: %s (deleting)", body)
- ch.basic_ack(r.delivery_tag)
+ channel.basic_ack(r.delivery_tag)
num_items_deleted += 1
return num_items_deleted
@@ -69,25 +88,31 @@ def generate_queue_names():
def main():
- parser = optparse.OptionParser(
- usage="usage: %prog [options] queue_name regex\n"
- "Pass `all` for queue_name to filter all queues"
+ parser = argparse.ArgumentParser(
+ description="""Filter queue based on a regex
+
+This script can be used whenever a new upload happens, obsoleting a previous
+one, and that previous upload still had a lot of tests scheduled. To avoid
+processing useless jobs, the queue can be filtered on the trigger of the
+obsolete upload.
+""",
+ formatter_class=argparse.RawTextHelpFormatter,
)
- parser.add_option(
+ parser.add_argument(
"-n",
"--dry-run",
default=False,
action="store_true",
help="only show the operations that would be performed",
)
- parser.add_option(
+ parser.add_argument(
"-v",
"--verbose",
default=False,
action="store_true",
help="additionally show queue items that are not removed",
)
- parser.add_option(
+ parser.add_argument(
"-a",
"--all-items-in-queue",
default=False,
@@ -97,25 +122,21 @@ def main():
"When using this option, the provided regex will be ignored."
),
)
- cp = configparser.ConfigParser()
- with open("/home/ubuntu/rabbitmq.cred", "r") as f:
- cp.read_string("[rabbit]\n" + f.read().replace('"', ""))
- creds = "amqp://%s:%s@%s" % (
- cp["rabbit"]["RABBIT_USER"],
- cp["rabbit"]["RABBIT_PASSWORD"],
- cp["rabbit"]["RABBIT_HOST"],
+ parser.add_argument(
+ "queue_name",
+ help="The name of the queue to filter. `all` is a valid value.",
+ )
+ parser.add_argument(
+ "regex", help="The regex with which to filter the queue"
)
- opts, args = parser.parse_args()
+ opts = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if opts.verbose else logging.INFO,
format="%(asctime)s - %(message)s",
)
- if len(args) != 2:
- parser.error("Need to specify queue name and regex")
-
if opts.all_items_in_queue:
print("""Do you really want to flush this queue? [yN]""", end="")
sys.stdout.flush()
@@ -123,16 +144,25 @@ def main():
if not response.strip().lower().startswith("y"):
print("""Exiting""")
sys.exit(1)
- queues = [args[0]] if args[0] != "all" else generate_queue_names()
+ queues = (
+ [opts.queue_name]
+ if opts.queue_name != "all"
+ else generate_queue_names()
+ )
- deletion_count_history = []
for this_queue in queues:
+ deletion_count_history = []
while True:
- num_deleted = filter_amqp(opts, creds, this_queue, args[1])
+ num_deleted = filter_amqp(opts, this_queue, opts.regex)
+ if num_deleted is None:
+ logging.info("Skipping")
+ break
deletion_count_history.append(num_deleted)
if opts.dry_run:
break
- if all([x == 0 for x in deletion_count_history[-5:]]):
+ if len(deletion_count_history) >= 5 and all(
+ [x == 0 for x in deletion_count_history[-5:]]
+ ):
logging.info(
"Finished filtering queue objects, run history:\n%s"
% "\n".join(str(x) for x in deletion_count_history)
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/filter-amqp-dupes-upstream b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/filter-amqp-dupes-upstream
index c255239..e7441cc 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/filter-amqp-dupes-upstream
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/filter-amqp-dupes-upstream
@@ -1,14 +1,14 @@
#!/usr/bin/python3
# Filter out all but the latest request for a given upstream PR
+import argparse
+import configparser
import json
import logging
-import optparse # pylint: disable=deprecated-module
import os
-import urllib.parse
from collections import defaultdict
-import amqplib.client_0_8 as amqp
+import amqp
import dateutil.parser
import distro_info
@@ -19,13 +19,23 @@ SUPPORTED_UBUNTU_RELEASES = sorted(
)
-def filter_amqp(options, host):
- url_parts = urllib.parse.urlsplit(host, allow_fragments=False)
- amqp_con = amqp.Connection(
- url_parts.hostname,
- userid=url_parts.username,
- password=url_parts.password,
- )
+def filter_amqp(options):
+ try:
+ cp = configparser.ConfigParser()
+ with open("/home/ubuntu/rabbitmq.cred", "r") as f:
+ cp.read_string("[rabbit]\n" + f.read().replace('"', ""))
+ amqp_con = amqp.Connection(
+ cp["rabbit"]["RABBIT_HOST"],
+ cp["rabbit"]["RABBIT_USER"],
+ cp["rabbit"]["RABBIT_PASSWORD"],
+ )
+ except FileNotFoundError:
+ amqp_con = amqp.Connection(
+ os.environ["RABBIT_HOST"],
+ userid=os.environ["RABBIT_USER"],
+ password=os.environ["RABBIT_PASSWORD"],
+ )
+ amqp_con.connect()
dry_run = "[dry-run] " if options.dry_run else ""
queues = (
@@ -40,10 +50,7 @@ def filter_amqp(options, host):
while True:
try:
r = ch.basic_get(queue_name)
- except amqp.AMQPChannelException as e:
- (code, _, _, _) = e.args
- if code != 404:
- raise
+ except amqp.NotFound:
logging.debug(f"No such queue {queue_name}")
break
if r is None:
@@ -52,7 +59,7 @@ def filter_amqp(options, host):
body = r.body.decode("UTF-8")
else:
body = r.body
- (pkg, params) = body.split(" ", 1)
+ (pkg, params) = body.split("\n", 1)
params_j = json.loads(params)
submit_time = dateutil.parser.parse(params_j["submit-time"])
pr = [
@@ -80,37 +87,35 @@ def filter_amqp(options, host):
def main():
- parser = optparse.OptionParser(
- usage="usage: %prog [options] amqp://user:pass@host queue_name regex"
+ parser = argparse.ArgumentParser(
+ description="""Deduplicates jobs in the upstream queue.
+
+The upstream integration is different than regular jobs pushed by Britney.
+If a developer pushes two times in a row on a pull request, then two test
+requests get queued. This script is here to deduplicate those requests.
+""",
+ formatter_class=argparse.RawTextHelpFormatter,
)
- parser.add_option(
- "-n",
+ parser.add_argument(
"--dry-run",
- default=False,
action="store_true",
help="only show the operations that would be performed",
)
- parser.add_option(
+ parser.add_argument(
"-v",
"--verbose",
- default=False,
action="store_true",
help="additionally show queue items that are not removed",
)
- # pylint: disable=unused-variable
- opts, args = parser.parse_args()
+ opts = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if opts.verbose else logging.INFO,
format="%(asctime)s - %(message)s",
)
- user = os.environ["RABBIT_USER"]
- password = os.environ["RABBIT_PASSWORD"]
- host = os.environ["RABBIT_HOST"]
- uri = f"amqp://{user}:{password}@{host}"
- filter_amqp(opts, uri)
+ filter_amqp(opts)
if __name__ == "__main__":
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/pull-amqp b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/pull-amqp
index cdda67a..fbd2092 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/pull-amqp
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/pull-amqp
@@ -2,10 +2,11 @@
import argparse
import configparser
+import os
import re
import sys
-import amqplib.client_0_8 as amqp
+import amqp
def parse_args():
@@ -45,14 +46,22 @@ You can alter the queue messages however you please, but be careful :)
def main():
args = parse_args()
- cp = configparser.ConfigParser()
- with open("/home/ubuntu/rabbitmq.cred", "r") as f:
- cp.read_string("[rabbit]\n" + f.read().replace('"', ""))
- amqp_con = amqp.Connection(
- cp["rabbit"]["RABBIT_HOST"],
- cp["rabbit"]["RABBIT_USER"],
- cp["rabbit"]["RABBIT_PASSWORD"],
- )
+ try:
+ cp = configparser.ConfigParser()
+ with open("/home/ubuntu/rabbitmq.cred", "r") as f:
+ cp.read_string("[rabbit]\n" + f.read().replace('"', ""))
+ amqp_con = amqp.Connection(
+ cp["rabbit"]["RABBIT_HOST"],
+ cp["rabbit"]["RABBIT_USER"],
+ cp["rabbit"]["RABBIT_PASSWORD"],
+ )
+ except FileNotFoundError:
+ amqp_con = amqp.Connection(
+ os.environ["RABBIT_HOST"],
+ userid=os.environ["RABBIT_USER"],
+ password=os.environ["RABBIT_PASSWORD"],
+ )
+ amqp_con.connect()
if args.regex is not None:
filter_re = re.compile(args.regex.encode("UTF-8"), re.DOTALL)
with amqp_con.channel() as ch:
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/push-amqp b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/push-amqp
index 80f70e2..10d94a1 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/push-amqp
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/push-amqp
@@ -3,9 +3,10 @@
import argparse
import ast
import configparser
+import os
import sys
-import amqplib.client_0_8 as amqp
+import amqp
def parse_args():
@@ -68,14 +69,22 @@ def main():
file=sys.stderr,
)
- cp = configparser.ConfigParser()
- with open("/home/ubuntu/rabbitmq.cred", "r") as f:
- cp.read_string("[rabbit]\n" + f.read().replace('"', ""))
- amqp_con = amqp.Connection(
- cp["rabbit"]["RABBIT_HOST"],
- cp["rabbit"]["RABBIT_USER"],
- cp["rabbit"]["RABBIT_PASSWORD"],
- )
+ try:
+ cp = configparser.ConfigParser()
+ with open("/home/ubuntu/rabbitmq.cred", "r") as f:
+ cp.read_string("[rabbit]\n" + f.read().replace('"', ""))
+ amqp_con = amqp.Connection(
+ cp["rabbit"]["RABBIT_HOST"],
+ cp["rabbit"]["RABBIT_USER"],
+ cp["rabbit"]["RABBIT_PASSWORD"],
+ )
+ except FileNotFoundError:
+ amqp_con = amqp.Connection(
+ os.environ["RABBIT_HOST"],
+ userid=os.environ["RABBIT_USER"],
+ password=os.environ["RABBIT_PASSWORD"],
+ )
+ amqp_con.connect()
ch = amqp_con.channel()
queue_name = args.queue_name
if args.message:
@@ -103,10 +112,7 @@ def main():
continue
try:
push(message, queue_name, ch)
- except (
- amqp.AMQPChannelException,
- amqp.AMQPConnectionException,
- ) as _:
+ except amqp.AMQPError:
print(
f"Pushing message `{message}` to queue {queue_name} failed.",
file=sys.stderr,
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/run-autopkgtest b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/run-autopkgtest
index 3dc8326..7a85403 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/run-autopkgtest
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/run-autopkgtest
@@ -9,9 +9,9 @@ import os
import sys
import urllib.parse
import uuid
-from datetime import datetime
+from datetime import datetime, timezone
-import amqplib.client_0_8 as amqp
+import amqp
my_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
@@ -172,7 +172,7 @@ if __name__ == "__main__":
if args.readable_by:
params["readable-by"] = args.readable_by
if args.all_proposed:
- params["all-proposed"] = True
+ params["all-proposed"] = "1"
if args.requester:
params["requester"] = args.requester
else:
@@ -181,7 +181,7 @@ if __name__ == "__main__":
except KeyError:
pass
params["submit-time"] = datetime.strftime(
- datetime.utcnow(), "%Y-%m-%d %H:%M:%S%z"
+ datetime.now().astimezone(timezone.utc), "%Y-%m-%d %H:%M:%S%z"
)
params["uuid"] = str(uuid.uuid4())
params = "\n" + json.dumps(params)
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/with-distributed-lock b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/with-distributed-lock
index 442e914..aaca3d2 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/with-distributed-lock
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/with-distributed-lock
@@ -15,7 +15,7 @@ import os
import subprocess
import sys
-import amqplib.client_0_8 as amqp
+import amqp
@contextlib.contextmanager
@@ -33,15 +33,14 @@ def amqp_lock(name):
userid=os.environ["RABBIT_USER"],
password=os.environ["RABBIT_PASSWORD"],
)
+ amqp_con.connect()
channel = amqp_con.channel()
- channel.queue_declare(
- name, arguments={"args.queue.x-single-active-consumer": True}
- )
+ channel.queue_declare(name, arguments={"x-single-active-consumer": True})
channel.basic_publish(amqp.Message(""), routing_key=name)
consumer_tag = channel.basic_consume(queue=name, callback=callback)
while channel.callbacks and not callback.called:
- channel.wait()
+ amqp_con.drain_events()
try:
yield
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
index 758ac0e..99203fd 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
@@ -2,7 +2,7 @@
# autopkgtest cloud worker
# Author: Martin Pitt <martin.pitt@xxxxxxxxxx>
#
-# Requirements: python3-amqplib python3-swiftclient python3-influxdb
+# Requirements: python3-amqp python3-swiftclient python3-influxdb
# Requirements for running autopkgtest from git: python3-debian libdpkg-perl
#
# pylint: disable=too-many-lines,line-too-long
@@ -27,7 +27,7 @@ import urllib.request
import uuid
from urllib.error import HTTPError
-import amqplib.client_0_8 as amqp
+import amqp
import distro_info
import novaclient.client
import novaclient.exceptions
@@ -562,7 +562,6 @@ def call_autopkgtest(
# set up status AMQP exchange
global amqp_con
status_amqp = amqp_con.channel()
- status_amqp.access_request("/data", active=True, read=False, write=True)
status_amqp.exchange_declare(
status_exchange_name, "fanout", durable=False, auto_delete=True
)
@@ -1576,9 +1575,6 @@ def request(msg):
global amqp_con
complete_amqp = amqp_con.channel()
- complete_amqp.access_request(
- "/complete", active=True, read=False, write=True
- )
complete_amqp.exchange_declare(
complete_exchange_name, "fanout", durable=True, auto_delete=False
)
@@ -1629,6 +1625,7 @@ def amqp_connect(cfg, callback):
password=os.environ["RABBIT_PASSWORD"],
confirm_publish=True,
)
+ amqp_con.connect()
queue = amqp_con.channel()
# avoids greedy grabbing of the entire queue while being too busy
queue.basic_qos(0, 1, True)
@@ -1665,7 +1662,7 @@ def amqp_connect(cfg, callback):
queue.queue_declare(queue_name, durable=True, auto_delete=False)
queue.basic_consume(queue=queue_name, callback=request)
- return queue
+ return amqp_con
def main():
@@ -1740,13 +1737,13 @@ def main():
swiftclient.Connection(**swift_creds).close()
# connect to AMQP queues
- queue = amqp_connect(cfg, request)
+ amqp_con = amqp_connect(cfg, request)
# process queues forever
try:
while exit_requested is None:
logging.info("Waiting for and processing AMQP requests")
- queue.wait()
+ amqp_con.drain_events()
except IOError:
if exit_requested is None:
raise
diff --git a/charms/focal/autopkgtest-cloud-worker/layer.yaml b/charms/focal/autopkgtest-cloud-worker/layer.yaml
index d8f8c0a..fd41b99 100644
--- a/charms/focal/autopkgtest-cloud-worker/layer.yaml
+++ b/charms/focal/autopkgtest-cloud-worker/layer.yaml
@@ -18,7 +18,7 @@ options:
- libdpkg-perl
- lxd-client
- make
- - python3-amqplib
+ - python3-amqp
- python3-debian
- python3-distro-info
- python3-glanceclient
diff --git a/charms/focal/autopkgtest-web/config.yaml b/charms/focal/autopkgtest-web/config.yaml
index 1cab2a2..0d1d94f 100644
--- a/charms/focal/autopkgtest-web/config.yaml
+++ b/charms/focal/autopkgtest-web/config.yaml
@@ -15,6 +15,10 @@ options:
type: string
default: autopkgtest.local
description: "Public host name of this web server"
+ archive-url:
+ type: string
+ default:
+ description: "URL of the Ubuntu archive, in case you want to use a mirror"
github-secrets:
type: string
default:
@@ -30,10 +34,6 @@ options:
default:
description: "project:user:token github credentials \
for POSTing to statuses_url"
- swift-web-credentials:
- type: string
- default:
- description: "SWIFT login credentials for the private-result web frontend"
https-proxy:
type: string
default:
@@ -43,12 +43,6 @@ options:
default:
description: "$no_proxy environment variable (for accessing sites \
other than github, like login.ubuntu.com and launchpad.net)"
- cookies:
- type: string
- default:
- description: "SRVNAME cookies for each autopkgtest-web unit. \
- Each web unit has a cookie assigned which tells the \
- haproxy server which web unit to redirect a request to."
indexed-packages-fp:
type: string
default:
diff --git a/charms/focal/autopkgtest-web/layer.yaml b/charms/focal/autopkgtest-web/layer.yaml
index 529b3c6..471439d 100644
--- a/charms/focal/autopkgtest-web/layer.yaml
+++ b/charms/focal/autopkgtest-web/layer.yaml
@@ -15,7 +15,7 @@ options:
- libjs-bootstrap
- libjs-jquery
- make
- - python3-amqplib
+ - python3-amqp
- python3-distro-info
- python3-flask
- python3-flask-openid
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index 2637e2c..282fae1 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -35,17 +35,7 @@ GITHUB_SECRETS_PATH = os.path.expanduser("~ubuntu/github-secrets.json")
GITHUB_STATUS_CREDENTIALS_PATH = os.path.expanduser(
"~ubuntu/github-status-credentials.txt"
)
-SWIFT_WEB_CREDENTIALS_PATH = os.path.expanduser(
- "~ubuntu/swift-web-credentials.conf"
-)
API_KEYS_PATH = "/home/ubuntu/external-web-requests-api-keys.json"
-CONFIG_DIR = pathlib.Path("/home/ubuntu/.config/autopkgtest-web/")
-if not CONFIG_DIR.exists():
- set_flag("autopkgtest-web.config-needs-writing")
-for parent in reversed(CONFIG_DIR.parents):
- parent.mkdir(mode=0o775, exist_ok=True)
-CONFIG_DIR.mkdir(mode=0o775, exist_ok=True)
-ALLOWED_REQUESTOR_TEAMS_PATH = CONFIG_DIR / "allowed-requestor-teams"
PUBLIC_SWIFT_CREDS_PATH = os.path.expanduser("~ubuntu/public-swift-creds")
@@ -128,14 +118,26 @@ def setup_rabbitmq(rabbitmq):
"amqp.available",
"config.set.storage-url-internal",
"config.set.hostname",
- "config.set.cookies",
"config.set.indexed-packages-fp",
+ "config.set.archive-url",
+ "config.set.public-swift-creds",
+ "config.set.allowed-requestor-teams",
)
def write_autopkgtest_cloud_conf(rabbitmq):
status.maintenance("Writing autopkgtest-cloud config")
swiftinternal = config().get("storage-url-internal")
hostname = config().get("hostname")
- cookies = config().get("cookies")
+ allowed_requestor_teams = ",".join(
+ config().get("allowed-requestor-teams").split("\n")
+ )
+ public_swift_creds = {
+ key.lower(): value
+ for key, value in [
+ line.split("=")
+ for line in config().get("public-swift-creds").strip().split("\n")
+ ]
+ }
+ archive_url = config().get("archive-url")
rabbituser = rabbitmq.username()
rabbitpassword = rabbitmq.password()
rabbithost = rabbitmq.private_address()
@@ -144,19 +146,31 @@ def write_autopkgtest_cloud_conf(rabbitmq):
with open(f"{AUTOPKGTEST_CLOUD_CONF}.new", "w") as f:
f.write(
dedent(
- """\
+ f"""\
[web]
database=/home/ubuntu/autopkgtest.db
database_ro=/home/ubuntu/public/autopkgtest.db
SwiftURL={swiftinternal}
ExternalURL=https://{hostname}/results
- cookies={cookies}
indexed_packages_fp={indexed_packages_fp}
+ archive_url={archive_url}
+ amqp_queue_cache=/home/ubuntu/queued.json
+ running_cache=/home/ubuntu/running.json
+ allowed_requestors={allowed_requestor_teams}
[amqp]
- uri=amqp://{rabbituser}:{rabbitpassword}@{rabbithost}""".format(
- **locals()
- )
+ uri=amqp://{rabbituser}:{rabbitpassword}@{rabbithost}
+
+ [swift]
+ os_region_name = {public_swift_creds['os_region_name']}
+ os_auth_url = {public_swift_creds['os_auth_url']}
+ os_project_domain_name = {public_swift_creds['os_project_domain_name']}
+ os_username = {public_swift_creds['os_username']}
+ os_user_domain_name = {public_swift_creds['os_user_domain_name']}
+ os_project_name = {public_swift_creds['os_project_name']}
+ os_password = {public_swift_creds['os_password']}
+ os_identity_api_version = {public_swift_creds['os_identity_api_version']}
+ """
)
)
os.rename(f"{AUTOPKGTEST_CLOUD_CONF}.new", AUTOPKGTEST_CLOUD_CONF)
@@ -345,19 +359,7 @@ def set_up_web_config(apache):
@when_any(
- "config.changed.allowed-requestor-teams",
- "config.set.allowed-requestor-teams",
- "autopkgtest-web.config-needs-writing",
-)
-def write_allowed_teams():
- allowed_requestor_teams = config().get("allowed-requestor-teams")
- allowed_teams_path = pathlib.Path(ALLOWED_REQUESTOR_TEAMS_PATH)
- allowed_teams_path.write_text(allowed_requestor_teams, encoding="utf-8")
-
-
-@when_any(
"config.changed.github-secrets",
- "autopkgtest-web.config-needs-writing",
)
def write_github_secrets():
status.maintenance("Writing github secrets")
@@ -407,26 +409,6 @@ def clear_github_secrets():
status.maintenance("Done clearing github secrets")
-@when_all(
- "config.changed.swift-web-credentials", "config.set.swift-web-credentials"
-)
-def write_swift_web_credentials():
- status.maintenance("Writing swift web credentials")
- swift_credentials = config().get("swift-web-credentials")
-
- with open(SWIFT_WEB_CREDENTIALS_PATH, "w") as f:
- f.write(swift_credentials)
-
- try:
- os.symlink(
- SWIFT_WEB_CREDENTIALS_PATH,
- os.path.expanduser("~www-data/swift-web-credentials.conf"),
- )
- except FileExistsError:
- pass
- status.maintenance("Done writing swift web credentials")
-
-
@when_all("config.changed.public-swift-creds", "config.set.public-swift-creds")
def write_openstack_creds():
status.maintenance("Writing openstack credentials")
@@ -437,21 +419,6 @@ def write_openstack_creds():
status.maintenance("Done writing openstack credentials")
-@when_not("config.set.swift-web-credentials")
-def clear_swift_web_credentials():
- status.maintenance("Clearing swift web credentials")
- try:
- os.unlink(SWIFT_WEB_CREDENTIALS_PATH)
- except FileNotFoundError:
- pass
-
- try:
- os.unlink(os.path.expanduser("~www-data/swift-web-credentials.conf"))
- except FileNotFoundError:
- pass
- status.maintenance("Done clearing swift web credentials")
-
-
@when_all(
"config.changed.github-status-credentials",
"config.set.github-status-credentials",
@@ -546,9 +513,7 @@ def symlink_running():
status.maintenance("Creating symlink to running.json")
try:
os.symlink(
- os.path.join(
- os.path.sep, "run", "amqp-status-collector", "running.json"
- ),
+ os.path.join(os.path.expanduser("~ubuntu"), "running.json"),
os.path.join(
os.path.expanduser("~ubuntu"),
"webcontrol",
@@ -591,13 +556,10 @@ def symlink_public_db():
symlink_file,
),
)
- set_flag(symlink_flag)
status.active(f"Done creating symlink for {symlink_file}")
except FileExistsError:
- clear_flag(symlink_flag)
- status.active(
- "symlinking public db and sha256 checksum already done"
- )
+ status.active(f"Symlink {symlink_file} already exists")
+ set_flag(symlink_flag)
@when("leadership.is_leader")
diff --git a/charms/focal/autopkgtest-web/units/amqp-status-collector.service b/charms/focal/autopkgtest-web/units/amqp-status-collector.service
index bfb92ae..5b98105 100644
--- a/charms/focal/autopkgtest-web/units/amqp-status-collector.service
+++ b/charms/focal/autopkgtest-web/units/amqp-status-collector.service
@@ -4,6 +4,7 @@ StartLimitIntervalSec=60s
StartLimitBurst=60
[Service]
+User=ubuntu
RuntimeDirectory=amqp-status-collector
RuntimeDirectoryPreserve=yes
ExecStart=/home/ubuntu/webcontrol/amqp-status-collector
diff --git a/charms/focal/autopkgtest-web/units/db-backup.service b/charms/focal/autopkgtest-web/units/db-backup.service
index 9c3d038..29f4763 100644
--- a/charms/focal/autopkgtest-web/units/db-backup.service
+++ b/charms/focal/autopkgtest-web/units/db-backup.service
@@ -4,5 +4,4 @@ Description=Backup sql database
[Service]
Type=oneshot
User=ubuntu
-EnvironmentFile=/home/ubuntu/public-swift-creds
ExecStart=/home/ubuntu/webcontrol/db-backup
diff --git a/charms/focal/autopkgtest-web/units/indexed-packages.service b/charms/focal/autopkgtest-web/units/indexed-packages.service
index e518740..858ff0f 100644
--- a/charms/focal/autopkgtest-web/units/indexed-packages.service
+++ b/charms/focal/autopkgtest-web/units/indexed-packages.service
@@ -2,5 +2,6 @@
Description=Get index of packages
[Service]
+User=ubuntu
Type=oneshot
ExecStart=/home/ubuntu/webcontrol/indexed-packages
diff --git a/charms/focal/autopkgtest-web/units/publish-db.service b/charms/focal/autopkgtest-web/units/publish-db.service
index 1c9c282..3a5728c 100644
--- a/charms/focal/autopkgtest-web/units/publish-db.service
+++ b/charms/focal/autopkgtest-web/units/publish-db.service
@@ -2,6 +2,7 @@
Description=publish autopkgtest.db
[Service]
+User=ubuntu
Type=oneshot
ExecStart=/home/ubuntu/webcontrol/publish-db
TimeoutSec=300
diff --git a/charms/focal/autopkgtest-web/units/sqlite-writer.service b/charms/focal/autopkgtest-web/units/sqlite-writer.service
index cf3b48e..3a47c08 100644
--- a/charms/focal/autopkgtest-web/units/sqlite-writer.service
+++ b/charms/focal/autopkgtest-web/units/sqlite-writer.service
@@ -5,7 +5,6 @@ StartLimitBurst=60
[Service]
User=ubuntu
-EnvironmentFile=/home/ubuntu/public-swift-creds
ExecStart=/home/ubuntu/webcontrol/sqlite-writer
Restart=on-failure
RestartSec=1s
diff --git a/charms/focal/autopkgtest-web/units/update-github-jobs.service b/charms/focal/autopkgtest-web/units/update-github-jobs.service
index 90de852..15627b7 100644
--- a/charms/focal/autopkgtest-web/units/update-github-jobs.service
+++ b/charms/focal/autopkgtest-web/units/update-github-jobs.service
@@ -2,8 +2,7 @@
Description=Update GitHub job status
[Service]
+User=ubuntu
Type=oneshot
-User=www-data
-Group=www-data
TimeoutStartSec=10m
ExecStart=/home/ubuntu/webcontrol/update-github-jobs
diff --git a/charms/focal/autopkgtest-web/webcontrol/.gitignore b/charms/focal/autopkgtest-web/webcontrol/.gitignore
new file mode 100644
index 0000000..dcdfe9b
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/.gitignore
@@ -0,0 +1 @@
+autopkgtest-cloud.conf
diff --git a/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector b/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector
index 1169f2e..ae2226c 100755
--- a/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector
+++ b/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector
@@ -7,15 +7,12 @@ import logging
import os
import socket
import time
-import urllib.parse
-import amqplib.client_0_8 as amqp
-from helpers.utils import get_autopkgtest_cloud_conf
+from helpers.utils import amqp_connect, get_autopkgtest_cloud_conf
exchange_name = "teststatus.fanout"
-running_name = os.path.join(
- os.path.sep, "run", "amqp-status-collector", "running.json"
-)
+cp = get_autopkgtest_cloud_conf()
+running_name = cp["web"]["running_cache"]
running_name_new = "{}.new".format(running_name)
# package -> runhash -> release -> arch -> (params, duration, logtail)
@@ -23,22 +20,6 @@ running_tests = {}
last_update = 0
-def amqp_connect():
- """Connect to AMQP server"""
-
- cp = get_autopkgtest_cloud_conf()
- amqp_uri = cp["amqp"]["uri"]
- parts = urllib.parse.urlsplit(amqp_uri, allow_fragments=False)
- amqp_con = amqp.Connection(
- parts.hostname, userid=parts.username, password=parts.password
- )
- logging.info(
- "Connected to AMQP server at %s@%s" % (parts.username, parts.hostname)
- )
-
- return amqp_con
-
-
def update_output(amqp_channel, force_update=False):
"""Update report"""
@@ -100,12 +81,11 @@ def process_message(msg):
#
logging.basicConfig(
- level=("DEBUG" in os.environ and logging.DEBUG or logging.INFO)
+ level=(logging.DEBUG if "DEBUG" in os.environ else logging.INFO)
)
amqp_con = amqp_connect()
status_ch = amqp_con.channel()
-status_ch.access_request("/data", active=True, read=True, write=False)
status_ch.exchange_declare(
exchange_name, "fanout", durable=False, auto_delete=True
)
@@ -116,4 +96,4 @@ status_ch.queue_bind(queue_name, exchange_name, queue_name)
logging.info("Listening to requests on %s" % queue_name)
status_ch.basic_consume("", callback=process_message, no_ack=True)
while status_ch.callbacks:
- status_ch.wait()
+ amqp_con.drain_events()
diff --git a/charms/focal/autopkgtest-web/webcontrol/autopkgtest-cloud.conf.example b/charms/focal/autopkgtest-web/webcontrol/autopkgtest-cloud.conf.example
new file mode 100644
index 0000000..8cfd2b4
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/autopkgtest-cloud.conf.example
@@ -0,0 +1,28 @@
+# For local development copy me to `./autopkgtest-cloud.conf`
+# All scripts in that folder should use that by default, provided you don't have
+# a `~/autopkgtest-cloud.conf` file existing.
+[web]
+database=/tmp/autopkgtest-cloud-web/autopkgtest.db
+database_ro=/tmp/autopkgtest-cloud-web/public/autopkgtest.db
+ExternalURL=http://127.0.0.1:5000/results
+indexed_packages_fp=/tmp/autopkgtest-cloud-web/indexed-packages.json
+amqp_queue_cache=/tmp/autopkgtest-cloud-web/queued.json
+running_cache=/tmp/autopkgtest-cloud-web/running.json
+archive_url=http://archive.ubuntu.com/ubuntu
+allowed_requestors=autopkgtest-requestors,canonical-ubuntu-qa
+
+[amqp]
+uri=amqp://guest:guest@127.0.0.1
+
+[swift]
+os_region_name = canonistack-bos01
+os_auth_url = https://keystone.bos01.canonistack.canonical.com:5000/v3
+os_project_domain_name = default
+# change this
+os_username = user
+os_user_domain_name = default
+# change this
+os_project_name = user_project
+# change this
+os_password = complicatedpassword
+os_identity_api_version = 3
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse-test.py b/charms/focal/autopkgtest-web/webcontrol/browse-test.py
index a01d944..72a5f44 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse-test.py
+++ b/charms/focal/autopkgtest-web/webcontrol/browse-test.py
@@ -56,40 +56,37 @@ def parse_args():
if __name__ == "__main__":
args = parse_args()
+ browse.init_config()
+ config = utils.get_autopkgtest_cloud_conf()
+
if args.data_dir:
- browse.AMQP_QUEUE_CACHE = Path(args.data_dir + "/queued.json")
- browse.RUNNING_CACHE = Path(args.data_dir + "/running.json")
- browse.db_con = utils.init_db(
- args.data_dir + "/autopkgtest.db",
- check_same_thread=False,
- )
+ browse.CONFIG["amqp_queue_cache"] = Path(args.data_dir) / "queued.json"
+ browse.CONFIG["running_cache"] = Path(args.data_dir) / "running.json"
+ browse.CONFIG["database"] = args.data_dir + "/autopkgtest.db"
else:
if args.database:
- browse.db_con = utils.init_db(
- args.database,
- check_same_thread=False,
- )
+ browse.CONFIG["database"] = args.database
else:
- browse.db_con = utils.init_db(
- ":memory:",
- check_same_thread=False,
- )
- with browse.db_con:
- tests.populate_dummy_db(browse.db_con)
+ # For convenience, the development Flask app uses database instead
+ # of database_ro.
+ # This is different from production deployment, where `publish-db`
+ # produces database_ro, that browse.cgi uses.
+ browse.CONFIG["database"] = config["web"]["database"]
if args.queue:
- browse.AMQP_QUEUE_CACHE = Path(args.queue)
+ browse.CONFIG["amqp_queue_cache"] = Path(args.queue)
else:
- browse.AMQP_QUEUE_CACHE = Path("/dev/shm/queue.json")
- tests.populate_dummy_amqp_cache(browse.AMQP_QUEUE_CACHE)
+ tests.populate_dummy_amqp_cache(browse.CONFIG["amqp_queue_cache"])
if args.running:
- browse.RUNNING_CACHE = Path(args.running)
+ browse.CONFIG["running_cache"] = Path(args.running)
else:
- browse.RUNNING_CACHE = Path("/dev/shm/running.json")
- tests.populate_dummy_running_cache(browse.RUNNING_CACHE)
+ tests.populate_dummy_running_cache(browse.CONFIG["running_cache"])
- browse.swift_container_url = "swift-%s"
+ browse.connect_db(browse.CONFIG["database"])
+ if utils.is_db_empty(browse.db_con):
+ with browse.db_con:
+ tests.populate_dummy_db(browse.db_con)
if activate_debugtoolbar:
browse.app.debug = True
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index e942f7c..7c13054 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -11,6 +11,7 @@ import sqlite3
import sys
import traceback
from collections import OrderedDict
+from pathlib import Path
from wsgiref.handlers import CGIHandler
import flask
@@ -25,6 +26,7 @@ from helpers.utils import (
get_all_releases,
get_autopkgtest_cloud_conf,
get_supported_releases,
+ init_db,
setup_key,
)
from werkzeug.middleware.proxy_fix import ProxyFix
@@ -44,31 +46,35 @@ secret_path = os.path.join(PATH, "secret_key")
setup_key(app, secret_path)
db_con = None
-swift_container_url = None
+CONFIG = {}
ALL_UBUNTU_RELEASES = get_all_releases()
SUPPORTED_UBUNTU_RELEASES = get_supported_releases()
-INDEXED_PACKAGES_FP = ""
-AMQP_QUEUE_CACHE = "/var/lib/cache-amqp/queued.json"
-RUNNING_CACHE = "/run/amqp-status-collector/running.json"
def init_config():
- global db_con, swift_container_url, INDEXED_PACKAGES_FP
+ global CONFIG
cp = get_autopkgtest_cloud_conf()
- db_con = sqlite3.connect(
- "file:%s?mode=ro" % cp["web"]["database_ro"],
- uri=True,
- check_same_thread=False,
- )
try:
- url = cp["web"]["ExternalURL"]
+ CONFIG["swift_container_url"] = (
+ cp["web"]["ExternalURL"] + "/autopkgtest-%s"
+ )
except KeyError:
- url = cp["web"]["SwiftURL"]
- INDEXED_PACKAGES_FP = cp["web"]["indexed_packages_fp"]
- swift_container_url = os.path.join(url, "autopkgtest-%s")
+ CONFIG["swift_container_url"] = (
+ cp["web"]["SwiftURL"] + "/autopkgtest-%s"
+ )
+ CONFIG["indexed_packages"] = Path(cp["web"]["indexed_packages_fp"])
+ CONFIG["amqp_queue_cache"] = Path(cp["web"]["amqp_queue_cache"])
+ CONFIG["running_cache"] = Path(cp["web"]["running_cache"])
+ CONFIG["database"] = Path(cp["web"]["database_ro"])
+
+
+def connect_db(path: str):
+ global db_con
+
+ db_con = init_db(path, uri=True, check_same_thread=False)
def get_package_release_arch(test_id):
@@ -101,7 +107,7 @@ def get_test_id(release, arch, src):
def get_running_jobs():
try:
- with open(RUNNING_CACHE) as f:
+ with open(CONFIG["running_cache"]) as f:
# package -> runhash -> release -> arch -> (params, duration, logtail)
return json.load(f)
except FileNotFoundError as e:
@@ -197,7 +203,7 @@ def get_queues_info():
Return (releases, arches, context -> release -> arch -> (queue_size, [requests])).
"""
- with open(AMQP_QUEUE_CACHE, "r") as json_file:
+ with open(CONFIG["amqp_queue_cache"], "r") as json_file:
queue_info_j = json.load(json_file)
arches = queue_info_j["arches"]
@@ -317,7 +323,7 @@ def get_results_for_user(user: str, limit: int, offset: int) -> list:
code = human_exitcode(row["exitcode"])
package, release, arch = get_package_release_arch(test_id)
url = os.path.join(
- swift_container_url % release,
+ CONFIG["swift_container_url"] % release,
release,
arch,
srchash(package),
@@ -635,7 +641,7 @@ def package_release_arch(package, release, arch, _=None):
show_retry = code != "pass" and identifier not in seen
seen.add(identifier)
url = os.path.join(
- swift_container_url % release,
+ CONFIG["swift_container_url"] % release,
release,
arch,
srchash(package),
@@ -716,9 +722,11 @@ def package_release_arch(package, release, arch, _=None):
(
"N/A",
item_info.get("triggers"),
- "all-proposed=1"
- if "all-proposed" in item_info.keys()
- else "",
+ (
+ "all-proposed=1"
+ if "all-proposed" in item_info.keys()
+ else ""
+ ),
human_date(item_info.get("submit-time")),
"N/A",
"-",
@@ -788,7 +796,7 @@ def get_by_uuid(uuid):
show_retry = code != "pass"
url = os.path.join(
- swift_container_url % release,
+ CONFIG["swift_container_url"] % release,
release,
arch,
srchash(package),
@@ -850,7 +858,7 @@ def release(release, arch=None):
# Version + triggers uniquely identifies this result
show_retry = code != "pass"
url = os.path.join(
- swift_container_url % release,
+ CONFIG["swift_container_url"] % release,
release,
run_arch,
srchash(package),
@@ -954,14 +962,16 @@ def queues_json():
@app.route("/queued.json")
def return_queued_exactly():
- return flask.send_file(AMQP_QUEUE_CACHE, mimetype="application/json")
+ return flask.send_file(
+ CONFIG["amqp_queue_cache"], mimetype="application/json"
+ )
@app.route("/testlist")
def testlist():
indexed_pkgs = {}
try:
- with open(INDEXED_PACKAGES_FP, "r") as f:
+ with open(CONFIG["indexed_packages"], "r") as f:
indexed_pkgs = json.load(f)
except FileNotFoundError:
indexed_pkgs = {}
@@ -1061,4 +1071,5 @@ if __name__ == "__main__":
app.config["DEBUG"] = True
init_config()
+ connect_db("file:%s?mode=ro" % CONFIG["database"])
CGIHandler().run(app)
diff --git a/charms/focal/autopkgtest-web/webcontrol/cache-amqp b/charms/focal/autopkgtest-web/webcontrol/cache-amqp
index e953c9d..aefae4b 100755
--- a/charms/focal/autopkgtest-web/webcontrol/cache-amqp
+++ b/charms/focal/autopkgtest-web/webcontrol/cache-amqp
@@ -10,9 +10,9 @@ import tempfile
import time
import urllib.parse
-import amqplib.client_0_8 as amqp
-from amqplib.client_0_8.exceptions import AMQPChannelException
-from helpers.utils import get_autopkgtest_cloud_conf, is_db_empty
+import amqp
+from amqp.exceptions import AMQPError
+from helpers.utils import amqp_connect, get_autopkgtest_cloud_conf, is_db_empty
AMQP_CONTEXTS = ["ubuntu", "huge", "ppa", "upstream"]
@@ -29,9 +29,7 @@ class AutopkgtestQueueContents:
# connect to AMQP
parts = urllib.parse.urlsplit(self.amqp_uri, allow_fragments=False)
- self.amqp_con = amqp.Connection(
- parts.hostname, userid=parts.username, password=parts.password
- )
+ self.amqp_con = amqp_connect()
self.amqp_channel = self.amqp_con.channel()
logger.info("Connected to AMQP host %s", parts.hostname)
@@ -50,8 +48,8 @@ class AutopkgtestQueueContents:
queue_name, durable=True, passive=True
)
logger.info(f"Semaphore queue '{queue_name}' exists")
- except AMQPChannelException as e:
- (code, _, _, _) = e.args
+ except AMQPError as e:
+ code = e.code
if code != 404:
raise
if os.path.exists("/run/autopkgtest-web-is-leader"):
@@ -182,8 +180,8 @@ class AutopkgtestQueueContents:
)
try:
requests = self.get_queue_requests(queue_name)
- except AMQPChannelException as e:
- (code, _, _, _) = e.args
+ except AMQPError as e:
+ code = e.code
if code != 404:
raise
requests = []
@@ -217,10 +215,9 @@ class AutopkgtestQueueContents:
if __name__ == "__main__":
- try:
- state_directory = os.environ["STATE_DIRECTORY"]
- except KeyError:
- state_directory = "/var/lib/cache-amqp/"
+ cp = get_autopkgtest_cloud_conf()
+
+ state_directory = cp["web"]["amqp_queue_cache"]
parser = argparse.ArgumentParser(
description="Fetch AMQP queues into a file"
@@ -244,13 +241,11 @@ if __name__ == "__main__":
"--output",
dest="output",
type=str,
- default=os.path.join(state_directory, "queued.json"),
+ default=state_directory,
)
args = parser.parse_args()
- cp = get_autopkgtest_cloud_conf()
-
formatter = logging.Formatter(
"%(asctime)s: %(message)s", "%Y-%m-%d %H:%M:%S"
)
diff --git a/charms/focal/autopkgtest-web/webcontrol/db-backup b/charms/focal/autopkgtest-web/webcontrol/db-backup
index 16cf7ee..f2bbba3 100755
--- a/charms/focal/autopkgtest-web/webcontrol/db-backup
+++ b/charms/focal/autopkgtest-web/webcontrol/db-backup
@@ -17,7 +17,7 @@ import swiftclient
from helpers.utils import (
get_autopkgtest_cloud_conf,
init_db,
- init_swift_con,
+ swift_connect,
zstd_compress,
)
@@ -104,7 +104,7 @@ def upload_backup_to_swift(
"Retry %i out of %i failed, exception: %s"
% (retry, SWIFT_RETRIES, str(e))
)
- swift_conn = init_swift_con()
+ swift_conn = swift_connect()
return swift_conn
@@ -135,7 +135,7 @@ def delete_old_backups(
"Retry %i out of %i failed, exception: %s"
% (retry, SWIFT_RETRIES, str(e))
)
- swift_conn = init_swift_con()
+ swift_conn = swift_connect()
return swift_conn
@@ -160,7 +160,7 @@ if __name__ == "__main__":
logging.info("Registering cleanup function")
atexit.register(cleanup)
logging.info("Setting up swift connection")
- swift_conn = init_swift_con()
+ swift_conn = swift_connect()
create_container_if_it_doesnt_exist(swift_conn)
logging.info("Uploading db to swift!")
swift_conn = upload_backup_to_swift(swift_conn)
diff --git a/charms/focal/autopkgtest-web/webcontrol/download-all-results b/charms/focal/autopkgtest-web/webcontrol/download-all-results
index dcfacf5..e9ce8ed 100755
--- a/charms/focal/autopkgtest-web/webcontrol/download-all-results
+++ b/charms/focal/autopkgtest-web/webcontrol/download-all-results
@@ -10,10 +10,8 @@
# notification of completed jobs, in case of bugs or network outages etc, this
# script can be used to find any results which were missed and insert them.
-import configparser
import datetime
import io
-import itertools
import json
import logging
import os
@@ -21,38 +19,24 @@ import sqlite3
import sys
import tarfile
import time
-import urllib.parse
-import amqplib.client_0_8 as amqp
+import amqp
import swiftclient
from distro_info import UbuntuDistroInfo
-from helpers.utils import SqliteWriterConfig, get_autopkgtest_cloud_conf
+from helpers.utils import (
+ SqliteWriterConfig,
+ amqp_connect,
+ get_db_path,
+ swift_connect,
+)
LOGGER = logging.getLogger(__name__)
-SWIFT_CREDS_FILE = "/home/ubuntu/public-swift-creds"
config = None
db_con = None
amqp_con = None
-def amqp_connect():
- """Connect to AMQP server"""
-
- cp = configparser.ConfigParser()
- cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
- amqp_uri = cp["amqp"]["uri"]
- parts = urllib.parse.urlsplit(amqp_uri, allow_fragments=False)
- amqp_con = amqp.Connection(
- parts.hostname, userid=parts.username, password=parts.password
- )
- logging.info(
- "Connected to AMQP server at %s@%s", parts.username, parts.hostname
- )
-
- return amqp_con
-
-
def list_remote_container(container_name, swift_conn, marker, limit=1000):
LOGGER.debug("Listing container %s", container_name)
_, list_of_test_results = swift_conn.get_container(
@@ -141,18 +125,15 @@ def fetch_one_result(container_name, object_name, swift_conn):
env_vars = []
env_spec = ["all-proposed"]
for env in env_spec:
- value = str(testinfo.get(env))
+ value = testinfo.get(env)
if value is not None:
- env_vars.append("=".join([env, value]))
+ env_vars.append("=".join([env, str(value)]))
start = datetime.datetime.now()
# Insert the write request into the queue
while True:
try:
complete_amqp = amqp_con.channel()
- complete_amqp.access_request(
- "/complete", active=True, read=False, write=True
- )
complete_amqp.exchange_declare(
SqliteWriterConfig.writer_exchange_name,
"fanout",
@@ -217,12 +198,12 @@ def fetch_container(release, swift_conn):
object_name=known_results[run_id],
swift_conn=swift_conn,
)
- except swiftclient.ClientException as e:
+ except swiftclient.exceptions.ClientException as e:
LOGGER.error(
"Something went wrong accessing container %s\nTraceback: %s"
% (container_name, str(e))
)
- raise
+ return
if __name__ == "__main__":
@@ -241,34 +222,11 @@ if __name__ == "__main__":
)
releases.sort(key=UbuntuDistroInfo().all.index, reverse=True)
- config = get_autopkgtest_cloud_conf()
amqp_con = amqp_connect()
db_con = sqlite3.connect(
- "file:%s%s" % (config["web"]["database"], "?mode=ro"), uri=True
+ "file:%s%s" % (get_db_path(), "?mode=ro"), uri=True
)
-
- swift_cfg = configparser.ConfigParser()
-
- with open(SWIFT_CREDS_FILE) as fp:
- swift_cfg.read_file(
- itertools.chain(["[swift]"], fp), source=SWIFT_CREDS_FILE
- )
-
- swift_creds = {
- "authurl": swift_cfg["swift"]["OS_AUTH_URL"],
- "user": swift_cfg["swift"]["OS_USERNAME"],
- "key": swift_cfg["swift"]["OS_PASSWORD"],
- "os_options": {
- "region_name": swift_cfg["swift"]["OS_REGION_NAME"],
- "project_domain_name": swift_cfg["swift"][
- "OS_PROJECT_DOMAIN_NAME"
- ],
- "project_name": swift_cfg["swift"]["OS_PROJECT_NAME"],
- "user_domain_name": swift_cfg["swift"]["OS_USER_DOMAIN_NAME"],
- },
- "auth_version": 3,
- }
- swift_conn = swiftclient.Connection(**swift_creds)
+ swift_conn = swift_connect()
try:
for release in releases:
diff --git a/charms/focal/autopkgtest-web/webcontrol/download-results b/charms/focal/autopkgtest-web/webcontrol/download-results
index 2c4ea47..02b98d2 100755
--- a/charms/focal/autopkgtest-web/webcontrol/download-results
+++ b/charms/focal/autopkgtest-web/webcontrol/download-results
@@ -7,31 +7,14 @@ import os
import socket
import sys
import time
-import urllib.parse
-import amqplib.client_0_8 as amqp
-from helpers.utils import SqliteWriterConfig, get_autopkgtest_cloud_conf
+import amqp
+from helpers.utils import SqliteWriterConfig, amqp_connect
EXCHANGE_NAME = "testcomplete.fanout"
amqp_con = None
-def amqp_connect():
- """Connect to AMQP server"""
-
- cp = get_autopkgtest_cloud_conf()
- amqp_uri = cp["amqp"]["uri"]
- parts = urllib.parse.urlsplit(amqp_uri, allow_fragments=False)
- amqp_con = amqp.Connection(
- parts.hostname, userid=parts.username, password=parts.password
- )
- logging.info(
- "Connected to AMQP server at %s@%s", parts.username, parts.hostname
- )
-
- return amqp_con
-
-
def process_message(msg):
global amqp_con
body = msg.body
@@ -77,9 +60,6 @@ def process_message(msg):
try:
# add to queue instead of writing to db
with amqp_con.channel() as complete_amqp:
- complete_amqp.access_request(
- "/complete", active=True, read=False, write=True
- )
complete_amqp.exchange_declare(
SqliteWriterConfig.writer_exchange_name,
"fanout",
@@ -120,12 +100,11 @@ def process_message(msg):
if __name__ == "__main__":
logging.basicConfig(
- level=("DEBUG" in os.environ and logging.DEBUG or logging.INFO)
+ level=(logging.DEBUG if "DEBUG" in os.environ else logging.INFO)
)
amqp_con = amqp_connect()
status_ch = amqp_con.channel()
- status_ch.access_request("/complete", active=True, read=True, write=False)
status_ch.exchange_declare(
EXCHANGE_NAME, "fanout", durable=True, auto_delete=False
)
@@ -136,4 +115,4 @@ if __name__ == "__main__":
logging.info("Listening to requests on %s" % queue_name)
status_ch.basic_consume("", callback=process_message)
while status_ch.callbacks:
- status_ch.wait()
+ amqp_con.drain_events()
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/tests.py b/charms/focal/autopkgtest-web/webcontrol/helpers/tests.py
index 8053282..51477bd 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/tests.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/tests.py
@@ -1,5 +1,6 @@
import json
from datetime import datetime
+from pathlib import Path
from uuid import uuid4
from .utils import get_supported_releases
@@ -44,8 +45,9 @@ def populate_dummy_db(db_con):
db_con.commit()
-def populate_dummy_amqp_cache(path):
+def populate_dummy_amqp_cache(path: Path):
supported_releases = get_supported_releases()
+ path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w") as f:
# pylint: disable=line-too-long
json.dump(
@@ -100,8 +102,9 @@ def populate_dummy_amqp_cache(path):
)
-def populate_dummy_running_cache(path):
+def populate_dummy_running_cache(path: Path):
supported_releases = get_supported_releases()
+ path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w") as f:
json.dump(
{
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
index b8d4ab0..b61cfcb 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
@@ -14,10 +14,12 @@ import sqlite3
import subprocess
import time
import typing
+import urllib.parse
# introduced in python3.7, we use 3.8
from dataclasses import dataclass
+import amqp
import distro_info
import swiftclient
@@ -87,7 +89,23 @@ def read_config_file(
def get_autopkgtest_cloud_conf():
- return read_config_file("/home/ubuntu/autopkgtest-cloud.conf")
+ try:
+ return read_config_file(
+ pathlib.Path("~/autopkgtest-cloud.conf").expanduser()
+ )
+ except FileNotFoundError:
+ try:
+ return read_config_file(
+ pathlib.Path(__file__).parent.parent / "autopkgtest-cloud.conf"
+ )
+ except FileNotFoundError as fnfe:
+ raise FileNotFoundError(
+ "No config file found. Have a look at %s"
+ % (
+ pathlib.Path(__file__).parent.parent
+ / "autopkgtest-cloud.conf.example"
+ )
+ ) from fnfe
def get_all_releases():
@@ -146,6 +164,8 @@ def setup_key(app, path):
def init_db(path, **kwargs):
"""Create DB if it does not exist, and connect to it"""
+ path = pathlib.Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
db = sqlite3.connect(path, **kwargs)
c = db.cursor()
@@ -205,6 +225,23 @@ def init_db(path, **kwargs):
return db
+def amqp_connect():
+ """Connect to AMQP server"""
+
+ cp = get_autopkgtest_cloud_conf()
+ amqp_uri = cp["amqp"]["uri"]
+ parts = urllib.parse.urlsplit(amqp_uri, allow_fragments=False)
+ amqp_con = amqp.Connection(
+ parts.hostname, userid=parts.username, password=parts.password
+ )
+ amqp_con.connect()
+ logging.info(
+ "Connected to AMQP server at %s@%s", parts.username, parts.hostname
+ )
+
+ return amqp_con
+
+
def get_test_id(db_con, release, arch, src):
"""
get id of test
@@ -269,24 +306,30 @@ def get_test_id(db_con, release, arch, src):
return test_id
-def init_swift_con() -> swiftclient.Connection:
+def swift_connect() -> swiftclient.Connection:
"""
Establish connection to swift storage
"""
- swift_creds = {
- "authurl": os.environ["OS_AUTH_URL"],
- "user": os.environ["OS_USERNAME"],
- "key": os.environ["OS_PASSWORD"],
- "os_options": {
- "region_name": os.environ["OS_REGION_NAME"],
- "project_domain_name": os.environ["OS_PROJECT_DOMAIN_NAME"],
- "project_name": os.environ["OS_PROJECT_NAME"],
- "user_domain_name": os.environ["OS_USER_DOMAIN_NAME"],
- },
- "auth_version": 3,
- }
- swift_conn = swiftclient.Connection(**swift_creds)
- return swift_conn
+ try:
+ config = get_autopkgtest_cloud_conf()
+ swift_creds = {
+ "authurl": config["swift"]["os_auth_url"],
+ "user": config["swift"]["os_username"],
+ "key": config["swift"]["os_password"],
+ "os_options": {
+ "region_name": config["swift"]["os_region_name"],
+ "project_domain_name": config["swift"][
+ "os_project_domain_name"
+ ],
+ "project_name": config["swift"]["os_project_name"],
+ "user_domain_name": config["swift"]["os_user_domain_name"],
+ },
+ "auth_version": config["swift"]["os_identity_api_version"],
+ }
+ swift_conn = swiftclient.Connection(**swift_creds)
+ return swift_conn
+ except KeyError as e:
+ raise swiftclient.ClientException(repr(e))
def is_db_empty(db_con):
@@ -304,9 +347,7 @@ def is_db_empty(db_con):
def get_db_path():
- cp = configparser.ConfigParser()
- cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
- return cp["web"]["database"]
+ return get_autopkgtest_cloud_conf()["web"]["database"]
get_test_id._cache = {}
diff --git a/charms/focal/autopkgtest-web/webcontrol/private_results/app.py b/charms/focal/autopkgtest-web/webcontrol/private_results/app.py
index 36b1fd0..ab43603 100644
--- a/charms/focal/autopkgtest-web/webcontrol/private_results/app.py
+++ b/charms/focal/autopkgtest-web/webcontrol/private_results/app.py
@@ -1,4 +1,5 @@
"""Test Result Fetcher Flask App"""
+
import logging
import os
import sys
@@ -14,11 +15,7 @@ from flask import (
session,
)
from flask_openid import OpenID
-from helpers.utils import (
- get_autopkgtest_cloud_conf,
- read_config_file,
- setup_key,
-)
+from helpers.utils import setup_key, swift_connect
from request.submit import Submit
from werkzeug.middleware.proxy_fix import ProxyFix
@@ -96,38 +93,8 @@ app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
secret_path = os.path.join(PATH, "secret_key")
setup_key(app, secret_path)
oid = OpenID(app, os.path.join(PATH, "openid"), safe_roots=[])
-# Load configuration
-cfg = read_config_file(
- os.path.expanduser("~ubuntu/swift-web-credentials.conf")
-)
-# The web configuration as well
-cfg_web = get_autopkgtest_cloud_conf()
-# Build swift credentials
-auth_url = cfg.get("swift", "auth_url")
-if "/v2.0" in auth_url:
- swift_creds = {
- "authurl": auth_url,
- "user": cfg.get("swift", "username"),
- "key": cfg.get("swift", "password"),
- "tenant_name": cfg.get("swift", "tenant"),
- "os_options": {"region_name": cfg.get("swift", "region_name")},
- "auth_version": "2.0",
- }
-else:
- swift_creds = {
- "authurl": auth_url,
- "user": cfg.get("swift", "username"),
- "key": cfg.get("swift", "password"),
- "os_options": {
- "region_name": cfg.get("swift", "region_name"),
- "project_name": cfg.get("swift", "project_name"),
- "object_storage_url": cfg_web["web"]["SwiftURL"],
- },
- "auth_version": "3.0",
- }
-cfg_web = None
# Connect to swift
-connection = swiftclient.Connection(**swift_creds)
+connection = swift_connect()
#
diff --git a/charms/focal/autopkgtest-web/webcontrol/publish-db b/charms/focal/autopkgtest-web/webcontrol/publish-db
index 78b4b92..4f97cda 100755
--- a/charms/focal/autopkgtest-web/webcontrol/publish-db
+++ b/charms/focal/autopkgtest-web/webcontrol/publish-db
@@ -14,21 +14,19 @@ import sqlite3
import sys
import tempfile
import urllib.request
+from pathlib import Path
import apt_pkg
from helpers.utils import get_autopkgtest_cloud_conf, is_db_empty
sqlite3.paramstyle = "named"
-config = None
-db_con = None
-
-archive_url = "http://ftpmaster.internal/ubuntu";
-components = ["main", "restricted", "universe", "multiverse"]
+COMPONENTS = ["main", "restricted", "universe", "multiverse"]
def init_db(path, path_current, path_rw):
"""Create DB if it does not exist, and connect to it"""
+ Path(path).parent.mkdir(parents=True, exist_ok=True)
db = sqlite3.connect(path)
db_rw = sqlite3.connect("file:%s?mode=ro" % path_rw, uri=True)
@@ -94,12 +92,7 @@ def init_db(path, path_current, path_rw):
logging.debug("Old current_versions copied over")
current_version_copied = True
except sqlite3.OperationalError as e:
- if "no such column: pocket" not in str(
- e
- ) and "no such column: component" not in str(
- e
- ): # schema upgrade
- raise
+ logging.debug("failed to copy current_version: %s", str(e))
current_version_copied = False
try:
@@ -143,8 +136,8 @@ def get_last_checked(db_con, url):
return None
-def get_sources(db_con, release):
- for component in components:
+def get_sources(db_con, release, archive_url):
+ for component in COMPONENTS:
for pocket in (release, release + "-updates"):
logging.debug("Processing %s/%s", pocket, component)
try:
@@ -200,8 +193,9 @@ def get_sources(db_con, release):
if __name__ == "__main__":
- if "DEBUG" in os.environ:
- logging.basicConfig(level=logging.DEBUG)
+ logging.basicConfig(
+ level=(logging.DEBUG if "DEBUG" in os.environ else logging.INFO)
+ )
config = get_autopkgtest_cloud_conf()
@@ -210,6 +204,8 @@ if __name__ == "__main__":
target_checksum = "{}.sha256".format(target)
target_checksum_new = "{}.new".format(target_checksum)
+ archive_url = config["web"]["archive_url"]
+
try:
# systemd makes sure to not run us in parallel, so we can safely
# delete any leftover file.
@@ -219,7 +215,7 @@ if __name__ == "__main__":
db_con = init_db(target_new, target, config["web"]["database"])
for row in db_con.execute("SELECT DISTINCT release FROM test"):
- get_sources(db_con, row[0])
+ get_sources(db_con, row[0], archive_url)
db_con.commit()
db_con.close()
diff --git a/charms/focal/autopkgtest-web/webcontrol/request/submit.py b/charms/focal/autopkgtest-web/webcontrol/request/submit.py
index 9635223..ce8f640 100644
--- a/charms/focal/autopkgtest-web/webcontrol/request/submit.py
+++ b/charms/focal/autopkgtest-web/webcontrol/request/submit.py
@@ -7,7 +7,6 @@ import base64
import json
import logging
import os
-import pathlib
import re
import sqlite3
import urllib.parse
@@ -17,7 +16,7 @@ from datetime import datetime, timedelta
from time import time
from urllib.error import HTTPError
-import amqplib.client_0_8 as amqp
+import amqp
from distro_info import UbuntuDistroInfo
from helpers.cache import KeyValueCache
from helpers.exceptions import (
@@ -29,7 +28,7 @@ from helpers.exceptions import (
RequestInQueue,
RequestRunning,
)
-from helpers.utils import get_autopkgtest_cloud_conf, timeout
+from helpers.utils import amqp_connect, get_autopkgtest_cloud_conf, timeout
# Launchpad REST API base
LP = "https://api.launchpad.net/1.0/";
@@ -40,35 +39,19 @@ ENV = re.compile(r"^[a-zA-Z][a-zA-Z0-9_]+=[a-zA-Z0-9.:~/ -=]*$")
# URL and optional branch name
GIT = re.compile(r"^https?://[a-zA-Z0-9._/~+-]+(#[a-zA-Z0-9._/-]+)?$")
-ALLOWED_REQUESTOR_TEAMS = []
-try:
- allowed_teams = pathlib.Path(
- "/home/ubuntu/.config/autopkgtest-web/allowed-requestor-teams"
- )
- ALLOWED_REQUESTOR_TEAMS = allowed_teams.read_text(
- encoding="utf-8"
- ).splitlines()
-except Exception as e:
- logging.warning(f"Reading allowed teams failed with {e}")
-
# not teams
ALLOWED_USERS_PERPACKAGE = {"snapcraft": ["snappy-m-o"]}
-# Path to json file detailing the queue
-QUEUE_FP = "/var/lib/cache-amqp/queued.json"
-# Path to json file detailing the running tests
-RUNNING_FP = "/run/amqp-status-collector/running.json"
-
ALLOWED_USER_CACHE_TIME = timedelta(hours=3)
class Submit:
def __init__(self):
- cp = get_autopkgtest_cloud_conf()
+ self.config = get_autopkgtest_cloud_conf()
# read valid releases and architectures from DB
self.db_con = sqlite3.connect(
- "file:%s?mode=ro" % cp["web"]["database_ro"], uri=True
+ "file:%s?mode=ro" % self.config["web"]["database_ro"], uri=True
)
self.releases = set(
UbuntuDistroInfo().supported() + UbuntuDistroInfo().supported_esm()
@@ -85,13 +68,6 @@ class Submit:
self.architectures.add(row[0])
logging.debug("Valid architectures: %s" % self.architectures)
- # dissect AMQP URL
- self.amqp_creds = urllib.parse.urlsplit(
- cp["amqp"]["uri"], allow_fragments=False
- )
- assert self.amqp_creds.scheme == "amqp"
- logging.debug("AMQP credentials: %s" % repr(self.amqp_creds))
-
self.allowed_user_cache = KeyValueCache(
"/dev/shm/autopkgtest_users.json"
)
@@ -330,11 +306,7 @@ class Submit:
count = 0
- with amqp.Connection(
- self.amqp_creds.hostname,
- userid=self.amqp_creds.username,
- password=self.amqp_creds.password,
- ) as amqp_con:
+ with amqp_connect() as amqp_con:
with amqp_con.channel() as ch:
while True:
message = ch.basic_get(queue)
@@ -370,11 +342,7 @@ class Submit:
body = "%s\n%s" % (package, json.dumps(params, sort_keys=True))
try:
with timeout(seconds=60):
- with amqp.Connection(
- self.amqp_creds.hostname,
- userid=self.amqp_creds.username,
- password=self.amqp_creds.password,
- ) as amqp_con:
+ with amqp_connect() as amqp_con:
with amqp_con.channel() as ch:
ch.basic_publish(
amqp.Message(body, delivery_mode=2), # persistent
@@ -542,8 +510,8 @@ class Submit:
return code >= 200 and code < 300
# pylint: disable=dangerous-default-value
- def in_allowed_team(self, person, teams=[]):
- """Check if person is in ALLOWED_REQUESTOR_TEAMS"""
+ def in_allowed_team(self, person):
+ """Check if person is allowed to queue tests"""
cached_entry = self.allowed_user_cache.get(person)
if cached_entry is not None:
cached_entry = datetime.fromtimestamp(float(cached_entry))
@@ -561,7 +529,7 @@ class Submit:
)
entries = response.get("entries")
for e in entries:
- for team in teams or ALLOWED_REQUESTOR_TEAMS:
+ for team in self.config["web"]["allowed_requestors"].split(","):
if team in e["team_link"]:
self.allowed_user_cache.set(person, time())
return True
@@ -615,10 +583,10 @@ class Submit:
ppas,
git,
):
- if not os.path.isfile(RUNNING_FP):
+ if not os.path.isfile(self.config["web"]["running_cache"]):
return False
data = {}
- with open(RUNNING_FP, "r") as f:
+ with open(self.config["web"]["running_cache"], "r") as f:
data = json.load(f)
if data == {}:
return False
@@ -673,10 +641,10 @@ class Submit:
ppas,
git,
):
- if not os.path.isfile(QUEUE_FP):
+ if not os.path.isfile(self.config["web"]["amqp_queue_cache"]):
return False
data = {}
- with open(QUEUE_FP, "r") as f:
+ with open(self.config["web"]["amqp_queue_cache"], "r") as f:
data = json.load(f)
data = data["queues"]
this_test = {
diff --git a/charms/focal/autopkgtest-web/webcontrol/request/tests/test_submit.py b/charms/focal/autopkgtest-web/webcontrol/request/tests/test_submit.py
index 6a3c7ed..c43df09 100644
--- a/charms/focal/autopkgtest-web/webcontrol/request/tests/test_submit.py
+++ b/charms/focal/autopkgtest-web/webcontrol/request/tests/test_submit.py
@@ -68,9 +68,6 @@ class DistroRequestValidationTests(SubmitTestBase):
releases.add("testy")
self.assertEqual(self.submit.releases, releases)
self.assertEqual(self.submit.architectures, {"6510", "C51", "hexium"})
- self.assertEqual(self.submit.amqp_creds.hostname, "1.2.3.4")
- self.assertEqual(self.submit.amqp_creds.username, "user")
- self.assertEqual(self.submit.amqp_creds.password, "s3kr1t")
def test_bad_release(self):
"""Unknown release"""
diff --git a/charms/focal/autopkgtest-web/webcontrol/sqlite-writer b/charms/focal/autopkgtest-web/webcontrol/sqlite-writer
index 50043cc..f7b2939 100755
--- a/charms/focal/autopkgtest-web/webcontrol/sqlite-writer
+++ b/charms/focal/autopkgtest-web/webcontrol/sqlite-writer
@@ -1,7 +1,6 @@
#!/usr/bin/python3
# pylint: disable=wrong-import-position
-import configparser
import datetime
import json
import logging
@@ -10,44 +9,25 @@ import socket
import sqlite3
import swiftclient
-
-sqlite3.paramstyle = "named"
-import urllib.parse
-
-import amqplib.client_0_8 as amqp
from helpers.utils import (
SqliteWriterConfig,
+ amqp_connect,
get_db_path,
get_test_id,
init_db,
- init_swift_con,
is_db_empty,
+ swift_connect,
zstd_decompress,
)
+sqlite3.paramstyle = "named"
+
LAST_CHECKPOINT = datetime.datetime.now()
config = None
db_con = None
-def amqp_connect():
- """Connect to AMQP server"""
-
- cp = configparser.ConfigParser()
- cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
- amqp_uri = cp["amqp"]["uri"]
- parts = urllib.parse.urlsplit(amqp_uri, allow_fragments=False)
- amqp_con = amqp.Connection(
- parts.hostname, userid=parts.username, password=parts.password
- )
- logging.info(
- "Connected to AMQP server at %s@%s", parts.username, parts.hostname
- )
-
- return amqp_con
-
-
def check_msg(queue_msg):
queue_keys = set(queue_msg.keys())
if set(SqliteWriterConfig.amqp_entry_fields) == queue_keys:
@@ -113,7 +93,8 @@ def restore_db_from_backup(db_con: sqlite3.Connection):
backups_container = "db-backups"
logging.info("Connecting to swift")
try:
- swift_conn = init_swift_con()
+ swift_conn = swift_connect()
+ _, objects = swift_conn.get_container(container=backups_container)
except swiftclient.ClientException as e:
logging.warning(
(
@@ -126,7 +107,7 @@ def restore_db_from_backup(db_con: sqlite3.Connection):
f"Connected to swift! Getting backups from container: {backups_container}"
)
db_con.execute("PRAGMA wal_checkpoint(TRUNCATE);")
- _, objects = swift_conn.get_container(container=backups_container)
+
latest = objects[-1]
_, compressed_db_dump = swift_conn.get_object(
container=backups_container, obj=latest["name"]
@@ -149,7 +130,9 @@ def restore_db_from_backup(db_con: sqlite3.Connection):
def main():
- logging.basicConfig(level=logging.INFO)
+ logging.basicConfig(
+ level=(logging.DEBUG if "DEBUG" in os.environ else logging.INFO)
+ )
db_con = init_db(get_db_path())
if is_db_empty(db_con):
logging.info(
@@ -159,7 +142,6 @@ def main():
restore_db_from_backup(db_con)
amqp_con = amqp_connect()
status_ch = amqp_con.channel()
- status_ch.access_request("/complete", active=True, read=True, write=False)
status_ch.exchange_declare(
SqliteWriterConfig.writer_exchange_name,
"fanout",
@@ -174,7 +156,7 @@ def main():
logging.info("Listening to requests on %s" % queue_name)
status_ch.basic_consume("", callback=lambda msg: msg_callback(msg, db_con))
while status_ch.callbacks:
- status_ch.wait()
+ amqp_con.drain_events()
if __name__ == "__main__":
diff --git a/charms/focal/autopkgtest-web/webcontrol/swift-cleanup b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
index 14578a4..8f58eed 100755
--- a/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
+++ b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
@@ -1,7 +1,5 @@
#!/usr/bin/python3
-import configparser
import io
-import itertools
import json
import logging
import os
@@ -12,8 +10,8 @@ import time
import swiftclient
from distro_info import UbuntuDistroInfo
+from helpers.utils import swift_connect
-SWIFT_CREDS_FILE = "/home/ubuntu/public-swift-creds"
RETRY_WAIT_TIME = 5
@@ -181,29 +179,6 @@ def fix_testinfo_jsons_for_release(release, swift_conn):
raise
-def get_swift_con():
- swift_cfg = configparser.ConfigParser()
- with open(SWIFT_CREDS_FILE) as fp:
- swift_cfg.read_file(
- itertools.chain(["[swift]"], fp), source=SWIFT_CREDS_FILE
- )
- swift_creds = {
- "authurl": swift_cfg["swift"]["OS_AUTH_URL"],
- "user": swift_cfg["swift"]["OS_USERNAME"],
- "key": swift_cfg["swift"]["OS_PASSWORD"],
- "os_options": {
- "region_name": swift_cfg["swift"]["OS_REGION_NAME"],
- "project_domain_name": swift_cfg["swift"][
- "OS_PROJECT_DOMAIN_NAME"
- ],
- "project_name": swift_cfg["swift"]["OS_PROJECT_NAME"],
- "user_domain_name": swift_cfg["swift"]["OS_USER_DOMAIN_NAME"],
- },
- "auth_version": 3,
- }
- return swiftclient.Connection(**swift_creds)
-
-
def get_releases():
releases = list(
set(
@@ -217,7 +192,7 @@ def get_releases():
def main():
logging.basicConfig(level=logging.INFO)
logging.info("Setting up swift connection")
- swift_conn = get_swift_con()
+ swift_conn = swift_connect()
releases = get_releases()
for release in releases:
fix_testinfo_jsons_for_release(release, swift_conn)
diff --git a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
index ed81915..a90a75e 100755
--- a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
+++ b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
@@ -9,17 +9,15 @@ import tarfile
from datetime import datetime, timedelta
from pathlib import Path
-import swiftclient
from helpers.utils import (
get_autopkgtest_cloud_conf,
get_github_context,
- read_config_file,
+ swift_connect,
)
from request.submit import Submit
PENDING_DIR = Path("/run/autopkgtest_webcontrol/github-pending")
RUNNING_CACHE = Path("/run/amqp-status-collector/running.json")
-SWIFT_CREDS_FILE = Path("/home/ubuntu/public-swift-creds")
MAX_DAY_DIFF = 30
swift_container_cache = None
@@ -271,24 +269,8 @@ if __name__ == "__main__":
config = get_autopkgtest_cloud_conf()
external_url = config["web"]["ExternalURL"]
- swift_cfg = read_config_file(filepath=SWIFT_CREDS_FILE, cfg_key="swift")
-
- swift_creds = {
- "authurl": swift_cfg["swift"]["OS_AUTH_URL"],
- "user": swift_cfg["swift"]["OS_USERNAME"],
- "key": swift_cfg["swift"]["OS_PASSWORD"],
- "os_options": {
- "region_name": swift_cfg["swift"]["OS_REGION_NAME"],
- "project_domain_name": swift_cfg["swift"][
- "OS_PROJECT_DOMAIN_NAME"
- ],
- "project_name": swift_cfg["swift"]["OS_PROJECT_NAME"],
- "user_domain_name": swift_cfg["swift"]["OS_USER_DOMAIN_NAME"],
- },
- "auth_version": 3,
- }
- swift_conn = swiftclient.Connection(**swift_creds)
+ swift_conn = swift_connect()
jobs = sys.argv[1:]
diff --git a/mojo/service-bundle b/mojo/service-bundle
index d23d61d..0a65ce3 100644
--- a/mojo/service-bundle
+++ b/mojo/service-bundle
@@ -4,12 +4,16 @@
{%- elif stage_name == "staging" or stage_name == "devel" %}
{%- set releases = "focal jammy noble oracular" %}
{%- set channel = "latest/edge" %}
+{%- else %}
+ {%- set releases = "noble" %}
{%- endif %}
{%- if stage_name == "production" %}
{%- set hostname = "autopkgtest.ubuntu.com" %}
{%- elif stage_name == "staging" %}
{%- set hostname = "autopkgtest.staging.ubuntu.com" %}
+{%- elif stage_name == "devel" %}
+ {%- set hostname = "autopkgtest.localhost" %}
{%- endif %}
{%- if stage_name == "production" or stage_name == "staging" %}
@@ -22,11 +26,13 @@ description: "autopkgtest-cloud"
series: {{ series }}
applications:
autopkgtest-cloud-worker:
+{%- if stage_name == "production" or stage_name == "staging" %}
charm: ubuntu-release-autopkgtest-cloud-worker
channel: {{ channel }}
-{%- if stage_name == "production" or stage_name == "staging" %}
num_units: 3
{%- else %}
+ # Don't use ~ here, it doesn't work!
+ charm: XXX/path/to/autopkgtest-cloud-git-repo/XXX/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud-worker_ubuntu-20.04-amd64.charm
num_units: 1
{%- endif %}
constraints: mem=16G cores=8 root-disk=40G
@@ -63,11 +69,14 @@ applications:
swift-project-name: stg-proposed-migration-environment_project
swift-user-domain-name: Default
{%- elif stage_name == "devel" %}
- swift-auth-url: XXX
- swift-username: XXX
- swift-region: XXX
+ influxdb-username: dev_proposed_migration
+ influxdb-context: devel
+ # Most Canonistack values can be found in your canonistack novarc file
+ swift-auth-url: https://keystone.bos01.canonistack.canonical.com:5000/v3
+ swift-username: XXX # canonistack username
+ swift-region: canonistack-bos01
swift-project-domain-name: default
- swift-project-name: XXX
+ swift-project-name: XXX_project # canonistack project
swift-user-domain-name: default
swift-auth-version: 3
{%- endif %}
@@ -114,17 +123,18 @@ applications:
ppc64el: net_stg-proposed-migration-ppc64el
s390x: net_stg-proposed-migration-s390x
{%- elif stage_name == "devel" %}
- net-name: net_instances
worker-flavor-config: |-
- default: stag-cpu2-ram4-disk20
- big: stag-cpu4-ram16-disk50
+ default: cpu2-ram4-disk20
+ big: cpu4-ram16-disk50
+ worker-net-names: |-
+ default: external-network
{%- endif %}
{%- if stage_name == "production" or stage_name == "staging" %}
mirror: http://ftpmaster.internal/ubuntu/
worker-args: ssh -s /CHECKOUTDIR//ssh-setup/nova -- --flavor $PACKAGESIZE --security-groups $SECGROUP --name adt-$RELEASE-$ARCHITECTURE-$PACKAGENAME-$TIMESTAMP-$HOSTNAME-$UUID --image adt/ubuntu-$RELEASE-$HOSTARCH-server --keyname testbed-/HOSTNAME/ --net-id=/NET_NAME/ -e TERM=linux -e 'http_proxy={{ http_proxy }}' -e 'https_proxy={{ https_proxy }}' -e 'no_proxy={{ no_proxy }}' --mirror=/MIRROR/
worker-setup-command: /AUTOPKGTEST_CLOUD_DIR//worker-config-production/setup-canonical.sh
{% else %}
- mirror: http://ports.ubuntu.com/ubuntu-ports/
+ mirror: http://archive.ubuntu.com/ubuntu/
worker-args: ssh -s /CHECKOUTDIR//ssh-setup/nova -- --flavor $PACKAGESIZE --security-groups $SECGROUP --name adt-$RELEASE-$ARCHITECTURE-$PACKAGENAME-$TIMESTAMP-$HOSTNAME-$UUID --image adt/ubuntu-$RELEASE-$ARCHITECTURE-server-.* --keyname testbed-/HOSTNAME/ --mirror=/MIRROR/
{% endif %}
{%- if stage_name == "production" %}
@@ -146,13 +156,21 @@ applications:
s390x: 1
{%- elif stage_name == "devel" %}
n-workers: |-
+ devstack:
+ amd64: 1
bos03:
+ amd64: 1
arm64: 0
ppc64el: 0
{%- endif %}
autopkgtest-lxd-worker:
+{%- if stage_name == "production" or stage_name == "staging" %}
charm: ubuntu-release-autopkgtest-cloud-worker
channel: {{ channel }}
+{%- else %}
+ # Don't use ~ here, it doesn't work!
+ charm: XXX/path/to/autopkgtest-cloud-git-repo/XXX/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud-worker_ubuntu-20.04-amd64.charm
+{%- endif %}
num_units: 1
constraints: mem=16G cores=8 root-disk=40G
{%- if stage_name == "production" or stage_name == "staging" %}
@@ -217,8 +235,13 @@ applications:
enable_modules: include cgi proxy proxy_http remoteip
mpm_type: prefork
autopkgtest-web:
+{%- if stage_name == "production" or stage_name == "staging" %}
charm: ubuntu-release-autopkgtest-web
channel: {{ channel }}
+{% else %}
+ # Don't use ~ here, it doesn't work!
+ charm: XXX/path/to/autopkgtest-cloud-git-repo/XXX/charms/focal/autopkgtest-web/autopkgtest-web_ubuntu-20.04-amd64.charm
+{%- endif %}
options:
hostname: {{ hostname }}
allowed-requestor-teams: |-
@@ -230,38 +253,38 @@ applications:
canonical-server
canonical-ubuntu-qa
{%- if stage_name == "production" %}
+ archive-url: http://ftpmaster.internal/ubuntu/
influxdb-context: production
influxdb-username: prod_proposed_migration
{%- set storage_host_internal = "objectstorage.prodstack5.canonical.com:443" %}
{%- set storage_path_internal = "/swift/v1/AUTH_0f9aae918d5b4744bf7b827671c86842" %}
{%- elif stage_name == "staging" %}
+ archive-url: http://ftpmaster.internal/ubuntu/
{%- set storage_host_internal = "objectstorage.prodstack5.canonical.com:443" %}
{%- set storage_path_internal = "/swift/v1/AUTH_cc509e38c54f4edebda2fd17557309bb" %}
influxdb-username: stg_proposed_migration
influxdb-context: staging
{%- elif stage_name == "devel" %}
+ archive-url: http://archive.ubuntu.com/ubuntu/
influxdb-context: XXX
influxdb-username: XXX
- storage_host_internal: XXX
- storage_path_internal: XXX
- influxdb-hostname: XXX
- influxdb-password: XXX
- influxdb-database: XXX
+ {# canonistack objectstorage URL, find this with `swift auth` #}
+ {%- set storage_host_internal = "swift-proxy.bos01.canonistack.canonical.com:8080" %}
+ {# canonistack swift path, find this with `swift auth` #}
+ {%- set storage_path_internal = "/v1/AUTH_0123456789abcdef0123456789abcdef" %}
{%- endif %}
storage-url-internal: https://{{ storage_host_internal }}{{ storage_path_internal }}
-{%- if stage_name == "production" or stage_name == "staging" %}
github-secrets: include-file://{{local_dir}}/github-secrets.json
- github-status-credentials: include-file://{{local_dir}}/github-status-credentials.txt
- swift-web-credentials: include-file://{{local_dir}}/swift-web-credentials.conf
- public-swift-creds: include-file://{{local_dir}}/public-swift-creds
external-web-requests-api-keys: include-file://{{local_dir}}/external-web-requests-api-keys.json
+ public-swift-creds: include-file://{{local_dir}}/public-swift-creds
+ indexed-packages-fp: /home/ubuntu/indexed-packages.json
+{%- if stage_name == "production" or stage_name == "staging" %}
+ github-status-credentials: include-file://{{local_dir}}/github-status-credentials.txt
influxdb-hostname: include-file://{{ local_dir }}/influx-hostname.txt
influxdb-password: include-file://{{ local_dir }}/influx-password.txt
influxdb-database: metrics
https-proxy: {{ https_proxy }}
no-proxy: {{ no_proxy }}
- cookies: S0 S1
- indexed-packages-fp: /home/ubuntu/indexed-packages.json
{%- endif %}
haproxy:
charm: cs:haproxy
diff --git a/mojorc b/mojorc
new file mode 100644
index 0000000..cc81516
--- /dev/null
+++ b/mojorc
@@ -0,0 +1,14 @@
+# This file will setup your environment for a local develop `mojo run`.
+# Please have a look at the "Deploying" documentation page to know how to use this file.
+
+base_dir="$(dirname "$(realpath "$0")")"
+
+export MOJO_ROOT=~/.local/share/mojo
+export MOJO_SERIES=focal
+export MOJO_PROJECT=autopkgtest-cloud
+export MOJO_WORKSPACE=autopkgtest-cloud
+export MOJO_SPEC="$base_dir/mojo/"
+export MOJO_STAGE=devel
+
+mojo project-new $MOJO_PROJECT -s $MOJO_SERIES --container containerless
+mojo workspace-new --project $MOJO_PROJECT -s $MOJO_SERIES $MOJO_SPEC $MOJO_WORKSPACE
