canonical-hw-cert team mailing list archive
-
canonical-hw-cert team
-
Mailing list archive
-
Message #18551
[Merge] ~jocave/hwcert-jenkins-tools:remove-trello-scripts into hwcert-jenkins-tools:master
Jonathan Cave has proposed merging ~jocave/hwcert-jenkins-tools:remove-trello-scripts into hwcert-jenkins-tools:master.
Requested reviews:
Canonical Hardware Certification (canonical-hw-cert)
For more details, see:
https://code.launchpad.net/~jocave/hwcert-jenkins-tools/+git/hwcert-jenkins-tools/+merge/429214
Remove the trello scripts from this repo. They are now hosted in this standalone project: https://github.com/canonical/certification-dashboard-manager
--
Your team Canonical Hardware Certification is requested to review the proposed merge of ~jocave/hwcert-jenkins-tools:remove-trello-scripts into hwcert-jenkins-tools:master.
diff --git a/trello-board-manager-desktop.py b/trello-board-manager-desktop.py
deleted file mode 100755
index 15c9c00..0000000
--- a/trello-board-manager-desktop.py
+++ /dev/null
@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2019 Canonical Ltd
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-# Written by:
-# Taihsiang Ho <taihsiang.ho@xxxxxxxxxxxxx>
-#
-#
-# The script uses py-trello package 0.9.0. You may want to fetch it from
-# source.
-#
-# This script will move cards between different lanes (-updates and -proposed)
-
-import sys
-import argparse
-import logging
-import os
-import re
-import requests
-import yaml
-
-from urllib.parse import urlparse
-from trello import TrelloClient
-
-
-repositories = ['proposed', 'updates']
-
-repository_promotion_map = {
- # repository -> next-repository
- 'proposed': 'updates'
-}
-
-codename_map = {'xenial': '16.04',
- 'bionic': '18.04',
- 'cosmic': '18.10',
- 'disco': '19.04',
- 'focal': '20.04',
- 'groovy': '20.10',
- 'hirsute': '21.04'}
-
-logger = logging.getLogger("trello-board-manager-desktop")
-
-
-def environ_or_required(key):
- """Mapping for argparse to supply required or default from $ENV."""
- if os.environ.get(key):
- return {'default': os.environ.get(key)}
- else:
- return {'required': True}
-
-
-def archive_card(card):
- logger.info('Archiving old revision: {}'.format(card))
- card.set_closed(True)
-
-
-def move_card(config, lane_name, card):
- """Move trello cards according to the current deb repository."""
- logger.info('Card {} in {}'.format(card.name, lane_name))
- m = re.match(
- r"(?P<stack>.*?)(?:\s+\-\s+)(?P<package>.*?)(?:\s+\-\s+)"
- r"\((?P<version>.*?)\)", card.name)
- if m:
- arch = config[m.group("stack")]["arch"]
- stack = m.group("stack")
- codename = m.group("stack").split('-')[0]
-
- # TODO: you may need to update this mapping in the future when
- # the oem image is based on the other distro
- if codename == 'oem':
- codename = 'bionic'
- elif codename == 'oem_focal':
- codename = 'focal'
-
- logger.debug('arch: {}'.format(arch))
- logger.debug('stack: {}'.format(stack))
- logger.debug('codename: {}'.format(codename))
-
- for repo in repositories:
- logger.debug('Working in repo: {}'.format(repo))
-
- jenkins_link = os.environ.get('BUILD_URL', '')
- uri = urlparse(jenkins_link)
- jenkins_host = '{uri.scheme}://{uri.netloc}/'.format(uri=uri)
- logger.debug('jenkins_link: {}'.format(jenkins_link))
- logger.debug('jenkins_host: {}'.format(jenkins_host))
- # TODO: we could merge main and universe repositories from the source
- # jenkins jobs
- # linux-oem is in universe rather than main
- if 'oem-osp1' in stack and not codename == 'xenial':
- text_template = '{}/job/cert-package-data/lastSuccessfulBuild/artifact/{}-universe-{}-{}.json'
- else:
- # packages of generic kernels
- # projects using these kernels:
- # stock images
- # oem image - xenial
- # oem images - shipped with oem-4.13
- # argos dgx-1/dgx-station images
- text_template = '{}/job/cert-package-data/lastSuccessfulBuild/artifact/{}-main-{}-{}.json'
-
- package_json_url = text_template.format(jenkins_host,
- codename, arch, repo)
- response = requests.get(url=package_json_url)
- pkg_data = response.json()
-
- # stack_version_full, svf, for example 4.4.0.150.158
- if 'oem' in stack and '5.6' in card.name:
- svf = pkg_data['linux-oem-20_04']
- elif 'oem' in stack and '5.10' in card.name:
- svf = pkg_data['linux-oem-20_04b']
- elif 'oem' in stack and '5.13' in card.name:
- svf = pkg_data['linux-oem-20_04c']
- elif 'oem' in stack and '5.14' in card.name:
- svf = pkg_data['linux-oem-20_04d']
- else:
- svf = pkg_data['linux-generic']
-
- if 'hwe' in stack:
- svf = re.match(r'\d+.\d+.\d+.\d+.\d+',
- pkg_data['linux-generic-hwe-' + codename_map[codename].replace('.', '_')]).group(0)
- # I only want 4_4_0-150
- sv = svf[:svf.rfind('.')].replace('.', '-')
- stack_version = sv.replace('-', '_', 2)
- if 'oem-osp1' in stack:
- deb_kernel_image = 'linux-image-' + stack_version + '-oem-osp1'
- elif "oem" in stack:
- deb_kernel_image = 'linux-image-' + stack_version + '-oem'
- else:
- deb_kernel_image = 'linux-image-' + stack_version + '-generic'
-
- logger.debug('stack_version: {}'.format(stack_version))
- logger.debug('deb_kernel_image: {}'.format(deb_kernel_image))
-
- deb_version = pkg_data[deb_kernel_image]
-
- ori = next_repo = repository_promotion_map.get(lane_name)
-
- logger.debug('deb_version: {}'.format(deb_version))
- logger.debug('next_repo: {}'.format(next_repo))
- logger.debug('m.group("stack"): '
- '{} {}'.format(m.group("stack"),
- type(m.group("stack"))))
- logger.debug('m.group("package"): '
- '{} {}'.format(m.group("package"),
- type(m.group("package"))))
- logger.debug('m.group("version"): '
- '{} {}'.format(m.group("version"),
- type(m.group("version"))))
-
- if (repo == lane_name and deb_version != m.group("version")):
- archive_card(card)
- continue
- if (repo == next_repo and
- deb_kernel_image == m.group("package") and
- deb_version == m.group("version")):
- for l in card.board.open_lists():
- if ori.capitalize() == l.name:
- msg = 'Moving the card {} to {}'.format(card.name,
- l.name)
- logger.debug(msg)
- card.change_list(l.id)
- return
-
-
-def load_config(configfile):
- if not configfile:
- return []
- try:
- data = yaml.safe_load(configfile)
- except (yaml.parser.ParserError, yaml.scanner.ScannerError):
- print('ERROR: Error parsing', configfile.name)
- sys.exit(1)
- return data
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--key', help="Trello API key",
- **environ_or_required('TRELLO_API_KEY'))
- parser.add_argument('--token', help="Trello OAuth token",
- **environ_or_required('TRELLO_TOKEN'))
- parser.add_argument('--board', help="Trello board identifier",
- **environ_or_required('TRELLO_BOARD'))
- parser.add_argument('--debug', help="Enable the debug mode",
- action="store_true", default=False)
- parser.add_argument('config', help="snaps configuration",
- type=argparse.FileType())
- args = parser.parse_args()
-
- format_str = "[ %(funcName)s() ] %(message)s"
- if args.debug:
- logging.basicConfig(format=format_str, level=logging.DEBUG)
- else:
- logging.basicConfig(format=format_str)
-
- client = TrelloClient(api_key=args.key, token=args.token)
- board = client.get_board(args.board)
-
- config = load_config(args.config)
-
- for lane in board.list_lists():
- lane_name = lane.name.lower()
- if lane_name in repositories:
- for card in lane.list_cards():
- try:
- move_card(config, lane_name, card)
- except Exception:
- logger.warning("WARNING", exc_info=True)
- continue
-
-
-if __name__ == "__main__":
- main()
diff --git a/trello-board-manager.py b/trello-board-manager.py
deleted file mode 100755
index fa715d9..0000000
--- a/trello-board-manager.py
+++ /dev/null
@@ -1,131 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2017 Canonical Ltd
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-# Written by:
-# Sylvain Pineau <sylvain.pineau@xxxxxxxxxxxxx>
-
-import argparse
-import logging
-import os
-import re
-import requests
-
-from trello import TrelloClient
-from yaml import safe_load
-from yaml.parser import ParserError
-
-
-channel_promotion_map = {
- # channel -> next-channel
- 'edge': 'beta',
- 'beta': 'candidate',
- 'candidate': 'stable',
- 'stable': 'stable',
-}
-
-
-def environ_or_required(key):
- """Mapping for argparse to supply required or default from $ENV."""
- if os.environ.get(key):
- return {'default': os.environ.get(key)}
- else:
- return {'required': True}
-
-
-def archive_card(card):
- print('Archiving old revision:', card)
- card.set_closed(True)
-
-
-def move_card(config, lane_name, card):
- """Move trello cards according to the snap current channel."""
- print(card.name)
- m = re.match(
- r"(?P<snap>.*?)(?:\s+\-\s+)(?P<version>.*?)(?:\s+\-\s+)"
- r"\((?P<revision>.*?)\)(?:\s+\-\s+\[(?P<track>.*?)\])?", card.name)
- if m:
- arch = config[m.group("snap")]["arch"]
- headers = {
- 'Snap-Device-Series': '16',
- 'Snap-Device-Architecture': arch,
- 'Snap-Device-Store': config[m.group("snap")]["store"],
- }
- req = requests.get(
- 'https://api.snapcraft.io/v2/'
- 'snaps/info/{}'.format(m.group("snap")),
- headers=headers)
- json_resp = req.json()
- track = m.group("track")
- if not track:
- track = 'latest'
- for channel_info in json_resp["channel-map"]:
- if (channel_info["channel"]["track"] == track and
- channel_info["channel"]["architecture"] == arch):
- risk = channel_info["channel"]["risk"]
- try:
- version = channel_info['version']
- revision = str(channel_info['revision'])
- except KeyError:
- continue
- ori = next_risk = channel_promotion_map[lane_name]
- # If the snap with this name, in this channel is a
- # differet revision, then this is old so archive it
- if (risk == lane_name and
- revision != m.group("revision")):
- archive_card(card)
- continue
- if (
- risk == next_risk and
- version == m.group("version") and
- revision == m.group("revision")
- ):
- for l in card.board.open_lists():
- if ori.capitalize() == l.name:
- card.change_list(l.id)
- return
-
-
-def main():
- logger = logging.getLogger("trello-board-manager")
- parser = argparse.ArgumentParser()
- parser.add_argument('--key', help="Trello API key",
- **environ_or_required('TRELLO_API_KEY'))
- parser.add_argument('--token', help="Trello OAuth token",
- **environ_or_required('TRELLO_TOKEN'))
- parser.add_argument('--board', help="Trello board identifier",
- **environ_or_required('TRELLO_BOARD'))
- parser.add_argument('config', help="snaps configuration",
- type=argparse.FileType())
- args = parser.parse_args()
- client = TrelloClient(api_key=args.key, token=args.token)
- board = client.get_board(args.board)
- try:
- config = safe_load(args.config)
- except ParserError:
- raise SystemExit('Error parsing %s' % args.config.name)
- for lane in board.list_lists():
- lane_name = lane.name.lower()
- if lane_name in channel_promotion_map.keys():
- for card in lane.list_cards():
- try:
- move_card(config, lane_name, card)
- except Exception:
- logger.warn("WARNING", exc_info=True)
- continue
-
-
-if __name__ == "__main__":
- main()
diff --git a/trello-board-updater-desktop.py b/trello-board-updater-desktop.py
deleted file mode 100755
index 719fc92..0000000
--- a/trello-board-updater-desktop.py
+++ /dev/null
@@ -1,623 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# Copyright 2019 Canonical Ltd.
-# All rights reserved.
-#
-# Written by:
-# Taihsiang Ho <taihsiang.ho@xxxxxxxxxxxxx>
-# Sylvain Pineau <sylvain.pineau@xxxxxxxxxxxxx>
-#
-#
-# The script uses py-trello package 0.9.0. You may want to fetch it from
-# source.
-#
-# This script will create or update corresponding cards for each kernel and
-# SUTs
-#
-import argparse
-import importlib
-import os
-import re
-import requests
-import sys
-import yaml
-import logging
-import collections
-
-import unittest
-import json
-import trello
-
-from urllib.parse import urlparse
-from datetime import datetime
-from trello import TrelloClient
-from trello.exceptions import ResourceUnavailable
-
-from unittest.mock import MagicMock
-
-tbu = importlib.import_module("trello-board-updater")
-
-format_str = "[ %(funcName)s() ] %(message)s"
-logging.basicConfig(level=logging.INFO, format=format_str)
-
-
-def run(args, board, c3_link, jenkins_link):
-
- kernel_stack = args.series
- if args.sru_type == 'stock-hwe':
- kernel_stack = args.series + '-hwe'
-
- # The current oem stack
- # For now it is bionic with linux-oem kernel
- # TODO: update the if statement when oem-osp1 is delivered
- if kernel_stack == 'bionic' and 'oem' in args.kernel:
- if 'osp1' in args.kernel:
- kernel_stack = 'oem-osp1'
- else:
- kernel_stack = 'oem_bionic'
- if kernel_stack == 'focal' and 'oem' in args.kernel:
- kernel_stack = 'oem_focal'
-
- uri = urlparse(jenkins_link)
- jenkins_host = '{uri.scheme}://{uri.netloc}/'.format(uri=uri)
-
- # TODO: we could merge main and universe repositories from the source
- # jenkins jobs
- # linux-raspi is in universe for bionic
- if args.series == 'bionic' and 'raspi' in args.kernel:
- package_json_name_template = '{}-universe-{}-proposed.json'
- else:
- # packages of generic kernels
- # projects using these kernels:
- # stock images
- # oem image - xenial
- # oem images - shipped with oem-4.13
- # argos dgx-1/dgx-station images
- package_json_name_template = '{}-main-{}-proposed.json'
-
- package_json_url_template = '{}/job/cert-package-data/'\
- 'lastSuccessfulBuild/artifact/' + \
- package_json_name_template
-
- package_json_url = package_json_url_template.format(jenkins_host,
- args.series,
- args.arch)
- logging.info('package json url: {}'.format(package_json_url))
- response = requests.get(url=package_json_url)
- package_data = response.json()
-
- # Since bionic-oem and bionic-oem-osp1 are retired, we should move oem item to generic card.
- # In order to distinguish them, we add a prefix to the item name.
- if 'oem' in args.kernel and 'oem' in args.sru_type and args.series == 'bionic':
- print(kernel_stack,args.kernel)
- kernel_stack = 'bionic-hwe'
- if 'osp1' in args.kernel:
- args.name = 'oem-osp1-'+ args.name + '-hwe'
- else:
- args.name = 'oem-'+ args.name + '-hwe'
- #since some focal-oem machines will upgrade to 5.8-hwe kernel, therefore we should move those item to focal-hwe card.
- if 'hwe' in args.kernel and 'oem' in args.sru_type and args.series == 'focal':
- print(kernel_stack,args.kernel)
- kernel_stack = 'focal-hwe'
- args.name = 'oem-'+ args.name + '-hwe'
-
- # linux deb version
- # e.g. linux-generic-hwe-16.04 which version is 4.15.0.50.71
- dlv = package_data[args.kernel].split('.')
- dlv_short = dlv[0] + '_' + dlv[1] + '_' + dlv[2] + '-' + dlv[3]
- logging.info("linux deb version: {}".format(dlv))
- logging.info("linux deb version (underscores): {}".format(dlv_short))
- kernel_suffix = kernel_stack.split('_')[0]
- if args.sru_type == 'stock' or args.sru_type == 'stock-hwe':
- # for stock images, it always uses generic kernels
- #
- # GA example:
- # linux-generic -->
- # linux-image-4_15_0-55-generic (4.15.0.55.57)
- #
- # hwe stack example:
- # linux-generic-hwe-18_04 -->
- # linux-image-5_0_0-21-generic (5.0.0-21.22~18.04.1)
- kernel_suffix = 'generic'
- elif args.sru_type == 'oem' and "xenial" in args.series:
- # very special case: oem xenial images
- # oem xenial 4.4 kernel is using generic kernel, besides,
- # some oem images are delivered as xenial + oem-4.13
- # when time goes by, it is updated to be xenial + generic xenial hwe
- #
- # this if condition includes the dgx-1 and dgx-station images
- #
- # TODO: we may need to add more conditions when more oem images
- # is updated to use generic kernel
- kernel_suffix = 'generic'
- elif args.sru_type == 'oem' and "bionic" in args.series:
- # bionic oem image has started using generic kernel.
- # bionic-oem and bionic-oem-osp1 will upgrade to 5.4 kernel.
- kernel_suffix = 'generic'
- elif args.sru_type == 'oem' and "focal" in args.series and 'hwe' in args.kernel:
- # some focal-oem machines will upgrade to 5.8-hwe kernel.
- kernel_suffix = 'generic'
-
- # For raspi kernels
- if 'raspi' in args.kernel:
- if "xenial" in args.series:
- kernel_suffix = 'raspi2'
- if "bionic" in args.series:
- if "hwe" in args.kernel:
- kernel_suffix = 'raspi'
- else:
- kernel_suffix = 'raspi2'
- else:
- kernel_suffix = 'raspi'
- kernel_stack = kernel_stack + '-' + kernel_suffix
-
- logging.info("kernel_suffix: {}".format(kernel_suffix))
-
- deb_kernel_image = 'linux-image-' + dlv_short + '-' + kernel_suffix
-
- deb_version = package_data[deb_kernel_image]
- pattern = "{} - {} - \({}\)".format(
- re.escape(kernel_stack),
- re.escape(deb_kernel_image),
- re.escape(deb_version))
- card = tbu.search_card(board, pattern)
- config = tbu.load_config(args.config, None)
- expected_tests = config.get(kernel_stack, {}).get('expected_tests', [])
-
- logging.info('SRU type: {}'.format(args.sru_type))
- logging.info('series: {}'.format(args.series))
- logging.info("kernel_stack: {}".format(kernel_stack))
- logging.info("deb_kernel_image: {}".format(deb_kernel_image))
- logging.info("deb_version: {}".format(deb_version))
- logging.info("expected_tests and SUTs: {}".format(expected_tests))
-
- lanes = ['Proposed', 'Updates']
-
- if not card:
- lane = None
- for l in board.open_lists():
- # TODO: not a reasonable condition, use better one later
- if l.name == lanes[0] and \
- package_data[deb_kernel_image] == deb_version:
- lane = l
- break
- if lane:
- print("No target card was found. Create an new one...")
- card = lane.add_card('{} - {} - ({})'.format(kernel_stack,
- deb_kernel_image,
- deb_version))
- else:
- print('No target card and lane was found. Give up to create an '
- 'new card.')
- sys.exit(1)
- if not args.cardonly:
- summary = '**[TESTFLINGER] {} {} {} {} ({})**\n---\n\n'.format(
- args.name, args.arch, args.kernel, deb_kernel_image, deb_version)
- summary += '- Jenkins build details: {}\n'.format(jenkins_link)
- summary += '- Full results at: {}\n\n```\n'.format(c3_link)
- summary_data = args.summary.read()
- summary += summary_data
- summary += '\n```\n'
- comment = card.comment(summary)
- comment_link = "{}#comment-{}".format(card.url, comment['id'])
- else:
- summary_data = ""
- checklist = tbu.find_or_create_checklist(card, 'Testflinger', expected_tests)
-
- # Need to test different arch on the same device, separate the results
- # TODO: We should support different arch for all devices, not just raspi
- if 'raspi' in args.kernel:
- args.name = args.name + '-' + args.arch
-
- if args.cardonly:
- item_content = "{} ({})".format(args.name, 'In progress')
- else:
- item_content = "[{}]({}) ({})".format(
- args.name, comment_link, datetime.utcnow().isoformat())
- if jenkins_link:
- item_content += " [[JENKINS]({})]".format(jenkins_link)
- if c3_link:
- item_content += " [[C3]({})]".format(c3_link)
- elif not args.cardonly:
- # If there was no c3_link, it's because the submission failed
- tbu.attach_labels(board, card, ['TESTFLINGER CRASH'])
-
- # debug message
- logging.info('checklist: {}'.format(checklist))
- logging.info('item_content: {}'.format(item_content))
- if not tbu.change_checklist_item(
- checklist, args.name, item_content,
- checked=tbu.no_new_fails_or_skips(summary_data)):
- checklist.add_checklist_item(item_content)
-
- if not [c for c in card.fetch_checklists() if c.name == 'Sign-Off']:
- checklist = tbu.find_or_create_checklist(card, 'Sign-Off')
- checklist.add_checklist_item('Ready for ' + lanes[0], True)
- checklist.add_checklist_item('Ready for ' + lanes[1])
- checklist.add_checklist_item('Can be Archived')
- checklist = tbu.find_or_create_checklist(card, 'Revisions')
- rev = '{} ({})'.format(deb_version, args.arch)
- if rev not in [item['name'] for item in checklist.items]:
- checklist.add_checklist_item(rev)
-
- # a read trello card object, useful for testing
- k_deb_card = collections.namedtuple("KernelDeb", ["kernel_stack",
- "deb_kernel_image",
- "deb_version",
- "expected_tests",
- "sut"])
- # card title
- k_deb_card.kernel_stack = kernel_stack
- k_deb_card.deb_kernel_image = deb_kernel_image
- k_deb_card.deb_version = deb_version
- # card content: SUTs
- k_deb_card.expected_tests = expected_tests
- k_deb_card.sut = args.name
-
- return k_deb_card
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--key', help="Trello API key",
- **tbu.environ_or_required('TRELLO_API_KEY'))
- parser.add_argument('--token', help="Trello OAuth token",
- **tbu.environ_or_required('TRELLO_TOKEN'))
- parser.add_argument('--board', help="Trello board identifier",
- **tbu.environ_or_required('TRELLO_BOARD'))
- parser.add_argument('--config', help="Pool configuration",
- type=argparse.FileType())
- parser.add_argument('-a', '--arch', help="deb architecture",
- required=True)
- parser.add_argument('-t', '--sru-type',
- help="SRU type, stock or oem etc.", required=True)
- parser.add_argument('-s', '--series',
- help="series code name, e.g. xenial etc.",
- required=True)
- parser.add_argument('-n', '--name', help="SUT name", required=True)
- parser.add_argument('-k', '--kernel', help="kernel type", required=True)
- # TODO: for now it is not required because I want to make it backward
- # compatible. If we could batch update the corresponding jenkins jobs then
- # we could make this argument required.
- parser.add_argument('-q', '--queue', help="kernel type", default="")
- parser.add_argument('summary', help="test results summary",
- type=argparse.FileType())
- parser.add_argument("--cardonly", help="Only create an empty card",
- action="store_true")
- args = parser.parse_args()
- client = TrelloClient(api_key=args.key, token=args.token)
- board = client.get_board(args.board)
- c3_link = os.environ.get('C3LINK', '')
- jenkins_link = os.environ.get('BUILD_URL', '')
-
- run(args, board, c3_link, jenkins_link)
-
-
-class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
-
- def _request_get(self):
- return requests.models.Response()
-
- def _get_package_data(self):
- with open(self.packages_info) as f:
- data = json.load(f)
-
- return data
-
- def _get_cards(self, board, card_id, name):
- card = trello.card.Card(board, card_id, name=name)
- return [card]
-
- def _mock_factory(self, jenkins_job_template, card_name, packages_info):
- parser = argparse.ArgumentParser()
- args = parser.parse_args([])
- args.__dict__.update(jenkins_job_template)
-
- self.args = args
- self.board = trello.board.Board("fake_board")
- self.c3_link = "fake_c3_link"
- self.jenkins_link = "fake_jenkins_link"
- self.packages_info = packages_info
-
- requests.get = MagicMock(return_value=self._request_get())
- requests.models.Response.json = MagicMock(
- side_effect=self._get_package_data)
- trello.board.Board.get_cards = MagicMock(return_value=self._get_cards(
- self.board,
- 9999,
- card_name))
- trello.board.Card.comment = MagicMock(return_value="fake_comment")
- trello.board.Card.fetch_checklists = MagicMock(
- return_value=[])
- mock_checklist = MagicMock()
- mock_checklist.add_checklist_item = print
-
- trello.board.Card.add_checklist = MagicMock(
- return_value=mock_checklist)
-
- def setUp(self):
- self.debs_yaml_stream = open('./data/debs.yaml')
- self.summary_stream = open('./data/raw_summary')
-
- def tearDown(self) -> None:
- self.debs_yaml_stream.close()
- self.summary_stream.close()
-
- def test_stock_xenial_4_4_kernel_stack(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'xenial-desktop-201606-22344',
- 'arch': 'amd64',
- 'kernel': 'linux-generic',
- 'series': 'xenial',
- 'sru_type': 'stock',
- 'queue': '',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "xenial - linux-image-4_4_0-167-generic - (4.4.0-167.196)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_xenial-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- self.assertEqual(kdeb_card.kernel_stack, "xenial")
-
- def test_stock_xenial_4_15_kernel_stack(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'xenial-hwe-desktop-201606-22344',
- 'arch': 'amd64',
- 'kernel': 'linux-generic-hwe-16_04',
- 'series': 'xenial',
- 'sru_type': 'stock',
- 'queue': '',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "xenial-hwe - linux-image-4_15_0-66-generic " \
- "- (4.15.0-66.75~16.04.1)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_xenial-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- self.assertEqual(kdeb_card.kernel_stack, "xenial-hwe")
-
- def test_stock_bionic_4_15_kernel_stack(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'bionic-desktop-201606-22344',
- 'arch': 'amd64',
- 'kernel': 'linux-generic',
- 'series': 'bionic',
- 'sru_type': 'stock',
- 'queue': '',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "bionic - linux-image-4_15_0-67-generic - (4.15.0-67.76)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_bionic-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- self.assertEqual(kdeb_card.kernel_stack, "bionic")
-
- def test_oem_xenial_4_4_kernel_stack(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'xenial-desktop-201610-25144',
- 'arch': 'amd64',
- 'kernel': 'linux-generic',
- 'series': 'xenial',
- 'sru_type': 'oem',
- 'queue': '',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "xenial - linux-image-4_4_0-167-generic - (4.4.0-167.196)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_xenial-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- self.assertEqual(kdeb_card.kernel_stack, "xenial")
-
- def test_oem_xenial_4_15_kernel_stack(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'xenial-hwe-desktop-201802-26107',
- 'arch': 'amd64',
- 'kernel': 'linux-generic-hwe-16_04',
- 'series': 'xenial',
- 'sru_type': 'oem',
- 'queue': '',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "xenial-hwe - linux-image-4_15_0-66-generic " \
- "- (4.15.0-66.75~16.04.1)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_xenial-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- self.assertEqual(kdeb_card.kernel_stack, "xenial-hwe")
-
- def test_oem_bionic_4_15_kernel_stack(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'bionic-desktop-201802-26107',
- 'arch': 'amd64',
- 'kernel': 'linux-oem',
- 'series': 'bionic',
- 'sru_type': 'oem',
- 'queue': '',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "oem - linux-image-4_15_0-1059-oem - (4.15.0-1059.68)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_bionic-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- self.assertEqual(kdeb_card.kernel_stack, "oem")
-
- def test_oem_osp1_bionic_4_15_kernel_stack(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'bionic-desktop-201906-27089',
- 'arch': 'amd64',
- 'kernel': 'linux-oem-osp1',
- 'series': 'bionic',
- 'sru_type': 'oem',
- 'queue': '',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "oem-osp1 - linux-image-5_0_0-1025-oem-osp1 " \
- "- (5.0.0-1025.28)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_bionic-universe-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- self.assertEqual(kdeb_card.kernel_stack, "oem-osp1")
-
- def test_argos_dgx_station_xenial_4_4(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'xenial-desktop-201711-25989',
- 'arch': 'amd64',
- 'kernel': 'linux-generic',
- 'series': 'xenial',
- 'sru_type': 'oem',
- 'queue': 'argos-201711-25989',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "xenial - linux-image-4_4_0-167-generic - (4.4.0-167.196)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_xenial-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- target_sut = "201711-25989-dgx-station"
- target_sut_kdc_id = kdeb_card.expected_tests.index(target_sut)
-
- self.assertEqual(kdeb_card.kernel_stack, "xenial")
- self.assertEqual(kdeb_card.sut, target_sut)
- self.assertEqual(kdeb_card.expected_tests[target_sut_kdc_id],
- target_sut)
-
- def test_argos_dgx_1_xenial_4_4(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'xenial-server-201802-26098',
- 'arch': 'amd64',
- 'kernel': 'linux-generic',
- 'series': 'xenial',
- 'sru_type': 'oem',
- 'queue': 'argos-201802-26098',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "xenial - linux-image-4_4_0-167-generic - (4.4.0-167.196)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_xenial-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- target_sut = "201802-26098-dgx-1"
- target_sut_kdc_id = kdeb_card.expected_tests.index(target_sut)
-
- self.assertEqual(kdeb_card.kernel_stack, "xenial")
- self.assertEqual(kdeb_card.sut, target_sut)
- self.assertEqual(kdeb_card.expected_tests[target_sut_kdc_id],
- target_sut)
-
- def test_argos_dgx_1_xenial_hwe(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'xenial-hwe-server-201802-26098',
- 'arch': 'amd64',
- 'kernel': 'linux-generic-hwe-16_04',
- 'series': 'xenial',
- 'sru_type': 'oem',
- 'queue': 'argos-201802-26098',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "xenial-hwe - linux-image-4_15_0-66-generic " \
- "- (4.15.0-66.75~16.04.1)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_xenial-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- target_sut = "201802-26098-dgx-1"
- target_sut_kdc_id = kdeb_card.expected_tests.index(target_sut)
-
- self.assertEqual(kdeb_card.kernel_stack, "xenial-hwe")
- self.assertEqual(kdeb_card.sut, target_sut)
- self.assertEqual(kdeb_card.expected_tests[target_sut_kdc_id],
- target_sut)
-
- def test_oem_focal_5_6_kernel_stack(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'focal-desktop-xps13-9310-c1',
- 'arch': 'amd64',
- 'kernel': 'linux-oem',
- 'series': 'focal',
- 'sru_type': 'oem',
- 'queue': '',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "oem_focal - linux-image-oem-20_04 - (5.6.0.1034.30)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_focal-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- self.assertEqual(kdeb_card.kernel_stack, "oem")
-
- def test_stock_focal_5_4_kernel_stack(self):
- jenkins_job_template = {
- 'cardonly': False,
- 'name': 'focal-desktop-inspiron20-3064',
- 'arch': 'amd64',
- 'kernel': 'linux-generic',
- 'series': 'focal',
- 'sru_type': 'stock',
- 'queue': '',
- 'config': self.debs_yaml_stream,
- 'summary': self.summary_stream
- }
- card_name = "focal - linux-image-5_4_0-54-generic - (5.4.0-54.60)"
-
- self._mock_factory(jenkins_job_template, card_name,
- "./data/deb-package_focal-main-amd64.json")
-
- kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link)
-
- self.assertEqual(kdeb_card.kernel_stack, "focal")
-
-
-if __name__ == "__main__":
- main()
diff --git a/trello-board-updater-image-testing.py b/trello-board-updater-image-testing.py
deleted file mode 100755
index ca97b92..0000000
--- a/trello-board-updater-image-testing.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2019-2020 Canonical Ltd.
-# All rights reserved.
-#
-# Written by:
-# Paul Larson <paul.larson@xxxxxxxxxxxxx>
-# Sylvain Pineau <sylvain.pineau@xxxxxxxxxxxxx>
-
-import argparse
-import importlib
-import os
-import re
-
-from datetime import datetime
-from trello import TrelloClient
-
-tbu = importlib.import_module("trello-board-updater")
-tbm = importlib.import_module("trello-board-manager")
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--key', help="Trello API key",
- **tbu.environ_or_required('TRELLO_API_KEY'))
- parser.add_argument('--token', help="Trello OAuth token",
- **tbu.environ_or_required('TRELLO_TOKEN'))
- parser.add_argument('--board', help="Trello board identifier",
- **tbu.environ_or_required('TRELLO_BOARD'))
- parser.add_argument('-n', '--name', help="SUT device name", required=True)
- parser.add_argument('-i', '--image', help="image name", required=True)
- parser.add_argument('-c', '--channel', help="image name", default="stable")
- parser.add_argument('-v', '--version', help="snap version", required=True)
- parser.add_argument('summary', help="test results summary",
- type=argparse.FileType())
- args = parser.parse_args()
- client = TrelloClient(api_key=args.key, token=args.token)
- board = client.get_board(args.board)
- c3_link = os.environ.get('C3LINK', '')
- jenkins_link = os.environ.get('BUILD_URL', '')
- pattern = "{} - {} - {}".format(
- re.escape(args.image),
- re.escape(args.channel),
- re.escape(args.version))
- # First, see if this exact card already exists
- card = tbu.search_card(board, pattern)
-
- # If not, see if there's an older one for this image
- if not card:
- pattern = "{} - {} - .*".format(re.escape(args.image), args.channel)
- card = tbu.search_card(board, pattern)
- if card:
- tbm.archive_card(card)
- # If we get here, then either we just archived the old card, or
- # it didn't exist. We need to create it either way
- lane = None
- for l in board.open_lists():
- if l.name == args.channel:
- lane = l
- break
- if not lane:
- lane = board.add_list(args.channel)
- card = lane.add_card('{} - {} - {}'.format(
- args.image, args.channel, args.version))
- summary = '**[TESTFLINGER] {} {} {}**\n---\n\n'.format(
- args.name, args.image, args.version)
- summary += '- Jenkins build details: {}\n'.format(jenkins_link)
- summary += '- Full results at: {}\n\n```\n'.format(c3_link)
- summary_data = args.summary.read()
- summary += summary_data
- summary += '\n```\n'
- comment = card.comment(summary)
- comment_link = "{}#comment-{}".format(card.url, comment['id'])
- checklist = tbu.find_or_create_checklist(card, 'Testflinger')
- item_content = "[{}]({}) ({})".format(
- args.name, comment_link, datetime.utcnow().isoformat())
- if jenkins_link:
- item_content += " [[JENKINS]({})]".format(jenkins_link)
- if c3_link:
- item_content += " [[C3]({})]".format(c3_link)
-
- if not tbu.change_checklist_item(
- checklist, args.name, item_content,
- checked=tbu.no_new_fails_or_skips(summary_data)):
- checklist.add_checklist_item(
- item_content, checked=tbu.no_new_fails_or_skips(summary_data))
-
-
-if __name__ == "__main__":
- main()
diff --git a/trello-board-updater.py b/trello-board-updater.py
deleted file mode 100755
index 9d9af50..0000000
--- a/trello-board-updater.py
+++ /dev/null
@@ -1,318 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2017 Canonical Ltd.
-# All rights reserved.
-#
-# Written by:
-# Sylvain Pineau <sylvain.pineau@xxxxxxxxxxxxx>
-
-import argparse
-import os
-import re
-import requests
-import sys
-import time
-import yaml
-
-from datetime import datetime
-from trello import TrelloClient
-from trello.exceptions import ResourceUnavailable
-from mailtool import send_mail
-
-
-def environ_or_required(key):
- if os.environ.get(key):
- return {'default': os.environ.get(key)}
- else:
- return {'required': True}
-
-
-def search_card(board, query, card_filter="open"):
- for card in board.get_cards(card_filter=card_filter):
- if re.match(query, card.name):
- return card
-
-
-def find_or_create_checklist(card, checklist_name, items=[]):
- existing_checklists = card.fetch_checklists()
- checklist = None
- for c in existing_checklists:
- if c.name == checklist_name:
- checklist = c
- break
- if not checklist:
- print("Creating checklist: {}".format(checklist_name))
- checklist = card.add_checklist(checklist_name, [])
- for item in items:
- checklist.add_checklist_item(item + ' (NO RESULTS)')
- return checklist
-
-
-def change_checklist_item(checklist, name, content, checked=False):
- """Attempt to rename and change details on a checklist item
-
- Return True if the item is located and changes are made
- Return False if no matching item was found in the specified checklist
-
- Since some cards have multiple results checklists, a False return
- may be expected in many cases
- """
- for item in checklist.items:
- if (
- name + ' ' in item.get('name') or
- name + ']' in item.get('name')
- ):
- checklist.rename_checklist_item(item.get('name'), content)
- checklist.set_checklist_item(content, checked)
- return True
- else:
- return False
-
-
-def no_new_fails_or_skips(summary_data):
- """Check summary data for new fails or skips
-
- Return True if there are no new fails or skips detected and if
- more than 1 test actually ran as a sanity check
- """
- return ("No new failed or skipped tests" in summary_data and
- "WARNING: Very small number of total tests" not in summary_data)
-
-
-def load_config(configfile, snapname):
- if not configfile:
- return []
- try:
- data = yaml.safe_load(configfile)
- except (yaml.parser.ParserError, yaml.scanner.ScannerError):
- print('ERROR: Error parsing', configfile.name)
- sys.exit(1)
- return data
-
-
-def attach_labels(board, card, label_list):
- for labelstr in label_list:
- for label in board.get_labels():
- if label.name == labelstr:
- labels = card.labels or []
- if label not in labels:
- # Avoid crash if checking labels fails to find it
- try:
- card.add_label(label)
- except ResourceUnavailable:
- pass
- break
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--key', help="Trello API key",
- **environ_or_required('TRELLO_API_KEY'))
- parser.add_argument('--token', help="Trello OAuth token",
- **environ_or_required('TRELLO_TOKEN'))
- parser.add_argument('--board', help="Trello board identifier",
- **environ_or_required('TRELLO_BOARD'))
- parser.add_argument('--config', help="Snaps configuration",
- type=argparse.FileType())
- parser.add_argument('-a', '--arch', help="snap architecture",
- required=True)
- parser.add_argument('-b', '--brandstore', help="brand store identifier",
- default='ubuntu')
- parser.add_argument('-e', '--email', help="Email recipients")
- parser.add_argument('-n', '--name', help="SUT name", required=True)
- parser.add_argument('-s', '--snap', help="snap name", required=True)
- parser.add_argument('-v', '--version', help="snap version", required=True)
- parser.add_argument('-r', '--revision', help="snap revision",
- required=True)
- parser.add_argument('-c', '--channel', help="snap channel", required=True)
- parser.add_argument('-t', '--track', help="snap track", required=True)
- parser.add_argument('summary', help="test results summary",
- type=argparse.FileType())
- parser.add_argument("--cardonly", help="Only create an empty card",
- action="store_true")
- args = parser.parse_args()
- client = TrelloClient(api_key=args.key, token=args.token)
- board = client.get_board(args.board)
- track = args.track.replace('__track__', '')
- c3_link = os.environ.get('C3LINK', '')
- jenkins_link = os.environ.get('BUILD_URL', '')
- pattern = "{} - {} - \({}\).*{}".format(
- re.escape(args.snap),
- re.escape(args.version),
- args.revision,
- re.escape(track))
- card = search_card(board, pattern)
- config = load_config(args.config, args.snap)
- snap_labels = config.get(args.snap, {}).get('labels', [])
- # Try to find the right card using other arch revision numbers
- # i.e. We could be recording a result for core rev 111 on armhf, but we
- # want to record it against the core rev 110 results with amd64, so find
- # that card by finding the arch/rev from the store to use in the search
- if not card:
- rev_list = dict()
- headers = {
- 'Snap-Device-Series': '16',
- 'Snap-Device-Store': args.brandstore,
- }
- try:
- json = requests.get(
- 'https://api.snapcraft.io/v2/'
- 'snaps/info/{}'.format(args.snap),
- headers=headers).json()
- except Exception:
- # Sometimes either the request is bad or it returns something that
- # .json() doesn't like. If it's a short-term issue, maybe we can
- # give it one more chance to pass
- print("WARNING: Bad store request, retrying...")
- time.sleep(30)
- json = requests.get(
- 'https://api.snapcraft.io/v2/'
- 'snaps/info/{}'.format(args.snap),
- headers=headers).json()
- # store_track is used for searching for the right track name in the
- # store, which would be latest if nothing is defined
- # track is used for search in the cards, which will be either the
- # default_track, the defined track for the run, or empty for 'latest'
- track = config.get(args.snap, {}).get('default_track', track)
- store_track = track or "latest"
- for channel_info in json['channel-map']:
- try:
- if channel_info['version'] != args.version:
- continue
- if (channel_info["channel"]["track"] == store_track and
- channel_info["channel"]["risk"] == args.channel):
- arch = channel_info["channel"]["architecture"]
- rev_list[arch] = channel_info['revision']
- except KeyError:
- continue
- # Try to find the right revision for the right architecture when a
- # channel follows the next one meaning nothing was pushed to it, e.g:
- #
- # channels:
- # stable: 0.7.1 2018-12-11 (329) 387MB -
- # candidate: ↑
- # beta: ↑
- if not rev_list:
- for risk in ['edge', 'beta', 'candidate', 'stable']:
- for channel_info in json['channel-map']:
- try:
- if channel_info['version'] != args.version:
- continue
- if channel_info["channel"]["track"] == store_track:
- if channel_info["channel"]["risk"] == risk:
- arch = channel_info["channel"]["architecture"]
- rev_list[arch] = channel_info['revision']
- except KeyError:
- continue
- if rev_list:
- break
- for rev in rev_list.values():
- pattern = "{} - {} - \({}\).*{}".format(
- re.escape(args.snap),
- re.escape(args.version),
- rev,
- re.escape(track))
- card = search_card(board, pattern)
- if card:
- # Prefer amd64 rev in card title
- if args.arch == 'amd64':
- if track:
- card.set_name('{} - {} - ({}) - [{}]'.format(
- args.snap, args.version, args.revision,
- track))
- else:
- card.set_name('{} - {} - ({})'.format(
- args.snap, args.version, args.revision))
- break
- print('Using card: {} ({})'.format(card.name, card.short_url))
- # Create the card in the right lane, since we still didn't find it
- # We only want one card for all architectures, so use the revision
- # declared for the default arch in snaps.yaml
- if not card:
- default_arch = config.get(args.snap, {}).get('arch', args.arch)
- default_rev = rev_list.get(default_arch, args.revision)
- channel = args.channel.capitalize()
- lane = None
- # Use the default_track if there is one, else use track name specified
- track = config.get(args.snap, {}).get('default_track', track)
- for l in board.open_lists():
- if channel == l.name:
- lane = l
- break
- if lane:
- if track:
- card = lane.add_card('{} - {} - ({}) - [{}]'.format(
- args.snap, args.version, default_rev, track))
- else:
- card = lane.add_card('{} - {} - ({})'.format(
- args.snap, args.version, default_rev))
- print('No card found!')
- print('Creating card: {} ({})'.format(card.name, card.short_url))
- if args.email:
- msg_subject = '[SUV-RESULT] {}'.format(card.name)
- msg_body = ('New results posted for {} version {}\n\n'
- '{}\n{}'.format(args.snap, args.version, card.name, card.short_url))
- try:
- send_mail(to=args.email, subject=msg_subject, body=msg_body)
- except Exception as e:
- print('WARNING: Unable to send email: {}'.format(e))
- if not args.cardonly:
- summary = '**[TESTFLINGER] {} {} {} ({}) {}**\n---\n\n'.format(
- args.name, args.snap, args.version, args.revision, args.channel)
- summary += '- Jenkins build details: {}\n'.format(jenkins_link)
- summary += '- Full results at: {}\n\n```\n'.format(c3_link)
- summary_data = args.summary.read()
- summary += summary_data
- summary += '\n```\n'
- comment = card.comment(summary)
- comment_link = "{}#comment-{}".format(card.url, comment['id'])
- else:
- summary_data = ""
- attach_labels(board, card, snap_labels)
- card_checklists = config.get(args.snap, {}).get('checklists', {})
- for checklist_name in card_checklists.keys():
- # expected_tests section in the yaml could be either a list of all
- # devices we want to see tests results on, or a dict of tracks with
- # those devices specified in a list under each track (ex. pi-kernel)
- expected_tests = card_checklists.get(
- checklist_name, {}).get('expected_tests', [])
- if isinstance(expected_tests, dict):
- # If multiple tracks are defined, only look at expected tests
- # for the track we care about
- expected_tests = expected_tests.get(track, [])
- checklist = find_or_create_checklist(
- card, checklist_name, expected_tests)
- if args.cardonly:
- item_content = "{} ({})".format(args.name, 'In progress')
- else:
- item_content = "[{}]({}) ({})".format(
- args.name, comment_link, datetime.utcnow().isoformat())
- if jenkins_link:
- item_content += " [[JENKINS]({})]".format(jenkins_link)
- if c3_link:
- item_content += " [[C3]({})]".format(c3_link)
- elif not args.cardonly:
- # If there was no c3_link, it's because the submission failed
- attach_labels(board, card, ['TESTFLINGER CRASH'])
-
- change_checklist_item(
- checklist, args.name, item_content,
- checked=no_new_fails_or_skips(summary_data))
-
- if not [c for c in card.fetch_checklists() if c.name == 'Sign-Off']:
- checklist = find_or_create_checklist(card, 'Sign-Off')
- checklist.add_checklist_item('Clear for Landing', True)
- checklist.add_checklist_item('Ready for Edge', True)
- checklist.add_checklist_item('Ready for Beta')
- if args.channel == 'beta':
- checklist.set_checklist_item('Ready for Beta', True)
- checklist.add_checklist_item('Ready for Candidate')
- checklist.add_checklist_item('Ready for Stable')
- checklist.add_checklist_item('Can be Archived')
- checklist = find_or_create_checklist(card, 'Revisions')
- rev = '{} ({})'.format(args.revision, args.arch)
- if rev not in [item['name'] for item in checklist.items]:
- checklist.add_checklist_item(rev)
-
-
-if __name__ == "__main__":
- main()
diff --git a/trello-sru-bug-manager.py b/trello-sru-bug-manager.py
deleted file mode 100755
index 6f0159b..0000000
--- a/trello-sru-bug-manager.py
+++ /dev/null
@@ -1,383 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2018 Canonical Ltd
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-# Written by:
-# Paul Larson <paul.larson@xxxxxxxxxxxxx>
-
-import argparse
-import datetime
-import os
-import re
-import requests
-import sys
-import traceback
-import json
-
-from launchpadlib.launchpad import Launchpad, uris
-from trello import TrelloClient
-from trello.exceptions import ResourceUnavailable
-
-
-class LPHelper:
- def __init__(self, credentials, project, staging):
- lp_server = uris.STAGING_SERVICE_ROOT if staging else uris.LPNET_SERVICE_ROOT
- self.lp = Launchpad.login_with(
- sys.argv[0], lp_server, credentials_file=credentials)
- self.project = self.lp.projects(project)
- self.srubugs = []
-
- def _build_sru_bugs(self):
- """Find a bug in this project with the specified search_text"""
- if not self.srubugs:
- try:
- # Take the kernel sru bug status data and make a list out of it
- # so we can use it more easily
- url = ('https://kernel.ubuntu.com/~kernel-ppa/status/'
- 'swm/status.json')
- bugdata = json.loads(requests.get(url).content)['trackers']
- for x in bugdata:
- # Add a 'bug' field that we can use later
- bugdata[x]['bug'] = x
- self.srubugs.append(bugdata[x])
- except Exception:
- print('ERROR: Importing bug data from {}'.format(url))
-
- def find_sru_bug(self, target, version, target_type="snap"):
- self._build_sru_bugs()
-
- if target_type == "snap":
- return self.find_sru_bug_snap(target, version)
- else:
- return self.find_sru_bug_deb(target, version)
-
- def find_sru_bug_snap(self, snap, version):
- try:
- # Only match if there's a valid task for this snap name, and
- # version is the same
- for bug in self.srubugs:
- if (bug.get('version') == version and
- bug.get('task', {}).get(
- 'snap-certification-testing', {}).get(
- 'target') == snap):
- return SruBug(
- self.lp.bugs(bug.get('bug')),
- bug.get('cycle'))
- # If we get this far, no bug was found
- raise LookupError
- except Exception:
- traceback.print_exc()
- print('WARNING: something went wrong, but continuing with other cards...')
- raise LookupError
-
- def find_sru_bug_deb(self, stack, version):
- series = stack.split('-')[0]
- package = 'linux'
- if 'hwe' in stack:
- if not 'xenial' in series:
- package += '-hwe-' + version.split('.')[0] + '.' + version.split('.')[1]
- else:
- package += '-hwe'
- # TODO: this will evolve when time goes by
- elif 'oem' in stack:
- series = series.split('_')[1]
- package += '-oem-' + version.split('.')[0] + '.' + version.split('.')[1]
-
- try:
- # Only match if there's a valid task for this stack name, and
- # version is the same
- for bug in self.srubugs:
- if (
- bug.get('version') == version and
- bug.get('package') == package and
- bug.get('series') == series and
- bug.get('variant') == 'debs' and
- bug.get('task').get("kernel-sru-workflow").get('status') == 'In Progress'
- ):
- return SruBug(
- self.lp.bugs(bug.get('bug')),
- bug.get('cycle'))
- # If we get this far, no bug was found
- raise LookupError
- except Exception:
- traceback.print_exc()
- print('WARNING: something went wrong, but continuing with other cards...')
- raise LookupError
-
-
-class TrelloHelper:
- def __init__(self, api_key, token, board):
- self.client = TrelloClient(api_key=api_key, token=token)
- self.board = self.client.get_board(board)
-
- def _get_lane(self, lane_id):
- """Search (case insensitive) for a lane called <lane_id>"""
- for l in self.board.list_lists():
- lane_name = l.name.lower()
- if lane_name == lane_id:
- return l
- raise LookupError('Trello lane "{}" not found!'.format(lane_id))
-
- def search_cards_in_lane(self, lane_id, search_text):
- """Yield cards with search_text in lane_id one at a time"""
- lane = self._get_lane(lane_id)
- for card in lane.list_cards():
- if search_text in card.name:
- yield card
-
-
-class SruBug:
- """Simplify operations we care about on an LP bug"""
- def __init__(self, bug, cycle):
- self.bug = bug
- self.id = bug.id
- self.web_link = bug.web_link
- self.cycle = cycle
- self.calculate_due_date()
-
- def __repr__(self):
- return self.bug.title
-
- def calculate_due_date(self):
- try:
- cycle_parts = tuple(
- int(x) for x in self.cycle.split('-')[0].split('.'))
- cycle_start = datetime.datetime(*cycle_parts, hour=12)
- # Since cycle starts on a Monday, +11 days is the due date
- # (Friday of the 2nd week in the cycle)
- self.due_date = cycle_start + datetime.timedelta(days=11)
- except Exception:
- self.due_date = None
- traceback.print_exc()
- print("WARNING: Unable to set due date for {}".format(self.id))
-
- def get_task_state(self, task_name):
- for task in self.bug.bug_tasks:
- if task_name in str(task):
- return task
- raise LookupError
-
- def set_task_state(self, task_name, state):
- for task in self.bug.bug_tasks:
- if task_name in str(task):
- task.status = state
- task.lp_save()
-
- def add_comment(self, comment_text):
- self.bug.newMessage(content=comment_text)
-
-
-def environ_or_required(key):
- """Mapping for argparse to supply required or default from $ENV."""
- if os.environ.get(key):
- return {'default': os.environ.get(key)}
- else:
- return {'required': True}
-
-
-def attach_labels(board, card, label_list):
- for labelstr in label_list:
- for label in board.get_labels():
- if label.name == labelstr:
- labels = card.labels or []
- if label not in labels:
- # Avoid crash if checking labels fails to find it
- try:
- card.add_label(label)
- except ResourceUnavailable:
- pass
- break
-
-
-def get_checklist_value(checklist, key):
- """Return the value of an item in a Trello checklist"""
- index = checklist._get_item_index(key)
- return checklist.items[index]['checked']
-
-
-def get_card_snap_version(card):
- """Return snap name and version for a card, if formatted as expected"""
- m = re.match(
- r"(?P<snap>.*?)(?:\s+\-\s+)(?P<version>.*?)(?:\s+\-\s+)"
- r"\((?P<revision>.*?)\)(?:\s+\-\s+\[(?P<track>.*?)\])?", card.name)
- if m:
- return (m.group('snap'), m.group('version'))
- raise ValueError
-
-
-def get_card_deb_version(card):
- """Return snap name and version for a card, if formatted as expected"""
- m = re.match(
- r"(?P<stack>.*?)(?:\s+\-\s+)(?P<package>.*?)(?:\s+\-\s+)"
- r"\((?P<version>.*?)\)", card.name)
- if m:
- return (m.group('stack'), m.group('version'))
- raise ValueError
-
-
-def get_args():
- parser = argparse.ArgumentParser()
- parser.add_argument('--project', help="Launchpad project to search",
- **environ_or_required('LAUNCHPAD_PROJECT'))
- parser.add_argument('--credentials', help="Specify launchpad credentials",
- **environ_or_required('LAUNCHPAD_CREDENTIALS'))
- parser.add_argument('--key', help="Trello API key",
- **environ_or_required('TRELLO_API_KEY'))
- parser.add_argument('--token', help="Trello OAuth token",
- **environ_or_required('TRELLO_TOKEN'))
- parser.add_argument('--board', help="Trello board identifier",
- **environ_or_required('TRELLO_BOARD'))
- parser.add_argument('--deb', action='store_true',
- help='Work on deb instead of snap')
- parser.add_argument('--staging', action='store_true',
- help='Use staging Launchpad for testing this script')
- return parser.parse_args()
-
-
-def card_ready_for_candidate(card):
- """Return True if the signoff checkbox 'Ready for Candidate' is checked"""
- # Find the Sign-Off checklist
- try:
- checklist = [x for x in card.fetch_checklists()
- if x.name == 'Sign-Off'][0]
- except IndexError:
- print("WARNING: No Sign-Off checklist found!")
- return False
- return get_checklist_value(checklist, 'Ready for Candidate')
-
-
-def card_ready_for_updates(card):
- """Return True if the signoff checkbox 'Ready for Updates' is checked"""
- # Find the Sign-Off checklist
- try:
- checklist = [x for x in card.fetch_checklists()
- if x.name == 'Sign-Off'][0]
- except IndexError:
- print("WARNING: No Sign-Off checklist found!")
- return False
- return get_checklist_value(checklist, 'Ready for Updates')
-
-
-def process_snaps(lp, trello):
- print("Processing SRU snaps ready for promotion...")
- for card in trello.search_cards_in_lane('beta', '-kernel'):
- print('{} ({})'.format(card.name, card.short_url))
- # If we can't get version from title, it's not formatted how we
- # expect, so ignore it
- try:
- snap, version = get_card_snap_version(card)
- except ValueError:
- continue
-
- try:
- bug = lp.find_sru_bug(snap, version)
- except LookupError:
- print(
- 'No bug found for {} or bug is already closed'.format(version))
- continue
-
- add_bug_description(bug, card)
-
- # Add due date to the card
- if bug.due_date:
- card.set_due(bug.due_date)
-
- # Automatically mark our task "In Progress" if it's "Confirmed"
- TARGET_TASK = 'snap-certification-testing'
- if bug.get_task_state(TARGET_TASK).status == 'Confirmed':
- bug.set_task_state(TARGET_TASK, 'In Progress')
-
- if not card_ready_for_candidate(card):
- continue
-
- update_lp(bug, 'snap-certification-testing', card)
-
-
-def process_debs(lp, trello):
- print("Processing SRU kernel debs ready for Updates repository...")
- for card in trello.search_cards_in_lane('proposed', 'linux-image'):
- print('{} ({})'.format(card.name, card.short_url))
- # If we can't get version from title, it's not formatted how we
- # expect, so ignore it
- try:
- stack, version = get_card_deb_version(card)
- except ValueError:
- continue
-
- try:
- bug = lp.find_sru_bug(stack, version, 'deb')
- except LookupError:
- print(
- 'No bug found for {} or bug is already closed'.format(version))
- continue
-
- add_bug_description(bug, card)
-
- # Add due date to the card
- if bug.due_date:
- card.set_due(bug.due_date)
-
- # Automatically mark our task "In Progress" if it's still "Confirmed"
- TARGET_TASK = 'certification-testing'
- if bug.get_task_state(TARGET_TASK).status == 'Confirmed':
- bug.set_task_state(TARGET_TASK, 'In Progress')
-
- if card_ready_for_updates(card):
- attach_labels(trello.board, card, ['READY FOR UPDATES'])
- else:
- continue
-
- update_lp(bug, 'certification-testing', card)
-
-
-def add_bug_description(bug, card):
- desc = "[[{}]({})] - {}".format(bug.id, bug.web_link, bug)
- card.set_description(desc)
-
-
-def update_lp(bug, target_task, card):
- print('- LP:{}'.format(bug.id))
- # If the bug is already fix-released, there's nothing more to do
- TARGET_TASK = target_task
- try:
- if bug.get_task_state(TARGET_TASK).status == 'Fix Released':
- print('- {} task already completed.'.format(TARGET_TASK))
- return
- except LookupError:
- print('ERROR: No task called "{}" found!'.format(TARGET_TASK))
- return
-
- # This is the bug for the card we found, mark the task complete and
- # add a comment
- print('- Marking {} task complete.'.format(TARGET_TASK))
- bug.set_task_state(TARGET_TASK, 'Fix Released')
- comment = ("Kernel deb testing completes, no regressions found. Ready "
- "for Updates. Results here: {}".format(card.url))
- bug.add_comment(comment)
-
-
-def main():
- args = get_args()
- lp = LPHelper(args.credentials, args.project, args.staging)
- trello = TrelloHelper(args.key, args.token, args.board)
-
- if args.deb:
- process_debs(lp, trello)
- else:
- process_snaps(lp, trello)
-
-
-if __name__ == "__main__":
- main()
Follow ups
