canonical-hw-cert team mailing list archive

[Kevin Yeh <mp+425997@xxxxxxxxxxxxxxxxxx>]

Kevin Yeh has proposed merging ~kevinyeh/cc-lab-manager:job_generator into cc-lab-manager:master.

Commit message:
add a script for generating jenkins jobs.
this version only targets on generic and oem kernel.

Requested reviews:
  Canonical Hardware Certification (canonical-hw-cert)

For more details, see:
https://code.launchpad.net/~kevinyeh/cc-lab-manager/+git/cc-lab-manager/+merge/425997
-- 
Your team Canonical Hardware Certification is requested to review the proposed merge of ~kevinyeh/cc-lab-manager:job_generator into cc-lab-manager:master.

diff --git a/cc_lab_manager/job_generator/__init__.py b/cc_lab_manager/job_generator/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cc_lab_manager/job_generator/__init__.py
diff --git a/cc_lab_manager/job_generator/job_generator.py b/cc_lab_manager/job_generator/job_generator.py
new file mode 100644
index 0000000..7f12976
--- /dev/null
+++ b/cc_lab_manager/job_generator/job_generator.py
@@ -0,0 +1,273 @@
+import sqlite3
+import logging
+import argparse
+import os
+import yaml
+import copy
+import sys
+import requests
+import re
+from cc_lab_manager.cc_lab_manager_maptable import release_image_mapping, kernel_to_check_table, oem_to_hwe_kerenl, test_cmds_pre_upgrade_actions_device, test_cmds_pre_test_actions_device, test_cmds_on_test_actions_device
+
+logger = logging.getLogger('cc-lab-manager')
+
+
+
+def read_data_from_db(path_to_db):
+    db = sqlite3.connect(path_to_db)
+    db.row_factory = sqlite3.Row
+    col_in_db = ["CID", "Provision","Device_ID", "Certified_OEM_Image", "SRU_Pool", "Provision", "IP","Project",
+                 "Controller_IP", "Lab", "Frame", "Shelf", "Partition", "CanonicalLabel"]
+    sqlcmd = 'select {} from lab_hw where CID not like \'dummy-%\' order by MAAS_Server'.format(",".join(col_in_db[0:]))
+    db_row_objs = db.execute(sqlcmd).fetchall()
+    db.close()
+    db_dict_list = []
+    for row_obj in db_row_objs:
+        db_dict_list.append({k: row_obj[k] for k in row_obj.keys()})
+    return db_row_objs
+
+def read_exclude_test_from_db(path_to_db):
+    
+    db = sqlite3.connect(path_to_db)
+    db.row_factory = sqlite3.Row
+    sqlcmd = 'select * from extra_option'
+    db_row_objs = db.execute(sqlcmd).fetchall()
+    db.close()
+    db_dict_list = []
+    for row_obj in db_row_objs:
+        db_dict_list.append({k: row_obj[k] for k in row_obj.keys()})
+
+    return db_row_objs
+
+def environ_or_required(key):
+    if os.environ.get(key):
+        return {'default': os.environ.get(key)}
+    else:
+        return {'required': True}
+
+def search_list_of_dict(slist, key):
+    
+    res = next((sub for sub in slist if sub['CID'] == key), None)
+    return dict(res)
+
+def selective_representer(dumper, data):
+    return dumper.represent_scalar(u"tag:yaml.org,2002:str", data,
+                                   style="|" if "\n" in data else None)
+
+
+def generate_generic_job_yaml(machine_list, exclude_test_list):
+    
+    sru_job_arch = [
+        {
+            "project":{
+                "name":None,
+                "srutype":None,
+                "series":None,
+                "image":None,
+                "flavor":None,
+                "kernel-to-check":None,
+                "jobs":[]
+            }
+        }
+    ]
+    
+    job_title = "{prefix}-{srutype}-sru-{series}-{flavor}-{system-id}"
+
+    job_list = {}
+    for machine in machine_list:
+        if machine["SRU_Pool"]:
+            release_list = machine["SRU_Pool"].split(",")
+            if release_list:
+                for rlist in release_list:
+                    if "oem" in rlist:
+                        continue
+                    if rlist not in job_list:
+                        if "oem" not in rlist:
+                            job_list[rlist] = []
+                    if machine["Provision"] == "maas_uefi":
+                        job_list[rlist].append({"cid":machine["CID"]})
+                    elif machine["Provision"] == "maas_legacy":
+                        job_list[rlist].append({"cid":machine["CID"],"image":release_image_mapping[rlist].format("legacy")})
+                    
+    for key in job_list:
+        if job_list[key]:
+            job_arch_tmp = copy.deepcopy(sru_job_arch)
+            job_arch_tmp[0]["project"]["name"] = key + "-desktop"
+            job_arch_tmp[0]["project"]["image"] = release_image_mapping[key].format("uefi")
+            job_arch_tmp[0]["project"]["kernel-to-check"] = kernel_to_check_table[key]
+            job_arch_tmp[0]["project"]["srutype"] = "stock"
+            job_arch_tmp[0]["project"]["series"] = key
+            if key in test_cmds_pre_upgrade_actions_device:
+                job_arch_tmp[0]["project"]["test_cmds_pre_upgrade_actions_device"] = test_cmds_pre_upgrade_actions_device[key]
+            if key in test_cmds_pre_test_actions_device:
+                job_arch_tmp[0]["project"]["test_cmds_pre_test_actions_device"] = test_cmds_pre_test_actions_device[key]
+            if key in test_cmds_on_test_actions_device:
+                job_arch_tmp[0]["project"]["test_cmds_on_test_actions_device"] = test_cmds_on_test_actions_device[key]
+            
+            if not job_arch_tmp[0]["project"]["flavor"]:
+                del job_arch_tmp[0]["project"]["flavor"]
+            
+            for job in job_list[key]:
+                machine = search_list_of_dict(machine_list,job["cid"])
+                extra_option = search_list_of_dict(exclude_test_list, job["cid"])
+                
+                tmp_dict = dict()
+                tmp_dict[job_title] = {}
+                tmp_dict[job_title]["system-id"] = machine["Device_ID"]
+                tmp_dict[job_title]["queue"] = machine["CID"]
+                if "image" in tmp_dict:
+                    tmp_dict[job_title]["image"] = job["image"]
+                if "kernel-to-check" in tmp_dict:
+                    tmp_dict[job_title]["kernel-to-check"] = job["kernel-to-check"]
+                if extra_option["Exclude_test"]:
+                    tmp_dict[job_title]["exclude_test"] = extra_option["Exclude_test"]
+                if extra_option["test_cmds_pre_upgrade_actions_device"]:
+                    extra_cmd = eval(extra_option["test_cmds_pre_upgrade_actions_device"])
+                    if key in extra_cmd:
+                        tmp_dict[job_title]["test_cmds_pre_upgrade_actions_device"] = extra_cmd[key]
+                        if key in test_cmds_pre_upgrade_actions_device:
+                            tmp_dict[job_title]["test_cmds_pre_upgrade_actions_device"] = extra_cmd[key]+"\n"+ test_cmds_pre_upgrade_actions_device[key]
+                if extra_option["test_cmds_pre_test_actions_device"]:
+                    extra_cmd = eval(extra_option["test_cmds_pre_test_actions_device"])
+                    if key in extra_cmd:
+                        tmp_dict[job_title]["test_cmds_pre_test_actions_device"] = extra_cmd[key]
+                        if key in test_cmds_pre_test_actions_device:
+                            tmp_dict[job_title]["test_cmds_pre_test_actions_device"] = extra_cmd[key]+"\n"+ test_cmds_pre_test_actions_device[key]
+                if extra_option["test_cmds_on_test_actions_device"]:
+                    extra_cmd = eval(extra_option["test_cmds_on_test_actions_device"])
+                    if key in extra_cmd:
+                        tmp_dict[job_title]["test_cmds_on_test_actions_device"] = extra_cmd[key]
+                        if key in test_cmds_on_test_actions_device:
+                            tmp_dict[job_title]["test_cmds_on_test_actions_device"] = extra_cmd[key]+"\n"+ test_cmds_on_test_actions_device[key]
+                job_arch_tmp[0]["project"]["jobs"].append(tmp_dict)
+            with open(key+".yaml", "w") as file:
+                yaml.add_representer(str, selective_representer)
+                yaml.dump(job_arch_tmp, file, default_flow_style=False, encoding='utf-8', allow_unicode=True, sort_keys=False, width=200)
+            
+def get_launchpad_tag(cid):
+    '''
+    this part we may need a new C3 API that can get the Launchpad tag for searching dependency of oem-meta package.
+    '''
+            
+def find_oem_meta_dependency(machine_list):
+    
+    for machine in machine_list:
+        if  "focal-oem" in machine["SRU_Pool"]:
+            lptag = get_launchpad_tag(machine["CID"])
+            if machine["Project"] == "Somerville":
+                url = "http://dell.archive.canonical.com/dists/focal-somerville-{}/public/binary-amd64/Packages".format(lptag)
+                re_pattern = 'oem-smoerville-(?:(?!factory)).*-meta'
+            elif machine["Project"] == "Stella":
+                url = "http://hp.archive.canonical.com/dists/focal-stella.cmit/public/binary-amd64/Packages";
+                re_pattern = 'oem-stella.cmit-(?:(?!factory)).*-meta'
+            
+            res = requests.get(url)
+            package_data = res.text.split("\n\n")
+            for data in package_data:
+                if data :
+                    data_yaml = yaml.safe_load(data)
+                    if re.match(re_pattern, data_yaml["Package"]):
+                        depend = data_yaml["Depends"].replace(" ","").split(",")
+                        for pkg in depend:
+                            if "linux-generic-hwe" in pkg:
+                                oem_to_hwe_kerenl["focal-oem"].append(machine["CID"])
+                
+def generate_oem_job_yaml(machine_list, exclude_test_list):
+    
+    sru_job_arch = [
+        {
+            "project":{
+                "name":None,
+                "srutype":None,
+                "series":None,
+                "image":None,
+                "flavor":None,
+                "kernel-to-check":None,
+                "jobs":[]
+            }
+        }
+    ]
+    
+    oem_job_list = {"focal-oem":[],"bionic-oem":[],"jammy-oem":[]}
+    job_title = "{prefix}-{srutype}-sru-{series}-{flavor}-{system-id}"
+    for machine in machine_list:
+        if machine["Certified_OEM_Image"]:
+            image_list = machine["Certified_OEM_Image"].split(",")
+            if image_list:
+                for ilist in image_list:
+                    if machine["Provision"] == "noprovision":
+                        image_name = ""
+                    else:
+                        image_name  = ilist.split('/')[-1].split('.')[0]
+                    if "focal" in ilist:
+                        if machine["CID"] in oem_to_hwe_kerenl["focal-oem"]:
+                            oem_job_list["focal-oem"].append({"cid":machine["CID"],"image":image_name,"kernel-to-check":"linux-generic_hwe-20_04"})
+                        else: 
+                            oem_job_list["focal-oem"].append({"cid":machine["CID"],"image":image_name,"kernel-to-check":"linux-oem-20_04"})
+                    elif "bionic" in ilist:
+                        oem_job_list["bionic-oem"].append({"cid":machine["CID"],"image":image_name,"kernel-to-check":"linux-oem"})
+    
+    for key in oem_job_list:
+        if oem_job_list[key]:
+            job_arch_tmp = copy.deepcopy(sru_job_arch)
+            job_arch_tmp[0]["project"]["name"] = key
+            job_arch_tmp[0]["project"]["srutype"] = "oem"
+            job_arch_tmp[0]["project"]["series"] = key.split("-")[0]
+            if key in test_cmds_pre_upgrade_actions_device:
+                job_arch_tmp[0]["project"]["test_cmds_pre_upgrade_actions_device"] = test_cmds_pre_upgrade_actions_device[key]
+            if key in test_cmds_pre_test_actions_device:
+                job_arch_tmp[0]["project"]["test_cmds_pre_test_actions_device"] = test_cmds_pre_test_actions_device[key]
+            if key in test_cmds_on_test_actions_device:
+                job_arch_tmp[0]["project"]["test_cmds_on_test_actions_device"] = test_cmds_on_test_actions_device[key]
+            for job in oem_job_list[key]:
+                machine = search_list_of_dict(machine_list,job["cid"])
+                extra_option = search_list_of_dict(exclude_test_list,job["cid"])
+                tmp_dict = dict()
+                tmp_dict[job_title] = {}
+                tmp_dict[job_title]["system-id"] = machine["Device_ID"]
+                tmp_dict[job_title]["image"] = job["image"]
+                tmp_dict[job_title]["kernel-to-check"] = job["kernel-to-check"]
+                if extra_option["Exclude_test"]:
+                    tmp_dict[job_title]["exclude_test"] = extra_option["Exclude_test"]
+                if extra_option["test_cmds_pre_upgrade_actions_device"]:
+                    extra_cmd = eval(extra_option["test_cmds_pre_upgrade_actions_device"])
+                    if key in extra_cmd:
+                        tmp_dict[job_title]["test_cmds_pre_upgrade_actions_device"] = extra_cmd[key]
+                        if key in test_cmds_pre_upgrade_actions_device:
+                            tmp_dict[job_title]["test_cmds_pre_upgrade_actions_device"] = extra_cmd[key]+"\n"+ test_cmds_pre_upgrade_actions_device[key]
+                if extra_option["test_cmds_pre_test_actions_device"]:
+                    extra_cmd = eval(extra_option["test_cmds_pre_test_actions_device"])
+                    if key in extra_cmd:
+                        tmp_dict[job_title]["test_cmds_pre_test_actions_device"] = extra_cmd[key]
+                        if key in test_cmds_pre_test_actions_device:
+                            tmp_dict[job_title]["test_cmds_pre_test_actions_device"] = extra_cmd[key]+"\n"+ test_cmds_pre_test_actions_device[key]
+                if extra_option["test_cmds_on_test_actions_device"]:
+                    extra_cmd = eval(extra_option["test_cmds_on_test_actions_device"])
+                    if key in extra_cmd:
+                        tmp_dict[job_title]["test_cmds_on_test_actions_device"] = extra_cmd[key]
+                        if key in test_cmds_on_test_actions_device:
+                            tmp_dict[job_title]["test_cmds_on_test_actions_device"] = extra_cmd[key]+"\n"+ test_cmds_on_test_actions_device[key]
+                job_arch_tmp[0]["project"]["jobs"].append(tmp_dict)
+            
+            for keys in job_arch_tmp[0]["project"].copy():
+                if not job_arch_tmp[0]["project"][keys]:
+                    job_arch_tmp[0]["project"].pop(keys)
+                    
+            with open(key+".yaml", "w") as file:
+                yaml.add_representer(str, selective_representer)
+                yaml.dump(job_arch_tmp, file, default_flow_style=False, encoding='utf-8', allow_unicode=True, sort_keys=False, width=200)
+
+
+def main():
+    
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--hwdb', help="Hardware db to read ip mapping", **environ_or_required('CCLM_HARDWARE_DB'))
+    args = parser.parse_args()
+    machine_list = read_data_from_db(args.hwdb)
+    exclude_test_list = read_exclude_test_from_db(args.hwdb)
+    generate_generic_job_yaml(machine_list, exclude_test_list)
+    # find_oem_meta_dependency(machine_list) # this funtciont is not completed
+    generate_oem_job_yaml(machine_list, exclude_test_list)
+
+if __name__ == '__main__':
+    main()