canonical-hw-cert team mailing list archive

[Gavin Lin <mp+425980@xxxxxxxxxxxxxxxxxx>]

Gavin Lin has proposed merging ~gavin.lin/cc-lab-manager:advertised into cc-lab-manager:master.

Commit message:
Add support to advertised queues in tf agent configs.

Requested reviews:
  Canonical Hardware Certification (canonical-hw-cert)

For more details, see:
https://code.launchpad.net/~gavin.lin/cc-lab-manager/+git/cc-lab-manager/+merge/425980
-- 
Your team Canonical Hardware Certification is requested to review the proposed merge of ~gavin.lin/cc-lab-manager:advertised into cc-lab-manager:master.

diff --git a/cc_lab_manager/gen_config/gen_agent_tf_config.py b/cc_lab_manager/gen_config/gen_agent_tf_config.py
index 2b08906..8acefe5 100644
--- a/cc_lab_manager/gen_config/gen_agent_tf_config.py
+++ b/cc_lab_manager/gen_config/gen_agent_tf_config.py
@@ -4,7 +4,7 @@ import yaml
 import argparse
 import re
 
-from cc_lab_manager.cc_lab_manager_maptable import lab_netboot_servers, provision_type_maptable, reboot_script_maptable
+from cc_lab_manager.cc_lab_manager_maptable import lab_netboot_servers, lab_file_servers, provision_type_maptable, reboot_script_maptable
 from cc_lab_manager.device_config_maptable import agent_config_platform_maptable, agent_config_device_maptable
 from cc_lab_manager.device_config_maptable import tf_config_platform_maptable, tf_config_device_maptable
 
@@ -63,22 +63,22 @@ tf_config = {"agent_id": None,
 
 
 def get_db_obj(path_to_db):
-    
     db = sqlite3.connect(path_to_db)
     db.row_factory = sqlite3.Row
     return db
 
+
 def get_cursor_obj(db_obj):
-    
     cursor = db_obj.cursor()
     return cursor
 
+
 def close_db(db):
     db.commit() 
     db.close()
-    
+
+
 def read_data_from_db(cursor):
-    
     col_in_c3 = ["CID","Lab","IP","MAAS_Node_ID", "Provision",
                  "Power", "Device_ID", "TF_Queue",
                  "Customized_agent_config", "Advertised", "PDU_IP",
@@ -127,7 +127,6 @@ def device_id_to_platform_name(device_id):
 
 
 def create_agent_config_dir(db_machine_list, cfg_path):
-    
     for machine in db_machine_list:
         if machine['lab'] == '':
             continue
@@ -139,29 +138,34 @@ def create_agent_config_dir(db_machine_list, cfg_path):
             print("dir exist for {}".format(agent_name))
 
 
-def insert_data_from_maptable(update_conf, device_data, maptable_name, device_key):
-    maptable = globals()[maptable_name]
-    if device_key in maptable:
-        for key in maptable[device_key]:
-            if isinstance(maptable[device_key][key], list):
-                update_conf[key] = []
-                for line in maptable[device_key][key]:
-                    try:
-                        update_conf[key].append(line.format(**device_data))
-                    except KeyError:
-                        update_conf[key].append(line)
-            elif isinstance(maptable[device_key][key], (bool, int, float, complex)):
-                update_conf[key] = maptable[device_key][key]
-            else:
-                try:
-                    update_conf[key] = maptable[device_key][key].format(**device_data)
-                except KeyError:
-                    update_conf[key] = maptable[device_key][key]
+def insert_config(update_conf, device_data, maptable, key):
+    if isinstance(maptable[key], dict):
+        update_conf[key] = {}
+        for sub_key in maptable[key]:
+            update_conf[key] = insert_config(update_conf[key], device_data, maptable[key], sub_key)
+    elif isinstance(maptable[key], list):
+        update_conf[key] = []
+        for line in maptable[key]:
+            try:
+                update_conf[key].append(str(line).format(**device_data))
+            except:
+                update_conf[key].append(line)
+    else:
+        try:
+            update_conf[key] = str(maptable[key]).format(**device_data)
+        except:
+            update_conf[key] = maptable[key]
+    return update_conf
+
+
+def insert_config_device(update_conf, device_data, maptable, identity):
+    if identity in maptable:
+        for key in maptable[identity]:
+            update_conf = insert_config(update_conf, device_data, maptable[identity], key)
     return update_conf
 
 
 def generate_agent_config(db_machine_list, cfg_path):
-    
     for machine in db_machine_list:
         if machine['lab'] == '' or machine['Customized_agent_config'] == 'TRUE':
             continue
@@ -220,17 +224,17 @@ def generate_agent_config(db_machine_list, cfg_path):
                 '| grep Serial| cut -d "," -f 3| cut -d " " -f 3) --dut')
 
         # Update reboot script for known type.
-        agent_conf["reboot_script"] = []
         if device_data["Power"] in reboot_script_maptable:
+            agent_conf["reboot_script"] = []
             for line in reboot_script_maptable[device_data["Power"]]:
                 agent_conf["reboot_script"].append(line.format(**device_data))
 
         # Insert or overwrite platform specific config from maptable if any.
         platform_name = device_id_to_platform_name(device_data['Device_ID'])
-        agent_conf = insert_data_from_maptable(agent_conf, device_data, 'agent_config_platform_maptable', platform_name)
+        agent_conf = insert_config_device(agent_conf, device_data, agent_config_platform_maptable, platform_name)
 
         # Insert or overwrite device specific config from maptable if any.
-        agent_conf = insert_data_from_maptable(agent_conf, device_data, 'agent_config_device_maptable', device_data["Device_ID"])
+        agent_conf = insert_config_device(agent_conf, device_data, agent_config_device_maptable, device_data["Device_ID"])
 
         empty_keys = []
         for key in agent_conf:
@@ -242,17 +246,25 @@ def generate_agent_config(db_machine_list, cfg_path):
         with open(os.path.join(cfg_path, 'data-' + machine['lab'].lower(), agent_name, 'default.yaml'), "w+") as f:
             yaml.dump(agent_conf, f, default_flow_style=False, encoding='utf-8', allow_unicode=True, sort_keys=False)
 
-        
+
 def generate_tf_config(db_machine_list, cfg_path):
-        
     for machine in db_machine_list:
         if machine['lab'] == '' or machine['Customized_agent_config'] == 'TRUE':
             continue
+        # Copy data from sqlite3.Row object to a dict so we can add more keys later
+        device_data = dict(machine)
+
+        try:
+            provision = provision_type_maptable[device_data["Provision"]]
+        except:
+            print("Unknown provision type for {}".format(device_data['Provision']))
+            continue
+
         tf_conf = tf_config.copy()
         
-        agent_name = device_id_to_agent_name(machine["Device_ID"])
+        agent_name = device_id_to_agent_name(device_data["Device_ID"])
         try:
-            provision = provision_type_maptable[machine["Provision"]]
+            provision = provision_type_maptable[device_data["Provision"]]
         except:
             print("Unknown provision type for {}".format(machine['cid']))
             continue
@@ -264,22 +276,25 @@ def generate_tf_config(db_machine_list, cfg_path):
 
         # Default queues: CID, Device_ID, and platform name(Device_ID without tailing CID or increment)
         tf_conf['job_queues'] = []
-        tf_conf['job_queues'].append(machine['CID'])
-        tf_conf['job_queues'].append(machine['Device_ID'])
-        platform_name = device_id_to_platform_name(machine['Device_ID'])
-        if platform_name != machine['Device_ID']:
+        tf_conf['job_queues'].append(device_data['CID'])
+        tf_conf['job_queues'].append(device_data['Device_ID'])
+        platform_name = device_id_to_platform_name(device_data['Device_ID'])
+        if platform_name != device_data['Device_ID']:
             tf_conf['job_queues'].append(platform_name)
 
+        # Add file server map
+        device_data['File_Server'] = lab_file_servers[device_data['Lab'].lower()]
+
         # Insert or overwrite platform specific config from maptable if any.
-        platform_name = device_id_to_platform_name(machine['Device_ID'])
-        agent_conf = insert_data_from_maptable(tf_conf, machine, 'tf_config_platform_maptable', platform_name)
+        platform_name = device_id_to_platform_name(device_data['Device_ID'])
+        tf_conf = insert_config_device(tf_conf, device_data, tf_config_platform_maptable, platform_name)
 
         # Insert or overwrite device specific config from maptable if any.
-        agent_conf = insert_data_from_maptable(tf_conf, machine, 'tf_config_device_maptable', machine["Device_ID"])
+        tf_conf = insert_config_device(tf_conf, device_data, tf_config_device_maptable, device_data["Device_ID"])
 
         # Add customized queues if any
-        if machine['TF_Queue'] != '':
-            cq = re.sub('[^A-Za-z0-9-,]+', '', machine['TF_Queue'])
+        if device_data['TF_Queue'] != '':
+            cq = re.sub('[^A-Za-z0-9-,]+', '', device_data['TF_Queue'])
             customized_queues = cq.lower().split(',')
             for tf_queue in customized_queues:
                 if tf_queue not in tf_conf['job_queues']:
@@ -294,9 +309,9 @@ def generate_tf_config(db_machine_list, cfg_path):
         
         with open(os.path.join(cfg_path, 'data-' + machine['lab'].lower(), agent_name, 'testflinger-agent.conf'), "w+") as f:
             yaml.dump(tf_conf, f, default_flow_style=False, encoding='utf-8', allow_unicode=True, sort_keys=False)
-                    
+
+
 def create_agent_yaml(db_machine_list, cfg_path):
-    
     # Separate agent configurations by labs
     labs = []
     for machine in db_machine_list:
@@ -334,16 +349,16 @@ def create_agent_yaml(db_machine_list, cfg_path):
                 }
         with open(os.path.join(cfg_path, 'ce-tf-agents-' + lab + '.yaml'),"w") as f:
             yaml.dump(data, f, default_flow_style=False, encoding='utf-8', allow_unicode=True, sort_keys=False)
-        
-        
+
+
 def environ_or_required(key):
     if os.environ.get(key):
         return {'default': os.environ.get(key)}
     else:
         return {'required': True}
 
+
 def main():
-    
     parser = argparse.ArgumentParser()
     parser.add_argument('--database', help="Database location",**environ_or_required('CCLM_HARDWARE_DB'))
     args = parser.parse_args()
@@ -356,5 +371,6 @@ def main():
     create_agent_yaml(machine_list, os.path.join(os.environ.get('CCLM_CERT_JUJU'), 'ce-cert'))
     close_db(db)
 
+
 if __name__ == '__main__':
         main()