opencompute-developers team mailing list archive

[Nelson Chu <Nelson.Chu@xxxxxxxxxxx>]

Nelson Chu has proposed merging lp:~nelson-chu/opencompute/add-ocp-cpu-memory-job into lp:opencompute/checkbox.

Requested reviews:
  Open Compute Developers (opencompute-developers)

For more details, see:
https://code.launchpad.net/~nelson-chu/opencompute/add-ocp-cpu-memory-job/+merge/196651
-- 
https://code.launchpad.net/~nelson-chu/opencompute/add-ocp-cpu-memory-job/+merge/196651
Your team Open Compute Developers is requested to review the proposed merge of lp:~nelson-chu/opencompute/add-ocp-cpu-memory-job into lp:opencompute/checkbox.

=== added file 'data/whitelists/opencompute-certify-local.whitelist'
--- data/whitelists/opencompute-certify-local.whitelist	1970-01-01 00:00:00 +0000
+++ data/whitelists/opencompute-certify-local.whitelist	2013-11-26 04:03:24 +0000
@@ -0,0 +1,44 @@
+# Resource Jobs
+block_device
+cdimage
+cpuinfo
+device
+dmi
+dpkg
+efi
+environment
+gconf
+lsb
+meminfo
+module
+optical_drive
+package
+sleep
+uname
+#Info attachment jobs
+__info__
+cpuinfo_attachment
+dmesg_attachment
+dmi_attachment
+dmidecode_attachment
+efi_attachment
+lspci_attachment
+lshw_attachment
+mcelog_attachment
+meminfo_attachment
+modprobe_attachment
+modules_attachment
+sysctl_attachment
+sysfs_attachment
+udev_attachment
+lsmod_attachment
+acpi_sleep_attachment
+info/hdparm
+info/hdparm_.*.txt
+installer_debug.gz
+info/disk_partitions
+# Actual test cases
+__TC-001-0001-CPU_Memory__
+TC-001-0001-001-CPU_information
+TC-001-0001-002-Processor_topology
+TC-001-0001-003-Memory_Information

=== added file 'jobs/TC-001-0001-CPU_Memory.txt.in'
--- jobs/TC-001-0001-CPU_Memory.txt.in	1970-01-01 00:00:00 +0000
+++ jobs/TC-001-0001-CPU_Memory.txt.in	2013-11-26 04:03:24 +0000
@@ -0,0 +1,25 @@
+plugin: shell
+name: TC-001-0001-001-CPU_information
+requires: package.name == 'lshw'
+command: cpu_info
+description:
+ 1. Gathering CPU information by using lshw command.
+ 2. Output CPU model and L1, L2, L3 cache size.
+ 3. Pass criteria: CPU model should belong to Intel Xeon processor E5-2600 product family and Last Level Cache (LLC) should be larger than 20MB
+
+plugin: shell
+name: TC-001-0001-002-Processor_topology
+command: processor_topology
+description:
+ 1. Gathering CPU information by using lscpu command.
+ 2. Output the total number of CPU, the number of Thread per core, the number of Core per Socket, and the number of Socket.
+ 3. Pass criteria: It should be 8 cores per CPU and 2 Threads per core (that is, up to 16 threads with Hyper-Threading Technology per CPU).
+
+plugin: shell
+name: TC-001-0001-003-Memory_Information
+requires: package.name == 'lshw'
+command: memory_info
+description:
+ 1. Gathering memory information by using lshw command.
+ 2. Output memory module, vendor, size and slot.
+ 3. Pass criteria: 2 DDR3 slots per channel per processor (total of 16 DIMMs on the motherboard).

=== modified file 'jobs/local.txt.in'
--- jobs/local.txt.in	2013-10-01 00:55:26 +0000
+++ jobs/local.txt.in	2013-11-26 04:03:24 +0000
@@ -109,3 +109,10 @@
 command:
  shopt -s extglob
  cat $CHECKBOX_SHARE/jobs/sniff.txt?(.in)
+
+name: __TC-001-0001-CPU_Memory__
+plugin: local
+_description: Verified CPU and Memory
+command:
+ shopt -s extglob
+ cat $CHECKBOX_SHARE/jobs/TC-001-0001-CPU_Memory.txt?(.in)

=== added file 'scripts/cpu_info'
--- scripts/cpu_info	1970-01-01 00:00:00 +0000
+++ scripts/cpu_info	2013-11-26 04:03:24 +0000
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+Copyright (C) 2010-2013 by Cloud Computing Center for Mobile Applications 
+Industrial Technology Research Institute
+
+cpu_info
+  Gathering CPU information by using lshw command.
+  And output CPU model and L1, L2, L3 cache size.
+  L3 Cache should be larger than 20MB.
+
+Authors
+  Nelson Chu <Nelson.Chu@xxxxxxxxxxx>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 3,
+as published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+
+import os
+import re
+import sys
+import xml.etree.ElementTree as ET
+from subprocess import check_output
+
+def main():
+    command = 'lshw -xml'
+    with open(os.devnull, "w") as NULL:
+       hwinfo_xml = check_output(command, stderr=NULL, shell=True)
+    root = ET.fromstring(hwinfo_xml)
+
+    # Parser lshw XML for gather processor information.
+    processor = root.findall(".//product/..[@class='processor']")
+
+    if not processor:
+        print("Can not parser any processor information.")
+        return 10
+
+    for cpu in processor:
+        if cpu.find('product') is None:
+            print("Can not found processor product.")
+            return 20
+        print(cpu.find('product').text)
+
+        cache_list = cpu.findall(".//size/..[@class='memory']")
+        if not cache_list:
+            print("Can not found any CPU cache.")
+            return 30
+
+        for cache in cache_list:
+            if cache.find('size') is None or cache.find('slot') is None:
+                print("Can not access Last Level Cache (LLC).")
+                return 40
+
+            cache_size = int(cache.find('size').text) / 1024
+            if cache_size > 1024:
+                cache_size = cache_size / 1024
+                print(('%s %d MB') %(cache.find('slot').text,
+                                     cache_size))
+                if re.search('L3', cache.find('slot').text):
+                    if cache_size < 20:
+                        print('L3 cache size less than 20MB.')
+                        return 50
+            else:
+                print(('%s %d KB') %(cache.find('slot').text,
+                                     cache_size))
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

=== added file 'scripts/memory_info'
--- scripts/memory_info	1970-01-01 00:00:00 +0000
+++ scripts/memory_info	2013-11-26 04:03:24 +0000
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""
+Copyright (C) 2010-2013 by Cloud Computing Center for Mobile Applications
+Industrial Technology Research Institute
+
+memory_info
+  Gathering memory information by using lshw command.
+  And output memory module, vendor, size and slot.
+
+Authors
+  Nelson Chu <Nelson.Chu@xxxxxxxxxxx>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 3,
+as published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+
+import sys
+import xml.etree.ElementTree as ET
+from subprocess import Popen, PIPE
+
+def main():
+    attribute = ['description', 'vendor', 'slot', 'size']
+    command = 'lshw -xml'
+    hwinfo_xml = Popen(command, stdout=PIPE, stderr=PIPE,
+                       shell=True).communicate()[0]
+    root = ET.fromstring(hwinfo_xml)
+
+    # Parser lshw XML for gather memory information.
+    memory_list = root.findall(".//clock/..[@class='memory']")
+
+    if not memory_list:
+        print("Can not parser any memory information.")
+        return 10
+
+    for dimm in memory_list:
+        for attr in attribute:
+            if dimm.find(attr) is None:
+                print(("Can not found memory %s") %attr)
+                return 20
+
+        memory_size = int(dimm.find('size').text) / (1024**3)
+        for attr in attribute:
+            if attr == 'description':
+                print(('%s,') %(dimm.find(attr).text)),
+                continue
+            elif attr == 'size':
+                print(('%s=%dGB.') %(attr, memory_size)),
+                continue
+            print(('%s=%s,') %(attr, dimm.find(attr).text)),
+        print()
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

=== added file 'scripts/processor_topology'
--- scripts/processor_topology	1970-01-01 00:00:00 +0000
+++ scripts/processor_topology	2013-11-26 04:03:24 +0000
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+'''
+cpu_topology
+Written by Jeffrey Lane <jeffrey.lane@xxxxxxxxxxxxx>
+'''
+import sys
+import os
+from subprocess import check_output
+
+class proc_cpuinfo():
+    '''
+    Class to get and handle information from /proc/cpuinfo
+    Creates a dictionary of data gleaned from that file.
+    '''
+    def __init__(self):
+        self.cpuinfo = {}
+        cpu_fh = open('/proc/cpuinfo', 'r')
+        try:
+            temp = cpu_fh.readlines()
+        finally:
+            cpu_fh.close()
+
+        for i in temp:
+            if i.startswith('processor'):
+                key = 'cpu' + (i.split(':')[1].strip())
+                self.cpuinfo[key] = {'core_id':'', 'physical_package_id':''}
+            elif i.startswith('core id'):
+                self.cpuinfo[key].update({'core_id': i.split(':')[1].strip()})
+            elif i.startswith('physical id'):
+                self.cpuinfo[key].update({'physical_package_id':
+                                          i.split(':')[1].strip()})
+            else:
+                continue
+
+
+class sysfs_cpu():
+    '''
+    Class to get and handle information from sysfs as relates to CPU topology
+    Creates an informational class to present information on various CPUs
+    '''
+
+    def __init__(self, proc):
+        self.syscpu = {}
+        self.path = '/sys/devices/system/cpu/' + proc + '/topology'
+        items = ['core_id', 'physical_package_id']
+        for i in items:
+            syscpu_fh = open(os.path.join(self.path, i), 'r')
+            try:
+                self.syscpu[i] = syscpu_fh.readline().strip()
+            finally:
+                syscpu_fh.close()
+
+
+def compare(proc_cpu, sys_cpu):
+    cpu_map = {}
+    '''
+    If there is only 1 CPU the test don't look for core_id
+    and physical_package_id because those information are absent in
+    /proc/cpuinfo on singlecore system
+    '''
+    for key in proc_cpu.keys():
+        if 'cpu1' not in proc_cpu:
+            cpu_map[key] = True
+        else:
+            for subkey in proc_cpu[key].keys():
+                if proc_cpu[key][subkey] == sys_cpu[key][subkey]:
+                    cpu_map[key] = True
+                else:
+                    cpu_map[key] = False
+    return cpu_map
+
+
+def main():
+    cpuinfo = proc_cpuinfo()
+    sys_cpu = {}
+    keys = cpuinfo.cpuinfo.keys()
+    for k in keys:
+        sys_cpu[k] = sysfs_cpu(k).syscpu
+    cpu_map = compare(cpuinfo.cpuinfo, sys_cpu)
+    if False in cpu_map.values() or len(cpu_map) < 1:
+        print("FAIL: CPU Topology is incorrect", file=sys.stderr)
+        print("-" * 52, file=sys.stderr)
+        print("{0}{1}".format("/proc/cpuinfo".center(30), "sysfs".center(25)),
+                file=sys.stderr)
+        print("{0}{1}{2}{3}{1}{2}".format(
+                "CPU".center(6),
+                "Physical ID".center(13),
+                "Core ID".center(9),
+                "|".center(3)), file=sys.stderr)
+        for key in sorted(sys_cpu.keys()):
+            print("{0}{1}{2}{3}{4}{5}".format(
+                  key.center(6),
+                  cpuinfo.cpuinfo[key]['physical_package_id'].center(13),
+                  cpuinfo.cpuinfo[key]['core_id'].center(9),
+                  "|".center(3),
+                  sys_cpu[key]['physical_package_id'].center(13),
+                  sys_cpu[key]['core_id'].center(9)), file=sys.stderr)
+        return 1
+    else:
+        # Output the total number of CPU, the number of Thread per core, 
+        # the number of Core per Socket, and the number of Socket.
+        # Revised by Nelson Chu <nelson.chu@xxxxxxxxxxx>
+        command = 'lscpu'
+
+        with open(os.devnull, "w") as NULL:
+            cpu_info = check_output(command, stderr=NULL, shell=True)
+
+        cpu_info = cpu_info.decode()
+
+        for cpu in cpu_info.split('\n'):
+            if cpu.startswith("CPU(s)"):
+                print(cpu + ',')
+            if cpu.startswith("Thread(s) per core"):
+                print(cpu + ',')
+            if cpu.startswith("Core(s) per socket"):
+                print(cpu + ',')
+            if cpu.startswith("Socket(s)"):
+                print(cpu + '.')
+
+        return 0
+
+if __name__ == '__main__':
+    sys.exit(main())