← Back to team overview

nagios-charmers team mailing list archive

[Merge] ~aluria/hw-health-charm/+git/hw-health-charm:unittests-engrot6 into hw-health-charm:master

 

Alvaro Uría has proposed merging ~aluria/hw-health-charm/+git/hw-health-charm:unittests-engrot6 into hw-health-charm:master.

Requested reviews:
  Nagios Charm developers (nagios-charmers)

For more details, see:
https://code.launchpad.net/~aluria/hw-health-charm/+git/hw-health-charm/+merge/361785
-- 
Your team Nagios Charm developers is requested to review the proposed merge of ~aluria/hw-health-charm/+git/hw-health-charm:unittests-engrot6 into hw-health-charm:master.
diff --git a/src/files/check_mdadm.py b/src/files/check_mdadm.py
index 741865e..22d9176 100644
--- a/src/files/check_mdadm.py
+++ b/src/files/check_mdadm.py
@@ -4,6 +4,11 @@
 import os
 import re
 import subprocess
+from nagios_plugin3 import (
+    CriticalError,
+    WarnError,
+    try_check,
+)
 
 
 def get_devices():
@@ -15,7 +20,8 @@ def get_devices():
                                            stderr=subprocess.PIPE)
             devices_re = re.compile('^ARRAY\s+([^ ]+) ')
             devices = set()
-            for line in devices_raw.stdout.readline().decode():
+            for line in devices_raw.stdout.readlines():
+                line = line.decode().strip()
                 device_re = devices_re.match(line)
                 if device_re is not None:
                     devices.add(device_re.group(1))
@@ -23,14 +29,13 @@ def get_devices():
         except Exception as e:
             # log error
             pass
-    return
+    return set()
 
 
-def main():
+def parse_output():
     devices = get_devices()
     if len(devices) == 0:
-        print('WARNING: unexpectedly checked no devices')
-        return 1
+        raise WarnError('WARNING: unexpectedly checked no devices')
 
     mdadm_detail = ['/sbin/mdadm', '--detail']
     mdadm_detail.extend(sorted(devices))
@@ -39,12 +44,11 @@ def main():
                                                stdout=subprocess.PIPE,
                                                stderr=subprocess.PIPE)
     except Exception as error:
-        print('WARNING: error executing mdadm: {}'.format(error))
-        return 1
+        raise WarnError('WARNING: error executing mdadm: {}'.format(error))
 
-    devices_re = '^(/[^ ]+):$'
-    state_re = '^\s*State :\s+(\S+)\s*'
-    status_re = '^\s*(Active|Working|Failed|Spare) Devices :\s+(\d+)'
+    devices_re = '^(/\S+):$'
+    state_re = '^\s*State\s+:\s+(\S+)$'
+    status_re = '^\s*(Active|Working|Failed|Spare) Devices\s+:\s+(\d+)$'
 
     devices_cre = re.compile(devices_re)
     state_cre = re.compile(state_re)
@@ -52,7 +56,8 @@ def main():
 
     device = None
     devices_stats = {}
-    for line in devices_details_raw.stdout.readline().decode():
+    for line in devices_details_raw.stdout.readlines():
+        line = line.decode().rstrip()
         m = devices_cre.match(line)
         if m:
             device = m.group(1)
@@ -75,35 +80,41 @@ def main():
 
     rc = devices_details_raw.wait()
     if rc:
-        print('WARNING: mdadm returned exit status {}'.format(int(rc)))
-        return 1
+        raise WarnError('WARNING: mdadm returned exit'
+                        ' status {}'.format(int(rc)))
 
     parts = []
     msg = []
     critical = False
     for device in devices_stats:
+        # Is device degraded?
         if devices_stats[device]['degraded']:
             critical = True
-            parts.append('{} degraded'.format(device))
+            parts = ['{} degraded'.format(device)]
         else:
-            parts.append('{} ok'.format(device))
-
-        for status in sorted(devices_stats[device]['stats']):
-            parts.append('{}[{}]'
-                         .format(status,
-                                 devices_stats[device]['stats'][status]))
-            if status == 'Failed' \
-                    and devices_stats[device]['stats'][status] > 0:
-                critical = True
+            parts = ['{} ok'.format(device)]
+
+        # If Failed drives are found, list counters (how many?)
+        failed_cnt = devices_stats[device]['stats'].get('Failed', 0)
+        if failed_cnt > 0:
+            critical = True
+            dev_stats = ['{}[{}]'.format(status,
+                                         devices_stats[device]['stats'][status]
+                                         )
+                         for status in sorted(devices_stats[device]['stats'])
+                         ]
+            parts.extend(dev_stats)
         msg.append(', '.join(parts))
 
     msg = '; '.join(msg)
     if critical:
-        print('CRITICAL: {}'.format(msg))
-        return 2
+        raise CriticalError('CRITICAL: {}'.format(msg))
     else:
         print('OK: {}'.format(msg))
-        return 0
+
+
+def main():
+    try_check(parse_output())
 
 
 if __name__ == '__main__':
diff --git a/src/files/megaraid/check_megacli.py b/src/files/megaraid/check_megacli.py
index 57e4219..41756d1 100755
--- a/src/files/megaraid/check_megacli.py
+++ b/src/files/megaraid/check_megacli.py
@@ -3,6 +3,11 @@
 
 import re
 import sys
+from nagios_plugin3 import (
+    CriticalError,
+    WarnError,
+    try_check,
+)
 
 INPUT_FILE = '/var/lib/nagios/megacli.out'
 
@@ -22,7 +27,7 @@ def parse_output(policy=False):
     npdrives_cre = re.compile(npdrives_re)
     w_policy_cre = re.compile(w_policy_re)
 
-    num_pdrive = num_ldrive = failed_ldrive = wrong_policy_ldrive = 0
+    num_pdrive = num_ldrive = failed_ld = wrg_policy_ld = 0
     nlines = 0
     errors = []
     match = critical = False
@@ -54,7 +59,7 @@ def parse_output(policy=False):
                 num_ldrive += 1
                 state = m.group(1)
                 if state != 'Optimal':
-                    failed_ldrive += 1
+                    failed_ld += 1
                     msg = 'adp({}):ld({}):state({})'.format(adapter_id,
                                                             ldrive_id,
                                                             state)
@@ -72,7 +77,7 @@ def parse_output(policy=False):
                 if m:
                     w_policy = m.group(1)
                     if w_policy != policy:
-                        wrong_policy_ldrive += 1
+                        wrg_policy_ld += 1
                         msg = 'adp({}):ld({}):policy({})'.format(adapter_id,
                                                                  ldrive_id,
                                                                  w_policy)
@@ -80,15 +85,22 @@ def parse_output(policy=False):
                         critical = True
                     continue
 
-    data = {'critical': critical}
-    if len(errors) > 0 and (failed_ldrive > 0 or wrong_policy_ldrive > 0):
-        data['errors'] = {'msgs': errors,
-                          'failed_ld': failed_ldrive,
-                          'wrg_policy_ld': wrong_policy_ldrive}
-    elif nlines == 0:
-        data['errors'] = {'msgs': 'WARNING: controller not found'}
+    if nlines == 0:
+        raise WarnError('WARNING: controller not found')
     elif not match:
-        data['errors'] = {'msgs': 'WARNING: megacli output, parsing error'}
+        raise WarnError('WARNING: megacli output, parsing error')
+    elif critical:
+        if len(errors) > 0:
+            msg = ', '.join(['{}({})'.format(cnt, eval(cnt))
+                             for counter in ('failed_ld', 'wrg_policy_ld')
+                             if eval(cnt) > 0])
+            msg += '; '.join(errors)
+        else:
+            msg = 'failure caught but no output available'
+        raise CriticalError('CRITICAL: {}'.format(msg))
+    elif len(errors) > 0:
+        raise WarnError('WARNING: {}'.format('; '.join(errors)))
+
     else:
         if num_ldrive == 0:
             msg = 'OK: no disks configured for RAID'
@@ -97,38 +109,11 @@ def parse_output(policy=False):
             if policy:
                 msg += ', policy[{}]'
             msg = msg.format(num_ldrive, num_pdrive, policy)
-        data['OK'] = msg
-
-    return data
-
-
-def main(policy=False):
-    data = parse_output(policy)
-
-    if data['critical']:
-        if 'errors' not in data:
-            print('CRITICAL: failure caught but no output available')
-            return 2
-
-        msg = 'CRITICAL: '
-        for counter in ('failed_ld', 'wrg_policy_ld'):
-            if data['errors'].get(counter, 0) > 0:
-                msg += '{}({}), '.format(counter,
-                                         data['errors'][counter])
-        msg += '; '.join(data['errors']['msgs'])
         print(msg)
-        return 2
 
-    if 'errors' in data:
-        print(data['errors']['msgs'])
-        return 1
 
-    if 'OK' in data:
-        print(data['OK'])
-        return 0
-
-    print('WARNING: no output')
-    return 1
+def main(policy=False):
+    try_check(parse_output(policy))
 
 
 if __name__ == '__main__':
diff --git a/src/files/megaraid/check_megacli.sh b/src/files/megaraid/check_megacli.sh
index 020cea3..cca1f9f 100755
--- a/src/files/megaraid/check_megacli.sh
+++ b/src/files/megaraid/check_megacli.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-PATH="/snap/bin:$PATH"
+PATH="/snap/bin:/usr/local/bin:$PATH"
 FILE=/var/lib/nagios/megacli.out
 TMP_FILE=/tmp/megacli.out
 
diff --git a/src/files/mpt/check_sas2ircu.py b/src/files/mpt/check_sas2ircu.py
index eb7d081..e142fb0 100755
--- a/src/files/mpt/check_sas2ircu.py
+++ b/src/files/mpt/check_sas2ircu.py
@@ -3,6 +3,11 @@
 
 import re
 import sys
+from nagios_plugin3 import (
+    CriticalError,
+    WarnError,
+    try_check,
+)
 
 INPUT_FILE = '/var/lib/nagios/sas2ircu.out'
 
@@ -37,20 +42,16 @@ def parse_output():
 
     msg = '; '.join(['{}[{}]'.format(state, ','.join(devices[state]))
                      for state in devices])
-    return (msg, critical)
-
-
-def main():
-    msg, critical = parse_output()
     if msg == '':
-        print('WARNING: no output')
-        return 1
+        raise WarnError('WARNING: no output')
     elif critical:
-        print('CRITICAL: {}'.format(msg))
-        return 2
+        raise CriticalError('CRITICAL: {}'.format(msg))
     else:
         print('OK: {}'.format(msg))
-        return 0
+
+
+def main():
+    try_check(parse_output())
 
 
 if __name__ == '__main__':
diff --git a/src/files/mpt/check_sas2ircu.sh b/src/files/mpt/check_sas2ircu.sh
index 1fe5127..23594a0 100755
--- a/src/files/mpt/check_sas2ircu.sh
+++ b/src/files/mpt/check_sas2ircu.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-PATH="/snap/bin:$PATH"
+PATH="/snap/bin:/usr/local/bin:$PATH"
 FILE=/var/lib/nagios/sas2ircu.out
 TMP_FILE=/tmp/sas2ircu.out
 
diff --git a/src/files/mpt/check_sas3ircu.py b/src/files/mpt/check_sas3ircu.py
index ff9b79a..ac09d76 100755
--- a/src/files/mpt/check_sas3ircu.py
+++ b/src/files/mpt/check_sas3ircu.py
@@ -3,6 +3,11 @@
 
 import re
 import sys
+from nagios_plugin3 import (
+    CriticalError,
+    WarnError,
+    try_check,
+)
 
 INPUT_FILE = '/var/lib/nagios/sas3ircu.out'
 
@@ -79,20 +84,17 @@ def parse_output():
     # msg = '; '.join(sorted(devices))
     msg = '; '.join(['{}[{}]'.format(state, ','.join(devices[state]))
                      for state in sorted(devices)])
-    return (msg, critical)
 
-
-def main():
-    msg, critical = parse_output()
     if msg == '':
-        print('WARNING: no output')
-        return 1
+        raise WarnError('WARNING: no output')
     elif critical:
-        print('CRITICAL: {}'.format(msg))
-        return 2
+        raise CriticalError('CRITICAL: {}'.format(msg))
     else:
         print('OK: {}'.format(msg))
-        return 0
+
+
+def main():
+    try_check(parse_output())
 
 
 if __name__ == '__main__':
diff --git a/src/files/mpt/check_sas3ircu.sh b/src/files/mpt/check_sas3ircu.sh
index 0995f4d..0b26e96 100755
--- a/src/files/mpt/check_sas3ircu.sh
+++ b/src/files/mpt/check_sas3ircu.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-PATH="/snap/bin:$PATH"
+PATH="/snap/bin:/usr/local/bin:$PATH"
 FILE=/var/lib/nagios/sas3ircu.out
 TMP_FILE=/tmp/sas3ircu.out
 
diff --git a/src/lib/utils/tooling.py b/src/lib/utils/tooling.py
index 387324f..e5ec8c3 100644
--- a/src/lib/utils/tooling.py
+++ b/src/lib/utils/tooling.py
@@ -7,9 +7,11 @@ from charmhelpers.core.hookenv import (
     ERROR,
     INFO
 )
-# from charms.layer import snap
-from shutil import copy2
+import glob
+import shutil
 import subprocess
+import tempfile
+import zipfile
 
 TOOLING = {
     'megacli': {'snap': 'megacli',
@@ -25,6 +27,51 @@ TOOLING = {
 }
 
 PLUGINS_DIR = '/usr/local/lib/nagios/plugins/'
+TOOLS_DIR = '/usr/local/bin/'
+TEMP_DIR = tempfile.mkdtemp()
+
+
+def install_resource(tools, resource):
+    ntools = len(tools)
+    if ntools == 0:
+        return False
+    elif 'mdadm' in tools:
+        tools = [tool for tool in tools if tool != 'mdadm']
+        ntools -= 1
+        if ntools == 0:
+            return True
+
+    if type(resource) != str \
+            or not resource.endswith('.zip') \
+            or not os.path.exists(resource):
+        return False
+
+    try:
+        with zipfile.ZipFile(resource, 'r') as zf:
+            zf.extractall(TEMP_DIR)
+    except zipfile.BadZipFile as error:
+        log('BadZipFile: {}'.format(error))
+        return False
+    except PermissionError as error:
+        log('Unable to unzip the resource: {}'.format(error),
+            ERROR)
+        return False
+
+    count = 0
+    for filepath in glob.glob(os.path.join(TEMP_DIR, '*')):
+        filebasename = os.path.basename(filepath)
+        if filebasename in tools \
+                and filebasename in TOOLING \
+                and os.path.isfile(filepath):
+            os.chmod(filepath, 0o755)
+            shutil.chown(filepath, user=0, group=0)
+            shutil.copy2(filepath, TOOLS_DIR)
+            count += 1
+
+    if os.path.isdir(TEMP_DIR):
+        shutil.rmtree(TEMP_DIR)
+
+    return ntools == count and count > 0
 
 
 def install_tools(tools, method=None):
@@ -41,6 +88,9 @@ def install_tools(tools, method=None):
                 except Exception as error:
                     log('Snap install error: {}'.format(error),
                         ERROR)
+        elif tool == 'mdadm':
+            # Note(aluria): mdadm is assumed to be installed by default
+            count += 1
 
     return len(tools) >= count > 0
 
@@ -62,7 +112,7 @@ def configure_tools(tools):
                 cronjob_scriptname = os.path.basename(TOOLING[tool]['cronjob'])
                 if filepath:
                     subprocess.call(filepath)
-                    copy2(filepath, PLUGINS_DIR)
+                    shutil.copy2(filepath, PLUGINS_DIR)
                     log('Cronjob script [{}]'
                         ' copied to {}'.format(filepath, PLUGINS_DIR),
                         DEBUG)
@@ -85,7 +135,7 @@ def configure_tools(tools):
             if 'filename' in TOOLING[tool]:
                 filepath = _get_filepath(TOOLING[tool]['filename'])
                 if filepath:
-                    copy2(filepath, PLUGINS_DIR)
+                    shutil.copy2(filepath, PLUGINS_DIR)
                     count += 1
                     log('NRPE script for tool [{}] configured'.format(tool),
                         INFO)
diff --git a/src/metadata.yaml b/src/metadata.yaml
index 8b056d3..70c845f 100644
--- a/src/metadata.yaml
+++ b/src/metadata.yaml
@@ -29,3 +29,9 @@ provides:
   nrpe-external-master:
     interface: nrpe-external-master
     scope: container
+resources:
+  tools:
+    type: file
+    filename: tools.zip
+    description: |
+      Archived 3rd party tool(s) not available by other means (apt, snap)
diff --git a/src/reactive/hw_health.py b/src/reactive/hw_health.py
index 730d405..6f95ef5 100644
--- a/src/reactive/hw_health.py
+++ b/src/reactive/hw_health.py
@@ -10,8 +10,7 @@ from charms.layer import status
 from utils.hwdiscovery import get_tools
 from utils.tooling import (
     configure_nrpe_checks,
-    # configure_tools,
-    install_tools,
+    install_resource,
 )
 
 
@@ -34,8 +33,26 @@ def install():
         set_flag('hw-health.unsupported')
         status.blocked('Hardware not supported')
     else:
-        # install vendor utilities
-        installed = install_tools(tools, method='snap')
+        resource = hookenv.resource_get('tools')
+        if resource:
+            hookenv.log('Installing from resource')
+            status.waiting('Installing from attached resource')
+            installed = install_resource(tools, resource)
+        else:
+            hookenv.log('Missing Juju resource: tools - alternative method'
+                        ' is not available yet')
+            status.blocked('Missing Juju resource: tools')
+            return
+            # TODO(aluria): install vendor utilities via snaps
+            # https://forum.snapcraft.io/t/request-for-classic-confinement-sas2ircu/9023
+            # Snap interface for hw tooling is not supported
+            # in strict mode (confined). Classic mode
+            # (unconfined snaps) need approval
+            #
+            # hookenv.log('Installing from snap')
+            # status.waiting('Installing from snap')
+            # installed = install_tools(tools, method='snap')
+
         if installed:
             kv = unitdata.kv()
             kv.set('tools', list(tools))
@@ -45,20 +62,6 @@ def install():
             status.blocked('Tools not found: {}'.format(', '.join(tools)))
 
 
-# @when('hw-health.installed')
-# @when_not('hw-health.configured')
-# def configure_scripts():
-#     kv = unitdata.kv()
-#     tools = kv.get('tools', set())
-#     # configure scripts from files/*
-#     if configure_tools(tools):
-#         status.active('ready')
-#         set_flag('hw-health.configured')
-#     else:
-#         status.waiting('Unable to configure tools: '
-#                        '{}'.format(', '.join(sorted(tools))))
-
-
 # update-status, config-changed, upgrade-charm, etc.
 @when('hw-health.installed')
 def any_change():
@@ -100,7 +103,8 @@ def configure_nrpe():
     # then, parse checks output for Nagios to alert on
     # configure_cronjobs(tools)
 
-    if configure_nrpe_checks(tools):
+    configured = configure_nrpe_checks(tools)
+    if configured:
         status.active('ready')
         set_flag('hw-health.nrpe')
     else:
diff --git a/src/tests/download_nagios_plugin3.sh b/src/tests/download_nagios_plugin3.sh
new file mode 100755
index 0000000..78ae784
--- /dev/null
+++ b/src/tests/download_nagios_plugin3.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+FILE=$(mktemp -p /tmp)
+MODULENAME=nagios_plugin3.py
+
+env > /tmp/env.tmp
+
+for i in .tox/py3/lib/python3*
+ do
+  if [[ -d $i && ! -e $i/$MODULENAME ]];then
+    SIZE=$(stat --printf="%s" $FILE)
+    if [[ $SIZE -eq 0 ]];then
+        curl -k -s https://git.launchpad.net/nrpe-charm/plain/files/nagios_plugin3.py > $FILE
+        SIZE=$(stat --printf="%s" $FILE)
+    fi
+    test $SIZE -gt 0 && cp $FILE ${i}/$MODULENAME || exit 1
+  fi
+ done
+test -e $FILE && rm -f $FILE 2>/dev/null
+exit 0
diff --git a/src/tests/hw-health-samples/mdadm.output b/src/tests/hw-health-samples/mdadm.output
new file mode 100644
index 0000000..9356757
--- /dev/null
+++ b/src/tests/hw-health-samples/mdadm.output
@@ -0,0 +1,100 @@
+/dev/md0:
+        Version : 1.2
+  Creation Time : Thu Aug  2 22:50:05 2018
+     Raid Level : raid1
+     Array Size : 523712 (511.52 MiB 536.28 MB)
+  Used Dev Size : 523712 (511.52 MiB 536.28 MB)
+   Raid Devices : 2
+  Total Devices : 2
+    Persistence : Superblock is persistent
+
+    Update Time : Sun Nov  4 00:57:07 2018
+          State : clean
+ Active Devices : 2
+Working Devices : 2
+ Failed Devices : 0
+  Spare Devices : 0
+
+           Name : fnos-nvme05:0  (local to host fnos-nvme05)
+           UUID : dfaf7413:7551b000:56dd7442:5b020adb
+         Events : 51
+
+    Number   Major   Minor   RaidDevice State
+       0       8        1        0      active sync   /dev/sda1
+       2       8       17        1      active sync   /dev/sdb1
+/dev/md1:
+        Version : 1.2
+  Creation Time : Thu Aug  2 22:50:09 2018
+     Raid Level : raid1
+     Array Size : 1948672 (1903.32 MiB 1995.44 MB)
+  Used Dev Size : 1948672 (1903.32 MiB 1995.44 MB)
+   Raid Devices : 2
+  Total Devices : 2
+    Persistence : Superblock is persistent
+
+    Update Time : Tue Nov 20 15:40:27 2018
+          State : clean
+ Active Devices : 2
+Working Devices : 2
+ Failed Devices : 0
+  Spare Devices : 0
+
+           Name : fnos-nvme05:1  (local to host fnos-nvme05)
+           UUID : 8946258c:a6246ca7:8d3dc20a:33bcabcb
+         Events : 51
+
+    Number   Major   Minor   RaidDevice State
+       0       8        2        0      active sync   /dev/sda2
+       2       8       18        1      active sync   /dev/sdb2
+/dev/md3:
+        Version : 1.2
+  Creation Time : Thu Aug  2 22:50:16 2018
+     Raid Level : raid1
+     Array Size : 488148992 (465.54 GiB 499.86 GB)
+  Used Dev Size : 488148992 (465.54 GiB 499.86 GB)
+   Raid Devices : 2
+  Total Devices : 2
+    Persistence : Superblock is persistent
+
+  Intent Bitmap : Internal
+
+    Update Time : Tue Nov 27 16:47:44 2018
+          State : active
+ Active Devices : 2
+Working Devices : 2
+ Failed Devices : 0
+  Spare Devices : 0
+
+           Name : fnos-nvme05:3  (local to host fnos-nvme05)
+           UUID : ae461b25:13219daa:75846e9e:2e5a52f1
+         Events : 1418
+
+    Number   Major   Minor   RaidDevice State
+       0       8        4        0      active sync   /dev/sda4
+       2       8       20        1      active sync   /dev/sdb4
+/dev/md2:
+        Version : 1.2
+  Creation Time : Thu Aug  2 22:50:13 2018
+     Raid Level : raid1
+     Array Size : 439320576 (418.97 GiB 449.86 GB)
+  Used Dev Size : 439320576 (418.97 GiB 449.86 GB)
+   Raid Devices : 2
+  Total Devices : 2
+    Persistence : Superblock is persistent
+
+  Intent Bitmap : Internal
+
+    Update Time : Tue Nov 27 16:47:41 2018
+          State : clean
+ Active Devices : 2
+Working Devices : 2
+ Failed Devices : 0
+  Spare Devices : 0
+
+           Name : fnos-nvme05:2  (local to host fnos-nvme05)
+           UUID : a83fda21:56cf85d4:99c802ed:c07c4f76
+         Events : 5715
+
+    Number   Major   Minor   RaidDevice State
+       0       8        3        0      active sync   /dev/sda3
+       2       8       19        1      active sync   /dev/sdb3
diff --git a/src/tests/hw-health-samples/mdadm.output.critical b/src/tests/hw-health-samples/mdadm.output.critical
new file mode 100644
index 0000000..4bdab35
--- /dev/null
+++ b/src/tests/hw-health-samples/mdadm.output.critical
@@ -0,0 +1,100 @@
+/dev/md0:
+        Version : 1.2
+  Creation Time : Thu Aug  2 22:50:05 2018
+     Raid Level : raid1
+     Array Size : 523712 (511.52 MiB 536.28 MB)
+  Used Dev Size : 523712 (511.52 MiB 536.28 MB)
+   Raid Devices : 2
+  Total Devices : 2
+    Persistence : Superblock is persistent
+
+    Update Time : Sun Nov  4 00:57:07 2018
+          State : clean
+ Active Devices : 2
+Working Devices : 2
+ Failed Devices : 0
+  Spare Devices : 0
+
+           Name : fnos-nvme05:0  (local to host fnos-nvme05)
+           UUID : dfaf7413:7551b000:56dd7442:5b020adb
+         Events : 51
+
+    Number   Major   Minor   RaidDevice State
+       0       8        1        0      active sync   /dev/sda1
+       2       8       17        1      active sync   /dev/sdb1
+/dev/md1:
+        Version : 1.2
+  Creation Time : Thu Aug  2 22:50:09 2018
+     Raid Level : raid1
+     Array Size : 1948672 (1903.32 MiB 1995.44 MB)
+  Used Dev Size : 1948672 (1903.32 MiB 1995.44 MB)
+   Raid Devices : 2
+  Total Devices : 2
+    Persistence : Superblock is persistent
+
+    Update Time : Tue Nov 20 15:40:27 2018
+          State : degraded
+ Active Devices : 1
+Working Devices : 1
+ Failed Devices : 1
+  Spare Devices : 0
+
+           Name : fnos-nvme05:1  (local to host fnos-nvme05)
+           UUID : 8946258c:a6246ca7:8d3dc20a:33bcabcb
+         Events : 51
+
+    Number   Major   Minor   RaidDevice State
+       0       8        2        0      active sync   /dev/sda2
+       2       8       18        1      active sync   /dev/sdb2
+/dev/md3:
+        Version : 1.2
+  Creation Time : Thu Aug  2 22:50:16 2018
+     Raid Level : raid1
+     Array Size : 488148992 (465.54 GiB 499.86 GB)
+  Used Dev Size : 488148992 (465.54 GiB 499.86 GB)
+   Raid Devices : 2
+  Total Devices : 2
+    Persistence : Superblock is persistent
+
+  Intent Bitmap : Internal
+
+    Update Time : Tue Nov 27 16:47:44 2018
+          State : active
+ Active Devices : 2
+Working Devices : 2
+ Failed Devices : 0
+  Spare Devices : 0
+
+           Name : fnos-nvme05:3  (local to host fnos-nvme05)
+           UUID : ae461b25:13219daa:75846e9e:2e5a52f1
+         Events : 1418
+
+    Number   Major   Minor   RaidDevice State
+       0       8        4        0      active sync   /dev/sda4
+       2       8       20        1      active sync   /dev/sdb4
+/dev/md2:
+        Version : 1.2
+  Creation Time : Thu Aug  2 22:50:13 2018
+     Raid Level : raid1
+     Array Size : 439320576 (418.97 GiB 449.86 GB)
+  Used Dev Size : 439320576 (418.97 GiB 449.86 GB)
+   Raid Devices : 2
+  Total Devices : 2
+    Persistence : Superblock is persistent
+
+  Intent Bitmap : Internal
+
+    Update Time : Tue Nov 27 16:47:41 2018
+          State : clean
+ Active Devices : 2
+Working Devices : 2
+ Failed Devices : 0
+  Spare Devices : 0
+
+           Name : fnos-nvme05:2  (local to host fnos-nvme05)
+           UUID : a83fda21:56cf85d4:99c802ed:c07c4f76
+         Events : 5715
+
+    Number   Major   Minor   RaidDevice State
+       0       8        3        0      active sync   /dev/sda3
+       2       8       19        1      active sync   /dev/sdb3
diff --git a/src/tests/hw-health-samples/megacli.output.1 b/src/tests/hw-health-samples/megacli.output.1
new file mode 100644
index 0000000..2fb339f
--- /dev/null
+++ b/src/tests/hw-health-samples/megacli.output.1
@@ -0,0 +1,31 @@
+
+
+Adapter 0 -- Virtual Drive Information:
+Virtual Drive: 0 (Target Id: 0)
+Name                :vg01
+RAID Level          : Primary-1, Secondary-0, RAID Level Qualifier-0
+Size                : 7.276 TB
+Sector Size         : 512
+Is VD emulated      : No
+Mirror Data         : 7.276 TB
+State               : Optimal
+Strip Size          : 64 KB
+Number Of Drives    : 4
+Span Depth          : 1
+Default Cache Policy: WriteBack, ReadAhead, Direct, No Write Cache if Bad BBU
+Current Cache Policy: WriteBack, ReadAhead, Direct, No Write Cache if Bad BBU
+Default Access Policy: Read/Write
+Current Access Policy: Read/Write
+Disk Cache Policy   : Disk's Default
+Encryption Type     : None
+Default Power Savings Policy: Controller Defined
+Current Power Savings Policy: None
+Can spin up in 1 minute: Yes
+LD has drives that support T10 power conditions: Yes
+LD's IO profile supports MAX power savings with cached writes: No
+Bad Blocks Exist: No
+Is VD Cached: No
+
+
+
+Exit Code: 0x00
diff --git a/src/tests/hw-health-samples/sas2ircu.huawei.output.1 b/src/tests/hw-health-samples/sas2ircu.huawei.output.1
new file mode 100644
index 0000000..02ac325
--- /dev/null
+++ b/src/tests/hw-health-samples/sas2ircu.huawei.output.1
@@ -0,0 +1,150 @@
+LSI Corporation SAS2 IR Configuration Utility.
+Version 20.00.00.00 (2014.09.18)
+Copyright (c) 2008-2014 LSI Corporation. All rights reserved.
+
+Read configuration has been initiated for controller 0
+------------------------------------------------------------------------
+Controller information
+------------------------------------------------------------------------
+  Controller type                         : SAS2308_2
+  BIOS version                            : 7.39.00.00
+  Firmware version                        : 18.00.04.00
+  Channel description                     : 1 Serial Attached SCSI
+  Initiator ID                            : 0
+  Maximum physical devices                : 255
+  Concurrent commands supported           : 3072
+  Slot                                    : Unknown
+  Segment                                 : 0
+  Bus                                     : 1
+  Device                                  : 0
+  Function                                : 0
+  RAID Support                            : Yes
+------------------------------------------------------------------------
+IR Volume information
+------------------------------------------------------------------------
+------------------------------------------------------------------------
+Physical device information
+------------------------------------------------------------------------
+Initiator at ID #0
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 0
+  SAS Address                             : 4433221-1-0000-0000
+  State                                   : Ready (RDY)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : HGST HUS724040AL
+  Firmware Revision                       : A8B0
+  Serial No                               : PN1334PEJ8VRBS
+  GUID                                    : 5000cca250e03649
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 1
+  SAS Address                             : 4433221-1-0100-0000
+  State                                   : Ready (RDY)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : HGST HUS724040AL
+  Firmware Revision                       : A8B0
+  Serial No                               : PN1334PEHMEH5S
+  GUID                                    : 5000cca250d6ed31
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 2
+  SAS Address                             : 4433221-1-0200-0000
+  State                                   : Ready (RDY)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : HGST HUS724040AL
+  Firmware Revision                       : A8B0
+  Serial No                               : PN1338P4HVWD8B
+  GUID                                    : 5000cca249da4ffe
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 3
+  SAS Address                             : 4433221-1-0300-0000
+  State                                   : Ready (RDY)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : HGST HUS724040AL
+  Firmware Revision                       : A8B0
+  Serial No                               : PN1338P4HVVHZB
+  GUID                                    : 5000cca249da4cb0
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 4
+  SAS Address                             : 4433221-1-0400-0000
+  State                                   : Ready (RDY)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : HGST HUS724040AL
+  Firmware Revision                       : A8B0
+  Serial No                               : PN1338P4HVPDGB
+  GUID                                    : 5000cca249da397e
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 5
+  SAS Address                             : 4433221-1-0500-0000
+  State                                   : Ready (RDY)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : HGST HUS724040AL
+  Firmware Revision                       : A8B0
+  Serial No                               : PN1338P4HVWWKB
+  GUID                                    : 5000cca249da51d8
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 6
+  SAS Address                             : 4433221-1-0600-0000
+  State                                   : Ready (RDY)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : HGST HUS724040AL
+  Firmware Revision                       : A8B0
+  Serial No                               : PN1334PEHPU1XX
+  GUID                                    : 5000cca250d80160
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 7
+  SAS Address                             : 4433221-1-0700-0000
+  State                                   : Ready (RDY)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : HGST HUS724040AL
+  Firmware Revision                       : A8B0
+  Serial No                               : PN1338P4HVWV7B
+  GUID                                    : 5000cca249da51af
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+------------------------------------------------------------------------
+Enclosure information
+------------------------------------------------------------------------
+  Enclosure#                              : 1
+  Logical ID                              : 5384c4f8:aa1c9000
+  Numslots                                : 8
+  StartSlot                               : 0
+------------------------------------------------------------------------
+SAS2IRCU: Command DISPLAY Completed Successfully.
+SAS2IRCU: Utility Completed Successfully.
diff --git a/src/tests/hw-health-samples/sas3ircu.supermicro.output.1 b/src/tests/hw-health-samples/sas3ircu.supermicro.output.1
new file mode 100644
index 0000000..03ca92e
--- /dev/null
+++ b/src/tests/hw-health-samples/sas3ircu.supermicro.output.1
@@ -0,0 +1,82 @@
+Avago Technologies SAS3 IR Configuration Utility.
+Version 17.00.00.00 (2018.04.02)
+Copyright (c) 2009-2018 Avago Technologies. All rights reserved.
+
+Read configuration has been initiated for controller 0
+------------------------------------------------------------------------
+Controller information
+------------------------------------------------------------------------
+  Controller type                         : SAS3008
+  PI Supported                            : Yes
+  PI Mixing                               : Disabled
+  BIOS version                            : 8.35.00.00
+  Firmware version                        : 15.00.00.00
+  Channel description                     : 1 Serial Attached SCSI
+  Initiator ID                            : 0
+  Maximum physical devices                : 255
+  Concurrent commands supported           : 3072
+  Slot                                    : 2
+  Segment                                 : 0
+  Bus                                     : 2
+  Device                                  : 0
+  Function                                : 0
+  RAID Support                            : Yes
+------------------------------------------------------------------------
+IR Volume information
+------------------------------------------------------------------------
+IR volume 1
+  Volume ID                               : 323
+  PI Supported                            : No
+  Status of volume                        : Okay (OKY)
+  Volume wwid                             : 017e97cb06987855
+  RAID level                              : RAID1
+  Size (in MB)                            : 3814697
+  Physical hard disks                     :
+  PHY[0] Enclosure#/Slot#                 : 1:0
+  PHY[1] Enclosure#/Slot#                 : 1:1
+------------------------------------------------------------------------
+Physical device information
+------------------------------------------------------------------------
+Initiator at ID #0
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 0
+  PI Supported                            : No
+  SAS Address                             : 4433221-1-0000-0000
+  State                                   : Optimal (OPT)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : ST4000NM115-1YZ1
+  Firmware Revision                       : SN02
+  Serial No                               : ZC11155N
+  Unit Serial No(VPD)                     : ZC11155N
+  GUID                                    : 5000c500a1d45b35
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+
+Device is a Hard disk
+  Enclosure #                             : 1
+  Slot #                                  : 1
+  PI Supported                            : No
+  SAS Address                             : 4433221-1-0100-0000
+  State                                   : Optimal (OPT)
+  Size (in MB)/(in sectors)               : 3815447/7814037167
+  Manufacturer                            : ATA
+  Model Number                            : ST4000NM115-1YZ1
+  Firmware Revision                       : SN02
+  Serial No                               : ZC111FVA
+  Unit Serial No(VPD)                     : ZC111FVA
+  GUID                                    : 5000c500a1d51834
+  Protocol                                : SATA
+  Drive Type                              : SATA_HDD
+------------------------------------------------------------------------
+Enclosure information
+------------------------------------------------------------------------
+  Enclosure#                              : 1
+  Logical ID                              : 50030480:22df6300
+  Numslots                                : 8
+  StartSlot                               : 0
+------------------------------------------------------------------------
+SAS3IRCU: Command DISPLAY Completed Successfully.
+SAS3IRCU: Utility Completed Successfully.
diff --git a/src/tests/unit/test_check_mdadm.py b/src/tests/unit/test_check_mdadm.py
new file mode 100644
index 0000000..6501143
--- /dev/null
+++ b/src/tests/unit/test_check_mdadm.py
@@ -0,0 +1,125 @@
+import mock
+import unittest
+import io
+import os
+import sys
+import nagios_plugin3
+
+sys.path.append('files')
+import check_mdadm
+
+
+class TestCheckMdadm(unittest.TestCase):
+    @mock.patch('os.path.exists')
+    @mock.patch('subprocess.Popen')
+    def test_get_devices_mdadm_exists(self, mdadm_details, path_exists):
+        class Test_Popen(object):
+            def __init__(cls):
+                data = (
+                    b'ARRAY /dev/md0 metadata=1.2 name=node00:0'
+                    b' UUID=dfaf7413:7551b000:56dd7442:5b020adb\n'
+                    b'ARRAY /dev/md1 metadata=1.2 name=node01:0'
+                    b' UUID=dfaf7413:7551b000:56dd7442:5b020adc\n'
+                )
+                cls.stdout = io.BufferedReader(io.BytesIO(data))
+
+        mdadm_details.return_value = Test_Popen()
+        path_exists.return_value = True
+
+        actual = check_mdadm.get_devices()
+        expected = set(['/dev/md0', '/dev/md1'])
+        self.assertEqual(actual, expected)
+
+    @mock.patch('os.path.exists')
+    def test_get_devices_mdadm_notfound(self, path_exists):
+        path_exists.return_value = False
+        actual = check_mdadm.get_devices()
+        expected = set()
+        self.assertEqual(actual, expected)
+
+    @mock.patch('os.path.exists')
+    @mock.patch('subprocess.Popen')
+    def test_get_devices_mdadm_exception(self, mdadm_details, path_exists):
+        path_exists.return_value = True
+        mdadm_details.side_effect = Exception
+        actual = check_mdadm.get_devices()
+        expected = set()
+        self.assertEqual(actual, expected)
+
+    @mock.patch('check_mdadm.get_devices')
+    @mock.patch('subprocess.Popen')
+    @mock.patch('sys.stdout', new_callable=io.StringIO)
+    def test_parse_output_ok(self, mock_print, mdadm_details, devices):
+        class Test_Popen(object):
+            def __init__(cls):
+                test_output = os.path.join(os.getcwd(),
+                                           'tests',
+                                           'hw-health-samples',
+                                           'mdadm.output')
+                cls.stdout = io.FileIO(test_output)
+                cls.wait = lambda: 0
+
+        devices.return_value = set(['/dev/md0', '/dev/md1', '/dev/md2'])
+        mdadm_details.return_value = Test_Popen()
+        check_mdadm.parse_output()
+        actual = mock_print.getvalue()
+        expected = 'OK: /dev/md0 ok; /dev/md1 ok; /dev/md3 ok; /dev/md2 ok\n'
+        self.assertEqual(actual, expected)
+
+    @mock.patch('check_mdadm.get_devices')
+    @mock.patch('subprocess.Popen')
+    def test_parse_output_wait_warning(self, mdadm_details, devices):
+        class Test_Popen(object):
+            def __init__(cls):
+                test_output = os.path.join(os.getcwd(),
+                                           'tests',
+                                           'hw-health-samples',
+                                           'mdadm.output')
+                cls.stdout = io.FileIO(test_output)
+                cls.wait = lambda: 2
+
+        devices.return_value = set(['/dev/md0', '/dev/md1', '/dev/md2'])
+        mdadm_details.return_value = Test_Popen()
+        expected = 'WARNING: mdadm returned exit status 2'
+        with self.assertRaises(nagios_plugin3.WarnError) as context:
+            check_mdadm.parse_output()
+        self.assertTrue(expected in str(context.exception))
+
+    @mock.patch('check_mdadm.get_devices')
+    def test_parse_output_nodevices(self, devices):
+        devices.return_value = set()
+        expected = 'WARNING: unexpectedly checked no devices'
+        with self.assertRaises(nagios_plugin3.WarnError) as context:
+            check_mdadm.parse_output()
+        self.assertTrue(expected in str(context.exception))
+
+    @mock.patch('check_mdadm.get_devices')
+    @mock.patch('subprocess.Popen')
+    def test_parse_output_mdadm_warning(self, mdadm_details, devices):
+        devices.return_value = set(['/dev/md0', '/dev/md1', '/dev/md2'])
+        mdadm_details.side_effect = Exception('ugly error')
+        expected = 'WARNING: error executing mdadm: ugly error'
+        with self.assertRaises(nagios_plugin3.WarnError) as context:
+            check_mdadm.parse_output()
+        self.assertTrue(expected in str(context.exception))
+
+    @mock.patch('check_mdadm.get_devices')
+    @mock.patch('subprocess.Popen')
+    def test_parse_output_degraded(self, mdadm_details, devices):
+        class Test_Popen(object):
+            def __init__(cls):
+                test_output = os.path.join(os.getcwd(),
+                                           'tests',
+                                           'hw-health-samples',
+                                           'mdadm.output.critical')
+                cls.stdout = io.FileIO(test_output)
+                cls.wait = lambda: 0
+
+        devices.return_value = set(['/dev/md0', '/dev/md1', '/dev/md2'])
+        mdadm_details.return_value = Test_Popen()
+        expected = ('CRITICAL: /dev/md0 ok; /dev/md1 degraded, Active[1],'
+                    ' Failed[1], Spare[0], Working[1]; /dev/md3 ok;'
+                    ' /dev/md2 ok')
+        with self.assertRaises(nagios_plugin3.CriticalError) as context:
+            check_mdadm.parse_output()
+        self.assertTrue(expected in str(context.exception))
diff --git a/src/tests/unit/test_check_megacli.py b/src/tests/unit/test_check_megacli.py
new file mode 100644
index 0000000..0b9f4ae
--- /dev/null
+++ b/src/tests/unit/test_check_megacli.py
@@ -0,0 +1,21 @@
+import io
+import mock
+import unittest
+import sys
+import os
+
+sys.path.append('files/megaraid')
+import check_megacli
+
+
+class TestCheckMegaCLI(unittest.TestCase):
+    @mock.patch('sys.stdout', new_callable=io.StringIO)
+    def test_parse_output(self, mock_print):
+        check_megacli.INPUT_FILE = os.path.join(os.getcwd(),
+                                                'tests',
+                                                'hw-health-samples',
+                                                'megacli.output.1')
+        check_megacli.parse_output()
+        actual = mock_print.getvalue()
+        expected = 'OK: Optimal, ldrives[1], pdrives[4]\n'
+        self.assertEqual(actual, expected)
diff --git a/src/tests/unit/test_check_sas2ircu.py b/src/tests/unit/test_check_sas2ircu.py
new file mode 100644
index 0000000..1dc1639
--- /dev/null
+++ b/src/tests/unit/test_check_sas2ircu.py
@@ -0,0 +1,21 @@
+import io
+import mock
+import unittest
+import sys
+import os
+
+sys.path.append('files/mpt')
+import check_sas2ircu
+
+
+class TestCheckMegaCLI(unittest.TestCase):
+    @mock.patch('sys.stdout', new_callable=io.StringIO)
+    def test_parse_output(self, mock_print):
+        check_sas2ircu.INPUT_FILE = os.path.join(os.getcwd(),
+                                                 'tests',
+                                                 'hw-health-samples',
+                                                 'sas2ircu.huawei.output.1')
+        check_sas2ircu.parse_output()
+        actual = mock_print.getvalue()
+        expected = 'OK: Ready[1:0,1:1,1:2,1:3,1:4,1:5,1:6,1:7]\n'
+        self.assertEqual(actual, expected)
diff --git a/src/tests/unit/test_check_sas3ircu.py b/src/tests/unit/test_check_sas3ircu.py
new file mode 100644
index 0000000..07960c6
--- /dev/null
+++ b/src/tests/unit/test_check_sas3ircu.py
@@ -0,0 +1,22 @@
+import io
+import mock
+import unittest
+import sys
+import os
+
+sys.path.append('files/mpt')
+import check_sas3ircu
+
+
+class TestCheckMegaCLI(unittest.TestCase):
+    @mock.patch('sys.stdout', new_callable=io.StringIO)
+    def test_parse_output(self, mock_print):
+        _filepath = os.path.join(os.getcwd(),
+                                 'tests',
+                                 'hw-health-samples',
+                                 'sas3ircu.supermicro.output.1')
+        check_sas3ircu.INPUT_FILE = _filepath
+        check_sas3ircu.parse_output()
+        actual = mock_print.getvalue()
+        expected = 'OK: Okay[323:(1:0,1:1)]; Optimal[1:0,1:1]\n'
+        self.assertEqual(actual, expected)
diff --git a/src/tests/unit/test_tooling.py b/src/tests/unit/test_tooling.py
index 78a6790..8310a6c 100644
--- a/src/tests/unit/test_tooling.py
+++ b/src/tests/unit/test_tooling.py
@@ -1,17 +1,141 @@
-# import mock
+import mock
 import unittest
+import shutil
 import sys
-# import os
+import os
+import zipfile
 # import io
 # import subprocess
 # import charmhelpers.core.host
+# from shutil import copy2
 
 sys.path.append('lib/utils')
 import tooling
 
 
 class TestTooling(unittest.TestCase):
-    def test_install_tools(self):
-        real = tooling.install_tools([])
+    def test_install_resource_notools(self):
+        self.assertEqual(tooling.install_resource([], False),
+                         False)
+        self.assertEqual(tooling.install_resource([], '/path/to/resource'),
+                         False)
+
+    def test_install_resource_noresource_nomdadm(self):
+        self.assertEqual(tooling.install_resource(['unittest'], False),
+                         False)
+
+    def test_install_resource_noresource_yesmdadm(self):
+        self.assertEqual(tooling.install_resource(['mdadm'], False),
+                         True)
+
+    @mock.patch('glob.glob')
+    @mock.patch('os.chmod')
+    @mock.patch('os.path.exists')
+    @mock.patch('os.path.isfile')
+    @mock.patch('shutil.chown')
+    @mock.patch('shutil.copy2')
+    @mock.patch('zipfile.ZipFile')
+    def test_install_resource_ok(self, zfile, shutil_copy2, shutil_chown,
+                                 path_isfile, path_exists, os_chmod,
+                                 glob_glob):
+        class _WrapCls(object):
+            def __init__(cls):
+                cls.extractall = lambda x: None
+                cls.close = lambda: None
+
+        class Test_resource_zipfile(object):
+            def __enter__(cls):
+                return _WrapCls()
+
+            def __exit__(cls, type, value, traceback):
+                pass
+
+        path_exists.return_value = True
+        zfile.return_value = Test_resource_zipfile()
+        glob_glob.return_value = ['/tmp/test/megacli']
+        os_chmod.return_value = None
+        path_isfile.return_value = True
+        shutil_chown.return_value = None
+        shutil_copy2.return_value = None
+        actual = tooling.install_resource(['megacli'], '/tmp/test.zip')
+        expected = True
+        self.assertEqual(actual, expected)
+
+    @mock.patch('os.path.exists')
+    @mock.patch('zipfile.ZipFile')
+    def test_install_resource_zipfile_error(self, zfile, path_exists):
+        path_exists.return_value = True
+        zfile.side_effect = zipfile.BadZipFile('ugly error')
+        actual = tooling.install_resource(['megacli'], '/tmp/test.zip')
+        expected = False
+        self.assertEqual(actual, expected)
+
+    @mock.patch('glob.glob')
+    @mock.patch('os.path.exists')
+    @mock.patch('os.path.isfile')
+    @mock.patch('zipfile.ZipFile')
+    def test_install_resource_nomatchedresource(self, zfile, path_isfile,
+                                                path_exists, glob_glob):
+        class _WrapCls(object):
+            def __init__(cls):
+                cls.extractall = lambda x: None
+                cls.close = lambda: None
+
+        class Test_resource_zipfile(object):
+            def __enter__(cls):
+                return _WrapCls()
+
+            def __exit__(cls, type, value, traceback):
+                pass
+
+        path_exists.return_value = True
+        zfile.return_value = Test_resource_zipfile()
+        glob_glob.return_value = ['/tmp/test/megacli']
+        path_isfile.return_value = True
+        self.assertEqual(tooling.install_resource(['unittest'],
+                                                  '/tmp/test.zip'),
+                         False)
+
+    def test_install_tools_notools(self):
+        self.assertEqual(tooling.install_tools([]),
+                         False)
+
+    def test_install_tools_unsupported_tools(self):
+        actual = tooling.install_tools(['unittest1', 'unittest2'])
         expected = False
-        self.assertEqual(real, expected)
+        self.assertEqual(actual, expected)
+
+    def test_install_tools_supported_tools(self):
+        self.assertEqual(tooling.install_tools(['megacli', 'sas2ircu']),
+                         True)
+
+    def test_install_tools_mdadm(self):
+        self.assertEqual(tooling.install_tools(['mdadm']),
+                         True)
+
+    @mock.patch('os.environ')
+    @mock.patch('os.path.exists')
+    @mock.patch('os.path.join')
+    def test_get_filepath(self, path_join, path_exists, environ):
+        expected = 'unittest1'
+        environ.return_value = {'CHARM_DIR': 'xxx'}
+        path_join.return_value = expected
+        path_exists.return_value = True
+        self.assertEqual(tooling._get_filepath('asdfasdf'),
+                         expected)
+
+    def test_configure_tools_notools(self):
+        self.assertEqual(tooling.configure_tools([]),
+                         False)
+
+    def test_configure_tools_unsupported_tools(self):
+        self.assertEqual(tooling.configure_tools(['unittest1', 'unittest2']),
+                         False)
+
+    @mock.patch('shutil.copy2')
+    @mock.patch('tooling._get_filepath')
+    def test_configure_tools_mdadm(self, get_filepath, copy_file):
+        get_filepath.return_value = 'unittest'
+        copy_file.return_value = None
+        self.assertEqual(tooling.configure_tools(['mdadm']),
+                         True)
diff --git a/src/tox.ini b/src/tox.ini
index fdc2f7d..c93091e 100644
--- a/src/tox.ini
+++ b/src/tox.ini
@@ -10,4 +10,6 @@ deps=
   nose
   mock
 # commands = {posargs}
-commands = nosetests tests/unit
+commands =
+  {toxworkdir}/../tests/download_nagios_plugin3.sh
+  nosetests tests/unit

Follow ups