cloud-init-dev team mailing list archive

[Chad Smith <chad.smith@xxxxxxxxxxxxx>]

Chad Smith has proposed merging ~chad.smith/cloud-init:collect-logs into cloud-init:master.

Requested reviews:
  cloud-init commiters (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/330626

cmdline: cloud-init collect-logs

Add a new collect-logs parameter to the cloud-init CLI. This script will
collect all logs pertinent to a cloud-init run and store them in a compressed
tar-gzipped file. this tarfile can be attached to any cloud-init bug filed in order
to aid in bug triage and resolution.
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:collect-logs into cloud-init:master.

diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py
new file mode 100644
index 0000000..396acc7
--- /dev/null
+++ b/cloudinit/cmd/devel/logs.py
@@ -0,0 +1,59 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Define 'collect-logs' utility and handler to include in cloud-init cmd."""
+
+import argparse
+from cloudinit.util import copy, del_dir, ensure_dir, subp, write_file
+from datetime import datetime
+import os
+import shutil
+
+
+CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log']
+CLOUDINIT_RUN_DIR = '/run/cloud-init'
+
+
+def get_parser(parser=None):
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog='collect-logs',
+            description='Collect and tar all cloud-init logs')
+    parser.add_argument(
+        "--tarfile", '-t', default='cloud-init.tar.gz',
+        help=('The tarfile to create containing all collected logs.'
+              ' Default: cloud-init.tar.gz'))
+    return parser
+
+
+def collect_logs(tarfile):
+    date = datetime.utcnow().date().strftime('%Y-%m-%d')
+    log_dir = 'cloud-init-logs-{0}'.format(date)
+    if os.path.exists(log_dir):
+        del_dir(log_dir)
+    ensure_dir(log_dir)
+    out, _ = subp(['dpkg-query', '-W', "-f='${Version}'", 'cloud-init'])
+    write_file(os.path.join(log_dir, 'version'), out)
+    out, _ = subp(['dmesg'])
+    write_file(os.path.join(log_dir, 'dmesg.txt'), out)
+    out, _ = subp(['journalctl', '-o', 'short-precise'])
+    write_file(os.path.join(log_dir, 'journal.txt'), out)
+    for log in CLOUDINIT_LOGS:
+        copy(log, log_dir)
+    run_dir = os.path.join(log_dir, 'run')
+    ensure_dir(run_dir)
+    shutil.copytree(CLOUDINIT_RUN_DIR, os.path.join(run_dir, 'cloud-init'))
+    subp(['tar', 'czvf', tarfile, log_dir])
+    del_dir(log_dir)
+
+
+def handle_collect_logs_args(name, args):
+    collect_logs(args.tarfile)
+
+
+def main():
+    """Tool to collect and tar all cloud-init related logs."""
+    parser = get_parser()
+    handle_collect_logs_args('cloudconfig-schema', parser.parse_args())
+    return 0
diff --git a/cloudinit/cmd/devel/tests/__init__.py b/cloudinit/cmd/devel/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cloudinit/cmd/devel/tests/__init__.py
diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py
new file mode 100644
index 0000000..3fa301a
--- /dev/null
+++ b/cloudinit/cmd/devel/tests/test_logs.py
@@ -0,0 +1,73 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.cmd.devel import logs
+from cloudinit.util import ensure_dir, load_file, subp, write_file
+from cloudinit.tests.helpers import FilesystemMockingTestCase, wrap_and_call
+from datetime import datetime
+import os
+
+
+class TestCollectLogs(FilesystemMockingTestCase):
+
+    def setUp(self):
+        super(TestCollectLogs, self).setUp()
+        self.new_root = self.tmp_dir()
+        self.run_dir = self.tmp_path('run', self.new_root)
+
+    def test_collect_logs_creates_tarfile(self):
+        """collect-logs creates a tarfile with all related cloud-init info."""
+        log1 = self.tmp_path('cloud-init.log', self.new_root)
+        write_file(log1, 'cloud-init-log')
+        log2 = self.tmp_path('cloud-init-output.log', self.new_root)
+        write_file(log2, 'cloud-init-output-log')
+        ensure_dir(self.run_dir)
+        write_file(self.tmp_path('results.json', self.run_dir), 'results')
+        output_tarfile = self.tmp_path('logs.tgz')
+
+        date = datetime.utcnow().date().strftime('%Y-%m-%d')
+        date_logdir = 'cloud-init-logs-{0}'.format(date)
+
+        expected_subp = {
+            ('dpkg-query', '-W', "-f='${Version}'", 'cloud-init'): '0.7fake',
+            ('dmesg',): 'dmesg-out',
+            ('journalctl', '-o', 'short-precise'): 'journal-out',
+            ('tar', 'czvf', output_tarfile, date_logdir): ''
+        }
+
+        def fake_subp(cmd):
+            cmd_tuple = tuple(cmd)
+            if cmd_tuple not in expected_subp:
+                raise AssertionError(
+                    'Unexpected command provided to subp: {0}'.format(cmd))
+            if cmd == ['tar', 'czvf', output_tarfile, date_logdir]:
+                subp(cmd)  # Pass through tar cmd so we can check output
+            return expected_subp[cmd_tuple], ''
+
+        wrap_and_call(
+            'cloudinit.cmd.devel.logs',
+            {'subp': {'side_effect': fake_subp},
+             'CLOUDINIT_LOGS': {'new': [log1, log2]},
+             'CLOUDINIT_RUN_DIR': {'new': self.run_dir}},
+            logs.collect_logs, output_tarfile)
+        # unpack the tarfile and check file contents
+        subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root])
+        out_logdir = self.tmp_path(date_logdir, self.new_root)
+        self.assertEqual(
+            '0.7fake',
+            load_file(os.path.join(out_logdir, 'version')))
+        self.assertEqual(
+            'cloud-init-log',
+            load_file(os.path.join(out_logdir, 'cloud-init.log')))
+        self.assertEqual(
+            'cloud-init-output-log',
+            load_file(os.path.join(out_logdir, 'cloud-init-output.log')))
+        self.assertEqual(
+            'dmesg-out',
+            load_file(os.path.join(out_logdir, 'dmesg.txt')))
+        self.assertEqual(
+            'journal-out',
+            load_file(os.path.join(out_logdir, 'journal.txt')))
+        self.assertEqual(
+            'results',
+            load_file(
+                os.path.join(out_logdir, 'run', 'cloud-init', 'results.json')))
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 68563e0..95bb4c2 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -764,16 +764,25 @@ def main(sysv_args=None):
     parser_devel = subparsers.add_parser(
         'devel', help='Run development tools')
 
+    parser_collect_logs = subparsers.add_parser(
+        'collect-logs', help='Collect and tar all cloud-init logs')
+
     if sysv_args:
         # Only load subparsers if subcommand is specified to avoid load cost
         if sysv_args[0] == 'analyze':
             from cloudinit.analyze.__main__ import get_parser as analyze_parser
             # Construct analyze subcommand parser
             analyze_parser(parser_analyze)
-        if sysv_args[0] == 'devel':
+        elif sysv_args[0] == 'devel':
             from cloudinit.cmd.devel.parser import get_parser as devel_parser
             # Construct devel subcommand parser
             devel_parser(parser_devel)
+        elif sysv_args[0] == 'collect-logs':
+            from cloudinit.cmd.devel.logs import (
+                get_parser as logs_parser, handle_collect_logs_args)
+            logs_parser(parser_collect_logs)
+            parser_collect_logs.set_defaults(
+                action=('collect-logs', handle_collect_logs_args))
 
     args = parser.parse_args(args=sysv_args)
 
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 495bdc9..258a9f0 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -72,18 +72,22 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
 
     def test_conditional_subcommands_from_entry_point_sys_argv(self):
         """Subcommands from entry-point are properly parsed from sys.argv."""
+        stdout = six.StringIO()
+        self.patchStdoutAndStderr(stdout=stdout)
+
         expected_errors = [
-            'usage: cloud-init analyze', 'usage: cloud-init devel']
-        conditional_subcommands = ['analyze', 'devel']
+            'usage: cloud-init analyze', 'usage: cloud-init collect-logs',
+            'usage: cloud-init devel']
+        conditional_subcommands = ['analyze', 'collect-logs', 'devel']
         # The cloud-init entrypoint calls main without passing sys_argv
         for subcommand in conditional_subcommands:
-            with mock.patch('sys.argv', ['cloud-init', subcommand]):
+            with mock.patch('sys.argv', ['cloud-init', subcommand, '-h']):
                 try:
                     cli.main()
                 except SystemExit as e:
-                    self.assertEqual(2, e.code)  # exit 2 on proper usage docs
+                    self.assertEqual(0, e.code)  # exit 2 on proper -h usage
         for error_message in expected_errors:
-            self.assertIn(error_message, self.stderr.getvalue())
+            self.assertIn(error_message, stdout.getvalue())
 
     def test_analyze_subcommand_parser(self):
         """The subcommand cloud-init analyze calls the correct subparser."""
@@ -94,6 +98,14 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         for subcommand in expected_subcommands:
             self.assertIn(subcommand, error)
 
+    def test_collect_logs_subcommand_parser(self):
+        """The subcommand cloud-init collect-logs calls the subparser."""
+        # Provide -h param to collect-logs to avoid having to mock behavior.
+        stdout = six.StringIO()
+        self.patchStdoutAndStderr(stdout=stdout)
+        self._call_main(['cloud-init', 'collect-logs', '-h'])
+        self.assertIn('usage: cloud-init collect-log', stdout.getvalue())
+
     def test_devel_subcommand_parser(self):
         """The subcommand cloud-init devel calls the correct subparser."""
         self._call_main(['cloud-init', 'devel'])