apport-hackers team mailing list archive

[James Hunt <james.hunt@xxxxxxxxxxxxx>]

James Hunt has proposed merging lp:~jamesodhunt/apport/bug-1256268 into lp:apport.

Requested reviews:
  Apport upstream developers (apport-hackers)
Related bugs:
  Bug #1256268 in Apport: "handle high cpu load and high memory usage issues"
  https://bugs.launchpad.net/apport/+bug/1256268

For more details, see:
https://code.launchpad.net/~jamesodhunt/apport/bug-1256268/+merge/198946

I've tried to specify "reasonable" default values, but we may need to tweak the following (or maybe even allow them to be configured via /etc/apport/)?:

# Consider this number of the top cpu-hogging processes.
max_hogs = 3

# Percentage threshold for cpu and memory hogs
cpu_threshold = 80
mem_threshold = 50


-- 
https://code.launchpad.net/~jamesodhunt/apport/bug-1256268/+merge/198946
Your team Apport upstream developers is requested to review the proposed merge of lp:~jamesodhunt/apport/bug-1256268 into lp:apport.

=== added file 'data/general-hooks/resource_hogs.py'
--- data/general-hooks/resource_hogs.py	1970-01-01 00:00:00 +0000
+++ data/general-hooks/resource_hogs.py	2013-12-13 14:17:25 +0000
@@ -0,0 +1,200 @@
+# Look for resource-hogging processes (currently CPU+Memory).
+#
+# Copyright (C) 2013 Canonical Ltd.
+# Author: James Hunt <james.hunt@xxxxxxxxxxxxx>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
+# the full text of the license.
+
+import os
+import re
+import apport.packaging
+from apport.hookutils import *
+
+def get_pid_path(pid):
+    '''
+    Determine the full path to the command running as the specified pid.
+
+    Returns: Full path, or None on error.
+    '''
+    fields = read_file('/proc/' + pid + '/cmdline').split('\0')
+
+    cmd_path = fields[0]
+
+    if not cmd_path:
+        # kernel thread?
+        return None
+
+    if cmd_path[0] == '/':
+        return cmd_path
+
+    # No absolute path so look up the command in the
+    # processes PATH
+    cmd = cmd_path
+
+    env = read_file('/proc/' + pid + '/environ').split('\0')
+    if not env:
+        return None
+
+    path = [x for x in env if x.startswith('PATH=') ]
+    if not path:
+        return None
+
+    path = path[0]
+
+    for elem in path.split(':'):
+        cmd_path = '{}/{}'.format(elem, cmd)
+        if os.path.exists(cmd_path):
+            return cmd_path
+
+    return None
+
+
+def run_top(max_lines, sort_field):
+    '''
+    Run top(1).
+
+    @max_lines: number of output lines to keep,
+    @sort_field: field to sort top outpu by.
+
+    Returns: List of (up to size @max_lines elements) with each element
+    containing a list of top output fields.
+    '''
+    lines = []
+    count = 0
+    args = ['top', '-b', '-n1' ]
+
+    args += [ '-o{}'.format(sort_field) ]
+
+    output = command_output(args).split('\n')
+
+    for line in output:
+        fields = line.split()
+
+        # ignore header lines
+        if (len(fields) != 12):
+            continue
+
+        # pid
+        result = re.match('\d+', fields[0])
+        if not result:
+            continue
+
+        # user
+        if not re.match('\w+', fields[1]):
+            continue
+
+        # time
+        if not re.match('\d+:\d+\.\d+', fields[10]):
+            continue
+
+        # command
+        cmd = fields[11]
+        if not cmd:
+            continue
+
+        if count == max_lines:
+            break
+
+        count += 1
+
+        lines.append(fields)
+
+    return lines
+
+
+def check_for_hogs(report, reported_package, max_hogs, cpu_threshold, mem_threshold):
+    '''
+    Check for cpu and memory hogs.
+    '''
+
+    cpu_hogs = get_hogs(report, reported_package, max_hogs, 'cpu-hog', '%CPU', 8, cpu_threshold)
+    mem_hogs = get_hogs(report, reported_package, max_hogs, 'memory-hog', '%MEM', 9, mem_threshold)
+
+    if len(cpu_hogs):
+        report['CPUHogs'] = str(cpu_hogs)
+
+    if len(mem_hogs):
+        report['MemoryHogs'] = str(mem_hogs)
+
+
+def get_hogs(report, reported_package, max_hogs, tag, top_sort_field, field_count, threshold):
+    '''
+    Determine a list of resource hogs.
+
+    @report: report,
+    @reported_package: name of package problem is being reported against,
+    @max_hogs: number of hogs to consider,
+    @top_sort_field: name of top(1) field for a specified resource,
+    @field_count: the field number of @top_sort_field (zero-indexed),
+    @threshold: float threshold value - any process above this will be
+     considered to be a hog.
+
+    Returns: List of tuples where each tuple is (full_command_path, package, percentage).
+    '''
+    hogs = []
+    count = 0
+
+    lines = run_top(max_hogs, top_sort_field)
+
+    for fields in lines:
+        pid = fields[0]
+
+        result = re.match('\d+\.\d+', fields[field_count])
+        if not result:
+            continue
+        hog_field = float(result.group(0))
+
+        count += 1
+
+        # top(1) displays sorted output in descending order so if the top
+        # offenders don't meet the threshold, there is no point
+        # looking any further.
+        if count == 1 and hog_field < threshold:
+            break
+
+        cmd = read_file('/proc/' + pid + '/cmdline').split('\0')[0]
+
+        if not cmd:
+            # kernel thread gone haywire?
+            continue
+
+        if cmd[0] != '/':
+            cmd = get_pid_path(pid)
+            if not cmd:
+                continue
+
+        pkg = apport.packaging.get_file_package(cmd)
+
+        # One of the hogs referred to the package the problem is being
+        # reported against.
+        if pkg == reported_package:
+            report['Tags'] += ' {}'.format(tag)
+
+        hogs.append((cmd, pkg, hog_field))
+
+    return hogs
+
+
+def add_info(report):
+    # Consider this number of the top cpu-hogging processes.
+    max_hogs = 3
+
+    # Percentage threshold for cpu and memory hogs
+    cpu_threshold = 80
+    mem_threshold = 50
+
+    # Only consider bugs reports for now
+    if report.get('ProblemType', None) != 'Bug':
+        return
+
+    # We need to know the package the user is reporting the bug against
+    if not 'Package' in report:
+        return
+    reported_package = report.get('Package').split()[0]
+
+    check_for_hogs(report, reported_package, max_hogs, cpu_threshold, mem_threshold)
+