launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #10790
[Merge] lp:~wgrant/launchpad/flaky-flaky into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/flaky-flaky into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/flaky-flaky/+merge/118892
This branch drops some ancient scripts from utilities:
* flaky: A pyflakes wrapper from 2006 which doesn't work with any vaguely modern pyflakes. Use bin/lint.sh instead.
* rocketfuel-flakes: Another pyflakes wrapper from ages ago. Use bin/lint.sh instead.
* check-scripts.py: Now integrated into the test suite properly as test_all_scripts.
* check-configs.py: Integrated into the test suite as test_config.
* summarizelogs.py: A crude top exception/timeout reporter from appserver logs. Probably doesn't work any more. Use OOPS instead.
--
https://code.launchpad.net/~wgrant/launchpad/flaky-flaky/+merge/118892
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/flaky-flaky into lp:launchpad.
=== modified file 'Makefile'
--- Makefile 2012-07-13 01:55:27 +0000
+++ Makefile 2012-08-09 08:01:20 +0000
@@ -131,12 +131,6 @@
xxxreport: $(PY)
${PY} -t ./utilities/xxxreport.py -f csv -o xxx-report.csv ./
-check-configs: $(PY)
- ${PY} utilities/check-configs.py
-
-pagetests: build
- env PYTHONPATH=$(PYTHONPATH) bin/test test_pages
-
codehosting-dir:
mkdir -p $(CODEHOSTING_ROOT)/mirrors
mkdir -p $(CODEHOSTING_ROOT)/config
=== removed file 'utilities/check-configs.py'
--- utilities/check-configs.py 2012-06-29 08:40:05 +0000
+++ utilities/check-configs.py 1970-01-01 00:00:00 +0000
@@ -1,117 +0,0 @@
-#! /usr/bin/python -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Check that all the launchpad.conf files can be loaded.
-
-Usage hint:
-
-% utilities/check-configs.py -v
-"""
-
-import _pythonpath
-
-import optparse
-import os
-import sys
-import traceback
-
-from lazr.config import ConfigSchema
-from lazr.config.interfaces import ConfigErrors
-import ZConfig
-
-# Calculate some landmark paths.
-import lp.services.config
-
-
-here = os.path.dirname(lp.services.config.__file__)
-lazr_schema_file = os.path.join(here, 'schema-lazr.conf')
-lazr_schema = ConfigSchema(lazr_schema_file)
-zconfig_schema_file = os.path.join(
- here, os.pardir, os.pardir, 'zope/app/server/schema.xml')
-zconfig_schema = ZConfig.loadSchema(zconfig_schema_file)
-
-
-def main():
- parser = optparse.OptionParser(usage="""\
-%prog [options] [zconfig_overrides]
-
-Parse all launchpad.conf and *-lazr.conf files found under the 'config'
-directory rooted at the current working directory. Warn about any
-problems loading the config file.
-
-With a specified directory, search there instead of the current working
-directory for launchpad.conf files. The search is always recursive.
-
-The environment variable LPCONFIG can be used to limit the search to only
-subdirectories of config that match $LPCONFIG, otherwise all are searched.
-
-zconfig_overrides are passed directly to ZConfig.loadConfig().
-""")
- parser.add_option('-v', '--verbose',
- action='count', dest='verbosity',
- help='Increase verbosity')
- options, arguments = parser.parse_args()
-
- # Are we searching for one config or for all configs?
- directory = os.path.join(here, os.pardir, os.pardir, os.pardir, 'configs')
- configs = []
- lpconfig = os.environ.get('LPCONFIG')
- if lpconfig is None:
- for dirpath, dirnames, filenames in os.walk(directory):
- for filename in filenames:
- if (filename == 'launchpad.conf'
- or filename.endswith('-lazr.conf')):
- configs.append(os.path.join(dirpath, filename))
- else:
- configs.append(os.path.join(directory, lpconfig, 'launchpad.conf'))
- configs.append(os.path.join(
- directory, lpconfig, 'launchpad-lazr.conf'))
-
- # Load each config and report any errors.
- summary = []
- for config in sorted(configs):
- if config.endswith('launchpad.conf'):
- # This is a ZConfig conf file.
- try:
- root, handlers = ZConfig.loadConfig(
- zconfig_schema, config, arguments)
- except ZConfig.ConfigurationSyntaxError as error:
- if options.verbosity > 2:
- traceback.print_exc()
- elif options.verbosity > 1:
- print error
- summary.append((config, False))
- else:
- summary.append((config, True))
- else:
- # This is a lazr.config conf file.
- lazr_config = lazr_schema.load(config)
- try:
- lazr_config.validate()
- except ConfigErrors as error:
- if options.verbosity > 2:
- messages = '\n'.join([str(er) for er in error.errors])
- print messages
- elif options.verbosity > 1:
- print error
- summary.append((config, False))
- else:
- summary.append((config, True))
-
- prefix_length = len(directory)
- for config, status in summary:
- path = config[prefix_length + 1:]
- if status:
- if options.verbosity > 0:
- print 'SUCCESS:', path
- else:
- print 'FAILURE:', path
-
- # Return a useful exit code. 0 == all success.
- return len([config for config, status in summary if not status])
-
-
-if __name__ == '__main__':
- sys.exit(main())
=== removed file 'utilities/check-scripts.py'
--- utilities/check-scripts.py 2010-04-27 19:48:39 +0000
+++ utilities/check-scripts.py 1970-01-01 00:00:00 +0000
@@ -1,47 +0,0 @@
-#! /usr/bin/python -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Check that all the launchpad scripts and cronscripts run.
-
-Usage hint:
-
-% utilities/check-scripts.py
-"""
-# pylint: disable-msg=W0403
-import _pythonpath
-
-import os
-import sys
-
-from lp.services.scripts.tests import find_lp_scripts
-from lp.testing import run_script
-
-
-def check_script(script_path):
- """Run the given script in a subprocess and report its result.
-
- Check if the script successfully runs if 'help' is requested via
- command line argument ('-h').
- """
- sys.stdout.write('Checking: %s ' % script_path)
- sys.stdout.flush()
- cmd_line = script_path + " -h"
- out, err, returncode = run_script(cmd_line)
- if returncode != os.EX_OK:
- sys.stdout.write('... FAILED\n')
- sys.stdout.write('%s\n' % err)
- else:
- sys.stdout.write('... OK\n')
- sys.stdout.flush()
-
-
-def main():
- """Walk over the specified script locations and check them."""
- for script_path in find_lp_scripts():
- check_script(script_path)
-
-
-if __name__ == '__main__':
- sys.exit(main())
=== removed file 'utilities/flaky.py'
--- utilities/flaky.py 2010-04-27 19:48:39 +0000
+++ utilities/flaky.py 1970-01-01 00:00:00 +0000
@@ -1,177 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Run pyflakes checks on a set of files."""
-
-import compiler
-import os
-import sys
-import traceback
-
-import pyflakes
-
-
-class Flakiness:
- COMPILE_FAILED = 0
- FLAKY = 1
- GOOD = 2
-
-
-class PyflakesResult:
- def __init__(self, filename, flakiness, messages):
- self.filename = filename
- self.flakiness = flakiness
- self.messages = messages
-
- def make_report(self):
- """Generate a text report for the result.
-
- Yields a line of text for each line of the report.
- """
-
- if self.flakiness == Flakiness.GOOD:
- return
-
- if self.flakiness == Flakiness.COMPILE_FAILED:
- yield '%s: Failed to compile:' % self.filename
-
- for message in self.messages:
- yield ' ' + str(message)
- elif self.flakiness == Flakiness.FLAKY:
- yield '%s:' % self.filename
-
- for message in self.messages:
- yield ' line ' + str(message)[len(self.filename)+1:]
-
- yield ''
-
-
-class PyflakesStatistics:
- """Counts of pyflakes messages over multiple Python files."""
-
- message_classes = {
- pyflakes.messages.UndefinedName: 'messages_undefined_name',
- pyflakes.messages.UnusedImport: 'messages_unused_import',
- pyflakes.messages.ImportStarUsed: 'messages_import_star',
- pyflakes.messages.RedefinedWhileUnused: 'messages_redefined_unused',
- }
-
- def __init__(self):
- self.files_total = 0
- self.files_compile_failed = 0
- self.files_flaky = 0
- self.files_good = 0
- self.messages_total = 0
- self.messages_undefined_name = 0
- self.messages_unused_import = 0
- self.messages_import_star = 0
- self.messages_redefined_unused = 0
-
- def add_result(self, result):
- self.files_total += 1
-
- if result.flakiness == Flakiness.GOOD:
- self.files_good += 1
- elif result.flakiness == Flakiness.COMPILE_FAILED:
- self.files_compile_failed += 1
- elif result.flakiness == Flakiness.FLAKY:
- self.files_flaky += 1
- statistic = None
-
- for message in result.messages:
- self.messages_total += 1
-
- # Increment the appropriate self.messages_* count.
- attr = PyflakesStatistics.message_classes[message.__class__]
- statistic = getattr(self, attr)
- setattr(self, attr, statistic + 1)
-
- def make_summary(self):
- return [
- 'Files checked: %d' % self.files_total,
- 'Files that failed to compile: %d' %
- self.files_compile_failed,
- 'Good files: %d' % self.files_good,
- 'Flaky files: %d' % self.files_flaky,
- ' - Undefined name: %d' % self.messages_undefined_name,
- ' - Unused imports: %d' % self.messages_unused_import,
- ' - * imported: %d' % self.messages_import_star,
- ' - Unused name redefined: %d' % self.messages_redefined_unused,
- ' - Problems total: %d' % self.messages_total,
- ]
-
-
-def find_python_files(top_path):
- for dirpath, dirnames, filenames in os.walk(top_path):
- for filename in filenames:
- if filename.endswith('.py'):
- yield os.path.join(dirpath, filename)
-
-
-def check_file(filename):
- """Return a list of pyflakes messages for a Python file."""
-
- source = open(filename).read()
-
- try:
- tree = compiler.parse(source)
- except (SyntaxError, IndentationError):
- tb_info = traceback.format_exception(*sys.exc_info())
- messages = [message[:-1] for message in tb_info]
- return PyflakesResult(filename, Flakiness.COMPILE_FAILED, messages)
- else:
- checker = pyflakes.Checker(tree, filename)
- messages = sorted(checker.messages, key=lambda message: message.lineno)
-
- if messages:
- return PyflakesResult(filename, Flakiness.FLAKY, messages)
- else:
- return PyflakesResult(filename, Flakiness.GOOD, messages)
-
-
-def main(argv):
- if len(argv) < 2:
- print >>sys.stderr, 'Usage: %s path...' % argv[0]
- return 1
-
- files = []
-
- for path in argv[1:]:
- if os.path.isdir(path):
- files.extend(find_python_files(argv[1]))
- else:
- files.append(path)
-
- sys.stderr.write('Running pyflakes checks\n')
- results = []
-
- for filename in files:
- results.append(check_file(filename))
- sys.stderr.write('.')
- sys.stderr.flush()
-
- sys.stderr.write('\nDone\n\n')
-
- for result in results:
- for line in result.make_report():
- print line
-
- statistics = PyflakesStatistics()
-
- for result in results:
- statistics.add_result(result)
-
- for line in statistics.make_summary():
- print line
-
- if statistics.files_compile_failed + statistics.files_flaky > 0:
- return 1
- else:
- return 0
-
-
-if __name__ == '__main__':
- sys.exit(main(sys.argv))
-
=== removed file 'utilities/modified-flaky.sh'
--- utilities/modified-flaky.sh 2009-06-24 20:15:50 +0000
+++ utilities/modified-flaky.sh 1970-01-01 00:00:00 +0000
@@ -1,10 +0,0 @@
-#!/bin/sh
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# Run pyflakes checks on files modified in working copy.
-(bzr status | sed -e '0,/^modified/d; /^[^ ]/,$d';
- bzr status | sed -e '0,/^added/d; /^[^ ]/,$d') |\
- grep '\.py$' |\
- xargs `dirname $0`/flaky.py
=== removed file 'utilities/rocketfuel-flakes'
--- utilities/rocketfuel-flakes 2009-06-24 20:15:50 +0000
+++ utilities/rocketfuel-flakes 1970-01-01 00:00:00 +0000
@@ -1,10 +0,0 @@
-#! /bin/bash
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-cd `bzr root`
-
-bzr diff -rbranch:../trunk | lsdiff| grep "\.py$" | xargs pyflakes | grep -v "__init__"
-cd -
-
=== removed file 'utilities/summarizelogs.py'
--- utilities/summarizelogs.py 2012-01-01 03:10:25 +0000
+++ utilities/summarizelogs.py 1970-01-01 00:00:00 +0000
@@ -1,96 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-#
-# parselogs.py
-# Christian Reis <kiko@xxxxxxxxxxxx>
-#
-# Parses Launchpad error logs and returns a list of most frequent errors
-
-import datetime
-import pprint
-import re
-import sys
-import time
-
-
-COUNT = 10
-LAST_DAYS = 7
-
-def init_or_set(d, v):
- if d.has_key(v):
- d[v] += 1
- else:
- d[v] = 1
-
-def init_list_or_set(d, v, e):
- if d.has_key(v):
- d[v][e] = 1
- else:
- d[v] = {e: 1}
-
-if len(sys.argv) == 1:
- lognames = ["launchpad1.log", "launchpad2.log"]
-else:
- lognames = sys.argv[1:]
-
-exceptions = {}
-expired = {}
-url_table = {}
-
-now = datetime.datetime.fromtimestamp(time.time())
-for logname in lognames:
- text = open(logname).read()
- errors = text.split("------")
- for error in errors:
- error = error.strip()
- if not error:
- continue
-
- fullerror = error
- error = error.split("\n")[-1].strip()
- first_line = fullerror.split("\n")[0]
-
- date = first_line.split(" ")[0]
- # XXX kiko 2005-10-17: handle timezone properly; it kinda sucks that
- # we have no way of knowing what timezone the log originates from.
- # For now I hack around this by assuming timezone is UTC.
- ts = time.strftime("%s", time.strptime(date, "%Y-%m-%dT%H:%M:%S"))
- then = datetime.datetime.fromtimestamp(float(ts))
- if now - then > datetime.timedelta(days=LAST_DAYS):
- continue
-
- if " WARNING " in error:
- continue
- extra = " ".join(first_line.split()[3:])
- if "RequestExpired:" in error:
- error = "RequestExpired: %s" % extra
- init_or_set(expired, error)
- continue
- if re.search("0x[abcdef0-9]+", error):
- error = re.sub("0x[abcdef0-9]+", "INSTANCE-ID", error)
- init_or_set(exceptions, error)
- init_list_or_set(url_table, error, extra)
-
-values = exceptions.items()
-values.sort(key=lambda x: x[1], reverse=True)
-
-print
-print "=== Top %d exceptions in the past %d days ===" % (COUNT, LAST_DAYS)
-print
-for exc, count in values[:COUNT]:
- print count, "\t", exc
- print "\t\t", "\n\t\t".join(url_table[exc].keys()[:10])
-
-values = expired.items()
-values.sort(key=lambda x: x[1], reverse=True)
-
-print
-print
-print "=== Top %d timed out pages in the past %d days ===" % (COUNT, LAST_DAYS)
-print
-for url, count in values[:COUNT]:
- print count, "\t", url
-
-
Follow ups
