launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #05250
[Merge] lp:~lifeless/python-oops-tools/amqp into lp:python-oops-tools
Robert Collins has proposed merging lp:~lifeless/python-oops-tools/amqp into lp:python-oops-tools.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~lifeless/python-oops-tools/amqp/+merge/79505
Build on the recent improvements in oops-datedir-repo and oops-amqp to add an amqp queue worker that will take oopses from amqp and load them straight into OOPS-tools. Also fix docs (0.6 is the current release) and stop telling folk to edit product.cfg in-place.
--
https://code.launchpad.net/~lifeless/python-oops-tools/amqp/+merge/79505
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~lifeless/python-oops-tools/amqp into lp:python-oops-tools.
=== modified file 'setup.py'
--- setup.py 2011-10-13 20:18:51 +0000
+++ setup.py 2011-10-16 22:38:22 +0000
@@ -57,6 +57,7 @@
'launchpadlib',
'lazr.config',
'oops',
+ 'oops-amqp',
'oops-datedir-repo',
'pytz',
'setuptools',
@@ -79,6 +80,7 @@
),
entry_points=dict(
console_scripts=[ # `console_scripts` is a magic name to setuptools
+ 'amqp2disk = oopstools.scripts.amqp2disk:main',
'analyse_error_reports = oopstools.scripts.analyse_error_reports:main',
'load_sample_data = oopstools.scripts.load_sample_data:main',
'update_db = oopstools.scripts.update_db:main',
=== modified file 'src/oopstools/NEWS.txt'
--- src/oopstools/NEWS.txt 2011-10-13 20:18:51 +0000
+++ src/oopstools/NEWS.txt 2011-10-16 22:38:22 +0000
@@ -2,8 +2,13 @@
NEWS for oopstools
===================
-0.5 (UNRELEASED)
-================
+NEXT
+====
+
+* Added AMQP support via the bin/amqp2disk script. (Robert Collins)
+
+0.6
+===
* Initial release
=== modified file 'src/oopstools/README.txt'
--- src/oopstools/README.txt 2011-10-13 20:18:51 +0000
+++ src/oopstools/README.txt 2011-10-16 22:38:22 +0000
@@ -51,19 +51,21 @@
Deployment using mod_wsgi
=========================
-Update the production.cfg file with your database configuration and the paths
-to your OOPS directories:
+Create a custom cfg file - start with production.cfg and take a copy. Update
+your copy with your database configuration and the paths to your OOPS
+directories:
[configuration]
db-name = /path/to/your/oops.db
index-template = 'index.html'
-oopsdir = /path/to/oops/reports
+oopsdir = /path/where/rsynced/oopses/are/found
+ /another/such/path
Update settings.py setting a custom SECRET_KEY
To deploy oops tools make sure all the dependecies are installed.
- * bin/buildout -c production.cfg
+ * bin/buildout -c yourfilename.cfg
* Run bin/django syncdb
@@ -71,6 +73,15 @@
* Copy apache/oops-tools.dev.mod_wsgi to /etc/apache2/sites-available/
+AMQP Integration
+================
+
+The script bin/amqp2disk is an AMQP handler that will receive OOPS reports over
+AMQP and publish them locally to disk as well as loading the metadata directly
+into the oops-tools database. To use this you will need to config your OOPS
+creation to publish over AMQP. If you are using Python then the oops-amqp
+module will help you do this.
+
Running locally
===============
=== modified file 'src/oopstools/oops/models.py'
--- src/oopstools/oops/models.py 2011-10-13 20:18:51 +0000
+++ src/oopstools/oops/models.py 2011-10-16 22:38:22 +0000
@@ -21,6 +21,8 @@
import datetime
import os.path
+from pytz import utc
+
from django.db import models
from django.db.models.signals import pre_save
import oops_datedir_repo.serializer
@@ -278,7 +280,11 @@
prefix = oops.get('reporter')
if not prefix:
# Legacy support for pre-reporter using OOPSes.
- prefix = oops_re.match(oopsid).group('oopsprefix')
+ prefix_match = oops_re.match(oopsid)
+ if prefix_match is not None:
+ prefix = prefix_match.group('oopsprefix')
+ else:
+ prefix = 'UNKNOWN'
prefix = prefix.upper()
try:
prefix = Prefix.objects.get(value__exact=prefix)
@@ -329,9 +335,10 @@
if total_time < 0:
total_time = 0
# Get the oops infestation
- exception_type = oops.get('type')
+ exception_type = oops.get('type') or ''
+ exception_value = oops.get('value') or ''
exception_value = _normalize_exception_value(
- exception_type, oops.get('value'), prefix)
+ exception_type, exception_value, prefix)
try:
infestation = Infestation.objects.get(
exception_type__exact=exception_type,
@@ -361,10 +368,13 @@
most_expensive_statement = conform(most_expensive_statement, 200)
url = conform(oops.get('url') or '', MAX_URL_LEN)
informational = oops.get('informational', 'False').lower() == 'true'
+ oops_date = oops.get('time')
+ if oops_date is None:
+ oops_date = datetime.datetime.now(utc)
data.update(
oopsid = oopsid,
prefix = prefix,
- date = oops.get('time').replace(microsecond=0),
+ date = oops_date.replace(microsecond=0),
# Missing pageids are urls because that suits our queries.
pageid = oops.get('topic') or url,
url = url,
@@ -377,7 +387,28 @@
is_bot = _robot_pat.search(data['user_agent']) is not None,
statements_count = len(statements),
)
- return data, req_vars, statements, oops.get('tb_text')
+ return data, req_vars, statements, oops.get('tb_text') or ''
+
+
+def parsed_oops_to_model_oops(parsed_oops, pathname):
+ """Convert an oops report dict to an Oops object."""
+ data, req_vars, statements, traceback = _get_oops_tuple(parsed_oops)
+ data['pathname'] = pathname
+ res = Oops(**data)
+ res.appinstance = res.get_appinstance()
+ res.save()
+ # Get it again. Otherwise we have discrepancies between old and
+ # new oops objects: old ones have unicode attributes, and new
+ # ones have string attributes, for instance. Ideally the message
+ # conversion would have converted everything to unicode, but it
+ # doesn't easily.
+ res = Oops.objects.get(oopsid__exact=parsed_oops['id'])
+ res.parsed_oops = parsed_oops
+ res.req_vars = req_vars
+ res.statements = statements
+ res.traceback = traceback
+ res.save()
+ return res
class Oops(models.Model):
@@ -432,21 +463,7 @@
try:
res = cls.objects.get(oopsid__exact=oopsid)
except cls.DoesNotExist:
- data, req_vars, statements, traceback = _get_oops_tuple(parsed_oops)
- data['pathname'] = pathname
- res = cls(**data)
- res.appinstance = res.get_appinstance()
- res.save()
- # Get it again. Otherwise we have discrepancies between old and
- # new oops objects: old ones have unicode attributes, and new
- # ones have string attributes, for instance. Ideally the message
- # conversion would have converted everything to unicode, but it
- # doesn't easily.
- res = cls.objects.get(oopsid__exact=oopsid)
- res.parsed_oops = parsed_oops
- res.req_vars = req_vars
- res.statements = statements
- res.traceback = traceback
+ res = parsed_oops_to_model_oops(parsed_oops, pathname)
return res
@readproperty
=== added file 'src/oopstools/oops/test/test_amqp2disk.py'
--- src/oopstools/oops/test/test_amqp2disk.py 1970-01-01 00:00:00 +0000
+++ src/oopstools/oops/test/test_amqp2disk.py 2011-10-16 22:38:22 +0000
@@ -0,0 +1,40 @@
+# Copyright 2005-2011 Canonical Ltd. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import os.path
+
+import bson
+from fixtures import TempDir
+from testtools import TestCase
+
+from oopstools.oops.models import Oops
+from oopstools.scripts import amqp2disk
+
+
+class TestOOPSConfig(TestCase):
+
+ def test_publishes_disk_and_DB(self):
+ self.root_dir = self.useFixture(TempDir()).path
+ config = amqp2disk.make_amqp_config(self.root_dir)
+ orig_report = {'id': '12345'}
+ report = dict(orig_report)
+ ids = config.publish(report)
+ self.assertEqual(['12345', '12345'], ids)
+ with open(report['datedir_repo_filepath'], 'rb') as fp:
+ disk_report = bson.loads(fp.read())
+ self.assertEqual(disk_report, orig_report)
+ model_report = Oops.objects.get(oopsid='12345')
+ self.assertNotEqual(None, model_report)
=== added file 'src/oopstools/scripts/amqp2disk.py'
--- src/oopstools/scripts/amqp2disk.py 1970-01-01 00:00:00 +0000
+++ src/oopstools/scripts/amqp2disk.py 2011-10-16 22:38:22 +0000
@@ -0,0 +1,137 @@
+# Copyright 2011 Canonical Ltd. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Receive OOPS reports from AMQP and publish them into the oops-tools
+# repository.
+
+__metaclass__ = type
+
+import sys
+import optparse
+import StringIO
+from textwrap import dedent
+
+import amqplib.client_0_8 as amqp
+from oops import Config
+import oops_amqp
+import oops_datedir_repo
+
+from oopstools.oops.helpers import parsedate, load_prefixes
+from oopstools.oops.models import (
+ Oops,
+ parsed_oops_to_model_oops,
+ Prefix,
+ Report,
+ )
+from oopstools.oops.oopsstore import OopsStore
+from oopstools.oops import dbsummaries
+from oopstools.oops.summaries import (
+ WebAppErrorSummary,
+ CheckwatchesErrorSummary,
+ CodeHostingWithRemoteSectionSummary,
+ GenericErrorSummary,
+)
+
+
+def main(argv=None):
+ if argv is None:
+ argv=sys.argv
+ usage = dedent("""\
+ %prog [options]
+
+ The following options must be supplied:
+ --output
+ --host
+ --username
+ --password
+ --vhost
+ --queue
+
+ e.g.
+ amqp2disk --output /srv/oops-tools/amqpoopses --host "localhost:3472" \\
+ --username "guest" --password "guest" --vhost "/" --queue "oops"
+
+ The AMQP environment should be setup in advance with a persistent queue
+ bound to your exchange : using transient queues would allow OOPSes to
+ be lost if the amqp2disk process were to be shutdown for a non-trivial
+ duration. The --bind-to option will cause the queue to be created and
+ bound to the given exchange. This is only needed the first time as it
+ is created persistently.
+ """)
+ description = "Load OOPS reports into oops-tools from AMQP."
+ parser = optparse.OptionParser(
+ description=description, usage=usage)
+ parser.add_option('--output', help="Root directory to store OOPSes in.")
+ parser.add_option('--host', help="AQMP host / host:port.")
+ parser.add_option('--username', help="AQMP username.")
+ parser.add_option('--password', help="AQMP password.")
+ parser.add_option('--vhost', help="AMQP vhost.")
+ parser.add_option('--queue', help="AMQP queue name.")
+ parser.add_option(
+ '--bind-to', help="AMQP exchange to bind to (only needed once).")
+ options, args = parser.parse_args(argv[1:])
+ def needed(optname):
+ if getattr(options, optname, None) is None:
+ raise ValueError('option "%s" must be supplied' % optname)
+ needed('host')
+ needed('output')
+ needed('username')
+ needed('password')
+ needed('vhost')
+ needed('queue')
+ connection = amqp.Connection(host=options.host, userid=options.username,
+ password=options.password, virtual_host=options.vhost)
+ channel = connection.channel()
+ if options.bind_to:
+ channel.queue_declare(options.queue, durable=True, auto_delete=False)
+ channel.queue_bind(options.queue, options.bind_to)
+ config = make_amqp_config(options.output)
+ receiver = oops_amqp.Receiver(config, channel, options.queue)
+ try:
+ receiver.run_forever()
+ except KeyboardInterrupt:
+ pass
+
+
+def db_publisher(report):
+ """Publish OOPS reports to the oops-tools django store."""
+ # the first publisher will either inherit or assign, so this should be
+ # impossible.
+ assert report['id'] is not None
+ # Some fallback methods could lead to duplicate paths into the DB: exit
+ # early if the OOPS is already loaded.
+ try:
+ res = Oops.objects.get(oopsid__exact=report['id'])
+ except Oops.DoesNotExist:
+ res = parsed_oops_to_model_oops(
+ report, report['datedir_repo_filepath'])
+ return res.oopsid
+ return None
+
+
+def make_amqp_config(output_dir):
+ """Create an OOPS Config for republishing amqp OOPSes.
+
+ An OOPS published to this config will be written to disk and then loaded
+ into the database.
+
+ :param output_dir: The directory to write OOPSes too.
+ """
+ config = Config()
+ disk_publisher = oops_datedir_repo.DateDirRepo(
+ output_dir, inherit_id=True, stash_path=True)
+ config.publishers.append(disk_publisher.publish)
+ config.publishers.append(db_publisher)
+ return config
=== modified file 'versions.cfg'
--- versions.cfg 2011-10-13 20:18:51 +0000
+++ versions.cfg 2011-10-16 22:38:22 +0000
@@ -19,8 +19,9 @@
launchpadlib = 1.6.0
lazr.config = 1.1.3
mechanize = 0.1.11
-oops = 0.0.7
-oops-datedir-repo = 0.0.7
+oops = 0.0.9
+oops-amqp = 0.0.1
+oops-datedir-repo = 0.0.9
setuptools = 0.6c11
z3c.recipe.filetemplate = 2.0.3
z3c.recipe.sphinxdoc = 0.0.8
