launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24920
[Merge] ~cjwatson/launchpad:hwdb-remove-submission-jobs into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:hwdb-remove-submission-jobs into launchpad:master.
Commit message:
Remove hardware DB submission processing jobs
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/386509
We stopped accepting new submissions a while ago.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:hwdb-remove-submission-jobs into launchpad:master.
diff --git a/cronscripts/process-hwdb-submissions.py b/cronscripts/process-hwdb-submissions.py
deleted file mode 100755
index 867b4cf..0000000
--- a/cronscripts/process-hwdb-submissions.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/python2 -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""
-Cron job that parses pending HWDB submissions.
-
-
-Options:
- -m, --max-submissions: (optional) The maximum number of submissions
- which will be processed.
-
-This script iterates over the HWDB submissions with the status
-SUBMITTED, beginning with the oldest submissions, populate the
-HWDB tables with the data from these submissions.
-
-Properly processed submissions are set to the status PROCESSED;
-submissions that cannot be processed are set to the status INVALID.
-"""
-
-import _pythonpath
-
-from lp.hardwaredb.scripts.hwdbsubmissions import process_pending_submissions
-from lp.services.scripts.base import LaunchpadCronScript
-
-
-class HWDBSubmissionProcessor(LaunchpadCronScript):
-
- def add_my_options(self):
- """See `LaunchpadScript`."""
- self.parser.add_option(
- '-m', '--max-submissions',
- help='Limit the number of submissions which will be processed.')
- self.parser.add_option(
- '-w', '--warnings', action="store_true", default=False,
- help='Include warnings.')
-
- def main(self):
- max_submissions = self.options.max_submissions
- if max_submissions is not None:
- try:
- max_submissions = int(self.options.max_submissions)
- except ValueError:
- self.logger.error(
- 'Invalid value for --max_submissions specified: %r.'
- % max_submissions)
- return
- if max_submissions <= 0:
- self.logger.error(
- '--max_submissions must be a positive integer.')
- return
-
- process_pending_submissions(
- self.txn, self.logger, max_submissions, self.options.warnings)
-
-if __name__ == '__main__':
- script = HWDBSubmissionProcessor(
- 'hwdbsubmissions', dbuser='hwdb-submission-processor')
- script.lock_and_run()
diff --git a/cronscripts/reprocess-hwdb-submissions.py b/cronscripts/reprocess-hwdb-submissions.py
deleted file mode 100755
index 55a3fd4..0000000
--- a/cronscripts/reprocess-hwdb-submissions.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/python2 -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""
-Cron job that parses pending HWDB submissions.
-
-
-Options:
- -m, --max-submissions: (optional) The maximum number of submissions
- which will be processed.
-
-This script iterates over the HWDB submissions with the status
-INVALID. It processes only submissions with an ID greater or equal
-than the number specified by the file given a option -s.
-
-When the script terminates, it writes the ID of the last processed
-submission into this file.
-
-Properly processed submissions are set to the status PROCESSED;
-submissions that cannot be processed retain the status INVALID.
-"""
-
-import _pythonpath
-
-from lp.hardwaredb.scripts.hwdbsubmissions import (
- reprocess_invalid_submissions,
- )
-from lp.services.scripts.base import LaunchpadCronScript
-
-
-class HWDBSubmissionProcessor(LaunchpadCronScript):
-
- def add_my_options(self):
- """See `LaunchpadScript`."""
- self.parser.add_option(
- '-m', '--max-submissions',
- help='Limit the number of submissions which will be processed.')
- self.parser.add_option(
- '-w', '--warnings', action="store_true", default=False,
- help='Include warnings.')
- self.parser.add_option(
- '-s', '--start-file', default=None,
- help=('The name of a file storing the smallest ID of a\n'
- 'hardware database submission that should be processed.\n'
- 'This script must have read and write access to the file.'))
-
- def main(self):
- max_submissions = self.options.max_submissions
- if max_submissions is not None:
- try:
- max_submissions = int(self.options.max_submissions)
- except ValueError:
- self.logger.error(
- 'Invalid value for --max_submissions specified: %r.'
- % max_submissions)
- return
- if max_submissions <= 0:
- self.logger.error(
- '--max_submissions must be a positive integer.')
- return
-
- if self.options.start_file is None:
- self.logger.error('Option --start-file not specified.')
- return
- try:
- start_file = open(self.options.start_file, 'r+')
- start_id = start_file.read().strip()
- except IOError as error:
- self.logger.error(
- 'Cannot access file %s: %s' % (
- self.options.start_file, error))
- return
- try:
- start_id = int(start_id)
- except ValueError:
- self.logger.error(
- '%s must contain only an integer' % self.options.start_file)
- return
- if start_id < 0:
- self.logger.error(
- '%s must contain a positive integer'
- % self.options.start_file)
- return
-
- next_start = reprocess_invalid_submissions(
- start_id, self.txn, self.logger,
- max_submissions, self.options.warnings)
-
- start_file.seek(0)
- start_file.write('%i' % next_start)
- start_file.close()
-
-if __name__ == '__main__':
- script = HWDBSubmissionProcessor(
- 'hwdbsubmissions', dbuser='hwdb-submission-processor')
- script.lock_and_run()
diff --git a/lib/lp/hardwaredb/doc/hwdb-submission.txt b/lib/lp/hardwaredb/doc/hwdb-submission.txt
index 229908f..13f3edc 100644
--- a/lib/lp/hardwaredb/doc/hwdb-submission.txt
+++ b/lib/lp/hardwaredb/doc/hwdb-submission.txt
@@ -213,211 +213,3 @@ Teams can be owners of submissions.
>>> submission = submission_set.getBySubmissionKey(u'unique-id-68')
>>> submission.owner.displayname
u'Ubuntu Team'
-
-
-Submission Processing
----------------------
-
-Submissions are processed by the cronscript process-hwdb-submissions.py.
-This script processes all submissions with the status SUBMITTED, checks
-the validity of each submission, populates the HWDB tables with data
-from a submission and sets the submission state to VALID. If any error
-occurs when a submission is processed, its status is set to INVALID and
-according error is logged. For details, see
-lp/hardwaredb/scripts/tests/test_hwdb_submission_processing.py.
-
-We have currently three unprocessed submissions in the database, one
-submission with the status PROCESSED and no submissions with the status
-INVALID.
-
- >>> from lp.hardwaredb.interfaces.hwdb import (
- ... HWSubmissionProcessingStatus)
- >>> new_submissions = submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.SUBMITTED)
- >>> for submission in new_submissions:
- ... print(submission.submission_key, submission.status.title)
- test_submission_id_1 Submitted
- unique-id-1 Submitted
- unique-id-68 Submitted
- >>> processed_submissions = submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.PROCESSED)
- >>> for submission in processed_submissions:
- ... print(submission.submission_key, submission.status.title)
- sample-submission Processed
- >>> invalid_submissions = submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.INVALID)
- >>> print(invalid_submissions.count())
- 0
-
-The script process-hwdb-submissions.py takes the optional parameter
--m or --max-submissions, so let's process just the first of these three
-submissions. We don't have a Librarian file for this submission, so
-let's add one. Let's add invalid data in order to see how invalid
-submissions are processed.
-
- >>> from lp.services.librarianserver.testing.server import fillLibrarianFile
- >>> submission = submission_set.getBySubmissionKey('test_submission_id_1')
- >>> fillLibrarianFile(
- ... submission.raw_submission.id, 'nonsense')
-
- # Commit the current transaction so that the script sees the
- # recently added data.
- >>> import transaction
- >>> transaction.commit()
-
- # Run the script.
- >>> from lp.testing.script import run_script
- >>> returnvalue, out, err = run_script(
- ... 'cronscripts/process-hwdb-submissions.py', ['-m1'])
- >>> returnvalue
- 0
- >>> print(err)
- INFO Creating lockfile: /var/lock/launchpad-hwdbsubmissions.lock
- ERROR Parsing submission test_submission_id_1: syntax error:
- line 1, column 0
- INFO OOPS-...
- INFO Processed 0 valid and 1 invalid HWDB submissions
- <BLANKLINE>
- >>> print(out)
- <BLANKLINE>
-
-Submission "test_submission_id_1" has now the state INVALID; the other
-submissions are unchanged.
-
- >>> new_submissions = submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.SUBMITTED)
- >>> for submission in new_submissions:
- ... print(submission.submission_key, submission.status.title)
- unique-id-1 Submitted
- unique-id-68 Submitted
- >>> processed_submissions = submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.PROCESSED)
- >>> for submission in processed_submissions:
- ... print(submission.submission_key, submission.status.title)
- sample-submission Processed
- >>> invalid_submissions = submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.INVALID)
- >>> for submission in invalid_submissions:
- ... print(submission.submission_key, submission.status.title)
- test_submission_id_1 Invalid submission
-
-From the remaining two not yet processed submissions, one has invalid
-data, the other submission is valid.
-
- >>> returnvalue, out, err = run_script(
- ... 'cronscripts/process-hwdb-submissions.py')
- >>> returnvalue
- 0
- >>> print(err)
- INFO Creating lockfile: /var/lock/launchpad-hwdbsubmissions.lock
- ERROR Parsing submission unique-id-1: syntax error: line 1, column 0
- INFO OOPS-...
- INFO Processed 1 valid and 1 invalid HWDB submissions
- <BLANKLINE>
- >>> print(out)
- <BLANKLINE>
-
-Now we have one valid, two invalid and no unprocessed submissions.
-
- >>> # We must start a new transaction in order to see the effects
- >>> # the script had on the database.
- >>> transaction.commit()
- >>> new_submissions = submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.SUBMITTED)
- >>> print(new_submissions.count())
- 0
- >>> processed_submissions = submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.PROCESSED)
- >>> for submission in processed_submissions:
- ... print(submission.submission_key, submission.status.title)
- sample-submission Processed
- unique-id-68 Processed
-
- >>> invalid_submissions = submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.INVALID)
- >>> for submission in invalid_submissions:
- ... print(submission.submission_key, submission.status.title)
- test_submission_id_1 Invalid submission
- unique-id-1 Invalid submission
-
-Larger numbers of submissions can be processed too. Add enough submissions
-that scripts.hwdbsubmissions.ProcessingLoop is called at least twice.
-
- >>> form['field.submission_data'] = valid_sample_data
- >>> for serial in range(80):
- ... form['field.submission_key'] = u'submission-%i' % serial
- ... submit_view = create_initialized_view(
- ... app, name='+submit', form=form)
-
-Now we have 80 new submissions and three submissions that were processed
-in previous tests.
-
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.SUBMITTED).count())
- 80
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.PROCESSED).count())
- 2
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.INVALID).count())
- 2
- >>> transaction.commit()
-
-Let's leave some submissions unprocessed in order to check if the
-processing loop terminates properly, when the parameter "-m" is given.
-
- >>> returnvalue, out, err = run_script(
- ... 'cronscripts/process-hwdb-submissions.py', ['-m60'])
- >>> returnvalue
- 0
- >>> print(err)
- INFO Creating lockfile: /var/lock/launchpad-hwdbsubmissions.lock
- INFO Processed 60 valid and 0 invalid HWDB submissions
- >>> print(out)
- <BLANKLINE>
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.SUBMITTED).count())
- 20
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.PROCESSED).count())
- 62
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.INVALID).count())
- 2
-
-Let's add more subscription so that we have more than max_chunk_size
-unprocessed submissions and process all of them.
-
- >>> for serial in range(80, 160):
- ... form['field.submission_key'] = u'submission-%i' % serial
- ... submit_view = create_initialized_view(
- ... app, name='+submit', form=form)
-
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.SUBMITTED).count())
- 100
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.PROCESSED).count())
- 62
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.INVALID).count())
- 2
- >>> transaction.commit()
- >>> returnvalue, out, err = run_script(
- ... 'cronscripts/process-hwdb-submissions.py')
- >>> returnvalue
- 0
- >>> print(err)
- INFO Creating lockfile: /var/lock/launchpad-hwdbsubmissions.lock
- INFO Processed 100 valid and 0 invalid HWDB submissions
- >>> print(out)
- <BLANKLINE>
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.SUBMITTED).count())
- 0
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.PROCESSED).count())
- 162
- >>> print(submission_set.getByStatus(
- ... HWSubmissionProcessingStatus.INVALID).count())
- 2
diff --git a/lib/lp/hardwaredb/scripts/hwdbsubmissions.py b/lib/lp/hardwaredb/scripts/hwdbsubmissions.py
index 78d6332..07ff095 100644
--- a/lib/lp/hardwaredb/scripts/hwdbsubmissions.py
+++ b/lib/lp/hardwaredb/scripts/hwdbsubmissions.py
@@ -10,9 +10,6 @@ data and for the community test submissions.
__metaclass__ = type
__all__ = [
'SubmissionParser',
- 'process_pending_submissions',
- 'ProcessingLoopForPendingSubmissions',
- 'ProcessingLoopForReprocessingBadSubmissions',
]
import bz2
@@ -24,39 +21,23 @@ import io
from logging import getLogger
import os
import re
-import sys
import defusedxml.cElementTree as etree
import pytz
from zope.component import getUtility
-from zope.interface import implementer
-from zope.security.proxy import removeSecurityProxy
-from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.hardwaredb.interfaces.hwdb import (
HWBus,
- HWSubmissionProcessingStatus,
IHWDeviceDriverLinkSet,
IHWDeviceSet,
IHWDriverSet,
IHWSubmissionDeviceSet,
- IHWSubmissionSet,
IHWVendorIDSet,
IHWVendorNameSet,
)
-from lp.hardwaredb.model.hwdb import HWSubmission
from lp.services.config import config
-from lp.services.librarian.interfaces.client import LibrarianServerError
-from lp.services.looptuner import (
- ITunableLoop,
- LoopTuner,
- )
from lp.services.propertycache import cachedproperty
from lp.services.scripts.base import disable_oops_handler
-from lp.services.webapp.errorlog import (
- ErrorReportingUtility,
- ScriptRequest,
- )
from lp.services.xml import RelaxNGValidator
@@ -2969,183 +2950,3 @@ class UdevDevice(BaseDevice):
@property
def id(self):
return self.udev['id']
-
-
-@implementer(ITunableLoop)
-class ProcessingLoopBase(object):
- """An `ITunableLoop` for processing HWDB submissions."""
-
- def __init__(self, transaction, logger, max_submissions, record_warnings):
- self.transaction = transaction
- self.logger = logger
- self.max_submissions = max_submissions
- self.valid_submissions = 0
- self.invalid_submissions = 0
- self.finished = False
- self.janitor = getUtility(ILaunchpadCelebrities).janitor
- self.record_warnings = record_warnings
-
- def _validateSubmission(self, submission):
- submission.status = HWSubmissionProcessingStatus.PROCESSED
- self.valid_submissions += 1
-
- def _invalidateSubmission(self, submission):
- submission.status = HWSubmissionProcessingStatus.INVALID
- self.invalid_submissions += 1
-
- def isDone(self):
- """See `ITunableLoop`."""
- return self.finished
-
- def reportOops(self, error_explanation):
- """Create an OOPS report and the OOPS ID."""
- info = sys.exc_info()
- properties = [('error-explanation', error_explanation)]
- request = ScriptRequest(properties)
- error_utility = ErrorReportingUtility()
- error_utility.raising(info, request)
- self.logger.error('%s (%s)' % (error_explanation, request.oopsid))
-
- def getUnprocessedSubmissions(self, chunk_size):
- raise NotImplementedError
-
- def __call__(self, chunk_size):
- """Process a batch of yet unprocessed HWDB submissions."""
- # chunk_size is a float; we compare it below with an int value,
- # which can lead to unexpected results. Since it is also used as
- # a limit for an SQL query, convert it into an integer.
- chunk_size = int(chunk_size)
- submissions = self.getUnprocessedSubmissions(chunk_size)
- # Listify the submissions, since we'll have to loop over each
- # one anyway. This saves a COUNT query for getting the number of
- # submissions
- submissions = list(submissions)
- if len(submissions) < chunk_size:
- self.finished = True
-
- # Note that we must either change the status of each submission
- # in the loop below or we must abort the submission processing
- # entirely: getUtility(IHWSubmissionSet).getByStatus() above
- # returns the oldest submissions first, so if one submission
- # would remain in the status SUBMITTED, it would be returned
- # in the next loop run again, leading to a potentially endless
- # loop.
- for submission in submissions:
- try:
- parser = SubmissionParser(self.logger, self.record_warnings)
- success = parser.processSubmission(submission)
- if success:
- self._validateSubmission(submission)
- else:
- self._invalidateSubmission(submission)
- except LibrarianServerError:
- # LibrarianServerError is raised when the server could
- # not be reaches for 30 minutes.
- #
- # In this case we can neither validate nor invalidate the
- # submission. Moreover, the attempt to process the next
- # submission will most likely also fail, so we should give
- # up for now.
- #
- # This exception is raised before any data for the current
- # submission is processed, hence we can commit submissions
- # processed in previous runs of this loop without causing
- # any inconsistencies.
- self.transaction.commit()
-
- self.reportOops(
- 'Could not reach the Librarian while processing HWDB '
- 'submission %s' % submission.submission_key)
- raise
- except Exception:
- self.transaction.abort()
- self.reportOops(
- 'Exception while processing HWDB submission %s'
- % submission.submission_key)
-
- self._invalidateSubmission(submission)
- # Ensure that this submission is marked as bad, even if
- # further submissions in this batch raise an exception.
- self.transaction.commit()
-
- self.start = submission.id + 1
- if self.max_submissions is not None:
- if self.max_submissions <= (
- self.valid_submissions + self.invalid_submissions):
- self.finished = True
- break
- self.transaction.commit()
-
-
-class ProcessingLoopForPendingSubmissions(ProcessingLoopBase):
-
- def getUnprocessedSubmissions(self, chunk_size):
- submissions = getUtility(IHWSubmissionSet).getByStatus(
- HWSubmissionProcessingStatus.SUBMITTED,
- user=self.janitor
- )[:chunk_size]
- submissions = list(submissions)
- return submissions
-
-
-class ProcessingLoopForReprocessingBadSubmissions(ProcessingLoopBase):
-
- def __init__(self, start, transaction, logger,
- max_submissions, record_warnings):
- super(ProcessingLoopForReprocessingBadSubmissions, self).__init__(
- transaction, logger, max_submissions, record_warnings)
- self.start = start
-
- def getUnprocessedSubmissions(self, chunk_size):
- submissions = getUtility(IHWSubmissionSet).getByStatus(
- HWSubmissionProcessingStatus.INVALID, user=self.janitor)
- submissions = removeSecurityProxy(submissions).find(
- HWSubmission.id >= self.start)
- submissions = list(submissions[:chunk_size])
- return submissions
-
-
-def process_pending_submissions(transaction, logger, max_submissions=None,
- record_warnings=True):
- """Process pending submissions.
-
- Parse pending submissions, store extracted data in HWDB tables and
- mark them as either PROCESSED or INVALID.
- """
- loop = ProcessingLoopForPendingSubmissions(
- transaction, logger, max_submissions, record_warnings)
- # It is hard to predict how long it will take to parse a submission.
- # we don't want to last a DB transaction too long but we also
- # don't want to commit more often than necessary. The LoopTuner
- # handles this for us. The loop's run time will be approximated to
- # 2 seconds, but will never handle more than 50 submissions.
- loop_tuner = LoopTuner(
- loop, 2, minimum_chunk_size=1, maximum_chunk_size=50)
- loop_tuner.run()
- logger.info(
- 'Processed %i valid and %i invalid HWDB submissions'
- % (loop.valid_submissions, loop.invalid_submissions))
-
-
-def reprocess_invalid_submissions(start, transaction, logger,
- max_submissions=None, record_warnings=True):
- """Reprocess invalid submissions.
-
- Parse submissions that have been marked as invalid. A newer
- variant of the parser might be able to process them.
- """
- loop = ProcessingLoopForReprocessingBadSubmissions(
- start, transaction, logger, max_submissions, record_warnings)
- # It is hard to predict how long it will take to parse a submission.
- # we don't want to last a DB transaction too long but we also
- # don't want to commit more often than necessary. The LoopTuner
- # handles this for us. The loop's run time will be approximated to
- # 2 seconds, but will never handle more than 50 submissions.
- loop_tuner = LoopTuner(
- loop, 2, minimum_chunk_size=1, maximum_chunk_size=50)
- loop_tuner.run()
- logger.info(
- 'Processed %i valid and %i invalid HWDB submissions'
- % (loop.valid_submissions, loop.invalid_submissions))
- logger.info('last processed: %i' % loop.start)
- return loop.start
diff --git a/lib/lp/hardwaredb/scripts/tests/test_hwdb_submission_processing.py b/lib/lp/hardwaredb/scripts/tests/test_hwdb_submission_processing.py
index 279a115..e0961d5 100644
--- a/lib/lp/hardwaredb/scripts/tests/test_hwdb_submission_processing.py
+++ b/lib/lp/hardwaredb/scripts/tests/test_hwdb_submission_processing.py
@@ -16,11 +16,9 @@ import pytz
from zope.component import getUtility
from zope.testing.loghandler import Handler
-from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.hardwaredb.interfaces.hwdb import (
HWBus,
HWSubmissionFormat,
- HWSubmissionProcessingStatus,
IHWDeviceDriverLinkSet,
IHWDeviceSet,
IHWDriverSet,
@@ -38,13 +36,10 @@ from lp.hardwaredb.scripts.hwdbsubmissions import (
PCI_SUBCLASS_BRIDGE_PCI,
PCI_SUBCLASS_SERIALBUS_USB,
PCI_SUBCLASS_STORAGE_SATA,
- process_pending_submissions,
SubmissionParser,
UdevDevice,
)
from lp.services.config import config
-from lp.services.librarian.interfaces.client import LibrarianServerError
-from lp.services.librarianserver.testing.server import fillLibrarianFile
from lp.testing import (
TestCase,
validate_mock_class,
@@ -5376,152 +5371,3 @@ class TestHWDBSubmissionTablePopulation(TestCaseHWDB):
'/devices/LNXSYSTM:00': 'A udev device',
}
self.assertEqual('A udev device', submission_parser.root_device)
-
- def testPendingSubmissionProcessing(self):
- """Test of process_pending_submissions().
-
- Run process_pending_submissions with three submissions; one
- of the submisisons contains invalid data.
- """
- # We have already one submisson with the status SUBMITTED in the
- # DB sample data; let's fill the associated Librarian file with
- # some test data.
- submission_set = getUtility(IHWSubmissionSet)
- submission = submission_set.getBySubmissionKey(
- 'test_submission_id_1')
- submission_data = self.getSampleData(
- 'simple_valid_hwdb_submission.xml')
- fillLibrarianFile(submission.raw_submission.id, submission_data)
-
- submission_data = self.getSampleData('real_hwdb_submission.xml.bz2')
- submission_key = 'submission-6'
- self.createSubmissionData(submission_data, False, submission_key)
-
- submission_key = 'private-submission'
- self.createSubmissionData(submission_data, False, submission_key,
- private=True)
-
- submission_key = 'submission-7'
- submission_data = """<?xml version="1.0" ?>
- <foo>
- This does not pass the RelaxNG validation.
- </foo>
- """
- self.createSubmissionData(submission_data, False, submission_key)
- process_pending_submissions(self.layer.txn, self.log)
-
- janitor = getUtility(ILaunchpadCelebrities).janitor
- valid_submissions = submission_set.getByStatus(
- HWSubmissionProcessingStatus.PROCESSED, user=janitor)
- valid_submission_keys = [
- submission.submission_key for submission in valid_submissions]
- self.assertEqual(
- valid_submission_keys,
- [u'test_submission_id_1', u'sample-submission', u'submission-6',
- u'private-submission'],
- 'Unexpected set of valid submissions: %r' % valid_submission_keys)
-
- invalid_submissions = submission_set.getByStatus(
- HWSubmissionProcessingStatus.INVALID, user=janitor)
- invalid_submission_keys = [
- submission.submission_key for submission in invalid_submissions]
- self.assertEqual(
- invalid_submission_keys, [u'submission-7'],
- 'Unexpected set of invalid submissions: %r'
- % invalid_submission_keys)
-
- new_submissions = submission_set.getByStatus(
- HWSubmissionProcessingStatus.SUBMITTED, user=janitor)
- new_submission_keys = [
- submission.submission_key for submission in new_submissions]
- self.assertEqual(
- new_submission_keys, [],
- 'Unexpected set of new submissions: %r' % new_submission_keys)
-
- messages = [record.getMessage() for record in self.handler.records]
- messages = '\n'.join(messages)
- self.assertEqual(
- messages,
- "Parsing submission submission-7: root node is not '<system>'\n"
- "Processed 3 valid and 1 invalid HWDB submissions",
- 'Unexpected log messages: %r' % messages)
-
- def testOopsLogging(self):
- """Test if OOPSes are properly logged."""
- def processSubmission(self, submission):
- x = 1
- x = x / 0
- process_submission_regular = SubmissionParser.processSubmission
- SubmissionParser.processSubmission = processSubmission
-
- process_pending_submissions(self.layer.txn, self.log)
-
- error_report = self.oopses[0]
- self.assertEqual('ZeroDivisionError', error_report['type'])
- self.assertStartsWith(
- error_report['req_vars']['error-explanation'],
- 'Exception while processing HWDB')
-
- messages = [record.getMessage() for record in self.handler.records]
- messages = '\n'.join(messages)
- expected_message = (
- 'Exception while processing HWDB submission '
- 'test_submission_id_1 (OOPS-')
- self.assertTrue(
- messages.startswith(expected_message),
- 'Unexpected log message: %r' % messages)
-
- SubmissionParser.processSubmission = process_submission_regular
-
- def testProcessingLoopExceptionHandling(self):
- """Test of the exception handling of ProcessingLoop.__call__()"""
- def processSubmission(self, submission):
- """Force failures during submission processing."""
- if submission.submission_key == 'submission-2':
- raise LibrarianServerError('Librarian does not respond')
- else:
- return True
-
- process_submission_regular = SubmissionParser.processSubmission
- SubmissionParser.processSubmission = processSubmission
-
- self.createSubmissionData(data='whatever', compress=False,
- submission_key='submission-1')
- self.createSubmissionData(data='whatever', compress=False,
- submission_key='submission-2')
-
- # When we call process_pending_submissions(), submission-2 will
- # cause an exception
- self.assertRaises(
- LibrarianServerError, process_pending_submissions,
- self.layer.txn, self.log)
- error_report = self.oopses[0]
- self.assertEqual('LibrarianServerError', error_report['type'])
- self.assertEqual('Librarian does not respond', error_report['value'])
-
- messages = [record.getMessage() for record in self.handler.records]
- messages = '\n'.join(messages)
- expected_message = (
- 'Could not reach the Librarian while processing HWDB '
- 'submission submission-2 (OOPS-')
- self.assertTrue(
- messages.startswith(expected_message),
- 'Unexpected log messages: %r' % messages)
-
- # Though processing the second submission caused an exception,
- # the first one has been corretly marked as being processed...
- self.layer.txn.begin()
- submission_set = getUtility(IHWSubmissionSet)
- submission_1 = submission_set.getBySubmissionKey('submission-1')
- self.assertEqual(
- submission_1.status, HWSubmissionProcessingStatus.PROCESSED,
- 'Unexpected status of submission 1: %s' % submission_1.status)
-
- # ... while the second submission has the status SUBMITTED.
- submission_set = getUtility(IHWSubmissionSet)
- submission_2 = submission_set.getBySubmissionKey('submission-2')
- self.assertEqual(
- submission_2.status, HWSubmissionProcessingStatus.SUBMITTED,
- 'Unexpected status of submission 1: %s' % submission_2.status)
-
- SubmissionParser.processSubmission = process_submission_regular
diff --git a/lib/lp/hardwaredb/scripts/tests/test_hwdbsubmissions.py b/lib/lp/hardwaredb/scripts/tests/test_hwdbsubmissions.py
deleted file mode 100644
index 4641dd7..0000000
--- a/lib/lp/hardwaredb/scripts/tests/test_hwdbsubmissions.py
+++ /dev/null
@@ -1,250 +0,0 @@
-# Copyright 2011 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tests for hwdbsubmissions script."""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-__metaclass__ = type
-
-from tempfile import mktemp
-
-from storm.store import Store
-import transaction
-
-from lp.hardwaredb.interfaces.hwdb import HWSubmissionProcessingStatus
-from lp.hardwaredb.scripts.hwdbsubmissions import (
- ProcessingLoopForPendingSubmissions,
- ProcessingLoopForReprocessingBadSubmissions,
- )
-from lp.testing import TestCaseWithFactory
-from lp.testing.layers import (
- DatabaseLayer,
- LaunchpadScriptLayer,
- )
-from lp.testing.matchers import Contains
-from lp.testing.script import run_script
-
-
-class TestProcessingLoops(TestCaseWithFactory):
- layer = LaunchpadScriptLayer
-
- def _makePendingSubmissionsLoop(self):
- """Parameters don't matter for these tests."""
- return ProcessingLoopForPendingSubmissions(None, None, 0, False)
-
- def test_PendingSubmissions_submitted_found(self):
- # The PendingSubmissions loop finds submitted entries.
- submission = self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.SUBMITTED)
- loop = self._makePendingSubmissionsLoop()
- # The sample data already contains one submission which we ignore.
- submissions = loop.getUnprocessedSubmissions(2)
- self.assertEqual([submission], submissions[1:])
-
- def test_PendingSubmissions_processed_not_found(self):
- # The PendingSubmissions loop ignores processed entries.
- submission = self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.PROCESSED)
- loop = self._makePendingSubmissionsLoop()
- # The sample data already contains one submission which we ignore.
- submissions = loop.getUnprocessedSubmissions(2)
- self.assertEqual([], submissions[1:])
- self.assertNotEqual([submission], submissions)
-
- def test_PendingSubmissions_invalid_not_found(self):
- # The PendingSubmissions loop ignores invalid entries.
- submission = self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.INVALID)
- loop = self._makePendingSubmissionsLoop()
- # The sample data already contains one submission which we ignore.
- submissions = loop.getUnprocessedSubmissions(2)
- self.assertEqual([], submissions[1:])
- self.assertNotEqual([submission], submissions)
-
- def test_PendingSubmissions_respects_chunk_size(self):
- # Only the requested number of entries are returned.
- self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.SUBMITTED)
- self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.SUBMITTED)
- loop = self._makePendingSubmissionsLoop()
- # The sample data already contains one submission.
- submissions = loop.getUnprocessedSubmissions(2)
- self.assertEqual(2, len(submissions))
-
- def _makeBadSubmissionsLoop(self, start=0):
- """Parameters don't matter for these tests."""
- return ProcessingLoopForReprocessingBadSubmissions(
- start, None, None, 0, False)
-
- def test_BadSubmissions_invalid_found(self):
- # The BadSubmissions loop finds invalid entries.
- submission = self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.INVALID)
- loop = self._makeBadSubmissionsLoop()
- submissions = loop.getUnprocessedSubmissions(2)
- self.assertEqual([submission], submissions)
-
- def test_BadSubmissions_processed_not_found(self):
- # The BadSubmissions loop ignores processed entries.
- self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.PROCESSED)
- loop = self._makeBadSubmissionsLoop()
- submissions = loop.getUnprocessedSubmissions(2)
- self.assertEqual([], submissions)
-
- def test_BadSubmissions_submitted_not_found(self):
- # The BadSubmissions loop ignores submitted entries.
- self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.SUBMITTED)
- loop = self._makeBadSubmissionsLoop()
- submissions = loop.getUnprocessedSubmissions(2)
- self.assertEqual([], submissions)
-
- def test_BadSubmissions_respects_chunk_size(self):
- # Only the requested number of entries are returned.
- self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.INVALID)
- self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.INVALID)
- loop = self._makeBadSubmissionsLoop()
- # The sample data already contains one submission.
- submissions = loop.getUnprocessedSubmissions(1)
- self.assertEqual(1, len(submissions))
-
- def test_BadSubmissions_respects_start(self):
- # It is possible to request a start id. Previous entries are ignored.
- submission1 = self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.INVALID)
- submission2 = self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.INVALID)
- self.assertTrue(submission1.id < submission2.id)
- loop = self._makeBadSubmissionsLoop(submission2.id)
- # The sample data already contains one submission.
- submissions = loop.getUnprocessedSubmissions(2)
- self.assertEqual([submission2], submissions)
- DatabaseLayer.force_dirty_database()
-
- def test_run_reprocessing_script_no_params(self):
- # cronscripts/reprocess-hwdb-submissions.py needs at least the
- # parameter --start-file
- retcode, stdout, stderr = run_script(
- 'cronscripts/reprocess-hwdb-submissions.py', [])
- self.assertThat(
- stderr, Contains('Option --start-file not specified.'))
- DatabaseLayer.force_dirty_database()
-
- def test_run_reprocessing_script_startfile_does_not_exist(self):
- # If the specified start file does not exist,
- # cronscripts/reprocess-hwdb-submissions.py reports an error.
- does_not_exist = mktemp()
- retcode, stdout, stderr = run_script(
- 'cronscripts/reprocess-hwdb-submissions.py',
- ['--start-file', does_not_exist])
- self.assertThat(
- stderr, Contains('Cannot access file %s' % does_not_exist))
- DatabaseLayer.force_dirty_database()
-
- def test_run_reprocessing_script_startfile_without_integer(self):
- # If the specified start file contains any non-integer string,
- # cronscripts/reprocess-hwdb-submissions.py reports an error.
- start_file_name = mktemp()
- start_file = open(start_file_name, 'w')
- start_file.write('nonsense')
- start_file.close()
- retcode, stdout, stderr = run_script(
- 'cronscripts/reprocess-hwdb-submissions.py',
- ['--start-file', start_file_name])
- self.assertThat(
- stderr,
- Contains('%s must contain only an integer' % start_file_name))
- DatabaseLayer.force_dirty_database()
-
- def test_run_reprocessing_script_startfile_with_negative_integer(self):
- # If the specified start file contains any non-integer string,
- # cronscripts/reprocess-hwdb-submissions.py reports an error.
- start_file_name = mktemp()
- start_file = open(start_file_name, 'w')
- start_file.write('-1')
- start_file.close()
- retcode, stdout, stderr = run_script(
- 'cronscripts/reprocess-hwdb-submissions.py',
- ['--start-file', start_file_name])
- self.assertThat(
- stderr,
- Contains('%s must contain a positive integer' % start_file_name))
- DatabaseLayer.force_dirty_database()
-
- def test_run_reprocessing_script_max_submission_not_integer(self):
- # If the parameter --max-submissions is not an integer,
- # cronscripts/reprocess-hwdb-submissions.py reports an error.
- retcode, stdout, stderr = run_script(
- 'cronscripts/reprocess-hwdb-submissions.py',
- ['--max-submissions', 'nonsense'])
- expected = "Invalid value for --max_submissions specified: 'nonsense'"
- self.assertThat(stderr, Contains(expected))
- DatabaseLayer.force_dirty_database()
-
- def test_run_reprocessing_script_two_batches(self):
- # cronscripts/reprocess-hwdb-submissions.py begings to process
- # submissions with IDs starting at the value stored in the
- # file given as the parameter --start-file. When is has
- # finished processing the number of submissions specified by
- # --max-submissions, it stores the ID of the last prcessed
- # submission in start-file.
- new_submissions = []
- for count in range(5):
- new_submissions.append(
- self.factory.makeHWSubmission(
- status=HWSubmissionProcessingStatus.INVALID))
-
- start_file_name = mktemp()
- start_file = open(start_file_name, 'w')
- start_file.write('%i' % new_submissions[1].id)
- start_file.close()
- transaction.commit()
- Store.of(new_submissions[0]).invalidate()
-
- retcode, stdout, stderr = run_script(
- 'cronscripts/reprocess-hwdb-submissions.py',
- ['--max-submissions', '2', '--start-file', start_file_name])
-
- # We started with the ID of the second submission created abvoe,
- # so the first submission still has the status INVALID.
- self.assertEqual(
- HWSubmissionProcessingStatus.INVALID,
- new_submissions[0].status)
- # We processed two submissions, they now have the status
- # PROCESSED.
- self.assertEqual(
- HWSubmissionProcessingStatus.PROCESSED,
- new_submissions[1].status)
- self.assertEqual(
- HWSubmissionProcessingStatus.PROCESSED,
- new_submissions[2].status)
- # The following submissions were not yet touched,
- self.assertEqual(
- HWSubmissionProcessingStatus.INVALID,
- new_submissions[3].status)
- self.assertEqual(
- HWSubmissionProcessingStatus.INVALID,
- new_submissions[4].status)
-
- # The start file now contains the ID of the 4th submission.
- new_start = int(open(start_file_name).read())
- self.assertEqual(new_submissions[3].id, new_start)
-
- # When we run the script again, for only one submission,
- # the 4th submission is processed.
- transaction.abort()
- Store.of(new_submissions[0]).invalidate()
- retcode, stdout, stderr = run_script(
- 'cronscripts/reprocess-hwdb-submissions.py',
- ['--max-submissions', '1', '--start-file', start_file_name])
- self.assertEqual(
- HWSubmissionProcessingStatus.PROCESSED,
- new_submissions[3].status)
- self.assertEqual(
- HWSubmissionProcessingStatus.INVALID,
- new_submissions[4].status)
