launchpad-reviewers team mailing list archive

[William Grant <me@xxxxxxxxxxxxxxxxxx>]

William Grant has proposed merging lp:~wgrant/launchpad/unscan-branch into lp:launchpad.

Commit message:
Introduce unscan-branch.py, a script to erase a branch's scan data in the DB. This should work around the scanner slowness bugs for now.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~wgrant/launchpad/unscan-branch/+merge/128859

Introduce unscan-branch.py, a script to erase a branch's scan data in the DB. This should work around the scanner slowness bugs for now by letting us reset cursed branches. I just erase the BranchRevisions, the last_scanned_id, and the revision_account; leaving linked bugs, RevisionCaches, and other potential scanning reside alone. They're already left around when revisions are removed normally, so this is no significant issue. The --rescan option also triggers a new branch scan job.
-- 
https://code.launchpad.net/~wgrant/launchpad/unscan-branch/+merge/128859
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/unscan-branch into lp:launchpad.

=== added file 'lib/lp/code/scripts/tests/test_unscanbranch.py'
--- lib/lp/code/scripts/tests/test_unscanbranch.py	1970-01-01 00:00:00 +0000
+++ lib/lp/code/scripts/tests/test_unscanbranch.py	2012-10-10 05:58:18 +0000
@@ -0,0 +1,34 @@
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from storm.exceptions import LostObjectError
+from storm.store import Store
+
+from lp.code.model.branchjob import BranchJob
+from lp.code.scripts.unscanbranch import UnscanBranchScript
+from lp.services.log.logger import DevNullLogger
+from lp.testing import TestCaseWithFactory
+from lp.testing.layers import ZopelessDatabaseLayer
+
+
+class TestUnscanBranchScript(TestCaseWithFactory):
+
+    layer = ZopelessDatabaseLayer
+
+    def test_unscanbranch_script(self):
+        branch = self.factory.makeAnyBranch()
+        self.factory.makeRevisionsForBranch(branch=branch)
+        head = branch.getBranchRevision(revision_id=branch.last_scanned_id)
+        self.assertEqual(5, head.sequence)
+        self.assertEqual(5, branch.revision_count)
+        self.assertEqual(
+            1, Store.of(branch).find(BranchJob, branch=branch).count())
+
+        UnscanBranchScript(
+            "unscan-branch", test_args=['--rescan', branch.displayname],
+            logger=DevNullLogger()).main()
+
+        self.assertIs(None, branch.last_scanned_id)
+        self.assertRaises(LostObjectError, getattr, head, 'sequence')
+        self.assertEqual(
+            2, Store.of(branch).find(BranchJob, branch=branch).count())

=== added file 'lib/lp/code/scripts/unscanbranch.py'
--- lib/lp/code/scripts/unscanbranch.py	1970-01-01 00:00:00 +0000
+++ lib/lp/code/scripts/unscanbranch.py	2012-10-10 05:58:18 +0000
@@ -0,0 +1,64 @@
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+__all__ = [
+    'UnscanBranchScript',
+    ]
+
+
+import transaction
+from zope.component import getUtility
+
+from lp.code.interfaces.branchlookup import IBranchLookup
+from lp.code.model.branchjob import BranchScanJob
+from lp.code.model.branchrevision import BranchRevision
+from lp.services.database.lpstorm import IStore
+from lp.services.scripts.base import (
+    LaunchpadScript,
+    LaunchpadScriptFailure,
+    )
+
+
+class UnscanBranchScript(LaunchpadScript):
+    """Unscan a branch.
+
+    Resets the database scan data (eg. BranchRevision records and
+    last_scanned_id) for a branch, and optionally requests a rescan.
+
+    Mostly useful for working around performance bugs in the branch scanner
+    that don't affect fresh branches.
+    """
+
+    description = __doc__
+    usage = "%prog <branch URL>"
+
+    def add_my_options(self):
+        self.parser.add_option(
+            "--rescan", dest="rescan", action="store_true", default=False,
+            help="Request a rescan of the branch after unscanning it.")
+
+    def main(self):
+        if len(self.args) != 1:
+            self.parser.error("Wrong number of arguments.")
+        branch = getUtility(IBranchLookup).getByUrl(self.args[0])
+        if branch is None:
+            raise LaunchpadScriptFailure(
+                "Branch does not exist: %s" % self.args[0])
+
+        self.logger.info(
+            "Unscanning %s (last scanned id: %s)", branch.displayname,
+            branch.last_scanned_id)
+        self.logger.info("Purging BranchRevisions.")
+        IStore(BranchRevision).find(BranchRevision, branch=branch).remove()
+
+        self.logger.info("Resetting scan data.")
+        branch.last_scanned = branch.last_scanned_id = None
+        branch.revision_count = 0
+
+        if self.options.rescan:
+            self.logger.info("Requesting rescan.")
+            job = BranchScanJob.create(branch)
+            job.celeryRunOnCommit()
+
+        transaction.commit()

=== added file 'scripts/unscan-branch.py'
--- scripts/unscan-branch.py	1970-01-01 00:00:00 +0000
+++ scripts/unscan-branch.py	2012-10-10 05:58:18 +0000
@@ -0,0 +1,12 @@
+#!/usr/bin/python -S
+#
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import _pythonpath
+
+from lp.code.scripts.unscanbranch import UnscanBranchScript
+
+
+if __name__ == '__main__':
+    UnscanBranchScript("unscan-branch", dbuser='branchscanner').run()