launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #06441
[Merge] lp:~linaro-infrastructure/launchpad/workitems-migration-script into lp:launchpad
Guilherme Salgado has proposed merging lp:~linaro-infrastructure/launchpad/workitems-migration-script into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~linaro-infrastructure/launchpad/workitems-migration-script/+merge/93883
This is the script to migrate work items from the whiteboard to the new table. We're not going to run it until we have the new UI to edit work items separately from the whiteboard, though.
--
https://code.launchpad.net/~linaro-infrastructure/launchpad/workitems-migration-script/+merge/93883
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~linaro-infrastructure/launchpad/workitems-migration-script into lp:launchpad.
=== added file 'lib/lp/blueprints/tests/test_workitem_migration.py'
--- lib/lp/blueprints/tests/test_workitem_migration.py 1970-01-01 00:00:00 +0000
+++ lib/lp/blueprints/tests/test_workitem_migration.py 2012-02-20 17:54:19 +0000
@@ -0,0 +1,299 @@
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+from textwrap import dedent
+
+from testtools.matchers import (
+ MatchesRegex,
+ MatchesStructure,
+ )
+
+import transaction
+from lp.testing import (
+ TestCase,
+ TestCaseWithFactory,
+ )
+from lp.testing.layers import DatabaseFunctionalLayer
+from lp.testing.script import run_script
+
+from lp.blueprints.enums import SpecificationWorkItemStatus
+from lp.blueprints.workitemmigration import (
+ extractWorkItemsFromWhiteboard,
+ WorkitemParser,
+ WorkItemParseError,
+ )
+
+
+class FakeSpecification(object):
+ assignee = None
+
+
+class TestWorkItemParser(TestCase):
+
+ def test_parse_line_basic(self):
+ parser = WorkitemParser(FakeSpecification())
+ assignee, description, status = parser.parse_blueprint_workitem(
+ "A single work item: TODO")
+ self.assertEqual(
+ [None, "A single work item", SpecificationWorkItemStatus.TODO],
+ [assignee, description, status])
+
+ def test_parse_line_with_assignee(self):
+ parser = WorkitemParser(FakeSpecification())
+ assignee, description, status = parser.parse_blueprint_workitem(
+ "[salgado] A single work item: TODO")
+ self.assertEqual(
+ ["salgado", "A single work item",
+ SpecificationWorkItemStatus.TODO],
+ [assignee, description, status])
+
+ def test_parse_line_with_missing_closing_bracket_for_assignee(self):
+ parser = WorkitemParser(FakeSpecification())
+ self.assertRaises(
+ WorkItemParseError, parser.parse_blueprint_workitem,
+ "[salgado A single work item: TODO")
+
+ def test_parse_line_without_status(self):
+ parser = WorkitemParser(FakeSpecification())
+ assignee, description, status = parser.parse_blueprint_workitem(
+ "A single work item")
+ self.assertEqual(
+ [None, "A single work item", SpecificationWorkItemStatus.TODO],
+ [assignee, description, status])
+
+ def test_parse_line_with_invalid_status(self):
+ parser = WorkitemParser(FakeSpecification())
+ self.assertRaises(
+ WorkItemParseError, parser.parse_blueprint_workitem,
+ "A single work item: FOO")
+
+ def test_parse_line_without_description(self):
+ parser = WorkitemParser(FakeSpecification())
+ self.assertRaises(
+ WorkItemParseError, parser.parse_blueprint_workitem,
+ " : TODO")
+
+ def test_parse_line_with_completed_status(self):
+ parser = WorkitemParser(FakeSpecification())
+ assignee, description, status = parser.parse_blueprint_workitem(
+ "A single work item: Completed")
+ self.assertEqual(
+ [None, "A single work item", SpecificationWorkItemStatus.DONE],
+ [assignee, description, status])
+
+ def test_parse_line_with_inprogress_status(self):
+ parser = WorkitemParser(FakeSpecification())
+ assignee, description, status = parser.parse_blueprint_workitem(
+ "A single work item: INPROGRESS")
+ self.assertEqual(
+ [None, "A single work item",
+ SpecificationWorkItemStatus.INPROGRESS],
+ [assignee, description, status])
+
+ def test_parse_line_with_postpone_status(self):
+ parser = WorkitemParser(FakeSpecification())
+ assignee, description, status = parser.parse_blueprint_workitem(
+ "A single work item: POSTPONE")
+ self.assertEqual(
+ [None, "A single work item",
+ SpecificationWorkItemStatus.POSTPONED],
+ [assignee, description, status])
+
+ def test_parse_line_with_drop_status(self):
+ parser = WorkitemParser(FakeSpecification())
+ assignee, description, status = parser.parse_blueprint_workitem(
+ "A single work item: DROP")
+ self.assertEqual(
+ [None, "A single work item",
+ SpecificationWorkItemStatus.POSTPONED],
+ [assignee, description, status])
+
+ def test_parse_line_with_dropped_status(self):
+ parser = WorkitemParser(FakeSpecification())
+ assignee, description, status = parser.parse_blueprint_workitem(
+ "A single work item: DROPPED")
+ self.assertEqual(
+ [None, "A single work item",
+ SpecificationWorkItemStatus.POSTPONED],
+ [assignee, description, status])
+
+ def test_parse_empty_line(self):
+ parser = WorkitemParser(FakeSpecification())
+ self.assertRaises(
+ AssertionError, parser.parse_blueprint_workitem, "")
+
+
+class TestSpecificationWorkItemExtractionFromWhiteboard(TestCaseWithFactory):
+ layer = DatabaseFunctionalLayer
+
+ def test_None_whiteboard(self):
+ spec = self.factory.makeSpecification(whiteboard=None)
+ work_items = extractWorkItemsFromWhiteboard(spec)
+ self.assertEqual([], work_items)
+
+ def test_empty_whiteboard(self):
+ spec = self.factory.makeSpecification(whiteboard='')
+ work_items = extractWorkItemsFromWhiteboard(spec)
+ self.assertEqual([], work_items)
+
+ def test_single_work_item(self):
+ whiteboard = dedent("""
+ Work items:
+ A single work item: TODO
+ """)
+ spec = self.factory.makeSpecification(whiteboard=whiteboard)
+ work_items = extractWorkItemsFromWhiteboard(spec)
+ self.assertEqual(1, len(work_items))
+ self.assertThat(work_items[0], MatchesStructure.byEquality(
+ assignee=None, title="A single work item",
+ status=SpecificationWorkItemStatus.TODO,
+ milestone=None,
+ specification=spec))
+
+ def test_multiple_work_items(self):
+ whiteboard = dedent("""
+ Work items:
+ A single work item: TODO
+ Another work item: DONE
+ """)
+ spec = self.factory.makeSpecification(whiteboard=whiteboard)
+ work_items = extractWorkItemsFromWhiteboard(spec)
+ self.assertEqual(2, len(work_items))
+ self.assertThat(work_items[0], MatchesStructure.byEquality(
+ assignee=None, title="A single work item",
+ status=SpecificationWorkItemStatus.TODO,
+ milestone=None,
+ specification=spec))
+ self.assertThat(work_items[1], MatchesStructure.byEquality(
+ assignee=None, title="Another work item",
+ status=SpecificationWorkItemStatus.DONE,
+ milestone=None,
+ specification=spec))
+
+ def test_work_item_with_assignee(self):
+ person = self.factory.makePerson()
+ whiteboard = dedent("""
+ Work items:
+ [%s] A single work item: TODO
+ """ % person.name)
+ spec = self.factory.makeSpecification(whiteboard=whiteboard)
+ work_items = extractWorkItemsFromWhiteboard(spec)
+ self.assertEqual(1, len(work_items))
+ self.assertThat(work_items[0], MatchesStructure.byEquality(
+ assignee=person, title="A single work item",
+ status=SpecificationWorkItemStatus.TODO,
+ milestone=None,
+ specification=spec))
+
+ def test_work_item_with_nonexistent_assignee(self):
+ whiteboard = dedent("""
+ Work items:
+ [nonono] A single work item: TODO
+ """)
+ spec = self.factory.makeSpecification(whiteboard=whiteboard)
+ self.assertRaises(ValueError, extractWorkItemsFromWhiteboard, spec)
+
+ def test_work_item_with_milestone(self):
+ milestone = self.factory.makeMilestone()
+ whiteboard = dedent("""
+ Work items for %s:
+ A single work item: TODO
+ """ % milestone.name)
+ spec = self.factory.makeSpecification(
+ whiteboard=whiteboard, product=milestone.product)
+ work_items = extractWorkItemsFromWhiteboard(spec)
+ self.assertEqual(1, len(work_items))
+ self.assertThat(work_items[0], MatchesStructure.byEquality(
+ assignee=None, title="A single work item",
+ status=SpecificationWorkItemStatus.TODO,
+ milestone=milestone,
+ specification=spec))
+
+ def test_work_item_with_unknown_milestone(self):
+ whiteboard = dedent("""
+ Work items for foo:
+ A single work item: TODO
+ """)
+ spec = self.factory.makeSpecification(whiteboard=whiteboard)
+ self.assertRaises(
+ WorkItemParseError, extractWorkItemsFromWhiteboard, spec)
+
+ def test_blank_line_signals_end_of_work_item_block(self):
+ whiteboard = dedent("""
+ Work items:
+ A single work item: TODO
+
+ Some random notes about this BP.
+ * This is what was discussed during UDS
+ * Oh, yeah, we need to do that too
+ """)
+ spec = self.factory.makeSpecification(whiteboard=whiteboard)
+ work_items = extractWorkItemsFromWhiteboard(spec)
+ self.assertEqual(1, len(work_items))
+ self.assertThat(work_items[0], MatchesStructure.byEquality(
+ assignee=None, title="A single work item",
+ status=SpecificationWorkItemStatus.TODO,
+ specification=spec))
+
+ def test_whiteboard_with_all_possible_sections(self):
+ whiteboard = dedent("""
+ Work items:
+ A single work item: TODO
+
+ Meta:
+ Headline: Foo bar
+ Acceptance: Baz foo
+
+ Complexity:
+ [user1] milestone1: 10
+ """)
+ spec = self.factory.makeSpecification(whiteboard=whiteboard)
+ work_items = extractWorkItemsFromWhiteboard(spec)
+ self.assertEqual(1, len(work_items))
+ self.assertThat(work_items[0], MatchesStructure.byEquality(
+ assignee=None, title="A single work item",
+ status=SpecificationWorkItemStatus.TODO,
+ milestone=None,
+ specification=spec))
+
+ # Now assert that the work items were removed from the whiteboard.
+ self.assertEqual(dedent("""
+ Meta:
+ Headline: Foo bar
+ Acceptance: Baz foo
+
+ Complexity:
+ [user1] milestone1: 10
+ """).strip(), spec.whiteboard.strip())
+
+
+class TestMigrationScript(TestCaseWithFactory):
+ layer = DatabaseFunctionalLayer
+
+ def test_script_run_as_subprocess(self):
+ whiteboard = dedent("""
+ Work items:
+ A single work item: TODO
+ Another work item: DONE
+ """)
+ spec = self.factory.makeSpecification(whiteboard=whiteboard)
+
+ # Make all this visible to the script we're about to run.
+ transaction.commit()
+
+ return_code, stdout, stderr = run_script(
+ 'scripts/migrate-workitems-from-whiteboard.py')
+ self.assertEqual(
+ 0, return_code,
+ "Script run failed; retval=%s, stdout=%s, stderr=%s " % (
+ return_code, stdout, stderr))
+ self.assertEqual('', stdout)
+ self.assertThat(stderr, MatchesRegex(
+ "INFO Creating lockfile:"
+ " /var/lock/launchpad-workitem-migration-script.lock\n"
+ "INFO Migrating work items from the whiteboard of 1 specs\n"
+ "INFO Migrated 2 work items from the whiteboard of"
+ " <Specification %d u'%s' for u'%s'>\n"
+ "INFO Done.\n" % (spec.id, spec.name, spec.product.name)))
=== added file 'lib/lp/blueprints/workitemmigration.py'
--- lib/lp/blueprints/workitemmigration.py 1970-01-01 00:00:00 +0000
+++ lib/lp/blueprints/workitemmigration.py 2012-02-20 17:54:19 +0000
@@ -0,0 +1,253 @@
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Helper functions for the migration of work items from whiteboards to the
+SpecificationWorkItem table.
+
+This will be removed once the migration is done.
+"""
+
+__metaclass__ = type
+__all__ = [
+ 'extractWorkItemsFromWhiteboard',
+ 'SpecificationWorkitemMigratorProcess',
+ ]
+
+import re
+
+from zope.component import getUtility
+from zope.interface import implements
+from zope.security.proxy import removeSecurityProxy
+
+from lp.services.database.lpstorm import IStore
+from lp.services.database.sqlbase import quote_like
+from lp.services.looptuner import DBLoopTuner, ITunableLoop
+
+from lp.blueprints.enums import SpecificationWorkItemStatus
+from lp.blueprints.model.specification import Specification
+
+from lp.registry.interfaces.person import IPersonSet
+
+
+class WorkItemParseError(Exception):
+ """An error when parsing a work item line from a blueprint's whiteboard."""
+
+
+class WorkitemParser(object):
+ """A parser to extract work items from Blueprint whiteboards."""
+
+ def __init__(self, blueprint):
+ self.blueprint = blueprint
+
+ def _normalize_status(self, status, desc):
+ status = status.strip().lower()
+ if not status:
+ status = SpecificationWorkItemStatus.TODO
+ elif status == u'completed':
+ status = SpecificationWorkItemStatus.DONE
+ elif status in (u'postpone', u'dropped', u'drop'):
+ status = SpecificationWorkItemStatus.POSTPONED
+ else:
+ valid_statuses = SpecificationWorkItemStatus.items
+ if status not in [item.name.lower() for item in valid_statuses]:
+ raise WorkItemParseError('Unknown status: %s' % status)
+ return valid_statuses[status.upper()]
+ return status
+
+ def _parse_line(self, line):
+ try:
+ desc, status = line.rsplit(':', 1)
+ except ValueError:
+ desc = line
+ status = ""
+ assignee_name = None
+ if desc.startswith('['):
+ if ']' in desc:
+ off = desc.index(']')
+ assignee_name = desc[1:off]
+ desc = desc[off + 1:].strip()
+ else:
+ raise WorkItemParseError('Missing closing "]" for assignee')
+ return assignee_name, desc, status
+
+ def parse_blueprint_workitem(self, line):
+ line = line.strip()
+ assert line, "Please don't give us an empty line"
+ assignee_name, desc, status = self._parse_line(line)
+ if not desc:
+ raise WorkItemParseError(
+ 'No work item description found on "%s"' % line)
+ status = self._normalize_status(status, desc)
+ return assignee_name, desc, status
+
+
+def milestone_extract(text, valid_milestones):
+ words = text.replace('(', ' ').replace(')', ' ').replace(
+ '[', ' ').replace(']', ' ').replace('<wbr></wbr>', '').split()
+ for milestone in valid_milestones:
+ for word in words:
+ if word == milestone.name:
+ return milestone
+ raise WorkItemParseError("No valid milestones found in %s" % words)
+
+
+def extractWorkItemsFromWhiteboard(spec):
+ work_items = []
+ if not spec.whiteboard:
+ return work_items
+ work_items_re = re.compile('^work items(.*)\s*:\s*$', re.I)
+ meta_re = re.compile('^Meta.*?:$', re.I)
+ complexity_re = re.compile('^Complexity.*?:$', re.I)
+ in_wi_block = False
+ new_whiteboard = []
+
+ target_milestones = list(spec.target.milestones)
+ wi_lines = []
+ # Iterate over all lines in the whiteboard and whenever we find a line
+ # matching work_items_re we 'continue' and store the following lines
+ # until we reach the end of the whiteboard or a line matching meta_re or
+ # complexity_re.
+ for line in spec.whiteboard.splitlines():
+ new_whiteboard.append(line)
+ wi_match = work_items_re.search(line)
+ if wi_match:
+ in_wi_block = True
+ milestone = None
+ milestone_part = wi_match.group(1).strip()
+ if milestone_part:
+ milestone = milestone_extract(
+ milestone_part, target_milestones)
+ new_whiteboard.pop()
+ continue
+ if meta_re.search(line):
+ milestone = None
+ in_wi_block = False
+ continue
+ if complexity_re.search(line):
+ milestone = None
+ in_wi_block = False
+ continue
+
+ if not in_wi_block:
+ # We only care about work-item lines.
+ continue
+
+ if line.strip() == '':
+ # An empty line signals the end of the work-item block:
+ # https://wiki.ubuntu.com/WorkItemsHowto.
+ in_wi_block = False
+ milestone = None
+ continue
+
+ # This is a work-item line, which we don't want in the new
+ # whiteboard because we're migrating them into the
+ # SpecificationWorkItem table.
+ new_whiteboard.pop()
+
+ wi_lines.append((line, milestone))
+
+ # Now parse the work item lines and store them in SpecificationWorkItem.
+ parser = WorkitemParser(spec)
+ sequence = 0
+ for line, milestone in wi_lines:
+ assignee_name, title, status = parser.parse_blueprint_workitem(line)
+ if assignee_name is not None:
+ assignee_name = assignee_name.strip()
+ assignee = getUtility(IPersonSet).getByName(assignee_name)
+ if assignee is None:
+ raise ValueError("Unknown person name: %s" % assignee_name)
+ else:
+ assignee = None
+ workitem = removeSecurityProxy(spec).newWorkItem(
+ status=status, title=title, assignee=assignee,
+ milestone=milestone, sequence=sequence)
+ work_items.append(workitem)
+ sequence += 1
+
+ removeSecurityProxy(spec).whiteboard = "\n".join(new_whiteboard)
+ return work_items
+
+
+class SpecificationWorkitemMigrator:
+ """Migrate work-items from Specification.whiteboard to
+ SpecificationWorkItem.
+
+ Migrating work items from the whiteboard is an all-or-nothing thing; if we
+ encounter any errors when parsing the whiteboard of a spec, we abort the
+ transaction and leave its whiteboard unchanged.
+
+ On a test with production data, only 100 whiteboards (out of almost 2500)
+ could not be migrated. On 24 of those the assignee in at least one work
+ item is not valid, on 33 the status of a work item is not valid and on 42
+ one or more milestones are not valid.
+ """
+ implements(ITunableLoop)
+
+ def __init__(self, transaction, logger, start_at=0):
+ self.transaction = transaction
+ self.logger = logger
+ self.start_at = start_at
+ # Get only the specs which contain "work items" in their whiteboard
+ # and which don't have any SpecificationWorkItems.
+ query = "whiteboard ilike '%%' || %s || '%%'" % quote_like(
+ 'work items')
+ query += (" and id not in (select distinct specification from "
+ "SpecificationWorkItem)")
+ self.specs = IStore(Specification).find(Specification, query)
+ self.total = self.specs.count()
+ self.logger.info(
+ "Migrating work items from the whiteboard of %d specs"
+ % self.total)
+
+ def getNextBatch(self, chunk_size):
+ end_at = self.start_at + int(chunk_size)
+ return self.specs[self.start_at:end_at]
+
+ def isDone(self):
+ # When the main loop hits the end of the Specifications with work
+ # items to migrate it sets start_at to None. Until we know we hit the
+ # end, it always has a numerical value.
+ return self.start_at is None
+
+ def __call__(self, chunk_size):
+ specs = self.getNextBatch(chunk_size)
+ specs_count = specs.count()
+ if specs_count == 0:
+ self.start_at = None
+ return
+
+ for spec in specs:
+ try:
+ work_items = extractWorkItemsFromWhiteboard(spec)
+ except Exception, e:
+ self.logger.info(
+ "Failed to parse whiteboard of %s: %s" % (
+ spec, unicode(e)))
+ self.transaction.abort()
+ self.transaction.begin()
+ continue
+
+ if len(work_items) > 0:
+ self.logger.info(
+ "Migrated %d work items from the whiteboard of %s" % (
+ len(work_items), spec))
+ self.transaction.commit()
+ self.transaction.begin()
+ else:
+ self.logger.info(
+ "No work items found on the whiteboard of %s" %
+ spec)
+
+ self.start_at += specs_count
+
+
+class SpecificationWorkitemMigratorProcess:
+
+ def __init__(self, transaction, logger):
+ self.transaction = transaction
+ self.logger = logger
+
+ def run(self):
+ loop = SpecificationWorkitemMigrator(self.transaction, self.logger)
+ DBLoopTuner(loop, 3, log=self.logger).run()
+ self.logger.info("Done.")
=== modified file 'lib/lp/testing/layers.py'
--- lib/lp/testing/layers.py 2012-02-03 06:54:05 +0000
+++ lib/lp/testing/layers.py 2012-02-20 17:54:19 +0000
@@ -269,7 +269,7 @@
class BaseLayer:
"""Base layer.
- All out layers should subclass Base, as this is where we will put
+ All our layers should subclass Base, as this is where we will put
test isolation checks to ensure that tests to not leave global
resources in a mess.
=== added file 'scripts/migrate-workitems-from-whiteboard.py'
--- scripts/migrate-workitems-from-whiteboard.py 1970-01-01 00:00:00 +0000
+++ scripts/migrate-workitems-from-whiteboard.py 2012-02-20 17:54:19 +0000
@@ -0,0 +1,26 @@
+#!/usr/bin/python -uS
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import _pythonpath
+
+from lp.services.scripts.base import LaunchpadScript
+
+from lp.blueprints.workitemmigration import (
+ SpecificationWorkitemMigratorProcess)
+
+
+class WorkitemMigrator(LaunchpadScript):
+
+ def main(self):
+ proc = SpecificationWorkitemMigratorProcess(self.txn, self.logger)
+ proc.run()
+
+
+if __name__ == '__main__':
+ # XXX: This is a throw-away script which we'll delete once the migration
+ # is complete, so I'd rather avoid setting up an extra DB user just for
+ # it, hence me using 'launchpad_main'.
+ script = WorkitemMigrator(
+ 'workitem-migration-script', dbuser='launchpad_main')
+ script.lock_and_run()
