launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #06273
[Merge] lp:~abentley/launchpad/upgrade-all into lp:launchpad
Aaron Bentley has proposed merging lp:~abentley/launchpad/upgrade-all into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~abentley/launchpad/upgrade-all/+merge/91894
= Summary =
Provide a script to upgrade all branches to 2a format, per RT 47986
== Proposed fix ==
This branch adds a script that can be used to upgrade all branches that aren't in 2a formats to 2a format. This will resolve bug 828409.
== Pre-implementation notes ==
Discussed pre-Epic, so can't really remember
== Implementation details ==
This branch changes the way Launchpad is started up, so that Bazaar plugins are always loaded. Not loading them sometimes was a mess. This caused config to be loaded more often, which meant that sys.argv wasn't always present when it was loaded, so sys.argv is now optional
== Tests ==
bin/test -t test_get_real_branch_path -t test_upgrade
== Demo and Q/A ==
None
= Launchpad lint =
Checking for conflicts and issues in changed files.
Linting changed files:
lib/lp/code/bzr.py
lib/lp/codehosting/tests/test_upgrade.py
lib/lp/codehosting/vfs/branchfs.py
scripts/upgrade_all_branches.py
lib/lp/codehosting/bzrutils.py
lib/lp/codehosting/scripts/tests/test_upgrade_all_branches.py
lib/lp/codehosting/upgrade.py
lib/lp/testing/__init__.py
lib/lp/codehosting/vfs/tests/test_branchfs.py
lib/lp_sitecustomize.py
lib/lp/services/config/__init__.py
./scripts/upgrade_all_branches.py
5: '_pythonpath' imported but unused
--
https://code.launchpad.net/~abentley/launchpad/upgrade-all/+merge/91894
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~abentley/launchpad/upgrade-all into lp:launchpad.
=== modified file 'lib/lp/code/bzr.py'
--- lib/lp/code/bzr.py 2011-07-22 12:35:31 +0000
+++ lib/lp/code/bzr.py 2012-02-07 18:29:31 +0000
@@ -5,6 +5,7 @@
__metaclass__ = type
__all__ = [
+ 'branch_changed',
'BranchFormat',
'ControlFormat',
'CURRENT_BRANCH_FORMATS',
@@ -17,6 +18,8 @@
# FIRST Ensure correct plugins are loaded. Do not delete this comment or the
# line below this comment.
import lp.codehosting
+# Silence lint warning.
+lp.codehosting
from bzrlib.branch import (
BranchReferenceFormat,
@@ -25,6 +28,10 @@
BzrBranchFormat7,
)
from bzrlib.bzrdir import BzrDirMetaFormat1
+from bzrlib.errors import (
+ NotStacked,
+ UnstackableBranchFormat,
+ )
from bzrlib.plugins.loom.branch import (
BzrBranchLoomFormat1,
BzrBranchLoomFormat6,
@@ -291,3 +298,21 @@
return (ControlFormat.get_enum(control_string),
BranchFormat.get_enum(branch_string),
RepositoryFormat.get_enum(repository_string))
+
+
+def branch_changed(db_branch, bzr_branch=None):
+ """Mark a database branch as changed.
+
+ :param db_branch: The branch to mark changed.
+ :param bzr_branch: (optional) The bzr branch to use to mark the branch
+ changed.
+ """
+ if bzr_branch is None:
+ bzr_branch = db_branch.getBzrBranch()
+ try:
+ stacked_on = bzr_branch.get_stacked_on_url()
+ except (NotStacked, UnstackableBranchFormat):
+ stacked_on = None
+ last_revision = bzr_branch.last_revision()
+ formats = get_branch_formats(bzr_branch)
+ db_branch.branchChanged(stacked_on, last_revision, *formats)
=== modified file 'lib/lp/codehosting/bzrutils.py'
--- lib/lp/codehosting/bzrutils.py 2011-12-24 17:49:30 +0000
+++ lib/lp/codehosting/bzrutils.py 2012-02-07 18:29:31 +0000
@@ -329,3 +329,12 @@
yield
finally:
branch.unlock()
+
+
+@contextmanager
+def server(server):
+ server.start_server()
+ try:
+ yield server
+ finally:
+ server.stop_server()
=== added file 'lib/lp/codehosting/scripts/tests/test_upgrade_all_branches.py'
--- lib/lp/codehosting/scripts/tests/test_upgrade_all_branches.py 1970-01-01 00:00:00 +0000
+++ lib/lp/codehosting/scripts/tests/test_upgrade_all_branches.py 2012-02-07 18:29:31 +0000
@@ -0,0 +1,81 @@
+# Copyright 2011-2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+
+import os.path
+import logging
+
+from bzrlib.repofmt.groupcompress_repo import RepositoryFormat2a
+import transaction
+
+from lp.code.bzr import branch_changed
+from lp.codehosting.upgrade import Upgrader
+from lp.testing import (
+ person_logged_in,
+ run_script,
+ TestCaseWithFactory,
+ )
+from lp.testing.layers import AppServerLayer
+
+
+class TestUpgradeAllBranchesScript(TestCaseWithFactory):
+
+ layer = AppServerLayer
+
+ def setUp(self):
+ super(TestUpgradeAllBranchesScript, self).setUp()
+ # useBzrBranches changes cwd
+ self.cwd = os.getcwd()
+
+ def upgrade_all_branches(self, target, finish=False):
+ """Run the script to upgrade all branches."""
+ transaction.commit()
+ if finish:
+ flags = ' --finish '
+ else:
+ flags = ' '
+ return run_script(
+ 'scripts/upgrade_all_branches.py' + flags + target, cwd=self.cwd)
+
+ def prepare(self):
+ """Prepare to run the script."""
+ self.useBzrBranches(direct_database=True)
+ branch, tree = self.create_branch_and_tree(format='pack-0.92')
+ tree.commit('foo')
+ with person_logged_in(branch.owner):
+ branch_changed(branch, tree.branch)
+ target = self.makeTemporaryDirectory()
+ upgrader = Upgrader(branch, target, logging.getLogger(), tree.branch)
+ return upgrader
+
+ def test_start_upgrade(self):
+ """Test that starting the upgrade behaves as expected."""
+ upgrader = self.prepare()
+ stdout, stderr, retcode = self.upgrade_all_branches(
+ upgrader.target_dir)
+ self.assertIn(
+ 'INFO Upgrading branch %s' % upgrader.branch.unique_name,
+ stderr)
+ self.assertIn(
+ 'INFO Converting repository with fetch.', stderr)
+ self.assertIn(
+ 'INFO Skipped 0 already-upgraded branches.', stderr)
+ self.assertEqual(0, retcode)
+ upgraded = upgrader.get_bzrdir().open_repository()
+ self.assertIs(RepositoryFormat2a, upgraded._format.__class__)
+
+ def test_finish_upgrade(self):
+ """Test that finishing the upgrade behaves as expected."""
+ upgrader = self.prepare()
+ upgrader.start_upgrade()
+ stdout, stderr, retcode = self.upgrade_all_branches(
+ upgrader.target_dir, finish=True)
+ self.assertIn(
+ 'INFO Upgrading branch %s' % upgrader.branch.unique_name,
+ stderr)
+ self.assertEqual(0, retcode)
+ upgraded = upgrader.branch.getBzrBranch()
+ self.assertIs(
+ RepositoryFormat2a, upgraded.repository._format.__class__)
=== added file 'lib/lp/codehosting/tests/test_upgrade.py'
--- lib/lp/codehosting/tests/test_upgrade.py 1970-01-01 00:00:00 +0000
+++ lib/lp/codehosting/tests/test_upgrade.py 2012-02-07 18:29:31 +0000
@@ -0,0 +1,264 @@
+__metaclass__ = type
+
+import logging
+
+from bzrlib.branch import Branch
+from bzrlib.bzrdir import BzrDir, format_registry
+from bzrlib.plugins.loom.branch import loomify
+from bzrlib.repofmt.groupcompress_repo import (
+ RepositoryFormat2a, RepositoryFormat2aSubtree)
+from bzrlib.revision import NULL_REVISION
+from bzrlib.transport import get_transport
+
+from lp.code.bzr import (
+ BranchFormat,
+ branch_changed,
+ get_branch_formats,
+ RepositoryFormat,
+ )
+from lp.codehosting.bzrutils import read_locked
+from lp.codehosting.upgrade import (
+ Upgrader,
+ )
+from lp.testing import (
+ temp_dir,
+ TestCaseWithFactory,
+ )
+from lp.testing.layers import ZopelessDatabaseLayer
+
+
+class TestUpgrader(TestCaseWithFactory):
+
+ layer = ZopelessDatabaseLayer
+
+ def prepare(self, format='pack-0.92', loomify_branch=False):
+ """Prepare an upgrade test.
+
+ :param format: The branch format to use, as a string.
+ :param loomify_branch: If true, convert the branch to a loom.
+ """
+ self.useBzrBranches(direct_database=True)
+ branch, tree = self.create_branch_and_tree(format=format)
+ tree.commit(
+ 'foo', rev_id='prepare-commit', committer='jrandom@xxxxxxxxxxx')
+ if loomify_branch:
+ loomify(tree.branch)
+ bzr_branch = tree.bzrdir.open_branch()
+ else:
+ bzr_branch = tree.branch
+ return self.getUpgrader(bzr_branch, branch)
+
+ def getUpgrader(self, bzr_branch, branch):
+ """Return an upgrader for the specified branches.
+
+ :param bzr_branch: the bzr branch to use.
+ :param branch: The DB branch to use.
+ """
+ target_dir = self.useContext(temp_dir())
+ return Upgrader(
+ branch, target_dir, logging.getLogger(), bzr_branch)
+
+ def addTreeReference(self, tree):
+ """Add a tree reference to a tree and commit.
+
+ :param tree: A Bazaar WorkingTree to add a tree to.
+ """
+ sub_branch = BzrDir.create_branch_convenience(
+ tree.bzrdir.root_transport.clone('sub').base)
+ tree.add_reference(sub_branch.bzrdir.open_workingtree())
+ tree.commit('added tree reference', committer='jrandom@xxxxxxxxxxx')
+
+ def check_branch(self, upgraded, branch_format=BranchFormat.BZR_BRANCH_7,
+ repository_format=RepositoryFormat.BZR_CHK_2A):
+ """Check that a branch matches expected post-upgrade formats."""
+ control, branch, repository = get_branch_formats(upgraded)
+ self.assertEqual(repository, repository_format)
+ self.assertEqual(branch, branch_format)
+
+ def test_simple_upgrade(self):
+ """Upgrade a pack-0.92 branch."""
+ upgrader = self.prepare()
+ upgrader.start_upgrade()
+ upgrader.finish_upgrade()
+ self.check_branch(
+ upgrader.branch.getBzrBranch())
+
+ def test_subtree_upgrade(self):
+ """Upgrade a pack-0.92-subtree branch."""
+ upgrader = self.prepare('pack-0.92-subtree')
+ upgrader.start_upgrade()
+ upgrader.finish_upgrade()
+ self.check_branch(upgrader.branch.getBzrBranch())
+
+ def test_upgrade_loom(self):
+ """Upgrade a loomified pack-0.92 branch."""
+ upgrader = self.prepare(loomify_branch=True)
+ upgrader.start_upgrade()
+ upgrader.finish_upgrade()
+ upgraded = upgrader.branch.getBzrBranch()
+ self.check_branch(upgraded, BranchFormat.BZR_LOOM_2)
+
+ def test_upgrade_subtree_loom(self):
+ """Upgrade a loomified pack-0.92-subtree branch."""
+ upgrader = self.prepare('pack-0.92-subtree', loomify_branch=True)
+ upgrader.start_upgrade()
+ upgrader.finish_upgrade()
+ upgraded = upgrader.branch.getBzrBranch()
+ self.check_branch(upgraded, BranchFormat.BZR_LOOM_2)
+
+ def test_default_repo_format(self):
+ """By default, the 2a repo format is selected."""
+ upgrader = self.prepare()
+ target_format = upgrader.get_target_format()
+ self.assertIs(
+ target_format._repository_format.__class__, RepositoryFormat2a)
+
+ def test_subtree_format_repo_format(self):
+ """Even subtree formats use 2a if they don't have tree references."""
+ self.useBzrBranches(direct_database=True)
+ format = format_registry.make_bzrdir('pack-0.92-subtree')
+ branch, tree = self.create_branch_and_tree(format=format)
+ upgrader = self.getUpgrader(tree.branch, branch)
+ with read_locked(upgrader.bzr_branch):
+ target_format = upgrader.get_target_format()
+ self.assertIs(
+ target_format._repository_format.__class__, RepositoryFormat2a)
+
+ def test_tree_reference_repo_format(self):
+ """Repos with tree references get 2aSubtree."""
+ self.useBzrBranches(direct_database=True)
+ format = format_registry.make_bzrdir('pack-0.92-subtree')
+ branch, tree = self.create_branch_and_tree(format=format)
+ upgrader = self.getUpgrader(tree.branch, branch)
+ self.addTreeReference(tree)
+ with read_locked(upgrader.bzr_branch):
+ target_format = upgrader.get_target_format()
+ self.assertIs(
+ target_format._repository_format.__class__,
+ RepositoryFormat2aSubtree)
+
+ def test_add_upgraded_branch_preserves_tip(self):
+ """Fetch-based upgrade preserves branch tip."""
+ upgrader = self.prepare('pack-0.92-subtree')
+ with read_locked(upgrader.bzr_branch):
+ upgrader.start_upgrade()
+ upgraded = upgrader.add_upgraded_branch().open_branch()
+ self.assertEqual('prepare-commit', upgraded.last_revision())
+
+ def test_create_upgraded_repository_preserves_dead_heads(self):
+ """Fetch-based upgrade preserves heads in the repository."""
+ upgrader = self.prepare('pack-0.92-subtree')
+ upgrader.bzr_branch.set_last_revision_info(0, NULL_REVISION)
+ with read_locked(upgrader.bzr_branch):
+ upgrader.create_upgraded_repository()
+ upgraded = upgrader.get_bzrdir().open_repository()
+ self.assertEqual(
+ 'foo', upgraded.get_revision('prepare-commit').message)
+
+ def test_create_upgraded_repository_uses_target_subdir(self):
+ """The repository is created in the right place."""
+ upgrader = self.prepare()
+ with read_locked(upgrader.bzr_branch):
+ upgrader.create_upgraded_repository()
+ upgrader.get_bzrdir().open_repository()
+
+ def test_add_upgraded_branch_preserves_tags(self):
+ """Fetch-based upgrade preserves heads in the repository."""
+ upgrader = self.prepare('pack-0.92-subtree')
+ upgrader.bzr_branch.tags.set_tag('steve', 'rev-id')
+ with read_locked(upgrader.bzr_branch):
+ upgrader.start_upgrade()
+ upgraded = upgrader.add_upgraded_branch().open_branch()
+ self.assertEqual('rev-id', upgraded.tags.lookup_tag('steve'))
+
+ def test_has_tree_references(self):
+ """Detects whether repo contains actual tree references."""
+ self.useBzrBranches(direct_database=True)
+ format = format_registry.make_bzrdir('pack-0.92-subtree')
+ branch, tree = self.create_branch_and_tree(format=format)
+ upgrader = self.getUpgrader(tree.branch, branch)
+ with read_locked(tree.branch.repository):
+ self.assertFalse(upgrader.has_tree_references())
+ self.addTreeReference(tree)
+ with read_locked(tree.branch.repository):
+ self.assertTrue(upgrader.has_tree_references())
+
+ def test_use_subtree_format_for_tree_references(self):
+ """Subtree references cause RepositoryFormat2aSubtree to be used."""
+ self.useBzrBranches(direct_database=True)
+ format = format_registry.make_bzrdir('pack-0.92-subtree')
+ branch, tree = self.create_branch_and_tree(format=format)
+ sub_branch = BzrDir.create_branch_convenience(
+ tree.bzrdir.root_transport.clone('sub').base, format=format)
+ tree.add_reference(sub_branch.bzrdir.open_workingtree())
+ tree.commit('added tree reference', committer='jrandom@xxxxxxxxxxx')
+ upgrader = self.getUpgrader(tree.branch, branch)
+ with read_locked(tree.branch):
+ upgrader.create_upgraded_repository()
+ upgraded = upgrader.get_bzrdir().open_repository()
+ self.assertIs(RepositoryFormat2aSubtree, upgraded._format.__class__)
+
+ def test_swap_in(self):
+ """Swap in swaps a branch into the original place."""
+ upgrader = self.prepare()
+ upgrader.start_upgrade()
+ upgrader.add_upgraded_branch()
+ upgrader.swap_in()
+ self.check_branch(upgrader.branch.getBzrBranch())
+
+ def test_swap_in_retains_original(self):
+ """Swap in retains the original branch in backup.bzr."""
+ upgrader = self.prepare()
+ upgrader.start_upgrade()
+ upgrader.add_upgraded_branch()
+ upgrader.swap_in()
+ t = get_transport(upgrader.branch.getInternalBzrUrl())
+ t = t.clone('backup.bzr')
+ branch = Branch.open_from_transport(t)
+ self.check_branch(branch, BranchFormat.BZR_BRANCH_6,
+ RepositoryFormat.BZR_KNITPACK_1)
+
+ def test_start_all_upgrades(self):
+ """Start all upgrades starts upgrading all branches."""
+ upgrader = self.prepare()
+ branch_changed(upgrader.branch, upgrader.bzr_branch)
+ Upgrader.start_all_upgrades(
+ upgrader.target_dir, upgrader.logger)
+ upgraded = upgrader.get_bzrdir().open_repository()
+ self.assertIs(RepositoryFormat2a, upgraded._format.__class__)
+ self.assertEqual(
+ 'foo', upgraded.get_revision('prepare-commit').message)
+
+ def test_finish_upgrade_fetches(self):
+ """finish_upgrade fetches new changes into the branch."""
+ upgrader = self.prepare()
+ upgrader.start_upgrade()
+ tree = upgrader.bzr_branch.create_checkout('tree', lightweight=True)
+ bar_id = tree.commit('bar', committer='jrandom@xxxxxxxxxxx')
+ upgrader.finish_upgrade()
+ upgraded = upgrader.branch.getBzrBranch()
+ self.assertEqual(
+ 'bar', upgraded.repository.get_revision(bar_id).message)
+
+ def test_finish_upgrade_updates_formats(self):
+ """finish_upgrade updates branch and repository formats."""
+ upgrader = self.prepare()
+ upgrader.start_upgrade()
+ upgrader.finish_upgrade()
+ self.assertEqual(
+ upgrader.branch.branch_format, BranchFormat.BZR_BRANCH_7)
+ self.assertEqual(
+ upgrader.branch.repository_format, RepositoryFormat.BZR_CHK_2A)
+
+ def test_finish_all_upgrades(self):
+ """Finish all upgrades behaves as expected."""
+ upgrader = self.prepare()
+ branch_changed(upgrader.branch, upgrader.bzr_branch)
+ upgrader.start_upgrade()
+ Upgrader.finish_all_upgrades(
+ upgrader.target_dir, upgrader.logger)
+ upgraded = upgrader.branch.getBzrBranch()
+ self.assertIs(RepositoryFormat2a,
+ upgraded.repository._format.__class__)
+ self.assertEqual(
+ 'foo', upgraded.repository.get_revision('prepare-commit').message)
=== added file 'lib/lp/codehosting/upgrade.py'
--- lib/lp/codehosting/upgrade.py 1970-01-01 00:00:00 +0000
+++ lib/lp/codehosting/upgrade.py 2012-02-07 18:29:31 +0000
@@ -0,0 +1,200 @@
+# Copyright 2011-2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Provide Upgrader to upgrade any branch to a 2a format.
+
+Provides special support for looms and subtree formats.
+
+Repositories that have no tree references are always upgraded to the standard
+2a format, even if they are in a subtree-supporting format. Repositories that
+actually have tree references are converted to RepositoryFormat2aSubtree.
+"""
+
+__metaclass__ = type
+
+__all__ = ['Upgrader']
+
+import os
+from shutil import rmtree
+from tempfile import mkdtemp
+
+from bzrlib.bzrdir import BzrDir, format_registry
+from bzrlib.errors import UpToDateFormat
+from bzrlib.plugins.loom.formats import (
+ NotALoom,
+ require_loom_branch,
+ )
+from bzrlib.repofmt.groupcompress_repo import RepositoryFormat2aSubtree
+from bzrlib.upgrade import upgrade
+
+from lp.services.database.lpstorm import IStore
+from lp.code.bzr import (
+ branch_changed,
+ RepositoryFormat,
+ )
+from lp.code.model.branch import Branch
+from lp.codehosting.bzrutils import read_locked
+from lp.codehosting.vfs.branchfs import get_real_branch_path
+
+
+class AlreadyUpgraded(Exception):
+ """Attempted to upgrade a branch that had already been upgraded."""
+
+
+class Upgrader:
+ """Upgrades branches to 2a-based formats if possible."""
+
+ def __init__(self, branch, target_dir, logger, bzr_branch=None):
+ self.branch = branch
+ self.bzr_branch = bzr_branch
+ if self.bzr_branch is None:
+ self.bzr_branch = self.branch.getBzrBranch()
+ self.target_dir = target_dir
+ self.target_subdir = os.path.join(
+ self.target_dir, str(self.branch.id))
+ self.logger = logger
+
+ def get_bzrdir(self):
+ """Return the target_subdir bzrdir."""
+ return BzrDir.open(self.target_subdir)
+
+ def get_target_format(self):
+ """Return the format to upgrade a branch to.
+
+ The repository format is always upgraded to a 2a format, but
+ the branch format is left alone if the branch is a loom.
+ :param branch: The bzr branch to upgrade
+ :return: A Metadir format instance.
+ """
+ format = format_registry.make_bzrdir('2a')
+ try:
+ require_loom_branch(self.bzr_branch)
+ except NotALoom:
+ pass
+ else:
+ format._branch_format = self.bzr_branch._format
+ if getattr(
+ self.bzr_branch.repository._format, 'supports_tree_reference',
+ False):
+ if self.has_tree_references():
+ format._repository_format = RepositoryFormat2aSubtree()
+ return format
+
+ @classmethod
+ def iter_upgraders(cls, target_dir, logger):
+ """Iterate through Upgraders given a target and logger."""
+ store = IStore(Branch)
+ branches = store.find(
+ Branch, Branch.repository_format != RepositoryFormat.BZR_CHK_2A)
+ branches.order_by(Branch.unique_name)
+ for branch in branches:
+ logger.info(
+ 'Upgrading branch %s (%d)', branch.unique_name,
+ branch.id)
+ yield cls(branch, target_dir, logger)
+
+ @classmethod
+ def start_all_upgrades(cls, target_dir, logger):
+ """Upgrade listed branches to a target directory.
+
+ :param branches: The Launchpad Branches to upgrade.
+ :param target_dir: The directory to store upgraded versions in.
+ """
+ skipped = 0
+ for upgrader in cls.iter_upgraders(target_dir, logger):
+ try:
+ upgrader.start_upgrade()
+ except AlreadyUpgraded:
+ skipped += 1
+ logger.info('Skipped %d already-upgraded branches.', skipped)
+
+ @classmethod
+ def finish_all_upgrades(cls, target_dir, logger):
+ """Upgrade listed branches to a target directory.
+
+ :param branches: The Launchpad Branches to upgrade.
+ :param target_dir: The directory to store upgraded versions in.
+ """
+ for upgrader in cls.iter_upgraders(target_dir, logger):
+ upgrader.finish_upgrade()
+
+ def finish_upgrade(self):
+ """Create an upgraded version of self.branch in self.target_dir."""
+ with read_locked(self.bzr_branch):
+ repository = self.get_bzrdir().open_repository()
+ self.add_upgraded_branch()
+ repository.fetch(self.bzr_branch.repository)
+ self.swap_in()
+ branch_changed(self.branch)
+
+ def add_upgraded_branch(self):
+ """Add an upgraded branch to the target_subdir.
+
+ self.branch's branch (but not repository) is mirrored to the BzrDir
+ and then the bzrdir is upgraded in the normal way.
+ """
+ bd = self.get_bzrdir()
+ self.mirror_branch(self.bzr_branch, bd)
+ try:
+ exceptions = upgrade(
+ bd.root_transport.base, self.get_target_format())
+ if exceptions:
+ if len(exceptions) == 1:
+ # Compatibility with historical behavior
+ raise exceptions[0]
+ else:
+ return 3
+ except UpToDateFormat:
+ pass
+ return bd
+
+ def start_upgrade(self):
+ """Do the slow part of the upgrade process."""
+ if os.path.exists(self.target_subdir):
+ raise AlreadyUpgraded
+ with read_locked(self.bzr_branch):
+ self.create_upgraded_repository()
+
+ def create_upgraded_repository(self):
+ """Create a repository in an upgraded format.
+
+ :param upgrade_dir: The directory to create the repository in.
+ :return: The created repository.
+ """
+ self.logger.info('Converting repository with fetch.')
+ upgrade_dir = mkdtemp(dir=self.target_dir)
+ try:
+ bzrdir = BzrDir.create(upgrade_dir, self.get_target_format())
+ repository = bzrdir.create_repository()
+ repository.fetch(self.bzr_branch.repository)
+ except:
+ rmtree(upgrade_dir)
+ raise
+ else:
+ os.rename(upgrade_dir, self.target_subdir)
+
+ def swap_in(self):
+ """Swap the upgraded branch into place."""
+ real_location = get_real_branch_path(self.branch.id)
+ backup_dir = os.path.join(self.target_subdir, 'backup.bzr')
+ os.rename(real_location, backup_dir)
+ os.rename(self.target_subdir, real_location)
+
+ def has_tree_references(self):
+ """Determine whether the repository contains tree references.
+
+ :return: True if it contains tree references, False otherwise.
+ """
+ repo = self.bzr_branch.repository
+ revision_ids = repo.all_revision_ids()
+ for tree in repo.revision_trees(revision_ids):
+ for path, entry in tree.iter_entries_by_dir():
+ if entry.kind == 'tree-reference':
+ return True
+ return False
+
+ def mirror_branch(self, bzr_branch, target_bd):
+ """Mirror the actual branch from a bzr_branch to a target bzrdir."""
+ target = target_bd.get_branch_transport(bzr_branch._format)
+ source = bzr_branch.bzrdir.get_branch_transport(bzr_branch._format)
+ source.copy_tree_to_transport(target)
=== modified file 'lib/lp/codehosting/vfs/branchfs.py'
--- lib/lp/codehosting/vfs/branchfs.py 2012-01-01 02:58:52 +0000
+++ lib/lp/codehosting/vfs/branchfs.py 2012-02-07 18:29:31 +0000
@@ -49,12 +49,14 @@
'branch_id_to_path',
'DirectDatabaseLaunchpadServer',
'get_lp_server',
+ 'get_real_branch_path',
'get_ro_server',
'get_rw_server',
'LaunchpadInternalServer',
'LaunchpadServer',
]
+import os.path
import sys
import xmlrpclib
@@ -201,6 +203,17 @@
'lp-internal:///', codehosting_endpoint, transport)
+def get_real_branch_path(branch_id):
+ """Return the on-disk location of a branch.
+
+ This should be used only when local filesystem operations are required.
+ For branch access, get_rw_server should be used.
+ :param branch_id: The integer id of the branch in the database.
+ """
+ root = config.codehosting.mirrored_branches_root
+ return os.path.join(root, branch_id_to_path(branch_id))
+
+
class ITransportDispatch(Interface):
"""Turns descriptions of transports into transports."""
=== modified file 'lib/lp/codehosting/vfs/tests/test_branchfs.py'
--- lib/lp/codehosting/vfs/tests/test_branchfs.py 2012-01-23 14:08:18 +0000
+++ lib/lp/codehosting/vfs/tests/test_branchfs.py 2012-02-07 18:29:31 +0000
@@ -64,12 +64,14 @@
BranchTransportDispatch,
DirectDatabaseLaunchpadServer,
get_lp_server,
+ get_real_branch_path,
LaunchpadInternalServer,
LaunchpadServer,
TransportDispatch,
UnknownTransportType,
)
from lp.codehosting.vfs.transport import AsyncVirtualTransport
+from lp.services.config import config
from lp.services.job.runner import TimeoutError
from lp.services.webapp import errorlog
from lp.testing import (
@@ -1196,3 +1198,14 @@
lp_server = get_lp_server(1, 'http://xmlrpc.example.invalid', '')
transport = lp_server._transport_dispatch._rw_dispatch.base_transport
self.assertIsInstance(transport, ChrootTransport)
+
+
+class TestRealBranchLocation(TestCase):
+
+ def test_get_real_branch_path(self):
+ """Correctly calculates the on-disk location of a branch."""
+ path = get_real_branch_path(0x00abcdef)
+ self.assertTrue(path.startswith(
+ config.codehosting.mirrored_branches_root))
+ tail = path[len(config.codehosting.mirrored_branches_root):]
+ self.assertEqual('/00/ab/cd/ef', tail)
=== modified file 'lib/lp/services/config/__init__.py'
--- lib/lp/services/config/__init__.py 2012-02-01 14:30:17 +0000
+++ lib/lp/services/config/__init__.py 2012-02-07 18:29:31 +0000
@@ -105,18 +105,26 @@
:param instance_name: the configuration instance to use. Defaults to
the value of the LPCONFIG environment variable.
:param process_name: the process configuration name to use. Defaults
- to the basename of sys.argv[0] without any extension.
+ to the basename of sys.argv[0] without any extension, or None if
+ sys.argv is not available.
"""
self._config = None
if instance_name is None:
instance_name = find_instance_name()
if process_name is None:
- process_name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
+ self._process_name = self._make_process_name()
+ else:
+ self._process_name = process_name
self._instance_name = instance_name
- self._process_name = process_name
self.root = TREE_ROOT
+ def _make_process_name(self):
+ if getattr(sys, 'argv', None) is None:
+ return None
+ basename = os.path.basename(sys.argv[0])
+ return os.path.splitext(basename)[0]
+
@property
def instance_name(self):
"""Return the config's instance name.
@@ -172,6 +180,8 @@
LaunchpadConfig loads the conf file named for the process. When
the conf file does not exist, it loads launchpad-lazr.conf instead.
"""
+ if self._process_name is None:
+ self._process_name = self._make_process_name()
return self._process_name
def setProcess(self, process_name):
=== modified file 'lib/lp/testing/__init__.py'
--- lib/lp/testing/__init__.py 2012-01-20 15:42:44 +0000
+++ lib/lp/testing/__init__.py 2012-02-07 18:29:31 +0000
@@ -1190,7 +1190,7 @@
now += delta
-def run_script(cmd_line, env=None):
+def run_script(cmd_line, env=None, cwd=None):
"""Run the given command line as a subprocess.
:param cmd_line: A command line suitable for passing to
@@ -1206,7 +1206,7 @@
env.pop('PYTHONPATH', None)
process = subprocess.Popen(
cmd_line, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, env=env)
+ stderr=subprocess.PIPE, env=env, cwd=cwd)
(out, err) = process.communicate()
return out, err, process.returncode
=== modified file 'lib/lp_sitecustomize.py'
--- lib/lp_sitecustomize.py 2012-01-03 05:05:39 +0000
+++ lib/lp_sitecustomize.py 2012-02-07 18:29:31 +0000
@@ -19,6 +19,12 @@
import zope.publisher.browser
from zope.security import checker
+# Load bzr plugins
+import lp.codehosting
+lp.codehosting
+# Force LoomBranch classes to be listed as subclasses of Branch
+import bzrlib.plugins.loom.branch
+bzrlib.plugins.loom.branch
from lp.services.log import loglevels
from lp.services.log.logger import LaunchpadLogger
from lp.services.log.mappingfilter import MappingFilter
=== added file 'scripts/upgrade_all_branches.py'
--- scripts/upgrade_all_branches.py 1970-01-01 00:00:00 +0000
+++ scripts/upgrade_all_branches.py 2012-02-07 18:29:31 +0000
@@ -0,0 +1,38 @@
+#!/usr/bin/python -S
+
+__metaclass__ = type
+
+import _pythonpath
+
+import sys
+from lp.codehosting.upgrade import Upgrader
+from lp.codehosting.bzrutils import server
+from lp.codehosting.vfs.branchfs import get_rw_server
+from lp.services.scripts.base import LaunchpadScript, LaunchpadScriptFailure
+
+
+class UpgradeAllBranches(LaunchpadScript):
+
+ def add_my_options(self):
+ self.parser.add_option(
+ '--finish', action="store_true",
+ help=("Finish the upgrade and move the new branches into place."))
+
+ def main(self):
+ if len(self.args) < 1:
+ raise LaunchpadScriptFailure('Please specify a target directory.')
+ if len(self.args) > 1:
+ raise LaunchpadScriptFailure('Too many arguments.')
+ target_dir = self.args[0]
+ with server(get_rw_server()):
+ if self.options.finish:
+ Upgrader.finish_all_upgrades(target_dir, self.logger)
+ else:
+ Upgrader.start_all_upgrades(target_dir, self.logger)
+
+
+if __name__ == "__main__":
+ sys.stderr.write(repr(sys.argv))
+ script = UpgradeAllBranches(
+ "upgrade-all-branches", dbuser='upgrade-branches')
+ script.lock_and_run()
Follow ups
