launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29043
[Merge] ~cjwatson/launchpad:remove-sourcedeps into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:remove-sourcedeps into launchpad:master with ~cjwatson/launchpad:brz-loom-3.0.0 as a prerequisite.
Commit message:
Remove the obsolete sourcedeps mechanism
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1016339 in Launchpad itself: "our bzr/brz plugins are in sourcecode/"
https://bugs.launchpad.net/launchpad/+bug/1016339
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/428729
All our Python dependencies are now either in the virtualenv or symlinked into the virtualenv from system-level packages.
`utilities/update-sourcecode` still exists as a no-op script for the time being, until we've ensured that no deployment machinery runs it any more.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:remove-sourcedeps into launchpad:master.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ddb984f..d1b37f1 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -31,9 +31,7 @@ repos:
exclude: |
(?x)^(
lib/contrib/.*
- |lib/devscripts/.*
|utilities/community-contributions\.py
- |utilities/update-sourcecode
)$
- repo: https://github.com/psf/black
rev: 22.3.0
@@ -42,9 +40,7 @@ repos:
exclude: |
(?x)^(
lib/contrib/.*
- |lib/devscripts/.*
|utilities/community-contributions\.py
- |utilities/update-sourcecode
)$
- repo: https://github.com/PyCQA/isort
rev: 5.9.2
diff --git a/Makefile b/Makefile
index af03cce..b2e2be3 100644
--- a/Makefile
+++ b/Makefile
@@ -190,8 +190,8 @@ inplace: build logs clean_logs codehosting-dir
.PHONY: build
build: compile apidoc jsbuild css_combine
-# Bootstrap download-cache and sourcecode. Useful for CI jobs that want to
-# set these up from scratch.
+# Bootstrap download-cache. Useful for CI jobs that want to set this up
+# from scratch.
.PHONY: bootstrap
bootstrap:
if [ -d download-cache/.git ]; then \
@@ -199,14 +199,13 @@ bootstrap:
else \
git clone --depth=1 $(DEPENDENCY_REPO) download-cache; \
fi
- utilities/update-sourcecode
-# LP_SOURCEDEPS_PATH should point to the sourcecode directory, but we
-# want the parent directory where the download-cache and env directories
-# are. We re-use the variable that is using for the rocketfuel-get script.
+# LP_PROJECT_ROOT/LP_SOURCEDEPS_DIR points to the parent directory where the
+# download-cache and env directories are. We reuse the variables that are
+# used for the rocketfuel-get script.
download-cache:
-ifdef LP_SOURCEDEPS_PATH
- utilities/link-external-sourcecode $(LP_SOURCEDEPS_PATH)/..
+ifneq (,$(LP_PROJECT_ROOT)$(LP_SOURCEDEPS_DIR))
+ utilities/link-external-sourcecode $(LP_PROJECT_ROOT)/$(LP_SOURCEDEPS_DIR)
else
@echo "Missing ./download-cache."
@echo "Developers: please run utilities/link-external-sourcecode."
diff --git a/doc/explanation/navigating.rst b/doc/explanation/navigating.rst
index 277d452..b4fd90f 100644
--- a/doc/explanation/navigating.rst
+++ b/doc/explanation/navigating.rst
@@ -58,10 +58,5 @@ of the ones that come up from time to time.
``brzplugins/``
Breezy plugins used in running Launchpad.
-``sourcecode/``
- A directory into which we symlink branches of some of Launchpad's
- dependencies that haven't yet been turned into proper Python
- dependencies.
-
``zcml/``
Various configuration files for the Zope services.
diff --git a/doc/explanation/running-details.rst b/doc/explanation/running-details.rst
index aad600b..09c62db 100644
--- a/doc/explanation/running-details.rst
+++ b/doc/explanation/running-details.rst
@@ -52,6 +52,5 @@ Get the code:
$ git clone https://git.launchpad.net/launchpad
$ cd launchpad
- $ utilities/update-sourcecode
$ git clone --depth=1 https://git.launchpad.net/lp-source-dependencies download-cache
$ make
diff --git a/lib/devscripts/__init__.py b/lib/devscripts/__init__.py
deleted file mode 100644
index 748fe27..0000000
--- a/lib/devscripts/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Scripts that are used in developing Launchpad."""
-
-import os
-
-
-def get_launchpad_root():
- return os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
diff --git a/lib/devscripts/sourcecode.py b/lib/devscripts/sourcecode.py
deleted file mode 100644
index 7293245..0000000
--- a/lib/devscripts/sourcecode.py
+++ /dev/null
@@ -1,425 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tools for maintaining the Launchpad source code."""
-
-from __future__ import absolute_import, print_function
-
-__all__ = [
- 'interpret_config',
- 'parse_config_file',
- 'plan_update',
-]
-
-import errno
-import json
-import optparse
-import os
-import shutil
-import sys
-
-try:
- from breezy import ui
- from breezy.branch import Branch
- from breezy.errors import (
- BzrError,
- IncompatibleRepositories,
- NotBranchError,
- )
- from breezy.plugin import load_plugins
- from breezy.revisionspec import RevisionSpec
- from breezy.trace import enable_default_logging, report_exception
- from breezy.upgrade import upgrade
- from breezy.workingtree import WorkingTree
-except ImportError:
- from bzrlib import ui
- from bzrlib.branch import Branch
- from bzrlib.errors import (
- BzrError,
- IncompatibleRepositories,
- NotBranchError,
- )
- from bzrlib.plugin import load_plugins
- from bzrlib.revisionspec import RevisionSpec
- from bzrlib.trace import enable_default_logging, report_exception
- from bzrlib.upgrade import upgrade
- from bzrlib.workingtree import WorkingTree
-
-from devscripts import get_launchpad_root
-
-
-def parse_config_file(file_handle):
- """Parse the source code config file 'file_handle'.
-
- :param file_handle: A file-like object containing sourcecode
- configuration.
- :return: A sequence of lines of either '[key, value]' or
- '[key, value, optional]'.
- """
- for line in file_handle:
- if line == '\n' or line.startswith('#'):
- continue
- yield line.split()
-
-
-def interpret_config_entry(entry, use_http=False):
- """Interpret a single parsed line from the config file."""
- branch_name = entry[0]
- components = entry[1].split(';revno=')
- branch_url = components[0]
- if use_http:
- branch_url = branch_url.replace('lp:', 'http://bazaar.launchpad.net/')
- if len(components) == 1:
- revision = None
- else:
- assert len(components) == 2, 'Bad branch URL: ' + entry[1]
- revision = components[1] or None
- if len(entry) > 2:
- assert len(entry) == 3 and entry[2].lower() == 'optional', (
- 'Bad configuration line: should be space delimited values of '
- 'sourcecode directory name, branch URL [, "optional"]\n' +
- ' '.join(entry))
- optional = True
- else:
- optional = False
- return branch_name, branch_url, revision, optional
-
-
-def load_cache(cache_filename):
- try:
- cache_file = open(cache_filename, 'r')
- except IOError as e:
- if e.errno == errno.ENOENT:
- return {}
- else:
- raise
- with cache_file:
- return json.load(cache_file)
-
-
-def interpret_config(config_entries, public_only, use_http=False):
- """Interpret a configuration stream, as parsed by 'parse_config_file'.
-
- :param configuration: A sequence of parsed configuration entries.
- :param public_only: If true, ignore private/optional branches.
- :param use_http: If True, force all branch URLs to use http://
- :return: A dict mapping the names of the sourcecode dependencies to a
- 2-tuple of their branches and whether or not they are optional.
- """
- config = {}
- for entry in config_entries:
- branch_name, branch_url, revision, optional = interpret_config_entry(
- entry, use_http)
- if not optional or not public_only:
- config[branch_name] = (branch_url, revision, optional)
- return config
-
-
-def _subset_dict(d, keys):
- """Return a dict that's a subset of 'd', based on the keys in 'keys'."""
- return {key: d[key] for key in keys}
-
-
-def plan_update(existing_branches, configuration):
- """Plan the update to existing branches based on 'configuration'.
-
- :param existing_branches: A sequence of branches that already exist.
- :param configuration: A dictionary of sourcecode configuration, such as is
- returned by `interpret_config`.
- :return: (new_branches, update_branches, removed_branches), where
- 'new_branches' are the branches in the configuration that don't exist
- yet, 'update_branches' are the branches in the configuration that do
- exist, and 'removed_branches' are the branches that exist locally, but
- not in the configuration. 'new_branches' and 'update_branches' are
- dicts of the same form as 'configuration', 'removed_branches' is a
- set of the same form as 'existing_branches'.
- """
- existing_branches = set(existing_branches)
- config_branches = set(configuration.keys())
- new_branches = config_branches - existing_branches
- removed_branches = existing_branches - config_branches
- update_branches = config_branches.intersection(existing_branches)
- return (
- _subset_dict(configuration, new_branches),
- _subset_dict(configuration, update_branches),
- removed_branches)
-
-
-def find_branches(directory):
- """List the directory names in 'directory' that are branches."""
- branches = []
- for name in os.listdir(directory):
- if name in ('.', '..'):
- continue
- try:
- Branch.open(os.path.join(directory, name))
- branches.append(name)
- except NotBranchError:
- pass
- return branches
-
-
-def get_revision_id(revision, from_branch, tip=False):
- """Return revision id for a revision number and a branch.
-
- If the revision is empty, the revision_id will be None.
-
- If ``tip`` is True, the revision value will be ignored.
- """
- if not tip and revision:
- spec = RevisionSpec.from_string(revision)
- return spec.as_revision_id(from_branch)
- # else return None
-
-
-def _format_revision_name(revision, tip=False):
- """Formatting helper to return human-readable identifier for revision.
-
- If ``tip`` is True, the revision value will be ignored.
- """
- if not tip and revision:
- return 'revision %s' % (revision,)
- else:
- return 'tip'
-
-
-def get_controldir(branch):
- try:
- # Breezy
- return branch.controldir
- except AttributeError:
- # Bazaar
- return branch.bzrdir
-
-
-def get_branches(sourcecode_directory, new_branches,
- possible_transports=None, tip=False, quiet=False):
- """Get the new branches into sourcecode."""
- for project, (branch_url, revision, optional) in new_branches.items():
- destination = os.path.join(sourcecode_directory, project)
- try:
- remote_branch = Branch.open(
- branch_url, possible_transports=possible_transports)
- except BzrError:
- if optional:
- report_exception(sys.exc_info(), sys.stderr)
- continue
- else:
- raise
- possible_transports.append(
- get_controldir(remote_branch).root_transport)
- if not quiet:
- print('Getting %s from %s at %s' %
- (project, branch_url, _format_revision_name(revision, tip)))
- # If the 'optional' flag is set, then it's a branch that shares
- # history with Launchpad, so we should share repositories. Otherwise,
- # we should avoid sharing repositories to avoid format
- # incompatibilities.
- force_new_repo = not optional
- revision_id = get_revision_id(revision, remote_branch, tip)
- get_controldir(remote_branch).sprout(
- destination, revision_id=revision_id, create_tree_if_local=True,
- source_branch=remote_branch, force_new_repo=force_new_repo,
- possible_transports=possible_transports)
-
-
-def find_stale(updated, cache, sourcecode_directory, quiet):
- """Find branches whose revision info doesn't match the cache."""
- new_updated = dict(updated)
- for project, (branch_url, revision, optional) in updated.items():
- cache_revision_info = cache.get(project)
- if cache_revision_info is None:
- continue
- cache_revno = cache_revision_info[0]
- cache_revision_id = cache_revision_info[1].encode('ASCII')
- if cache_revno != int(revision):
- continue
- destination = os.path.join(sourcecode_directory, project)
- try:
- branch = Branch.open(destination)
- except BzrError:
- continue
- last_revno, last_revision_id = branch.last_revision_info()
- if last_revno != cache_revno or last_revision_id != cache_revision_id:
- continue
- if not quiet:
- print('%s is already up to date.' % project)
- del new_updated[project]
- return new_updated
-
-
-def update_cache(cache, cache_filename, changed, sourcecode_directory, quiet):
- """Update the cache with the changed branches."""
- old_cache = dict(cache)
- for project, (branch_url, revision, optional) in changed.items():
- destination = os.path.join(sourcecode_directory, project)
- branch = Branch.open(destination)
- last_revno, last_revision_id = branch.last_revision_info()
- if not isinstance(last_revision_id, str): # Python 3
- last_revision_id = last_revision_id.decode('ASCII')
- cache[project] = [last_revno, last_revision_id]
- if cache == old_cache:
- return
- with open(cache_filename, 'w') as cache_file:
- # XXX cjwatson 2020-01-21: Stop explicitly specifying separators
- # once we require Python >= 3.4 (where this is the default).
- json.dump(
- cache, cache_file, indent=4, separators=(',', ': '),
- sort_keys=True)
- if not quiet:
- print('Cache updated. Please commit "%s".' % cache_filename)
-
-
-def update_branches(sourcecode_directory, update_branches,
- possible_transports=None, tip=False, quiet=False):
- """Update the existing branches in sourcecode."""
- if possible_transports is None:
- possible_transports = []
- # XXX: JonathanLange 2009-11-09: Rather than updating one branch after
- # another, we could instead try to get them in parallel.
- for project, (branch_url, revision, optional) in update_branches.items():
- # Update project from branch_url.
- destination = os.path.join(sourcecode_directory, project)
- if not quiet:
- print('Updating %s to %s' %
- (project, _format_revision_name(revision, tip)))
- local_tree = WorkingTree.open(destination)
- try:
- remote_branch = Branch.open(
- branch_url, possible_transports=possible_transports)
- except BzrError:
- if optional:
- report_exception(sys.exc_info(), sys.stderr)
- continue
- else:
- raise
- possible_transports.append(
- get_controldir(remote_branch).root_transport)
- revision_id = get_revision_id(revision, remote_branch, tip)
- try:
- result = local_tree.pull(
- remote_branch, stop_revision=revision_id, overwrite=True,
- possible_transports=possible_transports)
- except IncompatibleRepositories:
- # XXX JRV 20100407: Ideally get_controldir(remote_branch)._format
- # should be passed into upgrade() to ensure the format is the same
- # locally and remotely. Unfortunately smart server branches
- # have their _format set to RemoteFormat rather than an actual
- # format instance.
- upgrade(destination)
- # Upgraded, repoen working tree
- local_tree = WorkingTree.open(destination)
- result = local_tree.pull(
- remote_branch, stop_revision=revision_id, overwrite=True,
- possible_transports=possible_transports)
- if result.old_revid == result.new_revid:
- if not quiet:
- print(' (No change)')
- else:
- if result.old_revno < result.new_revno:
- change = 'Updated'
- else:
- change = 'Reverted'
- if not quiet:
- print(' (%s from %s to %s)' % (
- change, result.old_revno, result.new_revno))
-
-
-def remove_branches(sourcecode_directory, removed_branches, quiet=False):
- """Remove sourcecode that's no longer there."""
- for project in removed_branches:
- destination = os.path.join(sourcecode_directory, project)
- if not quiet:
- print('Removing %s' % project)
- try:
- shutil.rmtree(destination)
- except OSError:
- os.unlink(destination)
-
-
-def update_sourcecode(sourcecode_directory, config_filename, cache_filename,
- public_only, tip, dry_run, quiet=False, use_http=False):
- """Update the sourcecode."""
- config_file = open(config_filename)
- config = interpret_config(
- parse_config_file(config_file), public_only, use_http)
- config_file.close()
- cache = load_cache(cache_filename)
- if not os.path.exists(sourcecode_directory):
- os.makedirs(sourcecode_directory)
- branches = find_branches(sourcecode_directory)
- new, updated, removed = plan_update(branches, config)
- possible_transports = []
- if dry_run:
- print('Branches to fetch:', new.keys())
- print('Branches to update:', updated.keys())
- print('Branches to remove:', list(removed))
- else:
- get_branches(
- sourcecode_directory, new, possible_transports, tip, quiet)
- updated = find_stale(updated, cache, sourcecode_directory, quiet)
- update_branches(
- sourcecode_directory, updated, possible_transports, tip, quiet)
- changed = dict(updated)
- changed.update(new)
- update_cache(
- cache, cache_filename, changed, sourcecode_directory, quiet)
- remove_branches(sourcecode_directory, removed, quiet)
-
-
-# XXX: JonathanLange 2009-09-11: By default, the script will operate on the
-# current checkout. Most people only have symlinks to sourcecode in their
-# checkouts. This is fine for updating, but breaks for removing (you can't
-# shutil.rmtree a symlink) and breaks for adding, since it adds the new branch
-# to the checkout, rather than to the shared sourcecode area. Ideally, the
-# script would see that the sourcecode directory is full of symlinks and then
-# follow these symlinks to find the shared source directory. If the symlinks
-# differ from each other (because of developers fiddling with things), we can
-# take a survey of all of them, and choose the most popular.
-
-
-def main(args):
- parser = optparse.OptionParser("usage: %prog [options] [root [conffile]]")
- parser.add_option(
- '--public-only', action='store_true',
- help='Only fetch/update the public sourcecode branches.')
- parser.add_option(
- '--tip', action='store_true',
- help='Ignore revision constraints for all branches and pull tip')
- parser.add_option(
- '--dry-run', action='store_true',
- help='Do nothing, but report what would have been done.')
- parser.add_option(
- '--quiet', action='store_true',
- help="Don't print informational messages.")
- parser.add_option(
- '--use-http', action='store_true',
- help="Force bzr to use http to get the sourcecode branches "
- "rather than using bzr+ssh.")
- options, args = parser.parse_args(args)
- root = get_launchpad_root()
- if len(args) > 1:
- sourcecode_directory = args[1]
- else:
- sourcecode_directory = os.path.join(root, 'sourcecode')
- if len(args) > 2:
- config_filename = args[2]
- else:
- config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf')
- cache_filename = os.path.join(
- root, 'utilities', 'sourcedeps.cache')
- if len(args) > 3:
- parser.error("Too many arguments.")
- if not options.quiet:
- print('Sourcecode: %s' % (sourcecode_directory,))
- print('Config: %s' % (config_filename,))
- enable_default_logging()
- # Tell bzr to use the terminal (if any) to show progress bars
- ui.ui_factory = ui.make_ui_for_terminal(
- sys.stdin, sys.stdout, sys.stderr)
- load_plugins()
- update_sourcecode(
- sourcecode_directory, config_filename, cache_filename,
- options.public_only, options.tip, options.dry_run, options.quiet,
- options.use_http)
- return 0
diff --git a/lib/devscripts/tests/__init__.py b/lib/devscripts/tests/__init__.py
deleted file mode 100644
index 62d7904..0000000
--- a/lib/devscripts/tests/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tests for devscripts."""
diff --git a/lib/devscripts/tests/test_sourcecode.py b/lib/devscripts/tests/test_sourcecode.py
deleted file mode 100644
index e4c122c..0000000
--- a/lib/devscripts/tests/test_sourcecode.py
+++ /dev/null
@@ -1,226 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Module docstring goes here."""
-
-from __future__ import absolute_import, print_function
-
-import io
-import os
-import shutil
-import tempfile
-import unittest
-
-try:
- from breezy.bzr.bzrdir import BzrDir
- from breezy.tests import TestCase
- from breezy.transport import get_transport
-except ImportError:
- from bzrlib.bzrdir import BzrDir
- from bzrlib.tests import TestCase
- from bzrlib.transport import get_transport
-
-from devscripts import get_launchpad_root
-from devscripts.sourcecode import (
- find_branches,
- interpret_config,
- parse_config_file,
- plan_update,
-)
-
-
-class TestParseConfigFile(unittest.TestCase):
- """Tests for the config file parser."""
-
- def makeFile(self, contents):
- return io.StringIO(contents)
-
- def test_empty(self):
- # Parsing an empty config file returns an empty sequence.
- empty_file = self.makeFile("")
- self.assertEqual([], list(parse_config_file(empty_file)))
-
- def test_single_value(self):
- # Parsing a file containing a single key=value pair returns a sequence
- # containing the (key, value) as a list.
- config_file = self.makeFile("key value")
- self.assertEqual(
- [['key', 'value']], list(parse_config_file(config_file)))
-
- def test_comment_ignored(self):
- # If a line begins with a '#', then its a comment.
- comment_only = self.makeFile('# foo')
- self.assertEqual([], list(parse_config_file(comment_only)))
-
- def test_optional_value(self):
- # Lines in the config file can have a third optional entry.
- config_file = self.makeFile('key value optional')
- self.assertEqual(
- [['key', 'value', 'optional']],
- list(parse_config_file(config_file)))
-
- def test_whitespace_stripped(self):
- # Any whitespace around any of the tokens in the config file are
- # stripped out.
- config_file = self.makeFile(' key value optional ')
- self.assertEqual(
- [['key', 'value', 'optional']],
- list(parse_config_file(config_file)))
-
-
-class TestInterpretConfiguration(unittest.TestCase):
- """Tests for the configuration interpreter."""
-
- def test_empty(self):
- # An empty configuration stream means no configuration.
- config = interpret_config([], False)
- self.assertEqual({}, config)
-
- def test_key_value(self):
- # A (key, value) pair without a third optional value is returned in
- # the configuration as a dictionary entry under 'key' with '(value,
- # None, False)' as its value.
- config = interpret_config([['key', 'value']], False)
- self.assertEqual({'key': ('value', None, False)}, config)
-
- def test_key_value_public_only(self):
- # A (key, value) pair without a third optional value is returned in
- # the configuration as a dictionary entry under 'key' with '(value,
- # None, False)' as its value when public_only is true.
- config = interpret_config([['key', 'value']], True)
- self.assertEqual({'key': ('value', None, False)}, config)
-
- def test_key_value_optional(self):
- # A (key, value, optional) entry is returned in the configuration as a
- # dictionary entry under 'key' with '(value, True)' as its value.
- config = interpret_config([['key', 'value', 'optional']], False)
- self.assertEqual({'key': ('value', None, True)}, config)
-
- def test_key_value_optional_public_only(self):
- # A (key, value, optional) entry is not returned in the configuration
- # when public_only is true.
- config = interpret_config([['key', 'value', 'optional']], True)
- self.assertEqual({}, config)
-
- def test_key_value_revision(self):
- # A (key, value) pair without a third optional value when the
- # value has a suffix of ``;revno=[REVISION]`` is returned in the
- # configuration as a dictionary entry under 'key' with '(value,
- # None, False)' as its value.
- config = interpret_config([['key', 'value;revno=45']], False)
- self.assertEqual({'key': ('value', '45', False)}, config)
-
- def test_key_value_revision_with_multiple_revnos_raises_error(self):
- # A (key, value) pair without a third optional value when the
- # value has multiple suffixes of ``;revno=[REVISION]`` raises an
- # error.
- self.assertRaises(
- AssertionError,
- interpret_config, [['key', 'value;revno=45;revno=47']], False)
-
- def test_too_many_values(self):
- # A line with too many values raises an error.
- self.assertRaises(
- AssertionError,
- interpret_config, [['key', 'value', 'optional', 'extra']], False)
-
- def test_bad_optional_value(self):
- # A third value that is not the "optional" string raises an error.
- self.assertRaises(
- AssertionError,
- interpret_config, [['key', 'value', 'extra']], False)
-
- def test_use_http(self):
- # If use_http=True is passed to interpret_config, all lp: branch
- # URLs will be transformed into http:// URLs.
- config = interpret_config(
- [['key', 'lp:~sabdfl/foo/trunk']], False, use_http=True)
- expected_url = 'http://bazaar.launchpad.net/~sabdfl/foo/trunk'
- self.assertEqual(expected_url, config['key'][0])
-
-
-class TestPlanUpdate(unittest.TestCase):
- """Tests for how to plan the update."""
-
- def test_trivial(self):
- # In the trivial case, there are no existing branches and no
- # configured branches, so there are no branches to add, none to
- # update, and none to remove.
- new, existing, removed = plan_update([], {})
- self.assertEqual({}, new)
- self.assertEqual({}, existing)
- self.assertEqual(set(), removed)
-
- def test_all_new(self):
- # If there are no existing branches, then the all of the configured
- # branches are new, none are existing and none have been removed.
- new, existing, removed = plan_update([], {'a': ('b', False)})
- self.assertEqual({'a': ('b', False)}, new)
- self.assertEqual({}, existing)
- self.assertEqual(set(), removed)
-
- def test_all_old(self):
- # If there configuration is now empty, but there are existing
- # branches, then that means all the branches have been removed from
- # the configuration, none are new and none are updated.
- new, existing, removed = plan_update(['a', 'b', 'c'], {})
- self.assertEqual({}, new)
- self.assertEqual({}, existing)
- self.assertEqual({'a', 'b', 'c'}, removed)
-
- def test_all_same(self):
- # If the set of existing branches is the same as the set of
- # non-existing branches, then they all need to be updated.
- config = {'a': ('b', False), 'c': ('d', True)}
- new, existing, removed = plan_update(config.keys(), config)
- self.assertEqual({}, new)
- self.assertEqual(config, existing)
- self.assertEqual(set(), removed)
-
- def test_smoke_the_default_config(self):
- # Make sure we can parse, interpret and plan based on the default
- # config file.
- root = get_launchpad_root()
- config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf')
- config_file = open(config_filename)
- config = interpret_config(parse_config_file(config_file), False)
- config_file.close()
- plan_update([], config)
-
-
-class TestFindBranches(TestCase):
- """Tests the way that we find branches."""
-
- def setUp(self):
- TestCase.setUp(self)
- self.disable_directory_isolation()
-
- def makeBranch(self, path):
- transport = get_transport(path)
- transport.ensure_base()
- BzrDir.create_branch_convenience(
- transport.base, possible_transports=[transport])
-
- def makeDirectory(self):
- directory = tempfile.mkdtemp()
- self.addCleanup(shutil.rmtree, directory)
- return directory
-
- def test_empty_directory_has_no_branches(self):
- # An empty directory has no branches.
- empty = self.makeDirectory()
- self.assertEqual([], list(find_branches(empty)))
-
- def test_directory_with_branches(self):
- # find_branches finds branches in the directory.
- directory = self.makeDirectory()
- self.makeBranch('%s/a' % directory)
- self.assertEqual(['a'], list(find_branches(directory)))
-
- def test_ignores_files(self):
- # find_branches ignores any files in the directory.
- directory = self.makeDirectory()
- some_file = open('%s/a' % directory, 'w')
- some_file.write('hello\n')
- some_file.close()
- self.assertEqual([], list(find_branches(directory)))
diff --git a/lib/lp/scripts/utilities/test.py b/lib/lp/scripts/utilities/test.py
index f0d5f7f..bb852f0 100755
--- a/lib/lp/scripts/utilities/test.py
+++ b/lib/lp/scripts/utilities/test.py
@@ -179,7 +179,7 @@ defaults = {
# Find tests in the tests and ftests directories
"tests_pattern": "^f?tests$",
"test_path": [os.path.join(config.root, "lib")],
- "package": ["canonical", "lp", "devscripts", "launchpad_loggerhead"],
+ "package": ["canonical", "lp", "launchpad_loggerhead"],
"layer": ["!(YUIAppServerLayer)"],
"require_unique_ids": True,
}
diff --git a/setup.cfg b/setup.cfg
index e1a8c8f..367bb34 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -212,7 +212,7 @@ ignore =
[isort]
# database/* have some implicit relative imports.
-known_first_party = canonical,lp,launchpad_loggerhead,devscripts,fti,replication,preflight,security,upgrade,dbcontroller
+known_first_party = canonical,lp,launchpad_loggerhead,fti,replication,preflight,security,upgrade,dbcontroller
known_pythonpath = _pythonpath
line_length = 79
sections = FUTURE,PYTHONPATH,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
diff --git a/utilities/link-external-sourcecode b/utilities/link-external-sourcecode
index bc2a58b..06af63f 100755
--- a/utilities/link-external-sourcecode
+++ b/utilities/link-external-sourcecode
@@ -1,11 +1,11 @@
#!/usr/bin/python3
#
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the GNU
+# Copyright 2009-2023 Canonical Ltd. This software is licensed under the GNU
# Affero General Public License version 3 (see the file LICENSE).
import optparse
import subprocess
-from os import curdir, listdir, makedirs, symlink, unlink
+from os import curdir, symlink, unlink
from os.path import abspath, basename, exists, islink, join, realpath, relpath
from sys import stderr, stdout
@@ -21,18 +21,6 @@ def get_main_worktree(branch_dir):
return None
-def gen_missing_files(source, destination):
- """Generate info on every file in source not in destination.
-
- Yields `(source, destination)` tuples.
- """
- for name in listdir(source):
- destination_file = join(destination, name)
- if not exists(destination_file):
- source_file = join(source, name)
- yield source_file, destination_file,
-
-
def link(source, destination):
"""Symlink source to destination.
@@ -55,8 +43,8 @@ if __name__ == "__main__":
parser = optparse.OptionParser(
usage="%prog [options] [parent]",
description=(
- "Add a symlink in <target>/sourcecode for each corresponding "
- "file in <parent>/sourcecode."
+ "Add a symlink from <target>/download-cache to "
+ "<parent>/download-cache."
),
epilog=(
"Most of the time this does the right thing if run "
@@ -119,17 +107,6 @@ if __name__ == "__main__":
if options.parent is None:
parser.error("Parent tree not specified.")
- if not exists(join(options.target, "sourcecode")):
- makedirs(join(options.target, "sourcecode"))
-
- missing_files = gen_missing_files(
- abspath(join(options.parent, "sourcecode")),
- abspath(join(options.target, "sourcecode")),
- )
-
- for source, destination in missing_files:
- link(source, destination)
-
for folder_name in ("download-cache",):
source = abspath(join(options.parent, folder_name))
destination = abspath(join(options.target, folder_name))
diff --git a/utilities/rocketfuel-get b/utilities/rocketfuel-get
index 678ccb6..2c51408 100755
--- a/utilities/rocketfuel-get
+++ b/utilities/rocketfuel-get
@@ -1,24 +1,13 @@
#! /bin/bash
#
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2022 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
#
-# Update your copy of master and the necessary source dependencies, and make
-# sure all source dependencies are properly linked in to all the branches you
-# are working on.
+# Update your copy of master and the necessary source dependencies.
# Stop if there's an error, and treat unset variables as errors.
set -eu
-# A rough measure of how much stuff we can do in parallel.
-CPU_COUNT="$(grep -Ec '^processor\b' /proc/cpuinfo)"
-
-# Helper function to run a child process, indenting stdout to aid
-# readability.
-run-child() {
- "$@" | sed -e "s/^/ /"
-}
-
# Load local settings.
if [ -e "$HOME/.rocketfuel-env.sh" ]
then
@@ -39,7 +28,7 @@ git -C "$LP_TRUNK_PATH" pull
FINAL_REV=$(git -C "$LP_TRUNK_PATH" rev-parse HEAD)
# Make sure our directories are around.
-mkdir -p "$LP_SOURCEDEPS_PATH" "$YUI_PATH"
+mkdir -p "$(dirname "$LP_DOWNLOAD_CACHE_PATH")" "$YUI_PATH"
# Get/update the download cache.
if [ -d "$LP_DOWNLOAD_CACHE_PATH" ]
@@ -49,50 +38,6 @@ else
git clone --depth=1 lp:lp-source-dependencies "$LP_DOWNLOAD_CACHE_PATH"
fi
-# Add or update sourcepackages.
-sourcedeps_conf="$(dirname "$0")/sourcedeps.conf"
-if [ ! -e "$sourcedeps_conf" ]
-then
- # Use the global deps which are stable.
- echo "Could not find $sourcedeps_conf" >&2
- sourcedeps_conf="$LP_TRUNK_PATH/utilities/sourcedeps.conf"
-fi
-
-echo "Updating sourcecode dependencies in rocketfuel:"
-run-child \
- "$LP_TRUNK_PATH/utilities/update-sourcecode" \
- "$LP_SOURCEDEPS_PATH" "$sourcedeps_conf"
-
-# Update the current trees in the repo.
-echo "Updating sourcecode dependencies in current local branches:"
-
-# Find directories among local branches containing "sourcecode" directories.
-# Prints each as a null-terminated record (since Unix filenames may contain
-# newlines).
-find_branches_to_relink() {
- find "$LP_PROJECT_ROOT" \
- -mindepth 2 -maxdepth 2 -type d -name sourcecode -printf '%h\0'
-}
-
-# Some setups may have lp-sourcedeps mixed in with the local branches. Echo
-# stdin to stdout, with these filenames filtered out. Filenames must be
-# null-terminated on input, and remain null-terminated on output.
-filter_branches_to_relink() {
- grep -vz '/lp-sourcedeps$'
-}
-
-# Re-link the sourcecode directories for local branches. Takes the branch
-# paths from stdin, as null-terminated records.
-relink_branches() {
- run-child xargs --no-run-if-empty \
- --max-procs="${CPU_COUNT}" --max-args=1 --null \
- "$LP_TRUNK_PATH/utilities/link-external-sourcecode" \
- --parent "$LP_PROJECT_ROOT/$LP_SOURCEDEPS_DIR" --target
-}
-
-# Actually do it:
-find_branches_to_relink | filter_branches_to_relink | relink_branches
-
# Build launchpad if there were changes.
if [ "$FINAL_REV" != "$INITIAL_REV" ]
diff --git a/utilities/rocketfuel-setup b/utilities/rocketfuel-setup
index e03e8be..e4e1237 100755
--- a/utilities/rocketfuel-setup
+++ b/utilities/rocketfuel-setup
@@ -190,7 +190,6 @@ of the following steps:
$ git clone lp:launchpad
$ cd launchpad
$ git clone --depth=1 lp:lp-source-dependencies download-cache
-$ utilities/update-sourcecode
$ utilities/launchpad-database-setup
$ make schema
$ sudo make install
@@ -220,10 +219,6 @@ LP_TRUNK_NAME=\${LP_TRUNK_NAME:=launchpad}
LP_TRUNK_PATH=\$LP_PROJECT_ROOT/\$LP_TRUNK_NAME
LP_SOURCEDEPS_DIR=\${LP_SOURCEDEPS_DIR:=lp-sourcedeps}
-LP_SOURCEDEPS_PATH=\$LP_PROJECT_ROOT/\$LP_SOURCEDEPS_DIR/sourcecode
-
-# Force tilde expansion
-LP_SOURCEDEPS_PATH=\$(eval echo \${LP_SOURCEDEPS_PATH})
" > "$HOME/.rocketfuel-env.sh"
fi
diff --git a/utilities/snakefood/Makefile b/utilities/snakefood/Makefile
index f3bbec7..06d92cc 100644
--- a/utilities/snakefood/Makefile
+++ b/utilities/snakefood/Makefile
@@ -5,7 +5,7 @@ default: lp-clustered.svg
# Generate import dependency graph
lp.sfood:
sfood -i -u -I $(LIB_DIR)/sqlobject -I $(LIB_DIR)/schoolbell \
- -I $(LIB_DIR)/devscripts -I $(LIB_DIR)/contrib \
+ -I $(LIB_DIR)/contrib \
-I $(LIB_DIR)/canonical/not-used $(LIB_DIR)/canonical \
$(LIB_DIR)/lp 2>/dev/null | grep -v contrib/ \
| grep -v sqlobject | egrep -v 'BeautifulSoup|bs4' | grep -v psycopg \
diff --git a/utilities/sourcedeps.cache b/utilities/sourcedeps.cache
deleted file mode 100644
index 9e26dfe..0000000
--- a/utilities/sourcedeps.cache
+++ /dev/null
@@ -1 +0,0 @@
-{}
\ No newline at end of file
diff --git a/utilities/sourcedeps.conf b/utilities/sourcedeps.conf
deleted file mode 100644
index e37f376..0000000
--- a/utilities/sourcedeps.conf
+++ /dev/null
@@ -1,8 +0,0 @@
-# If you're adding items to this file you _must_ use the full path to
-# the branch after lp: rather than just trusting that bzr/lp will expand
-# the series for you. If you don't, update-sourcecode will break when
-# running parallelised tests inside lxc containers.
-
-#########################################################
-#### DEPRECATED. NO NEW ITEMS. NO NO NO NO NO NONONONONO
-#########################################################
diff --git a/utilities/sourcedeps.filter b/utilities/sourcedeps.filter
deleted file mode 100644
index d81ca05..0000000
--- a/utilities/sourcedeps.filter
+++ /dev/null
@@ -1,3 +0,0 @@
-P *.o
-P *.pyc
-P *.so
diff --git a/utilities/update-sourcecode b/utilities/update-sourcecode
index 1f2189a..8ea9e06 100755
--- a/utilities/update-sourcecode
+++ b/utilities/update-sourcecode
@@ -1,18 +1,15 @@
-#!/usr/bin/python2 -u
+#!/usr/bin/python3
#
# Copyright 2009 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
-"""Update the sourcecode managed in sourcecode/."""
+"""Update the sourcecode managed in sourcecode/.
-import os
-import sys
-
-sys.path.insert(
- 0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib")
-)
+This script is now a no-op, and will be removed once no deployment machinery
+uses it any more.
+"""
-from devscripts import sourcecode # noqa: E402
+import sys
if __name__ == "__main__":
- sys.exit(sourcecode.main(sys.argv))
+ sys.exit(0)
