launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #25931
[Merge] ~cjwatson/launchpad:remove-codeimport into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:remove-codeimport into launchpad:master with ~cjwatson/launchpad:move-instrument-method-helpers as a prerequisite.
Commit message:
Remove codeimport, which is now in a separate tree
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/395802
This now lives in https://git.launchpad.net/lp-codeimport.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:remove-codeimport into launchpad:master.
diff --git a/README b/README
index 9682c4c..d09ef4f 100644
--- a/README
+++ b/README
@@ -89,8 +89,8 @@ You can spend years hacking on Launchpad full-time and not know what all of
the files in the top-level directory are for. However, here's a guide to some
of the ones that come up from time to time.
- bzrplugins/
- Bazaar plugins used in running Launchpad.
+ brzplugins/
+ Breezy plugins used in running Launchpad.
sourcecode/
A directory into which we symlink branches of some of Launchpad's
diff --git a/bzrplugins/git b/bzrplugins/git
deleted file mode 120000
index cf14f27..0000000
--- a/bzrplugins/git
+++ /dev/null
@@ -1 +0,0 @@
-../sourcecode/bzr-git
\ No newline at end of file
diff --git a/bzrplugins/svn b/bzrplugins/svn
deleted file mode 120000
index 15208fc..0000000
--- a/bzrplugins/svn
+++ /dev/null
@@ -1 +0,0 @@
-../sourcecode/bzr-svn
\ No newline at end of file
diff --git a/cronscripts/code-import-dispatcher.py b/cronscripts/code-import-dispatcher.py
deleted file mode 100755
index 1225de7..0000000
--- a/cronscripts/code-import-dispatcher.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/python2 -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Look for and dispatch code import jobs as needed."""
-
-import _pythonpath
-
-from six.moves.xmlrpc_client import ServerProxy
-
-from lp.codehosting.codeimport.dispatcher import CodeImportDispatcher
-from lp.services.config import config
-from lp.services.scripts.base import LaunchpadScript
-from lp.services.webapp.errorlog import globalErrorUtility
-
-
-class CodeImportDispatcherScript(LaunchpadScript):
-
- def add_my_options(self):
- self.parser.add_option(
- "--max-jobs", dest="max_jobs", type=int,
- default=config.codeimportdispatcher.max_jobs_per_machine,
- help="The maximum number of jobs to run on this machine.")
-
- def run(self, use_web_security=False, isolation=None):
- """See `LaunchpadScript.run`.
-
- We override to avoid all of the setting up all of the component
- architecture and connecting to the database.
- """
- self.main()
-
- def main(self):
- globalErrorUtility.configure('codeimportdispatcher')
-
- dispatcher = CodeImportDispatcher(self.logger, self.options.max_jobs)
- dispatcher.findAndDispatchJobs(
- ServerProxy(config.codeimportdispatcher.codeimportscheduler_url))
-
-
-if __name__ == '__main__':
- script = CodeImportDispatcherScript("codeimportdispatcher")
- script.lock_and_run()
-
diff --git a/lib/lp/codehosting/__init__.py b/lib/lp/codehosting/__init__.py
index f41a997..0731704 100644
--- a/lib/lp/codehosting/__init__.py
+++ b/lib/lp/codehosting/__init__.py
@@ -3,8 +3,8 @@
"""Launchpad code-hosting system.
-NOTE: Importing this package will load any system Bazaar plugins, as well as
-all plugins in the bzrplugins/ directory underneath the rocketfuel checkout.
+NOTE: Importing this package will load any system Breezy plugins, as well as
+all plugins in the brzplugins/ directory underneath the rocketfuel checkout.
"""
from __future__ import absolute_import, print_function
@@ -12,7 +12,6 @@ from __future__ import absolute_import, print_function
__metaclass__ = type
__all__ = [
'get_brz_path',
- 'get_BZR_PLUGIN_PATH_for_subprocess',
]
@@ -25,39 +24,16 @@ from breezy.library_state import BzrLibraryState as BrzLibraryState
from breezy.plugin import load_plugins as brz_load_plugins
# This import is needed so that brz's logger gets registered.
import breezy.trace
-import six
from zope.security import checker
from lp.services.config import config
-if six.PY2:
- from bzrlib.plugin import load_plugins as bzr_load_plugins
- # This import is needed so that bzr's logger gets registered.
- import bzrlib.trace
-
def get_brz_path():
"""Find the path to the copy of Breezy for this rocketfuel instance"""
return os.path.join(config.root, 'bin', 'brz')
-def _get_bzr_plugins_path():
- """Find the path to the Bazaar plugins for this rocketfuel instance."""
- return os.path.join(config.root, 'bzrplugins')
-
-
-def get_BZR_PLUGIN_PATH_for_subprocess():
- """Calculate the appropriate value for the BZR_PLUGIN_PATH environment.
-
- The '-site' token tells bzrlib not to include the 'site specific plugins
- directory' (which is usually something like
- /usr/lib/pythonX.Y/dist-packages/bzrlib/plugins/) in the plugin search
- path, which would be inappropriate for Launchpad, which may be using a bzr
- egg of an incompatible version.
- """
- return ":".join((_get_bzr_plugins_path(), "-site"))
-
-
def _get_brz_plugins_path():
"""Find the path to the Breezy plugins for this rocketfuel instance."""
return os.path.join(config.root, 'brzplugins')
@@ -83,9 +59,6 @@ if breezy._global_state is None:
brz_state._start()
-# XXX cjwatson 2019-06-13: Remove BZR_PLUGIN_PATH and supporting code once
-# all of Launchpad has been ported to Breezy.
-os.environ['BZR_PLUGIN_PATH'] = get_BZR_PLUGIN_PATH_for_subprocess()
os.environ['BRZ_PLUGIN_PATH'] = get_BRZ_PLUGIN_PATH_for_subprocess()
# Disable some Breezy plugins that are likely to cause trouble if used on
@@ -100,8 +73,6 @@ os.environ['BRZ_DISABLE_PLUGINS'] = ':'.join([
# We want to have full access to Launchpad's Breezy plugins throughout the
# codehosting package.
-if six.PY2:
- bzr_load_plugins([_get_bzr_plugins_path()])
brz_load_plugins()
diff --git a/lib/lp/codehosting/codeimport/__init__.py b/lib/lp/codehosting/codeimport/__init__.py
deleted file mode 100644
index 094e67f..0000000
--- a/lib/lp/codehosting/codeimport/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# Make this directory a Python package
diff --git a/lib/lp/codehosting/codeimport/dispatcher.py b/lib/lp/codehosting/codeimport/dispatcher.py
deleted file mode 100644
index 2ab170f..0000000
--- a/lib/lp/codehosting/codeimport/dispatcher.py
+++ /dev/null
@@ -1,104 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""The code import dispatcher.
-
-The code import dispatcher is responsible for checking if any code
-imports need to be processed and launching child processes to handle
-them.
-"""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-__all__ = [
- 'CodeImportDispatcher',
- ]
-
-import os
-import socket
-import subprocess
-import time
-
-from lp.services.config import config
-
-
-class CodeImportDispatcher:
- """A CodeImportDispatcher kicks off the processing of a job if needed.
-
- The entry point is `findAndDispatchJob`.
-
- :ivar txn: A transaction manager.
- :ivar logger: A `Logger` object.
- """
-
- worker_script = os.path.join(
- config.root, 'scripts', 'code-import-worker-monitor.py')
-
- def __init__(self, logger, worker_limit, _sleep=time.sleep):
- """Initialize an instance.
-
- :param logger: A `Logger` object.
- """
- self.logger = logger
- self.worker_limit = worker_limit
- self._sleep = _sleep
-
- def getHostname(self):
- """Return the hostname of this machine.
-
- This usually calls `socket.gethostname` but it can be
- overridden by the config for tests and developer machines.
- """
- if config.codeimportdispatcher.forced_hostname:
- return config.codeimportdispatcher.forced_hostname
- else:
- return socket.gethostname()
-
- def dispatchJob(self, job_id):
- """Start the processing of job `job_id`."""
- # Just launch the process and forget about it.
- log_file = os.path.join(
- config.codeimportdispatcher.worker_log_dir,
- 'code-import-worker-%d.log' % (job_id,))
- # Return the Popen object to make testing easier.
- interpreter = "%s/bin/py" % config.root
- return subprocess.Popen(
- [interpreter, self.worker_script, str(job_id), '-vv',
- '--log-file', log_file])
-
- def findAndDispatchJob(self, scheduler_client):
- """Check for and dispatch a job if necessary.
-
- :return: A boolean, true if a job was found and dispatched.
- """
- job_id = scheduler_client.getJobForMachine(
- self.getHostname(), self.worker_limit)
-
- if job_id == 0:
- self.logger.info("No jobs pending.")
- return False
-
- self.logger.info("Dispatching job %d." % job_id)
-
- self.dispatchJob(job_id)
- return True
-
- def _getSleepInterval(self):
- """How long to sleep for until asking for a new job.
-
- The basic idea is to wait longer if the machine is more heavily
- loaded, so that less loaded slaves get a chance to grab some jobs.
-
- We assume worker_limit will be roughly the number of CPUs in the
- machine, so load/worker_limit is roughly how loaded the machine is.
- """
- return 5 * os.getloadavg()[0] / self.worker_limit
-
- def findAndDispatchJobs(self, scheduler_client):
- """Call findAndDispatchJob until no job is found."""
- while True:
- found = self.findAndDispatchJob(scheduler_client)
- if not found:
- break
- self._sleep(self._getSleepInterval())
diff --git a/lib/lp/codehosting/codeimport/foreigntree.py b/lib/lp/codehosting/codeimport/foreigntree.py
deleted file mode 100644
index e0fbe43..0000000
--- a/lib/lp/codehosting/codeimport/foreigntree.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Support for CVS branches."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-__all__ = ['CVSWorkingTree']
-
-import os
-
-import CVS
-import six
-
-
-class CVSWorkingTree:
- """Represents a CVS working tree."""
-
- def __init__(self, cvs_root, cvs_module, local_path):
- """Construct a CVSWorkingTree.
-
- :param cvs_root: The root of the CVS repository.
- :param cvs_module: The module in the CVS repository.
- :param local_path: The local path to check the working tree out to.
- """
- self.root = six.ensure_str(cvs_root)
- self.module = six.ensure_str(cvs_module)
- self.local_path = os.path.abspath(local_path)
-
- def checkout(self):
- repository = CVS.Repository(self.root, None)
- repository.get(self.module, self.local_path)
-
- def commit(self):
- tree = CVS.tree(self.local_path)
- tree.commit(log='Log message')
-
- def update(self):
- tree = CVS.tree(self.local_path)
- tree.update()
diff --git a/lib/lp/codehosting/codeimport/tarball.py b/lib/lp/codehosting/codeimport/tarball.py
deleted file mode 100644
index edabfbb..0000000
--- a/lib/lp/codehosting/codeimport/tarball.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Create and extract tarballs."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-__all__ = ['create_tarball', 'extract_tarball', 'TarError']
-
-
-import os
-import subprocess
-
-
-class TarError(Exception):
- """Raised when the `tar` command has failed for some reason."""
-
- _format = 'tar exited with status %(status)d'
-
- def __init__(self, status):
- Exception.__init__(self, self._format % {'status': status})
-
-
-class NotADirectory(Exception):
-
- _format = '%(path)s is not a directory.'
-
- def __init__(self, path):
- Exception.__init__(self, self._format % {'path': path})
-
-
-def _check_tar_retcode(retcode):
- if retcode != 0:
- raise TarError(retcode)
-
-
-def create_tarball(directory, tarball_name, filenames=None):
- """Create a tarball of `directory` called `tarball_name`.
-
- This creates a tarball of `directory` from its parent directory. This
- means that when untarred, it will create a new directory with the same
- name as `directory`. If `filenames` is not None, then the tarball will
- be limited to that list of directory entries under `directory`.
-
- Basically, this is the standard way of making tarballs.
- """
- if not os.path.isdir(directory):
- raise NotADirectory(directory)
- if filenames is None:
- filenames = ['.']
- retcode = subprocess.call(
- ['tar', '-C', directory, '-czf', tarball_name] + filenames)
- _check_tar_retcode(retcode)
-
-
-def extract_tarball(tarball_name, directory):
- """Extract contents of a tarball.
-
- Changes to `directory` and extracts the tarball at `tarball_name`.
- """
- if not os.path.isdir(directory):
- raise NotADirectory(directory)
- retcode = subprocess.call(['tar', 'xzf', tarball_name, '-C', directory])
- _check_tar_retcode(retcode)
diff --git a/lib/lp/codehosting/codeimport/tests/__init__.py b/lib/lp/codehosting/codeimport/tests/__init__.py
deleted file mode 100644
index 094e67f..0000000
--- a/lib/lp/codehosting/codeimport/tests/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# Make this directory a Python package
diff --git a/lib/lp/codehosting/codeimport/tests/servers.py b/lib/lp/codehosting/codeimport/tests/servers.py
deleted file mode 100644
index e3c3ae2..0000000
--- a/lib/lp/codehosting/codeimport/tests/servers.py
+++ /dev/null
@@ -1,427 +0,0 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Server classes that know how to create various kinds of foreign archive."""
-
-from __future__ import absolute_import, print_function
-
-__all__ = [
- 'BzrServer',
- 'CVSServer',
- 'GitServer',
- 'SubversionServer',
- ]
-
-__metaclass__ = type
-
-import errno
-import io
-import os
-import re
-import shutil
-import signal
-import stat
-import subprocess
-import tempfile
-import threading
-import time
-from wsgiref.simple_server import make_server
-
-from bzrlib.branch import Branch
-from bzrlib.branchbuilder import BranchBuilder
-from bzrlib.bzrdir import BzrDir
-from bzrlib.tests.test_server import (
- ReadonlySmartTCPServer_for_testing,
- TestServer,
- )
-from bzrlib.tests.treeshape import build_tree_contents
-from bzrlib.transport import Server
-from bzrlib.urlutils import (
- escape,
- join as urljoin,
- )
-import CVS
-from dulwich.errors import NotGitRepository
-import dulwich.index
-from dulwich.objects import Blob
-from dulwich.repo import Repo as GitRepo
-from dulwich.server import (
- Backend,
- Handler,
- )
-from dulwich.web import (
- GunzipFilter,
- handle_service_request,
- HTTPGitApplication,
- LimitedInputFilter,
- WSGIRequestHandlerLogger,
- WSGIServerLogger,
- )
-import six
-from subvertpy import SubversionException
-import subvertpy.ra
-import subvertpy.repos
-
-from lp.services.log.logger import BufferLogger
-
-
-def local_path_to_url(local_path):
- """Return a file:// URL to `local_path`.
-
- This implementation is unusual in that it returns a file://localhost/ URL.
- This is to work around the valid_vcs_details constraint on CodeImport.
- """
- return u'file://localhost' + escape(
- os.path.normpath(os.path.abspath(local_path)))
-
-
-def run_in_temporary_directory(function):
- """Decorate `function` to be run in a temporary directory.
-
- Creates a new temporary directory and changes to it for the duration of
- `function`.
- """
-
- def decorated(*args, **kwargs):
- old_cwd = os.getcwd()
- new_dir = tempfile.mkdtemp()
- os.chdir(new_dir)
- try:
- return function(*args, **kwargs)
- finally:
- os.chdir(old_cwd)
- shutil.rmtree(new_dir)
-
- decorated.__name__ = function.__name__
- decorated.__doc__ = function.__doc__
- return decorated
-
-
-class SubversionServer(Server):
- """A controller for an Subversion repository, used for testing."""
-
- def __init__(self, repository_path, use_svn_serve=False):
- super(SubversionServer, self).__init__()
- self.repository_path = os.path.abspath(repository_path)
- self._use_svn_serve = use_svn_serve
-
- def _get_ra(self, url):
- return subvertpy.ra.RemoteAccess(url,
- auth=subvertpy.ra.Auth([subvertpy.ra.get_username_provider()]))
-
- def createRepository(self, path):
- """Create a Subversion repository at `path`."""
- subvertpy.repos.create(path)
-
- def get_url(self):
- """Return a URL to the Subversion repository."""
- if self._use_svn_serve:
- return u'svn://localhost/'
- else:
- return local_path_to_url(self.repository_path)
-
- def start_server(self):
- super(SubversionServer, self).start_server()
- self.createRepository(self.repository_path)
- if self._use_svn_serve:
- conf_path = os.path.join(
- self.repository_path, 'conf/svnserve.conf')
- with open(conf_path, 'w') as conf_file:
- conf_file.write('[general]\nanon-access = write\n')
- self._svnserve = subprocess.Popen(
- ['svnserve', '--daemon', '--foreground', '--threads',
- '--root', self.repository_path])
- delay = 0.1
- for i in range(10):
- try:
- try:
- self._get_ra(self.get_url())
- except OSError as e:
- # Subversion < 1.9 just produces OSError.
- if e.errno == errno.ECONNREFUSED:
- time.sleep(delay)
- delay *= 1.5
- continue
- raise
- except SubversionException as e:
- # Subversion >= 1.9 turns the raw error into a
- # SubversionException. The code is
- # SVN_ERR_RA_CANNOT_CREATE_SESSION, which is not yet
- # in subvertpy.
- if e.args[1] == 170013:
- time.sleep(delay)
- delay *= 1.5
- continue
- raise
- else:
- break
- except Exception as e:
- self._kill_svnserve()
- raise
- else:
- self._kill_svnserve()
- raise AssertionError(
- "svnserve didn't start accepting connections")
-
- def _kill_svnserve(self):
- os.kill(self._svnserve.pid, signal.SIGINT)
- self._svnserve.communicate()
-
- def stop_server(self):
- super(SubversionServer, self).stop_server()
- if self._use_svn_serve:
- self._kill_svnserve()
-
- def makeBranch(self, branch_name, tree_contents):
- """Create a branch on the Subversion server called `branch_name`.
-
- :param branch_name: The name of the branch to create.
- :param tree_contents: The contents of the module. This is a list of
- tuples of (relative filename, file contents).
- """
- branch_url = self.makeDirectory(branch_name)
- ra = self._get_ra(branch_url)
- editor = ra.get_commit_editor({"svn:log": "Import"})
- root = editor.open_root()
- for filename, content in tree_contents:
- f = root.add_file(filename)
- try:
- subvertpy.delta.send_stream(
- io.BytesIO(content), f.apply_textdelta())
- finally:
- f.close()
- root.close()
- editor.close()
- return branch_url
-
- def makeDirectory(self, directory_name, commit_message=None):
- """Make a directory on the repository."""
- if commit_message is None:
- commit_message = 'Make %r' % (directory_name,)
- ra = self._get_ra(self.get_url())
- editor = ra.get_commit_editor({"svn:log": commit_message})
- root = editor.open_root()
- root.add_directory(directory_name).close()
- root.close()
- editor.close()
- return urljoin(self.get_url(), directory_name)
-
-
-class CVSServer(Server):
- """A CVS server for testing."""
-
- def __init__(self, repository_path):
- """Construct a `CVSServer`.
-
- :param repository_path: The path to the directory that will contain
- the CVS repository.
- """
- super(CVSServer, self).__init__()
- self._repository_path = os.path.abspath(repository_path)
-
- def createRepository(self, path):
- """Create a CVS repository at `path`.
-
- :param path: The local path to create a repository in.
- :return: A CVS.Repository`.
- """
- return CVS.init(path, BufferLogger())
-
- def getRoot(self):
- """Return the CVS root for this server."""
- return six.ensure_text(self._repository_path)
-
- @run_in_temporary_directory
- def makeModule(self, module_name, tree_contents):
- """Create a module on the CVS server called `module_name`.
-
- A 'module' in CVS roughly corresponds to a project.
-
- :param module_name: The name of the module to create.
- :param tree_contents: The contents of the module. This is a list of
- tuples of (relative filename, file contents).
- """
- build_tree_contents(tree_contents)
- self._repository.Import(
- module=module_name, log="import", vendor="vendor",
- release=['release'], dir='.')
-
- def start_server(self):
- # Initialize the repository.
- super(CVSServer, self).start_server()
- self._repository = self.createRepository(self._repository_path)
-
-
-class GitStoreBackend(Backend):
- """A backend that looks up repositories under a store directory."""
-
- def __init__(self, root):
- self.root = root
-
- def open_repository(self, path):
- full_path = os.path.normpath(os.path.join(self.root, path.lstrip("/")))
- if not full_path.startswith(self.root + "/"):
- raise NotGitRepository("Repository %s not under store" % path)
- return GitRepo(full_path)
-
-
-class TurnipSetSymbolicRefHandler(Handler):
- """Dulwich protocol handler for setting a symbolic ref.
-
- Transcribed from turnip.pack.git.PackBackendProtocol.
- """
-
- def __init__(self, backend, args, proto, http_req=None):
- super(TurnipSetSymbolicRefHandler, self).__init__(
- backend, proto, http_req=http_req)
- self.repo = backend.open_repository(args[0])
-
- def handle(self):
- line = self.proto.read_pkt_line()
- if line is None:
- self.proto.write_pkt_line(b"ERR Invalid set-symbolic-ref-line\n")
- return
- name, target = line.split(b" ", 1)
- if name != b"HEAD":
- self.proto.write_pkt_line(
- b'ERR Symbolic ref name must be "HEAD"\n')
- return
- if target.startswith(b"-"):
- self.proto.write_pkt_line(
- b'ERR Symbolic ref target may not start with "-"\n')
- return
- try:
- self.repo.refs.set_symbolic_ref(name, target)
- except Exception as e:
- self.proto.write_pkt_line(b'ERR %s\n' % e)
- else:
- self.proto.write_pkt_line(b'ACK %s\n' % name)
-
-
-class HTTPGitServerThread(threading.Thread):
- """Thread that runs an HTTP Git server."""
-
- def __init__(self, backend, address, port=None):
- super(HTTPGitServerThread, self).__init__()
- self.name = "HTTP Git server on %s:%s" % (address, port)
- app = HTTPGitApplication(
- backend,
- handlers={'turnip-set-symbolic-ref': TurnipSetSymbolicRefHandler})
- app.services[('POST', re.compile('/turnip-set-symbolic-ref$'))] = (
- handle_service_request)
- app = GunzipFilter(LimitedInputFilter(app))
- self.server = make_server(
- address, port, app, handler_class=WSGIRequestHandlerLogger,
- server_class=WSGIServerLogger)
-
- def run(self):
- self.server.serve_forever()
-
- def get_address(self):
- return self.server.server_address
-
- def stop(self):
- self.server.shutdown()
-
-
-class GitServer(Server):
-
- def __init__(self, repository_store, use_server=False):
- super(GitServer, self).__init__()
- self.repository_store = repository_store
- self._use_server = use_server
-
- def get_url(self, repository_name):
- """Return a URL to the Git repository."""
- if self._use_server:
- host, port = self._server.get_address()
- return u'http://%s:%d/%s' % (host, port, repository_name)
- else:
- return local_path_to_url(
- os.path.join(self.repository_store, repository_name))
-
- def createRepository(self, path, bare=False):
- if bare:
- GitRepo.init_bare(path)
- else:
- GitRepo.init(path)
-
- def start_server(self):
- super(GitServer, self).start_server()
- if self._use_server:
- self._server = HTTPGitServerThread(
- GitStoreBackend(self.repository_store), "localhost", 0)
- self._server.start()
-
- def stop_server(self):
- super(GitServer, self).stop_server()
- if self._use_server:
- self._server.stop()
-
- def makeRepo(self, repository_name, tree_contents):
- repository_path = os.path.join(self.repository_store, repository_name)
- os.makedirs(repository_path)
- self.createRepository(repository_path, bare=self._use_server)
- repo = GitRepo(repository_path)
- blobs = [
- (Blob.from_string(contents), filename) for (filename, contents)
- in tree_contents]
- repo.object_store.add_objects(blobs)
- root_id = dulwich.index.commit_tree(repo.object_store, [
- (filename, b.id, stat.S_IFREG | 0o644)
- for (b, filename) in blobs])
- repo.do_commit(committer='Joe Foo <joe@xxxxxxx>',
- message=u'<The commit message>', tree=root_id)
-
-
-class BzrServer(Server):
-
- def __init__(self, repository_path, use_server=False):
- super(BzrServer, self).__init__()
- self.repository_path = repository_path
- self._use_server = use_server
-
- def createRepository(self, path):
- BzrDir.create_branch_convenience(path)
-
- def makeRepo(self, tree_contents):
- branch = Branch.open(self.repository_path)
- branch.get_config().set_user_option("create_signatures", "never")
- builder = BranchBuilder(branch=branch)
- actions = [('add', ('', 'tree-root', 'directory', None))]
- actions += [
- ('add', (path, path + '-id', 'file', content))
- for (path, content) in tree_contents]
- builder.build_snapshot(
- None, None, actions, committer='Joe Foo <joe@xxxxxxx>',
- message=u'<The commit message>')
-
- def get_url(self):
- if self._use_server:
- return six.ensure_text(self._bzrserver.get_url())
- else:
- return local_path_to_url(self.repository_path)
-
- def start_server(self):
- super(BzrServer, self).start_server()
- self.createRepository(self.repository_path)
-
- class LocalURLServer(TestServer):
- def __init__(self, repository_path):
- self.repository_path = repository_path
-
- def start_server(self):
- pass
-
- def get_url(self):
- return local_path_to_url(self.repository_path)
-
- if self._use_server:
- self._bzrserver = ReadonlySmartTCPServer_for_testing()
- self._bzrserver.start_server(
- LocalURLServer(self.repository_path))
-
- def stop_server(self):
- super(BzrServer, self).stop_server()
- if self._use_server:
- self._bzrserver.stop_server()
diff --git a/lib/lp/codehosting/codeimport/tests/test_dispatcher.py b/lib/lp/codehosting/codeimport/tests/test_dispatcher.py
deleted file mode 100644
index a6dc82e..0000000
--- a/lib/lp/codehosting/codeimport/tests/test_dispatcher.py
+++ /dev/null
@@ -1,172 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tests for the code import dispatcher."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-
-
-from optparse import OptionParser
-import os
-import shutil
-import socket
-import tempfile
-
-from lp.codehosting.codeimport.dispatcher import CodeImportDispatcher
-from lp.services import scripts
-from lp.services.log.logger import BufferLogger
-from lp.testing import TestCase
-from lp.testing.layers import BaseLayer
-
-
-class StubSchedulerClient:
- """A scheduler client that returns a pre-arranged answer."""
-
- def __init__(self, ids_to_return):
- self.ids_to_return = ids_to_return
-
- def getJobForMachine(self, machine, limit):
- return self.ids_to_return.pop(0)
-
-
-class MockSchedulerClient:
- """A scheduler client that records calls to `getJobForMachine`."""
-
- def __init__(self):
- self.calls = []
-
- def getJobForMachine(self, machine, limit):
- self.calls.append((machine, limit))
- return 0
-
-
-class TestCodeImportDispatcherUnit(TestCase):
- """Unit tests for `CodeImportDispatcher`."""
-
- layer = BaseLayer
-
- def setUp(self):
- TestCase.setUp(self)
- self.pushConfig('codeimportdispatcher', forced_hostname='none')
-
- def makeDispatcher(self, worker_limit=10, _sleep=lambda delay: None):
- """Make a `CodeImportDispatcher`."""
- return CodeImportDispatcher(
- BufferLogger(), worker_limit, _sleep=_sleep)
-
- def test_getHostname(self):
- # By default, getHostname return the same as socket.gethostname()
- dispatcher = self.makeDispatcher()
- self.assertEqual(socket.gethostname(), dispatcher.getHostname())
-
- def test_getHostnameOverride(self):
- # getHostname can be overridden by the config for testing, however.
- dispatcher = self.makeDispatcher()
- self.pushConfig('codeimportdispatcher', forced_hostname='test-value')
- self.assertEqual('test-value', dispatcher.getHostname())
-
- def writePythonScript(self, script_path, script_body):
- """Write out an executable Python script.
-
- This method writes a script header and `script_body` (which should be
- a list of lines of Python source) to `script_path` and makes the file
- executable.
- """
- script = open(script_path, 'w')
- for script_line in script_body:
- script.write(script_line + '\n')
-
- def filterOutLoggingOptions(self, arglist):
- """Remove the standard logging options from a list of arguments."""
-
- # Calling parser.parse_args as we do below is dangerous,
- # as if a callback invokes parser.error the test suite
- # terminates. This hack removes the dangerous argument manually.
- arglist = [
- arg for arg in arglist if not arg.startswith('--log-file=')]
- while '--log-file' in arglist:
- index = arglist.index('--log-file')
- del arglist[index] # Delete the argument
- del arglist[index] # And its parameter
-
- parser = OptionParser()
- scripts.logger_options(parser)
- options, args = parser.parse_args(arglist)
- return args
-
- def test_dispatchJob(self):
- # dispatchJob launches a process described by its
- # worker_script attribute with a given job id as an argument.
-
- # We create a script that writes its command line arguments to
- # some a temporary file and examine that.
- dispatcher = self.makeDispatcher()
- tmpdir = tempfile.mkdtemp()
- self.addCleanup(shutil.rmtree, tmpdir)
- script_path = os.path.join(tmpdir, 'script.py')
- output_path = os.path.join(tmpdir, 'output.txt')
- self.writePythonScript(
- script_path,
- ['import sys',
- 'open(%r, "w").write(str(sys.argv[1:]))' % output_path])
- dispatcher.worker_script = script_path
- proc = dispatcher.dispatchJob(10)
- proc.wait()
- with open(output_path) as f:
- arglist = self.filterOutLoggingOptions(eval(f.read()))
- self.assertEqual(['10'], arglist)
-
- def test_findAndDispatchJob_jobWaiting(self):
- # If there is a job to dispatch, then we call dispatchJob with its id
- # and the worker_limit supplied to the dispatcher.
- calls = []
- dispatcher = self.makeDispatcher()
- dispatcher.dispatchJob = lambda job_id: calls.append(job_id)
- found = dispatcher.findAndDispatchJob(StubSchedulerClient([10]))
- self.assertEqual(([10], True), (calls, found))
-
- def test_findAndDispatchJob_noJobWaiting(self):
- # If there is no job to dispatch, then we just exit quietly.
- calls = []
- dispatcher = self.makeDispatcher()
- dispatcher.dispatchJob = lambda job_id: calls.append(job_id)
- found = dispatcher.findAndDispatchJob(StubSchedulerClient([0]))
- self.assertEqual(([], False), (calls, found))
-
- def test_findAndDispatchJob_calls_getJobForMachine_with_limit(self):
- # findAndDispatchJob calls getJobForMachine on the scheduler client
- # with the hostname and supplied worker limit.
- worker_limit = self.factory.getUniqueInteger()
- dispatcher = self.makeDispatcher(worker_limit)
- scheduler_client = MockSchedulerClient()
- dispatcher.findAndDispatchJob(scheduler_client)
- self.assertEqual(
- [(dispatcher.getHostname(), worker_limit)],
- scheduler_client.calls)
-
- def test_findAndDispatchJobs(self):
- # findAndDispatchJobs calls getJobForMachine on the scheduler_client,
- # dispatching jobs, until it indicates that there are no more jobs to
- # dispatch.
- calls = []
- dispatcher = self.makeDispatcher()
- dispatcher.dispatchJob = lambda job_id: calls.append(job_id)
- dispatcher.findAndDispatchJobs(StubSchedulerClient([10, 9, 0]))
- self.assertEqual([10, 9], calls)
-
- def test_findAndDispatchJobs_sleeps(self):
- # After finding a job, findAndDispatchJobs sleeps for an interval as
- # returned by _getSleepInterval.
- sleep_calls = []
- interval = self.factory.getUniqueInteger()
-
- def _sleep(delay):
- sleep_calls.append(delay)
-
- dispatcher = self.makeDispatcher(_sleep=_sleep)
- dispatcher.dispatchJob = lambda job_id: None
- dispatcher._getSleepInterval = lambda: interval
- dispatcher.findAndDispatchJobs(StubSchedulerClient([10, 0]))
- self.assertEqual([interval], sleep_calls)
diff --git a/lib/lp/codehosting/codeimport/tests/test_foreigntree.py b/lib/lp/codehosting/codeimport/tests/test_foreigntree.py
deleted file mode 100644
index 161f83b..0000000
--- a/lib/lp/codehosting/codeimport/tests/test_foreigntree.py
+++ /dev/null
@@ -1,117 +0,0 @@
-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tests for foreign branch support."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-
-import os
-import time
-
-from bzrlib.tests import TestCaseWithTransport
-import CVS
-
-from lp.codehosting.codeimport.foreigntree import CVSWorkingTree
-from lp.codehosting.codeimport.tests.servers import CVSServer
-from lp.testing.layers import BaseLayer
-
-
-class TestCVSWorkingTree(TestCaseWithTransport):
-
- layer = BaseLayer
-
- def assertHasCheckout(self, cvs_working_tree):
- """Assert that `cvs_working_tree` has a checkout of its CVS module."""
- tree = CVS.tree(os.path.abspath(cvs_working_tree.local_path))
- repository = tree.repository()
- self.assertEqual(repository.root, cvs_working_tree.root)
- self.assertEqual(tree.module().name(), cvs_working_tree.module)
-
- def makeCVSWorkingTree(self, local_path):
- """Make a CVS working tree for testing."""
- return CVSWorkingTree(
- self.cvs_server.getRoot(), self.module_name, local_path)
-
- def setUp(self):
- super(TestCVSWorkingTree, self).setUp()
- self.cvs_server = CVSServer('repository_path')
- self.cvs_server.start_server()
- self.module_name = 'test_module'
- self.cvs_server.makeModule(
- self.module_name, [('README', 'Random content\n')])
- self.addCleanup(self.cvs_server.stop_server)
-
- def test_path(self):
- # The local path is passed to the constructor and available as
- # 'local_path'.
- tree = CVSWorkingTree('root', 'module', 'path')
- self.assertEqual(tree.local_path, os.path.abspath('path'))
-
- def test_module(self):
- # The module is passed to the constructor and available as 'module'.
- tree = CVSWorkingTree('root', 'module', 'path')
- self.assertEqual(tree.module, 'module')
-
- def test_root(self):
- # The root is passed to the constructor and available as 'root'.
- tree = CVSWorkingTree('root', 'module', 'path')
- self.assertEqual(tree.root, 'root')
-
- def test_checkout(self):
- # checkout() checks out an up-to-date working tree.
- tree = self.makeCVSWorkingTree('working_tree')
- tree.checkout()
- self.assertHasCheckout(tree)
-
- def test_commit(self):
- # commit() makes local changes available to other checkouts.
- tree = self.makeCVSWorkingTree('working_tree')
- tree.checkout()
-
- # If you write to a file in the same second as the previous commit,
- # CVS will not think that it has changed.
- time.sleep(1)
-
- # Make a change.
- new_content = 'Comfort ye\n'
- readme = open(os.path.join(tree.local_path, 'README'), 'w')
- readme.write(new_content)
- readme.close()
- self.assertFileEqual(new_content, 'working_tree/README')
-
- # Commit the change.
- tree.commit()
-
- tree2 = self.makeCVSWorkingTree('working_tree2')
- tree2.checkout()
-
- self.assertFileEqual(new_content, 'working_tree2/README')
-
- def test_update(self):
- # update() fetches any changes to the branch from the remote branch.
- # We test this by checking out the same branch twice, making
- # modifications in one, then updating the other. If the modifications
- # appear, then update() works.
- tree = self.makeCVSWorkingTree('working_tree')
- tree.checkout()
-
- tree2 = self.makeCVSWorkingTree('working_tree2')
- tree2.checkout()
-
- # If you write to a file in the same second as the previous commit,
- # CVS will not think that it has changed.
- time.sleep(1)
-
- # Make a change.
- new_content = 'Comfort ye\n'
- self.build_tree_contents([('working_tree/README', new_content)])
-
- # Commit the change.
- tree.commit()
-
- # Update.
- tree2.update()
- readme_path = os.path.join(tree2.local_path, 'README')
- self.assertFileEqual(new_content, readme_path)
diff --git a/lib/lp/codehosting/codeimport/tests/test_uifactory.py b/lib/lp/codehosting/codeimport/tests/test_uifactory.py
deleted file mode 100644
index 5c93900..0000000
--- a/lib/lp/codehosting/codeimport/tests/test_uifactory.py
+++ /dev/null
@@ -1,162 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tests for `LoggingUIFactory`."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-
-from lp.codehosting.codeimport.uifactory import LoggingUIFactory
-from lp.services.log.logger import BufferLogger
-from lp.testing import (
- FakeTime,
- TestCase,
- )
-
-
-class TestLoggingUIFactory(TestCase):
- """Tests for `LoggingUIFactory`."""
-
- def setUp(self):
- TestCase.setUp(self)
- self.fake_time = FakeTime(12345)
- self.logger = BufferLogger()
-
- def makeLoggingUIFactory(self):
- """Make a `LoggingUIFactory` with fake time and contained output."""
- return LoggingUIFactory(
- time_source=self.fake_time.now, logger=self.logger)
-
- def test_first_progress_updates(self):
- # The first call to progress generates some output.
- factory = self.makeLoggingUIFactory()
- bar = factory.nested_progress_bar()
- bar.update("hi")
- self.assertEqual('INFO hi\n', self.logger.getLogBuffer())
-
- def test_second_rapid_progress_doesnt_update(self):
- # The second of two progress calls that are less than the factory's
- # interval apart does not generate output.
- factory = self.makeLoggingUIFactory()
- bar = factory.nested_progress_bar()
- bar.update("hi")
- self.fake_time.advance(factory.interval / 2)
- bar.update("there")
- self.assertEqual('INFO hi\n', self.logger.getLogBuffer())
-
- def test_second_slow_progress_updates(self):
- # The second of two progress calls that are more than the factory's
- # interval apart does generate output.
- factory = self.makeLoggingUIFactory()
- bar = factory.nested_progress_bar()
- bar.update("hi")
- self.fake_time.advance(factory.interval * 2)
- bar.update("there")
- self.assertEqual(
- 'INFO hi\n'
- 'INFO there\n',
- self.logger.getLogBuffer())
-
- def test_first_progress_on_new_bar_updates(self):
- # The first progress on a new progress task always generates output.
- factory = self.makeLoggingUIFactory()
- bar = factory.nested_progress_bar()
- bar.update("hi")
- self.fake_time.advance(factory.interval / 2)
- bar2 = factory.nested_progress_bar()
- bar2.update("there")
- self.assertEqual(
- 'INFO hi\nINFO hi:there\n', self.logger.getLogBuffer())
-
- def test_update_with_count_formats_nicely(self):
- # When more details are passed to update, they are formatted nicely.
- factory = self.makeLoggingUIFactory()
- bar = factory.nested_progress_bar()
- bar.update("hi", 1, 8)
- self.assertEqual('INFO hi 1/8\n', self.logger.getLogBuffer())
-
- def test_report_transport_activity_reports_bytes_since_last_update(self):
- # If there is no call to _progress_updated for 'interval' seconds, the
- # next call to report_transport_activity will report however many
- # bytes have been transferred since the update.
- factory = self.makeLoggingUIFactory()
- bar = factory.nested_progress_bar()
- bar.update("hi", 1, 10)
- self.fake_time.advance(factory.interval / 2)
- # The bytes in this call will not be reported:
- factory.report_transport_activity(None, 1, 'read')
- self.fake_time.advance(factory.interval)
- bar.update("hi", 2, 10)
- self.fake_time.advance(factory.interval / 2)
- factory.report_transport_activity(None, 10, 'read')
- self.fake_time.advance(factory.interval)
- factory.report_transport_activity(None, 100, 'read')
- self.fake_time.advance(factory.interval * 2)
- # This call will cause output that does not include the transport
- # activity info.
- bar.update("hi", 3, 10)
- self.assertEqual(
- 'INFO hi 1/10\n'
- 'INFO hi 2/10\n'
- 'INFO 110 bytes transferred | hi 2/10\n'
- 'INFO hi 3/10\n',
- self.logger.getLogBuffer())
-
- def test_note(self):
- factory = self.makeLoggingUIFactory()
- factory.note("Banja Luka")
- self.assertEqual('INFO Banja Luka\n', self.logger.getLogBuffer())
-
- def test_show_error(self):
- factory = self.makeLoggingUIFactory()
- factory.show_error("Exploding Peaches")
- self.assertEqual(
- "ERROR Exploding Peaches\n", self.logger.getLogBuffer())
-
- def test_confirm_action(self):
- factory = self.makeLoggingUIFactory()
- self.assertTrue(factory.confirm_action(
- "How are you %(when)s?", "wellness", {"when": "today"}))
-
- def test_show_message(self):
- factory = self.makeLoggingUIFactory()
- factory.show_message("Peaches")
- self.assertEqual("INFO Peaches\n", self.logger.getLogBuffer())
-
- def test_get_username(self):
- factory = self.makeLoggingUIFactory()
- self.assertIs(
- None, factory.get_username("Who are you %(when)s?", when="today"))
-
- def test_get_password(self):
- factory = self.makeLoggingUIFactory()
- self.assertIs(
- None,
- factory.get_password("How is your %(drink)s", drink="coffee"))
-
- def test_show_warning(self):
- factory = self.makeLoggingUIFactory()
- factory.show_warning("Peaches")
- self.assertEqual("WARNING Peaches\n", self.logger.getLogBuffer())
-
- def test_show_warning_unicode(self):
- factory = self.makeLoggingUIFactory()
- factory.show_warning(u"Peach\xeas")
- self.assertEqual(
- "WARNING Peach\xc3\xaas\n", self.logger.getLogBuffer())
-
- def test_user_warning(self):
- factory = self.makeLoggingUIFactory()
- factory.show_user_warning('cross_format_fetch',
- from_format="athing", to_format="anotherthing")
- message = factory._user_warning_templates['cross_format_fetch'] % {
- "from_format": "athing",
- "to_format": "anotherthing",
- }
- self.assertEqual("WARNING %s\n" % message, self.logger.getLogBuffer())
-
- def test_clear_term(self):
- factory = self.makeLoggingUIFactory()
- factory.clear_term()
- self.assertEqual("", self.logger.getLogBuffer())
diff --git a/lib/lp/codehosting/codeimport/tests/test_worker.py b/lib/lp/codehosting/codeimport/tests/test_worker.py
deleted file mode 100644
index 50fa83c..0000000
--- a/lib/lp/codehosting/codeimport/tests/test_worker.py
+++ /dev/null
@@ -1,1550 +0,0 @@
-# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tests for the code import worker."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-
-import logging
-import os
-import shutil
-import subprocess
-import tempfile
-import time
-from uuid import uuid4
-
-from bzrlib import (
- trace,
- urlutils,
- )
-from bzrlib.branch import (
- Branch,
- BranchReferenceFormat,
- )
-from bzrlib.branchbuilder import BranchBuilder
-from bzrlib.bzrdir import (
- BzrDir,
- BzrDirFormat,
- format_registry,
- )
-from bzrlib.errors import NoSuchFile
-from bzrlib.tests import (
- http_utils,
- TestCaseWithTransport,
- )
-from bzrlib.transport import (
- get_transport,
- get_transport_from_url,
- )
-from bzrlib.url_policy_open import (
- _BlacklistPolicy,
- AcceptAnythingPolicy,
- BadUrl,
- BranchOpener,
- BranchOpenPolicy,
- )
-from bzrlib.urlutils import (
- join as urljoin,
- local_path_from_url,
- )
-from CVS import (
- Repository,
- tree as CVSTree,
- )
-from dulwich.repo import Repo as GitRepo
-from fixtures import FakeLogger
-from pymacaroons import Macaroon
-import scandir
-import six
-import subvertpy
-import subvertpy.client
-import subvertpy.ra
-from testtools.matchers import (
- ContainsAll,
- LessThan,
- )
-
-import lp.codehosting
-from lp.codehosting.codeimport.tarball import (
- create_tarball,
- extract_tarball,
- )
-from lp.codehosting.codeimport.tests.servers import (
- BzrServer,
- CVSServer,
- GitServer,
- SubversionServer,
- )
-from lp.codehosting.codeimport.worker import (
- BazaarBranchStore,
- BzrImportWorker,
- BzrSvnImportWorker,
- CodeImportBranchOpenPolicy,
- CodeImportSourceDetails,
- CodeImportWorkerExitCode,
- CSCVSImportWorker,
- ForeignTreeStore,
- get_default_bazaar_branch_store,
- GitImportWorker,
- GitToGitImportWorker,
- ImportDataStore,
- ToBzrImportWorker,
- )
-from lp.codehosting.tests.helpers import create_branch_with_one_revision
-from lp.services.config import config
-from lp.services.log.logger import BufferLogger
-from lp.testing import TestCase
-from lp.testing.layers import BaseLayer
-
-
-class ForeignBranchPluginLayer(BaseLayer):
- """Ensure only specific tests are run with foreign branch plugins loaded.
- """
-
- @classmethod
- def setUp(cls):
- pass
-
- @classmethod
- def tearDown(cls):
- # Raise NotImplementedError to signal that this layer cannot be torn
- # down. This means that the test runner will run subsequent tests in
- # a different process.
- raise NotImplementedError
-
- @classmethod
- def testSetUp(cls):
- pass
-
- @classmethod
- def testTearDown(cls):
- pass
-
-
-default_format = BzrDirFormat.get_default_format()
-
-
-class WorkerTest(TestCaseWithTransport, TestCase):
- """Base test case for things that test the code import worker.
-
- Provides Bazaar testing features, access to Launchpad objects and
- factories for some code import objects.
- """
-
- layer = ForeignBranchPluginLayer
-
- def setUp(self):
- super(WorkerTest, self).setUp()
- self.disable_directory_isolation()
- BranchOpener.install_hook()
-
- def assertDirectoryTreesEqual(self, directory1, directory2):
- """Assert that `directory1` has the same structure as `directory2`.
-
- That is, assert that all of the files and directories beneath
- `directory1` are laid out in the same way as `directory2`.
- """
- def list_files(directory):
- for path, ignored, ignored in scandir.walk(directory):
- yield path[len(directory):]
- self.assertEqual(
- sorted(list_files(directory1)), sorted(list_files(directory2)))
-
- def makeTemporaryDirectory(self):
- directory = tempfile.mkdtemp()
- self.addCleanup(shutil.rmtree, directory)
- return directory
-
-
-class TestBazaarBranchStore(WorkerTest):
- """Tests for `BazaarBranchStore`."""
-
- def setUp(self):
- WorkerTest.setUp(self)
- # XXX: JonathanLange 2010-12-24 bug=694140: Avoid spurious "No
- # handlers for logger 'bzr'" messages.
- trace._bzr_logger = logging.getLogger('bzr')
- self.temp_dir = self.makeTemporaryDirectory()
- self.arbitrary_branch_id = 10
-
- def makeBranchStore(self):
- return BazaarBranchStore(self.get_transport())
-
- def test_defaultStore(self):
- # The default store is at config.codeimport.bazaar_branch_store.
- store = get_default_bazaar_branch_store()
- self.assertEqual(
- store.transport.base.rstrip('/'),
- config.codeimport.bazaar_branch_store.rstrip('/'))
-
- def test__getMirrorURL(self):
- # _getMirrorURL returns a URL for the branch with the given id.
- store = BazaarBranchStore(get_transport_from_url(
- 'sftp://storage.example/branches'))
- self.assertEqual(
- 'sftp://storage.example/branches/000186a0',
- store._getMirrorURL(100000))
-
- def test__getMirrorURL_push(self):
- # _getMirrorURL prefers bzr+ssh over sftp when constructing push
- # URLs.
- store = BazaarBranchStore(get_transport_from_url(
- 'sftp://storage.example/branches'))
- self.assertEqual(
- 'bzr+ssh://storage.example/branches/000186a0',
- store._getMirrorURL(100000, push=True))
-
- def test_getNewBranch(self):
- # If there's no Bazaar branch of this id, then pull creates a new
- # Bazaar branch.
- store = self.makeBranchStore()
- bzr_branch = store.pull(
- self.arbitrary_branch_id, self.temp_dir, default_format)
- self.assertEqual(0, bzr_branch.revno())
-
- def test_getNewBranch_without_tree(self):
- # If pull() with needs_tree=False creates a new branch, it doesn't
- # create a working tree.
- store = self.makeBranchStore()
- bzr_branch = store.pull(
- self.arbitrary_branch_id, self.temp_dir, default_format, False)
- self.assertFalse(bzr_branch.bzrdir.has_workingtree())
-
- def test_getNewBranch_with_tree(self):
- # If pull() with needs_tree=True creates a new branch, it creates a
- # working tree.
- store = self.makeBranchStore()
- bzr_branch = store.pull(
- self.arbitrary_branch_id, self.temp_dir, default_format, True)
- self.assertTrue(bzr_branch.bzrdir.has_workingtree())
-
- def test_pushBranchThenPull(self):
- # After we've pushed up a branch to the store, we can then pull it
- # from the store.
- store = self.makeBranchStore()
- tree = create_branch_with_one_revision('original')
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
- new_branch = store.pull(
- self.arbitrary_branch_id, self.temp_dir, default_format)
- self.assertEqual(
- tree.branch.last_revision(), new_branch.last_revision())
-
- def test_pull_without_needs_tree_doesnt_create_tree(self):
- # pull with needs_tree=False doesn't spend the time to create a
- # working tree.
- store = self.makeBranchStore()
- tree = create_branch_with_one_revision('original')
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
- new_branch = store.pull(
- self.arbitrary_branch_id, self.temp_dir, default_format, False)
- self.assertFalse(new_branch.bzrdir.has_workingtree())
-
- def test_pull_needs_tree_creates_tree(self):
- # pull with needs_tree=True creates a working tree.
- store = self.makeBranchStore()
- tree = create_branch_with_one_revision('original')
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
- new_branch = store.pull(
- self.arbitrary_branch_id, self.temp_dir, default_format, True)
- self.assertTrue(new_branch.bzrdir.has_workingtree())
-
- def test_pullUpgradesFormat(self):
- # A branch should always be in the most up-to-date format before a
- # pull is performed.
- store = self.makeBranchStore()
- target_url = store._getMirrorURL(self.arbitrary_branch_id)
- knit_format = format_registry.get('knit')()
- tree = create_branch_with_one_revision(target_url, format=knit_format)
- self.assertNotEqual(
- tree.bzrdir._format.repository_format.network_name(),
- default_format.repository_format.network_name())
-
- # The fetched branch is in the default format.
- new_branch = store.pull(
- self.arbitrary_branch_id, self.temp_dir, default_format)
- # Make sure backup.bzr is removed, as it interferes with CSCVS.
- self.assertEqual(os.listdir(self.temp_dir), [".bzr"])
- self.assertEqual(new_branch.repository._format.network_name(),
- default_format.repository_format.network_name())
-
- def test_pushUpgradesFormat(self):
- # A branch should always be in the most up-to-date format before a
- # pull is performed.
- store = self.makeBranchStore()
- target_url = store._getMirrorURL(self.arbitrary_branch_id)
- knit_format = format_registry.get('knit')()
- create_branch_with_one_revision(target_url, format=knit_format)
-
- # The fetched branch is in the default format.
- new_branch = store.pull(
- self.arbitrary_branch_id, self.temp_dir, default_format)
- self.assertEqual(
- default_format, new_branch.bzrdir._format)
-
- # The remote branch is still in the old format at this point.
- target_branch = Branch.open(target_url)
- self.assertEqual(
- knit_format.get_branch_format(),
- target_branch._format)
-
- store.push(self.arbitrary_branch_id, new_branch, default_format)
-
- # The remote branch is now in the new format.
- target_branch = Branch.open(target_url)
- # Only .bzr is left behind. The scanner removes branches
- # in which invalid directories (such as .bzr.retire.
- # exist). (bug #798560)
- self.assertEqual(
- target_branch.user_transport.list_dir("."),
- [".bzr"])
- self.assertEqual(
- default_format.get_branch_format(),
- target_branch._format)
- self.assertEqual(
- target_branch.last_revision_info(),
- new_branch.last_revision_info())
-
- def test_pushTwiceThenPull(self):
- # We can push up a branch to the store twice and then pull it from the
- # store.
- store = self.makeBranchStore()
- tree = create_branch_with_one_revision('original')
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
- new_branch = store.pull(
- self.arbitrary_branch_id, self.temp_dir, default_format)
- self.assertEqual(
- tree.branch.last_revision(), new_branch.last_revision())
-
- def test_push_divergant_branches(self):
- # push() uses overwrite=True, so divergent branches (rebased) can be
- # pushed.
- store = self.makeBranchStore()
- tree = create_branch_with_one_revision('original')
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
- tree = create_branch_with_one_revision('divergant')
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
-
- def fetchBranch(self, from_url, target_path):
- """Pull a branch from `from_url` to `target_path`.
-
- This uses the Bazaar API for pulling a branch, and is used to test
- that `push` indeed pushes a branch to a specific location.
-
- :return: The working tree of the branch.
- """
- bzr_dir = BzrDir.open(from_url)
- bzr_dir.sprout(target_path)
- return BzrDir.open(target_path).open_workingtree()
-
- def test_makesDirectories(self):
- # push() tries to create the base directory of the branch store if it
- # doesn't already exist.
- store = BazaarBranchStore(self.get_transport('doesntexist'))
- tree = create_branch_with_one_revision('original')
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
- self.assertIsDirectory('doesntexist', self.get_transport())
-
- def test_storedLocation(self):
- # push() puts the branch in a directory named after the branch ID on
- # the BazaarBranchStore's transport.
- store = self.makeBranchStore()
- tree = create_branch_with_one_revision('original')
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
- new_branch = self.fetchBranch(
- urljoin(store.transport.base, '%08x' % self.arbitrary_branch_id),
- 'new_tree')
- self.assertEqual(
- tree.branch.last_revision(), new_branch.last_revision())
-
- def test_sftpPrefix(self):
- # Since branches are mirrored by importd via sftp, _getMirrorURL must
- # support sftp urls. There was once a bug that made it incorrect with
- # sftp.
- sftp_prefix = 'sftp://example/base/'
- store = BazaarBranchStore(get_transport(sftp_prefix))
- self.assertEqual(
- store._getMirrorURL(self.arbitrary_branch_id),
- sftp_prefix + '%08x' % self.arbitrary_branch_id)
-
- def test_sftpPrefixNoSlash(self):
- # If the prefix has no trailing slash, one should be added. It's very
- # easy to forget a trailing slash in the importd configuration.
- sftp_prefix_noslash = 'sftp://example/base'
- store = BazaarBranchStore(get_transport(sftp_prefix_noslash))
- self.assertEqual(
- store._getMirrorURL(self.arbitrary_branch_id),
- sftp_prefix_noslash + '/' + '%08x' % self.arbitrary_branch_id)
-
- def test_all_revisions_saved(self):
- # All revisions in the branch's repo are transferred, not just those
- # in the ancestry of the tip.
- # Consider a branch with two heads in its repo:
- # revid
- # / \
- # revid1 revid2 <- branch tip
- # A naive push/pull would just store 'revid' and 'revid2' in the
- # branch store -- we need to make sure all three revisions are stored
- # and retrieved.
- builder = self.make_branch_builder('tree')
- revid = builder.build_snapshot(
- None, None, [('add', ('', 'root-id', 'directory', ''))])
- revid1 = builder.build_snapshot(None, [revid], [])
- revid2 = builder.build_snapshot(None, [revid], [])
- store = self.makeBranchStore()
- store.push(
- self.arbitrary_branch_id, builder.get_branch(), default_format)
- retrieved_branch = store.pull(
- self.arbitrary_branch_id, 'pulled', default_format)
- self.assertEqual(
- set([revid, revid1, revid2]),
- set(retrieved_branch.repository.all_revision_ids()))
-
- def test_pull_doesnt_bring_backup_directories(self):
- # If the branch has been upgraded in the branch store, `pull` does not
- # copy the backup.bzr directory to `target_path`, just the .bzr
- # directory.
- store = self.makeBranchStore()
- tree = create_branch_with_one_revision('original')
- store.push(self.arbitrary_branch_id, tree.branch, default_format)
- t = get_transport(store._getMirrorURL(self.arbitrary_branch_id))
- t.mkdir('backup.bzr')
- retrieved_branch = store.pull(
- self.arbitrary_branch_id, 'pulled', default_format,
- needs_tree=False)
- self.assertEqual(
- ['.bzr'], retrieved_branch.bzrdir.root_transport.list_dir('.'))
-
-
-class TestImportDataStore(WorkerTest):
- """Tests for `ImportDataStore`."""
-
- def test_fetch_returnsFalseIfNotFound(self):
- # If the requested file does not exist on the transport, fetch returns
- # False.
- filename = '%s.tar.gz' % (self.factory.getUniqueString(),)
- source_details = self.factory.makeCodeImportSourceDetails()
- store = ImportDataStore(self.get_transport(), source_details)
- ret = store.fetch(filename)
- self.assertFalse(ret)
-
- def test_fetch_doesntCreateFileIfNotFound(self):
- # If the requested file does not exist on the transport, no local file
- # is created.
- filename = '%s.tar.gz' % (self.factory.getUniqueString(),)
- source_details = self.factory.makeCodeImportSourceDetails()
- store = ImportDataStore(self.get_transport(), source_details)
- store.fetch(filename)
- self.assertFalse(os.path.exists(filename))
-
- def test_fetch_returnsTrueIfFound(self):
- # If the requested file exists on the transport, fetch returns True.
- source_details = self.factory.makeCodeImportSourceDetails()
- # That the remote name is like this is part of the interface of
- # ImportDataStore.
- remote_name = '%08x.tar.gz' % (source_details.target_id,)
- local_name = '%s.tar.gz' % (self.factory.getUniqueString(),)
- transport = self.get_transport()
- transport.put_bytes(remote_name, b'')
- store = ImportDataStore(transport, source_details)
- ret = store.fetch(local_name)
- self.assertTrue(ret)
-
- def test_fetch_retrievesFileIfFound(self):
- # If the requested file exists on the transport, fetch copies its
- # content to the filename given to fetch.
- source_details = self.factory.makeCodeImportSourceDetails()
- # That the remote name is like this is part of the interface of
- # ImportDataStore.
- remote_name = '%08x.tar.gz' % (source_details.target_id,)
- content = self.factory.getUniqueString()
- transport = self.get_transport()
- transport.put_bytes(remote_name, content)
- store = ImportDataStore(transport, source_details)
- local_name = '%s.tar.gz' % (self.factory.getUniqueString('tarball'),)
- store.fetch(local_name)
- self.assertEqual(content, open(local_name).read())
-
- def test_fetch_with_dest_transport(self):
- # The second, optional, argument to fetch is the transport in which to
- # place the retrieved file.
- source_details = self.factory.makeCodeImportSourceDetails()
- # That the remote name is like this is part of the interface of
- # ImportDataStore.
- remote_name = '%08x.tar.gz' % (source_details.target_id,)
- content = self.factory.getUniqueString()
- transport = self.get_transport()
- transport.put_bytes(remote_name, content)
- store = ImportDataStore(transport, source_details)
- local_prefix = self.factory.getUniqueString()
- self.get_transport(local_prefix).ensure_base()
- local_name = '%s.tar.gz' % (self.factory.getUniqueString(),)
- store.fetch(local_name, self.get_transport(local_prefix))
- self.assertEqual(
- content, open(os.path.join(local_prefix, local_name)).read())
-
- def test_put_copiesFileToTransport(self):
- # Put copies the content of the passed filename to the remote
- # transport.
- local_name = '%s.tar.gz' % (self.factory.getUniqueString(),)
- source_details = self.factory.makeCodeImportSourceDetails()
- content = self.factory.getUniqueString()
- get_transport('.').put_bytes(local_name, content)
- transport = self.get_transport()
- store = ImportDataStore(transport, source_details)
- store.put(local_name)
- # That the remote name is like this is part of the interface of
- # ImportDataStore.
- remote_name = '%08x.tar.gz' % (source_details.target_id,)
- self.assertEqual(content, transport.get_bytes(remote_name))
-
- def test_put_ensures_base(self):
- # Put ensures that the directory pointed to by the transport exists.
- local_name = '%s.tar.gz' % (self.factory.getUniqueString(),)
- subdir_name = self.factory.getUniqueString()
- source_details = self.factory.makeCodeImportSourceDetails()
- get_transport('.').put_bytes(local_name, b'')
- transport = self.get_transport()
- store = ImportDataStore(transport.clone(subdir_name), source_details)
- store.put(local_name)
- self.assertTrue(transport.has(subdir_name))
-
- def test_put_with_source_transport(self):
- # The second, optional, argument to put is the transport from which to
- # read the retrieved file.
- local_prefix = self.factory.getUniqueString()
- local_name = '%s.tar.gz' % (self.factory.getUniqueString(),)
- source_details = self.factory.makeCodeImportSourceDetails()
- content = self.factory.getUniqueString()
- os.mkdir(local_prefix)
- get_transport(local_prefix).put_bytes(local_name, content)
- transport = self.get_transport()
- store = ImportDataStore(transport, source_details)
- store.put(local_name, self.get_transport(local_prefix))
- # That the remote name is like this is part of the interface of
- # ImportDataStore.
- remote_name = '%08x.tar.gz' % (source_details.target_id,)
- self.assertEqual(content, transport.get_bytes(remote_name))
-
-
-class MockForeignWorkingTree:
- """Working tree that records calls to checkout and update."""
-
- def __init__(self, local_path):
- self.local_path = local_path
- self.log = []
-
- def checkout(self):
- self.log.append('checkout')
-
- def update(self):
- self.log.append('update')
-
-
-class TestForeignTreeStore(WorkerTest):
- """Tests for the `ForeignTreeStore` object."""
-
- def assertCheckedOut(self, tree):
- self.assertEqual(['checkout'], tree.log)
-
- def assertUpdated(self, tree):
- self.assertEqual(['update'], tree.log)
-
- def setUp(self):
- """Set up a code import for an SVN working tree."""
- super(TestForeignTreeStore, self).setUp()
- self.temp_dir = self.makeTemporaryDirectory()
-
- def makeForeignTreeStore(self, source_details=None):
- """Make a foreign tree store.
-
- The store is in a different directory to the local working directory.
- """
- def _getForeignTree(target_path):
- return MockForeignWorkingTree(target_path)
- fake_it = False
- if source_details is None:
- fake_it = True
- source_details = self.factory.makeCodeImportSourceDetails()
- transport = self.get_transport('remote')
- store = ForeignTreeStore(ImportDataStore(transport, source_details))
- if fake_it:
- store._getForeignTree = _getForeignTree
- return store
-
- def test_getForeignTreeCVS(self):
- # _getForeignTree() returns a CVS working tree for CVS code imports.
- source_details = self.factory.makeCodeImportSourceDetails(
- rcstype='cvs')
- store = self.makeForeignTreeStore(source_details)
- working_tree = store._getForeignTree('path')
- self.assertIsSameRealPath(working_tree.local_path, 'path')
- self.assertEqual(working_tree.root, source_details.cvs_root)
- self.assertEqual(working_tree.module, source_details.cvs_module)
-
- def test_getNewWorkingTree(self):
- # If the foreign tree store doesn't have an archive of the foreign
- # tree, then fetching the tree actually pulls in from the original
- # site.
- store = self.makeForeignTreeStore()
- tree = store.fetchFromSource(self.temp_dir)
- self.assertCheckedOut(tree)
-
- def test_archiveTree(self):
- # Once we have a foreign working tree, we can archive it so that we
- # can retrieve it more reliably in the future.
- store = self.makeForeignTreeStore()
- foreign_tree = store.fetchFromSource(self.temp_dir)
- store.archive(foreign_tree)
- transport = store.import_data_store._transport
- source_details = store.import_data_store.source_details
- self.assertTrue(
- transport.has('%08x.tar.gz' % source_details.target_id),
- "Couldn't find '%08x.tar.gz'" % source_details.target_id)
-
- def test_fetchFromArchiveFailure(self):
- # If a tree has not been archived yet, but we try to retrieve it from
- # the archive, we get a NoSuchFile error.
- store = self.makeForeignTreeStore()
- self.assertRaises(
- NoSuchFile,
- store.fetchFromArchive, self.temp_dir)
-
- def test_fetchFromArchive(self):
- # After archiving a tree, we can retrieve it from the store -- the
- # tarball gets downloaded and extracted.
- store = self.makeForeignTreeStore()
- foreign_tree = store.fetchFromSource(self.temp_dir)
- store.archive(foreign_tree)
- new_temp_dir = self.makeTemporaryDirectory()
- foreign_tree2 = store.fetchFromArchive(new_temp_dir)
- self.assertEqual(new_temp_dir, foreign_tree2.local_path)
- self.assertDirectoryTreesEqual(self.temp_dir, new_temp_dir)
-
- def test_fetchFromArchiveUpdates(self):
- # The local working tree is updated with changes from the remote
- # branch after it has been fetched from the archive.
- store = self.makeForeignTreeStore()
- foreign_tree = store.fetchFromSource(self.temp_dir)
- store.archive(foreign_tree)
- new_temp_dir = self.makeTemporaryDirectory()
- foreign_tree2 = store.fetchFromArchive(new_temp_dir)
- self.assertUpdated(foreign_tree2)
-
-
-class TestWorkerCore(WorkerTest):
- """Tests for the core (VCS-independent) part of the code import worker."""
-
- def setUp(self):
- WorkerTest.setUp(self)
- self.source_details = self.factory.makeCodeImportSourceDetails()
-
- def makeBazaarBranchStore(self):
- """Make a Bazaar branch store."""
- return BazaarBranchStore(self.get_transport('bazaar_branches'))
-
- def makeImportWorker(self):
- """Make an ImportWorker."""
- return ToBzrImportWorker(
- self.source_details, self.get_transport('import_data'),
- self.makeBazaarBranchStore(), logging.getLogger("silent"),
- AcceptAnythingPolicy())
-
- def test_construct(self):
- # When we construct an ImportWorker, it has a CodeImportSourceDetails
- # object.
- worker = self.makeImportWorker()
- self.assertEqual(self.source_details, worker.source_details)
-
- def test_getBazaarWorkingBranchMakesEmptyBranch(self):
- # getBazaarBranch returns a brand-new working tree for an initial
- # import.
- worker = self.makeImportWorker()
- bzr_branch = worker.getBazaarBranch()
- self.assertEqual(0, bzr_branch.revno())
-
- def test_bazaarBranchLocation(self):
- # getBazaarBranch makes the working tree under the current working
- # directory.
- worker = self.makeImportWorker()
- bzr_branch = worker.getBazaarBranch()
- self.assertIsSameRealPath(
- os.path.abspath(worker.BZR_BRANCH_PATH),
- os.path.abspath(local_path_from_url(bzr_branch.base)))
-
-
-class TestCSCVSWorker(WorkerTest):
- """Tests for methods specific to CSCVSImportWorker."""
-
- def setUp(self):
- WorkerTest.setUp(self)
- self.source_details = self.factory.makeCodeImportSourceDetails()
-
- def makeImportWorker(self):
- """Make a CSCVSImportWorker."""
- return CSCVSImportWorker(
- self.source_details, self.get_transport('import_data'), None,
- logging.getLogger("silent"), opener_policy=AcceptAnythingPolicy())
-
- def test_getForeignTree(self):
- # getForeignTree returns an object that represents the 'foreign'
- # branch (i.e. a CVS or Subversion branch).
- worker = self.makeImportWorker()
-
- def _getForeignTree(target_path):
- return MockForeignWorkingTree(target_path)
-
- worker.foreign_tree_store._getForeignTree = _getForeignTree
- working_tree = worker.getForeignTree()
- self.assertIsSameRealPath(
- os.path.abspath(worker.FOREIGN_WORKING_TREE_PATH),
- working_tree.local_path)
-
-
-class TestGitImportWorker(WorkerTest):
- """Test for behaviour particular to `GitImportWorker`."""
-
- def makeBazaarBranchStore(self):
- """Make a Bazaar branch store."""
- t = self.get_transport('bazaar_branches')
- t.ensure_base()
- return BazaarBranchStore(self.get_transport('bazaar_branches'))
-
- def makeImportWorker(self):
- """Make an GitImportWorker."""
- source_details = self.factory.makeCodeImportSourceDetails()
- return GitImportWorker(
- source_details, self.get_transport('import_data'),
- self.makeBazaarBranchStore(), logging.getLogger("silent"),
- opener_policy=AcceptAnythingPolicy())
-
- def test_pushBazaarBranch_saves_git_cache(self):
- # GitImportWorker.pushBazaarBranch saves a tarball of the git cache
- # from the tree's repository in the worker's ImportDataStore.
- content = self.factory.getUniqueString()
- branch = self.make_branch('.')
- branch.repository._transport.mkdir('git')
- branch.repository._transport.put_bytes('git/cache', content)
- import_worker = self.makeImportWorker()
- import_worker.pushBazaarBranch(branch)
- import_worker.import_data_store.fetch('git-cache.tar.gz')
- extract_tarball('git-cache.tar.gz', '.')
- self.assertEqual(content, open('cache').read())
-
- def test_getBazaarBranch_fetches_legacy_git_db(self):
- # GitImportWorker.getBazaarBranch fetches the legacy git.db file, if
- # present, from the worker's ImportDataStore into the tree's
- # repository.
- import_worker = self.makeImportWorker()
- # Store the git.db file in the store.
- content = self.factory.getUniqueString()
- open('git.db', 'w').write(content)
- import_worker.import_data_store.put('git.db')
- # Make sure there's a Bazaar branch in the branch store.
- branch = self.make_branch('branch')
- ToBzrImportWorker.pushBazaarBranch(import_worker, branch)
- # Finally, fetching the tree gets the git.db file too.
- branch = import_worker.getBazaarBranch()
- self.assertEqual(
- content, branch.repository._transport.get('git.db').read())
-
- def test_getBazaarBranch_fetches_git_cache(self):
- # GitImportWorker.getBazaarBranch fetches the tarball of the git
- # cache from the worker's ImportDataStore and expands it into the
- # tree's repository.
- import_worker = self.makeImportWorker()
- # Store a tarred-up cache in the store.x
- content = self.factory.getUniqueString()
- os.mkdir('cache')
- open('cache/git-cache', 'w').write(content)
- create_tarball('cache', 'git-cache.tar.gz')
- import_worker.import_data_store.put('git-cache.tar.gz')
- # Make sure there's a Bazaar branch in the branch store.
- branch = self.make_branch('branch')
- ToBzrImportWorker.pushBazaarBranch(import_worker, branch)
- # Finally, fetching the tree gets the git.db file too.
- new_branch = import_worker.getBazaarBranch()
- self.assertEqual(
- content,
- new_branch.repository._transport.get('git/git-cache').read())
-
-
-def clean_up_default_stores_for_import(target_id):
- """Clean up the default branch and foreign tree stores for an import.
-
- This checks for an existing branch and/or other import data corresponding
- to the passed in import and deletes them if they are found.
-
- If there are tarballs or branches in the default stores that might
- conflict with working on our job, life gets very, very confusing.
-
- :source_details: A `CodeImportSourceDetails` describing the import.
- """
- tree_transport = get_transport(config.codeimport.foreign_tree_store)
- prefix = '%08x' % target_id
- if tree_transport.has('.'):
- for filename in tree_transport.list_dir('.'):
- if filename.startswith(prefix):
- tree_transport.delete(filename)
- branchstore = get_default_bazaar_branch_store()
- branch_name = '%08x' % target_id
- if branchstore.transport.has(branch_name):
- branchstore.transport.delete_tree(branch_name)
-
-
-class TestActualImportMixin:
- """Mixin for tests that check the actual importing."""
-
- def setUpImport(self):
- """Set up the objects required for an import.
-
- This means a BazaarBranchStore, CodeImport and a CodeImportJob.
- """
- self.bazaar_store = BazaarBranchStore(
- self.get_transport('bazaar_store'))
- self.foreign_commit_count = 0
-
- def makeImportWorker(self, source_details, opener_policy):
- """Make a new `ImportWorker`.
-
- Override this in your subclass.
- """
- raise NotImplementedError(
- "Override this with a VCS-specific implementation.")
-
- def makeForeignCommit(self, source_details):
- """Commit a revision to the repo described by `self.source_details`.
-
- Increment `self.foreign_commit_count` as appropriate.
-
- Override this in your subclass.
- """
- raise NotImplementedError(
- "Override this with a VCS-specific implementation.")
-
- def makeWorkerArguments(self, module_name, files, stacked_on_url=None):
- """Make a list of worker arguments pointing to a real repository.
-
- This should set `self.foreign_commit_count` to an appropriate value.
-
- Override this in your subclass.
- """
- raise NotImplementedError(
- "Override this with a VCS-specific implementation.")
-
- def makeSourceDetails(self, module_name, files, stacked_on_url=None):
- """Make a `CodeImportSourceDetails` pointing to a real repository."""
- return CodeImportSourceDetails.fromArguments(self.makeWorkerArguments(
- module_name, files, stacked_on_url=stacked_on_url))
-
- def getStoredBazaarBranch(self, worker):
- """Get the Bazaar branch 'worker' stored into its BazaarBranchStore.
- """
- branch_url = worker.bazaar_branch_store._getMirrorURL(
- worker.source_details.target_id)
- return Branch.open(branch_url)
-
- def clearCaches(self):
- """Clear any caches between worker runs, if necessary.
-
- Override this in your subclass if you need it.
- """
-
- def test_exclude_hosts(self):
- details = CodeImportSourceDetails.fromArguments(
- self.makeWorkerArguments(
- 'trunk', [('README', b'Original contents')]) +
- ['--exclude-host', 'bad.example.com',
- '--exclude-host', 'worse.example.com'])
- self.assertEqual(
- ['bad.example.com', 'worse.example.com'], details.exclude_hosts)
-
- def test_import(self):
- # Running the worker on a branch that hasn't been imported yet imports
- # the branch.
- worker = self.makeImportWorker(self.makeSourceDetails(
- 'trunk', [('README', b'Original contents')]),
- opener_policy=AcceptAnythingPolicy())
- worker.run()
- branch = self.getStoredBazaarBranch(worker)
- self.assertEqual(self.foreign_commit_count, branch.revno())
-
- def test_sync(self):
- # Do an import.
- worker = self.makeImportWorker(self.makeSourceDetails(
- 'trunk', [('README', b'Original contents')]),
- opener_policy=AcceptAnythingPolicy())
- worker.run()
- branch = self.getStoredBazaarBranch(worker)
- self.assertEqual(self.foreign_commit_count, branch.revno())
-
- # Change the remote branch.
- self.makeForeignCommit(worker.source_details)
-
- # Run the same worker again.
- self.clearCaches()
- worker.run()
-
- # Check that the new revisions are in the Bazaar branch.
- branch = self.getStoredBazaarBranch(worker)
- self.assertEqual(self.foreign_commit_count, branch.revno())
-
- def test_import_script(self):
- # Like test_import, but using the code-import-worker.py script
- # to perform the import.
- arguments = self.makeWorkerArguments(
- 'trunk', [('README', b'Original contents')])
- source_details = CodeImportSourceDetails.fromArguments(arguments)
-
- clean_up_default_stores_for_import(source_details.target_id)
-
- script_path = os.path.join(
- config.root, 'scripts', 'code-import-worker.py')
- output = tempfile.TemporaryFile()
- retcode = subprocess.call(
- [script_path, '--access-policy=anything', '--'] + arguments,
- stderr=output, stdout=output)
- self.assertEqual(retcode, 0)
-
- # It's important that the subprocess writes to stdout or stderr
- # regularly to let the worker monitor know it's still alive. That
- # specifically is hard to test, but we can at least test that the
- # process produced _some_ output.
- output.seek(0, 2)
- self.assertPositive(output.tell())
-
- self.addCleanup(
- lambda: clean_up_default_stores_for_import(
- source_details.target_id))
-
- tree_path = tempfile.mkdtemp()
- self.addCleanup(lambda: shutil.rmtree(tree_path))
-
- branch_url = get_default_bazaar_branch_store()._getMirrorURL(
- source_details.target_id)
- branch = Branch.open(branch_url)
-
- self.assertEqual(self.foreign_commit_count, branch.revno())
-
- def test_script_exit_codes(self):
- # After a successful import that imports revisions, the worker exits
- # with a code of CodeImportWorkerExitCode.SUCCESS. After a successful
- # import that does not import revisions, the worker exits with a code
- # of CodeImportWorkerExitCode.SUCCESS_NOCHANGE.
- arguments = self.makeWorkerArguments(
- 'trunk', [('README', b'Original contents')])
- source_details = CodeImportSourceDetails.fromArguments(arguments)
-
- clean_up_default_stores_for_import(source_details.target_id)
-
- script_path = os.path.join(
- config.root, 'scripts', 'code-import-worker.py')
- output = tempfile.TemporaryFile()
- retcode = subprocess.call(
- [script_path, '--access-policy=anything', '--'] + arguments,
- stderr=output, stdout=output)
- self.assertEqual(retcode, CodeImportWorkerExitCode.SUCCESS)
- retcode = subprocess.call(
- [script_path, '--access-policy=anything', '--'] + arguments,
- stderr=output, stdout=output)
- self.assertEqual(retcode, CodeImportWorkerExitCode.SUCCESS_NOCHANGE)
-
-
-class CSCVSActualImportMixin(TestActualImportMixin):
-
- def setUpImport(self):
- """Set up the objects required for an import.
-
- This sets up a ForeignTreeStore in addition to what
- TestActualImportMixin.setUpImport does.
- """
- TestActualImportMixin.setUpImport(self)
-
- def makeImportWorker(self, source_details, opener_policy):
- """Make a new `ImportWorker`."""
- return CSCVSImportWorker(
- source_details, self.get_transport('foreign_store'),
- self.bazaar_store, logging.getLogger(), opener_policy)
-
-
-class TestCVSImport(WorkerTest, CSCVSActualImportMixin):
- """Tests for the worker importing and syncing a CVS module."""
-
- def setUp(self):
- super(TestCVSImport, self).setUp()
- self.useFixture(FakeLogger())
- self.setUpImport()
-
- def makeForeignCommit(self, source_details):
- # If you write to a file in the same second as the previous commit,
- # CVS will not think that it has changed.
- time.sleep(1)
- repo = Repository(
- six.ensure_str(source_details.cvs_root), BufferLogger())
- repo.get(six.ensure_str(source_details.cvs_module), 'working_dir')
- wt = CVSTree('working_dir')
- self.build_tree_contents([('working_dir/README', 'New content')])
- wt.commit(log='Log message')
- self.foreign_commit_count += 1
- shutil.rmtree('working_dir')
-
- def makeWorkerArguments(self, module_name, files, stacked_on_url=None):
- """Make CVS worker arguments pointing at a real CVS repo."""
- cvs_server = CVSServer(self.makeTemporaryDirectory())
- cvs_server.start_server()
- self.addCleanup(cvs_server.stop_server)
-
- cvs_server.makeModule('trunk', [('README', 'original\n')])
-
- self.foreign_commit_count = 2
-
- return [
- str(self.factory.getUniqueInteger()), 'cvs', 'bzr',
- cvs_server.getRoot(), '--cvs-module', 'trunk',
- ]
-
-
-class SubversionImportHelpers:
- """Implementations of `makeForeignCommit` and `makeSourceDetails` for svn.
- """
-
- def makeForeignCommit(self, source_details):
- """Change the foreign tree."""
- auth = subvertpy.ra.Auth([subvertpy.ra.get_username_provider()])
- auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_USERNAME, "lptest2")
- client = subvertpy.client.Client(auth=auth)
- client.checkout(source_details.url, 'working_tree', "HEAD")
- file = open('working_tree/newfile', 'w')
- file.write('No real content\n')
- file.close()
- client.add('working_tree/newfile')
- client.log_msg_func = lambda c: 'Add a file'
- (revnum, date, author) = client.commit(['working_tree'], recurse=True)
- # CSCVS breaks on commits without an author, so make sure there
- # is one.
- self.assertIsNot(None, author)
- self.foreign_commit_count += 1
- shutil.rmtree('working_tree')
-
- def makeWorkerArguments(self, branch_name, files, stacked_on_url=None):
- """Make SVN worker arguments pointing at a real SVN repo."""
- svn_server = SubversionServer(self.makeTemporaryDirectory())
- svn_server.start_server()
- self.addCleanup(svn_server.stop_server)
-
- svn_branch_url = svn_server.makeBranch(branch_name, files)
- svn_branch_url = svn_branch_url.replace('://localhost/', ':///')
- self.foreign_commit_count = 2
- arguments = [
- str(self.factory.getUniqueInteger()), self.rcstype, 'bzr',
- svn_branch_url,
- ]
- if stacked_on_url is not None:
- arguments.extend(['--stacked-on', stacked_on_url])
- return arguments
-
-
-class PullingImportWorkerTests:
- """Tests for the PullingImportWorker subclasses."""
-
- def createBranchReference(self):
- """Create a pure branch reference that points to a branch.
- """
- branch = self.make_branch('branch')
- t = get_transport(self.get_url('.'))
- t.mkdir('reference')
- a_bzrdir = BzrDir.create(self.get_url('reference'))
- BranchReferenceFormat().initialize(a_bzrdir, target_branch=branch)
- return a_bzrdir.root_transport.base, branch.base
-
- def test_reject_branch_reference(self):
- # URLs that point to other branch types than that expected by the
- # import should be rejected.
- args = {'rcstype': self.rcstype}
- reference_url, target_url = self.createBranchReference()
- if self.rcstype in ('git', 'bzr-svn'):
- args['url'] = reference_url
- else:
- raise AssertionError("unexpected rcs_type %r" % self.rcstype)
- source_details = self.factory.makeCodeImportSourceDetails(**args)
- worker = self.makeImportWorker(source_details,
- opener_policy=AcceptAnythingPolicy())
- self.assertEqual(
- CodeImportWorkerExitCode.FAILURE_INVALID, worker.run())
-
- def test_invalid(self):
- # If there is no branch in the target URL, exit with FAILURE_INVALID
- worker = self.makeImportWorker(
- self.factory.makeCodeImportSourceDetails(
- rcstype=self.rcstype,
- url="http://localhost/path/non/existant";),
- opener_policy=AcceptAnythingPolicy())
- self.assertEqual(
- CodeImportWorkerExitCode.FAILURE_INVALID, worker.run())
-
- def test_forbidden(self):
- # If the branch specified is using an invalid scheme, exit with
- # FAILURE_FORBIDDEN
- worker = self.makeImportWorker(
- self.factory.makeCodeImportSourceDetails(
- rcstype=self.rcstype, url="file:///local/path"),
- opener_policy=CodeImportBranchOpenPolicy("bzr", "bzr"))
- self.assertEqual(
- CodeImportWorkerExitCode.FAILURE_FORBIDDEN, worker.run())
-
- def test_unsupported_feature(self):
- # If there is no branch in the target URL, exit with FAILURE_INVALID
- worker = self.makeImportWorker(self.makeSourceDetails(
- 'trunk', [('bzr\\doesnt\\support\\this', b'Original contents')]),
- opener_policy=AcceptAnythingPolicy())
- self.assertEqual(
- CodeImportWorkerExitCode.FAILURE_UNSUPPORTED_FEATURE,
- worker.run())
-
- def test_partial(self):
- # Only config.codeimport.revisions_import_limit will be imported
- # in a given run.
- worker = self.makeImportWorker(self.makeSourceDetails(
- 'trunk', [('README', b'Original contents')]),
- opener_policy=AcceptAnythingPolicy())
- self.makeForeignCommit(worker.source_details)
- self.assertTrue(self.foreign_commit_count > 1)
- import_limit = self.foreign_commit_count - 1
- self.pushConfig(
- 'codeimport',
- git_revisions_import_limit=import_limit,
- svn_revisions_import_limit=import_limit)
- self.assertEqual(
- CodeImportWorkerExitCode.SUCCESS_PARTIAL, worker.run())
- self.clearCaches()
- self.assertEqual(
- CodeImportWorkerExitCode.SUCCESS, worker.run())
-
- def test_stacked(self):
- stacked_on = self.make_branch('stacked-on')
- source_details = self.makeSourceDetails(
- 'trunk', [('README', b'Original contents')],
- stacked_on_url=stacked_on.base)
- stacked_on.fetch(Branch.open(source_details.url))
- base_rev_count = self.foreign_commit_count
- # There should only be one revision there, the other
- # one is in the stacked-on repository.
- self.addCleanup(stacked_on.lock_read().unlock)
- self.assertEqual(
- base_rev_count,
- len(stacked_on.repository.revisions.keys()))
- worker = self.makeImportWorker(
- source_details,
- opener_policy=AcceptAnythingPolicy())
- self.makeForeignCommit(source_details)
- self.assertEqual(
- CodeImportWorkerExitCode.SUCCESS, worker.run())
- branch = self.getStoredBazaarBranch(worker)
- self.assertEqual(
- base_rev_count,
- len(stacked_on.repository.revisions.keys()))
- # There should only be one revision there, the other
- # one is in the stacked-on repository.
- self.addCleanup(branch.lock_read().unlock)
- self.assertEqual(1,
- len(branch.repository.revisions.without_fallbacks().keys()))
- self.assertEqual(stacked_on.base, branch.get_stacked_on_url())
-
-
-class TestGitImport(WorkerTest, TestActualImportMixin,
- PullingImportWorkerTests):
-
- rcstype = 'git'
-
- def setUp(self):
- super(TestGitImport, self).setUp()
- self.setUpImport()
-
- def tearDown(self):
- self.clearCaches()
- super(TestGitImport, self).tearDown()
-
- def clearCaches(self):
- """Clear bzr-git's cache of sqlite connections.
-
- This is rather obscure: different test runs tend to re-use the same
- paths on disk, which confuses bzr-git as it keeps a cache that maps
- paths to database connections, which happily returns the connection
- that corresponds to a path that no longer exists.
- """
- from bzrlib.plugins.git.cache import mapdbs
- mapdbs().clear()
-
- def makeImportWorker(self, source_details, opener_policy):
- """Make a new `ImportWorker`."""
- return GitImportWorker(
- source_details, self.get_transport('import_data'),
- self.bazaar_store, logging.getLogger(),
- opener_policy=opener_policy)
-
- def makeForeignCommit(self, source_details, message=None, ref="HEAD"):
- """Change the foreign tree, generating exactly one commit."""
- repo = GitRepo(local_path_from_url(source_details.url))
- if message is None:
- message = self.factory.getUniqueString()
- repo.do_commit(message=message,
- committer="Joe Random Hacker <joe@xxxxxxxxxxx>", ref=ref)
- self.foreign_commit_count += 1
-
- def makeWorkerArguments(self, branch_name, files, stacked_on_url=None):
- """Make Git worker arguments pointing at a real Git repo."""
- repository_store = self.makeTemporaryDirectory()
- git_server = GitServer(repository_store)
- git_server.start_server()
- self.addCleanup(git_server.stop_server)
-
- git_server.makeRepo('source', files)
- self.foreign_commit_count = 1
-
- arguments = [
- str(self.factory.getUniqueInteger()), 'git', 'bzr',
- git_server.get_url('source'),
- ]
- if stacked_on_url is not None:
- arguments.extend(['--stacked-on', stacked_on_url])
- return arguments
-
- def test_non_master(self):
- # non-master branches can be specified in the import URL.
- source_details = self.makeSourceDetails(
- 'trunk', [('README', b'Original contents')])
- self.makeForeignCommit(source_details, ref="refs/heads/other",
- message="Message for other")
- self.makeForeignCommit(source_details, ref="refs/heads/master",
- message="Message for master")
- source_details.url = urlutils.join_segment_parameters(
- source_details.url, {"branch": "other"})
- source_transport = get_transport_from_url(source_details.url)
- self.assertEqual(
- {"branch": "other"},
- source_transport.get_segment_parameters())
- worker = self.makeImportWorker(source_details,
- opener_policy=AcceptAnythingPolicy())
- self.assertTrue(self.foreign_commit_count > 1)
- self.assertEqual(
- CodeImportWorkerExitCode.SUCCESS, worker.run())
- branch = worker.getBazaarBranch()
- lastrev = branch.repository.get_revision(branch.last_revision())
- self.assertEqual(lastrev.message, "Message for other")
-
-
-class TestBzrSvnImport(WorkerTest, SubversionImportHelpers,
- TestActualImportMixin, PullingImportWorkerTests):
-
- rcstype = 'bzr-svn'
-
- def setUp(self):
- super(TestBzrSvnImport, self).setUp()
- self.setUpImport()
-
- def makeImportWorker(self, source_details, opener_policy):
- """Make a new `ImportWorker`."""
- return BzrSvnImportWorker(
- source_details, self.get_transport('import_data'),
- self.bazaar_store, logging.getLogger(),
- opener_policy=opener_policy)
-
- def test_pushBazaarBranch_saves_bzr_svn_cache(self):
- # BzrSvnImportWorker.pushBazaarBranch saves a tarball of the bzr-svn
- # cache in the worker's ImportDataStore.
- from bzrlib.plugins.svn.cache import get_cache
- worker = self.makeImportWorker(self.makeSourceDetails(
- 'trunk', [('README', b'Original contents')]),
- opener_policy=AcceptAnythingPolicy())
- uuid = subvertpy.ra.RemoteAccess(worker.source_details.url).get_uuid()
- cache_dir = get_cache(uuid).create_cache_dir()
- cache_dir_contents = os.listdir(cache_dir)
- self.assertNotEqual([], cache_dir_contents)
- opener = BranchOpener(worker._opener_policy, worker.probers)
- remote_branch = opener.open(six.ensure_str(worker.source_details.url))
- worker.pushBazaarBranch(
- self.make_branch('.'), remote_branch=remote_branch)
- worker.import_data_store.fetch('svn-cache.tar.gz')
- extract_tarball('svn-cache.tar.gz', '.')
- self.assertContentEqual(cache_dir_contents, os.listdir(uuid))
-
- def test_getBazaarBranch_fetches_bzr_svn_cache(self):
- # BzrSvnImportWorker.getBazaarBranch fetches the tarball of the
- # bzr-svn cache from the worker's ImportDataStore and expands it
- # into the appropriate cache directory.
- from bzrlib.plugins.svn.cache import get_cache
- worker = self.makeImportWorker(self.makeSourceDetails(
- 'trunk', [('README', b'Original contents')]),
- opener_policy=AcceptAnythingPolicy())
- # Store a tarred-up cache in the store.
- content = self.factory.getUniqueString()
- uuid = str(uuid4())
- os.makedirs(os.path.join('cache', uuid))
- with open(os.path.join('cache', uuid, 'svn-cache'), 'w') as cache_file:
- cache_file.write(content)
- create_tarball('cache', 'svn-cache.tar.gz', filenames=[uuid])
- worker.import_data_store.put('svn-cache.tar.gz')
- # Make sure there's a Bazaar branch in the branch store.
- branch = self.make_branch('branch')
- ToBzrImportWorker.pushBazaarBranch(worker, branch)
- # Finally, fetching the tree gets the cache too.
- worker.getBazaarBranch()
- cache_dir = get_cache(uuid).create_cache_dir()
- with open(os.path.join(cache_dir, 'svn-cache')) as cache_file:
- self.assertEqual(content, cache_file.read())
-
-
-class TestBzrImport(WorkerTest, TestActualImportMixin,
- PullingImportWorkerTests):
-
- rcstype = 'bzr'
-
- def setUp(self):
- super(TestBzrImport, self).setUp()
- self.setUpImport()
-
- def makeImportWorker(self, source_details, opener_policy):
- """Make a new `ImportWorker`."""
- return BzrImportWorker(
- source_details, self.get_transport('import_data'),
- self.bazaar_store, logging.getLogger(), opener_policy)
-
- def makeForeignCommit(self, source_details):
- """Change the foreign tree, generating exactly one commit."""
- branch = Branch.open(source_details.url)
- builder = BranchBuilder(branch=branch)
- builder.build_commit(message=self.factory.getUniqueString(),
- committer="Joe Random Hacker <joe@xxxxxxxxxxx>")
- self.foreign_commit_count += 1
-
- def makeWorkerArguments(self, branch_name, files, stacked_on_url=None):
- """Make Bzr worker arguments pointing at a real Bzr repo."""
- repository_path = self.makeTemporaryDirectory()
- bzr_server = BzrServer(repository_path)
- bzr_server.start_server()
- self.addCleanup(bzr_server.stop_server)
-
- bzr_server.makeRepo(files)
- self.foreign_commit_count = 1
-
- arguments = [
- str(self.factory.getUniqueInteger()), 'bzr', 'bzr',
- bzr_server.get_url(),
- ]
- if stacked_on_url is not None:
- arguments.extend(['--stacked-on', stacked_on_url])
- return arguments
-
- def test_partial(self):
- self.skipTest(
- "Partial fetching is not supported for native bzr branches "
- "at the moment.")
-
- def test_unsupported_feature(self):
- self.skipTest("All Bazaar features are supported by Bazaar.")
-
- def test_reject_branch_reference(self):
- # Branch references are allowed in the BzrImporter, but their URL
- # should be checked.
- reference_url, target_url = self.createBranchReference()
- source_details = self.factory.makeCodeImportSourceDetails(
- url=reference_url, rcstype='bzr')
- policy = _BlacklistPolicy(True, [target_url])
- worker = self.makeImportWorker(source_details, opener_policy=policy)
- self.assertEqual(
- CodeImportWorkerExitCode.FAILURE_FORBIDDEN, worker.run())
-
- def test_support_branch_reference(self):
- # Branch references are allowed in the BzrImporter.
- reference_url, target_url = self.createBranchReference()
- target_branch = Branch.open(target_url)
- builder = BranchBuilder(branch=target_branch)
- builder.build_commit(message=self.factory.getUniqueString(),
- committer="Some Random Hacker <jane@xxxxxxxxxxx>")
- source_details = self.factory.makeCodeImportSourceDetails(
- url=reference_url, rcstype='bzr')
- worker = self.makeImportWorker(source_details,
- opener_policy=AcceptAnythingPolicy())
- self.assertEqual(
- CodeImportWorkerExitCode.SUCCESS, worker.run())
- branch = self.getStoredBazaarBranch(worker)
- self.assertEqual(1, branch.revno())
- self.assertEqual(
- "Some Random Hacker <jane@xxxxxxxxxxx>",
- branch.repository.get_revision(branch.last_revision()).committer)
-
-
-class TestGitToGitImportWorker(TestCase):
-
- def makeWorkerArguments(self):
- """Make Git worker arguments, pointing at a fake URL for now."""
- return [
- 'git-unique-name', 'git', 'git',
- self.factory.getUniqueURL(scheme='git'),
- '--macaroon', Macaroon().serialize(),
- ]
-
- def test_throttleProgress(self):
- source_details = CodeImportSourceDetails.fromArguments(
- self.makeWorkerArguments())
- logger = BufferLogger()
- worker = GitToGitImportWorker(
- source_details, logger, AcceptAnythingPolicy())
- read_fd, write_fd = os.pipe()
- pid = os.fork()
- if pid == 0: # child
- os.close(read_fd)
- with os.fdopen(write_fd, "wb") as write:
- write.write(b"Starting\n")
- for i in range(50):
- time.sleep(0.1)
- write.write(("%d ...\r" % i).encode("UTF-8"))
- if (i % 10) == 9:
- write.write(
- ("Interval %d\n" % (i // 10)).encode("UTF-8"))
- write.write(b"Finishing\n")
- os._exit(0)
- else: # parent
- os.close(write_fd)
- with os.fdopen(read_fd, "rb") as read:
- lines = list(worker._throttleProgress(read, timeout=0.5))
- os.waitpid(pid, 0)
- # Matching the exact sequence of lines would be too brittle, but
- # we require some things to be true:
- # All the non-progress lines must be present, in the right
- # order.
- self.assertEqual(
- [u"Starting\n", u"Interval 0\n", u"Interval 1\n",
- u"Interval 2\n", u"Interval 3\n", u"Interval 4\n",
- u"Finishing\n"],
- [line for line in lines if not line.endswith(u"\r")])
- # No more than 15 progress lines may be present (allowing some
- # slack for the child process being slow).
- progress_lines = [line for line in lines if line.endswith(u"\r")]
- self.assertThat(len(progress_lines), LessThan(16))
- # All the progress lines immediately before interval markers
- # must be present.
- self.assertThat(
- progress_lines,
- ContainsAll([u"%d ...\r" % i for i in (9, 19, 29, 39, 49)]))
-
-
-class CodeImportBranchOpenPolicyTests(TestCase):
-
- def setUp(self):
- super(CodeImportBranchOpenPolicyTests, self).setUp()
- self.policy = CodeImportBranchOpenPolicy("bzr", "bzr")
-
- def test_follows_references(self):
- self.assertEqual(True, self.policy.should_follow_references())
-
- def assertBadUrl(self, url):
- self.assertRaises(BadUrl, self.policy.check_one_url, url)
-
- def assertGoodUrl(self, url):
- self.policy.check_one_url(url)
-
- def test_check_one_url(self):
- self.assertBadUrl("sftp://somehost/";)
- self.assertBadUrl("/etc/passwd")
- self.assertBadUrl("file:///etc/passwd")
- self.assertBadUrl("bzr+ssh://devpad/")
- self.assertBadUrl("bzr+ssh://devpad/")
- self.assertBadUrl("unknown-scheme://devpad/")
- self.assertGoodUrl("http://svn.example/branches/trunk";)
- self.assertGoodUrl("http://user:password@svn.example/branches/trunk";)
- self.assertBadUrl("svn+ssh://svn.example.com/bla")
- self.assertGoodUrl("bzr://bzr.example.com/somebzrurl/")
-
- def test_check_one_url_git_to_bzr(self):
- self.policy = CodeImportBranchOpenPolicy("git", "bzr")
- self.assertBadUrl("/etc/passwd")
- self.assertBadUrl("file:///etc/passwd")
- self.assertBadUrl("unknown-scheme://devpad/")
- self.assertGoodUrl("git://git.example.com/repo")
-
- def test_check_one_url_git_to_git(self):
- self.policy = CodeImportBranchOpenPolicy("git", "git")
- self.assertBadUrl("/etc/passwd")
- self.assertBadUrl("file:///etc/passwd")
- self.assertBadUrl("unknown-scheme://devpad/")
- self.assertGoodUrl("git://git.example.com/repo")
-
- def test_check_one_url_exclude_hosts(self):
- self.policy = CodeImportBranchOpenPolicy(
- "bzr", "bzr",
- exclude_hosts=["bad.example.com", "worse.example.com"])
- self.assertGoodUrl("git://good.example.com/repo")
- self.assertBadUrl("git://bad.example.com/repo")
- self.assertBadUrl("git://worse.example.com/repo")
-
-
-class RedirectTests(http_utils.TestCaseWithRedirectedWebserver, TestCase):
-
- layer = ForeignBranchPluginLayer
-
- def setUp(self):
- http_utils.TestCaseWithRedirectedWebserver.setUp(self)
- self.disable_directory_isolation()
- BranchOpener.install_hook()
- tree = self.make_branch_and_tree('.')
- self.revid = tree.commit("A commit")
- self.bazaar_store = BazaarBranchStore(
- self.get_transport('bazaar_store'))
-
- def makeImportWorker(self, url, opener_policy):
- """Make a new `ImportWorker`."""
- source_details = self.factory.makeCodeImportSourceDetails(
- rcstype='bzr', url=url)
- return BzrImportWorker(
- source_details, self.get_transport('import_data'),
- self.bazaar_store, logging.getLogger(), opener_policy)
-
- def test_follow_redirect(self):
- worker = self.makeImportWorker(
- self.get_old_url(), AcceptAnythingPolicy())
- self.assertEqual(
- CodeImportWorkerExitCode.SUCCESS, worker.run())
- branch_url = self.bazaar_store._getMirrorURL(
- worker.source_details.target_id)
- branch = Branch.open(branch_url)
- self.assertEqual(self.revid, branch.last_revision())
-
- def test_redirect_to_forbidden_url(self):
- class NewUrlBlacklistPolicy(BranchOpenPolicy):
-
- def __init__(self, new_url):
- self.new_url = new_url
-
- def should_follow_references(self):
- return True
-
- def check_one_url(self, url):
- if url.startswith(self.new_url):
- raise BadUrl(url)
-
- def transform_fallback_location(self, branch, url):
- return urlutils.join(branch.base, url), False
-
- policy = NewUrlBlacklistPolicy(self.get_new_url())
- worker = self.makeImportWorker(self.get_old_url(), policy)
- self.assertEqual(
- CodeImportWorkerExitCode.FAILURE_FORBIDDEN, worker.run())
-
- def test_too_many_redirects(self):
- # Make the server redirect to itself
- self.old_server = http_utils.HTTPServerRedirecting(
- protocol_version=self._protocol_version)
- self.old_server.redirect_to(self.old_server.host,
- self.old_server.port)
- self.old_server._url_protocol = self._url_protocol
- self.old_server.start_server()
- try:
- worker = self.makeImportWorker(
- self.old_server.get_url(), AcceptAnythingPolicy())
- finally:
- self.old_server.stop_server()
- self.assertEqual(
- CodeImportWorkerExitCode.FAILURE_INVALID, worker.run())
diff --git a/lib/lp/codehosting/codeimport/tests/test_workermonitor.py b/lib/lp/codehosting/codeimport/tests/test_workermonitor.py
deleted file mode 100644
index f60d19a..0000000
--- a/lib/lp/codehosting/codeimport/tests/test_workermonitor.py
+++ /dev/null
@@ -1,899 +0,0 @@
-# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tests for the CodeImportWorkerMonitor and related classes."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-
-import io
-import os
-import shutil
-import subprocess
-import tempfile
-from textwrap import dedent
-
-from bzrlib.branch import Branch
-from bzrlib.tests import TestCaseInTempDir
-from dulwich.repo import Repo as GitRepo
-from fixtures import MockPatchObject
-import oops_twisted
-from pymacaroons import Macaroon
-from six.moves import xmlrpc_client
-from testtools.matchers import (
- AnyMatch,
- Equals,
- IsInstance,
- MatchesListwise,
- )
-from testtools.twistedsupport import (
- assert_fails_with,
- AsynchronousDeferredRunTest,
- )
-from twisted.internet import (
- defer,
- error,
- protocol,
- reactor,
- )
-from twisted.web import (
- server,
- xmlrpc,
- )
-
-from lp.code.enums import CodeImportResultStatus
-from lp.code.tests.helpers import GitHostingFixture
-from lp.codehosting.codeimport.tests.servers import (
- BzrServer,
- CVSServer,
- GitServer,
- SubversionServer,
- )
-from lp.codehosting.codeimport.tests.test_worker import (
- clean_up_default_stores_for_import,
- )
-from lp.codehosting.codeimport.worker import (
- CodeImportWorkerExitCode,
- get_default_bazaar_branch_store,
- )
-from lp.codehosting.codeimport.workermonitor import (
- CodeImportWorkerMonitor,
- CodeImportWorkerMonitorProtocol,
- ExitQuietly,
- )
-from lp.services.config import config
-from lp.services.config.fixture import (
- ConfigFixture,
- ConfigUseFixture,
- )
-from lp.services.log.logger import BufferLogger
-from lp.services.twistedsupport import suppress_stderr
-from lp.services.twistedsupport.tests.test_processmonitor import (
- makeFailure,
- ProcessTestsMixin,
- )
-from lp.services.webapp import errorlog
-from lp.testing import TestCase
-from lp.xmlrpc.faults import NoSuchCodeImportJob
-
-
-class TestWorkerMonitorProtocol(ProcessTestsMixin, TestCase):
-
- class StubWorkerMonitor:
-
- def __init__(self):
- self.calls = []
-
- def updateHeartbeat(self, tail):
- self.calls.append(('updateHeartbeat', tail))
-
- def setUp(self):
- self.worker_monitor = self.StubWorkerMonitor()
- self.log_file = io.BytesIO()
- super(TestWorkerMonitorProtocol, self).setUp()
-
- def makeProtocol(self):
- """See `ProcessTestsMixin.makeProtocol`."""
- return CodeImportWorkerMonitorProtocol(
- self.termination_deferred, self.worker_monitor, self.log_file,
- self.clock)
-
- def test_callsUpdateHeartbeatInConnectionMade(self):
- # The protocol calls updateHeartbeat() as it is connected to the
- # process.
- # connectionMade() is called during setUp().
- self.assertEqual(
- self.worker_monitor.calls,
- [('updateHeartbeat', '')])
-
- def test_callsUpdateHeartbeatRegularly(self):
- # The protocol calls 'updateHeartbeat' on the worker_monitor every
- # config.codeimportworker.heartbeat_update_interval seconds.
- # Forget the call in connectionMade()
- self.worker_monitor.calls = []
- # Advance the simulated time a little to avoid fencepost errors.
- self.clock.advance(0.1)
- # And check that updateHeartbeat is called at the frequency we expect:
- for i in range(4):
- self.protocol.resetTimeout()
- self.assertEqual(
- self.worker_monitor.calls,
- [('updateHeartbeat', '')] * i)
- self.clock.advance(
- config.codeimportworker.heartbeat_update_interval)
-
- def test_updateHeartbeatStopsOnProcessExit(self):
- # updateHeartbeat is not called after the process has exited.
- # Forget the call in connectionMade()
- self.worker_monitor.calls = []
- self.simulateProcessExit()
- # Advance the simulated time past the time the next update is due.
- self.clock.advance(
- config.codeimportworker.heartbeat_update_interval + 1)
- # Check that updateHeartbeat was not called.
- self.assertEqual(self.worker_monitor.calls, [])
-
- def test_outReceivedWritesToLogFile(self):
- # outReceived writes the data it is passed into the log file.
- output = [b'some data\n', b'some more data\n']
- self.protocol.outReceived(output[0])
- self.assertEqual(self.log_file.getvalue(), output[0])
- self.protocol.outReceived(output[1])
- self.assertEqual(self.log_file.getvalue(), output[0] + output[1])
-
- def test_outReceivedUpdatesTail(self):
- # outReceived updates the tail of the log, currently and arbitrarily
- # defined to be the last 5 lines of the log.
- lines = ['line %d' % number for number in range(1, 7)]
- self.protocol.outReceived('\n'.join(lines[:3]) + '\n')
- self.assertEqual(
- self.protocol._tail, 'line 1\nline 2\nline 3\n')
- self.protocol.outReceived('\n'.join(lines[3:]) + '\n')
- self.assertEqual(
- self.protocol._tail, 'line 3\nline 4\nline 5\nline 6\n')
-
-
-class FakeCodeImportScheduler(xmlrpc.XMLRPC, object):
- """A fake implementation of `ICodeImportScheduler`.
-
- The constructor takes a dictionary mapping job ids to information that
- should be returned by getImportDataForJobID and the fault to return if
- getImportDataForJobID is called with a job id not in the passed-in
- dictionary, defaulting to a fault with the same code as
- NoSuchCodeImportJob (because the class of the fault is lost when you go
- through XML-RPC serialization).
- """
-
- def __init__(self, jobs_dict, no_such_job_fault=None):
- super(FakeCodeImportScheduler, self).__init__(allowNone=True)
- self.calls = []
- self.jobs_dict = jobs_dict
- if no_such_job_fault is None:
- no_such_job_fault = xmlrpc.Fault(
- faultCode=NoSuchCodeImportJob.error_code, faultString='')
- self.no_such_job_fault = no_such_job_fault
-
- def xmlrpc_getImportDataForJobID(self, job_id):
- self.calls.append(('getImportDataForJobID', job_id))
- if job_id in self.jobs_dict:
- return self.jobs_dict[job_id]
- else:
- return self.no_such_job_fault
-
- def xmlrpc_updateHeartbeat(self, job_id, log_tail):
- self.calls.append(('updateHeartbeat', job_id, log_tail))
- return 0
-
- def xmlrpc_finishJobID(self, job_id, status_name, log_file):
- self.calls.append(('finishJobID', job_id, status_name, log_file))
-
-
-class FakeCodeImportSchedulerMixin:
-
- def makeFakeCodeImportScheduler(self, jobs_dict, no_such_job_fault=None):
- """Start a `FakeCodeImportScheduler` and return its URL."""
- scheduler = FakeCodeImportScheduler(
- jobs_dict, no_such_job_fault=no_such_job_fault)
- scheduler_listener = reactor.listenTCP(0, server.Site(scheduler))
- self.addCleanup(scheduler_listener.stopListening)
- scheduler_port = scheduler_listener.getHost().port
- return scheduler, 'http://localhost:%d/' % scheduler_port
-
-
-class TestWorkerMonitorUnit(FakeCodeImportSchedulerMixin, TestCase):
- """Unit tests for most of the `CodeImportWorkerMonitor` class.
-
- We have to pay attention to the fact that several of the methods of the
- `CodeImportWorkerMonitor` class are wrapped in decorators that create and
- commit a transaction, and have to start our own transactions to check what
- they did.
- """
-
- run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=20)
-
- def makeWorkerMonitorWithJob(self, job_id=1, job_data={}):
- self.scheduler, scheduler_url = self.makeFakeCodeImportScheduler(
- {job_id: job_data})
- return CodeImportWorkerMonitor(
- job_id, BufferLogger(), xmlrpc.Proxy(scheduler_url), "anything")
-
- def makeWorkerMonitorWithoutJob(self, fault=None):
- self.scheduler, scheduler_url = self.makeFakeCodeImportScheduler(
- {}, fault)
- return CodeImportWorkerMonitor(
- 1, BufferLogger(), xmlrpc.Proxy(scheduler_url), None)
-
- def test_getWorkerArguments(self):
- # getWorkerArguments returns a deferred that fires with the
- # 'arguments' part of what getImportDataForJobID returns.
- args = [self.factory.getUniqueString(),
- self.factory.getUniqueString()]
- data = {'arguments': args}
- worker_monitor = self.makeWorkerMonitorWithJob(1, data)
- return worker_monitor.getWorkerArguments().addCallback(
- self.assertEqual, args)
-
- def test_getWorkerArguments_job_not_found_raises_exit_quietly(self):
- # When getImportDataForJobID signals a fault indicating that
- # getWorkerArguments didn't find the supplied job, getWorkerArguments
- # translates this to an 'ExitQuietly' exception.
- worker_monitor = self.makeWorkerMonitorWithoutJob()
- return assert_fails_with(
- worker_monitor.getWorkerArguments(), ExitQuietly)
-
- def test_getWorkerArguments_endpoint_failure_raises(self):
- # When getImportDataForJobID raises an arbitrary exception, it is not
- # handled in any special way by getWorkerArguments.
- self.useFixture(MockPatchObject(
- xmlrpc_client, 'loads', side_effect=ZeroDivisionError()))
- worker_monitor = self.makeWorkerMonitorWithoutJob()
- return assert_fails_with(
- worker_monitor.getWorkerArguments(), ZeroDivisionError)
-
- def test_getWorkerArguments_arbitrary_fault_raises(self):
- # When getImportDataForJobID signals an arbitrary fault, it is not
- # handled in any special way by getWorkerArguments.
- worker_monitor = self.makeWorkerMonitorWithoutJob(
- fault=xmlrpc.Fault(1, ''))
- return assert_fails_with(
- worker_monitor.getWorkerArguments(), xmlrpc.Fault)
-
- def test_updateHeartbeat(self):
- # updateHeartbeat calls the updateHeartbeat XML-RPC method.
- log_tail = self.factory.getUniqueString()
- job_id = self.factory.getUniqueInteger()
- worker_monitor = self.makeWorkerMonitorWithJob(job_id)
-
- def check_updated_details(result):
- self.assertEqual(
- [('updateHeartbeat', job_id, log_tail)],
- self.scheduler.calls)
-
- return worker_monitor.updateHeartbeat(log_tail).addCallback(
- check_updated_details)
-
- def test_finishJob_calls_finishJobID_empty_log_file(self):
- # When the log file is empty, finishJob calls finishJobID with the
- # name of the status enum and an empty binary string.
- job_id = self.factory.getUniqueInteger()
- worker_monitor = self.makeWorkerMonitorWithJob(job_id)
- self.assertEqual(worker_monitor._log_file.tell(), 0)
-
- def check_finishJob_called(result):
- self.assertEqual(
- [('finishJobID', job_id, 'SUCCESS',
- xmlrpc_client.Binary(b''))],
- self.scheduler.calls)
-
- return worker_monitor.finishJob(
- CodeImportResultStatus.SUCCESS).addCallback(
- check_finishJob_called)
-
- def test_finishJob_sends_nonempty_file_to_scheduler(self):
- # finishJob method calls finishJobID with the contents of the log
- # file.
- job_id = self.factory.getUniqueInteger()
- log_bytes = self.factory.getUniqueBytes()
- worker_monitor = self.makeWorkerMonitorWithJob(job_id)
- worker_monitor._log_file.write(log_bytes)
-
- def check_finishJob_called(result):
- self.assertEqual(
- [('finishJobID', job_id, 'SUCCESS',
- xmlrpc_client.Binary(log_bytes))],
- self.scheduler.calls)
-
- return worker_monitor.finishJob(
- CodeImportResultStatus.SUCCESS).addCallback(
- check_finishJob_called)
-
- def patchOutFinishJob(self, worker_monitor):
- """Replace `worker_monitor.finishJob` with a `FakeMethod`-alike stub.
-
- :param worker_monitor: CodeImportWorkerMonitor to patch up.
- :return: A list of statuses that `finishJob` has been called with.
- Future calls will be appended to this list.
- """
- calls = []
-
- def finishJob(status):
- calls.append(status)
- return defer.succeed(None)
-
- worker_monitor.finishJob = finishJob
- return calls
-
- def test_callFinishJobCallsFinishJobSuccess(self):
- # callFinishJob calls finishJob with CodeImportResultStatus.SUCCESS if
- # its argument is not a Failure.
- worker_monitor = self.makeWorkerMonitorWithJob()
- calls = self.patchOutFinishJob(worker_monitor)
- worker_monitor.callFinishJob(None)
- self.assertEqual(calls, [CodeImportResultStatus.SUCCESS])
-
- @suppress_stderr
- def test_callFinishJobCallsFinishJobFailure(self):
- # callFinishJob calls finishJob with CodeImportResultStatus.FAILURE
- # and swallows the failure if its argument indicates that the
- # subprocess exited with an exit code of
- # CodeImportWorkerExitCode.FAILURE.
- worker_monitor = self.makeWorkerMonitorWithJob()
- calls = self.patchOutFinishJob(worker_monitor)
- ret = worker_monitor.callFinishJob(
- makeFailure(
- error.ProcessTerminated,
- exitCode=CodeImportWorkerExitCode.FAILURE))
- self.assertEqual(calls, [CodeImportResultStatus.FAILURE])
- # We return the deferred that callFinishJob returns -- if
- # callFinishJob did not swallow the error, this will fail the test.
- return ret
-
- def test_callFinishJobCallsFinishJobSuccessNoChange(self):
- # If the argument to callFinishJob indicates that the subprocess
- # exited with a code of CodeImportWorkerExitCode.SUCCESS_NOCHANGE, it
- # calls finishJob with a status of SUCCESS_NOCHANGE.
- worker_monitor = self.makeWorkerMonitorWithJob()
- calls = self.patchOutFinishJob(worker_monitor)
- ret = worker_monitor.callFinishJob(
- makeFailure(
- error.ProcessTerminated,
- exitCode=CodeImportWorkerExitCode.SUCCESS_NOCHANGE))
- self.assertEqual(calls, [CodeImportResultStatus.SUCCESS_NOCHANGE])
- # We return the deferred that callFinishJob returns -- if
- # callFinishJob did not swallow the error, this will fail the test.
- return ret
-
- @suppress_stderr
- def test_callFinishJobCallsFinishJobArbitraryFailure(self):
- # If the argument to callFinishJob indicates that there was some other
- # failure that had nothing to do with the subprocess, it records
- # failure.
- worker_monitor = self.makeWorkerMonitorWithJob()
- calls = self.patchOutFinishJob(worker_monitor)
- ret = worker_monitor.callFinishJob(makeFailure(RuntimeError))
- self.assertEqual(calls, [CodeImportResultStatus.FAILURE])
- # We return the deferred that callFinishJob returns -- if
- # callFinishJob did not swallow the error, this will fail the test.
- return ret
-
- def test_callFinishJobCallsFinishJobPartial(self):
- # If the argument to callFinishJob indicates that the subprocess
- # exited with a code of CodeImportWorkerExitCode.SUCCESS_PARTIAL, it
- # calls finishJob with a status of SUCCESS_PARTIAL.
- worker_monitor = self.makeWorkerMonitorWithJob()
- calls = self.patchOutFinishJob(worker_monitor)
- ret = worker_monitor.callFinishJob(
- makeFailure(
- error.ProcessTerminated,
- exitCode=CodeImportWorkerExitCode.SUCCESS_PARTIAL))
- self.assertEqual(calls, [CodeImportResultStatus.SUCCESS_PARTIAL])
- # We return the deferred that callFinishJob returns -- if
- # callFinishJob did not swallow the error, this will fail the test.
- return ret
-
- def test_callFinishJobCallsFinishJobInvalid(self):
- # If the argument to callFinishJob indicates that the subprocess
- # exited with a code of CodeImportWorkerExitCode.FAILURE_INVALID, it
- # calls finishJob with a status of FAILURE_INVALID.
- worker_monitor = self.makeWorkerMonitorWithJob()
- calls = self.patchOutFinishJob(worker_monitor)
- ret = worker_monitor.callFinishJob(
- makeFailure(
- error.ProcessTerminated,
- exitCode=CodeImportWorkerExitCode.FAILURE_INVALID))
- self.assertEqual(calls, [CodeImportResultStatus.FAILURE_INVALID])
- # We return the deferred that callFinishJob returns -- if
- # callFinishJob did not swallow the error, this will fail the test.
- return ret
-
- def test_callFinishJobCallsFinishJobUnsupportedFeature(self):
- # If the argument to callFinishJob indicates that the subprocess
- # exited with a code of FAILURE_UNSUPPORTED_FEATURE, it
- # calls finishJob with a status of FAILURE_UNSUPPORTED_FEATURE.
- worker_monitor = self.makeWorkerMonitorWithJob()
- calls = self.patchOutFinishJob(worker_monitor)
- ret = worker_monitor.callFinishJob(makeFailure(
- error.ProcessTerminated,
- exitCode=CodeImportWorkerExitCode.FAILURE_UNSUPPORTED_FEATURE))
- self.assertEqual(
- calls, [CodeImportResultStatus.FAILURE_UNSUPPORTED_FEATURE])
- # We return the deferred that callFinishJob returns -- if
- # callFinishJob did not swallow the error, this will fail the test.
- return ret
-
- def test_callFinishJobCallsFinishJobRemoteBroken(self):
- # If the argument to callFinishJob indicates that the subprocess
- # exited with a code of FAILURE_REMOTE_BROKEN, it
- # calls finishJob with a status of FAILURE_REMOTE_BROKEN.
- worker_monitor = self.makeWorkerMonitorWithJob()
- calls = self.patchOutFinishJob(worker_monitor)
- ret = worker_monitor.callFinishJob(
- makeFailure(
- error.ProcessTerminated,
- exitCode=CodeImportWorkerExitCode.FAILURE_REMOTE_BROKEN))
- self.assertEqual(
- calls, [CodeImportResultStatus.FAILURE_REMOTE_BROKEN])
- # We return the deferred that callFinishJob returns -- if
- # callFinishJob did not swallow the error, this will fail the test.
- return ret
-
- @suppress_stderr
- def test_callFinishJobLogsTracebackOnFailure(self):
- # When callFinishJob is called with a failure, it dumps the traceback
- # of the failure into the log file.
- worker_monitor = self.makeWorkerMonitorWithJob()
- ret = worker_monitor.callFinishJob(makeFailure(RuntimeError))
-
- def check_log_file(ignored):
- worker_monitor._log_file.seek(0)
- log_bytes = worker_monitor._log_file.read()
- self.assertIn(b'Traceback (most recent call last)', log_bytes)
- self.assertIn(b'RuntimeError', log_bytes)
- return ret.addCallback(check_log_file)
-
- def test_callFinishJobRespects_call_finish_job(self):
- # callFinishJob does not call finishJob if _call_finish_job is False.
- # This is to support exiting without fuss when the job we are working
- # on is deleted in the web UI.
- worker_monitor = self.makeWorkerMonitorWithJob()
- calls = self.patchOutFinishJob(worker_monitor)
- worker_monitor._call_finish_job = False
- worker_monitor.callFinishJob(None)
- self.assertEqual(calls, [])
-
-
-class TestWorkerMonitorRunNoProcess(FakeCodeImportSchedulerMixin, TestCase):
- """Tests for `CodeImportWorkerMonitor.run` that don't launch a subprocess.
- """
-
- run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=20)
-
- class WorkerMonitor(CodeImportWorkerMonitor):
- """See `CodeImportWorkerMonitor`.
-
- Override _launchProcess to return a deferred that we can
- callback/errback as we choose. Passing ``has_job=False`` to the
- constructor will cause getWorkerArguments() to raise ExitQuietly (this
- bit is tested above).
- """
-
- def __init__(self, job_id, logger, codeimport_endpoint, access_policy,
- process_deferred):
- CodeImportWorkerMonitor.__init__(
- self, job_id, logger, codeimport_endpoint, access_policy)
- self.result_status = None
- self.process_deferred = process_deferred
-
- def _launchProcess(self, worker_arguments):
- return self.process_deferred
-
- def finishJob(self, status):
- assert self.result_status is None, "finishJob called twice!"
- self.result_status = status
- return defer.succeed(None)
-
- def makeWorkerMonitor(self, process_deferred, has_job=True):
- if has_job:
- job_data = {1: {'arguments': []}}
- else:
- job_data = {}
- _, scheduler_url = self.makeFakeCodeImportScheduler(job_data)
- return self.WorkerMonitor(
- 1, BufferLogger(), xmlrpc.Proxy(scheduler_url), "anything",
- process_deferred)
-
- def assertFinishJobCalledWithStatus(self, ignored, worker_monitor,
- status):
- """Assert that finishJob was called with the given status."""
- self.assertEqual(worker_monitor.result_status, status)
-
- def assertFinishJobNotCalled(self, ignored, worker_monitor):
- """Assert that finishJob was called with the given status."""
- self.assertFinishJobCalledWithStatus(ignored, worker_monitor, None)
-
- def test_success(self):
- # In the successful case, finishJob is called with
- # CodeImportResultStatus.SUCCESS.
- worker_monitor = self.makeWorkerMonitor(defer.succeed(None))
- return worker_monitor.run().addCallback(
- self.assertFinishJobCalledWithStatus, worker_monitor,
- CodeImportResultStatus.SUCCESS)
-
- def test_failure(self):
- # If the process deferred is fired with a failure, finishJob is called
- # with CodeImportResultStatus.FAILURE, but the call to run() still
- # succeeds.
- # Need a twisted error reporting stack (normally set up by
- # loggingsupport.set_up_oops_reporting).
- errorlog.globalErrorUtility.configure(
- config_factory=oops_twisted.Config,
- publisher_adapter=oops_twisted.defer_publisher,
- publisher_helpers=oops_twisted.publishers)
- self.addCleanup(errorlog.globalErrorUtility.configure)
- worker_monitor = self.makeWorkerMonitor(defer.fail(RuntimeError()))
- return worker_monitor.run().addCallback(
- self.assertFinishJobCalledWithStatus, worker_monitor,
- CodeImportResultStatus.FAILURE)
-
- def test_quiet_exit(self):
- # If the process deferred fails with ExitQuietly, the call to run()
- # succeeds, and finishJob is not called at all.
- worker_monitor = self.makeWorkerMonitor(
- defer.succeed(None), has_job=False)
- return worker_monitor.run().addCallback(
- self.assertFinishJobNotCalled, worker_monitor)
-
- def test_quiet_exit_from_finishJob(self):
- # If finishJob fails with ExitQuietly, the call to run() still
- # succeeds.
- worker_monitor = self.makeWorkerMonitor(defer.succeed(None))
-
- def finishJob(reason):
- raise ExitQuietly
- worker_monitor.finishJob = finishJob
- return worker_monitor.run()
-
- def test_callFinishJob_logs_failure(self):
- # callFinishJob logs a failure from the child process.
- errorlog.globalErrorUtility.configure(
- config_factory=oops_twisted.Config,
- publisher_adapter=oops_twisted.defer_publisher,
- publisher_helpers=oops_twisted.publishers)
- self.addCleanup(errorlog.globalErrorUtility.configure)
- failure_msg = b"test_callFinishJob_logs_failure expected failure"
- worker_monitor = self.makeWorkerMonitor(
- defer.fail(RuntimeError(failure_msg)))
- d = worker_monitor.run()
-
- def check_log_file(ignored):
- worker_monitor._log_file.seek(0)
- log_bytes = worker_monitor._log_file.read()
- self.assertIn(
- b"Failure: exceptions.RuntimeError: " + failure_msg,
- log_bytes)
-
- d.addCallback(check_log_file)
- return d
-
-
-class CIWorkerMonitorProtocolForTesting(CodeImportWorkerMonitorProtocol):
- """A `CodeImportWorkerMonitorProtocol` that counts `resetTimeout` calls.
- """
-
- def __init__(self, deferred, worker_monitor, log_file, clock=None):
- """See `CodeImportWorkerMonitorProtocol.__init__`."""
- CodeImportWorkerMonitorProtocol.__init__(
- self, deferred, worker_monitor, log_file, clock)
- self.reset_calls = 0
-
- def resetTimeout(self):
- """See `ProcessMonitorProtocolWithTimeout.resetTimeout`."""
- CodeImportWorkerMonitorProtocol.resetTimeout(self)
- self.reset_calls += 1
-
-
-class CIWorkerMonitorForTesting(CodeImportWorkerMonitor):
- """A `CodeImportWorkerMonitor` that hangs on to the process protocol."""
-
- def _makeProcessProtocol(self, deferred):
- """See `CodeImportWorkerMonitor._makeProcessProtocol`.
-
- We hang on to the constructed object for later inspection -- see
- `TestWorkerMonitorIntegration.assertImported`.
- """
- protocol = CIWorkerMonitorProtocolForTesting(
- deferred, self, self._log_file)
- self._protocol = protocol
- return protocol
-
-
-class TestWorkerMonitorIntegration(FakeCodeImportSchedulerMixin,
- TestCaseInTempDir, TestCase):
-
- run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=60)
-
- def setUp(self):
- super(TestWorkerMonitorIntegration, self).setUp()
- self.repo_path = tempfile.mkdtemp()
- self.disable_directory_isolation()
- self.addCleanup(shutil.rmtree, self.repo_path)
- self.foreign_commit_count = 0
-
- def makeCVSCodeImport(self):
- """Make arguments that point to a real CVS repository."""
- cvs_server = CVSServer(self.repo_path)
- cvs_server.start_server()
- self.addCleanup(cvs_server.stop_server)
-
- cvs_server.makeModule('trunk', [('README', 'original\n')])
- self.foreign_commit_count = 2
-
- return [
- str(self.factory.getUniqueInteger()), 'cvs', 'bzr',
- cvs_server.getRoot(), '--cvs-module', 'trunk',
- ]
-
- def makeBzrSvnCodeImport(self):
- """Make arguments that point to a real Subversion repository."""
- self.subversion_server = SubversionServer(
- self.repo_path, use_svn_serve=True)
- self.subversion_server.start_server()
- self.addCleanup(self.subversion_server.stop_server)
- url = self.subversion_server.makeBranch(
- 'trunk', [('README', b'contents')])
- self.foreign_commit_count = 2
-
- return [
- str(self.factory.getUniqueInteger()), 'bzr-svn', 'bzr',
- url,
- ]
-
- def makeGitCodeImport(self, target_rcs_type='bzr'):
- """Make arguments that point to a real Git repository."""
- self.git_server = GitServer(self.repo_path, use_server=False)
- self.git_server.start_server()
- self.addCleanup(self.git_server.stop_server)
-
- self.git_server.makeRepo('source', [('README', 'contents')])
- self.foreign_commit_count = 1
-
- target_id = (
- str(self.factory.getUniqueInteger()) if target_rcs_type == 'bzr'
- else self.factory.getUniqueUnicode())
- arguments = [
- target_id, 'git', target_rcs_type,
- self.git_server.get_url('source'),
- ]
- if target_rcs_type == 'git':
- arguments.extend(['--macaroon', Macaroon().serialize()])
- return arguments
-
- def makeBzrCodeImport(self):
- """Make arguments that point to a real Bazaar branch."""
- self.bzr_server = BzrServer(self.repo_path)
- self.bzr_server.start_server()
- self.addCleanup(self.bzr_server.stop_server)
-
- self.bzr_server.makeRepo([('README', 'contents')])
- self.foreign_commit_count = 1
-
- return [
- str(self.factory.getUniqueInteger()), 'bzr', 'bzr',
- self.bzr_server.get_url(),
- ]
-
- def getStartedJobForImport(self, arguments):
- """Get a started `CodeImportJob` for `code_import`.
-
- This method returns a job ID and job data, imitating an approved job
- on Launchpad. It also makes sure there are no branches or foreign
- trees in the default stores to interfere with processing this job.
- """
- if arguments[2] == 'bzr':
- target_id = int(arguments[0])
- clean_up_default_stores_for_import(target_id)
- self.addCleanup(clean_up_default_stores_for_import, target_id)
- return (1, {'arguments': arguments})
-
- def makeTargetGitServer(self):
- """Set up a target Git server that can receive imports."""
- self.target_store = tempfile.mkdtemp()
- self.addCleanup(shutil.rmtree, self.target_store)
- self.target_git_server = GitServer(self.target_store, use_server=True)
- self.target_git_server.start_server()
- self.addCleanup(self.target_git_server.stop_server)
- config_name = self.factory.getUniqueUnicode()
- config_fixture = self.useFixture(ConfigFixture(
- config_name, os.environ['LPCONFIG']))
- setting_lines = [
- "[codehosting]",
- "git_browse_root: %s" % self.target_git_server.get_url(""),
- "",
- "[launchpad]",
- "internal_macaroon_secret_key: some-secret",
- ]
- config_fixture.add_section("\n" + "\n".join(setting_lines))
- self.useFixture(ConfigUseFixture(config_name))
- self.useFixture(GitHostingFixture())
-
- def assertBranchImportedOKForCodeImport(self, target_id):
- """Assert that a branch was pushed into the default branch store."""
- if target_id.isdigit():
- url = get_default_bazaar_branch_store()._getMirrorURL(
- int(target_id))
- branch = Branch.open(url)
- commit_count = branch.revno()
- else:
- repo_path = os.path.join(self.target_store, target_id)
- commit_count = int(subprocess.check_output(
- ["git", "rev-list", "--count", "HEAD"],
- cwd=repo_path, universal_newlines=True))
- self.assertEqual(self.foreign_commit_count, commit_count)
-
- def assertImported(self, job_id, job_data):
- """Assert that the code import with the given job id was imported.
-
- Since we don't have a full Launchpad appserver instance here, we
- just check that the code import worker has made the correct XML-RPC
- calls via the given worker monitor.
- """
- # In the in-memory tests, check that resetTimeout on the
- # CodeImportWorkerMonitorProtocol was called at least once.
- if self._protocol is not None:
- self.assertPositive(self._protocol.reset_calls)
- self.assertThat(self.scheduler.calls, AnyMatch(
- MatchesListwise([
- Equals('finishJobID'),
- Equals(job_id),
- Equals('SUCCESS'),
- IsInstance(xmlrpc_client.Binary),
- ])))
- self.assertBranchImportedOKForCodeImport(job_data['arguments'][0])
-
- @defer.inlineCallbacks
- def performImport(self, job_id, job_data):
- """Perform the import job with ID job_id and data job_data.
-
- Return a Deferred that fires when the job is done.
-
- This implementation does it in-process.
- """
- logger = BufferLogger()
- self.scheduler, scheduler_url = self.makeFakeCodeImportScheduler(
- {job_id: job_data})
- worker_monitor = CIWorkerMonitorForTesting(
- job_id, logger, xmlrpc.Proxy(scheduler_url), "anything")
- result = yield worker_monitor.run()
- self._protocol = worker_monitor._protocol
- defer.returnValue(result)
-
- @defer.inlineCallbacks
- def test_import_cvs(self):
- # Create a CVS CodeImport and import it.
- job_id, job_data = self.getStartedJobForImport(
- self.makeCVSCodeImport())
- yield self.performImport(job_id, job_data)
- self.assertImported(job_id, job_data)
-
- @defer.inlineCallbacks
- def test_import_git(self):
- # Create a Git CodeImport and import it.
- job_id, job_data = self.getStartedJobForImport(
- self.makeGitCodeImport())
- yield self.performImport(job_id, job_data)
- self.assertImported(job_id, job_data)
-
- @defer.inlineCallbacks
- def test_import_git_to_git(self):
- # Create a Git-to-Git CodeImport and import it.
- self.makeTargetGitServer()
- job_id, job_data = self.getStartedJobForImport(
- self.makeGitCodeImport(target_rcs_type='git'))
- target_repo_path = os.path.join(
- self.target_store, job_data['arguments'][0])
- os.makedirs(target_repo_path)
- self.target_git_server.createRepository(target_repo_path, bare=True)
- yield self.performImport(job_id, job_data)
- self.assertImported(job_id, job_data)
- target_repo = GitRepo(target_repo_path)
- self.assertContentEqual(
- ["heads/master"], target_repo.refs.keys(base="refs"))
- self.assertEqual(
- "ref: refs/heads/master", target_repo.refs.read_ref("HEAD"))
-
- @defer.inlineCallbacks
- def test_import_git_to_git_refs_changed(self):
- # Create a Git-to-Git CodeImport and import it incrementally with
- # ref and HEAD changes.
- self.makeTargetGitServer()
- job_id, job_data = self.getStartedJobForImport(
- self.makeGitCodeImport(target_rcs_type='git'))
- source_repo = GitRepo(os.path.join(self.repo_path, "source"))
- commit = source_repo.refs["refs/heads/master"]
- source_repo.refs["refs/heads/one"] = commit
- source_repo.refs["refs/heads/two"] = commit
- source_repo.refs.set_symbolic_ref("HEAD", "refs/heads/one")
- del source_repo.refs["refs/heads/master"]
- target_repo_path = os.path.join(
- self.target_store, job_data['arguments'][0])
- self.target_git_server.makeRepo(
- job_data['arguments'][0], [("NEWS", "contents")])
- yield self.performImport(job_id, job_data)
- self.assertImported(job_id, job_data)
- target_repo = GitRepo(target_repo_path)
- self.assertContentEqual(
- ["heads/one", "heads/two"], target_repo.refs.keys(base="refs"))
- self.assertEqual(
- "ref: refs/heads/one",
- GitRepo(target_repo_path).refs.read_ref("HEAD"))
-
- @defer.inlineCallbacks
- def test_import_bzr(self):
- # Create a Bazaar CodeImport and import it.
- job_id, job_data = self.getStartedJobForImport(
- self.makeBzrCodeImport())
- yield self.performImport(job_id, job_data)
- self.assertImported(job_id, job_data)
-
- @defer.inlineCallbacks
- def test_import_bzrsvn(self):
- # Create a Subversion-via-bzr-svn CodeImport and import it.
- job_id, job_data = self.getStartedJobForImport(
- self.makeBzrSvnCodeImport())
- yield self.performImport(job_id, job_data)
- self.assertImported(job_id, job_data)
-
-
-class DeferredOnExit(protocol.ProcessProtocol):
-
- def __init__(self, deferred):
- self._deferred = deferred
-
- def processEnded(self, reason):
- if reason.check(error.ProcessDone):
- self._deferred.callback(None)
- else:
- self._deferred.errback(reason)
-
-
-class TestWorkerMonitorIntegrationScript(TestWorkerMonitorIntegration):
- """Tests for CodeImportWorkerMonitor that execute a child process."""
-
- def setUp(self):
- super(TestWorkerMonitorIntegrationScript, self).setUp()
- self._protocol = None
-
- def performImport(self, job_id, job_data):
- """Perform the import job with ID job_id and data job_data.
-
- Return a Deferred that fires when the job is done.
-
- This implementation does it in a child process.
- """
- self.scheduler, scheduler_url = self.makeFakeCodeImportScheduler(
- {job_id: job_data})
- config_name = self.factory.getUniqueUnicode()
- config_fixture = self.useFixture(
- ConfigFixture(config_name, os.environ['LPCONFIG']))
- config_fixture.add_section(dedent("""
- [codeimportdispatcher]
- codeimportscheduler_url: %s
- """) % scheduler_url)
- self.useFixture(ConfigUseFixture(config_name))
- script_path = os.path.join(
- config.root, 'scripts', 'code-import-worker-monitor.py')
- process_end_deferred = defer.Deferred()
- # The "childFDs={0:0, 1:1, 2:2}" means that any output from the script
- # goes to the test runner's console rather than to pipes that noone is
- # listening too.
- interpreter = '%s/bin/py' % config.root
- reactor.spawnProcess(
- DeferredOnExit(process_end_deferred), interpreter, [
- interpreter,
- script_path,
- '--access-policy=anything',
- str(job_id),
- '-q',
- ], childFDs={0: 0, 1: 1, 2: 2}, env=os.environ)
- return process_end_deferred
diff --git a/lib/lp/codehosting/codeimport/uifactory.py b/lib/lp/codehosting/codeimport/uifactory.py
deleted file mode 100644
index 2b8bfd8..0000000
--- a/lib/lp/codehosting/codeimport/uifactory.py
+++ /dev/null
@@ -1,207 +0,0 @@
-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""A UIFactory useful for code imports."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-__all__ = ['LoggingUIFactory']
-
-
-import sys
-import time
-
-from bzrlib.ui import NoninteractiveUIFactory
-from bzrlib.ui.text import TextProgressView
-import six
-
-
-class LoggingUIFactory(NoninteractiveUIFactory):
- """A UI Factory that produces reasonably sparse logging style output.
-
- The goal is to produce a line of output no more often than once a minute
- (by default).
- """
-
- # XXX: JelmerVernooij 2011-08-02 bug=820127: This seems generic enough to
- # live in bzrlib.ui
-
- def __init__(self, time_source=time.time, logger=None, interval=60.0):
- """Construct a `LoggingUIFactory`.
-
- :param time_source: A callable that returns time in seconds since the
- epoch. Defaults to ``time.time`` and should be replaced with
- something deterministic in tests.
- :param logger: The logger object to write to
- :param interval: Don't produce output more often than once every this
- many seconds. Defaults to 60 seconds.
- """
- NoninteractiveUIFactory.__init__(self)
- self.interval = interval
- self.logger = logger
- self._progress_view = LoggingTextProgressView(
- time_source, lambda m: logger.info("%s", m), interval)
-
- def show_user_warning(self, warning_id, **message_args):
- self.logger.warning(
- "%s", self.format_user_warning(warning_id, message_args))
-
- def show_warning(self, msg):
- self.logger.warning("%s", six.ensure_binary(msg))
-
- def get_username(self, prompt, **kwargs):
- return None
-
- def get_password(self, prompt, **kwargs):
- return None
-
- def show_message(self, msg):
- self.logger.info("%s", msg)
-
- def note(self, msg):
- self.logger.info("%s", msg)
-
- def show_error(self, msg):
- self.logger.error("%s", msg)
-
- def _progress_updated(self, task):
- """A task has been updated and wants to be displayed.
- """
- if not self._task_stack:
- self.logger.warning("%r updated but no tasks are active", task)
- self._progress_view.show_progress(task)
-
- def _progress_all_finished(self):
- self._progress_view.clear()
-
- def report_transport_activity(self, transport, byte_count, direction):
- """Called by transports as they do IO.
-
- This may update a progress bar, spinner, or similar display.
- By default it does nothing.
- """
- self._progress_view.show_transport_activity(transport,
- direction, byte_count)
-
-
-class LoggingTextProgressView(TextProgressView):
- """Support class for `LoggingUIFactory`. """
-
- def __init__(self, time_source, writer, interval):
- """See `LoggingUIFactory.__init__` for descriptions of the parameters.
- """
- # If anything refers to _term_file, that's a bug for us.
- TextProgressView.__init__(self, term_file=None)
- self._writer = writer
- self.time_source = time_source
- if writer is None:
- self.write = sys.stdout.write
- else:
- self.write = writer
- # _transport_expire_time is how long to keep the transport activity in
- # the progress bar for when show_progress is called. We opt for
- # always just showing the task info.
- self._transport_expire_time = 0
- # We repaint only after 'interval' seconds whether we're being told
- # about task progress or transport activity.
- self._update_repaint_frequency = interval
- self._transport_repaint_frequency = interval
-
- def _show_line(self, s):
- # This is a bit hackish: even though this method is just expected to
- # produce output, we reset the _bytes_since_update so that transport
- # activity is reported as "since last log message" and
- # _transport_update_time so that transport activity doesn't cause an
- # update until it occurs more than _transport_repaint_frequency
- # seconds after the last update of any kind.
- self._bytes_since_update = 0
- self._transport_update_time = self.time_source()
- self._writer(s)
-
- def _render_bar(self):
- # There's no point showing a progress bar in a flat log.
- return ''
-
- def _render_line(self):
- bar_string = self._render_bar()
- if self._last_task:
- task_part, counter_part = self._format_task(self._last_task)
- else:
- task_part = counter_part = ''
- if self._last_task and not self._last_task.show_transport_activity:
- trans = ''
- else:
- trans = self._last_transport_msg
- # the bar separates the transport activity from the message, so even
- # if there's no bar or spinner, we must show something if both those
- # fields are present
- if (task_part and trans) and not bar_string:
- bar_string = ' | '
- s = trans + bar_string + task_part + counter_part
- return s
-
- def _format_transport_msg(self, scheme, dir_char, rate):
- # We just report the amount of data transferred.
- return '%s bytes transferred' % self._bytes_since_update
-
- # What's below is copied and pasted from bzrlib.ui.text.TextProgressView
- # and changed to (a) get its notion of time from self.time_source (which
- # can be replaced by a deterministic time source in tests) rather than
- # time.time and (b) respect the _update_repaint_frequency,
- # _transport_expire_time and _transport_repaint_frequency instance
- # variables rather than having these as hard coded constants. These
- # changes could and should be ported upstream and then we won't have to
- # carry our version of this code around any more.
-
- def show_progress(self, task):
- """Called by the task object when it has changed.
-
- :param task: The top task object; its parents are also included
- by following links.
- """
- must_update = task is not self._last_task
- self._last_task = task
- now = self.time_source()
- if ((not must_update) and
- (now < self._last_repaint + self._update_repaint_frequency)):
- return
- if now > self._transport_update_time + self._transport_expire_time:
- # no recent activity; expire it
- self._last_transport_msg = ''
- self._last_repaint = now
- self._repaint()
-
- def show_transport_activity(self, transport, direction, byte_count):
- """Called by transports via the ui_factory, as they do IO.
-
- This may update a progress bar, spinner, or similar display.
- By default it does nothing.
- """
- # XXX: Probably there should be a transport activity model, and that
- # too should be seen by the progress view, rather than being poked in
- # here.
- self._total_byte_count += byte_count
- self._bytes_since_update += byte_count
- now = self.time_source()
- if self._transport_update_time is None:
- self._transport_update_time = now
- elif now >= (self._transport_update_time
- + self._transport_repaint_frequency):
- # guard against clock stepping backwards, and don't update too
- # often
- rate = self._bytes_since_update / (
- now - self._transport_update_time)
- scheme = getattr(transport, '_scheme', None) or repr(transport)
- if direction == 'read':
- dir_char = '>'
- elif direction == 'write':
- dir_char = '<'
- else:
- dir_char = '?'
- msg = self._format_transport_msg(scheme, dir_char, rate)
- self._transport_update_time = now
- self._last_repaint = now
- self._bytes_since_update = 0
- self._last_transport_msg = msg
- self._repaint()
diff --git a/lib/lp/codehosting/codeimport/worker.py b/lib/lp/codehosting/codeimport/worker.py
deleted file mode 100644
index 43f6938..0000000
--- a/lib/lp/codehosting/codeimport/worker.py
+++ /dev/null
@@ -1,1221 +0,0 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""The code import worker. This imports code from foreign repositories."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-__all__ = [
- 'BazaarBranchStore',
- 'BzrImportWorker',
- 'BzrSvnImportWorker',
- 'CSCVSImportWorker',
- 'CodeImportBranchOpenPolicy',
- 'CodeImportSourceDetails',
- 'CodeImportWorkerExitCode',
- 'ForeignTreeStore',
- 'GitImportWorker',
- 'ImportWorker',
- 'ToBzrImportWorker',
- 'get_default_bazaar_branch_store',
- ]
-
-from argparse import ArgumentParser
-import io
-import os
-import shutil
-import signal
-import subprocess
-
-# FIRST Ensure correct plugins are loaded. Do not delete this comment or the
-# line below this comment.
-import lp.codehosting
-
-from bzrlib.branch import (
- Branch,
- InterBranch,
- )
-from bzrlib.bzrdir import (
- BzrDir,
- BzrDirFormat,
- )
-from bzrlib.errors import (
- ConnectionError,
- InvalidEntryName,
- NoRepositoryPresent,
- NoSuchFile,
- NotBranchError,
- TooManyRedirections,
- )
-from bzrlib.transport import (
- get_transport_from_path,
- get_transport_from_url,
- )
-import bzrlib.ui
-from bzrlib.upgrade import upgrade
-from bzrlib.url_policy_open import (
- BadUrl,
- BranchOpener,
- BranchOpenPolicy,
- )
-from bzrlib.urlutils import (
- join as urljoin,
- local_path_from_url,
- )
-import cscvs
-from cscvs.cmds import totla
-import CVS
-from dulwich.errors import GitProtocolError
-from dulwich.protocol import (
- pkt_line,
- Protocol,
- )
-from lazr.uri import (
- InvalidURIError,
- URI,
- )
-from pymacaroons import Macaroon
-import SCM
-import six
-from six.moves.urllib.parse import (
- urlsplit,
- urlunsplit,
- )
-
-from lp.codehosting.codeimport.foreigntree import CVSWorkingTree
-from lp.codehosting.codeimport.tarball import (
- create_tarball,
- extract_tarball,
- )
-from lp.codehosting.codeimport.uifactory import LoggingUIFactory
-from lp.services.config import config
-from lp.services.propertycache import cachedproperty
-from lp.services.timeout import (
- raise_for_status_redacted,
- urlfetch,
- )
-from lp.services.utils import sanitise_urls
-
-
-class CodeImportBranchOpenPolicy(BranchOpenPolicy):
- """Branch open policy for code imports.
-
- In summary:
- - follow references,
- - only open non-Launchpad URLs for imports from Bazaar to Bazaar or
- from Git to Git
- - only open the allowed schemes
- """
-
- allowed_schemes = ['http', 'https', 'svn', 'git', 'ftp', 'bzr']
-
- def __init__(self, rcstype, target_rcstype, exclude_hosts=None):
- self.rcstype = rcstype
- self.target_rcstype = target_rcstype
- self.exclude_hosts = list(exclude_hosts or [])
-
- def should_follow_references(self):
- """See `BranchOpenPolicy.should_follow_references`.
-
- We traverse branch references for MIRRORED branches because they
- provide a useful redirection mechanism and we want to be consistent
- with the bzr command line.
- """
- return True
-
- def transform_fallback_location(self, branch, url):
- """See `BranchOpenPolicy.transform_fallback_location`.
-
- For mirrored branches, we stack on whatever the remote branch claims
- to stack on, but this URL still needs to be checked.
- """
- return urljoin(branch.base, url), True
-
- def check_one_url(self, url):
- """See `BranchOpenPolicy.check_one_url`.
-
- We refuse to mirror Bazaar branches from Launchpad, or any branches
- from a ssh-like or file URL.
- """
- try:
- uri = URI(url)
- except InvalidURIError:
- raise BadUrl(url)
- for hostname in self.exclude_hosts:
- if uri.underDomain(hostname):
- raise BadUrl(url)
- if uri.scheme not in self.allowed_schemes:
- raise BadUrl(url)
-
-
-class CodeImportWorkerExitCode:
- """Exit codes used by the code import worker script."""
-
- SUCCESS = 0
- FAILURE = 1
- SUCCESS_NOCHANGE = 2
- SUCCESS_PARTIAL = 3
- FAILURE_INVALID = 4
- FAILURE_UNSUPPORTED_FEATURE = 5
- FAILURE_FORBIDDEN = 6
- FAILURE_REMOTE_BROKEN = 7
-
-
-class BazaarBranchStore:
- """A place where Bazaar branches of code imports are kept."""
-
- def __init__(self, transport):
- """Construct a Bazaar branch store based at `transport`."""
- self.transport = transport
-
- def _getMirrorURL(self, db_branch_id, push=False):
- """Return the URL that `db_branch` is stored at."""
- base_url = self.transport.base
- if push:
- # Pulling large branches over sftp is less CPU-intensive, but
- # pushing over bzr+ssh seems to be more reliable.
- split = urlsplit(base_url)
- if split.scheme == 'sftp':
- base_url = urlunsplit([
- 'bzr+ssh', split.netloc, split.path, split.query,
- split.fragment])
- return urljoin(base_url, '%08x' % db_branch_id)
-
- def pull(self, db_branch_id, target_path, required_format,
- needs_tree=False, stacked_on_url=None):
- """Pull down the Bazaar branch of an import to `target_path`.
-
- :return: A Bazaar branch for the code import corresponding to the
- database branch with id `db_branch_id`.
- """
- remote_url = self._getMirrorURL(db_branch_id)
- try:
- remote_bzr_dir = BzrDir.open(remote_url)
- except NotBranchError:
- local_branch = BzrDir.create_branch_and_repo(
- target_path, format=required_format)
- if needs_tree:
- local_branch.bzrdir.create_workingtree()
- if stacked_on_url:
- local_branch.set_stacked_on_url(stacked_on_url)
- return local_branch
- # The proper thing to do here would be to call
- # "remote_bzr_dir.sprout()". But 2a fetch slowly checks which
- # revisions are in the ancestry of the tip of the remote branch, which
- # we strictly don't care about, so we just copy the whole thing down
- # at the vfs level.
- control_dir = remote_bzr_dir.root_transport.relpath(
- remote_bzr_dir.transport.abspath('.'))
- target = get_transport_from_path(target_path)
- target_control = target.clone(control_dir)
- target_control.create_prefix()
- remote_bzr_dir.transport.copy_tree_to_transport(target_control)
- local_bzr_dir = BzrDir.open_from_transport(target)
- if local_bzr_dir.needs_format_conversion(format=required_format):
- try:
- local_bzr_dir.root_transport.delete_tree('backup.bzr')
- except NoSuchFile:
- pass
- upgrade(target_path, required_format, clean_up=True)
- if needs_tree:
- local_bzr_dir.create_workingtree()
- return local_bzr_dir.open_branch()
-
- def push(self, db_branch_id, bzr_branch, required_format,
- stacked_on_url=None):
- """Push up `bzr_branch` as the Bazaar branch for `code_import`.
-
- :return: A boolean that is true if the push was non-trivial
- (i.e. actually transferred revisions).
- """
- self.transport.create_prefix()
- target_url = self._getMirrorURL(db_branch_id, push=True)
- try:
- remote_branch = Branch.open(target_url)
- except NotBranchError:
- remote_branch = BzrDir.create_branch_and_repo(
- target_url, format=required_format)
- old_branch = None
- else:
- if remote_branch.bzrdir.needs_format_conversion(
- required_format):
- # For upgrades, push to a new branch in
- # the new format. When done pushing,
- # retire the old .bzr directory and rename
- # the new one in place.
- old_branch = remote_branch
- upgrade_url = urljoin(target_url, "backup.bzr")
- try:
- remote_branch.bzrdir.root_transport.delete_tree(
- 'backup.bzr')
- except NoSuchFile:
- pass
- remote_branch = BzrDir.create_branch_and_repo(
- upgrade_url, format=required_format)
- else:
- old_branch = None
- # This can be done safely, since only modern formats are used to
- # import to.
- if stacked_on_url is not None:
- remote_branch.set_stacked_on_url(stacked_on_url)
- pull_result = remote_branch.pull(bzr_branch, overwrite=True)
- # Because of the way we do incremental imports, there may be revisions
- # in the branch's repo that are not in the ancestry of the branch tip.
- # We need to transfer them too.
- remote_branch.repository.fetch(bzr_branch.repository)
- if old_branch is not None:
- # The format has changed; move the new format
- # branch in place.
- base_transport = old_branch.bzrdir.root_transport
- base_transport.delete_tree('.bzr')
- base_transport.rename("backup.bzr/.bzr", ".bzr")
- base_transport.rmdir("backup.bzr")
- return pull_result.old_revid != pull_result.new_revid
-
-
-def get_default_bazaar_branch_store():
- """Return the default `BazaarBranchStore`."""
- return BazaarBranchStore(
- get_transport_from_url(config.codeimport.bazaar_branch_store))
-
-
-class CodeImportSourceDetails:
- """The information needed to process an import.
-
- As the worker doesn't talk to the database, we don't use
- `CodeImport` objects for this.
-
- The 'fromArguments' method builds an instance of this class from a form
- of the information suitable for passing around on executables' command
- lines.
-
- :ivar target_id: The id of the Bazaar branch or the path of the Git
- repository associated with this code import, used for locating the
- existing import and the foreign tree.
- :ivar rcstype: 'cvs', 'git', 'bzr-svn', 'bzr' as appropriate.
- :ivar target_rcstype: 'bzr' or 'git' as appropriate.
- :ivar url: The branch URL if rcstype in ['bzr-svn', 'git', 'bzr'], None
- otherwise.
- :ivar cvs_root: The $CVSROOT if rcstype == 'cvs', None otherwise.
- :ivar cvs_module: The CVS module if rcstype == 'cvs', None otherwise.
- :ivar stacked_on_url: The URL of the branch that the associated branch
- is stacked on, if any.
- :ivar macaroon: A macaroon granting authority to push to the target
- repository if target_rcstype == 'git', None otherwise.
- :ivar exclude_hosts: A list of host names from which we should not
- mirror branches, or None.
- """
-
- def __init__(self, target_id, rcstype, target_rcstype, url=None,
- cvs_root=None, cvs_module=None, stacked_on_url=None,
- macaroon=None, exclude_hosts=None):
- self.target_id = target_id
- self.rcstype = rcstype
- self.target_rcstype = target_rcstype
- self.url = url
- self.cvs_root = cvs_root
- self.cvs_module = cvs_module
- self.stacked_on_url = stacked_on_url
- self.macaroon = macaroon
- self.exclude_hosts = exclude_hosts
-
- @classmethod
- def fromArguments(cls, arguments):
- """Convert command line-style arguments to an instance."""
- # Keep this in sync with CodeImportJob.makeWorkerArguments.
- parser = ArgumentParser(description='Code import worker.')
- parser.add_argument(
- 'target_id', help='Target branch ID or repository unique name')
- parser.add_argument(
- 'rcstype', choices=('bzr', 'bzr-svn', 'cvs', 'git'),
- help='Source revision control system type')
- parser.add_argument(
- 'target_rcstype', choices=('bzr', 'git'),
- help='Target revision control system type')
- parser.add_argument(
- 'url', help='Source URL (or $CVSROOT for rcstype cvs)')
- parser.add_argument(
- '--cvs-module', help='CVS module (only valid for rcstype cvs)')
- parser.add_argument(
- '--stacked-on',
- help='Stacked-on URL (only valid for target_rcstype bzr)')
- parser.add_argument(
- '--macaroon', type=Macaroon.deserialize,
- help=(
- 'Macaroon authorising push (only valid for target_rcstype '
- 'git)'))
- parser.add_argument(
- '--exclude-host', action='append', dest='exclude_hosts',
- help='Refuse to mirror branches from this host')
- args = parser.parse_args(arguments)
- target_id = args.target_id
- rcstype = args.rcstype
- target_rcstype = args.target_rcstype
- url = args.url if rcstype in ('bzr', 'bzr-svn', 'git') else None
- if rcstype == 'cvs':
- if args.cvs_module is None:
- parser.error('rcstype cvs requires --cvs-module')
- cvs_root = args.url
- cvs_module = args.cvs_module
- else:
- cvs_root = None
- cvs_module = None
- if target_rcstype == 'bzr':
- try:
- target_id = int(target_id)
- except ValueError:
- parser.error(
- 'rcstype bzr requires target_id to be an integer')
- stacked_on_url = args.stacked_on
- macaroon = None
- elif target_rcstype == 'git':
- if args.macaroon is None:
- parser.error('target_rcstype git requires --macaroon')
- stacked_on_url = None
- macaroon = args.macaroon
- return cls(
- target_id, rcstype, target_rcstype, url, cvs_root, cvs_module,
- stacked_on_url, macaroon, exclude_hosts=args.exclude_hosts)
-
-
-class ImportDataStore:
- """A store for data associated with an import.
-
- Import workers can store and retreive files into and from the store using
- `put()` and `fetch()`.
-
- So this store can find files stored by previous versions of this code, the
- files are stored at ``<BRANCH ID IN HEX>.<EXT>`` where BRANCH ID comes
- from the CodeImportSourceDetails used to construct the instance and EXT
- comes from the local name passed to `put` or `fetch`.
- """
-
- def __init__(self, transport, source_details):
- """Initialize an `ImportDataStore`.
-
- :param transport: The transport files will be stored on.
- :param source_details: The `CodeImportSourceDetails` object, used to
- know where to store files on the remote transport.
- """
- self.source_details = source_details
- self._transport = transport
- self._target_id = source_details.target_id
-
- def _getRemoteName(self, local_name):
- """Convert `local_name` to the name used to store a file.
-
- The algorithm is a little stupid for historical reasons: we chop off
- the extension and stick that on the end of the branch id from the
- source_details we were constructed with, in hex padded to 8
- characters. For example 'tree.tar.gz' might become '0000a23d.tar.gz'
- or 'git.db' might become '00003e4.db'.
-
- :param local_name: The local name of the file to be stored.
- :return: The name to store the file as on the remote transport.
- """
- if '/' in local_name:
- raise AssertionError("local_name must be a name, not a path")
- dot_index = local_name.index('.')
- if dot_index < 0:
- raise AssertionError("local_name must have an extension.")
- ext = local_name[dot_index:]
- return '%08x%s' % (self._target_id, ext)
-
- def fetch(self, filename, dest_transport=None):
- """Retrieve `filename` from the store.
-
- :param filename: The name of the file to retrieve (must be a filename,
- not a path).
- :param dest_transport: The transport to retrieve the file to,
- defaulting to ``get_transport_from_path('.')``.
- :return: A boolean, true if the file was found and retrieved, false
- otherwise.
- """
- if dest_transport is None:
- dest_transport = get_transport_from_path('.')
- remote_name = self._getRemoteName(filename)
- if self._transport.has(remote_name):
- dest_transport.put_file(
- filename, self._transport.get(remote_name))
- return True
- else:
- return False
-
- def put(self, filename, source_transport=None):
- """Put `filename` into the store.
-
- :param filename: The name of the file to store (must be a filename,
- not a path).
- :param source_transport: The transport to look for the file on,
- defaulting to ``get_transport('.')``.
- """
- if source_transport is None:
- source_transport = get_transport_from_path('.')
- remote_name = self._getRemoteName(filename)
- local_file = source_transport.get(filename)
- self._transport.create_prefix()
- try:
- self._transport.put_file(remote_name, local_file)
- finally:
- local_file.close()
-
-
-class ForeignTreeStore:
- """Manages retrieving and storing foreign working trees.
-
- The code import system stores tarballs of CVS and SVN working trees on
- another system. The tarballs are kept in predictable locations based on
- the ID of the branch associated to the `CodeImport`.
-
- The tarballs are all kept in one directory. The filename of a tarball is
- XXXXXXXX.tar.gz, where 'XXXXXXXX' is the ID of the `CodeImport`'s branch
- in hex.
- """
-
- def __init__(self, import_data_store):
- """Construct a `ForeignTreeStore`.
-
- :param transport: A writable transport that points to the base
- directory where the tarballs are stored.
- :ptype transport: `bzrlib.transport.Transport`.
- """
- self.import_data_store = import_data_store
-
- def _getForeignTree(self, target_path):
- """Return a foreign tree object for `target_path`."""
- source_details = self.import_data_store.source_details
- if source_details.rcstype == 'cvs':
- return CVSWorkingTree(
- source_details.cvs_root, source_details.cvs_module,
- target_path)
- else:
- raise AssertionError(
- "unknown RCS type: %r" % source_details.rcstype)
-
- def archive(self, foreign_tree):
- """Archive the foreign tree."""
- local_name = 'foreign_tree.tar.gz'
- create_tarball(foreign_tree.local_path, 'foreign_tree.tar.gz')
- self.import_data_store.put(local_name)
-
- def fetch(self, target_path):
- """Fetch the foreign branch for `source_details` to `target_path`.
-
- If there is no tarball archived for `source_details`, then try to
- download (i.e. checkout) the foreign tree from its source repository,
- generally on a third party server.
- """
- try:
- return self.fetchFromArchive(target_path)
- except NoSuchFile:
- return self.fetchFromSource(target_path)
-
- def fetchFromSource(self, target_path):
- """Fetch the foreign tree for `source_details` to `target_path`."""
- branch = self._getForeignTree(target_path)
- branch.checkout()
- return branch
-
- def fetchFromArchive(self, target_path):
- """Fetch the foreign tree for `source_details` from the archive."""
- local_name = 'foreign_tree.tar.gz'
- if not self.import_data_store.fetch(local_name):
- raise NoSuchFile(local_name)
- extract_tarball(local_name, target_path)
- tree = self._getForeignTree(target_path)
- tree.update()
- return tree
-
-
-class ImportWorker:
- """Oversees the actual work of a code import."""
-
- def __init__(self, source_details, logger, opener_policy):
- """Construct an `ImportWorker`.
-
- :param source_details: A `CodeImportSourceDetails` object.
- :param logger: A `Logger` to pass to cscvs.
- :param opener_policy: Policy object that decides what branches can
- be imported
- """
- self.source_details = source_details
- self._logger = logger
- self._opener_policy = opener_policy
-
- def getWorkingDirectory(self):
- """The directory we should change to and store all scratch files in.
- """
- base = config.codeimportworker.working_directory_root
- dirname = 'worker-for-branch-%s' % self.source_details.target_id
- return os.path.join(base, dirname)
-
- def run(self):
- """Run the code import job.
-
- This is the primary public interface to the `ImportWorker`. This
- method:
-
- 1. Retrieves an up-to-date foreign tree to import.
- 2. Gets the Bazaar branch to import into.
- 3. Imports the foreign tree into the Bazaar branch. If we've
- already imported this before, we synchronize the imported Bazaar
- branch with the latest changes to the foreign tree.
- 4. Publishes the newly-updated Bazaar branch, making it available to
- Launchpad users.
- 5. Archives the foreign tree, so that we can update it quickly next
- time.
- """
- working_directory = self.getWorkingDirectory()
- if os.path.exists(working_directory):
- shutil.rmtree(working_directory)
- os.makedirs(working_directory)
- saved_pwd = os.getcwd()
- os.chdir(working_directory)
- try:
- return self._doImport()
- finally:
- shutil.rmtree(working_directory)
- os.chdir(saved_pwd)
-
- def _doImport(self):
- """Perform the import.
-
- :return: A CodeImportWorkerExitCode
- """
- raise NotImplementedError()
-
-
-class ToBzrImportWorker(ImportWorker):
- """Oversees the actual work of a code import to Bazaar."""
-
- # Where the Bazaar working tree will be stored.
- BZR_BRANCH_PATH = 'bzr_branch'
-
- # Should `getBazaarBranch` create a working tree?
- needs_bzr_tree = True
-
- required_format = BzrDirFormat.get_default_format()
-
- def __init__(self, source_details, import_data_transport,
- bazaar_branch_store, logger, opener_policy):
- """Construct a `ToBzrImportWorker`.
-
- :param source_details: A `CodeImportSourceDetails` object.
- :param bazaar_branch_store: A `BazaarBranchStore`. The import worker
- uses this to fetch and store the Bazaar branches that are created
- and updated during the import process.
- :param logger: A `Logger` to pass to cscvs.
- :param opener_policy: Policy object that decides what branches can
- be imported
- """
- super(ToBzrImportWorker, self).__init__(
- source_details, logger, opener_policy)
- self.bazaar_branch_store = bazaar_branch_store
- self.import_data_store = ImportDataStore(
- import_data_transport, self.source_details)
-
- def getBazaarBranch(self):
- """Return the Bazaar `Branch` that we are importing into."""
- if os.path.isdir(self.BZR_BRANCH_PATH):
- shutil.rmtree(self.BZR_BRANCH_PATH)
- return self.bazaar_branch_store.pull(
- self.source_details.target_id, self.BZR_BRANCH_PATH,
- self.required_format, self.needs_bzr_tree,
- stacked_on_url=self.source_details.stacked_on_url)
-
- def pushBazaarBranch(self, bazaar_branch, remote_branch=None):
- """Push the updated Bazaar branch to the server.
-
- :return: True if revisions were transferred.
- """
- return self.bazaar_branch_store.push(
- self.source_details.target_id, bazaar_branch,
- self.required_format,
- stacked_on_url=self.source_details.stacked_on_url)
-
-
-class CSCVSImportWorker(ToBzrImportWorker):
- """An ImportWorker for imports that use CSCVS.
-
- As well as invoking cscvs to do the import, this class also needs to
- manage a foreign working tree.
- """
-
- # Where the foreign working tree will be stored.
- FOREIGN_WORKING_TREE_PATH = 'foreign_working_tree'
-
- @cachedproperty
- def foreign_tree_store(self):
- return ForeignTreeStore(self.import_data_store)
-
- def getForeignTree(self):
- """Return the foreign branch object that we are importing from.
-
- :return: A `CVSWorkingTree`.
- """
- if os.path.isdir(self.FOREIGN_WORKING_TREE_PATH):
- shutil.rmtree(self.FOREIGN_WORKING_TREE_PATH)
- os.mkdir(self.FOREIGN_WORKING_TREE_PATH)
- return self.foreign_tree_store.fetch(self.FOREIGN_WORKING_TREE_PATH)
-
- def importToBazaar(self, foreign_tree, bazaar_branch):
- """Actually import `foreign_tree` into `bazaar_branch`.
-
- :param foreign_tree: A `CVSWorkingTree`.
- :param bazaar_tree: A `bzrlib.branch.Branch`, which must have a
- colocated working tree.
- """
- foreign_directory = foreign_tree.local_path
- bzr_directory = str(bazaar_branch.bzrdir.open_workingtree().basedir)
-
- scm_branch = SCM.branch(bzr_directory)
- last_commit = cscvs.findLastCscvsCommit(scm_branch)
-
- # If branch in `bazaar_tree` doesn't have any identifiable CSCVS
- # revisions, CSCVS "initializes" the branch.
- if last_commit is None:
- self._runToBaz(
- foreign_directory, "-SI", "MAIN.1", bzr_directory)
-
- # Now we synchronise the branch, that is, import all new revisions
- # from the foreign branch into the Bazaar branch. If we've just
- # initialized the Bazaar branch, then this means we import *all*
- # revisions.
- last_commit = cscvs.findLastCscvsCommit(scm_branch)
- self._runToBaz(
- foreign_directory, "-SC", "%s::" % last_commit, bzr_directory)
-
- def _runToBaz(self, source_dir, flags, revisions, bazpath):
- """Actually run the CSCVS utility that imports revisions.
-
- :param source_dir: The directory containing the foreign working tree
- that we are importing from.
- :param flags: Flags to pass to `totla.totla`.
- :param revisions: The revisions to import.
- :param bazpath: The directory containing the Bazaar working tree that
- we are importing into.
- """
- # XXX: JonathanLange 2008-02-08: We need better documentation for
- # `flags` and `revisions`.
- config = CVS.Config(source_dir)
- config.args = ["--strict", "-b", bazpath,
- flags, revisions, bazpath]
- totla.totla(config, self._logger, config.args, SCM.tree(source_dir))
-
- def _doImport(self):
- foreign_tree = self.getForeignTree()
- bazaar_branch = self.getBazaarBranch()
- self.importToBazaar(foreign_tree, bazaar_branch)
- non_trivial = self.pushBazaarBranch(bazaar_branch)
- self.foreign_tree_store.archive(foreign_tree)
- if non_trivial:
- return CodeImportWorkerExitCode.SUCCESS
- else:
- return CodeImportWorkerExitCode.SUCCESS_NOCHANGE
-
-
-class PullingImportWorker(ToBzrImportWorker):
- """An import worker for imports that can be done by a bzr plugin.
-
- Subclasses need to implement `probers`.
- """
-
- needs_bzr_tree = False
-
- @property
- def invalid_branch_exceptions(self):
- """Exceptions that indicate no (valid) remote branch is present."""
- raise NotImplementedError
-
- @property
- def unsupported_feature_exceptions(self):
- """The exceptions to consider for unsupported features."""
- raise NotImplementedError
-
- @property
- def broken_remote_exceptions(self):
- """The exceptions to consider for broken remote branches."""
- raise NotImplementedError
-
- @property
- def probers(self):
- """The probers that should be tried for this import."""
- raise NotImplementedError
-
- def getRevisionLimit(self):
- """Return maximum number of revisions to fetch (None for no limit).
- """
- return None
-
- def _doImport(self):
- self._logger.info("Starting job.")
- saved_factory = bzrlib.ui.ui_factory
- opener = BranchOpener(self._opener_policy, self.probers)
- bzrlib.ui.ui_factory = LoggingUIFactory(logger=self._logger)
- try:
- self._logger.info(
- "Getting exising bzr branch from central store.")
- bazaar_branch = self.getBazaarBranch()
- try:
- remote_branch = opener.open(
- six.ensure_str(self.source_details.url))
- except TooManyRedirections:
- self._logger.info("Too many redirections.")
- return CodeImportWorkerExitCode.FAILURE_INVALID
- except NotBranchError:
- self._logger.info("No branch found at remote location.")
- return CodeImportWorkerExitCode.FAILURE_INVALID
- except BadUrl as e:
- self._logger.info("Invalid URL: %s" % e)
- return CodeImportWorkerExitCode.FAILURE_FORBIDDEN
- except ConnectionError as e:
- self._logger.info("Unable to open remote branch: %s" % e)
- return CodeImportWorkerExitCode.FAILURE_INVALID
- try:
- remote_branch_tip = remote_branch.last_revision()
- inter_branch = InterBranch.get(remote_branch, bazaar_branch)
- self._logger.info("Importing branch.")
- revision_limit = self.getRevisionLimit()
- inter_branch.fetch(limit=revision_limit)
- if bazaar_branch.repository.has_revision(remote_branch_tip):
- pull_result = inter_branch.pull(overwrite=True)
- if pull_result.old_revid != pull_result.new_revid:
- result = CodeImportWorkerExitCode.SUCCESS
- else:
- result = CodeImportWorkerExitCode.SUCCESS_NOCHANGE
- else:
- result = CodeImportWorkerExitCode.SUCCESS_PARTIAL
- except Exception as e:
- if e.__class__ in self.unsupported_feature_exceptions:
- self._logger.info(
- "Unable to import branch because of limitations in "
- "Bazaar.")
- self._logger.info(str(e))
- return (
- CodeImportWorkerExitCode.FAILURE_UNSUPPORTED_FEATURE)
- elif e.__class__ in self.invalid_branch_exceptions:
- self._logger.info("Branch invalid: %s", str(e))
- return CodeImportWorkerExitCode.FAILURE_INVALID
- elif e.__class__ in self.broken_remote_exceptions:
- self._logger.info("Remote branch broken: %s", str(e))
- return CodeImportWorkerExitCode.FAILURE_REMOTE_BROKEN
- else:
- raise
- self._logger.info("Pushing local import branch to central store.")
- self.pushBazaarBranch(bazaar_branch, remote_branch=remote_branch)
- self._logger.info("Job complete.")
- return result
- finally:
- bzrlib.ui.ui_factory = saved_factory
-
-
-class GitImportWorker(PullingImportWorker):
- """An import worker for Git imports.
-
- The only behaviour we add is preserving the 'git.db' shamap between runs.
- """
-
- @property
- def invalid_branch_exceptions(self):
- return [
- NoRepositoryPresent,
- NotBranchError,
- ConnectionError,
- ]
-
- @property
- def unsupported_feature_exceptions(self):
- from bzrlib.plugins.git.fetch import SubmodulesRequireSubtrees
- return [
- InvalidEntryName,
- SubmodulesRequireSubtrees,
- ]
-
- @property
- def broken_remote_exceptions(self):
- return []
-
- @property
- def probers(self):
- """See `PullingImportWorker.probers`."""
- from bzrlib.plugins.git import (
- LocalGitProber, RemoteGitProber)
- return [LocalGitProber, RemoteGitProber]
-
- def getRevisionLimit(self):
- """See `PullingImportWorker.getRevisionLimit`."""
- return config.codeimport.git_revisions_import_limit
-
- def getBazaarBranch(self):
- """See `ToBzrImportWorker.getBazaarBranch`.
-
- In addition to the superclass' behaviour, we retrieve bzr-git's
- caches, both legacy and modern, from the import data store and put
- them where bzr-git will find them in the Bazaar tree, that is at
- '.bzr/repository/git.db' and '.bzr/repository/git'.
- """
- branch = PullingImportWorker.getBazaarBranch(self)
- # Fetch the legacy cache from the store, if present.
- self.import_data_store.fetch(
- 'git.db', branch.repository._transport)
- # The cache dir from newer bzr-gits is stored as a tarball.
- local_name = 'git-cache.tar.gz'
- if self.import_data_store.fetch(local_name):
- repo_transport = branch.repository._transport
- repo_transport.mkdir('git')
- git_db_dir = os.path.join(
- local_path_from_url(repo_transport.base), 'git')
- extract_tarball(local_name, git_db_dir)
- return branch
-
- def pushBazaarBranch(self, bazaar_branch, remote_branch=None):
- """See `ToBzrImportWorker.pushBazaarBranch`.
-
- In addition to the superclass' behaviour, we store bzr-git's cache
- directory at .bzr/repository/git in the import data store.
- """
- non_trivial = PullingImportWorker.pushBazaarBranch(
- self, bazaar_branch)
- repo_base = bazaar_branch.repository._transport.base
- git_db_dir = os.path.join(local_path_from_url(repo_base), 'git')
- local_name = 'git-cache.tar.gz'
- create_tarball(git_db_dir, local_name)
- self.import_data_store.put(local_name)
- return non_trivial
-
-
-class BzrSvnImportWorker(PullingImportWorker):
- """An import worker for importing Subversion via bzr-svn."""
-
- @property
- def invalid_branch_exceptions(self):
- return [
- NoRepositoryPresent,
- NotBranchError,
- ConnectionError,
- ]
-
- @property
- def unsupported_feature_exceptions(self):
- from bzrlib.plugins.svn.errors import InvalidFileName
- return [
- InvalidEntryName,
- InvalidFileName,
- ]
-
- @property
- def broken_remote_exceptions(self):
- from bzrlib.plugins.svn.errors import IncompleteRepositoryHistory
- return [IncompleteRepositoryHistory]
-
- def getRevisionLimit(self):
- """See `PullingImportWorker.getRevisionLimit`."""
- return config.codeimport.svn_revisions_import_limit
-
- @property
- def probers(self):
- """See `PullingImportWorker.probers`."""
- from bzrlib.plugins.svn import SvnRemoteProber
- return [SvnRemoteProber]
-
- def getBazaarBranch(self):
- """See `ToBzrImportWorker.getBazaarBranch`.
-
- In addition to the superclass' behaviour, we retrieve bzr-svn's
- cache from the import data store and put it where bzr-svn will find
- it.
- """
- from bzrlib.plugins.svn.cache import create_cache_dir
- branch = super(BzrSvnImportWorker, self).getBazaarBranch()
- local_name = 'svn-cache.tar.gz'
- if self.import_data_store.fetch(local_name):
- extract_tarball(local_name, create_cache_dir())
- return branch
-
- def pushBazaarBranch(self, bazaar_branch, remote_branch=None):
- """See `ToBzrImportWorker.pushBazaarBranch`.
-
- In addition to the superclass' behaviour, we store bzr-svn's cache
- directory in the import data store.
- """
- from bzrlib.plugins.svn.cache import get_cache
- non_trivial = super(BzrSvnImportWorker, self).pushBazaarBranch(
- bazaar_branch)
- if remote_branch is not None:
- cache = get_cache(remote_branch.repository.uuid)
- cache_dir = cache.create_cache_dir()
- local_name = 'svn-cache.tar.gz'
- create_tarball(
- os.path.dirname(cache_dir), local_name,
- filenames=[os.path.basename(cache_dir)])
- self.import_data_store.put(local_name)
- # XXX cjwatson 2019-02-06: Once this is behaving well on
- # production, consider removing the local cache after pushing a
- # copy of it to the import data store.
- return non_trivial
-
-
-class BzrImportWorker(PullingImportWorker):
- """An import worker for importing Bazaar branches."""
-
- invalid_branch_exceptions = [
- NotBranchError,
- ConnectionError,
- ]
- unsupported_feature_exceptions = []
- broken_remote_exceptions = []
-
- def getRevisionLimit(self):
- """See `PullingImportWorker.getRevisionLimit`."""
- # For now, just grab the whole branch at once.
- # bzr does support fetch(limit=) but it isn't very efficient at
- # the moment.
- return None
-
- @property
- def probers(self):
- """See `PullingImportWorker.probers`."""
- from bzrlib.bzrdir import BzrProber, RemoteBzrProber
- return [BzrProber, RemoteBzrProber]
-
-
-class GitToGitImportWorker(ImportWorker):
- """An import worker for imports from Git to Git."""
-
- def _throttleProgress(self, file_obj, timeout=15.0):
- """Throttle progress messages from a file object.
-
- git can produce progress output on stderr, but it produces rather a
- lot of it and we don't want it all to end up in logs. Throttle this
- so that we only produce output every `timeout` seconds, or when we
- see a line terminated with a newline rather than a carriage return.
-
- :param file_obj: a file-like object opened in binary mode.
- :param timeout: emit progress output only after this many seconds
- have elapsed.
- :return: an iterator of interesting text lines read from the file.
- """
- # newline="" requests universal newlines mode, but without
- # translation.
- if six.PY2 and isinstance(file_obj, file):
- # A Python 2 file object can't be used directly to construct an
- # io.TextIOWrapper.
- class _ReadableFileWrapper:
- def __init__(self, raw):
- self._raw = raw
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, exc_tb):
- pass
-
- def readable(self):
- return True
-
- def writable(self):
- return False
-
- def seekable(self):
- return True
-
- def __getattr__(self, name):
- return getattr(self._raw, name)
-
- with _ReadableFileWrapper(file_obj) as readable:
- with io.BufferedReader(readable) as buffered:
- for line in self._throttleProgress(
- buffered, timeout=timeout):
- yield line
- return
-
- class ReceivedAlarm(Exception):
- pass
-
- def alarm_handler(signum, frame):
- raise ReceivedAlarm()
-
- old_alarm = signal.signal(signal.SIGALRM, alarm_handler)
- try:
- progress_buffer = None
- with io.TextIOWrapper(
- file_obj, encoding="UTF-8", errors="replace",
- newline="") as wrapped_file:
- while True:
- try:
- signal.setitimer(signal.ITIMER_REAL, timeout)
- line = next(wrapped_file)
- signal.setitimer(signal.ITIMER_REAL, 0)
- if line.endswith(u"\r"):
- progress_buffer = line
- else:
- if progress_buffer is not None:
- yield progress_buffer
- progress_buffer = None
- yield line
- except ReceivedAlarm:
- if progress_buffer is not None:
- yield progress_buffer
- progress_buffer = None
- except StopIteration:
- break
- finally:
- signal.setitimer(signal.ITIMER_REAL, 0)
- signal.signal(signal.SIGALRM, old_alarm)
-
- def _runGit(self, *args, **kwargs):
- """Run git with arguments, sending output to the logger."""
- cmd = ["git"] + list(args)
- git_process = subprocess.Popen(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
- for line in self._throttleProgress(git_process.stdout):
- self._logger.info(sanitise_urls(line.rstrip("\r\n")))
- retcode = git_process.wait()
- if retcode:
- raise subprocess.CalledProcessError(retcode, cmd)
-
- def _getHead(self, repository, remote_name):
- """Get HEAD from a configured remote in a local repository.
-
- The returned ref name will be adjusted in such a way that it can be
- passed to `_setHead` (e.g. refs/remotes/origin/master ->
- refs/heads/master).
- """
- # This is a bit weird, but set-head will bail out if the target
- # doesn't exist in the correct remotes namespace. git 2.8.0 has
- # "git ls-remote --symref <repository> HEAD" which would involve
- # less juggling.
- self._runGit(
- "fetch", "-q", ".", "refs/heads/*:refs/remotes/%s/*" % remote_name,
- cwd=repository)
- self._runGit(
- "remote", "set-head", remote_name, "--auto", cwd=repository)
- ref_prefix = "refs/remotes/%s/" % remote_name
- target_ref = subprocess.check_output(
- ["git", "symbolic-ref", ref_prefix + "HEAD"],
- cwd=repository, universal_newlines=True).rstrip("\n")
- if not target_ref.startswith(ref_prefix):
- raise GitProtocolError(
- "'git remote set-head %s --auto' did not leave remote HEAD "
- "under %s" % (remote_name, ref_prefix))
- real_target_ref = "refs/heads/" + target_ref[len(ref_prefix):]
- # Ensure the result is a valid ref name, just in case.
- self._runGit("check-ref-format", real_target_ref, cwd="repository")
- return real_target_ref
-
- def _setHead(self, target_url, target_ref):
- """Set HEAD on a remote repository.
-
- This relies on the turnip-set-symbolic-ref extension.
- """
- service = "turnip-set-symbolic-ref"
- url = urljoin(target_url, service)
- headers = {
- "Content-Type": "application/x-%s-request" % service,
- }
- body = pkt_line("HEAD %s" % target_ref) + pkt_line(None)
- try:
- response = urlfetch(url, method="POST", headers=headers, data=body)
- raise_for_status_redacted(response)
- except Exception as e:
- raise GitProtocolError(str(e))
- content_type = response.headers.get("Content-Type")
- if content_type != ("application/x-%s-result" % service):
- raise GitProtocolError(
- "Invalid Content-Type from server: %s" % content_type)
- content = io.BytesIO(response.content)
- proto = Protocol(content.read, None)
- pkt = proto.read_pkt_line()
- if pkt is None:
- raise GitProtocolError("Unexpected flush-pkt from server")
- elif pkt.rstrip(b"\n") == b"ACK HEAD":
- pass
- elif pkt.startswith(b"ERR "):
- raise GitProtocolError(
- pkt[len(b"ERR "):].rstrip(b"\n").decode("UTF-8"))
- else:
- raise GitProtocolError("Unexpected packet %r from server" % pkt)
-
- def _deleteRefs(self, repository, pattern):
- """Delete all refs in `repository` matching `pattern`."""
- # XXX cjwatson 2016-11-08: We might ideally use something like:
- # "git for-each-ref --format='delete %(refname)%00%(objectname)%00' \
- # <pattern> | git update-ref --stdin -z
- # ... which would be faster, but that requires git 1.8.5.
- remote_refs = subprocess.check_output(
- ["git", "for-each-ref", "--format=%(refname)", pattern],
- cwd="repository").splitlines()
- for remote_ref in remote_refs:
- self._runGit("update-ref", "-d", remote_ref, cwd="repository")
-
- def _doImport(self):
- self._logger.info("Starting job.")
- try:
- self._opener_policy.check_one_url(self.source_details.url)
- except BadUrl as e:
- self._logger.info("Invalid URL: %s" % e)
- return CodeImportWorkerExitCode.FAILURE_FORBIDDEN
- unauth_target_url = urljoin(
- config.codehosting.git_browse_root, self.source_details.target_id)
- split = urlsplit(unauth_target_url)
- target_netloc = ":%s@%s" % (
- self.source_details.macaroon.serialize(), split.hostname)
- if split.port:
- target_netloc += ":%s" % split.port
- target_url = urlunsplit([
- split.scheme, target_netloc, split.path, "", ""])
- # XXX cjwatson 2016-10-11: Ideally we'd put credentials in a
- # credentials store instead. However, git only accepts credentials
- # that have both a non-empty username and a non-empty password.
- self._logger.info("Getting existing repository from hosting service.")
- try:
- self._runGit("clone", "--mirror", target_url, "repository")
- except subprocess.CalledProcessError as e:
- self._logger.info(
- "Unable to get existing repository from hosting service: "
- "git clone exited %s" % e.returncode)
- return CodeImportWorkerExitCode.FAILURE
- self._logger.info("Fetching remote repository.")
- try:
- self._runGit("config", "gc.auto", "0", cwd="repository")
- # Remove any stray remote-tracking refs from the last time round.
- self._deleteRefs("repository", "refs/remotes/source/**")
- self._runGit(
- "remote", "add", "source", self.source_details.url,
- cwd="repository")
- self._runGit(
- "fetch", "--prune", "source", "+refs/*:refs/*",
- cwd="repository")
- try:
- new_head = self._getHead("repository", "source")
- except (subprocess.CalledProcessError, GitProtocolError) as e2:
- self._logger.info("Unable to fetch default branch: %s" % e2)
- new_head = None
- self._runGit("remote", "rm", "source", cwd="repository")
- # XXX cjwatson 2016-11-03: For some reason "git remote rm"
- # doesn't actually remove the refs.
- self._deleteRefs("repository", "refs/remotes/source/**")
- except subprocess.CalledProcessError as e:
- self._logger.info("Unable to fetch remote repository: %s" % e)
- return CodeImportWorkerExitCode.FAILURE_INVALID
- self._logger.info("Pushing repository to hosting service.")
- try:
- if new_head is not None:
- # Push the target of HEAD first to ensure that it is always
- # available.
- self._runGit(
- "push", "--progress", target_url,
- "+%s:%s" % (new_head, new_head), cwd="repository")
- try:
- self._setHead(target_url, new_head)
- except GitProtocolError as e:
- self._logger.info("Unable to set default branch: %s" % e)
- self._runGit(
- "push", "--progress", "--mirror", target_url, cwd="repository")
- except subprocess.CalledProcessError as e:
- self._logger.info(
- "Unable to push to hosting service: git push exited %s" %
- e.returncode)
- return CodeImportWorkerExitCode.FAILURE
- return CodeImportWorkerExitCode.SUCCESS
diff --git a/lib/lp/codehosting/codeimport/workermonitor.py b/lib/lp/codehosting/codeimport/workermonitor.py
deleted file mode 100644
index 6fa304b..0000000
--- a/lib/lp/codehosting/codeimport/workermonitor.py
+++ /dev/null
@@ -1,252 +0,0 @@
-# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Code to talk to the database about what the worker script is doing."""
-
-from __future__ import absolute_import, print_function
-
-__metaclass__ = type
-__all__ = []
-
-
-import os
-import tempfile
-
-import six
-from six.moves import xmlrpc_client
-from twisted.internet import (
- defer,
- error,
- reactor,
- task,
- )
-from twisted.python import failure
-from twisted.web import xmlrpc
-
-from lp.code.enums import CodeImportResultStatus
-from lp.codehosting.codeimport.worker import CodeImportWorkerExitCode
-from lp.services.config import config
-from lp.services.twistedsupport.processmonitor import (
- ProcessMonitorProtocolWithTimeout,
- )
-from lp.xmlrpc.faults import NoSuchCodeImportJob
-
-
-class CodeImportWorkerMonitorProtocol(ProcessMonitorProtocolWithTimeout):
- """The protocol by which the child process talks to the monitor.
-
- In terms of bytes, the protocol is extremely simple: any output is stored
- in the log file and seen as timeout-resetting activity. Every
- config.codeimportworker.heartbeat_update_interval seconds we ask the
- monitor to update the heartbeat of the job we are working on and pass the
- tail of the log output.
- """
-
- def __init__(self, deferred, worker_monitor, log_file, clock=None):
- """Construct an instance.
-
- :param deferred: See `ProcessMonitorProtocol.__init__` -- the deferred
- that will be fired when the process has exited.
- :param worker_monitor: A `CodeImportWorkerMonitor` instance.
- :param log_file: A file object that the output of the child
- process will be logged to.
- :param clock: A provider of Twisted's IReactorTime. This parameter
- exists to allow testing that does not depend on an external clock.
- If a clock is not passed in explicitly the reactor is used.
- """
- ProcessMonitorProtocolWithTimeout.__init__(
- self, deferred, clock=clock,
- timeout=config.codeimport.worker_inactivity_timeout)
- self.worker_monitor = worker_monitor
- self._tail = b''
- self._log_file = log_file
- self._looping_call = task.LoopingCall(self._updateHeartbeat)
- self._looping_call.clock = self._clock
-
- def connectionMade(self):
- """See `BaseProtocol.connectionMade`.
-
- We call updateHeartbeat for the first time when we are connected to
- the process and every
- config.codeimportworker.heartbeat_update_interval seconds thereafter.
- """
- ProcessMonitorProtocolWithTimeout.connectionMade(self)
- self._looping_call.start(
- config.codeimportworker.heartbeat_update_interval)
-
- def _updateHeartbeat(self):
- """Ask the monitor to update the heartbeat.
-
- We use runNotification() to serialize the updates and ensure
- that any errors are handled properly. We do not return the
- deferred, as we want this function to be called at a frequency
- independent of how long it takes to update the heartbeat."""
- self.runNotification(
- self.worker_monitor.updateHeartbeat, self._tail)
-
- def outReceived(self, data):
- """See `ProcessProtocol.outReceived`.
-
- Any output resets the timeout, is stored in the logfile and
- updates the tail of the log.
- """
- self.resetTimeout()
- self._log_file.write(data)
- self._tail = b'\n'.join((self._tail + data).split(b'\n')[-5:])
-
- errReceived = outReceived
-
- def processEnded(self, reason):
- """See `ProcessMonitorProtocolWithTimeout.processEnded`.
-
- We stop updating the heartbeat when the process exits.
- """
- ProcessMonitorProtocolWithTimeout.processEnded(self, reason)
- self._looping_call.stop()
-
-
-class ExitQuietly(Exception):
- """Raised to indicate that we should abort and exit without fuss.
-
- Raised when the job we are working on disappears, as we assume
- this is the result of the job being killed or reclaimed.
- """
- pass
-
-
-class CodeImportWorkerMonitor:
- """Controller for a single import job.
-
- An instance of `CodeImportWorkerMonitor` runs a child process to
- perform an import and communicates status to the database.
- """
-
- path_to_script = os.path.join(
- config.root, 'scripts', 'code-import-worker.py')
-
- def __init__(self, job_id, logger, codeimport_endpoint, access_policy):
- """Construct an instance.
-
- :param job_id: The ID of the CodeImportJob we are to work on.
- :param logger: A `Logger` object.
- """
- self._job_id = job_id
- self._logger = logger
- self.codeimport_endpoint = codeimport_endpoint
- self._call_finish_job = True
- self._log_file = tempfile.TemporaryFile()
- self._access_policy = access_policy
-
- def _trap_nosuchcodeimportjob(self, failure):
- failure.trap(xmlrpc.Fault)
- if failure.value.faultCode == NoSuchCodeImportJob.error_code:
- self._call_finish_job = False
- raise ExitQuietly
- else:
- raise failure.value
-
- def getWorkerArguments(self):
- """Get arguments for the worker for the import we are working on."""
- deferred = self.codeimport_endpoint.callRemote(
- 'getImportDataForJobID', self._job_id)
-
- def _processResult(result):
- code_import_arguments = result['arguments']
- self._logger.info(
- 'Found source details: %s', code_import_arguments)
- return code_import_arguments
- return deferred.addCallbacks(
- _processResult, self._trap_nosuchcodeimportjob)
-
- def updateHeartbeat(self, tail):
- """Call the updateHeartbeat method for the job we are working on."""
- self._logger.debug("Updating heartbeat.")
- # The log tail is really bytes, but it's stored in the database as a
- # text column, so it's easiest to convert it to text now; passing
- # text over XML-RPC requires less boilerplate than bytes anyway.
- deferred = self.codeimport_endpoint.callRemote(
- 'updateHeartbeat', self._job_id,
- six.ensure_text(tail, errors='replace'))
- return deferred.addErrback(self._trap_nosuchcodeimportjob)
-
- def finishJob(self, status):
- """Call the finishJobID method for the job we are working on."""
- self._log_file.seek(0)
- return self.codeimport_endpoint.callRemote(
- 'finishJobID', self._job_id, status.name,
- xmlrpc_client.Binary(self._log_file.read())
- ).addErrback(self._trap_nosuchcodeimportjob)
-
- def _makeProcessProtocol(self, deferred):
- """Make an `CodeImportWorkerMonitorProtocol` for a subprocess."""
- return CodeImportWorkerMonitorProtocol(deferred, self, self._log_file)
-
- def _launchProcess(self, worker_arguments):
- """Launch the code-import-worker.py child process."""
- deferred = defer.Deferred()
- protocol = self._makeProcessProtocol(deferred)
- interpreter = '%s/bin/py' % config.root
- args = [interpreter, self.path_to_script]
- if self._access_policy is not None:
- args.append("--access-policy=%s" % self._access_policy)
- args.append('--')
- command = args + worker_arguments
- self._logger.info(
- "Launching worker child process %s.", command)
- reactor.spawnProcess(
- protocol, interpreter, command, env=os.environ, usePTY=True)
- return deferred
-
- def run(self):
- """Perform the import."""
- return self.getWorkerArguments().addCallback(
- self._launchProcess).addBoth(
- self.callFinishJob).addErrback(
- self._silenceQuietExit)
-
- def _silenceQuietExit(self, failure):
- """Quietly swallow a ExitQuietly failure."""
- failure.trap(ExitQuietly)
- return None
-
- def _reasonToStatus(self, reason):
- """Translate the 'reason' for process exit into a result status.
-
- Different exit codes are presumed by Twisted to be errors, but are
- different kinds of success for us.
- """
- exit_code_map = {
- CodeImportWorkerExitCode.SUCCESS_NOCHANGE:
- CodeImportResultStatus.SUCCESS_NOCHANGE,
- CodeImportWorkerExitCode.SUCCESS_PARTIAL:
- CodeImportResultStatus.SUCCESS_PARTIAL,
- CodeImportWorkerExitCode.FAILURE_UNSUPPORTED_FEATURE:
- CodeImportResultStatus.FAILURE_UNSUPPORTED_FEATURE,
- CodeImportWorkerExitCode.FAILURE_INVALID:
- CodeImportResultStatus.FAILURE_INVALID,
- CodeImportWorkerExitCode.FAILURE_FORBIDDEN:
- CodeImportResultStatus.FAILURE_FORBIDDEN,
- CodeImportWorkerExitCode.FAILURE_REMOTE_BROKEN:
- CodeImportResultStatus.FAILURE_REMOTE_BROKEN,
- }
- if isinstance(reason, failure.Failure):
- if reason.check(error.ProcessTerminated):
- return exit_code_map.get(reason.value.exitCode,
- CodeImportResultStatus.FAILURE)
- return CodeImportResultStatus.FAILURE
- else:
- return CodeImportResultStatus.SUCCESS
-
- def callFinishJob(self, reason):
- """Call finishJob() with the appropriate status."""
- if not self._call_finish_job:
- return reason
- status = self._reasonToStatus(reason)
- if status == CodeImportResultStatus.FAILURE:
- self._log_file.write(b"Import failed:\n")
- self._log_file.write(reason.getTraceback().encode("UTF-8"))
- self._logger.info(
- "Import failed: %s: %s" % (reason.type, reason.value))
- else:
- self._logger.info('Import succeeded.')
- return self.finishJob(status)
diff --git a/lib/lp/codehosting/puller/worker.py b/lib/lp/codehosting/puller/worker.py
index c0b426f..7dd7073 100644
--- a/lib/lp/codehosting/puller/worker.py
+++ b/lib/lp/codehosting/puller/worker.py
@@ -495,8 +495,7 @@ class PullerWorkerUIFactory(SilentUIFactory):
def confirm_action(self, prompt, confirmation_id, args):
"""If we're asked to break a lock like a stale lock of ours, say yes.
"""
- if confirmation_id not in (
- 'bzrlib.lockdir.break', 'breezy.lockdir.break'):
+ if confirmation_id != 'breezy.lockdir.break':
raise AssertionError(
"Didn't expect confirmation id %r" % (confirmation_id,))
branch_id = self.puller_worker_protocol.branch_id
diff --git a/lib/lp/codehosting/tests/helpers.py b/lib/lp/codehosting/tests/helpers.py
index 295acaa..81fbea1 100644
--- a/lib/lp/codehosting/tests/helpers.py
+++ b/lib/lp/codehosting/tests/helpers.py
@@ -16,6 +16,7 @@ __all__ = [
import os
+from breezy.controldir import ControlDir
from breezy.errors import FileExists
from breezy.plugins.loom import branch as loom_branch
from breezy.tests import (
@@ -86,13 +87,10 @@ class LoomTestMixin:
def create_branch_with_one_revision(branch_dir, format=None):
"""Create a dummy Bazaar branch at the given directory."""
- # XXX cjwatson 2019-06-13: This still uses bzrlib until such time as the
- # code import workers are ported to Breezy.
- from bzrlib.bzrdir import BzrDir
if not os.path.exists(branch_dir):
os.makedirs(branch_dir)
try:
- tree = BzrDir.create_standalone_workingtree(branch_dir, format)
+ tree = ControlDir.create_standalone_workingtree(branch_dir, format)
except FileExists:
return
f = open(os.path.join(branch_dir, 'hello'), 'w')
diff --git a/lib/lp/scripts/utilities/test.py b/lib/lp/scripts/utilities/test.py
index 644c496..3d6753f 100755
--- a/lib/lp/scripts/utilities/test.py
+++ b/lib/lp/scripts/utilities/test.py
@@ -120,9 +120,6 @@ def filter_warnings():
warnings.filterwarnings(
'ignore', 'twisted.python.plugin', DeprecationWarning,
)
- warnings.filterwarnings(
- 'ignore', 'bzrlib.*was deprecated', DeprecationWarning,
- )
# XXX cjwatson 2019-10-18: This can be dropped once the port to Breezy
# is complete.
warnings.filterwarnings(
diff --git a/lib/lp/testing/__init__.py b/lib/lp/testing/__init__.py
index b9f7c58..55759e8 100644
--- a/lib/lp/testing/__init__.py
+++ b/lib/lp/testing/__init__.py
@@ -179,9 +179,6 @@ from lp.testing.fixture import (
from lp.testing.karma import KarmaRecorder
from lp.testing.mail_helpers import pop_notifications
-if six.PY2:
- from bzrlib import trace as bzr_trace
-
# The following names have been imported for the purpose of being
# exported. They are referred to here to silence lint warnings.
admin_logged_in
@@ -842,8 +839,6 @@ class TestCaseWithFactory(TestCase):
# make it so in order to avoid "No handlers for "brz" logger'
# messages.
trace._brz_logger = logging.getLogger('brz')
- if six.PY2:
- bzr_trace._bzr_logger = logging.getLogger('bzr')
def getUserBrowser(self, url=None, user=None):
"""Return a Browser logged in as a fresh user, maybe opened at `url`.
diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
index 685a142..7d60973 100644
--- a/lib/lp/testing/factory.py
+++ b/lib/lp/testing/factory.py
@@ -520,44 +520,6 @@ class ObjectFactory(
epoch = datetime(2009, 1, 1, tzinfo=pytz.UTC)
return epoch + timedelta(minutes=self.getUniqueInteger())
- def makeCodeImportSourceDetails(self, target_id=None, rcstype=None,
- target_rcstype=None, url=None,
- cvs_root=None, cvs_module=None,
- stacked_on_url=None, macaroon=None):
- if not six.PY2:
- raise NotImplementedError(
- "Code imports do not yet work on Python 3.")
-
- # XXX cjwatson 2020-08-07: Move this back up to module level once
- # codeimport has been ported to Breezy.
- from lp.codehosting.codeimport.worker import CodeImportSourceDetails
-
- if target_id is None:
- target_id = self.getUniqueInteger()
- if rcstype is None:
- rcstype = 'bzr-svn'
- if target_rcstype is None:
- target_rcstype = 'bzr'
- if rcstype in ['bzr-svn', 'bzr']:
- assert cvs_root is cvs_module is None
- if url is None:
- url = self.getUniqueURL()
- elif rcstype == 'cvs':
- assert url is None
- if cvs_root is None:
- cvs_root = self.getUniqueUnicode()
- if cvs_module is None:
- cvs_module = self.getUniqueUnicode()
- elif rcstype == 'git':
- assert cvs_root is cvs_module is None
- if url is None:
- url = self.getUniqueURL(scheme='git')
- else:
- raise AssertionError("Unknown rcstype %r." % rcstype)
- return CodeImportSourceDetails(
- target_id, rcstype, target_rcstype, url, cvs_root, cvs_module,
- stacked_on_url=stacked_on_url, macaroon=macaroon)
-
class BareLaunchpadObjectFactory(ObjectFactory):
"""Factory methods for creating Launchpad objects.
diff --git a/requirements/launchpad.txt b/requirements/launchpad.txt
index c80a759..d280824 100644
--- a/requirements/launchpad.txt
+++ b/requirements/launchpad.txt
@@ -21,8 +21,6 @@ breezy==3.0.1
bson==0.5.9
boto3==1.16.2
botocore==1.19.2
-# lp:~launchpad/bzr/lp
-bzr==2.6.0.lp.4
celery==4.1.1
Chameleon==3.6.2
configobj==5.0.6
@@ -146,7 +144,6 @@ statsd==3.3.0
# lp:~launchpad-committers/storm/lp
storm==0.24+lp416
subprocess32==3.2.6
-subvertpy==0.9.1
tenacity==6.1.0
testresources==0.2.7
testscenarios==0.4
diff --git a/scripts/code-import-worker-monitor.py b/scripts/code-import-worker-monitor.py
deleted file mode 100755
index a505225..0000000
--- a/scripts/code-import-worker-monitor.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/python2 -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""When passed a CodeImportJob id on the command line, process that job.
-
-The actual work of processing a job is done by the code-import-worker.py
-script which this process runs as a child process and updates the database on
-its progress and result.
-
-This script is usually run by the code-import-dispatcher cronscript.
-"""
-
-__metaclass__ = type
-
-
-import _pythonpath
-
-import os
-
-from twisted.internet import (
- defer,
- reactor,
- )
-from twisted.python import log
-from twisted.web import xmlrpc
-
-from lp.codehosting.codeimport.workermonitor import CodeImportWorkerMonitor
-from lp.services.config import config
-from lp.services.scripts.base import LaunchpadScript
-from lp.services.twistedsupport.loggingsupport import set_up_oops_reporting
-
-
-class CodeImportWorker(LaunchpadScript):
-
- def __init__(self, name, dbuser=None, test_args=None):
- LaunchpadScript.__init__(self, name, dbuser, test_args)
- # The logfile changes its name according to the code in
- # CodeImportDispatcher, so we pull it from the command line
- # options.
- set_up_oops_reporting(
- self.name, 'codeimportworker', logfile=self.options.log_file)
-
- def add_my_options(self):
- """See `LaunchpadScript`."""
- self.parser.add_option(
- "--access-policy", type="choice", metavar="ACCESS_POLICY",
- choices=["anything", "default"], default=None)
-
- def _init_db(self, isolation):
- # This script doesn't access the database.
- pass
-
- def main(self):
- arg, = self.args
- job_id = int(arg)
- # XXX: MichaelHudson 2008-05-07 bug=227586: Setting up the component
- # architecture overrides $GNUPGHOME to something stupid.
- os.environ['GNUPGHOME'] = ''
- reactor.callWhenRunning(self._do_import, job_id)
- reactor.run()
-
- def _do_import(self, job_id):
- defer.maybeDeferred(self._main, job_id).addErrback(
- log.err).addCallback(
- lambda ignored: reactor.stop())
-
- def _main(self, job_id):
- worker = CodeImportWorkerMonitor(
- job_id, self.logger,
- xmlrpc.Proxy(
- config.codeimportdispatcher.codeimportscheduler_url.encode(
- 'UTF-8')),
- self.options.access_policy)
- return worker.run()
-
-if __name__ == '__main__':
- script = CodeImportWorker('codeimportworker')
- script.run()
diff --git a/scripts/code-import-worker.py b/scripts/code-import-worker.py
deleted file mode 100755
index 434b449..0000000
--- a/scripts/code-import-worker.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/python2 -S
-#
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Process a code import described by the command line arguments.
-
-By 'processing a code import' we mean importing or updating code from a
-remote, non-Bazaar, repository.
-
-This script is usually run by the code-import-worker-monitor.py script that
-communicates progress and results to the database.
-"""
-
-__metaclass__ = type
-
-
-import _pythonpath
-
-from optparse import OptionParser
-import sys
-
-from bzrlib.transport import get_transport
-from bzrlib.url_policy_open import AcceptAnythingPolicy
-
-from lp.codehosting.codeimport.worker import (
- BzrImportWorker,
- BzrSvnImportWorker,
- CodeImportBranchOpenPolicy,
- CodeImportSourceDetails,
- CSCVSImportWorker,
- get_default_bazaar_branch_store,
- GitImportWorker,
- GitToGitImportWorker,
- )
-from lp.services import scripts
-from lp.services.config import config
-from lp.services.timeout import set_default_timeout_function
-
-
-opener_policies = {
- "anything": (
- lambda rcstype, target_rcstype, exclude_hosts=None:
- AcceptAnythingPolicy()),
- "default": CodeImportBranchOpenPolicy,
- }
-
-
-def force_bzr_to_use_urllib():
- """Prevent bzr from using pycurl to connect to http: urls.
-
- We want this because pycurl rejects self signed certificates, which
- prevents a significant number of import branchs from updating. Also see
- https://bugs.launchpad.net/bzr/+bug/516222.
- """
- from bzrlib.transport import register_lazy_transport
- register_lazy_transport('http://', 'bzrlib.transport.http._urllib',
- 'HttpTransport_urllib')
- register_lazy_transport('https://', 'bzrlib.transport.http._urllib',
- 'HttpTransport_urllib')
-
-
-class CodeImportWorker:
-
- def __init__(self):
- parser = OptionParser()
- scripts.logger_options(parser)
- parser.add_option(
- "--access-policy", type="choice", metavar="ACCESS_POLICY",
- choices=["anything", "default"], default="default",
- help="Access policy to use when accessing branches to import.")
- self.options, self.args = parser.parse_args()
- self.logger = scripts.logger(self.options, 'code-import-worker')
-
- def main(self):
- force_bzr_to_use_urllib()
- set_default_timeout_function(lambda: 60.0)
- source_details = CodeImportSourceDetails.fromArguments(self.args)
- if source_details.rcstype == 'git':
- if source_details.target_rcstype == 'bzr':
- import_worker_cls = GitImportWorker
- else:
- import_worker_cls = GitToGitImportWorker
- elif source_details.rcstype == 'bzr-svn':
- import_worker_cls = BzrSvnImportWorker
- elif source_details.rcstype == 'bzr':
- import_worker_cls = BzrImportWorker
- elif source_details.rcstype == 'cvs':
- import_worker_cls = CSCVSImportWorker
- else:
- raise AssertionError(
- 'unknown rcstype %r' % source_details.rcstype)
- opener_policy = opener_policies[self.options.access_policy](
- source_details.rcstype, source_details.target_rcstype,
- exclude_hosts=source_details.exclude_hosts)
- if source_details.target_rcstype == 'bzr':
- import_worker = import_worker_cls(
- source_details,
- get_transport(config.codeimport.foreign_tree_store),
- get_default_bazaar_branch_store(), self.logger, opener_policy)
- else:
- import_worker = import_worker_cls(
- source_details, self.logger, opener_policy)
- return import_worker.run()
-
-
-if __name__ == '__main__':
- script = CodeImportWorker()
- sys.exit(script.main())
diff --git a/setup.py b/setup.py
index de4fafa..b5696f3 100644
--- a/setup.py
+++ b/setup.py
@@ -155,10 +155,6 @@ setup(
'beautifulsoup4[lxml]',
'boto3',
'breezy',
- # XXX cjwatson 2020-08-07: This should eventually be removed
- # entirely, but we need to retain it until codeimport has been
- # ported to Breezy.
- 'bzr; python_version < "3"',
'celery',
'contextlib2; python_version < "3.3"',
'cssselect',
@@ -237,9 +233,6 @@ setup(
'Sphinx',
'statsd',
'storm',
- # XXX cjwatson 2020-08-07: Temporarily dropped on Python 3 until
- # codeimport can be ported to Breezy.
- 'subvertpy; python_version < "3"',
'tenacity',
'testscenarios',
'testtools',
diff --git a/system-packages.txt b/system-packages.txt
index 6812444..11a6f29 100644
--- a/system-packages.txt
+++ b/system-packages.txt
@@ -31,6 +31,3 @@ PIL
# Dependency of cscvs.
sqlite; python_version < "3"
_sqlite; python_version < "3"
-
-# Used by bzr-git and bzr-svn.
-tdb
diff --git a/utilities/mock-code-import b/utilities/mock-code-import
deleted file mode 100755
index 524289e..0000000
--- a/utilities/mock-code-import
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/python2 -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Make a Subversion repostiory and then make a CodeImportJob for it.
-
-USAGE: ./utilities/mock-code-import
-
-Run 'make schema' first! This utility mutates the DB and doesn't restore
-afterwards.
-
-This means that the valid_vcs_details constraint on CodeImport is lost and
-that there are new, crappy test objects in the DB. The utility will bork on
-these when run again.
-
-Details of the Subversion server are printed to stdout.
-"""
-
-# XXX: JonathanLange 2008-01-03: This is deliberately horrible.
-# You can make it nicer if you want.
-
-from __future__ import absolute_import, print_function
-
-import _pythonpath
-
-import os
-from subprocess import (
- PIPE,
- Popen,
- )
-import tempfile
-
-import transaction
-
-from lp.codehosting.codeimport.tests.test_foreigntree import SubversionServer
-from lp.services.scripts import execute_zcml_for_scripts
-from lp.services.webapp import canonical_url
-from lp.testing.factory import LaunchpadObjectFactory
-
-
-def shell(*args):
- print(' '.join(args))
- return Popen(args, stdout=PIPE).communicate()[0]
-
-
-def make_import_job(svn_url):
- factory = LaunchpadObjectFactory()
- code_import = factory.makeCodeImport(svn_branch_url=svn_url)
- return factory.makeCodeImportJob(code_import)
-
-
-def main():
- execute_zcml_for_scripts(use_web_security=False)
- temp_directory = tempfile.mkdtemp()
- svn_repo_path = os.path.join(temp_directory, 'svn-repository')
- svn_server = SubversionServer(svn_repo_path)
- svn_server.setUp()
- try:
- svn_url = svn_server.makeBranch(
- 'trunk', [('README', 'No real content\n.')])
- job = make_import_job(svn_url)
- transaction.commit()
- print("CodeImportJob.id:", job.id)
- print("Code Import URL:", canonical_url(job.code_import))
- print("Subversion Repository:", svn_repo_path)
- print("Subversion branch URL:", job.code_import.svn_branch_url)
- print("Launchpad branch URL:", canonical_url(job.code_import.branch))
- print()
- finally:
- svn_server.tearDown()
-
-
-if __name__ == '__main__':
- main()
diff --git a/utilities/sourcedeps.cache b/utilities/sourcedeps.cache
index 906459e..99f67fb 100644
--- a/utilities/sourcedeps.cache
+++ b/utilities/sourcedeps.cache
@@ -7,14 +7,6 @@
166,
"jelmer@xxxxxxxxx-20190822193925-ydrq7fgdi78lpgm7"
],
- "bzr-git": [
- 280,
- "launchpad@xxxxxxxxxxxxxxxxx-20171222005919-u98ut0f5z2g618um"
- ],
- "bzr-svn": [
- 2725,
- "launchpad@xxxxxxxxxxxxxxxxx-20130816045016-wzr810hu2z459t4y"
- ],
"cscvs": [
433,
"launchpad@xxxxxxxxxxxxxxxxx-20130816043319-bts3l3bckmx431q1"
diff --git a/utilities/sourcedeps.conf b/utilities/sourcedeps.conf
index 59773ad..7755ec7 100644
--- a/utilities/sourcedeps.conf
+++ b/utilities/sourcedeps.conf
@@ -9,8 +9,6 @@
brz-builder lp:~jelmer/brz-builder/trunk;revno=183
brz-loom lp:~jelmer/brz-loom/trunk;revno=166
-bzr-git lp:~launchpad-pqm/bzr-git/devel;revno=280
-bzr-svn lp:~launchpad-pqm/bzr-svn/devel;revno=2725
cscvs lp:~launchpad-pqm/launchpad-cscvs/devel;revno=433
difftacular lp:~launchpad/difftacular/trunk;revno=11
loggerhead lp:~loggerhead-team/loggerhead/trunk-rich;revno=511
