launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #21080
[Merge] lp:~cjwatson/launchpad/codeimport-git-worker into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/codeimport-git-worker into lp:launchpad with lp:~cjwatson/launchpad/codeimport-create-hosting as a prerequisite.
Commit message:
Add a Git-to-Git import worker.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1469459 in Launchpad itself: "import external code into a LP git repo (natively)"
https://bugs.launchpad.net/launchpad/+bug/1469459
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/codeimport-git-worker/+merge/308142
This takes the easy approach for now: clone existing imported repository, fetch-mirror remote repository into it, and push it back. Once turnip has more advanced plumbing then we can always switch over to use that (although it would involve some work setting up a turnip fixture).
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/codeimport-git-worker into lp:launchpad.
=== modified file 'configs/development/launchpad-lazr.conf'
--- configs/development/launchpad-lazr.conf 2016-06-30 16:05:11 +0000
+++ configs/development/launchpad-lazr.conf 2016-10-11 15:36:32 +0000
@@ -55,6 +55,7 @@
[codeimport]
bazaar_branch_store: file:///tmp/bazaar-branches
+git_repository_store: https://git.launchpad.dev/
foreign_tree_store: file:///tmp/foreign-branches
[codeimportdispatcher]
=== modified file 'configs/testrunner-appserver/launchpad-lazr.conf'
--- configs/testrunner-appserver/launchpad-lazr.conf 2014-02-27 08:39:44 +0000
+++ configs/testrunner-appserver/launchpad-lazr.conf 2016-10-11 15:36:32 +0000
@@ -8,6 +8,9 @@
[codehosting]
launch: False
+[codeimport]
+macaroon_secret_key: dev-macaroon-secret
+
[google_test_service]
launch: False
=== modified file 'lib/lp/codehosting/codeimport/tests/servers.py'
--- lib/lp/codehosting/codeimport/tests/servers.py 2016-02-05 16:51:12 +0000
+++ lib/lp/codehosting/codeimport/tests/servers.py 2016-10-11 15:36:32 +0000
@@ -253,12 +253,15 @@
else:
return local_path_to_url(self.repository_path)
- def createRepository(self, path):
- GitRepo.init(path)
+ def createRepository(self, path, bare=False):
+ if bare:
+ GitRepo.init_bare(path)
+ else:
+ GitRepo.init(path)
- def start_server(self):
+ def start_server(self, bare=False):
super(GitServer, self).start_server()
- self.createRepository(self.repository_path)
+ self.createRepository(self.repository_path, bare=bare)
if self._use_server:
repo = GitRepo(self.repository_path)
self._server = TCPGitServerThread(
=== modified file 'lib/lp/codehosting/codeimport/tests/test_worker.py'
--- lib/lp/codehosting/codeimport/tests/test_worker.py 2016-10-11 15:36:32 +0000
+++ lib/lp/codehosting/codeimport/tests/test_worker.py 2016-10-11 15:36:32 +0000
@@ -45,16 +45,26 @@
)
from dulwich.repo import Repo as GitRepo
from fixtures import FakeLogger
+from pymacaroons import Macaroon
import subvertpy
import subvertpy.client
import subvertpy.ra
-from testtools.matchers import Equals
+from testtools.matchers import (
+ Equals,
+ Matcher,
+ MatchesListwise,
+ Mismatch,
+ )
+from zope.component import getUtility
from lp.app.enums import InformationType
+from lp.code.enums import TargetRevisionControlSystems
from lp.code.interfaces.codehosting import (
branch_id_alias,
compose_public_url,
)
+from lp.code.interfaces.codeimportjob import ICodeImportJobWorkflow
+from lp.code.tests.helpers import GitHostingFixture
import lp.codehosting
from lp.codehosting.codeimport.tarball import (
create_tarball,
@@ -90,7 +100,9 @@
from lp.codehosting.tests.helpers import create_branch_with_one_revision
from lp.services.config import config
from lp.services.log.logger import BufferLogger
+from lp.services.macaroons.interfaces import IMacaroonIssuer
from lp.testing import (
+ celebrity_logged_in,
TestCase,
TestCaseWithFactory,
)
@@ -1049,7 +1061,7 @@
worker = self.makeImportWorker(
self.factory.makeCodeImportSourceDetails(
rcstype=self.rcstype, url="file:///local/path"),
- opener_policy=CodeImportBranchOpenPolicy("bzr"))
+ opener_policy=CodeImportBranchOpenPolicy("bzr", "bzr"))
self.assertEqual(
CodeImportWorkerExitCode.FAILURE_FORBIDDEN, worker.run())
@@ -1285,7 +1297,7 @@
def setUp(self):
super(CodeImportBranchOpenPolicyTests, self).setUp()
- self.policy = CodeImportBranchOpenPolicy("bzr")
+ self.policy = CodeImportBranchOpenPolicy("bzr", "bzr")
def test_follows_references(self):
self.assertEquals(True, self.policy.shouldFollowReferences())
@@ -1309,14 +1321,22 @@
self.assertGoodUrl("bzr://bzr.example.com/somebzrurl/")
self.assertBadUrl("bzr://bazaar.launchpad.dev/example")
- def test_checkOneURL_git(self):
- self.policy = CodeImportBranchOpenPolicy("git")
+ def test_checkOneURL_git_to_bzr(self):
+ self.policy = CodeImportBranchOpenPolicy("git", "bzr")
self.assertBadUrl("/etc/passwd")
self.assertBadUrl("file:///etc/passwd")
self.assertBadUrl("unknown-scheme://devpad/")
self.assertGoodUrl("git://git.example.com/repo")
self.assertGoodUrl("git://git.launchpad.dev/example")
+ def test_checkOneURL_git_to_git(self):
+ self.policy = CodeImportBranchOpenPolicy("git", "git")
+ self.assertBadUrl("/etc/passwd")
+ self.assertBadUrl("file:///etc/passwd")
+ self.assertBadUrl("unknown-scheme://devpad/")
+ self.assertGoodUrl("git://git.example.com/repo")
+ self.assertBadUrl("git://git.launchpad.dev/example")
+
class RedirectTests(http_utils.TestCaseWithRedirectedWebserver, TestCase):
@@ -1387,6 +1407,19 @@
CodeImportWorkerExitCode.FAILURE_INVALID, worker.run())
+class CodeImportJobMacaroonVerifies(Matcher):
+ """Matches if a code-import-job macaroon can be verified."""
+
+ def __init__(self, context):
+ self.context = context
+
+ def match(self, macaroon_raw):
+ issuer = getUtility(IMacaroonIssuer, 'code-import-job')
+ macaroon = Macaroon.deserialize(macaroon_raw)
+ if not issuer.verifyMacaroon(macaroon, self.context):
+ return Mismatch("Macaroon '%s' does not verify" % macaroon_raw)
+
+
class CodeImportSourceDetailsTests(TestCaseWithFactory):
layer = DatabaseFunctionalLayer
@@ -1395,8 +1428,12 @@
# Use an admin user as we aren't checking edit permissions here.
TestCaseWithFactory.setUp(self, 'admin@xxxxxxxxxxxxx')
- def assertArgumentsMatch(self, code_import, matcher):
+ def assertArgumentsMatch(self, code_import, matcher, start_job=False):
job = self.factory.makeCodeImportJob(code_import=code_import)
+ if start_job:
+ machine = self.factory.makeCodeImportMachine(set_online=True)
+ with celebrity_logged_in("vcs_imports"):
+ getUtility(ICodeImportJobWorkflow).startJob(job, machine)
details = CodeImportSourceDetails.fromCodeImportJob(job)
self.assertThat(details.asArguments(), matcher)
@@ -1416,6 +1453,19 @@
str(code_import.branch.id), 'git',
'git://git.example.com/project.git']))
+ def test_git_to_git_arguments(self):
+ self.pushConfig('codeimport', macaroon_secret_key='some-secret')
+ self.useFixture(GitHostingFixture())
+ code_import = self.factory.makeCodeImport(
+ git_repo_url="git://git.example.com/project.git",
+ target_rcs_type=TargetRevisionControlSystems.GIT)
+ self.assertArgumentsMatch(
+ code_import, MatchesListwise([
+ Equals(code_import.git_repository.unique_name),
+ Equals('git:git'), Equals('git://git.example.com/project.git'),
+ CodeImportJobMacaroonVerifies(code_import.git_repository)]),
+ start_job=True)
+
def test_cvs_arguments(self):
code_import = self.factory.makeCodeImport(
cvs_root=':pserver:foo@xxxxxxxxxxx/bar', cvs_module='bar')
=== modified file 'lib/lp/codehosting/codeimport/tests/test_workermonitor.py'
--- lib/lp/codehosting/codeimport/tests/test_workermonitor.py 2016-10-11 15:36:32 +0000
+++ lib/lp/codehosting/codeimport/tests/test_workermonitor.py 2016-10-11 15:36:32 +0000
@@ -11,8 +11,10 @@
import os
import shutil
import StringIO
+import subprocess
import tempfile
import urllib
+from urlparse import urlsplit
from bzrlib.branch import Branch
from bzrlib.tests import (
@@ -40,11 +42,14 @@
CodeImportResultStatus,
CodeImportReviewStatus,
RevisionControlSystems,
+ TargetRevisionControlSystems,
)
+from lp.code.interfaces.branch import IBranch
from lp.code.interfaces.codeimport import ICodeImportSet
from lp.code.interfaces.codeimportjob import ICodeImportJobSet
from lp.code.model.codeimport import CodeImport
from lp.code.model.codeimportjob import CodeImportJob
+from lp.code.tests.helpers import GitHostingFixture
from lp.codehosting.codeimport.tests.servers import (
BzrServer,
CVSServer,
@@ -65,6 +70,10 @@
ExitQuietly,
)
from lp.services.config import config
+from lp.services.config.fixture import (
+ ConfigFixture,
+ ConfigUseFixture,
+ )
from lp.services.log.logger import BufferLogger
from lp.services.twistedsupport import suppress_stderr
from lp.services.twistedsupport.tests.test_processmonitor import (
@@ -669,7 +678,7 @@
return protocol
-class TestWorkerMonitorIntegration(TestCaseInTempDir):
+class TestWorkerMonitorIntegration(TestCaseInTempDir, TestCase):
layer = ZopelessAppServerLayer
run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=60)
@@ -724,7 +733,7 @@
return self.factory.makeCodeImport(
svn_branch_url=url, rcs_type=RevisionControlSystems.BZR_SVN)
- def makeGitCodeImport(self):
+ def makeGitCodeImport(self, target_rcs_type=None):
"""Make a `CodeImport` that points to a real Git repository."""
self.git_server = GitServer(self.repo_path, use_server=False)
self.git_server.start_server()
@@ -734,7 +743,8 @@
self.foreign_commit_count = 1
return self.factory.makeCodeImport(
- git_repo_url=self.git_server.get_url())
+ git_repo_url=self.git_server.get_url(),
+ target_rcs_type=target_rcs_type)
def makeBzrCodeImport(self):
"""Make a `CodeImport` that points to a real Bazaar branch."""
@@ -761,8 +771,9 @@
job = getUtility(ICodeImportJobSet).getJobForMachine('machine', 10)
self.assertEqual(code_import, job.code_import)
source_details = CodeImportSourceDetails.fromCodeImportJob(job)
- clean_up_default_stores_for_import(source_details)
- self.addCleanup(clean_up_default_stores_for_import, source_details)
+ if IBranch.providedBy(code_import.target):
+ clean_up_default_stores_for_import(source_details)
+ self.addCleanup(clean_up_default_stores_for_import, source_details)
return job
def assertCodeImportResultCreated(self, code_import):
@@ -773,10 +784,19 @@
def assertBranchImportedOKForCodeImport(self, code_import):
"""Assert that a branch was pushed into the default branch store."""
- url = get_default_bazaar_branch_store()._getMirrorURL(
- code_import.branch.id)
- branch = Branch.open(url)
- self.assertEqual(self.foreign_commit_count, branch.revno())
+ if IBranch.providedBy(code_import.target):
+ url = get_default_bazaar_branch_store()._getMirrorURL(
+ code_import.branch.id)
+ branch = Branch.open(url)
+ commit_count = branch.revno()
+ else:
+ repo_path = os.path.join(
+ urlsplit(config.codeimport.git_repository_store).path,
+ code_import.target.unique_name)
+ commit_count = int(subprocess.check_output(
+ ["git", "rev-list", "--count", "HEAD"],
+ cwd=repo_path, universal_newlines=True))
+ self.assertEqual(self.foreign_commit_count, commit_count)
def assertImported(self, ignored, code_import_id):
"""Assert that the `CodeImport` of the given id was imported."""
@@ -841,6 +861,35 @@
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
+ def test_import_git_to_git(self):
+ # Create a Git-to-Git CodeImport and import it.
+ target_store = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, target_store)
+ config_name = self.getUniqueString()
+ config_fixture = self.useFixture(ConfigFixture(
+ config_name, self.layer.config_fixture.instance_name))
+ setting_lines = [
+ "[codeimport]",
+ "git_repository_store: file://%s" % target_store,
+ "macaroon_secret_key: some-secret",
+ ]
+ config_fixture.add_section("\n" + "\n".join(setting_lines))
+ self.useFixture(ConfigUseFixture(config_name))
+ self.useFixture(GitHostingFixture())
+ job = self.getStartedJobForImport(self.makeGitCodeImport(
+ target_rcs_type=TargetRevisionControlSystems.GIT))
+ code_import_id = job.code_import.id
+ job_id = job.id
+ self.layer.txn.commit()
+ target_repo_path = os.path.join(
+ target_store, job.code_import.target.unique_name)
+ os.makedirs(target_repo_path)
+ target_git_server = GitServer(target_repo_path, use_server=False)
+ target_git_server.start_server(bare=True)
+ self.addCleanup(target_git_server.stop_server)
+ result = self.performImport(job_id)
+ return result.addCallback(self.assertImported, code_import_id)
+
def test_import_bzr(self):
# Create a Bazaar CodeImport and import it.
job = self.getStartedJobForImport(self.makeBzrCodeImport())
=== modified file 'lib/lp/codehosting/codeimport/worker.py'
--- lib/lp/codehosting/codeimport/worker.py 2016-10-11 15:36:32 +0000
+++ lib/lp/codehosting/codeimport/worker.py 2016-10-11 15:36:32 +0000
@@ -21,7 +21,13 @@
import os
+import re
import shutil
+import subprocess
+from urlparse import (
+ urlsplit,
+ urlunsplit,
+ )
# FIRST Ensure correct plugins are loaded. Do not delete this comment or the
# line below this comment.
@@ -60,10 +66,16 @@
InvalidURIError,
URI,
)
+from pymacaroons import Macaroon
import SCM
+from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
from lp.code.enums import RevisionControlSystems
-from lp.code.interfaces.branch import get_blacklisted_hostnames
+from lp.code.interfaces.branch import (
+ get_blacklisted_hostnames,
+ IBranch,
+ )
from lp.code.interfaces.codehosting import (
branch_id_alias,
compose_public_url,
@@ -80,6 +92,7 @@
SafeBranchOpener,
)
from lp.services.config import config
+from lp.services.macaroons.interfaces import IMacaroonIssuer
from lp.services.propertycache import cachedproperty
@@ -88,14 +101,16 @@
In summary:
- follow references,
- - only open non-Launchpad URLs for imports from Bazaar
+ - only open non-Launchpad URLs for imports from Bazaar to Bazaar or
+ from Git to Git
- only open the allowed schemes
"""
allowed_schemes = ['http', 'https', 'svn', 'git', 'ftp', 'bzr']
- def __init__(self, rcstype):
+ def __init__(self, rcstype, target_rcstype):
self.rcstype = rcstype
+ self.target_rcstype = target_rcstype
def shouldFollowReferences(self):
"""See `BranchOpenPolicy.shouldFollowReferences`.
@@ -124,9 +139,7 @@
uri = URI(url)
except InvalidURIError:
raise BadUrl(url)
- # XXX cjwatson 2015-06-12: Once we have imports into Git, this
- # should be extended to prevent Git-to-Git self-imports as well.
- if self.rcstype == "bzr":
+ if self.rcstype == self.target_rcstype:
launchpad_domain = config.vhost.mainsite.hostname
if uri.underDomain(launchpad_domain):
raise BadUrl(url)
@@ -269,36 +282,50 @@
of the information suitable for passing around on executables' command
lines.
- :ivar target_id: The id of the Bazaar branch associated with this code
- import, used for locating the existing import and the foreign tree.
+ :ivar target_id: The id of the Bazaar branch or the path of the Git
+ repository associated with this code import, used for locating the
+ existing import and the foreign tree.
:ivar rcstype: 'cvs', 'git', 'bzr-svn', 'bzr' as appropriate.
+ :ivar target_rcstype: 'bzr' or 'git' as appropriate.
:ivar url: The branch URL if rcstype in ['bzr-svn', 'git', 'bzr'], None
otherwise.
:ivar cvs_root: The $CVSROOT if rcstype == 'cvs', None otherwise.
:ivar cvs_module: The CVS module if rcstype == 'cvs', None otherwise.
:ivar stacked_on_url: The URL of the branch that the associated branch
is stacked on, if any.
+ :ivar macaroon: A macaroon granting authority to push to the target
+ repository if target_rcstype == 'git', None otherwise.
"""
- def __init__(self, target_id, rcstype, url=None, cvs_root=None,
- cvs_module=None, stacked_on_url=None):
+ def __init__(self, target_id, rcstype, target_rcstype, url=None,
+ cvs_root=None, cvs_module=None, stacked_on_url=None,
+ macaroon=None):
self.target_id = target_id
self.rcstype = rcstype
+ self.target_rcstype = target_rcstype
self.url = url
self.cvs_root = cvs_root
self.cvs_module = cvs_module
self.stacked_on_url = stacked_on_url
+ self.macaroon = macaroon
@classmethod
def fromArguments(cls, arguments):
"""Convert command line-style arguments to an instance."""
- target_id = int(arguments.pop(0))
+ target_id = arguments.pop(0)
rcstype = arguments.pop(0)
+ if ':' in rcstype:
+ rcstype, target_rcstype = rcstype.split(':', 1)
+ else:
+ target_rcstype = 'bzr'
if rcstype in ['bzr-svn', 'git', 'bzr']:
url = arguments.pop(0)
- try:
- stacked_on_url = arguments.pop(0)
- except IndexError:
+ if target_rcstype == 'bzr':
+ try:
+ stacked_on_url = arguments.pop(0)
+ except IndexError:
+ stacked_on_url = None
+ else:
stacked_on_url = None
cvs_root = cvs_module = None
elif rcstype == 'cvs':
@@ -307,35 +334,54 @@
[cvs_root, cvs_module] = arguments
else:
raise AssertionError("Unknown rcstype %r." % rcstype)
+ if target_rcstype == 'bzr':
+ target_id = int(target_id)
+ macaroon = None
+ elif target_rcstype == 'git':
+ macaroon = Macaroon.deserialize(arguments.pop(0))
+ else:
+ raise AssertionError("Unknown target_rcstype %r." % target_rcstype)
return cls(
- target_id, rcstype, url, cvs_root, cvs_module, stacked_on_url)
+ target_id, rcstype, target_rcstype, url, cvs_root, cvs_module,
+ stacked_on_url, macaroon)
@classmethod
def fromCodeImportJob(cls, job):
"""Convert a `CodeImportJob` to an instance."""
code_import = job.code_import
target = code_import.target
- if target.stacked_on is not None and not target.stacked_on.private:
- stacked_path = branch_id_alias(target.stacked_on)
- stacked_on_url = compose_public_url('http', stacked_path)
+ if IBranch.providedBy(target):
+ if target.stacked_on is not None and not target.stacked_on.private:
+ stacked_path = branch_id_alias(target.stacked_on)
+ stacked_on_url = compose_public_url('http', stacked_path)
+ else:
+ stacked_on_url = None
+ target_id = target.id
else:
- stacked_on_url = None
+ target_id = target.unique_name
if code_import.rcs_type == RevisionControlSystems.BZR_SVN:
return cls(
- target.id, 'bzr-svn', str(code_import.url),
+ target_id, 'bzr-svn', 'bzr', str(code_import.url),
stacked_on_url=stacked_on_url)
elif code_import.rcs_type == RevisionControlSystems.CVS:
return cls(
- target.id, 'cvs',
+ target_id, 'cvs', 'bzr',
cvs_root=str(code_import.cvs_root),
cvs_module=str(code_import.cvs_module))
elif code_import.rcs_type == RevisionControlSystems.GIT:
- return cls(
- target.id, 'git', str(code_import.url),
- stacked_on_url=stacked_on_url)
+ if IBranch.providedBy(target):
+ return cls(
+ target_id, 'git', 'bzr', str(code_import.url),
+ stacked_on_url=stacked_on_url)
+ else:
+ issuer = getUtility(IMacaroonIssuer, 'code-import-job')
+ macaroon = removeSecurityProxy(issuer).issueMacaroon(job)
+ return cls(
+ target_id, 'git', 'git', str(code_import.url),
+ macaroon=macaroon)
elif code_import.rcs_type == RevisionControlSystems.BZR:
return cls(
- target.id, 'bzr', str(code_import.url),
+ target_id, 'bzr', 'bzr', str(code_import.url),
stacked_on_url=stacked_on_url)
else:
raise AssertionError("Unknown rcstype %r." % code_import.rcs_type)
@@ -343,7 +389,14 @@
def asArguments(self):
"""Return a list of arguments suitable for passing to a child process.
"""
- result = [str(self.target_id), self.rcstype]
+ result = [str(self.target_id)]
+ if self.target_rcstype == 'bzr':
+ result.append(self.rcstype)
+ elif self.target_rcstype == 'git':
+ result.append('%s:%s' % (self.rcstype, self.target_rcstype))
+ else:
+ raise AssertionError(
+ "Unknown target_rcstype %r." % self.target_rcstype)
if self.rcstype in ['bzr-svn', 'git', 'bzr']:
result.append(self.url)
if self.stacked_on_url is not None:
@@ -353,6 +406,8 @@
result.append(self.cvs_module)
else:
raise AssertionError("Unknown rcstype %r." % self.rcstype)
+ if self.target_rcstype == 'git':
+ result.append(self.macaroon.serialize())
return result
@@ -918,3 +973,67 @@
"""See `PullingImportWorker.probers`."""
from bzrlib.bzrdir import BzrProber, RemoteBzrProber
return [BzrProber, RemoteBzrProber]
+
+
+class GitToGitImportWorker(ImportWorker):
+ """An import worker for imports from Git to Git."""
+
+ def _runGit(self, *args, **kwargs):
+ """Run git with arguments, sending output to the logger."""
+ cmd = ["git"] + list(args)
+ git_process = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
+ for line in git_process.stdout:
+ line = line.decode("UTF-8", "replace").rstrip("\n")
+ # Remove any user/password data from URLs.
+ line = re.sub(r"://([^:]*:[^@]*@)(\S+)", r"://\2", line)
+ self._logger.info(line)
+ retcode = git_process.wait()
+ if retcode:
+ raise subprocess.CalledProcessError(retcode, cmd)
+
+ def _doImport(self):
+ self._logger.info("Starting job.")
+ self._logger.info(config.codeimport.git_repository_store)
+ try:
+ self._opener_policy.checkOneURL(self.source_details.url)
+ except BadUrl as e:
+ self._logger.info("Invalid URL: %s" % e)
+ return CodeImportWorkerExitCode.FAILURE_FORBIDDEN
+ unauth_target_url = urljoin(
+ config.codeimport.git_repository_store,
+ self.source_details.target_id)
+ split = urlsplit(unauth_target_url)
+ if split.hostname:
+ target_netloc = ":%s@%s" % (
+ self.source_details.macaroon.serialize(), split.hostname)
+ else:
+ target_netloc = ""
+ target_url = urlunsplit([
+ split.scheme, target_netloc, split.path, "", ""])
+ # XXX cjwatson 2016-10-11: Ideally we'd put credentials in a
+ # credentials store instead. However, git only accepts credentials
+ # that have both a non-empty username and a non-empty password.
+ self._logger.info("Getting existing repository from hosting service.")
+ try:
+ self._runGit("clone", "--bare", target_url, "repository")
+ except subprocess.CalledProcessError as e:
+ self._logger.info(
+ "Unable to get existing repository from hosting service: "
+ "%s" % e)
+ return CodeImportWorkerExitCode.FAILURE
+ self._logger.info("Fetching remote repository.")
+ try:
+ self._runGit(
+ "remote", "add", "-f", "--mirror=fetch",
+ "mirror", self.source_details.url, cwd="repository")
+ except subprocess.CalledProcessError as e:
+ self._logger.info("Unable to fetch remote repository: %s" % e)
+ return CodeImportWorkerExitCode.FAILURE_INVALID
+ self._logger.info("Pushing repository to hosting service.")
+ try:
+ self._runGit("push", "--mirror", target_url, cwd="repository")
+ except subprocess.CalledProcessError as e:
+ self._logger.info("Unable to push to hosting service: %s" % e)
+ return CodeImportWorkerExitCode.FAILURE
+ return CodeImportWorkerExitCode.SUCCESS
=== modified file 'lib/lp/services/config/fixture.py'
--- lib/lp/services/config/fixture.py 2011-12-29 05:29:36 +0000
+++ lib/lp/services/config/fixture.py 2016-10-11 15:36:32 +0000
@@ -1,4 +1,4 @@
-# Copyright 2010 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2016 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Fixtures related to configs.
@@ -50,11 +50,11 @@
def setUp(self):
super(ConfigFixture, self).setUp()
- root = 'configs/' + self.instance_name
+ root = os.path.join(config.root, 'configs', self.instance_name)
os.mkdir(root)
self.absroot = os.path.abspath(root)
self.addCleanup(shutil.rmtree, self.absroot)
- source = 'configs/' + self.copy_from_instance
+ source = os.path.join(config.root, 'configs', self.copy_from_instance)
for basename in os.listdir(source):
if basename == 'launchpad-lazr.conf':
self.add_section(self._extend_str % self.copy_from_instance)
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2016-10-11 15:36:32 +0000
+++ lib/lp/services/config/schema-lazr.conf 2016-10-11 15:36:32 +0000
@@ -378,6 +378,10 @@
# datatype: string
bazaar_branch_store: sftp://hoover@escudero/srv/importd/www/
+# Where the Git imports are stored.
+# datatype: string
+git_repository_store: none
+
# The default value of the update interval of a code import from
# Subversion, in seconds.
# datatype: integer
@@ -417,7 +421,7 @@
svn_revisions_import_limit: 500
# Secret key for macaroons used to grant git push permission to workers.
-macaroon_secret_key:
+macaroon_secret_key: none
[codeimportdispatcher]
# The directory where the code import worker should be directed to
=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py 2016-10-11 15:36:32 +0000
+++ lib/lp/testing/factory.py 2016-10-11 15:36:32 +0000
@@ -493,12 +493,15 @@
return epoch + timedelta(minutes=self.getUniqueInteger())
def makeCodeImportSourceDetails(self, target_id=None, rcstype=None,
- url=None, cvs_root=None, cvs_module=None,
- stacked_on_url=None):
+ target_rcstype=None, url=None,
+ cvs_root=None, cvs_module=None,
+ stacked_on_url=None, macaroon=None):
if target_id is None:
target_id = self.getUniqueInteger()
if rcstype is None:
rcstype = 'bzr-svn'
+ if target_rcstype is None:
+ target_rcstype = 'bzr'
if rcstype in ['bzr-svn', 'bzr']:
assert cvs_root is cvs_module is None
if url is None:
@@ -516,8 +519,8 @@
else:
raise AssertionError("Unknown rcstype %r." % rcstype)
return CodeImportSourceDetails(
- target_id, rcstype, url, cvs_root, cvs_module,
- stacked_on_url=stacked_on_url)
+ target_id, rcstype, target_rcstype, url, cvs_root, cvs_module,
+ stacked_on_url=stacked_on_url, macaroon=macaroon)
class BareLaunchpadObjectFactory(ObjectFactory):
=== modified file 'scripts/code-import-worker.py'
--- scripts/code-import-worker.py 2015-06-12 14:20:12 +0000
+++ scripts/code-import-worker.py 2016-10-11 15:36:32 +0000
@@ -1,6 +1,6 @@
#!/usr/bin/python -S
#
-# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Process a code import described by the command line arguments.
@@ -30,6 +30,7 @@
CSCVSImportWorker,
get_default_bazaar_branch_store,
GitImportWorker,
+ GitToGitImportWorker,
)
from lp.codehosting.safe_open import AcceptAnythingPolicy
from lp.services import scripts
@@ -37,7 +38,7 @@
opener_policies = {
- "anything": lambda rcstype: AcceptAnythingPolicy(),
+ "anything": lambda rcstype, target_rcstype: AcceptAnythingPolicy(),
"default": CodeImportBranchOpenPolicy,
}
@@ -72,7 +73,10 @@
force_bzr_to_use_urllib()
source_details = CodeImportSourceDetails.fromArguments(self.args)
if source_details.rcstype == 'git':
- import_worker_cls = GitImportWorker
+ if source_details.target_rcstype == 'bzr':
+ import_worker_cls = GitImportWorker
+ else:
+ import_worker_cls = GitToGitImportWorker
elif source_details.rcstype == 'bzr-svn':
import_worker_cls = BzrSvnImportWorker
elif source_details.rcstype == 'bzr':
@@ -83,11 +87,15 @@
raise AssertionError(
'unknown rcstype %r' % source_details.rcstype)
opener_policy = opener_policies[self.options.access_policy](
- source_details.rcstype)
- import_worker = import_worker_cls(
- source_details,
- get_transport(config.codeimport.foreign_tree_store),
- get_default_bazaar_branch_store(), self.logger, opener_policy)
+ source_details.rcstype, source_details.target_rcstype)
+ if source_details.target_rcstype == 'bzr':
+ import_worker = import_worker_cls(
+ source_details,
+ get_transport(config.codeimport.foreign_tree_store),
+ get_default_bazaar_branch_store(), self.logger, opener_policy)
+ else:
+ import_worker = import_worker_cls(
+ source_details, self.logger, opener_policy)
return import_worker.run()
Follow ups
