launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24856
[Merge] ~pappacena/turnip:autocreate-repo-on-push into turnip:master
Thiago F. Pappacena has proposed merging ~pappacena/turnip:autocreate-repo-on-push into turnip:master.
Commit message:
Create repository when translatePath indicates that it doesn't yet exist on Launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~pappacena/turnip/+git/turnip/+merge/385158
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~pappacena/turnip:autocreate-repo-on-push into turnip:master.
diff --git a/requirements.txt b/requirements.txt
index a298c0a..cb86dfe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -26,6 +26,7 @@ linecache2==1.0.0
m2r==0.1.14
mccabe==0.3
mistune==0.8.3
+mock==3.0.5
Paste==2.0.2
PasteDeploy==2.1.0
pbr==5.4.4
diff --git a/setup.py b/setup.py
index cd8f4fb..b5821f7 100755
--- a/setup.py
+++ b/setup.py
@@ -34,6 +34,7 @@ test_requires = [
'docutils',
'fixtures',
'flake8',
+ 'mock',
'testtools',
'webtest',
]
diff --git a/turnip/pack/git.py b/turnip/pack/git.py
index 28bc2da..8370bd6 100644
--- a/turnip/pack/git.py
+++ b/turnip/pack/git.py
@@ -7,8 +7,10 @@ from __future__ import (
unicode_literals,
)
+import json
import uuid
+import six
from twisted.internet import (
defer,
error,
@@ -20,6 +22,8 @@ from twisted.logger import Logger
from twisted.web import xmlrpc
from zope.interface import implementer
+from turnip.api.store import init_repo, delete_repo
+from turnip.config import config
from turnip.helpers import compose_path
from turnip.pack.helpers import (
decode_packet,
@@ -407,12 +411,31 @@ class PackBackendProtocol(PackServerProtocol):
hookrpc_key = None
expect_set_symbolic_ref = False
+ @defer.inlineCallbacks
def requestReceived(self, command, raw_pathname, params):
self.extractRequestMeta(command, raw_pathname, params)
self.command = command
self.raw_pathname = raw_pathname
self.path = compose_path(self.factory.root, self.raw_pathname)
+ auth_params = self.createAuthParams(params)
+
+ # If any operation should be executed before running the requested
+ # command, we should run it here.
+ if 'turnip-pre-execution' in params:
+ pre_exec_details = json.loads(params['turnip-pre-execution'])
+ operation = pre_exec_details['operation']
+ pre_exec_params = pre_exec_details['params']
+ method = {
+ 'turnip-create-repo': self._createRepo,
+ }
+ try:
+ yield method[operation](
+ auth_params=auth_params, **pre_exec_params)
+ except Exception as e:
+ self.die(b'Could not create repository: %s' % e)
+ return
+
if command == b'turnip-set-symbolic-ref':
self.expect_set_symbolic_ref = True
self.resumeProducing()
@@ -434,7 +457,6 @@ class PackBackendProtocol(PackServerProtocol):
if params.pop(b'turnip-advertise-refs', None):
args.append(b'--advertise-refs')
args.append(self.path)
- auth_params = self.createAuthParams(params)
self.spawnGit(subcmd,
args,
write_operation=write_operation,
@@ -468,6 +490,44 @@ class PackBackendProtocol(PackServerProtocol):
def spawnProcess(self, cmd, args, env=None):
default_reactor.spawnProcess(self.peer, cmd, args, env=env)
+ @defer.inlineCallbacks
+ def _createRepo(self, xmlrpc_endpoint, pathname, creation_params,
+ auth_params):
+ """Creates a repository locally, and asks Launchpad to initialize
+ database objects too.
+
+ :param xmlrpc_endpoint: The XML-RCP proxy address to launchpad.
+ :param pathname: Local path for the repository.
+ :param creation_params: Repo creation parameters returned from
+ translatePath call."""
+ xmlrpc_endpoint = six.ensure_binary(xmlrpc_endpoint, 'utf8')
+ proxy = xmlrpc.Proxy(xmlrpc_endpoint, allowNone=True)
+ clone_from = creation_params.get("clone_from")
+ repository_id = creation_params["repository_id"]
+ xmlrpc_timeout = config.get("virtinfo_timeout")
+ try:
+ repo_path = compose_path(self.factory.root, pathname)
+ if clone_from:
+ clone_path = compose_path(self.factory.root, clone_from)
+ else:
+ clone_path = None
+ self._init_repo(repo_path, clone_path)
+ yield proxy.callRemote(
+ "confirmRepoCreation", repository_id, auth_params).addTimeout(
+ xmlrpc_timeout, default_reactor)
+ except Exception as e:
+ yield proxy.callRemote(
+ "abortRepoCreation", repository_id, auth_params).addTimeout(
+ xmlrpc_timeout, default_reactor)
+ self._delete_repo(repo_path)
+ raise e
+
+ def _init_repo(self, repo_path, clone_path):
+ init_repo(repo_path, clone_path)
+
+ def _delete_repo(self, pathname):
+ delete_repo(pathname)
+
def packetReceived(self, data):
if self.expect_set_symbolic_ref:
if data is None:
@@ -571,6 +631,18 @@ class PackVirtServerProtocol(PackProxyServerProtocol):
VIRT_ERROR_PREFIX +
b'NOT_FOUND Repository does not exist.')
pathname = translated['path']
+
+ # Repository doesn't exist, and should be created.
+ creation_params = translated.get("creation_params")
+ if creation_params:
+ params[b'turnip-pre-execution'] = json.dumps({
+ 'operation': 'turnip-create-repo',
+ 'params': {
+ 'xmlrpc_endpoint': self.factory.virtinfo_endpoint,
+ 'pathname': pathname,
+ 'creation_params': creation_params}})
+ params[b'turnip-pre-execution'] = six.ensure_binary(
+ params[b'turnip-pre-execution'], 'utf8')
except xmlrpc.Fault as e:
fault_type = translate_xmlrpc_fault(
e.faultCode).name.encode('UTF-8')
@@ -583,7 +655,8 @@ class PackVirtServerProtocol(PackProxyServerProtocol):
VIRT_ERROR_PREFIX +
b'GATEWAY_TIMEOUT Path translation timed out.')
except Exception as e:
- self.die(VIRT_ERROR_PREFIX + b'INTERNAL_SERVER_ERROR ' + str(e))
+ msg = str(e).encode("UTF-8")
+ self.die(VIRT_ERROR_PREFIX + b'INTERNAL_SERVER_ERROR ' + msg)
else:
try:
yield self.connectToBackend(command, pathname, params)
diff --git a/turnip/pack/helpers.py b/turnip/pack/helpers.py
index 533c967..cd07558 100644
--- a/turnip/pack/helpers.py
+++ b/turnip/pack/helpers.py
@@ -100,6 +100,8 @@ def decode_request(data):
def encode_request(command, pathname, params):
"""Encode a command, pathname and parameters into a turnip-proto-request.
"""
+ command = six.ensure_binary(command)
+ pathname = six.ensure_binary(pathname)
if b' ' in command or b'\0' in pathname:
raise ValueError('Metacharacter in arguments')
bits = [pathname]
diff --git a/turnip/pack/tests/fake_servers.py b/turnip/pack/tests/fake_servers.py
index c0c4015..6e4712c 100644
--- a/turnip/pack/tests/fake_servers.py
+++ b/turnip/pack/tests/fake_servers.py
@@ -19,6 +19,8 @@ __all__ = [
"FakeVirtInfoService",
]
+from twisted.web.xmlrpc import Binary
+
class FakeAuthServerService(xmlrpc.XMLRPC):
"""A fake version of the Launchpad authserver service."""
@@ -53,7 +55,17 @@ class FakeVirtInfoService(xmlrpc.XMLRPC):
"""A trivial virt information XML-RPC service.
Translates a path to its SHA-256 hash. The repo is writable if the
- path is prefixed with '/+rw'
+ path is prefixed with '/+rw', and is new if its name contains '-new'.
+
+ For new repositories, to fake a response with "clone_from", include in
+ its name the pattern "/clone-from:REPO_NAME" in the end of pathname.
+
+ Examples of repositories:
+ - /example: Simple read-only repo
+ - /+rwexample: Read & write repo
+ - /example-new: Non-existing repository, read only
+ - /+rwexample-new/clone-from:foo: New repository called "example-new",
+ that can be written and cloned from "foo"
"""
def __init__(self, *args, **kwargs):
@@ -67,20 +79,34 @@ class FakeVirtInfoService(xmlrpc.XMLRPC):
self.ref_permissions_checks = []
self.ref_permissions = {}
self.ref_permissions_fault = None
+ self.confirm_repo_creation_call_args = []
+ self.abort_repo_creation_call_args = []
def xmlrpc_translatePath(self, pathname, permission, auth_params):
if self.require_auth and 'user' not in auth_params:
raise xmlrpc.Fault(3, "Unauthorized")
+ if isinstance(pathname, Binary):
+ pathname = pathname.data
self.translations.append((pathname, permission, auth_params))
writable = False
- if pathname.startswith('/+rw'):
+ if pathname.startswith(b'/+rw'):
writable = True
pathname = pathname[4:]
if permission != b'read' and not writable:
raise xmlrpc.Fault(2, "Repository is read-only")
- return {'path': hashlib.sha256(pathname).hexdigest()}
+ retval = {'path': hashlib.sha256(pathname).hexdigest()}
+
+ if b"-new" in pathname:
+ if b"/clone-from:" in pathname:
+ clone_path = pathname.split(b"/clone-from:", 1)[1]
+ clone_from = hashlib.sha256(clone_path).hexdigest()
+ else:
+ clone_from = None
+ retval["creation_params"] = {
+ "repository_id": 66, "clone_from": clone_from}
+ return retval
def xmlrpc_authenticateWithPassword(self, username, password):
self.authentications.append((username, password))
@@ -102,3 +128,9 @@ class FakeVirtInfoService(xmlrpc.XMLRPC):
raise self.merge_proposal_url_fault
else:
return self.merge_proposal_url
+
+ def xmlrpc_confirmRepoCreation(self, repository_id, auth_params):
+ self.confirm_repo_creation_call_args.append((repository_id, ))
+
+ def xmlrpc_abortRepoCreation(self, repository_id, auth_params):
+ self.abort_repo_creation_call_args.append((repository_id, ))
diff --git a/turnip/pack/tests/test_git.py b/turnip/pack/tests/test_git.py
index e5738a4..09c83f0 100644
--- a/turnip/pack/tests/test_git.py
+++ b/turnip/pack/tests/test_git.py
@@ -8,6 +8,7 @@ from __future__ import (
)
import hashlib
+import json
import os.path
from fixtures import TempDir
@@ -27,6 +28,7 @@ from twisted.internet import (
testing,
)
from twisted.web import server
+from twisted.web.xmlrpc import Fault
from turnip.pack import (
git,
@@ -34,6 +36,7 @@ from turnip.pack import (
)
from turnip.pack.tests.fake_servers import FakeVirtInfoService
from turnip.pack.tests.test_hooks import MockHookRPCHandler
+from turnip.tests.compat import mock
class DummyPackServerProtocol(git.PackServerProtocol):
@@ -109,10 +112,21 @@ class TestPackServerProtocol(TestCase):
self.assertKilledWith(b'Bad request: flush-pkt instead')
-class DummyPackBackendProtocol(git.PackBackendProtocol):
+class DummyPackBackendProtocol(git.PackBackendProtocol, object):
test_process = None
+ def __init__(self):
+ super(DummyPackBackendProtocol, self).__init__()
+ self.initiated_repos = []
+ self.deleted_repos = []
+
+ def _init_repo(self, repo_path, clone_path):
+ self.initiated_repos.append((repo_path, clone_path))
+
+ def _delete_repo(self, pathname):
+ self.deleted_repos.append((pathname, ))
+
def spawnProcess(self, cmd, args, env=None):
if self.test_process is not None:
raise AssertionError('Process already spawned.')
@@ -169,6 +183,8 @@ class TestPackFrontendServerProtocol(TestCase):
class TestPackBackendProtocol(TestCase):
"""Test the Git pack backend protocol."""
+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
+
def setUp(self):
super(TestPackBackendProtocol, self).setUp()
self.root = self.useFixture(TempDir()).path
@@ -182,12 +198,77 @@ class TestPackBackendProtocol(TestCase):
self.transport.protocol = self.proto
self.proto.makeConnection(self.transport)
+ # XML-RPC server
+ self.virtinfo = FakeVirtInfoService(allowNone=True)
+ self.virtinfo_listener = default_reactor.listenTCP(0, server.Site(
+ self.virtinfo))
+ self.virtinfo_port = self.virtinfo_listener.getHost().port
+ self.virtinfo_url = b'http://localhost:%d/' % self.virtinfo_port
+ self.addCleanup(self.virtinfo_listener.stopListening)
+
def assertKilledWith(self, message):
self.assertFalse(self.transport.connected)
self.assertEqual(
(b'ERR ' + message + b'\n', b''),
helpers.decode_packet(self.transport.value()))
+ @defer.inlineCallbacks
+ def test_create_repo_pre_execution_command(self):
+ # If command params has 'turnip-pre-execution', it should run what
+ # is described there before spawning process
+ pre_exec_operation = {
+ "operation": "turnip-create-repo",
+ "params": {
+ 'xmlrpc_endpoint': str(self.virtinfo_url),
+ 'pathname': 'foo.git',
+ 'creation_params': {"clone_from": None, "repository_id": 9}}}
+ yield self.proto.requestReceived(
+ b'git-upload-pack', b'/foo.git', {
+ b'host': b'example.com',
+ b'turnip-pre-execution': json.dumps(pre_exec_operation)
+ })
+
+ full_path = os.path.join(six.ensure_binary(self.root), b'foo.git')
+ self.assertEqual([(full_path, None)], self.proto.initiated_repos)
+ self.assertEqual([], self.proto.deleted_repos)
+
+ self.assertEqual(
+ [(9, )], self.virtinfo.confirm_repo_creation_call_args)
+
+ self.assertEqual(
+ (b'git',
+ [b'git', b'upload-pack', full_path],
+ {}),
+ self.proto.test_process)
+
+ @defer.inlineCallbacks
+ def test_create_repo_fails_to_confirm(self):
+ self.virtinfo.xmlrpc_confirmRepoCreation = mock.Mock()
+ self.virtinfo.xmlrpc_confirmRepoCreation.side_effect = Fault(1, "?")
+
+ pre_exec_operation = {
+ "operation": "turnip-create-repo",
+ "params": {
+ 'xmlrpc_endpoint': str(self.virtinfo_url),
+ 'pathname': 'foo.git',
+ 'creation_params': {"clone_from": None, "repository_id": 9}}}
+ params = {
+ b'host': b'example.com',
+ b'turnip-pre-execution': json.dumps(pre_exec_operation)}
+ yield self.proto.requestReceived(
+ b'git-upload-pack', b'/foo.git', params)
+
+ full_path = os.path.join(six.ensure_binary(self.root), b'foo.git')
+ self.assertEqual([(full_path, None)], self.proto.initiated_repos)
+ self.assertEqual([(full_path, )], self.proto.deleted_repos)
+
+ self.assertEqual([(9, )], self.virtinfo.abort_repo_creation_call_args)
+ self.assertEqual(
+ [mock.call(mock.ANY, 9, self.proto.createAuthParams(params))],
+ self.virtinfo.xmlrpc_confirmRepoCreation.call_args_list)
+
+ self.assertIsNone(self.proto.test_process)
+
def test_git_upload_pack_calls_spawnProcess(self):
# If the command is git-upload-pack, requestReceived calls
# spawnProcess with appropriate arguments.
@@ -286,35 +367,62 @@ class TestPackVirtServerProtocol(TestCase):
(b'ERR turnip virt error: ' + message + b'\n', b''),
helpers.decode_packet(self.transport.value()))
- @defer.inlineCallbacks
- def test_translatePath(self):
- root = self.useFixture(TempDir()).path
- hookrpc_handler = MockHookRPCHandler()
- hookrpc_sock = os.path.join(root, 'hookrpc_sock')
- backend_factory = DummyPackBackendFactory(
- root, hookrpc_handler, hookrpc_sock)
- backend_factory.protocol = DummyPackBackendProtocol
- backend_listener = default_reactor.listenTCP(0, backend_factory)
- backend_port = backend_listener.getHost().port
- self.addCleanup(backend_listener.stopListening)
- virtinfo = FakeVirtInfoService(allowNone=True)
- virtinfo_listener = default_reactor.listenTCP(0, server.Site(virtinfo))
- virtinfo_port = virtinfo_listener.getHost().port
- virtinfo_url = b'http://localhost:%d/' % virtinfo_port
- self.addCleanup(virtinfo_listener.stopListening)
+ def setUp(self):
+ super(TestPackVirtServerProtocol, self).setUp()
+ self.root = self.useFixture(TempDir()).path
+ self.hookrpc_handler = MockHookRPCHandler()
+ self.hookrpc_sock = os.path.join(self.root, 'hookrpc_sock')
+ self.backend_factory = DummyPackBackendFactory(
+ self.root, self.hookrpc_handler, self.hookrpc_sock)
+ self.backend_factory.protocol = DummyPackBackendProtocol
+ self.backend_listener = default_reactor.listenTCP(
+ 0, self.backend_factory)
+ self.backend_port = self.backend_listener.getHost().port
+ self.addCleanup(self.backend_listener.stopListening)
+
+ self.virtinfo = FakeVirtInfoService(allowNone=True)
+ self.virtinfo_listener = default_reactor.listenTCP(0, server.Site(
+ self.virtinfo))
+ self.virtinfo_port = self.virtinfo_listener.getHost().port
+ self.virtinfo_url = b'http://localhost:%d/' % self.virtinfo_port
+ self.addCleanup(self.virtinfo_listener.stopListening)
factory = git.PackVirtFactory(
- b'localhost', backend_port, virtinfo_url, 5)
- proto = git.PackVirtServerProtocol()
- proto.factory = factory
+ b'localhost', self.backend_port, self.virtinfo_url, 5)
+ self.proto = git.PackVirtServerProtocol()
+ self.proto.factory = factory
self.transport = testing.StringTransportWithDisconnection()
- self.transport.protocol = proto
- proto.makeConnection(self.transport)
- proto.pauseProducing()
- proto.got_request = True
- yield proto.requestReceived(b'git-upload-pack', b'/example', {})
+ self.transport.protocol = self.proto
+ self.proto.makeConnection(self.transport)
+ self.proto.pauseProducing()
+ self.proto.got_request = True
+
+ @defer.inlineCallbacks
+ def test_translatePath(self):
+ yield self.proto.requestReceived(b'git-upload-pack', b'/example', {})
self.assertEqual(
- hashlib.sha256(b'/example').hexdigest(), proto.pathname)
- backend_factory.test_protocol.transport.loseConnection()
+ hashlib.sha256(b'/example').hexdigest(), self.proto.pathname)
+ self.backend_factory.test_protocol.transport.loseConnection()
+
+ @defer.inlineCallbacks
+ def test_git_push_for_new_repository_adds_pre_execution_step(self):
+ path = b'/+rwexample-new/clone-from:foo-repo'
+
+ yield self.proto.requestReceived(b'git-upload-pack', path, {})
+ self.backend_factory.test_protocol.transport.loseConnection()
+
+ digest = hashlib.sha256(b'example-new/clone-from:foo-repo').hexdigest()
+ clone_digest = hashlib.sha256(b'foo-repo').hexdigest()
+ self.assertEqual({
+ 'turnip-pre-execution': json.dumps({
+ "operation": "turnip-create-repo",
+ "params": {
+ "xmlrpc_endpoint": str(self.virtinfo_url),
+ "pathname": digest,
+ "creation_params": {
+ "repository_id": 66,
+ "clone_from": clone_digest}
+ }})
+ }, self.proto.params)
def test_translatePath_timeout(self):
root = self.useFixture(TempDir()).path
diff --git a/turnip/tests/compat.py b/turnip/tests/compat.py
new file mode 100644
index 0000000..daa21d8
--- /dev/null
+++ b/turnip/tests/compat.py
@@ -0,0 +1,19 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+
+from __future__ import (
+ absolute_import,
+ print_function,
+ unicode_literals,
+)
+
+try:
+ from unittest import mock
+except ImportError:
+ import mock
+
+
+__all__ = [
+ 'mock'
+]
