launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #19932
[Merge] lp:~blr/launchpad/build-manager-proxy-client into lp:launchpad
Kit Randel has proposed merging lp:~blr/launchpad/build-manager-proxy-client into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~blr/launchpad/build-manager-proxy-client/+merge/283372
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~blr/launchpad/build-manager-proxy-client into lp:launchpad.
=== modified file 'configs/development/launchpad-lazr.conf'
--- configs/development/launchpad-lazr.conf 2015-02-19 17:33:35 +0000
+++ configs/development/launchpad-lazr.conf 2016-02-01 20:55:49 +0000
@@ -43,7 +43,9 @@
rewrite_script_log_file: /var/tmp/bazaar.launchpad.dev/rewrite.log
host_key_pair_path: lib/lp/codehosting/sshserver/tests/keys
port: tcp:5022:interface=127.0.0.88
-bzr_lp_prefix: lp://dev/
+#port: tcp:5022:interface:0.0.0.0
+#bzr_lp_prefix: lp://dev/
+bzr_lp_prefix: lp:
lp_url_hosts: dev
access_log: /var/tmp/bazaar.launchpad.dev/codehosting-access.log
blacklisted_hostnames:
@@ -96,7 +98,7 @@
enable_test_openid_provider: True
openid_provider_root: https://testopenid.dev/
code_domain: code.launchpad.dev
-default_batch_size: 5
+default_batch_size: 50
max_attachment_size: 2097152
branchlisting_batch_size: 6
mugshot_batch_size: 8
@@ -184,6 +186,13 @@
password: guest
virtual_host: /
+[snappy]
+builder_proxy_auth_api_admin_username: admin-launchpad.net
+builder_proxy_auth_api_endpoint: http://snap-proxy.launchpad.dev:8080/tokens
+builder_proxy_host: snap-proxy.launchpad.dev
+builder_proxy_port: 3128
+tools_source: deb http://ppa.launchpad.net/snappy-dev/snapcraft-daily/ubuntu %(series)s main
+
[txlongpoll]
launch: True
frontend_port: 22435
=== modified file 'lib/lp/buildmaster/model/buildfarmjobbehaviour.py'
--- lib/lp/buildmaster/model/buildfarmjobbehaviour.py 2015-02-17 09:33:33 +0000
+++ lib/lp/buildmaster/model/buildfarmjobbehaviour.py 2016-02-01 20:55:49 +0000
@@ -77,7 +77,7 @@
"Preparing job %s (%s) on %s."
% (cookie, self.build.title, self._builder.url))
- builder_type, das, files, args = self.composeBuildRequest(logger)
+ builder_type, das, files, args = yield self.composeBuildRequest(logger)
# First cache the chroot and any other files that the job needs.
chroot = das.getChroot()
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2015-12-02 11:27:02 +0000
+++ lib/lp/services/config/schema-lazr.conf 2016-02-01 20:55:49 +0000
@@ -1751,6 +1751,23 @@
http_proxy: none
[snappy]
+# Admin secret for requesting tokens from the builder proxy service.
+# For a mojo deployed proxy service, the secret is generated and will reside in
+# /srv/mojo/${MOJO_PROJECT}/${SERIES}/${MOJO_WORKSPACE}/local/admin-api-secret
+builder_proxy_auth_api_admin_secret: none
+
+# Admin username for builder proxy service.
+builder_proxy_auth_api_admin_username: none
+
+# Endpoint for builder proxy authentication service
+builder_proxy_auth_api_endpoint: none
+
+# Builder http proxy host
+builder_proxy_host: none
+
+# Builder http proxy port
+builder_proxy_port: none
+
# Optional sources.list entry to send to build slaves when doing snap
# package builds. Use this form so that the series is set:
# deb http://foo %(series)s main
=== modified file 'lib/lp/snappy/model/snapbuildbehaviour.py'
--- lib/lp/snappy/model/snapbuildbehaviour.py 2015-09-10 17:18:00 +0000
+++ lib/lp/snappy/model/snapbuildbehaviour.py 2016-02-01 20:55:49 +0000
@@ -11,6 +11,13 @@
'SnapBuildBehaviour',
]
+import base64
+from datetime import datetime
+import json
+from urllib import urlencode
+
+from twisted.internet import defer
+from twisted.web.client import getPage
from zope.component import adapter
from zope.interface import implementer
@@ -70,12 +77,24 @@
raise CannotBuild(
"Missing chroot for %s" % build.distro_arch_series.displayname)
+ @defer.inlineCallbacks
def _extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
build = self.build
args = {}
+ token = yield self._requestProxyToken()
+ args["proxy_url"] = (
+ "http://{username}:{password}@{host}:{port}".format(
+ username=token['username'],
+ password=token['secret'],
+ host=config.snappy.builder_proxy_host,
+ port=config.snappy.builder_proxy_port))
+ args["revocation_endpoint"] = (
+ "{endpoint}/{token}".format(
+ endpoint=config.snappy.builder_proxy_auth_api_endpoint,
+ token=token['username']))
args["name"] = build.snap.name
args["arch_tag"] = build.distro_arch_series.architecturetag
# XXX cjwatson 2015-08-03: Allow tools_source to be overridden at
@@ -96,12 +115,36 @@
raise CannotBuild(
"Source branch/repository for ~%s/%s has been deleted." %
(build.snap.owner.name, build.snap.name))
- return args
-
+ defer.returnValue(args)
+
+ @defer.inlineCallbacks
+ def _requestProxyToken(self):
+ admin_username = config.snappy.builder_proxy_auth_api_admin_username
+ secret = config.snappy.builder_proxy_auth_api_admin_secret
+ url = config.snappy.builder_proxy_auth_api_endpoint
+ timestamp = (datetime.utcnow() - datetime(1970, 1, 1)).total_seconds()
+ proxy_username = '{build_id}-{timestamp}'.format(
+ build_id=self.build.build_cookie,
+ timestamp=timestamp)
+ auth_string = '{}:{}'.format(admin_username, secret).strip()
+ auth_header = 'Basic ' + base64.encodestring(auth_string)
+ data = json.dumps({'username': proxy_username})
+
+ result = yield getPage(
+ url,
+ method='POST',
+ postdata=data,
+ headers={
+ 'Authorization': auth_header,
+ 'Content-Type': 'application/json'}
+ )
+ token = json.loads(result)
+ defer.returnValue(token)
+
+ @defer.inlineCallbacks
def composeBuildRequest(self, logger):
- return (
- "snap", self.build.distro_arch_series, {},
- self._extraBuildArgs(logger=logger))
+ args = yield self._extraBuildArgs(logger=logger)
+ defer.returnValue(("snap", self.build.distro_arch_series, {}, args))
def verifySuccessfulBuild(self):
"""See `IBuildFarmJobBehaviour`."""
=== modified file 'lib/lp/snappy/tests/test_snapbuildbehaviour.py'
--- lib/lp/snappy/tests/test_snapbuildbehaviour.py 2015-09-16 19:07:43 +0000
+++ lib/lp/snappy/tests/test_snapbuildbehaviour.py 2016-02-01 20:55:49 +0000
@@ -5,8 +5,18 @@
__metaclass__ = type
+from datetime import datetime
+from mock import (
+ patch,
+ Mock,
+ )
+
import fixtures
import transaction
+from testtools import ExpectedException
+from testtools.deferredruntest import AsynchronousDeferredRunTest
+from testtools.matchers import IsInstance
+from twisted.internet import defer
from twisted.trial.unittest import TestCase as TrialTestCase
from zope.component import getUtility
@@ -27,6 +37,7 @@
)
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.interfaces.series import SeriesStatus
+from lp.services.config import config
from lp.services.features.testing import FeatureFixture
from lp.services.log.logger import BufferLogger
from lp.snappy.interfaces.snap import (
@@ -42,12 +53,11 @@
from lp.testing.layers import LaunchpadZopelessLayer
-class TestSnapBuildBehaviour(TestCaseWithFactory):
-
+class TestSnapBuildBehaviourBase(TestCaseWithFactory):
layer = LaunchpadZopelessLayer
def setUp(self):
- super(TestSnapBuildBehaviour, self).setUp()
+ super(TestSnapBuildBehaviourBase, self).setUp()
self.useFixture(FeatureFixture({SNAP_FEATURE_FLAG: u"on"}))
self.pushConfig("snappy", tools_source=None)
@@ -65,6 +75,10 @@
name=u"test-snap", **kwargs)
return IBuildFarmJobBehaviour(build)
+
+class TestSnapBuildBehaviour(TestSnapBuildBehaviourBase):
+ layer = LaunchpadZopelessLayer
+
def test_provides_interface(self):
# SnapBuildBehaviour provides IBuildFarmJobBehaviour.
job = SnapBuildBehaviour(None)
@@ -154,6 +168,42 @@
e = self.assertRaises(CannotBuild, job.verifyBuildRequest, logger)
self.assertIn("Missing chroot", str(e))
+
+class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
+ run_tests_with = AsynchronousDeferredRunTest
+
+ def setUp(self):
+ super(TestAsyncSnapBuildBehaviour, self).setUp()
+ self.token = {'secret': '03368addc7994647ace69e7ac2eb1ae9',
+ 'username': 'SNAPBUILD-1',
+ 'timestamp': datetime.utcnow().isoformat()}
+ self.proxy_url = ("http://{username}:{password}";
+ "@{host}:{port}".format(
+ username=self.token['username'],
+ password=self.token['secret'],
+ host=config.snappy.builder_proxy_host,
+ port=config.snappy.builder_proxy_port))
+ self.patcher = patch.object(
+ SnapBuildBehaviour, '_requestProxyToken',
+ Mock(return_value=self.mockRequestProxyToken())).start()
+
+ def tearDown(self):
+ super(TestAsyncSnapBuildBehaviour, self).tearDown()
+ self.patcher.stop()
+
+ def mockRequestProxyToken(self):
+ return defer.succeed(self.token)
+
+ @defer.inlineCallbacks
+ def test_composeBuildRequest(self):
+ job = self.makeJob()
+ lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ build_request = yield job.composeBuildRequest(None)
+ self.assertEqual(build_request[1], job.build.distro_arch_series)
+ self.assertThat(build_request[3], IsInstance(dict))
+
+ @defer.inlineCallbacks
def test_extraBuildArgs_bzr(self):
# _extraBuildArgs returns appropriate arguments if asked to build a
# job for a Bazaar branch.
@@ -161,14 +211,17 @@
job = self.makeJob(branch=branch)
expected_archives = get_sources_list_for_building(
job.build, job.build.distro_arch_series, None)
+ args = yield job._extraBuildArgs()
self.assertEqual({
"archive_private": False,
"archives": expected_archives,
"arch_tag": "i386",
"branch": branch.bzr_identity,
"name": u"test-snap",
- }, job._extraBuildArgs())
+ "proxy_url": self.proxy_url,
+ }, args)
+ @defer.inlineCallbacks
def test_extraBuildArgs_git(self):
# _extraBuildArgs returns appropriate arguments if asked to build a
# job for a Git branch.
@@ -176,6 +229,7 @@
job = self.makeJob(git_ref=ref)
expected_archives = get_sources_list_for_building(
job.build, job.build.distro_arch_series, None)
+ args = yield job._extraBuildArgs()
self.assertEqual({
"archive_private": False,
"archives": expected_archives,
@@ -183,17 +237,22 @@
"git_repository": ref.repository.git_https_url,
"git_path": ref.name,
"name": u"test-snap",
- }, job._extraBuildArgs())
+ "proxy_url": self.proxy_url,
+ }, args)
- def test_composeBuildRequest(self):
+ @defer.inlineCallbacks
+ def test_extraBuildArgs_proxy_url_set(self):
job = self.makeJob()
- lfa = self.factory.makeLibraryFileAlias(db_only=True)
- job.build.distro_arch_series.addOrUpdateChroot(lfa)
- self.assertEqual(
- ('snap', job.build.distro_arch_series, {},
- job._extraBuildArgs()),
- job.composeBuildRequest(None))
+ build_request = yield job.composeBuildRequest(None)
+ proxy_url = ("http://{username}:{password}";
+ "@{host}:{port}".format(
+ username=self.token['username'],
+ password=self.token['secret'],
+ host=config.snappy.builder_proxy_host,
+ port=config.snappy.builder_proxy_port))
+ self.assertEqual(proxy_url, build_request[3]['proxy_url'])
+ @defer.inlineCallbacks
def test_composeBuildRequest_deleted(self):
# If the source branch/repository has been deleted,
# composeBuildRequest raises CannotBuild.
@@ -203,12 +262,12 @@
branch.destroySelf(break_references=True)
self.assertIsNone(job.build.snap.branch)
self.assertIsNone(job.build.snap.git_repository)
- self.assertRaisesWithContent(
- CannotBuild,
- "Source branch/repository for ~snap-owner/test-snap has been "
- "deleted.",
- job.composeBuildRequest, None)
+ expected_exception_msg = ("Source branch/repository for "
+ "~snap-owner/test-snap has been deleted.")
+ with ExpectedException(CannotBuild, expected_exception_msg):
+ yield job.composeBuildRequest(None)
+ @defer.inlineCallbacks
def test_composeBuildRequest_git_ref_deleted(self):
# If the source Git reference has been deleted, composeBuildRequest
# raises CannotBuild.
@@ -218,11 +277,10 @@
job = self.makeJob(registrant=owner, owner=owner, git_ref=ref)
repository.removeRefs([ref.path])
self.assertIsNone(job.build.snap.git_ref)
- self.assertRaisesWithContent(
- CannotBuild,
- "Source branch/repository for ~snap-owner/test-snap has been "
- "deleted.",
- job.composeBuildRequest, None)
+ expected_exception_msg = ("Source branch/repository for "
+ "~snap-owner/test-snap has been deleted.")
+ with ExpectedException(CannotBuild, expected_exception_msg):
+ yield job.composeBuildRequest(None)
class MakeSnapBuildMixin:
Follow ups
