launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #22703
[Merge] lp:~cjwatson/launchpad/treq into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/treq into lp:launchpad.
Commit message:
Port from the deprecated twisted.web.client.getPage to treq.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/treq/+merge/348808
twisted.web.client.getPage was deprecated in Twisted 17.1.0, so we should stop using it.
treq is disappointingly more verbose in places, though I've done my best to keep that to a minimum. That said, it definitely has its good points (e.g. decent JSON handling), and the changes to the SnapBuildBehaviour tests make them clearly better by using a double for the proxy authentication API rather than just mocking the client.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/treq into lp:launchpad.
=== modified file 'constraints.txt'
--- constraints.txt 2018-06-06 12:46:56 +0000
+++ constraints.txt 2018-07-02 00:28:14 +0000
@@ -359,6 +359,7 @@
testresources==0.2.7
testscenarios==0.4
timeline==0.0.3
+treq==18.6.0
Twisted[conch,tls]==17.9.0
txAMQP==0.6.2
txfixtures==0.4.2
=== modified file 'lib/lp/buildmaster/tests/test_interactor.py'
--- lib/lp/buildmaster/tests/test_interactor.py 2018-06-06 08:47:46 +0000
+++ lib/lp/buildmaster/tests/test_interactor.py 2018-07-02 00:28:14 +0000
@@ -29,13 +29,13 @@
AsynchronousDeferredRunTestForBrokenTwisted,
SynchronousDeferredRunTest,
)
+import treq
from twisted.internet import (
defer,
reactor as default_reactor,
)
from twisted.internet.task import Clock
from twisted.python.failure import Failure
-from twisted.web.client import getPage
from zope.security.proxy import removeSecurityProxy
from lp.buildmaster.enums import (
@@ -66,6 +66,8 @@
WaitingSlave,
)
from lp.services.config import config
+from lp.services.twistedsupport.testing import TReqFixture
+from lp.services.twistedsupport.treq import check_status
from lp.soyuz.model.binarypackagebuildbehaviour import (
BinaryPackageBuildBehaviour,
)
@@ -763,10 +765,12 @@
d.addCallback(self.assertEqual, [True, 'Download'])
return d
+ @defer.inlineCallbacks
def test_retrieve_files_from_filecache(self):
# Files that are present on the slave can be downloaded with a
# filename made from the sha1 of the content underneath the
# 'filecache' directory.
+ from twisted.internet import reactor
content = "Hello World"
lf = self.factory.makeLibraryFileAlias(
'HelloWorld.txt', content=content)
@@ -775,13 +779,11 @@
self.slave_helper.base_url, lf.content.sha1)
self.slave_helper.getServerSlave()
slave = self.slave_helper.getClientSlave()
- d = slave.ensurepresent(lf.content.sha1, lf.http_url, "", "")
-
- def check_file(ignored):
- d = getPage(expected_url.encode('utf8'))
- return d.addCallback(self.assertEqual, content)
-
- return d.addCallback(check_file)
+ yield slave.ensurepresent(lf.content.sha1, lf.http_url, "", "")
+ client = self.useFixture(TReqFixture(reactor)).client
+ response = yield client.get(expected_url).addCallback(check_status)
+ got_content = yield treq.content(response)
+ self.assertEqual(content, got_content)
def test_getFiles(self):
# Test BuilderSlave.getFiles().
=== added file 'lib/lp/services/twistedsupport/testing.py'
--- lib/lp/services/twistedsupport/testing.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/twistedsupport/testing.py 2018-07-02 00:28:14 +0000
@@ -0,0 +1,33 @@
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test-specific Twisted utilities."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'TReqFixture',
+ ]
+
+from fixtures import Fixture
+from treq.client import HTTPClient
+from twisted.web.client import (
+ Agent,
+ HTTPConnectionPool,
+ )
+
+
+class TReqFixture(Fixture):
+ """A `treq` client that handles test cleanup."""
+
+ def __init__(self, reactor, pool=None):
+ super(TReqFixture, self).__init__()
+ self.reactor = reactor
+ if pool is None:
+ pool = HTTPConnectionPool(reactor, persistent=False)
+ self.pool = pool
+
+ def _setUp(self):
+ self.client = HTTPClient(Agent(self.reactor, pool=self.pool))
+ self.addCleanup(self.pool.closeCachedConnections)
=== added file 'lib/lp/services/twistedsupport/treq.py'
--- lib/lp/services/twistedsupport/treq.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/twistedsupport/treq.py 2018-07-02 00:28:14 +0000
@@ -0,0 +1,37 @@
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Utilities for HTTP request handling with Twisted and treq."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'check_status',
+ ]
+
+import treq
+from twisted.internet import defer
+from twisted.web.error import Error
+
+
+# This can be removed once something like
+# https://github.com/twisted/treq/pull/159 is merged into treq.
+def check_status(response):
+ """Fail with an error if the response has an error status.
+
+ An error status is a 4xx or 5xx code (RFC 7231).
+
+ :rtype: A `Deferred` that fires with the response itself if the status
+ is a known non-error code, or that fails it with
+ :exc:`twisted.web.error.Error` otherwise.
+ """
+ if 100 <= response.code < 400:
+ d = defer.succeed(response)
+ else:
+ def raise_error(body):
+ raise Error(response.code, response=body)
+
+ d = treq.content(response).addCallback(raise_error)
+
+ return d
=== modified file 'lib/lp/snappy/model/snapbuildbehaviour.py'
--- lib/lp/snappy/model/snapbuildbehaviour.py 2018-05-02 12:45:12 +0000
+++ lib/lp/snappy/model/snapbuildbehaviour.py 2018-07-02 00:28:14 +0000
@@ -12,11 +12,10 @@
]
import base64
-import json
import time
+import treq
from twisted.internet import defer
-from twisted.web.client import getPage
from zope.component import adapter
from zope.interface import implementer
from zope.security.proxy import removeSecurityProxy
@@ -30,6 +29,7 @@
)
from lp.registry.interfaces.series import SeriesStatus
from lp.services.config import config
+from lp.services.twistedsupport.treq import check_status
from lp.snappy.interfaces.snap import SnapBuildArchiveOwnerMismatch
from lp.snappy.interfaces.snapbuild import ISnapBuild
from lp.soyuz.adapters.archivedependencies import (
@@ -152,18 +152,14 @@
build_id=self.build.build_cookie,
timestamp=timestamp)
auth_string = '{}:{}'.format(admin_username, secret).strip()
- auth_header = 'Basic ' + base64.b64encode(auth_string)
- data = json.dumps({'username': proxy_username})
+ auth_header = b'Basic ' + base64.b64encode(auth_string)
- result = yield getPage(
- url,
- method='POST',
- postdata=data,
- headers={
- 'Authorization': auth_header,
- 'Content-Type': 'application/json'}
- )
- token = json.loads(result)
+ response = yield treq.post(
+ url, headers={'Authorization': auth_header},
+ json={'username': proxy_username},
+ reactor=self._slave.reactor,
+ pool=self._slave.pool).addCallback(check_status)
+ token = yield treq.json_content(response)
defer.returnValue(token)
def verifySuccessfulBuild(self):
=== modified file 'lib/lp/snappy/tests/test_snapbuildbehaviour.py'
--- lib/lp/snappy/tests/test_snapbuildbehaviour.py 2018-05-02 13:22:17 +0000
+++ lib/lp/snappy/tests/test_snapbuildbehaviour.py 2018-07-02 00:28:14 +0000
@@ -12,24 +12,40 @@
import json
import os.path
from textwrap import dedent
+import time
import uuid
import fixtures
-from mock import patch
from pymacaroons import Macaroon
+from six.moves.urllib_parse import urlsplit
from testtools import ExpectedException
from testtools.matchers import (
AfterPreprocessing,
+ ContainsDict,
Equals,
+ HasLength,
+ Is,
IsInstance,
MatchesDict,
MatchesListwise,
+ MatchesStructure,
StartsWith,
)
-from testtools.twistedsupport import AsynchronousDeferredRunTest
+from testtools.twistedsupport import (
+ AsynchronousDeferredRunTestForBrokenTwisted,
+ )
import transaction
-from twisted.internet import defer
+from twisted.internet import (
+ defer,
+ endpoints,
+ reactor,
+ )
+from twisted.python.compat import nativeString
from twisted.trial.unittest import TestCase as TrialTestCase
+from twisted.web import (
+ resource,
+ server,
+ )
from zope.component import getUtility
from zope.proxy import isProxy
from zope.security.proxy import removeSecurityProxy
@@ -46,6 +62,7 @@
from lp.buildmaster.tests.mock_slaves import (
MockBuilder,
OkSlave,
+ SlaveTestHelpers,
)
from lp.buildmaster.tests.test_buildfarmjobbehaviour import (
TestGetUploadMethodsMixin,
@@ -66,12 +83,74 @@
from lp.soyuz.interfaces.archive import ArchiveDisabled
from lp.soyuz.tests.soyuz import Base64KeyMatches
from lp.testing import TestCaseWithFactory
-from lp.testing.fakemethod import FakeMethod
from lp.testing.gpgkeys import gpgkeysdir
from lp.testing.keyserver import InProcessKeyServerFixture
from lp.testing.layers import LaunchpadZopelessLayer
+class ProxyAuthAPITokensResource(resource.Resource):
+ """A test tokens resource for the proxy authentication API."""
+
+ isLeaf = True
+
+ def __init__(self):
+ resource.Resource.__init__(self)
+ self.requests = []
+
+ def render_POST(self, request):
+ content = request.content.read()
+ self.requests.append({
+ "method": request.method,
+ "uri": request.uri,
+ "headers": dict(request.requestHeaders.getAllRawHeaders()),
+ "content": content,
+ })
+ username = json.loads(content)["username"]
+ return json.dumps({
+ "username": username,
+ "secret": uuid.uuid4().hex,
+ "timestamp": datetime.utcnow().isoformat(),
+ })
+
+
+class InProcessProxyAuthAPIFixture(fixtures.Fixture):
+ """A fixture that pretends to be the proxy authentication API.
+
+ Users of this fixture must call the `start` method, which returns a
+ `Deferred`, and arrange for that to get back to the reactor. This is
+ necessary because the basic fixture API does not allow `setUp` to return
+ anything. For example:
+
+ class TestSomething(TestCase):
+
+ run_tests_with = AsynchronousDeferredRunTest.make_factory(
+ timeout=10)
+
+ @defer.inlineCallbacks
+ def setUp(self):
+ super(TestSomething, self).setUp()
+ yield self.useFixture(InProcessProxyAuthAPIFixture()).start()
+ """
+
+ @defer.inlineCallbacks
+ def start(self):
+ root = resource.Resource()
+ self.tokens = ProxyAuthAPITokensResource()
+ root.putChild("tokens", self.tokens)
+ endpoint = endpoints.serverFromString(reactor, nativeString("tcp:0"))
+ site = server.Site(self.tokens)
+ self.addCleanup(site.stopFactory)
+ port = yield endpoint.listen(site)
+ self.addCleanup(port.stopListening)
+ config.push("in-process-proxy-auth-api-fixture", dedent("""
+ [snappy]
+ builder_proxy_auth_api_admin_secret: admin-secret
+ builder_proxy_auth_api_endpoint: http://%s:%s/tokens
+ """) %
+ (port.getHost().host, port.getHost().port))
+ self.addCleanup(config.pop, "in-process-proxy-auth-api-fixture")
+
+
class TestSnapBuildBehaviourBase(TestCaseWithFactory):
layer = LaunchpadZopelessLayer
@@ -80,7 +159,7 @@
self.pushConfig("snappy", tools_source=None, tools_fingerprint=None)
def makeJob(self, archive=None, pocket=PackagePublishingPocket.UPDATES,
- with_builder=False, **kwargs):
+ **kwargs):
"""Create a sample `ISnapBuildBehaviour`."""
if archive is None:
distribution = self.factory.makeDistribution(name="distro")
@@ -95,12 +174,7 @@
build = self.factory.makeSnapBuild(
archive=archive, distroarchseries=distroarchseries, pocket=pocket,
name="test-snap", **kwargs)
- job = IBuildFarmJobBehaviour(build)
- if with_builder:
- builder = MockBuilder()
- builder.processor = processor
- job.setBuilder(builder, None)
- return job
+ return IBuildFarmJobBehaviour(build)
class TestSnapBuildBehaviour(TestSnapBuildBehaviourBase):
@@ -197,8 +271,10 @@
class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
- run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=10)
+ run_tests_with = AsynchronousDeferredRunTestForBrokenTwisted.make_factory(
+ timeout=10)
+ @defer.inlineCallbacks
def setUp(self):
super(TestAsyncSnapBuildBehaviour, self).setUp()
build_username = 'SNAPBUILD-1'
@@ -211,23 +287,41 @@
password=self.token['secret'],
host=config.snappy.builder_proxy_host,
port=config.snappy.builder_proxy_port))
- self.revocation_endpoint = "{endpoint}/{username}".format(
- endpoint=config.snappy.builder_proxy_auth_api_endpoint,
- username=build_username)
- self.api_admin_secret = "admin-secret"
- self.pushConfig(
- "snappy",
- builder_proxy_auth_api_admin_secret=self.api_admin_secret)
- self.mock_proxy_api = FakeMethod(
- result=defer.succeed(json.dumps(self.token)))
- patcher = patch(
- "lp.snappy.model.snapbuildbehaviour.getPage", self.mock_proxy_api)
- patcher.start()
- self.addCleanup(patcher.stop)
+ self.proxy_api = self.useFixture(InProcessProxyAuthAPIFixture())
+ yield self.proxy_api.start()
+ self.now = time.time()
+ self.useFixture(fixtures.MockPatch(
+ "time.time", return_value=self.now))
+
+ def makeJob(self, **kwargs):
+ # We need a builder slave in these tests, in order that requesting a
+ # proxy token can piggyback on its reactor and pool.
+ job = super(TestAsyncSnapBuildBehaviour, self).makeJob(**kwargs)
+ builder = MockBuilder()
+ builder.processor = job.build.processor
+ slave = self.useFixture(SlaveTestHelpers()).getClientSlave()
+ job.setBuilder(builder, slave)
+ self.addCleanup(slave.pool.closeCachedConnections)
+ return job
+
+ def getProxyURLMatcher(self, job):
+ return AfterPreprocessing(urlsplit, MatchesStructure(
+ scheme=Equals("http"),
+ username=Equals("{}-{}".format(
+ job.build.build_cookie, int(self.now))),
+ password=HasLength(32),
+ hostname=Equals(config.snappy.builder_proxy_host),
+ port=Equals(config.snappy.builder_proxy_port),
+ path=Equals("")))
+
+ def getRevocationEndpointMatcher(self, job):
+ return Equals("{}/{}-{}".format(
+ config.snappy.builder_proxy_auth_api_endpoint,
+ job.build.build_cookie, int(self.now)))
@defer.inlineCallbacks
def test_composeBuildRequest(self):
- job = self.makeJob(with_builder=True)
+ job = self.makeJob()
lfa = self.factory.makeLibraryFileAlias(db_only=True)
job.build.distro_arch_series.addOrUpdateChroot(lfa)
build_request = yield job.composeBuildRequest(None)
@@ -238,7 +332,7 @@
def test_requestProxyToken_unconfigured(self):
self.pushConfig("snappy", builder_proxy_auth_api_admin_secret=None)
branch = self.factory.makeBranch()
- job = self.makeJob(branch=branch, with_builder=True)
+ job = self.makeJob(branch=branch)
expected_exception_msg = (
"builder_proxy_auth_api_admin_secret is not configured.")
with ExpectedException(CannotBuild, expected_exception_msg):
@@ -247,25 +341,25 @@
@defer.inlineCallbacks
def test_requestProxyToken(self):
branch = self.factory.makeBranch()
- job = self.makeJob(branch=branch, with_builder=True)
+ job = self.makeJob(branch=branch)
yield job.extraBuildArgs()
- self.assertThat(self.mock_proxy_api.calls, MatchesListwise([
- MatchesListwise([
- MatchesListwise([
- Equals(config.snappy.builder_proxy_auth_api_endpoint),
- ]),
- MatchesDict({
- "method": Equals(b"POST"),
- "postdata": AfterPreprocessing(json.loads, MatchesDict({
- "username": StartsWith(job.build.build_cookie + "-"),
- })),
- "headers": MatchesDict({
- b"Authorization": Equals(b"Basic " + base64.b64encode(
- b"admin-launchpad.dev:admin-secret")),
- b"Content-Type": Equals(b"application/json"),
- }),
+ self.assertThat(self.proxy_api.tokens.requests, MatchesListwise([
+ MatchesDict({
+ "method": Equals("POST"),
+ "uri": Equals(urlsplit(
+ config.snappy.builder_proxy_auth_api_endpoint).path),
+ "headers": ContainsDict({
+ b"Authorization": MatchesListwise([
+ Equals(b"Basic " + base64.b64encode(
+ b"admin-launchpad.dev:admin-secret"))]),
+ b"Content-Type": MatchesListwise([
+ Equals(b"application/json; charset=UTF-8"),
+ ]),
}),
- ]),
+ "content": AfterPreprocessing(json.loads, MatchesDict({
+ "username": StartsWith(job.build.build_cookie + "-"),
+ })),
+ }),
]))
@defer.inlineCallbacks
@@ -273,51 +367,51 @@
# extraBuildArgs returns appropriate arguments if asked to build a
# job for a Bazaar branch.
branch = self.factory.makeBranch()
- job = self.makeJob(branch=branch, with_builder=True)
+ job = self.makeJob(branch=branch)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
args = yield job.extraBuildArgs()
- self.assertEqual({
- "archive_private": False,
- "archives": expected_archives,
- "arch_tag": "i386",
- "branch": branch.bzr_identity,
- "build_source_tarball": False,
- "build_url": canonical_url(job.build),
- "fast_cleanup": True,
- "name": "test-snap",
- "proxy_url": self.proxy_url,
- "revocation_endpoint": self.revocation_endpoint,
- "series": "unstable",
- "trusted_keys": expected_trusted_keys,
- }, args)
+ self.assertThat(args, MatchesDict({
+ "archive_private": Is(False),
+ "archives": Equals(expected_archives),
+ "arch_tag": Equals("i386"),
+ "branch": Equals(branch.bzr_identity),
+ "build_source_tarball": Is(False),
+ "build_url": Equals(canonical_url(job.build)),
+ "fast_cleanup": Is(True),
+ "name": Equals("test-snap"),
+ "proxy_url": self.getProxyURLMatcher(job),
+ "revocation_endpoint": self.getRevocationEndpointMatcher(job),
+ "series": Equals("unstable"),
+ "trusted_keys": Equals(expected_trusted_keys),
+ }))
@defer.inlineCallbacks
def test_extraBuildArgs_git(self):
# extraBuildArgs returns appropriate arguments if asked to build a
# job for a Git branch.
[ref] = self.factory.makeGitRefs()
- job = self.makeJob(git_ref=ref, with_builder=True)
+ job = self.makeJob(git_ref=ref)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
args = yield job.extraBuildArgs()
- self.assertEqual({
- "archive_private": False,
- "archives": expected_archives,
- "arch_tag": "i386",
- "build_source_tarball": False,
- "build_url": canonical_url(job.build),
- "fast_cleanup": True,
- "git_repository": ref.repository.git_https_url,
- "git_path": ref.name,
- "name": "test-snap",
- "proxy_url": self.proxy_url,
- "revocation_endpoint": self.revocation_endpoint,
- "series": "unstable",
- "trusted_keys": expected_trusted_keys,
- }, args)
+ self.assertThat(args, MatchesDict({
+ "archive_private": Is(False),
+ "archives": Equals(expected_archives),
+ "arch_tag": Equals("i386"),
+ "build_source_tarball": Is(False),
+ "build_url": Equals(canonical_url(job.build)),
+ "fast_cleanup": Is(True),
+ "git_repository": Equals(ref.repository.git_https_url),
+ "git_path": Equals(ref.name),
+ "name": Equals("test-snap"),
+ "proxy_url": self.getProxyURLMatcher(job),
+ "revocation_endpoint": self.getRevocationEndpointMatcher(job),
+ "series": Equals("unstable"),
+ "trusted_keys": Equals(expected_trusted_keys),
+ }))
@defer.inlineCallbacks
def test_extraBuildArgs_git_HEAD(self):
@@ -325,26 +419,25 @@
# job for the default branch in a Launchpad-hosted Git repository.
[ref] = self.factory.makeGitRefs()
removeSecurityProxy(ref.repository)._default_branch = ref.path
- job = self.makeJob(
- git_ref=ref.repository.getRefByPath("HEAD"), with_builder=True)
+ job = self.makeJob(git_ref=ref.repository.getRefByPath("HEAD"))
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
args = yield job.extraBuildArgs()
- self.assertEqual({
- "archive_private": False,
- "archives": expected_archives,
- "arch_tag": "i386",
- "build_source_tarball": False,
- "build_url": canonical_url(job.build),
- "fast_cleanup": True,
- "git_repository": ref.repository.git_https_url,
- "name": "test-snap",
- "proxy_url": self.proxy_url,
- "revocation_endpoint": self.revocation_endpoint,
- "series": "unstable",
- "trusted_keys": expected_trusted_keys,
- }, args)
+ self.assertThat(args, MatchesDict({
+ "archive_private": Is(False),
+ "archives": Equals(expected_archives),
+ "arch_tag": Equals("i386"),
+ "build_source_tarball": Is(False),
+ "build_url": Equals(canonical_url(job.build)),
+ "fast_cleanup": Is(True),
+ "git_repository": Equals(ref.repository.git_https_url),
+ "name": Equals("test-snap"),
+ "proxy_url": self.getProxyURLMatcher(job),
+ "revocation_endpoint": self.getRevocationEndpointMatcher(job),
+ "series": Equals("unstable"),
+ "trusted_keys": Equals(expected_trusted_keys),
+ }))
@defer.inlineCallbacks
def test_extraBuildArgs_git_url(self):
@@ -353,26 +446,26 @@
url = "https://git.example.org/foo";
ref = self.factory.makeGitRefRemote(
repository_url=url, path="refs/heads/master")
- job = self.makeJob(git_ref=ref, with_builder=True)
+ job = self.makeJob(git_ref=ref)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
args = yield job.extraBuildArgs()
- self.assertEqual({
- "archive_private": False,
- "archives": expected_archives,
- "arch_tag": "i386",
- "build_source_tarball": False,
- "build_url": canonical_url(job.build),
- "fast_cleanup": True,
- "git_repository": url,
- "git_path": "master",
- "name": "test-snap",
- "proxy_url": self.proxy_url,
- "revocation_endpoint": self.revocation_endpoint,
- "series": "unstable",
- "trusted_keys": expected_trusted_keys,
- }, args)
+ self.assertThat(args, MatchesDict({
+ "archive_private": Is(False),
+ "archives": Equals(expected_archives),
+ "arch_tag": Equals("i386"),
+ "build_source_tarball": Is(False),
+ "build_url": Equals(canonical_url(job.build)),
+ "fast_cleanup": Is(True),
+ "git_repository": Equals(url),
+ "git_path": Equals("master"),
+ "name": Equals("test-snap"),
+ "proxy_url": self.getProxyURLMatcher(job),
+ "revocation_endpoint": self.getRevocationEndpointMatcher(job),
+ "series": Equals("unstable"),
+ "trusted_keys": Equals(expected_trusted_keys),
+ }))
@defer.inlineCallbacks
def test_extraBuildArgs_git_url_HEAD(self):
@@ -380,31 +473,31 @@
# job for the default branch in an external Git repository.
url = "https://git.example.org/foo";
ref = self.factory.makeGitRefRemote(repository_url=url, path="HEAD")
- job = self.makeJob(git_ref=ref, with_builder=True)
+ job = self.makeJob(git_ref=ref)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
args = yield job.extraBuildArgs()
- self.assertEqual({
- "archive_private": False,
- "archives": expected_archives,
- "arch_tag": "i386",
- "build_source_tarball": False,
- "build_url": canonical_url(job.build),
- "fast_cleanup": True,
- "git_repository": url,
- "name": "test-snap",
- "proxy_url": self.proxy_url,
- "revocation_endpoint": self.revocation_endpoint,
- "series": "unstable",
- "trusted_keys": expected_trusted_keys,
- }, args)
+ self.assertThat(args, MatchesDict({
+ "archive_private": Is(False),
+ "archives": Equals(expected_archives),
+ "arch_tag": Equals("i386"),
+ "build_source_tarball": Is(False),
+ "build_url": Equals(canonical_url(job.build)),
+ "fast_cleanup": Is(True),
+ "git_repository": Equals(url),
+ "name": Equals("test-snap"),
+ "proxy_url": self.getProxyURLMatcher(job),
+ "revocation_endpoint": self.getRevocationEndpointMatcher(job),
+ "series": Equals("unstable"),
+ "trusted_keys": Equals(expected_trusted_keys),
+ }))
@defer.inlineCallbacks
def test_extraBuildArgs_prefers_store_name(self):
# For the "name" argument, extraBuildArgs prefers Snap.store_name
# over Snap.name if the former is set.
- job = self.makeJob(store_name="something-else", with_builder=True)
+ job = self.makeJob(store_name="something-else")
args = yield job.extraBuildArgs()
self.assertEqual("something-else", args["name"])
@@ -416,7 +509,7 @@
key_path = os.path.join(gpgkeysdir, "ppa-sample@xxxxxxxxxxxxxxxxx")
yield IArchiveSigningKey(archive).setSigningKey(
key_path, async_keyserver=True)
- job = self.makeJob(archive=archive, with_builder=True)
+ job = self.makeJob(archive=archive)
self.factory.makeBinaryPackagePublishingHistory(
distroarchseries=job.build.distro_arch_series,
pocket=job.build.pocket, archive=archive,
@@ -429,7 +522,7 @@
@defer.inlineCallbacks
def test_extraBuildArgs_channels(self):
# If the build needs particular channels, extraBuildArgs sends them.
- job = self.makeJob(channels={"snapcraft": "edge"}, with_builder=True)
+ job = self.makeJob(channels={"snapcraft": "edge"})
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
@@ -441,7 +534,7 @@
def test_extraBuildArgs_disallow_internet(self):
# If external network access is not allowed for the snap,
# extraBuildArgs does not dispatch a proxy token.
- job = self.makeJob(allow_internet=False, with_builder=True)
+ job = self.makeJob(allow_internet=False)
args = yield job.extraBuildArgs()
self.assertNotIn("proxy_url", args)
self.assertNotIn("revocation_endpoint", args)
@@ -450,21 +543,16 @@
def test_extraBuildArgs_build_source_tarball(self):
# If the snap requests building of a source tarball, extraBuildArgs
# sends the appropriate arguments.
- job = self.makeJob(build_source_tarball=True, with_builder=True)
+ job = self.makeJob(build_source_tarball=True)
args = yield job.extraBuildArgs()
self.assertTrue(args["build_source_tarball"])
@defer.inlineCallbacks
def test_composeBuildRequest_proxy_url_set(self):
- job = self.makeJob(with_builder=True)
+ job = self.makeJob()
build_request = yield job.composeBuildRequest(None)
- proxy_url = ("http://{username}:{password}";
- "@{host}:{port}".format(
- username=self.token['username'],
- password=self.token['secret'],
- host=config.snappy.builder_proxy_host,
- port=config.snappy.builder_proxy_port))
- self.assertEqual(proxy_url, build_request[3]['proxy_url'])
+ self.assertThat(
+ build_request[3]["proxy_url"], self.getProxyURLMatcher(job))
@defer.inlineCallbacks
def test_composeBuildRequest_deleted(self):
@@ -472,8 +560,7 @@
# composeBuildRequest raises CannotBuild.
branch = self.factory.makeBranch()
owner = self.factory.makePerson(name="snap-owner")
- job = self.makeJob(
- registrant=owner, owner=owner, branch=branch, with_builder=True)
+ job = self.makeJob(registrant=owner, owner=owner, branch=branch)
branch.destroySelf(break_references=True)
self.assertIsNone(job.build.snap.branch)
self.assertIsNone(job.build.snap.git_repository)
@@ -489,8 +576,7 @@
repository = self.factory.makeGitRepository()
[ref] = self.factory.makeGitRefs(repository=repository)
owner = self.factory.makePerson(name="snap-owner")
- job = self.makeJob(
- registrant=owner, owner=owner, git_ref=ref, with_builder=True)
+ job = self.makeJob(registrant=owner, owner=owner, git_ref=ref)
repository.removeRefs([ref.path])
self.assertIsNone(job.build.snap.git_ref)
expected_exception_msg = ("Source branch/repository for "
=== modified file 'lib/lp/testing/keyserver/tests/test_inprocess.py'
--- lib/lp/testing/keyserver/tests/test_inprocess.py 2017-12-19 17:22:44 +0000
+++ lib/lp/testing/keyserver/tests/test_inprocess.py 2018-07-02 00:28:14 +0000
@@ -1,4 +1,4 @@
-# Copyright 2017 Canonical Ltd. This software is licensed under the
+# Copyright 2017-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""In-process keyserver fixture tests."""
@@ -10,10 +10,12 @@
from testtools.twistedsupport import (
AsynchronousDeferredRunTestForBrokenTwisted,
)
+import treq
from twisted.internet import defer
-from twisted.web.client import getPage
from lp.services.config import config
+from lp.services.twistedsupport.testing import TReqFixture
+from lp.services.twistedsupport.treq import check_status
from lp.testing import TestCase
from lp.testing.keyserver import InProcessKeyServerFixture
from lp.testing.keyserver.web import GREETING
@@ -38,7 +40,10 @@
@defer.inlineCallbacks
def test_starts_properly(self):
# The fixture starts properly and we can load the page.
+ from twisted.internet import reactor
fixture = self.useFixture(InProcessKeyServerFixture())
yield fixture.start()
- content = yield getPage(fixture.url)
+ client = self.useFixture(TReqFixture(reactor)).client
+ response = yield client.get(fixture.url).addCallback(check_status)
+ content = yield treq.content(response)
self.assertEqual(GREETING, content)
=== modified file 'lib/lp/testing/keyserver/tests/test_web.py'
--- lib/lp/testing/keyserver/tests/test_web.py 2017-12-19 17:22:44 +0000
+++ lib/lp/testing/keyserver/tests/test_web.py 2018-07-02 00:28:14 +0000
@@ -1,4 +1,4 @@
-# Copyright 2011 Canonical Ltd. This software is licensed under the
+# Copyright 2011-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Tests for the web resources of the testkeyserver."""
@@ -9,11 +9,13 @@
import shutil
from testtools.twistedsupport import AsynchronousDeferredRunTest
+import treq
from twisted.internet.endpoints import serverFromString
from twisted.python.failure import Failure
-from twisted.web.client import getPage
from twisted.web.server import Site
+from lp.services.twistedsupport.testing import TReqFixture
+from lp.services.twistedsupport.treq import check_status
from lp.testing import TestCase
from lp.testing.keyserver.harness import KEYS_DIR
from lp.testing.keyserver.web import KeyServerResource
@@ -46,10 +48,13 @@
def fetchResource(self, listening_port, path):
"""GET the content at 'path' from the web server at 'listening_port'.
"""
+ from twisted.internet import reactor
url = 'http://localhost:%s/%s' % (
listening_port.getHost().port,
path.lstrip('/'))
- return getPage(url)
+ client = self.useFixture(TReqFixture(reactor)).client
+ return client.get(url).addCallback(check_status).addCallback(
+ treq.content)
def getURL(self, path):
"""Start a test key server and get the content at 'path'."""
=== modified file 'setup.py'
--- setup.py 2018-06-26 10:59:34 +0000
+++ setup.py 2018-07-02 00:28:14 +0000
@@ -224,6 +224,7 @@
'testtools',
'timeline',
'transaction',
+ 'treq',
'Twisted[conch,tls]',
'txfixtures',
'txlongpoll',
Follow ups
