launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27289
[Merge] ~cjwatson/launchpad-buildd:rename-builder-proxy into launchpad-buildd:master
Colin Watson has proposed merging ~cjwatson/launchpad-buildd:rename-builder-proxy into launchpad-buildd:master.
Commit message:
Rename "snap proxy" to "builder proxy"
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/+git/launchpad-buildd/+merge/406005
It's used for several different build types nowadays, and is too easily confused with the snap store proxy.
The XML-RPC protocol between Launchpad and launchpad-buildd remains unchanged, but the launchpad-buildd configuration file needs modifying (done for typical cases by debian/upgrade-config).
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad-buildd:rename-builder-proxy into launchpad-buildd:master.
diff --git a/MANIFEST.in b/MANIFEST.in
index 44e8248..f061118 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,8 +3,8 @@ include Makefile
include bin/builder-prep
include bin/buildrecipe
include bin/in-target
+include bin/lpbuildd-git-proxy
include bin/sbuild-package
-include bin/snap-git-proxy
include bin/test_buildd_generatetranslationtemplates
include bin/test_buildd_recipe
include buildd-genconfig
diff --git a/bin/snap-git-proxy b/bin/lpbuildd-git-proxy
similarity index 100%
rename from bin/snap-git-proxy
rename to bin/lpbuildd-git-proxy
diff --git a/buildd-genconfig b/buildd-genconfig
index 467dee4..2615b88 100755
--- a/buildd-genconfig
+++ b/buildd-genconfig
@@ -37,8 +37,8 @@ parser.add_option("-t", "--template", dest="TEMPLATE",
metavar="FILE",
default="/usr/share/launchpad-buildd/template-buildd-slave.conf")
-parser.add_option("--snap-proxy-port", dest="SNAPPROXYPORT",
- help="the port the local snap proxy binds to",
+parser.add_option("--proxy-port", dest="PROXYPORT",
+ help="the port the local builder proxy binds to",
metavar="PORT",
default="8222")
@@ -51,7 +51,7 @@ replacements = {
"@BINDHOST@": options.BINDHOST,
"@ARCHTAG@": options.ARCHTAG,
"@BINDPORT@": options.BINDPORT,
- "@SNAPPROXYPORT@": options.SNAPPROXYPORT,
+ "@PROXYPORT@": options.PROXYPORT,
}
for replacement_key in replacements:
diff --git a/debian/changelog b/debian/changelog
index 8bcab81..01907c6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+launchpad-buildd (199) UNRELEASED; urgency=medium
+
+ * Rename "snap proxy" to "builder proxy", since it's used for several
+ different build types nowadays.
+
+ -- Colin Watson <cjwatson@xxxxxxxxxx> Wed, 21 Jul 2021 09:31:39 +0100
+
launchpad-buildd (198) bionic; urgency=medium
* Run charmcraft in verbose mode.
diff --git a/debian/launchpad-buildd.install b/debian/launchpad-buildd.install
index 981c763..95db295 100644
--- a/debian/launchpad-buildd.install
+++ b/debian/launchpad-buildd.install
@@ -2,8 +2,8 @@ bin/builder-prep usr/share/launchpad-buildd/bin
bin/buildrecipe usr/share/launchpad-buildd/bin
bin/check-implicit-pointer-functions usr/share/launchpad-buildd/bin
bin/in-target usr/share/launchpad-buildd/bin
+bin/lpbuildd-git-proxy usr/share/launchpad-buildd/bin
bin/sbuild-package usr/share/launchpad-buildd/bin
-bin/snap-git-proxy usr/share/launchpad-buildd/bin
buildd-genconfig usr/share/launchpad-buildd
debian/upgrade-config usr/share/launchpad-buildd
default/launchpad-buildd etc/default
diff --git a/debian/postinst b/debian/postinst
index 7118bb1..1ad6123 100644
--- a/debian/postinst
+++ b/debian/postinst
@@ -14,7 +14,7 @@ BUILDDUID=${BUILDD_UID:-2001}
make_buildd()
{
- /usr/share/launchpad-buildd/buildd-genconfig --name=default --host=0.0.0.0 --port=8221 --snap-proxy-port=8222 > \
+ /usr/share/launchpad-buildd/buildd-genconfig --name=default --host=0.0.0.0 --port=8221 --proxy-port=8222 > \
/etc/launchpad-buildd/default
echo Default buildd created.
}
diff --git a/debian/upgrade-config b/debian/upgrade-config
index 3c7b9d4..659d620 100755
--- a/debian/upgrade-config
+++ b/debian/upgrade-config
@@ -7,6 +7,18 @@
from __future__ import print_function
+try:
+ from configparser import (
+ ConfigParser as SafeConfigParser,
+ NoOptionError,
+ NoSectionError,
+ )
+except ImportError:
+ from ConfigParser import (
+ SafeConfigParser,
+ NoOptionError,
+ NoSectionError,
+ )
import os
import re
import sys
@@ -219,6 +231,48 @@ def upgrade_to_190():
line = "[builder]\n"
out_file.write(line)
+def upgrade_to_199():
+ print("Upgrading %s to version 199" % conf_file)
+
+ # We need to move snapmanager.proxyport to builder.proxyport, so start
+ # by parsing the existing file to find the current value.
+ conf = SafeConfigParser()
+ conf.read(conf_file)
+ try:
+ proxyport = conf.get("snapmanager", "proxyport")
+ except (NoOptionError, NoSectionError):
+ proxyport = None
+
+ os.rename(conf_file, conf_file + "-prev199~")
+ with open(conf_file + "-prev199~", "r") as in_file:
+ with open(conf_file, "w") as out_file:
+ in_builder = False
+ in_snapmanager = False
+ wrote_blank = False
+ for line in in_file:
+ if line.startswith("[builder]"):
+ in_builder = True
+ elif in_builder and (line.startswith("[") or not line.strip()):
+ if proxyport is not None:
+ out_file.write("proxyport = %s\n" % proxyport)
+ in_builder = False
+ elif line.startswith("[snapmanager]"):
+ # Delete this section.
+ in_snapmanager = True
+
+ if not line.strip():
+ wrote_blank = True
+ elif not in_snapmanager:
+ if wrote_blank:
+ out_file.write("\n")
+ out_file.write(line)
+ wrote_blank = False
+
+ if (in_snapmanager and
+ not line.startswith("[snapmanager]") and
+ (line.startswith("[") or not line.strip())):
+ in_snapmanager = False
+
if __name__ == "__main__":
old_version = re.sub(r'[~-].*', '', old_version)
if apt_pkg.version_compare(old_version, "12") < 0:
@@ -249,3 +303,5 @@ if __name__ == "__main__":
upgrade_to_162()
if apt_pkg.version_compare(old_version, "190") < 0:
upgrade_to_190()
+ if apt_pkg.version_compare(old_version, "199") < 0:
+ upgrade_to_199()
diff --git a/lpbuildd/builder.py b/lpbuildd/builder.py
index 5e459be..d83c10f 100644
--- a/lpbuildd/builder.py
+++ b/lpbuildd/builder.py
@@ -49,12 +49,12 @@ def _sanitizeURLs(bytes_seq):
# This regular expression will be used to remove authentication
# credentials from URLs.
password_re = re.compile(br'://([^:]+:[^@]+@)(\S+)')
- # Snap proxy passwords are UUIDs.
- snap_proxy_auth_re = re.compile(br',proxyauth=[^:]+:[A-Za-z0-9-]+')
+ # Builder proxy passwords are UUIDs.
+ proxy_auth_re = re.compile(br',proxyauth=[^:]+:[A-Za-z0-9-]+')
for line in bytes_seq:
sanitized_line = password_re.sub(br'://\2', line)
- sanitized_line = snap_proxy_auth_re.sub(b'', sanitized_line)
+ sanitized_line = proxy_auth_re.sub(b'', sanitized_line)
yield sanitized_line
diff --git a/lpbuildd/charm.py b/lpbuildd/charm.py
index dcfc546..520b97c 100644
--- a/lpbuildd/charm.py
+++ b/lpbuildd/charm.py
@@ -11,7 +11,7 @@ from lpbuildd.debian import (
DebianBuildState,
DebianBuildManager,
)
-from lpbuildd.snap import SnapBuildProxyMixin
+from lpbuildd.proxy import BuildManagerProxyMixin
RETCODE_SUCCESS = 0
@@ -23,7 +23,7 @@ class CharmBuildState(DebianBuildState):
BUILD_CHARM = "BUILD_CHARM"
-class CharmBuildManager(SnapBuildProxyMixin, DebianBuildManager):
+class CharmBuildManager(BuildManagerProxyMixin, DebianBuildManager):
"""Build a charm."""
backend_name = "lxd"
diff --git a/lpbuildd/oci.py b/lpbuildd/oci.py
index 53198c8..c0aa6c2 100644
--- a/lpbuildd/oci.py
+++ b/lpbuildd/oci.py
@@ -21,7 +21,7 @@ from lpbuildd.debian import (
DebianBuildManager,
DebianBuildState,
)
-from lpbuildd.snap import SnapBuildProxyMixin
+from lpbuildd.proxy import BuildManagerProxyMixin
RETCODE_SUCCESS = 0
@@ -33,7 +33,7 @@ class OCIBuildState(DebianBuildState):
BUILD_OCI = "BUILD_OCI"
-class OCIBuildManager(SnapBuildProxyMixin, DebianBuildManager):
+class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
"""Build an OCI Image."""
backend_name = "lxd"
diff --git a/lpbuildd/proxy.py b/lpbuildd/proxy.py
new file mode 100644
index 0000000..908770d
--- /dev/null
+++ b/lpbuildd/proxy.py
@@ -0,0 +1,261 @@
+# Copyright 2015-2021 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+import base64
+import io
+
+from six.moves.urllib.error import (
+ HTTPError,
+ URLError,
+ )
+from six.moves.urllib.parse import urlparse
+from six.moves.urllib.request import (
+ Request,
+ urlopen,
+ )
+from twisted.application import strports
+from twisted.internet import reactor
+from twisted.internet.interfaces import IHalfCloseableProtocol
+from twisted.python.compat import intToBytes
+from twisted.web import (
+ http,
+ proxy,
+ )
+from zope.interface import implementer
+
+
+class BuilderProxyClient(proxy.ProxyClient):
+
+ def __init__(self, command, rest, version, headers, data, father):
+ proxy.ProxyClient.__init__(
+ self, command, rest, version, headers, data, father)
+ # Why doesn't ProxyClient at least store this?
+ self.version = version
+ # We must avoid calling self.father.finish in the event that its
+ # connection was already lost, i.e. if the original client
+ # disconnects first (which is particularly likely in the case of
+ # CONNECT).
+ d = self.father.notifyFinish()
+ d.addBoth(self.requestFinished)
+
+ def connectionMade(self):
+ proxy.ProxyClient.connectionMade(self)
+ self.father.setChildClient(self)
+
+ def sendCommand(self, command, path):
+ # For some reason, HTTPClient.sendCommand doesn't preserve the
+ # protocol version.
+ self.transport.writeSequence(
+ [command, b' ', path, b' ', self.version, b'\r\n'])
+
+ def handleEndHeaders(self):
+ self.father.handleEndHeaders()
+
+ def sendData(self, data):
+ self.transport.write(data)
+
+ def endData(self):
+ if self.transport is not None:
+ self.transport.loseWriteConnection()
+
+ def requestFinished(self, result):
+ self._finished = True
+ self.transport.loseConnection()
+
+
+class BuilderProxyClientFactory(proxy.ProxyClientFactory):
+
+ protocol = BuilderProxyClient
+
+
+class BuilderProxyRequest(http.Request):
+
+ child_client = None
+ _request_buffer = None
+ _request_data_done = False
+
+ def setChildClient(self, child_client):
+ self.child_client = child_client
+ if self._request_buffer is not None:
+ self.child_client.sendData(self._request_buffer.getvalue())
+ self._request_buffer = None
+ if self._request_data_done:
+ self.child_client.endData()
+
+ def allHeadersReceived(self, command, path, version):
+ # Normally done in `requestReceived`, but we disable that since it
+ # does other things we don't want.
+ self.method, self.uri, self.clientproto = command, path, version
+ self.client = self.channel.transport.getPeer()
+ self.host = self.channel.transport.getHost()
+
+ remote_parsed = urlparse(self.channel.factory.remote_url)
+ request_parsed = urlparse(path)
+ headers = self.getAllHeaders().copy()
+ if b"host" not in headers and request_parsed.netloc:
+ headers[b"host"] = request_parsed.netloc
+ if remote_parsed.username:
+ auth = (remote_parsed.username + ":" +
+ remote_parsed.password).encode("ASCII")
+ authHeader = b"Basic " + base64.b64encode(auth)
+ headers[b"proxy-authorization"] = authHeader
+ self.client_factory = BuilderProxyClientFactory(
+ command, path, version, headers, b"", self)
+ reactor.connectTCP(
+ remote_parsed.hostname, remote_parsed.port, self.client_factory)
+
+ def requestReceived(self, command, path, version):
+ # We do most of our work in `allHeadersReceived` instead.
+ pass
+
+ def rawDataReceived(self, data):
+ if self.child_client is not None:
+ if not self._request_data_done:
+ self.child_client.sendData(data)
+ else:
+ if self._request_buffer is None:
+ self._request_buffer = io.BytesIO()
+ self._request_buffer.write(data)
+
+ def handleEndHeaders(self):
+ # Cut-down version of Request.write. We must avoid switching to
+ # chunked encoding for the sake of CONNECT; since our actual
+ # response data comes from another proxy, we can cut some corners.
+ if self.startedWriting:
+ return
+ self.startedWriting = 1
+ l = []
+ l.append(
+ self.clientproto + b" " + intToBytes(self.code) + b" " +
+ self.code_message + b"\r\n")
+ for name, values in self.responseHeaders.getAllRawHeaders():
+ for value in values:
+ l.extend([name, b": ", value, b"\r\n"])
+ l.append(b"\r\n")
+ self.transport.writeSequence(l)
+
+ def write(self, data):
+ if self.channel is not None:
+ self.channel.resetTimeout()
+ http.Request.write(self, data)
+
+ def endData(self):
+ if self.child_client is not None:
+ self.child_client.endData()
+ self._request_data_done = True
+
+
+@implementer(IHalfCloseableProtocol)
+class BuilderProxy(http.HTTPChannel):
+ """A channel that streams request data.
+
+ The stock HTTPChannel isn't quite suitable for our needs, because it
+ expects to read the entire request data before passing control to the
+ request. This doesn't work well for CONNECT.
+ """
+
+ requestFactory = BuilderProxyRequest
+
+ def checkPersistence(self, request, version):
+ # ProxyClient.__init__ forces "Connection: close".
+ return False
+
+ def allHeadersReceived(self):
+ http.HTTPChannel.allHeadersReceived(self)
+ self.requests[-1].allHeadersReceived(
+ self._command, self._path, self._version)
+ if self._command == b"CONNECT":
+ # This is a lie, but we don't want HTTPChannel to decide that
+ # the request is finished just because a CONNECT request
+ # (naturally) has no Content-Length.
+ self.length = -1
+
+ def rawDataReceived(self, data):
+ self.resetTimeout()
+ if self.requests:
+ self.requests[-1].rawDataReceived(data)
+
+ def readConnectionLost(self):
+ for request in self.requests:
+ request.endData()
+
+ def writeConnectionLost(self):
+ pass
+
+
+class BuilderProxyFactory(http.HTTPFactory):
+
+ protocol = BuilderProxy
+
+ def __init__(self, manager, remote_url, *args, **kwargs):
+ http.HTTPFactory.__init__(self, *args, **kwargs)
+ self.manager = manager
+ self.remote_url = remote_url
+
+ def log(self, request):
+ # Log requests to the build log rather than to Twisted.
+ # Reimplement log formatting because there's no point logging the IP
+ # here.
+ referrer = http._escape(request.getHeader(b"referer") or b"-")
+ agent = http._escape(request.getHeader(b"user-agent") or b"-")
+ line = (
+ u'%(timestamp)s "%(method)s %(uri)s %(protocol)s" '
+ u'%(code)d %(length)s "%(referrer)s" "%(agent)s"\n' % {
+ 'timestamp': self._logDateTime,
+ 'method': http._escape(request.method),
+ 'uri': http._escape(request.uri),
+ 'protocol': http._escape(request.clientproto),
+ 'code': request.code,
+ 'length': request.sentLength or "-",
+ 'referrer': referrer,
+ 'agent': agent,
+ })
+ self.manager._builder.log(line.encode("UTF-8"))
+
+
+class BuildManagerProxyMixin:
+
+ def startProxy(self):
+ """Start the local builder proxy, if necessary."""
+ if not self.proxy_url:
+ return []
+ proxy_port = self._builder._config.get("builder", "proxyport")
+ proxy_factory = BuilderProxyFactory(self, self.proxy_url, timeout=60)
+ self.proxy_service = strports.service(
+ "tcp:%s" % proxy_port, proxy_factory)
+ self.proxy_service.setServiceParent(self._builder.service)
+ if self.backend_name == "lxd":
+ proxy_host = self.backend.ipv4_network.ip
+ else:
+ proxy_host = "localhost"
+ return ["--proxy-url", "http://{}:{}/".format(proxy_host, proxy_port)]
+
+ def stopProxy(self):
+ """Stop the local builder proxy, if necessary."""
+ if self.proxy_service is None:
+ return
+ self.proxy_service.disownServiceParent()
+ self.proxy_service = None
+
+ def revokeProxyToken(self):
+ """Revoke builder proxy token."""
+ if not self.revocation_endpoint:
+ return
+ self._builder.log("Revoking proxy token...\n")
+ url = urlparse(self.proxy_url)
+ auth = "{}:{}".format(url.username, url.password)
+ headers = {
+ "Authorization": "Basic {}".format(
+ base64.b64encode(auth.encode('utf-8')).decode('utf-8'))
+ }
+ req = Request(self.revocation_endpoint, None, headers)
+ req.get_method = lambda: "DELETE"
+ try:
+ urlopen(req)
+ except (HTTPError, URLError) as e:
+ self._builder.log(
+ "Unable to revoke token for %s: %s" % (url.username, e))
diff --git a/lpbuildd/snap.py b/lpbuildd/snap.py
index d720d49..2a5ffc4 100644
--- a/lpbuildd/snap.py
+++ b/lpbuildd/snap.py
@@ -5,8 +5,6 @@ from __future__ import print_function
__metaclass__ = type
-import base64
-import io
import json
import os
import sys
@@ -15,30 +13,13 @@ from six.moves.configparser import (
NoOptionError,
NoSectionError,
)
-from six.moves.urllib.error import (
- HTTPError,
- URLError,
- )
-from six.moves.urllib.parse import urlparse
-from six.moves.urllib.request import (
- Request,
- urlopen,
- )
-from twisted.application import strports
-from twisted.internet import reactor
-from twisted.internet.interfaces import IHalfCloseableProtocol
-from twisted.python.compat import intToBytes
-from twisted.web import (
- http,
- proxy,
- )
-from zope.interface import implementer
from lpbuildd.debian import (
DebianBuildManager,
DebianBuildState,
get_build_path,
)
+from lpbuildd.proxy import BuildManagerProxyMixin
RETCODE_SUCCESS = 0
@@ -46,244 +27,11 @@ RETCODE_FAILURE_INSTALL = 200
RETCODE_FAILURE_BUILD = 201
-class SnapProxyClient(proxy.ProxyClient):
-
- def __init__(self, command, rest, version, headers, data, father):
- proxy.ProxyClient.__init__(
- self, command, rest, version, headers, data, father)
- # Why doesn't ProxyClient at least store this?
- self.version = version
- # We must avoid calling self.father.finish in the event that its
- # connection was already lost, i.e. if the original client
- # disconnects first (which is particularly likely in the case of
- # CONNECT).
- d = self.father.notifyFinish()
- d.addBoth(self.requestFinished)
-
- def connectionMade(self):
- proxy.ProxyClient.connectionMade(self)
- self.father.setChildClient(self)
-
- def sendCommand(self, command, path):
- # For some reason, HTTPClient.sendCommand doesn't preserve the
- # protocol version.
- self.transport.writeSequence(
- [command, b' ', path, b' ', self.version, b'\r\n'])
-
- def handleEndHeaders(self):
- self.father.handleEndHeaders()
-
- def sendData(self, data):
- self.transport.write(data)
-
- def endData(self):
- if self.transport is not None:
- self.transport.loseWriteConnection()
-
- def requestFinished(self, result):
- self._finished = True
- self.transport.loseConnection()
-
-
-class SnapProxyClientFactory(proxy.ProxyClientFactory):
-
- protocol = SnapProxyClient
-
-
-class SnapProxyRequest(http.Request):
-
- child_client = None
- _request_buffer = None
- _request_data_done = False
-
- def setChildClient(self, child_client):
- self.child_client = child_client
- if self._request_buffer is not None:
- self.child_client.sendData(self._request_buffer.getvalue())
- self._request_buffer = None
- if self._request_data_done:
- self.child_client.endData()
-
- def allHeadersReceived(self, command, path, version):
- # Normally done in `requestReceived`, but we disable that since it
- # does other things we don't want.
- self.method, self.uri, self.clientproto = command, path, version
- self.client = self.channel.transport.getPeer()
- self.host = self.channel.transport.getHost()
-
- remote_parsed = urlparse(self.channel.factory.remote_url)
- request_parsed = urlparse(path)
- headers = self.getAllHeaders().copy()
- if b"host" not in headers and request_parsed.netloc:
- headers[b"host"] = request_parsed.netloc
- if remote_parsed.username:
- auth = (remote_parsed.username + ":" +
- remote_parsed.password).encode("ASCII")
- authHeader = b"Basic " + base64.b64encode(auth)
- headers[b"proxy-authorization"] = authHeader
- self.client_factory = SnapProxyClientFactory(
- command, path, version, headers, b"", self)
- reactor.connectTCP(
- remote_parsed.hostname, remote_parsed.port, self.client_factory)
-
- def requestReceived(self, command, path, version):
- # We do most of our work in `allHeadersReceived` instead.
- pass
-
- def rawDataReceived(self, data):
- if self.child_client is not None:
- if not self._request_data_done:
- self.child_client.sendData(data)
- else:
- if self._request_buffer is None:
- self._request_buffer = io.BytesIO()
- self._request_buffer.write(data)
-
- def handleEndHeaders(self):
- # Cut-down version of Request.write. We must avoid switching to
- # chunked encoding for the sake of CONNECT; since our actual
- # response data comes from another proxy, we can cut some corners.
- if self.startedWriting:
- return
- self.startedWriting = 1
- l = []
- l.append(
- self.clientproto + b" " + intToBytes(self.code) + b" " +
- self.code_message + b"\r\n")
- for name, values in self.responseHeaders.getAllRawHeaders():
- for value in values:
- l.extend([name, b": ", value, b"\r\n"])
- l.append(b"\r\n")
- self.transport.writeSequence(l)
-
- def write(self, data):
- if self.channel is not None:
- self.channel.resetTimeout()
- http.Request.write(self, data)
-
- def endData(self):
- if self.child_client is not None:
- self.child_client.endData()
- self._request_data_done = True
-
-
-@implementer(IHalfCloseableProtocol)
-class SnapProxy(http.HTTPChannel):
- """A channel that streams request data.
-
- The stock HTTPChannel isn't quite suitable for our needs, because it
- expects to read the entire request data before passing control to the
- request. This doesn't work well for CONNECT.
- """
-
- requestFactory = SnapProxyRequest
-
- def checkPersistence(self, request, version):
- # ProxyClient.__init__ forces "Connection: close".
- return False
-
- def allHeadersReceived(self):
- http.HTTPChannel.allHeadersReceived(self)
- self.requests[-1].allHeadersReceived(
- self._command, self._path, self._version)
- if self._command == b"CONNECT":
- # This is a lie, but we don't want HTTPChannel to decide that
- # the request is finished just because a CONNECT request
- # (naturally) has no Content-Length.
- self.length = -1
-
- def rawDataReceived(self, data):
- self.resetTimeout()
- if self.requests:
- self.requests[-1].rawDataReceived(data)
-
- def readConnectionLost(self):
- for request in self.requests:
- request.endData()
-
- def writeConnectionLost(self):
- pass
-
-
-class SnapProxyFactory(http.HTTPFactory):
-
- protocol = SnapProxy
-
- def __init__(self, manager, remote_url, *args, **kwargs):
- http.HTTPFactory.__init__(self, *args, **kwargs)
- self.manager = manager
- self.remote_url = remote_url
-
- def log(self, request):
- # Log requests to the build log rather than to Twisted.
- # Reimplement log formatting because there's no point logging the IP
- # here.
- referrer = http._escape(request.getHeader(b"referer") or b"-")
- agent = http._escape(request.getHeader(b"user-agent") or b"-")
- line = (
- u'%(timestamp)s "%(method)s %(uri)s %(protocol)s" '
- u'%(code)d %(length)s "%(referrer)s" "%(agent)s"\n' % {
- 'timestamp': self._logDateTime,
- 'method': http._escape(request.method),
- 'uri': http._escape(request.uri),
- 'protocol': http._escape(request.clientproto),
- 'code': request.code,
- 'length': request.sentLength or "-",
- 'referrer': referrer,
- 'agent': agent,
- })
- self.manager._builder.log(line.encode("UTF-8"))
-
-
class SnapBuildState(DebianBuildState):
BUILD_SNAP = "BUILD_SNAP"
-class SnapBuildProxyMixin():
-
- def startProxy(self):
- """Start the local snap proxy, if necessary."""
- if not self.proxy_url:
- return []
- proxy_port = self._builder._config.get("snapmanager", "proxyport")
- proxy_factory = SnapProxyFactory(self, self.proxy_url, timeout=60)
- self.proxy_service = strports.service(
- "tcp:%s" % proxy_port, proxy_factory)
- self.proxy_service.setServiceParent(self._builder.service)
- if self.backend_name == "lxd":
- proxy_host = self.backend.ipv4_network.ip
- else:
- proxy_host = "localhost"
- return ["--proxy-url", "http://{}:{}/".format(proxy_host, proxy_port)]
-
- def stopProxy(self):
- """Stop the local snap proxy, if necessary."""
- if self.proxy_service is None:
- return
- self.proxy_service.disownServiceParent()
- self.proxy_service = None
-
- def revokeProxyToken(self):
- """Revoke builder proxy token."""
- if not self.revocation_endpoint:
- return
- self._builder.log("Revoking proxy token...\n")
- url = urlparse(self.proxy_url)
- auth = "{}:{}".format(url.username, url.password)
- headers = {
- "Authorization": "Basic {}".format(
- base64.b64encode(auth.encode('utf-8')).decode('utf-8'))
- }
- req = Request(self.revocation_endpoint, None, headers)
- req.get_method = lambda: "DELETE"
- try:
- urlopen(req)
- except (HTTPError, URLError) as e:
- self._builder.log(
- "Unable to revoke token for %s: %s" % (url.username, e))
-
-
-class SnapBuildManager(SnapBuildProxyMixin, DebianBuildManager):
+class SnapBuildManager(BuildManagerProxyMixin, DebianBuildManager):
"""Build a snap."""
backend_name = "lxd"
diff --git a/lpbuildd/target/build_charm.py b/lpbuildd/target/build_charm.py
index b2cb740..ec4ba0f 100644
--- a/lpbuildd/target/build_charm.py
+++ b/lpbuildd/target/build_charm.py
@@ -13,7 +13,7 @@ import sys
from lpbuildd.target.backend import check_path_escape
from lpbuildd.target.build_snap import SnapChannelsAction
from lpbuildd.target.operation import Operation
-from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin
+from lpbuildd.target.proxy import BuilderProxyOperationMixin
from lpbuildd.target.snapstore import SnapStoreOperationMixin
from lpbuildd.target.vcs import VCSOperationMixin
@@ -25,7 +25,7 @@ RETCODE_FAILURE_BUILD = 201
logger = logging.getLogger(__name__)
-class BuildCharm(SnapBuildProxyOperationMixin, VCSOperationMixin,
+class BuildCharm(BuilderProxyOperationMixin, VCSOperationMixin,
SnapStoreOperationMixin, Operation):
description = "Build a charm."
@@ -124,7 +124,7 @@ class BuildCharm(SnapBuildProxyOperationMixin, VCSOperationMixin,
if self.args.proxy_url:
env["http_proxy"] = self.args.proxy_url
env["https_proxy"] = self.args.proxy_url
- env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
+ env["GIT_PROXY_COMMAND"] = "/usr/local/bin/lpbuildd-git-proxy"
args = [
"charmcraft", "build", "-v",
"-p", build_context_path,
diff --git a/lpbuildd/target/build_oci.py b/lpbuildd/target/build_oci.py
index 5615325..437932c 100644
--- a/lpbuildd/target/build_oci.py
+++ b/lpbuildd/target/build_oci.py
@@ -14,7 +14,7 @@ from textwrap import dedent
from lpbuildd.target.backend import check_path_escape
from lpbuildd.target.operation import Operation
-from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin
+from lpbuildd.target.proxy import BuilderProxyOperationMixin
from lpbuildd.target.snapstore import SnapStoreOperationMixin
from lpbuildd.target.vcs import VCSOperationMixin
@@ -26,7 +26,7 @@ RETCODE_FAILURE_BUILD = 201
logger = logging.getLogger(__name__)
-class BuildOCI(SnapBuildProxyOperationMixin, VCSOperationMixin,
+class BuildOCI(BuilderProxyOperationMixin, VCSOperationMixin,
SnapStoreOperationMixin, Operation):
description = "Build an OCI image."
diff --git a/lpbuildd/target/build_snap.py b/lpbuildd/target/build_snap.py
index 8b08cfa..e8495f8 100644
--- a/lpbuildd/target/build_snap.py
+++ b/lpbuildd/target/build_snap.py
@@ -17,7 +17,7 @@ from textwrap import dedent
from six.moves.urllib.parse import urlparse
from lpbuildd.target.operation import Operation
-from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin
+from lpbuildd.target.proxy import BuilderProxyOperationMixin
from lpbuildd.target.snapstore import SnapStoreOperationMixin
from lpbuildd.target.vcs import VCSOperationMixin
@@ -47,7 +47,7 @@ class SnapChannelsAction(argparse.Action):
getattr(namespace, self.dest)[snap] = channel
-class BuildSnap(SnapBuildProxyOperationMixin, VCSOperationMixin,
+class BuildSnap(BuilderProxyOperationMixin, VCSOperationMixin,
SnapStoreOperationMixin, Operation):
description = "Build a snap."
@@ -189,7 +189,7 @@ class BuildSnap(SnapBuildProxyOperationMixin, VCSOperationMixin,
if self.args.proxy_url:
env["http_proxy"] = self.args.proxy_url
env["https_proxy"] = self.args.proxy_url
- env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
+ env["GIT_PROXY_COMMAND"] = "/usr/local/bin/lpbuildd-git-proxy"
self.run_build_command(
["snapcraft", "pull"],
cwd=os.path.join("/build", self.args.name),
@@ -213,7 +213,7 @@ class BuildSnap(SnapBuildProxyOperationMixin, VCSOperationMixin,
if self.args.proxy_url:
env["http_proxy"] = self.args.proxy_url
env["https_proxy"] = self.args.proxy_url
- env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
+ env["GIT_PROXY_COMMAND"] = "/usr/local/bin/lpbuildd-git-proxy"
self.run_build_command(
["snapcraft"],
cwd=os.path.join("/build", self.args.name),
diff --git a/lpbuildd/target/snapbuildproxy.py b/lpbuildd/target/proxy.py
similarity index 81%
rename from lpbuildd/target/snapbuildproxy.py
rename to lpbuildd/target/proxy.py
index 6ab8097..4834e46 100644
--- a/lpbuildd/target/snapbuildproxy.py
+++ b/lpbuildd/target/proxy.py
@@ -9,12 +9,12 @@ from collections import OrderedDict
import os
-class SnapBuildProxyOperationMixin:
- """Methods supporting the build time HTTP proxy for snap and OCI builds."""
+class BuilderProxyOperationMixin:
+ """Methods supporting the build time HTTP proxy for certain build types."""
@classmethod
def add_arguments(cls, parser):
- super(SnapBuildProxyOperationMixin, cls).add_arguments(parser)
+ super(BuilderProxyOperationMixin, cls).add_arguments(parser)
parser.add_argument("--proxy-url", help="builder proxy url")
parser.add_argument(
"--revocation-endpoint",
@@ -26,8 +26,8 @@ class SnapBuildProxyOperationMixin:
def install_git_proxy(self):
self.backend.copy_in(
- os.path.join(self.bin, "snap-git-proxy"),
- "/usr/local/bin/snap-git-proxy")
+ os.path.join(self.bin, "lpbuildd-git-proxy"),
+ "/usr/local/bin/lpbuildd-git-proxy")
def build_proxy_environment(self, proxy_url=None, env=None):
"""Extend a command environment to include http proxy variables."""
@@ -37,5 +37,5 @@ class SnapBuildProxyOperationMixin:
if proxy_url:
full_env["http_proxy"] = self.args.proxy_url
full_env["https_proxy"] = self.args.proxy_url
- full_env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
+ full_env["GIT_PROXY_COMMAND"] = "/usr/local/bin/lpbuildd-git-proxy"
return full_env
diff --git a/lpbuildd/target/tests/test_build_charm.py b/lpbuildd/target/tests/test_build_charm.py
index ae1da6a..61e8602 100644
--- a/lpbuildd/target/tests/test_build_charm.py
+++ b/lpbuildd/target/tests/test_build_charm.py
@@ -198,7 +198,7 @@ class TestBuildCharm(TestCase):
build_charm.bin = "/builderbin"
self.useFixture(FakeFilesystem()).add("/builderbin")
os.mkdir("/builderbin")
- with open("/builderbin/snap-git-proxy", "w") as proxy_script:
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
proxy_script.write("proxy script\n")
os.fchmod(proxy_script.fileno(), 0o755)
build_charm.install()
@@ -211,7 +211,8 @@ class TestBuildCharm(TestCase):
]))
self.assertEqual(
(b"proxy script\n", stat.S_IFREG | 0o755),
- build_charm.backend.backend_fs["/usr/local/bin/snap-git-proxy"])
+ build_charm.backend.backend_fs[
+ "/usr/local/bin/lpbuildd-git-proxy"])
def test_repo_bzr(self):
args = [
@@ -328,7 +329,7 @@ class TestBuildCharm(TestCase):
env = {
"http_proxy": "http://proxy.example:3128/";,
"https_proxy": "http://proxy.example:3128/";,
- "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
}
self.assertThat(build_charm.backend.run.calls, MatchesListwise([
RanBuildCommand(
@@ -394,7 +395,7 @@ class TestBuildCharm(TestCase):
"CHARMCRAFT_MANAGED_MODE": "1",
"http_proxy": "http://proxy.example:3128/";,
"https_proxy": "http://proxy.example:3128/";,
- "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
}
self.assertThat(build_charm.backend.run.calls, MatchesListwise([
RanBuildCommand(
diff --git a/lpbuildd/target/tests/test_build_oci.py b/lpbuildd/target/tests/test_build_oci.py
index 3b9966a..4b95221 100644
--- a/lpbuildd/target/tests/test_build_oci.py
+++ b/lpbuildd/target/tests/test_build_oci.py
@@ -177,7 +177,7 @@ class TestBuildOCI(TestCase):
build_oci.bin = "/builderbin"
self.useFixture(FakeFilesystem()).add("/builderbin")
os.mkdir("/builderbin")
- with open("/builderbin/snap-git-proxy", "w") as proxy_script:
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
proxy_script.write("proxy script\n")
os.fchmod(proxy_script.fileno(), 0o755)
build_oci.install()
@@ -190,7 +190,7 @@ class TestBuildOCI(TestCase):
]))
self.assertEqual(
(b"proxy script\n", stat.S_IFREG | 0o755),
- build_oci.backend.backend_fs["/usr/local/bin/snap-git-proxy"])
+ build_oci.backend.backend_fs["/usr/local/bin/lpbuildd-git-proxy"])
def test_repo_bzr(self):
args = [
@@ -279,7 +279,7 @@ class TestBuildOCI(TestCase):
env = {
"http_proxy": "http://proxy.example:3128/";,
"https_proxy": "http://proxy.example:3128/";,
- "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
}
self.assertThat(build_oci.backend.run.calls, MatchesListwise([
RanBuildCommand(
diff --git a/lpbuildd/target/tests/test_build_snap.py b/lpbuildd/target/tests/test_build_snap.py
index 3e9f320..1a529ea 100644
--- a/lpbuildd/target/tests/test_build_snap.py
+++ b/lpbuildd/target/tests/test_build_snap.py
@@ -183,7 +183,7 @@ class TestBuildSnap(TestCase):
build_snap.bin = "/builderbin"
self.useFixture(FakeFilesystem()).add("/builderbin")
os.mkdir("/builderbin")
- with open("/builderbin/snap-git-proxy", "w") as proxy_script:
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
proxy_script.write("proxy script\n")
os.fchmod(proxy_script.fileno(), 0o755)
build_snap.install()
@@ -193,7 +193,7 @@ class TestBuildSnap(TestCase):
]))
self.assertEqual(
(b"proxy script\n", stat.S_IFREG | 0o755),
- build_snap.backend.backend_fs["/usr/local/bin/snap-git-proxy"])
+ build_snap.backend.backend_fs["/usr/local/bin/lpbuildd-git-proxy"])
self.assertEqual(
(b"[global]\n"
b"http-proxy-host = proxy.example\n"
@@ -333,7 +333,7 @@ class TestBuildSnap(TestCase):
env = {
"http_proxy": "http://proxy.example:3128/";,
"https_proxy": "http://proxy.example:3128/";,
- "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
}
self.assertThat(build_snap.backend.run.calls, MatchesListwise([
RanBuildCommand(
@@ -389,7 +389,7 @@ class TestBuildSnap(TestCase):
"SNAPCRAFT_BUILD_ENVIRONMENT": "host",
"http_proxy": "http://proxy.example:3128/";,
"https_proxy": "http://proxy.example:3128/";,
- "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
}
self.assertThat(build_snap.backend.run.calls, MatchesListwise([
RanBuildCommand(
@@ -473,7 +473,7 @@ class TestBuildSnap(TestCase):
"SNAPCRAFT_BUILD_ENVIRONMENT": "host",
"http_proxy": "http://proxy.example:3128/";,
"https_proxy": "http://proxy.example:3128/";,
- "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
}
self.assertThat(build_snap.backend.run.calls, MatchesListwise([
RanBuildCommand(["snapcraft"], cwd="/build/test-snap", **env),
diff --git a/lpbuildd/tests/buildlog b/lpbuildd/tests/buildlog
index bc033f6..4b64e85 100644
--- a/lpbuildd/tests/buildlog
+++ b/lpbuildd/tests/buildlog
@@ -21,4 +21,4 @@ The following packages will be upgraded:
Need to get 0B/2832kB of archives.
After unpacking 94.2kB of additional disk space will be used.
(Reading database ... 8942 files and directories currently installed.)
-socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
+socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
diff --git a/lpbuildd/tests/buildlog.long b/lpbuildd/tests/buildlog.long
index 8de21c7..9ca5464 100644
--- a/lpbuildd/tests/buildlog.long
+++ b/lpbuildd/tests/buildlog.long
@@ -21,7 +21,7 @@ The following packages will be upgraded:
Need to get 0B/2832kB of archives.
After unpacking 94.2kB of additional disk space will be used.
(Reading database ... 8942 files and directories currently installed.)
-socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
+socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
RUN: /usr/share/launchpad-buildd/bin/unpack-chroot 370614-896976 /home/buildd/filecache-default/a40e3c410938399b35051833fe5244f9ac6f3774
Unpacking chroot for build 370614-896976
RUN: /usr/share/launchpad-buildd/bin/mount-chroot 370614-896976
@@ -81,4 +81,4 @@ Get:2 http://must:go@ftpmaster.internal gutsy Release [65.9kB]
Get:3 http://scrub:this@ftpmaster.internal gutsy/main Packages [1085kB]
Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
Fetched 5142kB in 5s (1012kB/s)
-socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
+socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
diff --git a/lpbuildd/tests/test_1.diff b/lpbuildd/tests/test_1.diff
index 249b2af..d4f9b70 100644
--- a/lpbuildd/tests/test_1.diff
+++ b/lpbuildd/tests/test_1.diff
@@ -20,5 +20,5 @@
Need to get 0B/2832kB of archives.
After unpacking 94.2kB of additional disk space will be used.
(Reading database ... 8942 files and directories currently installed.)
--socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
-+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128
\ No newline at end of file
+-socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
++socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128
\ No newline at end of file
diff --git a/lpbuildd/tests/test_2.diff b/lpbuildd/tests/test_2.diff
index 8cccd43..77362dd 100644
--- a/lpbuildd/tests/test_2.diff
+++ b/lpbuildd/tests/test_2.diff
@@ -4,7 +4,7 @@
@@ -1,13 +1,12 @@
--or build 370614-896976
+-build 370614-896976
RUN: /usr/share/launchpad-buildd/bin/mount-chroot 370614-896976
Mounting chroot for build 370614-896976
RUN: /usr/share/launchpad-buildd/bin/apply-ogre-model 370614-896976 universe
@@ -32,5 +32,5 @@
+Get:3 http://ftpmaster.internal gutsy/main Packages [1085kB]
Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
Fetched 5142kB in 5s (1012kB/s)
--socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
-+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128
\ No newline at end of file
+-socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
++socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128
\ No newline at end of file
diff --git a/lpbuildd/tests/test_charm.py b/lpbuildd/tests/test_charm.py
index 8610bad..18e01a1 100644
--- a/lpbuildd/tests/test_charm.py
+++ b/lpbuildd/tests/test_charm.py
@@ -142,7 +142,7 @@ class TestCharmBuildManagerIteration(TestCase):
self.buildmanager.iterate, self.buildmanager.iterators[-1])
self.assertFalse(self.builder.wasCalled("buildFail"))
- @mock.patch('lpbuildd.snap.urlopen')
+ @mock.patch('lpbuildd.proxy.urlopen')
def test_revokeProxyToken(self, urlopen_mock):
self.buildmanager.revocation_endpoint = "http://revoke_endpoint";
self.buildmanager.proxy_url = "http://username:password@proxy_url";
diff --git a/lpbuildd/tests/test_snap.py b/lpbuildd/tests/test_snap.py
index 5b7053d..b018a53 100644
--- a/lpbuildd/tests/test_snap.py
+++ b/lpbuildd/tests/test_snap.py
@@ -29,10 +29,10 @@ from twisted.web import (
static,
)
+from lpbuildd.proxy import BuilderProxyFactory
from lpbuildd.snap import (
SnapBuildManager,
SnapBuildState,
- SnapProxyFactory,
)
from lpbuildd.tests.fakebuilder import FakeBuilder
from lpbuildd.tests.matchers import HasWaitingFiles
@@ -411,7 +411,7 @@ class TestSnapBuildManagerIteration(TestCase):
return remote_proxy_listener
def startLocalProxy(self, remote_url):
- proxy_factory = SnapProxyFactory(
+ proxy_factory = BuilderProxyFactory(
self.buildmanager, remote_url, timeout=60)
proxy_listener = reactor.listenTCP(0, proxy_factory)
self.addCleanup(proxy_listener.stopListening)
@@ -453,7 +453,7 @@ class TestSnapBuildManagerIteration(TestCase):
# the code under test since the stock twisted.web.proxy doesn't support
# CONNECT.
- @mock.patch('lpbuildd.snap.urlopen')
+ @mock.patch('lpbuildd.proxy.urlopen')
def test_revokeProxyToken(self, urlopen_mock):
self.buildmanager.revocation_endpoint = "http://revoke_endpoint";
self.buildmanager.proxy_url = "http://username:password@proxy_url";
diff --git a/template-buildd-slave.conf b/template-buildd-slave.conf
index b195c82..be99a61 100644
--- a/template-buildd-slave.conf
+++ b/template-buildd-slave.conf
@@ -14,4 +14,4 @@ sharepath = /usr/share/launchpad-buildd
resultarchive = translation-templates.tar.gz
[snapmanager]
-proxyport = @SNAPPROXYPORT@
+proxyport = @PROXYPORT@
