launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #26074
[Merge] ~pappacena/launchpad:gunicorn-appserver-tests into launchpad:master
Thiago F. Pappacena has proposed merging ~pappacena/launchpad:gunicorn-appserver-tests into launchpad:master.
Commit message:
Run tests with gunicorn instead of zope server (disabled)
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~pappacena/launchpad/+git/launchpad/+merge/396691
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~pappacena/launchpad:gunicorn-appserver-tests into launchpad:master.
diff --git a/configs/development/launchpad-lazr.conf b/configs/development/launchpad-lazr.conf
index a4956f3..8409e5c 100644
--- a/configs/development/launchpad-lazr.conf
+++ b/configs/development/launchpad-lazr.conf
@@ -88,6 +88,7 @@ public_host: keyserver.launchpad.test
public_https: False
[launchpad]
+devmode: true
enable_test_openid_provider: True
openid_canonical_root: https://testopenid.test/
openid_provider_root: https://testopenid.test/
@@ -223,6 +224,7 @@ hostname: xmlrpc.launchpad.test
[vhost.xmlrpc_private]
hostname: xmlrpc-private.launchpad.test
+private_port: 8087
[vhost.feeds]
hostname: feeds.launchpad.test
diff --git a/configs/testrunner-appserver/gunicorn.conf.py b/configs/testrunner-appserver/gunicorn.conf.py
new file mode 100644
index 0000000..ba22ec1
--- /dev/null
+++ b/configs/testrunner-appserver/gunicorn.conf.py
@@ -0,0 +1,12 @@
+import os
+config_dir = os.path.dirname(__file__)
+log_dir = os.path.join(config_dir, '..', '..', 'logs')
+
+bind = [":8085", ":8087"]
+workers = 1
+threads = 10
+log_level = "DEBUG"
+
+log_file = os.path.join(log_dir, 'gunicorn.log')
+error_logfile = os.path.join(log_dir, 'gunicorn-error.log')
+access_logfile = os.path.join(log_dir, 'gunicorn-access.log')
diff --git a/configs/testrunner/gunicorn.conf.py b/configs/testrunner/gunicorn.conf.py
new file mode 100644
index 0000000..ba22ec1
--- /dev/null
+++ b/configs/testrunner/gunicorn.conf.py
@@ -0,0 +1,12 @@
+import os
+config_dir = os.path.dirname(__file__)
+log_dir = os.path.join(config_dir, '..', '..', 'logs')
+
+bind = [":8085", ":8087"]
+workers = 1
+threads = 10
+log_level = "DEBUG"
+
+log_file = os.path.join(log_dir, 'gunicorn.log')
+error_logfile = os.path.join(log_dir, 'gunicorn-error.log')
+access_logfile = os.path.join(log_dir, 'gunicorn-access.log')
diff --git a/lib/lp/scripts/runlaunchpad.py b/lib/lp/scripts/runlaunchpad.py
index 0557ee8..79ba3d1 100644
--- a/lib/lp/scripts/runlaunchpad.py
+++ b/lib/lp/scripts/runlaunchpad.py
@@ -6,11 +6,13 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = ['start_launchpad']
+import tempfile
try:
from contextlib import ExitStack
except ImportError:
from contextlib2 import ExitStack
import os
+import re
import signal
import subprocess
import sys
@@ -332,7 +334,31 @@ def start_testapp(argv=list(sys.argv)):
pass
+def gunicornify_zope_config_file():
+ """Creates a new launchpad.config file removing directives related to
+ Zope Server that shouldn't be used when running on gunicorn.
+ """
+ original_filename = config.zope_config_file
+ with open(original_filename) as fd:
+ content = fd.read()
+
+ # Remove unwanted tags.
+ for tag in ['server', 'accesslog']:
+ content = re.sub(
+ r"<%s>(.*)</%s>" % (tag, tag), "", content, flags=re.S)
+
+ # Remove unwanted single-line directives.
+ for directive in ['interrupt-check-interval']:
+ content = re.sub(r"%s .*" % directive, "", content)
+
+ new_filename = tempfile.mktemp(prefix='launchpad.conf-gunicorn-')
+ with open(new_filename, 'w') as fd:
+ fd.write(content)
+ config.zope_config_file = new_filename
+
+
def gunicorn_main():
+ gunicornify_zope_config_file()
orig_argv = sys.argv
try:
sys.argv = [
diff --git a/lib/lp/scripts/tests/test_runlaunchpad.py b/lib/lp/scripts/tests/test_runlaunchpad.py
index 4a7222c..9587583 100644
--- a/lib/lp/scripts/tests/test_runlaunchpad.py
+++ b/lib/lp/scripts/tests/test_runlaunchpad.py
@@ -15,11 +15,13 @@ __all__ = [
import os
import shutil
import tempfile
+from textwrap import dedent
import testtools
from lp.scripts.runlaunchpad import (
get_services_to_run,
+ gunicornify_zope_config_file,
process_config_arguments,
SERVICES,
split_out_runlaunchpad_arguments,
@@ -191,3 +193,56 @@ class TestAppServerStart(lp.testing.TestCase):
start_launchpad([])
self.assertEqual(0, gmain.call_count)
self.assertEqual(1, zmain.call_count)
+
+ def test_gunicornify_config(self):
+ content = dedent("""
+ site-definition zcml/webapp.zcml
+ # With some comment
+ devmode off
+ interrupt-check-interval 200
+ <server>
+ type HTTP
+ address 8085
+ </server>
+ <server>
+ type XXX
+ address 123
+ </server>
+
+ <zodb>
+ <mappingstorage/>
+ </zodb>
+
+ <accesslog>
+ <logfile>
+ path logs/test-appserver-layer.log
+ </logfile>
+ </accesslog>
+ """)
+ config_filename = tempfile.mktemp()
+ with open(config_filename, "w") as fd:
+ fd.write(content)
+
+ patched_cfg = mock.patch(
+ 'lp.services.config.LaunchpadConfig.zope_config_file',
+ new_callable=mock.PropertyMock)
+ with patched_cfg as mock_zope_config_file:
+ mock_zope_config_file.return_value = config_filename
+
+ gunicornify_zope_config_file()
+ self.assertEqual(2, mock_zope_config_file.call_count)
+ new_file = mock_zope_config_file.call_args[0][0]
+ with open(new_file) as fd:
+ self.assertEqual(dedent("""
+ site-definition zcml/webapp.zcml
+ # With some comment
+ devmode off
+
+
+
+ <zodb>
+ <mappingstorage/>
+ </zodb>
+
+
+ """), fd.read())
diff --git a/lib/lp/services/config/__init__.py b/lib/lp/services/config/__init__.py
index d4bd705..c2ed613 100644
--- a/lib/lp/services/config/__init__.py
+++ b/lib/lp/services/config/__init__.py
@@ -125,6 +125,12 @@ class LaunchpadConfig:
self._process_name = process_name
self._instance_name = instance_name
self.root = TREE_ROOT
+ # Allow overriding Zope's config file.
+ # XXX pappacena 2021-01-21: We allow overriding this at runtime so
+ # we can do a painless transition to gunicorn. Once we are fully
+ # running gunicorn instead of zope server, this shouldn't be
+ # necessary anymore.
+ self._zope_config_file = None
def _make_process_name(self):
if getattr(sys, 'argv', None) is None:
@@ -152,7 +158,7 @@ class LaunchpadConfig:
"""When running launchpad server, shall we use gunicorn?"""
# XXX pappacena: 2021-01-20: Forced False until we have everything
# in place.
- return False
+ return True
def setInstance(self, instance_name):
"""Set the instance name where the conf files are stored.
@@ -251,10 +257,22 @@ class LaunchpadConfig:
@property
def zope_config_file(self):
"""Return the path to the ZConfig file for this instance."""
+ if self._zope_config_file is not None:
+ return self._zope_config_file
return os.path.join(self.config_dir, 'launchpad.conf')
+ @zope_config_file.setter
+ def zope_config_file(self, value):
+ self._zope_config_file = value
+
def _getZConfig(self):
- """Modify the config, adding automatically generated settings"""
+ """Modify the config, adding automatically generated settings
+
+ XXX pappacena 2021-01-21 This method should probably be deprecated
+ once we move to gunicorn since we read zope's configs here mostly to
+ get "devmode" directive, which is moved to launchpad-lazr.conf when
+ running launchpad server on gunicorn.
+ """
with resources.path('zope.app.server', 'schema.xml') as schemafile:
schema = ZConfig.loadSchema(str(schemafile))
root_options, handlers = ZConfig.loadConfig(
@@ -268,6 +286,10 @@ class LaunchpadConfig:
Copied here for ease of access.
"""
+ if self.use_gunicorn:
+ return self.launchpad.devmode
+ # XXX pappacena 2021-01-21: Read only from launchpad-lazr.conf file
+ # once we will be running on gunicorn.
if self._devmode is None:
self._getZConfig()
return self._devmode
@@ -279,6 +301,8 @@ class LaunchpadConfig:
@property
def servers(self):
"""The defined servers."""
+ if self.use_gunicorn:
+ return []
if self._servers is None:
self._getZConfig()
return self._servers
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index aaa9d30..6b54929 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -843,6 +843,9 @@ max_scaling: 500
[launchpad]
+# Is this server running in dev mode?
+devmode: false
+
# A directory of files from which *-lazr.conf will be loaded and
# overlaid in order on top of the main config.
# Note that this is relative to the top-level config file, not
@@ -1640,6 +1643,10 @@ althostnames: none
# datatype: string
rooturl: none
+# Is this vhost enabled?
+# Used to create the correct publishers
+private_port: None
+
[vhost.mainsite]
# Can the profile page act as a OpenID delegated identity?
# data-type: boolean
diff --git a/lib/lp/services/webapp/publication.py b/lib/lp/services/webapp/publication.py
index 7325d9f..25bc2de 100644
--- a/lib/lp/services/webapp/publication.py
+++ b/lib/lp/services/webapp/publication.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -7,6 +7,7 @@ __all__ = [
'LaunchpadBrowserPublication',
]
+import logging
import re
import sys
import threading
@@ -871,6 +872,11 @@ def tracelog(request, prefix, msg):
easier. ``prefix`` should be unique and contain no spaces, and
preferably a single character to save space.
"""
- tracelog = ITraceLog(request, None)
- if tracelog is not None:
- tracelog.log('%s %s' % (prefix, six.ensure_str(msg, 'US-ASCII')))
+ msg = '%s %s' % (prefix, six.ensure_str(msg, 'US-ASCII'))
+ if not config.use_gunicorn:
+ tracelog = ITraceLog(request, None)
+ if tracelog is not None:
+ tracelog.log(msg)
+ else:
+ logger = logging.getLogger()
+ logger.info(msg)
diff --git a/lib/lp/services/webapp/servers.py b/lib/lp/services/webapp/servers.py
index d7bcad2..0e2bf17 100644
--- a/lib/lp/services/webapp/servers.py
+++ b/lib/lp/services/webapp/servers.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Definition of the internet servers that Launchpad uses."""
@@ -1110,7 +1110,8 @@ class LaunchpadAccessLogger(CommonAccessLogger):
)
)
-
+# XXX pappacena 2021-01-21: These 4 server definitions can be removed once
+# we are using only gunicorn (and not Zope Server).
http = wsgi.ServerType(
ZServerTracelogServer, # subclass of WSGIHTTPServer
WSGIPublisherApplication,
@@ -1577,10 +1578,13 @@ def register_launchpad_request_publication_factories():
# We may also have a private XML-RPC server.
private_port = None
- for server in config.servers:
- if server.type == 'PrivateXMLRPC':
- ip, private_port = server.address
- break
+ if config.use_gunicorn:
+ private_port = config.vhost.xmlrpc_private.private_port
+ else:
+ for server in config.servers:
+ if server.type == 'PrivateXMLRPC':
+ ip, private_port = server.address
+ break
if private_port is not None:
factories.append(XMLRPCRequestPublicationFactory(
diff --git a/lib/lp/testing/layers.py b/lib/lp/testing/layers.py
index 9089c60..98c8149 100644
--- a/lib/lp/testing/layers.py
+++ b/lib/lp/testing/layers.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Layers used by Launchpad tests.
@@ -55,6 +55,7 @@ from functools import partial
import gc
import logging
import os
+import select
import signal
import socket
import subprocess
@@ -74,7 +75,10 @@ from fixtures import (
MonkeyPatch,
)
import psycopg2
-from six.moves.urllib.error import URLError
+from six.moves.urllib.error import (
+ HTTPError,
+ URLError,
+ )
from six.moves.urllib.parse import (
quote,
urlparse,
@@ -84,6 +88,7 @@ import transaction
from webob.request import environ_from_url as orig_environ_from_url
import wsgi_intercept
from wsgi_intercept import httplib2_intercept
+import zcml
from zope.app.publication.httpfactory import HTTPPublicationRequestFactory
from zope.app.wsgi import WSGIPublisherApplication
from zope.component import (
@@ -155,7 +160,6 @@ from lp.testing import (
reset_logging,
)
from lp.testing.pgsql import PgTestSetup
-import zcml
COMMA = ','
@@ -1794,6 +1798,14 @@ class LayerProcessController:
raise LayerIsolationError(
"App server died in this test (status=%s):\n%s" % (
cls.appserver.returncode, cls.appserver.stdout.read()))
+ # Cleanup the app server's output buffer between tests.
+ while True:
+ # Read while we have something available at the stdout.
+ r, w, e = select.select([cls.appserver.stdout], [], [], 0)
+ if cls.appserver.stdout in r:
+ cls.appserver.stdout.readline()
+ else:
+ break
DatabaseLayer.force_dirty_database()
@classmethod
@@ -1836,6 +1848,12 @@ class LayerProcessController:
try:
connection = urlopen(root_url)
connection.read()
+ except HTTPError as error:
+ if error.code == 503:
+ raise RuntimeError(
+ "App server is returning unknown error code %s. Is "
+ "there another instance running in the same port?" %
+ error.code)
except URLError as error:
# We are interested in a wrapped socket.error.
if not isinstance(error.reason, socket.error):
Follow ups
