launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27509
[Merge] ~cjwatson/launchpad:faster-librarian-startup into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:faster-librarian-startup into launchpad:master.
Commit message:
Speed up librarian startup
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/408492
The librarian doesn't need to load most of Launchpad, nor to set up Zope security, and with a few adjustments it's relatively easy to avoid it doing so. This takes librarian startup time from 9.2s to 2.7s on my system, with a corresponding decrease in `LibrarianLayer` setup time.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:faster-librarian-startup into launchpad:master.
diff --git a/daemons/librarian.tac b/daemons/librarian.tac
index a09f873..1a7bd5a 100644
--- a/daemons/librarian.tac
+++ b/daemons/librarian.tac
@@ -1,4 +1,4 @@
-# Copyright 2009-2014 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
# Twisted Application Configuration file.
@@ -51,7 +51,8 @@ from lp.services.twistedsupport.loggingsupport import set_up_oops_reporting
dbconfig.override(
dbuser=config.librarian.dbuser,
isolation_level=config.librarian.isolation_level)
-execute_zcml_for_scripts()
+execute_zcml_for_scripts(
+ scriptzcmlfilename='librarian.zcml', setup_interaction=False)
if os.environ.get('LP_TEST_INSTANCE'):
# Running in ephemeral mode: get the root dir from the environment and
diff --git a/lib/lp/services/librarianserver/storage.py b/lib/lp/services/librarianserver/storage.py
index cebcd59..aec7ff1 100644
--- a/lib/lp/services/librarianserver/storage.py
+++ b/lib/lp/services/librarianserver/storage.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -14,11 +14,15 @@ from twisted.internet import defer
from twisted.internet.threads import deferToThread
from twisted.python import log
from twisted.web.static import StaticProducer
+from zope.component import getUtility
-from lp.registry.model.product import Product
from lp.services.config import dbconfig
from lp.services.database import write_transaction
-from lp.services.database.interfaces import IStore
+from lp.services.database.interfaces import (
+ DEFAULT_FLAVOR,
+ IStoreSelector,
+ MAIN_STORE,
+ )
from lp.services.database.postgresql import ConnectionString
from lp.services.features import getFeatureFlag
from lp.services.librarianserver import swift
@@ -248,7 +252,9 @@ class LibraryFileUpload(object):
config_dbname = ConnectionString(
dbconfig.rw_main_master).dbname
- result = IStore(Product).execute("SELECT current_database()")
+ store = getUtility(IStoreSelector).get(
+ MAIN_STORE, DEFAULT_FLAVOR)
+ result = store.execute("SELECT current_database()")
real_dbname = result.get_one()[0]
if self.databaseName not in (config_dbname, real_dbname):
raise WrongDatabaseError(
diff --git a/lib/lp/services/scripts/__init__.py b/lib/lp/services/scripts/__init__.py
index be7059e..56557eb 100644
--- a/lib/lp/services/scripts/__init__.py
+++ b/lib/lp/services/scripts/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Library functions for use in all scripts.
@@ -44,7 +44,8 @@ from lp.services.webapp.interaction import (
)
-def execute_zcml_for_scripts(use_web_security=False):
+def execute_zcml_for_scripts(use_web_security=False, scriptzcmlfilename=None,
+ setup_interaction=True):
"""Execute the zcml rooted at launchpad/script.zcml
If use_web_security is True, the same security policy as the web
@@ -68,10 +69,11 @@ def execute_zcml_for_scripts(use_web_security=False):
Instead, your test should use the Zopeless layer.
"""
- if config.isTestRunner():
- scriptzcmlfilename = 'script-testing.zcml'
- else:
- scriptzcmlfilename = 'script.zcml'
+ if scriptzcmlfilename is None:
+ if config.isTestRunner():
+ scriptzcmlfilename = 'script-testing.zcml'
+ else:
+ scriptzcmlfilename = 'script.zcml'
scriptzcmlfilename = os.path.abspath(
os.path.join(config.root, 'zcml', scriptzcmlfilename))
@@ -109,9 +111,11 @@ def execute_zcml_for_scripts(use_web_security=False):
"QueueProcessorThread did not shut down")
atexit.register(kill_queue_processor_threads)
- # This is a convenient hack to set up a zope interaction, before we get
- # the proper API for having a principal / user running in scripts.
- setupInteractionByEmail(ANONYMOUS)
+ if setup_interaction:
+ # This is a convenient hack to set up a zope interaction, before we
+ # get the proper API for having a principal / user running in
+ # scripts.
+ setupInteractionByEmail(ANONYMOUS)
def db_options(parser):
diff --git a/lib/lp/services/webapp/configure.zcml b/lib/lp/services/webapp/configure.zcml
index b349be9..c894270 100644
--- a/lib/lp/services/webapp/configure.zcml
+++ b/lib/lp/services/webapp/configure.zcml
@@ -1,4 +1,4 @@
-<!-- Copyright 2009-2020 Canonical Ltd. This software is licensed under the
+<!-- Copyright 2009-2021 Canonical Ltd. This software is licensed under the
GNU Affero General Public License version 3 (see the file LICENSE).
-->
@@ -9,6 +9,7 @@
i18n_domain="launchpad">
<include file="errorlog.zcml" />
+ <include file="database.zcml" />
<browser:defaultView name="index.html" />
@@ -32,26 +33,6 @@
/>
<adapter
- for="zope.publisher.interfaces.http.IHTTPRequest"
- provides="lp.services.database.interfaces.IDatabasePolicy"
- factory="lp.services.database.policy.LaunchpadDatabasePolicyFactory"
- />
- <adapter
- for="zope.publisher.interfaces.xmlrpc.IXMLRPCRequest"
- provides="lp.services.database.interfaces.IDatabasePolicy"
- factory="lp.services.database.policy.MasterDatabasePolicy"
- />
- <adapter
- for="lp.layers.WebServiceLayer"
- provides="lp.services.database.interfaces.IDatabasePolicy"
- factory="lp.services.database.policy.WebServiceDatabasePolicyFactory"
- />
- <adapter
- for="lp.layers.FeedsLayer"
- provides="lp.services.database.interfaces.IDatabasePolicy"
- factory="lp.services.database.policy.SlaveOnlyDatabasePolicy"
- />
- <adapter
for="lp.layers.WebServiceLayer"
provides="lazr.restful.interfaces.IWebBrowserOriginatingRequest"
factory="lp.services.webapp.servers.web_service_request_to_browser_request"
@@ -175,47 +156,6 @@
provides="zope.session.interfaces.ISessionDataContainer"
/>
- <!-- Storm Store selector. -->
- <utility
- component="lp.services.webapp.adapter.StoreSelector"
- provides="lp.services.database.interfaces.IStoreSelector">
- </utility>
-
- <!-- Storm Store adapters, adapting a Storm subclass or instance
- to the correct Store for that tables replication set.
- -->
- <adapter
- provides="lp.services.database.interfaces.IStore"
- for="zope.interface.Interface"
- factory="lp.services.webapp.adapter.get_store"
- />
- <adapter
- provides="lp.services.database.interfaces.IMasterStore"
- for="zope.interface.Interface"
- factory="lp.services.webapp.adapter.get_master_store"
- />
- <adapter
- provides="lp.services.database.interfaces.ISlaveStore"
- for="zope.interface.Interface"
- factory="lp.services.webapp.adapter.get_slave_store"
- />
- <!-- Universal adapter needed here per Bug #591841.
- We have no way of specifying that all subclasses of
- storm.locals.Storm implement an Interface. -->
- <adapter
- provides="lp.services.database.interfaces.IMasterObject"
- for="zope.interface.Interface"
- trusted="yes"
- factory="lp.services.webapp.adapter.get_object_from_master_store"
- />
- <class class="storm.store.Store">
- <implements interface="lp.services.database.interfaces.IStore" />
- <allow attributes="get" />
- </class>
- <class class="lp.services.database.sqlbase.SQLBase">
- <implements interface="lp.services.database.interfaces.IDBObject" />
- </class>
-
<!-- Default favicon -->
<browser:favicon for="*" file="../../../canonical/launchpad/images/launchpad.png" />
diff --git a/lib/lp/services/webapp/database.zcml b/lib/lp/services/webapp/database.zcml
new file mode 100644
index 0000000..0655a5f
--- /dev/null
+++ b/lib/lp/services/webapp/database.zcml
@@ -0,0 +1,70 @@
+<!-- Copyright 2009-2021 Canonical Ltd. This software is licensed under the
+ GNU Affero General Public License version 3 (see the file LICENSE).
+-->
+
+<configure
+ xmlns="http://namespaces.zope.org/zope";>
+
+ <adapter
+ for="zope.publisher.interfaces.http.IHTTPRequest"
+ provides="lp.services.database.interfaces.IDatabasePolicy"
+ factory="lp.services.database.policy.LaunchpadDatabasePolicyFactory"
+ />
+ <adapter
+ for="zope.publisher.interfaces.xmlrpc.IXMLRPCRequest"
+ provides="lp.services.database.interfaces.IDatabasePolicy"
+ factory="lp.services.database.policy.MasterDatabasePolicy"
+ />
+ <adapter
+ for="lp.layers.WebServiceLayer"
+ provides="lp.services.database.interfaces.IDatabasePolicy"
+ factory="lp.services.database.policy.WebServiceDatabasePolicyFactory"
+ />
+ <adapter
+ for="lp.layers.FeedsLayer"
+ provides="lp.services.database.interfaces.IDatabasePolicy"
+ factory="lp.services.database.policy.SlaveOnlyDatabasePolicy"
+ />
+
+ <!-- Storm Store selector. -->
+ <utility
+ component="lp.services.webapp.adapter.StoreSelector"
+ provides="lp.services.database.interfaces.IStoreSelector">
+ </utility>
+
+ <!-- Storm Store adapters, adapting a Storm subclass or instance
+ to the correct Store for that tables replication set.
+ -->
+ <adapter
+ provides="lp.services.database.interfaces.IStore"
+ for="zope.interface.Interface"
+ factory="lp.services.webapp.adapter.get_store"
+ />
+ <adapter
+ provides="lp.services.database.interfaces.IMasterStore"
+ for="zope.interface.Interface"
+ factory="lp.services.webapp.adapter.get_master_store"
+ />
+ <adapter
+ provides="lp.services.database.interfaces.ISlaveStore"
+ for="zope.interface.Interface"
+ factory="lp.services.webapp.adapter.get_slave_store"
+ />
+ <!-- Universal adapter needed here per Bug #591841.
+ We have no way of specifying that all subclasses of
+ storm.locals.Storm implement an Interface. -->
+ <adapter
+ provides="lp.services.database.interfaces.IMasterObject"
+ for="zope.interface.Interface"
+ trusted="yes"
+ factory="lp.services.webapp.adapter.get_object_from_master_store"
+ />
+ <class class="storm.store.Store">
+ <implements interface="lp.services.database.interfaces.IStore" />
+ <allow attributes="get" />
+ </class>
+ <class class="lp.services.database.sqlbase.SQLBase">
+ <implements interface="lp.services.database.interfaces.IDBObject" />
+ </class>
+
+</configure>
diff --git a/lib/lp/xmlrpc/faults.py b/lib/lp/xmlrpc/faults.py
index 5b7d95b..4427b0c 100644
--- a/lib/lp/xmlrpc/faults.py
+++ b/lib/lp/xmlrpc/faults.py
@@ -52,7 +52,6 @@ __all__ = [
]
-from lp.registry.interfaces.projectgroup import IProjectGroup
from lp.services.xmlrpc import LaunchpadFault
@@ -293,10 +292,9 @@ class CannotHaveLinkedBranch(LaunchpadFault):
"%(component_type)s cannot have a default branch.")
def __init__(self, component):
- if IProjectGroup.providedBy(component):
+ component_type = component.__class__.__name__.lower()
+ if component_type == 'projectgroup':
component_type = 'project group'
- else:
- component_type = component.__class__.__name__.lower()
LaunchpadFault.__init__(
self, component_name=component.displayname,
component_type=component_type)
diff --git a/zcml/librarian.zcml b/zcml/librarian.zcml
new file mode 100644
index 0000000..6dc5336
--- /dev/null
+++ b/zcml/librarian.zcml
@@ -0,0 +1,26 @@
+<!-- Copyright 2021 Canonical Ltd. This software is licensed under the
+ GNU Affero General Public License version 3 (see the file LICENSE).
+-->
+
+<configure
+ xmlns="http://namespaces.zope.org/zope";
+ xmlns:i18n="http://namespaces.zope.org/i18n";
+ i18n_domain="launchpad">
+
+ <include file="zopeapp.zcml" />
+
+ <include package="storm.zope" file="meta.zcml"/>
+ <include package="storm.zope" />
+
+ <includeOverrides
+ package="lp.services.webapp"
+ file="meta-overrides.zcml" />
+ <include package="lp.services.webapp" file="meta.zcml" />
+
+ <include package="lp.services.webapp" file="errorlog.zcml" />
+ <include package="lp.services.webapp" file="database.zcml" />
+ <include package="lp.services.librarian" />
+
+ <include package="lp" file="permissions.zcml" />
+
+</configure>
