launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24120
[Merge] ~cjwatson/launchpad:decouple-mailman into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:decouple-mailman into launchpad:master.
Commit message:
Decouple Mailman from the database
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/375623
The only part of Launchpad's Mailman integration that requires the same deployment to have access to both Mailman and the Launchpad database was the mlist-sync script, which runs on staging and needs to update the team email addresses corresponding to each mailing list to refer to the staging hostname rather than the production hostname.
Like the rest of Mailman, mlist-sync now talks to the database via the XML-RPC API exposed by Launchpad. This will allow extracting Launchpad's customised version of Mailman from the Launchpad tree and running it separately. I'd like to do this for two reasons: firstly, because Mailman 2 runs exclusively on Python 2, so extracting it will simplify porting Launchpad to Python 3; and secondly, to avoid having to build Mailman as part of every Launchpad tree.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:decouple-mailman into launchpad:master.
diff --git a/lib/lp/registry/interfaces/mailinglist.py b/lib/lp/registry/interfaces/mailinglist.py
index e9d1045..ac660cd 100644
--- a/lib/lp/registry/interfaces/mailinglist.py
+++ b/lib/lp/registry/interfaces/mailinglist.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Mailing list interfaces."""
@@ -535,6 +535,15 @@ class IMailingListSet(Interface):
value_type=Object(schema=IMailingList),
readonly=True)
+ def updateTeamAddresses(old_hostname):
+ """Update team addresses to refer to a different Launchpad instance.
+
+ :param old_hostname: The mailing list hostname of the Launchpad
+ instance from which this instance syncs mailing list data. Any
+ teams with this address will be updated to refer to the current
+ mailing list hostname.
+ """
+
class IMailingListAPIView(Interface):
"""XMLRPC API that Mailman polls for mailing list actions."""
@@ -645,6 +654,18 @@ class IMailingListAPIView(Interface):
either the string 'accept' or 'decline'.
"""
+ def updateTeamAddresses(old_hostname):
+ """Update team addresses to refer to a different Launchpad instance.
+
+ The mlist-sync script syncs Mailman data between different instances
+ of Launchpad, which requires fixing up team mailing list addresses
+ to refer to the new hostname. This does so in bulk.
+
+ This endpoint only works on non-production instances.
+
+ :return: True
+ """
+
class IMailingListSubscription(Interface):
"""A mailing list subscription."""
diff --git a/lib/lp/registry/model/mailinglist.py b/lib/lp/registry/model/mailinglist.py
index f15039a..a4c4aa7 100644
--- a/lib/lp/registry/model/mailinglist.py
+++ b/lib/lp/registry/model/mailinglist.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -25,8 +25,10 @@ from sqlobject import (
)
from storm.expr import (
And,
+ Func,
Join,
Or,
+ Select,
)
from storm.info import ClassAlias
from storm.store import Store
@@ -73,6 +75,7 @@ from lp.services.database.sqlbase import (
SQLBase,
sqlvalues,
)
+from lp.services.database.stormexpr import Concatenate
from lp.services.identity.interfaces.account import AccountStatus
from lp.services.identity.interfaces.emailaddress import (
EmailAddressStatus,
@@ -701,6 +704,31 @@ class MailingListSet:
return MailingList.select('status IN %s' % sqlvalues(
(MailingListStatus.CONSTRUCTING, MailingListStatus.UPDATING)))
+ def updateTeamAddresses(self, old_hostname):
+ """See `IMailingListSet`."""
+ # Circular imports.
+ from lp.registry.model.mailinglist import MailingList
+ from lp.registry.model.person import Person
+ # This is really an operation on EmailAddress rows, but it's so
+ # specific to mailing lists that it seems better to keep it here.
+ old_suffix = u"@" + old_hostname
+ if config.mailman.build_host_name:
+ new_suffix = u"@" + config.mailman.build_host_name
+ else:
+ new_suffix = u"@" + getfqdn()
+ clauses = [
+ EmailAddress.person == Person.id,
+ Person.teamowner != None,
+ Person.id == MailingList.teamID,
+ EmailAddress.email.endswith(old_suffix),
+ ]
+ addresses = IMasterStore(EmailAddress).find(
+ EmailAddress,
+ EmailAddress.id.is_in(Select(EmailAddress.id, And(*clauses))))
+ addresses.set(email=Concatenate(
+ Func("left", EmailAddress.email, -len(old_suffix)),
+ new_suffix))
+
@implementer(IMailingListSubscription)
class MailingListSubscription(SQLBase):
diff --git a/lib/lp/registry/tests/mailinglists_helper.py b/lib/lp/registry/tests/mailinglists_helper.py
index 9c7e56f..4a51736 100644
--- a/lib/lp/registry/tests/mailinglists_helper.py
+++ b/lib/lp/registry/tests/mailinglists_helper.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Helper functions for testing XML-RPC services."""
@@ -31,24 +31,16 @@ from lp.services.database.sqlbase import flush_database_updates
def fault_catcher(func):
"""Decorator for displaying Faults in a cross-compatible way.
- When running the same doctest with the ServerProxy, faults are turned into
+ When running tests with the ServerProxy, faults are turned into
exceptions by the XMLRPC machinery, but with the direct view the faults
- are just returned. This causes an impedance mismatch with exception
- display in the doctest that cannot be papered over by using ellipses. So
- to make this work in a consistent way, a subclass of the view class is
- used which prints faults to match the output of ServerProxy (proper
- exceptions aren't really necessary).
+ are just returned. To paper over the resulting impedance mismatch,
+ check whether the result is a fault and if so raise it.
"""
def caller(self, *args, **kws):
result = func(self, *args, **kws)
if isinstance(result, xmlrpclib.Fault):
- # Fake this to look like exception output. The second line is
- # necessary to match ellipses in the doctest, but its contents are
- # completely ignored; /something/ just has to be there.
- print 'Traceback (most recent call last):'
- print 'ignore'
- print 'Fault:', result
+ raise result
else:
return result
return caller
@@ -141,3 +133,8 @@ class MailingListXMLRPCTestProxy(MailingListAPIView):
@fault_catcher
def isTeamPublic(self, team_name):
return super(MailingListXMLRPCTestProxy, self).isTeamPublic(team_name)
+
+ @fault_catcher
+ def updateTeamAddresses(self, old_hostname):
+ return super(MailingListXMLRPCTestProxy, self).updateTeamAddresses(
+ old_hostname)
diff --git a/lib/lp/registry/tests/test_mailinglistapi.py b/lib/lp/registry/tests/test_mailinglistapi.py
index bb4c204..8b90ce2 100644
--- a/lib/lp/registry/tests/test_mailinglistapi.py
+++ b/lib/lp/registry/tests/test_mailinglistapi.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Unit tests for the private MailingList API."""
@@ -10,8 +10,14 @@ from email import message_from_string
from textwrap import dedent
import xmlrpclib
+from testtools.matchers import (
+ Equals,
+ MatchesDict,
+ MatchesSetwise,
+ )
import transaction
from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
from lp.registry.enums import TeamMembershipPolicy
from lp.registry.interfaces.mailinglist import (
@@ -32,7 +38,10 @@ from lp.registry.xmlrpc.mailinglist import (
)
from lp.services.config import config
from lp.services.identity.interfaces.account import AccountStatus
-from lp.services.identity.interfaces.emailaddress import EmailAddressStatus
+from lp.services.identity.interfaces.emailaddress import (
+ EmailAddressStatus,
+ IEmailAddressSet,
+ )
from lp.services.messages.interfaces.message import IMessageSet
from lp.testing import (
admin_logged_in,
@@ -158,6 +167,85 @@ class MailingListAPITestCase(TestCaseWithFactory):
self.assertIs(True, self.api.inGoodStanding('yes@xxxxxx'))
self.assertIs(False, self.api.inGoodStanding('no@xxxxxx'))
+ def test_updateTeamAddresses(self):
+ production_config_name = self.factory.getUniqueString()
+ config.push(
+ production_config_name,
+ '\n[mailman]\nbuild_host_name: lists.launchpad.net\n')
+ try:
+ teams = [
+ self.factory.makeTeam(name='team-%d' % i) for i in range(3)]
+ # A person with an ordinary email address.
+ example_person = self.factory.makePerson(
+ email='person@xxxxxxxxxxx')
+ # A person who has weirdly ended up with a lists email address.
+ lists_person = self.factory.makePerson(
+ email='person@xxxxxxxxxxxxxxxxxxx')
+ for i in (0, 1):
+ self.factory.makeMailingList(teams[i], teams[i].teamowner)
+ for i in (1, 2):
+ # Give some teams another email address; updateTeamAddresses
+ # leaves these alone.
+ self.factory.makeEmail('team-%d@xxxxxxxxxxx' % i, teams[i])
+ finally:
+ config.pop(production_config_name)
+
+ def getEmails(people):
+ email_address_set = getUtility(IEmailAddressSet)
+ return {
+ person.name: set(
+ removeSecurityProxy(email_address).email
+ for email_address in email_address_set.getByPerson(person))
+ for person in people}
+
+ self.assertThat(
+ getEmails(teams + [example_person, lists_person]),
+ MatchesDict({
+ teams[0].name: MatchesSetwise(
+ Equals('team-0@xxxxxxxxxxxxxxxxxxx')),
+ teams[1].name: MatchesSetwise(
+ Equals('team-1@xxxxxxxxxxxxxxxxxxx'),
+ Equals('team-1@xxxxxxxxxxx')),
+ teams[2].name: MatchesSetwise(
+ Equals('team-2@xxxxxxxxxxx')),
+ example_person.name: MatchesSetwise(
+ Equals('person@xxxxxxxxxxx')),
+ lists_person.name: MatchesSetwise(
+ Equals('person@xxxxxxxxxxxxxxxxxxx')),
+ }))
+
+ staging_config_name = self.factory.getUniqueString()
+ config.push(
+ staging_config_name,
+ '\n[launchpad]\nis_demo: True\n'
+ '\n[mailman]\nbuild_host_name: lists.launchpad.test\n')
+ try:
+ self.api.updateTeamAddresses('lists.launchpad.net')
+ finally:
+ config.pop(staging_config_name)
+
+ self.assertThat(
+ getEmails(teams + [example_person, lists_person]),
+ MatchesDict({
+ teams[0].name: MatchesSetwise(
+ Equals('team-0@xxxxxxxxxxxxxxxxxxxx')),
+ teams[1].name: MatchesSetwise(
+ Equals('team-1@xxxxxxxxxxxxxxxxxxxx'),
+ Equals('team-1@xxxxxxxxxxx')),
+ teams[2].name: MatchesSetwise(
+ Equals('team-2@xxxxxxxxxxx')),
+ example_person.name: MatchesSetwise(
+ Equals('person@xxxxxxxxxxx')),
+ lists_person.name: MatchesSetwise(
+ Equals('person@xxxxxxxxxxxxxxxxxxx')),
+ }))
+
+ def test_updateTeamAddresses_refuses_on_production(self):
+ self.pushConfig('launchpad', is_demo=False)
+ self.assertIsInstance(
+ self.api.updateTeamAddresses('lists.launchpad.net'),
+ faults.PermissionDenied)
+
class MailingListAPIWorkflowTestCase(TestCaseWithFactory):
"""Tests for MailingListAPIView workflows.
diff --git a/lib/lp/registry/xmlrpc/mailinglist.py b/lib/lp/registry/xmlrpc/mailinglist.py
index f8064f9..7277200 100644
--- a/lib/lp/registry/xmlrpc/mailinglist.py
+++ b/lib/lp/registry/xmlrpc/mailinglist.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""XMLRPC APIs for mailing lists."""
@@ -29,9 +29,7 @@ from lp.registry.interfaces.person import (
)
from lp.services.config import config
from lp.services.encoding import escape_nonascii_uniquely
-from lp.services.identity.interfaces.account import (
- INACTIVE_ACCOUNT_STATUSES,
- )
+from lp.services.identity.interfaces.account import INACTIVE_ACCOUNT_STATUSES
from lp.services.identity.interfaces.emailaddress import (
EmailAddressStatus,
IEmailAddressSet,
@@ -285,3 +283,11 @@ class MailingListAPIView(LaunchpadXMLRPCView):
response[message_id] = (team_name, disposition)
message_set.acknowledgeMessagesWithStatus(status)
return response
+
+ def updateTeamAddresses(self, old_hostname):
+ """See `IMailingListAPIView`."""
+ # For safety, we only permit this on non-production sites.
+ if not config.launchpad.is_demo:
+ return faults.PermissionDenied()
+ getUtility(IMailingListSet).updateTeamAddresses(old_hostname)
+ return True
diff --git a/lib/lp/services/mailman/scripts/__init__.py b/lib/lp/services/mailman/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/lp/services/mailman/scripts/__init__.py
diff --git a/lib/lp/services/mailman/scripts/mlist_sync.py b/lib/lp/services/mailman/scripts/mlist_sync.py
new file mode 100644
index 0000000..2aa245e
--- /dev/null
+++ b/lib/lp/services/mailman/scripts/mlist_sync.py
@@ -0,0 +1,197 @@
+#!/usr/bin/python -S
+
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Sync Mailman data from one Launchpad to another."""
+
+# XXX BarryWarsaw 2008-02-12:
+# Things this script does NOT do correctly.
+#
+# - Fix up the deactivated lists. This isn't done because that data lives in
+# the backed up tar file, so handling this would mean untar'ing, tricking
+# Mailman into loading the pickle (or manually loading and patching), then
+# re-tar'ing. I don't think it's worth it because the only thing that will
+# be broken is if a list that's deactivated on production is re-activated on
+# staging.
+#
+# - Backpatch all the message footers and RFC 2369 headers of the messages in
+# the archive. To do this, we'd have to iterate through all messages,
+# tweaking the List-* headers (easy) and ripping apart the footers,
+# recalculating them and reattaching them (difficult). Doing the iteration
+# and update is quite painful in Python 2.4, but would be easier with Python
+# 2.5's new mailbox module. /Then/ we'd have to regenerate the archives.
+# Not doing this means that some of the links in staging's MHonArc archive
+# will point to production archives.
+
+import logging
+import os
+import subprocess
+import sys
+import textwrap
+
+from six.moves.xmlrpc_client import Fault
+
+from lp.services.config import config
+from lp.services.mailman.config import configure_prefix
+from lp.services.scripts.base import LaunchpadScript
+from lp.xmlrpc import faults
+
+
+RSYNC_OPTIONS = ('-avz', '--delete')
+RSYNC_COMMAND = '/usr/bin/rsync'
+RSYNC_SUBDIRECTORIES = ('archives', 'backups', 'lists', 'mhonarc')
+SPACE = ' '
+
+
+class MailingListSyncScript(LaunchpadScript):
+ """
+ %prog [options] source_url
+
+ Sync the Mailman data structures between production and staging. This
+ takes the most efficient route by rsync'ing over the list pickles, raw
+ archive mboxes, and mhonarc files, then it fixes up anything that needs
+ fixing. This does /not/ sync over any qfiles because staging doesn't send
+ emails anyway.
+
+ source_url is required and it is the rsync source url which contains
+ mailman's var directory. The destination is taken from the launchpad.conf
+ file.
+ """
+
+ loglevel = logging.INFO
+ description = 'Sync the Mailman data structures with the database.'
+
+ def __init__(self, *args, **kwargs):
+ self.usage = textwrap.dedent(self.__doc__)
+ super(MailingListSyncScript, self).__init__(*args, **kwargs)
+
+ def add_my_options(self):
+ """See `LaunchpadScript`."""
+ # Add optional override of production mailing list host name. In real
+ # use, it's always going to be lists.launchpad.net so that makes a
+ # reasonable default. Some testing environments may override this.
+ self.parser.add_option('--hostname', default='lists.launchpad.net',
+ help=('The hostname for the production '
+ 'mailing list system. This is used to '
+ 'resolve teams which have multiple '
+ 'email addresses.'))
+
+ def syncMailmanDirectories(self, source_url):
+ """Synchronize the Mailman directories.
+
+ :param source_url: the base url of the source
+ """
+ # This can't be done at module global scope.
+ from Mailman import mm_cfg
+ # Start by rsync'ing over the entire $vardir/lists, $vardir/archives,
+ # $vardir/backups, and $vardir/mhonarc directories. We specifically
+ # do not rsync the data, locks, logs, qfiles, or spam directories.
+ destination_url = config.mailman.build_var_dir
+ # Do one rsync for all subdirectories.
+ rsync_command = [RSYNC_COMMAND]
+ rsync_command.extend(RSYNC_OPTIONS)
+ rsync_command.append('--exclude=%s' % mm_cfg.MAILMAN_SITE_LIST)
+ rsync_command.append('--exclude=%s.mbox' % mm_cfg.MAILMAN_SITE_LIST)
+ rsync_command.extend(os.path.join(source_url, subdirectory)
+ for subdirectory in RSYNC_SUBDIRECTORIES)
+ rsync_command.append(destination_url)
+ self.logger.info('executing: %s', SPACE.join(rsync_command))
+ process = subprocess.Popen(rsync_command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate()
+ if process.returncode == 0:
+ self.logger.debug('%s', stdout)
+ else:
+ self.logger.error('rsync command failed with exit status: %s',
+ process.returncode)
+ self.logger.error('STDOUT:\n%s\nSTDERR:\n%s', stdout, stderr)
+ return process.returncode
+
+ def fixHostnames(self):
+ """Fix up the host names in Mailman and the LP database."""
+ # These can't be done at module global scope.
+ from Mailman import Utils
+ from Mailman import mm_cfg
+ from Mailman.MailList import MailList
+ from Mailman.Queue import XMLRPCRunner
+
+ # Ask Launchpad to update all the team email addresses.
+ proxy = XMLRPCRunner.get_mailing_list_api_proxy()
+ proxy.updateTeamAddresses(self.options.hostname)
+
+ # Clean things up per mailing list.
+ for list_name in Utils.list_names():
+ # Skip the site list.
+ if list_name == mm_cfg.MAILMAN_SITE_LIST:
+ continue
+
+ # The first thing to clean up is the mailing list pickles. There
+ # are things like host names in some attributes that need to be
+ # converted. The following opens a locked list.
+ mailing_list = MailList(list_name)
+ try:
+ mailing_list.host_name = mm_cfg.DEFAULT_EMAIL_HOST
+ mailing_list.web_page_url = (
+ mm_cfg.DEFAULT_URL_PATTERN % mm_cfg.DEFAULT_URL_HOST)
+ mailing_list.Save()
+ finally:
+ mailing_list.Unlock()
+
+ try:
+ proxy.isTeamPublic(list_name)
+ except Fault as fault:
+ if fault.faultCode == faults.NoSuchPersonWithName.error_code:
+ # We found a mailing list in Mailman that does not exist
+ # in the Launchpad database. This can happen if we
+ # rsynced the Mailman directories after the lists were
+ # created, but we copied the LP database /before/ the
+ # lists were created. If we don't delete the Mailman
+ # lists, we won't be able to create the mailing lists on
+ # staging.
+ self.logger.error('No LP mailing list for: %s', list_name)
+ self.deleteMailmanList(list_name)
+ continue
+
+ def deleteMailmanList(self, list_name):
+ """Delete all Mailman data structures for `list_name`."""
+ mailman_bindir = os.path.normpath(os.path.join(
+ configure_prefix(config.mailman.build_prefix), 'bin'))
+ process = subprocess.Popen(('./rmlist', '-a', list_name),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=mailman_bindir)
+ stdout, stderr = process.communicate()
+ if process.returncode == 0:
+ self.logger.info('%s', stdout)
+ else:
+ self.logger.error('rmlist command failed with exit status: %s',
+ process.returncode)
+ self.logger.error('STDOUT:\n%s\nSTDERR:\n%s', stdout, stderr)
+ # Keep going.
+
+ def main(self):
+ """See `LaunchpadScript`."""
+ source_url = None
+ if len(self.args) == 0:
+ self.parser.error('Missing source_url')
+ elif len(self.args) > 1:
+ self.parser.error('Too many arguments')
+ else:
+ source_url = self.args[0]
+
+ # We need to get to the Mailman API. Set up the paths so that Mailman
+ # can be imported. This can't be done at module global scope.
+ mailman_path = configure_prefix(config.mailman.build_prefix)
+ sys.path.append(mailman_path)
+
+ retcode = self.syncMailmanDirectories(source_url)
+ if retcode != 0:
+ return retcode
+
+ self.fixHostnames()
+
+ # All done; commit the database changes.
+ self.txn.commit()
+ return 0
diff --git a/lib/lp/services/mailman/tests/test_mlist_sync.py b/lib/lp/services/mailman/tests/test_mlist_sync.py
index 8d43c4a..adf1123 100644
--- a/lib/lp/services/mailman/tests/test_mlist_sync.py
+++ b/lib/lp/services/mailman/tests/test_mlist_sync.py
@@ -1,4 +1,4 @@
-# Copyright 2011 Canonical Ltd. This software is licensed under the
+# Copyright 2011-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Test the mlist-sync script."""
@@ -8,23 +8,22 @@ __all__ = []
from contextlib import contextmanager
import os
import shutil
-from subprocess import (
- PIPE,
- Popen,
- )
import tempfile
from Mailman import mm_cfg
from Mailman.MailList import MailList
from Mailman.Utils import list_names
-from transaction import commit
+import transaction
from lp.services.config import config
from lp.services.database.interfaces import IStore
from lp.services.identity.model.emailaddress import EmailAddressSet
+from lp.services.log.logger import BufferLogger
+from lp.services.mailman.scripts.mlist_sync import MailingListSyncScript
from lp.services.mailman.tests import MailmanTestCase
from lp.testing import person_logged_in
-from lp.testing.layers import DatabaseFunctionalLayer
+from lp.testing.dbuser import dbuser
+from lp.testing.layers import ZopelessDatabaseLayer
@contextmanager
@@ -46,10 +45,23 @@ def production_config(host_name):
config.pop('production')
+@contextmanager
+def staging_config():
+ """Simulate a staging Launchpad config."""
+ config.push('staging', """\
+ [launchpad]
+ is_demo: True
+ """)
+ try:
+ yield
+ finally:
+ config.pop('staging')
+
+
class TestMListSync(MailmanTestCase):
"""Test mlist-sync script."""
- layer = DatabaseFunctionalLayer
+ layer = ZopelessDatabaseLayer
def setUp(self):
super(TestMListSync, self).setUp()
@@ -79,16 +91,17 @@ class TestMListSync(MailmanTestCase):
"""Run mlist-sync.py."""
store = IStore(self.team)
store.flush()
- commit()
+ transaction.commit()
store.invalidate()
- proc = Popen(
- ('scripts/mlist-sync.py', '--hostname',
- self.host_name, source_dir),
- stdout=PIPE, stderr=PIPE,
- cwd=config.root,
- env=dict(LPCONFIG=DatabaseFunctionalLayer.appserver_config_name))
- stdout, stderr = proc.communicate()
- return proc.returncode, stderr
+ script = MailingListSyncScript(
+ test_args=['--hostname', self.host_name, source_dir],
+ logger=BufferLogger())
+ script.txn = transaction
+ try:
+ with dbuser('mlist-sync'), staging_config():
+ return script.main()
+ finally:
+ self.addDetail('log', script.logger.content)
def getListInfo(self):
"""Return a list of 4-tuples of Mailman mailing list info."""
@@ -110,8 +123,7 @@ class TestMListSync(MailmanTestCase):
def test_staging_sync(self):
# List is synced with updated URLs and email addresses.
source_dir = self.setupProductionFiles()
- returncode, stderr = self.runMListSync(source_dir)
- self.assertEqual(0, returncode, stderr)
+ self.assertEqual(0, self.runMListSync(source_dir))
list_summary = [(
'team-1',
'lists.launchpad.test',
@@ -129,8 +141,7 @@ class TestMListSync(MailmanTestCase):
mm_cfg.VAR_PREFIX, 'mhonarc', 'no-team'))
self.addCleanup(self.cleanMailmanList, None, 'no-team')
source_dir = self.setupProductionFiles()
- returncode, stderr = self.runMListSync(source_dir)
- self.assertEqual(0, returncode, stderr)
+ self.assertEqual(0, self.runMListSync(source_dir))
list_summary = [(
'team-1',
'lists.launchpad.test',
@@ -144,8 +155,7 @@ class TestMListSync(MailmanTestCase):
with production_config(self.host_name):
self.team.setContactAddress(email)
source_dir = self.setupProductionFiles()
- returncode, stderr = self.runMListSync(source_dir)
- self.assertEqual(0, returncode, stderr)
+ self.assertEqual(0, self.runMListSync(source_dir))
list_summary = [(
'team-1',
'lists.launchpad.test',
diff --git a/scripts/mlist-sync.py b/scripts/mlist-sync.py
index 56d8f77..503ba57 100755
--- a/scripts/mlist-sync.py
+++ b/scripts/mlist-sync.py
@@ -1,226 +1,18 @@
#!/usr/bin/python -S
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Sync Mailman data from one Launchpad to another."""
import _pythonpath
-# XXX BarryWarsaw 2008-02-12:
-# Things this script does NOT do correctly.
-#
-# - Fix up the deactivated lists. This isn't done because that data lives in
-# the backed up tar file, so handling this would mean untar'ing, tricking
-# Mailman into loading the pickle (or manually loading and patching), then
-# re-tar'ing. I don't think it's worth it because the only thing that will
-# be broken is if a list that's deactivated on production is re-activated on
-# staging.
-#
-# - Backpatch all the message footers and RFC 2369 headers of the messages in
-# the archive. To do this, we'd have to iterate through all messages,
-# tweaking the List-* headers (easy) and ripping apart the footers,
-# recalculating them and reattaching them (difficult). Doing the iteration
-# and update is quite painful in Python 2.4, but would be easier with Python
-# 2.5's new mailbox module. /Then/ we'd have to regenerate the archives.
-# Not doing this means that some of the links in staging's MHonArc archive
-# will point to production archives.
-
-import logging
-import os
-import subprocess
import sys
-import textwrap
-
-from zope.component import getUtility
-from zope.security.proxy import removeSecurityProxy
-
-from lp.registry.interfaces.mailinglist import IMailingListSet
-from lp.registry.interfaces.person import IPersonSet
-from lp.services.config import config
-from lp.services.identity.interfaces.emailaddress import IEmailAddressSet
-from lp.services.mailman.config import configure_prefix
-from lp.services.scripts.base import LaunchpadScript
-
-
-RSYNC_OPTIONS = ('-avz', '--delete')
-RSYNC_COMMAND = '/usr/bin/rsync'
-RSYNC_SUBDIRECTORIES = ('archives', 'backups', 'lists', 'mhonarc')
-SPACE = ' '
-
-
-class MailingListSyncScript(LaunchpadScript):
- """
- %prog [options] source_url
-
- Sync the Mailman data structures between production and staging. This
- takes the most efficient route by rsync'ing over the list pickles, raw
- archive mboxes, and mhonarc files, then it fixes up anything that needs
- fixing. This does /not/ sync over any qfiles because staging doesn't send
- emails anyway.
-
- source_url is required and it is the rsync source url which contains
- mailman's var directory. The destination is taken from the launchpad.conf
- file.
- """
-
- loglevel = logging.INFO
- description = 'Sync the Mailman data structures with the database.'
-
- def __init__(self, name, dbuser=None):
- self.usage = textwrap.dedent(self.__doc__)
- super(MailingListSyncScript, self).__init__(name, dbuser)
-
- def add_my_options(self):
- """See `LaunchpadScript`."""
- # Add optional override of production mailing list host name. In real
- # use, it's always going to be lists.launchpad.net so that makes a
- # reasonable default. Some testing environments may override this.
- self.parser.add_option('--hostname', default='lists.launchpad.net',
- help=('The hostname for the production '
- 'mailing list system. This is used to '
- 'resolve teams which have multiple '
- 'email addresses.'))
-
- def syncMailmanDirectories(self, source_url):
- """Synchronize the Mailman directories.
-
- :param source_url: the base url of the source
- """
- # This can't be done at module global scope.
- from Mailman import mm_cfg
- # Start by rsync'ing over the entire $vardir/lists, $vardir/archives,
- # $vardir/backups, and $vardir/mhonarc directories. We specifically
- # do not rsync the data, locks, logs, qfiles, or spam directories.
- destination_url = config.mailman.build_var_dir
- # Do one rsync for all subdirectories.
- rsync_command = [RSYNC_COMMAND]
- rsync_command.extend(RSYNC_OPTIONS)
- rsync_command.append('--exclude=%s' % mm_cfg.MAILMAN_SITE_LIST)
- rsync_command.append('--exclude=%s.mbox' % mm_cfg.MAILMAN_SITE_LIST)
- rsync_command.extend(os.path.join(source_url, subdirectory)
- for subdirectory in RSYNC_SUBDIRECTORIES)
- rsync_command.append(destination_url)
- self.logger.info('executing: %s', SPACE.join(rsync_command))
- process = subprocess.Popen(rsync_command,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- stdout, stderr = process.communicate()
- if process.returncode == 0:
- self.logger.debug('%s', stdout)
- else:
- self.logger.error('rsync command failed with exit status: %s',
- process.returncode)
- self.logger.error('STDOUT:\n%s\nSTDERR:\n%s', stdout, stderr)
- return process.returncode
-
- def fixHostnames(self):
- """Fix up the host names in Mailman and the LP database."""
- # These can't be done at module global scope.
- from Mailman import Utils
- from Mailman import mm_cfg
- from Mailman.MailList import MailList
-
- # Grab a couple of useful components.
- email_address_set = getUtility(IEmailAddressSet)
- mailing_list_set = getUtility(IMailingListSet)
-
- # Clean things up per mailing list.
- for list_name in Utils.list_names():
- # Skip the site list.
- if list_name == mm_cfg.MAILMAN_SITE_LIST:
- continue
-
- # The first thing to clean up is the mailing list pickles. There
- # are things like host names in some attributes that need to be
- # converted. The following opens a locked list.
- mailing_list = MailList(list_name)
- try:
- mailing_list.host_name = mm_cfg.DEFAULT_EMAIL_HOST
- mailing_list.web_page_url = (
- mm_cfg.DEFAULT_URL_PATTERN % mm_cfg.DEFAULT_URL_HOST)
- mailing_list.Save()
- finally:
- mailing_list.Unlock()
-
- # Patch up the email address for the list in the Launchpad
- # database.
- lp_mailing_list = mailing_list_set.get(list_name)
- if lp_mailing_list is None:
- # We found a mailing list in Mailman that does not exist in
- # the Launchpad database. This can happen if we rsync'd the
- # Mailman directories after the lists were created, but we
- # copied the LP database /before/ the lists were created.
- # If we don't delete the Mailman lists, we won't be able to
- # create the mailing lists on staging.
- self.logger.error('No LP mailing list for: %s', list_name)
- self.deleteMailmanList(list_name)
- continue
-
- # Clean up the team email addresses corresponding to their mailing
- # lists. Note that teams can have two email addresses if they
- # have a different contact address.
- team = getUtility(IPersonSet).getByName(list_name)
- mlist_addresses = email_address_set.getByPerson(team)
- if mlist_addresses.count() == 0:
- self.logger.error('No LP email address for: %s', list_name)
- else:
- # Teams can have both a mailing list and a contact address.
- old_address = '%s@%s' % (list_name, self.options.hostname)
- for email_address in mlist_addresses:
- if email_address.email == old_address:
- new_address = lp_mailing_list.address
- removeSecurityProxy(email_address).email = new_address
- self.logger.info('%s -> %s', old_address, new_address)
- break
- else:
- self.logger.error('No change to LP email address for: %s',
- list_name)
-
- def deleteMailmanList(self, list_name):
- """Delete all Mailman data structures for `list_name`."""
- mailman_bindir = os.path.normpath(os.path.join(
- configure_prefix(config.mailman.build_prefix), 'bin'))
- process = subprocess.Popen(('./rmlist', '-a', list_name),
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- cwd=mailman_bindir)
- stdout, stderr = process.communicate()
- if process.returncode == 0:
- self.logger.info('%s', stdout)
- else:
- self.logger.error('rmlist command failed with exit status: %s',
- process.returncode)
- self.logger.error('STDOUT:\n%s\nSTDERR:\n%s', stdout, stderr)
- # Keep going.
-
- def main(self):
- """See `LaunchpadScript`."""
- source_url = None
- if len(self.args) == 0:
- self.parser.error('Missing source_url')
- elif len(self.args) > 1:
- self.parser.error('Too many arguments')
- else:
- source_url = self.args[0]
-
- # We need to get to the Mailman API. Set up the paths so that Mailman
- # can be imported. This can't be done at module global scope.
- mailman_path = configure_prefix(config.mailman.build_prefix)
- sys.path.append(mailman_path)
-
- retcode = self.syncMailmanDirectories(source_url)
- if retcode != 0:
- return retcode
-
- self.fixHostnames()
- # All done; commit the database changes.
- self.txn.commit()
- return 0
+from lp.services.mailman.scripts.mlist_sync import MailingListSyncScript
if __name__ == '__main__':
- script = MailingListSyncScript('scripts.mlist-sync', 'mlist-sync')
+ script = MailingListSyncScript('mlist-sync', dbuser='mlist-sync')
status = script.lock_and_run()
sys.exit(status)
