launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27015
[Merge] ~cjwatson/launchpad:garbo-archive-tokens into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:garbo-archive-tokens into launchpad:master.
Commit message:
Move remaining generate-ppa-htaccess functions to garbo
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/402375
Now that archive authentication is handled dynamically, we no longer need to deal with expiring archive subscriptions and deactivating archive authorization tokens via the relatively heavyweight mechanism of a standalone script run once per minute on the PPA publisher system: it's really just a particular kind of garbage collection (with the only quirk being that it sends emails), so it can reasonably be run from garbo.
Deactivating tokens depends on expiring subscriptions, but the latter is quite lightweight and so can be run from garbo-frequently, so in practice this won't delay things for long, and it isn't vital for the cancellation emails sent when tokens are deactivated to be sent immediately.
We can remove generate-ppa-htaccess entirely once this commit lands on production and once we've removed it from production crontabs.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:garbo-archive-tokens into launchpad:master.
diff --git a/database/schema/security.cfg b/database/schema/security.cfg
index 1bbb845..c3aba65 100644
--- a/database/schema/security.cfg
+++ b/database/schema/security.cfg
@@ -2423,6 +2423,8 @@ public.accesspolicyartifact = SELECT, DELETE
public.accesspolicygrant = SELECT, DELETE
public.account = SELECT, DELETE
public.answercontact = SELECT, DELETE
+public.archiveauthtoken = SELECT, UPDATE
+public.archivesubscriber = SELECT, UPDATE
public.branch = SELECT, UPDATE
public.branchjob = SELECT, DELETE
public.branchmergeproposal = SELECT, UPDATE, DELETE
diff --git a/lib/lp/archivepublisher/scripts/generate_ppa_htaccess.py b/lib/lp/archivepublisher/scripts/generate_ppa_htaccess.py
index 26e8db8..5ddcc0e 100644
--- a/lib/lp/archivepublisher/scripts/generate_ppa_htaccess.py
+++ b/lib/lp/archivepublisher/scripts/generate_ppa_htaccess.py
@@ -3,34 +3,17 @@
# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
-from datetime import datetime
-
-import pytz
-
-from lp.registry.model.teammembership import TeamParticipation
-from lp.services.config import config
-from lp.services.database.interfaces import IStore
-from lp.services.mail.helpers import get_email_template
-from lp.services.mail.mailwrapper import MailWrapper
-from lp.services.mail.sendmail import (
- format_address,
- simple_sendmail,
- )
from lp.services.scripts.base import LaunchpadCronScript
-from lp.services.webapp import canonical_url
-from lp.soyuz.enums import ArchiveSubscriberStatus
-from lp.soyuz.model.archiveauthtoken import ArchiveAuthToken
-from lp.soyuz.model.archivesubscriber import ArchiveSubscriber
class HtaccessTokenGenerator(LaunchpadCronScript):
"""Expire archive subscriptions and deactivate invalid tokens."""
- # XXX cjwatson 2021-04-21: This script and class are now misnamed, as we
- # no longer generate .htaccess or .htpasswd files, but instead check
- # archive authentication dynamically. We can remove this script once we
- # stop running it on production and move its remaining functions
- # elsewhere (probably garbo).
+ # XXX cjwatson 2021-05-06: This script does nothing. We no longer
+ # generate .htaccess or .htpasswd files, but instead check archive
+ # authentication dynamically; and garbo now handles expiring
+ # subscriptions and deactivating tokens. We can remove this script once
+ # we stop running it on production.
def add_my_options(self):
"""Add script command line options."""
@@ -44,145 +27,6 @@ class HtaccessTokenGenerator(LaunchpadCronScript):
dest="no_deactivation", default=False,
help="If set, tokens are not deactivated.")
- def sendCancellationEmail(self, token):
- """Send an email to the person whose subscription was cancelled."""
- if token.archive.suppress_subscription_notifications:
- # Don't send an email if they should be suppresed for the
- # archive
- return
- send_to_person = token.person
- ppa_name = token.archive.displayname
- ppa_owner_url = canonical_url(token.archive.owner)
- subject = "PPA access cancelled for %s" % ppa_name
- template = get_email_template(
- "ppa-subscription-cancelled.txt", app='soyuz')
-
- assert not send_to_person.is_team, (
- "Token.person is a team, it should always be individuals.")
-
- if send_to_person.preferredemail is None:
- # The person has no preferred email set, so we don't
- # email them.
- return
-
- to_address = [send_to_person.preferredemail.email]
- replacements = {
- 'recipient_name': send_to_person.displayname,
- 'ppa_name': ppa_name,
- 'ppa_owner_url': ppa_owner_url,
- }
- body = MailWrapper(72).format(
- template % replacements, force_wrap=True)
-
- from_address = format_address(
- ppa_name,
- config.canonical.noreply_from_address)
-
- headers = {
- 'Sender': config.canonical.bounce_address,
- }
-
- simple_sendmail(from_address, to_address, subject, body, headers)
-
- def _getInvalidTokens(self):
- """Return all invalid tokens.
-
- A token is invalid if it is active and the token owner is *not* a
- subscriber to the archive that the token is for. The subscription can
- be either direct or through a team.
- """
- # First we grab all the active tokens for which there is a
- # matching current archive subscription for a team of which the
- # token owner is a member.
- store = IStore(ArchiveSubscriber)
- valid_tokens = store.find(
- ArchiveAuthToken,
- ArchiveAuthToken.name == None,
- ArchiveAuthToken.date_deactivated == None,
- ArchiveAuthToken.archive_id == ArchiveSubscriber.archive_id,
- ArchiveSubscriber.status == ArchiveSubscriberStatus.CURRENT,
- ArchiveSubscriber.subscriber_id == TeamParticipation.teamID,
- TeamParticipation.personID == ArchiveAuthToken.person_id)
-
- # We can then evaluate the invalid tokens by the difference of
- # all active tokens and valid tokens.
- all_active_tokens = store.find(
- ArchiveAuthToken,
- ArchiveAuthToken.name == None,
- ArchiveAuthToken.date_deactivated == None)
-
- return all_active_tokens.difference(valid_tokens)
-
- def deactivateTokens(self, tokens, send_email=False):
- """Deactivate the given tokens.
-
- :return: A set of PPAs affected by the deactivations.
- """
- affected_ppas = set()
- num_tokens = 0
- for token in tokens:
- if send_email:
- self.sendCancellationEmail(token)
- # Deactivate tokens one at a time, as 'tokens' is the result of a
- # set expression and storm does not allow setting on such things.
- token.deactivate()
- affected_ppas.add(token.archive)
- num_tokens += 1
- self.logger.debug(
- "Deactivated %s tokens, %s PPAs affected"
- % (num_tokens, len(affected_ppas)))
- return affected_ppas
-
- def deactivateInvalidTokens(self, send_email=False):
- """Deactivate tokens as necessary.
-
- If an active token for a PPA no longer has any subscribers,
- we deactivate the token.
-
- :param send_email: Whether to send a cancellation email to the owner
- of the token. This defaults to False to speed up the test
- suite.
- :return: the set of ppas affected by token deactivations.
- """
- invalid_tokens = self._getInvalidTokens()
- return self.deactivateTokens(invalid_tokens, send_email=send_email)
-
- def expireSubscriptions(self):
- """Expire subscriptions as necessary.
-
- If an `ArchiveSubscriber`'s date_expires has passed, then
- set its status to EXPIRED.
- """
- now = datetime.now(pytz.UTC)
-
- store = IStore(ArchiveSubscriber)
- newly_expired_subscriptions = store.find(
- ArchiveSubscriber,
- ArchiveSubscriber.status == ArchiveSubscriberStatus.CURRENT,
- ArchiveSubscriber.date_expires != None,
- ArchiveSubscriber.date_expires <= now)
-
- subscription_names = [
- subs.displayname for subs in newly_expired_subscriptions]
- if subscription_names:
- newly_expired_subscriptions.set(
- status=ArchiveSubscriberStatus.EXPIRED)
- self.logger.info(
- "Expired subscriptions: %s" % ", ".join(subscription_names))
-
def main(self):
"""Script entry point."""
- self.logger.info('Starting the PPA .htaccess generation')
- self.expireSubscriptions()
- affected_ppas = self.deactivateInvalidTokens(send_email=True)
- self.logger.debug(
- '%s PPAs with deactivated tokens' % len(affected_ppas))
-
- if self.options.no_deactivation or self.options.dryrun:
- self.logger.info('Dry run, so not committing transaction.')
- self.txn.abort()
- else:
- self.logger.info('Committing transaction...')
- self.txn.commit()
-
- self.logger.info('Finished PPA .htaccess generation')
+ pass
diff --git a/lib/lp/archivepublisher/tests/test_generate_ppa_htaccess.py b/lib/lp/archivepublisher/tests/test_generate_ppa_htaccess.py
deleted file mode 100644
index 472b7bf..0000000
--- a/lib/lp/archivepublisher/tests/test_generate_ppa_htaccess.py
+++ /dev/null
@@ -1,337 +0,0 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Test the generate_ppa_htaccess.py script. """
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-from datetime import (
- datetime,
- timedelta,
- )
-import os
-import subprocess
-import sys
-
-import pytz
-from zope.component import getUtility
-
-from lp.archivepublisher.scripts.generate_ppa_htaccess import (
- HtaccessTokenGenerator,
- )
-from lp.registry.interfaces.distribution import IDistributionSet
-from lp.registry.interfaces.person import IPersonSet
-from lp.registry.interfaces.teammembership import TeamMembershipStatus
-from lp.services.config import config
-from lp.services.features.testing import FeatureFixture
-from lp.services.log.logger import BufferLogger
-from lp.soyuz.enums import ArchiveSubscriberStatus
-from lp.soyuz.interfaces.archive import NAMED_AUTH_TOKEN_FEATURE_FLAG
-from lp.testing import TestCaseWithFactory
-from lp.testing.dbuser import (
- lp_dbuser,
- switch_dbuser,
- )
-from lp.testing.layers import LaunchpadZopelessLayer
-from lp.testing.mail_helpers import pop_notifications
-
-
-class TestPPAHtaccessTokenGeneration(TestCaseWithFactory):
- """Test the generate_ppa_htaccess.py script."""
-
- layer = LaunchpadZopelessLayer
- dbuser = config.generateppahtaccess.dbuser
-
- SCRIPT_NAME = 'test tokens'
-
- def setUp(self):
- super(TestPPAHtaccessTokenGeneration, self).setUp()
- self.owner = self.factory.makePerson(
- name="joe", displayname="Joe Smith")
- self.ppa = self.factory.makeArchive(
- owner=self.owner, name="myppa", private=True)
-
- # "Ubuntu" doesn't have a proper publisher config but Ubuntutest
- # does, so override the PPA's distro here.
- ubuntutest = getUtility(IDistributionSet)['ubuntutest']
- self.ppa.distribution = ubuntutest
-
- # Enable named auth tokens.
- self.useFixture(FeatureFixture({NAMED_AUTH_TOKEN_FEATURE_FLAG: "on"}))
-
- def getScript(self, test_args=None):
- """Return a HtaccessTokenGenerator instance."""
- if test_args is None:
- test_args = []
- script = HtaccessTokenGenerator(self.SCRIPT_NAME, test_args=test_args)
- script.logger = BufferLogger()
- script.txn = self.layer.txn
- switch_dbuser(self.dbuser)
- return script
-
- def runScript(self):
- """Run the expiry script.
-
- :return: a tuple of return code, stdout and stderr.
- """
- script = os.path.join(
- config.root, "cronscripts", "generate-ppa-htaccess.py")
- args = [sys.executable, script, "-v"]
- process = subprocess.Popen(
- args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = process.communicate()
- return process.returncode, stdout, stderr
-
- def assertDeactivated(self, token):
- """Helper function to test token deactivation state."""
- return self.assertNotEqual(token.date_deactivated, None)
-
- def assertNotDeactivated(self, token):
- """Helper function to test token deactivation state."""
- self.assertEqual(token.date_deactivated, None)
-
- def setupSubscriptionsAndTokens(self):
- """Set up a few subscriptions and test tokens and return them."""
- # Set up some teams. We need to test a few scenarios:
- # - someone in one subscribed team and leaving that team loses
- # their token.
- # - someone in two subscribed teams leaving one team does not
- # lose their token.
- # - All members of a team lose their tokens when a team of a
- # subscribed team leaves it.
-
- persons1 = []
- persons2 = []
- name12 = getUtility(IPersonSet).getByName("name12")
- team1 = self.factory.makeTeam(owner=name12)
- team2 = self.factory.makeTeam(owner=name12)
- for count in range(5):
- person = self.factory.makePerson()
- team1.addMember(person, name12)
- persons1.append(person)
- person = self.factory.makePerson()
- team2.addMember(person, name12)
- persons2.append(person)
-
- all_persons = persons1 + persons2
-
- parent_team = self.factory.makeTeam(owner=name12)
- # This needs to be forced or TeamParticipation is not updated.
- parent_team.addMember(team2, name12, force_team_add=True)
-
- promiscuous_person = self.factory.makePerson()
- team1.addMember(promiscuous_person, name12)
- team2.addMember(promiscuous_person, name12)
- all_persons.append(promiscuous_person)
-
- lonely_person = self.factory.makePerson()
- all_persons.append(lonely_person)
-
- # At this point we have team1, with 5 people in it, team2 with 5
- # people in it, team3 with only team2 in it, promiscuous_person
- # who is in team1 and team2, and lonely_person who is in no
- # teams.
-
- # Ok now do some subscriptions and ensure everyone has a token.
- self.ppa.newSubscription(team1, self.ppa.owner)
- self.ppa.newSubscription(parent_team, self.ppa.owner)
- self.ppa.newSubscription(lonely_person, self.ppa.owner)
- tokens = {}
- for person in all_persons:
- tokens[person] = self.ppa.newAuthToken(person)
-
- return (
- team1, team2, parent_team, lonely_person,
- promiscuous_person, all_persons, persons1, persons2, tokens)
-
- def testDeactivatingTokens(self):
- """Test that token deactivation happens properly."""
- data = self.setupSubscriptionsAndTokens()
- (team1, team2, parent_team, lonely_person, promiscuous_person,
- all_persons, persons1, persons2, tokens) = data
- team1_person = persons1[0]
-
- # Named tokens should be ignored for deactivation.
- self.ppa.newNamedAuthToken("tokenname1")
- named_token = self.ppa.newNamedAuthToken("tokenname2")
- named_token.deactivate()
-
- # Initially, nothing is eligible for deactivation.
- script = self.getScript()
- script.deactivateInvalidTokens()
- for person in tokens:
- self.assertNotDeactivated(tokens[person])
-
- # Now remove someone from team1. They will lose their token but
- # everyone else keeps theirs.
- with lp_dbuser():
- team1_person.leave(team1)
- # Clear out emails generated when leaving a team.
- pop_notifications()
-
- script.deactivateInvalidTokens(send_email=True)
- self.assertDeactivated(tokens[team1_person])
- del tokens[team1_person]
- for person in tokens:
- self.assertNotDeactivated(tokens[person])
-
- # Ensure that a cancellation email was sent.
- self.assertEmailQueueLength(1)
-
- # Promiscuous_person now leaves team1, but does not lose their
- # token because they're also in team2. No other tokens are
- # affected.
- with lp_dbuser():
- promiscuous_person.leave(team1)
- # Clear out emails generated when leaving a team.
- pop_notifications()
- script.deactivateInvalidTokens(send_email=True)
- self.assertNotDeactivated(tokens[promiscuous_person])
- for person in tokens:
- self.assertNotDeactivated(tokens[person])
-
- # Ensure that a cancellation email was not sent.
- self.assertEmailQueueLength(0)
-
- # Team 2 now leaves parent_team, and all its members lose their
- # tokens.
- with lp_dbuser():
- name12 = getUtility(IPersonSet).getByName("name12")
- parent_team.setMembershipData(
- team2, TeamMembershipStatus.APPROVED, name12)
- parent_team.setMembershipData(
- team2, TeamMembershipStatus.DEACTIVATED, name12)
- self.assertFalse(team2.inTeam(parent_team))
- script.deactivateInvalidTokens()
- for person in persons2:
- self.assertDeactivated(tokens[person])
-
- # promiscuous_person also loses the token because they're not in
- # either team now.
- self.assertDeactivated(tokens[promiscuous_person])
-
- # lonely_person still has their token; they're not in any teams.
- self.assertNotDeactivated(tokens[lonely_person])
-
- def setupDummyTokens(self):
- """Helper function to set up some tokens."""
- name12 = getUtility(IPersonSet).getByName("name12")
- name16 = getUtility(IPersonSet).getByName("name16")
- sub1 = self.ppa.newSubscription(name12, self.ppa.owner)
- sub2 = self.ppa.newSubscription(name16, self.ppa.owner)
- token1 = self.ppa.newAuthToken(name12)
- token2 = self.ppa.newAuthToken(name16)
- token3 = self.ppa.newNamedAuthToken("tokenname3")
- self.layer.txn.commit()
- return (sub1, sub2), (token1, token2, token3)
-
- def testSubscriptionExpiry(self):
- """Ensure subscriptions' statuses are set to EXPIRED properly."""
- subs, tokens = self.setupDummyTokens()
- now = datetime.now(pytz.UTC)
-
- # Expire the first subscription.
- subs[0].date_expires = now - timedelta(minutes=3)
- self.assertEqual(subs[0].status, ArchiveSubscriberStatus.CURRENT)
-
- # Set the expiry in the future for the second.
- subs[1].date_expires = now + timedelta(minutes=3)
- self.assertEqual(subs[0].status, ArchiveSubscriberStatus.CURRENT)
-
- # Run the script and make sure only the first was expired.
- script = self.getScript()
- script.main()
- self.assertEqual(subs[0].status, ArchiveSubscriberStatus.EXPIRED)
- self.assertEqual(subs[1].status, ArchiveSubscriberStatus.CURRENT)
-
- def _setupOptionsData(self):
- """Setup test data for option testing."""
- subs, tokens = self.setupDummyTokens()
-
- # Cancel the first subscription.
- subs[0].cancel(self.ppa.owner)
- self.assertNotDeactivated(tokens[0])
- return subs, tokens
-
- def testDryrunOption(self):
- """Test that the dryrun and no-deactivation option works."""
- subs, tokens = self._setupOptionsData()
-
- script = self.getScript(test_args=["--dry-run"])
- script.main()
-
- # Assert that the cancelled subscription did not cause the token
- # to get deactivated.
- self.assertNotDeactivated(tokens[0])
-
- def testNoDeactivationOption(self):
- """Test that the --no-deactivation option works."""
- subs, tokens = self._setupOptionsData()
- script = self.getScript(test_args=["--no-deactivation"])
- script.main()
- self.assertNotDeactivated(tokens[0])
- script = self.getScript()
- script.main()
- self.assertDeactivated(tokens[0])
-
- def testSendingCancellationEmail(self):
- """Test that when a token is deactivated, its user gets an email.
-
- The email must contain the right headers and text.
- """
- subs, tokens = self.setupDummyTokens()
- script = self.getScript()
-
- # Clear out any existing email.
- pop_notifications()
-
- script.sendCancellationEmail(tokens[0])
-
- [email] = pop_notifications()
- self.assertEqual(
- email['Subject'],
- "PPA access cancelled for PPA named myppa for Joe Smith")
- self.assertEqual(email['To'], "test@xxxxxxxxxxxxx")
- self.assertEqual(
- email['From'],
- "PPA named myppa for Joe Smith <noreply@xxxxxxxxxxxxx>")
- self.assertEqual(email['Sender'], "bounces@xxxxxxxxxxxxx")
-
- body = email.get_payload()
- self.assertEqual(
- body,
- "Hello Sample Person,\n\n"
- "Launchpad: cancellation of archive access\n"
- "-----------------------------------------\n\n"
- "Your access to the private software archive "
- "\"PPA named myppa for Joe\nSmith\", "
- "which is hosted by Launchpad, has been "
- "cancelled.\n\n"
- "You will now no longer be able to download software from this "
- "archive.\n"
- "If you think this cancellation is in error, you should contact "
- "the owner\n"
- "of the archive to verify it.\n\n"
- "You can contact the archive owner by visiting their Launchpad "
- "page here:\n\n"
- "<http://launchpad.test/~joe>\n\n"
- "If you have any concerns you can contact the Launchpad team by "
- "emailing\n"
- "feedback@xxxxxxxxxxxxx\n\n"
- "Regards,\n"
- "The Launchpad team")
-
- def testNoEmailOnCancellationForSuppressedArchive(self):
- """No email should be sent if the archive has
- suppress_subscription_notifications set."""
- subs, tokens = self.setupDummyTokens()
- token = tokens[0]
- token.archive.suppress_subscription_notifications = True
- script = self.getScript()
-
- # Clear out any existing email.
- pop_notifications()
-
- script.sendCancellationEmail(token)
-
- self.assertEmailQueueLength(0)
diff --git a/lib/lp/scripts/garbo.py b/lib/lp/scripts/garbo.py
index 4492911..1d1a8da 100644
--- a/lib/lp/scripts/garbo.py
+++ b/lib/lp/scripts/garbo.py
@@ -36,12 +36,14 @@ import six
from storm.expr import (
And,
Cast,
+ Except,
In,
Join,
Max,
Min,
Or,
Row,
+ Select,
SQL,
)
from storm.info import ClassAlias
@@ -74,19 +76,25 @@ from lp.code.model.revision import (
RevisionCache,
)
from lp.oci.model.ocirecipebuild import OCIFile
+from lp.registry.interfaces.person import IPersonSet
from lp.registry.model.person import Person
from lp.registry.model.product import Product
from lp.registry.model.sourcepackagename import SourcePackageName
-from lp.registry.model.teammembership import TeamMembership
+from lp.registry.model.teammembership import (
+ TeamMembership,
+ TeamParticipation,
+ )
from lp.services.config import config
from lp.services.database import postgresql
from lp.services.database.bulk import (
create,
dbify_value,
+ load_related,
)
from lp.services.database.constants import UTC_NOW
from lp.services.database.interfaces import IMasterStore
from lp.services.database.sqlbase import (
+ convert_storm_clause_to_string,
cursor,
session_store,
sqlvalues,
@@ -109,6 +117,13 @@ from lp.services.job.model.job import Job
from lp.services.librarian.model import TimeLimitedToken
from lp.services.log.logger import PrefixFilter
from lp.services.looptuner import TunableLoop
+from lp.services.mail.helpers import get_email_template
+from lp.services.mail.mailwrapper import MailWrapper
+from lp.services.mail.sendmail import (
+ format_address,
+ set_immediate_mail_delivery,
+ simple_sendmail,
+ )
from lp.services.openid.model.openidconsumer import OpenIDConsumerNonce
from lp.services.propertycache import cachedproperty
from lp.services.scripts.base import (
@@ -118,12 +133,16 @@ from lp.services.scripts.base import (
)
from lp.services.session.model import SessionData
from lp.services.verification.model.logintoken import LoginToken
+from lp.services.webapp.publisher import canonical_url
from lp.services.webhooks.interfaces import IWebhookJobSource
from lp.services.webhooks.model import WebhookJob
from lp.snappy.model.snapbuild import SnapFile
from lp.snappy.model.snapbuildjob import SnapBuildJobType
+from lp.soyuz.enums import ArchiveSubscriberStatus
from lp.soyuz.interfaces.publishing import active_publishing_status
from lp.soyuz.model.archive import Archive
+from lp.soyuz.model.archiveauthtoken import ArchiveAuthToken
+from lp.soyuz.model.archivesubscriber import ArchiveSubscriber
from lp.soyuz.model.distributionsourcepackagecache import (
DistributionSourcePackageCache,
)
@@ -1556,6 +1575,150 @@ class GitRepositoryPruner(TunableLoop):
transaction.commit()
+class ArchiveSubscriptionExpirer(BulkPruner):
+ """Expire archive subscriptions as necessary.
+
+ If an `ArchiveSubscriber`'s date_expires has passed, then set its status
+ to EXPIRED.
+ """
+ target_table_class = ArchiveSubscriber
+
+ ids_to_prune_query = convert_storm_clause_to_string(Select(
+ ArchiveSubscriber.id,
+ where=And(
+ ArchiveSubscriber.status == ArchiveSubscriberStatus.CURRENT,
+ ArchiveSubscriber.date_expires != None,
+ ArchiveSubscriber.date_expires <= UTC_NOW)))
+
+ maximum_chunk_size = 1000
+
+ _num_removed = None
+
+ def __call__(self, chunk_size):
+ """See `ITunableLoop`."""
+ chunk_size = int(chunk_size + 0.5)
+ newly_expired_subscriptions = list(self.store.find(
+ ArchiveSubscriber,
+ ArchiveSubscriber.id.is_in(SQL(
+ "SELECT * FROM cursor_fetch(%s, %s) AS f(id integer)",
+ params=(self.cursor_name, chunk_size)))))
+ load_related(Archive, newly_expired_subscriptions, ["archive_id"])
+ load_related(Person, newly_expired_subscriptions, ["subscriber_id"])
+ subscription_names = [
+ sub.displayname for sub in newly_expired_subscriptions]
+ if subscription_names:
+ self.store.find(
+ ArchiveSubscriber,
+ ArchiveSubscriber.id.is_in(
+ [sub.id for sub in newly_expired_subscriptions]),
+ ).set(status=ArchiveSubscriberStatus.EXPIRED)
+ self.log.info(
+ "Expired subscriptions: %s" % ", ".join(subscription_names))
+ self._num_removed = len(subscription_names)
+ transaction.commit()
+
+
+class ArchiveAuthTokenDeactivator(BulkPruner):
+ """Deactivate archive auth tokens as necessary.
+
+ If an active token for a PPA no longer has any subscribers, we
+ deactivate the token, and send an email to the person whose subscription
+ was cancelled.
+ """
+ target_table_class = ArchiveAuthToken
+
+ # A token is invalid if it is active and the token owner is *not* a
+ # subscriber to the archive that the token is for. The subscription can
+ # be either direct or through a team.
+ ids_to_prune_query = convert_storm_clause_to_string(Except(
+ # All valid tokens.
+ Select(
+ ArchiveAuthToken.id, tables=[ArchiveAuthToken],
+ where=And(
+ ArchiveAuthToken.name == None,
+ ArchiveAuthToken.date_deactivated == None)),
+ # Active tokens for which there is a matching current archive
+ # subscription for a team of which the token owner is a member.
+ # Removing these from the set of all valid tokens leaves only the
+ # invalid tokens.
+ Select(
+ ArchiveAuthToken.id,
+ tables=[ArchiveAuthToken, ArchiveSubscriber, TeamParticipation],
+ where=And(
+ ArchiveAuthToken.name == None,
+ ArchiveAuthToken.date_deactivated == None,
+ ArchiveAuthToken.archive_id == ArchiveSubscriber.archive_id,
+ ArchiveSubscriber.status == ArchiveSubscriberStatus.CURRENT,
+ ArchiveSubscriber.subscriber_id == TeamParticipation.teamID,
+ TeamParticipation.personID == ArchiveAuthToken.person_id))))
+
+ maximum_chunk_size = 10
+
+ def _sendCancellationEmail(self, token):
+ """Send an email to the person whose subscription was cancelled."""
+ if token.archive.suppress_subscription_notifications:
+ # Don't send an email if they should be suppressed for the
+ # archive.
+ return
+ send_to_person = token.person
+ ppa_name = token.archive.displayname
+ ppa_owner_url = canonical_url(token.archive.owner)
+ subject = "PPA access cancelled for %s" % ppa_name
+ template = get_email_template(
+ "ppa-subscription-cancelled.txt", app="soyuz")
+
+ if send_to_person.is_team:
+ raise AssertionError(
+ "Token.person is a team, it should always be individuals.")
+
+ if send_to_person.preferredemail is None:
+ # The person has no preferred email set, so we don't email them.
+ return
+
+ to_address = [send_to_person.preferredemail.email]
+ replacements = {
+ "recipient_name": send_to_person.display_name,
+ "ppa_name": ppa_name,
+ "ppa_owner_url": ppa_owner_url,
+ }
+ body = MailWrapper(72).format(
+ template % replacements, force_wrap=True)
+
+ from_address = format_address(
+ ppa_name,
+ config.canonical.noreply_from_address)
+
+ headers = {
+ "Sender": config.canonical.bounce_address,
+ }
+
+ simple_sendmail(from_address, to_address, subject, body, headers)
+
+ def __call__(self, chunk_size):
+ """See `ITunableLoop`."""
+ chunk_size = int(chunk_size + 0.5)
+ tokens = list(self.store.find(
+ ArchiveAuthToken,
+ ArchiveAuthToken.id.is_in(SQL(
+ "SELECT * FROM cursor_fetch(%s, %s) AS f(id integer)",
+ params=(self.cursor_name, chunk_size)))))
+ affected_ppas = load_related(Archive, tokens, ["archive_id"])
+ load_related(Person, affected_ppas, ["ownerID"])
+ getUtility(IPersonSet).getPrecachedPersonsFromIDs(
+ [token.person_id for token in tokens], need_preferred_email=True)
+ for token in tokens:
+ self._sendCancellationEmail(token)
+ self.store.find(
+ ArchiveAuthToken,
+ ArchiveAuthToken.id.is_in([token.id for token in tokens]),
+ ).set(date_deactivated=UTC_NOW)
+ self.log.info(
+ "Deactivated %s tokens, %s PPAs affected" %
+ (len(tokens), len(affected_ppas)))
+ self._num_removed = len(tokens)
+ transaction.commit()
+
+
class BaseDatabaseGarbageCollector(LaunchpadCronScript):
"""Abstract base class to run a collection of TunableLoops."""
script_name = None # Script name for locking and database user. Override.
@@ -1600,6 +1763,10 @@ class BaseDatabaseGarbageCollector(LaunchpadCronScript):
def main(self):
self.start_time = time.time()
+ # Any email we send can safely be queued until the transaction is
+ # committed.
+ set_immediate_mail_delivery(False)
+
# Stores the number of failed tasks.
self.failure_count = 0
@@ -1783,6 +1950,7 @@ class FrequentDatabaseGarbageCollector(BaseDatabaseGarbageCollector):
script_name = 'garbo-frequently'
tunable_loops = [
AntiqueSessionPruner,
+ ArchiveSubscriptionExpirer,
BugSummaryJournalRollup,
BugWatchScheduler,
OpenIDConsumerAssociationPruner,
@@ -1805,6 +1973,7 @@ class HourlyDatabaseGarbageCollector(BaseDatabaseGarbageCollector):
"""
script_name = 'garbo-hourly'
tunable_loops = [
+ ArchiveAuthTokenDeactivator,
BugHeatUpdater,
DuplicateSessionPruner,
GitRepositoryPruner,
diff --git a/lib/lp/scripts/tests/test_garbo.py b/lib/lp/scripts/tests/test_garbo.py
index 2b5fa34..fc7fb34 100644
--- a/lib/lp/scripts/tests/test_garbo.py
+++ b/lib/lp/scripts/tests/test_garbo.py
@@ -12,8 +12,11 @@ from datetime import (
datetime,
timedelta,
)
+from functools import partial
import hashlib
import logging
+import re
+from textwrap import dedent
import time
from pytz import UTC
@@ -33,6 +36,7 @@ from storm.store import Store
from testtools.content import text_content
from testtools.matchers import (
AfterPreprocessing,
+ ContainsDict,
Equals,
GreaterThan,
Is,
@@ -133,7 +137,11 @@ from lp.snappy.model.snapbuildjob import (
SnapBuildJob,
SnapStoreUploadJob,
)
-from lp.soyuz.enums import PackagePublishingStatus
+from lp.soyuz.enums import (
+ ArchiveSubscriberStatus,
+ PackagePublishingStatus,
+ )
+from lp.soyuz.interfaces.archive import NAMED_AUTH_TOKEN_FEATURE_FLAG
from lp.soyuz.interfaces.livefs import LIVEFS_FEATURE_FLAG
from lp.soyuz.model.distributionsourcepackagecache import (
DistributionSourcePackageCache,
@@ -154,6 +162,7 @@ from lp.testing.layers import (
LaunchpadZopelessLayer,
ZopelessDatabaseLayer,
)
+from lp.testing.mail_helpers import pop_notifications
from lp.translations.model.pofile import POFile
from lp.translations.model.potmsgset import POTMsgSet
from lp.translations.model.translationtemplateitem import (
@@ -1739,6 +1748,238 @@ class TestGarbo(FakeAdapterMixin, TestCaseWithFactory):
# retained.
self._test_SnapFilePruner('foo.snap', None, 7, expected_count=1)
+ def test_ArchiveSubscriptionExpirer_expires_subscriptions(self):
+ # Archive subscriptions with expiry dates in the past have their
+ # statuses set to EXPIRED.
+ switch_dbuser('testadmin')
+ ppa = self.factory.makeArchive(private=True)
+ subs = [
+ ppa.newSubscription(self.factory.makePerson(), ppa.owner)
+ for _ in range(2)]
+ now = datetime.now(UTC)
+ subs[0].date_expires = now - timedelta(minutes=3)
+ self.assertEqual(ArchiveSubscriberStatus.CURRENT, subs[0].status)
+ subs[1].date_expires = now + timedelta(minutes=3)
+ self.assertEqual(ArchiveSubscriberStatus.CURRENT, subs[1].status)
+ Store.of(subs[0]).flush()
+
+ self.runFrequently()
+
+ switch_dbuser('testadmin')
+ self.assertEqual(ArchiveSubscriberStatus.EXPIRED, subs[0].status)
+ self.assertEqual(ArchiveSubscriberStatus.CURRENT, subs[1].status)
+
+ def test_ArchiveAuthTokenDeactivator_ignores_named_tokens(self):
+ switch_dbuser('testadmin')
+ self.useFixture(FeatureFixture({NAMED_AUTH_TOKEN_FEATURE_FLAG: 'on'}))
+ ppa = self.factory.makeArchive(private=True)
+ named_tokens = [
+ ppa.newNamedAuthToken(self.factory.getUniqueUnicode())
+ for _ in range(2)]
+ named_tokens[1].deactivate()
+
+ self.runHourly()
+
+ switch_dbuser('testadmin')
+ self.assertIsNone(named_tokens[0].date_deactivated)
+
+ def test_ArchiveAuthTokenDeactivator_leave_subscribed_team(self):
+ # Somebody who leaves a subscribed team loses their token, but other
+ # team members keep theirs.
+ switch_dbuser('testadmin')
+ ppa = self.factory.makeArchive(private=True)
+ team = self.factory.makeTeam()
+ people = []
+ for _ in range(3):
+ person = self.factory.makePerson()
+ team.addMember(person, team.teamowner)
+ people.append(person)
+ ppa.newSubscription(team, ppa.owner)
+ tokens = {person: ppa.newAuthToken(person) for person in people}
+
+ self.runHourly()
+ switch_dbuser('testadmin')
+ for person in tokens:
+ self.assertIsNone(tokens[person].date_deactivated)
+
+ people[0].leave(team)
+ # Clear out emails generated when leaving a team.
+ pop_notifications()
+
+ self.runHourly()
+ switch_dbuser('testadmin')
+ self.assertIsNotNone(tokens[people[0]].date_deactivated)
+ del tokens[people[0]]
+ for person in tokens:
+ self.assertIsNone(tokens[person].date_deactivated)
+
+ # A cancellation email was sent.
+ self.assertEmailQueueLength(1)
+
+ def test_ArchiveAuthTokenDeactivator_leave_only_one_subscribed_team(self):
+ # Somebody who leaves a subscribed team retains their token if they
+ # are still subscribed via another team.
+ switch_dbuser('testadmin')
+ ppa = self.factory.makeArchive(private=True)
+ teams = [self.factory.makeTeam() for _ in range(2)]
+ people = []
+ for _ in range(3):
+ for team in teams:
+ person = self.factory.makePerson()
+ team.addMember(person, team.teamowner)
+ people.append(person)
+ parent_team = self.factory.makeTeam()
+ parent_team.addMember(
+ teams[1], parent_team.teamowner, force_team_add=True)
+ multiple_teams_person = self.factory.makePerson()
+ for team in teams:
+ team.addMember(multiple_teams_person, team.teamowner)
+ people.append(multiple_teams_person)
+ ppa.newSubscription(teams[0], ppa.owner)
+ ppa.newSubscription(parent_team, ppa.owner)
+ tokens = {person: ppa.newAuthToken(person) for person in people}
+
+ self.runHourly()
+ switch_dbuser('testadmin')
+ for person in tokens:
+ self.assertIsNone(tokens[person].date_deactivated)
+
+ multiple_teams_person.leave(teams[0])
+ # Clear out emails generated when leaving a team.
+ pop_notifications()
+
+ self.runHourly()
+ switch_dbuser('testadmin')
+ self.assertIsNone(tokens[multiple_teams_person].date_deactivated)
+ for person in tokens:
+ self.assertIsNone(tokens[person].date_deactivated)
+
+ # A cancellation email was not sent.
+ self.assertEmailQueueLength(0)
+
+ def test_ArchiveAuthTokenDeactivator_leave_indirect_subscription(self):
+ # Members of a team that leaves a subscribed parent team lose their
+ # tokens.
+ switch_dbuser('testadmin')
+ ppa = self.factory.makeArchive(private=True)
+ child_team = self.factory.makeTeam()
+ people = []
+ for _ in range(3):
+ person = self.factory.makePerson()
+ child_team.addMember(person, child_team.teamowner)
+ people.append(person)
+ parent_team = self.factory.makeTeam()
+ parent_team.addMember(
+ child_team, parent_team.teamowner, force_team_add=True)
+ directly_subscribed_person = self.factory.makePerson()
+ people.append(directly_subscribed_person)
+ ppa.newSubscription(parent_team, ppa.owner)
+ ppa.newSubscription(directly_subscribed_person, ppa.owner)
+ tokens = {person: ppa.newAuthToken(person) for person in people}
+
+ self.runHourly()
+ switch_dbuser('testadmin')
+ for person in tokens:
+ self.assertIsNone(tokens[person].date_deactivated)
+
+ # child_team now leaves parent_team, and all its members lose their
+ # tokens.
+ parent_team.setMembershipData(
+ child_team, TeamMembershipStatus.APPROVED, parent_team.teamowner)
+ parent_team.setMembershipData(
+ child_team, TeamMembershipStatus.DEACTIVATED,
+ parent_team.teamowner)
+ self.assertFalse(child_team.inTeam(parent_team))
+ self.runHourly()
+ switch_dbuser('testadmin')
+ for person in people[:3]:
+ self.assertIsNotNone(tokens[person].date_deactivated)
+
+ # directly_subscribed_person still has their token; they're not in
+ # any teams.
+ self.assertIsNone(tokens[directly_subscribed_person].date_deactivated)
+
+ def test_ArchiveAuthTokenDeactivator_cancellation_email(self):
+ # When a token is deactivated, its user gets an email, which
+ # contains the correct headers and body.
+ switch_dbuser('testadmin')
+ # Avoid hyphens in owner and archive names; while these work fine,
+ # MailWrapper may wrap at hyphens, which makes it inconvenient to
+ # write a precise test for the email body text.
+ owner = self.factory.makePerson(
+ name='someperson', displayname='Some Person')
+ ppa = self.factory.makeArchive(
+ owner=owner, name='someppa', private=True)
+ subscriber = self.factory.makePerson()
+ subscription = ppa.newSubscription(subscriber, owner)
+ token = ppa.newAuthToken(subscriber)
+ subscription.cancel(owner)
+ pop_notifications()
+
+ self.runHourly()
+
+ switch_dbuser('testadmin')
+ self.assertIsNotNone(token.date_deactivated)
+ [email] = pop_notifications()
+ self.assertThat(dict(email), ContainsDict({
+ 'From': AfterPreprocessing(
+ partial(re.sub, r'\n[\t ]', ' '),
+ Equals('%s <noreply@xxxxxxxxxxxxx>' % ppa.displayname)),
+ 'To': Equals(subscriber.preferredemail.email),
+ 'Subject': AfterPreprocessing(
+ partial(re.sub, r'\n[\t ]', ' '),
+ Equals('PPA access cancelled for %s' % ppa.displayname)),
+ 'Sender': Equals('bounces@xxxxxxxxxxxxx'),
+ }))
+ expected_body = dedent("""\
+ Hello {subscriber},
+
+ Launchpad: cancellation of archive access
+ -----------------------------------------
+
+ Your access to the private software archive "{archive}", which
+ is hosted by Launchpad, has been cancelled.
+
+ You will now no longer be able to download software from this
+ archive. If you think this cancellation is in error, you should
+ contact the owner of the archive to verify it.
+
+ You can contact the archive owner by visiting their Launchpad
+ page here:
+
+ <http://launchpad\\.test/~{archive_owner_name}>
+
+ If you have any concerns you can contact the Launchpad team by
+ emailing feedback@launchpad\\.net
+
+
+ Regards,
+ The Launchpad team
+ """).format(
+ subscriber=subscriber.display_name,
+ archive=ppa.displayname,
+ archive_owner_name=owner.name)
+ self.assertTextMatchesExpressionIgnoreWhitespace(
+ expected_body, email.get_payload())
+
+ def test_ArchiveAuthTokenDeactivator_suppressed_archive(self):
+ # When a token is deactivated for an archive with
+ # suppress_subscription_notifications set, no email is sent.
+ switch_dbuser('testadmin')
+ ppa = self.factory.makeArchive(
+ private=True, suppress_subscription_notifications=True)
+ subscriber = self.factory.makePerson()
+ subscription = ppa.newSubscription(subscriber, ppa.owner)
+ token = ppa.newAuthToken(subscriber)
+ subscription.cancel(ppa.owner)
+ pop_notifications()
+
+ self.runHourly()
+
+ switch_dbuser('testadmin')
+ self.assertIsNotNone(token.date_deactivated)
+ self.assertEmailQueueLength(0)
+
class TestGarboTasks(TestCaseWithFactory):
layer = LaunchpadZopelessLayer
