launchpad-reviewers team mailing list archive

[Ian Booth <ian.booth@xxxxxxxxxxxxx>]

Ian Booth has proposed merging lp:~wallyworld/launchpad/project-sharingpolicy-garbo-job2 into lp:launchpad.

Requested reviews:
  Curtis Hovey (sinzui)
Related bugs:
  Bug #1037886 in Launchpad itself: "Exisitng, non commercial projects need to have their bug and branch sharing policies set up"
  https://bugs.launchpad.net/launchpad/+bug/1037886

For more details, see:
https://code.launchpad.net/~wallyworld/launchpad/project-sharingpolicy-garbo-job2/+merge/120063

== Implementation ==

A garbo job to iterate over all non commercial, public projects which do not have bug or branch sharing policies set and set these to PUBLIC.

This is a second attempt branch. William pointed out a flaw in the first version - projects with private bugs = true, or projects with non public BVPs should be excluded from the migration.

I have aborted the ec2 land of the first version and will land this instead.

== Tests ==

Add a test for the new garbo job to test_garbo

== Lint ==

Checking for conflicts and issues in changed files.

Linting changed files:
  database/schema/security.cfg
  lib/lp/scripts/garbo.py
  lib/lp/scripts/tests/test_garbo.py
-- 
https://code.launchpad.net/~wallyworld/launchpad/project-sharingpolicy-garbo-job2/+merge/120063
Your team Launchpad code reviewers is subscribed to branch lp:launchpad.

=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg	2012-08-06 03:47:42 +0000
+++ database/schema/security.cfg	2012-08-17 07:21:20 +0000
@@ -2227,6 +2227,7 @@
 public.binarypackagerelease             = SELECT
 public.binarypackagepublishinghistory   = SELECT, UPDATE
 public.bug                              = SELECT, UPDATE
+public.branchvisibilitypolicy           = SELECT
 public.bugaffectsperson                 = SELECT
 public.bugattachment                    = SELECT, DELETE
 public.bugmessage                       = SELECT, UPDATE
@@ -2248,6 +2249,7 @@
 public.codeimportevent                  = SELECT, DELETE
 public.codeimporteventdata              = SELECT, DELETE
 public.codeimportresult                 = SELECT, DELETE
+public.commercialsubscription           = SELECT
 public.emailaddress                     = SELECT, UPDATE, DELETE
 public.hwsubmission                     = SELECT, UPDATE
 public.job                              = SELECT, INSERT, DELETE
@@ -2258,6 +2260,7 @@
 public.openidconsumerassociation        = SELECT, DELETE
 public.openidconsumernonce              = SELECT, DELETE
 public.person                           = SELECT, DELETE
+public.product                          = SELECT, UPDATE
 public.pofiletranslator                 = SELECT, INSERT, UPDATE, DELETE
 public.potranslation                    = SELECT, DELETE
 public.potmsgset                        = SELECT, DELETE

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2012-07-23 11:25:26 +0000
+++ lib/lp/scripts/garbo.py	2012-08-17 07:21:20 +0000
@@ -26,7 +26,14 @@
 import iso8601
 from psycopg2 import IntegrityError
 import pytz
-from storm.expr import In
+from storm.expr import (
+    And,
+    Exists,
+    In,
+    Not,
+    Select,
+    Update,
+    Or)
 from storm.locals import (
     Max,
     Min,
@@ -47,7 +54,9 @@
     BugWatchScheduler,
     MAX_SAMPLE_SIZE,
     )
+from lp.code.enums import BranchVisibilityRule
 from lp.code.interfaces.revision import IRevisionSet
+from lp.code.model.branchvisibilitypolicy import BranchVisibilityTeamPolicy
 from lp.code.model.codeimportevent import CodeImportEvent
 from lp.code.model.codeimportresult import CodeImportResult
 from lp.code.model.revision import (
@@ -55,7 +64,9 @@
     RevisionCache,
     )
 from lp.hardwaredb.model.hwdb import HWSubmission
+from lp.registry.model.commercialsubscription import CommercialSubscription
 from lp.registry.model.person import Person
+from lp.registry.model.product import Product
 from lp.services.config import config
 from lp.services.database import postgresql
 from lp.services.database.constants import UTC_NOW
@@ -990,6 +1001,59 @@
         transaction.commit()
 
 
+class PopulateProjectSharingPolicies(TunableLoop):
+    """Sets bug and branch sharing policies for non commercial projects."""
+
+    maximum_chunk_size = 5000
+
+    def __init__(self, log, abort_time=None):
+        super(PopulateProjectSharingPolicies, self).__init__(log, abort_time)
+        self.store = IMasterStore(Product)
+
+    def getProducts(self):
+        """ Load the products to process.
+
+        We only want products which:
+            - are non-commercial products which have neither bug nor
+              branch sharing policy set
+            - have private_bugs = false
+            - have no branch visibility policies other than public
+        """
+        return self.store.find(
+            Product.id,
+            Not(
+                Or(
+                    Exists(Select(1, tables=[CommercialSubscription],
+                        where=And(
+                            CommercialSubscription.product == Product.id,
+                            CommercialSubscription.date_expires > datetime.now(
+                            pytz.UTC)))),
+                    Product.private_bugs == True,
+                    Exists(Select(1, tables=[BranchVisibilityTeamPolicy],
+                        where=And(
+                            BranchVisibilityTeamPolicy.product == Product.id,
+                            BranchVisibilityTeamPolicy.rule !=
+                                BranchVisibilityRule.PUBLIC))),
+                )),
+            And(Product.bug_sharing_policy == None,
+                Product.branch_sharing_policy == None)).order_by(Product.id)
+
+    def isDone(self):
+        return self.getProducts().is_empty()
+
+    def __call__(self, chunk_size):
+        products_to_process = self.getProducts()[:chunk_size]
+        changes = {
+            Product.bug_sharing_policy: 1,
+            Product.branch_sharing_policy: 1
+        }
+        expr = Update(
+            changes,
+            where=Product.id.is_in(products_to_process))
+        self.store.execute(expr, noresult=True)
+        transaction.commit()
+
+
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None  # Script name for locking and database user. Override.
@@ -1243,6 +1307,7 @@
         UnusedSessionPruner,
         DuplicateSessionPruner,
         BugHeatUpdater,
+        PopulateProjectSharingPolicies,
         ]
     experimental_tunable_loops = []
 

=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2012-07-23 11:25:26 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2012-08-17 07:21:20 +0000
@@ -19,6 +19,7 @@
     In,
     Min,
     Not,
+    Or,
     SQL,
     )
 from storm.locals import (
@@ -43,7 +44,10 @@
     BranchFormat,
     RepositoryFormat,
     )
-from lp.code.enums import CodeImportResultStatus
+from lp.code.enums import (
+    BranchVisibilityRule,
+    CodeImportResultStatus,
+    )
 from lp.code.interfaces.codeimportevent import ICodeImportEventSet
 from lp.code.model.branchjob import (
     BranchJob,
@@ -51,7 +55,13 @@
     )
 from lp.code.model.codeimportevent import CodeImportEvent
 from lp.code.model.codeimportresult import CodeImportResult
+from lp.registry.enums import (
+    BranchSharingPolicy,
+    BugSharingPolicy,
+    )
 from lp.registry.interfaces.person import IPersonSet
+from lp.registry.interfaces.product import IProductSet
+from lp.registry.model.product import Product
 from lp.scripts.garbo import (
     AntiqueSessionPruner,
     BulkPruner,
@@ -102,7 +112,10 @@
     TestCase,
     TestCaseWithFactory,
     )
-from lp.testing.dbuser import switch_dbuser
+from lp.testing.dbuser import (
+    dbuser,
+    switch_dbuser,
+    )
 from lp.testing.layers import (
     DatabaseLayer,
     LaunchpadScriptLayer,
@@ -1016,6 +1029,60 @@
         self.runHourly()
         self.assertNotEqual(old_update, naked_bug.heat_last_updated)
 
+    def test_PopulateProjectSharingPolicies(self):
+        # Non commercial projects have their bug and branch sharing policies
+        # set.
+        with dbuser('testadmin'):
+            non_commercial_products = [
+                self.factory.makeProduct()
+                for i in range(10)]
+            commercial_project = self.factory.makeProduct()
+            self.factory.makeCommercialSubscription(commercial_project)
+            configured_project = self.factory.makeProduct(
+                bug_sharing_policy=BugSharingPolicy.PROPRIETARY)
+            private_project = self.factory.makeProduct(private_bugs=True)
+            project_with_bvp = self.factory.makeProduct()
+            project_with_bvp.setBranchVisibilityTeamPolicy(
+                None, BranchVisibilityRule.FORBIDDEN)
+
+
+        def get_non_migrated_products():
+            return IMasterStore(Product).find(
+                Product,
+                Or(
+                    Product.bug_sharing_policy == None,
+                    Product.branch_sharing_policy == None))
+
+        self.runHourly()
+
+        # Check only the expected projects have been migrated.
+        # landscape and launchpad are projects in the test database which have
+        # non public branch visibility policies so are also not migrated.
+        product_set = getUtility(IProductSet)
+        landscape = product_set.getByName('landscape')
+        launchpad = product_set.getByName('launchpad')
+        self.assertContentEqual(
+            [commercial_project, configured_project, private_project,
+             project_with_bvp, landscape, launchpad],
+            get_non_migrated_products())
+        # The non migrated projects still have their original policies.
+        self.assertIsNone(commercial_project.bug_sharing_policy)
+        self.assertIsNone(commercial_project.branch_sharing_policy)
+        self.assertIsNone(private_project.bug_sharing_policy)
+        self.assertIsNone(private_project.branch_sharing_policy)
+        self.assertIsNone(project_with_bvp.bug_sharing_policy)
+        self.assertIsNone(project_with_bvp.branch_sharing_policy)
+        self.assertIsNone(configured_project.branch_sharing_policy)
+        self.assertEquals(
+            BugSharingPolicy.PROPRIETARY,
+            configured_project.bug_sharing_policy)
+        # The migrated projects have the expected policies.
+        for product in non_commercial_products:
+            self.assertEqual(
+                BranchSharingPolicy.PUBLIC, product.branch_sharing_policy)
+            self.assertEqual(
+                BugSharingPolicy.PUBLIC, product.bug_sharing_policy)
+
 
 class TestGarboTasks(TestCaseWithFactory):
     layer = LaunchpadZopelessLayer