launchpad-reviewers team mailing list archive

[Steve Kowalik <stevenk@xxxxxxxxxx>]

Steve Kowalik has proposed merging lp:~stevenk/launchpad/populate-bprc into lp:launchpad.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)
Related bugs:
  Bug #796997 in Launchpad itself: "Contents generation should be done from the database"
  https://bugs.launchpad.net/launchpad/+bug/796997

For more details, see:
https://code.launchpad.net/~stevenk/launchpad/populate-bprc/+merge/69412

Following on from https://code.launchpad.net/~stevenk/launchpad/db-add-bprc/+merge/64783, this branch adds a garbo-hourly job to start population of the BinaryPackageReleaseContents and BinaryPackagePath tables.
-- 
https://code.launchpad.net/~stevenk/launchpad/populate-bprc/+merge/69412
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~stevenk/launchpad/populate-bprc into lp:launchpad.

=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg	2011-07-23 02:49:38 +0000
+++ database/schema/security.cfg	2011-07-27 09:13:19 +0000
@@ -118,6 +118,7 @@
 public.archivesubscriber                = SELECT, INSERT, UPDATE
 public.authtoken                        = SELECT, INSERT, UPDATE, DELETE
 public.binaryandsourcepackagenameview   = SELECT
+public.binarypackagefile                = SELECT, INSERT, DELETE
 public.binarypackagepath                = SELECT, INSERT, DELETE
 public.binarypackagepublishinghistory   = SELECT
 public.binarypackagereleasecontents     = SELECT, INSERT, DELETE
@@ -2143,6 +2144,10 @@
 
 [garbo]
 groups=script,read
+public.binarypackagefile                = SELECT, DELETE
+public.binarypackagepath                = SELECT, INSERT
+public.binarypackagerelease             = SELECT
+public.binarypackagereleasecontents     = INSERT
 public.branchjob                        = SELECT, DELETE
 public.bug                              = SELECT, UPDATE
 public.bugaffectsperson                 = SELECT
@@ -2166,6 +2171,7 @@
 public.codeimportresult                 = SELECT, DELETE
 public.emailaddress                     = SELECT, UPDATE
 public.hwsubmission                     = SELECT, UPDATE
+public.libraryfilealias                 = SELECT
 public.job                              = SELECT, INSERT, DELETE
 public.mailinglistsubscription          = SELECT, DELETE
 public.oauthnonce                       = SELECT, DELETE

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2011-07-05 05:46:02 +0000
+++ lib/lp/scripts/garbo.py	2011-07-27 09:13:19 +0000
@@ -25,7 +25,9 @@
 import multiprocessing
 from psycopg2 import IntegrityError
 import pytz
+from storm.expr import LeftJoin
 from storm.locals import (
+    And,
     Max,
     Min,
     SQL,
@@ -42,7 +44,10 @@
     sqlvalues,
     )
 from canonical.launchpad.database.emailaddress import EmailAddress
-from canonical.launchpad.database.librarian import TimeLimitedToken
+from canonical.launchpad.database.librarian import (
+    LibraryFileAlias,
+    TimeLimitedToken,
+    )
 from canonical.launchpad.database.oauth import OAuthNonce
 from canonical.launchpad.database.openidconsumer import OpenIDConsumerNonce
 from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus
@@ -74,12 +79,18 @@
 from lp.registry.model.person import Person
 from lp.services.job.model.job import Job
 from lp.services.log.logger import PrefixFilter
+from lp.services.memcache.interfaces import IMemcacheClient
 from lp.services.scripts.base import (
     LaunchpadCronScript,
     LOCK_PATH,
     SilentLaunchpadScriptFailure,
     )
 from lp.services.session.model import SessionData
+from lp.soyuz.interfaces.binarypackagereleasecontents import (
+    IBinaryPackageReleaseContentsSet,
+    )
+from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease
+from lp.soyuz.model.files import BinaryPackageFile
 from lp.translations.interfaces.potemplate import IPOTemplateSet
 from lp.translations.model.potranslation import POTranslation
 
@@ -859,6 +870,52 @@
         self.done = True
 
 
+class PopulateBinaryPackageReleaseContents(TunableLoop):
+    maximum_chunk_size = 1
+
+    def __init__(self, log, abort_time=None):
+        super(PopulateBinaryPackageReleaseContents, self).__init__(
+            log, abort_time)
+        value = getUtility(IMemcacheClient).get('populate-bprc')
+        if not value:
+            self.start_at = 0
+        else:
+            self.start_at = value
+        self.finish_at = self.getCandidateBPRs(0).last()
+
+    def getCandidateBPRs(self, start_at):
+        return IMasterStore(BinaryPackageRelease).using(
+            BinaryPackageRelease,
+            LeftJoin(
+                BinaryPackageFile,
+                BinaryPackageFile.binarypackagereleaseID ==
+                    BinaryPackageRelease.id),
+            LeftJoin(
+                LibraryFileAlias,
+                And(LibraryFileAlias.id == BinaryPackageFile.libraryfileID,
+                    LibraryFileAlias.content != None
+            ))).find(
+                BinaryPackageRelease.id,
+                BinaryPackageRelease.id >= start_at
+            ).order_by(BinaryPackageRelease.id)
+
+    def isDone(self):
+        return self.start_at > self.finish_at
+
+    def __call__(self, chunk_size):
+        for bprid in self.getCandidateBPRs(self.start_at)[:chunk_size]:
+            bpr = BinaryPackageRelease.get(bprid)
+            result = getUtility(IBinaryPackageReleaseContentsSet).add(bpr)
+            if result:
+                self.log.debug('BPR %d added successfully.' % bpr.id)
+            else:
+                self.log.debug('BPR %d unable to be added.' % bpr.id)
+        self.start_at = bpr.id + 1
+        result = getUtility(IMemcacheClient).set(
+            'populate-bprc', self.start_at)
+        transaction.commit()
+
+
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None # Script name for locking and database user. Override.
@@ -1086,6 +1143,7 @@
         UnusedSessionPruner,
         DuplicateSessionPruner,
         BugHeatUpdater,
+        PopulateBinaryPackageReleaseContents,
         ]
     experimental_tunable_loops = []
 

=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2011-07-05 05:46:02 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2011-07-27 09:13:19 +0000
@@ -55,6 +55,7 @@
     LaunchpadZopelessLayer,
     ZopelessDatabaseLayer,
     )
+from lp.archiveuploader.tests import datadir
 from lp.bugs.model.bugmessage import BugMessage
 from lp.bugs.model.bugnotification import (
     BugNotification,
@@ -91,6 +92,10 @@
     SessionData,
     SessionPkgData,
     )
+from lp.soyuz.model.binarypackagereleasecontents import (
+    BinaryPackageReleaseContents,
+    )
+from lp.soyuz.model.files import BinaryPackageFile
 from lp.testing import (
     TestCase,
     TestCaseWithFactory,
@@ -110,6 +115,11 @@
 
     def test_hourly_script(self):
         """Ensure garbo-hourly.py actually runs."""
+        # Our sampledata doesn't contain anythng that
+        # PopulateBinaryPackageReleaseContents can process without errors,
+        # so it's easier to just remove every BinaryPackageFile.
+        IMasterStore(BinaryPackageFile).find(BinaryPackageFile).remove()
+        transaction.commit()
         rv, out, err = run_script(
             "cronscripts/garbo-hourly.py", ["-q"], expect_returncode=0)
         self.failIf(out.strip(), "Output to stdout: %s" % out)
@@ -355,6 +365,12 @@
         self.log.addHandler(NullHandler())
         self.log.propagate = 0
 
+        # Our sampledata doesn't contain anythng that
+        # PopulateBinaryPackageReleaseContents can process without errors,
+        # so it's easier to just remove every BinaryPackageFile.
+        IMasterStore(BinaryPackageFile).find(BinaryPackageFile).remove()
+        transaction.commit()
+
         # Run the garbage collectors to remove any existing garbage,
         # starting us in a known state.
         self.runDaily()
@@ -897,3 +913,30 @@
         self.assertNotEqual(0, unmigrated())
         self.runHourly()
         self.assertEqual(0, unmigrated())
+
+    def test_populate_bprc(self):
+        LaunchpadZopelessLayer.switchDbUser('testadmin')
+        bpr = self.factory.makeBinaryPackageRelease()
+        deb = open(datadir('pmount_0.9.7-2ubuntu2_amd64.deb'), 'r')
+        lfa = self.factory.makeLibraryFileAlias(
+            filename='pmount_0.9.7-2ubuntu2_amd64.deb', content=deb.read())
+        deb.close()
+        transaction.commit()
+        bpr.addFile(lfa)
+        transaction.commit()
+        self.runHourly()
+        bprc = IMasterStore(BinaryPackageReleaseContents).find(
+            BinaryPackageReleaseContents)
+        self.assertEqual(13, bprc.count())
+        paths = map(lambda x: x.binarypackagepath.path, bprc)
+        expected_paths = [
+            'etc/pmount.allow', 'usr/bin/pumount', 'usr/bin/pmount-hal',
+            'usr/bin/pmount', 'usr/share/doc/pmount/TODO',
+            'usr/share/doc/pmount/README.Debian',
+            'usr/share/doc/pmount/AUTHORS', 'usr/share/doc/pmount/copyright',
+            'usr/share/doc/pmount/changelog.gz',
+            'usr/share/doc/pmount/changelog.Debian.gz',
+            'usr/share/man/man1/pmount-hal.1.gz',
+            'usr/share/man/man1/pmount.1.gz',
+            'usr/share/man/man1/pumount.1.gz']
+        self.assertContentEqual(expected_paths, paths)