launchpad-reviewers team mailing list archive

[Steve Kowalik <stevenk@xxxxxxxxxx>]

Steve Kowalik has proposed merging lp:~stevenk/launchpad/set-spph-packageupload into lp:launchpad.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)
Related bugs:
  Bug #885739 in Launchpad itself: "queue and override manipulations should have an audit trail"
  https://bugs.launchpad.net/launchpad/+bug/885739

For more details, see:
https://code.launchpad.net/~stevenk/launchpad/set-spph-packageupload/+merge/108513

Write a garbo job to try and set SourcePackagePublishingHistory.packageupload for existing publications. Since every publication isn't going to have a packageupload, we make sure to sort by id and use memcache to keep track of where we are up to. Since a publication may have multiple packageuploads related to a source (due to delayed copies or other close madness), we make sure the archive, distroseries, pocket, component and section all match.
-- 
https://code.launchpad.net/~stevenk/launchpad/set-spph-packageupload/+merge/108513
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~stevenk/launchpad/set-spph-packageupload into lp:launchpad.

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2012-05-24 05:43:49 +0000
+++ lib/lp/scripts/garbo.py	2012-06-04 01:22:22 +0000
@@ -98,6 +98,12 @@
     MAIN_STORE,
     MASTER_FLAVOR,
     )
+from lp.soyuz.model.publishing import SourcePackagePublishingHistory
+from lp.soyuz.model.queue import (
+    PackageUpload,
+    PackageUploadSource,
+    )
+from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
 from lp.translations.interfaces.potemplate import IPOTemplateSet
 from lp.translations.model.potmsgset import POTMsgSet
 from lp.translations.model.potranslation import POTranslation
@@ -1137,6 +1143,59 @@
         transaction.commit()
 
 
+class PopulateSourcePackagePublishingHistoryPackageUpload(TunableLoop):
+
+    maximum_chunk_size = 5000
+
+    def __init__(self, log, abort_time=None):
+        super(
+            PopulateSourcePackagePublishingHistoryPackageUpload,
+            self).__init__(log, abort_time)
+        self.store = IMasterStore(SourcePackagePublishingHistory)
+        self.memcache_key = '%s:populate-spph-pu' % config.instance_name
+        watermark = getUtility(IMemcacheClient).get(self.memcache_key)
+        self.start_at = watermark or 0
+
+    def findSPPHs(self):
+        return self.store.find(
+            SourcePackagePublishingHistory.id,
+            SourcePackagePublishingHistory.packageuploadID == None,
+            SourcePackagePublishingHistory.id >= self.start_at).order_by(
+                SourcePackagePublishingHistory.id)
+
+    def isDone(self):
+        return self.findSPPHs().is_empty()
+
+    def __call__(self, chunk_size):
+        ids = [spph_id for spph_id in self.findSPPHs()[:chunk_size]]
+        packageuploads = self.store.find(
+            (SourcePackagePublishingHistory, PackageUpload),
+            SourcePackagePublishingHistory.sourcepackagereleaseID ==
+                PackageUploadSource.sourcepackagereleaseID,
+            SourcePackagePublishingHistory.sourcepackagereleaseID ==
+                SourcePackageRelease.id,
+            PackageUploadSource.packageuploadID == PackageUpload.id,
+            SourcePackagePublishingHistory.archiveID ==
+                PackageUpload.archiveID,
+            SourcePackagePublishingHistory.distroseriesID ==
+                PackageUpload.distroseriesID,
+            SourcePackagePublishingHistory.pocket == PackageUpload.pocket,
+            SourcePackagePublishingHistory.componentID ==
+                SourcePackageRelease.componentID,
+            SourcePackagePublishingHistory.sectionID ==
+                SourcePackageRelease.sectionID,
+            SourcePackagePublishingHistory.id.is_in(ids))
+        for (spph, pu) in packageuploads:
+            if pu is not None:
+                spph.packageupload = pu
+        self.start_at = ids[-1] + 1
+        result = getUtility(IMemcacheClient).set(
+            self.memcache_key, self.start_at)
+        if not result:
+            self.log.warning('Failed to set start_at in memcache.')
+        transaction.commit()
+
+
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None  # Script name for locking and database user. Override.
@@ -1392,6 +1451,7 @@
         DuplicateSessionPruner,
         BugHeatUpdater,
         BugTaskFlattener,
+        PopulateSourcePackagePublishingHistoryPackageUpload,
         ]
     experimental_tunable_loops = []
 

=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2012-05-09 01:35:41 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2012-06-04 01:22:22 +0000
@@ -1149,6 +1149,28 @@
         self.runHourly()
         self.assertEqual((task.id,), get_flat())
 
+    def test_PopulateSPPHPU_no_pu(self):
+        with dbuser('testadmin'):
+            spph = self.factory.makeSourcePackagePublishingHistory()
+        self.runHourly()
+        with dbuser('testadmin'):
+            self.assertIs(None, spph.packageupload)
+
+    def test_PopulateSourcePackagePublishingHistoryPackageUpload(self):
+        with dbuser('testadmin'):
+            distroseries = self.factory.makeDistroSeries()
+            spph = self.factory.makeSourcePackagePublishingHistory(
+                distroseries=distroseries, archive=distroseries.main_archive,
+                pocket=self.factory.getAnyPocket())
+            pu = self.factory.makePackageUpload(
+                distroseries=distroseries, archive=distroseries.main_archive,
+                pocket=self.factory.getAnyPocket())
+            pu.addSource(spph.sourcepackagerelease)
+            self.assertIs(None, spph.packageupload)
+        self.runHourly()
+        with dbuser('testadmin'):
+            self.assertEqual(pu, spph.packageupload)
+
 
 class TestGarboTasks(TestCaseWithFactory):
     layer = LaunchpadZopelessLayer