launchpad-reviewers team mailing list archive

[Ioana Lasc <mp+414551@xxxxxxxxxxxxxxxxxx>]

Ioana Lasc has proposed merging ~ilasc/launchpad:status-reports-garbo-job into launchpad:master.

Commit message:
Add garbo job for old status reports

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~ilasc/launchpad/+git/launchpad/+merge/414551
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~ilasc/launchpad:status-reports-garbo-job into launchpad:master.

diff --git a/lib/lp/code/model/revisionstatus.py b/lib/lp/code/model/revisionstatus.py
index de9bfff..03713c8 100644
--- a/lib/lp/code/model/revisionstatus.py
+++ b/lib/lp/code/model/revisionstatus.py
@@ -2,6 +2,7 @@
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __all__ = [
+    'RevisionStatusArtifact',
     'RevisionStatusReport',
     ]
 
diff --git a/lib/lp/scripts/garbo.py b/lib/lp/scripts/garbo.py
index 5eabeb6..c33e7d6 100644
--- a/lib/lp/scripts/garbo.py
+++ b/lib/lp/scripts/garbo.py
@@ -73,6 +73,10 @@ from lp.code.model.revision import (
     RevisionAuthor,
     RevisionCache,
     )
+from lp.code.model.revisionstatus import (
+    RevisionStatusArtifact,
+    RevisionStatusReport,
+    )
 from lp.oci.model.ocirecipebuild import OCIFile
 from lp.registry.interfaces.person import IPersonSet
 from lp.registry.model.person import Person
@@ -1762,6 +1766,52 @@ class PopulateSnapBuildStoreRevision(TunableLoop):
         transaction.commit()
 
 
+class RevisionStatusReportPruner(TunableLoop):
+    """Removes old revision status reports and their artifacts."""
+    minimum_chunk_size = 5
+    maximum_chunk_size = 100
+
+    def isDone(self):
+        """We are done when there are no old reports to delete."""
+        epoch = datetime.now(pytz.UTC) - timedelta(days=30)
+        store = IMasterStore(RevisionStatusReport)
+        results = store.find(
+            RevisionStatusReport, RevisionStatusReport.date_created < epoch)
+        return results.count() == 0
+
+    def __call__(self, chunk_size):
+        # Storm doesn't handle remove a limited result set,
+        # we limit to the chunk_size of the garbo job with the subquery
+        epoch = datetime.now(pytz.UTC) - timedelta(days=30)
+        subquery = Select(
+            [RevisionStatusReport.id, ],
+            where=[RevisionStatusReport.date_created < epoch, ],
+            limit=chunk_size)
+        clauses = [
+            RevisionStatusArtifact.report == RevisionStatusReport.id,
+            RevisionStatusReport.id.is_in(subquery), ]
+        where = convert_storm_clause_to_string(And(*clauses))
+
+        # Delete all artifacts for reports in the current batch
+        result = IMasterStore(RevisionStatusArtifact).execute("""
+            DELETE FROM RevisionStatusArtifact
+            USING RevisionStatusReport
+            WHERE """ + where)
+        self.log.info(
+            "Deleted %s status artifacts." % result.rowcount)
+
+        # Now delete reports in current batch
+        clauses = [RevisionStatusReport.id.is_in(subquery), ]
+        where = convert_storm_clause_to_string(*clauses)
+        result = IMasterStore(RevisionStatusArtifact).execute("""
+            DELETE FROM RevisionStatusReport WHERE """ + where)
+
+        self.log.info(
+            "Deleted %s status reports." % result.rowcount)
+
+        transaction.commit()
+
+
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None  # Script name for locking and database user. Override.
@@ -2058,6 +2108,7 @@ class DailyDatabaseGarbageCollector(BaseDatabaseGarbageCollector):
         PreviewDiffPruner,
         ProductVCSPopulator,
         RevisionAuthorEmailLinker,
+        RevisionStatusReportPruner,
         ScrubPOFileTranslator,
         SnapBuildJobPruner,
         SnapFilePruner,
diff --git a/lib/lp/scripts/tests/test_garbo.py b/lib/lp/scripts/tests/test_garbo.py
index 36c64e6..1ddc474 100644
--- a/lib/lp/scripts/tests/test_garbo.py
+++ b/lib/lp/scripts/tests/test_garbo.py
@@ -62,6 +62,10 @@ from lp.code.enums import (
     )
 from lp.code.interfaces.codeimportevent import ICodeImportEventSet
 from lp.code.interfaces.gitrepository import IGitRepositorySet
+from lp.code.interfaces.revisionstatus import (
+    IRevisionStatusArtifactSet,
+    IRevisionStatusReportSet,
+    )
 from lp.code.model.branchjob import (
     BranchJob,
     BranchUpgradeJob,
@@ -2045,6 +2049,57 @@ class TestGarbo(FakeAdapterMixin, TestCaseWithFactory):
         switch_dbuser('testadmin')
         self.assertEqual(build1._store_upload_revision, 1)
 
+    def test_RevisionStatusReportPruner(self):
+        # report1 and its artifacts are older than 30 days
+        # and we expect the gabo job to delete them.
+        # report2 and its artifact are newer then 30 days and
+        # we expect them to survive the garbo job.
+        switch_dbuser('testadmin')
+        now = datetime.now(UTC)
+        report1 = removeSecurityProxy(self.factory.makeRevisionStatusReport())
+        report2 = self.factory.makeRevisionStatusReport()
+        report1.date_created = now - timedelta(days=60)
+        repo1 = report1.git_repository
+        repo2 = report2.git_repository
+        artifact1_1 = self.factory.makeRevisionStatusArtifact(report=report1)
+        artifact1_2 = self.factory.makeRevisionStatusArtifact(report=report1)
+        for i in range(0, 5):
+            self.factory.makeRevisionStatusArtifact(report=report2)
+        reports_in_db = list(getUtility(
+                IRevisionStatusReportSet).findByRepository(repo1))
+        self.assertEqual(1, len(reports_in_db))
+        artifacts_db = list(getUtility(
+            IRevisionStatusArtifactSet).findByReport(report1))
+        self.assertEqual(2, len(artifacts_db))
+        self.assertTrue(artifact1_1, artifact1_2 in artifacts_db)
+
+        self.runDaily()
+
+        log = self.log_buffer.getvalue()
+        self.assertIn(
+            '[RevisionStatusReportPruner] '
+            'Deleted 2 status artifacts.',
+            log)
+        self.assertIn(
+            '[RevisionStatusReportPruner] Deleted 1 status reports.',
+            log)
+        artifacts_db = list(getUtility(
+            IRevisionStatusArtifactSet).findByReport(report1))
+        self.assertEqual(0, len(artifacts_db))
+        self.assertEqual(
+            0,
+            len(list(getUtility(
+                IRevisionStatusReportSet).findByRepository(repo1))))
+
+        # assert report2 and its artifacts remained intact
+        artifacts_db = list(getUtility(
+            IRevisionStatusArtifactSet).findByReport(report2))
+        self.assertEqual(5, len(artifacts_db))
+        self.assertEqual(
+            1,
+            len(list(getUtility(
+                IRevisionStatusReportSet).findByRepository(repo2))))
+
 
 class TestGarboTasks(TestCaseWithFactory):
     layer = LaunchpadZopelessLayer