launchpad-reviewers team mailing list archive

["Thiago F. Pappacena" <pappacena@xxxxxxxxx>]

Thiago F. Pappacena has proposed merging ~pappacena/launchpad:gitrepo-garbo-status into launchpad:master with ~pappacena/launchpad:gitrepo-status as a prerequisite.

Commit message:
Garbage collection job to remove repositories being created for too long.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~pappacena/launchpad/+git/launchpad/+merge/386109
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~pappacena/launchpad:gitrepo-garbo-status into launchpad:master.

diff --git a/database/schema/security.cfg b/database/schema/security.cfg
index 93dc9f8..ba3c197 100644
--- a/database/schema/security.cfg
+++ b/database/schema/security.cfg
@@ -2431,8 +2431,13 @@ public.distroarchseriesfilter           = SELECT
 public.distroseries                     = SELECT, UPDATE
 public.emailaddress                     = SELECT, UPDATE, DELETE
 public.garbojobstate                    = SELECT, INSERT, UPDATE, DELETE
-public.gitjob                           = SELECT, DELETE
-public.gitrepository                    = SELECT, UPDATE
+public.gitactivity                      = SELECT, DELETE
+public.gitjob                           = SELECT, INSERT, DELETE
+public.gitref                           = SELECT, DELETE
+public.gitrepository                    = SELECT, UPDATE, DELETE
+public.gitrule                          = SELECT, DELETE
+public.gitrulegrant                     = SELECT, DELETE
+public.gitsubscription                  = SELECT, DELETE
 public.hwsubmission                     = SELECT, UPDATE
 public.job                              = SELECT, INSERT, DELETE
 public.latestpersonsourcepackagereleasecache = SELECT, INSERT, UPDATE
@@ -2471,6 +2476,7 @@ public.teammembership                   = SELECT, DELETE
 public.teamparticipation                = SELECT, DELETE
 public.translationmessage               = SELECT, DELETE
 public.translationtemplateitem          = SELECT, DELETE
+public.webhook                          = SELECT, DELETE
 public.webhookjob                       = SELECT, DELETE
 public.xref                             = SELECT, INSERT
 type=user
diff --git a/lib/lp/scripts/garbo.py b/lib/lp/scripts/garbo.py
index cf0bc4f..75fa093 100644
--- a/lib/lp/scripts/garbo.py
+++ b/lib/lp/scripts/garbo.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Database garbage collection."""
@@ -58,6 +58,7 @@ from lp.bugs.scripts.checkwatches.scheduler import (
     BugWatchScheduler,
     MAX_SAMPLE_SIZE,
     )
+from lp.code.enums import GitRepositoryStatus
 from lp.code.interfaces.revision import IRevisionSet
 from lp.code.model.codeimportevent import CodeImportEvent
 from lp.code.model.codeimportresult import CodeImportResult
@@ -65,6 +66,7 @@ from lp.code.model.diff import (
     Diff,
     PreviewDiff,
     )
+from lp.code.model.gitrepository import GitRepository
 from lp.code.model.revision import (
     RevisionAuthor,
     RevisionCache,
@@ -1583,6 +1585,33 @@ class OCIFilePruner(BulkPruner):
         """
 
 
+class GitRepositoryPruner(TunableLoop):
+    """Remove GitRepositories that are "CREATING" for far too long."""
+
+    maximum_chunk_size = 500
+    repository_creation_timeout = timedelta(minutes=30)
+
+    def __init__(self, log, abort_time=None):
+        super(GitRepositoryPruner, self).__init__(log, abort_time)
+        self.store = IMasterStore(GitRepository)
+
+    def findRepositories(self):
+        min_date = datetime.now(pytz.UTC) - self.repository_creation_timeout
+        repositories = self.store.find(
+            GitRepository,
+            GitRepository.status == GitRepositoryStatus.CREATING,
+            GitRepository.date_created < min_date)
+        return repositories.order_by(GitRepository.date_created)
+
+    def isDone(self):
+        return self.findRepositories().is_empty()
+
+    def __call__(self, chunk_size):
+        for repository in self.findRepositories()[:chunk_size]:
+            repository.destroySelf()
+        transaction.commit()
+
+
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None  # Script name for locking and database user. Override.
@@ -1812,6 +1841,7 @@ class FrequentDatabaseGarbageCollector(BaseDatabaseGarbageCollector):
         AntiqueSessionPruner,
         BugSummaryJournalRollup,
         BugWatchScheduler,
+        GitRepositoryPruner,
         OpenIDConsumerAssociationPruner,
         OpenIDConsumerNoncePruner,
         PopulateDistributionSourcePackageCache,
diff --git a/lib/lp/scripts/tests/test_garbo.py b/lib/lp/scripts/tests/test_garbo.py
index b74d0a8..1540fa1 100644
--- a/lib/lp/scripts/tests/test_garbo.py
+++ b/lib/lp/scripts/tests/test_garbo.py
@@ -56,7 +56,10 @@ from lp.code.bzr import (
     BranchFormat,
     RepositoryFormat,
     )
-from lp.code.enums import CodeImportResultStatus
+from lp.code.enums import (
+    CodeImportResultStatus,
+    GitRepositoryStatus,
+    )
 from lp.code.interfaces.codeimportevent import ICodeImportEventSet
 from lp.code.interfaces.gitrepository import IGitRepositorySet
 from lp.code.model.branchjob import (
@@ -70,6 +73,7 @@ from lp.code.model.gitjob import (
     GitJob,
     GitRefScanJob,
     )
+from lp.code.model.gitrepository import GitRepository
 from lp.oci.interfaces.ocirecipe import OCI_RECIPE_ALLOW_CREATE
 from lp.oci.model.ocirecipebuild import OCIFile
 from lp.registry.enums import (
@@ -1107,6 +1111,46 @@ class TestGarbo(FakeAdapterMixin, TestCaseWithFactory):
         switch_dbuser('testadmin')
         self.assertEqual(1, store.find(OCIFile).count())
 
+    def test_GitRepositoryPruner_removes_staled_creations(self):
+        # Garbo removes GitRepository with status = CREATING for too long.
+        switch_dbuser('testadmin')
+        store = IMasterStore(GitRepository)
+        now = datetime.now(UTC)
+        recently = now - timedelta(minutes=1)
+        long_ago = now - timedelta(minutes=40)
+
+        # Creating a bunch of old staled repositories to be deleted,
+        # to make sure the chunk size is beign respected.
+        for i in range(5):
+            repo = removeSecurityProxy(self.factory.makeGitRepository())
+            repo.date_created = long_ago
+            repo.status = GitRepositoryStatus.CREATING
+            long_ago += timedelta(seconds=1)
+
+        recent_creating, old_available, recent_available = [
+            removeSecurityProxy(self.factory.makeGitRepository())
+            for _ in range(3)]
+
+        recent_creating.date_created = recently
+        recent_creating.status = GitRepositoryStatus.CREATING
+
+        old_available.date_created = long_ago
+        old_available.status = GitRepositoryStatus.AVAILABLE
+
+        recent_available.date_created = recently
+        recent_available.status = GitRepositoryStatus.AVAILABLE
+
+        self.assertEqual(8, store.find(GitRepository).count())
+
+        self.runFrequently(maximum_chunk_size=2)
+
+        switch_dbuser('testadmin')
+        remaining_repos = store.find(GitRepository)
+        self.assertEqual(3, remaining_repos.count())
+        self.assertEqual(
+            {old_available, recent_available, recent_creating},
+            set(remaining_repos))
+
     def test_WebhookJobPruner(self):
         # Garbo should remove jobs completed over 30 days ago.
         switch_dbuser('testadmin')