launchpad-reviewers team mailing list archive

[Colin Watson <mp+425099@xxxxxxxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/launchpad:ci-build-upload-job-retry into launchpad:master.

Commit message:
Retry CIBuildUploadJobs on librarian server errors

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/425099

These can happen due to network problems or similar, so let's be a bit more resilient.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:ci-build-upload-job-retry into launchpad:master.

diff --git a/lib/lp/soyuz/model/archivejob.py b/lib/lp/soyuz/model/archivejob.py
index 9f2e131..e3948d8 100644
--- a/lib/lp/soyuz/model/archivejob.py
+++ b/lib/lp/soyuz/model/archivejob.py
@@ -39,6 +39,7 @@ from lp.services.job.model.job import (
     Job,
     )
 from lp.services.job.runner import BaseRunnableJob
+from lp.services.librarian.interfaces.client import LibrarianServerError
 from lp.services.librarian.utils import copy_and_close
 from lp.soyuz.enums import (
     ArchiveJobType,
@@ -200,6 +201,9 @@ class CIBuildUploadJob(ArchiveJobDerived):
 
     class_job_type = ArchiveJobType.CI_BUILD_UPLOAD
 
+    retry_error_types = (LibrarianServerError,)
+    max_retries = 3
+
     config = config.ICIBuildUploadJobSource
 
     # XXX cjwatson 2022-06-10: There doesn't seem to be a very clear
diff --git a/lib/lp/soyuz/tests/test_archivejob.py b/lib/lp/soyuz/tests/test_archivejob.py
index 5c07954..a0c39b4 100644
--- a/lib/lp/soyuz/tests/test_archivejob.py
+++ b/lib/lp/soyuz/tests/test_archivejob.py
@@ -4,6 +4,7 @@
 import os.path
 
 from debian.deb822 import Changes
+from fixtures import MockPatch
 from testtools.matchers import (
     Equals,
     Is,
@@ -23,6 +24,7 @@ from lp.services.job.tests import (
     block_on_job,
     pop_remote_notifications,
     )
+from lp.services.librarian.interfaces.client import LibrarianServerError
 from lp.services.mail.sendmail import format_address_for_person
 from lp.soyuz.enums import (
     ArchiveJobType,
@@ -551,6 +553,40 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
                 channel=Equals(channel))
             for channel in ("edge", "0.0.1/edge"))))
 
+    def test_librarian_server_error_retries(self):
+        # A run that gets an error from the librarian server schedules
+        # itself to be retried.
+        archive = self.factory.makeArchive()
+        distroseries = self.factory.makeDistroSeries(
+            distribution=archive.distribution)
+        das = self.factory.makeDistroArchSeries(distroseries=distroseries)
+        build = self.factory.makeCIBuild(distro_arch_series=das)
+        report = build.getOrCreateRevisionStatusReport("build:0")
+        path = "wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl"
+        with open(datadir(path), mode="rb") as f:
+            report.attach(name=os.path.basename(path), data=f.read())
+        job = CIBuildUploadJob.create(
+            build, build.git_repository.owner, archive, distroseries,
+            PackagePublishingPocket.RELEASE, target_channel="0.0.1/edge")
+        transaction.commit()
+
+        def fake_open(*args):
+            raise LibrarianServerError
+
+        with dbuser(job.config.dbuser):
+            with MockPatch(
+                "lp.services.librarian.model.LibraryFileAlias.open", fake_open
+            ):
+                JobRunner([job]).runAll()
+
+        self.assertEqual(JobStatus.WAITING, job.job.status)
+
+        # Try again.  The job is retried, and this time it succeeds.
+        with dbuser(job.config.dbuser):
+            JobRunner([job]).runAll()
+
+        self.assertEqual(JobStatus.COMPLETED, job.job.status)
+
 
 class TestViaCelery(TestCaseWithFactory):