launchpad-reviewers team mailing list archive

[İlker Emre Koç <mp+493288@xxxxxxxxxxxxxxxxxx>]

İlker Emre Koç has proposed merging ~ilkeremrekoc/launchpad:exportvulnerability-modify into launchpad:master.

Commit message:
Add ExportVulnerabilityJob to slow queue

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~ilkeremrekoc/launchpad/+git/launchpad/+merge/493288

Added ExportVulnerabilityJob to slow queue with modified soft time
limit and lease time limit.

Also added a set of logs to be able to follow the progress of the job
during its run.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~ilkeremrekoc/launchpad:exportvulnerability-modify into launchpad:master.

diff --git a/lib/lp/bugs/model/exportvulnerabilityjob.py b/lib/lp/bugs/model/exportvulnerabilityjob.py
index 1e45aac..254b6e6 100644
--- a/lib/lp/bugs/model/exportvulnerabilityjob.py
+++ b/lib/lp/bugs/model/exportvulnerabilityjob.py
@@ -45,6 +45,7 @@ from lp.services.librarian.interfaces import ILibraryFileAliasSet
 from lp.services.utils import utc_now
 
 EXPIRATION_WEEK_INTERVAL = 4
+PROCESSED_CVE_LOG_INTERVAL = 500
 logger = logging.getLogger(__name__)
 
 HANDLER_EXPORTER_MAP = {VulnerabilityHandlerEnum.SOSS: SOSSExporter}
@@ -57,6 +58,13 @@ class ExportVulnerabilityJob(VulnerabilityJobDerived):
 
     user_error_types = (VulnerabilityJobException,)
 
+    # This is a known slow task that will exceed the timeouts for
+    # the normal job queue, so put it on a queue with longer timeouts
+    task_queue = "launchpad_job_slow"
+
+    soft_time_limit = timedelta(minutes=120)
+    lease_duration = timedelta(minutes=120)
+
     config = config.IExportVulnerabilityJobSource
 
     @property
@@ -231,6 +239,9 @@ class ExportVulnerabilityJob(VulnerabilityJobDerived):
             "export and storing them in Records"
         )
 
+        # Log progress every PROCESSED_CVE_LOG_INTERVAL CVEs
+        num_parsed_cves = 0
+
         exported_cves: List[Tuple[SOSSRecord, str]] = []
         for vul in distribution_vulnerability_set:
             cve: CveModel = vul.cve
@@ -260,11 +271,26 @@ class ExportVulnerabilityJob(VulnerabilityJobDerived):
 
             exported_cves.append((record, record.candidate))
 
+            if num_parsed_cves % PROCESSED_CVE_LOG_INTERVAL == 0:
+                logger.debug(
+                    f"[ExportVulnerabilityJob] Processed "
+                    f"{num_parsed_cves} CVEs so far..."
+                )
+            num_parsed_cves += 1
+
+        logger.debug(
+            f"[ExportVulnerabilityJob] Finished processing CVEs. "
+            f"Processed {num_parsed_cves} CVEs in total."
+        )
+
         if exported_cves == []:
             exception = VulnerabilityJobException("No CVEs to export")
             self.notifyUserError(exception)
             raise exception
 
+        # Log progress every PROCESSED_CVE_LOG_INTERVAL CVEs
+        num_written_cves = 0
+
         # Create a temporary folder
         with tempfile.TemporaryDirectory() as temp_dir:
 
@@ -304,6 +330,18 @@ class ExportVulnerabilityJob(VulnerabilityJobDerived):
                                 filename
                             )
 
+                        if num_written_cves % PROCESSED_CVE_LOG_INTERVAL == 0:
+                            logger.debug(
+                                f"[ExportVulnerabilityJob] Wrote "
+                                f"{num_written_cves} CVEs to zip so far..."
+                            )
+                        num_written_cves += 1
+
+        logger.debug(
+            f"[ExportVulnerabilityJob] Finished writing CVEs to zip. "
+            f"Wrote {num_written_cves} CVEs in total."
+        )
+
         zip_name = f"exported_vulnerabilities_{self.context.id}.zip"
         contentType = "application/zip"
         file_size = buf.getbuffer().nbytes