sts-sponsors team mailing list archive

[Adam Collard <mp+437666@xxxxxxxxxxxxxxxxxx>]

Adam Collard has proposed merging ~adam-collard/maas-ci/+git/maas-ci-internal:threaded-reviewable into ~maas-committers/maas-ci/+git/maas-ci-internal:main.

Commit message:
[job-reviewer] Use futures.ThreadPoolExecutor to maximise throughput



Requested reviews:
  MAAS Committers (maas-committers)

For more details, see:
https://code.launchpad.net/~adam-collard/maas-ci/+git/maas-ci-internal/+merge/437666
-- 
Your team MAAS Committers is requested to review the proposed merge of ~adam-collard/maas-ci/+git/maas-ci-internal:threaded-reviewable into ~maas-committers/maas-ci/+git/maas-ci-internal:main.

diff --git a/jobs/launchpad-ci/launchpad b/jobs/launchpad-ci/launchpad
index 2f66a8c..11e580f 100755
--- a/jobs/launchpad-ci/launchpad
+++ b/jobs/launchpad-ci/launchpad
@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 
 import argparse
+from concurrent import futures
 from dataclasses import dataclass
-from itertools import islice, chain
+from itertools import chain
 import json
 import logging
 import os
@@ -135,7 +136,24 @@ def generate_mergable_proposals(lp, git_repo):
         yield proposal
 
 
-def should_review(lp, proposal, repo_logger):
+def _get_mp(lp, git_repo_path, proposal_address, repo_logger):
+    git_repo = lp.git_repositories.getByPath(path=git_repo_path)
+    proposal_idx = {
+        mp["address"]: i for i, mp in enumerate(git_repo.landing_candidates.entries)
+    }
+    try:
+        return git_repo.landing_candidates[proposal_idx[proposal_address]]
+    except IndexError:
+        repo_logger.warning(f"Having to hydrate proposals for {git_repo.display_name}")
+        for proposal in git_repo.landing_candidates:
+            if proposal.address == proposal_address:
+                return proposal
+
+
+def should_review(args, git_repo_path, proposal_address, repo_logger):
+    lp = login_to_lp(args.credentials)
+    proposal = _get_mp(lp, git_repo_path, proposal_address, repo_logger)
+    repo_logger.debug(f"Considering {proposal.web_link}")
     hit_test_marker = False
     reviewed_commits = []
     for comment in proposal.all_comments:
@@ -167,14 +185,21 @@ def should_review(lp, proposal, repo_logger):
         return True
 
 
-def generate_reviewable_proposals(lp, git_repo, repo_logger):
+def generate_reviewable_proposals(args, git_repo, repo_logger):
     needs_review_proposals = list(git_repo.getMergeProposals(status="Needs review"))
     wip_proposals = list(git_repo.getMergeProposals(status="Work in progress"))
     shuffle(needs_review_proposals)
     shuffle(wip_proposals)
-    for proposal in chain(needs_review_proposals, wip_proposals):
-        repo_logger.debug(f"Considering {proposal.web_link}")
-        if should_review(lp, proposal, repo_logger):
+    ex = futures.ThreadPoolExecutor()
+    results = ex.map(
+        lambda mp: mp
+        if should_review(args, git_repo.unique_name, mp.address, repo_logger)
+        else None,
+        chain(needs_review_proposals, wip_proposals),
+    )
+
+    for proposal in results:
+        if proposal:
             repo_logger.debug(f"Testing {proposal.web_link} !")
             yield proposal
 
@@ -349,7 +374,7 @@ def generate_repos(jobs_cfg_dir):
             with yaml_file.open() as fh:
                 data = yaml.safe_load(fh)[0]
         except (yaml.YAMLError, IndexError):
-            data_logger.error(f"Unable to load job config")
+            data_logger.error("Unable to load job config")
             continue
         else:
             try:
@@ -376,7 +401,7 @@ def handle_reviewable_jobs(args, lp):
         if git_repo is None:
             repo_logger.error(f"Unable to load git repo at {repo.lp_path}")
             continue
-        for proposal in generate_reviewable_proposals(lp, git_repo, repo_logger):
+        for proposal in generate_reviewable_proposals(args, git_repo, repo_logger):
             job = get_job_info(proposal)
             del job["MP"]
             job["NAME"] = repo.name