sts-sponsors team mailing list archive
-
sts-sponsors team
-
Mailing list archive
-
Message #05260
[Merge] ~adam-collard/maas-ci/+git/maas-ci-internal:unify-lander into ~maas-committers/maas-ci/+git/maas-ci-internal:main
Adam Collard has proposed merging ~adam-collard/maas-ci/+git/maas-ci-internal:unify-lander into ~maas-committers/maas-ci/+git/maas-ci-internal:main.
Commit message:
[launchpad-ci] Unify all the reviewers
Replace {name}-reviewer with a single reviewer job that looks across
all the projects setup for launchpad-ci.
Requested reviews:
MAAS Lander (maas-lander): unittests
MAAS Committers (maas-committers)
For more details, see:
https://code.launchpad.net/~adam-collard/maas-ci/+git/maas-ci-internal/+merge/437574
Recursive diff of output XML
https://paste.ubuntu.com/p/94vgYPXXDf/
--
Your team MAAS Committers is requested to review the proposed merge of ~adam-collard/maas-ci/+git/maas-ci-internal:unify-lander into ~maas-committers/maas-ci/+git/maas-ci-internal:main.
diff --git a/jobs/launchpad-ci/job-lander.groovy b/jobs/launchpad-ci/job-lander.groovy
index c5c01e8..67829ce 100644
--- a/jobs/launchpad-ci/job-lander.groovy
+++ b/jobs/launchpad-ci/job-lander.groovy
@@ -65,7 +65,7 @@ pipeline {
]) {
script {
env.GIT_SSH_COMMAND = "ssh -i $SSHKEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
- def output = run('ci/launchpad --credentials $CREDS jobs --mergable --limit 1 {{ repo_lp_path }}')
+ def output = run('ci/launchpad --credentials $CREDS mergeable-jobs --limit 1 {{ repo_lp_path }}')
def jobs = parseJobs(output)
if(jobs.size() > 0) {
def builds = [:]
diff --git a/jobs/launchpad-ci/job-reviewer.groovy b/jobs/launchpad-ci/job-reviewer.groovy
index e7c5971..54ff611 100644
--- a/jobs/launchpad-ci/job-reviewer.groovy
+++ b/jobs/launchpad-ci/job-reviewer.groovy
@@ -1,12 +1,9 @@
-{% import 'launchpad-ci.groovy' as common %}
-{% include 'launchpad-ci-utils.groovy' %}
-
def createBuildContextForJob(job) {
ret_build = {
def local_job = job
- def series = '{{ ubuntu_series | default }}'
+ def series = local_job.SERIES
sh "ci/launchpad --credentials $CREDS mark-mp --start-review --repo-src $local_job.LP_REPO_SRC --branch-src $local_job.LP_BRANCH_SRC --repo-dest $local_job.LP_REPO_DEST --branch-dest $local_job.LP_BRANCH_DEST"
- def test_build = makeTestBuild('{{ name }}', series, local_job)
+ def test_build = makeTestBuild($local_job.NAME, series, local_job)
def test_result = test_build.getResult()
def test_build_url = test_build.getAbsoluteUrl()
if (test_result != 'SUCCESS') {
@@ -20,44 +17,54 @@ def createBuildContextForJob(job) {
}
pipeline {
- agent {
- label '{{ reviewer_agent_label }}'
- }
+ agent {
+ label 'reviewer'
+ }
- options {
- buildDiscarder(logRotator(numToKeepStr: '25'))
- disableConcurrentBuilds()
- }
+ options {
+ buildDiscarder(logRotator(numToKeepStr: '25'))
+ disableConcurrentBuilds()
+ }
- stages {
- stage('Prepare') {
- steps {
- cleanWs()
- {{ common.clone_ci_repo_step() }}
- }
- }
+ stages {
+ stage('Prepare') {
+ steps {
+ cleanWs()
+ withCredentials([
+ file(credentialsId: 'lp-lander-credentials', variable: 'CREDS'),
+ sshUserPrivateKey(credentialsId: 'launchpad-ci-ssh-key', keyFileVariable: 'SSHKEY', usernameVariable: 'SSHUSER')
+ ]) {
+ sh '''
+ rm -rf ci maas-ci-internal
+ export GIT_SSH_COMMAND="ssh -i $SSHKEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+ git clone git+ssh://${SSHUSER}@git.launchpad.net/~maas-committers/maas-ci/+git/maas-ci-internal --branch main --depth 1
+ mv maas-ci-internal/jobs/launchpad-ci ci
+ '''
+ }
+ }
+ }
- stage('Unit Test Branches') {
- steps {
- withCredentials([
- file(credentialsId: 'lp-lander-credentials', variable: 'CREDS'),
- sshUserPrivateKey(credentialsId: 'launchpad-ci-ssh-key', keyFileVariable: 'SSHKEY', usernameVariable: 'SSHUSER')
- ]) {
- script {
- env.GIT_SSH_COMMAND = "ssh -i $SSHKEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
- def output = run('ci/launchpad --credentials $CREDS jobs --limit 1 {{ repo_lp_path }}')
- def jobs = parseJobs(output)
- if(jobs.size() > 0) {
- def builds = [:]
- for(job in jobs) {
- builds[job.LP_BRANCH_SRC] = createBuildContextForJob(job)
- }
- parallel builds
- }
- }
- }
+ stage('Unit Test Branches') {
+ steps {
+ withCredentials([
+ file(credentialsId: 'lp-lander-credentials', variable: 'CREDS'),
+ sshUserPrivateKey(credentialsId: 'launchpad-ci-ssh-key', keyFileVariable: 'SSHKEY', usernameVariable: 'SSHUSER')
+ ]) {
+ script {
+ env.GIT_SSH_COMMAND = "ssh -i $SSHKEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+ def output = run('ci/launchpad --credentials $CREDS reviewable-jobs maas-ci-internal/jobs/')
+ def jobs = parseJobs(output)
+ if(jobs.size() > 0) {
+ def builds = [:]
+ for(job in jobs) {
+ builds[job.LP_BRANCH_SRC] = createBuildContextForJob(job)
+ }
+ parallel builds
}
+ }
}
- }
+ }
+ }
+ }
}
diff --git a/jobs/launchpad-ci/launchpad b/jobs/launchpad-ci/launchpad
index 84a5a69..0cdbba4 100755
--- a/jobs/launchpad-ci/launchpad
+++ b/jobs/launchpad-ci/launchpad
@@ -1,10 +1,12 @@
#!/usr/bin/env python3
import argparse
+from dataclasses import dataclass
from itertools import islice, chain
import json
import logging
import os
+from pathlib import Path
from random import shuffle
import re
import subprocess
@@ -12,6 +14,7 @@ import sys
import tempfile
from datetime import datetime
+import yaml
from launchpadlib.launchpad import Launchpad
logging.basicConfig(level=logging.DEBUG)
@@ -157,15 +160,15 @@ def generate_mergable_proposals(lp, git_repo):
yield proposal
-def generate_reviewable_proposals(lp, git_repo):
+def generate_reviewable_proposals(lp, git_repo, repo_logger):
needs_review_proposals = list(git_repo.getMergeProposals(status="Needs review"))
wip_proposals = list(git_repo.getMergeProposals(status="Work in progress"))
shuffle(needs_review_proposals)
shuffle(wip_proposals)
for proposal in chain(needs_review_proposals, wip_proposals):
- logger.debug(f"Considering {proposal.web_link}")
+ repo_logger.debug(f"Considering {proposal.web_link}")
if should_unconditionally_test(lp, proposal):
- logger.debug(f"Found marker on {proposal.web_link}; Testing!")
+ repo_logger.debug(f"Found marker on {proposal.web_link}; Testing!")
yield proposal
continue
@@ -173,16 +176,16 @@ def generate_reviewable_proposals(lp, git_repo):
latest_commit = get_latest_commit_sha1(proposal)
reviewed_commits = get_all_reviewed_commit_sha1(lp, proposal)
if latest_commit in reviewed_commits:
- logger.debug(
+ repo_logger.debug(
f"Skipping {proposal.web_link}: {latest_commit} has already been reviewed"
)
continue
# If WIP, see if this has been marked to be tested.
if proposal.queue_status == "Work in progress":
- logger.debug(f"Skipping {proposal.web_link}: WiP but no marker found")
+ repo_logger.debug(f"Skipping {proposal.web_link}: WiP but no marker found")
continue
- logger.debug(f"Testing {proposal.web_link} !")
+ repo_logger.debug(f"Testing {proposal.web_link} !")
yield proposal
@@ -323,13 +326,78 @@ def find_target_milestone(project, now):
return latest_before
-def handle_jobs(args, lp):
- repo = get_repository(lp, args.repo_lp_path)
- generator = (
- generate_mergable_proposals if args.mergable else generate_reviewable_proposals
+def _get_launchpad_ci_files(jobs_cfg_dir: Path):
+ launchpad_ci_files = []
+ grep = subprocess.run(
+ ["grep", "-l", "--", "-launchpad-ci"] + list(jobs_cfg_dir.glob("*.yaml")),
+ text=True,
+ check=True,
+ capture_output=True,
)
+ for line in grep.stdout.splitlines():
+ launchpad_ci_files.append(Path(line))
+ return launchpad_ci_files
+
+
+@dataclass
+class Repo:
+ name: str
+ series: str
+ lp_path: str
+
+
+def generate_repos(jobs_cfg_dir):
+ yaml_files = _get_launchpad_ci_files(jobs_cfg_dir)
+ shuffle(yaml_files)
+ for yaml_file in yaml_files:
+
+ data_logger = logging.getLogger(yaml_file.name)
+ try:
+ with yaml_file.open() as fh:
+ data = yaml.safe_load(fh)[0]
+ except (yaml.YAMLError, IndexError):
+ data_logger.error(f"Unable to load job config")
+ continue
+ else:
+ try:
+ project = data["project"]
+ name = project["name"]
+ lp_path = project["repo_lp_path"]
+ series = project.get("ubuntu_series")
+ except KeyError:
+ data_logger.error(
+ "...doesn't look like a launchpad-ci config, skipping"
+ )
+ continue
+ else:
+ repo = Repo(name, lp_path=lp_path, series=series)
+ data_logger.debug(f"Found {repo}")
+ yield repo, data_logger
+
+
+def handle_reviewable_jobs(args, lp):
+ """Check for MPs that can be reviewed."""
+ job_list = []
+ for repo, repo_logger in generate_repos(args.jobs_cfg_dir):
+ git_repo = get_repository(lp, repo.lp_path)
+ if git_repo is None:
+ repo_logger.error(f"Unable to load git repo at {repo.lp_path}")
+ continue
+ for proposal in generate_reviewable_proposals(lp, git_repo, repo_logger):
+ job = get_job_info(proposal)
+ del job["MP"]
+ job["NAME"] = repo.name
+ job["SERIES"] = repo.series
+ job_list.append(job)
+ print(json.dumps(job_list))
+ return 0
+
+
+def handle_mergable_jobs(args, lp):
+ """Check for MPs that can be merged."""
+ repo = get_repository(lp, args.repo_lp_path)
job_list = []
- for proposal in islice(generator(lp, repo), args.limit):
+ for proposal in islice(generate_mergable_proposals(lp, repo), args.limit):
job = get_job_info(proposal)
del job["MP"]
job_list.append(job)
@@ -451,20 +519,22 @@ def main():
)
subcommands = parser.add_subparsers(help="sub-command help")
- jobs_parser = subcommands.add_parser("jobs")
- jobs_parser.add_argument(
- "repo_lp_path", help="The path to the repo, for which to get jobs for."
+ reviewable_jobs_parser = subcommands.add_parser("reviewable-jobs")
+ reviewable_jobs_parser.add_argument(
+ "jobs_cfg_dir",
+ type=Path,
+ help="Path to directory containing yaml launchpad-ci configs listing the repos to look for open MPs in.",
)
- jobs_parser.add_argument(
- "--mergable",
- action="store_true",
- default=False,
- help="Return jobs that are mergable.",
+ reviewable_jobs_parser.set_defaults(func=handle_reviewable_jobs)
+
+ mergeable_jobs_parser = subcommands.add_parser("mergeable-jobs")
+ mergeable_jobs_parser.add_argument(
+ "repo_lp_path", help="The path to the repo, for which to get jobs for."
)
- jobs_parser.add_argument(
+ mergeable_jobs_parser.add_argument(
"--limit", type=int, default=None, help="Maximum number of jobs to return."
)
- jobs_parser.set_defaults(func=handle_jobs)
+ mergeable_jobs_parser.set_defaults(func=handle_mergable_jobs)
mark_mp_parser = subcommands.add_parser("mark-mp")
mark_mp_parser.add_argument(
diff --git a/jobs/launchpad-ci/launchpad-ci-utils.groovy b/jobs/launchpad-ci/launchpad-ci-utils.groovy
index bd8d6b2..8808e3f 100644
--- a/jobs/launchpad-ci/launchpad-ci-utils.groovy
+++ b/jobs/launchpad-ci/launchpad-ci-utils.groovy
@@ -18,6 +18,8 @@ def parseJobs(jsonString) {
// This is only set in the reviewer job
job['LP_COMMIT_SHA1'] = parsed_jobs[i]['LP_COMMIT_SHA1']
job['LP_MP_LINK'] = parsed_jobs[i]['LP_MP_LINK']
+ job['SERIES'] = parsed_jobs[i]['SERIES']
+ job['NAME'] = parsed_jobs[i]['NAME']
jobs[i] = job
}
return jobs
@@ -33,7 +35,7 @@ def makeTestBuild(name, series, job) {
}
println("Running job for $job.LP_BRANCH_SRC into $job.LP_BRANCH_DEST on $series")
return build(
- job: "${name}-tester",
+ job: "$job.NAME-tester",
propagate: false,
parameters: [
[$class: 'StringParameterValue', name: 'LP_REPO_SRC', value: job.LP_REPO_SRC],
diff --git a/jobs/launchpad-ci/launchpad-ci.yaml b/jobs/launchpad-ci/launchpad-ci.yaml
index e14e14d..11a3835 100644
--- a/jobs/launchpad-ci/launchpad-ci.yaml
+++ b/jobs/launchpad-ci/launchpad-ci.yaml
@@ -2,9 +2,25 @@
name: '{name}-launchpad-ci'
jobs:
- '{name}-tester'
- - '{name}-reviewer'
+ - 'reviewer'
- '{name}-lander'
+- job:
+ name: 'reviewer'
+ description: |
+ Launchpad CI reviewer.
+
+ Maintained in lp:~maas-committers/maas-ci/+git/maas-ci-internal.
+ project-type: pipeline
+ # Can't use #!include-jinja2 on a job, just a job-template since
+ # a job doesn't have context variables to render the template
+ dsl: !include-raw:
+ - launchpad-ci-utils.groovy
+ - job-reviewer.groovy
+ triggers:
+ - timed: 'H/2 * * * *'
+
+
- job-template:
name: '{name}-tester'
description: |
@@ -46,21 +62,6 @@
- job-template:
- name: '{name}-reviewer'
- description: |
- Launchpad CI reviewer for {name}.
-
- Maintained in lp:~maas-committers/maas-ci/+git/maas-ci-internal.
- project-type: pipeline
- dsl: !include-jinja2: job-reviewer.groovy
- triggers:
- - timed: 'H/2 * * * *'
-
- # overridable parameters
- reviewer_agent_label: ''
-
-
-- job-template:
name: '{name}-lander'
description: |
Launchpad CI lander for {name}.
@@ -83,4 +84,3 @@
job-name:
- '{name}-lander'
- '{name}-tester'
- - '{name}-reviewer'
References
