sts-sponsors team mailing list archive

[Adam Collard <mp+437574@xxxxxxxxxxxxxxxxxx>]

Diff comments:

> diff --git a/jobs/launchpad-ci/job-reviewer.groovy b/jobs/launchpad-ci/job-reviewer.groovy
> index e7c5971..b58a81f 100644
> --- a/jobs/launchpad-ci/job-reviewer.groovy
> +++ b/jobs/launchpad-ci/job-reviewer.groovy
> @@ -20,44 +17,54 @@ def createBuildContextForJob(job) {
>  }
>  
>  pipeline {
> -    agent {
> -        label '{{ reviewer_agent_label }}'
> -    }
> +  agent {
> +    label 'reviewer'
> +  }
>  
> -    options {
> -        buildDiscarder(logRotator(numToKeepStr: '25'))
> -        disableConcurrentBuilds()
> -    }
> +  options {
> +    buildDiscarder(logRotator(numToKeepStr: '25'))
> +    disableConcurrentBuilds()
> +  }
>  
>  
> -    stages {
> -        stage('Prepare') {
> -            steps {
> -                cleanWs()
> -                {{ common.clone_ci_repo_step() }}
> -            }
> -        }
> +  stages {
> +    stage('Prepare') {
> +      steps {
> +        cleanWs()
> +	withCredentials([
> +	  file(credentialsId: 'lp-lander-credentials', variable: 'CREDS'),
> +	  sshUserPrivateKey(credentialsId: 'launchpad-ci-ssh-key', keyFileVariable: 'SSHKEY', usernameVariable: 'SSHUSER')
> +	]) {
> +	  sh '''
> +        rm -rf ci maas-ci-internal
> +        export GIT_SSH_COMMAND="ssh -i $SSHKEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
> +        git clone git+ssh://${SSHUSER}@git.launchpad.net/~maas-committers/maas-ci/+git/maas-ci-internal --branch main --depth 1
> +        mv maas-ci-internal/jobs/launchpad-ci ci
> +    '''

good catch, fixed

> +	}
> +      }
> +    }
>  
> -        stage('Unit Test Branches') {
> -            steps {
> -                withCredentials([
> -                    file(credentialsId: 'lp-lander-credentials', variable: 'CREDS'),
> -                    sshUserPrivateKey(credentialsId: 'launchpad-ci-ssh-key', keyFileVariable: 'SSHKEY', usernameVariable: 'SSHUSER')
> -                ]) {
> -                    script {
> -                        env.GIT_SSH_COMMAND = "ssh -i $SSHKEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
> -                        def output = run('ci/launchpad --credentials $CREDS jobs --limit 1 {{ repo_lp_path }}')
> -                        def jobs = parseJobs(output)
> -                        if(jobs.size() > 0) {
> -                            def builds = [:]
> -                            for(job in jobs) {
> -                                builds[job.LP_BRANCH_SRC] = createBuildContextForJob(job)
> -                            }
> -                            parallel builds
> -                        }
> -                    }
> -                }
> +    stage('Unit Test Branches') {
> +      steps {
> +        withCredentials([
> +          file(credentialsId: 'lp-lander-credentials', variable: 'CREDS'),
> +          sshUserPrivateKey(credentialsId: 'launchpad-ci-ssh-key', keyFileVariable: 'SSHKEY', usernameVariable: 'SSHUSER')
> +        ]) {
> +          script {
> +            env.GIT_SSH_COMMAND = "ssh -i $SSHKEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
> +            def output = run('ci/launchpad --credentials $CREDS reviewable-jobs maas-ci-internal/jobs/')
> +            def jobs = parseJobs(output)
> +            if(jobs.size() > 0) {
> +              def builds = [:]
> +              for(job in jobs) {
> +                builds[job.LP_BRANCH_SRC] = createBuildContextForJob(job)
> +              }
> +              parallel builds
>              }
> +          }
>          }
> -    } 
> +      }
> +    }
> +  }
>  }
> diff --git a/jobs/launchpad-ci/launchpad b/jobs/launchpad-ci/launchpad
> index 84a5a69..0cdbba4 100755
> --- a/jobs/launchpad-ci/launchpad
> +++ b/jobs/launchpad-ci/launchpad
> @@ -323,13 +326,78 @@ def find_target_milestone(project, now):
>          return latest_before
>  
>  
> -def handle_jobs(args, lp):
> -    repo = get_repository(lp, args.repo_lp_path)
> -    generator = (
> -        generate_mergable_proposals if args.mergable else generate_reviewable_proposals
> +def _get_launchpad_ci_files(jobs_cfg_dir: Path):
> +    launchpad_ci_files = []
> +    grep = subprocess.run(

Done

> +        ["grep", "-l", "--", "-launchpad-ci"] + list(jobs_cfg_dir.glob("*.yaml")),
> +        text=True,
> +        check=True,
> +        capture_output=True,
>      )
> +    for line in grep.stdout.splitlines():
> +        launchpad_ci_files.append(Path(line))
> +    return launchpad_ci_files
> +
> +
> +@dataclass
> +class Repo:
> +    name: str
> +    series: str
> +    lp_path: str
> +
> +
> +def generate_repos(jobs_cfg_dir):
> +    yaml_files = _get_launchpad_ci_files(jobs_cfg_dir)
> +    shuffle(yaml_files)
> +    for yaml_file in yaml_files:
> +
> +        data_logger = logging.getLogger(yaml_file.name)
> +        try:
> +            with yaml_file.open() as fh:
> +                data = yaml.safe_load(fh)[0]
> +        except (yaml.YAMLError, IndexError):
> +            data_logger.error(f"Unable to load job config")
> +            continue
> +        else:
> +            try:
> +                project = data["project"]
> +                name = project["name"]
> +                lp_path = project["repo_lp_path"]
> +                series = project.get("ubuntu_series")
> +            except KeyError:
> +                data_logger.error(
> +                    "...doesn't look like a launchpad-ci config, skipping"
> +                )
> +                continue
> +            else:
> +                repo = Repo(name, lp_path=lp_path, series=series)
> +                data_logger.debug(f"Found {repo}")
> +                yield repo, data_logger
> +
> +
> +def handle_reviewable_jobs(args, lp):
> +    """Check for MPs that can be reviewed."""
> +    job_list = []
> +    for repo, repo_logger in generate_repos(args.jobs_cfg_dir):
> +        git_repo = get_repository(lp, repo.lp_path)
> +        if git_repo is None:
> +            repo_logger.error(f"Unable to load git repo at {repo.lp_path}")
> +            continue
> +        for proposal in generate_reviewable_proposals(lp, git_repo, repo_logger):
> +            job = get_job_info(proposal)
> +            del job["MP"]
> +            job["NAME"] = repo.name
> +            job["SERIES"] = repo.series
> +            job_list.append(job)
> +    print(json.dumps(job_list))
> +    return 0
> +
> +
> +def handle_mergable_jobs(args, lp):
> +    """Check for MPs that can be merged."""
> +    repo = get_repository(lp, args.repo_lp_path)
>      job_list = []
> -    for proposal in islice(generator(lp, repo), args.limit):
> +    for proposal in islice(generate_mergable_proposals(lp, repo), args.limit):
>          job = get_job_info(proposal)
>          del job["MP"]
>          job_list.append(job)


-- 
https://code.launchpad.net/~adam-collard/maas-ci/+git/maas-ci-internal/+merge/437574
Your team MAAS Committers is subscribed to branch ~maas-committers/maas-ci/+git/maas-ci-internal:main.