sts-sponsors team mailing list archive

[Jack Lloyd-Walters <mp+438683@xxxxxxxxxxxxxxxxxx>]

Jack Lloyd-Walters has proposed merging ~maas-committers/maas-ci/+git/maas-ci-config:Jenkins-autojob-ansible into ~maas-committers/maas-ci/+git/maas-ci-config:master.

Commit message:
[Systemstests] Automatically create ansible-playbook-tests jobs from GitHub PRs

Requested reviews:
  MAAS Committers (maas-committers)

For more details, see:
https://code.launchpad.net/~maas-committers/maas-ci/+git/maas-ci-config/+merge/438683
-- 
Your team MAAS Committers is requested to review the proposed merge of ~maas-committers/maas-ci/+git/maas-ci-config:Jenkins-autojob-ansible into ~maas-committers/maas-ci/+git/maas-ci-config:master.

diff --git a/jenkins/jobs/ansible_job_creator.groovy b/jenkins/jobs/ansible_job_creator.groovy
new file mode 100644
index 0000000..f60002f
--- /dev/null
+++ b/jenkins/jobs/ansible_job_creator.groovy
@@ -0,0 +1,100 @@
+import groovy.json.JsonSlurper
+
+@NonCPS
+def parseJobs(jsonString) {
+    def parsed_jobs = new JsonSlurper().parseText(jsonString)
+    def jobs = []
+
+    if parsed_jobs.size() > 0 {
+        for(parsed_job in parsed_jobs) {
+            def job = [:]
+
+            source_repo = parsed_job["head"]["repo"]["html_url"]
+            source_branch = parsed_job["head"]["ref"]
+            source_latest_commit = sh(script: "git ls-remote ${source_repo} ${source_branch} | awk \'{print \$1}\'", returnStdout: true).trim()
+
+            target_repo = parsed_job["base"]["repo"]["html_url"]
+            target_branch = parsed_job["base"]["ref"]
+
+            def pr_updated = Date.parse("yyyy-MM-dd'T'HH:mm:ss'Z'", parsed_job["updated_at"])
+
+            // Don't test PRs that aren't targeting the main branch
+            target_repo_name = parsed_job["base"]["repo"]["full_name"]
+            default_branch = parsed_job["base"]["repo"]["default_branch"]
+            correct_repo = false
+            if (target_branch==default_branch && target_repo_name=="maas/maas-ansible-playbook") {
+                correct_repo = true
+            }
+
+            // Don't test PRs that have the WIP label applied
+            parsed_pr_labels = parsed_job["labels"]
+            if (parsed_pr_labels.size() > 0) {
+                def wip = false
+                for (label in parsed_pr_labels) {
+                    if (label["name"]=="WIP") {
+                        wip = true
+                    }
+                }
+            }
+
+            // Don't test PRs that have already been tested (ie: they have a status applied to the commit)
+            def commit_status_out = sh(script: "curl -s https://api.github.com/repos/maas/maas-ansible-playbook/commits/${source_latest_commit}/statuses";, returnStdout: true).trim()
+            commit_status = new JsonSlurper().parseText(commit_status_out)
+            tested = false
+            if (commit_status.size() > 0) {
+                tested = true
+            }
+
+            // add this commit to be tested
+            if (correct_repo && !wip && !tested) {
+                job["updated"] = pr_updated
+                job["ANSIBLE_REPO"] = source_repo
+                job["ANSIBLE_BRANCH"] = source_branch
+                jobs.add(job)
+            }
+        }
+    }
+    return jobs
+}
+
+def makeBuild(job) {
+    println("Running job for ${job.ANSIBLE_REPO} ${job.ANSIBLE_BRANCH}")
+    return build(
+        job: "ansible-playbook-tests",
+        propogate: false,
+        parameters: [
+            [$class: 'StringParameterValue', name: 'ANSIBLE_REPO', value: job.ANSIBLE_REPO],
+            [$class: 'StringParameterValue', name: 'ANSIBLE_BRANCH', value: job.ANSIBLE_BRANCH],
+        ],
+    )
+}
+
+pipeline {
+    agent {
+        label 'ci-lab'
+    }
+    stages {
+        stage("Fetch Pull Requests") {
+            steps {
+                script {
+                    split_repo_url = "${params.MAIN_REPO}".split("/")
+                    repo_name = split_repo_url[split_repo_url.length-1]
+                    user_name = split_repo_url[split_repo_url.length-2]
+                    def output = sh(script: "curl -s https://api.github.com/repos/${user_name}/${repo_name}/pulls?state=open";, returnStdout: true).trim()
+                    env._pr_to_review = output
+                }
+            }
+        }
+        stage("Test PRs") {
+            when { not { environment name: "_pr_to_review", value: "[]" }}
+            steps {
+                script {
+                    def jobs = parseJobs(env._pr_to_review)
+                    // start with the least-recently updated job
+                    jobs.sort{a,b-> a.updated<=>b.updated}
+                    makeBuild(jobs[0])
+                }
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/jenkins/jobs/ansible_playbook.groovy b/jenkins/jobs/ansible_playbook.groovy
index 699eba9..8b94288 100644
--- a/jenkins/jobs/ansible_playbook.groovy
+++ b/jenkins/jobs/ansible_playbook.groovy
@@ -65,7 +65,7 @@ pipeline {
     post {
         always {
             archiveArtifacts artifacts:'system-tests/*.log,system-tests/config.yaml,system-tests/junit*.xml', fingerprint: true
-            junit allowEmptyResults: true, testResults: 'system-tests/junit*.xml'
+            def summary = junit allowEmptyResults: true, testResults: 'system-tests/junit*.xml'
             sh """
                 lxc delete ansible-main --force || true
                 lxc delete ansible-host-1 --force || true
@@ -74,11 +74,11 @@ pipeline {
         }
         success {
             mattermostSend (color: 'green', message: "[${env.JOB_NAME} #${env.BUILD_NUMBER}](${env.BUILD_URL}) [${env.ANSIBLE_BRANCH}](${params.ANSIBLE_REPO}/commit/${env.COMMIT_SHA}) :success: was successful")
-            updateCommitStatus("${params.MAIN_REPO}", "${env.COMMIT_SHA}", "SUCCESS", "[Test output](${env.BUILD_URL})")
+            updateCommitStatus("${params.MAIN_REPO}", "${env.COMMIT_SHA}", "SUCCESS", "Success")
         }
         failure {
             mattermostSend (color: 'red', message: "[${env.JOB_NAME} #${env.BUILD_NUMBER}](${env.BUILD_URL}) [${env.ANSIBLE_BRANCH}](${params.ANSIBLE_REPO}/commit/${env.COMMIT_SHA}) :fire: failed")
-            updateCommitStatus("${params.MAIN_REPO}", "${env.COMMIT_SHA}", "FAILURE", "[Test output](${env.BUILD_URL})")
+            updateCommitStatus("${params.MAIN_REPO}", "${env.COMMIT_SHA}", "FAILURE", "Failed ${summary.failCount}/${summary.totalCount-summary.skipCount}")
         }
     }
 }
diff --git a/jenkins/jobs/ansible_playbook.yaml b/jenkins/jobs/ansible_playbook.yaml
index 41c1ecb..9206bba 100644
--- a/jenkins/jobs/ansible_playbook.yaml
+++ b/jenkins/jobs/ansible_playbook.yaml
@@ -1,8 +1,23 @@
 - project:
     name: ansible-playbook-tests
     jobs:
+      - ansible-job-creator
       - ansible-playbook-tests
 
+- job-template::
+    name: ansible-job-creator
+    description: |
+      check GitHub for testable commits
+    project-type: pipeline
+    parameters:
+      - string:
+          name: MAIN_REPO
+          description: Git repo to merge Ansible playbook into
+          default: https://github.com/maas/maas-ansible-playbook
+    triggers:
+      - timed: "H * * * *"
+    dsl: !include-jinja2: ansible_job_creator.groovy
+
 - job-template:
     name: ansible-playbook-tests
     description: |