launchpad-reviewers team mailing list archive

[Jürgen Gmach <mp+415782@xxxxxxxxxxxxxxxxxx>]

Jürgen Gmach has proposed merging ~jugmac00/launchpad:attach-build-output-to-revision-status-reports into launchpad:master.

Commit message:
Attach output of completed lpcraft builds

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~jugmac00/launchpad/+git/launchpad/+merge/415782
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~jugmac00/launchpad:attach-build-output-to-revision-status-reports into launchpad:master.

diff --git a/lib/lp/archiveuploader/ciupload.py b/lib/lp/archiveuploader/ciupload.py
new file mode 100644
index 0000000..9a8b9ef
--- /dev/null
+++ b/lib/lp/archiveuploader/ciupload.py
@@ -0,0 +1,96 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Process a CI upload."""
+
+__all__ = [
+    "CIUpload",
+    ]
+
+import json
+import os
+
+from zope.component import getUtility
+
+from lp.archiveuploader.utils import UploadError
+from lp.buildmaster.enums import BuildStatus
+from lp.code.enums import RevisionStatusResult
+from lp.code.interfaces.revisionstatus import IRevisionStatusReportSet
+
+
+class CIUpload:
+    """An upload from a pipeline of CI jobs."""
+
+    def __init__(self, upload_path, logger):
+        """Create a `CIUpload`.
+
+        :param upload_path: A directory containing files to upload.
+        :param logger: The logger to be used.
+        """
+        self.upload_path = upload_path
+        self.logger = logger
+
+    def process(self, build):
+        """Process this upload, loading it into the database."""
+        self.logger.debug("Beginning processing.")
+
+        jobs_path = os.path.join(self.upload_path, "jobs.json")
+        try:
+            with open(jobs_path) as jobs_file:
+                jobs = json.load(jobs_file)
+        except FileNotFoundError:
+            raise UploadError("Build did not run any jobs.")
+
+        # collect all artifacts
+        artifacts = {}
+        # we assume first level directories are job directories
+        job_directories = (
+            d.name for d in os.scandir(self.upload_path) if d.is_dir()
+        )
+        for job_directory in job_directories:
+            artifacts[job_directory] = []
+            for dirpath, _, filenames in os.walk(os.path.join(
+                self.upload_path, job_directory
+            )):
+                for filename in filenames:
+                    artifacts[job_directory].append(os.path.join(
+                        dirpath, filename
+                    ))
+
+        for job_name in jobs:
+            report = getUtility(IRevisionStatusReportSet).getByCIBuildAndTitle(
+                build, job_name)
+            if not report:
+                # the report should normally exist, since the build request
+                # logic will eventually create report rows for the jobs it
+                # expects to run, but for robustness it's a good idea to
+                # ensure its existence here
+                report = getUtility(IRevisionStatusReportSet).new(
+                    creator=build.git_repository.owner,
+                    title=job_name,
+                    git_repository=build.git_repository,
+                    commit_sha1=build.commit_sha1,
+                    ci_build=build,
+                )
+
+            # attach log file
+            log_file = os.path.join(self.upload_path, job_name + ".log")
+            with open(log_file, mode="rb") as f:
+                report.setLog(f.read())
+
+            # attach artifacts
+            for file_path in artifacts[job_name]:
+                with open(file_path, mode="rb") as f:
+                    report.attach(
+                        name=os.path.basename(file_path), data=f.read()
+                    )
+
+            # set status
+            report.update(
+                result=getattr(RevisionStatusResult, jobs[job_name]["result"])
+            )
+
+        self.logger.debug("Updating %s" % build.title)
+        build.updateStatus(BuildStatus.FULLYBUILT)
+
+        self.logger.debug("Finished upload.")
diff --git a/lib/lp/archiveuploader/tests/test_ciupload.py b/lib/lp/archiveuploader/tests/test_ciupload.py
new file mode 100644
index 0000000..d0f0a11
--- /dev/null
+++ b/lib/lp/archiveuploader/tests/test_ciupload.py
@@ -0,0 +1,135 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test uploads of CIBuilds."""
+
+import json
+import os
+
+from storm.store import Store
+from zope.component import getUtility
+
+from lp.archiveuploader.tests.test_uploadprocessor import (
+    TestUploadProcessorBase,
+    )
+from lp.archiveuploader.uploadprocessor import (
+    UploadHandler,
+    UploadStatusEnum,
+    )
+from lp.buildmaster.enums import BuildStatus
+from lp.code.interfaces.revisionstatus import IRevisionStatusReportSet
+from lp.services.osutils import write_file
+
+
+class TestCIUBuildUploads(TestUploadProcessorBase):
+    """End-to-end tests of CIBuild uploads."""
+
+    def setUp(self):
+        super().setUp()
+        self.switchToAdmin()
+        self.build = self.factory.makeCIBuild()
+        self.build.updateStatus(BuildStatus.UPLOADING)
+        Store.of(self.build).flush()
+        self.switchToUploader()
+        self.uploadprocessor = self.getUploadProcessor(
+            self.layer.txn, builds=True
+        )
+
+    def test_requires_CI_job(self):
+        """If no jobs run, no `jobs.json` will be created.
+
+        This results in an `UploadError` / rejected upload."""
+        handler = UploadHandler.forProcessor(
+            self.uploadprocessor,
+            self.incoming_folder,
+            "test",
+            self.build,
+        )
+
+        result = handler.processCIResult(self.log)
+
+        self.assertEqual(
+            UploadStatusEnum.REJECTED, result
+        )
+
+    def test_triggers_store_upload_for_ci(self):
+        # create "jobs.json"
+        path = os.path.join(self.incoming_folder, "test", "jobs.json")
+        content = {
+            'build:0':
+                {
+                    'log': 'test_file_hash',
+                    'result': 'SUCCEEDED',
+                }
+        }
+        write_file(path, json.dumps(content).encode("utf-8"))
+
+        # create log file
+        path = os.path.join(self.incoming_folder, "test", "build:0.log")
+        content = "some log content"
+        write_file(path, content.encode("utf-8"))
+
+        # create artifact
+        path = os.path.join(
+            self.incoming_folder, "test", "build:0", "ci.whl")
+        content = b"abc"
+        write_file(path, content)
+
+        revision_status_report = self.factory.makeRevisionStatusReport(
+            title="build:0",
+            ci_build=self.build,
+        )
+        Store.of(revision_status_report).flush()
+
+        handler = UploadHandler.forProcessor(
+            self.uploadprocessor,
+            self.incoming_folder,
+            "test",
+            self.build,
+        )
+
+        result = handler.processCIResult(self.log)
+
+        self.assertEqual(UploadStatusEnum.ACCEPTED, result)
+        self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
+
+    def test_creates_revision_status_report_if_not_present(self):
+        # create "jobs.json"
+        path = os.path.join(self.incoming_folder, "test", "jobs.json")
+        content = {
+            'build:0':
+                {
+                    'log': 'test_file_hash',
+                    'result': 'SUCCEEDED',
+                }
+        }
+        write_file(path, json.dumps(content).encode("utf-8"))
+
+        # create log file
+        path = os.path.join(self.incoming_folder, "test", "build:0.log")
+        content = "some log content"
+        write_file(path, content.encode("utf-8"))
+
+        # create artifact
+        path = os.path.join(
+            self.incoming_folder, "test", "build:0", "ci.whl")
+        content = b"abc"
+        write_file(path, content)
+
+        handler = UploadHandler.forProcessor(
+            self.uploadprocessor,
+            self.incoming_folder,
+            "test",
+            self.build,
+        )
+
+        result = handler.processCIResult(self.log)
+
+        self.assertEqual(
+            self.build,
+            getUtility(
+                IRevisionStatusReportSet
+            ).getByCIBuildAndTitle(self.build, "build:0").ci_build
+        )
+        self.assertEqual(UploadStatusEnum.ACCEPTED, result)
+        self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
diff --git a/lib/lp/archiveuploader/uploadprocessor.py b/lib/lp/archiveuploader/uploadprocessor.py
index b71d0aa..3b6820d 100644
--- a/lib/lp/archiveuploader/uploadprocessor.py
+++ b/lib/lp/archiveuploader/uploadprocessor.py
@@ -53,6 +53,7 @@ from zope.component import getUtility
 
 from lp.app.errors import NotFoundError
 from lp.archiveuploader.charmrecipeupload import CharmRecipeUpload
+from lp.archiveuploader.ciupload import CIUpload
 from lp.archiveuploader.livefsupload import LiveFSUpload
 from lp.archiveuploader.nascentupload import (
     EarlyReturnUploadError,
@@ -68,6 +69,7 @@ from lp.archiveuploader.utils import UploadError
 from lp.buildmaster.enums import BuildStatus
 from lp.buildmaster.interfaces.buildfarmjob import ISpecificBuildFarmJobSource
 from lp.charms.interfaces.charmrecipebuild import ICharmRecipeBuild
+from lp.code.interfaces.cibuild import ICIBuild
 from lp.code.interfaces.sourcepackagerecipebuild import (
     ISourcePackageRecipeBuild,
     )
@@ -584,6 +586,31 @@ class BuildUploadHandler(UploadHandler):
                 "Unable to find %s with id %d. Skipping." %
                 (job_type, job_id))
 
+    def processCIResult(self, logger=None):
+        """Process a CI result upload."""
+        assert ICIBuild.providedBy(self.build)
+        if logger is None:
+            logger = self.processor.log
+        try:
+            logger.info("Processing CI result upload %s" % self.upload_path)
+            CIUpload(self.upload_path, logger).process(self.build)
+
+            if self.processor.dry_run:
+                logger.info("Dry run, aborting transaction.")
+                self.processor.ztm.abort()
+            else:
+                logger.info(
+                    "Committing the transaction and any mails associated "
+                    "with this upload.")
+                self.processor.ztm.commit()
+            return UploadStatusEnum.ACCEPTED
+        except UploadError as e:
+            logger.error(str(e))
+            return UploadStatusEnum.REJECTED
+        except BaseException:
+            self.processor.ztm.abort()
+            raise
+
     def processLiveFS(self, logger=None):
         """Process a live filesystem upload."""
         assert ILiveFSBuild.providedBy(self.build)
@@ -727,6 +754,8 @@ class BuildUploadHandler(UploadHandler):
                 result = self.processOCIRecipe(logger)
             elif ICharmRecipeBuild.providedBy(self.build):
                 result = self.processCharmRecipe(logger)
+            elif ICIBuild.providedBy(self.build):
+                result = self.processCIResult(logger)
             else:
                 self.processor.log.debug("Build %s found" % self.build.id)
                 [changes_file] = self.locateChangesFiles()
diff --git a/lib/lp/code/interfaces/revisionstatus.py b/lib/lp/code/interfaces/revisionstatus.py
index 7db816b..90d4e02 100644
--- a/lib/lp/code/interfaces/revisionstatus.py
+++ b/lib/lp/code/interfaces/revisionstatus.py
@@ -231,6 +231,9 @@ class IRevisionStatusReportSet(Interface):
     def findByCommit(repository, commit_sha1):
         """Returns all `RevisionStatusReport` for a repository and commit."""
 
+    def getByCIBuildAndTitle(ci_build, title):
+        """Return the `RevisionStatusReport` for a given CI build and title."""
+
     def deleteForRepository(repository):
         """Delete all `RevisionStatusReport` for a repository."""
 
diff --git a/lib/lp/code/model/revisionstatus.py b/lib/lp/code/model/revisionstatus.py
index 624fa77..8ef5cba 100644
--- a/lib/lp/code/model/revisionstatus.py
+++ b/lib/lp/code/model/revisionstatus.py
@@ -170,6 +170,11 @@ class RevisionStatusReportSet:
                 RevisionStatusReport.date_created,
                 RevisionStatusReport.id)
 
+    def getByCIBuildAndTitle(self, ci_build, title):
+        """See `IRevisionStatusReportSet`."""
+        return IStore(RevisionStatusReport).find(
+            RevisionStatusReport, ci_build=ci_build, title=title).one()
+
     def deleteForRepository(self, repository):
         clauses = [
             RevisionStatusArtifact.report == RevisionStatusReport.id,