launchpad-reviewers team mailing list archive

[Quentin Debhi <mp+473999@xxxxxxxxxxxxxxxxxx>]

Quentin Debhi has proposed merging ~ruinedyourlife/launchpad:add-build-upload-processing-for-craft-recipes into launchpad:master with ~ruinedyourlife/launchpad:add-build-behaviour-for-craft-recipes as a prerequisite.

Commit message:
Add build upload processing for craft recipes

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad/+git/launchpad/+merge/473999
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad:add-build-upload-processing-for-craft-recipes into launchpad:master.

diff --git a/lib/lp/archiveuploader/craftrecipeupload.py b/lib/lp/archiveuploader/craftrecipeupload.py
new file mode 100644
index 0000000..bb6e97f
--- /dev/null
+++ b/lib/lp/archiveuploader/craftrecipeupload.py
@@ -0,0 +1,68 @@
+# Copyright 2024 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Process a craft recipe upload."""
+
+__all__ = [
+    "CraftRecipeUpload",
+]
+
+import os
+from contextlib import closing
+
+from zope.component import getUtility
+
+from lp.archiveuploader.utils import UploadError
+from lp.buildmaster.enums import BuildStatus
+from lp.services.helpers import filenameToContentType
+from lp.services.librarian.interfaces import ILibraryFileAliasSet
+
+
+class CraftRecipeUpload:
+    """A craft recipe upload."""
+
+    def __init__(self, upload_path, logger):
+        """Create a `CraftRecipeUpload`.
+
+        :param upload_path: A directory containing files to upload.
+        :param logger: The logger to be used.
+        """
+        self.upload_path = upload_path
+        self.logger = logger
+
+        self.librarian = getUtility(ILibraryFileAliasSet)
+
+    def process(self, build):
+        """Process this upload, loading it into the database."""
+        self.logger.debug("Beginning processing.")
+
+        found_craft = False
+        craft_paths = []
+        for dirpath, _, filenames in os.walk(self.upload_path):
+            if dirpath == self.upload_path:
+                # All relevant files will be in a subdirectory.
+                continue
+            for craft_file in sorted(filenames):
+                if craft_file.endswith(".craft"):
+                    found_craft = True
+                craft_paths.append(os.path.join(dirpath, craft_file))
+
+        if not found_craft:
+            raise UploadError("Build did not produce any craft files.")
+
+        for craft_path in craft_paths:
+            with closing(open(craft_path, "rb")) as file:
+                libraryfile = self.librarian.create(
+                    os.path.basename(craft_path),
+                    os.stat(craft_path).st_size,
+                    file,
+                    filenameToContentType(craft_path),
+                    restricted=build.is_private,
+                )
+            build.addFile(libraryfile)
+
+        # The master verifies the status to confirm successful upload.
+        self.logger.debug("Updating %s" % build.title)
+        build.updateStatus(BuildStatus.FULLYBUILT)
+
+        self.logger.debug("Finished upload.")
diff --git a/lib/lp/archiveuploader/tests/test_craftrecipeupload.py b/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
new file mode 100644
index 0000000..d07093a
--- /dev/null
+++ b/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
@@ -0,0 +1,79 @@
+# Copyright 2024 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for `CraftRecipeUpload`."""
+
+import os
+
+from storm.store import Store
+
+from lp.archiveuploader.tests.test_uploadprocessor import (
+    TestUploadProcessorBase,
+)
+from lp.archiveuploader.uploadprocessor import UploadHandler, UploadStatusEnum
+from lp.buildmaster.enums import BuildStatus
+from lp.crafts.interfaces.craftrecipe import CRAFT_RECIPE_ALLOW_CREATE
+from lp.services.features.testing import FeatureFixture
+from lp.services.osutils import write_file
+
+
+class TestCraftRecipeUploads(TestUploadProcessorBase):
+    """End-to-end tests of craft recipe uploads."""
+
+    def setUp(self):
+        super().setUp()
+        self.useFixture(FeatureFixture({CRAFT_RECIPE_ALLOW_CREATE: "on"}))
+
+        self.setupBreezy()
+
+        self.switchToAdmin()
+        self.build = self.factory.makeCraftRecipeBuild(
+            distro_arch_series=self.breezy["i386"]
+        )
+        self.build.updateStatus(BuildStatus.UPLOADING)
+        Store.of(self.build).flush()
+        self.switchToUploader()
+        self.options.context = "buildd"
+
+        self.uploadprocessor = self.getUploadProcessor(
+            self.layer.txn, builds=True
+        )
+
+    def test_sets_build_and_state(self):
+        # The upload processor uploads files and sets the correct status.
+        self.assertFalse(self.build.verifySuccessfulUpload())
+        upload_dir = os.path.join(
+            self.incoming_folder, "test", str(self.build.id), "ubuntu"
+        )
+        write_file(os.path.join(upload_dir, "foo_0_all.craft"), b"craft")
+        write_file(os.path.join(upload_dir, "foo_0_all.manifest"), b"manifest")
+        handler = UploadHandler.forProcessor(
+            self.uploadprocessor, self.incoming_folder, "test", self.build
+        )
+        result = handler.processCraftRecipe(self.log)
+        self.assertEqual(
+            UploadStatusEnum.ACCEPTED,
+            result,
+            "Craft upload failed\nGot: %s" % self.log.getLogBuffer(),
+        )
+        self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
+        self.assertTrue(self.build.verifySuccessfulUpload())
+
+    def test_requires_craft(self):
+        # The upload processor fails if the upload does not contain any
+        # .craft files.
+        self.assertFalse(self.build.verifySuccessfulUpload())
+        upload_dir = os.path.join(
+            self.incoming_folder, "test", str(self.build.id), "ubuntu"
+        )
+        write_file(os.path.join(upload_dir, "foo_0_all.manifest"), b"manifest")
+        handler = UploadHandler.forProcessor(
+            self.uploadprocessor, self.incoming_folder, "test", self.build
+        )
+        result = handler.processCraftRecipe(self.log)
+        self.assertEqual(UploadStatusEnum.REJECTED, result)
+        self.assertIn(
+            "ERROR Build did not produce any craft files.",
+            self.log.getLogBuffer(),
+        )
+        self.assertFalse(self.build.verifySuccessfulUpload())
diff --git a/lib/lp/archiveuploader/uploadprocessor.py b/lib/lp/archiveuploader/uploadprocessor.py
index 1289241..72dec7b 100644
--- a/lib/lp/archiveuploader/uploadprocessor.py
+++ b/lib/lp/archiveuploader/uploadprocessor.py
@@ -54,6 +54,7 @@ from zope.component import getUtility
 from lp.app.errors import NotFoundError
 from lp.archiveuploader.charmrecipeupload import CharmRecipeUpload
 from lp.archiveuploader.ciupload import CIUpload
+from lp.archiveuploader.craftrecipeupload import CraftRecipeUpload
 from lp.archiveuploader.livefsupload import LiveFSUpload
 from lp.archiveuploader.nascentupload import (
     EarlyReturnUploadError,
@@ -74,6 +75,7 @@ from lp.code.interfaces.cibuild import ICIBuild
 from lp.code.interfaces.sourcepackagerecipebuild import (
     ISourcePackageRecipeBuild,
 )
+from lp.crafts.interfaces.craftrecipebuild import ICraftRecipeBuild
 from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild
 from lp.registry.interfaces.distribution import IDistributionSet
 from lp.registry.interfaces.person import IPersonSet
@@ -776,6 +778,32 @@ class BuildUploadHandler(UploadHandler):
             self.processor.ztm.abort()
             raise
 
+    def processCraftRecipe(self, logger=None):
+        """Process a craft recipe upload."""
+        assert ICraftRecipeBuild.providedBy(self.build)
+        if logger is None:
+            logger = self.processor.log
+        try:
+            logger.info("Processing craft upload %s" % self.upload_path)
+            CraftRecipeUpload(self.upload_path, logger).process(self.build)
+
+            if self.processor.dry_run:
+                logger.info("Dry run, aborting transaction.")
+                self.processor.ztm.abort()
+            else:
+                logger.info(
+                    "Committing the transaction and any mails associated "
+                    "with this upload."
+                )
+                self.processor.ztm.commit()
+            return UploadStatusEnum.ACCEPTED
+        except UploadError as e:
+            logger.error(str(e))
+            return UploadStatusEnum.REJECTED
+        except BaseException:
+            self.processor.ztm.abort()
+            raise
+
     def processRockRecipe(self, logger=None):
         """Process a rock recipe upload."""
         assert IRockRecipeBuild.providedBy(self.build)
@@ -859,6 +887,8 @@ class BuildUploadHandler(UploadHandler):
                 result = self.processCharmRecipe(logger)
             elif IRockRecipeBuild.providedBy(self.build):
                 result = self.processRockRecipe(logger)
+            elif ICraftRecipeBuild.providedBy(self.build):
+                result = self.processCraftRecipe(logger)
             elif ICIBuild.providedBy(self.build):
                 result = self.processCIResult(logger)
             else: