launchpad-reviewers team mailing list archive

[Quentin Debhi <mp+477899@xxxxxxxxxxxxxxxxxx>]

Quentin Debhi has proposed merging ~ruinedyourlife/launchpad:craft-build-rust-scan into launchpad:master.

Commit message:
Scan crates inside rust build archives

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad/+git/launchpad/+merge/477899

This will find `.crate` files inside `sourcecraft` outputted archives and add it as the build output.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad:craft-build-rust-scan into launchpad:master.

diff --git a/lib/lp/archivepublisher/artifactory.py b/lib/lp/archivepublisher/artifactory.py
index 08eb83a..e435aa7 100644
--- a/lib/lp/archivepublisher/artifactory.py
+++ b/lib/lp/archivepublisher/artifactory.py
@@ -617,7 +617,7 @@ class ArtifactoryPool:
             ]
         elif repository_format == ArchiveRepositoryFormat.RUST:
             return [
-                "*.tar.xz",
+                "*.crate",
             ]
         elif repository_format == ArchiveRepositoryFormat.GENERIC:
             return ["*"]
diff --git a/lib/lp/archivepublisher/tests/test_artifactory.py b/lib/lp/archivepublisher/tests/test_artifactory.py
index 3fc2aa5..a5212b7 100644
--- a/lib/lp/archivepublisher/tests/test_artifactory.py
+++ b/lib/lp/archivepublisher/tests/test_artifactory.py
@@ -290,7 +290,7 @@ class TestArtifactoryPool(TestCase):
     def test_getArtifactPatterns_rust(self):
         pool = self.makePool()
         self.assertEqual(
-            ["*.tar.xz"],
+            ["*.crate"],
             pool.getArtifactPatterns(ArchiveRepositoryFormat.RUST),
         )
 
diff --git a/lib/lp/archiveuploader/craftrecipeupload.py b/lib/lp/archiveuploader/craftrecipeupload.py
index 5e9d285..a7ed62f 100644
--- a/lib/lp/archiveuploader/craftrecipeupload.py
+++ b/lib/lp/archiveuploader/craftrecipeupload.py
@@ -8,7 +8,10 @@ __all__ = [
 ]
 
 import os
+import tarfile
+import tempfile
 
+import yaml
 from zope.component import getUtility
 
 from lp.archiveuploader.utils import UploadError
@@ -45,6 +48,7 @@ class CraftRecipeUpload:
                 # All relevant files will be in a subdirectory.
                 continue
             for craft_file in sorted(filenames):
+                # Look for .tar.xz archives (which might contain .crate files)
                 if craft_file.endswith(".tar.xz"):
                     found_craft = True
                 craft_paths.append(os.path.join(dirpath, craft_file))
@@ -53,18 +57,52 @@ class CraftRecipeUpload:
             raise UploadError("Build did not produce any craft files.")
 
         for craft_path in craft_paths:
-            with open(craft_path, "rb") as file:
-                libraryfile = self.librarian.create(
-                    os.path.basename(craft_path),
-                    os.stat(craft_path).st_size,
-                    file,
-                    filenameToContentType(craft_path),
-                    restricted=build.is_private,
-                )
-            build.addFile(libraryfile)
+            # Extract and process .crate files from archive
+            self._process_rust_archive(build, craft_path)
 
         # The master verifies the status to confirm successful upload.
         self.logger.debug("Updating %s" % build.title)
         build.updateStatus(BuildStatus.FULLYBUILT)
-
         self.logger.debug("Finished upload.")
+
+    def _process_rust_archive(self, build, archive_path):
+        """Process a .tar.xz archive that may contain .crate files."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with tarfile.open(archive_path, "r:xz") as tar:
+                tar.extractall(path=tmpdir)
+
+            # Read metadata.yaml for crate info
+            # XXX: ruinedyourlife 2024-12-06
+            # We will need this later to give the crate a name and version to
+            # artifactory. This is a placeholder for now, which will be changed
+            # when we find a way to send that information to artifactory.
+            metadata_path = os.path.join(tmpdir, "metadata.yaml")
+            if os.path.exists(metadata_path):
+                with open(metadata_path) as f:
+                    try:
+                        metadata = yaml.safe_load(f)
+                        _ = metadata.get("name")
+                        _ = metadata.get("version")
+                    except Exception as e:
+                        self.logger.warning(
+                            "Failed to parse metadata.yaml: %s", e
+                        )
+
+            # Look for .crate files in extracted contents
+            for root, _, files in os.walk(tmpdir):
+                for filename in files:
+                    if filename.endswith(".crate"):
+                        crate_path = os.path.join(root, filename)
+                        self._process_crate_file(build, crate_path)
+
+    def _process_crate_file(self, build, crate_path):
+        """Process a single .crate file."""
+        with open(crate_path, "rb") as file:
+            libraryfile = self.librarian.create(
+                os.path.basename(crate_path),
+                os.stat(crate_path).st_size,
+                file,
+                filenameToContentType(crate_path),
+                restricted=build.is_private,
+            )
+        build.addFile(libraryfile)
diff --git a/lib/lp/archiveuploader/tests/test_craftrecipeupload.py b/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
index 5f7f737..c137c78 100644
--- a/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
+++ b/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
@@ -4,8 +4,12 @@
 """Tests for `CraftRecipeUpload`."""
 
 import os
+import tarfile
+import tempfile
 
+import yaml
 from storm.store import Store
+from zope.security.proxy import removeSecurityProxy
 
 from lp.archiveuploader.tests.test_uploadprocessor import (
     TestUploadProcessorBase,
@@ -39,16 +43,45 @@ class TestCraftRecipeUploads(TestUploadProcessorBase):
             self.layer.txn, builds=True
         )
 
+    def _createArchiveWithCrate(
+        self, upload_dir, crate_name="test-crate", crate_version="0.1.0"
+    ):
+        """Helper to create a tar.xz archive containing a crate & metadata."""
+        # Create a temporary directory to build our archive
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # Create metadata.yaml
+            metadata = {
+                "name": crate_name,
+                "version": crate_version,
+            }
+            metadata_path = os.path.join(tmpdir, "metadata.yaml")
+            with open(metadata_path, "w") as f:
+                yaml.safe_dump(metadata, f)
+
+            # Create dummy crate file
+            crate_path = os.path.join(
+                tmpdir, f"{crate_name}-{crate_version}.crate"
+            )
+            with open(crate_path, "wb") as f:
+                f.write(b"dummy crate contents")
+
+            # Create tar.xz archive
+            archive_path = os.path.join(upload_dir, "output.tar.xz")
+            with tarfile.open(archive_path, "w:xz") as tar:
+                tar.add(metadata_path, arcname="metadata.yaml")
+                tar.add(crate_path, arcname=os.path.basename(crate_path))
+
+            return archive_path
+
     def test_sets_build_and_state(self):
         # The upload processor uploads files and sets the correct status.
         self.assertFalse(self.build.verifySuccessfulUpload())
         upload_dir = os.path.join(
             self.incoming_folder, "test", str(self.build.id), "ubuntu"
         )
-        write_file(
-            os.path.join(upload_dir, "foo_0_all.craft.tar.xz"), b"craft"
-        )
-        write_file(os.path.join(upload_dir, "foo_0_all.manifest"), b"manifest")
+        os.makedirs(upload_dir, exist_ok=True)
+        self._createArchiveWithCrate(upload_dir)
+
         handler = UploadHandler.forProcessor(
             self.uploadprocessor, self.incoming_folder, "test", self.build
         )
@@ -61,6 +94,42 @@ class TestCraftRecipeUploads(TestUploadProcessorBase):
         self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
         self.assertTrue(self.build.verifySuccessfulUpload())
 
+        # Verify that the crate file was properly extracted and stored
+        build = removeSecurityProxy(self.build)
+        files = list(build.getFiles())
+        self.assertEqual(1, len(files))
+        stored_file = files[0][1]
+        self.assertTrue(stored_file.filename.endswith(".crate"))
+
+    def test_processes_crate_from_archive(self):
+        """Test extracting/processing crates within .tar.xz archives."""
+        upload_dir = os.path.join(
+            self.incoming_folder, "test", str(self.build.id), "ubuntu"
+        )
+        os.makedirs(upload_dir, exist_ok=True)
+
+        # Create archive with specific crate name and version
+        crate_name = "test-crate"
+        crate_version = "0.2.0"
+        self._createArchiveWithCrate(upload_dir, crate_name, crate_version)
+
+        handler = UploadHandler.forProcessor(
+            self.uploadprocessor, self.incoming_folder, "test", self.build
+        )
+        result = handler.processCraftRecipe(self.log)
+
+        # Verify upload succeeded
+        self.assertEqual(UploadStatusEnum.ACCEPTED, result)
+        self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
+
+        # Verify the crate file was properly stored
+        build = removeSecurityProxy(self.build)
+        files = list(build.getFiles())
+        self.assertEqual(1, len(files))
+        stored_file = files[0][1]
+        expected_filename = f"{crate_name}-{crate_version}.crate"
+        self.assertEqual(expected_filename, stored_file.filename)
+
     def test_requires_craft(self):
         # The upload processor fails if the upload does not contain any
         # .craft files.
diff --git a/lib/lp/soyuz/model/archivejob.py b/lib/lp/soyuz/model/archivejob.py
index 100e3ba..65d2861 100644
--- a/lib/lp/soyuz/model/archivejob.py
+++ b/lib/lp/soyuz/model/archivejob.py
@@ -330,6 +330,9 @@ class CIBuildUploadJob(ArchiveJobDerived):
             SourcePackageFileType.GO_MODULE_MOD,
             SourcePackageFileType.GO_MODULE_ZIP,
         },
+        # XXX: ruinedyourlife 2024-12-06
+        # Remove the Rust format and it's scanner as we don't need it from
+        # CI builds. We only care about crates in craft recipe uploads.
         ArchiveRepositoryFormat.RUST: {
             BinaryPackageFormat.CRATE,
         },