launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32119
[Merge] ~ruinedyourlife/launchpad:revert-craft-build-rust-scan into launchpad:master
Quentin Debhi has proposed merging ~ruinedyourlife/launchpad:revert-craft-build-rust-scan into launchpad:master.
Commit message:
Revert "Scan crates inside rust build archives"
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad/+git/launchpad/+merge/479850
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad:revert-craft-build-rust-scan into launchpad:master.
diff --git a/lib/lp/archivepublisher/artifactory.py b/lib/lp/archivepublisher/artifactory.py
index 93d8efe..7cf7982 100644
--- a/lib/lp/archivepublisher/artifactory.py
+++ b/lib/lp/archivepublisher/artifactory.py
@@ -617,7 +617,7 @@ class ArtifactoryPool:
]
elif repository_format == ArchiveRepositoryFormat.RUST:
return [
- "*.crate",
+ "*.tar.xz",
]
elif repository_format == ArchiveRepositoryFormat.GENERIC:
return ["*"]
diff --git a/lib/lp/archivepublisher/tests/test_artifactory.py b/lib/lp/archivepublisher/tests/test_artifactory.py
index a5212b7..3fc2aa5 100644
--- a/lib/lp/archivepublisher/tests/test_artifactory.py
+++ b/lib/lp/archivepublisher/tests/test_artifactory.py
@@ -290,7 +290,7 @@ class TestArtifactoryPool(TestCase):
def test_getArtifactPatterns_rust(self):
pool = self.makePool()
self.assertEqual(
- ["*.crate"],
+ ["*.tar.xz"],
pool.getArtifactPatterns(ArchiveRepositoryFormat.RUST),
)
diff --git a/lib/lp/archiveuploader/craftrecipeupload.py b/lib/lp/archiveuploader/craftrecipeupload.py
index 75ab430..5e9d285 100644
--- a/lib/lp/archiveuploader/craftrecipeupload.py
+++ b/lib/lp/archiveuploader/craftrecipeupload.py
@@ -8,11 +8,7 @@ __all__ = [
]
import os
-import tarfile
-import tempfile
-from pathlib import Path
-import yaml
from zope.component import getUtility
from lp.archiveuploader.utils import UploadError
@@ -42,62 +38,33 @@ class CraftRecipeUpload:
"""Process this upload, loading it into the database."""
self.logger.debug("Beginning processing.")
- # Find all .tar.xz files in subdirectories
- upload_path = Path(self.upload_path)
- craft_paths = list(upload_path.rglob("*.tar.xz"))
-
- # Skip files directly in upload_path
- craft_paths = [p for p in craft_paths if p.parent != upload_path]
-
- if not craft_paths:
+ found_craft = False
+ craft_paths = []
+ for dirpath, _, filenames in os.walk(self.upload_path):
+ if dirpath == self.upload_path:
+ # All relevant files will be in a subdirectory.
+ continue
+ for craft_file in sorted(filenames):
+ if craft_file.endswith(".tar.xz"):
+ found_craft = True
+ craft_paths.append(os.path.join(dirpath, craft_file))
+
+ if not found_craft:
raise UploadError("Build did not produce any craft files.")
- for craft_path in sorted(craft_paths):
- # Extract and process .crate files from archive
- self._process_rust_archive(build, str(craft_path))
+ for craft_path in craft_paths:
+ with open(craft_path, "rb") as file:
+ libraryfile = self.librarian.create(
+ os.path.basename(craft_path),
+ os.stat(craft_path).st_size,
+ file,
+ filenameToContentType(craft_path),
+ restricted=build.is_private,
+ )
+ build.addFile(libraryfile)
# The master verifies the status to confirm successful upload.
self.logger.debug("Updating %s" % build.title)
build.updateStatus(BuildStatus.FULLYBUILT)
- self.logger.debug("Finished upload.")
- def _process_rust_archive(self, build, archive_path):
- """Process a .tar.xz archive that may contain .crate files."""
- with tempfile.TemporaryDirectory() as tmpdir:
- with tarfile.open(archive_path, "r:xz") as tar:
- tar.extractall(path=tmpdir)
-
- # Read metadata.yaml for crate info
- metadata_path = Path(tmpdir) / "metadata.yaml"
- if metadata_path.exists():
- try:
- metadata = yaml.safe_load(metadata_path.read_text())
- # XXX: ruinedyourlife 2024-12-06
- # We will need this later to give the crate a name and
- # version to artifactory. This is a placeholder for now,
- # which will be changed when we find a way to send that
- # information to artifactory.
- _ = metadata.get("name")
- _ = metadata.get("version")
- except Exception as e:
- self.logger.warning("Failed to parse metadata.yaml: %s", e)
- else:
- self.logger.debug(
- "No metadata.yaml found at %s", metadata_path
- )
-
- # Look for .crate files in extracted contents
- for crate_path in Path(tmpdir).rglob("*.crate"):
- self._process_crate_file(build, str(crate_path))
-
- def _process_crate_file(self, build, crate_path):
- """Process a single .crate file."""
- with open(crate_path, "rb") as file:
- libraryfile = self.librarian.create(
- os.path.basename(crate_path),
- os.stat(crate_path).st_size,
- file,
- filenameToContentType(crate_path),
- restricted=build.is_private,
- )
- build.addFile(libraryfile)
+ self.logger.debug("Finished upload.")
diff --git a/lib/lp/archiveuploader/tests/test_craftrecipeupload.py b/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
index c137c78..5f7f737 100644
--- a/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
+++ b/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
@@ -4,12 +4,8 @@
"""Tests for `CraftRecipeUpload`."""
import os
-import tarfile
-import tempfile
-import yaml
from storm.store import Store
-from zope.security.proxy import removeSecurityProxy
from lp.archiveuploader.tests.test_uploadprocessor import (
TestUploadProcessorBase,
@@ -43,45 +39,16 @@ class TestCraftRecipeUploads(TestUploadProcessorBase):
self.layer.txn, builds=True
)
- def _createArchiveWithCrate(
- self, upload_dir, crate_name="test-crate", crate_version="0.1.0"
- ):
- """Helper to create a tar.xz archive containing a crate & metadata."""
- # Create a temporary directory to build our archive
- with tempfile.TemporaryDirectory() as tmpdir:
- # Create metadata.yaml
- metadata = {
- "name": crate_name,
- "version": crate_version,
- }
- metadata_path = os.path.join(tmpdir, "metadata.yaml")
- with open(metadata_path, "w") as f:
- yaml.safe_dump(metadata, f)
-
- # Create dummy crate file
- crate_path = os.path.join(
- tmpdir, f"{crate_name}-{crate_version}.crate"
- )
- with open(crate_path, "wb") as f:
- f.write(b"dummy crate contents")
-
- # Create tar.xz archive
- archive_path = os.path.join(upload_dir, "output.tar.xz")
- with tarfile.open(archive_path, "w:xz") as tar:
- tar.add(metadata_path, arcname="metadata.yaml")
- tar.add(crate_path, arcname=os.path.basename(crate_path))
-
- return archive_path
-
def test_sets_build_and_state(self):
# The upload processor uploads files and sets the correct status.
self.assertFalse(self.build.verifySuccessfulUpload())
upload_dir = os.path.join(
self.incoming_folder, "test", str(self.build.id), "ubuntu"
)
- os.makedirs(upload_dir, exist_ok=True)
- self._createArchiveWithCrate(upload_dir)
-
+ write_file(
+ os.path.join(upload_dir, "foo_0_all.craft.tar.xz"), b"craft"
+ )
+ write_file(os.path.join(upload_dir, "foo_0_all.manifest"), b"manifest")
handler = UploadHandler.forProcessor(
self.uploadprocessor, self.incoming_folder, "test", self.build
)
@@ -94,42 +61,6 @@ class TestCraftRecipeUploads(TestUploadProcessorBase):
self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
self.assertTrue(self.build.verifySuccessfulUpload())
- # Verify that the crate file was properly extracted and stored
- build = removeSecurityProxy(self.build)
- files = list(build.getFiles())
- self.assertEqual(1, len(files))
- stored_file = files[0][1]
- self.assertTrue(stored_file.filename.endswith(".crate"))
-
- def test_processes_crate_from_archive(self):
- """Test extracting/processing crates within .tar.xz archives."""
- upload_dir = os.path.join(
- self.incoming_folder, "test", str(self.build.id), "ubuntu"
- )
- os.makedirs(upload_dir, exist_ok=True)
-
- # Create archive with specific crate name and version
- crate_name = "test-crate"
- crate_version = "0.2.0"
- self._createArchiveWithCrate(upload_dir, crate_name, crate_version)
-
- handler = UploadHandler.forProcessor(
- self.uploadprocessor, self.incoming_folder, "test", self.build
- )
- result = handler.processCraftRecipe(self.log)
-
- # Verify upload succeeded
- self.assertEqual(UploadStatusEnum.ACCEPTED, result)
- self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
-
- # Verify the crate file was properly stored
- build = removeSecurityProxy(self.build)
- files = list(build.getFiles())
- self.assertEqual(1, len(files))
- stored_file = files[0][1]
- expected_filename = f"{crate_name}-{crate_version}.crate"
- self.assertEqual(expected_filename, stored_file.filename)
-
def test_requires_craft(self):
# The upload processor fails if the upload does not contain any
# .craft files.
diff --git a/lib/lp/soyuz/model/archivejob.py b/lib/lp/soyuz/model/archivejob.py
index 9242e69..ee23fee 100644
--- a/lib/lp/soyuz/model/archivejob.py
+++ b/lib/lp/soyuz/model/archivejob.py
@@ -331,9 +331,6 @@ class CIBuildUploadJob(ArchiveJobDerived):
SourcePackageFileType.GO_MODULE_MOD,
SourcePackageFileType.GO_MODULE_ZIP,
},
- # XXX: ruinedyourlife 2024-12-06
- # Remove the Rust format and it's scanner as we don't need it from
- # CI builds. We only care about crates in craft recipe uploads.
ArchiveRepositoryFormat.RUST: {
BinaryPackageFormat.CRATE,
},
diff --git a/requirements/types.txt b/requirements/types.txt
index 3c51cbb..0b7a7d7 100644
--- a/requirements/types.txt
+++ b/requirements/types.txt
@@ -5,7 +5,6 @@ types-bleach==3.3.1
types-oauthlib==3.1.0
types-psycopg2==2.9.21.4
types-python-dateutil==2.9.0.20241003
-types-PyYAML==6.0.12.20241230
types-requests==0.1.13
types-six==0.1.9
types-urllib3==1.26.25.4
