launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28575
[Merge] ~cjwatson/launchpad:scan-conda into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:scan-conda into launchpad:master.
Commit message:
Implement metadata scanning for Conda packages
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/424343
This allows `CIBuildUploadJob` to work for Conda v1 (`.tar.bz2`) and v2 (`.conda`) packages.
Dependencies MP: https://code.launchpad.net/~cjwatson/lp-source-dependencies/+git/lp-source-dependencies/+merge/424342
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:scan-conda into launchpad:master.
diff --git a/lib/lp/soyuz/enums.py b/lib/lp/soyuz/enums.py
index 96acc2e..5bc948a 100644
--- a/lib/lp/soyuz/enums.py
+++ b/lib/lp/soyuz/enums.py
@@ -271,6 +271,19 @@ class BinaryPackageFormat(DBEnumeratedType):
U{https://peps.python.org/pep-0427/}.
""")
+ CONDA_V1 = DBItem(7, """
+ Conda Package v1
+
+ Version 1 of the Conda package format, with the ".tar.bz2" extension.
+ """)
+
+ CONDA_V2 = DBItem(8, """
+ Conda Package v2
+
+ Version 2 of the Conda package format, with the ".conda" extension;
+ introduced in Conda 4.7.
+ """)
+
class PackageCopyPolicy(DBEnumeratedType):
"""Package copying policy.
diff --git a/lib/lp/soyuz/model/archivejob.py b/lib/lp/soyuz/model/archivejob.py
index c8c9cb5..a183ae8 100644
--- a/lib/lp/soyuz/model/archivejob.py
+++ b/lib/lp/soyuz/model/archivejob.py
@@ -2,9 +2,12 @@
# GNU Affero General Public License version 3 (see the file LICENSE).
import io
+import json
import logging
import os.path
+import tarfile
import tempfile
+import zipfile
from lazr.delegates import delegate_to
from pkginfo import Wheel
@@ -20,6 +23,7 @@ from zope.interface import (
implementer,
provider,
)
+import zstandard
from lp.code.enums import RevisionStatusArtifactType
from lp.code.interfaces.cibuild import ICIBuildSet
@@ -241,7 +245,19 @@ class CIBuildUploadJob(ArchiveJobDerived):
def target_channel(self):
return self.metadata["target_channel"]
+ def _scanCondaMetadata(self, index, about):
+ return {
+ "name": index["name"],
+ "version": index["version"],
+ "summary": about.get("summary", ""),
+ "description": about.get("description", ""),
+ "architecturespecific": index["platform"] is not None,
+ "homepage": about.get("home", ""),
+ }
+
def _scanFile(self, path):
+ # XXX cjwatson 2022-06-10: We should probably start splitting this
+ # up some more.
if path.endswith(".whl"):
try:
parsed_path = parse_wheel_filename(path)
@@ -257,6 +273,37 @@ class CIBuildUploadJob(ArchiveJobDerived):
"architecturespecific": "any" not in parsed_path.platform_tags,
"homepage": wheel.home_page or "",
}
+ elif path.endswith(".tar.bz2"):
+ try:
+ with tarfile.open(path) as tar:
+ index = json.loads(
+ tar.extractfile("info/index.json").read().decode())
+ about = json.loads(
+ tar.extractfile("info/about.json").read().decode())
+ except Exception as e:
+ raise ScanException("Failed to scan %s" % path) from e
+ scanned = {"binpackageformat": BinaryPackageFormat.CONDA_V1}
+ scanned.update(self._scanCondaMetadata(index, about))
+ return scanned
+ elif path.endswith(".conda"):
+ try:
+ with zipfile.ZipFile(path) as zipf:
+ base_name = os.path.basename(path)[:-len(".conda")]
+ info = io.BytesIO()
+ with zipf.open("info-%s.tar.zst" % base_name) as raw_info:
+ zstandard.ZstdDecompressor().copy_stream(
+ raw_info, info)
+ info.seek(0)
+ with tarfile.open(fileobj=info) as tar:
+ index = json.loads(
+ tar.extractfile("info/index.json").read().decode())
+ about = json.loads(
+ tar.extractfile("info/about.json").read().decode())
+ except Exception as e:
+ raise ScanException("Failed to scan %s" % path) from e
+ scanned = {"binpackageformat": BinaryPackageFormat.CONDA_V2}
+ scanned.update(self._scanCondaMetadata(index, about))
+ return scanned
else:
return None
diff --git a/lib/lp/soyuz/tests/data/conda-arch/.launchpad.yaml b/lib/lp/soyuz/tests/data/conda-arch/.launchpad.yaml
new file mode 100644
index 0000000..9e49d8c
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/conda-arch/.launchpad.yaml
@@ -0,0 +1,12 @@
+pipeline:
+ - build
+
+jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ build-target: .
+ output:
+ paths:
+ - dist/*/*.tar.bz2
diff --git a/lib/lp/soyuz/tests/data/conda-arch/dist/linux-64/conda-arch-0.1-0.tar.bz2 b/lib/lp/soyuz/tests/data/conda-arch/dist/linux-64/conda-arch-0.1-0.tar.bz2
new file mode 100644
index 0000000..088f0bf
Binary files /dev/null and b/lib/lp/soyuz/tests/data/conda-arch/dist/linux-64/conda-arch-0.1-0.tar.bz2 differ
diff --git a/lib/lp/soyuz/tests/data/conda-arch/meta.yaml b/lib/lp/soyuz/tests/data/conda-arch/meta.yaml
new file mode 100644
index 0000000..2b4016b
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/conda-arch/meta.yaml
@@ -0,0 +1,7 @@
+package:
+ name: conda-arch
+ version: 0.1
+about:
+ home: http://example.com/
+ summary: Example summary
+ description: Example description
diff --git a/lib/lp/soyuz/tests/data/conda-indep/.launchpad.yaml b/lib/lp/soyuz/tests/data/conda-indep/.launchpad.yaml
new file mode 100644
index 0000000..9e49d8c
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/conda-indep/.launchpad.yaml
@@ -0,0 +1,12 @@
+pipeline:
+ - build
+
+jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ build-target: .
+ output:
+ paths:
+ - dist/*/*.tar.bz2
diff --git a/lib/lp/soyuz/tests/data/conda-indep/dist/noarch/conda-indep-0.1-0.tar.bz2 b/lib/lp/soyuz/tests/data/conda-indep/dist/noarch/conda-indep-0.1-0.tar.bz2
new file mode 100644
index 0000000..a83afae
Binary files /dev/null and b/lib/lp/soyuz/tests/data/conda-indep/dist/noarch/conda-indep-0.1-0.tar.bz2 differ
diff --git a/lib/lp/soyuz/tests/data/conda-indep/meta.yaml b/lib/lp/soyuz/tests/data/conda-indep/meta.yaml
new file mode 100644
index 0000000..fe0261d
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/conda-indep/meta.yaml
@@ -0,0 +1,8 @@
+package:
+ name: conda-indep
+ version: 0.1
+build:
+ noarch: generic
+about:
+ summary: Example summary
+ description: Example description
diff --git a/lib/lp/soyuz/tests/data/conda-v2-arch/.launchpad.yaml b/lib/lp/soyuz/tests/data/conda-v2-arch/.launchpad.yaml
new file mode 100644
index 0000000..ab9a167
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/conda-v2-arch/.launchpad.yaml
@@ -0,0 +1,12 @@
+pipeline:
+ - build
+
+jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ build-target: .
+ output:
+ paths:
+ - dist/*/*.conda
diff --git a/lib/lp/soyuz/tests/data/conda-v2-arch/dist/linux-64/conda-v2-arch-0.1-0.conda b/lib/lp/soyuz/tests/data/conda-v2-arch/dist/linux-64/conda-v2-arch-0.1-0.conda
new file mode 100644
index 0000000..4d1136f
Binary files /dev/null and b/lib/lp/soyuz/tests/data/conda-v2-arch/dist/linux-64/conda-v2-arch-0.1-0.conda differ
diff --git a/lib/lp/soyuz/tests/data/conda-v2-arch/meta.yaml b/lib/lp/soyuz/tests/data/conda-v2-arch/meta.yaml
new file mode 100644
index 0000000..3db1dd5
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/conda-v2-arch/meta.yaml
@@ -0,0 +1,10 @@
+package:
+ name: conda-v2-arch
+ version: 0.1
+outputs:
+ - name: conda-v2-arch
+ type: conda_v2
+about:
+ home: http://example.com/
+ summary: Example summary
+ description: Example description
diff --git a/lib/lp/soyuz/tests/data/conda-v2-indep/.launchpad.yaml b/lib/lp/soyuz/tests/data/conda-v2-indep/.launchpad.yaml
new file mode 100644
index 0000000..ab9a167
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/conda-v2-indep/.launchpad.yaml
@@ -0,0 +1,12 @@
+pipeline:
+ - build
+
+jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ build-target: .
+ output:
+ paths:
+ - dist/*/*.conda
diff --git a/lib/lp/soyuz/tests/data/conda-v2-indep/dist/noarch/conda-v2-indep-0.1-0.conda b/lib/lp/soyuz/tests/data/conda-v2-indep/dist/noarch/conda-v2-indep-0.1-0.conda
new file mode 100644
index 0000000..734a3e6
Binary files /dev/null and b/lib/lp/soyuz/tests/data/conda-v2-indep/dist/noarch/conda-v2-indep-0.1-0.conda differ
diff --git a/lib/lp/soyuz/tests/data/conda-v2-indep/meta.yaml b/lib/lp/soyuz/tests/data/conda-v2-indep/meta.yaml
new file mode 100644
index 0000000..306fd64
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/conda-v2-indep/meta.yaml
@@ -0,0 +1,11 @@
+package:
+ name: conda-v2-indep
+ version: 0.1
+build:
+ noarch: generic
+outputs:
+ - name: conda-v2-indep
+ type: conda_v2
+about:
+ summary: Example summary
+ description: Example description
diff --git a/lib/lp/soyuz/tests/test_archivejob.py b/lib/lp/soyuz/tests/test_archivejob.py
index 1b68478..b96c8b5 100644
--- a/lib/lp/soyuz/tests/test_archivejob.py
+++ b/lib/lp/soyuz/tests/test_archivejob.py
@@ -231,6 +231,86 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
}
self.assertEqual(expected, job._scanFile(datadir(path)))
+ def test__scanFile_conda_indep(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ path = "conda-indep/dist/noarch/conda-indep-0.1-0.tar.bz2"
+ expected = {
+ "name": "conda-indep",
+ "version": "0.1",
+ "summary": "Example summary",
+ "description": "Example description",
+ "binpackageformat": BinaryPackageFormat.CONDA_V1,
+ "architecturespecific": False,
+ "homepage": "",
+ }
+ self.assertEqual(expected, job._scanFile(datadir(path)))
+
+ def test__scanFile_conda_arch(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ path = "conda-arch/dist/linux-64/conda-arch-0.1-0.tar.bz2"
+ expected = {
+ "name": "conda-arch",
+ "version": "0.1",
+ "summary": "Example summary",
+ "description": "Example description",
+ "binpackageformat": BinaryPackageFormat.CONDA_V1,
+ "architecturespecific": True,
+ "homepage": "http://example.com/";,
+ }
+ self.assertEqual(expected, job._scanFile(datadir(path)))
+
+ def test__scanFile_conda_v2_indep(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ path = "conda-v2-indep/dist/noarch/conda-v2-indep-0.1-0.conda"
+ expected = {
+ "name": "conda-v2-indep",
+ "version": "0.1",
+ "summary": "Example summary",
+ "description": "Example description",
+ "binpackageformat": BinaryPackageFormat.CONDA_V2,
+ "architecturespecific": False,
+ "homepage": "",
+ }
+ self.assertEqual(expected, job._scanFile(datadir(path)))
+
+ def test__scanFile_conda_v2_arch(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ path = "conda-v2-arch/dist/linux-64/conda-v2-arch-0.1-0.conda"
+ expected = {
+ "name": "conda-v2-arch",
+ "version": "0.1",
+ "summary": "Example summary",
+ "description": "Example description",
+ "binpackageformat": BinaryPackageFormat.CONDA_V2,
+ "architecturespecific": True,
+ "homepage": "http://example.com/";,
+ }
+ self.assertEqual(expected, job._scanFile(datadir(path)))
+
def test_run_indep(self):
archive = self.factory.makeArchive()
distroseries = self.factory.makeDistroSeries(
diff --git a/requirements/launchpad.txt b/requirements/launchpad.txt
index 2846cc9..e66e855 100644
--- a/requirements/launchpad.txt
+++ b/requirements/launchpad.txt
@@ -197,3 +197,4 @@ zope.testbrowser==5.5.1
# lp:~launchpad-committers/zope.testrunner:launchpad
zope.testrunner==5.3.0+lp1
zope.vocabularyregistry==1.1.1
+zstandard==0.15.2
diff --git a/setup.cfg b/setup.cfg
index 145a832..e23f16b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -154,6 +154,7 @@ install_requires =
zope.testrunner[subunit]
zope.traversing
zope.vocabularyregistry
+ zstandard
# Loggerhead dependencies. These should be removed once bug 383360 is
# fixed and we include it as a source dist.
bleach
Follow ups
