launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28584
[Merge] ~cjwatson/launchpad:ci-build-upload-job-filetype into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:ci-build-upload-job-filetype into launchpad:master with ~cjwatson/launchpad:scan-conda as a prerequisite.
Commit message:
Extend filetype detection in CIBuildUploadJob
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/424403
`BinaryPackageRelease.addFile` attempts to detect the `BinaryPackageFileType` based on the file name, but this isn't likely to be reliably possible for all new package types. Determine this in `CIBuildUploadJob` and pass it down.
The intent of the distinction between `BinaryPackageFormat` and `BinaryPackageFileType` is unclear to me, but their values don't completely coincide so merging them would be non-trivial. For now, we can at least maintain a one-to-one mapping for the values relevant to `CIBuildUploadJob`.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:ci-build-upload-job-filetype into launchpad:master.
diff --git a/lib/lp/soyuz/enums.py b/lib/lp/soyuz/enums.py
index 5bc948a..93f4a70 100644
--- a/lib/lp/soyuz/enums.py
+++ b/lib/lp/soyuz/enums.py
@@ -221,6 +221,19 @@ class BinaryPackageFileType(DBEnumeratedType):
U{https://peps.python.org/pep-0427/}.
""")
+ CONDA_V1 = DBItem(7, """
+ Conda Package v1
+
+ Version 1 of the Conda package format, with the ".tar.bz2" extension.
+ """)
+
+ CONDA_V2 = DBItem(8, """
+ Conda Package v2
+
+ Version 2 of the Conda package format, with the ".conda" extension;
+ introduced in Conda 4.7.
+ """)
+
class BinaryPackageFormat(DBEnumeratedType):
"""Binary Package Format
diff --git a/lib/lp/soyuz/interfaces/binarypackagerelease.py b/lib/lp/soyuz/interfaces/binarypackagerelease.py
index a4c9e4b..2b243cb 100644
--- a/lib/lp/soyuz/interfaces/binarypackagerelease.py
+++ b/lib/lp/soyuz/interfaces/binarypackagerelease.py
@@ -96,9 +96,12 @@ class IBinaryPackageRelease(Interface):
sourcepackageversion = Attribute(
"The version of the source package from where this binary was built.")
- def addFile(file):
+ def addFile(file, filetype=None):
"""Create a BinaryPackageFile record referencing this build
and attach the provided library file alias (file).
+
+ If filetype is None, then the file type is automatically detected
+ based on the file name, if possible.
"""
def override(component=None, section=None, priority=None):
diff --git a/lib/lp/soyuz/model/archivejob.py b/lib/lp/soyuz/model/archivejob.py
index a183ae8..cc9b75b 100644
--- a/lib/lp/soyuz/model/archivejob.py
+++ b/lib/lp/soyuz/model/archivejob.py
@@ -42,6 +42,7 @@ from lp.services.job.runner import BaseRunnableJob
from lp.services.librarian.utils import copy_and_close
from lp.soyuz.enums import (
ArchiveJobType,
+ BinaryPackageFileType,
BinaryPackageFormat,
PackageUploadStatus,
)
@@ -201,6 +202,18 @@ class CIBuildUploadJob(ArchiveJobDerived):
config = config.ICIBuildUploadJobSource
+ # XXX cjwatson 2022-06-10: There doesn't seem to be a very clear
+ # conceptual distinction between BinaryPackageFormat and
+ # BinaryPackageFileType, but we end up having to add entries to both for
+ # each new package type because they're used in different database
+ # columns. Try to minimize the hassle involved in this by maintaining a
+ # mapping here of all the formats we're interested in.
+ filetype_by_format = {
+ BinaryPackageFormat.WHL: BinaryPackageFileType.WHL,
+ BinaryPackageFormat.CONDA_V1: BinaryPackageFileType.CONDA_V1,
+ BinaryPackageFormat.CONDA_V2: BinaryPackageFileType.CONDA_V2,
+ }
+
@classmethod
def create(cls, ci_build, requester, target_archive, target_distroseries,
target_pocket, target_channel=None):
@@ -332,8 +345,10 @@ class CIBuildUploadJob(ArchiveJobDerived):
metadata["binarypackagename"] = (
getUtility(IBinaryPackageNameSet).ensure(metadata["name"]))
del metadata["name"]
+ filetype = self.filetype_by_format[
+ metadata["binpackageformat"]]
bpr = self.ci_build.createBinaryPackageRelease(**metadata)
- bpr.addFile(artifact.library_file)
+ bpr.addFile(artifact.library_file, filetype=filetype)
# The publishBinaries interface was designed for .debs,
# which need extra per-binary "override" information
# (component, etc.). None of this is relevant here.
diff --git a/lib/lp/soyuz/model/binarypackagerelease.py b/lib/lp/soyuz/model/binarypackagerelease.py
index 7134895..15192c2 100644
--- a/lib/lp/soyuz/model/binarypackagerelease.py
+++ b/lib/lp/soyuz/model/binarypackagerelease.py
@@ -146,26 +146,26 @@ class BinaryPackageRelease(SQLBase):
return list(
Store.of(self).find(BinaryPackageFile, binarypackagerelease=self))
- def addFile(self, file):
+ def addFile(self, file, filetype=None):
"""See `IBinaryPackageRelease`."""
- determined_filetype = None
- if file.filename.endswith(".deb"):
- determined_filetype = BinaryPackageFileType.DEB
- elif file.filename.endswith(".rpm"):
- determined_filetype = BinaryPackageFileType.RPM
- elif file.filename.endswith(".udeb"):
- determined_filetype = BinaryPackageFileType.UDEB
- elif file.filename.endswith(".ddeb"):
- determined_filetype = BinaryPackageFileType.DDEB
- elif file.filename.endswith(".whl"):
- determined_filetype = BinaryPackageFileType.WHL
- else:
- raise AssertionError(
- 'Unsupported file type: %s' % file.filename)
+ if filetype is None:
+ if file.filename.endswith(".deb"):
+ filetype = BinaryPackageFileType.DEB
+ elif file.filename.endswith(".rpm"):
+ filetype = BinaryPackageFileType.RPM
+ elif file.filename.endswith(".udeb"):
+ filetype = BinaryPackageFileType.UDEB
+ elif file.filename.endswith(".ddeb"):
+ filetype = BinaryPackageFileType.DDEB
+ elif file.filename.endswith(".whl"):
+ filetype = BinaryPackageFileType.WHL
+ else:
+ raise AssertionError(
+ 'Unsupported file type: %s' % file.filename)
del get_property_cache(self).files
return BinaryPackageFile(binarypackagerelease=self,
- filetype=determined_filetype,
+ filetype=filetype,
libraryfile=file)
def override(self, component=None, section=None, priority=None):
diff --git a/lib/lp/soyuz/tests/test_archivejob.py b/lib/lp/soyuz/tests/test_archivejob.py
index b96c8b5..d599c48 100644
--- a/lib/lp/soyuz/tests/test_archivejob.py
+++ b/lib/lp/soyuz/tests/test_archivejob.py
@@ -404,6 +404,98 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
binarypackageformat=Equals(BinaryPackageFormat.WHL),
distroarchseries=Equals(dases[0]))))
+ def test_run_conda(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ dases = [
+ self.factory.makeDistroArchSeries(distroseries=distroseries)
+ for _ in range(2)]
+ build = self.factory.makeCIBuild(distro_arch_series=dases[0])
+ report = build.getOrCreateRevisionStatusReport("build:0")
+ report.setLog(b"log data")
+ path = "conda-arch/dist/linux-64/conda-arch-0.1-0.tar.bz2"
+ with open(datadir(path), mode="rb") as f:
+ report.attach(name=os.path.basename(path), data=f.read())
+ artifact = IStore(RevisionStatusArtifact).find(
+ RevisionStatusArtifact,
+ report=report,
+ artifact_type=RevisionStatusArtifactType.BINARY).one()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ transaction.commit()
+
+ with dbuser(job.config.dbuser):
+ JobRunner([job]).runAll()
+
+ self.assertThat(archive.getAllPublishedBinaries(), MatchesSetwise(
+ MatchesStructure(
+ binarypackagename=MatchesStructure.byEquality(
+ name="conda-arch"),
+ binarypackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ binarypackagename=MatchesStructure.byEquality(
+ name="conda-arch"),
+ version=Equals("0.1"),
+ summary=Equals("Example summary"),
+ description=Equals("Example description"),
+ binpackageformat=Equals(BinaryPackageFormat.CONDA_V1),
+ architecturespecific=Is(True),
+ homepage=Equals("http://example.com/";),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifact.library_file,
+ filetype=BinaryPackageFileType.CONDA_V1))),
+ binarypackageformat=Equals(BinaryPackageFormat.CONDA_V1),
+ distroarchseries=Equals(dases[0]))))
+
+ def test_run_conda_v2(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ dases = [
+ self.factory.makeDistroArchSeries(distroseries=distroseries)
+ for _ in range(2)]
+ build = self.factory.makeCIBuild(distro_arch_series=dases[0])
+ report = build.getOrCreateRevisionStatusReport("build:0")
+ report.setLog(b"log data")
+ path = "conda-v2-arch/dist/linux-64/conda-v2-arch-0.1-0.conda"
+ with open(datadir(path), mode="rb") as f:
+ report.attach(name=os.path.basename(path), data=f.read())
+ artifact = IStore(RevisionStatusArtifact).find(
+ RevisionStatusArtifact,
+ report=report,
+ artifact_type=RevisionStatusArtifactType.BINARY).one()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ transaction.commit()
+
+ with dbuser(job.config.dbuser):
+ JobRunner([job]).runAll()
+
+ self.assertThat(archive.getAllPublishedBinaries(), MatchesSetwise(
+ MatchesStructure(
+ binarypackagename=MatchesStructure.byEquality(
+ name="conda-v2-arch"),
+ binarypackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ binarypackagename=MatchesStructure.byEquality(
+ name="conda-v2-arch"),
+ version=Equals("0.1"),
+ summary=Equals("Example summary"),
+ description=Equals("Example description"),
+ binpackageformat=Equals(BinaryPackageFormat.CONDA_V2),
+ architecturespecific=Is(True),
+ homepage=Equals("http://example.com/";),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifact.library_file,
+ filetype=BinaryPackageFileType.CONDA_V2))),
+ binarypackageformat=Equals(BinaryPackageFormat.CONDA_V2),
+ distroarchseries=Equals(dases[0]))))
+
class TestViaCelery(TestCaseWithFactory):
