launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29195
[Merge] ~cjwatson/launchpad:generic-artifactory-publishing into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:generic-artifactory-publishing into launchpad:master.
Commit message:
Support generic Artifactory repositories
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/429956
This follows the plan in https://docs.google.com/document/d/1fqfQhX2qR7oyIB3OBCXWw16pp_2xTSbKl6FU_jERblE: CI builds can produce artifacts that have the `name` and `version` (and optionally `source`) properties set, which are then used to construct publishing paths. Generic repositories have no indexing support.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:generic-artifactory-publishing into launchpad:master.
diff --git a/lib/lp/archivepublisher/artifactory.py b/lib/lp/archivepublisher/artifactory.py
index 4bd7f0a..f56799e 100644
--- a/lib/lp/archivepublisher/artifactory.py
+++ b/lib/lp/archivepublisher/artifactory.py
@@ -75,6 +75,11 @@ def _path_for(
raise ValueError("Cannot publish a Go module with no module-path")
# Base path required by https://go.dev/ref/mod#module-proxy.
path = rootpath / module_path / "@v"
+ elif repository_format == ArchiveRepositoryFormat.GENERIC:
+ package_name = release.getUserDefinedField("name")
+ if package_name is None:
+ package_name = source_name
+ path = rootpath / package_name / source_version
else:
raise AssertionError(
"Unsupported repository format: %r" % repository_format
@@ -550,6 +555,8 @@ class ArtifactoryPool:
"*.mod",
"*.zip",
]
+ elif repository_format == ArchiveRepositoryFormat.GENERIC:
+ return ["*"]
else:
raise AssertionError(
"Unknown repository format %r" % repository_format
diff --git a/lib/lp/archivepublisher/tests/test_artifactory.py b/lib/lp/archivepublisher/tests/test_artifactory.py
index 9aea55c..8cea8dd 100644
--- a/lib/lp/archivepublisher/tests/test_artifactory.py
+++ b/lib/lp/archivepublisher/tests/test_artifactory.py
@@ -142,6 +142,26 @@ class TestArtifactoryPool(TestCase):
pool.pathFor(None, "go-module", "v1.0", pub_file),
)
+ def test_pathFor_generic_with_file(self):
+ pool = self.makePool(ArchiveRepositoryFormat.GENERIC)
+ pub_file = FakePackageReleaseFile(
+ b"source artifact",
+ "foo-1.0.tar.gz",
+ release_type=FakeReleaseType.SOURCE,
+ user_defined_fields=[
+ ("name", "foo"),
+ ("version", "1.0"),
+ ("source", True),
+ ],
+ )
+ self.assertEqual(
+ ArtifactoryPath(
+ "https://foo.example.com/artifactory/repository/";
+ "foo/1.0/foo-1.0.tar.gz"
+ ),
+ pool.pathFor(None, "foo-generic", "1.0", pub_file),
+ )
+
def test_addFile(self):
pool = self.makePool()
foo = ArtifactoryPoolTestingFile(
@@ -251,6 +271,12 @@ class TestArtifactoryPool(TestCase):
pool.getArtifactPatterns(ArchiveRepositoryFormat.GO_PROXY),
)
+ def test_getArtifactPatterns_generic(self):
+ pool = self.makePool()
+ self.assertEqual(
+ ["*"], pool.getArtifactPatterns(ArchiveRepositoryFormat.GENERIC)
+ )
+
def test_getAllArtifacts_debian(self):
# getAllArtifacts mostly relies on constructing a correct AQL query,
# which we can't meaningfully test without a real Artifactory
@@ -363,6 +389,34 @@ class TestArtifactoryPool(TestCase):
),
)
+ def test_getAllArtifacts_generic(self):
+ pool = self.makePool(ArchiveRepositoryFormat.GENERIC)
+ ArtifactoryPoolTestingFile(
+ pool=pool,
+ source_name="bar",
+ source_version="1.0",
+ filename="bar-1.0.tar.gz",
+ release_type=FakeReleaseType.SOURCE,
+ release_id=1,
+ user_defined_fields=[
+ ("name", "bar"),
+ ("version", "1.0"),
+ ("source", True),
+ ],
+ ).addToPool()
+ self.assertEqual(
+ {
+ PurePath("bar/1.0/bar-1.0.tar.gz"): {
+ "launchpad.release-id": ["source:1"],
+ "launchpad.source-name": ["bar"],
+ "launchpad.source-version": ["1.0"],
+ },
+ },
+ pool.getAllArtifacts(
+ self.repository_name, ArchiveRepositoryFormat.GENERIC
+ ),
+ )
+
def test_getAllArtifacts_handles_empty_properties(self):
# AQL queries seem to return empty properties as something like
# `{"key": "pypi.requires.python"}` rather than `{"key":
@@ -1102,6 +1156,168 @@ class TestArtifactoryPoolFromLibrarian(TestCaseWithFactory):
path.properties,
)
+ def test_updateProperties_generic_source(self):
+ pool = self.makePool(ArchiveRepositoryFormat.GENERIC)
+ dses = [
+ self.factory.makeDistroSeries(
+ distribution=pool.archive.distribution
+ )
+ for _ in range(2)
+ ]
+ das = self.factory.makeDistroArchSeries(distroseries=dses[0])
+ ci_build = self.factory.makeCIBuild(distro_arch_series=das)
+ spr = self.factory.makeSourcePackageRelease(
+ archive=pool.archive,
+ sourcepackagename="foo-package",
+ version="1.0",
+ format=SourcePackageType.CI_BUILD,
+ ci_build=ci_build,
+ user_defined_fields=[
+ ("name", "foo"),
+ ("version", "1.0"),
+ ("source", True),
+ ],
+ )
+ spph = self.factory.makeSourcePackagePublishingHistory(
+ archive=pool.archive,
+ sourcepackagerelease=spr,
+ distroseries=dses[0],
+ pocket=PackagePublishingPocket.RELEASE,
+ component="main",
+ sourcepackagename="foo-package",
+ version="1.0",
+ channel="edge",
+ format=SourcePackageType.CI_BUILD,
+ )
+ spr = spph.sourcepackagerelease
+ sprf = self.factory.makeSourcePackageReleaseFile(
+ sourcepackagerelease=spr,
+ library_file=self.factory.makeLibraryFileAlias(
+ filename="foo-1.0.tar.gz"
+ ),
+ filetype=SourcePackageFileType.GENERIC,
+ )
+ spphs = [spph]
+ spphs.append(
+ spph.copyTo(dses[1], PackagePublishingPocket.RELEASE, pool.archive)
+ )
+ transaction.commit()
+ pool.addFile(None, spr.name, spr.version, sprf)
+ path = pool.rootpath / "foo" / "1.0" / "foo-1.0.tar.gz"
+ self.assertTrue(path.exists())
+ self.assertFalse(path.is_symlink())
+ self.assertEqual(
+ {
+ "launchpad.release-id": ["source:%d" % spr.id],
+ "launchpad.source-name": ["foo-package"],
+ "launchpad.source-version": ["1.0"],
+ "soss.source_url": [
+ ci_build.git_repository.getCodebrowseUrl()
+ ],
+ "soss.commit_id": [ci_build.commit_sha1],
+ },
+ path.properties,
+ )
+ pool.updateProperties(spr.name, spr.version, [sprf], spphs)
+ self.assertEqual(
+ {
+ "launchpad.release-id": ["source:%d" % spr.id],
+ "launchpad.source-name": ["foo-package"],
+ "launchpad.source-version": ["1.0"],
+ "launchpad.channel": list(
+ sorted("%s:edge" % ds.name for ds in dses)
+ ),
+ "soss.source_url": [
+ ci_build.git_repository.getCodebrowseUrl()
+ ],
+ "soss.commit_id": [ci_build.commit_sha1],
+ },
+ path.properties,
+ )
+
+ def test_updateProperties_generic_binary(self):
+ pool = self.makePool(ArchiveRepositoryFormat.GENERIC)
+ dses = [
+ self.factory.makeDistroSeries(
+ distribution=pool.archive.distribution
+ )
+ for _ in range(2)
+ ]
+ processor = self.factory.makeProcessor()
+ dases = [
+ self.factory.makeDistroArchSeries(
+ distroseries=ds, architecturetag=processor.name
+ )
+ for ds in dses
+ ]
+ ci_build = self.factory.makeCIBuild(distro_arch_series=dases[0])
+ bpn = self.factory.makeBinaryPackageName(name="foo")
+ bpr = self.factory.makeBinaryPackageRelease(
+ binarypackagename=bpn,
+ version="1.0",
+ ci_build=ci_build,
+ binpackageformat=BinaryPackageFormat.GENERIC,
+ user_defined_fields=[("name", "foo"), ("version", "1.0")],
+ )
+ bpf = self.factory.makeBinaryPackageFile(
+ binarypackagerelease=bpr,
+ library_file=self.factory.makeLibraryFileAlias(
+ filename="test-binary"
+ ),
+ filetype=BinaryPackageFileType.GENERIC,
+ )
+ bpph = self.factory.makeBinaryPackagePublishingHistory(
+ binarypackagerelease=bpr,
+ archive=pool.archive,
+ distroarchseries=dases[0],
+ pocket=PackagePublishingPocket.RELEASE,
+ architecturespecific=True,
+ channel="edge",
+ )
+ bpphs = [bpph]
+ bpphs.append(
+ getUtility(IPublishingSet).copyBinaries(
+ pool.archive,
+ dses[1],
+ PackagePublishingPocket.RELEASE,
+ [bpph],
+ channel="edge",
+ )[0]
+ )
+ transaction.commit()
+ pool.addFile(None, bpph.pool_name, bpph.pool_version, bpf)
+ path = pool.rootpath / "foo" / "1.0" / "test-binary"
+ self.assertTrue(path.exists())
+ self.assertFalse(path.is_symlink())
+ self.assertEqual(
+ {
+ "launchpad.release-id": ["binary:%d" % bpr.id],
+ "launchpad.source-name": ["foo"],
+ "launchpad.source-version": ["1.0"],
+ "soss.source_url": [
+ ci_build.git_repository.getCodebrowseUrl()
+ ],
+ "soss.commit_id": [ci_build.commit_sha1],
+ },
+ path.properties,
+ )
+ pool.updateProperties(bpph.pool_name, bpph.pool_version, [bpf], bpphs)
+ self.assertEqual(
+ {
+ "launchpad.release-id": ["binary:%d" % bpr.id],
+ "launchpad.source-name": ["foo"],
+ "launchpad.source-version": ["1.0"],
+ "launchpad.channel": list(
+ sorted("%s:edge" % ds.name for ds in dses)
+ ),
+ "soss.source_url": [
+ ci_build.git_repository.getCodebrowseUrl()
+ ],
+ "soss.commit_id": [ci_build.commit_sha1],
+ },
+ path.properties,
+ )
+
def test_updateProperties_preserves_externally_set_properties(self):
# Artifactory sets some properties by itself as part of scanning
# packages. We leave those untouched.
diff --git a/lib/lp/registry/interfaces/sourcepackage.py b/lib/lp/registry/interfaces/sourcepackage.py
index d2a4ad2..6baff50 100644
--- a/lib/lp/registry/interfaces/sourcepackage.py
+++ b/lib/lp/registry/interfaces/sourcepackage.py
@@ -531,6 +531,16 @@ class SourcePackageFileType(DBEnumeratedType):
""",
)
+ GENERIC = DBItem(
+ 15,
+ """
+ Generic source file
+
+ This file is a generic source file without any particular known
+ metadata of its own, produced by a CI build.
+ """,
+ )
+
class SourcePackageType(DBEnumeratedType):
"""Source Package Format
diff --git a/lib/lp/soyuz/enums.py b/lib/lp/soyuz/enums.py
index 920975c..b3b63c9 100644
--- a/lib/lp/soyuz/enums.py
+++ b/lib/lp/soyuz/enums.py
@@ -297,6 +297,16 @@ class BinaryPackageFileType(DBEnumeratedType):
""",
)
+ GENERIC = DBItem(
+ 9,
+ """
+ Generic binary file
+
+ This file is a generic binary file without any particular known
+ metadata of its own, produced by a CI build.
+ """,
+ )
+
class BinaryPackageFormat(DBEnumeratedType):
"""Binary Package Format
@@ -384,6 +394,16 @@ class BinaryPackageFormat(DBEnumeratedType):
""",
)
+ GENERIC = DBItem(
+ 9,
+ """
+ Generic binary
+
+ This is a generic binary format without any particular known
+ metadata of its own, produced by a CI build.
+ """,
+ )
+
class PackageCopyPolicy(DBEnumeratedType):
"""Package copying policy.
@@ -947,3 +967,12 @@ class ArchiveRepositoryFormat(DBEnumeratedType):
(https://go.dev/ref/mod#module-proxy).
""",
)
+
+ GENERIC = DBItem(
+ 4,
+ """
+ Generic
+
+ A generic repository with a basic name/version layout and no indexing.
+ """,
+ )
diff --git a/lib/lp/soyuz/model/archivejob.py b/lib/lp/soyuz/model/archivejob.py
index 2bf29f5..f73cec1 100644
--- a/lib/lp/soyuz/model/archivejob.py
+++ b/lib/lp/soyuz/model/archivejob.py
@@ -26,6 +26,7 @@ from lp.code.interfaces.cibuild import ICIBuildSet
from lp.code.interfaces.revisionstatus import (
IRevisionStatusArtifact,
IRevisionStatusArtifactSet,
+ IRevisionStatusReport,
)
from lp.registry.interfaces.distributionsourcepackage import (
IDistributionSourcePackage,
@@ -305,6 +306,7 @@ class CIBuildUploadJob(ArchiveJobDerived):
BinaryPackageFormat.WHL: BinaryPackageFileType.WHL,
BinaryPackageFormat.CONDA_V1: BinaryPackageFileType.CONDA_V1,
BinaryPackageFormat.CONDA_V2: BinaryPackageFileType.CONDA_V2,
+ BinaryPackageFormat.GENERIC: BinaryPackageFileType.GENERIC,
}
# We're only interested in uploading certain kinds of packages to
@@ -328,6 +330,10 @@ class CIBuildUploadJob(ArchiveJobDerived):
SourcePackageFileType.GO_MODULE_MOD,
SourcePackageFileType.GO_MODULE_ZIP,
},
+ ArchiveRepositoryFormat.GENERIC: {
+ SourcePackageFileType.GENERIC,
+ BinaryPackageFormat.GENERIC,
+ },
}
@classmethod
@@ -411,7 +417,9 @@ class CIBuildUploadJob(ArchiveJobDerived):
def target_channel(self):
return self.metadata["target_channel"]
- def _scanWheel(self, paths: Iterable[Path]) -> Dict[str, ArtifactMetadata]:
+ def _scanWheel(
+ self, report: IRevisionStatusReport, paths: Iterable[Path]
+ ) -> Dict[str, ArtifactMetadata]:
all_metadata = {}
for path in paths:
if not path.name.endswith(".whl"):
@@ -436,7 +444,9 @@ class CIBuildUploadJob(ArchiveJobDerived):
)
return all_metadata
- def _scanSDist(self, paths: Iterable[Path]) -> Dict[str, ArtifactMetadata]:
+ def _scanSDist(
+ self, report: IRevisionStatusReport, paths: Iterable[Path]
+ ) -> Dict[str, ArtifactMetadata]:
all_metadata = {}
for path in paths:
if not path.name.endswith((".tar.gz", ".zip")):
@@ -484,7 +494,7 @@ class CIBuildUploadJob(ArchiveJobDerived):
)
def _scanCondaV1(
- self, paths: Iterable[Path]
+ self, report: IRevisionStatusReport, paths: Iterable[Path]
) -> Dict[str, ArtifactMetadata]:
all_metadata = {}
for path in paths:
@@ -510,7 +520,7 @@ class CIBuildUploadJob(ArchiveJobDerived):
return all_metadata
def _scanCondaV2(
- self, paths: Iterable[Path]
+ self, report: IRevisionStatusReport, paths: Iterable[Path]
) -> Dict[str, ArtifactMetadata]:
all_metadata = {}
for path in paths:
@@ -542,7 +552,9 @@ class CIBuildUploadJob(ArchiveJobDerived):
)
return all_metadata
- def _scanGoMod(self, paths: Iterable[Path]) -> Dict[str, ArtifactMetadata]:
+ def _scanGoMod(
+ self, report: IRevisionStatusReport, paths: Iterable[Path]
+ ) -> Dict[str, ArtifactMetadata]:
all_metadata = {}
for path in paths:
if not path.name.endswith(".mod"):
@@ -601,18 +613,54 @@ class CIBuildUploadJob(ArchiveJobDerived):
)
return all_metadata
- def _scanFiles(self, directory: Path) -> Dict[str, ArtifactMetadata]:
+ def _scanGeneric(
+ self, report: IRevisionStatusReport, paths: Iterable[Path]
+ ) -> Dict[str, ArtifactMetadata]:
+ properties = report.properties
+ if (
+ properties is None
+ or "name" not in properties
+ or "version" not in properties
+ ):
+ return {}
+
+ all_metadata = {}
+ for path in paths:
+ if properties.get("source", False):
+ logger.info("%s is a generic source artifact", path.name)
+ all_metadata[path.name] = SourceArtifactMetadata(
+ format=SourcePackageFileType.GENERIC,
+ name=properties["name"],
+ version=properties["version"],
+ )
+ else:
+ logger.info("%s is a generic binary artifact", path.name)
+ all_metadata[path.name] = BinaryArtifactMetadata(
+ format=BinaryPackageFormat.GENERIC,
+ name=properties["name"],
+ version=properties["version"],
+ summary="",
+ description="",
+ architecturespecific=True,
+ homepage="",
+ )
+ return all_metadata
+
+ def _scanFiles(
+ self, report: IRevisionStatusReport, directory: Path
+ ) -> Dict[str, ArtifactMetadata]:
scanners = (
self._scanWheel,
self._scanSDist,
self._scanCondaV1,
self._scanCondaV2,
self._scanGoMod,
+ self._scanGeneric,
)
paths = [directory / child for child in directory.iterdir()]
all_metadata = OrderedDict()
for scanner in scanners:
- for name, metadata in scanner(paths).items():
+ for name, metadata in scanner(report, paths).items():
all_metadata[name] = metadata
paths = [path for path in paths if path.name not in all_metadata]
return all_metadata
@@ -635,15 +683,24 @@ class CIBuildUploadJob(ArchiveJobDerived):
with tempfile.TemporaryDirectory(prefix="ci-build-copy-job") as tmpdir:
tmpdirpath = Path(tmpdir)
artifact_by_name = {}
+ report_by_id = {}
for artifact in artifacts:
if artifact.artifact_type == RevisionStatusArtifactType.LOG:
continue
name = artifact.library_file.filename
- contents = str(tmpdirpath / name)
+ contents = tmpdirpath / str(artifact.report.id) / name
+ if not contents.parent.exists():
+ contents.parent.mkdir()
artifact.library_file.open()
- copy_and_close(artifact.library_file, open(contents, "wb"))
+ copy_and_close(
+ artifact.library_file, open(str(contents), "wb")
+ )
artifact_by_name[name] = artifact
- all_metadata = self._scanFiles(tmpdirpath)
+ report_by_id[str(artifact.report.id)] = artifact.report
+ all_metadata = {}
+ for report_path in sorted(tmpdirpath.iterdir()):
+ report = report_by_id[report_path.name]
+ all_metadata.update(self._scanFiles(report, report_path))
for name, metadata in all_metadata.items():
if metadata.format not in allowed_formats:
logger.info(
diff --git a/lib/lp/soyuz/tests/test_archivejob.py b/lib/lp/soyuz/tests/test_archivejob.py
index 8d36944..623c7cb 100644
--- a/lib/lp/soyuz/tests/test_archivejob.py
+++ b/lib/lp/soyuz/tests/test_archivejob.py
@@ -273,6 +273,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
distribution=archive.distribution
)
build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(ci_build=build)
job = CIBuildUploadJob.create(
build,
build.git_repository.owner,
@@ -284,7 +285,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
path = Path("wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl")
tmpdir = Path(self.useFixture(TempDir()).path)
shutil.copy2(datadir(str(path)), str(tmpdir))
- all_metadata = job._scanFiles(tmpdir)
+ all_metadata = job._scanFiles(report, tmpdir)
self.assertThat(
all_metadata,
MatchesDict(
@@ -308,6 +309,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
distribution=archive.distribution
)
build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(ci_build=build)
job = CIBuildUploadJob.create(
build,
build.git_repository.owner,
@@ -321,7 +323,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
)
tmpdir = Path(self.useFixture(TempDir()).path)
shutil.copy2(datadir(str(path)), str(tmpdir))
- all_metadata = job._scanFiles(tmpdir)
+ all_metadata = job._scanFiles(report, tmpdir)
self.assertThat(
all_metadata,
MatchesDict(
@@ -345,6 +347,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
distribution=archive.distribution
)
build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(ci_build=build)
job = CIBuildUploadJob.create(
build,
build.git_repository.owner,
@@ -356,7 +359,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
path = Path("wheel-arch/dist/wheel-arch-0.0.1.tar.gz")
tmpdir = Path(self.useFixture(TempDir()).path)
shutil.copy2(datadir(str(path)), str(tmpdir))
- all_metadata = job._scanFiles(tmpdir)
+ all_metadata = job._scanFiles(report, tmpdir)
self.assertThat(
all_metadata,
MatchesDict(
@@ -377,6 +380,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
distribution=archive.distribution
)
build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(ci_build=build)
job = CIBuildUploadJob.create(
build,
build.git_repository.owner,
@@ -388,7 +392,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
path = Path("conda-indep/dist/noarch/conda-indep-0.1-0.tar.bz2")
tmpdir = Path(self.useFixture(TempDir()).path)
shutil.copy2(datadir(str(path)), str(tmpdir))
- all_metadata = job._scanFiles(tmpdir)
+ all_metadata = job._scanFiles(report, tmpdir)
self.assertThat(
all_metadata,
MatchesDict(
@@ -413,6 +417,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
distribution=archive.distribution
)
build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(ci_build=build)
job = CIBuildUploadJob.create(
build,
build.git_repository.owner,
@@ -424,7 +429,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
path = Path("conda-arch/dist/linux-64/conda-arch-0.1-0.tar.bz2")
tmpdir = Path(self.useFixture(TempDir()).path)
shutil.copy2(datadir(str(path)), str(tmpdir))
- all_metadata = job._scanFiles(tmpdir)
+ all_metadata = job._scanFiles(report, tmpdir)
self.assertThat(
all_metadata,
MatchesDict(
@@ -449,6 +454,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
distribution=archive.distribution
)
build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(ci_build=build)
job = CIBuildUploadJob.create(
build,
build.git_repository.owner,
@@ -460,7 +466,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
path = Path("conda-v2-indep/dist/noarch/conda-v2-indep-0.1-0.conda")
tmpdir = Path(self.useFixture(TempDir()).path)
shutil.copy2(datadir(str(path)), str(tmpdir))
- all_metadata = job._scanFiles(tmpdir)
+ all_metadata = job._scanFiles(report, tmpdir)
self.assertThat(
all_metadata,
MatchesDict(
@@ -485,6 +491,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
distribution=archive.distribution
)
build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(ci_build=build)
job = CIBuildUploadJob.create(
build,
build.git_repository.owner,
@@ -496,7 +503,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
path = Path("conda-v2-arch/dist/linux-64/conda-v2-arch-0.1-0.conda")
tmpdir = Path(self.useFixture(TempDir()).path)
shutil.copy2(datadir(str(path)), str(tmpdir))
- all_metadata = job._scanFiles(tmpdir)
+ all_metadata = job._scanFiles(report, tmpdir)
self.assertThat(
all_metadata,
MatchesDict(
@@ -522,6 +529,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
distribution=archive.distribution
)
build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(ci_build=build)
job = CIBuildUploadJob.create(
build,
build.git_repository.owner,
@@ -537,7 +545,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
shutil.copy2(datadir(str(info_path)), str(tmpdir))
shutil.copy2(datadir(str(mod_path)), str(tmpdir))
shutil.copy2(datadir(str(zip_path)), str(tmpdir))
- all_metadata = job._scanFiles(tmpdir)
+ all_metadata = job._scanFiles(report, tmpdir)
self.assertThat(
all_metadata,
MatchesDict(
@@ -564,6 +572,85 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
),
)
+ def test__scanFiles_generic_source(self):
+ self.useFixture(FakeLogger())
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution
+ )
+ build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(
+ title="build-source", ci_build=build
+ )
+ report.update(
+ properties={
+ "name": "foo",
+ "version": "1.0",
+ "source": True,
+ }
+ )
+ job = CIBuildUploadJob.create(
+ build,
+ build.git_repository.owner,
+ archive,
+ distroseries,
+ PackagePublishingPocket.RELEASE,
+ target_channel="edge",
+ )
+ tmpdir = Path(self.useFixture(TempDir()).path)
+ (tmpdir / "foo-1.0.tar.gz").write_bytes(b"source artifact")
+ all_metadata = job._scanFiles(report, tmpdir)
+ self.assertThat(
+ all_metadata,
+ MatchesDict(
+ {
+ "foo-1.0.tar.gz": MatchesStructure.byEquality(
+ format=SourcePackageFileType.GENERIC,
+ name="foo",
+ version="1.0",
+ )
+ }
+ ),
+ )
+
+ def test__scanFiles_generic_binary(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution
+ )
+ build = self.makeCIBuild(archive.distribution)
+ report = self.factory.makeRevisionStatusReport(
+ title="build-binary", ci_build=build
+ )
+ report.update(properties={"name": "foo", "version": "1.0"})
+ job = CIBuildUploadJob.create(
+ build,
+ build.git_repository.owner,
+ archive,
+ distroseries,
+ PackagePublishingPocket.RELEASE,
+ target_channel="edge",
+ )
+ tmpdir = Path(self.useFixture(TempDir()).path)
+ (tmpdir / "test-binary").write_bytes(b"binary artifact")
+ all_metadata = job._scanFiles(report, tmpdir)
+ self.assertThat(
+ all_metadata,
+ MatchesDict(
+ {
+ "test-binary": MatchesStructure.byEquality(
+ format=BinaryPackageFormat.GENERIC,
+ name="foo",
+ version="1.0",
+ summary="",
+ description="",
+ architecturespecific=True,
+ homepage="",
+ )
+ }
+ ),
+ )
+
def test_run_indep(self):
archive = self.factory.makeArchive(
repository_format=ArchiveRepositoryFormat.PYTHON
@@ -1046,6 +1133,126 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
)
self.assertContentEqual([], archive.getAllPublishedBinaries())
+ def test_run_generic(self):
+ self.useFixture(FakeLogger())
+ archive = self.factory.makeArchive(
+ repository_format=ArchiveRepositoryFormat.GENERIC
+ )
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution
+ )
+ dases = [
+ self.factory.makeDistroArchSeries(distroseries=distroseries)
+ for _ in range(2)
+ ]
+ build = self.makeCIBuild(
+ archive.distribution, distro_arch_series=dases[0]
+ )
+ source_report = build.getOrCreateRevisionStatusReport("build-source:0")
+ source_report.setLog(b"log data")
+ source_report.update(
+ properties={
+ "name": "foo",
+ "version": "1.0",
+ "source": True,
+ }
+ )
+ source_report.attach(name="foo-1.0.tar.gz", data=b"source artifact")
+ binary_report = build.getOrCreateRevisionStatusReport("build-binary:0")
+ binary_report.setLog(b"log data")
+ binary_report.update(properties={"name": "foo", "version": "1.0"})
+ binary_report.attach(name="test-binary", data=b"binary artifact")
+ artifacts = (
+ IStore(RevisionStatusArtifact)
+ .find(
+ RevisionStatusArtifact,
+ RevisionStatusArtifact.report_id.is_in(
+ {source_report.id, binary_report.id}
+ ),
+ RevisionStatusArtifact.artifact_type
+ == RevisionStatusArtifactType.BINARY,
+ )
+ .order_by("id")
+ )
+ job = CIBuildUploadJob.create(
+ build,
+ build.git_repository.owner,
+ archive,
+ distroseries,
+ PackagePublishingPocket.RELEASE,
+ target_channel="edge",
+ )
+ transaction.commit()
+
+ with dbuser(job.config.dbuser):
+ JobRunner([job]).runAll()
+
+ self.assertThat(
+ archive.getPublishedSources(),
+ MatchesSetwise(
+ MatchesStructure(
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name
+ ),
+ sourcepackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name
+ ),
+ version=Equals("1.0"),
+ format=Equals(SourcePackageType.CI_BUILD),
+ architecturehintlist=Equals(""),
+ creator=Equals(build.git_repository.owner),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifacts[0].library_file,
+ filetype=SourcePackageFileType.GENERIC,
+ )
+ ),
+ user_defined_fields=MatchesSetwise(
+ Equals(["name", "foo"]),
+ Equals(["version", "1.0"]),
+ Equals(["source", True]),
+ ),
+ ),
+ format=Equals(SourcePackageType.CI_BUILD),
+ distroseries=Equals(distroseries),
+ )
+ ),
+ )
+ self.assertThat(
+ archive.getAllPublishedBinaries(),
+ MatchesSetwise(
+ MatchesStructure(
+ binarypackagename=MatchesStructure.byEquality(name="foo"),
+ binarypackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ binarypackagename=MatchesStructure.byEquality(
+ name="foo"
+ ),
+ version=Equals("1.0"),
+ summary=Equals(""),
+ description=Equals(""),
+ binpackageformat=Equals(BinaryPackageFormat.GENERIC),
+ architecturespecific=Is(True),
+ homepage=Equals(""),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifacts[1].library_file,
+ filetype=BinaryPackageFileType.GENERIC,
+ )
+ ),
+ user_defined_fields=MatchesSetwise(
+ Equals(["name", "foo"]),
+ Equals(["version", "1.0"]),
+ ),
+ ),
+ binarypackageformat=Equals(BinaryPackageFormat.GENERIC),
+ distroarchseries=Equals(dases[0]),
+ )
+ ),
+ )
+
def test_run_attaches_properties(self):
# The upload process attaches properties from the report as
# `SourcePackageRelease.user_defined_fields` or
