launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28793
[Merge] ~cjwatson/launchpad:go-proxy-archive-layout into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:go-proxy-archive-layout into launchpad:master with ~cjwatson/launchpad:ci-build-upload-make-spr as a prerequisite.
Commit message:
Handle publishing to the Go module proxy archive layout
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/426764
This follows the description of the `GOPROXY` protocol in https://go.dev/ref/mod#module-proxy. (The naming is confusing: module proxies seem to have a layout in their own right, rather than proxying to some kind of repository stored elsewhere as one might expect from the name.)
I anticipated some decisions that I expect to have to take elsewhere. Go module paths are essentially the equivalent of package names for other package types, but they typically contain slashes, which is inconvenient for Launchpad source package names since they're involved in URL traversal. Fortunately, they have a relatively restrictive set of valid characters, so I expect to mangle them into "," for source package name purposes (chosen as a character without too many unpleasant properties which isn't itself permitted in Go module paths), and I arranged to perform the corresponding demangling here.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:go-proxy-archive-layout into launchpad:master.
diff --git a/lib/lp/archivepublisher/artifactory.py b/lib/lp/archivepublisher/artifactory.py
index 2ee56ab..6970f9a 100644
--- a/lib/lp/archivepublisher/artifactory.py
+++ b/lib/lp/archivepublisher/artifactory.py
@@ -59,6 +59,14 @@ def _path_for(
"Cannot publish a Conda package with no subdir"
)
path = rootpath / subdir
+ elif repository_format == ArchiveRepositoryFormat.GO_PROXY:
+ # Go module paths contain "/" characters
+ # (https://go.dev/ref/mod#go-mod-file-ident), which are awkward to
+ # include in Launchpad source package names since they would need
+ # special URL traversal handling. "," is disallowed in Go module
+ # paths, so we use that to mangle them into source package names.
+ # Reverse this mangling for publication.
+ path = rootpath / source_name.replace(",", "/") / "@v"
else:
raise AssertionError(
"Unsupported repository format: %r" % repository_format
@@ -504,6 +512,12 @@ class ArtifactoryPool:
"*.tar.bz2",
"*.conda",
]
+ elif repository_format == ArchiveRepositoryFormat.GO_PROXY:
+ return [
+ "*.info",
+ "*.mod",
+ "*.zip",
+ ]
else:
raise AssertionError(
"Unknown repository format %r" % repository_format
diff --git a/lib/lp/archivepublisher/tests/test_artifactory.py b/lib/lp/archivepublisher/tests/test_artifactory.py
index 18af683..ee7799a 100644
--- a/lib/lp/archivepublisher/tests/test_artifactory.py
+++ b/lib/lp/archivepublisher/tests/test_artifactory.py
@@ -126,6 +126,17 @@ class TestArtifactoryPool(TestCase):
pool.pathFor(None, "foo", "1.0", pub_file),
)
+ def test_pathFor_go_proxy_with_file(self):
+ pool = self.makePool(ArchiveRepositoryFormat.GO_PROXY)
+ pub_file = FakePackageReleaseFile(b"go-module", "v1.0.zip")
+ self.assertEqual(
+ ArtifactoryPath(
+ "https://foo.example.com/artifactory/repository/";
+ "launchpad.net/go-module/@v/v1.0.zip"
+ ),
+ pool.pathFor(None, "launchpad.net,go-module", "v1.0", pub_file),
+ )
+
def test_addFile(self):
pool = self.makePool()
foo = ArtifactoryPoolTestingFile(
@@ -224,13 +235,24 @@ class TestArtifactoryPool(TestCase):
pool.getArtifactPatterns(ArchiveRepositoryFormat.CONDA),
)
- def test_getAllArtifacts(self):
+ def test_getArtifactPatterns_go_proxy(self):
+ pool = self.makePool()
+ self.assertEqual(
+ [
+ "*.info",
+ "*.mod",
+ "*.zip",
+ ],
+ pool.getArtifactPatterns(ArchiveRepositoryFormat.GO_PROXY),
+ )
+
+ def test_getAllArtifacts_debian(self):
# getAllArtifacts mostly relies on constructing a correct AQL query,
# which we can't meaningfully test without a real Artifactory
# instance, although `FakeArtifactoryFixture` tries to do something
- # with it. This test mainly ensures that we transform the response
+ # with it. These tests mainly ensure that we transform the response
# correctly.
- pool = self.makePool()
+ pool = self.makePool(ArchiveRepositoryFormat.DEBIAN)
ArtifactoryPoolTestingFile(
pool=pool,
source_name="foo",
@@ -247,22 +269,6 @@ class TestArtifactoryPool(TestCase):
release_type=FakeReleaseType.BINARY,
release_id=2,
).addToPool()
- ArtifactoryPoolTestingFile(
- pool=pool,
- source_name="bar",
- source_version="1.0",
- filename="bar-1.0.whl",
- release_type=FakeReleaseType.BINARY,
- release_id=3,
- ).addToPool()
- ArtifactoryPoolTestingFile(
- pool=pool,
- source_name="qux",
- source_version="1.0",
- filename="qux-1.0.conda",
- release_type=FakeReleaseType.BINARY,
- release_id=4,
- ).addToPool()
self.assertEqual(
{
PurePath("pool/f/foo/foo-1.0.deb"): {
@@ -280,9 +286,20 @@ class TestArtifactoryPool(TestCase):
self.repository_name, ArchiveRepositoryFormat.DEBIAN
),
)
+
+ def test_getAllArtifacts_python(self):
+ pool = self.makePool(ArchiveRepositoryFormat.PYTHON)
+ ArtifactoryPoolTestingFile(
+ pool=pool,
+ source_name="bar",
+ source_version="1.0",
+ filename="bar-1.0.whl",
+ release_type=FakeReleaseType.BINARY,
+ release_id=3,
+ ).addToPool()
self.assertEqual(
{
- PurePath("pool/b/bar/bar-1.0.whl"): {
+ PurePath("bar/1.0/bar-1.0.whl"): {
"launchpad.release-id": ["binary:3"],
"launchpad.source-name": ["bar"],
"launchpad.source-version": ["1.0"],
@@ -292,9 +309,21 @@ class TestArtifactoryPool(TestCase):
self.repository_name, ArchiveRepositoryFormat.PYTHON
),
)
+
+ def test_getAllArtifacts_conda(self):
+ pool = self.makePool(ArchiveRepositoryFormat.CONDA)
+ ArtifactoryPoolTestingFile(
+ pool=pool,
+ source_name="qux",
+ source_version="1.0",
+ filename="qux-1.0.conda",
+ release_type=FakeReleaseType.BINARY,
+ release_id=4,
+ user_defined_fields=[("subdir", "linux-64")],
+ ).addToPool()
self.assertEqual(
{
- PurePath("pool/q/qux/qux-1.0.conda"): {
+ PurePath("linux-64/qux-1.0.conda"): {
"launchpad.release-id": ["binary:4"],
"launchpad.source-name": ["qux"],
"launchpad.source-version": ["1.0"],
@@ -305,6 +334,29 @@ class TestArtifactoryPool(TestCase):
),
)
+ def test_getAllArtifacts_go_proxy(self):
+ pool = self.makePool(ArchiveRepositoryFormat.GO_PROXY)
+ ArtifactoryPoolTestingFile(
+ pool=pool,
+ source_name="launchpad.net,go-module",
+ source_version="v0.0.1",
+ filename="v0.0.1.zip",
+ release_type=FakeReleaseType.BINARY,
+ release_id=5,
+ ).addToPool()
+ self.assertEqual(
+ {
+ PurePath("launchpad.net/go-module/@v/v0.0.1.zip"): {
+ "launchpad.release-id": ["binary:5"],
+ "launchpad.source-name": ["launchpad.net,go-module"],
+ "launchpad.source-version": ["v0.0.1"],
+ },
+ },
+ pool.getAllArtifacts(
+ self.repository_name, ArchiveRepositoryFormat.GO_PROXY
+ ),
+ )
+
def test_getAllArtifacts_handles_empty_properties(self):
# AQL queries seem to return empty properties as something like
# `{"key": "pypi.requires.python"}` rather than `{"key":
diff --git a/lib/lp/archivepublisher/tests/test_pool.py b/lib/lp/archivepublisher/tests/test_pool.py
index 25c17ff..fe99e20 100644
--- a/lib/lp/archivepublisher/tests/test_pool.py
+++ b/lib/lp/archivepublisher/tests/test_pool.py
@@ -104,6 +104,7 @@ class PoolTestingFile:
filename,
release_type=FakeReleaseType.BINARY,
release_id=1,
+ user_defined_fields=None,
):
self.pool = pool
self.source_name = source_name
@@ -113,6 +114,7 @@ class PoolTestingFile:
filename,
release_type=release_type,
release_id=release_id,
+ user_defined_fields=user_defined_fields,
)
def addToPool(self, component: str):
diff --git a/lib/lp/soyuz/enums.py b/lib/lp/soyuz/enums.py
index 93f4a70..aafde4c 100644
--- a/lib/lp/soyuz/enums.py
+++ b/lib/lp/soyuz/enums.py
@@ -715,3 +715,10 @@ class ArchiveRepositoryFormat(DBEnumeratedType):
A Conda channel
(https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/channels.html).
""")
+
+ GO_PROXY = DBItem(3, """
+ Go
+
+ A Go registry, laid out as a module proxy
+ (https://go.dev/ref/mod#module-proxy).
+ """)
Follow ups
