launchpad-reviewers team mailing list archive

[Colin Watson <mp+419405@xxxxxxxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/launchpad:gitlab-blob into launchpad:master.

Commit message:
Support fetching individual files from GitLab repositories

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)
Related bugs:
  Bug #1968904 in Launchpad itself: "Snap builder does not detect base series properly for repositories hosted on gitlab.com"
  https://bugs.launchpad.net/launchpad/+bug/1968904

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/419405

In order to work out how to dispatch snap builds, we need the contents of `snapcraft.yaml`, but in the context where we need that we can't really clone the git repository.  As a result, we need to know how to fetch individual files from git repositories, which is non-standard and has to be implemented separately for each hosting provider.  Implement this for GitLab.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:gitlab-blob into launchpad:master.

diff --git a/lib/lp/code/model/gitref.py b/lib/lp/code/model/gitref.py
index 08bb3bc..7a0d828 100644
--- a/lib/lp/code/model/gitref.py
+++ b/lib/lp/code/model/gitref.py
@@ -790,6 +790,32 @@ def _fetch_blob_from_github(repository_url, ref_path, filename):
     return response.content
 
 
+def _fetch_blob_from_gitlab(repository_url, ref_path, filename):
+    repo_url = repository_url.strip("/")
+    if repo_url.endswith(".git"):
+        repo_url = repo_url[:-len(".git")]
+    try:
+        response = urlfetch(
+            "%s/-/raw/%s/%s" % (
+                repo_url,
+                # GitLab supports either branch or tag names here, but both
+                # must be shortened.  (If both a branch and a tag exist with
+                # the same name, it appears to pick the tag.)
+                quote(re.sub(r"^refs/(?:heads|tags)/", "", ref_path)),
+                quote(filename)),
+            use_proxy=True)
+    except requests.RequestException as e:
+        if (e.response is not None and
+                e.response.status_code == requests.codes.NOT_FOUND):
+            raise GitRepositoryBlobNotFound(
+                repository_url, filename, rev=ref_path)
+        else:
+            raise GitRepositoryScanFault(
+                "Failed to get file from Git repository at %s: %s" %
+                (repository_url, str(e)))
+    return response.content
+
+
 def _fetch_blob_from_launchpad(repository_url, ref_path, filename):
     repo_path = urlsplit(repository_url).path.strip("/")
     try:
@@ -924,6 +950,9 @@ class GitRefRemote(GitRefMixin):
                 len(url.path.strip("/").split("/")) == 2):
             return _fetch_blob_from_github(
                 self.repository_url, self.path, filename)
+        if url.hostname == "gitlab.com":
+            return _fetch_blob_from_gitlab(
+                self.repository_url, self.path, filename)
         if (url.hostname == "git.launchpad.net" and
                 config.vhost.mainsite.hostname != "launchpad.net"):
             # Even if this isn't launchpad.net, we can still retrieve files
diff --git a/lib/lp/code/model/tests/test_gitref.py b/lib/lp/code/model/tests/test_gitref.py
index f18287a..c1b9c97 100644
--- a/lib/lp/code/model/tests/test_gitref.py
+++ b/lib/lp/code/model/tests/test_gitref.py
@@ -479,6 +479,62 @@ class TestGitRefGetBlob(TestCaseWithFactory):
         self.assertRaises(
             GitRepositoryBlobUnsupportedRemote, ref.getBlob, "dir/file")
 
+    @responses.activate
+    def test_remote_gitlab_branch(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://gitlab.com/owner/name";,
+            path="refs/heads/path")
+        responses.add(
+            "GET", "https://gitlab.com/owner/name/-/raw/path/dir/file";,
+            body=b"foo")
+        self.assertEqual(b"foo", ref.getBlob("dir/file"))
+
+    @responses.activate
+    def test_remote_gitlab_tag(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://gitlab.com/owner/name";,
+            path="refs/tags/1.0")
+        responses.add(
+            "GET", "https://gitlab.com/owner/name/-/raw/1.0/dir/file";,
+            body=b"foo")
+        self.assertEqual(b"foo", ref.getBlob("dir/file"))
+
+    @responses.activate
+    def test_remote_gitlab_HEAD(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://gitlab.com/owner/name";, path="HEAD")
+        responses.add(
+            "GET", "https://gitlab.com/owner/name/-/raw/HEAD/dir/file";,
+            body=b"foo")
+        self.assertEqual(b"foo", ref.getBlob("dir/file"))
+
+    @responses.activate
+    def test_remote_gitlab_trailing_dot_git(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://gitlab.com/owner/name.git";, path="HEAD")
+        responses.add(
+            "GET", "https://gitlab.com/owner/name/-/raw/HEAD/dir/file";,
+            body=b"foo")
+        self.assertEqual(b"foo", ref.getBlob("dir/file"))
+
+    @responses.activate
+    def test_remote_gitlab_404(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://gitlab.com/owner/name";, path="HEAD")
+        responses.add(
+            "GET", "https://gitlab.com/owner/name/-/raw/HEAD/dir/file";,
+            status=404)
+        self.assertRaises(GitRepositoryBlobNotFound, ref.getBlob, "dir/file")
+
+    @responses.activate
+    def test_remote_gitlab_error(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://gitlab.com/owner/name";, path="HEAD")
+        responses.add(
+            "GET", "https://gitlab.com/owner/name/-/raw/HEAD/dir/file";,
+            status=500)
+        self.assertRaises(GitRepositoryScanFault, ref.getBlob, "dir/file")
+
     def test_remote_unknown_host(self):
         ref = self.factory.makeGitRefRemote(
             repository_url="https://example.com/foo";)