launchpad-reviewers team mailing list archive

[Ioana Lasc <ioana.lasc@xxxxxxxxxxxxx>]

Ioana Lasc has proposed merging ~ilasc/launchpad:upload-gzipped-oci-layers into launchpad:master.

Commit message:
Upload oci layers to dockerhub registry as tar.gz

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~ilasc/launchpad/+git/launchpad/+merge/404373
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~ilasc/launchpad:upload-gzipped-oci-layers into launchpad:master.

diff --git a/lib/lp/oci/model/ociregistryclient.py b/lib/lp/oci/model/ociregistryclient.py
index 98b925e..7261b50 100644
--- a/lib/lp/oci/model/ociregistryclient.py
+++ b/lib/lp/oci/model/ociregistryclient.py
@@ -165,20 +165,27 @@ class OCIRegistryClient:
         try:
             with tarfile.open(fileobj=lfa, mode='r|gz') as un_zipped:
                 for tarinfo in un_zipped:
-                    if tarinfo.name != 'layer.tar':
-                        continue
-                    fileobj = un_zipped.extractfile(tarinfo)
-                    # XXX Work around requests handling of objects that have
-                    # fileno, but error on access in python3:
-                    # https://github.com/psf/requests/pull/5239
-                    fileobj.len = tarinfo.size
-                    try:
+                    if tarinfo.name == 'layer.tar':
+                        fileobj = un_zipped.extractfile(tarinfo)
+                        # XXX Work around requests handling of objects that have
+                        # fileno, but error on access in python3:
+                        # https://github.com/psf/requests/pull/5239
+                        fileobj.len = tarinfo.size
+                        try:
+                            cls._upload(
+                                digest, push_rule, fileobj, tarinfo.size,
+                                http_client)
+                        finally:
+                            fileobj.close()
+                        return tarinfo.size
+                    else:
+                        size = lfa.content.filesize
+                        wrapper = LibraryFileAliasWrapper(lfa)
+                        wrapper.len = size
                         cls._upload(
-                            digest, push_rule, fileobj, tarinfo.size,
+                            digest, push_rule, wrapper, size,
                             http_client)
-                    finally:
-                        fileobj.close()
-                    return tarinfo.size
+                        return size
         finally:
             lfa.close()
 
@@ -688,6 +695,25 @@ class BearerTokenRegistryClient(RegistryHTTPClient):
             raise
 
 
+class LibraryFileAliasWrapper:
+
+    def __init__(self, lfa):
+        self.lfa = lfa
+        self.position = 0
+
+    def __len__(self):
+        return self.lfa.content.filesize - self.position
+
+    def read(self, length=-1):
+        chunksize = None if length == -1 else length
+        data = self.lfa.read(chunksize=chunksize)
+        if chunksize is None:
+            self.position = self.lfa.content.filesize
+        else:
+            self.position += length
+        return data
+
+
 class AWSAuthenticatorMixin:
     """Basic class to override the way we get credentials, exchanging
     registered aws_access_key_id and aws_secret_access_key with the
diff --git a/lib/lp/oci/tests/test_ociregistryclient.py b/lib/lp/oci/tests/test_ociregistryclient.py
index 286bdce..974eb5a 100644
--- a/lib/lp/oci/tests/test_ociregistryclient.py
+++ b/lib/lp/oci/tests/test_ociregistryclient.py
@@ -23,7 +23,6 @@ from requests.exceptions import (
     HTTPError,
     )
 import responses
-from tenacity import RetryError
 from testtools.matchers import (
     AfterPreprocessing,
     ContainsDict,
@@ -1006,6 +1005,30 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
             HTTPError, self.client.uploadManifestList,
             build_request, [self.build])
 
+    @responses.activate
+    def test_upload_layer_put_gziped_blob(self):
+        lfa = self.factory.makeLibraryFileAlias(
+            content=LaunchpadWriteTarFile.files_to_bytes(
+                {"6d56becb66b184f.tar.gz": b"test gzipped layer"}))
+        transaction.commit()
+        push_rule = self.build.recipe.push_rules[0]
+        http_client = RegistryHTTPClient(push_rule)
+        blobs_url = "{}/blobs/{}".format(
+            http_client.api_url, "test-digest")
+        uploads_url = "{}/blobs/uploads/".format(http_client.api_url)
+        upload_url = "{}/blobs/uploads/{}".format(
+            http_client.api_url, uuid.uuid4())
+        responses.add("HEAD", blobs_url, status=404)
+        responses.add("POST", uploads_url, headers={"Location": upload_url})
+        responses.add("PUT", upload_url, status=201)
+        self.client._upload_layer("test-digest", push_rule, lfa, http_client)
+        self.assertThat(responses.calls[2].request, MatchesStructure(
+            method=Equals("PUT"),
+            headers=ContainsDict({
+                "Content-Length": Equals(str(lfa.content.filesize)),
+                }),
+            ))
+
 
 class TestRegistryHTTPClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
                              TestCaseWithFactory):