duplicity-team team mailing list archive

[David Chin <david.w.h.chin@xxxxxxxxx>]

David Chin has proposed merging lp:~david-w-h-chin/duplicity/azurebackend-bug into lp:duplicity.

Requested reviews:
  duplicity-team (duplicity-team)
Related bugs:
  Bug #1694770 in Duplicity: "azure backend not working with latest Azure Storage SDK for Python"
  https://bugs.launchpad.net/duplicity/+bug/1694770

For more details, see:
https://code.launchpad.net/~david-w-h-chin/duplicity/azurebackend-bug/+merge/330478

Updates duplicity/backends/azurebackend.py to work with azure-storage 0.36.0 (current as of 2017-09-09)
-- 
Your team duplicity-team is requested to review the proposed merge of lp:~david-w-h-chin/duplicity/azurebackend-bug into lp:duplicity.

=== modified file 'duplicity/backends/azurebackend.py'
--- duplicity/backends/azurebackend.py	2017-07-11 14:55:38 +0000
+++ duplicity/backends/azurebackend.py	2017-09-09 20:18:17 +0000
@@ -22,7 +22,6 @@
 import os
 
 import duplicity.backend
-from duplicity import globals
 from duplicity import log
 from duplicity.errors import BackendException
 
@@ -44,12 +43,8 @@
                 self.AzureMissingResourceError = azure.WindowsAzureMissingResourceError
                 self.AzureConflictError = azure.WindowsAzureConflictError
             else:
-                # v1.0.0 and above
-                import azure.storage.blob
-                if hasattr(azure.storage.blob, 'BlobService'):
-                    from azure.storage.blob import BlobService
-                else:
-                    from azure.storage.blob.blockblobservice import BlockBlobService as BlobService
+                # v 0.36.0
+                from azure.storage.blob import BlockBlobService
                 self.AzureMissingResourceError = azure.common.AzureMissingResourceHttpError
                 self.AzureConflictError = azure.common.AzureConflictHttpError
         except ImportError as e:
@@ -57,44 +52,17 @@
 Azure backend requires Microsoft Azure Storage SDK for Python (https://pypi.python.org/pypi/azure-storage/).
 Exception: %s""" % str(e))
 
+        if 'AZURE_STORAGE_ACCOUNT' not in os.environ:
+            raise BackendException('AZURE_STORAGE_ACCOUNT environment variable not set.')
+        if 'AZURE_STORAGE_KEY' not in os.environ:
+            raise BackendException('AZURE_STORAGE_KEY environment variable not set.')
+        self.block_blob_service = BlockBlobService(account_name=os.environ['AZURE_STORAGE_ACCOUNT'],
+                                        account_key=os.environ['AZURE_STORAGE_KEY'])
+
         # TODO: validate container name
         self.container = parsed_url.path.lstrip('/')
-
-        if 'AZURE_ACCOUNT_NAME' not in os.environ:
-            raise BackendException('AZURE_ACCOUNT_NAME environment variable not set.')
-
-        if 'AZURE_ACCOUNT_KEY' in os.environ:
-            self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'],
-                                            account_key=os.environ['AZURE_ACCOUNT_KEY'])
-            self._create_container()
-        elif 'AZURE_SHARED_ACCESS_SIGNATURE' in os.environ:
-            self.blob_service = BlobService(account_name=os.environ['AZURE_ACCOUNT_NAME'],
-                                            sas_token=os.environ['AZURE_SHARED_ACCESS_SIGNATURE'])
-        else:
-            raise BackendException(
-                'Neither AZURE_ACCOUNT_KEY nor AZURE_SHARED_ACCESS_SIGNATURE environment variable not set.')
-
-        if globals.azure_max_single_put_size:
-            # check if we use azure-storage>=0.30.0
-            try:
-                _ = self.blob_service.MAX_SINGLE_PUT_SIZE
-                self.blob_service.MAX_SINGLE_PUT_SIZE = globals.azure_max_single_put_size
-            # fallback for azure-storage<0.30.0
-            except AttributeError:
-                self.blob_service._BLOB_MAX_DATA_SIZE = globals.azure_max_single_put_size
-
-        if globals.azure_max_block_size:
-            # check if we use azure-storage>=0.30.0
-            try:
-                _ = self.blob_service.MAX_BLOCK_SIZE
-                self.blob_service.MAX_BLOCK_SIZE = globals.azure_max_block_size
-            # fallback for azure-storage<0.30.0
-            except AttributeError:
-                self.blob_service._BLOB_MAX_CHUNK_DATA_SIZE = globals.azure_max_block_size
-
-    def _create_container(self):
         try:
-            self.blob_service.create_container(self.container, fail_on_exist=True)
+            self.block_blob_service.create_container(self.container, fail_on_exist=True)
         except self.AzureConflictError:
             # Indicates that the resource could not be created because it already exists.
             pass
@@ -104,48 +72,29 @@
                            log.ErrorCode.connection_failed)
 
     def _put(self, source_path, remote_filename):
-        kwargs = {}
-        if globals.azure_max_connections:
-            kwargs['max_connections'] = globals.azure_max_connections
-
-        # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#upload-a-blob-into-a-container
-        try:
-            self.blob_service.create_blob_from_path(self.container, remote_filename, source_path.name, **kwargs)
-        except AttributeError:  # Old versions use a different method name
-            self.blob_service.put_block_blob_from_path(self.container, remote_filename, source_path.name, **kwargs)
+        # https://docs.microsoft.com/en-us/azure/storage/blobs/storage-python-how-to-use-blob-storage#upload-a-blob-into-a-container
+        self.block_blob_service.create_blob_from_path(self.container, remote_filename, source_path.name)
 
     def _get(self, remote_filename, local_path):
-        # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#download-blobs
-        self.blob_service.get_blob_to_path(self.container, remote_filename, local_path.name)
+        # https://docs.microsoft.com/en-us/azure/storage/blobs/storage-python-how-to-use-blob-storage#download-blobs
+        self.block_blob_service.get_blob_to_path(self.container, remote_filename, local_path.name)
 
     def _list(self):
-        # https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#list-the-blobs-in-a-container
-        blobs = []
+        # https://docs.microsoft.com/en-us/azure/storage/blobs/storage-python-how-to-use-blob-storage#list-the-blobs-in-a-container
         marker = None
-        while True:
-            batch = self.blob_service.list_blobs(self.container, marker=marker)
-            blobs.extend(batch)
-            if not batch.next_marker:
-                break
-            marker = batch.next_marker
-        return [blob.name for blob in blobs]
+        batch = self.block_blob_service.list_blobs(self.container, marker=marker)
+        return [blob.name for blob in batch]
 
     def _delete(self, filename):
-        # http://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-blob-storage/#delete-blobs
-        self.blob_service.delete_blob(self.container, filename)
+        # https://docs.microsoft.com/en-us/azure/storage/blobs/storage-python-how-to-use-blob-storage#delete-a-blob
+        self.block_blob_service.delete_blob(self.container, filename)
 
     def _query(self, filename):
-        prop = self.blob_service.get_blob_properties(self.container, filename)
-        try:
-            info = {'size': int(prop.properties.content_length)}
-        except AttributeError:
-            # old versions directly returned the properties
-            info = {'size': int(prop['content-length'])}
-        return info
+        prop = self.block_blob_service.get_blob_properties(self.container, filename)
+        return {'size': prop.properties.content_length}
 
     def _error_code(self, operation, e):
         if isinstance(e, self.AzureMissingResourceError):
             return log.ErrorCode.backend_not_found
 
-
 duplicity.backend.register_backend('azure', AzureBackend)