duplicity-team team mailing list archive

[Xavier Lucas <xavier.lucas@xxxxxxxxxxxx>]

Xavier Lucas has proposed merging lp:~xlucas/duplicity/pca-backend into lp:duplicity.

Requested reviews:
  duplicity-team (duplicity-team)

For more details, see:
https://code.launchpad.net/~xlucas/duplicity/pca-backend/+merge/326618
-- 
Your team duplicity-team is requested to review the proposed merge of lp:~xlucas/duplicity/pca-backend into lp:duplicity.

=== modified file 'bin/duplicity.1'
--- bin/duplicity.1	2017-06-04 16:05:58 +0000
+++ bin/duplicity.1	2017-06-30 17:02:06 +0000
@@ -1262,6 +1262,15 @@
 .B "A NOTE ON SWIFT (OPENSTACK OBJECT STORAGE) ACCESS"
 .RE
 .PP
+.BR "Public Cloud Archive" " (OVH)"
+.PP
+.RS
+pca://container_name[/prefix]
+.PP
+See also
+.B "A NOTE ON PCA ACCESS"
+.RE
+.PP
 .B "Tahoe-LAFS"
 .PP
 .RS
@@ -2074,6 +2083,40 @@
 .B SWIFT_AUTHVERSION
 is unspecified, it will default to version 1.
 
+.SH A NOTE ON PCA ACCESS
+PCA is a long-term data archival solution by OVH. It runs a slightly modified
+version of Openstack Swift introducing latency in the data retrieval process.
+It is a good pick for a
+.BR "multi backend "
+configuration where receiving volumes while an other backend is used to store
+manifests and signatures.
+
+.br
+The backend requires python-switclient to be installed on the system.
+python-keystoneclient is also needed to interact with OpenStack's Keystone
+Identity service.
+See
+.B REQUIREMENTS
+above.
+
+It uses following environment variables for authentification:
+.BR PCA_USERNAME " (required),"
+.BR PCA_PASSWORD " (required),"
+.BR PCA_AUTHURL " (required),"
+.BR PCA_USERID " (optional),"
+.BR PCA_TENANTID " (optional, but either the tenant name or tenant id must be supplied)"
+.BR PCA_REGIONNAME " (optional),"
+.BR PCA_TENANTNAME " (optional, but either the tenant name or tenant id must be supplied)"
+
+If the user was previously authenticated, the following environment
+variables can be used instead:
+.BR PCA_PREAUTHURL " (required),"
+.BR PCA_PREAUTHTOKEN " (required)"
+
+If
+.B PCA_AUTHVERSION
+is unspecified, it will default to version 2.
+
 .SH A NOTE ON MEDIAFIRE BACKEND
 This backend requires
 .B mediafire

=== added file 'duplicity/backends/pcabackend.py'
--- duplicity/backends/pcabackend.py	1970-01-01 00:00:00 +0000
+++ duplicity/backends/pcabackend.py	2017-06-30 17:02:06 +0000
@@ -0,0 +1,209 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2013 Matthieu Huin <mhu@xxxxxxxxxxxx>
+# Copyright 2017 Xavier Lucas <xavier.lucas@xxxxxxxxxxxx>
+#
+# This file is part of duplicity.
+#
+# Duplicity is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# Duplicity is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with duplicity; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+
+import duplicity.backend
+from duplicity import log
+from duplicity import util
+from duplicity.errors import BackendException
+import time
+
+
+class PCABackend(duplicity.backend.Backend):
+    """
+    Backend for OVH PCA
+    """
+    def __init__(self, parsed_url):
+        duplicity.backend.Backend.__init__(self, parsed_url)
+
+        try:
+            from swiftclient import Connection
+            from swiftclient import ClientException
+        except ImportError as e:
+            raise BackendException("""\
+PCA backend requires the python-swiftclient library.
+Exception: %s""" % str(e))
+
+        self.resp_exc = ClientException
+        self.conn_cls = Connection
+        conn_kwargs = {}
+
+        # if the user has already authenticated
+        if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ:
+            conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL']
+            conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN']
+
+        else:
+            if 'PCA_USERNAME' not in os.environ:
+                raise BackendException('PCA_USERNAME environment variable '
+                                       'not set.')
+
+            if 'PCA_PASSWORD' not in os.environ:
+                raise BackendException('PCA_PASSWORD environment variable '
+                                       'not set.')
+
+            if 'PCA_AUTHURL' not in os.environ:
+                raise BackendException('PCA_AUTHURL environment variable '
+                                       'not set.')
+
+            conn_kwargs['user'] = os.environ['PCA_USERNAME']
+            conn_kwargs['key'] = os.environ['PCA_PASSWORD']
+            conn_kwargs['authurl'] = os.environ['PCA_AUTHURL']
+
+        os_options = {}
+
+        if 'PCA_AUTHVERSION' in os.environ:
+            conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION']
+            if os.environ['PCA_AUTHVERSION'] == '3':
+                if 'PCA_USER_DOMAIN_NAME' in os.environ:
+                    os_options.update({'user_domain_name': os.environ['PCA_USER_DOMAIN_NAME']})
+                if 'PCA_USER_DOMAIN_ID' in os.environ:
+                    os_options.update({'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']})
+                if 'PCA_PROJECT_DOMAIN_NAME' in os.environ:
+                    os_options.update({'project_domain_name': os.environ['PCA_PROJECT_DOMAIN_NAME']})
+                if 'PCA_PROJECT_DOMAIN_ID' in os.environ:
+                    os_options.update({'project_domain_id': os.environ['PCA_PROJECT_DOMAIN_ID']})
+                if 'PCA_TENANTNAME' in os.environ:
+                    os_options.update({'tenant_name': os.environ['PCA_TENANTNAME']})
+                if 'PCA_ENDPOINT_TYPE' in os.environ:
+                    os_options.update({'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']})
+                if 'PCA_USERID' in os.environ:
+                    os_options.update({'user_id': os.environ['PCA_USERID']})
+                if 'PCA_TENANTID' in os.environ:
+                    os_options.update({'tenant_id': os.environ['PCA_TENANTID']})
+                if 'PCA_REGIONNAME' in os.environ:
+                    os_options.update({'region_name': os.environ['PCA_REGIONNAME']})
+
+        else:
+            conn_kwargs['auth_version'] = '2'
+        if 'PCA_TENANTNAME' in os.environ:
+            conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME']
+        if 'PCA_REGIONNAME' in os.environ:
+            os_options.update({'region_name': os.environ['PCA_REGIONNAME']})
+
+        conn_kwargs['os_options'] = os_options
+        conn_kwargs['retries'] = 0
+
+        self.conn_kwargs = conn_kwargs
+
+        # This folds the null prefix and all null parts, which means that:
+        #  //MyContainer/ and //MyContainer are equivalent.
+        #  //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
+        url_parts = [x for x in parsed_url.path.split('/') if x != '']
+
+        self.container = url_parts.pop(0)
+        if url_parts:
+            self.prefix = '%s/' % '/'.join(url_parts)
+        else:
+            self.prefix = ''
+
+        policy = 'PCA'
+        policy_header = 'X-Storage-Policy'
+
+        container_metadata = None
+        try:
+            self.conn = Connection(**self.conn_kwargs)
+            container_metadata = self.conn.head_container(self.container)
+        except ClientException:
+            pass
+        except Exception as e:
+            log.FatalError("Connection failed: %s %s"
+                           % (e.__class__.__name__, str(e)),
+                           log.ErrorCode.connection_failed)
+
+        if container_metadata is None:
+            log.Info("Creating container %s" % self.container)
+            try:
+                headers = dict([[policy_header, policy]])
+                self.conn.put_container(self.container, headers=headers)
+            except Exception as e:
+                log.FatalError("Container creation failed: %s %s"
+                               % (e.__class__.__name__, str(e)),
+                               log.ErrorCode.connection_failed)
+        elif policy and container_metadata[policy_header.lower()] != policy:
+            log.FatalError("Container '%s' exists but its storage policy is '%s' not '%s'."
+                           % (self.container, container_metadata[policy_header.lower()], policy))
+
+    def _error_code(self, operation, e):
+        if isinstance(e, self.resp_exc):
+            if e.http_status == 404:
+                return log.ErrorCode.backend_not_found
+
+    def _put(self, source_path, remote_filename):
+        self.conn.put_object(self.container, self.prefix + remote_filename,
+                             file(source_path.name))
+
+    def _get(self, remote_filename, local_path):
+        body = self.preprocess_download(remote_filename, 60)
+        if body:
+            with open(local_path.name, 'wb') as f:
+                for chunk in body:
+                    f.write(chunk)
+
+    def _list(self):
+        headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix)
+        # removes prefix from return values. should check for the prefix ?
+        return [o['name'][len(self.prefix):] for o in objs]
+
+    def _delete(self, filename):
+        self.conn.delete_object(self.container, self.prefix + filename)
+
+    def _query(self, filename):
+        sobject = self.conn.head_object(self.container, self.prefix + filename)
+        return {'size': int(sobject['content-length'])}
+
+    def preprocess_download(self, remote_filename, retry_period, wait=True):
+        body = self.unseal(remote_filename)
+        try:
+            if wait:
+                while not body:
+                    time.sleep(retry_period)
+                    self.conn = self.conn_cls(**self.conn_kwargs)
+                    body = self.unseal(remote_filename)
+                    self.conn.close()
+        except Exception as e:
+            log.FatalError("Connection failed: %s %s" % (e.__class__.__name__, str(e)),
+                           log.ErrorCode.connection_failed)
+        return body
+
+    def unseal(self, remote_filename):
+        try:
+            _, body = self.conn.get_object(self.container, self.prefix + remote_filename,
+                                           resp_chunk_size=1024)
+            log.Info("File %s was successfully unsealed." % remote_filename)
+            return body
+        except self.resp_exc, e:
+            # The object is sealed but being released.
+            if e.http_status == 429:
+                # The retry-after header contains the remaining duration before
+                # the unsealing operation completes.
+                duration = int(e.http_response_headers['Retry-After'])
+                m, s = divmod(duration, 60)
+                h, m = divmod(m, 60)
+                eta = "%dh%02dm%02ds" % (h, m, s)
+                log.Info("File %s is being unsealed, operation ETA is %s." %
+                         (remote_filename, eta))
+            else:
+                raise
+
+
+duplicity.backend.register_backend("pca", PCABackend)

=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py	2017-05-12 21:55:05 +0000
+++ duplicity/commandline.py	2017-06-30 17:02:06 +0000
@@ -947,6 +947,7 @@
   scp://%(user)s[:%(password)s]@%(other_host)s[:%(port)s]/%(some_dir)s
   ssh://%(user)s[:%(password)s]@%(other_host)s[:%(port)s]/%(some_dir)s
   swift://%(container_name)s
+  pca://%(container_name)s
   tahoe://%(alias)s/%(directory)s
   webdav://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s
   webdavs://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s