duplicity-team team mailing list archive
-
duplicity-team team
-
Mailing list archive
-
Message #01682
[Merge] lp:~mhu-s/duplicity/swiftbackend into lp:duplicity
Matthieu Huin has proposed merging lp:~mhu-s/duplicity/swiftbackend into lp:duplicity.
Requested reviews:
duplicity-team (duplicity-team)
For more details, see:
https://code.launchpad.net/~mhu-s/duplicity/swiftbackend/+merge/169517
Hello,
This branch adds support for Swift, the OpenStack Object Storage service. See https://blueprints.launchpad.net/duplicity/+spec/swiftbackend
--
https://code.launchpad.net/~mhu-s/duplicity/swiftbackend/+merge/169517
Your team duplicity-team is requested to review the proposed merge of lp:~mhu-s/duplicity/swiftbackend into lp:duplicity.
=== modified file 'bin/duplicity.1'
--- bin/duplicity.1 2013-04-27 14:10:11 +0000
+++ bin/duplicity.1 2013-06-14 18:32:28 +0000
@@ -24,6 +24,13 @@
.B Cloud Files Python API
- http://www.rackspace.com/knowledge_center/article/python-api-installation-for-cloud-files
.TP
+.BR "OpenStack Swift backend"
+.B Python swiftclient module
+- https://github.com/openstack/python-swiftclient/
+.br
+.B Python keystoneclient module
+- https://github.com/openstack/python-keystoneclient/
+.TP
.B "ftp backend"
.B NcFTP Client
- http://www.ncftp.com/
@@ -991,6 +998,11 @@
and
.BR "A NOTE ON SSH BACKENDS" .
.PP
+swift://container_name
+.br
+See also
+.B "A NOTE ON OPENSTACK SWIFT ACCESS"
+.PP
tahoe://alias/directory
.PP
.BI "Ubuntu One"
@@ -1327,6 +1339,33 @@
.I must
be set in order to use other cloud files providers.
+.SH A NOTE ON OPENSTACK SWIFT ACCESS
+
+Swift is the OpenStack Object Storage service.
+
+The backend requires python-switclient to be installed on the system.
+python-keystoneclient is also needed to use OpenStack's Keystone Identity service.
+See
+.B REQUIREMENTS
+above.
+
+It uses four environment variables for authentification:
+.BR SWIFT_USERNAME " (required),"
+.BR SWIFT_PASSWORD " (required),"
+.BR SWIFT_AUTHURL " (required),"
+.BR SWIFT_TENANTNAME " (optional, the tenant can be included in the username)"
+
+If the user was previously authenticated, the following environment
+variables can be used instead:
+.BR SWIFT_PREAUTHURL " (required),"
+.BR SWIFT_PREAUTHTOKEN " (required)"
+
+If
+.B SWIFT_AUTHVERSION
+is unspecified, it will default to version 1.
+
+
+
.SH A NOTE ON EUROPEAN S3 BUCKETS
Amazon S3 provides the ability to choose the location of a bucket upon
its creation. The purpose is to enable the user to choose a location
=== added file 'duplicity/backends/swiftbackend.py'
--- duplicity/backends/swiftbackend.py 1970-01-01 00:00:00 +0000
+++ duplicity/backends/swiftbackend.py 2013-06-14 18:32:28 +0000
@@ -0,0 +1,202 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2013 Matthieu Huin <mhu@xxxxxxxxxxxx>
+#
+# This file is part of duplicity.
+#
+# Duplicity is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# Duplicity is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with duplicity; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+import time
+
+import duplicity.backend
+from duplicity import globals
+from duplicity import log
+from duplicity.errors import * #@UnusedWildImport
+from duplicity.util import exception_traceback
+from duplicity.backend import retry
+
+class SwiftBackend(duplicity.backend.Backend):
+ """
+ Backend for Swift
+ """
+ def __init__(self, parsed_url):
+ try:
+ from swiftclient import Connection
+ from swiftclient import ClientException
+ except ImportError:
+ raise BackendException("This backend requires "
+ "the python-swiftclient library.")
+
+ self.resp_exc = ClientException
+ conn_kwargs = {}
+
+ # if the user has already authenticated
+ if os.environ.has_key('SWIFT_PREAUTHURL') and os.environ.has_key('SWIFT_PREAUTHTOKEN'):
+ conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
+ conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
+
+ else:
+ if not os.environ.has_key('SWIFT_USERNAME'):
+ raise BackendException('SWIFT_USERNAME environment variable '
+ 'not set.')
+
+ if not os.environ.has_key('SWIFT_PASSWORD'):
+ raise BackendException('SWIFT_PASSWORD environment variable '
+ 'not set.')
+
+ if not os.environ.has_key('SWIFT_AUTHURL'):
+ raise BackendException('SWIFT_AUTHURL environment variable '
+ 'not set.')
+
+ conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
+ conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
+ conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
+
+ if os.environ.has_key('SWIFT_AUTHVERSION'):
+ conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
+ else:
+ conn_kwargs['auth_version'] = '1'
+ if os.environ.has_key('SWIFT_TENANTNAME'):
+ conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
+
+ self.container = parsed_url.path.lstrip('/')
+
+ try:
+ self.conn = Connection(**conn_kwargs)
+ self.conn.put_container(self.container)
+ except Exception, e:
+ log.FatalError("Connection failed: %s %s"
+ % (e.__class__.__name__, str(e)),
+ log.ErrorCode.connection_failed)
+
+ def put(self, source_path, remote_filename = None):
+ if not remote_filename:
+ remote_filename = source_path.get_filename()
+
+ for n in range(1, globals.num_retries+1):
+ log.Info("Uploading '%s/%s' " % (self.container, remote_filename))
+ try:
+ self.conn.put_object(self.container,
+ remote_filename,
+ file(source_path.name))
+ return
+ except self.resp_exc, error:
+ log.Warn("Upload of '%s' failed (attempt %d): Swift server returned: %s %s"
+ % (remote_filename, n, error.http_status, error.message))
+ except Exception, e:
+ log.Warn("Upload of '%s' failed (attempt %s): %s: %s"
+ % (remote_filename, n, e.__class__.__name__, str(e)))
+ log.Debug("Backtrace of previous error: %s"
+ % exception_traceback())
+ time.sleep(30)
+ log.Warn("Giving up uploading '%s' after %s attempts"
+ % (remote_filename, globals.num_retries))
+ raise BackendException("Error uploading '%s'" % remote_filename)
+
+ def get(self, remote_filename, local_path):
+ for n in range(1, globals.num_retries+1):
+ log.Info("Downloading '%s/%s'" % (self.container, remote_filename))
+ try:
+ headers, body = self.conn.get_object(self.container,
+ remote_filename)
+ f = open(local_path.name, 'w')
+ for chunk in body:
+ f.write(chunk)
+ local_path.setdata()
+ return
+ except self.resp_exc, resperr:
+ log.Warn("Download of '%s' failed (attempt %s): Swift server returned: %s %s"
+ % (remote_filename, n, resperr.http_status, resperr.message))
+ except Exception, e:
+ log.Warn("Download of '%s' failed (attempt %s): %s: %s"
+ % (remote_filename, n, e.__class__.__name__, str(e)))
+ log.Debug("Backtrace of previous error: %s"
+ % exception_traceback())
+ time.sleep(30)
+ log.Warn("Giving up downloading '%s' after %s attempts"
+ % (remote_filename, globals.num_retries))
+ raise BackendException("Error downloading '%s/%s'"
+ % (self.container, remote_filename))
+
+ def list(self):
+ for n in range(1, globals.num_retries+1):
+ log.Info("Listing '%s'" % (self.container))
+ try:
+ # Cloud Files will return a max of 10,000 objects. We have
+ # to make multiple requests to get them all.
+ headers, objs = self.conn.get_container(self.container)
+ return [ o['name'] for o in objs ]
+ except self.resp_exc, resperr:
+ log.Warn("Listing of '%s' failed (attempt %s): Swift server returned: %s %s"
+ % (self.container, n, resperr.http_status, resperr.message))
+ except Exception, e:
+ log.Warn("Listing of '%s' failed (attempt %s): %s: %s"
+ % (self.container, n, e.__class__.__name__, str(e)))
+ log.Debug("Backtrace of previous error: %s"
+ % exception_traceback())
+ time.sleep(30)
+ log.Warn("Giving up listing of '%s' after %s attempts"
+ % (self.container, globals.num_retries))
+ raise BackendException("Error listing '%s'"
+ % (self.container))
+
+ def delete_one(self, remote_filename):
+ for n in range(1, globals.num_retries+1):
+ log.Info("Deleting '%s/%s'" % (self.container, remote_filename))
+ try:
+ self.conn.delete_object(self.container, remote_filename)
+ return
+ except self.resp_exc, resperr:
+ if n > 1 and resperr.http_status == 404:
+ # We failed on a timeout, but delete succeeded on the server
+ log.Warn("Delete of '%s' missing after retry - must have succeded earlier" % remote_filename )
+ return
+ log.Warn("Delete of '%s' failed (attempt %s): Swift server returned: %s %s"
+ % (remote_filename, n, resperr.http_status, resperr.message))
+ except Exception, e:
+ log.Warn("Delete of '%s' failed (attempt %s): %s: %s"
+ % (remote_filename, n, e.__class__.__name__, str(e)))
+ log.Debug("Backtrace of previous error: %s"
+ % exception_traceback())
+ time.sleep(30)
+ log.Warn("Giving up deleting '%s' after %s attempts"
+ % (remote_filename, globals.num_retries))
+ raise BackendException("Error deleting '%s/%s'"
+ % (self.container, remote_filename))
+
+ def delete(self, filename_list):
+ for file in filename_list:
+ self.delete_one(file)
+ log.Debug("Deleted '%s/%s'" % (self.container, file))
+
+ @retry
+ def _query_file_info(self, filename, raise_errors=False):
+ try:
+ sobject = self.conn.head_object(self.container, filename)
+ return {'size': long(sobject['content-length'])}
+ except self.resp_exc:
+ return {'size': -1}
+ except Exception, e:
+ log.Warn("Error querying '%s/%s': %s"
+ "" % (self.container,
+ filename,
+ str(e)))
+ if raise_errors:
+ raise e
+ else:
+ return {'size': None}
+
+duplicity.backend.register_backend("swift", SwiftBackend)
=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py 2013-04-27 14:48:39 +0000
+++ duplicity/commandline.py 2013-06-14 18:32:28 +0000
@@ -826,6 +826,7 @@
s3+http://%(bucket_name)s[/%(prefix)s]
scp://%(user)s[:%(password)s]@%(other_host)s[:%(port)s]/%(some_dir)s
ssh://%(user)s[:%(password)s]@%(other_host)s[:%(port)s]/%(some_dir)s
+ swift://%(container_name)s
tahoe://%(alias)s/%(directory)s
webdav://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s
webdavs://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s
Follow ups
