duplicity-team team mailing list archive
-
duplicity-team team
-
Mailing list archive
-
Message #01536
[Merge] lp:~ed.so/duplicity/webdav.fix-retry into lp:duplicity
edso has proposed merging lp:~ed.so/duplicity/webdav.fix-retry into lp:duplicity.
Requested reviews:
duplicity-team (duplicity-team)
For more details, see:
https://code.launchpad.net/~ed.so/duplicity/webdav.fix-retry/+merge/142759
webdav
- added ssl certificate verification (see man page)
- more robust retry routine to survive ssl errors, broken pipe errors
- added http redirect support
--
https://code.launchpad.net/~ed.so/duplicity/webdav.fix-retry/+merge/142759
Your team duplicity-team is requested to review the proposed merge of lp:~ed.so/duplicity/webdav.fix-retry into lp:duplicity.
=== modified file 'bin/duplicity.1'
--- bin/duplicity.1 2012-12-25 11:07:32 +0000
+++ bin/duplicity.1 2013-01-10 19:14:20 +0000
@@ -64,6 +64,7 @@
.B ssh pexpect backend
.B sftp/scp client binaries
OpenSSH - http://www.openssh.com/
+<<<<<<< TREE
.TP
.B "Ubuntu One"
.B httplib2
@@ -73,6 +74,13 @@
.B oauthlib
(python OAuth request-signing logic)
- http://pypi.python.org/pypi/oauthlib
+=======
+.TP
+.B "webdav backend"
+.B ca cert file
+for ssl certificate verification of HTTPS connections
+- http://curl.haxx.se/docs/caextract.html
+>>>>>>> MERGE-SOURCE
.SH SYNOPSIS
.B duplicity [full|incremental]
@@ -773,6 +781,22 @@
.RE
.TP
+.BI "--ssl-cacert-file " file
+.B (only webdav backend)
+Provide a cacert file for ssl certificate verification.
+.br
+See also
+.BR "A NOTE ON SSL CERTIFICATE VERIFICATION" .
+
+.TP
+.B --ssl-no-check-certificate
+.B (only webdav backend)
+Disable ssl certificate verification.
+.br
+See also
+.BR "A NOTE ON SSL CERTIFICATE VERIFICATION" .
+
+.TP
.BI "--tempdir " directory
Use this existing directory for duplicity temporary files instead of
the system default, which is usually the /tmp directory. This option
@@ -1375,6 +1399,7 @@
in addition for this backend mode to work properly. Sftp does not have these limitations but needs
an sftp service running on the backend server, which is sometimes not an option.
+<<<<<<< TREE
.SH A NOTE ON UBUNTU ONE
To use Ubuntu One you must have an Ubuntu One OAuth access token. Such
@@ -1389,6 +1414,40 @@
acquired token is then printed to the console.
.PP
See https://one.ubuntu.com/ for more information about Ubuntu One.
+=======
+.SH A NOTE ON SSL CERTIFICATE VERIFICATION
+Certificate verification as implemented right now [01.2013] only in the webdav backend needs a file
+based database of certification authority certificates (cacert file). It has to be a
+.B PEM
+formatted text file as currently provided by the
+.B CURL
+project. See
+.PP
+.RS
+http://curl.haxx.se/docs/caextract.html
+.PP
+.RE
+After creating/retrieving a valid cacert file you should copy it to either
+.PP
+.RS
+~/.duplicity/cacert.pem
+.br
+~/duplicity_cacert.pem
+.br
+/etc/duplicity/cacert.pem
+.PP
+.RE
+Duplicity searches it there in the same order and will fail if it can't find it.
+You can however specify the option
+.BI --ssl-cacert-file " <file>"
+to point duplicity to a copy in a different location.
+.PP
+Finally there is the
+.B --ssl-no-check-certificate
+option to disable certificate verification alltogether, in case some ssl library
+is missing or verification is not wanted. Use it with care, as even with self signed
+servers manually providing the private ca certificate is definitely the safer option.
+>>>>>>> MERGE-SOURCE
.SH A NOTE ON SYMMETRIC ENCRYPTION AND SIGNING
Signing and symmetrically encrypt at the same time with the gpg binary on the
@@ -1409,6 +1468,20 @@
.BI PASSPHRASE
for symmetric encryption and the passphrase of the signing key are identical.
+.SH A NOTE ON UBUNTU ONE
+
+To use Ubuntu One you must have an Ubuntu One OAuth access token. Such
+OAuth tokens have a practically unlimited lifetime; you can have multiple
+active tokens and you can revoke tokens using the Ubuntu One web interface.
+.PP
+duplicity expects th token in the environment variable FTP_PASSWORD
+(in the format "consumer_key:consumer_secret:token:token_secret"). If no
+token is present, duplicity asks for your Ubuntu One email address and password
+and requests an access token from the Ubuntu SSO service. The newly
+acquired token is then printed to the console.
+.PP
+See https://one.ubuntu.com/ for more information about Ubuntu One.
+
.SH KNOWN ISSUES / BUGS
Hard links currently unsupported (they will be treated as non-linked
regular files).
=== modified file 'duplicity/backend.py'
--- duplicity/backend.py 2012-12-12 17:45:38 +0000
+++ duplicity/backend.py 2013-01-10 19:14:20 +0000
@@ -42,7 +42,7 @@
from duplicity.util import exception_traceback
-from duplicity.errors import BackendException
+from duplicity.errors import BackendException, FatalBackendError
from duplicity.errors import TemporaryLoadException
from duplicity.errors import ConflictingScheme
from duplicity.errors import InvalidBackendURL
@@ -333,26 +333,35 @@
# as we don't know what the underlying code comes up with and we really *do*
# want to retry globals.num_retries times under all circumstances
def retry_fatal(fn):
- def iterate(*args):
- for n in range(1, globals.num_retries):
- try:
- return fn(*args)
- except Exception, e:
- log.Warn("Attempt %s failed. %s: %s"
- % (n, e.__class__.__name__, str(e)))
- log.Debug("Backtrace of previous error: %s"
- % exception_traceback())
- time.sleep(10) # wait a bit before trying again
+ def _retry_fatal(self, *args):
+ try:
+ n = 0
+ for n in range(1, globals.num_retries):
+ try:
+ self.retry_count = n
+ return fn(self, *args)
+ except FatalBackendError, e:
+ # die on fatal errors
+ raise e
+ except Exception, e:
+ # retry on anything else
+ log.Warn("Attempt %s failed. %s: %s"
+ % (n, e.__class__.__name__, str(e)))
+ log.Debug("Backtrace of previous error: %s"
+ % exception_traceback())
+ time.sleep(10) # wait a bit before trying again
# final trial, die on exception
- try:
- return fn(*args)
+ self.retry_count = n+1
+ return fn(self, *args)
except Exception, e:
log.FatalError("Giving up after %s attempts. %s: %s"
- % (globals.num_retries, e.__class__.__name__, str(e)),
+ % (self.retry_count, e.__class__.__name__, str(e)),
log.ErrorCode.backend_error)
log.Debug("Backtrace of previous error: %s"
% exception_traceback())
- return iterate
+ self.retry_count = 0
+
+ return _retry_fatal
class Backend:
"""
@@ -371,6 +380,7 @@
- move
"""
+
def __init__(self, parsed_url):
self.parsed_url = parsed_url
=== modified file 'duplicity/backends/webdavbackend.py'
--- duplicity/backends/webdavbackend.py 2012-12-09 14:36:19 +0000
+++ duplicity/backends/webdavbackend.py 2013-01-10 19:14:20 +0000
@@ -2,6 +2,8 @@
#
# Copyright 2002 Ben Escoto <ben@xxxxxxxxxxx>
# Copyright 2007 Kenneth Loafman <kenneth@xxxxxxxxxxx>
+# Copyright 2013 Edgar Soldin
+# - ssl cert verification, some robustness enhancements
#
# This file is part of duplicity.
#
@@ -20,7 +22,7 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import base64
-import httplib
+import httplib, os
import re
import urllib
import urllib2
@@ -46,6 +48,58 @@
def get_method(self):
return self.method
+class VerifiedHTTPSConnection(httplib.HTTPSConnection):
+ def __init__(self, *args, **kwargs):
+ try:
+ global socket, ssl
+ import socket, ssl
+ except ImportError:
+ raise FatalBackendError("Missing socket or ssl libraries.")
+
+ httplib.HTTPSConnection.__init__(self, *args, **kwargs)
+
+ self.cacert_file = globals.ssl_cacert_file
+ cacert_candidates = [ "~/.duplicity/cacert.pem", \
+ "~/duplicity_cacert.pem", \
+ "/etc/duplicity/cacert.pem" ]
+ #
+ if not self.cacert_file:
+ for path in cacert_candidates :
+ path = os.path.expanduser(path)
+ if (os.path.isfile(path)):
+ self.cacert_file = path
+ break
+ # still no cacert file, inform user
+ if not self.cacert_file:
+ raise FatalBackendError("""For certificate verification a cacert database file is needed in one of these locations: %s
+Hints:
+ Consult the man page, chapter 'SSL Certificate Verification'.
+ Consider using the options --ssl-cacert-file, --ssl-no-check-certificate .""" % ", ".join(cacert_candidates) )
+ # check if file is accessible (libssl errors are not very detailed)
+ if not os.access(self.cacert_file, os.R_OK):
+ raise FatalBackendError("Cacert database file '%s' is not readable." % cacert_file)
+
+ def connect(self):
+ # create new socket
+ sock = socket.create_connection((self.host, self.port),
+ self.timeout)
+ if self._tunnel_host:
+ self.sock = sock
+ self._tunnel()
+
+ # wrap the socket in ssl using verification
+ self.sock = ssl.wrap_socket(sock,
+ cert_reqs=ssl.CERT_REQUIRED,
+ ca_certs=self.cacert_file,
+ )
+
+ def request(self, *args, **kwargs):
+ try:
+ return httplib.HTTPSConnection.request(self, *args, **kwargs)
+ except ssl.SSLError, e:
+ # encapsulate ssl errors
+ raise BackendException("SSL failed: %s" % str(e),log.ErrorCode.backend_error)
+
class WebDAVBackend(duplicity.backend.Backend):
"""Backend for accessing a WebDAV repository.
@@ -70,23 +124,22 @@
self.digest_challenge = None
self.digest_auth_handler = None
- if parsed_url.path:
+ self.username = parsed_url.username
+ self.password = self.get_password()
+ self.directory = self._sanitize_path(parsed_url.path)
+
+ log.Info("Using WebDAV protocol %s" % (globals.webdav_proto,))
+ log.Info("Using WebDAV host %s port %s" % (parsed_url.hostname, parsed_url.port))
+ log.Info("Using WebDAV directory %s" % (self.directory,))
+
+ self.conn = None
+
+ def _sanitize_path(self,path):
+ if path:
foldpath = re.compile('/+')
- self.directory = foldpath.sub('/', parsed_url.path + '/' )
- else:
- self.directory = '/'
-
- log.Info("Using WebDAV host %s" % (parsed_url.hostname,))
- log.Info("Using WebDAV port %s" % (parsed_url.port,))
- log.Info("Using WebDAV directory %s" % (self.directory,))
- log.Info("Using WebDAV protocol %s" % (globals.webdav_proto,))
-
- if parsed_url.scheme == 'webdav':
- self.conn = httplib.HTTPConnection(parsed_url.hostname, parsed_url.port)
- elif parsed_url.scheme == 'webdavs':
- self.conn = httplib.HTTPSConnection(parsed_url.hostname, parsed_url.port)
- else:
- raise BackendException("Unknown URI scheme: %s" % (parsed_url.scheme))
+ return foldpath.sub('/', path + '/' )
+ else:
+ return '/'
def _getText(self,nodelist):
rc = ""
@@ -94,29 +147,76 @@
if node.nodeType == node.TEXT_NODE:
rc = rc + node.data
return rc
+
+ def _connect(self, forced=False):
+ """
+ Connect or re-connect to the server, updates self.conn
+ # reconnect on errors as a precaution, there are errors e.g.
+ # "[Errno 32] Broken pipe" or SSl errors that render the connection unusable
+ """
+ if self.retry_count<=1 and self.conn \
+ and self.conn.host == self.parsed_url.hostname: return
+
+ log.Info("WebDAV create connection on '%s' (retry %s) " % (self.parsed_url.hostname,self.retry_count) )
+ if self.conn: self.conn.close()
+ # http schemes needed for redirect urls from servers
+ if self.parsed_url.scheme in ['webdav','http']:
+ self.conn = httplib.HTTPConnection(self.parsed_url.hostname, self.parsed_url.port)
+ elif self.parsed_url.scheme in ['webdavs','https']:
+ if globals.ssl_no_check_certificate:
+ self.conn = httplib.HTTPSConnection(self.parsed_url.hostname, self.parsed_url.port)
+ else:
+ self.conn = VerifiedHTTPSConnection(self.parsed_url.hostname, self.parsed_url.port)
+ else:
+ raise FatalBackendError("WebDAV Unknown URI scheme: %s" % (self.parsed_url.scheme))
def close(self):
self.conn.close()
- def request(self, method, path, data=None):
+ def request(self, method, path, data=None, redirected=0):
"""
Wraps the connection.request method to retry once if authentication is
required
"""
- quoted_path = urllib.quote(path)
+ self._connect()
+
+ quoted_path = urllib.quote(path,"/:~")
if self.digest_challenge is not None:
self.headers['Authorization'] = self.get_digest_authorization(path)
+
+ log.Info("WebDAV %s %s request with headers: %s " % (method,quoted_path,self.headers))
+ log.Info("WebDAV data length: %s " % len(str(data)) )
self.conn.request(method, quoted_path, data, self.headers)
response = self.conn.getresponse()
- if response.status == 401:
+ log.Info("WebDAV response status %s with reason '%s'." % (response.status,response.reason))
+ # resolve redirects and reset url on listing requests (they usually come before everything else)
+ if response.status in [301,302] and method == 'PROPFIND':
+ redirect_url = response.getheader('location',None)
+ response.close()
+ if redirect_url:
+ log.Notice("WebDAV redirect to: %s " % urllib.unquote(redirect_url) )
+ if redirected > 10:
+ raise FatalBackendError("WebDAV redirected 10 times. Giving up.")
+ self.parsed_url = duplicity.backend.ParsedUrl(redirect_url)
+ self.directory = self._sanitize_path(self.parsed_url.path)
+ return self.request(method,self.directory,data,redirected+1)
+ else:
+ raise FatalBackendError("WebDAV missing location header in redirect response.")
+ elif response.status == 401:
response.close()
self.headers['Authorization'] = self.get_authorization(response, quoted_path)
+ log.Info("WebDAV retry request with authentification headers.")
+ log.Info("WebDAV %s %s request2 with headers: %s " % (method,quoted_path,self.headers))
+ log.Info("WebDAV data length: %s " % len(str(data)) )
self.conn.request(method, quoted_path, data, self.headers)
response = self.conn.getresponse()
-
+ log.Info("WebDAV response2 status %s with reason '%s'." % (response.status,response.reason))
+
return response
+
+
def get_authorization(self, response, path):
"""
Fetches the auth header based on the requested method (basic or digest)
@@ -139,7 +239,7 @@
"""
Returns the basic auth header
"""
- auth_string = '%s:%s' % (self.parsed_url.username, self.get_password())
+ auth_string = '%s:%s' % (self.username, self.password)
return 'Basic %s' % base64.encodestring(auth_string).strip()
def get_digest_authorization(self, path):
@@ -149,7 +249,7 @@
u = self.parsed_url
if self.digest_auth_handler is None:
pw_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
- pw_manager.add_password(None, self.conn.host, u.username, self.get_password())
+ pw_manager.add_password(None, self.conn.host, self.username, self.password)
self.digest_auth_handler = urllib2.HTTPDigestAuthHandler(pw_manager)
# building a dummy request that gets never sent,
@@ -165,6 +265,7 @@
def list(self):
"""List files in directory"""
log.Info("Listing directory %s on WebDAV server" % (self.directory,))
+ response = None
try:
self.headers['Depth'] = "1"
response = self.request("PROPFIND", self.directory, self.listbody)
@@ -178,7 +279,7 @@
response.close()
# just created folder is so return empty
return []
- elif response.status == 207:
+ elif response.status in [200, 207]:
document = response.read()
response.close()
else:
@@ -195,9 +296,10 @@
if filename:
result.append(filename)
return result
- except Exception, cause:
- e = BackendException("Listing directory %s on WebDAV server failed. %s" % (self.directory,cause))
+ except Exception, e:
raise e
+ finally:
+ if response: response.close()
def __taste_href(self, href):
"""
@@ -241,11 +343,15 @@
"""Get remote filename, saving it to local_path"""
url = self.directory + remote_filename
log.Info("Retrieving %s from WebDAV server" % (url ,))
+ response = None
try:
target_file = local_path.open("wb")
response = self.request("GET", url)
if response.status == 200:
+ #data=response.read()
target_file.write(response.read())
+ #import hashlib
+ #log.Info("WebDAV GOT %s bytes with md5=%s" % (len(data),hashlib.md5(data).hexdigest()) )
assert not target_file.close()
local_path.setdata()
response.close()
@@ -254,9 +360,10 @@
reason = response.reason
response.close()
raise BackendException("Bad status code %s reason %s." % (status,reason))
- except Exception, cause:
- e = BackendException("Getting %s from WebDAV server failed. %s" % (url,cause))
+ except Exception, e:
raise e
+ finally:
+ if response: response.close()
@retry_fatal
def put(self, source_path, remote_filename = None):
@@ -265,10 +372,11 @@
remote_filename = source_path.get_filename()
url = self.directory + remote_filename
log.Info("Saving %s on WebDAV server" % (url ,))
+ response = None
try:
source_file = source_path.open("rb")
response = self.request("PUT", url, source_file.read())
- if response.status == 201:
+ if response.status in [201, 204]:
response.read()
response.close()
else:
@@ -276,9 +384,10 @@
reason = response.reason
response.close()
raise BackendException("Bad status code %s reason %s." % (status,reason))
- except Exception, cause:
- e = BackendException("Putting %s on WebDAV server failed. %s" % (url,cause))
+ except Exception, e:
raise e
+ finally:
+ if response: response.close()
@retry_fatal
def delete(self, filename_list):
@@ -286,6 +395,7 @@
for filename in filename_list:
url = self.directory + filename
log.Info("Deleting %s from WebDAV server" % (url ,))
+ response = None
try:
response = self.request("DELETE", url)
if response.status == 204:
@@ -296,9 +406,10 @@
reason = response.reason
response.close()
raise BackendException("Bad status code %s reason %s." % (status,reason))
- except Exception, cause:
- e = BackendException("Deleting %s on WebDAV server failed. %s" % (url,cause))
+ except Exception, e:
raise e
+ finally:
+ if response: response.close()
duplicity.backend.register_backend("webdav", WebDAVBackend)
duplicity.backend.register_backend("webdavs", WebDAVBackend)
=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py 2012-11-21 01:27:35 +0000
+++ duplicity/commandline.py 2013-01-10 19:14:20 +0000
@@ -502,6 +502,11 @@
# user added ssh options
parser.add_option("--ssh-options", action="extend", metavar=_("options"))
+ # user added ssl options (webdav backend)
+ parser.add_option("--ssl-cacert-file", metavar=_("pem formatted bundle of certificate authorities"))
+
+ parser.add_option("--ssl-no-check-certificate", action="store_true")
+
# Working directory for the tempfile module. Defaults to /tmp on most systems.
parser.add_option("--tempdir", dest="temproot", type="file", metavar=_("path"))
=== modified file 'duplicity/errors.py'
--- duplicity/errors.py 2011-08-03 19:47:29 +0000
+++ duplicity/errors.py 2013-01-10 19:14:20 +0000
@@ -70,6 +70,12 @@
"""
pass
+class FatalBackendError(DuplicityError):
+ """
+ Raised to indicate a backend failed fatally.
+ """
+ pass
+
class TemporaryLoadException(BackendException):
"""
Raised to indicate a temporary issue on the backend.
=== modified file 'duplicity/globals.py'
--- duplicity/globals.py 2012-03-13 20:53:35 +0000
+++ duplicity/globals.py 2013-01-10 19:14:20 +0000
@@ -214,6 +214,10 @@
# whether to use scp for put/get, sftp is default
use_scp = False
+# HTTPS ssl optons (currently only webdav)
+ssl_cacert_file = None
+ssl_no_check_certificate = False
+
# user added rsync options
rsync_options = ""
Follow ups
