duplicity-team team mailing list archive
-
duplicity-team team
-
Mailing list archive
-
Message #03409
[Merge] lp:~ed.so/duplicity/webdav.lftp.ssl-overhaul into lp:duplicity
edso has proposed merging lp:~ed.so/duplicity/webdav.lftp.ssl-overhaul into lp:duplicity.
Requested reviews:
duplicity-team (duplicity-team)
For more details, see:
https://code.launchpad.net/~ed.so/duplicity/webdav.lftp.ssl-overhaul/+merge/287654
duplicity.1, commandline.py, globals.py
- added --ssl-cacert-path parameter
backend.py
- make sure url path component is properly url decoded,
in case it contains special chars (eg. @ or space)
lftpbackend.py
- quote _all_ cmd line params
- added missing lftp+ftpes protocol
- fix empty list result when chdir failed silently
- added ssl_cacert_path support
webdavbackend.py
- add ssl default context support for python 2.7.9+
(using system certs eg. in /etc/ssl/certs)
- added ssl_cacert_path support for python 2.7.9+
- gettext wrapped all log messages
- minor refinements
--
Your team duplicity-team is requested to review the proposed merge of lp:~ed.so/duplicity/webdav.lftp.ssl-overhaul into lp:duplicity.
=== modified file 'bin/duplicity.1'
--- bin/duplicity.1 2016-02-18 18:22:35 +0000
+++ bin/duplicity.1 2016-03-01 15:46:05 +0000
@@ -854,15 +854,23 @@
.TP
.BI "--ssl-cacert-file " file
-.B (only webdav backend)
+.B (only webdav & lftp backend)
Provide a cacert file for ssl certificate verification.
.br
See also
.BR "A NOTE ON SSL CERTIFICATE VERIFICATION" .
.TP
+.BI "--ssl-cacert-path " path/to/certs/
+.B (only webdav backend and python 2.7.9+ OR lftp+webdavs and a recent lftp)
+Provide a path to a folder containing cacert files for ssl certificate verification.
+.br
+See also
+.BR "A NOTE ON SSL CERTIFICATE VERIFICATION" .
+
+.TP
.BI --ssl-no-check-certificate
-.B (only webdav backend)
+.B (only webdav & lftp backend)
Disable ssl certificate verification.
.br
See also
@@ -1873,8 +1881,16 @@
an sftp service running on the backend server, which is sometimes not an option.
.SH A NOTE ON SSL CERTIFICATE VERIFICATION
-Certificate verification as implemented right now [01.2013] only in the webdav backend needs a file
-based database of certification authority certificates (cacert file). It has to be a
+Certificate verification as implemented right now [02.2016] only in the webdav
+and lftp backends. older pythons 2.7.8- and older lftp binaries need a file
+based database of certification authority certificates (cacert file).
+.br
+Newer python 2.7.9+ and recent lftp versions however support the system default
+certificates (usually in /etc/ssl/certs) and also giving an alternative ca cert
+folder via
+.BR --ssl-cacert-path .
+.PP
+The cacert file has to be a
.B PEM
formatted text file as currently provided by the
.B CURL
=== modified file 'duplicity/backend.py'
--- duplicity/backend.py 2016-01-24 17:56:38 +0000
+++ duplicity/backend.py 2016-03-01 15:46:05 +0000
@@ -265,7 +265,7 @@
raise InvalidBackendURL("Syntax error (netloc) in: %s" % url_string)
try:
- self.path = pu.path
+ self.path = urllib.unquote(pu.path)
except Exception:
raise InvalidBackendURL("Syntax error (path) in: %s" % url_string)
=== modified file 'duplicity/backends/lftpbackend.py'
--- duplicity/backends/lftpbackend.py 2016-01-24 17:30:02 +0000
+++ duplicity/backends/lftpbackend.py 2016-03-01 15:46:05 +0000
@@ -95,24 +95,21 @@
cacert_candidates = ["~/.duplicity/cacert.pem",
"~/duplicity_cacert.pem",
"/etc/duplicity/cacert.pem"]
- #
+ # look for a default cacert file
if not self.cacert_file:
for path in cacert_candidates:
path = os.path.expanduser(path)
if (os.path.isfile(path)):
self.cacert_file = path
break
- # still no cacert file, inform user
- if not self.cacert_file:
- raise duplicity.errors.FatalBackendException("""For certificate verification a cacert database file is needed in one of these locations: %s
-Hints:
- Consult the man page, chapter 'SSL Certificate Verification'.
- Consider using the options --ssl-cacert-file, --ssl-no-check-certificate .""" % ", ".join(cacert_candidates))
+ # save config into a reusable temp file
self.tempfile, self.tempname = tempdir.default().mkstemp()
os.write(self.tempfile, "set ssl:verify-certificate " + ("false" if globals.ssl_no_check_certificate else "true") + "\n")
- if globals.ssl_cacert_file:
- os.write(self.tempfile, "set ssl:ca-file '" + globals.ssl_cacert_file + "'\n")
+ if self.cacert_file:
+ os.write(self.tempfile, "set ssl:ca-file " + cmd_quote(self.cacert_file) + "\n")
+ if globals.ssl_cacert_path:
+ os.write(self.tempfile, "set ssl:ca-path " + cmd_quote(globals.ssl_cacert_path) + "\n")
if self.parsed_url.scheme == 'ftps':
os.write(self.tempfile, "set ftp:ssl-allow true\n")
os.write(self.tempfile, "set ftp:ssl-protect-data true\n")
@@ -134,13 +131,14 @@
else:
os.write(self.tempfile, "open %s %s\n" % (self.authflag, self.url_string))
os.close(self.tempfile)
+ # print settings in debug mode
if log.getverbosity() >= log.DEBUG:
f = open(self.tempname, 'r')
log.Debug("SETTINGS: \n"
- "%s" % f.readlines())
+ "%s" % f.read())
def _put(self, source_path, remote_filename):
- commandline = "lftp -c 'source %s; mkdir -p %s; put %s -o %s'" % (
+ commandline = "lftp -c \"source %s; mkdir -p %s; put %s -o %s\"" % (
self.tempname,
cmd_quote(self.remote_path),
cmd_quote(source_path.name),
@@ -155,8 +153,8 @@
"%s" % (l))
def _get(self, remote_filename, local_path):
- commandline = "lftp -c 'source %s; get %s -o %s'" % (
- self.tempname,
+ commandline = "lftp -c \"source %s; get %s -o %s\"" % (
+ cmd_quote(self.tempname),
cmd_quote(self.remote_path) + remote_filename,
cmd_quote(local_path.name)
)
@@ -172,9 +170,11 @@
# remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
remote_dir = urllib.unquote(self.parsed_url.path)
# print remote_dir
- commandline = "lftp -c 'source %s; cd %s || exit 0; ls'" % (
- self.tempname,
- cmd_quote(self.remote_path)
+ quoted_path = cmd_quote(self.remote_path);
+ # failing to cd into the folder might be because it was not created already
+ commandline = "lftp -c \"source %s; ( cd %s && ls ) || ( mkdir -p %s && cd %s && ls )\"" % (
+ cmd_quote(self.tempname),
+ quoted_path, quoted_path, quoted_path
)
log.Debug("CMD: %s" % commandline)
_, l, e = self.subprocess_popen(commandline)
@@ -187,10 +187,10 @@
return [x.split()[-1] for x in l.split('\n') if x]
def _delete(self, filename):
- commandline = "lftp -c 'source %s; cd %s; rm %s'" % (
- self.tempname,
+ commandline = "lftp -c \"source %s; cd %s; rm %s\"" % (
+ cmd_quote(self.tempname),
cmd_quote(self.remote_path),
- filename
+ cmd_quote(filename)
)
log.Debug("CMD: %s" % commandline)
_, l, e = self.subprocess_popen(commandline)
@@ -204,10 +204,10 @@
duplicity.backend.register_backend("fish", LFTPBackend)
duplicity.backend.register_backend("ftpes", LFTPBackend)
-
duplicity.backend.register_backend("lftp+ftp", LFTPBackend)
duplicity.backend.register_backend("lftp+ftps", LFTPBackend)
duplicity.backend.register_backend("lftp+fish", LFTPBackend)
+duplicity.backend.register_backend("lftp+ftpes", LFTPBackend)
duplicity.backend.register_backend("lftp+sftp", LFTPBackend)
duplicity.backend.register_backend("lftp+webdav", LFTPBackend)
duplicity.backend.register_backend("lftp+webdavs", LFTPBackend)
@@ -216,7 +216,8 @@
duplicity.backend.uses_netloc.extend(['ftp', 'ftps', 'fish', 'ftpes',
'lftp+ftp', 'lftp+ftps',
- 'lftp+fish', 'lftp+sftp',
+ 'lftp+fish', 'lftp+ftpes',
+ 'lftp+sftp',
'lftp+webdav', 'lftp+webdavs',
'lftp+http', 'lftp+https']
)
=== modified file 'duplicity/backends/webdavbackend.py'
--- duplicity/backends/webdavbackend.py 2016-02-05 09:58:57 +0000
+++ duplicity/backends/webdavbackend.py 2016-03-01 15:46:05 +0000
@@ -58,31 +58,25 @@
import socket
import ssl
except ImportError:
- raise FatalBackendException("Missing socket or ssl libraries.")
+ raise FatalBackendException(_("Missing socket or ssl python modules."))
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
self.cacert_file = globals.ssl_cacert_file
- cacert_candidates = ["~/.duplicity/cacert.pem",
+ self.cacert_candidates = ["~/.duplicity/cacert.pem",
"~/duplicity_cacert.pem",
"/etc/duplicity/cacert.pem"]
- #
+ # if no cacert file was given search default locations
if not self.cacert_file:
- for path in cacert_candidates:
+ for path in self.cacert_candidates:
path = os.path.expanduser(path)
if (os.path.isfile(path)):
self.cacert_file = path
break
- # still no cacert file, inform user
- if not self.cacert_file:
- raise FatalBackendException("""\
-For certificate verification a cacert database file is needed in one of these locations: %s
-Hints:
- Consult the man page, chapter 'SSL Certificate Verification'.
- Consider using the options --ssl-cacert-file, --ssl-no-check-certificate .""" % ", ".join(cacert_candidates))
+
# check if file is accessible (libssl errors are not very detailed)
- if not os.access(self.cacert_file, os.R_OK):
- raise FatalBackendException("Cacert database file '%s' is not readable." % self.cacert_file)
+ if self.cacert_file and not os.access(self.cacert_file, os.R_OK):
+ raise FatalBackendException(_("Cacert database file '%s' is not readable.") % self.cacert_file)
def connect(self):
# create new socket
@@ -92,8 +86,24 @@
self.sock = sock
self.tunnel()
- # wrap the socket in ssl using verification
- self.sock = ssl.wrap_socket(sock,
+ # python 2.7.9+ supports default system certs now
+ if "create_default_context" in dir(ssl):
+ context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=self.cacert_file, capath=globals.ssl_cacert_path)
+ self.sock = context.wrap_socket(sock, server_hostname=self.host)
+ # the legacy way needing a cert file
+ else:
+ if globals.ssl_cacert_path:
+ raise FatalBackendException(_("Option '--ssl-cacert-path' is not supported with python 2.7.8 and below."))
+
+ if not self.cacert_file:
+ raise FatalBackendException(_("""\
+For certificate verification with python 2.7.8 or earlier a cacert database file is needed in one of these locations: %s
+Hints:
+ Consult the man page, chapter 'SSL Certificate Verification'.
+ Consider using the options --ssl-cacert-file, --ssl-no-check-certificate .""") % ", ".join(self.cacert_candidates))
+
+ # wrap the socket in ssl using verification
+ self.sock = ssl.wrap_socket(sock,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=self.cacert_file,
)
@@ -129,9 +139,9 @@
self.password = self.get_password()
self.directory = self.sanitize_path(parsed_url.path)
- log.Info("Using WebDAV protocol %s" % (globals.webdav_proto,))
- log.Info("Using WebDAV host %s port %s" % (parsed_url.hostname, parsed_url.port))
- log.Info("Using WebDAV directory %s" % (self.directory,))
+ log.Info(_("Using WebDAV protocol %s") % (globals.webdav_proto,))
+ log.Info(_("Using WebDAV host %s port %s") % (parsed_url.hostname, parsed_url.port))
+ log.Info(_("Using WebDAV directory %s") % (self.directory,))
self.conn = None
@@ -162,7 +172,7 @@
and self.conn.host == self.parsed_url.hostname:
return
- log.Info("WebDAV create connection on '%s'" % (self.parsed_url.hostname))
+ log.Info(_("WebDAV create connection on '%s'") % (self.parsed_url.hostname))
self._close()
# http schemes needed for redirect urls from servers
if self.parsed_url.scheme in ['webdav', 'http']:
@@ -173,7 +183,7 @@
else:
self.conn = VerifiedHTTPSConnection(self.parsed_url.hostname, self.parsed_url.port)
else:
- raise FatalBackendException("WebDAV Unknown URI scheme: %s" % (self.parsed_url.scheme))
+ raise FatalBackendException(_("WebDAV Unknown URI scheme: %s") % (self.parsed_url.scheme))
def _close(self):
if self.conn:
@@ -192,34 +202,34 @@
if self.digest_challenge is not None:
self.headers['Authorization'] = self.get_digest_authorization(path)
- log.Info("WebDAV %s %s request with headers: %s " % (method, quoted_path, self.headers))
- log.Info("WebDAV data length: %s " % len(str(data)))
+ log.Info(_("WebDAV %s %s request with headers: %s ") % (method, quoted_path, self.headers))
+ log.Info(_("WebDAV data length: %s ") % len(str(data)))
self.conn.request(method, quoted_path, data, self.headers)
response = self.conn.getresponse()
- log.Info("WebDAV response status %s with reason '%s'." % (response.status, response.reason))
+ log.Info(_("WebDAV response status %s with reason '%s'.") % (response.status, response.reason))
# resolve redirects and reset url on listing requests (they usually come before everything else)
if response.status in [301, 302] and method == 'PROPFIND':
redirect_url = response.getheader('location', None)
response.close()
if redirect_url:
- log.Notice("WebDAV redirect to: %s " % urllib.unquote(redirect_url))
+ log.Notice(_("WebDAV redirect to: %s ") % urllib.unquote(redirect_url))
if redirected > 10:
- raise FatalBackendException("WebDAV redirected 10 times. Giving up.")
+ raise FatalBackendException(_("WebDAV redirected 10 times. Giving up."))
self.parsed_url = duplicity.backend.ParsedUrl(redirect_url)
self.directory = self.sanitize_path(self.parsed_url.path)
return self.request(method, self.directory, data, redirected + 1)
else:
- raise FatalBackendException("WebDAV missing location header in redirect response.")
+ raise FatalBackendException(_("WebDAV missing location header in redirect response."))
elif response.status == 401:
response.read()
response.close()
self.headers['Authorization'] = self.get_authorization(response, quoted_path)
- log.Info("WebDAV retry request with authentification headers.")
- log.Info("WebDAV %s %s request2 with headers: %s " % (method, quoted_path, self.headers))
- log.Info("WebDAV data length: %s " % len(str(data)))
+ log.Info(_("WebDAV retry request with authentification headers."))
+ log.Info(_("WebDAV %s %s request2 with headers: %s ") % (method, quoted_path, self.headers))
+ log.Info(_("WebDAV data length: %s ") % len(str(data)))
self.conn.request(method, quoted_path, data, self.headers)
response = self.conn.getresponse()
- log.Info("WebDAV response2 status %s with reason '%s'." % (response.status, response.reason))
+ log.Info(_("WebDAV response2 status %s with reason '%s'.") % (response.status, response.reason))
return response
@@ -337,11 +347,11 @@
log.Info("Checking existence dir %s: %d" % (d, response.status))
if response.status == 404:
- log.Info("Creating missing directory %s" % d)
+ log.Info(_("Creating missing directory %s") % d)
res = self.request("MKCOL", d)
if res.status != 201:
- raise BackendException("WebDAV MKCOL %s failed: %s %s" % (d, res.status, res.reason))
+ raise BackendException(_("WebDAV MKCOL %s failed: %s %s") % (d, res.status, res.reason))
def taste_href(self, href):
"""
@@ -353,8 +363,8 @@
raw_filename = self.getText(href.childNodes).strip()
parsed_url = urlparse.urlparse(urllib.unquote(raw_filename))
filename = parsed_url.path
- log.Debug("webdav path decoding and translation: "
- "%s -> %s" % (raw_filename, filename))
+ log.Debug(_("WebDAV path decoding and translation: "
+ "%s -> %s") % (raw_filename, filename))
# at least one WebDAV server returns files in the form
# of full URL:s. this may or may not be
@@ -397,7 +407,7 @@
status = response.status
reason = response.reason
response.close()
- raise BackendException("Bad status code %s reason %s." % (status, reason))
+ raise BackendException(_("WebDAV GET Bad status code %s reason %s.") % (status, reason))
except Exception as e:
raise e
finally:
@@ -418,7 +428,7 @@
status = response.status
reason = response.reason
response.close()
- raise BackendException("Bad status code %s reason %s." % (status, reason))
+ raise BackendException(_("WebDAV PUT Bad status code %s reason %s.") % (status, reason))
except Exception as e:
raise e
finally:
@@ -437,7 +447,7 @@
status = response.status
reason = response.reason
response.close()
- raise BackendException("Bad status code %s reason %s." % (status, reason))
+ raise BackendException(_("WebDAV DEL Bad status code %s reason %s.") % (status, reason))
except Exception as e:
raise e
finally:
=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py 2016-02-10 17:15:49 +0000
+++ duplicity/commandline.py 2016-03-01 15:46:05 +0000
@@ -582,9 +582,9 @@
# user added ssh options
parser.add_option("--ssh-options", action="extend", metavar=_("options"))
- # user added ssl options (webdav backend)
+ # user added ssl options (used by webdav, lftp backend)
parser.add_option("--ssl-cacert-file", metavar=_("pem formatted bundle of certificate authorities"))
-
+ parser.add_option("--ssl-cacert-path", metavar=_("path to a folder with certificate authority files"))
parser.add_option("--ssl-no-check-certificate", action="store_true")
# Working directory for the tempfile module. Defaults to /tmp on most systems.
=== modified file 'duplicity/globals.py'
--- duplicity/globals.py 2015-10-10 00:02:35 +0000
+++ duplicity/globals.py 2016-03-01 15:46:05 +0000
@@ -242,8 +242,9 @@
# default cf backend is pyrax
cf_backend = "pyrax"
-# HTTPS ssl optons (currently only webdav)
+# HTTPS ssl options (currently only webdav, lftp)
ssl_cacert_file = None
+ssl_cacert_path = None
ssl_no_check_certificate = False
# user added rsync options
Follow ups
