duplicity-team team mailing list archive

[edso <edgar.soldin@xxxxxx>]

edso has proposed merging lp:~ed.so/duplicity/readd.optional.ncftp into lp:duplicity.

Requested reviews:
  duplicity-team (duplicity-team)

For more details, see:
https://code.launchpad.net/~ed.so/duplicity/readd.optional.ncftp/+merge/240057

readd ncftp as optional ftp backend
-- 
https://code.launchpad.net/~ed.so/duplicity/readd.optional.ncftp/+merge/240057
Your team duplicity-team is requested to review the proposed merge of lp:~ed.so/duplicity/readd.optional.ncftp into lp:duplicity.

=== modified file 'bin/duplicity.1'
--- bin/duplicity.1	2014-10-23 11:56:13 +0000
+++ bin/duplicity.1	2014-10-29 20:44:39 +0000
@@ -77,13 +77,16 @@
 - https://github.com/shazow/urllib3
 .TP
 .B "ftp backend"
+.B LFTP Client
+(default - supports ftp, ftps)
+- http://lftp.yar.ru/
+.br
+or
+.br
 .B NcFTP Client
+(supports ftp, select via ncftp+ftp://)
 - http://www.ncftp.com/
 .TP
-.B "ftps backend"
-.B LFTP Client
-- http://lftp.yar.ru/
-.TP
 .BR "gdocs backend" " (Google Docs)"
 .B Google Data APIs Python Client Library
 - http://code.google.com/p/gdata-python-client/
@@ -1084,43 +1087,59 @@
 Make sure to read
 .BR "A NOTE ON DROPBOX ACCESS" " first!"
 .PP
+.B "Copy cloud storage"
+.br
 copy://user[:password]@copy.com/some_dir
 .PP
-.PP
+.B "Local file path"
+.br
 file://[relative|/absolute]/local/path
 .PP
+.B "FTP"
+.br
 ftp[s]://user[:password]@other.host[:port]/some_dir
+.br
+.B NOTE:
+use lftp+, ncftp+ prefixes to enforce a specific backend, e.g. ncftp+ftp://...
 .PP
+.B "Google Docs"
+.br
 gdocs://user[:password]@other.host/some_dir
 .PP
-.BI "Google Cloud Storage"
+.B "Google Cloud Storage"
 .br
 gs://bucket[/prefix]
 .PP
 hsi://user[:password]@other.host/some_dir
 .PP
+.B "IMAP email storage"
+.br
 imap[s]://user[:password]@host.com[/from_address_prefix]
 .br
 See also
 .B "A NOTE ON IMAP"
 .PP
+.B "Mega cloud storage"
+.br
 mega://user[:password]@mega.co.nz/some_dir
 .PP
-.BI "Par2 Wrapper Backend"
+.B "Par2 Wrapper Backend"
 .br
 par2+scheme://[user[:password]@]host[:port]/[/]path
 .br
 See also
 .B "A NOTE ON PAR2 WRAPPER BACKEND"
 .PP
-.B "using rsync daemon"
+.B "Rsync via daemon"
 .br
 rsync://user[:password]@host.com[:port]::[/]module/some_dir
 .br
-.B "using rsync over ssh (only key auth)"
+.B "Rsync over ssh (only key auth)"
 .br
 rsync://user@xxxxxxxx[:port]/[relative|/absolute]_path
 .PP
+.B "Amazon S3 storage"
+.br
 s3://host/bucket_name[/prefix]
 .br
 s3+http://bucket_name[/prefix]
@@ -1128,6 +1147,8 @@
 See also
 .B "A NOTE ON EUROPEAN S3 BUCKETS"
 .PP
+.B "SCP/SFTP access"
+.br
 scp://.. or ssh://.. are synonymous with
 .br
 sftp://user[:password]@other.host[:port]/[/]some_dir
@@ -1140,13 +1161,19 @@
 and
 .BR "A NOTE ON SSH BACKENDS" .
 .PP
+.B "Openstack Swift"
+.br
 swift://container_name
 .br
 See also
 .B "A NOTE ON SWIFT (OPENSTACK OBJECT STORAGE) ACCESS"
 .PP
+.B "Tahoe-LAFS"
+.br
 tahoe://alias/directory
 .PP
+.B "WebDAV"
+.br
 webdav[s]://user[:password]@other.host[:port]/some_dir
 .RE
 

=== modified file 'duplicity/backend.py'
--- duplicity/backend.py	2014-10-27 02:27:36 +0000
+++ duplicity/backend.py	2014-10-29 20:44:39 +0000
@@ -32,6 +32,7 @@
 import re
 import getpass
 import gettext
+import re
 import types
 import urllib
 import urlparse
@@ -164,6 +165,11 @@
 
     _backend_prefixes[scheme] = backend_factory
 
+def strip_prefix(url_string, prefix_scheme):
+    """
+    strip the prefix from a string e.g. par2+ftp://... -> ftp://...
+    """
+    return re.sub('(?i)^'+re.escape(prefix_scheme)+'\+','',url_string)
 
 def is_backend_url(url_string):
     """
@@ -198,7 +204,7 @@
     for prefix in _backend_prefixes:
         if url_string.startswith(prefix + '+'):
             factory = _backend_prefixes[prefix]
-            pu = ParsedUrl(url_string.lstrip(prefix + '+'))
+            pu = ParsedUrl(strip_prefix(url_string,prefix))
             break
 
     if factory is None:
@@ -337,11 +343,8 @@
 def strip_auth_from_url(parsed_url):
     """Return a URL from a urlparse object without a username or password."""
 
-    # Get a copy of the network location without the username or password.
-    straight_netloc = parsed_url.netloc.split('@')[-1]
-
-    # Replace the full network location with the stripped copy.
-    return parsed_url.geturl().replace(parsed_url.netloc, straight_netloc, 1)
+    clean_url = re.sub('^([^:/]+://)(.*@)?(.*)',r'\1\3',parsed_url.geturl())
+    return clean_url
 
 def _get_code_from_exception(backend, operation, e):
     if isinstance(e, BackendException) and e.code != log.ErrorCode.backend_error:

=== renamed file 'duplicity/backends/ftpbackend.py' => 'duplicity/backends/lftpbackend.py'
--- duplicity/backends/ftpbackend.py	2014-10-01 20:35:16 +0000
+++ duplicity/backends/lftpbackend.py	2014-10-29 20:44:39 +0000
@@ -3,7 +3,6 @@
 # Copyright 2002 Ben Escoto <ben@xxxxxxxxxxx>
 # Copyright 2007 Kenneth Loafman <kenneth@xxxxxxxxxxx>
 # Copyright 2010 Marcel Pennewiss <opensource@xxxxxxxxxxxx>
-# Copyright 2014 Moritz Maisel <moritz@xxxxxxxxxxx>
 #
 # This file is part of duplicity.
 #
@@ -31,7 +30,7 @@
 from duplicity import log
 from duplicity import tempdir
 
-class FTPBackend(duplicity.backend.Backend):
+class LFTPBackend(duplicity.backend.Backend):
     """Connect to remote store using File Transfer Protocol"""
     def __init__(self, parsed_url):
         duplicity.backend.Backend.__init__(self, parsed_url)
@@ -56,6 +55,9 @@
 
         self.url_string = duplicity.backend.strip_auth_from_url(self.parsed_url)
 
+        # strip lftp+ prefix
+        self.url_string = duplicity.backend.strip_prefix(self.url_string, 'lftp')
+
         # Use an explicit directory name.
         if self.url_string[-1] != '/':
             self.url_string += '/'
@@ -110,5 +112,8 @@
         commandline = "lftp -c 'source %s;cd \'%s\';rm \'%s\''" % (self.tempname, remote_dir, filename)
         self.subprocess_popen(commandline)
 
-duplicity.backend.register_backend("ftp", FTPBackend)
-duplicity.backend.register_backend("ftps", FTPBackend)
+duplicity.backend.register_backend("ftp", LFTPBackend)
+duplicity.backend.register_backend("ftps", LFTPBackend)
+duplicity.backend.register_backend("lftp+ftp", LFTPBackend)
+duplicity.backend.register_backend("lftp+ftps", LFTPBackend)
+duplicity.backend.uses_netloc.extend([ 'ftp', 'ftps', 'lftp+ftp', 'lftp+ftps' ])
\ No newline at end of file

=== added file 'duplicity/backends/ncftpbackend.py'
--- duplicity/backends/ncftpbackend.py	1970-01-01 00:00:00 +0000
+++ duplicity/backends/ncftpbackend.py	2014-10-29 20:44:39 +0000
@@ -0,0 +1,118 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2002 Ben Escoto <ben@xxxxxxxxxxx>
+# Copyright 2007 Kenneth Loafman <kenneth@xxxxxxxxxxx>
+#
+# This file is part of duplicity.
+#
+# Duplicity is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# Duplicity is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with duplicity; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os.path
+import urllib
+
+import duplicity.backend
+from duplicity import globals
+from duplicity import log
+from duplicity import tempdir
+
+class NCFTPBackend(duplicity.backend.Backend):
+    """Connect to remote store using File Transfer Protocol"""
+    def __init__(self, parsed_url):
+        duplicity.backend.Backend.__init__(self, parsed_url)
+
+        # we expect an error return, so go low-level and ignore it
+        try:
+            p = os.popen("ncftpls -v")
+            fout = p.read()
+            ret = p.close()
+        except Exception:
+            pass
+        # the expected error is 8 in the high-byte and some output
+        if ret != 0x0800 or not fout:
+            log.FatalError("NcFTP not found:  Please install NcFTP version 3.1.9 or later",
+                           log.ErrorCode.ftp_ncftp_missing)
+
+        # version is the second word of the first line
+        version = fout.split('\n')[0].split()[1]
+        if version < "3.1.9":
+            log.FatalError("NcFTP too old:  Duplicity requires NcFTP version 3.1.9,"
+                           "3.2.1 or later.  Version 3.2.0 will not work properly.",
+                           log.ErrorCode.ftp_ncftp_too_old)
+        elif version == "3.2.0":
+            log.Warn("NcFTP (ncftpput) version 3.2.0 may fail with duplicity.\n"
+                     "see: http://www.ncftpd.com/ncftp/doc/changelog.html\n";
+                     "If you have trouble, please upgrade to 3.2.1 or later",
+                     log.WarningCode.ftp_ncftp_v320)
+        log.Notice("NcFTP version is %s" % version)
+
+        self.parsed_url = parsed_url
+
+        self.url_string = duplicity.backend.strip_auth_from_url(self.parsed_url)
+
+        # strip ncftp+ prefix
+        self.url_string = duplicity.backend.strip_prefix(self.url_string, 'ncftp')
+
+        # This squelches the "file not found" result from ncftpls when
+        # the ftp backend looks for a collection that does not exist.
+        # version 3.2.2 has error code 5, 1280 is some legacy value
+        self.popen_breaks[ 'ncftpls' ] = [ 5, 1280 ]
+
+        # Use an explicit directory name.
+        if self.url_string[-1] != '/':
+            self.url_string += '/'
+
+        self.password = self.get_password()
+
+        if globals.ftp_connection == 'regular':
+            self.conn_opt = '-E'
+        else:
+            self.conn_opt = '-F'
+
+        self.tempfile, self.tempname = tempdir.default().mkstemp()
+        os.write(self.tempfile, "host %s\n" % self.parsed_url.hostname)
+        os.write(self.tempfile, "user %s\n" % self.parsed_url.username)
+        os.write(self.tempfile, "pass %s\n" % self.password)
+        os.close(self.tempfile)
+        self.flags = "-f %s %s -t %s -o useCLNT=0,useHELP_SITE=0 " % \
+            (self.tempname, self.conn_opt, globals.timeout)
+        if parsed_url.port != None and parsed_url.port != 21:
+            self.flags += " -P '%s'" % (parsed_url.port)
+
+    def _put(self, source_path, remote_filename):
+        remote_path = os.path.join(urllib.unquote(self.parsed_url.path.lstrip('/')), remote_filename).rstrip()
+        commandline = "ncftpput %s -m -V -C '%s' '%s'" % \
+            (self.flags, source_path.name, remote_path)
+        self.subprocess_popen(commandline)
+
+    def _get(self, remote_filename, local_path):
+        remote_path = os.path.join(urllib.unquote(self.parsed_url.path), remote_filename).rstrip()
+        commandline = "ncftpget %s -V -C '%s' '%s' '%s'" % \
+            (self.flags, self.parsed_url.hostname, remote_path.lstrip('/'), local_path.name)
+        self.subprocess_popen(commandline)
+
+    def _list(self):
+        # Do a long listing to avoid connection reset
+        commandline = "ncftpls %s -l '%s'" % (self.flags, self.url_string)
+        _, l, _ = self.subprocess_popen(commandline)
+        # Look for our files as the last element of a long list line
+        return [x.split()[-1] for x in l.split('\n') if x and not x.startswith("total ")]
+
+    def _delete(self, filename):
+        commandline = "ncftpls %s -l -X 'DELE %s' '%s'" % \
+            (self.flags, filename, self.url_string)
+        self.subprocess_popen(commandline)
+
+duplicity.backend.register_backend("ncftp+ftp", NCFTPBackend)
+duplicity.backend.uses_netloc.extend([ 'ncftp+ftp' ])
\ No newline at end of file