duplicity-team team mailing list archive

[Germar <germar.reitze@xxxxxxxxx>]

Germar has proposed merging lp:~germar/duplicity/par2 into lp:duplicity.

Requested reviews:
  duplicity-team (duplicity-team)
Related bugs:
  Bug #426282 in Duplicity: "Please add support for par2 creating"
  https://bugs.launchpad.net/duplicity/+bug/426282

For more details, see:
https://code.launchpad.net/~germar/duplicity/par2/+merge/210103

This branch adds Par2 recovery files to duplicity. It is a wrapper backend which will create the recovery files and upload them all together with the wrapped backend. Corrupt archives will be detected and repaired (if possible) on the fly during restore.
It can be used with url-string par2+webdavs://USER@HOST/PATH
-- 
https://code.launchpad.net/~germar/duplicity/par2/+merge/210103
Your team duplicity-team is requested to review the proposed merge of lp:~germar/duplicity/par2 into lp:duplicity.

=== modified file 'bin/duplicity.1'
--- bin/duplicity.1	2014-02-26 19:25:17 +0000
+++ bin/duplicity.1	2014-03-09 22:19:55 +0000
@@ -98,6 +98,10 @@
 .BR "mega backend" " (mega.co.nz)"
 .B Python library for mega API
 - https://github.com/ckornacker/mega.py, ubuntu ppa - ppa:ckornacker/backup
+.TP
+.B "Par2 Wrapper Backend"
+.B par2cmdline
+- http://parchive.sourceforge.net/
 .PP
 There are two
 .B ssh backends
@@ -712,6 +716,12 @@
 the new filename format.
 
 .TP
+.BI "--par2-redundancy " percent
+Adjust the level of redundancy in
+.I percent
+for Par2 recovery files (default 10%)
+
+.TP
 .B --progress
 When selected, duplicity will output the current upload progress and estimated
 upload time. To annotate changes, it will perform a first dry-run before a full
@@ -1080,6 +1090,13 @@
 .PP
 mega://user[:password]@mega.co.nz/some_dir
 .PP
+.BI "Par2 Wrapper Backend"
+.br
+par2+scheme://[user[:password]@]host[:port]/[/]path
+.br
+See also
+.B "A NOTE ON PAR2 WRAPPER BACKEND"
+.PP
 .B "using rsync daemon"
 .br
 rsync://user[:password]@host.com[:port]::[/]module/some_dir
@@ -1534,6 +1551,24 @@
 .B from_address_prefix
 will distinguish between different backups.
 
+.SH A NOTE ON PAR2 WRAPPER BACKEND
+Par2 Wrapper Backend can be used in combination with all other backends to
+create recovery files. Just add
+.BR par2+
+before a regular scheme (e.g.
+.IR par2+ftp://user@host/dir " or"
+.I par2+s3+http://bucket_name
+). This will create par2 recovery files for each archive and upload them all to
+the wrapped backend.
+.PP
+Before restoring, archives will be verified. Corrupt archives will be repaired
+on the fly if there are enough recovery blocks available.
+.PP
+Use
+.BI "--par2-redundancy " percent
+to adjust the size (and redundancy) of recovery files in
+.I percent.
+
 .SH A NOTE ON SSH BACKENDS
 The
 .I ssh backends

=== modified file 'duplicity/backend.py'
--- duplicity/backend.py	2013-12-27 06:39:00 +0000
+++ duplicity/backend.py	2014-03-09 22:19:55 +0000
@@ -71,6 +71,7 @@
     assert path.endswith("duplicity/backends"), duplicity.backends.__path__
 
     files = os.listdir(path)
+    files.sort()
     for fn in files:
         if fn.endswith("backend.py"):
             fn = fn[:-3]

=== added file 'duplicity/backends/~par2wrapperbackend.py'
--- duplicity/backends/~par2wrapperbackend.py	1970-01-01 00:00:00 +0000
+++ duplicity/backends/~par2wrapperbackend.py	2014-03-09 22:19:55 +0000
@@ -0,0 +1,194 @@
+# Copyright 2013 Germar Reitze <germar.reitze@xxxxxxxxx>
+#
+# This file is part of duplicity.
+#
+# Duplicity is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# Duplicity is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with duplicity; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+import re
+from duplicity import backend
+from duplicity.errors import UnsupportedBackendScheme, BackendException
+from duplicity.pexpect import run
+from duplicity import log
+from duplicity import globals
+
+class Par2WrapperBackend(backend.Backend):
+    """This backend wrap around other backends and create Par2 recovery files
+    before the file and the Par2 files are transfered with the wrapped backend.
+    
+    If a received file is corrupt it will try to repair it on the fly.
+    """
+    def __init__(self, parsed_url):
+        backend.Backend.__init__(self, parsed_url)
+        self.parsed_url = parsed_url
+        try:
+            self.redundancy = globals.par2_redundancy
+        except AttributeError:
+            self.redundancy = 10
+
+        try:
+            url_string = self.parsed_url.url_string.lstrip('par2+')
+            self.wrapped_backend = backend.get_backend(url_string)
+        except:
+            raise UnsupportedBackendScheme(self.parsed_url.url_string)
+
+    def put(self, source_path, remote_filename = None):
+        """create Par2 files and transfer the given file and the Par2 files
+        with the wrapped backend.
+        
+        Par2 must run on the real filename or it would restore the
+        temp-filename later on. So first of all create a tempdir and symlink
+        the soure_path with remote_filename into this. 
+        """
+        if remote_filename is None:
+            remote_filename = source_path.get_filename()
+
+        par2temp = source_path.get_temp_in_same_dir()
+        par2temp.mkdir()
+        source_symlink = par2temp.append(remote_filename)
+        os.symlink(source_path.get_canonical(), source_symlink.get_canonical())
+        source_symlink.setdata()
+
+        log.Info("Create Par2 recovery files")
+        par2create = 'par2 c -r%d -n1 -q -q %s' % (self.redundancy, source_symlink.get_canonical())
+        out, returncode = run(par2create, -1, True)
+        source_symlink.delete()
+        files_to_transfer = []
+        if not returncode:
+            for file in par2temp.listdir():
+                files_to_transfer.append(par2temp.append(file))
+
+        ret = self.wrapped_backend.put(source_path, remote_filename)
+        for file in files_to_transfer:
+            self.wrapped_backend.put(file, file.get_filename())
+
+        par2temp.deltree()
+        return ret
+
+    def move(self, source_path, remote_filename = None):
+        self.put(source_path, remote_filename)
+        source_path.delete()
+
+    def get(self, remote_filename, local_path):
+        """transfer remote_filename and the related .par2 file into
+        a temp-dir. remote_filename will be renamed into local_path before
+        finishing.
+        
+        If "par2 verify" detect an error transfer the Par2-volumes into the
+        temp-dir and try to repair.
+        """
+        par2temp = local_path.get_temp_in_same_dir()
+        par2temp.mkdir()
+        local_path_temp = par2temp.append(remote_filename)
+
+        ret = self.wrapped_backend.get(remote_filename, local_path_temp)
+
+        try:
+            par2file = par2temp.append(remote_filename + '.par2')
+            self.wrapped_backend.get(par2file.get_filename(), par2file)
+
+            par2verify = 'par2 v -q -q %s %s' % (par2file.get_canonical(), local_path_temp.get_canonical())
+            out, returncode = run(par2verify, -1, True)
+
+            if returncode:
+                log.Warn("File is corrupt. Try to repair %s" % remote_filename)
+                par2volumes = self.list(re.compile(r'%s\.vol[\d+]*\.par2' % remote_filename))
+
+                for filename in par2volumes:
+                    file = par2temp.append(filename)
+                    self.wrapped_backend.get(filename, file)
+
+                par2repair = 'par2 r -q -q %s %s' % (par2file.get_canonical(), local_path_temp.get_canonical())
+                out, returncode = run(par2repair, -1, True)
+
+                if returncode:
+                    log.Error("Failed to repair %s" % remote_filename)
+                else:
+                    log.Warn("Repair successful %s" % remote_filename)
+        except BackendException:
+            #par2 file not available
+            pass
+        finally:
+            local_path_temp.rename(local_path)
+            par2temp.deltree()
+        return ret
+
+    def list(self, filter = re.compile(r'(?!.*\.par2$)')):
+        """default filter all files that ends with ".par"
+        filter can be a re.compile instance or False for all remote files
+        """
+        list = self.wrapped_backend.list()
+        if not filter:
+            return list
+        filtered_list = []
+        for item in list:
+            if filter.match(item):
+                filtered_list.append(item)
+        return filtered_list
+
+    def delete(self, filename_list):
+        """delete given filename_list and all .par2 files that belong to them
+        """
+        remote_list = self.list(False)
+
+        for filename in filename_list[:]:
+            c =  re.compile(r'%s(?:\.vol[\d+]*)?\.par2' % filename)
+            for remote_filename in remote_list:
+                if c.match(remote_filename):
+                    filename_list.append(remote_filename)
+
+        return self.wrapped_backend.delete(filename_list)
+
+    """just return the output of coresponding wrapped backend
+    for all other functions
+    """
+    def query_info(self, filename_list, raise_errors=True):
+        return self.wrapped_backend.query_info(filename_list, raise_errors)
+
+    def get_password(self):
+        return self.wrapped_backend.get_password()
+
+    def munge_password(self, commandline):
+        return self.wrapped_backend.munge_password(commandline)
+
+    def run_command(self, commandline):
+        return self.wrapped_backend.run_command(commandline)
+    def run_command_persist(self, commandline):
+        return self.wrapped_backend.run_command_persist(commandline)
+
+    def popen(self, commandline):
+        return self.wrapped_backend.popen(commandline)
+    def popen_persist(self, commandline):
+        return self.wrapped_backend.popen_persist(commandline)
+
+    def _subprocess_popen(self, commandline):
+        return self.wrapped_backend._subprocess_popen(commandline)
+
+    def subprocess_popen(self, commandline):
+        return self.wrapped_backend.subprocess_popen(commandline)
+
+    def subprocess_popen_persist(self, commandline):
+        return self.wrapped_backend.subprocess_popen_persist(commandline)
+
+    def close(self):
+        return self.wrapped_backend.close()
+
+"""register this backend with leading "par2+" for all already known backends
+
+files must be sorted in duplicity.backend.import_backends to catch
+all supported backends
+"""
+for item in backend._backends.keys():
+    backend.register_backend('par2+' + item, Par2WrapperBackend)

=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py	2014-02-21 17:35:24 +0000
+++ duplicity/commandline.py	2014-03-09 22:19:55 +0000
@@ -450,6 +450,9 @@
                       callback = lambda o, s, v, p: (setattr(p.values, o.dest, True),
                                                    old_fn_deprecation(s)))
 
+    # Level of Redundancy in % for Par2 files
+    parser.add_option("--par2-redundancy", type="int", metavar=_("number"))
+
     # Used to display the progress for the full and incremental backup operations
     parser.add_option("--progress", action = "store_true")
 

=== modified file 'duplicity/globals.py'
--- duplicity/globals.py	2014-02-21 17:35:24 +0000
+++ duplicity/globals.py	2014-03-09 22:19:55 +0000
@@ -278,3 +278,6 @@
 # Controls the upload progress messages refresh rate. Default: update each
 # 3 seconds
 progress_rate = 3
+
+# Level of Redundancy in % for Par2 files
+par2_redundancy = 10