duplicity-team team mailing list archive

[Yajo <yajo.sk8@xxxxxxxxx>]

Yajo has proposed merging lp:~yajo/duplicity/duplicity into lp:duplicity/0.7-series.

Requested reviews:
  duplicity-team (duplicity-team)

For more details, see:
https://code.launchpad.net/~yajo/duplicity/duplicity/+merge/366358

Support partial metadata sync.

Fixes bug #1823858 by letting the user to choose partial syncing. Only the metadata for the target chain will be downloaded. If older (or newer) chains are encrypted with a different passphrase, the user will be able to restore to a given time by supplying only the passphrase for the chain selected by the `--restore-time` option when using this new option.

A side effect is that using this flag reduces dramatically the sync time when moving files from one to another location, in cases where big amounts of chains are found. 
-- 
Your team duplicity-team is requested to review the proposed merge of lp:~yajo/duplicity/duplicity into lp:duplicity/0.7-series.

=== modified file 'bin/duplicity'
--- bin/duplicity	2018-09-28 13:55:53 +0000
+++ bin/duplicity	2019-04-22 12:44:43 +0000
@@ -1020,7 +1020,7 @@
                    _("Rerun command with --force option to actually delete."))
 
 
-def sync_archive():
+def sync_archive(col_stats):
     """
     Synchronize local archive manifest file and sig chains to remote archives.
     Copy missing files from remote to local as needed to make sure the local
@@ -1031,6 +1031,27 @@
     """
     suffixes = [".g", ".gpg", ".z", ".gz", ".part"]
 
+    def is_needed(filename):
+        """Indicates if the metadata file should be synced.
+
+        In full sync mode, or if there's a collection misbehavior, all files
+        are needed.
+
+        Otherwise, only the metadata for the target chain needs sync.
+        """
+        if globals.metadata_sync_mode == "full":
+            return True
+        assert globals.metadata_sync_mode == "partial"
+        parsed = file_naming.parse(filename)
+        try:
+            target_chain = col_stats.get_backup_chain_at_time(
+                globals.restore_time or dup_time.curtime)
+        except collections.CollectionsError:
+            # With zero or multiple chains at this time, do a full sync
+            return True
+        return parsed.end_time >= target_chain.start_time and \
+            parsed.start_time <= target_chain.end_time
+
     def get_metafiles(filelist):
         """
         Return metafiles of interest from the file list.
@@ -1174,13 +1195,14 @@
 
     local_missing = []
     local_spurious = []
+    import pudb; pudb.set_trace()
 
     for key in remote_keys:
         # If we lost our cache, re-get the remote file.  But don't do it if we
         # already have a local partial.  The local partial will already be
         # complete in this case (seems we got interrupted before we could move
         # it to its final location).
-        if key not in local_keys and key not in local_partials:
+        if key not in local_keys and key not in local_partials and is_needed(key):
             local_missing.append(remote_metafiles[key])
 
     for key in local_keys:
@@ -1414,15 +1436,15 @@
     # check for disk space and available file handles
     check_resources(action)
 
-    # check archive synch with remote, fix if needed
-    if action not in ["collection-status"]:
-        sync_archive()
-
     # get current collection status
     col_stats = collections.CollectionsStatus(globals.backend,
                                               globals.archive_dir,
                                               action).set_values()
 
+    # check archive synch with remote, fix if needed
+    if action not in ["collection-status"]:
+        sync_archive(col_stats)
+
     while True:
         # if we have to clean up the last partial, then col_stats are invalidated
         # and we have to start the process all over again until clean.
@@ -1488,7 +1510,7 @@
     elif action == "remove-all-but-n-full" or action == "remove-all-inc-of-but-n-full":
         remove_all_but_n_full(col_stats)
     elif action == "sync":
-        sync_archive()
+        sync_archive(col_stats)
     else:
         assert action == "inc" or action == "full", action
         # the passphrase for full and inc is used by --sign-key

=== modified file 'bin/duplicity.1'
--- bin/duplicity.1	2019-01-26 16:37:54 +0000
+++ bin/duplicity.1	2019-04-22 12:44:43 +0000
@@ -882,6 +882,14 @@
 .BR "A NOTE ON SSL CERTIFICATE VERIFICATION" .
 
 .TP
+.BI "--metadata-sync-mode " mode
+This option defaults to 'full', but you can set it to 'partial'
+to avoid syncing metadata for backup chains that you are not going to use.
+This saves time when restoring for the first time, and lets you restore an
+old backup that was encrypted with a different passphrase by supplying only
+the target passphrase.
+
+.TP
 .BI "--tempdir " directory
 Use this existing directory for duplicity temporary files instead of
 the system default, which is usually the /tmp directory. This option

=== modified file 'duplicity/collections.py'
--- duplicity/collections.py	2018-02-01 21:17:08 +0000
+++ duplicity/collections.py	2019-04-22 12:44:43 +0000
@@ -24,7 +24,7 @@
 from future_builtins import filter, map
 
 import types
-import gettext
+from gettext import gettext as _
 
 
 from duplicity import log

=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py	2018-02-01 21:17:08 +0000
+++ duplicity/commandline.py	2019-04-22 12:44:43 +0000
@@ -491,6 +491,11 @@
                       callback=lambda o, s, v, p: (setattr(p.values, o.dest, True),
                                                    old_fn_deprecation(s)))
 
+    # Sync only required metadata
+    parser.add_option("--metadata-sync-mode",
+                      default="full",
+                      choices=("full", "partial"))
+
     # Level of Redundancy in % for Par2 files
     parser.add_option("--par2-redundancy", type="int", metavar=_("number"))
 

=== modified file 'duplicity/globals.py'
--- duplicity/globals.py	2018-02-01 21:17:08 +0000
+++ duplicity/globals.py	2019-04-22 12:44:43 +0000
@@ -225,6 +225,9 @@
 # Can be changed with a command line argument.
 imap_mailbox = "INBOX"
 
+# Sync all metadata by default
+metadata_sync_mode = "full"
+
 # Whether the old filename format is in effect.
 old_filenames = False