deja-dup-team team mailing list archive
-
deja-dup-team team
-
Mailing list archive
-
Message #00477
[Merge] lp:~mterry/deja-dup/verify into lp:deja-dup
Michael Terry has proposed merging lp:~mterry/deja-dup/verify into lp:deja-dup.
Requested reviews:
Ken VanDine (ken-vandine)
For more details, see:
https://code.launchpad.net/~mterry/deja-dup/verify/+merge/119176
This branch does two things:
1) Every backup now also backs up a file created by Deja Dup: ~/.cache/deja-dup/metadata/README which holds the following:
This folder can be safely deleted.
@SECONDS_SINCE_EPOCH@
Where obviously, @SECONDS_SINCE_EPOCH@ is a number.
Intentionally, part of it is always the same (the safely deleted part) and part of it is always different. This means duplicity will always back it up, so it will be present in every backup. And the part that is the same can be sanity-checked.
It doesn't matter if the user sets their cache directory in some weird place, since we always currently use this file right after a backup, so the cache directory will be in the same place we expect it.
2) After each backup, we restore this file and sanity check it. This will have to download one volume from each incremental backup since the start. The idea is to test that the backup can be restored if needed. This isn't 100% foolproof, but will test for a lot of different problems.
The next step in this plan (and a different branch altogether) will be to add logic that every now and then tries to restore without using saved knowledge like the user's encryption password or duplicity cached data, to really test that the user remembers enough to restore if needed.
--
https://code.launchpad.net/~mterry/deja-dup/verify/+merge/119176
Your team Déjà Dup Developers is subscribed to branch lp:deja-dup.
=== modified file 'common/Makefile.am'
--- common/Makefile.am 2012-08-08 01:12:17 +0000
+++ common/Makefile.am 2012-08-10 18:43:21 +0000
@@ -48,10 +48,14 @@
Network.vala \
Operation.vala \
OperationBackup.vala \
+ OperationFiles.vala \
OperationRestore.vala \
OperationStatus.vala \
- OperationFiles.vala \
+ OperationVerify.vala \
PythonChecker.vala \
+ RecursiveDelete.vala \
+ RecursiveMove.vala \
+ RecursiveOp.vala \
SimpleSettings.vala \
ToolPlugin.vala
=== modified file 'common/Operation.vala'
--- common/Operation.vala 2012-08-06 22:41:13 +0000
+++ common/Operation.vala 2012-08-10 18:43:21 +0000
@@ -95,12 +95,13 @@
backend = Backend.get_default();
}
- public async virtual void start()
+ public async virtual void start(bool try_claim_bus = true)
{
action_desc_changed(_("Preparing…"));
try {
- claim_bus();
+ if (try_claim_bus)
+ claim_bus();
}
catch (Error e) {
raise_error(e.message, null);
@@ -182,7 +183,7 @@
job.done.connect((d, o, c, detail) => {operation_finished.begin(d, o, c, detail);});
job.raise_error.connect((d, s, detail) => {raise_error(s, detail);});
job.action_desc_changed.connect((d, s) => {action_desc_changed(s);});
- job.action_file_changed.connect((d, f, b) => {action_file_changed(f, b);});
+ job.action_file_changed.connect((d, f, b) => {send_action_file_changed(f, b);});
job.progress.connect((d, p) => {progress(p);});
job.question.connect((d, t, m) => {question(t, m);});
job.is_full.connect((first) => {is_full(first);});
@@ -195,6 +196,11 @@
});
}
+ protected virtual void send_action_file_changed(File file, bool actual)
+ {
+ action_file_changed(file, actual);
+ }
+
public void set_passphrase(string? passphrase)
{
needs_password = false;
@@ -222,7 +228,33 @@
*/
return null;
}
-
+
+ protected async void chain_op(Operation subop, string desc)
+ {
+ /**
+ * Sometimes an operation wants to chain to a separate operation.
+ * Here is the glue to make that happen.
+ */
+ subop.ref();
+ subop.done.connect((s, c, d) => {done(s, c, d); subop.unref();});
+ subop.raise_error.connect((e, d) => {raise_error(e, d);});
+ subop.progress.connect((p) => {progress(p);});
+ subop.passphrase_required.connect(() => {
+ passphrase_required();
+ subop.needs_password = needs_password;
+ subop.passphrase = passphrase;
+ });
+ subop.question.connect((t, m) => {question(t, m);});
+
+ subop.set_state(get_state());
+ job = subop.job;
+
+ action_desc_changed(desc);
+ progress(0);
+
+ yield subop.start(false);
+ }
+
uint bus_id = 0;
void claim_bus() throws BackupError
{
@@ -239,7 +271,8 @@
void unclaim_bus()
{
- Bus.unown_name(bus_id);
+ if (bus_id > 0)
+ Bus.unown_name(bus_id);
}
}
=== modified file 'common/OperationBackup.vala'
--- common/OperationBackup.vala 2012-08-06 22:41:13 +0000
+++ common/OperationBackup.vala 2012-08-10 18:43:21 +0000
@@ -23,6 +23,8 @@
public class OperationBackup : Operation
{
+ File metadir;
+
public OperationBackup() {
Object(mode: ToolJob.Mode.BACKUP);
}
@@ -32,10 +34,21 @@
/* If successfully completed, update time of last backup and run base operation_finished */
if (success)
DejaDup.update_last_run_timestamp(DejaDup.TimestampType.BACKUP);
-
- yield base.operation_finished(job, success, cancelled, detail);
- }
-
+
+ if (metadir != null)
+ new RecursiveDelete(metadir).start();
+
+ yield chain_op(new OperationVerify(), _("Verifying backup…"));
+ }
+
+ protected override void send_action_file_changed(File file, bool actual)
+ {
+ // Intercept action_file_changed signals and ignore them if they are
+ // metadata file, the user doesn't need to see them.
+ if (!file.has_prefix(metadir))
+ base.send_action_file_changed(file, actual);
+ }
+
protected override List<string>? make_argv()
{
var settings = get_settings();
@@ -54,9 +67,20 @@
job.excludes.prepend(s);
foreach (File s in include_list)
job.includes.prepend(s);
+
+ // Insert deja-dup meta info directory
+ string cachedir = Environment.get_user_cache_dir();
+ try {
+ metadir = File.new_for_path(Path.build_filename(cachedir, Config.PACKAGE, "metadata"));
+ fill_metadir();
+ job.includes.prepend(metadir);
+ }
+ catch (Error e) {
+ warning("%s\n", e.message);
+ }
job.local = File.new_for_path("/");
-
+
return null;
}
@@ -99,6 +123,24 @@
return rv;
}
+
+ void fill_metadir() throws Error
+ {
+ if (metadir == null)
+ return;
+
+ // Delete old dir, if any, and replace it
+ new RecursiveDelete(metadir).start();
+ metadir.make_directory_with_parents(null);
+
+ // Put a file in there that is one part always constant, and one part
+ // always different, for basic sanity checking. This way, it will be
+ // included in every backup, but we can still check its contents for
+ // corruption. We'll stuff seconds-since-epoch in it.
+ var now = new DateTime.now_utc();
+ var msg = "This folder can be safely deleted.\n%s".printf(now.format("%s"));
+ FileUtils.set_contents(Path.build_filename(metadir.get_path(), "README"), msg);
+ }
}
} // end namespace
=== modified file 'common/OperationRestore.vala'
--- common/OperationRestore.vala 2012-08-06 22:41:13 +0000
+++ common/OperationRestore.vala 2012-08-10 18:43:21 +0000
@@ -46,20 +46,15 @@
mode: ToolJob.Mode.RESTORE);
}
- public async override void start()
+ public async override void start(bool try_claim_bus = true)
{
action_desc_changed(_("Restoring files…"));
- yield base.start();
+ yield base.start(try_claim_bus);
}
- protected override void connect_to_job()
+ protected override List<string>? make_argv()
{
- base.connect_to_job();
job.restore_files = restore_files;
- }
-
- protected override List<string>? make_argv()
- {
job.time = time;
job.local = File.new_for_path(dest);
return null;
=== added file 'common/OperationVerify.vala'
--- common/OperationVerify.vala 1970-01-01 00:00:00 +0000
+++ common/OperationVerify.vala 2012-08-10 18:43:21 +0000
@@ -0,0 +1,85 @@
+/* -*- Mode: Vala; indent-tabs-mode: nil; tab-width: 2 -*- */
+/*
+ This file is part of Déjà Dup.
+ For copyright information, see AUTHORS.
+
+ Déjà Dup is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Déjà Dup is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Déjà Dup. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using GLib;
+
+namespace DejaDup {
+
+/* This is meant to be used right after a successful OperationBackup to
+ verify the results. */
+
+public class OperationVerify : Operation
+{
+ File metadir;
+ File destdir;
+
+ public OperationVerify() {
+ Object(mode: ToolJob.Mode.RESTORE);
+ }
+
+ public async override void start(bool try_claim_bus = true)
+ {
+ action_desc_changed(_("Verifying backup…"));
+ yield base.start(try_claim_bus);
+ }
+
+ protected override void connect_to_job()
+ {
+ string cachedir = Environment.get_user_cache_dir();
+ metadir = File.new_for_path(Path.build_filename(cachedir, Config.PACKAGE, "metadata"));
+ job.restore_files.append(metadir);
+
+ destdir = File.new_for_path("/");
+ job.local = destdir;
+
+ base.connect_to_job();
+ }
+
+ internal async override void operation_finished(ToolJob job, bool success, bool cancelled, string? detail)
+ {
+ // Verify results
+ if (success) {
+ var verified = true;
+ string contents;
+ try {
+ FileUtils.get_contents(Path.build_filename(metadir.get_path(), "README"), out contents);
+ }
+ catch (Error e) {
+ verified = false;
+ }
+
+ if (verified) {
+ var lines = contents.split("\n");
+ verified = (lines[0] == "This folder can be safely deleted.");
+ }
+
+ if (!verified) {
+ raise_error(_("Your backup appears to be corrupted. You should delete the backup and try again."), null);
+ success = false;
+ }
+ }
+
+ new RecursiveDelete(metadir).start();
+
+ yield base.operation_finished(job, success, cancelled, detail);
+ }
+}
+
+} // end namespace
+
=== renamed file 'tools/duplicity/RecursiveDelete.vala' => 'common/RecursiveDelete.vala'
--- tools/duplicity/RecursiveDelete.vala 2012-04-30 00:18:18 +0000
+++ common/RecursiveDelete.vala 2012-08-10 18:43:21 +0000
@@ -19,7 +19,9 @@
using GLib;
-internal class RecursiveDelete : RecursiveOp
+namespace DejaDup {
+
+public class RecursiveDelete : RecursiveOp
{
public RecursiveDelete(File source)
{
@@ -57,3 +59,4 @@
}
}
+} // namespace
=== renamed file 'tools/duplicity/RecursiveMove.vala' => 'common/RecursiveMove.vala'
--- tools/duplicity/RecursiveMove.vala 2012-04-30 00:18:18 +0000
+++ common/RecursiveMove.vala 2012-08-10 18:43:21 +0000
@@ -19,6 +19,8 @@
using GLib;
+namespace DejaDup {
+
/**
* Recursively moves one directory into another, merging files. And by merge,
* I mean it overwrites. It skips any files it can't move and reports an
@@ -27,7 +29,7 @@
* This is not optimized for remote files. It's mostly async, but it does the
* occasional sync operation.
*/
-internal class RecursiveMove : RecursiveOp
+public class RecursiveMove : RecursiveOp
{
public RecursiveMove(File source, File dest)
{
@@ -162,3 +164,4 @@
}
}
+} // namespace
=== renamed file 'tools/duplicity/RecursiveOp.vala' => 'common/RecursiveOp.vala'
--- tools/duplicity/RecursiveOp.vala 2012-08-07 13:33:53 +0000
+++ common/RecursiveOp.vala 2012-08-10 18:43:21 +0000
@@ -19,7 +19,9 @@
using GLib;
-internal abstract class RecursiveOp : Object
+namespace DejaDup {
+
+public abstract class RecursiveOp : Object
{
public signal void done();
public signal void raise_error(File src, File dst, string errstr);
@@ -129,3 +131,4 @@
}
}
+} // namespace
=== modified file 'po/POTFILES.in'
--- po/POTFILES.in 2012-04-30 00:18:18 +0000
+++ po/POTFILES.in 2012-08-10 18:43:21 +0000
@@ -22,7 +22,11 @@
common/OperationFiles.vala
common/OperationRestore.vala
common/OperationStatus.vala
+common/OperationVerify.vala
common/Operation.vala
+common/RecursiveDelete.vala
+common/RecursiveMove.vala
+common/RecursiveOp.vala
common/SimpleSettings.vala
deja-dup/AssistantBackup.vala
deja-dup/AssistantOperation.vala
@@ -39,9 +43,6 @@
tools/duplicity/DuplicityInstance.vala
tools/duplicity/DuplicityJob.vala
tools/duplicity/DuplicityPlugin.vala
-tools/duplicity/RecursiveDelete.vala
-tools/duplicity/RecursiveMove.vala
-tools/duplicity/RecursiveOp.vala
widgets/ConfigBool.vala
widgets/ConfigChoice.vala
widgets/ConfigDelete.vala
=== modified file 'po/POTFILES.skip'
--- po/POTFILES.skip 2012-04-30 00:18:18 +0000
+++ po/POTFILES.skip 2012-08-10 18:43:21 +0000
@@ -11,7 +11,11 @@
common/OperationFiles.c
common/OperationRestore.c
common/OperationStatus.c
+common/OperationVerify.c
common/Operation.c
+common/RecursiveDelete.c
+common/RecursiveMove.c
+common/RecursiveOp.c
common/SimpleSettings.c
deja-dup/AssistantBackup.c
deja-dup/AssistantOperation.c
@@ -28,9 +32,6 @@
tools/duplicity/DuplicityInstance.c
tools/duplicity/DuplicityJob.c
tools/duplicity/DuplicityPlugin.c
-tools/duplicity/RecursiveDelete.c
-tools/duplicity/RecursiveMove.c
-tools/duplicity/RecursiveOp.c
widgets/ConfigBool.c
widgets/ConfigChoice.c
widgets/ConfigDelete.c
=== modified file 'tests/runner/runner.vala'
--- tests/runner/runner.vala 2012-08-10 18:01:39 +0000
+++ tests/runner/runner.vala 2012-08-10 18:43:21 +0000
@@ -96,6 +96,7 @@
STATUS,
DRY,
BACKUP,
+ VERIFY,
CLEANUP,
RESTORE,
RESTORE_STATUS,
@@ -113,6 +114,8 @@
return "cleanup '--force' 'file://%s' '--gio' '--no-encryption' '--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s/deja-dup' '--log-fd=?'".printf(backupdir, cachedir);
else if (mode == Mode.RESTORE)
return "'restore' '--gio' '--force' 'file://%s' '%s' '--no-encryption' '--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s/deja-dup' '--log-fd=?'".printf(backupdir, restoredir, cachedir);
+ else if (mode == Mode.VERIFY)
+ return "'restore' '--file-to-restore=%s/deja-dup/metadata' '--gio' '--force' 'file://%s' '%s/deja-dup/metadata' '--no-encryption' '--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s/deja-dup' '--log-fd=?'".printf(cachedir.substring(1), backupdir, cachedir, cachedir);
else if (mode == Mode.LIST)
return "'list-current-files' '--gio' 'file://%s' '--no-encryption' '--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s/deja-dup' '--log-fd=?'".printf(backupdir, cachedir);
@@ -137,7 +140,7 @@
args += "collection-status ";
if (mode == Mode.STATUS || mode == Mode.NONE || mode == Mode.DRY || mode == Mode.BACKUP) {
- args += "'--exclude=%s' ".printf(backupdir);
+ args += "'--exclude=%s' '--include=%s/deja-dup/metadata' ".printf(backupdir, cachedir);
string[] excludes1 = {"~/Downloads", "~/.local/share/Trash", "~/.xsession-errors", "~/.thumbnails", "~/.Private", "~/.gvfs", "~/.adobe/Flash_Player/AssetCache"};
@@ -279,7 +282,8 @@
string replace_keywords(string in)
{
var home = Environment.get_home_dir();
- return in.replace("@HOME@", home);
+ var cachedir = Environment.get_variable("XDG_CACHE_HOME");
+ return in.replace("@HOME@", home).replace("@XDG_CACHE_HOME@", cachedir);
}
string run_script(string in)
@@ -369,6 +373,8 @@
mode = Mode.LIST;
else if (type == "backup")
mode = Mode.BACKUP;
+ else if (type == "verify")
+ mode = Mode.VERIFY;
else if (type == "restore")
mode = Mode.RESTORE;
else if (type == "cleanup")
@@ -376,6 +382,8 @@
else
assert_not_reached();
+ var cachedir = Environment.get_variable("XDG_CACHE_HOME");
+
var dupscript = "ARGS: " + default_args(br, mode, encrypted, extra_args);
if (cancel) {
@@ -394,6 +402,8 @@
if (script != null)
dupscript += "\n" + "SCRIPT: " + script;
+ else if (mode == Mode.VERIFY)
+ dupscript += "\n" + "SCRIPT: mkdir -p %s/deja-dup/metadata; echo 'This folder can be safely deleted.\\n0' > %s/deja-dup/metadata/README".printf(cachedir, cachedir);
if (outputscript != null && outputscript != "")
dupscript += "\n\n" + outputscript + "\n";
=== modified file 'tests/scripts/bad-hostname.test'
--- tests/scripts/bad-hostname.test 2012-04-18 11:45:19 +0000
+++ tests/scripts/bad-hostname.test 2012-08-10 18:43:21 +0000
@@ -2,7 +2,7 @@
Type=backup
[Duplicity]
-Runs=status;dry 1;dry 2;backup;
+Runs=status;dry 1;dry 2;backup;status-restore;list;verify;
[Duplicity dry 1]
#ERROR 3 new old
=== modified file 'tests/scripts/clean-incomplete.test'
--- tests/scripts/clean-incomplete.test 2012-07-27 19:04:29 +0000
+++ tests/scripts/clean-incomplete.test 2012-08-10 18:43:21 +0000
@@ -6,7 +6,7 @@
Type=backup
[Duplicity]
-Runs=status;dry;backup 1;cleanup;backup 2;
+Runs=status;dry;backup 1;cleanup;backup 2;status-restore;list;verify;
[Duplicity backup 1]
#WARNING 2
=== modified file 'tests/scripts/read-error.test'
--- tests/scripts/read-error.test 2012-04-18 13:40:40 +0000
+++ tests/scripts/read-error.test 2012-08-10 18:43:21 +0000
@@ -5,7 +5,7 @@
Detail=Could not back up the following files. Please make sure you are able to open them.\n\n@HOME@/1\n@HOME@/2
[Duplicity]
-Runs=status;dry;backup;
+Runs=status;dry;backup;status-restore;list;verify;
[Duplicity backup]
#WARNING 10 '/blarg'
=== modified file 'tests/scripts/threshold-full.test'
--- tests/scripts/threshold-full.test 2012-06-21 17:47:50 +0000
+++ tests/scripts/threshold-full.test 2012-08-10 18:43:21 +0000
@@ -6,7 +6,7 @@
IsFull=true
[Duplicity]
-Runs=status;dry;backup;
+Runs=status;dry;backup;status-restore;list;verify;
[Duplicity status]
#echo "INFO 3"
=== modified file 'tests/scripts/threshold-inc.test'
--- tests/scripts/threshold-inc.test 2012-06-21 17:47:50 +0000
+++ tests/scripts/threshold-inc.test 2012-08-10 18:43:21 +0000
@@ -7,7 +7,7 @@
IsFull=false
[Duplicity]
-Runs=status;dry;backup;
+Runs=status;dry;backup;status-restore;list;verify;
[Duplicity status]
#echo "INFO 3"
=== added file 'tests/scripts/verify.test'
--- tests/scripts/verify.test 1970-01-01 00:00:00 +0000
+++ tests/scripts/verify.test 2012-08-10 18:43:21 +0000
@@ -0,0 +1,10 @@
+[Operation]
+Type=backup
+Success=false
+Error=Your backup appears to be corrupted. You should delete the backup and try again.
+
+[Duplicity]
+Runs=status;dry;backup;status-restore;list;verify;
+
+[Duplicity verify]
+Script=mkdir -p @XDG_CACHE_HOME@/deja-dup/metadata; echo 'Nope' > @XDG_CACHE_HOME@/deja-dup/metadata/README
=== modified file 'tools/duplicity/DuplicityInstance.vala'
--- tools/duplicity/DuplicityInstance.vala 2012-08-07 13:33:53 +0000
+++ tools/duplicity/DuplicityInstance.vala 2012-08-10 18:43:21 +0000
@@ -40,7 +40,10 @@
if (!settings.get_boolean(DejaDup.ROOT_PROMPT_KEY))
as_root = false;
}
-
+
+ if (as_root && Environment.get_variable("DEJA_DUP_TESTING") != null)
+ as_root = false;
+
// Copy current environment, add custom variables
var myenv = Environment.list_variables();
int myenv_len = 0;
=== modified file 'tools/duplicity/DuplicityJob.vala'
--- tools/duplicity/DuplicityJob.vala 2012-08-07 13:33:53 +0000
+++ tools/duplicity/DuplicityJob.vala 2012-08-10 18:43:21 +0000
@@ -826,7 +826,7 @@
return;
var cachedir = Path.build_filename(dir, Config.PACKAGE);
- var del = new RecursiveDelete(File.new_for_path(cachedir));
+ var del = new DejaDup.RecursiveDelete(File.new_for_path(cachedir));
del.start();
}
=== modified file 'tools/duplicity/Makefile.am'
--- tools/duplicity/Makefile.am 2012-08-08 01:12:17 +0000
+++ tools/duplicity/Makefile.am 2012-08-10 18:43:21 +0000
@@ -37,10 +37,7 @@
libduplicity_la_VALASOURCES = \
DuplicityInstance.vala \
DuplicityJob.vala \
- DuplicityPlugin.vala \
- RecursiveDelete.vala \
- RecursiveMove.vala \
- RecursiveOp.vala
+ DuplicityPlugin.vala
libduplicity_la_SOURCES = \
$(libduplicity_la_VALASOURCES)
Follow ups
