deja-dup-team team mailing list archive
-
deja-dup-team team
-
Mailing list archive
-
Message #00537
[Merge] lp:~mterry/deja-dup/tempdir into lp:deja-dup
Michael Terry has proposed merging lp:~mterry/deja-dup/tempdir into lp:deja-dup.
Requested reviews:
Robert Bruce Park (robru)
Related bugs:
Bug #1100092 in Déjà Dup: "Try to use a tempdir on same partition as source files"
https://bugs.launchpad.net/deja-dup/+bug/1100092
For more details, see:
https://code.launchpad.net/~mterry/deja-dup/tempdir/+merge/144032
If /tmp is a tmpfs or /home is on a separate partition, it is easy for duplicity to exhaust the available temporary space. This is because it needs at most twice the size of the largest source file. Which could be over a gig.
Rather than fix duplicity's use of temp space, which would not be trivial, we can at least tell it to use a more sensible temporary directory. Ideally something on the same partition as the source files (that way we know it should have an appropriate amount of space).
This branch tries three possible locations for temporary directories: /tmp, /var/tmp, and ~/.cache/deja-dup/tmp. The first one that matches the source files is used.
In case a backup is interrupted, those locations are cleared of folders that start with duplicity-* on startup and when we are done with a backup or restore.
Common scenarios for this branch are tmpfs (likely would use /var/tmp) or ecryptfs/separate home partition (likely would use ~/.cache/deja-dup/tmp).
--
https://code.launchpad.net/~mterry/deja-dup/tempdir/+merge/144032
Your team Déjà Dup Developers is subscribed to branch lp:deja-dup.
=== modified file 'common/CommonUtils.vala'
--- common/CommonUtils.vala 2012-10-25 20:20:47 +0000
+++ common/CommonUtils.vala 2013-01-20 20:14:20 +0000
@@ -383,6 +383,9 @@
var unused_backend = DejaDup.Backend.get_default();
unused_backend = null;
+ // And cleanup from any previous runs
+ clean_tempdirs.begin();
+
return true;
}
@@ -531,5 +534,127 @@
"type", Secret.SchemaAttributeType.STRING);
}
+public bool ensure_directory_exists(string path)
+{
+ var gfile = File.new_for_path(path);
+ try {
+ if (gfile.make_directory_with_parents())
+ return true;
+ }
+ catch (IOError.EXISTS e) {
+ return true; // ignore
+ }
+ catch (Error e) {
+ warning("%s\n", e.message);
+ }
+ return false;
+}
+
+// By default, duplicity uses normal tmp folders like /tmp to store its
+// in-process files. These can get quite large, especially when restoring.
+// You may need up to twice the size of the largest source file.
+// Because /tmp may not be super large, especially on systems that use
+// tmpfs by default (e.g. Fedora 18), we try to use a tempdir that is on
+// the same partition as the source files.
+public async string get_tempdir()
+{
+ var tempdirs = get_tempdirs();
+
+ // First, decide the "main include". Assume that if $HOME
+ // is present, that is it. Else, the first include we find decides it.
+ // This is admittedly fast and loose, but our primary concern is just
+ // avoiding silly choices like tmpfs or tiny special /tmp partitions.
+ var settings = get_settings();
+ var include_val = settings.get_value(INCLUDE_LIST_KEY);
+ var include_list = parse_dir_list(include_val.get_strv());
+ File main_include = null;
+ var home = File.new_for_path(Environment.get_home_dir());
+ foreach (var include in include_list) {
+ if (include.equal(home)) {
+ main_include = include;
+ break;
+ }
+ else if (main_include == null)
+ main_include = include;
+ }
+ if (main_include == null)
+ return tempdirs[0];
+
+ // Grab that include's fs ID
+ string filesystem_id;
+ try {
+ var info = yield main_include.query_info_async(FileAttribute.ID_FILESYSTEM,
+ FileQueryInfoFlags.NONE);
+ filesystem_id = info.get_attribute_string(FileAttribute.ID_FILESYSTEM);
+ }
+ catch (Error e) {
+ return tempdirs[0];
+ }
+
+ // Then, see which of our possible tempdirs matches that partition.
+ foreach (var tempdir in tempdirs) {
+ string temp_id;
+ ensure_directory_exists(tempdir);
+ try {
+ var gfile = File.new_for_path(tempdir);
+ var info = yield gfile.query_info_async(FileAttribute.ID_FILESYSTEM,
+ FileQueryInfoFlags.NONE);
+ temp_id = info.get_attribute_string(FileAttribute.ID_FILESYSTEM);
+ }
+ catch (Error e) {
+ continue;
+ }
+ if (temp_id == filesystem_id)
+ return tempdir;
+ }
+
+ // Fallback to simply using the highest preferred tempdir
+ return tempdirs[0];
+}
+
+public string[] get_tempdirs()
+{
+ var tempdir = Environment.get_variable("DEJA_DUP_TEMPDIR");
+ if (tempdir != null && tempdir != "")
+ return {tempdir};
+ // Prefer directories that have their own cleanup logic in case ours isn't
+ // run for a while. (e.g. /tmp every boot, /var/tmp every now and then)
+ return {Environment.get_tmp_dir(), "/var/tmp",
+ Path.build_filename(Environment.get_user_cache_dir(), Config.PACKAGE,
+ "tmp")};
+}
+
+public async void clean_tempdirs()
+{
+ var tempdirs = get_tempdirs();
+ const int NUM_ENUMERATED = 16;
+ foreach (var tempdir in tempdirs) {
+ var gfile = File.new_for_path(tempdir);
+
+ // Now try to find and delete all files that start with "duplicity-"
+ try {
+ var enumerator = yield gfile.enumerate_children_async(
+ FileAttribute.STANDARD_NAME,
+ FileQueryInfoFlags.NOFOLLOW_SYMLINKS,
+ Priority.DEFAULT, null);
+ while (true) {
+ var infos = yield enumerator.next_files_async(NUM_ENUMERATED,
+ Priority.DEFAULT, null);
+ foreach (FileInfo info in infos) {
+ if (info.get_name().has_prefix("duplicity-")) {
+ var child = gfile.get_child(info.get_name());
+ yield new DejaDup.RecursiveDelete(child).start_async();
+ }
+ }
+ if (infos.length() != NUM_ENUMERATED)
+ break;
+ }
+ }
+ catch (Error e) {
+ // No worries
+ }
+ }
+}
+
} // end namespace
=== modified file 'common/Operation.vala'
--- common/Operation.vala 2012-08-20 01:16:15 +0000
+++ common/Operation.vala 2013-01-20 20:14:20 +0000
@@ -223,6 +223,7 @@
finished = true;
unclaim_bus();
+ yield DejaDup.clean_tempdirs();
done(success, cancelled, detail);
}
=== modified file 'common/OperationBackup.vala'
--- common/OperationBackup.vala 2012-11-26 17:04:31 +0000
+++ common/OperationBackup.vala 2013-01-20 20:14:20 +0000
@@ -117,11 +117,11 @@
rv.append(Path.build_filename(dir, ".xsession-errors"));
}
- // Some problematic directories like /tmp and /proc should be left alone
- dir = Environment.get_tmp_dir();
- if (dir != null)
- rv.append(dir);
-
+ // Skip all of our temporary directories
+ foreach (var tempdir in DejaDup.get_tempdirs())
+ rv.append(tempdir);
+
+ // Skip kernel directories
rv.append("/proc");
rv.append("/sys");
=== modified file 'tests/runner.vala'
--- tests/runner.vala 2012-12-04 21:40:51 +0000
+++ tests/runner.vala 2013-01-20 20:14:20 +0000
@@ -72,6 +72,10 @@
Environment.set_variable("XDG_CACHE_HOME", Path.build_filename(dir, "cache"), true);
Environment.set_variable("PATH", get_srcdir() + "/mock:" + Environment.get_variable("PATH"), true);
+ var tempdir = Path.build_filename(dir, "tmp");
+ DejaDup.ensure_directory_exists(tempdir);
+ Environment.set_variable("DEJA_DUP_TEMPDIR", tempdir, true);
+
var settings = DejaDup.get_settings();
settings.set_string(DejaDup.BACKEND_KEY, "file");
settings = DejaDup.get_settings(DejaDup.FILE_ROOT);
@@ -138,22 +142,30 @@
var archive = tmp_archive ? "?" : "%s/deja-dup".printf(cachedir);
+ string enc_str = "";
+ if (!encrypted)
+ enc_str = "--no-encryption ";
+
+ var tempdir = Path.build_filename(test_home, "tmp");
+
+ var end_str = "%s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '--tempdir=%s' '%s'".printf(enc_str, archive, tempdir, make_fd_arg(as_root));
+
if (mode == Mode.CLEANUP)
- return "cleanup '--force' 'file://%s' '--gio' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '%s'".printf(backupdir, encrypted ? "" : "'--no-encryption' ", archive, make_fd_arg(as_root));
+ return "cleanup '--force' 'file://%s' '--gio' %s".printf(backupdir, end_str);
else if (mode == Mode.RESTORE) {
string file_arg = "", dest_arg = "";
if (file_to_restore != null) {
file_arg = "'--file-to-restore=%s' ".printf(file_to_restore.substring(1)); // skip root /
dest_arg = file_to_restore;
}
- return "'restore' '--gio' %s%s'--force' 'file://%s' '%s%s' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '%s'".printf(file_arg, extra, backupdir, restoredir, dest_arg, encrypted ? "" : "'--no-encryption' ", archive, make_fd_arg(as_root));
+ return "'restore' '--gio' %s%s'--force' 'file://%s' '%s%s' %s".printf(file_arg, extra, backupdir, restoredir, dest_arg, end_str);
}
else if (mode == Mode.VERIFY)
- return "'restore' '--gio' '--file-to-restore=%s/deja-dup/metadata' '--force' 'file://%s' '%s/deja-dup/metadata' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '%s'".printf(cachedir.substring(1), backupdir, cachedir, encrypted ? "" : "'--no-encryption' ", archive, make_fd_arg(as_root));
+ return "'restore' '--gio' '--file-to-restore=%s/deja-dup/metadata' '--force' 'file://%s' '%s/deja-dup/metadata' %s".printf(cachedir.substring(1), backupdir, cachedir, end_str);
else if (mode == Mode.LIST)
- return "'list-current-files' '--gio' 'file://%s' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '%s'".printf(backupdir, encrypted ? "" : "'--no-encryption' ", archive, make_fd_arg(as_root));
+ return "'list-current-files' '--gio' 'file://%s' %s".printf(backupdir, end_str);
else if (mode == Mode.REMOVE)
- return "'remove-all-but-n-full' '%d' '--force' 'file://%s' '--gio' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '%s'".printf(remove_n, backupdir, encrypted ? "" : "'--no-encryption' ", archive, make_fd_arg(as_root));
+ return "'remove-all-but-n-full' '%d' '--force' 'file://%s' '--gio' %s".printf(remove_n, backupdir, end_str);
string source_str = "";
if (mode == Mode.DRY || mode == Mode.BACKUP)
@@ -163,10 +175,6 @@
if (mode == Mode.DRY)
dry_str = "--dry-run ";
- string enc_str = "";
- if (!encrypted)
- enc_str = "--no-encryption ";
-
string args = "";
if (br.is_full && !br.is_first && (mode == Mode.BACKUP || mode == Mode.DRY))
@@ -207,7 +215,7 @@
args += "'--include=%s' ".printf(Environment.get_home_dir());
args += include_args;
- string[] excludes2 = {"/sys", "/proc", Environment.get_tmp_dir()};
+ string[] excludes2 = {"/sys", "/proc", tempdir};
foreach (string ex in excludes2) {
if (FileUtils.test (ex, FileTest.EXISTS))
args += "'--exclude=%s' ".printf(ex);
@@ -226,7 +234,7 @@
args += "'--exclude=**' ";
}
- args += "%s'--gio' %s%s'file://%s' %s'--verbosity=9' '--gpg-options=--no-use-agent' '--archive-dir=%s' '%s'".printf(extra, dry_str, source_str, backupdir, enc_str, archive, make_fd_arg(as_root));
+ args += "%s'--gio' %s%s'file://%s' %s".printf(extra, dry_str, source_str, backupdir, end_str);
return args;
}
@@ -569,10 +577,15 @@
if (as_root)
dupscript += "\n" + "AS_ROOT";
- if (script != null)
- dupscript += "\n" + "SCRIPT: " + script;
- else if (mode == Mode.VERIFY)
- dupscript += "\n" + "SCRIPT: mkdir -p %s/deja-dup/metadata; echo 'This folder can be safely deleted.\\n0' > %s/deja-dup/metadata/README".printf(cachedir, cachedir);
+ var verify_script = "mkdir -p %s/deja-dup/metadata && echo 'This folder can be safely deleted.\\n0' > %s/deja-dup/metadata/README".printf(cachedir, cachedir);
+ if (mode == Mode.VERIFY)
+ dupscript += "\n" + "SCRIPT: " + verify_script;
+ if (script != null) {
+ if (mode == Mode.VERIFY)
+ dupscript += " && " + script;
+ else
+ dupscript += "\n" + "SCRIPT: " + script;
+ }
if (passphrase)
dupscript += "\n" + "PASSPHRASE: test";
=== added file 'tests/scripts/clean-tempdir.test'
--- tests/scripts/clean-tempdir.test 1970-01-01 00:00:00 +0000
+++ tests/scripts/clean-tempdir.test 2013-01-20 20:14:20 +0000
@@ -0,0 +1,12 @@
+# Make sure we clean up the tempdir that we give to duplicity after we're done
+
+[Operation]
+Type=backup
+Script=mkdir -p @TEST_HOME@/tmp/duplicity-blarg @TEST_HOME@/tmp/nope
+
+[Duplicity]
+Runs=status;dry;backup;status-restore;list;verify;
+
+# Check at end of everything that we cleared the temp files
+[Duplicity verify]
+Script=test ! -e @TEST_HOME@/tmp/duplicity-blarg -a -d @TEST_HOME@/tmp/nope
=== modified file 'tests/scripts/verify.test'
--- tests/scripts/verify.test 2012-08-10 18:33:29 +0000
+++ tests/scripts/verify.test 2013-01-20 20:14:20 +0000
@@ -7,4 +7,4 @@
Runs=status;dry;backup;status-restore;list;verify;
[Duplicity verify]
-Script=mkdir -p @XDG_CACHE_HOME@/deja-dup/metadata; echo 'Nope' > @XDG_CACHE_HOME@/deja-dup/metadata/README
+Script=echo 'Nope' > @XDG_CACHE_HOME@/deja-dup/metadata/README
=== modified file 'tools/duplicity/DuplicityInstance.vala'
--- tools/duplicity/DuplicityInstance.vala 2012-10-30 00:19:38 +0000
+++ tools/duplicity/DuplicityInstance.vala 2013-01-20 20:14:20 +0000
@@ -29,8 +29,29 @@
public bool verbose {get; private set; default = false;}
public string forced_cache_dir {get; set; default = null;}
- public virtual void start(List<string> argv_in, List<string>? envp_in,
- bool as_root = false) throws Error
+ public async void start(List<string> argv_in, List<string>? envp_in,
+ bool as_root = false)
+ {
+ try {
+ /* Make deep copies of the lists, so if our caller doesn't yield, the
+ lists won't be invalidated. */
+ var argv = new List<string>();
+ foreach (var arg in argv_in)
+ argv.append(arg);
+ var envp = new List<string>();
+ foreach (var env in envp_in)
+ envp.append(env);
+ if (!yield start_internal(argv, envp, as_root))
+ done(false, false);
+ }
+ catch (Error e) {
+ warning("%s\n", e.message);
+ done(false, false);
+ }
+ }
+
+ async bool start_internal(List<string> argv_in, List<string>? envp_in,
+ bool as_root) throws Error
{
var verbose_str = Environment.get_variable("DEJA_DUP_DEBUG");
if (verbose_str != null && int.parse(verbose_str) > 0)
@@ -73,46 +94,29 @@
if (cache_dir == null)
cache_dir = Environment.get_user_cache_dir();
if (cache_dir != null) {
- bool add_dir = false;
- var cache_file = File.new_for_path(cache_dir);
- cache_file = cache_file.get_child(Config.PACKAGE);
- try {
- if (cache_file.make_directory_with_parents(null))
- add_dir = true;
- }
- catch (IOError.EXISTS e) {
- add_dir = true; // ignore
- }
- catch (Error e) {
- warning("%s\n", e.message);
- }
- if (add_dir)
- argv.append("--archive-dir=" + cache_file.get_path());
+ cache_dir = Path.build_filename(cache_dir, Config.PACKAGE);
+ if (DejaDup.ensure_directory_exists(cache_dir))
+ argv.append("--archive-dir=" + cache_dir);
}
-
+
+ // Specify tempdir
+ var tempdir = yield DejaDup.get_tempdir();
+ if (DejaDup.ensure_directory_exists(tempdir))
+ argv.append("--tempdir=%s".printf(tempdir));
+
// Add logging argument
if (as_root) {
// Make log file
int logfd = 0;
- try {
- string logname;
- logfd = FileUtils.open_tmp(Config.PACKAGE + "-XXXXXX", out logname);
- logfile = File.new_for_path(logname);
- }
- catch (Error e) {
- warning("%s\n", e.message);
- done(false, false);
- return;
- }
-
+ string logname;
+ logfd = FileUtils.open_tmp(Config.PACKAGE + "-XXXXXX", out logname);
+ logfile = File.new_for_path(logname);
argv.append("--log-file=%s".printf(logfile.get_path()));
}
else {
// Open pipes to communicate with subprocess
- if (Posix.pipe(pipes) != 0) {
- done(false, false);
- return;
- }
+ if (Posix.pipe(pipes) != 0)
+ return false;
argv.append("--log-fd=%d".printf(pipes[1]));
}
@@ -192,9 +196,10 @@
if (pipes[1] != -1)
Posix.close(pipes[1]);
- read_log.begin();
+ yield read_log();
+ return true;
}
-
+
public bool is_started()
{
return (int)child_pid > 0;
=== modified file 'tools/duplicity/DuplicityJob.vala'
--- tools/duplicity/DuplicityJob.vala 2012-10-31 12:09:38 +0000
+++ tools/duplicity/DuplicityJob.vala 2013-01-20 20:14:20 +0000
@@ -1460,13 +1460,7 @@
}
/* Start duplicity instance */
- try {
- inst.start(argv, envp, needs_root);
- }
- catch (Error e) {
- show_error(e.message);
- done(false, false, null);
- }
+ inst.start.begin(argv, envp, needs_root);
}
}
Follow ups
