duplicity-team team mailing list archive

[Michael Terry <michael.terry@xxxxxxxxxxxxx>]

Michael Terry has proposed merging lp:~mterry/duplicity/gpg-encode into lp:duplicity.

Requested reviews:
  duplicity-team (duplicity-team)
Related bugs:
  Bug #1276327 in duplicity (Ubuntu): "deja-dup/duplicity fails gpg encrypted backup in trusty"
  https://bugs.launchpad.net/ubuntu/+source/duplicity/+bug/1276327

For more details, see:
https://code.launchpad.net/~mterry/duplicity/gpg-encode/+merge/204808

getpass.getpass(prompt) eventually calls str(prompt).  Which is a no go, if the prompt contains unicode.  Here's a patch to always pass getpass() a byte string.

Our tests didn't catch this because they always set PASSPHRASE.  I've added a test that passes the passphrase via stdin.

In order to actually test the bug condition (_() returning unicode strings that can't convert to ascii), I added a gettext.py override that always adds an invisible unicode character to returned strings when testing.  This way, we should be able to catch future ascii translation problems in any of the code under test.
-- 
https://code.launchpad.net/~mterry/duplicity/gpg-encode/+merge/204808
Your team duplicity-team is requested to review the proposed merge of lp:~mterry/duplicity/gpg-encode into lp:duplicity.

=== modified file 'bin/duplicity'
--- bin/duplicity	2014-01-21 21:04:27 +0000
+++ bin/duplicity	2014-02-05 02:57:13 +0000
@@ -27,7 +27,7 @@
 # Please send mail to me or the mailing list if you find bugs or have
 # any suggestions.
 
-import getpass, gzip, os, sys, time, types
+import gzip, os, sys, time, types
 import traceback, platform, statvfs, resource, re
 import threading
 from datetime import datetime
@@ -37,9 +37,6 @@
 if os.path.exists(os.path.join(pwd, "../duplicity")):
     sys.path.insert(0, os.path.abspath(os.path.join(pwd, "../.")))
 
-import gettext
-gettext.install('duplicity', codeset='utf8')
-
 from duplicity import log
 log.setup()
 
@@ -65,6 +62,13 @@
 # If exit_val is not None, exit with given value at end.
 exit_val = None
 
+def getpass_safe(message):
+    # getpass() in Python 2.x will call str() on our prompt.  So we can't pass
+    # in non-ascii characters.
+    import getpass, locale
+    message = message.encode(locale.getpreferredencoding(), 'replace')
+    return getpass.getpass(message)
+
 def get_passphrase(n, action, for_signing = False):
     """
     Check to make sure passphrase is indeed needed, then get
@@ -160,19 +164,19 @@
                     if use_cache and globals.gpg_profile.signing_passphrase:
                         pass1 = globals.gpg_profile.signing_passphrase
                     else:
-                        pass1 = getpass.getpass(_("GnuPG passphrase for signing key:")+" ")
+                        pass1 = getpass_safe(_("GnuPG passphrase for signing key:")+" ")
                 else:
                     if use_cache and globals.gpg_profile.passphrase:
                         pass1 = globals.gpg_profile.passphrase
                     else:
-                        pass1 = getpass.getpass(_("GnuPG passphrase:")+" ")
+                        pass1 = getpass_safe(_("GnuPG passphrase:")+" ")
 
             if n == 1:
                 pass2 = pass1
             elif for_signing:
-                pass2 = getpass.getpass(_("Retype passphrase for signing key to confirm: "))
+                pass2 = getpass_safe(_("Retype passphrase for signing key to confirm: "))
             else:
-                pass2 = getpass.getpass(_("Retype passphrase to confirm: "))
+                pass2 = getpass_safe(_("Retype passphrase to confirm: "))
 
             if not pass1 == pass2:
                 print _("First and second passphrases do not match!  Please try again.")

=== modified file 'bin/rdiffdir'
--- bin/rdiffdir	2013-12-27 06:39:00 +0000
+++ bin/rdiffdir	2014-02-05 02:57:13 +0000
@@ -27,9 +27,6 @@
 
 import sys, getopt, gzip, os
 
-import gettext
-gettext.install('duplicity', codeset='utf8')
-
 from duplicity import diffdir
 from duplicity import patchdir
 from duplicity import log

=== added directory 'testing/overrides'
=== added file 'testing/overrides/gettext.py'
--- testing/overrides/gettext.py	1970-01-01 00:00:00 +0000
+++ testing/overrides/gettext.py	2014-02-05 02:57:13 +0000
@@ -0,0 +1,34 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4; encoding:utf8 -*-
+#
+# Copyright 2014 Michael Terry <mike@xxxxxxxxxxx>
+#
+# This file is part of duplicity.
+#
+# Duplicity is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# Duplicity is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with duplicity; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+# This is just a small override to the system gettext.py which allows us to
+# always return a string with fancy unicode characters, which will notify us
+# if we ever get a unicode->ascii translation by accident.
+
+def translation(*args, **kwargs):
+    class Translation:
+        ZWSP = u"​" # ZERO WIDTH SPACE, basically an invisible space separator
+        def install(self, **kwargs):
+            import __builtin__
+            __builtin__.__dict__['_'] = lambda x: x + self.ZWSP
+        def ungettext(self, one, more, n):
+            if n == 1: return one + self.ZWSP
+            else:      return more + self.ZWSP
+    return Translation()

=== modified file 'testing/run-tests'
--- testing/run-tests	2011-11-24 01:49:53 +0000
+++ testing/run-tests	2014-02-05 02:57:13 +0000
@@ -25,9 +25,9 @@
 
 THISDIR=$(pwd)
 export TZ=US/Central
-export LANG=
+export LANG=en_US.UTF-8
 # up for 'duplicity' module and here for 'helper.py'
-export PYTHONPATH="$(dirname $THISDIR):$THISDIR/helpers"
+export PYTHONPATH="$THISDIR/overrides:$(dirname $THISDIR):$THISDIR/helpers"
 export GNUPGHOME="$THISDIR/gnupg"
 export PATH="$(dirname $THISDIR)/bin:$PATH"
 

=== modified file 'testing/tests/finaltest.py'
--- testing/tests/finaltest.py	2012-11-24 19:45:09 +0000
+++ testing/tests/finaltest.py	2014-02-05 02:57:13 +0000
@@ -20,6 +20,7 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
 import helper
+import pexpect
 import sys, os, unittest
 
 import duplicity.backend
@@ -50,7 +51,8 @@
     """
     Test backup/restore using duplicity binary
     """
-    def run_duplicity(self, arglist, options = [], current_time = None):
+    def run_duplicity(self, arglist, options = [], current_time = None,
+                      passphrase_input = None):
         """Run duplicity binary with given arguments and options"""
         options.append("--archive-dir testfiles/cache")
         cmd_list = ["duplicity"]
@@ -62,22 +64,23 @@
         cmd_list.extend(arglist)
         cmdline = " ".join(cmd_list)
         #print "Running '%s'." % cmdline
-        if not os.environ.has_key('PASSPHRASE'):
+        if passphrase_input is None and not os.environ.has_key('PASSPHRASE'):
             os.environ['PASSPHRASE'] = 'foobar'
-        return_val = os.system(cmdline)
+        (output, return_val) = pexpect.run(cmdline, withexitstatus=True,
+            events={'passphrase.*:': passphrase_input}) 
         if return_val:
             raise CmdError(return_val)
 
-    def backup(self, type, input_dir, options = [], current_time = None):
+    def backup(self, type, input_dir, options = [], **kwargs):
         """Run duplicity backup to default directory"""
         options = options[:]
         if type == "full":
             options.insert(0, 'full')
         args = [input_dir, "'%s'" % backend_url]
-        self.run_duplicity(args, options, current_time)
+        self.run_duplicity(args, options, **kwargs)
 
     def restore(self, file_to_restore = None, time = None, options = [],
-                current_time = None):
+                **kwargs):
         options = options[:] # just nip any mutability problems in bud
         assert not os.system("rm -rf testfiles/restore_out")
         args = ["'%s'" % backend_url, "testfiles/restore_out"]
@@ -85,17 +88,17 @@
             options.extend(['--file-to-restore', file_to_restore])
         if time:
             options.extend(['--restore-time', str(time)])
-        self.run_duplicity(args, options, current_time)
+        self.run_duplicity(args, options, **kwargs)
 
     def verify(self, dirname, file_to_verify = None, time = None, options = [],
-               current_time = None):
+               **kwargs):
         options = ["verify"] + options[:]
         args = ["'%s'" % backend_url, dirname]
         if file_to_verify:
             options.extend(['--file-to-restore', file_to_verify])
         if time:
             options.extend(['--restore-time', str(time)])
-        self.run_duplicity(args, options, current_time)
+        self.run_duplicity(args, options, **kwargs)
 
     def deltmp(self):
         """Delete temporary directories"""
@@ -255,6 +258,12 @@
         assert chain.start_time == 30000, chain.start_time
         assert chain.end_time == 40000, chain.end_time
 
+    def test_piped_password(self):
+        """Make sure that prompting for a password works"""
+        self.backup("full", "testfiles/empty_dir",
+                    passphrase_input="foobar\nfoobar\n")
+        self.restore(passphrase_input="foobar\n")
+
 class FinalTest1(FinalTest, unittest.TestCase):
     def setUp(self):
         assert not os.system("tar xzf testfiles.tar.gz > /dev/null 2>&1")