duplicity-team team mailing list archive

[nils <nils@xxxxxxxxxxx>]

nils has proposed merging lp:~dernils/duplicity/robust-dropbox-backend into lp:duplicity.

Requested reviews:
  duplicity-team (duplicity-team)

For more details, see:
https://code.launchpad.net/~dernils/duplicity/robust-dropbox-backend/+merge/311073

Added new command line option --backend-retry-delay that allows to determine the time that duplicity sleeps before retrying after an error has occured. 

Added some robustness to dpbxbackend.py that ensures re-authentication happens in case that a socket is changed (e.g. due to a forced reconnect of a dynamic internet connection).
-- 
Your team duplicity-team is requested to review the proposed merge of lp:~dernils/duplicity/robust-dropbox-backend into lp:duplicity.

=== modified file 'duplicity/backend.py'
--- duplicity/backend.py	2016-06-28 21:03:46 +0000
+++ duplicity/backend.py	2016-11-16 20:35:41 +0000
@@ -395,9 +395,9 @@
                                  % (n, e.__class__.__name__, util.uexc(e)))
                     if not at_end:
                         if isinstance(e, TemporaryLoadException):
-                            time.sleep(90)  # wait longer before trying again
+                            time.sleep(3*globals.backend_retry_delay)  # wait longer before trying again
                         else:
-                            time.sleep(30)  # wait a bit before trying again
+                            time.sleep(globals.backend_retry_delay)  # wait a bit before trying again
                         if hasattr(self.backend, '_retry_cleanup'):
                             self.backend._retry_cleanup()
 

=== modified file 'duplicity/backends/dpbxbackend.py'
--- duplicity/backends/dpbxbackend.py	2016-06-28 21:03:46 +0000
+++ duplicity/backends/dpbxbackend.py	2016-11-16 20:35:41 +0000
@@ -98,7 +98,14 @@
         self.auth_flow = None
 
         self.login()
-
+    def user_authenticated(self):
+        try:
+            account = self.api_client.users_get_current_account()
+            log.Debug("User authenticated as ,%s" % account)
+            return True
+        except:
+            log.Debug('User not authenticated')
+            return False
     def load_access_token(self):
         return os.environ.get('DPBX_ACCESS_TOKEN', None)
 
@@ -196,10 +203,14 @@
                                    (res_metadata.size, file_size))
 
     def put_file_small(self, source_path, remote_path):
+        if not self.user_authenticated():
+            self.login()
+
         file_size = os.path.getsize(source_path.name)
         f = source_path.open('rb')
         try:
             log.Debug('dpbx,files_upload(%s, [%d bytes])' % (remote_path, file_size))
+
             res_metadata = self.api_client.files_upload(f, remote_path,
                                                         mode=WriteMode.overwrite,
                                                         autorename=False,
@@ -212,6 +223,9 @@
             f.close()
 
     def put_file_chunked(self, source_path, remote_path):
+        if not self.user_authenticated():
+            self.login()
+
         file_size = os.path.getsize(source_path.name)
         f = source_path.open('rb')
         try:
@@ -295,6 +309,10 @@
                     log.Debug('dpbx,files_upload_session_append: %s' % e)
 
                     retry_number -= 1
+
+                    if not self.user_authenticated():
+                        self.login()
+
                     if retry_number == 0:
                         raise
 
@@ -320,6 +338,9 @@
 
     @command()
     def _get(self, remote_filename, local_path):
+        if not self.user_authenticated():
+            self.login()
+
         remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
         remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
 
@@ -353,6 +374,8 @@
     @command()
     def _list(self):
         # Do a long listing to avoid connection reset
+        if not self.user_authenticated():
+            self.login()
         remote_dir = '/' + urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
 
         log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
@@ -373,6 +396,9 @@
 
     @command()
     def _delete(self, filename):
+        if not self.user_authenticated():
+            self.login()
+
         remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
         remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
 
@@ -390,6 +416,8 @@
 
     @command()
     def _query(self, filename):
+        if not self.user_authenticated():
+            self.login()
         remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
         remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
 
@@ -399,6 +427,8 @@
         return {'size': info.size}
 
     def check_renamed_files(self, file_list):
+        if not self.user_authenticated():
+            self.login()
         bad_list = [x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None]
         if len(bad_list) == 0:
             return

=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py	2016-03-01 15:38:58 +0000
+++ duplicity/commandline.py	2016-11-16 20:35:41 +0000
@@ -624,6 +624,11 @@
                       metavar=_("path"), dest="file_changed",
                       callback=lambda o, s, v, p: setattr(p.values, "file_changed", v.rstrip('/')))
 
+    # delay time before next try after a failure of a backend operation
+    # TRANSL: Used in usage help. Example:
+    # --backend-retry-delay <seconds>
+    parser.add_option("--backend-retry-delay", type="int", metavar=_("seconds"))
+                      
     # parse the options
     (options, args) = parser.parse_args()
 

=== modified file 'duplicity/globals.py'
--- duplicity/globals.py	2016-06-12 13:13:57 +0000
+++ duplicity/globals.py	2016-11-16 20:35:41 +0000
@@ -293,3 +293,6 @@
 
 # If set, collect only the file status, not the whole root.
 file_changed = None
+
+# delay (in seconds) before next operation after failure
+backend_retry_delay = 30