duplicity-team team mailing list archive
-
duplicity-team team
-
Mailing list archive
-
Message #02695
[Merge] lp:~stapelberg+ubuntu/duplicity/add-onedrive-backend into lp:duplicity
Michael Stapelberg has proposed merging lp:~stapelberg+ubuntu/duplicity/add-onedrive-backend into lp:duplicity.
Commit message:
Add a Microsoft OneDrive backend
Requested reviews:
duplicity-team (duplicity-team)
For more details, see:
https://code.launchpad.net/~stapelberg+ubuntu/duplicity/add-onedrive-backend/+merge/245534
Add a Microsoft OneDrive backend
--
Your team duplicity-team is requested to review the proposed merge of lp:~stapelberg+ubuntu/duplicity/add-onedrive-backend into lp:duplicity.
=== modified file 'bin/duplicity.1'
--- bin/duplicity.1 2014-12-10 19:09:28 +0000
+++ bin/duplicity.1 2015-01-04 21:06:25 +0000
@@ -91,6 +91,13 @@
.B Python library for mega API
- https://github.com/ckornacker/mega.py, ubuntu ppa - ppa:ckornacker/backup
.TP
+.BR "OneDrive backend" " (Microsoft OneDrive)"
+.B python-requests
+- http://python-requests.org
+.br
+.B python-requests-oauthlib
+- https://github.com/requests/requests-oauthlib
+.TP
.BR "ncftp backend" " (ftp, select via ncftp+ftp://)"
.B NcFTP
- http://www.ncftp.com/
@@ -1178,6 +1185,12 @@
mega://user[:password]@mega.co.nz/some_dir
.RE
.PP
+.B "OneDrive Backend"
+.PP
+.RS
+onedrive://some_dir
+.RE
+.PP
.B "Par2 Wrapper Backend"
.PP
.RS
=== added file 'duplicity/backends/onedrivebackend.py'
--- duplicity/backends/onedrivebackend.py 1970-01-01 00:00:00 +0000
+++ duplicity/backends/onedrivebackend.py 2015-01-04 21:06:25 +0000
@@ -0,0 +1,330 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+# vim:tabstop=4:shiftwidth=4:expandtab
+#
+# Copyright 2014 Google Inc.
+# Contact Michael Stapelberg <stapelberg+duplicity@xxxxxxxxxx>
+# This is NOT a Google product.
+#
+# This file is part of duplicity.
+#
+# Duplicity is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# Duplicity is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with duplicity; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import time
+import json
+import os
+import sys
+# On debian (and derivatives), get these dependencies using:
+# apt-get install python-requests python-requests-oauthlib
+# On fedora (and derivatives), get these dependencies using:
+# yum install python-requests python-requests-oauthlib
+import requests
+from requests_oauthlib import OAuth2Session
+
+import duplicity.backend
+from duplicity.errors import BackendException
+from duplicity import globals
+from duplicity import log
+
+# For documentation on the API, see
+# http://msdn.microsoft.com/en-us/library/dn659752.aspx
+# http://msdn.microsoft.com/en-us/library/dn631844.aspx
+# https://gist.github.com/rgregg/37ba8929768a62131e85
+class OneDriveBackend(duplicity.backend.Backend):
+ """Uses Microsoft OneDrive (formerly SkyDrive) for backups."""
+
+ OAUTH_TOKEN_PATH = os.path.expanduser(
+ '~/.duplicity_onedrive_oauthtoken.json')
+
+ API_URI = 'https://apis.live.net/v5.0/'
+ MAXIMUM_FRAGMENT_SIZE = 60 * 1024 * 1024
+ BITS_1_5_UPLOAD_PROTOCOL = '{7df0354d-249b-430f-820d-3d2a9bef4931}'
+ CLIENT_ID = '000000004C12E85D'
+ CLIENT_SECRET = 'k1oR0CbtbvTG9nK1PEDeVW2dzvAaiN4d'
+ OAUTH_TOKEN_URI = 'https://login.live.com/oauth20_token.srf'
+ OAUTH_AUTHORIZE_URI = 'https://login.live.com/oauth20_authorize.srf'
+ OAUTH_REDIRECT_URI = 'https://login.live.com/oauth20_desktop.srf'
+ # wl.skydrive is for reading files,
+ # wl.skydrive_update is for creating/writing files,
+ # wl.offline_access is necessary for duplicity to access onedrive without
+ # the user being logged in right now.
+ OAUTH_SCOPE = ['wl.skydrive', 'wl.skydrive_update', 'wl.offline_access']
+
+ def __init__(self, parsed_url):
+ duplicity.backend.Backend.__init__(self, parsed_url)
+ self.names_to_ids = None
+ self.user_id = None
+ self.directory = parsed_url.path.lstrip('/')
+ if self.directory == "":
+ raise BackendException((
+ 'You did not specify a path. '
+ 'Please specify a path, e.g. onedrive://duplicity_backups'))
+ if globals.volsize > (10 * 1024 * 1024 * 1024):
+ raise BackendException((
+ 'Your --volsize is bigger than 10 GiB, which is the maximum '
+ 'file size on OneDrive.'))
+ self.initialize_oauth2_session()
+ self.resolve_directory()
+
+ def initialize_oauth2_session(self):
+ def token_updater(token):
+ try:
+ with open(self.OAUTH_TOKEN_PATH, 'w') as f:
+ json.dump(token, f)
+ except Exception as e:
+ log.Error(('Could not save the OAuth2 token to %s. '
+ 'This means you may need to do the OAuth2 '
+ 'authorization process again soon. '
+ 'Original error: %s' % (
+ self.OAUTH_TOKEN_PATH, e)))
+
+ token = None
+ try:
+ with open(self.OAUTH_TOKEN_PATH) as f:
+ token = json.load(f)
+ except IOError as e:
+ log.Error(('Could not load OAuth2 token. '
+ 'Trying to create a new one. (original error: %s)' % e))
+
+ self.http_client = OAuth2Session(
+ self.CLIENT_ID,
+ scope=self.OAUTH_SCOPE,
+ redirect_uri=self.OAUTH_REDIRECT_URI,
+ token=token,
+ auto_refresh_kwargs={
+ 'client_id': self.CLIENT_ID,
+ 'client_secret': self.CLIENT_SECRET,
+ },
+ auto_refresh_url=self.OAUTH_TOKEN_URI,
+ token_updater=token_updater)
+
+ # Send a request to make sure the token is valid (or could at least be
+ # refreshed successfully, which will happen under the covers). In case
+ # this request fails, the provided token was too old (i.e. expired),
+ # and we need to get a new token.
+ user_info_response = self.http_client.get(self.API_URI + 'me')
+ if user_info_response.status_code != requests.codes.ok:
+ token = None
+
+ if token is None:
+ if not sys.stdout.isatty() or not sys.stdin.isatty():
+ log.FatalError(('The OAuth2 token could not be loaded from %s '
+ 'and you are not running duplicity '
+ 'interactively, so duplicity cannot possibly '
+ 'access OneDrive.' % self.OAUTH_TOKEN_PATH))
+ authorization_url, state = self.http_client.authorization_url(
+ self.OAUTH_AUTHORIZE_URI, display='touch')
+
+ print ''
+ print ('In order to authorize duplicity to access your OneDrive, '
+ 'please open %s in a browser and copy the URL of the blank '
+ 'page the dialog leads to.' % authorization_url)
+ print ''
+
+ redirected_to = raw_input('URL of the blank page: ')
+
+ token = self.http_client.fetch_token(self.OAUTH_TOKEN_URI,
+ client_secret=self.CLIENT_SECRET,
+ authorization_response=redirected_to)
+
+ user_info_response = self.http_client.get(self.API_URI + 'me')
+ user_info_response.raise_for_status()
+
+ try:
+ with open(self.OAUTH_TOKEN_PATH, 'w') as f:
+ json.dump(token, f)
+ except Exception as e:
+ log.Error(('Could not save the OAuth2 token to %s. '
+ 'This means you need to do the OAuth2 authorization '
+ 'process on every start of duplicity. '
+ 'Original error: %s' % (
+ self.OAUTH_TOKEN_PATH, e)))
+
+ if not 'id' in user_info_response.json():
+ log.Error('user info response lacks the "id" field.')
+
+ self.user_id = user_info_response.json()['id']
+
+ def resolve_directory(self):
+ """Ensures self.directory_id contains the folder id for the path.
+
+ There is no API call to resolve a logical path (e.g.
+ /backups/duplicity/notebook/), so we recursively list directories
+ until we get the object id of the configured directory, creating
+ directories as necessary.
+ """
+ object_id = 'me/skydrive'
+ for component in self.directory.split('/'):
+ tried_mkdir = False
+ while True:
+ files = self.get_files(object_id)
+ names_to_ids = {x['name']: x['id'] for x in files}
+ if component not in names_to_ids:
+ if not tried_mkdir:
+ self.mkdir(object_id, component)
+ tried_mkdir = True
+ continue
+ raise BackendException((
+ 'Could not resolve/create directory "%s" on '
+ 'OneDrive: %s not in %s (files of folder %s)' % (
+ self.directory, component,
+ names_to_ids.keys(), object_id)))
+ break
+ object_id = names_to_ids[component]
+ self.directory_id = object_id
+ log.Debug('OneDrive id for the configured directory "%s" is "%s"' % (
+ self.directory, self.directory_id))
+
+ def mkdir(self, object_id, folder_name):
+ data = {'name': folder_name, 'description': 'Created by duplicity'}
+ headers = {'Content-Type': 'application/json'}
+ response = self.http_client.post(
+ self.API_URI + object_id,
+ data=json.dumps(data),
+ headers=headers)
+ response.raise_for_status()
+
+ def get_files(self, path):
+ response = self.http_client.get(self.API_URI + path + '/files')
+ response.raise_for_status()
+ if 'data' not in response.json():
+ raise BackendException((
+ 'Malformed JSON: expected "data" member in %s' % (
+ response.json())))
+ return response.json()['data']
+
+ def _list(self):
+ files = self.get_files(self.directory_id)
+ self.names_to_ids = {x['name']: x['id'] for x in files}
+ return [x['name'] for x in files]
+
+ def get_file_id(self, remote_filename):
+ """Returns the file id from cache, updating the cache if necessary."""
+ if (self.names_to_ids is None or
+ remote_filename not in self.names_to_ids):
+ self._list()
+ return self.names_to_ids.get(remote_filename)
+
+ def _get(self, remote_filename, local_path):
+ with local_path.open('wb') as f:
+ file_id = self.get_file_id(remote_filename)
+ if file_id is None:
+ raise BackendException((
+ 'File "%s" cannot be downloaded: it does not exist' % (
+ remote_filename)))
+ response = self.http_client.get(
+ self.API_URI + file_id + '/content', stream=True)
+ response.raise_for_status()
+ for chunk in response.iter_content(chunk_size=4096):
+ if chunk:
+ f.write(chunk)
+ f.flush()
+
+ def _put(self, source_path, remote_filename):
+ # Check if the user has enough space available on OneDrive before even
+ # attempting to upload the file.
+ source_size = os.path.getsize(source_path.name)
+ start = time.time()
+ response = self.http_client.get(self.API_URI + 'me/skydrive/quota')
+ response.raise_for_status()
+ if ('available' in response.json() and
+ source_size > response.json()['available']):
+ raise BackendException((
+ 'Out of space: trying to store "%s" (%d bytes), but only '
+ '%d bytes available on OneDrive.' % (
+ source_path.name, source_size,
+ response.json()['available'])))
+ log.Debug("Checked quota in %fs" % (time.time() - start))
+
+ with source_path.open() as source_file:
+ start = time.time()
+ # Create a BITS session, so that we can upload large files.
+ short_directory_id = self.directory_id.split('.')[-1]
+ url = 'https://cid-%s.users.storage.live.com/items/%s/%s' % (
+ self.user_id, short_directory_id, remote_filename)
+ headers = {
+ 'X-Http-Method-Override': 'BITS_POST',
+ 'BITS-Packet-Type': 'Create-Session',
+ 'BITS-Supported-Protocols': self.BITS_1_5_UPLOAD_PROTOCOL,
+ }
+
+ response = self.http_client.post(
+ url,
+ headers=headers)
+ response.raise_for_status()
+ if (not 'bits-packet-type' in response.headers or
+ response.headers['bits-packet-type'].lower() != 'ack'):
+ raise BackendException((
+ 'File "%s" cannot be uploaded: '
+ 'Could not create BITS session: '
+ 'Server response did not include BITS-Packet-Type: ACK' % (
+ remote_filename)))
+ bits_session_id = response.headers['bits-session-id']
+ log.Debug('BITS session id is "%s"' % bits_session_id)
+
+ # Send fragments (with a maximum size of 60 MB each).
+ offset = 0
+ while True:
+ chunk = source_file.read(self.MAXIMUM_FRAGMENT_SIZE)
+ if len(chunk) == 0:
+ break
+ headers = {
+ 'X-Http-Method-Override': 'BITS_POST',
+ 'BITS-Packet-Type': 'Fragment',
+ 'BITS-Session-Id': bits_session_id,
+ 'Content-Range': 'bytes %d-%d/%d' % (offset, offset + len(chunk) - 1, source_size),
+ }
+ response = self.http_client.post(
+ url,
+ headers=headers,
+ data=chunk)
+ response.raise_for_status()
+ offset += len(chunk)
+
+ # Close the BITS session to commit the file.
+ headers = {
+ 'X-Http-Method-Override': 'BITS_POST',
+ 'BITS-Packet-Type': 'Close-Session',
+ 'BITS-Session-Id': bits_session_id,
+ }
+ response = self.http_client.post(url, headers=headers)
+ response.raise_for_status()
+
+ log.Debug("PUT file in %fs" % (time.time() - start))
+
+ def _delete(self, remote_filename):
+ file_id = self.get_file_id(remote_filename)
+ if file_id is None:
+ raise BackendException((
+ 'File "%s" cannot be deleted: it does not exist' % (
+ remote_filename)))
+ response = self.http_client.delete(self.API_URI + file_id)
+ response.raise_for_status()
+
+ def _query(self, remote_filename):
+ file_id = self.get_file_id(remote_filename)
+ if file_id is None:
+ return {'size': -1}
+ response = self.http_client.get(self.API_URI + file_id)
+ response.raise_for_status()
+ if 'size' not in response.json():
+ raise BackendException((
+ 'Malformed JSON: expected "size" member in %s' % (
+ response.json())))
+ return {'size': response.json()['size']}
+
+ def _retry_cleanup(self):
+ self.initialize_oauth2_session()
+
+duplicity.backend.register_backend('onedrive', OneDriveBackend)
=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py 2014-12-12 14:54:56 +0000
+++ duplicity/commandline.py 2015-01-04 21:06:25 +0000
@@ -859,6 +859,7 @@
mega://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s
copy://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s
dpbx:///%(some_dir)s
+ onedrive://%(some_dir)s
""" % dict
Follow ups
