ladon-dev-team team mailing list archive
-
ladon-dev-team team
-
Mailing list archive
-
Message #00171
[Merge] lp:~luca-vs800/ladon/ladon into lp:ladon
ellethee has proposed merging lp:~luca-vs800/ladon/ladon into lp:ladon.
Requested reviews:
Ladon Developer (ladon-dev-team)
For more details, see:
https://code.launchpad.net/~luca-vs800/ladon/ladon/+merge/215657
Replaces the standard file object with a file-like object that supports a callback function.
So we can set up a function that can manage a progress bar or some other stuff.
to set your callback, use:
from ladon.tools.callback_file import set_callback
set_callback(up_n_down_callback)
the callback should manage these events:
*init* *start* *read* *write* *end* *close*
the callback structure should be like this:
def up_n_down_callback(event, attachment, **kwargs):
if event == 'init':
"""
Put here the progressbar initialization (file is open)
"""
elif event == 'start':
"""
Whatever you need when your file starts to read or write.
"""
elif event == 'read':
"""
Whatever you need when file reads.
"""
elif event == 'write':
"""
Whatever you need when file writes.
"""
elif event == 'end':
"""
When the file progress reach the file size (ends read/write process).
"""
elif event == 'close':
"""
When the file is closed.
"""
this could be and example:
import progressbar
pbar = progressbar.ProgressBar(
widgets=[
"File Name placeholder", progressbar.Percentage(), ' ',
progressbar.Bar('#'), ' ', progressbar.ETA(), ' ',
progressbar.FileTransferSpeed()])
def up_n_down_callback(event, attachment, **kwargs):
"""
Progress callback.
"""
if event == 'init':
pbar.maxval = attachment.size
pbar.widgets[0] = attachment.save_name
elif event == 'start':
pbar.start()
elif event in ['read', 'write']:
pbar.update(attachment.progress)
elif event == 'end':
pbar.finish()
elif event == 'close':
pass
--
https://code.launchpad.net/~luca-vs800/ladon/ladon/+merge/215657
Your team Ladon Developer is requested to review the proposed merge of lp:~luca-vs800/ladon/ladon into lp:ladon.
=== modified file 'frameworks/python/examples/services/transferservice.py'
--- frameworks/python/examples/services/transferservice.py 2013-11-08 12:44:11 +0000
+++ frameworks/python/examples/services/transferservice.py 2014-04-14 13:05:34 +0000
@@ -75,6 +75,7 @@
f = File()
f.name = name
f.data = attachment(open(join(upload_dir,name),'rb'))
+ f.data.headers.update({'x-filename': name})
response += [f]
return response
=== added file 'frameworks/python/examples/test_callback.py'
--- frameworks/python/examples/test_callback.py 1970-01-01 00:00:00 +0000
+++ frameworks/python/examples/test_callback.py 2014-04-14 13:05:34 +0000
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ test
+"""
+__file_name__ = "test_callback.py"
+__author__ = "luca"
+__version__ = "1.0.0"
+__date__ = "2014-04-09"
+
+
+import os
+import pprint
+from os.path import join, dirname, abspath
+from ladon.clients.jsonwsp import JSONWSPClient
+
+base_url = 'http://localhost:8080'
+files_dir = join(dirname(abspath(__file__)), 'files')
+download_dir = join(dirname(abspath(__file__)), 'download')
+
+
+# We need a progressbar to show
+import progressbar
+pbar = progressbar.ProgressBar(
+ widgets=[
+ "Trasferimento",
+ progressbar.Percentage(),
+ ' ', progressbar.Bar('#'),
+ ' ', progressbar.ETA(),
+ ' ', progressbar.FileTransferSpeed()],
+)
+
+
+# And a silly callback that can manage our events.
+def up_n_down_callback(event, attachment, **kwargs):
+ """
+ Callback function
+
+ :param event: one of the events 'init', 'start', 'read', 'write', 'end'
+ and 'close'.
+ :param attachment: a CallBackFile object which have also 'size',
+ 'save_name' and 'progress' as properties.
+ :param **kwargs: In case of...
+ """
+ if event == 'init':
+ pass
+ elif event == 'start':
+ pbar.maxval = attachment.size
+ pbar.widgets[0] = attachment.save_name
+ pbar.start()
+ elif event in ['read', 'write']:
+ pbar.update(attachment.progress)
+ elif event == 'end':
+ pbar.finish()
+ pass
+ elif event == 'close':
+ pass
+
+
+# Let's set up the callback
+from ladon.tools.callback_file import set_callback
+set_callback(up_n_down_callback)
+
+
+def print_result(jsonwsp_resp):
+ if jsonwsp_resp.status == 200:
+ if 'result' in jsonwsp_resp.response_dict:
+ pprint.pprint(jsonwsp_resp.response_dict['result'], indent=2)
+ else:
+ pprint.pprint(jsonwsp_resp.response_dict)
+ else:
+ print("A problem occured while communicating with the service:\n")
+ print(jsonwsp_resp.response_body)
+
+
+def test_callback():
+ global files_dir, download_dir
+
+ print("\n\nTesting TransferService:\n")
+ # Load the TransferService description
+ transfer_client = JSONWSPClient(
+ base_url +
+ '/TransferService/jsonwsp/description')
+
+ # File list for the upload() call
+ file_list = []
+ # list of file names fot the download() call
+ name_list = []
+
+ # Get a list of files in the "files" folder
+ files = os.listdir(files_dir)
+ for f in files:
+ fpath = join(files_dir, f)
+ # Check if the entry is a file
+ if os.path.isfile(fpath):
+ # Add the file as a TransferService.File object (for the upload()
+ # call)
+ file_list += [{
+ # Attach the file using an open file-handle
+ 'data': open(fpath, 'rb'),
+ 'name': f # The file name
+ }]
+
+ # Add the file name to the list of file names (for the download()
+ # call)
+ name_list += [f]
+
+ # Upload multiple files (all files found in the "files" directory) in one
+ # request
+ jsonwsp_resp = transfer_client.upload(incomming=file_list)
+ print_result(jsonwsp_resp)
+ # Download all the files we just uploaded in one request
+ jsonwsp_resp = transfer_client.download(names=name_list)
+ print_result(jsonwsp_resp)
+ ## The attachments are referenced as open file-handles in the response object
+ ## read their content and save it as files in the "download" folder.
+ if jsonwsp_resp.status == 200:
+ jsonwsp_resp.save_all(download_dir)
+
+if __name__ == '__main__':
+ test_callback()
=== modified file 'frameworks/python/src/ladon/clients/jsonwsp.py'
--- frameworks/python/src/ladon/clients/jsonwsp.py 2013-11-07 10:30:11 +0000
+++ frameworks/python/src/ladon/clients/jsonwsp.py 2014-04-14 13:05:34 +0000
@@ -15,6 +15,9 @@
from urlparse import urlparse
from httplib import HTTPConnection, HTTPSConnection
+# CallBackFile
+from ladon.tools.callback_file import CallBackFile
+
rx_ctype_charset = re.compile('charset\s*=\s*([-_.a-zA-Z0-9]+)',re.I)
rx_detect_multipart = re.compile('multipart/([^; ]+)',re.I)
rx_detect_boundary = re.compile('boundary=([^; ]+)',re.I)
@@ -130,6 +133,15 @@
if not attachment == None:
result[key] = attachment
+ def save_all(self, path):
+ """
+ Save all attachments.
+ """
+ if not os.path.isdir(path):
+ raise IOError("Path must be a existing folder.")
+ for attach in self.attachments.values():
+ attach.save(path)
+
class JSONWSPClient(object):
@@ -245,7 +257,7 @@
boundary_match = rx_detect_boundary.findall(content_type)
if len(boundary_match):
boundary = boundary_match[0]
- mpr = MultiPartReader(20000,boundary.encode(jsonwsp_charset),response)
+ mpr = MultiPartReader(20000,boundary.encode(jsonwsp_charset),response, size=int(jsonwsp_response.headers.get('CONTENT-LENGTH', '0')))
mpr.read_chunk()
while not mpr.eos:
mpr.read_chunk()
@@ -297,8 +309,14 @@
else:
conn = HTTPConnection(self.hostname,self.port)
req_path = self.path + '/' + extra_path
- buffer_fp = open(buffer_fname,'rb')
- if extra_headers!=None:
+ # CallBackFile
+ if len(files) > 1:
+ save_name = "Multiple files"
+ else:
+ save_name = os.path.basename([f.name for f in files.values()][0])
+ buffer_fp = CallBackFile(path=buffer_fname, save_name=save_name)
+ # /CallBackFile
+ if extra_headers is not None:
headers.update(extra_headers)
conn.request("POST", req_path, buffer_fp, headers)
buffer_fp.close()
=== added file 'frameworks/python/src/ladon/tools/callback_file.py'
--- frameworks/python/src/ladon/tools/callback_file.py 1970-01-01 00:00:00 +0000
+++ frameworks/python/src/ladon/tools/callback_file.py 2014-04-14 13:05:34 +0000
@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ New Attachment file to handle progressbars.
+"""
+__file_name__ = "callback_file.py"
+__author__ = "luca"
+__version__ = "1.0.0"
+__date__ = "2014-04-08"
+
+
+import tempfile
+from os.path import join, isdir
+import os
+from shutil import copy2
+
+try:
+ O_BINARY = os.O_BINARY
+except AttributeError:
+ O_BINARY = 0
+
+MODES = {
+ 'a': os.O_APPEND,
+ 'w': os.O_WRONLY,
+ 'r': os.O_RDONLY,
+ '+': os.O_RDWR,
+ 'b': O_BINARY,
+}
+
+
+def void_callback(event, attachment, **kwargs):
+ """
+ A void callback.
+ """
+ return (event, attachment, kwargs)
+
+
+CALLBACK = void_callback
+
+
+def set_callback(new_callback):
+ """
+ Set global callback.
+ """
+ global CALLBACK
+ CALLBACK = new_callback
+
+
+class CallBackFile(object):
+ """
+ File-Like Object that supports callbacks too.
+
+ :param istemp: This will be a tmp file.
+ :param attach_id: A simple id for the file.
+ :param save_name: The real name that you'd like to use in tmp case.
+ :param size: The size of the file in tmp case.
+ :param path: Path of the file or tmp dir in tmp case.
+ :param mode: mode in NON tmp case.
+ :param callback: callback to use.
+ :param **kwargs: Add kwargs to pass to callback.
+ """
+ fdesc = None
+
+ def __init__(
+ self, istemp=False, attach_id=None, save_name=None, size=0L,
+ suffix='', prefix='tmp', text=False, removeit=True, path=None,
+ mode='rb', callback=None, **kwargs):
+ rmode = 0
+ for char in mode:
+ rmode |= MODES.get(char, 0)
+ self.last_chunk = 0
+ self.istemp = istemp
+ self.removeit = removeit
+ self.file_id = attach_id or id(self)
+ if self.istemp:
+ self.fdesc, self.name = tempfile.mkstemp(
+ suffix, prefix, path, text)
+ self.size = size
+ else:
+ self.name = path
+ if not os.path.exists(path):
+ rmode |= os.O_CREAT
+ self.fdesc = os.open(path, rmode)
+ self.seek(0, os.SEEK_END)
+ self.size = self.tell() or size
+ self.save_name = save_name or os.path.basename(self.name)
+ self.seek(0)
+ self._callback = callback or CALLBACK
+ self._kwargs = kwargs
+ self._callback('init', self, **self._kwargs)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exception_type, exception_val, trace):
+ self.close()
+
+ @staticmethod
+ def mkstemp(
+ suffix='', prefix='tmp', tmpdir=None, text=False, save_name=None):
+ """
+ Simulate tempfile.mkstemp
+ """
+ the_file = CallBackFile(
+ istemp=True, suffix=suffix, prefix=prefix, path=tmpdir, text=text,
+ save_name=save_name)
+ return the_file
+
+ @staticmethod
+ def get_uploaditem(filename):
+ """
+ Ritorna un *upload_item* da usare per l'invio di un file.
+ """
+ return {
+ 'name': os.path.basename(filename),
+ 'data': open(filename, 'rb'),
+ 'size': os.path.getsize(filename)
+ }
+
+ @property
+ def progress(self):
+ """
+ Returns progress position.
+ """
+ return self.tell()
+
+ def __len__(self):
+ return self.size
+
+ def seek(self, pos, how=os.SEEK_SET):
+ """
+ Seek wrapper.
+ """
+ return os.lseek(self.fdesc, pos, how)
+
+ def tell(self):
+ """
+ tell wrapper.
+ """
+ return os.lseek(self.fdesc, 0, os.SEEK_CUR)
+
+ def read(self, size):
+ """
+ read wrapper.
+ """
+ if self.progress == 0:
+ self._callback('start', self, **self._kwargs)
+ data = os.read(self.fdesc, size)
+ part = len(data)
+ self._callback('read', self, **self._kwargs)
+ if part == 0 or part == self.size:
+ self._callback('end', self, **self._kwargs)
+ return data
+
+ def write(self, data):
+ """
+ Write wrapper.
+ """
+ if self.istemp and self.size and (
+ self.progress + len(data)) > self.size:
+ data = data[:-((self.progress + len(data)) - self.size)]
+ self.last_chunk = len(data)
+ if self.progress == 0:
+ self._callback('start', self, **self._kwargs)
+ os.write(self.fdesc, data)
+ self._callback('write', self, **self._kwargs)
+ if self.progress + len(data) >= self.size:
+ self._callback('end', self, **self._kwargs)
+
+ def save(self, path):
+ """
+ Save attachment.
+ """
+ if isdir(path):
+ path = join(path, self.save_name)
+ copy2(self.name, path)
+
+ def __del__(self):
+ pass
+
+ def close(self):
+ """
+ Close wrapper.
+ """
+ try:
+ os.close(self.fdesc)
+ self._callback('close', self, **self._kwargs)
+ except:
+ pass
=== modified file 'frameworks/python/src/ladon/tools/multiparthandler.py'
--- frameworks/python/src/ladon/tools/multiparthandler.py 2011-11-01 11:50:14 +0000
+++ frameworks/python/src/ladon/tools/multiparthandler.py 2014-04-14 13:05:34 +0000
@@ -13,6 +13,10 @@
import re,tempfile,os,time,hashlib
from ladon.compat import PORTABLE_BYTES, PORTABLE_STRING
+# CallBackFile
+from ladon.tools.callback_file import CallBackFile
+import json
+
rx_2linefeeds = re.compile(b'\n\n|\r\n\r\n',re.M)
rx_headers = re.compile(b'^([-_a-zA-Z0-9]+): (.*)$',re.M)
@@ -20,9 +24,11 @@
return (pair[0].upper().replace(b'-',b'_'),pair[1])
class MultiPartReader(object):
- def __init__(self,chunk_size,boundary,req,content_size=None):
+
+ def __init__(self, chunk_size, boundary, req, content_size=None, size=0):
self.chunk_size = chunk_size
self.content_size=content_size
+ self.size = size
self.boundary = boundary
rx_boundary = re.escape(boundary)
self.rx_boundary_split = re.compile(b'--<b>\n|\n--<b>\n|\n--<b>--|--<b>\r\n|\r\n--<b>\r\n|\r\n--<b>--'.replace(b'<b>',boundary) ,re.M)
@@ -39,6 +45,7 @@
self.attachment_headers_parsed = False
self.tmpdir = None
self.fd = None
+ self.fobj = None
self.request_bytes_read = 0
self.attachment_bytes_size = 0
self.raw_dump_rest = False
@@ -65,11 +72,15 @@
self.interface_request += data
self.interface_request_headers = dict(map(upperkey,rx_headers.findall(self.attachment_headers)))
else:
- os.write(self.fd,data)
+ # CallBackFile
+ self.fobj.write(data)
+ # /CallBackFile
if eop==True:
- if self.fd != None:
- os.close(self.fd)
+ if self.fobj != None:
+ # CallBackFile
+ self.fobj.close()
+ # /CallBackFile
headers_dict = dict(map(upperkey,rx_headers.findall(self.attachment_headers)))
self.attachments[self.part_count-1]['size'] = self.attachment_bytes_size
self.attachments[self.part_count-1]['headers'] = headers_dict
@@ -78,7 +89,16 @@
if not self.eos:
self.attachments[self.part_count] = {}
- self.fd,self.attachments[self.part_count]['path'] = tempfile.mkstemp('','content_',self.tmpdir)
+ # CallBackFile
+ try:
+ save_name = json.loads(data)['result']['file']['name']
+ except:
+ save_name = None
+ self.fobj = CallBackFile(
+ istemp=True, prefix='content_', path=self.tmpdir,
+ save_name=save_name, size=self.size)
+ self.attachments[self.part_count]['path'] = self.fobj.name
+ # /CallBackFile
self.part_count += 1
self.attachment_bytes_size = 0
self.attachment_headers = b''
@@ -108,7 +128,9 @@
self.eos = True
self.write(parts[-1:][0][:-self.len_rest_chunk])
if self.eos:
- os.close(self.fd)
+ # CallBackFile
+ self.fobj.close()
+ # /CallBackFile
os.unlink(self.attachments[self.part_count-1]['path'])
self.rest_chunk = parts[-1:][0][-self.len_rest_chunk:]
=== modified file 'frameworks/python/src/ladon/types/attachment.py'
--- frameworks/python/src/ladon/types/attachment.py 2011-12-11 22:51:56 +0000
+++ frameworks/python/src/ladon/types/attachment.py 2014-04-14 13:05:34 +0000
@@ -1,5 +1,7 @@
# -*- coding: utf-8 -*-
import os,re
+from os.path import isdir, join, basename
+from shutil import copy2
from ladon.exceptions.dispatcher import NonExistingAttachment,InvalidAttachmentReference
class attachment(object):
@@ -36,6 +38,16 @@
self.closed = self.bufferobj.closed
self.close = self.bufferobj.close
+ def save(self, path):
+ """
+ Save the attachment.
+ """
+ save_name = self.headers.get(
+ 'X_FILENAME', basename(self.bufferobj.name))
+ if isdir(path):
+ path = join(path, save_name)
+ copy2(self.bufferobj.name, path)
+
def __del__(self):
if self.bufferobj:
self.bufferobj.close()
