openerp-community-reviewer team mailing list archive
-
openerp-community-reviewer team
-
Mailing list archive
-
Message #00905
lp:~camptocamp/account-financial-tools/account-financial-tools-add_async_move_line_import-nbi into lp:account-financial-tools
Nicolas Bessi - Camptocamp has proposed merging lp:~camptocamp/account-financial-tools/account-financial-tools-add_async_move_line_import-nbi into lp:account-financial-tools.
Requested reviews:
Frederic Clementi - Camptocamp.com (frederic-clementi)
fclementi (fclementi)
Account Core Editors (account-core-editors)
For more details, see:
https://code.launchpad.net/~camptocamp/account-financial-tools/account-financial-tools-add_async_move_line_import-nbi/+merge/186563
Add asyn_move_line_importer module
--
https://code.launchpad.net/~camptocamp/account-financial-tools/account-financial-tools-add_async_move_line_import-nbi/+merge/186563
Your team OpenERP Community Reviewer is subscribed to branch lp:account-financial-tools.
=== added directory 'async_move_line_importer'
=== added file 'async_move_line_importer/__init__.py'
--- async_move_line_importer/__init__.py 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/__init__.py 2013-11-01 12:38:22 +0000
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Author: Nicolas Bessi
+# Copyright 2013 Camptocamp SA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from . import model
=== added file 'async_move_line_importer/__openerp__.py'
--- async_move_line_importer/__openerp__.py 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/__openerp__.py 2013-11-01 12:38:22 +0000
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Author: Nicolas Bessi
+# Copyright 2013 Camptocamp SA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+{'name': 'Asynchronous move/move line CSV importer',
+ 'version': '0.1.1',
+ 'author': 'Camptocamp',
+ 'maintainer': 'Camptocamp',
+ 'category': 'Accounting',
+ 'complexity': 'normal',
+ 'depends': ['base', 'account'],
+ 'description': """
+This module allows you to import moves / move lines via CSV asynchronously.
+
+You can access model in the journal entries menu -> Moves/ Move lines importer.
+User must be an account manger.
+
+To import a CSV simply save an UTF8 CSV file in the "file" field.
+Then you can choose a CSV separator.
+
+If volumetry is important you can tick "Fast import" check box.
+When enabled import will be faster but it will not use orm and may
+not support all CSV canvas.
+
+- Entry posted option of journal will be skipped.
+- AA lines will only be created when moves are posted.
+- Tax lines computation will be skipped until the move are posted.
+
+This option should be used with caution and preferably in conjunction with provided canvas in tests/data
+
+Then simply press import file button. The process will be run in background
+and you will be able to continue your work.
+
+When the import is finished you will received a notification and an
+import report will be available on the record
+""",
+ 'website': 'http://www.camptocamp.com',
+ 'data': ['data.xml',
+ 'view/move_line_importer_view.xml',
+ 'security/ir.model.access.csv',
+ 'security/multi_company.xml'],
+ 'demo': [],
+ 'test': [],
+ 'installable': True,
+ 'auto_install': False,
+ 'license': 'AGPL-3',
+ 'application': False,
+ }
=== added file 'async_move_line_importer/data.xml'
--- async_move_line_importer/data.xml 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/data.xml 2013-11-01 12:38:22 +0000
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<openerp>
+ <data>
+ <record id="mvl_imported" model="mail.message.subtype">
+ <field name="name">Import successfully finished</field>
+ <field name="res_model">move.line.importer</field>
+ <field name="default" eval="True"/>
+ <field name="description">Import successfully finished</field>
+ </record>
+ <record id="mvl_error" model="mail.message.subtype">
+ <field name="name">Import failed</field>
+ <field name="res_model">move.line.importer</field>
+ <field name="default" eval="True"/>
+ <field name="description">Import failed</field>
+ </record>
+ </data>
+</openerp>
=== added directory 'async_move_line_importer/i18n'
=== added directory 'async_move_line_importer/model'
=== added file 'async_move_line_importer/model/__init__.py'
--- async_move_line_importer/model/__init__.py 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/model/__init__.py 2013-11-01 12:38:22 +0000
@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Author: Nicolas Bessi
+# Copyright 2013 Camptocamp SA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from . import move_line_importer
+from . import account
=== added file 'async_move_line_importer/model/account.py'
--- async_move_line_importer/model/account.py 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/model/account.py 2013-11-01 12:38:22 +0000
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Author: Nicolas Bessi
+# Copyright 2013 Camptocamp SA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import psycopg2
+import logging
+from openerp.osv import orm
+from openerp.tools.float_utils import float_compare
+_logger = logging.getLogger(__name__)
+
+
+def _format_inserts_values(vals):
+ cols = vals.keys()
+ if 'line_id' in cols:
+ cols.remove('line_id')
+ return (', '.join(cols), ', '.join(['%%(%s)s' % i for i in cols]))
+
+
+class account_move(orm.Model):
+ """redefine account move create to bypass orm.
+
+ Async_bypass_create must be set to True in context.
+
+ """
+
+ _inherit = "account.move"
+
+ def _prepare_line(self, cr, uid, move_id, line, vals, context=None):
+ """Take incomming move vals and complete move line dict with missing data
+
+ :param move_id: parent move id
+ :param line: dict of vals of move line
+ :param vals: dict of vals of move
+ :returns: dict of val of move line completed
+
+ """
+ if isinstance(line, tuple):
+ line = line[2]
+ line['journal_id'] = vals.get('journal_id')
+ line['date'] = vals.get('date')
+ line['period_id'] = vals.get('period_id')
+ line['company_id'] = vals.get('company_id')
+ line['state'] = vals['state']
+ line['move_id'] = move_id
+ if line['debit'] and line['credit']:
+ raise ValueError('debit and credit set on same line')
+ if not line.get('analytic_account_id'):
+ line['analytic_account_id'] = None
+ for key in line:
+ if line[key] is False:
+ line[key] = None
+ return line
+
+ def _check_balance(self, vals):
+ """Check if move is balanced"""
+ line_dicts = [y[2] for y in vals['line_id']]
+ debit = sum(x.get('debit') or 0.0 for x in line_dicts)
+ credit = sum(x.get('credit') or 0.0 for x in line_dicts)
+ if float_compare(debit, credit, precision_digits=2):
+ raise ValueError('Move is not balanced %s %s' % (debit, credits))
+
+ def _bypass_create(self, cr, uid, vals, context=None):
+ """Create entries using cursor directly
+
+ :returns: created id
+
+ """
+ mvl_obj = self.pool['account.move.line']
+ vals['company_id'] = context.get('company_id', False)
+ vals['state'] = 'draft'
+ if not vals.get('name'):
+ vals['name'] = "/"
+ sql = u"Insert INTO account_move (%s) VALUES (%s) RETURNING id"
+ sql = sql % _format_inserts_values(vals)
+ try:
+ cr.execute(sql, vals)
+ except psycopg2.Error:
+ _logger.exception('ORM by pass error for move')
+ raise
+ created_id = cr.fetchone()[0]
+ if vals.get('line_id'):
+ self._check_balance(vals)
+ for line in vals['line_id']:
+ l_vals = self._prepare_line(cr, uid, created_id,
+ line, vals, context=context)
+ mvl_obj.create(cr, uid, l_vals, context=context)
+ return created_id
+
+ def create(self, cr, uid, vals, context=None):
+ """Please refer to orm.BaseModel.create documentation"""
+ if context is None:
+ context = {}
+ if context.get('async_bypass_create'):
+ return self._bypass_create(cr, uid, vals, context=context)
+ return super(account_move, self).create(cr, uid, vals, context=context)
+
+
+class account_move_line(orm.Model):
+ """Redefine account move line create to bypass orm.
+
+ Async_bypass_create must be set to True in context
+
+ """
+
+ _inherit = "account.move.line"
+
+ def create(self, cr, uid, vals, context=None):
+ """Please refer to orm.BaseModel.create documentation"""
+ if context is None:
+ context = {}
+ if context.get('async_bypass_create'):
+ return self._bypass_create(cr, uid, vals, context=context)
+ return super(account_move_line, self).create(cr, uid, vals, context=context)
+
+ def _bypass_create(self, cr, uid, vals, context=None):
+ """Create entries using cursor directly
+
+ :returns: created id
+
+ """
+ sql = u"Insert INTO account_move_line (%s) VALUES (%s) RETURNING id"
+ sql = sql % _format_inserts_values(vals)
+ try:
+ cr.execute(sql, vals)
+ except psycopg2.Error:
+ _logger.exception('ORM by pass error for move line')
+ raise
+ return cr.fetchone()[0]
=== added file 'async_move_line_importer/model/move_line_importer.py'
--- async_move_line_importer/model/move_line_importer.py 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/model/move_line_importer.py 2013-11-01 12:38:22 +0000
@@ -0,0 +1,337 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Author: Nicolas Bessi
+# Copyright 2013 Camptocamp SA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import sys
+import traceback
+import logging
+import base64
+import threading
+import csv
+import tempfile
+
+import psycopg2
+
+import openerp.pooler as pooler
+from openerp.osv import orm, fields
+from openerp.tools.translate import _
+
+
+_logger = logging.getLogger(__name__)
+
+
+class move_line_importer(orm.Model):
+ """Asynchrone move / move line importer.
+
+ It will parse the saved CSV file using orm.BaseModel.load
+ in a thread. If you set bypass_orm to True then the load function
+ will use a totally overriden create function that is a lot faster
+ but that totally bypass the ORM
+
+ """
+
+ _name = "move.line.importer"
+ _inherit = ['mail.thread']
+
+ def copy(self, cr, uid, id, default=None, context=None):
+ if default is None:
+ default = {}
+ default.update(state='draft', report=False)
+ return super(move_line_importer, self).copy(cr, uid, id, default=default,
+ context=context)
+
+ def track_success(sef, cr, uid, obj, context=None):
+ """Used by mail subtype"""
+ return obj['state'] == 'done'
+
+ def track_error(sef, cr, uid, obj, context=None):
+ """Used by mail subtype"""
+ return obj['state'] == 'error'
+
+ _track = {
+ 'state': {
+ 'async_move_line_importer.mvl_imported': track_success,
+ 'async_move_line_importer.mvl_error': track_error,
+ },
+ }
+
+ _columns = {'name': fields.datetime('Name',
+ required=True,
+ readonly=True),
+ 'state': fields.selection([('draft', 'New'),
+ ('running', 'Running'),
+ ('done', 'Success'),
+ ('error', 'Error')],
+ readonly=True,
+ string='Status'),
+ 'report': fields.text('Report',
+ readonly=True),
+ 'file': fields.binary('File',
+ required=True),
+ 'delimiter': fields.selection([(',', ','), (';', ';'), ('|', '|')],
+ string="CSV delimiter",
+ required=True),
+ 'company_id': fields.many2one('res.company',
+ 'Company'),
+ 'bypass_orm': fields.boolean('Fast import (use with caution)',
+ help="When enabled import will be faster but"
+ " it will not use orm and may"
+ " not support all CSV canvas. \n"
+ "Entry posted option will be skipped. \n"
+ "AA lines will only be created when"
+ " moves are posted. \n"
+ "Tax lines computation will be skipped. \n"
+ "This option should be used with caution"
+ " and in conjonction with provided canvas."),
+ }
+
+ def _get_current_company(self, cr, uid, context=None, model="move.line.importer"):
+ return self.pool.get('res.company')._company_default_get(cr, uid, model,
+ context=context)
+
+ _defaults = {'state': 'draft',
+ 'name': fields.datetime.now(),
+ 'company_id': _get_current_company,
+ 'delimiter': ',',
+ 'bypass_orm': False}
+
+ def _parse_csv(self, cr, uid, imp_id):
+ """Parse stored CSV file in order to be usable by BaseModel.load method.
+
+ Manage base 64 decoding.
+
+ :param imp_id: current importer id
+ :returns: (head [list of first row], data [list of list])
+
+ """
+ # We use tempfile in order to avoid memory error with large files
+ with tempfile.TemporaryFile() as src:
+ imp = self.read(cr, uid, imp_id, ['file', 'delimiter'])
+ content = imp['file']
+ delimiter = imp['delimiter']
+ src.write(content)
+ with tempfile.TemporaryFile() as decoded:
+ src.seek(0)
+ base64.decode(src, decoded)
+ decoded.seek(0)
+ return self._prepare_csv_data(decoded, delimiter)
+
+ def _prepare_csv_data(self, csv_file, delimiter=","):
+ """Parse a decoded CSV file and return head list and data list
+
+ :param csv_file: decoded CSV file
+ :param delimiter: CSV file delimiter char
+ :returns: (head [list of first row], data [list of list])
+
+ """
+ try:
+ data = csv.reader(csv_file, delimiter=str(delimiter))
+ except csv.Error as error:
+ raise orm.except_orm(_('CSV file is malformed'),
+ _("Maybe you have not choose correct separator \n"
+ "the error detail is : \n %s") % repr(error))
+ head = data.next()
+ head = [x.replace(' ', '') for x in head]
+ # Generator does not work with orm.BaseModel.load
+ values = [tuple(x) for x in data if x]
+ return (head, values)
+
+ def format_messages(self, messages):
+ """Format error messages generated by the BaseModel.load method
+
+ :param messages: return of BaseModel.load messages key
+ :returns: formatted string
+
+ """
+ res = []
+ for msg in messages:
+ rows = msg.get('rows', {})
+ res.append(_("%s. -- Field: %s -- rows %s to %s") % (msg.get('message', 'N/A'),
+ msg.get('field', 'N/A'),
+ rows.get('from', 'N/A'),
+ rows.get('to', 'N/A')))
+ return "\n \n".join(res)
+
+ def _manage_load_results(self, cr, uid, imp_id, result, _do_commit=True, context=None):
+ """Manage the BaseModel.load function output and store exception.
+
+ Will generate success/failure report and store it into report field.
+ Manage commit and rollback even if load method uses PostgreSQL
+ Savepoints.
+
+ :param imp_id: current importer id
+ :param result: BaseModel.laod return {ids: list(int)|False, messages: [Message]}
+ :param _do_commit: toggle commit management only used for testing purpose only
+ :returns: current importer id
+
+ """
+ # Import sucessful
+ state = msg = None
+ if not result['messages']:
+ msg = _("%s lines imported" % len(result['ids'] or []))
+ state = 'done'
+ else:
+ if _do_commit:
+ cr.rollback()
+ msg = self.format_messages(result['messages'])
+ state = 'error'
+ return (imp_id, state, msg)
+
+ def _write_report(self, cr, uid, imp_id, state, msg, _do_commit=True,
+ max_tries=5, context=None):
+ """Commit report in a separated transaction.
+
+ It will avoid concurrent update error due to mail.message.
+ If transaction trouble happen we try 5 times to rewrite report
+
+ :param imp_id: current importer id
+ :param state: import state
+ :param msg: report summary
+ :returns: current importer id
+
+ """
+ if _do_commit:
+ db_name = cr.dbname
+ local_cr = pooler.get_db(db_name).cursor()
+ try:
+ self.write(local_cr, uid, [imp_id],
+ {'state': state, 'report': msg},
+ context=context)
+ local_cr.commit()
+ # We handle concurrent error troubles
+ except psycopg2.OperationalError as pg_exc:
+ _logger.error('Can not write report. System will retry %s time(s)' % max_tries)
+ if pg_exc.pg_code in orm.PG_CONCURRENCY_ERRORS_TO_RETRY and max_tries >= 0:
+ local_cr.rollback()
+ local_cr.close()
+ remaining_try = max_tries - 1
+ self._write_report(cr, uid, imp_id, cr, _do_commit=_do_commit,
+ max_tries=remaining_try, context=context)
+ else:
+ _logger.exception('Can not log report - Operational update error')
+ raise
+ except Exception:
+ _logger.exception('Can not log report')
+ local_cr.rollback()
+ raise
+ finally:
+ if not local_cr.closed:
+ local_cr.close()
+ else:
+ self.write(cr, uid, [imp_id], {'state': state, 'report': msg}, context=context)
+ return imp_id
+
+ def _load_data(self, cr, uid, imp_id, head, data, _do_commit=True, context=None):
+ """Function that does the load of parsed CSV file.
+
+ If will log exception and susccess into the report fields.
+
+ :param imp_id: current importer id
+ :param head: CSV file head (list of header)
+ :param data: CSV file content (list of data list)
+ :param _do_commit: toggle commit management only used for testing purpose only
+ :returns: current importer id
+
+ """
+ state = msg = None
+ try:
+ res = self.pool['account.move'].load(cr, uid, head, data, context=context)
+ r_id, state, msg = self._manage_load_results(cr, uid, imp_id, res,
+ _do_commit=_do_commit,
+ context=context)
+ except Exception as exc:
+ if _do_commit:
+ cr.rollback()
+ ex_type, sys_exc, tb = sys.exc_info()
+ tb_msg = ''.join(traceback.format_tb(tb, 30))
+ _logger.error(tb_msg)
+ _logger.error(repr(exc))
+ msg = _("Unexpected exception.\n %s \n %s" % (repr(exc), tb_msg))
+ state = 'error'
+ finally:
+ self._write_report(cr, uid, imp_id, state, msg,
+ _do_commit=_do_commit, context=context)
+ if _do_commit:
+ try:
+ cr.commit()
+ except psycopg2.Error:
+ _logger.exception('Can not do final commit')
+ cr.close()
+ return imp_id
+
+ def _allows_thread(self, imp_id):
+ """Check if there is a async import of this file running
+
+ :param imp_id: current importer id
+ :returns: void
+ :raise: orm.except in case on failure
+
+ """
+ for th in threading.enumerate():
+ if th.getName() == 'async_move_line_import_%s' % imp_id:
+ raise orm.except_orm(_('An import of this file is already running'),
+ _('Please try latter'))
+
+ def _check_permissions(self, cr, uid, context=None):
+ """Ensure that user is allowed to create move / move line"""
+ move_obj = self.pool['account.move']
+ move_line_obj = self.pool['account.move.line']
+ move_obj.check_access_rule(cr, uid, [], 'create')
+ move_obj.check_access_rights(cr, uid, 'create', raise_exception=True)
+ move_line_obj.check_access_rule(cr, uid, [], 'create')
+ move_line_obj.check_access_rights(cr, uid, 'create', raise_exception=True)
+
+ def import_file(self, cr, uid, imp_id, context=None):
+ """ Will do an asynchronous load of a CSV file.
+
+ Will generate an success/failure report and generate some
+ maile threads. It uses BaseModel.load to lookup CSV.
+ If you set bypass_orm to True then the load function
+ will use a totally overriden create function that is a lot faster
+ but that totally bypass the ORM
+
+ """
+
+ if isinstance(imp_id, list):
+ imp_id = imp_id[0]
+ if context is None:
+ context = {}
+ current = self.read(cr, uid, imp_id, ['bypass_orm', 'company_id'],
+ load='_classic_write')
+ context['company_id'] = current['company_id']
+ bypass_orm = current['bypass_orm']
+ if bypass_orm:
+ # Tells create funtion to bypass orm
+ # As we bypass orm we ensure that
+ # user is allowed to creat move / move line
+ self._check_permissions(cr, uid, context=context)
+ context['async_bypass_create'] = True
+ head, data = self._parse_csv(cr, uid, imp_id)
+ self.write(cr, uid, [imp_id], {'state': 'running',
+ 'report': _('Import is running')})
+ self._allows_thread(imp_id)
+ db_name = cr.dbname
+ local_cr = pooler.get_db(db_name).cursor()
+ thread = threading.Thread(target=self._load_data,
+ name='async_move_line_import_%s' % imp_id,
+ args=(local_cr, uid, imp_id, head, data),
+ kwargs={'context': context.copy()})
+ thread.start()
+
+ return {}
=== added directory 'async_move_line_importer/security'
=== added file 'async_move_line_importer/security/ir.model.access.csv'
--- async_move_line_importer/security/ir.model.access.csv 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/security/ir.model.access.csv 2013-11-01 12:38:22 +0000
@@ -0,0 +1,2 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_move_line_importer_manager,access_move_line_importer_manager,model_move_line_importer,account.group_account_manager,1,1,1,0
=== added file 'async_move_line_importer/security/multi_company.xml'
--- async_move_line_importer/security/multi_company.xml 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/security/multi_company.xml 2013-11-01 12:38:22 +0000
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<openerp>
+ <data noupdate="1">
+ <record model="ir.rule" id="move_line_import_mc_rule">
+ <field name="name">Move line importer company rule</field>
+ <field name="model_id" ref="model_move_line_importer"/>
+ <field name="global" eval="True"/>
+ <field name="domain_force">['|',('company_id','=',False),('company_id','child_of',[user.company_id.id])]</field>
+ </record>
+ </data>
+</openerp>
=== added directory 'async_move_line_importer/tests'
=== added file 'async_move_line_importer/tests/__init__.py'
--- async_move_line_importer/tests/__init__.py 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/tests/__init__.py 2013-11-01 12:38:22 +0000
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Author: Nicolas Bessi
+# Copyright 2013 Camptocamp SA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from . import test_async_import
+
+checks = [test_async_import]
=== added directory 'async_move_line_importer/tests/data'
=== added file 'async_move_line_importer/tests/data/faulty_moves.csv'
--- async_move_line_importer/tests/data/faulty_moves.csv 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/tests/data/faulty_moves.csv 2013-11-01 12:38:22 +0000
@@ -0,0 +1,7 @@
+ref;date;period_id;journal_id;line_id / account_id;line_id / partner_id;line_id / name;line_id / debit;line_id / credit;line_id/tax_code_id
+test_3;2013-10-01;X 01/2013;MISC;X2001;Camptocamp;TEST C2C;;1000;Tax Received
+;;;;X11003;Camptocamp;TEST C2C;;200;Tax Paid
+;;;;X11002;Camptocamp;TEST C2C;1200;;
+test_3b;2013-10-01;X 01/2013;MISC;X2001;Camptocamp;TEST C2C;;1000;Tax Received
+;;;;X11003;Camptocamp;TEST C2C;;200;Faulty code
+;;;;X11002;Camptocamp;TEST C2C;1200;;
\ No newline at end of file
=== added file 'async_move_line_importer/tests/data/one_move.csv'
--- async_move_line_importer/tests/data/one_move.csv 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/tests/data/one_move.csv 2013-11-01 12:38:22 +0000
@@ -0,0 +1,4 @@
+ref;date;period_id;journal_id;line_id / account_id;line_id / partner_id;line_id / name;line_id / debit;line_id / credit;line_id/tax_code_id
+éöüàè_test_1;2013-10-01;X 01/2013;MISC;X2001;Camptocamp;TEST C2C;;1000;Tax Received
+;;;;X11003;Camptocamp;TEST C2C;;200;Tax Paid
+;;;;X11002;Camptocamp;TEST C2C;1200;;
=== added file 'async_move_line_importer/tests/data/one_move2.csv'
--- async_move_line_importer/tests/data/one_move2.csv 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/tests/data/one_move2.csv 2013-11-01 12:38:22 +0000
@@ -0,0 +1,4 @@
+ref;date;period_id;journal_id;line_id / account_id;line_id / partner_id;line_id / name;line_id / debit;line_id / credit;line_id/tax_code_id
+test_2;2013-10-01;X 01/2013;MISC;X2001;Camptocamp;TEST C2C;;1000;Tax Received
+;;;;X11003;Camptocamp;TEST C2C;;200;Tax Paid
+;;;;X11002;Camptocamp;TEST C2C;1200;;
=== added file 'async_move_line_importer/tests/test_async_import.py'
--- async_move_line_importer/tests/test_async_import.py 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/tests/test_async_import.py 2013-11-01 12:38:22 +0000
@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Author: Nicolas Bessi
+# Copyright 2013 Camptocamp SA
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import base64
+import tempfile
+
+import openerp.tests.common as test_common
+from openerp import addons
+
+
+class TestMoveLineImporter(test_common.SingleTransactionCase):
+
+ def get_file(self, filename):
+ """Retrive file from test data"""
+ path = addons.get_module_resource('async_move_line_importer',
+ 'tests', 'data', filename)
+ with open(path) as test_data:
+ with tempfile.TemporaryFile() as out:
+ base64.encode(test_data, out)
+ out.seek(0)
+ return out.read()
+
+ def setUp(self):
+ super(TestMoveLineImporter, self).setUp()
+ self.importer_model = self.registry('move.line.importer')
+ self.move_model = self.registry('account.move')
+
+ def tearDown(self):
+ super(TestMoveLineImporter, self).tearDown()
+
+ def test_01_one_line_without_orm_bypass(self):
+ """Test one line import without bypassing orm"""
+ cr, uid = self.cr, self.uid
+ importer_id = self.importer_model.create(cr, uid,
+ {'file': self.get_file('one_move.csv'),
+ 'delimiter': ';'})
+ importer = self.importer_model.browse(cr, uid, importer_id)
+ self.assertTrue(importer.company_id, 'Not default company set')
+ self.assertFalse(importer.bypass_orm, 'Bypass orm must not be active')
+ self.assertEqual(importer.state, 'draft')
+ head, data = self.importer_model._parse_csv(cr, uid, importer.id)
+ self.importer_model._load_data(cr, uid, importer.id, head, data, _do_commit=False, context={})
+ importer = self.importer_model.browse(cr, uid, importer_id)
+ self.assertEquals(importer.state, 'done',
+ 'Exception %s during import' % importer.report)
+ created_move_ids = self.move_model.search(cr, uid, [('ref', '=', 'éöüàè_test_1')])
+ self.assertTrue(created_move_ids, 'No move imported')
+ created_move = self.move_model.browse(cr, uid, created_move_ids[0])
+ self.assertTrue(len(created_move.line_id) == 3, 'Wrong number of move line imported')
+ debit = credit = 0.0
+ for line in created_move.line_id:
+ debit += line.debit if line.debit else 0.0
+ credit += line.credit if line.credit else 0.0
+ self.assertEqual(debit, 1200.00)
+ self.assertEqual(credit, 1200.00)
+ self.assertEqual(created_move.state, 'draft', 'Wrong move state')
+
+ def test_02_one_line_using_orm_bypass(self):
+ """Test one line import using orm bypass"""
+ cr, uid = self.cr, self.uid
+ importer_id = self.importer_model.create(cr, uid,
+ {'file': self.get_file('one_move2.csv'),
+ 'delimiter': ';',
+ 'bypass_orm': True})
+ importer = self.importer_model.browse(cr, uid, importer_id)
+ self.assertTrue(importer.company_id, 'Not default company set')
+ self.assertTrue(importer.bypass_orm, 'Bypass orm must be active')
+ self.assertEqual(importer.state, 'draft')
+ head, data = self.importer_model._parse_csv(cr, uid, importer.id)
+ context = {'async_bypass_create': True,
+ 'company_id': 1}
+ self.importer_model._load_data(cr, uid, importer.id, head, data,
+ _do_commit=False, context=context)
+ importer = self.importer_model.browse(cr, uid, importer_id)
+ self.assertEquals(importer.state, 'done',
+ 'Exception %s during import' % importer.report)
+ created_move_ids = self.move_model.search(cr, uid, [('ref', '=', 'test_2')])
+ self.assertTrue(created_move_ids, 'No move imported')
+ created_move = self.move_model.browse(cr, uid, created_move_ids[0])
+ self.assertTrue(len(created_move.line_id) == 3, 'Wrong number of move line imported')
+ debit = credit = 0.0
+ for line in created_move.line_id:
+ debit += line.debit if line.debit else 0.0
+ credit += line.credit if line.credit else 0.0
+ self.assertEqual(debit, 1200.00)
+ self.assertEqual(credit, 1200.00)
+ self.assertEqual(created_move.state, 'draft', 'Wrong move state')
+
+ def test_03_one_line_failing(self):
+ """Test one line import with faulty CSV file"""
+ cr, uid = self.cr, self.uid
+ importer_id = self.importer_model.create(cr, uid,
+ {'file': self.get_file('faulty_moves.csv'),
+ 'delimiter': ';'})
+ importer = self.importer_model.browse(cr, uid, importer_id)
+ self.assertTrue(importer.company_id, 'Not default company set')
+ self.assertFalse(importer.bypass_orm, 'Bypass orm must not be active')
+ self.assertEqual(importer.state, 'draft')
+ head, data = self.importer_model._parse_csv(cr, uid, importer.id)
+ self.importer_model._load_data(cr, uid, importer.id, head, data, _do_commit=False, context={})
+ importer = self.importer_model.browse(cr, uid, importer_id)
+ self.assertEquals(importer.state, 'error',
+ 'No exception %s during import' % importer.report)
+ created_move_ids = self.move_model.search(cr, uid, [('ref', '=', 'test_3')])
+ self.assertFalse(created_move_ids, 'Move was imported but it should not be the case')
=== added directory 'async_move_line_importer/view'
=== added file 'async_move_line_importer/view/move_line_importer_view.xml'
--- async_move_line_importer/view/move_line_importer_view.xml 1970-01-01 00:00:00 +0000
+++ async_move_line_importer/view/move_line_importer_view.xml 2013-11-01 12:38:22 +0000
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="utf-8"?>
+<openerp>
+ <data>
+ <record id="move_line_importer_form" model="ir.ui.view">
+ <field name="name">move line importer form</field>
+ <field name="model">move.line.importer</field>
+ <field name="arch" type="xml">
+ <form version="7.0" string="Move / Move lines importer">
+ <header>
+ <button name="import_file"
+ type="object"
+ states="draft,error"
+ string="Import File"
+ class="oe_highlight"/>
+ <field name="state"
+ widget="statusbar"
+ nolabel="1"
+ statusbar_visible="draft,running,done"
+ statusbar_colors='{"draft": "blue", "running": "blue", "done": "blue", "error": "red"}'/>
+ </header>
+ <group>
+ <field name="name" />
+ </group>
+ <group>
+ <div>
+ <field name="file"
+ attrs="{'readonly': [('state', '=', 'done')]}"
+ class="oe_inline"/>
+ <label string="Delimiter" class="oe_inline"/>
+ <field name="delimiter"
+ attrs="{'readonly': [('state', '=', 'done')]}"
+ class="oe_inline"/>
+ </div>
+ </group>
+ <group>
+ <field name="bypass_orm"/>
+ </group>
+ <group>
+ <field name="company_id" groups="base.group_multi_company"/>
+ </group>
+ <notebook>
+ <page string="Report">
+ <field name="report"
+ nolabel="1"
+ colspan="4"/>
+ </page>
+ </notebook>
+ <div class="oe_chatter">
+ <field name="message_follower_ids" widget="mail_followers"/>
+ <field name="message_ids" widget="mail_thread"/>
+ </div>
+ </form>
+ </field>
+ </record>
+ <record id="move_line_importer_tree" model="ir.ui.view">
+ <field name="name">move line importer tree</field>
+ <field name="model">move.line.importer</field>
+ <field name="arch" type="xml">
+ <tree version="7.0" string="Imported files">
+ <field name="name" />
+ <field name="state"/>
+ <field name="company_id" readonly="1"/>
+ </tree>
+ </field>
+ </record>
+
+ <record model="ir.actions.act_window" id="move_line_importer_action">
+ <field name="name">Import Move lines</field>
+ <field name="type">ir.actions.act_window</field>
+ <field name="res_model">move.line.importer</field>
+ <field name="domain"></field>
+ <field name="view_type">form</field>
+ <field name="view_mode">tree,form</field>
+ <field name="view_id" ref="move_line_importer_tree"/>
+ </record>
+
+ <menuitem
+ name="Import Move lines"
+ parent="account.menu_finance_entries"
+ action="move_line_importer_action"
+ id="move_line_importer_action_menu"/>
+ </data>
+</openerp>
Follow ups
