openerp-dev-web team mailing list archive
-
openerp-dev-web team
-
Mailing list archive
-
Message #06995
[Merge] lp:~openerp-dev/openobject-server/trunk-modules-loading-vmt into lp:openobject-server
Vo Minh Thu (OpenERP) has proposed merging lp:~openerp-dev/openobject-server/trunk-modules-loading-vmt into lp:openobject-server.
Requested reviews:
OpenERP Core Team (openerp)
For more details, see:
https://code.launchpad.net/~openerp-dev/openobject-server/trunk-modules-loading-vmt/+merge/61525
--
https://code.launchpad.net/~openerp-dev/openobject-server/trunk-modules-loading-vmt/+merge/61525
Your team OpenERP R&D Team is subscribed to branch lp:~openerp-dev/openobject-server/trunk-modules-loading-vmt.
=== modified file 'openerp-server'
--- openerp-server 2011-03-24 09:50:12 +0000
+++ openerp-server 2011-05-19 09:56:30 +0000
@@ -39,10 +39,11 @@
import sys
import threading
import traceback
+import time
-import openerp.release as release
-__author__ = release.author
-__version__ = release.version
+import openerp
+__author__ = openerp.release.author
+__version__ = openerp.release.version
if os.name == 'posix':
import pwd
@@ -53,80 +54,58 @@
sys.exit(1)
#-----------------------------------------------------------------------
-# import the tools module so that the commandline parameters are parsed
+# parse the command line
#-----------------------------------------------------------------------
-import openerp.tools as tools
-tools.config.parse_config(sys.argv[1:])
+openerp.tools.config.parse_config(sys.argv[1:])
+config = openerp.tools.config
#----------------------------------------------------------
# get logger
#----------------------------------------------------------
-import openerp.netsvc as netsvc
-netsvc.init_logger()
+openerp.netsvc.init_logger()
logger = logging.getLogger('server')
-logger.info("OpenERP version - %s", release.version)
-for name, value in [('addons_path', tools.config['addons_path']),
- ('database hostname', tools.config['db_host'] or 'localhost'),
- ('database port', tools.config['db_port'] or '5432'),
- ('database user', tools.config['db_user'])]:
+logger.info("OpenERP version - %s", __version__)
+for name, value in [('addons_path', config['addons_path']),
+ ('database hostname', config['db_host'] or 'localhost'),
+ ('database port', config['db_port'] or '5432'),
+ ('database user', config['db_user'])]:
logger.info("%s - %s", name, value)
# Don't allow if the connection to PostgreSQL done by postgres user
-if tools.config['db_user'] == 'postgres':
+if config['db_user'] == 'postgres':
logger.error("Connecting to the database as 'postgres' user is forbidden, as it present major security issues. Shutting down.")
sys.exit(1)
-import time
-
#----------------------------------------------------------
# init net service
#----------------------------------------------------------
logger.info('initialising distributed objects services')
-#---------------------------------------------------------------
-# connect to the database and initialize it with base if needed
-#---------------------------------------------------------------
-import openerp.pooler as pooler
-
-#----------------------------------------------------------
-# import basic modules
-#----------------------------------------------------------
-import openerp.osv as osv
-import openerp.workflow as workflow
-import openerp.report as report
-import openerp.service as service
-
-#----------------------------------------------------------
-# import addons
-#----------------------------------------------------------
-
-import openerp.addons as addons
-
#----------------------------------------------------------
# Load and update databases if requested
#----------------------------------------------------------
-import openerp.service.http_server as service_http_server
-
-if not ( tools.config["stop_after_init"] or \
- tools.config["translate_in"] or \
- tools.config["translate_out"] ):
- service_http_server.init_servers()
- service_http_server.init_xmlrpc()
- service_http_server.init_static_http()
-
- import openerp.service.netrpc_server as service_netrpc_server
- service_netrpc_server.init_servers()
-
-if tools.config['db_name']:
- for dbname in tools.config['db_name'].split(','):
- db,pool = pooler.get_db_and_pool(dbname, update_module=tools.config['init'] or tools.config['update'], pooljobs=False)
+if not ( config["stop_after_init"] or \
+ config["translate_in"] or \
+ config["translate_out"] ):
+ openerp.osv.osv.start_object_proxy()
+ openerp.service.web_services.start_web_services()
+ http_server = openerp.service.http_server
+ netrpc_server = openerp.service.netrpc_server
+ http_server.init_servers()
+ http_server.init_xmlrpc()
+ http_server.init_static_http()
+ netrpc_server.init_servers()
+
+if config['db_name']:
+ for dbname in config['db_name'].split(','):
+ db, pool = openerp.pooler.get_db_and_pool(dbname, update_module=config['init'] or config['update'], pooljobs=False)
cr = db.cursor()
- if tools.config["test_file"]:
- logger.info('loading test file %s', tools.config["test_file"])
- tools.convert_yaml_import(cr, 'base', file(tools.config["test_file"]), {}, 'test', True)
+ if config["test_file"]:
+ logger.info('loading test file %s', config["test_file"])
+ openerp.tools.convert_yaml_import(cr, 'base', file(config["test_file"]), {}, 'test', True)
cr.rollback()
pool.get('ir.cron')._poolJobs(db.dbname)
@@ -136,35 +115,33 @@
#----------------------------------------------------------
# translation stuff
#----------------------------------------------------------
-if tools.config["translate_out"]:
- import csv
-
- if tools.config["language"]:
- msg = "language %s" % (tools.config["language"],)
+if config["translate_out"]:
+ if config["language"]:
+ msg = "language %s" % (config["language"],)
else:
msg = "new language"
- logger.info('writing translation file for %s to %s', msg, tools.config["translate_out"])
+ logger.info('writing translation file for %s to %s', msg, config["translate_out"])
- fileformat = os.path.splitext(tools.config["translate_out"])[-1][1:].lower()
- buf = file(tools.config["translate_out"], "w")
- dbname = tools.config['db_name']
- cr = pooler.get_db(dbname).cursor()
- tools.trans_export(tools.config["language"], tools.config["translate_modules"] or ["all"], buf, fileformat, cr)
+ fileformat = os.path.splitext(config["translate_out"])[-1][1:].lower()
+ buf = file(config["translate_out"], "w")
+ dbname = config['db_name']
+ cr = openerp.pooler.get_db(dbname).cursor()
+ openerp.tools.trans_export(config["language"], config["translate_modules"] or ["all"], buf, fileformat, cr)
cr.close()
buf.close()
logger.info('translation file written successfully')
sys.exit(0)
-if tools.config["translate_in"]:
- context = {'overwrite': tools.config["overwrite_existing_translations"]}
- dbname = tools.config['db_name']
- cr = pooler.get_db(dbname).cursor()
- tools.trans_load(cr,
- tools.config["translate_in"],
- tools.config["language"],
+if config["translate_in"]:
+ context = {'overwrite': config["overwrite_existing_translations"]}
+ dbname = config['db_name']
+ cr = openerp.pooler.get_db(dbname).cursor()
+ openerp.tools.trans_load(cr,
+ config["translate_in"],
+ config["language"],
context=context)
- tools.trans_update_res_ids(cr)
+ openerp.tools.trans_update_res_ids(cr)
cr.commit()
cr.close()
sys.exit(0)
@@ -172,9 +149,10 @@
#----------------------------------------------------------------------------------
# if we don't want the server to continue to run after initialization, we quit here
#----------------------------------------------------------------------------------
-if tools.config["stop_after_init"]:
+if config["stop_after_init"]:
sys.exit(0)
+openerp.netsvc.start_agent()
#----------------------------------------------------------
# Launch Servers
@@ -186,15 +164,16 @@
[(getattr(signal, sign), sign) for sign in LST_SIGNALS]
)
-netsvc.quit_signals_received = 0
+quit_signals_received = 0
def handler(signum, frame):
"""
:param signum: the signal number
:param frame: the interrupted stack frame or None
"""
- netsvc.quit_signals_received += 1
- if netsvc.quit_signals_received > 1:
+ global quit_signals_received
+ quit_signals_received += 1
+ if quit_signals_received > 1:
sys.stderr.write("Forced shutdown.\n")
os._exit(0)
@@ -219,10 +198,10 @@
signal.signal(signal.SIGQUIT, dumpstacks)
def quit():
- netsvc.Agent.quit()
- netsvc.Server.quitAll()
- if tools.config['pidfile']:
- os.unlink(tools.config['pidfile'])
+ openerp.netsvc.Agent.quit()
+ openerp.netsvc.Server.quitAll()
+ if config['pidfile']:
+ os.unlink(config['pidfile'])
logger = logging.getLogger('shutdown')
logger.info("Initiating OpenERP Server shutdown")
logger.info("Hit CTRL-C again or send a second signal to immediately terminate the server...")
@@ -240,17 +219,17 @@
time.sleep(0.05)
sys.exit(0)
-if tools.config['pidfile']:
- fd = open(tools.config['pidfile'], 'w')
+if config['pidfile']:
+ fd = open(config['pidfile'], 'w')
pidtext = "%d" % (os.getpid())
fd.write(pidtext)
fd.close()
-netsvc.Server.startAll()
+openerp.netsvc.Server.startAll()
logger.info('OpenERP server is running, waiting for connections...')
-while netsvc.quit_signals_received == 0:
+while quit_signals_received == 0:
time.sleep(60)
quit()
=== modified file 'openerp/__init__.py'
--- openerp/__init__.py 2011-02-08 09:29:23 +0000
+++ openerp/__init__.py 2011-05-19 09:56:30 +0000
@@ -19,9 +19,15 @@
#
##############################################################################
+""" OpenERP core library.
+
+"""
+
import addons
+import conf
import ir
import loglevels
+import modules
import netsvc
import osv
import pooler
=== added file 'openerp/addons/__init__.py'
--- openerp/addons/__init__.py 1970-01-01 00:00:00 +0000
+++ openerp/addons/__init__.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
+# Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+""" Addons module.
+
+This module only serves to contain OpenERP addons. For the code to
+manage those addons, see openerp.modules. This module conveniently
+reexports some symbols from openerp.modules. Importing them from here
+is deprecated.
+
+"""
+
+# get_module_path is used only by base_module_quality
+from openerp.modules import get_module_resource, get_module_path
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/addons/base/ir/ir_ui_menu.py'
--- openerp/addons/base/ir/ir_ui_menu.py 2011-05-16 15:05:34 +0000
+++ openerp/addons/base/ir/ir_ui_menu.py 2011-05-19 09:56:30 +0000
@@ -24,7 +24,7 @@
import re
import tools
-import addons
+import openerp.modules
from osv import fields, osv
from tools.translate import _
@@ -229,7 +229,7 @@
if not path:
return False
path_info = path.split(',')
- icon_path = addons.get_module_resource(path_info[0],path_info[1])
+ icon_path = openerp.modules.get_module_resource(path_info[0],path_info[1])
icon_image = False
if icon_path:
try:
=== modified file 'openerp/addons/base/module/module.py'
--- openerp/addons/base/module/module.py 2011-05-12 13:45:53 +0000
+++ openerp/addons/base/module/module.py 2011-05-19 09:56:30 +0000
@@ -30,7 +30,7 @@
import zipfile
import zipimport
-import addons
+import openerp.modules as addons
import pooler
import release
import tools
@@ -79,8 +79,7 @@
info = {}
try:
info = addons.load_information_from_description_file(name)
- if 'version' in info:
- info['version'] = release.major_version + '.' + info['version']
+ info['version'] = release.major_version + '.' + info['version']
except Exception:
cls.__logger.debug('Error when trying to fetch informations for '
'module %s', name, exc_info=True)
=== added directory 'openerp/conf'
=== added file 'openerp/conf/__init__.py'
--- openerp/conf/__init__.py 1970-01-01 00:00:00 +0000
+++ openerp/conf/__init__.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2011 OpenERP s.a. (<http://openerp.com>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+""" Library-wide configuration variables.
+
+For now, configuration code is in openerp.tools.config. It is in mainly
+unprocessed form, e.g. addons_path is a string with commas-separated
+paths. The aim is to have code related to configuration (command line
+parsing, configuration file loading and saving, ...) in this module
+and provide real Python variables, e.g. addons_paths is really a list
+of paths.
+
+"""
+
+import deprecation
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== added file 'openerp/conf/deprecation.py'
--- openerp/conf/deprecation.py 1970-01-01 00:00:00 +0000
+++ openerp/conf/deprecation.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2011 OpenERP s.a. (<http://openerp.com>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+""" Regroup variables for deprecated features.
+
+To keep the OpenERP server backward compatible with older modules, some
+additional code is needed throughout the core library. This module keeps
+track of those specific measures by providing variables that can be unset
+by the user to check if her code is future proof.
+
+"""
+
+# If True, the Python modules inside the openerp namespace are made available
+# without the 'openerp.' prefix. E.g. openerp.osv.osv and osv.osv refer to the
+# same module.
+# Introduced around 2011.02.
+open_openerp_namespace = True
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== added directory 'openerp/db'
=== added file 'openerp/db/__init__.py'
--- openerp/db/__init__.py 1970-01-01 00:00:00 +0000
+++ openerp/db/__init__.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2011 OpenERP s.a. (<http://openerp.com>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+""" Lower-level database access.
+
+This module provides access to the underlying database without going
+through the ORM. The goal is to gather sql_db.py and other various db
+code.
+
+"""
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== added directory 'openerp/modules'
=== added file 'openerp/modules/__init__.py'
--- openerp/modules/__init__.py 1970-01-01 00:00:00 +0000
+++ openerp/modules/__init__.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
+# Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+""" Modules (also called addons) management.
+
+"""
+
+import openerp.modules.db
+import openerp.modules.graph
+import openerp.modules.loading
+import openerp.modules.migration
+import openerp.modules.module
+
+# TODO temporarily expose those things
+from openerp.modules.module import \
+ get_modules, get_modules_with_version, \
+ load_information_from_description_file, \
+ get_module_resource, zip_directory, \
+ get_module_path, initialize_sys_path, \
+ register_module_classes, init_module_models
+
+from openerp.modules.loading import load_modules
+
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== added file 'openerp/modules/db.py'
--- openerp/modules/db.py 1970-01-01 00:00:00 +0000
+++ openerp/modules/db.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
+# Copyright (C) 2010 OpenERP s.a. (<http://openerp.com>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+import openerp.modules
+
+def is_initialized(cr):
+ """ Check if a database has been initialized for the ORM.
+
+ The database can be initialized with the 'initialize' function below.
+
+ """
+ cr.execute("SELECT relname FROM pg_class WHERE relkind='r' AND relname='ir_module_module'")
+ return len(cr.fetchall()) > 0
+
+def initialize(cr):
+ """ Initialize a database with for the ORM.
+
+ This executes base/base.sql, creates the ir_module_categories (taken
+ from each module descriptor file), and creates the ir_module_module
+ and ir_model_data entries.
+
+ """
+ f = openerp.modules.get_module_resource('base', 'base.sql')
+ base_sql_file = openerp.tools.misc.file_open(f)
+ try:
+ cr.execute(base_sql_file.read())
+ cr.commit()
+ finally:
+ base_sql_file.close()
+
+ for i in openerp.modules.get_modules():
+ mod_path = openerp.modules.get_module_path(i)
+ if not mod_path:
+ continue
+
+ # This will raise an exception if no/unreadable descriptor file.
+ info = openerp.modules.load_information_from_description_file(i)
+
+ if not info:
+ continue
+ categories = info['category'].split('/')
+ category_id = create_categories(cr, categories)
+
+ if info['installable']:
+ if info['active']:
+ state = 'to install'
+ else:
+ state = 'uninstalled'
+ else:
+ state = 'uninstallable'
+
+ cr.execute('INSERT INTO ir_module_module \
+ (author, website, name, shortdesc, description, \
+ category_id, state, certificate, web, license) \
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id', (
+ info['author'],
+ info['website'], i, info['name'],
+ info['description'], category_id, state, info['certificate'],
+ info['web'],
+ info['license']))
+ id = cr.fetchone()[0]
+ cr.execute('INSERT INTO ir_model_data \
+ (name,model,module, res_id, noupdate) VALUES (%s,%s,%s,%s,%s)', (
+ 'module_meta_information', 'ir.module.module', i, id, True))
+ dependencies = info['depends']
+ for d in dependencies:
+ cr.execute('INSERT INTO ir_module_module_dependency \
+ (module_id,name) VALUES (%s, %s)', (id, d))
+ cr.commit()
+
+def create_categories(cr, categories):
+ """ Create the ir_module_category entries for some categories.
+
+ categories is a list of strings forming a single category with its
+ parent categories, like ['Grand Parent', 'Parent', 'Child'].
+
+ Return the database id of the (last) category.
+
+ """
+ p_id = None
+ while categories:
+ if p_id is not None:
+ cr.execute('SELECT id \
+ FROM ir_module_category \
+ WHERE name=%s AND parent_id=%s', (categories[0], p_id))
+ else:
+ cr.execute('SELECT id \
+ FROM ir_module_category \
+ WHERE name=%s AND parent_id IS NULL', (categories[0],))
+ c_id = cr.fetchone()
+ if not c_id:
+ cr.execute('INSERT INTO ir_module_category \
+ (name, parent_id) \
+ VALUES (%s, %s) RETURNING id', (categories[0], p_id))
+ c_id = cr.fetchone()[0]
+ else:
+ c_id = c_id[0]
+ p_id = c_id
+ categories = categories[1:]
+ return p_id
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== added file 'openerp/modules/graph.py'
--- openerp/modules/graph.py 1970-01-01 00:00:00 +0000
+++ openerp/modules/graph.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
+# Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+""" Modules dependency graph. """
+
+import os, sys, imp
+from os.path import join as opj
+import itertools
+import zipimport
+
+import openerp
+
+import openerp.osv as osv
+import openerp.tools as tools
+import openerp.tools.osutil as osutil
+from openerp.tools.safe_eval import safe_eval as eval
+import openerp.pooler as pooler
+from openerp.tools.translate import _
+
+import openerp.netsvc as netsvc
+
+import zipfile
+import openerp.release as release
+
+import re
+import base64
+from zipfile import PyZipFile, ZIP_DEFLATED
+from cStringIO import StringIO
+
+import logging
+
+logger = netsvc.Logger()
+
+
+class Graph(dict):
+ """ Modules dependency graph.
+
+ The graph is a mapping from module name to Nodes.
+
+ """
+
+ def add_node(self, name, deps):
+ max_depth, father = 0, None
+ for n in [Node(x, self) for x in deps]:
+ if n.depth >= max_depth:
+ father = n
+ max_depth = n.depth
+ if father:
+ return father.add_child(name)
+ else:
+ return Node(name, self)
+
+ def update_from_db(self, cr):
+ if not len(self):
+ return
+ # update the graph with values from the database (if exist)
+ ## First, we set the default values for each package in graph
+ additional_data = dict.fromkeys(self.keys(), {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None})
+ ## Then we get the values from the database
+ cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version'
+ ' FROM ir_module_module'
+ ' WHERE name IN %s',(tuple(additional_data),)
+ )
+
+ ## and we update the default values with values from the database
+ additional_data.update(dict([(x.pop('name'), x) for x in cr.dictfetchall()]))
+
+ for package in self.values():
+ for k, v in additional_data[package.name].items():
+ setattr(package, k, v)
+
+ def add_module(self, cr, module, force=None):
+ self.add_modules(cr, [module], force)
+
+ def add_modules(self, cr, module_list, force=None):
+ if force is None:
+ force = []
+ packages = []
+ len_graph = len(self)
+ for module in module_list:
+ # This will raise an exception if no/unreadable descriptor file.
+ # NOTE The call to load_information_from_description_file is already
+ # done by db.initialize, so it is possible to not do it again here.
+ info = openerp.modules.module.load_information_from_description_file(module)
+ if info['installable']:
+ packages.append((module, info)) # TODO directly a dict, like in get_modules_with_version
+ else:
+ logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not installable, skipped' % (module))
+
+ dependencies = dict([(p, info['depends']) for p, info in packages])
+ current, later = set([p for p, info in packages]), set()
+
+ while packages and current > later:
+ package, info = packages[0]
+ deps = info['depends']
+
+ # if all dependencies of 'package' are already in the graph, add 'package' in the graph
+ if reduce(lambda x, y: x and y in self, deps, True):
+ if not package in current:
+ packages.pop(0)
+ continue
+ later.clear()
+ current.remove(package)
+ node = self.add_node(package, deps)
+ node.data = info
+ for kind in ('init', 'demo', 'update'):
+ if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
+ setattr(node, kind, True)
+ else:
+ later.add(package)
+ packages.append((package, info))
+ packages.pop(0)
+
+ self.update_from_db(cr)
+
+ for package in later:
+ unmet_deps = filter(lambda p: p not in self, dependencies[package])
+ logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: Unmet dependencies: %s' % (package, ', '.join(unmet_deps)))
+
+ result = len(self) - len_graph
+ if result != len(module_list):
+ logger.notifyChannel('init', netsvc.LOG_WARNING, 'Not all modules have loaded.')
+ return result
+
+
+ def __iter__(self):
+ level = 0
+ done = set(self.keys())
+ while done:
+ level_modules = [(name, module) for name, module in self.items() if module.depth==level]
+ for name, module in level_modules:
+ done.remove(name)
+ yield module
+ level += 1
+
+
+class Singleton(object):
+ def __new__(cls, name, graph):
+ if name in graph:
+ inst = graph[name]
+ else:
+ inst = object.__new__(cls)
+ inst.name = name
+ graph[name] = inst
+ return inst
+
+
+class Node(Singleton):
+ """ One module in the modules dependency graph.
+
+ Node acts as a per-module singleton.
+
+ """
+
+ def __init__(self, name, graph):
+ self.graph = graph
+ if not hasattr(self, 'children'):
+ self.children = []
+ if not hasattr(self, 'depth'):
+ self.depth = 0
+
+ def add_child(self, name):
+ node = Node(name, self.graph)
+ node.depth = self.depth + 1
+ if node not in self.children:
+ self.children.append(node)
+ for attr in ('init', 'update', 'demo'):
+ if hasattr(self, attr):
+ setattr(node, attr, True)
+ self.children.sort(lambda x, y: cmp(x.name, y.name))
+ return node
+
+ def __setattr__(self, name, value):
+ super(Singleton, self).__setattr__(name, value)
+ if name in ('init', 'update', 'demo'):
+ tools.config[name][self.name] = 1
+ for child in self.children:
+ setattr(child, name, value)
+ if name == 'depth':
+ for child in self.children:
+ setattr(child, name, value + 1)
+
+ def __iter__(self):
+ return itertools.chain(iter(self.children), *map(iter, self.children))
+
+ def __str__(self):
+ return self._pprint()
+
+ def _pprint(self, depth=0):
+ s = '%s\n' % self.name
+ for c in self.children:
+ s += '%s`-> %s' % (' ' * depth, c._pprint(depth+1))
+ return s
+
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== renamed file 'openerp/addons/__init__.py' => 'openerp/modules/loading.py'
--- openerp/addons/__init__.py 2011-02-09 10:08:45 +0000
+++ openerp/modules/loading.py 2011-05-19 09:56:30 +0000
@@ -20,11 +20,17 @@
#
##############################################################################
+""" Modules (also called addons) management.
+
+"""
+
import os, sys, imp
from os.path import join as opj
import itertools
import zipimport
+import openerp
+
import openerp.osv as osv
import openerp.tools as tools
import openerp.tools.osutil as osutil
@@ -44,581 +50,29 @@
import logging
+import openerp.modules.db
+import openerp.modules.graph
+import openerp.modules.migration
+
+from openerp.modules.module import \
+ get_modules, get_modules_with_version, \
+ load_information_from_description_file, \
+ get_module_resource, zip_directory, \
+ get_module_path, initialize_sys_path, \
+ register_module_classes, init_module_models
+
logger = netsvc.Logger()
-_ad = os.path.dirname(__file__) # default addons path (base)
-ad_paths = []
-
-# Modules already loaded
-loaded = []
-
-def initialize_sys_path():
- global ad_paths
-
- if ad_paths:
- return
-
- ad_paths = map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(','))
-
- sys.path.insert(1, _ad)
-
- ad_cnt=1
- for adp in ad_paths:
- if adp != _ad:
- sys.path.insert(ad_cnt, adp)
- ad_cnt+=1
-
- ad_paths.append(_ad) # for get_module_path
-
-class Graph(dict):
-
- def addNode(self, name, deps):
- max_depth, father = 0, None
- for n in [Node(x, self) for x in deps]:
- if n.depth >= max_depth:
- father = n
- max_depth = n.depth
- if father:
- father.addChild(name)
- else:
- Node(name, self)
-
- def update_from_db(self, cr):
- if not len(self):
- return
- # update the graph with values from the database (if exist)
- ## First, we set the default values for each package in graph
- additional_data = dict.fromkeys(self.keys(), {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None})
- ## Then we get the values from the database
- cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version'
- ' FROM ir_module_module'
- ' WHERE name IN %s',(tuple(additional_data),)
- )
-
- ## and we update the default values with values from the database
- additional_data.update(dict([(x.pop('name'), x) for x in cr.dictfetchall()]))
-
- for package in self.values():
- for k, v in additional_data[package.name].items():
- setattr(package, k, v)
-
- def __iter__(self):
- level = 0
- done = set(self.keys())
- while done:
- level_modules = [(name, module) for name, module in self.items() if module.depth==level]
- for name, module in level_modules:
- done.remove(name)
- yield module
- level += 1
-
-class Singleton(object):
- def __new__(cls, name, graph):
- if name in graph:
- inst = graph[name]
- else:
- inst = object.__new__(cls)
- inst.name = name
- graph[name] = inst
- return inst
-
-
-class Node(Singleton):
-
- def __init__(self, name, graph):
- self.graph = graph
- if not hasattr(self, 'children'):
- self.children = []
- if not hasattr(self, 'depth'):
- self.depth = 0
-
- def addChild(self, name):
- node = Node(name, self.graph)
- node.depth = self.depth + 1
- if node not in self.children:
- self.children.append(node)
- for attr in ('init', 'update', 'demo'):
- if hasattr(self, attr):
- setattr(node, attr, True)
- self.children.sort(lambda x, y: cmp(x.name, y.name))
-
- def __setattr__(self, name, value):
- super(Singleton, self).__setattr__(name, value)
- if name in ('init', 'update', 'demo'):
- tools.config[name][self.name] = 1
- for child in self.children:
- setattr(child, name, value)
- if name == 'depth':
- for child in self.children:
- setattr(child, name, value + 1)
-
- def __iter__(self):
- return itertools.chain(iter(self.children), *map(iter, self.children))
-
- def __str__(self):
- return self._pprint()
-
- def _pprint(self, depth=0):
- s = '%s\n' % self.name
- for c in self.children:
- s += '%s`-> %s' % (' ' * depth, c._pprint(depth+1))
- return s
-
-
-def get_module_path(module, downloaded=False):
- """Return the path of the given module."""
- initialize_sys_path()
- for adp in ad_paths:
- if os.path.exists(opj(adp, module)) or os.path.exists(opj(adp, '%s.zip' % module)):
- return opj(adp, module)
-
- if downloaded:
- return opj(_ad, module)
- logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: module not found' % (module,))
- return False
-
-
-def get_module_filetree(module, dir='.'):
- path = get_module_path(module)
- if not path:
- return False
-
- dir = os.path.normpath(dir)
- if dir == '.':
- dir = ''
- if dir.startswith('..') or (dir and dir[0] == '/'):
- raise Exception('Cannot access file outside the module')
-
- if not os.path.isdir(path):
- # zipmodule
- zip = zipfile.ZipFile(path + ".zip")
- files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()]
- else:
- files = osutil.listdir(path, True)
-
- tree = {}
- for f in files:
- if not f.startswith(dir):
- continue
-
- if dir:
- f = f[len(dir)+int(not dir.endswith('/')):]
- lst = f.split(os.sep)
- current = tree
- while len(lst) != 1:
- current = current.setdefault(lst.pop(0), {})
- current[lst.pop(0)] = None
-
- return tree
-
-def zip_directory(directory, b64enc=True, src=True):
- """Compress a directory
-
- @param directory: The directory to compress
- @param base64enc: if True the function will encode the zip file with base64
- @param src: Integrate the source files
-
- @return: a string containing the zip file
- """
-
- RE_exclude = re.compile('(?:^\..+\.swp$)|(?:\.py[oc]$)|(?:\.bak$)|(?:\.~.~$)', re.I)
-
- def _zippy(archive, path, src=True):
- path = os.path.abspath(path)
- base = os.path.basename(path)
- for f in osutil.listdir(path, True):
- bf = os.path.basename(f)
- if not RE_exclude.search(bf) and (src or bf in ('__openerp__.py', '__terp__.py') or not bf.endswith('.py')):
- archive.write(os.path.join(path, f), os.path.join(base, f))
-
- archname = StringIO()
- archive = PyZipFile(archname, "w", ZIP_DEFLATED)
-
- # for Python 2.5, ZipFile.write() still expects 8-bit strings (2.6 converts to utf-8)
- directory = tools.ustr(directory).encode('utf-8')
-
- archive.writepy(directory)
- _zippy(archive, directory, src=src)
- archive.close()
- archive_data = archname.getvalue()
- archname.close()
-
- if b64enc:
- return base64.encodestring(archive_data)
-
- return archive_data
-
-def get_module_as_zip(modulename, b64enc=True, src=True):
- """Generate a module as zip file with the source or not and can do a base64 encoding
-
- @param modulename: The module name
- @param b64enc: if True the function will encode the zip file with base64
- @param src: Integrate the source files
-
- @return: a stream to store in a file-like object
- """
-
- ap = get_module_path(str(modulename))
- if not ap:
- raise Exception('Unable to find path for module %s' % modulename)
-
- ap = ap.encode('utf8')
- if os.path.isfile(ap + '.zip'):
- val = file(ap + '.zip', 'rb').read()
- if b64enc:
- val = base64.encodestring(val)
- else:
- val = zip_directory(ap, b64enc, src)
-
- return val
-
-
-def get_module_resource(module, *args):
- """Return the full path of a resource of the given module.
-
- @param module: the module
- @param args: the resource path components
-
- @return: absolute path to the resource
- """
- a = get_module_path(module)
- if not a: return False
- resource_path = opj(a, *args)
- if zipfile.is_zipfile( a +'.zip') :
- zip = zipfile.ZipFile( a + ".zip")
- files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()]
- resource_path = '/'.join(args)
- if resource_path in files:
- return opj(a, resource_path)
- elif os.path.exists(resource_path):
- return resource_path
- return False
-
-def get_modules():
- """Returns the list of module names
- """
- def listdir(dir):
- def clean(name):
- name = os.path.basename(name)
- if name[-4:] == '.zip':
- name = name[:-4]
- return name
-
- def is_really_module(name):
- name = opj(dir, name)
- return os.path.isdir(name) or zipfile.is_zipfile(name)
- return map(clean, filter(is_really_module, os.listdir(dir)))
-
- plist = []
- initialize_sys_path()
- for ad in ad_paths:
- plist.extend(listdir(ad))
- return list(set(plist))
-
-def load_information_from_description_file(module):
- """
- :param module: The name of the module (sale, purchase, ...)
- """
-
- for filename in ['__openerp__.py', '__terp__.py']:
- description_file = get_module_resource(module, filename)
- if description_file :
- desc_f = tools.file_open(description_file)
- try:
- return eval(desc_f.read())
- finally:
- desc_f.close()
-
- #TODO: refactor the logger in this file to follow the logging guidelines
- # for 6.0
- logging.getLogger('addons').debug('The module %s does not contain a description file:'\
- '__openerp__.py or __terp__.py (deprecated)', module)
- return {}
-
-def get_modules_with_version():
- modules = get_modules()
- res = {}
- for module in modules:
- try:
- info = load_information_from_description_file(module)
- res[module] = "%s.%s" % (release.major_version, info['version'])
- except Exception, e:
- continue
- return res
-
-def create_graph(cr, module_list, force=None):
- graph = Graph()
- upgrade_graph(graph, cr, module_list, force)
- return graph
-
-def upgrade_graph(graph, cr, module_list, force=None):
- if force is None:
- force = []
- packages = []
- len_graph = len(graph)
- for module in module_list:
- mod_path = get_module_path(module)
- terp_file = get_module_resource(module, '__openerp__.py')
- if not terp_file:
- terp_file = get_module_resource(module, '__terp__.py')
- if not mod_path or not terp_file:
- logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not found, skipped' % (module))
- continue
-
- if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path+'.zip'):
- terp_f = tools.file_open(terp_file)
- try:
- info = eval(terp_f.read())
- except Exception:
- logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: eval file %s' % (module, terp_file))
- raise
- finally:
- terp_f.close()
- if info.get('installable', True):
- packages.append((module, info.get('depends', []), info))
- else:
- logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not installable, skipped' % (module))
-
- dependencies = dict([(p, deps) for p, deps, data in packages])
- current, later = set([p for p, dep, data in packages]), set()
-
- while packages and current > later:
- package, deps, data = packages[0]
-
- # if all dependencies of 'package' are already in the graph, add 'package' in the graph
- if reduce(lambda x, y: x and y in graph, deps, True):
- if not package in current:
- packages.pop(0)
- continue
- later.clear()
- current.remove(package)
- graph.addNode(package, deps)
- node = Node(package, graph)
- node.data = data
- for kind in ('init', 'demo', 'update'):
- if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
- setattr(node, kind, True)
- else:
- later.add(package)
- packages.append((package, deps, data))
- packages.pop(0)
-
- graph.update_from_db(cr)
-
- for package in later:
- unmet_deps = filter(lambda p: p not in graph, dependencies[package])
- logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: Unmet dependencies: %s' % (package, ', '.join(unmet_deps)))
-
- result = len(graph) - len_graph
- if result != len(module_list):
- logger.notifyChannel('init', netsvc.LOG_WARNING, 'Not all modules have loaded.')
- return result
-
-
-def init_module_objects(cr, module_name, obj_list):
- logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: creating or updating database tables' % module_name)
- todo = []
- for obj in obj_list:
- try:
- result = obj._auto_init(cr, {'module': module_name})
- except Exception, e:
- raise
- if result:
- todo += result
- if hasattr(obj, 'init'):
- obj.init(cr)
- cr.commit()
- todo.sort()
- for t in todo:
- t[1](cr, *t[2])
- cr.commit()
-
-
-def register_class(m):
- """
- Register module named m, if not already registered
- """
-
- def log(e):
- mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or ''
- msg = "Couldn't load %smodule %s" % (mt, m)
- logger.notifyChannel('init', netsvc.LOG_CRITICAL, msg)
- logger.notifyChannel('init', netsvc.LOG_CRITICAL, e)
-
- global loaded
- if m in loaded:
- return
- logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: registering objects' % m)
- mod_path = get_module_path(m)
-
- initialize_sys_path()
- try:
- zip_mod_path = mod_path + '.zip'
- if not os.path.isfile(zip_mod_path):
- fm = imp.find_module(m, ad_paths)
- try:
- imp.load_module(m, *fm)
- finally:
- if fm[0]:
- fm[0].close()
- else:
- zimp = zipimport.zipimporter(zip_mod_path)
- zimp.load_module(m)
- except Exception, e:
- log(e)
- raise
- else:
- loaded.append(m)
-
-
-class MigrationManager(object):
- """
- This class manage the migration of modules
- Migrations files must be python files containing a "migrate(cr, installed_version)" function.
- Theses files must respect a directory tree structure: A 'migrations' folder which containt a
- folder by version. Version can be 'module' version or 'server.module' version (in this case,
- the files will only be processed by this version of the server). Python file names must start
- by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation
- Example:
-
- <moduledir>
- `-- migrations
- |-- 1.0
- | |-- pre-update_table_x.py
- | |-- pre-update_table_y.py
- | |-- post-clean-data.py
- | `-- README.txt # not processed
- |-- 5.0.1.1 # files in this folder will be executed only on a 5.0 server
- | |-- pre-delete_table_z.py
- | `-- post-clean-data.py
- `-- foo.py # not processed
-
- This similar structure is generated by the maintenance module with the migrations files get by
- the maintenance contract
-
- """
- def __init__(self, cr, graph):
- self.cr = cr
- self.graph = graph
- self.migrations = {}
- self._get_files()
-
- def _get_files(self):
-
- """
- import addons.base.maintenance.utils as maintenance_utils
- maintenance_utils.update_migrations_files(self.cr)
- #"""
-
- for pkg in self.graph:
- self.migrations[pkg.name] = {}
- if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'):
- continue
-
- self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {}
- self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {}
-
- def migrate_module(self, pkg, stage):
- assert stage in ('pre', 'post')
- stageformat = {'pre': '[>%s]',
- 'post': '[%s>]',
- }
-
- if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'):
- return
-
- def convert_version(version):
- if version.startswith(release.major_version) and version != release.major_version:
- return version # the version number already containt the server version
- return "%s.%s" % (release.major_version, version)
-
- def _get_migration_versions(pkg):
- def __get_dir(tree):
- return [d for d in tree if tree[d] is not None]
-
- versions = list(set(
- __get_dir(self.migrations[pkg.name]['module']) +
- __get_dir(self.migrations[pkg.name]['maintenance'])
- ))
- versions.sort(key=lambda k: parse_version(convert_version(k)))
- return versions
-
- def _get_migration_files(pkg, version, stage):
- """ return a list of tuple (module, file)
- """
- m = self.migrations[pkg.name]
- lst = []
-
- mapping = {'module': opj(pkg.name, 'migrations'),
- 'maintenance': opj('base', 'maintenance', 'migrations', pkg.name),
- }
-
- for x in mapping.keys():
- if version in m[x]:
- for f in m[x][version]:
- if m[x][version][f] is not None:
- continue
- if not f.startswith(stage + '-'):
- continue
- lst.append(opj(mapping[x], version, f))
- lst.sort()
- return lst
-
- def mergedict(a, b):
- a = a.copy()
- a.update(b)
- return a
-
- from openerp.tools.parse_version import parse_version
-
- parsed_installed_version = parse_version(pkg.installed_version or '')
- current_version = parse_version(convert_version(pkg.data.get('version', '0')))
-
- versions = _get_migration_versions(pkg)
-
- for version in versions:
- if parsed_installed_version < parse_version(convert_version(version)) <= current_version:
-
- strfmt = {'addon': pkg.name,
- 'stage': stage,
- 'version': stageformat[stage] % version,
- }
-
- for pyfile in _get_migration_files(pkg, version, stage):
- name, ext = os.path.splitext(os.path.basename(pyfile))
- if ext.lower() != '.py':
- continue
- mod = fp = fp2 = None
- try:
- fp = tools.file_open(pyfile)
-
- # imp.load_source need a real file object, so we create
- # one from the file-like object we get from file_open
- fp2 = os.tmpfile()
- fp2.write(fp.read())
- fp2.seek(0)
- try:
- mod = imp.load_source(name, pyfile, fp2)
- logger.notifyChannel('migration', netsvc.LOG_INFO, 'module %(addon)s: Running migration %(version)s %(name)s' % mergedict({'name': mod.__name__}, strfmt))
- mod.migrate(self.cr, pkg.installed_version)
- except ImportError:
- logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % mergedict({'file': pyfile}, strfmt))
- raise
- except AttributeError:
- logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt)
- except:
- raise
- finally:
- if fp:
- fp.close()
- if fp2:
- fp2.close()
- if mod:
- del mod
-
-log = logging.getLogger('init')
-
-def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, **kwargs):
+
+def open_openerp_namespace():
+ # See comment for open_openerp_namespace.
+ if openerp.conf.deprecation.open_openerp_namespace:
+ for k, v in list(sys.modules.items()):
+ if k.startswith('openerp.') and sys.modules.get(k[8:]) is None:
+ sys.modules[k[8:]] = v
+
+
+def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: status dictionary for keeping track of progress
@@ -634,50 +88,26 @@
if new_query:
cr.execute(new_query)
- def load_init_update_xml(cr, m, idref, mode, kind):
- for filename in package.data.get('%s_xml' % kind, []):
- logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, filename))
- _, ext = os.path.splitext(filename)
- fp = tools.file_open(opj(m, filename))
- try:
- if ext == '.csv':
- noupdate = (kind == 'init')
- tools.convert_csv_import(cr, m, os.path.basename(filename), fp.read(), idref, mode=mode, noupdate=noupdate)
- elif ext == '.sql':
- process_sql_file(cr, fp)
- elif ext == '.yml':
- tools.convert_yaml_import(cr, m, fp, idref, mode=mode, **kwargs)
- else:
- tools.convert_xml_import(cr, m, fp, idref, mode=mode, **kwargs)
- finally:
- fp.close()
+ def load_init_xml(cr, m, idref, mode):
+ _load_data(cr, m, idref, mode, 'init_xml')
+
+ def load_update_xml(cr, m, idref, mode):
+ _load_data(cr, m, idref, mode, 'update_xml')
def load_demo_xml(cr, m, idref, mode):
- for xml in package.data.get('demo_xml', []):
- name, ext = os.path.splitext(xml)
- logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, xml))
- fp = tools.file_open(opj(m, xml))
- try:
- if ext == '.csv':
- tools.convert_csv_import(cr, m, os.path.basename(xml), fp.read(), idref, mode=mode, noupdate=True)
- elif ext == '.yml':
- tools.convert_yaml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs)
- else:
- tools.convert_xml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs)
- finally:
- fp.close()
-
- def load_data(cr, module_name, id_map, mode):
- _load_data(cr, module_name, id_map, mode, 'data')
-
- def load_demo(cr, module_name, id_map, mode):
- _load_data(cr, module_name, id_map, mode, 'demo')
-
- def load_test(cr, module_name, id_map, mode):
+ _load_data(cr, m, idref, mode, 'demo_xml')
+
+ def load_data(cr, module_name, idref, mode):
+ _load_data(cr, module_name, idref, mode, 'data')
+
+ def load_demo(cr, module_name, idref, mode):
+ _load_data(cr, module_name, idref, mode, 'demo')
+
+ def load_test(cr, module_name, idref, mode):
cr.commit()
if not tools.config.options['test_disable']:
try:
- _load_data(cr, module_name, id_map, mode, 'test')
+ _load_data(cr, module_name, idref, mode, 'test')
except Exception, e:
logging.getLogger('test').exception('Tests failed to execute in module %s', module_name)
finally:
@@ -686,49 +116,61 @@
else:
cr.rollback()
- def _load_data(cr, module_name, id_map, mode, kind):
- for filename in package.data.get(kind, []):
- noupdate = (kind == 'demo')
- _, ext = os.path.splitext(filename)
+ def _load_data(cr, module_name, idref, mode, kind):
+ """
+
+ kind: data, demo, test, init_xml, update_xml, demo_xml.
+
+ noupdate is False, unless it is demo data or it is csv data in
+ init mode.
+
+ """
+ for filename in package.data[kind]:
+ log = logging.getLogger('init')
log.info("module %s: loading %s", module_name, filename)
+ _, ext = os.path.splitext(filename)
pathname = os.path.join(module_name, filename)
- file = tools.file_open(pathname)
+ fp = tools.file_open(pathname)
+ noupdate = False
+ if kind in ('demo', 'demo_xml'):
+ noupdate = True
try:
- if ext == '.sql':
- process_sql_file(cr, file)
- elif ext == '.csv':
- noupdate = (kind == 'init')
- tools.convert_csv_import(cr, module_name, pathname, file.read(), id_map, mode, noupdate)
+ if ext == '.csv':
+ if kind in ('init', 'init_xml'):
+ noupdate = True
+ tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate)
+ elif ext == '.sql':
+ process_sql_file(cr, fp)
elif ext == '.yml':
- tools.convert_yaml_import(cr, module_name, file, id_map, mode, noupdate)
+ tools.convert_yaml_import(cr, module_name, fp, idref, mode, noupdate)
else:
- tools.convert_xml_import(cr, module_name, file, id_map, mode, noupdate)
+ tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report)
finally:
- file.close()
+ fp.close()
- # **kwargs is passed directly to convert_xml_import
- if not status:
+ if status is None:
status = {}
- status = status.copy()
processed_modules = []
statusi = 0
pool = pooler.get_pool(cr.dbname)
- migrations = MigrationManager(cr, graph)
- modobj = None
+ migrations = openerp.modules.migration.MigrationManager(cr, graph)
logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph))
+ # register, instanciate and initialize models for each modules
for package in graph:
if skip_modules and package.name in skip_modules:
continue
logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name)
migrations.migrate_module(package, 'pre')
- register_class(package.name)
- modules = pool.instanciate(package.name, cr)
+ register_module_classes(package.name)
+ models = pool.instanciate(package.name, cr)
if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
- init_module_objects(cr, package.name, modules)
+ init_module_models(cr, package.name, models)
cr.commit()
+ # load data for each modules
+ modobj = pool.get('ir.module.module')
for package in graph:
status['progress'] = (float(statusi)+0.1) / len(graph)
m = package.name
@@ -737,10 +179,7 @@
if skip_modules and m in skip_modules:
continue
- if modobj is None:
- modobj = pool.get('ir.module.module')
-
- if modobj and perform_checks:
+ if perform_checks:
modobj.check(cr, 1, [mid])
idref = {}
@@ -751,11 +190,11 @@
mode = 'init'
if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
- for kind in ('init', 'update'):
- if package.state=='to upgrade':
- # upgrading the module information
- modobj.write(cr, 1, [mid], modobj.get_values_from_terp(package.data))
- load_init_update_xml(cr, m, idref, mode, kind)
+ if package.state=='to upgrade':
+ # upgrading the module information
+ modobj.write(cr, 1, [mid], modobj.get_values_from_terp(package.data))
+ load_init_xml(cr, m, idref, mode)
+ load_update_xml(cr, m, idref, mode)
load_data(cr, m, idref, mode)
if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'):
status['progress'] = (float(statusi)+0.75) / len(graph)
@@ -773,14 +212,13 @@
migrations.migrate_module(package, 'post')
- if modobj:
- ver = release.major_version + '.' + package.data.get('version', '1.0')
- # Set new modules and dependencies
- modobj.write(cr, 1, [mid], {'state': 'installed', 'latest_version': ver})
- cr.commit()
- # Update translations for all installed languages
- modobj.update_translations(cr, 1, [mid], None, {'overwrite': tools.config['overwrite_existing_translations']})
- cr.commit()
+ ver = release.major_version + '.' + package.data['version']
+ # Set new modules and dependencies
+ modobj.write(cr, 1, [mid], {'state': 'installed', 'latest_version': ver})
+ cr.commit()
+ # Update translations for all installed languages
+ modobj.update_translations(cr, 1, [mid], None)
+ cr.commit()
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
@@ -808,34 +246,31 @@
logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
def load_modules(db, force_demo=False, status=None, update_module=False):
+ # TODO status['progress'] reporting is broken: used twice (and reset each
+ # time to zero) in load_module_graph, not fine-grained enough.
+ # It should be a method exposed by the pool.
initialize_sys_path()
- # Backward compatibility: addons don't have to import openerp.xxx, they still can import xxx
- for k, v in list(sys.modules.items()):
- if k.startswith('openerp.') and sys.modules.get(k[8:]) is None:
- sys.modules[k[8:]] = v
-
- if not status:
- status = {}
+ open_openerp_namespace()
+
+ force = []
+ if force_demo:
+ force.append('demo')
+
cr = db.cursor()
- if cr:
- cr.execute("SELECT relname FROM pg_class WHERE relkind='r' AND relname='ir_module_module'")
- if len(cr.fetchall())==0:
+ try:
+ if not openerp.modules.db.is_initialized(cr):
logger.notifyChannel("init", netsvc.LOG_INFO, "init db")
- tools.init_db(cr)
+ openerp.modules.db.initialize(cr)
tools.config["init"]["all"] = 1
tools.config['update']['all'] = 1
if not tools.config['without_demo']:
tools.config["demo"]['all'] = 1
- force = []
- if force_demo:
- force.append('demo')
-
- # This is a brand new pool, just created in pooler.get_db_and_pool()
- pool = pooler.get_pool(cr.dbname)
-
- try:
+
+ # This is a brand new pool, just created in pooler.get_db_and_pool()
+ pool = pooler.get_pool(cr.dbname)
+
processed_modules = []
report = tools.assertion_report()
# NOTE: Try to also load the modules that have been marked as uninstallable previously...
@@ -844,7 +279,8 @@
cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
- graph = create_graph(cr, ['base'], force)
+ graph = openerp.modules.graph.Graph()
+ graph.add_module(cr, 'base', force)
if not graph:
logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)')
raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
@@ -857,8 +293,8 @@
# STEP 2: Mark other modules to be loaded/updated
if update_module:
modobj = pool.get('ir.module.module')
- logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list')
if ('base' in tools.config['init']) or ('base' in tools.config['update']):
+ logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list')
modobj.update_list(cr, 1)
_check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
@@ -892,7 +328,7 @@
if not module_list:
break
- new_modules_in_graph = upgrade_graph(graph, cr, module_list, force)
+ new_modules_in_graph = graph.add_modules(cr, module_list, force)
if new_modules_in_graph == 0:
# nothing to load
break
@@ -940,6 +376,8 @@
cr.commit()
if update_module:
+ # Remove records referenced from ir_model_data for modules to be
+ # removed (and removed the references from ir_model_data).
cr.execute("select id,name from ir_module_module where state=%s", ('to remove',))
for mod_id, mod_name in cr.fetchall():
cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,))
@@ -947,14 +385,17 @@
uid = 1
rmod_module= pool.get(rmod)
if rmod_module:
+ # TODO group by module so that we can delete multiple ids in a call
rmod_module.unlink(cr, uid, [rid])
else:
logger.notifyChannel('init', netsvc.LOG_ERROR, 'Could not locate %s to remove res=%d' % (rmod,rid))
cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,))
cr.commit()
- #
+
+ # Remove menu items that are not referenced by any of other
+ # (child) menu item, ir_values, or ir_model_data.
+ # This code could be a method of ir_ui_menu.
# TODO: remove menu without actions of children
- #
while True:
cr.execute('''delete from
ir_ui_menu
@@ -970,6 +411,7 @@
else:
logger.notifyChannel('init', netsvc.LOG_INFO, 'removed %d unused menus' % (cr.rowcount,))
+ # Pretend that modules to be removed are actually uninstalled.
cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',))
cr.commit()
finally:
=== added file 'openerp/modules/migration.py'
--- openerp/modules/migration.py 1970-01-01 00:00:00 +0000
+++ openerp/modules/migration.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
+# Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+""" Modules migration handling. """
+
+import os, sys, imp
+from os.path import join as opj
+import itertools
+import zipimport
+
+import openerp
+
+import openerp.osv as osv
+import openerp.tools as tools
+import openerp.tools.osutil as osutil
+from openerp.tools.safe_eval import safe_eval as eval
+import openerp.pooler as pooler
+from openerp.tools.translate import _
+
+import openerp.netsvc as netsvc
+
+import zipfile
+import openerp.release as release
+
+import re
+import base64
+from zipfile import PyZipFile, ZIP_DEFLATED
+from cStringIO import StringIO
+
+import logging
+
+import openerp.modules.db
+import openerp.modules.graph
+
+logger = netsvc.Logger()
+
+
+class MigrationManager(object):
+ """
+ This class manage the migration of modules
+ Migrations files must be python files containing a "migrate(cr, installed_version)" function.
+ Theses files must respect a directory tree structure: A 'migrations' folder which containt a
+ folder by version. Version can be 'module' version or 'server.module' version (in this case,
+ the files will only be processed by this version of the server). Python file names must start
+ by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation
+ Example:
+
+ <moduledir>
+ `-- migrations
+ |-- 1.0
+ | |-- pre-update_table_x.py
+ | |-- pre-update_table_y.py
+ | |-- post-clean-data.py
+ | `-- README.txt # not processed
+ |-- 5.0.1.1 # files in this folder will be executed only on a 5.0 server
+ | |-- pre-delete_table_z.py
+ | `-- post-clean-data.py
+ `-- foo.py # not processed
+
+ This similar structure is generated by the maintenance module with the migrations files get by
+ the maintenance contract
+
+ """
+ def __init__(self, cr, graph):
+ self.cr = cr
+ self.graph = graph
+ self.migrations = {}
+ self._get_files()
+
+ def _get_files(self):
+
+ """
+ import addons.base.maintenance.utils as maintenance_utils
+ maintenance_utils.update_migrations_files(self.cr)
+ #"""
+
+ for pkg in self.graph:
+ self.migrations[pkg.name] = {}
+ if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'):
+ continue
+
+ get_module_filetree = openerp.modules.module.get_module_filetree
+ self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {}
+ self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {}
+
+ def migrate_module(self, pkg, stage):
+ assert stage in ('pre', 'post')
+ stageformat = {'pre': '[>%s]',
+ 'post': '[%s>]',
+ }
+
+ if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'):
+ return
+
+ def convert_version(version):
+ if version.startswith(release.major_version) and version != release.major_version:
+ return version # the version number already containt the server version
+ return "%s.%s" % (release.major_version, version)
+
+ def _get_migration_versions(pkg):
+ def __get_dir(tree):
+ return [d for d in tree if tree[d] is not None]
+
+ versions = list(set(
+ __get_dir(self.migrations[pkg.name]['module']) +
+ __get_dir(self.migrations[pkg.name]['maintenance'])
+ ))
+ versions.sort(key=lambda k: parse_version(convert_version(k)))
+ return versions
+
+ def _get_migration_files(pkg, version, stage):
+ """ return a list of tuple (module, file)
+ """
+ m = self.migrations[pkg.name]
+ lst = []
+
+ mapping = {'module': opj(pkg.name, 'migrations'),
+ 'maintenance': opj('base', 'maintenance', 'migrations', pkg.name),
+ }
+
+ for x in mapping.keys():
+ if version in m[x]:
+ for f in m[x][version]:
+ if m[x][version][f] is not None:
+ continue
+ if not f.startswith(stage + '-'):
+ continue
+ lst.append(opj(mapping[x], version, f))
+ lst.sort()
+ return lst
+
+ def mergedict(a, b):
+ a = a.copy()
+ a.update(b)
+ return a
+
+ from openerp.tools.parse_version import parse_version
+
+ parsed_installed_version = parse_version(pkg.installed_version or '')
+ current_version = parse_version(convert_version(pkg.data['version']))
+
+ versions = _get_migration_versions(pkg)
+
+ for version in versions:
+ if parsed_installed_version < parse_version(convert_version(version)) <= current_version:
+
+ strfmt = {'addon': pkg.name,
+ 'stage': stage,
+ 'version': stageformat[stage] % version,
+ }
+
+ for pyfile in _get_migration_files(pkg, version, stage):
+ name, ext = os.path.splitext(os.path.basename(pyfile))
+ if ext.lower() != '.py':
+ continue
+ mod = fp = fp2 = None
+ try:
+ fp = tools.file_open(pyfile)
+
+ # imp.load_source need a real file object, so we create
+ # one from the file-like object we get from file_open
+ fp2 = os.tmpfile()
+ fp2.write(fp.read())
+ fp2.seek(0)
+ try:
+ mod = imp.load_source(name, pyfile, fp2)
+ logger.notifyChannel('migration', netsvc.LOG_INFO, 'module %(addon)s: Running migration %(version)s %(name)s' % mergedict({'name': mod.__name__}, strfmt))
+ mod.migrate(self.cr, pkg.installed_version)
+ except ImportError:
+ logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % mergedict({'file': pyfile}, strfmt))
+ raise
+ except AttributeError:
+ logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt)
+ except:
+ raise
+ finally:
+ if fp:
+ fp.close()
+ if fp2:
+ fp2.close()
+ if mod:
+ del mod
+
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== added file 'openerp/modules/module.py'
--- openerp/modules/module.py 1970-01-01 00:00:00 +0000
+++ openerp/modules/module.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,379 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
+# Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+import os, sys, imp
+from os.path import join as opj
+import itertools
+import zipimport
+
+import openerp
+
+import openerp.osv as osv
+import openerp.tools as tools
+import openerp.tools.osutil as osutil
+from openerp.tools.safe_eval import safe_eval as eval
+import openerp.pooler as pooler
+from openerp.tools.translate import _
+
+import openerp.netsvc as netsvc
+
+import zipfile
+import openerp.release as release
+
+import re
+import base64
+from zipfile import PyZipFile, ZIP_DEFLATED
+from cStringIO import StringIO
+
+import logging
+
+import openerp.modules.db
+import openerp.modules.graph
+
+_ad = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'addons') # default addons path (base)
+ad_paths = []
+
+# Modules already loaded
+loaded = []
+
+logger = netsvc.Logger()
+
+def initialize_sys_path():
+ global ad_paths
+
+ if ad_paths:
+ return
+
+ ad_paths = map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(','))
+
+ sys.path.insert(1, _ad)
+
+ ad_cnt=1
+ for adp in ad_paths:
+ if adp != _ad:
+ sys.path.insert(ad_cnt, adp)
+ ad_cnt+=1
+
+ ad_paths.append(_ad) # for get_module_path
+
+
+def get_module_path(module, downloaded=False):
+ """Return the path of the given module.
+
+ Search the addons paths and return the first path where the given
+ module is found. If downloaded is True, return the default addons
+ path if nothing else is found.
+
+ """
+ initialize_sys_path()
+ for adp in ad_paths:
+ if os.path.exists(opj(adp, module)) or os.path.exists(opj(adp, '%s.zip' % module)):
+ return opj(adp, module)
+
+ if downloaded:
+ return opj(_ad, module)
+ logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: module not found' % (module,))
+ return False
+
+
+def get_module_filetree(module, dir='.'):
+ path = get_module_path(module)
+ if not path:
+ return False
+
+ dir = os.path.normpath(dir)
+ if dir == '.':
+ dir = ''
+ if dir.startswith('..') or (dir and dir[0] == '/'):
+ raise Exception('Cannot access file outside the module')
+
+ if not os.path.isdir(path):
+ # zipmodule
+ zip = zipfile.ZipFile(path + ".zip")
+ files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()]
+ else:
+ files = osutil.listdir(path, True)
+
+ tree = {}
+ for f in files:
+ if not f.startswith(dir):
+ continue
+
+ if dir:
+ f = f[len(dir)+int(not dir.endswith('/')):]
+ lst = f.split(os.sep)
+ current = tree
+ while len(lst) != 1:
+ current = current.setdefault(lst.pop(0), {})
+ current[lst.pop(0)] = None
+
+ return tree
+
+def zip_directory(directory, b64enc=True, src=True):
+ """Compress a directory
+
+ @param directory: The directory to compress
+ @param base64enc: if True the function will encode the zip file with base64
+ @param src: Integrate the source files
+
+ @return: a string containing the zip file
+ """
+
+ RE_exclude = re.compile('(?:^\..+\.swp$)|(?:\.py[oc]$)|(?:\.bak$)|(?:\.~.~$)', re.I)
+
+ def _zippy(archive, path, src=True):
+ path = os.path.abspath(path)
+ base = os.path.basename(path)
+ for f in osutil.listdir(path, True):
+ bf = os.path.basename(f)
+ if not RE_exclude.search(bf) and (src or bf in ('__openerp__.py', '__terp__.py') or not bf.endswith('.py')):
+ archive.write(os.path.join(path, f), os.path.join(base, f))
+
+ archname = StringIO()
+ archive = PyZipFile(archname, "w", ZIP_DEFLATED)
+
+ # for Python 2.5, ZipFile.write() still expects 8-bit strings (2.6 converts to utf-8)
+ directory = tools.ustr(directory).encode('utf-8')
+
+ archive.writepy(directory)
+ _zippy(archive, directory, src=src)
+ archive.close()
+ archive_data = archname.getvalue()
+ archname.close()
+
+ if b64enc:
+ return base64.encodestring(archive_data)
+
+ return archive_data
+
+def get_module_as_zip(modulename, b64enc=True, src=True):
+ """Generate a module as zip file with the source or not and can do a base64 encoding
+
+ @param modulename: The module name
+ @param b64enc: if True the function will encode the zip file with base64
+ @param src: Integrate the source files
+
+ @return: a stream to store in a file-like object
+ """
+
+ ap = get_module_path(str(modulename))
+ if not ap:
+ raise Exception('Unable to find path for module %s' % modulename)
+
+ ap = ap.encode('utf8')
+ if os.path.isfile(ap + '.zip'):
+ val = file(ap + '.zip', 'rb').read()
+ if b64enc:
+ val = base64.encodestring(val)
+ else:
+ val = zip_directory(ap, b64enc, src)
+
+ return val
+
+
+def get_module_resource(module, *args):
+ """Return the full path of a resource of the given module.
+
+ @param module: the module
+ @param args: the resource path components
+
+ @return: absolute path to the resource
+
+ TODO name it get_resource_path
+ TODO make it available inside on osv object (self.get_resource_path)
+ """
+ a = get_module_path(module)
+ if not a: return False
+ resource_path = opj(a, *args)
+ if zipfile.is_zipfile( a +'.zip') :
+ zip = zipfile.ZipFile( a + ".zip")
+ files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()]
+ resource_path = '/'.join(args)
+ if resource_path in files:
+ return opj(a, resource_path)
+ elif os.path.exists(resource_path):
+ return resource_path
+ return False
+
+
+def load_information_from_description_file(module):
+ """
+ :param module: The name of the module (sale, purchase, ...)
+ """
+
+ terp_file = get_module_resource(module, '__openerp__.py')
+ if not terp_file:
+ terp_file = get_module_resource(module, '__terp__.py')
+ mod_path = get_module_path(module)
+ if terp_file:
+ info = {}
+ if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path+'.zip'):
+ terp_f = tools.file_open(terp_file)
+ try:
+ info = eval(terp_f.read())
+ except Exception:
+ logger.notifyChannel('modules', netsvc.LOG_ERROR,
+ 'module %s: exception while evaluating file %s' %
+ (module, terp_file))
+ raise
+ finally:
+ terp_f.close()
+ # TODO the version should probably be mandatory
+ info.setdefault('version', '0')
+ info.setdefault('category', 'Uncategorized')
+ info.setdefault('depends', [])
+ info.setdefault('author', '')
+ info.setdefault('website', '')
+ info.setdefault('name', False)
+ info.setdefault('description', '')
+ info['certificate'] = info.get('certificate') or None
+ info['web'] = info.get('web') or False
+ info['license'] = info.get('license') or 'AGPL-3'
+ info.setdefault('installable', True)
+ info.setdefault('active', False)
+ for kind in ['data', 'demo', 'test',
+ 'init_xml', 'update_xml', 'demo_xml']:
+ info.setdefault(kind, [])
+ return info
+
+ #TODO: refactor the logger in this file to follow the logging guidelines
+ # for 6.0
+ logging.getLogger('modules').debug('module %s: no descriptor file'
+ ' found: __openerp__.py or __terp__.py (deprecated)', module)
+ return {}
+
+
+def init_module_models(cr, module_name, obj_list):
+ """ Initialize a list of models.
+
+ Call _auto_init and init on each model to create or update the
+ database tables supporting the models.
+
+ TODO better explanation of _auto_init and init.
+
+ """
+
+ logger.notifyChannel('init', netsvc.LOG_INFO,
+ 'module %s: creating or updating database tables' % module_name)
+ # TODO _auto_init doesn't seem to return anything
+ # so this todo list would be useless.
+ todo = []
+ for obj in obj_list:
+ try:
+ # TODO the module in the context doesn't seem usefull:
+ # it is available (at least) in the class' _module attribute.
+ # (So module_name would be useless too.)
+ result = obj._auto_init(cr, {'module': module_name})
+ except Exception, e:
+ raise
+ if result:
+ todo += result
+ if hasattr(obj, 'init'):
+ obj.init(cr)
+ cr.commit()
+ todo.sort()
+ for t in todo:
+ t[1](cr, *t[2])
+ cr.commit()
+
+
+def load_module(module_name):
+ """ Load a Python module found on the addons paths."""
+ fm = imp.find_module(module_name, ad_paths)
+ try:
+ imp.load_module(module_name, *fm)
+ finally:
+ if fm[0]:
+ fm[0].close()
+
+
+def register_module_classes(m):
+ """ Register module named m, if not already registered.
+
+ This will load the module and register all of its models. (Actually, the
+ explicit constructor call of each of the models inside the module will
+ register them.)
+
+ """
+
+ def log(e):
+ mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or ''
+ msg = "Couldn't load %smodule %s" % (mt, m)
+ logger.notifyChannel('init', netsvc.LOG_CRITICAL, msg)
+ logger.notifyChannel('init', netsvc.LOG_CRITICAL, e)
+
+ global loaded
+ if m in loaded:
+ return
+ logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: registering objects' % m)
+ mod_path = get_module_path(m)
+
+ initialize_sys_path()
+ try:
+ zip_mod_path = mod_path + '.zip'
+ if not os.path.isfile(zip_mod_path):
+ load_module(m)
+ else:
+ zimp = zipimport.zipimporter(zip_mod_path)
+ zimp.load_module(m)
+ except Exception, e:
+ log(e)
+ raise
+ else:
+ loaded.append(m)
+
+
+def get_modules():
+ """Returns the list of module names
+ """
+ def listdir(dir):
+ def clean(name):
+ name = os.path.basename(name)
+ if name[-4:] == '.zip':
+ name = name[:-4]
+ return name
+
+ def is_really_module(name):
+ name = opj(dir, name)
+ return os.path.isdir(name) or zipfile.is_zipfile(name)
+ return map(clean, filter(is_really_module, os.listdir(dir)))
+
+ plist = []
+ initialize_sys_path()
+ for ad in ad_paths:
+ plist.extend(listdir(ad))
+ return list(set(plist))
+
+
+def get_modules_with_version():
+ modules = get_modules()
+ res = {}
+ for module in modules:
+ try:
+ info = load_information_from_description_file(module)
+ res[module] = "%s.%s" % (release.major_version, info['version'])
+ except Exception, e:
+ continue
+ return res
+
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== added file 'openerp/modules/registry.py'
--- openerp/modules/registry.py 1970-01-01 00:00:00 +0000
+++ openerp/modules/registry.py 2011-05-19 09:56:30 +0000
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# OpenERP, Open Source Management Solution
+# Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+""" Models registries.
+
+"""
+
+import openerp.sql_db
+
+
+class BoundRegistry(object):
+ """ Model registry/database connection pair."""
+ def __init__(self, db, registry):
+ self.db = db
+ self.registry = registry
+
+
+class RegistryManager(object):
+ """ Model registries manager.
+
+ The manager is responsible for creation and deletion of bound model
+ registries (essentially database connection/model registry pairs).
+
+ """
+
+ # TODO maybe should receive the addons paths
+ def __init__(self):
+ # Mapping between db name and bound model registry.
+ # Accessed through the methods below.
+ self.bound_registries = {}
+
+
+ def get(self, db_name, force_demo=False, status=None, update_module=False,
+ pooljobs=True):
+ """ Return a bound registry for a given database name."""
+
+ if db_name in self.bound_registries:
+ bound_registry = self.bound_registries[db_name]
+ else:
+ bound_registry = self.new(db_name, force_demo, status,
+ update_module, pooljobs)
+ return bound_registry
+
+
+ def new(self, db_name, force_demo=False, status=None,
+ update_module=False, pooljobs=True):
+ """ Create and return a new bound registry for a given database name.
+
+ The (possibly) previous bound registry for that database name is
+ discarded.
+
+ """
+
+ import openerp.modules
+ import openerp.osv.osv as osv_osv
+ db = openerp.sql_db.db_connect(db_name)
+ pool = osv_osv.osv_pool()
+
+ # Initializing a registry will call general code which will in turn
+ # call registries.get (this object) to obtain the registry being
+ # initialized. Make it available in the bound_registries dictionary
+ # then remove it if an exception is raised.
+ self.delete(db_name)
+ bound_registry = BoundRegistry(db, pool)
+ self.bound_registries[db_name] = bound_registry
+ try:
+ # This should be a method on BoundRegistry
+ openerp.modules.load_modules(db, force_demo, status, update_module)
+ except Exception:
+ del self.bound_registries[db_name]
+ raise
+
+ cr = db.cursor()
+ try:
+ pool.init_set(cr, False)
+ pool.get('ir.actions.report.xml').register_all(cr)
+ cr.commit()
+ finally:
+ cr.close()
+
+ if pooljobs:
+ pool.get('ir.cron').restart(db.dbname)
+
+ return bound_registry
+
+
+ def delete(self, db_name):
+ if db_name in self.bound_registries:
+ del self.bound_registries[db_name]
+
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/netsvc.py'
--- openerp/netsvc.py 2011-05-13 13:35:14 +0000
+++ openerp/netsvc.py 2011-05-19 09:56:30 +0000
@@ -213,6 +213,19 @@
logger.addHandler(handler)
logger.setLevel(int(tools.config['log_level'] or '0'))
+# A alternative logging scheme for automated runs of the
+# server intended to test it.
+def init_alternative_logger():
+ class H(logging.Handler):
+ def emit(self, record):
+ if record.levelno > 20:
+ print record.levelno, record.pathname, record.msg
+ handler = H()
+ logger = logging.getLogger()
+ logger.handlers = []
+ logger.addHandler(handler)
+ logger.setLevel(logging.ERROR)
+
class Agent(object):
"""Singleton that keeps track of cancellable tasks to run at a given
timestamp.
@@ -277,12 +290,13 @@
time.sleep(1)
time.sleep(60)
-agent_runner = threading.Thread(target=Agent.runner, name="netsvc.Agent.runner")
-# the agent runner is a typical daemon thread, that will never quit and must be
-# terminated when the main process exits - with no consequence (the processing
-# threads it spawns are not marked daemon)
-agent_runner.setDaemon(True)
-agent_runner.start()
+def start_agent():
+ agent_runner = threading.Thread(target=Agent.runner, name="netsvc.Agent.runner")
+ # the agent runner is a typical daemon thread, that will never quit and must be
+ # terminated when the main process exits - with no consequence (the processing
+ # threads it spawns are not marked daemon)
+ agent_runner.setDaemon(True)
+ agent_runner.start()
import traceback
=== modified file 'openerp/osv/orm.py'
--- openerp/osv/orm.py 2011-05-16 15:05:34 +0000
+++ openerp/osv/orm.py 2011-05-19 09:56:30 +0000
@@ -63,6 +63,36 @@
from openerp.tools import SKIPPED_ELEMENT_TYPES
regex_order = re.compile('^(([a-z0-9_]+|"[a-z0-9_]+")( *desc| *asc)?( *, *|))+$', re.I)
+regex_object_name = re.compile(r'^[a-z0-9_.]+$')
+
+def check_object_name(name):
+ """ Check if the given name is a valid openerp object name.
+
+ The _name attribute in osv and osv_memory object is subject to
+ some restrictions. This function returns True or False whether
+ the given name is allowed or not.
+
+ TODO: this is an approximation. The goal in this approximation
+ is to disallow uppercase characters (in some places, we quote
+ table/column names and in other not, which leads to this kind
+ of errors:
+
+ psycopg2.ProgrammingError: relation "xxx" does not exist).
+
+ The same restriction should apply to both osv and osv_memory
+ objects for consistency.
+
+ """
+ if regex_object_name.match(name) is None:
+ return False
+ return True
+
+def raise_on_invalid_object_name(name):
+ if not check_object_name(name):
+ msg = "The _name attribute %s is not valid." % name
+ logger = netsvc.Logger()
+ logger.notifyChannel('orm', netsvc.LOG_ERROR, msg)
+ raise except_orm('ValueError', msg)
POSTGRES_CONFDELTYPES = {
'RESTRICT': 'r',
@@ -508,6 +538,7 @@
cr.commit()
def _auto_init(self, cr, context=None):
+ raise_on_invalid_object_name(self._name)
self._field_create(cr, context=context)
def __init__(self, cr):
@@ -2363,6 +2394,7 @@
self._table, column['attname'])
def _auto_init(self, cr, context=None):
+ raise_on_invalid_object_name(self._name)
if context is None:
context = {}
store_compute = False
=== modified file 'openerp/osv/osv.py'
--- openerp/osv/osv.py 2011-04-29 08:51:45 +0000
+++ openerp/osv/osv.py 2011-05-19 09:56:30 +0000
@@ -23,18 +23,20 @@
# OSV: Objects Services
#
+import sys
+import inspect
import orm
import openerp.netsvc as netsvc
import openerp.pooler as pooler
+import openerp.sql_db as sql_db
import copy
import logging
from psycopg2 import IntegrityError, errorcodes
from openerp.tools.func import wraps
from openerp.tools.translate import translate
-module_list = []
+# Mapping between openerp module names and their osv classes.
module_class_list = {}
-class_pool = {}
class except_osv(Exception):
def __init__(self, name, value, exc_type='warning'):
@@ -92,7 +94,7 @@
ids = args[3]
else:
ids = []
- cr = pooler.get_db_only(dbname).cursor()
+ cr = sql_db.db_connect(db_name).cursor()
return src(obj, cr, uid, ids, context=(ctx or {}))
except Exception:
pass
@@ -103,7 +105,7 @@
# be returned, it is the best we have.
try:
- cr = pooler.get_db_only(dbname).cursor()
+ cr = sql_db.db_connect(db_name).cursor()
res = translate(cr, name=False, source_type=ttype,
lang=lang, source=src)
if res:
@@ -202,19 +204,22 @@
cr.close()
return res
-object_proxy()
class osv_pool(object):
+ """ Model registry for a particular database.
+
+ The registry is essentially a mapping between model names and model
+ instances. There is one registry instance per database.
+
+ """
+
def __init__(self):
self._ready = False
- self.obj_pool = {}
- self.module_object_list = {}
- self.created = []
+ self.obj_pool = {} # model name/model instance mapping
self._sql_error = {}
self._store_function = {}
self._init = True
self._init_parent = {}
- self.logger = logging.getLogger("pool")
def init_set(self, cr, mode):
different = mode != self._init
@@ -229,46 +234,74 @@
self._ready = True
return different
-
def obj_list(self):
+ """ Return the list of model names in this registry."""
return self.obj_pool.keys()
- # adds a new object instance to the object pool.
- # if it already existed, the instance is replaced
- def add(self, name, obj_inst):
- if name in self.obj_pool:
- del self.obj_pool[name]
- self.obj_pool[name] = obj_inst
- module = obj_inst.__class__.__module__.split('.')[0]
- self.module_object_list.setdefault(module, []).append(obj_inst)
+ def add(self, model_name, model):
+ """ Add or replace a model in the registry."""
+ self.obj_pool[model_name] = model
- # Return None if object does not exist
def get(self, name):
- obj = self.obj_pool.get(name, None)
- return obj
+ """ Return a model for a given name or None if it doesn't exist."""
+ return self.obj_pool.get(name)
#TODO: pass a list of modules to load
def instanciate(self, module, cr):
+ """ Instanciate all the classes of a given module for a particular db."""
+
res = []
- class_list = module_class_list.get(module, [])
- for klass in class_list:
+
+ # instanciate classes registered through their constructor
+ for klass in module_class_list.get(module, []):
res.append(klass.createInstance(self, module, cr))
+
return res
class osv_base(object):
+ """ Base class for openerp models.
+
+ OpenERP models are created by inheriting from this class (although
+ not directly; more specifically by inheriting from osv or
+ osv_memory). The constructor is called once, usually directly
+ after the class definition, e.g.:
+
+ class user(osv):
+ ...
+ user()
+
+ The system will later instanciate the class once per database (on
+ which the class' module is installed).
+
+ """
+
def __init__(self, pool, cr):
+ """ Initialize a model and make it part of the given registry."""
pool.add(self._name, self)
self.pool = pool
super(osv_base, self).__init__(cr)
def __new__(cls):
+ """ Register this model.
+
+ This doesn't create an instance but simply register the model
+ as being part of the module where it is defined.
+
+ TODO make it possible to not even have to call the constructor
+ to be registered.
+
+ """
+
+ # Set the module name (e.g. base, sale, accounting, ...) on the class.
module = cls.__module__.split('.')[0]
if not hasattr(cls, '_module'):
cls._module = module
+
+ # Remember which models to instanciate for this module.
module_class_list.setdefault(cls._module, []).append(cls)
- class_pool[cls._name] = cls
- if module not in module_list:
- module_list.append(cls._module)
+
+ # Since we don't return an instance here, the __init__
+ # method won't be called.
return None
class osv_memory(osv_base, orm.orm_memory):
@@ -355,5 +388,8 @@
return obj
createInstance = classmethod(createInstance)
+def start_object_proxy():
+ object_proxy()
+
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/pooler.py'
--- openerp/pooler.py 2011-02-07 12:57:23 +0000
+++ openerp/pooler.py 2011-05-19 09:56:30 +0000
@@ -19,64 +19,50 @@
#
##############################################################################
-pool_dic = {}
+""" Functions kept for backward compatibility.
+
+ They are simple wrappers around a global RegistryManager methods.
+
+"""
+
+from openerp.modules.registry import RegistryManager
+
+_Registries = None
+
+
+def ensure_registries():
+ global _Registries
+ if _Registries is None:
+ _Registries = RegistryManager()
+
def get_db_and_pool(db_name, force_demo=False, status=None, update_module=False, pooljobs=True):
- if not status:
- status={}
-
- db = get_db_only(db_name)
-
- if db_name in pool_dic:
- pool = pool_dic[db_name]
- else:
- import openerp.addons as addons
- import openerp.osv.osv as osv_osv
- pool = osv_osv.osv_pool()
- pool_dic[db_name] = pool
-
- try:
- addons.load_modules(db, force_demo, status, update_module)
- except Exception:
- del pool_dic[db_name]
- raise
-
- cr = db.cursor()
- try:
- pool.init_set(cr, False)
- pool.get('ir.actions.report.xml').register_all(cr)
- cr.commit()
- finally:
- cr.close()
-
- if pooljobs:
- pool.get('ir.cron').restart(db.dbname)
- return db, pool
+ """Create and return a database connection and a newly initialized registry."""
+ ensure_registries()
+ bound_registry = _Registries.get(db_name, force_demo, status, update_module, pooljobs)
+ return bound_registry.db, bound_registry.registry
+
+
+def delete_pool(db_name):
+ """Delete an existing registry."""
+ ensure_registries()
+ _Registries.delete(db_name)
def restart_pool(db_name, force_demo=False, status=None, update_module=False):
- if db_name in pool_dic:
- del pool_dic[db_name]
- return get_db_and_pool(db_name, force_demo, status, update_module=update_module)
-
-
-def get_db_only(db_name):
- # ATTENTION:
- # do not put this import outside this function
- # sql_db must not be loaded before the logger is initialized.
- # sql_db import psycopg2.tool which create a default logger if there is not.
- # this resulting of having the logs outputed twice...
- import openerp.sql_db as sql_db
- db = sql_db.db_connect(db_name)
- return db
+ """Delete an existing registry and return a database connection and a newly initialized registry."""
+ ensure_registries()
+ bound_registry = _Registries.new(db_name, force_demo, status, update_module, True)
+ return bound_registry.db, bound_registry.registry
def get_db(db_name):
+ """Return a database connection. The corresponding registry is initialized."""
return get_db_and_pool(db_name)[0]
def get_pool(db_name, force_demo=False, status=None, update_module=False):
- pool = get_db_and_pool(db_name, force_demo, status, update_module)[1]
- return pool
+ """Return a model registry."""
+ return get_db_and_pool(db_name, force_demo, status, update_module)[1]
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/report/interface.py'
--- openerp/report/interface.py 2011-05-12 12:52:39 +0000
+++ openerp/report/interface.py 2011-05-19 09:56:30 +0000
@@ -27,7 +27,7 @@
import openerp.pooler as pooler
import openerp.tools as tools
-import openerp.addons as addons
+import openerp.modules
import print_xml
import render
import urllib
@@ -119,7 +119,7 @@
pos_xml = i.end()
doc = print_xml.document(cr, uid, {}, {})
- tmpl_path = addons.get_module_resource('base', 'report', 'corporate_defaults.xml')
+ tmpl_path = openerp.modules.get_module_resource('base', 'report', 'corporate_defaults.xml')
doc.parse(tmpl_path, [uid], 'res.users', context)
corporate_header = doc.xml_get()
doc.close()
=== modified file 'openerp/report/render/__init__.py'
--- openerp/report/render/__init__.py 2010-12-14 16:28:26 +0000
+++ openerp/report/render/__init__.py 2011-05-19 09:56:30 +0000
@@ -27,7 +27,7 @@
import Image
except ImportError:
import logging
- logging.getLogger('init').warning('Python Imaging not installed, you can use only .JPG pictures !')
+ logging.warning('Python Imaging not installed, you can use only .JPG pictures !')
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/service/__init__.py'
--- openerp/service/__init__.py 2009-10-20 10:52:23 +0000
+++ openerp/service/__init__.py 2011-05-19 09:56:30 +0000
@@ -19,6 +19,8 @@
#
##############################################################################
+import http_server
+import netrpc_server
import web_services
=== modified file 'openerp/service/web_services.py'
--- openerp/service/web_services.py 2011-02-07 12:57:23 +0000
+++ openerp/service/web_services.py 2011-05-19 09:56:30 +0000
@@ -28,13 +28,14 @@
import sys
import platform
from openerp.tools.translate import _
-import openerp.addons as addons
+import openerp.modules
import openerp.ir
import openerp.netsvc as netsvc
import openerp.pooler as pooler
import openerp.release as release
import openerp.sql_db as sql_db
import openerp.tools as tools
+import openerp.modules
import locale
import logging
from cStringIO import StringIO
@@ -91,12 +92,7 @@
cr = None
try:
serv.actions[id]['progress'] = 0
- cr = sql_db.db_connect(db_name).cursor()
- tools.init_db(cr)
tools.config['lang'] = lang
- cr.commit()
- cr.close()
- cr = None
pool = pooler.restart_pool(db_name, demo, serv.actions[id],
update_module=True)[1]
@@ -139,7 +135,7 @@
def exp_get_progress(self, id):
if self.actions[id]['thread'].isAlive():
-# return addons.init_progress[db_name]
+# return openerp.modules.init_progress[db_name]
return (min(self.actions[id].get('progress', 0),0.95), [])
else:
clean = self.actions[id]['clean']
@@ -154,6 +150,7 @@
def exp_drop(self, db_name):
sql_db.close_db(db_name)
+ openerp.netsvc.Agent.cancel(db_name)
logger = netsvc.Logger()
db = sql_db.db_connect('template1')
@@ -256,6 +253,7 @@
def exp_rename(self, old_name, new_name):
sql_db.close_db(old_name)
+ openerp.netsvc.Agent.cancel(db_name)
logger = netsvc.Logger()
db = sql_db.db_connect('template1')
@@ -346,7 +344,6 @@
l.notifyChannel('web-services', netsvc.LOG_ERROR, tb_s)
raise
return True
-db()
class _ObjectService(netsvc.ExportService):
"A common base class for those who have fn(db, uid, password,...) "
@@ -447,7 +444,7 @@
if not rc.id:
raise tm.RemoteContractException('This contract does not exist or is not active')
- return rc.get_available_updates(rc.id, addons.get_modules_with_version())
+ return rc.get_available_updates(rc.id, openerp.modules.get_modules_with_version())
except tm.RemoteContractException, e:
self.abortResponse(1, 'Migration Error', 'warning', str(e))
@@ -465,7 +462,7 @@
l.notifyChannel('migration', netsvc.LOG_INFO, 'starting migration with contract %s' % (rc.name,))
- zips = rc.retrieve_updates(rc.id, addons.get_modules_with_version())
+ zips = rc.retrieve_updates(rc.id, openerp.modules.get_modules_with_version())
from shutil import rmtree, copytree, copy
@@ -477,7 +474,7 @@
for module in zips:
l.notifyChannel('migration', netsvc.LOG_INFO, 'upgrade module %s' % (module,))
- mp = addons.get_module_path(module)
+ mp = openerp.modules.get_module_path(module)
if mp:
if os.path.isdir(mp):
copytree(mp, os.path.join(backup_directory, module))
@@ -578,7 +575,6 @@
logger.warning("Counters of SQL will not be reliable unless DEBUG_SQL is set at the server's config.")
return sql_db.sql_counter
-common()
class objects_proxy(netsvc.ExportService):
def __init__(self, name="object"):
@@ -602,8 +598,6 @@
def new_dispatch(self,method,auth,params):
pass
-objects_proxy()
-
#
# Wizard ID: 1
@@ -664,7 +658,6 @@
raise Exception, 'AccessDenied'
else:
raise Exception, 'WizardNotFound'
-wizard()
#
# TODO: set a maximum report number per user to avoid DOS attacks
@@ -778,7 +771,13 @@
else:
raise Exception, 'ReportNotFound'
-report_spool()
+
+def start_web_services():
+ db()
+ common()
+ objects_proxy()
+ wizard()
+ report_spool()
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/sql_db.py'
--- openerp/sql_db.py 2011-05-12 12:52:39 +0000
+++ openerp/sql_db.py 2011-05-19 09:56:30 +0000
@@ -54,7 +54,6 @@
import tools
from tools.func import wraps, frame_codeinfo
-from netsvc import Agent
from datetime import datetime as mdt
from datetime import timedelta
import threading
@@ -68,17 +67,22 @@
class Cursor(object):
IN_MAX = 1000 # decent limit on size of IN queries - guideline = Oracle limit
- __logger = logging.getLogger('db.cursor')
+ __logger = None
def check(f):
@wraps(f)
def wrapper(self, *args, **kwargs):
if self.__closed:
- raise psycopg2.OperationalError('Unable to use the cursor after having closed it')
+ msg = 'Unable to use a closed cursor.'
+ if self.__closer:
+ msg += ' It was closed at %s, line %s' % self.__closer
+ raise psycopg2.OperationalError(msg)
return f(self, *args, **kwargs)
return wrapper
def __init__(self, pool, dbname, serialized=False):
+ if self.__class__.__logger is None:
+ self.__class__.__logger = logging.getLogger('db.cursor')
self.sql_from_log = {}
self.sql_into_log = {}
@@ -100,6 +104,7 @@
self.__caller = frame_codeinfo(currentframe(),2)
else:
self.__caller = False
+ self.__closer = False
def __del__(self):
if not self.__closed:
@@ -198,6 +203,8 @@
if not self._obj:
return
+ if self.sql_log:
+ self.__closer = frame_codeinfo(currentframe(),3)
self.print_log()
if not self._serialized:
@@ -375,15 +382,18 @@
return key(first) == key(second)
-_Pool = ConnectionPool(int(tools.config['db_maxconn']))
+_Pool = None
def db_connect(db_name):
+ global _Pool
+ if _Pool is None:
+ _Pool = ConnectionPool(int(tools.config['db_maxconn']))
currentThread().dbname = db_name
return Connection(_Pool, db_name)
def close_db(db_name):
+ """ You might want to call openerp.netsvc.Agent.cancel(db_name) along this function."""
_Pool.close_all(dsn(db_name))
- Agent.cancel(db_name)
tools.cache.clean_caches_for_db(db_name)
ct = currentThread()
if hasattr(ct, 'dbname'):
=== modified file 'openerp/tools/config.py'
--- openerp/tools/config.py 2011-05-12 13:45:53 +0000
+++ openerp/tools/config.py 2011-05-19 09:56:30 +0000
@@ -39,67 +39,21 @@
class configmanager(object):
def __init__(self, fname=None):
+ # Options not exposed on the command line. Command line options will be added
+ # from optparse's parser.
self.options = {
- 'email_from':False,
- 'xmlrpc_interface': '', # this will bind the server to all interfaces
- 'xmlrpc_port': 8069,
- 'netrpc_interface': '',
- 'netrpc_port': 8070,
- 'xmlrpcs_interface': '', # this will bind the server to all interfaces
- 'xmlrpcs_port': 8071,
- 'db_host': False,
- 'db_port': False,
- 'db_name': False,
- 'db_user': False,
- 'db_password': False,
- 'db_maxconn': 64,
- 'reportgz': False,
- 'netrpc': True,
- 'xmlrpc': True,
- 'xmlrpcs': True,
- 'translate_in': None,
- 'translate_out': None,
- 'overwrite_existing_translations': False,
- 'load_language': None,
- 'language': None,
- 'pg_path': None,
'admin_passwd': 'admin',
'csv_internal_sep': ',',
- 'addons_path': None,
- 'root_path': None,
- 'debug_mode': False,
- 'import_partial': "",
- 'pidfile': None,
- 'logfile': None,
- 'logrotate': True,
- 'smtp_server': 'localhost',
- 'smtp_user': False,
- 'smtp_port':25,
- 'smtp_ssl':False,
- 'smtp_password': False,
- 'stop_after_init': False, # this will stop the server after initialization
- 'syslog' : False,
- 'log_level': logging.INFO,
- 'assert_exit_level': logging.ERROR, # level above which a failed assert will be raised
- 'cache_timeout': 100000,
'login_message': False,
- 'list_db' : True,
- 'timezone' : False, # to override the default TZ
- 'test_file' : False,
- 'test_report_directory' : False,
- 'test_disable' : False,
- 'test_commit' : False,
- 'static_http_enable': False,
- 'static_http_document_root': None,
- 'static_http_url_prefix': None,
- 'secure_cert_file': 'server.cert',
- 'secure_pkey_file': 'server.pkey',
'publisher_warranty_url': 'http://services.openerp.com/publisher-warranty/',
- 'osv_memory_count_limit': None, # number of records in each osv_memory virtual table
- 'osv_memory_age_limit': 1, # hours
+ 'reportgz': False,
+ 'root_path': None,
}
- self.blacklist_for_save = set(["publisher_warranty_url", "load_language", "root_path"])
+ # Not exposed in the configuration file.
+ self.blacklist_for_save = set(
+ ['publisher_warranty_url', 'load_language', 'root_path',
+ 'init', 'save', 'config', 'update'])
self.misc = {}
self.config_file = fname
@@ -122,15 +76,18 @@
group.add_option("--without-demo", dest="without_demo",
help="disable loading demo data for modules to be installed (comma-separated, use \"all\" for all modules). Requires -d and -i. Default is %default",
default=False)
- group.add_option("-P", "--import-partial", dest="import_partial",
- help="Use this for big data importation, if it crashes you will be able to continue at the current state. Provide a filename to store intermediate importation states.", default=False)
+ group.add_option("-P", "--import-partial", dest="import_partial", default='',
+ help="Use this for big data importation, if it crashes you will be able to continue at the current state. Provide a filename to store intermediate importation states.")
group.add_option("--pidfile", dest="pidfile", help="file where the server pid will be stored")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "XML-RPC Configuration")
- group.add_option("--xmlrpc-interface", dest="xmlrpc_interface", help="specify the TCP IP address for the XML-RPC protocol")
- group.add_option("--xmlrpc-port", dest="xmlrpc_port", help="specify the TCP port for the XML-RPC protocol", type="int")
- group.add_option("--no-xmlrpc", dest="xmlrpc", action="store_false", help="disable the XML-RPC protocol")
+ group.add_option("--xmlrpc-interface", dest="xmlrpc_interface", default='',
+ help="Specify the TCP IP address for the XML-RPC protocol. The empty string binds to all interfaces.")
+ group.add_option("--xmlrpc-port", dest="xmlrpc_port", default=8069,
+ help="specify the TCP port for the XML-RPC protocol", type="int")
+ group.add_option("--no-xmlrpc", dest="xmlrpc", action="store_false", default=True,
+ help="disable the XML-RPC protocol")
parser.add_option_group(group)
title = "XML-RPC Secure Configuration"
@@ -138,18 +95,26 @@
title += " (disabled as ssl is unavailable)"
group = optparse.OptionGroup(parser, title)
- group.add_option("--xmlrpcs-interface", dest="xmlrpcs_interface", help="specify the TCP IP address for the XML-RPC Secure protocol")
- group.add_option("--xmlrpcs-port", dest="xmlrpcs_port", help="specify the TCP port for the XML-RPC Secure protocol", type="int")
- group.add_option("--no-xmlrpcs", dest="xmlrpcs", action="store_false", help="disable the XML-RPC Secure protocol")
- group.add_option("--cert-file", dest="secure_cert_file", help="specify the certificate file for the SSL connection")
- group.add_option("--pkey-file", dest="secure_pkey_file", help="specify the private key file for the SSL connection")
+ group.add_option("--xmlrpcs-interface", dest="xmlrpcs_interface", default='',
+ help="Specify the TCP IP address for the XML-RPC Secure protocol. The empty string binds to all interfaces.")
+ group.add_option("--xmlrpcs-port", dest="xmlrpcs_port", default=8071,
+ help="specify the TCP port for the XML-RPC Secure protocol", type="int")
+ group.add_option("--no-xmlrpcs", dest="xmlrpcs", action="store_false", default=True,
+ help="disable the XML-RPC Secure protocol")
+ group.add_option("--cert-file", dest="secure_cert_file", default='server.cert',
+ help="specify the certificate file for the SSL connection")
+ group.add_option("--pkey-file", dest="secure_pkey_file", default='server.pkey',
+ help="specify the private key file for the SSL connection")
parser.add_option_group(group)
# NET-RPC
group = optparse.OptionGroup(parser, "NET-RPC Configuration")
- group.add_option("--netrpc-interface", dest="netrpc_interface", help="specify the TCP IP address for the NETRPC protocol")
- group.add_option("--netrpc-port", dest="netrpc_port", help="specify the TCP port for the NETRPC protocol", type="int")
- group.add_option("--no-netrpc", dest="netrpc", action="store_false", help="disable the NETRPC protocol")
+ group.add_option("--netrpc-interface", dest="netrpc_interface", default='',
+ help="specify the TCP IP address for the NETRPC protocol")
+ group.add_option("--netrpc-port", dest="netrpc_port", default=8070,
+ help="specify the TCP port for the NETRPC protocol", type="int")
+ group.add_option("--no-netrpc", dest="netrpc", action="store_false", default=True,
+ help="disable the NETRPC protocol")
parser.add_option_group(group)
# Static HTTP
@@ -161,45 +126,60 @@
# Testing Group
group = optparse.OptionGroup(parser, "Testing Configuration")
- group.add_option("--test-file", dest="test_file", help="Launch a YML test file.")
- group.add_option("--test-report-directory", dest="test_report_directory", help="If set, will save sample of all reports in this directory.")
+ group.add_option("--test-file", dest="test_file", default=False,
+ help="Launch a YML test file.")
+ group.add_option("--test-report-directory", dest="test_report_directory", default=False,
+ help="If set, will save sample of all reports in this directory.")
group.add_option("--test-disable", action="store_true", dest="test_disable",
default=False, help="Disable loading test files.")
group.add_option("--test-commit", action="store_true", dest="test_commit",
default=False, help="Commit database changes performed by tests.")
group.add_option("--assert-exit-level", dest='assert_exit_level', type="choice", choices=self._LOGLEVELS.keys(),
- help="specify the level at which a failed assertion will stop the server. Accepted values: %s" % (self._LOGLEVELS.keys(),))
+ default='error',
+ help="specify the level at which a failed assertion will stop the server. Accepted values: %s" % (self._LOGLEVELS.keys(),))
parser.add_option_group(group)
# Logging Group
group = optparse.OptionGroup(parser, "Logging Configuration")
group.add_option("--logfile", dest="logfile", help="file where the server log will be stored")
- group.add_option("--no-logrotate", dest="logrotate", action="store_false",
+ group.add_option("--no-logrotate", dest="logrotate", action="store_false", default=True,
help="do not rotate the logfile")
group.add_option("--syslog", action="store_true", dest="syslog",
default=False, help="Send the log to the syslog server")
group.add_option('--log-level', dest='log_level', type='choice', choices=self._LOGLEVELS.keys(),
+ default='info',
help='specify the level of the logging. Accepted values: ' + str(self._LOGLEVELS.keys()))
parser.add_option_group(group)
# SMTP Group
group = optparse.OptionGroup(parser, "SMTP Configuration")
- group.add_option('--email-from', dest='email_from', help='specify the SMTP email address for sending email')
- group.add_option('--smtp', dest='smtp_server', help='specify the SMTP server for sending email')
- group.add_option('--smtp-port', dest='smtp_port', help='specify the SMTP port', type="int")
- group.add_option('--smtp-ssl', dest='smtp_ssl', action='store_true', help='specify the SMTP server support SSL or not')
- group.add_option('--smtp-user', dest='smtp_user', help='specify the SMTP username for sending email')
- group.add_option('--smtp-password', dest='smtp_password', help='specify the SMTP password for sending email')
+ group.add_option('--email-from', dest='email_from', default=False,
+ help='specify the SMTP email address for sending email')
+ group.add_option('--smtp', dest='smtp_server', default='localhost',
+ help='specify the SMTP server for sending email')
+ group.add_option('--smtp-port', dest='smtp_port', default=25,
+ help='specify the SMTP port', type="int")
+ group.add_option('--smtp-ssl', dest='smtp_ssl', action='store_true', default=False,
+ help='specify the SMTP server support SSL or not')
+ group.add_option('--smtp-user', dest='smtp_user', default=False,
+ help='specify the SMTP username for sending email')
+ group.add_option('--smtp-password', dest='smtp_password', default=False,
+ help='specify the SMTP password for sending email')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Database related options")
- group.add_option("-d", "--database", dest="db_name", help="specify the database name")
- group.add_option("-r", "--db_user", dest="db_user", help="specify the database user name")
- group.add_option("-w", "--db_password", dest="db_password", help="specify the database password")
+ group.add_option("-d", "--database", dest="db_name", default=False,
+ help="specify the database name")
+ group.add_option("-r", "--db_user", dest="db_user", default=False,
+ help="specify the database user name")
+ group.add_option("-w", "--db_password", dest="db_password", default=False,
+ help="specify the database password")
group.add_option("--pg_path", dest="pg_path", help="specify the pg executable path")
- group.add_option("--db_host", dest="db_host", help="specify the database host")
- group.add_option("--db_port", dest="db_port", help="specify the database port", type="int")
- group.add_option("--db_maxconn", dest="db_maxconn", type='int',
+ group.add_option("--db_host", dest="db_host", default=False,
+ help="specify the database host")
+ group.add_option("--db_port", dest="db_port", default=False,
+ help="specify the database port", type="int")
+ group.add_option("--db_maxconn", dest="db_maxconn", type='int', default=64,
help="specify the the maximum number of physical connections to posgresql")
parser.add_option_group(group)
@@ -211,6 +191,7 @@
group.add_option('--load-language', dest="load_language",
help="specifies the languages for the translations you want to be loaded")
group.add_option('-l', "--language", dest="language",
+ default=None,
help="specify the language of the translation file. Use it with --i18n-export or --i18n-import")
group.add_option("--i18n-export", dest="translate_out",
help="export all sentences to be translated to a CSV file, a PO file or a TGZ archive and exit")
@@ -226,17 +207,19 @@
parser.add_option_group(group)
security = optparse.OptionGroup(parser, 'Security-related options')
- security.add_option('--no-database-list', action="store_false", dest='list_db', help="disable the ability to return the list of databases")
+ security.add_option('--no-database-list', action="store_false", dest='list_db', default=True,
+ help="disable the ability to return the list of databases")
parser.add_option_group(security)
# Advanced options
group = optparse.OptionGroup(parser, "Advanced options")
- group.add_option("--cache-timeout", dest="cache_timeout",
+ group.add_option("--cache-timeout", dest="cache_timeout", default=100000,
help="set the timeout for the cache system", type="int")
group.add_option('--debug', dest='debug_mode', action='store_true', default=False, help='enable debug mode')
group.add_option("--stop-after-init", action="store_true", dest="stop_after_init", default=False,
- help="stop the server after it initializes")
- group.add_option("-t", "--timezone", dest="timezone", help="specify reference timezone for the server (e.g. Europe/Brussels")
+ help="stop the server after its initialization")
+ group.add_option("-t", "--timezone", dest="timezone", default=False,
+ help="specify reference timezone for the server (e.g. Europe/Brussels")
group.add_option("--osv-memory-count-limit", dest="osv_memory_count_limit", default=False,
help="Force a limit on the maximum number of records kept in the virtual "
"osv_memory tables. The default is False, which means no count-based limit.",
@@ -248,6 +231,14 @@
type="float")
parser.add_option_group(group)
+ # Copy all optparse options into self.options.
+ for group in parser.option_groups:
+ for option in group.option_list:
+ if option.default == ('NO', 'DEFAULT'):
+ self.options[option.dest] = None
+ else:
+ self.options[option.dest] = option.default
+
self.parse_config()
def parse_config(self, args=[]):
=== modified file 'openerp/tools/misc.py'
--- openerp/tools/misc.py 2011-05-09 09:44:48 +0000
+++ openerp/tools/misc.py 2011-05-19 09:56:30 +0000
@@ -69,76 +69,6 @@
# We include the *Base ones just in case, currently they seem to be subclasses of the _* ones.
SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase)
-# initialize a database with base/base.sql
-def init_db(cr):
- import openerp.addons as addons
- f = addons.get_module_resource('base', 'base.sql')
- base_sql_file = file_open(f)
- try:
- cr.execute(base_sql_file.read())
- cr.commit()
- finally:
- base_sql_file.close()
-
- for i in addons.get_modules():
- mod_path = addons.get_module_path(i)
- if not mod_path:
- continue
-
- info = addons.load_information_from_description_file(i)
-
- if not info:
- continue
- categs = info.get('category', 'Uncategorized').split('/')
- p_id = None
- while categs:
- if p_id is not None:
- cr.execute('SELECT id \
- FROM ir_module_category \
- WHERE name=%s AND parent_id=%s', (categs[0], p_id))
- else:
- cr.execute('SELECT id \
- FROM ir_module_category \
- WHERE name=%s AND parent_id IS NULL', (categs[0],))
- c_id = cr.fetchone()
- if not c_id:
- cr.execute('INSERT INTO ir_module_category \
- (name, parent_id) \
- VALUES (%s, %s) RETURNING id', (categs[0], p_id))
- c_id = cr.fetchone()[0]
- else:
- c_id = c_id[0]
- p_id = c_id
- categs = categs[1:]
-
- active = info.get('active', False)
- installable = info.get('installable', True)
- if installable:
- if active:
- state = 'to install'
- else:
- state = 'uninstalled'
- else:
- state = 'uninstallable'
- cr.execute('INSERT INTO ir_module_module \
- (author, website, name, shortdesc, description, \
- category_id, state, certificate, web, license) \
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id', (
- info.get('author', ''),
- info.get('website', ''), i, info.get('name', False),
- info.get('description', ''), p_id, state, info.get('certificate') or None,
- info.get('web') or False,
- info.get('license') or 'AGPL-3'))
- id = cr.fetchone()[0]
- cr.execute('INSERT INTO ir_model_data \
- (name,model,module, res_id, noupdate) VALUES (%s,%s,%s,%s,%s)', (
- 'module_meta_information', 'ir.module.module', i, id, True))
- dependencies = info.get('depends', [])
- for d in dependencies:
- cr.execute('INSERT INTO ir_module_module_dependency \
- (module_id,name) VALUES (%s, %s)', (id, d))
- cr.commit()
-
def find_in_path(name):
try:
return which(name)
@@ -204,8 +134,8 @@
@return: fileobject if pathinfo is False else (fileobject, filepath)
"""
- import openerp.addons as addons
- adps = addons.ad_paths
+ import openerp.modules as addons
+ adps = addons.module.ad_paths
rtp = os.path.normcase(os.path.abspath(config['root_path']))
if name.replace(os.path.sep, '/').startswith('addons/'):
@@ -717,7 +647,28 @@
except TypeError:
return False
-class cache(object):
+class dummy_cache(object):
+ """ Cache decorator replacement to actually do no caching.
+
+ This can be useful to benchmark and/or track memory leak.
+
+ """
+
+ def __init__(self, timeout=None, skiparg=2, multi=None, size=8192):
+ pass
+
+ def clear(self, dbname, *args, **kwargs):
+ pass
+
+ @classmethod
+ def clean_caches_for_db(cls, dbname):
+ pass
+
+ def __call__(self, fn):
+ fn.clear_cache = self.clear
+ return fn
+
+class real_cache(object):
"""
Use it as a decorator of the function you plan to cache
Timeout: 0 = no timeout, otherwise in seconds
@@ -842,6 +793,9 @@
cached_result.clear_cache = self.clear
return cached_result
+# TODO make it an option
+cache = real_cache
+
def to_xml(s):
return s.replace('&','&').replace('<','<').replace('>','>')
=== modified file 'openerp/tools/translate.py'
--- openerp/tools/translate.py 2011-04-21 09:34:00 +0000
+++ openerp/tools/translate.py 2011-05-19 09:56:30 +0000
@@ -27,6 +27,7 @@
import locale
import os
import openerp.pooler as pooler
+import openerp.sql_db as sql_db
import re
import logging
import tarfile
@@ -160,7 +161,7 @@
# find current DB based on thread/worker db name (see netsvc)
db_name = getattr(threading.currentThread(), 'dbname', None)
if db_name:
- return pooler.get_db_only(db_name)
+ return sql_db.db_connect(db_name)
def _get_cr(self, frame):
is_new_cr = False
=== modified file 'openerp/tools/yaml_import.py'
--- openerp/tools/yaml_import.py 2011-05-13 12:52:18 +0000
+++ openerp/tools/yaml_import.py 2011-05-19 09:56:30 +0000
@@ -792,7 +792,7 @@
is_preceded_by_comment = False
return is_preceded_by_comment
-def yaml_import(cr, module, yamlfile, idref=None, mode='init', noupdate=False, report=None):
+def yaml_import(cr, module, yamlfile, idref=None, mode='init', noupdate=False):
if idref is None:
idref = {}
yaml_string = yamlfile.read()
=== modified file 'tools/module_graph.py'
--- tools/module_graph.py 2010-05-20 14:37:04 +0000
+++ tools/module_graph.py 2011-05-19 09:56:30 +0000
@@ -27,6 +27,7 @@
import sys
import glob
+# TODO use the same function provided in openerp.modules
def load_information_from_description_file(module):
"""
:param module: The name of the module (sale, purchase, ...)
Follow ups
