openlp-core team mailing list archive
-
openlp-core team
-
Mailing list archive
-
Message #30204
[Merge] lp:~phill-ridout/openlp/more-bible-refactors into lp:openlp
Phill has proposed merging lp:~phill-ridout/openlp/more-bible-refactors into lp:openlp.
Requested reviews:
Tim Bentley (trb143)
For more details, see:
https://code.launchpad.net/~phill-ridout/openlp/more-bible-refactors/+merge/302478
Some more bible refactors. Mainly focusing on xml parsing and the csv importer
lp:~phill-ridout/openlp/more-bible-refactors (revision 2699)
[SUCCESS] https://ci.openlp.io/job/Branch-01-Pull/1708/
[SUCCESS] https://ci.openlp.io/job/Branch-02-Functional-Tests/1619/
[SUCCESS] https://ci.openlp.io/job/Branch-03-Interface-Tests/1557/
[SUCCESS] https://ci.openlp.io/job/Branch-04a-Windows_Functional_Tests/1319/
[SUCCESS] https://ci.openlp.io/job/Branch-04b-Windows_Interface_Tests/909/
[SUCCESS] https://ci.openlp.io/job/Branch-05a-Code_Analysis/977/
[SUCCESS] https://ci.openlp.io/job/Branch-05b-Test_Coverage/845/
[SUCCESS] https://ci.openlp.io/job/Branch-05c-Code_Analysis2/23/
--
Your team OpenLP Core is subscribed to branch lp:openlp.
=== modified file 'openlp/core/common/__init__.py'
--- openlp/core/common/__init__.py 2016-07-08 19:19:08 +0000
+++ openlp/core/common/__init__.py 2016-08-09 21:01:19 +0000
@@ -30,6 +30,7 @@
import re
import sys
import traceback
+from chardet.universaldetector import UniversalDetector
from ipaddress import IPv4Address, IPv6Address, AddressValueError
from shutil import which
from subprocess import check_output, CalledProcessError, STDOUT
@@ -416,3 +417,24 @@
runlog = ''
log.debug('check_output returned: {text}'.format(text=runlog))
return runlog
+
+
+def get_file_encoding(filename):
+ """
+ Utility function to incrementally detect the file encoding.
+
+ :param filename: Filename for the file to determine the encoding for. Str
+ :return: A dict with the keys 'encoding' and 'confidence'
+ """
+ detector = UniversalDetector()
+ try:
+ with open(filename, 'rb') as detect_file:
+ while not detector.done:
+ chunk = detect_file.read(1024)
+ if not chunk:
+ break
+ detector.feed(chunk)
+ detector.close()
+ return detector.result
+ except OSError:
+ log.exception('Error detecting file encoding')
=== modified file 'openlp/core/lib/exceptions.py'
--- openlp/core/lib/exceptions.py 2015-12-31 22:46:06 +0000
+++ openlp/core/lib/exceptions.py 2016-08-09 21:01:19 +0000
@@ -24,9 +24,15 @@
"""
+# TODO: Test __init__ & __str__
class ValidationError(Exception):
"""
The :class:`~openlp.core.lib.exceptions.ValidationError` exception provides a custom exception for validating
import files.
"""
- pass
+
+ def __init__(self, msg="Validation Error"):
+ self.msg = msg
+
+ def __str__(self):
+ return '{error_message}'.format(error_message=self.msg)
=== modified file 'openlp/plugins/bibles/bibleplugin.py'
--- openlp/plugins/bibles/bibleplugin.py 2016-06-05 11:03:28 +0000
+++ openlp/plugins/bibles/bibleplugin.py 2016-08-09 21:01:19 +0000
@@ -22,12 +22,9 @@
import logging
-from PyQt5 import QtWidgets
-
from openlp.core.common.actions import ActionList
from openlp.core.lib import Plugin, StringContent, build_icon, translate
from openlp.core.lib.ui import UiStrings, create_action
-from openlp.plugins.bibles.forms import BibleUpgradeForm
from openlp.plugins.bibles.lib import BibleManager, BiblesTab, BibleMediaItem, LayoutStyle, DisplayStyle, \
LanguageSelection
from openlp.plugins.bibles.lib.mediaitem import BibleSearch
@@ -90,7 +87,6 @@
action_list.add_action(self.import_bible_item, UiStrings().Import)
# Set to invisible until we can export bibles
self.export_bible_item.setVisible(False)
- self.tools_upgrade_item.setVisible(bool(self.manager.old_bible_databases))
def finalise(self):
"""
@@ -104,20 +100,6 @@
self.import_bible_item.setVisible(False)
self.export_bible_item.setVisible(False)
- def app_startup(self):
- """
- Perform tasks on application startup
- """
- super(BiblePlugin, self).app_startup()
- if self.manager.old_bible_databases:
- if QtWidgets.QMessageBox.information(
- self.main_window, translate('OpenLP', 'Information'),
- translate('OpenLP', 'Bible format has changed.\nYou have to upgrade your '
- 'existing Bibles.\nShould OpenLP upgrade now?'),
- QtWidgets.QMessageBox.StandardButtons(QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No)) == \
- QtWidgets.QMessageBox.Yes:
- self.on_tools_upgrade_item_triggered()
-
def add_import_menu_item(self, import_menu):
"""
Add an import menu item
@@ -139,30 +121,6 @@
text=translate('BiblesPlugin', '&Bible'), visible=False)
export_menu.addAction(self.export_bible_item)
- def add_tools_menu_item(self, tools_menu):
- """
- Give the bible plugin the opportunity to add items to the **Tools** menu.
-
- :param tools_menu: The actual **Tools** menu item, so that your actions can use it as their parent.
- """
- log.debug('add tools menu')
- self.tools_upgrade_item = create_action(
- tools_menu, 'toolsUpgradeItem',
- text=translate('BiblesPlugin', '&Upgrade older Bibles'),
- statustip=translate('BiblesPlugin', 'Upgrade the Bible databases to the latest format.'),
- visible=False, triggers=self.on_tools_upgrade_item_triggered)
- tools_menu.addAction(self.tools_upgrade_item)
-
- def on_tools_upgrade_item_triggered(self):
- """
- Upgrade older bible databases.
- """
- if not hasattr(self, 'upgrade_wizard'):
- self.upgrade_wizard = BibleUpgradeForm(self.main_window, self.manager, self)
- # If the import was not cancelled then reload.
- if self.upgrade_wizard.exec():
- self.media_item.reload_bibles()
-
def on_bible_import_click(self):
"""
Show the Bible Import wizard
=== modified file 'openlp/plugins/bibles/forms/__init__.py'
--- openlp/plugins/bibles/forms/__init__.py 2015-12-31 22:46:06 +0000
+++ openlp/plugins/bibles/forms/__init__.py 2016-08-09 21:01:19 +0000
@@ -21,30 +21,12 @@
###############################################################################
"""
-Forms in OpenLP are made up of two classes. One class holds all the graphical elements, like buttons and lists, and the
-other class holds all the functional code, like slots and loading and saving.
-
-The first class, commonly known as the **Dialog** class, is typically named ``Ui_<name>Dialog``. It is a slightly
-modified version of the class that the ``pyuic5`` command produces from Qt5's .ui file. Typical modifications will be
-converting most strings from "" to '' and using OpenLP's ``translate()`` function for translating strings.
-
-The second class, commonly known as the **Form** class, is typically named ``<name>Form``. This class is the one which
-is instantiated and used. It uses dual inheritance to inherit from (usually) QtWidgets.QDialog and the Ui class
-mentioned above, like so::
-
- class BibleImportForm(QtWidgets.QWizard, Ui_BibleImportWizard):
-
- def __init__(self, parent, manager, bible_plugin):
- super(BibleImportForm, self).__init__(parent)
- self.setupUi(self)
-
-This allows OpenLP to use ``self.object`` for all the GUI elements while keeping them separate from the functionality,
-so that it is easier to recreate the GUI from the .ui files later if necessary.
+The :mod:`forms` module contains all the ui functionality for the bibles
+plugin.
"""
from .booknameform import BookNameForm
from .languageform import LanguageForm
from .bibleimportform import BibleImportForm
-from .bibleupgradeform import BibleUpgradeForm
from .editbibleform import EditBibleForm
-__all__ = ['BookNameForm', 'LanguageForm', 'BibleImportForm', 'BibleUpgradeForm', 'EditBibleForm']
+__all__ = ['BookNameForm', 'LanguageForm', 'BibleImportForm', 'EditBibleForm']
=== modified file 'openlp/plugins/bibles/forms/bibleimportform.py'
--- openlp/plugins/bibles/forms/bibleimportform.py 2016-05-21 08:31:24 +0000
+++ openlp/plugins/bibles/forms/bibleimportform.py 2016-08-09 21:01:19 +0000
@@ -40,7 +40,7 @@
from openlp.core.common.languagemanager import get_locale_key
from openlp.plugins.bibles.lib.manager import BibleFormat
from openlp.plugins.bibles.lib.db import clean_filename
-from openlp.plugins.bibles.lib.http import CWExtract, BGExtract, BSExtract
+from openlp.plugins.bibles.lib.importers.http import CWExtract, BGExtract, BSExtract
log = logging.getLogger(__name__)
=== removed file 'openlp/plugins/bibles/forms/bibleupgradeform.py'
--- openlp/plugins/bibles/forms/bibleupgradeform.py 2016-05-21 08:31:24 +0000
+++ openlp/plugins/bibles/forms/bibleupgradeform.py 1970-01-01 00:00:00 +0000
@@ -1,575 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
-
-###############################################################################
-# OpenLP - Open Source Lyrics Projection #
-# --------------------------------------------------------------------------- #
-# Copyright (c) 2008-2016 OpenLP Developers #
-# --------------------------------------------------------------------------- #
-# This program is free software; you can redistribute it and/or modify it #
-# under the terms of the GNU General Public License as published by the Free #
-# Software Foundation; version 2 of the License. #
-# #
-# This program is distributed in the hope that it will be useful, but WITHOUT #
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
-# more details. #
-# #
-# You should have received a copy of the GNU General Public License along #
-# with this program; if not, write to the Free Software Foundation, Inc., 59 #
-# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
-###############################################################################
-"""
-The bible import functions for OpenLP
-"""
-import logging
-import os
-import shutil
-from tempfile import gettempdir
-
-from PyQt5 import QtCore, QtWidgets
-
-from openlp.core.common import Registry, AppLocation, UiStrings, Settings, check_directory_exists, translate, \
- delete_file
-from openlp.core.lib.ui import critical_error_message_box
-from openlp.core.ui.lib.wizard import OpenLPWizard, WizardStrings
-from openlp.plugins.bibles.lib.db import BibleDB, BibleMeta, OldBibleDB, BiblesResourcesDB
-from openlp.plugins.bibles.lib.http import BSExtract, BGExtract, CWExtract
-
-log = logging.getLogger(__name__)
-
-
-class BibleUpgradeForm(OpenLPWizard):
- """
- This is the Bible Upgrade Wizard, which allows easy importing of Bibles into OpenLP from older OpenLP2 database
- versions.
- """
- log.info('BibleUpgradeForm loaded')
-
- def __init__(self, parent, manager, bible_plugin):
- """
- Instantiate the wizard, and run any extra setup we need to.
-
- :param parent: The QWidget-derived parent of the wizard.
- :param manager: The Bible manager.
- :param bible_plugin: The Bible plugin.
- """
- self.manager = manager
- self.media_item = bible_plugin.media_item
- self.suffix = '.sqlite'
- self.settings_section = 'bibles'
- self.path = AppLocation.get_section_data_path(self.settings_section)
- self.temp_dir = os.path.join(gettempdir(), 'openlp')
- self.files = self.manager.old_bible_databases
- self.success = {}
- self.new_bibles = {}
- super(BibleUpgradeForm, self).__init__(
- parent, bible_plugin, 'bibleUpgradeWizard', ':/wizards/wizard_importbible.bmp')
-
- def setupUi(self, image):
- """
- Set up the UI for the bible wizard.
- """
- super(BibleUpgradeForm, self).setupUi(image)
- Registry().register_function('openlp_stop_wizard', self.stop_import)
-
- def stop_import(self):
- """
- Stops the import of the Bible.
- """
- log.debug('Stopping import')
- self.stop_import_flag = True
-
- def reject(self):
- """
- Stop the wizard on cancel button, close button or ESC key.
- """
- log.debug('Wizard cancelled by user')
- self.stop_import_flag = True
- if not self.currentPage() == self.progress_page:
- self.done(QtWidgets.QDialog.Rejected)
-
- def onCurrentIdChanged(self, page_id):
- """
- Perform necessary functions depending on which wizard page is active.
- """
- if self.page(page_id) == self.progress_page:
- self.pre_wizard()
- self.perform_wizard()
- self.post_wizard()
- elif self.page(page_id) == self.selectPage and not self.files:
- self.next()
-
- def on_backup_browse_button_clicked(self):
- """
- Show the file open dialog for the OSIS file.
- """
- filename = QtWidgets.QFileDialog.getExistingDirectory(self, translate('BiblesPlugin.UpgradeWizardForm',
- 'Select a Backup Directory'), '')
- if filename:
- self.backupDirectoryEdit.setText(filename)
-
- def on_no_backup_check_box_toggled(self, checked):
- """
- Enable or disable the backup directory widgets.
- """
- self.backupDirectoryEdit.setEnabled(not checked)
- self.backupBrowseButton.setEnabled(not checked)
-
- def backup_old_bibles(self, backup_directory):
- """
- Backup old bible databases in a given folder.
- """
- check_directory_exists(backup_directory)
- success = True
- for filename in self.files:
- try:
- shutil.copy(os.path.join(self.path, filename[0]), backup_directory)
- except:
- success = False
- return success
-
- def custom_init(self):
- """
- Perform any custom initialisation for bible upgrading.
- """
- self.manager.set_process_dialog(self)
- self.restart()
-
- def custom_signals(self):
- """
- Set up the signals used in the bible importer.
- """
- self.backupBrowseButton.clicked.connect(self.on_backup_browse_button_clicked)
- self.noBackupCheckBox.toggled.connect(self.on_no_backup_check_box_toggled)
-
- def add_custom_pages(self):
- """
- Add the bible import specific wizard pages.
- """
- # Backup Page
- self.backup_page = QtWidgets.QWizardPage()
- self.backup_page.setObjectName('BackupPage')
- self.backupLayout = QtWidgets.QVBoxLayout(self.backup_page)
- self.backupLayout.setObjectName('BackupLayout')
- self.backupInfoLabel = QtWidgets.QLabel(self.backup_page)
- self.backupInfoLabel.setOpenExternalLinks(True)
- self.backupInfoLabel.setTextFormat(QtCore.Qt.RichText)
- self.backupInfoLabel.setWordWrap(True)
- self.backupInfoLabel.setObjectName('backupInfoLabel')
- self.backupLayout.addWidget(self.backupInfoLabel)
- self.selectLabel = QtWidgets.QLabel(self.backup_page)
- self.selectLabel.setObjectName('select_label')
- self.backupLayout.addWidget(self.selectLabel)
- self.formLayout = QtWidgets.QFormLayout()
- self.formLayout.setContentsMargins(0, 0, 0, 0)
- self.formLayout.setObjectName('FormLayout')
- self.backupDirectoryLabel = QtWidgets.QLabel(self.backup_page)
- self.backupDirectoryLabel.setObjectName('backupDirectoryLabel')
- self.backupDirectoryLayout = QtWidgets.QHBoxLayout()
- self.backupDirectoryLayout.setObjectName('BackupDirectoryLayout')
- self.backupDirectoryEdit = QtWidgets.QLineEdit(self.backup_page)
- self.backupDirectoryEdit.setObjectName('BackupFolderEdit')
- self.backupDirectoryLayout.addWidget(self.backupDirectoryEdit)
- self.backupBrowseButton = QtWidgets.QToolButton(self.backup_page)
- self.backupBrowseButton.setIcon(self.open_icon)
- self.backupBrowseButton.setObjectName('BackupBrowseButton')
- self.backupDirectoryLayout.addWidget(self.backupBrowseButton)
- self.formLayout.addRow(self.backupDirectoryLabel, self.backupDirectoryLayout)
- self.backupLayout.addLayout(self.formLayout)
- self.noBackupCheckBox = QtWidgets.QCheckBox(self.backup_page)
- self.noBackupCheckBox.setObjectName('NoBackupCheckBox')
- self.backupLayout.addWidget(self.noBackupCheckBox)
- self.spacer = QtWidgets.QSpacerItem(10, 0, QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Minimum)
- self.backupLayout.addItem(self.spacer)
- self.addPage(self.backup_page)
- # Select Page
- self.selectPage = QtWidgets.QWizardPage()
- self.selectPage.setObjectName('SelectPage')
- self.pageLayout = QtWidgets.QVBoxLayout(self.selectPage)
- self.pageLayout.setObjectName('pageLayout')
- self.scrollArea = QtWidgets.QScrollArea(self.selectPage)
- self.scrollArea.setWidgetResizable(True)
- self.scrollArea.setObjectName('scrollArea')
- self.scrollArea.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
- self.scrollAreaContents = QtWidgets.QWidget(self.scrollArea)
- self.scrollAreaContents.setObjectName('scrollAreaContents')
- self.formLayout = QtWidgets.QVBoxLayout(self.scrollAreaContents)
- self.formLayout.setSpacing(2)
- self.formLayout.setObjectName('formLayout')
- self.addScrollArea()
- self.pageLayout.addWidget(self.scrollArea)
- self.addPage(self.selectPage)
-
- def addScrollArea(self):
- """
- Add the content to the scrollArea.
- """
- self.checkBox = {}
- for number, filename in enumerate(self.files):
- bible = OldBibleDB(self.media_item, path=self.path, file=filename[0])
- self.checkBox[number] = QtWidgets.QCheckBox(self.scrollAreaContents)
- self.checkBox[number].setObjectName('checkBox[{count:d}]'.format(count=number))
- self.checkBox[number].setText(bible.get_name())
- self.checkBox[number].setCheckState(QtCore.Qt.Checked)
- self.formLayout.addWidget(self.checkBox[number])
- self.spacer_item = QtWidgets.QSpacerItem(20, 5, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding)
- self.formLayout.addItem(self.spacer_item)
- self.scrollArea.setWidget(self.scrollAreaContents)
-
- def clearScrollArea(self):
- """
- Remove the content from the scrollArea.
- """
- for number, filename in enumerate(self.files):
- self.formLayout.removeWidget(self.checkBox[number])
- self.checkBox[number].setParent(None)
- self.formLayout.removeItem(self.spacer_item)
-
- def retranslateUi(self):
- """
- Allow for localisation of the bible import wizard.
- """
- self.setWindowTitle(translate('BiblesPlugin.UpgradeWizardForm', 'Bible Upgrade Wizard'))
- self.title_label.setText(WizardStrings.HeaderStyle % translate('OpenLP.Ui',
- 'Welcome to the Bible Upgrade Wizard'))
- self.information_label.setText(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'This wizard will help you to upgrade your existing Bibles from a prior version of OpenLP 2. '
- 'Click the next button below to start the upgrade process.'))
- self.backup_page.setTitle(translate('BiblesPlugin.UpgradeWizardForm', 'Select Backup Directory'))
- self.backup_page.setSubTitle(
- translate('BiblesPlugin.UpgradeWizardForm', 'Please select a backup directory for your Bibles'))
- self.backupInfoLabel.setText(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Previous releases of OpenLP 2.0 are unable to use upgraded Bibles.'
- ' This will create a backup of your current Bibles so that you can '
- 'simply copy the files back to your OpenLP data directory if you '
- 'need to revert to a previous release of OpenLP. Instructions on '
- 'how to restore the files can be found in our <a href="'
- 'http://wiki.openlp.org/faq";>Frequently Asked Questions</a>.'))
- self.selectLabel.setText(
- translate('BiblesPlugin.UpgradeWizardForm', 'Please select a backup location for your Bibles.'))
- self.backupDirectoryLabel.setText(translate('BiblesPlugin.UpgradeWizardForm', 'Backup Directory:'))
- self.noBackupCheckBox.setText(
- translate('BiblesPlugin.UpgradeWizardForm', 'There is no need to backup my Bibles'))
- self.selectPage.setTitle(translate('BiblesPlugin.UpgradeWizardForm', 'Select Bibles'))
- self.selectPage.setSubTitle(
- translate('BiblesPlugin.UpgradeWizardForm', 'Please select the Bibles to upgrade'))
- self.progress_page.setTitle(translate('BiblesPlugin.UpgradeWizardForm', 'Upgrading'))
- self.progress_page.setSubTitle(
- translate('BiblesPlugin.UpgradeWizardForm', 'Please wait while your Bibles are upgraded.'))
- self.progress_label.setText(WizardStrings.Ready)
- self.progress_bar.setFormat('%p%')
-
- def validateCurrentPage(self):
- """
- Validate the current page before moving on to the next page.
- """
- if self.currentPage() == self.welcome_page:
- return True
- elif self.currentPage() == self.backup_page:
- if not self.noBackupCheckBox.checkState() == QtCore.Qt.Checked:
- backup_path = self.backupDirectoryEdit.text()
- if not backup_path:
- critical_error_message_box(
- UiStrings().EmptyField,
- translate('BiblesPlugin.UpgradeWizardForm', 'You need to specify a backup directory for '
- 'your Bibles.'))
- self.backupDirectoryEdit.setFocus()
- return False
- else:
- if not self.backup_old_bibles(backup_path):
- critical_error_message_box(
- UiStrings().Error,
- translate('BiblesPlugin.UpgradeWizardForm', 'The backup was not successful.\nTo backup '
- 'your Bibles you need permission to write to the given directory.'))
- return False
- return True
- elif self.currentPage() == self.selectPage:
- check_directory_exists(self.temp_dir)
- for number, filename in enumerate(self.files):
- if not self.checkBox[number].checkState() == QtCore.Qt.Checked:
- continue
- # Move bibles to temp dir.
- if not os.path.exists(os.path.join(self.temp_dir, filename[0])):
- shutil.move(os.path.join(self.path, filename[0]), self.temp_dir)
- else:
- delete_file(os.path.join(self.path, filename[0]))
- return True
- if self.currentPage() == self.progress_page:
- return True
-
- def set_defaults(self):
- """
- Set default values for the wizard pages.
- """
- log.debug('BibleUpgrade set_defaults')
- settings = Settings()
- settings.beginGroup(self.plugin.settings_section)
- self.stop_import_flag = False
- self.success.clear()
- self.new_bibles.clear()
- self.clearScrollArea()
- self.files = self.manager.old_bible_databases
- self.addScrollArea()
- self.retranslateUi()
- for number, filename in enumerate(self.files):
- self.checkBox[number].setCheckState(QtCore.Qt.Checked)
- self.progress_bar.show()
- self.restart()
- self.finish_button.setVisible(False)
- self.cancel_button.setVisible(True)
- settings.endGroup()
-
- def pre_wizard(self):
- """
- Prepare the UI for the upgrade.
- """
- super(BibleUpgradeForm, self).pre_wizard()
- self.progress_label.setText(translate('BiblesPlugin.UpgradeWizardForm', 'Starting upgrade...'))
- self.application.process_events()
-
- def perform_wizard(self):
- """
- Perform the actual upgrade.
- """
- self.includeWebBible = False
- proxy_server = None
- if not self.files:
- self.progress_label.setText(
- translate('BiblesPlugin.UpgradeWizardForm', 'There are no Bibles that need to be upgraded.'))
- self.progress_bar.hide()
- return
- max_bibles = 0
- for number, file in enumerate(self.files):
- if self.checkBox[number].checkState() == QtCore.Qt.Checked:
- max_bibles += 1
- old_bible = None
- for number, filename in enumerate(self.files):
- # Close the previous bible's connection.
- if old_bible is not None:
- old_bible.close_connection()
- # Set to None to make obvious that we have already closed the
- # database.
- old_bible = None
- if self.stop_import_flag:
- self.success[number] = False
- break
- if not self.checkBox[number].checkState() == QtCore.Qt.Checked:
- self.success[number] = False
- continue
- self.progress_bar.reset()
- old_bible = OldBibleDB(self.media_item, path=self.temp_dir, file=filename[0])
- name = filename[1]
- self.progress_label.setText(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible {count} of {total}: "{name}"\n'
- 'Upgrading ...').format(count=number + 1,
- total=max_bibles,
- name=name))
- self.new_bibles[number] = BibleDB(self.media_item, path=self.path, name=name, file=filename[0])
- self.new_bibles[number].register(self.plugin.upgrade_wizard)
- metadata = old_bible.get_metadata()
- web_bible = False
- meta_data = {}
- for meta in metadata:
- # Upgrade the names of the metadata keys
- if meta['key'] == 'Version':
- meta['key'] = 'name'
- if meta['key'] == 'Bookname language':
- meta['key'] = 'book_name_language'
- meta['key'] = meta['key'].lower().replace(' ', '_')
- # Copy the metadata
- meta_data[meta['key']] = meta['value']
- if meta['key'] != 'name' and meta['key'] != 'dbversion':
- self.new_bibles[number].save_meta(meta['key'], meta['value'])
- if meta['key'] == 'download_source':
- web_bible = True
- self.includeWebBible = True
- proxy_server = meta.get('proxy_server')
- if web_bible:
- if meta_data['download_source'].lower() == 'crosswalk':
- handler = CWExtract(proxy_server)
- elif meta_data['download_source'].lower() == 'biblegateway':
- handler = BGExtract(proxy_server)
- elif meta_data['download_source'].lower() == 'bibleserver':
- handler = BSExtract(proxy_server)
- books = handler.get_books_from_http(meta_data['download_name'])
- if not books:
- log.error('Upgrading books from {uri} - '
- 'download name: "{name}" failed'.format(uri=meta_data['download_source'],
- name=meta_data['download_name']))
- self.new_bibles[number].session.close()
- del self.new_bibles[number]
- critical_error_message_box(
- translate('BiblesPlugin.UpgradeWizardForm', 'Download Error'),
- translate('BiblesPlugin.UpgradeWizardForm',
- 'To upgrade your Web Bibles an Internet connection is required.'))
- text = translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible {count} of {total}: "{name}"\n'
- 'Failed').format(count=number + 1, total=max_bibles, name=name)
- self.increment_progress_bar(text, self.progress_bar.maximum() - self.progress_bar.value())
- self.success[number] = False
- continue
- bible = BiblesResourcesDB.get_webbible(
- meta_data['download_name'],
- meta_data['download_source'].lower())
- if bible and bible['language_id']:
- language_id = bible['language_id']
- self.new_bibles[number].save_meta('language_id', language_id)
- else:
- language_id = self.new_bibles[number].get_language(name)
- if not language_id:
- log.warning('Upgrading from "{name}" failed'.format(name=filename[0]))
- self.new_bibles[number].session.close()
- del self.new_bibles[number]
- self.increment_progress_bar(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible {count} of {total}: "{name}"\n'
- 'Failed').format(count=number + 1, total=max_bibles, name=name),
- self.progress_bar.maximum() - self.progress_bar.value())
- self.success[number] = False
- continue
- self.progress_bar.setMaximum(len(books))
- for book in books:
- if self.stop_import_flag:
- self.success[number] = False
- break
- self.increment_progress_bar(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible {count} of {total}: "{name}"\n'
- 'Upgrading {book} ...').format(count=number + 1, total=max_bibles,
- name=name, book=book))
- book_ref_id = self.new_bibles[number].\
- get_book_ref_id_by_name(book, len(books), language_id)
- if not book_ref_id:
- log.warning('Upgrading books from {source} - download name: "{name}" '
- 'aborted by user'.format(source=meta_data['download_source'],
- name=meta_data['download_name']))
- self.new_bibles[number].session.close()
- del self.new_bibles[number]
- self.success[number] = False
- break
- book_details = BiblesResourcesDB.get_book_by_id(book_ref_id)
- db_book = self.new_bibles[number].create_book(book, book_ref_id, book_details['testament_id'])
- # Try to import already downloaded verses.
- oldbook = old_bible.get_book(book)
- if oldbook:
- verses = old_bible.get_verses(oldbook['id'])
- if not verses:
- log.warning('No verses found to import for book "{book}"'.format(book=book))
- continue
- for verse in verses:
- if self.stop_import_flag:
- self.success[number] = False
- break
- self.new_bibles[number].create_verse(db_book.id, int(verse['chapter']),
- int(verse['verse']), str(verse['text']))
- self.application.process_events()
- self.new_bibles[number].session.commit()
- else:
- language_id = self.new_bibles[number].get_object(BibleMeta, 'language_id')
- if not language_id:
- language_id = self.new_bibles[number].get_language(name)
- if not language_id:
- log.warning('Upgrading books from "{name}" failed'.format(name=name))
- self.new_bibles[number].session.close()
- del self.new_bibles[number]
- self.increment_progress_bar(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible {count} of {total}: "{name}"\n'
- 'Failed').format(count=number + 1, total=max_bibles, name=name),
- self.progress_bar.maximum() - self.progress_bar.value())
- self.success[number] = False
- continue
- books = old_bible.get_books()
- self.progress_bar.setMaximum(len(books))
- for book in books:
- if self.stop_import_flag:
- self.success[number] = False
- break
- self.increment_progress_bar(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible {count} of {total}: "{name}"\n'
- 'Upgrading {book} ...').format(count=number + 1, total=max_bibles,
- name=name, book=book['name']))
- book_ref_id = self.new_bibles[number].get_book_ref_id_by_name(book['name'], len(books), language_id)
- if not book_ref_id:
- log.warning('Upgrading books from {name} " failed - aborted by user'.format(name=name))
- self.new_bibles[number].session.close()
- del self.new_bibles[number]
- self.success[number] = False
- break
- book_details = BiblesResourcesDB.get_book_by_id(book_ref_id)
- db_book = self.new_bibles[number].create_book(book['name'], book_ref_id,
- book_details['testament_id'])
- verses = old_bible.get_verses(book['id'])
- if not verses:
- log.warning('No verses found to import for book "{book}"'.format(book=book['name']))
- self.new_bibles[number].delete_book(db_book)
- continue
- for verse in verses:
- if self.stop_import_flag:
- self.success[number] = False
- break
- self.new_bibles[number].create_verse(db_book.id, int(verse['chapter']), int(verse['verse']),
- str(verse['text']))
- self.application.process_events()
- self.new_bibles[number].session.commit()
- if not self.success.get(number, True):
- self.increment_progress_bar(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible {count} of {total}: "{name}"\n'
- 'Failed').format(count=number + 1, total=max_bibles, name=name),
- self.progress_bar.maximum() - self.progress_bar.value())
- else:
- self.success[number] = True
- self.new_bibles[number].save_meta('name', name)
- self.increment_progress_bar(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible {count} of {total}: "{name}"\n'
- 'Complete').format(count=number + 1, total=max_bibles, name=name))
- if number in self.new_bibles:
- self.new_bibles[number].session.close()
- # Close the last bible's connection if possible.
- if old_bible is not None:
- old_bible.close_connection()
-
- def post_wizard(self):
- """
- Clean up the UI after the import has finished.
- """
- successful_import = 0
- failed_import = 0
- for number, filename in enumerate(self.files):
- if self.success.get(number):
- successful_import += 1
- elif self.checkBox[number].checkState() == QtCore.Qt.Checked:
- failed_import += 1
- # Delete upgraded (but not complete, corrupted, ...) bible.
- delete_file(os.path.join(self.path, filename[0]))
- # Copy not upgraded bible back.
- shutil.move(os.path.join(self.temp_dir, filename[0]), self.path)
- if failed_import > 0:
- failed_import_text = translate('BiblesPlugin.UpgradeWizardForm',
- ', {name} failed').format(name=failed_import)
- else:
- failed_import_text = ''
- if successful_import > 0:
- if self.includeWebBible:
- self.progress_label.setText(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible(s): {count:d} successful{failed}\nPlease note that verses '
- 'from Web Bibles will be downloaded on demand and so an Internet connection is required.'
- ).format(count=successful_import, failed=failed_import_text))
- else:
- self.progress_label.setText(
- translate('BiblesPlugin.UpgradeWizardForm',
- 'Upgrading Bible(s): {count:d} successful{failed}').format(count=successful_import,
- failed=failed_import_text))
- else:
- self.progress_label.setText(translate('BiblesPlugin.UpgradeWizardForm', 'Upgrade failed.'))
- # Remove temp directory.
- shutil.rmtree(self.temp_dir, True)
- super(BibleUpgradeForm, self).post_wizard()
=== added file 'openlp/plugins/bibles/lib/bibleimport.py'
--- openlp/plugins/bibles/lib/bibleimport.py 1970-01-01 00:00:00 +0000
+++ openlp/plugins/bibles/lib/bibleimport.py 2016-08-09 21:01:19 +0000
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2016 OpenLP Developers #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it #
+# under the terms of the GNU General Public License as published by the Free #
+# Software Foundation; version 2 of the License. #
+# #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
+# more details. #
+# #
+# You should have received a copy of the GNU General Public License along #
+# with this program; if not, write to the Free Software Foundation, Inc., 59 #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
+###############################################################################
+
+import logging
+
+from lxml import etree, objectify
+
+from openlp.core.common import OpenLPMixin, languages
+from openlp.core.lib import ValidationError
+from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB
+
+log = logging.getLogger(__name__)
+
+
+class BibleImport(OpenLPMixin, BibleDB):
+ """
+ Helper class to import bibles from a third party source into OpenLP
+ """
+ # TODO: Test
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.filename = kwargs['filename'] if 'filename' in kwargs else None
+
+ def get_language_id(self, file_language=None, bible_name=None):
+ """
+ Get the language_id for the language of the bible. Fallback to user input if we cannot do this pragmatically.
+
+ :param file_language: Language of the bible. Possibly retrieved from the file being imported. Str
+ :param bible_name: Name of the bible to display on the get_language dialog. Str
+ :return: The id of a language Int or None
+ """
+ language_id = None
+ if file_language:
+ language = languages.get_language(file_language)
+ if language and language.id:
+ language_id = language.id
+ if not language_id:
+ # The language couldn't be detected, ask the user
+ language_id = self.get_language(bible_name)
+ if not language_id:
+ # User cancelled get_language dialog
+ log.error('Language detection failed when importing from "{name}". User aborted language selection.'
+ .format(name=bible_name))
+ return None
+ self.save_meta('language_id', language_id)
+ return language_id
+
+ def find_and_create_book(self, name, no_of_books, language_id, guess_id=None):
+ """
+ Find the OpenLP book id and then create the book in this bible db
+
+ :param name: Name of the book. If None, then fall back to the guess_id Str
+ :param no_of_books: The total number of books contained in this bible Int
+ :param language_id: The OpenLP id of the language of this bible Int
+ :param guess_id: The guessed id of the book, used if name is None Int
+ :return:
+ """
+ if name:
+ book_ref_id = self.get_book_ref_id_by_name(name, no_of_books, language_id)
+ else:
+ log.debug('No book name supplied. Falling back to guess_id')
+ book_ref_id = guess_id
+ if not book_ref_id:
+ raise ValidationError(msg='Could not resolve book_ref_id in "{}"'.format(self.filename))
+ book_details = BiblesResourcesDB.get_book_by_id(book_ref_id)
+ if book_details is None:
+ raise ValidationError(msg='book_ref_id: {book_ref} Could not be found in the BibleResourcesDB while '
+ 'importing {file}'.format(book_ref=book_ref_id, file=self.filename))
+ return self.create_book(name, book_ref_id, book_details['testament_id'])
+
+ @staticmethod
+ def parse_xml(filename, use_objectify=False, elements=None, tags=None):
+ """
+ Parse and clean the supplied file by removing any elements or tags we don't use.
+ :param filename: The filename of the xml file to parse. Str
+ :param use_objectify: Use the objectify parser rather than the etree parser. (Bool)
+ :param elements: A tuple of element names (Str) to remove along with their content.
+ :param tags: A tuple of element names (Str) to remove, preserving their content.
+ :return: The root element of the xml document
+ """
+ with open(filename, 'rb') as import_file:
+ # NOTE: We don't need to do any of the normal encoding detection here, because lxml does it's own encoding
+ # detection, and the two mechanisms together interfere with each other.
+ if not use_objectify:
+ tree = etree.parse(import_file, parser=etree.XMLParser(recover=True))
+ else:
+ tree = objectify.parse(import_file, parser=objectify.makeparser(recover=True))
+ if elements:
+ # Strip tags we don't use - remove content
+ etree.strip_elements(tree, elements, with_tail=False)
+ if tags:
+ # Strip tags we don't use - keep content
+ etree.strip_tags(tree, tags)
+ return tree.getroot()
=== modified file 'openlp/plugins/bibles/lib/db.py'
--- openlp/plugins/bibles/lib/db.py 2016-08-04 20:37:04 +0000
+++ openlp/plugins/bibles/lib/db.py 2016-08-09 21:01:19 +0000
@@ -477,7 +477,7 @@
combo_box = language_form.language_combo_box
language_id = combo_box.itemData(combo_box.currentIndex())
if not language_id:
- return False
+ return None
self.save_meta('language_id', language_id)
return language_id
@@ -864,138 +864,3 @@
return AlternativeBookNamesDB.run_sql(
'INSERT INTO alternative_book_names(book_reference_id, language_id, name) '
'VALUES (?, ?, ?)', (book_reference_id, language_id, name), True)
-
-
-class OldBibleDB(QtCore.QObject, Manager):
- """
- This class connects to the old bible databases to reimport them to the new
- database scheme.
- """
- cursor = None
-
- def __init__(self, parent, **kwargs):
- """
- The constructor loads up the database and creates and initialises the tables if the database doesn't exist.
-
- **Required keyword arguments:**
-
- ``path``
- The path to the bible database file.
-
- ``name``
- The name of the database. This is also used as the file name for SQLite databases.
- """
- log.info('OldBibleDB loaded')
- QtCore.QObject.__init__(self)
- if 'path' not in kwargs:
- raise KeyError('Missing keyword argument "path".')
- if 'file' not in kwargs:
- raise KeyError('Missing keyword argument "file".')
- if 'path' in kwargs:
- self.path = kwargs['path']
- if 'file' in kwargs:
- self.file = kwargs['file']
-
- def get_cursor(self):
- """
- Return the cursor object. Instantiate one if it doesn't exist yet.
- """
- if self.cursor is None:
- file_path = os.path.join(self.path, self.file)
- self.connection = sqlite3.connect(file_path)
- self.cursor = self.connection.cursor()
- return self.cursor
-
- def run_sql(self, query, parameters=()):
- """
- Run an SQL query on the database, returning the results.
-
- :param query: The actual SQL query to run.
- :param parameters: Any variable parameters to add to the query.
- """
- cursor = self.get_cursor()
- cursor.execute(query, parameters)
- return cursor.fetchall()
-
- def get_name(self):
- """
- Returns the version name of the Bible.
- """
- self.name = None
- version_name = self.run_sql('SELECT value FROM metadata WHERE key = "name"')
- if version_name:
- self.name = version_name[0][0]
- else:
- # Fallback to old way of naming
- version_name = self.run_sql('SELECT value FROM metadata WHERE key = "Version"')
- if version_name:
- self.name = version_name[0][0]
- return self.name
-
- def get_metadata(self):
- """
- Returns the metadata of the Bible.
- """
- metadata = self.run_sql('SELECT key, value FROM metadata ORDER BY rowid')
- if metadata:
- return [{
- 'key': str(meta[0]),
- 'value': str(meta[1])
- } for meta in metadata]
- else:
- return None
-
- def get_book(self, name):
- """
- Return a book by name or abbreviation.
-
- ``name``
- The name or abbreviation of the book.
- """
- if not isinstance(name, str):
- name = str(name)
- books = self.run_sql(
- 'SELECT id, testament_id, name, abbreviation FROM book WHERE LOWER(name) = ? OR '
- 'LOWER(abbreviation) = ?', (name.lower(), name.lower()))
- if books:
- return {
- 'id': books[0][0],
- 'testament_id': books[0][1],
- 'name': str(books[0][2]),
- 'abbreviation': str(books[0][3])
- }
- else:
- return None
-
- def get_books(self):
- """
- Returns the books of the Bible.
- """
- books = self.run_sql('SELECT name, id FROM book ORDER BY id')
- if books:
- return [{
- 'name': str(book[0]),
- 'id':int(book[1])
- } for book in books]
- else:
- return None
-
- def get_verses(self, book_id):
- """
- Returns the verses of the Bible.
- """
- verses = self.run_sql(
- 'SELECT book_id, chapter, verse, text FROM verse WHERE book_id = ? ORDER BY id', (book_id, ))
- if verses:
- return [{
- 'book_id': int(verse[0]),
- 'chapter': int(verse[1]),
- 'verse': int(verse[2]),
- 'text': str(verse[3])
- } for verse in verses]
- else:
- return None
-
- def close_connection(self):
- self.cursor.close()
- self.connection.close()
=== added directory 'openlp/plugins/bibles/lib/importers'
=== renamed file 'openlp/plugins/bibles/lib/csvbible.py' => 'openlp/plugins/bibles/lib/importers/csvbible.py'
--- openlp/plugins/bibles/lib/csvbible.py 2016-05-21 08:31:24 +0000
+++ openlp/plugins/bibles/lib/importers/csvbible.py 2016-08-09 21:01:19 +0000
@@ -49,125 +49,138 @@
All CSV files are expected to use a comma (',') as the delimiter and double quotes ('"') as the quote symbol.
"""
+import csv
import logging
-import chardet
-import csv
+from collections import namedtuple
-from openlp.core.common import translate
-from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB
+from openlp.core.common import get_file_encoding, translate
+from openlp.core.lib.exceptions import ValidationError
+from openlp.plugins.bibles.lib.bibleimport import BibleImport
log = logging.getLogger(__name__)
-
-class CSVBible(BibleDB):
+Book = namedtuple('Book', 'id, testament_id, name, abbreviation')
+Verse = namedtuple('Verse', 'book_id_name, chapter_number, number, text')
+
+
+class CSVBible(BibleImport):
"""
This class provides a specialisation for importing of CSV Bibles.
"""
log.info('CSVBible loaded')
- def __init__(self, parent, **kwargs):
+ def __init__(self, *args, **kwargs):
"""
Loads a Bible from a set of CSV files. This class assumes the files contain all the information and a clean
bible is being loaded.
"""
log.info(self.__class__.__name__)
- BibleDB.__init__(self, parent, **kwargs)
+ super().__init__(*args, **kwargs)
self.books_file = kwargs['booksfile']
self.verses_file = kwargs['versefile']
+ @staticmethod
+ def get_book_name(name, books):
+ """
+ Normalize a book name or id.
+
+ :param name: The name, or id of a book. Str
+ :param books: A dict of books parsed from the books file.
+ :return: The normalized name. Str
+ """
+ try:
+ book_name = books[int(name)]
+ except ValueError:
+ book_name = name
+ return book_name
+
+ @staticmethod
+ def parse_csv_file(filename, results_tuple):
+ """
+ Parse the supplied CSV file.
+
+ :param filename: The name of the file to parse. Str
+ :param results_tuple: The namedtuple to use to store the results. namedtuple
+ :return: An iterable yielding namedtuples of type results_tuple
+ """
+ try:
+ encoding = get_file_encoding(filename)['encoding']
+ with open(filename, 'r', encoding=encoding, newline='') as csv_file:
+ csv_reader = csv.reader(csv_file, delimiter=',', quotechar='"')
+ return [results_tuple(*line) for line in csv_reader]
+ except (OSError, csv.Error):
+ raise ValidationError(msg='Parsing "{file}" failed'.format(file=filename))
+
+ def process_books(self, books):
+ """
+ Process the books parsed from the books file.
+
+ :param books: An a list Book namedtuples
+ :return: A dict of books or None
+ """
+ book_list = {}
+ number_of_books = len(books)
+ for book in books:
+ if self.stop_import_flag:
+ return None
+ self.wizard.increment_progress_bar(
+ translate('BiblesPlugin.CSVBible', 'Importing books... {book}').format(book=book.name))
+ self.find_and_create_book(book.name, number_of_books, self.language_id)
+ book_list.update({int(book.id): book.name})
+ self.application.process_events()
+ return book_list
+
+ def process_verses(self, verses, books):
+ """
+ Process the verses parsed from the verses file.
+
+ :param verses: A list of Verse namedtuples
+ :param books: A dict of books
+ :return: None
+ """
+ book_ptr = None
+ for verse in verses:
+ if self.stop_import_flag:
+ return None
+ verse_book = self.get_book_name(verse.book_id_name, books)
+ if book_ptr != verse_book:
+ book = self.get_book(verse_book)
+ book_ptr = book.name
+ self.wizard.increment_progress_bar(
+ translate('BiblesPlugin.CSVBible', 'Importing verses from {book}...',
+ 'Importing verses from <book name>...').format(book=book.name))
+ self.session.commit()
+ self.create_verse(book.id, verse.chapter_number, verse.number, verse.text)
+ self.wizard.increment_progress_bar(translate('BiblesPlugin.CSVBible', 'Importing verses... done.'))
+ self.application.process_events()
+ self.session.commit()
+
def do_import(self, bible_name=None):
"""
- Import the bible books and verses.
+ Import a bible from the CSV files.
+
+ :param bible_name: Optional name of the bible being imported. Str or None
+ :return: True if the import was successful, False if it failed or was cancelled
"""
- self.wizard.progress_bar.setValue(0)
- self.wizard.progress_bar.setMinimum(0)
- self.wizard.progress_bar.setMaximum(66)
- success = True
- language_id = self.get_language(bible_name)
- if not language_id:
- log.error('Importing books from "{name}" failed'.format(name=self.filename))
- return False
- books_file = None
- book_list = {}
- # Populate the Tables
- try:
- details = get_file_encoding(self.books_file)
- books_file = open(self.books_file, 'r', encoding=details['encoding'])
- books_reader = csv.reader(books_file, delimiter=',', quotechar='"')
- for line in books_reader:
- if self.stop_import_flag:
- break
- self.wizard.increment_progress_bar(translate('BiblesPlugin.CSVBible',
- 'Importing books... {text}').format(text=line[2]))
- book_ref_id = self.get_book_ref_id_by_name(line[2], 67, language_id)
- if not book_ref_id:
- log.error('Importing books from "{name}" failed'.format(name=self.books_file))
- return False
- book_details = BiblesResourcesDB.get_book_by_id(book_ref_id)
- self.create_book(line[2], book_ref_id, book_details['testament_id'])
- book_list.update({int(line[0]): line[2]})
- self.application.process_events()
- except (IOError, IndexError):
- log.exception('Loading books from file failed')
- success = False
- finally:
- if books_file:
- books_file.close()
- if self.stop_import_flag or not success:
- return False
- self.wizard.progress_bar.setValue(0)
- self.wizard.progress_bar.setMaximum(67)
- verse_file = None
- try:
- book_ptr = None
- details = get_file_encoding(self.verses_file)
- verse_file = open(self.verses_file, 'r', encoding=details['encoding'])
- verse_reader = csv.reader(verse_file, delimiter=',', quotechar='"')
- for line in verse_reader:
- if self.stop_import_flag:
- break
- try:
- line_book = book_list[int(line[0])]
- except ValueError:
- line_book = line[0]
- if book_ptr != line_book:
- book = self.get_book(line_book)
- book_ptr = book.name
- # TODO: Check out this conversion in translations
- self.wizard.increment_progress_bar(
- translate('BiblesPlugin.CSVBible',
- 'Importing verses from {name}...'.format(name=book.name),
- 'Importing verses from <book name>...'))
- self.session.commit()
- verse_text = line[3]
- self.create_verse(book.id, line[1], line[2], verse_text)
- self.wizard.increment_progress_bar(translate('BiblesPlugin.CSVBible', 'Importing verses... done.'))
- self.application.process_events()
- self.session.commit()
- except IOError:
- log.exception('Loading verses from file failed')
- success = False
- finally:
- if verse_file:
- verse_file.close()
- if self.stop_import_flag:
- return False
- else:
- return success
-
-
-def get_file_encoding(filename):
- """
- Utility function to get the file encoding.
- """
- detect_file = None
- try:
- detect_file = open(filename, 'rb')
- details = chardet.detect(detect_file.read(1024))
- except IOError:
- log.exception('Error detecting file encoding')
- finally:
- if detect_file:
- detect_file.close()
- return details
+ try:
+ self.language_id = self.get_language(bible_name)
+ if not self.language_id:
+ raise ValidationError(msg='Invalid language selected')
+ books = self.parse_csv_file(self.books_file, Book)
+ self.wizard.progress_bar.setValue(0)
+ self.wizard.progress_bar.setMinimum(0)
+ self.wizard.progress_bar.setMaximum(len(books))
+ book_list = self.process_books(books)
+ if self.stop_import_flag:
+ return False
+ verses = self.parse_csv_file(self.verses_file, Verse)
+ self.wizard.progress_bar.setValue(0)
+ self.wizard.progress_bar.setMaximum(len(books) + 1)
+ self.process_verses(verses, book_list)
+ if self.stop_import_flag:
+ return False
+ except ValidationError:
+ log.exception('Could not import CSV bible')
+ return False
+ return True
=== renamed file 'openlp/plugins/bibles/lib/http.py' => 'openlp/plugins/bibles/lib/importers/http.py'
--- openlp/plugins/bibles/lib/http.py 2016-07-24 19:49:29 +0000
+++ openlp/plugins/bibles/lib/importers/http.py 2016-08-09 21:01:19 +0000
@@ -34,6 +34,7 @@
from openlp.core.lib.ui import critical_error_message_box
from openlp.core.lib.webpagereader import get_web_page
from openlp.plugins.bibles.lib import SearchResults
+from openlp.plugins.bibles.lib.bibleimport import BibleImport
from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB, Book
CLEANER_REGEX = re.compile(r' |<br />|\'\+\'')
@@ -576,10 +577,10 @@
return bibles
-class HTTPBible(BibleDB, RegistryProperties):
+class HTTPBible(BibleImport, RegistryProperties):
log.info('{name} HTTPBible loaded'.format(name=__name__))
- def __init__(self, parent, **kwargs):
+ def __init__(self, *args, **kwargs):
"""
Finds all the bibles defined for the system. Creates an Interface Object for each bible containing connection
information.
@@ -588,7 +589,7 @@
Init confirms the bible exists and stores the database path.
"""
- BibleDB.__init__(self, parent, **kwargs)
+ super().__init__(*args, **kwargs)
self.download_source = kwargs['download_source']
self.download_name = kwargs['download_name']
# TODO: Clean up proxy stuff. We probably want one global proxy per connection type (HTTP and HTTPS) at most.
@@ -638,12 +639,8 @@
return False
self.wizard.progress_bar.setMaximum(len(books) + 2)
self.wizard.increment_progress_bar(translate('BiblesPlugin.HTTPBible', 'Registering Language...'))
- if self.language_id:
- self.save_meta('language_id', self.language_id)
- else:
- self.language_id = self.get_language(bible_name)
+ self.language_id = self.get_language_id(bible_name=bible_name)
if not self.language_id:
- log.error('Importing books from {name} failed'.format(name=self.filename))
return False
for book in books:
if self.stop_import_flag:
=== renamed file 'openlp/plugins/bibles/lib/opensong.py' => 'openlp/plugins/bibles/lib/importers/opensong.py'
--- openlp/plugins/bibles/lib/opensong.py 2016-07-05 20:31:29 +0000
+++ openlp/plugins/bibles/lib/importers/opensong.py 2016-08-09 21:01:19 +0000
@@ -25,25 +25,17 @@
from openlp.core.common import translate, trace_error_handler
from openlp.core.lib.ui import critical_error_message_box
+from openlp.plugins.bibles.lib.bibleimport import BibleImport
from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB
log = logging.getLogger(__name__)
-class OpenSongBible(BibleDB):
- """
- OpenSong Bible format importer class.
- """
- def __init__(self, parent, **kwargs):
- """
- Constructor to create and set up an instance of the OpenSongBible class. This class is used to import Bibles
- from OpenSong's XML format.
- """
- log.debug(self.__class__.__name__)
- BibleDB.__init__(self, parent, **kwargs)
- self.filename = kwargs['filename']
-
+class OpenSongBible(BibleImport):
+ """
+ OpenSong Bible format importer class. This class is used to import Bibles from OpenSong's XML format.
+ """
def get_text(self, element):
"""
Recursively get all text in an objectify element and its child elements.
@@ -64,16 +56,9 @@
Loads a Bible from file.
"""
log.debug('Starting OpenSong import from "{name}"'.format(name=self.filename))
- if not isinstance(self.filename, str):
- self.filename = str(self.filename, 'utf8')
- import_file = None
success = True
try:
- # NOTE: We don't need to do any of the normal encoding detection here, because lxml does it's own encoding
- # detection, and the two mechanisms together interfere with each other.
- import_file = open(self.filename, 'rb')
- opensong = objectify.parse(import_file)
- bible = opensong.getroot()
+ bible = self.parse_xml(self.filename, use_objectify=True)
# Check that we're not trying to import a Zefania XML bible, it is sometimes refered to as 'OpenSong'
if bible.tag.upper() == 'XMLBIBLE':
critical_error_message_box(
@@ -82,9 +67,8 @@
'please use the Zefania import option.'))
return False
# No language info in the opensong format, so ask the user
- language_id = self.get_language(bible_name)
+ language_id = self.get_language_id(bible_name=self.filename)
if not language_id:
- log.error('Importing books from "{name}" failed'.format(name=self.filename))
return False
for book in bible.b:
if self.stop_import_flag:
@@ -138,9 +122,6 @@
except (IOError, AttributeError):
log.exception('Loading Bible from OpenSong file failed')
success = False
- finally:
- if import_file:
- import_file.close()
if self.stop_import_flag:
return False
else:
=== renamed file 'openlp/plugins/bibles/lib/osis.py' => 'openlp/plugins/bibles/lib/importers/osis.py'
--- openlp/plugins/bibles/lib/osis.py 2016-08-03 17:26:10 +0000
+++ openlp/plugins/bibles/lib/importers/osis.py 2016-08-09 21:01:19 +0000
@@ -23,105 +23,83 @@
import logging
from lxml import etree
-from openlp.core.common import languages, translate, trace_error_handler
-from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB
+from openlp.core.common import translate, trace_error_handler
from openlp.core.lib.ui import critical_error_message_box
+from openlp.plugins.bibles.lib.bibleimport import BibleImport
+from openlp.plugins.bibles.lib.db import BiblesResourcesDB
log = logging.getLogger(__name__)
+NS = {'ns': 'http://www.bibletechnologies.net/2003/OSIS/namespace'}
+# Tags we don't use and can remove the content
+REMOVABLE_ELEMENTS = ('{http://www.bibletechnologies.net/2003/OSIS/namespace}note',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}milestone',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}title',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}abbr',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}catchWord',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}index',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}rdg',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}rdgGroup',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}figure')
+# Tags we don't use but need to keep the content
+REMOVABLE_TAGS = ('{http://www.bibletechnologies.net/2003/OSIS/namespace}p',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}l',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}lg',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}q',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}a',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}w',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}divineName',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}foreign',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}hi',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}inscription',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}mentioned',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}name',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}reference',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}seg',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}transChange',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}salute',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}signed',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}closer',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}speech',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}speaker',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}list',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}item',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}table',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}head',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}row',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}cell',
+ '{http://www.bibletechnologies.net/2003/OSIS/namespace}caption')
+
def replacement(match):
return match.group(2).upper()
-class OSISBible(BibleDB):
+class OSISBible(BibleImport):
"""
`OSIS <http://www.bibletechnologies.net/>`_ Bible format importer class.
"""
- log.info('BibleOSISImpl loaded')
-
- def __init__(self, parent, **kwargs):
- log.debug(self.__class__.__name__)
- BibleDB.__init__(self, parent, **kwargs)
- self.filename = kwargs['filename']
-
def do_import(self, bible_name=None):
"""
Loads a Bible from file.
"""
log.debug('Starting OSIS import from "{name}"'.format(name=self.filename))
- if not isinstance(self.filename, str):
- self.filename = str(self.filename, 'utf8')
- import_file = None
success = True
try:
- # NOTE: We don't need to do any of the normal encoding detection here, because lxml does it's own encoding
- # detection, and the two mechanisms together interfere with each other.
- import_file = open(self.filename, 'rb')
- osis_bible_tree = etree.parse(import_file, parser=etree.XMLParser(recover=True))
- namespace = {'ns': 'http://www.bibletechnologies.net/2003/OSIS/namespace'}
- # Find bible language
- language_id = None
- lang = osis_bible_tree.xpath("//ns:osisText/@xml:lang", namespaces=namespace)
- if lang:
- language = languages.get_language(lang[0])
- if hasattr(language, 'id'):
- language_id = language.id
- # The language couldn't be detected, ask the user
- if not language_id:
- language_id = self.get_language(bible_name)
- if not language_id:
- log.error('Importing books from "{name}" failed'.format(name=self.filename))
- return False
- self.save_meta('language_id', language_id)
- num_books = int(osis_bible_tree.xpath("count(//ns:div[@type='book'])", namespaces=namespace))
self.wizard.increment_progress_bar(translate('BiblesPlugin.OsisImport',
'Removing unused tags (this may take a few minutes)...'))
- # We strip unused tags from the XML, this should leave us with only chapter, verse and div tags.
- # Strip tags we don't use - remove content
- etree.strip_elements(osis_bible_tree, ('{http://www.bibletechnologies.net/2003/OSIS/namespace}note',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}milestone',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}title',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}abbr',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}catchWord',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}index',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}rdg',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}rdgGroup',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}figure'),
- with_tail=False)
- # Strip tags we don't use - keep content
- etree.strip_tags(osis_bible_tree, ('{http://www.bibletechnologies.net/2003/OSIS/namespace}p',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}l',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}lg',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}q',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}a',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}w',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}divineName',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}foreign',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}hi',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}inscription',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}mentioned',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}name',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}reference',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}seg',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}transChange',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}salute',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}signed',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}closer',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}speech',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}speaker',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}list',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}item',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}table',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}head',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}row',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}cell',
- '{http://www.bibletechnologies.net/2003/OSIS/namespace}caption'))
+ osis_bible_tree = self.parse_xml(self.filename, elements=REMOVABLE_ELEMENTS, tags=REMOVABLE_TAGS)
+ # Find bible language]
+ language = osis_bible_tree.xpath("//ns:osisText/@xml:lang", namespaces=NS)
+ language_id = self.get_language_id(language[0] if language else None, bible_name=self.filename)
+ if not language_id:
+ return False
+ num_books = int(osis_bible_tree.xpath("count(//ns:div[@type='book'])", namespaces=NS))
# Precompile a few xpath-querys
- verse_in_chapter = etree.XPath('count(//ns:chapter[1]/ns:verse)', namespaces=namespace)
- text_in_verse = etree.XPath('count(//ns:verse[1]/text())', namespaces=namespace)
+ verse_in_chapter = etree.XPath('count(//ns:chapter[1]/ns:verse)', namespaces=NS)
+ text_in_verse = etree.XPath('count(//ns:verse[1]/text())', namespaces=NS)
# Find books in the bible
- bible_books = osis_bible_tree.xpath("//ns:div[@type='book']", namespaces=namespace)
+ bible_books = osis_bible_tree.xpath("//ns:div[@type='book']", namespaces=NS)
for book in bible_books:
if self.stop_import_flag:
break
@@ -189,9 +167,6 @@
critical_error_message_box(message=translate('BiblesPlugin.OsisImport',
'The file is not a valid OSIS-XML file:'
'\n{text}').format(text=e.msg))
- finally:
- if import_file:
- import_file.close()
if self.stop_import_flag:
return False
else:
=== renamed file 'openlp/plugins/bibles/lib/sword.py' => 'openlp/plugins/bibles/lib/importers/sword.py'
--- openlp/plugins/bibles/lib/sword.py 2016-08-03 17:26:10 +0000
+++ openlp/plugins/bibles/lib/importers/sword.py 2016-08-09 21:01:19 +0000
@@ -23,25 +23,26 @@
import logging
from pysword import modules
-from openlp.core.common import languages, translate
+from openlp.core.common import translate
from openlp.core.lib.ui import critical_error_message_box
-from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB
+from openlp.plugins.bibles.lib.bibleimport import BibleImport
+from openlp.plugins.bibles.lib.db import BiblesResourcesDB
log = logging.getLogger(__name__)
-class SwordBible(BibleDB):
+class SwordBible(BibleImport):
"""
SWORD Bible format importer class.
"""
- def __init__(self, parent, **kwargs):
+ def __init__(self, *args, **kwargs):
"""
Constructor to create and set up an instance of the SwordBible class. This class is used to import Bibles
from SWORD bible modules.
"""
log.debug(self.__class__.__name__)
- BibleDB.__init__(self, parent, **kwargs)
+ super().__init__(*args, **kwargs)
self.sword_key = kwargs['sword_key']
self.sword_path = kwargs['sword_path']
if self.sword_path == '':
@@ -57,13 +58,11 @@
pysword_modules = modules.SwordModules(self.sword_path)
pysword_module_json = pysword_modules.parse_modules()[self.sword_key]
bible = pysword_modules.get_bible_from_module(self.sword_key)
- language_id = None
language = pysword_module_json['lang']
language = language[language.find('.') + 1:]
- language = languages.get_language(language)
- if hasattr(language, 'id'):
- language_id = language.id
- self.save_meta('language_id', language_id)
+ language_id = self.get_language_id(language, bible_name=self.filename)
+ if not language_id:
+ return False
books = bible.get_structure().get_books()
# Count number of books
num_books = 0
=== renamed file 'openlp/plugins/bibles/lib/zefania.py' => 'openlp/plugins/bibles/lib/importers/zefania.py'
--- openlp/plugins/bibles/lib/zefania.py 2016-08-03 17:26:10 +0000
+++ openlp/plugins/bibles/lib/importers/zefania.py 2016-08-09 21:01:19 +0000
@@ -21,64 +21,41 @@
###############################################################################
import logging
-from lxml import etree
-from openlp.core.common import languages, translate
+from openlp.core.common import translate
from openlp.core.lib.ui import critical_error_message_box
-from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB
+from openlp.plugins.bibles.lib.bibleimport import BibleImport
+from openlp.plugins.bibles.lib.db import BiblesResourcesDB
log = logging.getLogger(__name__)
-
-class ZefaniaBible(BibleDB):
- """
- Zefania Bible format importer class.
- """
- def __init__(self, parent, **kwargs):
- """
- Constructor to create and set up an instance of the ZefaniaBible class. This class is used to import Bibles
- from ZefaniaBible's XML format.
- """
- log.debug(self.__class__.__name__)
- BibleDB.__init__(self, parent, **kwargs)
- self.filename = kwargs['filename']
+# Tags we don't use and can remove the content
+REMOVABLE_ELEMENTS = ('PROLOG', 'REMARK', 'CAPTION', 'MEDIA')
+# Tags we don't use but need to keep the content
+REMOVABLE_TAGS = ('STYLE', 'GRAM', 'NOTE', 'SUP', 'XREF')
+
+
+class ZefaniaBible(BibleImport):
+ """
+ Zefania Bible format importer class. This class is used to import Bibles from ZefaniaBible's XML format.
+ """
def do_import(self, bible_name=None):
"""
Loads a Bible from file.
"""
log.debug('Starting Zefania import from "{name}"'.format(name=self.filename))
- if not isinstance(self.filename, str):
- self.filename = str(self.filename, 'utf8')
- import_file = None
success = True
try:
- # NOTE: We don't need to do any of the normal encoding detection here, because lxml does it's own encoding
- # detection, and the two mechanisms together interfere with each other.
- import_file = open(self.filename, 'rb')
- zefania_bible_tree = etree.parse(import_file, parser=etree.XMLParser(recover=True))
+ xmlbible = self.parse_xml(self.filename, elements=REMOVABLE_ELEMENTS, tags=REMOVABLE_TAGS)
# Find bible language
- language_id = None
- language = zefania_bible_tree.xpath("/XMLBIBLE/INFORMATION/language/text()")
- if language:
- language = languages.get_language(language[0])
- if hasattr(language, 'id'):
- language_id = language.id
- # The language couldn't be detected, ask the user
- if not language_id:
- language_id = self.get_language(bible_name)
- if not language_id:
- log.error('Importing books from "{name}" failed'.format(name=self.filename))
+ language = xmlbible.xpath("/XMLBIBLE/INFORMATION/language/text()")
+ language_id = self.get_language_id(language[0] if language else None, bible_name=self.filename)
+ if not language_id:
return False
- self.save_meta('language_id', language_id)
- num_books = int(zefania_bible_tree.xpath('count(//BIBLEBOOK)'))
- self.wizard.progress_bar.setMaximum(int(zefania_bible_tree.xpath('count(//CHAPTER)')))
- # Strip tags we don't use - keep content
- etree.strip_tags(zefania_bible_tree, ('STYLE', 'GRAM', 'NOTE', 'SUP', 'XREF'))
- # Strip tags we don't use - remove content
- etree.strip_elements(zefania_bible_tree, ('PROLOG', 'REMARK', 'CAPTION', 'MEDIA'), with_tail=False)
- xmlbible = zefania_bible_tree.getroot()
+ num_books = int(xmlbible.xpath('count(//BIBLEBOOK)'))
+ self.wizard.progress_bar.setMaximum(int(xmlbible.xpath('count(//CHAPTER)')))
for BIBLEBOOK in xmlbible:
if self.stop_import_flag:
break
@@ -116,9 +93,6 @@
'compressed. You must decompress them before import.'))
log.exception(str(e))
success = False
- finally:
- if import_file:
- import_file.close()
if self.stop_import_flag:
return False
else:
=== modified file 'openlp/plugins/bibles/lib/manager.py'
--- openlp/plugins/bibles/lib/manager.py 2016-07-28 19:57:21 +0000
+++ openlp/plugins/bibles/lib/manager.py 2016-08-09 21:01:19 +0000
@@ -26,13 +26,13 @@
from openlp.core.common import RegistryProperties, AppLocation, Settings, translate, delete_file, UiStrings
from openlp.plugins.bibles.lib import parse_reference, LanguageSelection
from openlp.plugins.bibles.lib.db import BibleDB, BibleMeta
-from .csvbible import CSVBible
-from .http import HTTPBible
-from .opensong import OpenSongBible
-from .osis import OSISBible
-from .zefania import ZefaniaBible
+from .importers.csvbible import CSVBible
+from .importers.http import HTTPBible
+from .importers.opensong import OpenSongBible
+from .importers.osis import OSISBible
+from .importers.zefania import ZefaniaBible
try:
- from .sword import SwordBible
+ from .importers.sword import SwordBible
except:
pass
@@ -124,7 +124,6 @@
files.remove('alternative_book_names.sqlite')
log.debug('Bible Files {text}'.format(text=files))
self.db_cache = {}
- self.old_bible_databases = []
for filename in files:
bible = BibleDB(self.parent, path=self.path, file=filename)
if not bible.session:
=== modified file 'tests/functional/openlp_core_common/test_init.py'
--- tests/functional/openlp_core_common/test_init.py 2016-05-31 21:40:13 +0000
+++ tests/functional/openlp_core_common/test_init.py 2016-08-09 21:01:19 +0000
@@ -23,11 +23,12 @@
Functional tests to test the AppLocation class and related methods.
"""
import os
+from io import BytesIO
from unittest import TestCase
-from openlp.core.common import add_actions, get_uno_instance, get_uno_command, delete_file, get_filesystem_encoding, \
- split_filename, clean_filename
-from tests.functional import MagicMock, patch
+from openlp.core.common import add_actions, clean_filename, delete_file, get_file_encoding, get_filesystem_encoding, \
+ get_uno_command, get_uno_instance, split_filename
+from tests.functional import MagicMock, PropertyMock, call, patch
from tests.helpers.testmixin import TestMixin
@@ -340,3 +341,63 @@
# THEN: delete_file should log and exception and return False
self.assertEqual(mocked_log.exception.call_count, 1)
self.assertFalse(result, 'delete_file should return False when os.remove raises an OSError')
+
+ def test_get_file_name_encoding_done_test(self):
+ """
+ Test get_file_encoding when the detector sets done to True
+ """
+ # GIVEN: A mocked UniversalDetector instance with done attribute set to True after first iteration
+ with patch('openlp.core.common.UniversalDetector') as mocked_universal_detector, \
+ patch('builtins.open', return_value=BytesIO(b"data" * 260)) as mocked_open:
+ encoding_result = {'encoding': 'UTF-8', 'confidence': 0.99}
+ mocked_universal_detector_inst = MagicMock(result=encoding_result)
+ type(mocked_universal_detector_inst).done = PropertyMock(side_effect=[False, True])
+ mocked_universal_detector.return_value = mocked_universal_detector_inst
+
+ # WHEN: Calling get_file_encoding
+ result = get_file_encoding('file name')
+
+ # THEN: The feed method of UniversalDetector should only br called once before returning a result
+ mocked_open.assert_called_once_with('file name', 'rb')
+ self.assertEqual(mocked_universal_detector_inst.feed.mock_calls, [call(b"data" * 256)])
+ mocked_universal_detector_inst.close.assert_called_once_with()
+ self.assertEqual(result, encoding_result)
+
+ def test_get_file_name_encoding_eof_test(self):
+ """
+ Test get_file_encoding when the end of the file is reached
+ """
+ # GIVEN: A mocked UniversalDetector instance which isn't set to done and a mocked open, with 1040 bytes of test
+ # data (enough to run the iterator twice)
+ with patch('openlp.core.common.UniversalDetector') as mocked_universal_detector, \
+ patch('builtins.open', return_value=BytesIO(b"data" * 260)) as mocked_open:
+ encoding_result = {'encoding': 'UTF-8', 'confidence': 0.99}
+ mocked_universal_detector_inst = MagicMock(mock=mocked_universal_detector,
+ **{'done': False, 'result': encoding_result})
+ mocked_universal_detector.return_value = mocked_universal_detector_inst
+
+ # WHEN: Calling get_file_encoding
+ result = get_file_encoding('file name')
+
+ # THEN: The feed method of UniversalDetector should have been called twice before returning a result
+ mocked_open.assert_called_once_with('file name', 'rb')
+ self.assertEqual(mocked_universal_detector_inst.feed.mock_calls, [call(b"data" * 256), call(b"data" * 4)])
+ mocked_universal_detector_inst.close.assert_called_once_with()
+ self.assertEqual(result, encoding_result)
+
+ def test_get_file_name_encoding_oserror_test(self):
+ """
+ Test get_file_encoding when the end of the file is reached
+ """
+ # GIVEN: A mocked UniversalDetector instance which isn't set to done and a mocked open, with 1040 bytes of test
+ # data (enough to run the iterator twice)
+ with patch('openlp.core.common.UniversalDetector'), \
+ patch('builtins.open', side_effect=OSError), \
+ patch('openlp.core.common.log') as mocked_log:
+
+ # WHEN: Calling get_file_encoding
+ result = get_file_encoding('file name')
+
+ # THEN: log.exception should be called and get_file_encoding should return None
+ mocked_log.exception.assert_called_once_with('Error detecting file encoding')
+ self.assertIsNone(result)
=== added file 'tests/functional/openlp_plugins/bibles/test_bibleimport.py'
--- tests/functional/openlp_plugins/bibles/test_bibleimport.py 1970-01-01 00:00:00 +0000
+++ tests/functional/openlp_plugins/bibles/test_bibleimport.py 2016-08-09 21:01:19 +0000
@@ -0,0 +1,212 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2015 OpenLP Developers #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it #
+# under the terms of the GNU General Public License as published by the Free #
+# Software Foundation; version 2 of the License. #
+# #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
+# more details. #
+# #
+# You should have received a copy of the GNU General Public License along #
+# with this program; if not, write to the Free Software Foundation, Inc., 59 #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
+###############################################################################
+"""
+This module contains tests for the bibleimport module.
+"""
+
+from io import BytesIO
+from lxml import etree, objectify
+
+from unittest import TestCase
+
+from openlp.core.common.languages import Language
+from openlp.plugins.bibles.lib.bibleimport import BibleImport
+from tests.functional import MagicMock, patch
+
+
+class TestBibleImport(TestCase):
+ """
+ Test the functions in the :mod:`bibleimport` module.
+ """
+
+ def setUp(self):
+ test_file = BytesIO(b'<?xml version="1.0" encoding="UTF-8" ?>\n'
+ b'<root>\n'
+ b' <data><div>Test<p>data</p><a>to</a>keep</div></data>\n'
+ b' <data><unsupported>Test<x>data</x><y>to</y>discard</unsupported></data>\n'
+ b'</root>')
+ self.file_patcher = patch('builtins.open', return_value=test_file)
+ self.log_patcher = patch('openlp.plugins.bibles.lib.bibleimport.log')
+ self.setup_patcher = patch('openlp.plugins.bibles.lib.db.BibleDB._setup')
+
+ self.addCleanup(self.file_patcher.stop)
+ self.addCleanup(self.log_patcher.stop)
+ self.addCleanup(self.setup_patcher.stop)
+
+ self.file_patcher.start()
+ self.mock_log = self.log_patcher.start()
+ self.setup_patcher.start()
+
+ def get_language_id_language_found_test(self):
+ """
+ Test get_language_id() when called with a name found in the languages list
+ """
+ # GIVEN: A mocked languages.get_language which returns language and an instance of BibleImport
+ with patch('openlp.core.common.languages.get_language', return_value=Language(30, 'English', 'en')) \
+ as mocked_languages_get_language, \
+ patch('openlp.plugins.bibles.lib.db.BibleDB.get_language') as mocked_db_get_language:
+ instance = BibleImport(MagicMock())
+ instance.save_meta = MagicMock()
+
+ # WHEN: Calling get_language_id() with a language name and bible name
+ result = instance.get_language_id('English', 'KJV')
+
+ # THEN: The id of the language returned from languages.get_language should be returned
+ mocked_languages_get_language.assert_called_once_with('English')
+ self.assertFalse(mocked_db_get_language.called)
+ instance.save_meta.assert_called_once_with('language_id', 30)
+ self.assertEqual(result, 30)
+
+ def get_language_id_language_not_found_test(self):
+ """
+ Test get_language_id() when called with a name not found in the languages list
+ """
+ # GIVEN: A mocked languages.get_language which returns language and an instance of BibleImport
+ with patch('openlp.core.common.languages.get_language', return_value=None) \
+ as mocked_languages_get_language, \
+ patch('openlp.plugins.bibles.lib.db.BibleDB.get_language', return_value=20) as mocked_db_get_language:
+ instance = BibleImport(MagicMock())
+ instance.save_meta = MagicMock()
+
+ # WHEN: Calling get_language_id() with a language name and bible name
+ result = instance.get_language_id('RUS', 'KJV')
+
+ # THEN: The id of the language returned from languages.get_language should be returned
+ mocked_languages_get_language.assert_called_once_with('RUS')
+ mocked_db_get_language.assert_called_once_with('KJV')
+ instance.save_meta.assert_called_once_with('language_id', 20)
+ self.assertEqual(result, 20)
+
+ def get_language_id_user_choice_test(self):
+ """
+ Test get_language_id() when the language is not found and the user is asked for the language
+ """
+ # GIVEN: A mocked languages.get_language which returns None a mocked BibleDB.get_language which returns a
+ # language id.
+ with patch('openlp.core.common.languages.get_language', return_value=None) as mocked_languages_get_language, \
+ patch('openlp.plugins.bibles.lib.db.BibleDB.get_language', return_value=40) as mocked_db_get_language:
+ self.mock_log.error.reset_mock()
+ instance = BibleImport(MagicMock())
+ instance.save_meta = MagicMock()
+
+ # WHEN: Calling get_language_id() with a language name and bible name
+ result = instance.get_language_id('English', 'KJV')
+
+ # THEN: The id of the language returned from BibleDB.get_language should be returned
+ mocked_languages_get_language.assert_called_once_with('English')
+ mocked_db_get_language.assert_called_once_with('KJV')
+ self.assertFalse(self.mock_log.error.called)
+ instance.save_meta.assert_called_once_with('language_id', 40)
+ self.assertEqual(result, 40)
+
+ def get_language_id_user_choice_rejected_test(self):
+ """
+ Test get_language_id() when the language is not found and the user rejects the dilaog box
+ """
+ # GIVEN: A mocked languages.get_language which returns None a mocked BibleDB.get_language which returns a
+ # language id.
+ with patch('openlp.core.common.languages.get_language', return_value=None) as mocked_languages_get_language, \
+ patch('openlp.plugins.bibles.lib.db.BibleDB.get_language', return_value=None) as mocked_db_get_language:
+ self.mock_log.error.reset_mock()
+ instance = BibleImport(MagicMock())
+ instance.save_meta = MagicMock()
+
+ # WHEN: Calling get_language_id() with a language name and bible name
+ result = instance.get_language_id('Qwerty', 'KJV')
+
+ # THEN: None should be returned and an error should be logged
+ mocked_languages_get_language.assert_called_once_with('Qwerty')
+ mocked_db_get_language.assert_called_once_with('KJV')
+ self.mock_log.error.assert_called_once_with('Language detection failed when importing from "KJV". '
+ 'User aborted language selection.')
+ self.assertFalse(instance.save_meta.called)
+ self.assertIsNone(result)
+
+ def parse_xml_etree_test(self):
+ """
+ Test BibleImport.parse_xml() when called with the use_objectify default value
+ """
+ # GIVEN: A sample "file" to parse
+ # WHEN: Calling parse_xml
+ result = BibleImport.parse_xml('file.tst')
+
+ # THEN: The result returned should contain the correct data, and should be an instance of eetree_Element
+ self.assertEqual(etree.tostring(result),
+ b'<root>\n <data><div>Test<p>data</p><a>to</a>keep</div></data>\n'
+ b' <data><unsupported>Test<x>data</x><y>to</y>discard</unsupported></data>\n</root>')
+ self.assertIsInstance(result, etree._Element)
+
+ def parse_xml_etree_use_objectify_test(self):
+ """
+ Test BibleImport.parse_xml() when called with use_objectify set to True
+ """
+ # GIVEN: A sample "file" to parse
+ # WHEN: Calling parse_xml
+ result = BibleImport.parse_xml('file.tst', use_objectify=True)
+
+ # THEN: The result returned should contain the correct data, and should be an instance of ObjectifiedElement
+ self.assertEqual(etree.tostring(result),
+ b'<root><data><div>Test<p>data</p><a>to</a>keep</div></data>'
+ b'<data><unsupported>Test<x>data</x><y>to</y>discard</unsupported></data></root>')
+ self.assertIsInstance(result, objectify.ObjectifiedElement)
+
+ def parse_xml_elements_test(self):
+ """
+ Test BibleImport.parse_xml() when given a tuple of elements to remove
+ """
+ # GIVEN: A tuple of elements to remove
+ elements = ('unsupported', 'x', 'y')
+
+ # WHEN: Calling parse_xml, with a test file
+ result = BibleImport.parse_xml('file.tst', elements=elements)
+
+ # THEN: The result returned should contain the correct data
+ self.assertEqual(etree.tostring(result),
+ b'<root>\n <data><div>Test<p>data</p><a>to</a>keep</div></data>\n <data/>\n</root>')
+
+ def parse_xml_tags_test(self):
+ """
+ Test BibleImport.parse_xml() when given a tuple of tags to remove
+ """
+ # GIVEN: A tuple of tags to remove
+ tags = ('div', 'p', 'a')
+
+ # WHEN: Calling parse_xml, with a test file
+ result = BibleImport.parse_xml('file.tst', tags=tags)
+
+ # THEN: The result returned should contain the correct data
+ self.assertEqual(etree.tostring(result), b'<root>\n <data>Testdatatokeep</data>\n <data><unsupported>Test'
+ b'<x>data</x><y>to</y>discard</unsupported></data>\n</root>')
+
+ def parse_xml_elements_tags_test(self):
+ """
+ Test BibleImport.parse_xml() when given a tuple of elements and of tags to remove
+ """
+ # GIVEN: A tuple of elements and of tags to remove
+ elements = ('unsupported', 'x', 'y')
+ tags = ('div', 'p', 'a')
+
+ # WHEN: Calling parse_xml, with a test file
+ result = BibleImport.parse_xml('file.tst', elements=elements, tags=tags)
+
+ # THEN: The result returned should contain the correct data
+ self.assertEqual(etree.tostring(result), b'<root>\n <data>Testdatatokeep</data>\n <data/>\n</root>')
=== modified file 'tests/functional/openlp_plugins/bibles/test_csvimport.py'
--- tests/functional/openlp_plugins/bibles/test_csvimport.py 2016-05-31 21:40:13 +0000
+++ tests/functional/openlp_plugins/bibles/test_csvimport.py 2016-08-09 21:01:19 +0000
@@ -23,13 +23,17 @@
This module contains tests for the CSV Bible importer.
"""
+import csv
+import json
import os
-import json
+from collections import namedtuple
from unittest import TestCase
-from tests.functional import MagicMock, patch
-from openlp.plugins.bibles.lib.csvbible import CSVBible
-from openlp.plugins.bibles.lib.db import BibleDB
+from tests.functional import ANY, MagicMock, PropertyMock, call, patch
+from openlp.core.lib.exceptions import ValidationError
+from openlp.plugins.bibles.lib.bibleimport import BibleImport
+from openlp.plugins.bibles.lib.importers.csvbible import Book, CSVBible, Verse
+
TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..', '..', 'resources', 'bibles'))
@@ -41,14 +45,12 @@
"""
def setUp(self):
+ self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager')
self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry')
+ self.addCleanup(self.manager_patcher.stop)
+ self.addCleanup(self.registry_patcher.stop)
+ self.manager_patcher.start()
self.registry_patcher.start()
- self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager')
- self.manager_patcher.start()
-
- def tearDown(self):
- self.registry_patcher.stop()
- self.manager_patcher.stop()
def test_create_importer(self):
"""
@@ -58,12 +60,331 @@
mocked_manager = MagicMock()
# WHEN: An importer object is created
- importer = CSVBible(mocked_manager, path='.', name='.', booksfile='.', versefile='.')
-
- # THEN: The importer should be an instance of BibleDB
- self.assertIsInstance(importer, BibleDB)
-
- def test_file_import(self):
+ importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv')
+
+ # THEN: The importer should be an instance of BibleImport
+ self.assertIsInstance(importer, BibleImport)
+ self.assertEqual(importer.books_file, 'books.csv')
+ self.assertEqual(importer.verses_file, 'verse.csv')
+
+ def book_namedtuple_test(self):
+ """
+ Test that the Book namedtuple is created as expected
+ """
+ # GIVEN: The Book namedtuple
+ # WHEN: Creating an instance of Book
+ result = Book('id', 'testament_id', 'name', 'abbreviation')
+
+ # THEN: The attributes should match up with the data we used
+ self.assertEqual(result.id, 'id')
+ self.assertEqual(result.testament_id, 'testament_id')
+ self.assertEqual(result.name, 'name')
+ self.assertEqual(result.abbreviation, 'abbreviation')
+
+ def verse_namedtuple_test(self):
+ """
+ Test that the Verse namedtuple is created as expected
+ """
+ # GIVEN: The Verse namedtuple
+ # WHEN: Creating an instance of Verse
+ result = Verse('book_id_name', 'chapter_number', 'number', 'text')
+
+ # THEN: The attributes should match up with the data we used
+ self.assertEqual(result.book_id_name, 'book_id_name')
+ self.assertEqual(result.chapter_number, 'chapter_number')
+ self.assertEqual(result.number, 'number')
+ self.assertEqual(result.text, 'text')
+
+ def get_book_name_id_test(self):
+ """
+ Test that get_book_name() returns the correct book when called with an id
+ """
+ # GIVEN: A dictionary of books with their id as the keys
+ books = {1: 'Book 1', 2: 'Book 2', 3: 'Book 3'}
+
+ # WHEN: Calling get_book_name() and the name is an integer represented as a string
+ test_data = [['1', 'Book 1'], ['2', 'Book 2'], ['3', 'Book 3']]
+ for name, expected_result in test_data:
+ actual_result = CSVBible.get_book_name(name, books)
+
+ # THEN: get_book_name() should return the book name associated with that id from the books dictionary
+ self.assertEqual(actual_result, expected_result)
+
+ def get_book_name_test(self):
+ """
+ Test that get_book_name() returns the name when called with a non integer value
+ """
+ # GIVEN: A dictionary of books with their id as the keys
+ books = {1: 'Book 1', 2: 'Book 2', 3: 'Book 3'}
+
+ # WHEN: Calling get_book_name() and the name is not an integer represented as a string
+ test_data = [['Book 4', 'Book 4'], ['Book 5', 'Book 5'], ['Book 6', 'Book 6']]
+ for name, expected_result in test_data:
+ actual_result = CSVBible.get_book_name(name, books)
+
+ # THEN: get_book_name() should return the input
+ self.assertEqual(actual_result, expected_result)
+
+ def parse_csv_file_test(self):
+ """
+ Test the parse_csv_file() with sample data
+ """
+ # GIVEN: A mocked csv.reader which returns an iterator with test data
+ test_data = [['1', 'Line 1', 'Data 1'], ['2', 'Line 2', 'Data 2'], ['3', 'Line 3', 'Data 3']]
+ TestTuple = namedtuple('TestTuple', 'line_no line_description line_data')
+
+ with patch('openlp.plugins.bibles.lib.importers.csvbible.get_file_encoding',
+ return_value={'encoding': 'utf-8', 'confidence': 0.99}),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.open', create=True) as mocked_open,\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.csv.reader',
+ return_value=iter(test_data)) as mocked_reader:
+
+ # WHEN: Calling the CSVBible parse_csv_file method with a file name and TestTuple
+ result = CSVBible.parse_csv_file('file.csv', TestTuple)
+
+ # THEN: A list of TestTuple instances with the parsed data should be returned
+ self.assertEqual(result, [TestTuple('1', 'Line 1', 'Data 1'), TestTuple('2', 'Line 2', 'Data 2'),
+ TestTuple('3', 'Line 3', 'Data 3')])
+ mocked_open.assert_called_once_with('file.csv', 'r', encoding='utf-8', newline='')
+ mocked_reader.assert_called_once_with(ANY, delimiter=',', quotechar='"')
+
+ def parse_csv_file_oserror_test(self):
+ """
+ Test the parse_csv_file() handles an OSError correctly
+ """
+ # GIVEN: Mocked a mocked open object which raises an OSError
+ with patch('openlp.plugins.bibles.lib.importers.csvbible.get_file_encoding',
+ return_value={'encoding': 'utf-8', 'confidence': 0.99}),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.open', side_effect=OSError, create=True):
+
+ # WHEN: Calling CSVBible.parse_csv_file
+ # THEN: A ValidationError should be raised
+ with self.assertRaises(ValidationError) as context:
+ CSVBible.parse_csv_file('file.csv', None)
+ self.assertEqual(context.exception.msg, 'Parsing "file.csv" failed')
+
+ def parse_csv_file_csverror_test(self):
+ """
+ Test the parse_csv_file() handles an csv.Error correctly
+ """
+ # GIVEN: Mocked a csv.reader which raises an csv.Error
+ with patch('openlp.plugins.bibles.lib.importers.csvbible.get_file_encoding',
+ return_value={'encoding': 'utf-8', 'confidence': 0.99}),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.open', create=True),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.csv.reader', side_effect=csv.Error):
+
+ # WHEN: Calling CSVBible.parse_csv_file
+ # THEN: A ValidationError should be raised
+ with self.assertRaises(ValidationError) as context:
+ CSVBible.parse_csv_file('file.csv', None)
+ self.assertEqual(context.exception.msg, 'Parsing "file.csv" failed')
+
+ def process_books_stopped_import_test(self):
+ """
+ Test process books when the import is stopped
+ """
+ # GIVEN: An instance of CSVBible with the stop_import_flag set to True
+ mocked_manager = MagicMock()
+ with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'):
+ importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv')
+ type(importer).application = PropertyMock()
+ importer.stop_import_flag = True
+ importer.wizard = MagicMock()
+
+ # WHEN: Calling process_books
+ result = importer.process_books(['Book 1'])
+
+ # THEN: increment_progress_bar should not be called and the return value should be None
+ self.assertFalse(importer.wizard.increment_progress_bar.called)
+ self.assertIsNone(result)
+
+ def process_books_test(self):
+ """
+ Test process books when it completes successfully
+ """
+ # GIVEN: An instance of CSVBible with the stop_import_flag set to False, and some sample data
+ mocked_manager = MagicMock()
+ with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.translate'):
+ importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv')
+ type(importer).application = PropertyMock()
+ importer.find_and_create_book = MagicMock()
+ importer.language_id = 10
+ importer.stop_import_flag = False
+ importer.wizard = MagicMock()
+
+ books = [Book('1', '1', '1. Mosebog', '1Mos'), Book('2', '1', '2. Mosebog', '2Mos')]
+
+ # WHEN: Calling process_books
+ result = importer.process_books(books)
+
+ # THEN: translate and find_and_create_book should have been called with both book names.
+ # The returned data should be a dictionary with both song's id and names.
+ self.assertEqual(importer.find_and_create_book.mock_calls,
+ [call('1. Mosebog', 2, 10), call('2. Mosebog', 2, 10)])
+ importer.application.process_events.assert_called_once_with()
+ self.assertDictEqual(result, {1: '1. Mosebog', 2: '2. Mosebog'})
+
+ def process_verses_stopped_import_test(self):
+ """
+ Test process_verses when the import is stopped
+ """
+ # GIVEN: An instance of CSVBible with the stop_import_flag set to True
+ mocked_manager = MagicMock()
+ with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'):
+ importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv')
+ type(importer).application = PropertyMock()
+ importer.get_book_name = MagicMock()
+ importer.session = MagicMock()
+ importer.stop_import_flag = True
+ importer.wizard = MagicMock()
+
+ # WHEN: Calling process_verses
+ result = importer.process_verses([], [])
+
+ # THEN: get_book_name should not be called and the return value should be None
+ self.assertFalse(importer.get_book_name.called)
+ importer.wizard.increment_progress_bar.assert_called_once_with('Importing verses... done.')
+ importer.application.process_events.assert_called_once_with()
+ self.assertIsNone(result)
+
+ def process_verses_successful_test(self):
+ """
+ Test process_verses when the import is successful
+ """
+ # GIVEN: An instance of CSVBible with the application and wizard attributes mocked out, and some test data.
+ mocked_manager = MagicMock()
+ with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.translate'):
+ importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv')
+ type(importer).application = PropertyMock()
+ importer.create_verse = MagicMock()
+ importer.get_book = MagicMock(return_value=Book('1', '1', '1. Mosebog', '1Mos'))
+ importer.get_book_name = MagicMock(return_value='1. Mosebog')
+ importer.session = MagicMock()
+ importer.stop_import_flag = False
+ importer.wizard = MagicMock()
+ verses = [Verse(1, 1, 1, 'I Begyndelsen skabte Gud Himmelen og Jorden.'),
+ Verse(1, 1, 2, 'Og Jorden var øde og tom, og der var Mørke over Verdensdybet. '
+ 'Men Guds Ånd svævede over Vandene.')]
+ books = {1: '1. Mosebog'}
+
+ # WHEN: Calling process_verses
+ importer.process_verses(verses, books)
+
+ # THEN: create_verse is called with the test data
+ self.assertEqual(importer.get_book_name.mock_calls, [call(1, books), call(1, books)])
+ importer.get_book.assert_called_once_with('1. Mosebog')
+ self.assertEqual(importer.session.commit.call_count, 2)
+ self.assertEqual(importer.create_verse.mock_calls,
+ [call('1', 1, 1, 'I Begyndelsen skabte Gud Himmelen og Jorden.'),
+ call('1', 1, 2, 'Og Jorden var øde og tom, og der var Mørke over Verdensdybet. '
+ 'Men Guds Ånd svævede over Vandene.')])
+ importer.application.process_events.assert_called_once_with()
+
+ def do_import_invalid_language_id_test(self):
+ """
+ Test do_import when the user cancels the language selection dialog box
+ """
+ # GIVEN: An instance of CSVBible and a mocked get_language which simulates the user cancelling the language box
+ mocked_manager = MagicMock()
+ with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.log') as mocked_log:
+ importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv')
+ importer.get_language = MagicMock(return_value=None)
+
+ # WHEN: Calling do_import
+ result = importer.do_import('Bible Name')
+
+ # THEN: The log.exception method should have been called to show that it reached the except clause.
+ # False should be returned.
+ importer.get_language.assert_called_once_with('Bible Name')
+ mocked_log.exception.assert_called_once_with('Could not import CSV bible')
+ self.assertFalse(result)
+
+ def do_import_stop_import_test(self):
+ """
+ Test do_import when the import is stopped
+ """
+ # GIVEN: An instance of CSVBible with stop_import set to True
+ mocked_manager = MagicMock()
+ with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.log') as mocked_log:
+ importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv')
+ importer.get_language = MagicMock(return_value=10)
+ importer.parse_csv_file = MagicMock(return_value=['Book 1', 'Book 2', 'Book 3'])
+ importer.process_books = MagicMock()
+ importer.stop_import_flag = True
+ importer.wizard = MagicMock()
+
+ # WHEN: Calling do_import
+ result = importer.do_import('Bible Name')
+
+ # THEN: log.exception should not be called, parse_csv_file should only be called once,
+ # and False should be returned.
+ self.assertFalse(mocked_log.exception.called)
+ importer.parse_csv_file.assert_called_once_with('books.csv', Book)
+ importer.process_books.assert_called_once_with(['Book 1', 'Book 2', 'Book 3'])
+ self.assertFalse(result)
+
+ def do_import_stop_import_2_test(self):
+ """
+ Test do_import when the import is stopped
+ """
+ # GIVEN: An instance of CSVBible with stop_import which is True the second time of calling
+ mocked_manager = MagicMock()
+ with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.log') as mocked_log:
+ CSVBible.stop_import_flag = PropertyMock(side_effect=[False, True])
+ importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verses.csv')
+ importer.get_language = MagicMock(return_value=10)
+ importer.parse_csv_file = MagicMock(side_effect=[['Book 1'], ['Verse 1']])
+ importer.process_books = MagicMock(return_value=['Book 1'])
+ importer.process_verses = MagicMock(return_value=['Verse 1'])
+ importer.wizard = MagicMock()
+
+ # WHEN: Calling do_import
+ result = importer.do_import('Bible Name')
+
+ # THEN: log.exception should not be called, parse_csv_file should be called twice,
+ # and False should be returned.
+ self.assertFalse(mocked_log.exception.called)
+ self.assertEqual(importer.parse_csv_file.mock_calls, [call('books.csv', Book), call('verses.csv', Verse)])
+ importer.process_verses.assert_called_once_with(['Verse 1'], ['Book 1'])
+ self.assertFalse(result)
+
+ # Cleanup
+ del CSVBible.stop_import_flag
+
+ def do_import_success_test(self):
+ """
+ Test do_import when the import succeeds
+ """
+ # GIVEN: An instance of CSVBible
+ mocked_manager = MagicMock()
+ with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\
+ patch('openlp.plugins.bibles.lib.importers.csvbible.log') as mocked_log:
+ importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verses.csv')
+ importer.get_language = MagicMock(return_value=10)
+ importer.parse_csv_file = MagicMock(side_effect=[['Book 1'], ['Verse 1']])
+ importer.process_books = MagicMock(return_value=['Book 1'])
+ importer.process_verses = MagicMock(return_value=['Verse 1'])
+ importer.session = MagicMock()
+ importer.stop_import_flag = False
+ importer.wizard = MagicMock()
+
+ # WHEN: Calling do_import
+ result = importer.do_import('Bible Name')
+
+ # THEN: log.exception should not be called, parse_csv_file should be called twice,
+ # and True should be returned.
+ self.assertFalse(mocked_log.exception.called)
+ self.assertEqual(importer.parse_csv_file.mock_calls, [call('books.csv', Book), call('verses.csv', Verse)])
+ importer.process_books.assert_called_once_with(['Book 1'])
+ importer.process_verses.assert_called_once_with(['Verse 1'], ['Book 1'])
+ self.assertTrue(result)
+
+ def file_import_test(self):
"""
Test the actual import of CSV Bible file
"""
@@ -73,7 +394,7 @@
test_data = json.loads(result_file.read().decode())
books_file = os.path.join(TEST_PATH, 'dk1933-books.csv')
verses_file = os.path.join(TEST_PATH, 'dk1933-verses.csv')
- with patch('openlp.plugins.bibles.lib.csvbible.CSVBible.application'):
+ with patch('openlp.plugins.bibles.lib.importers.csvbible.CSVBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = CSVBible(mocked_manager, path='.', name='.', booksfile=books_file, versefile=verses_file)
=== modified file 'tests/functional/openlp_plugins/bibles/test_http.py'
--- tests/functional/openlp_plugins/bibles/test_http.py 2016-05-31 21:40:13 +0000
+++ tests/functional/openlp_plugins/bibles/test_http.py 2016-08-09 21:01:19 +0000
@@ -26,7 +26,7 @@
from bs4 import BeautifulSoup
from tests.functional import patch, MagicMock
-from openlp.plugins.bibles.lib.http import BSExtract
+from openlp.plugins.bibles.lib.importers.http import BSExtract
# TODO: Items left to test
# BGExtract
@@ -68,11 +68,11 @@
# get_books_from_http
# _get_application
def setUp(self):
- self.get_soup_for_bible_ref_patcher = patch('openlp.plugins.bibles.lib.http.get_soup_for_bible_ref')
- self.log_patcher = patch('openlp.plugins.bibles.lib.http.log')
- self.send_error_message_patcher = patch('openlp.plugins.bibles.lib.http.send_error_message')
- self.socket_patcher = patch('openlp.plugins.bibles.lib.http.socket')
- self.urllib_patcher = patch('openlp.plugins.bibles.lib.http.urllib')
+ self.get_soup_for_bible_ref_patcher = patch('openlp.plugins.bibles.lib.importers.http.get_soup_for_bible_ref')
+ self.log_patcher = patch('openlp.plugins.bibles.lib.importers.http.log')
+ self.send_error_message_patcher = patch('openlp.plugins.bibles.lib.importers.http.send_error_message')
+ self.socket_patcher = patch('openlp.plugins.bibles.lib.importers.http.socket')
+ self.urllib_patcher = patch('openlp.plugins.bibles.lib.importers.http.urllib')
self.mock_get_soup_for_bible_ref = self.get_soup_for_bible_ref_patcher.start()
self.mock_log = self.log_patcher.start()
=== modified file 'tests/functional/openlp_plugins/bibles/test_opensongimport.py'
--- tests/functional/openlp_plugins/bibles/test_opensongimport.py 2016-05-31 21:40:13 +0000
+++ tests/functional/openlp_plugins/bibles/test_opensongimport.py 2016-08-09 21:01:19 +0000
@@ -28,7 +28,7 @@
from unittest import TestCase
from tests.functional import MagicMock, patch
-from openlp.plugins.bibles.lib.opensong import OpenSongBible
+from openlp.plugins.bibles.lib.importers.opensong import OpenSongBible
from openlp.plugins.bibles.lib.db import BibleDB
TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
@@ -72,7 +72,7 @@
result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'opensong-dk1933.xml'
- with patch('openlp.plugins.bibles.lib.opensong.OpenSongBible.application'):
+ with patch('openlp.plugins.bibles.lib.importers.opensong.OpenSongBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = OpenSongBible(mocked_manager, path='.', name='.', filename='')
@@ -98,7 +98,7 @@
Test that we give an error message if trying to import a zefania bible
"""
# GIVEN: A mocked out "manager" and mocked out critical_error_message_box and an import
- with patch('openlp.plugins.bibles.lib.opensong.critical_error_message_box') as \
+ with patch('openlp.plugins.bibles.lib.importers.opensong.critical_error_message_box') as \
mocked_critical_error_message_box:
mocked_manager = MagicMock()
importer = OpenSongBible(mocked_manager, path='.', name='.', filename='')
=== modified file 'tests/functional/openlp_plugins/bibles/test_osisimport.py'
--- tests/functional/openlp_plugins/bibles/test_osisimport.py 2016-05-31 21:40:13 +0000
+++ tests/functional/openlp_plugins/bibles/test_osisimport.py 2016-08-09 21:01:19 +0000
@@ -28,7 +28,7 @@
from unittest import TestCase
from tests.functional import MagicMock, patch
-from openlp.plugins.bibles.lib.osis import OSISBible
+from openlp.plugins.bibles.lib.importers.osis import OSISBible
from openlp.plugins.bibles.lib.db import BibleDB
TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
@@ -72,7 +72,7 @@
result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'osis-dk1933.xml'
- with patch('openlp.plugins.bibles.lib.osis.OSISBible.application'):
+ with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = OSISBible(mocked_manager, path='.', name='.', filename='')
@@ -102,7 +102,7 @@
result_file = open(os.path.join(TEST_PATH, 'kjv.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'osis-kjv.xml'
- with patch('openlp.plugins.bibles.lib.osis.OSISBible.application'):
+ with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = OSISBible(mocked_manager, path='.', name='.', filename='')
@@ -132,7 +132,7 @@
result_file = open(os.path.join(TEST_PATH, 'web.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'osis-web.xml'
- with patch('openlp.plugins.bibles.lib.osis.OSISBible.application'):
+ with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = OSISBible(mocked_manager, path='.', name='.', filename='')
@@ -162,7 +162,7 @@
result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'osis-dk1933-empty-verse.xml'
- with patch('openlp.plugins.bibles.lib.osis.OSISBible.application'):
+ with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = OSISBible(mocked_manager, path='.', name='.', filename='')
=== modified file 'tests/functional/openlp_plugins/bibles/test_swordimport.py'
--- tests/functional/openlp_plugins/bibles/test_swordimport.py 2016-08-03 17:26:10 +0000
+++ tests/functional/openlp_plugins/bibles/test_swordimport.py 2016-08-09 21:01:19 +0000
@@ -29,7 +29,7 @@
from tests.functional import MagicMock, patch
try:
- from openlp.plugins.bibles.lib.sword import SwordBible
+ from openlp.plugins.bibles.lib.importers.sword import SwordBible
HAS_PYSWORD = True
except ImportError:
HAS_PYSWORD = False
@@ -68,8 +68,8 @@
# THEN: The importer should be an instance of BibleDB
self.assertIsInstance(importer, BibleDB)
- @patch('openlp.plugins.bibles.lib.sword.SwordBible.application')
- @patch('openlp.plugins.bibles.lib.sword.modules')
+ @patch('openlp.plugins.bibles.lib.importers.sword.SwordBible.application')
+ @patch('openlp.plugins.bibles.lib.importers.sword.modules')
@patch('openlp.core.common.languages')
def test_simple_import(self, mocked_languages, mocked_pysword_modules, mocked_application):
"""
=== modified file 'tests/functional/openlp_plugins/bibles/test_zefaniaimport.py'
--- tests/functional/openlp_plugins/bibles/test_zefaniaimport.py 2016-05-31 21:40:13 +0000
+++ tests/functional/openlp_plugins/bibles/test_zefaniaimport.py 2016-08-09 21:01:19 +0000
@@ -28,7 +28,7 @@
from unittest import TestCase
from tests.functional import MagicMock, patch
-from openlp.plugins.bibles.lib.zefania import ZefaniaBible
+from openlp.plugins.bibles.lib.importers.zefania import ZefaniaBible
from openlp.plugins.bibles.lib.db import BibleDB
TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
@@ -72,7 +72,7 @@
result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'zefania-dk1933.xml'
- with patch('openlp.plugins.bibles.lib.zefania.ZefaniaBible.application'):
+ with patch('openlp.plugins.bibles.lib.importers.zefania.ZefaniaBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = ZefaniaBible(mocked_manager, path='.', name='.', filename='')
@@ -102,7 +102,7 @@
result_file = open(os.path.join(TEST_PATH, 'rst.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'zefania-rst.xml'
- with patch('openlp.plugins.bibles.lib.zefania.ZefaniaBible.application'):
+ with patch('openlp.plugins.bibles.lib.importers.zefania.ZefaniaBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = ZefaniaBible(mocked_manager, path='.', name='.', filename='')
=== modified file 'tests/interfaces/openlp_plugins/bibles/test_lib_http.py'
--- tests/interfaces/openlp_plugins/bibles/test_lib_http.py 2016-07-24 19:49:29 +0000
+++ tests/interfaces/openlp_plugins/bibles/test_lib_http.py 2016-08-09 21:01:19 +0000
@@ -25,7 +25,7 @@
from unittest import TestCase, skip
from openlp.core.common import Registry
-from openlp.plugins.bibles.lib.http import BGExtract, CWExtract, BSExtract
+from openlp.plugins.bibles.lib.importers.http import BGExtract, CWExtract, BSExtract
from tests.interfaces import MagicMock
Follow ups
