openlp-core team mailing list archive
-
openlp-core team
-
Mailing list archive
-
Message #31376
[Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
Gary Talent has proposed merging lp:~gtalent/openlp/easyworship6 into lp:openlp.
Requested reviews:
Tomas Groth (tomasgroth)
Raoul Snyman (raoul-snyman)
Related bugs:
Bug #1675641 in OpenLP: "EasyWorship 6 uses SQLite "
https://bugs.launchpad.net/openlp/+bug/1675641
For more details, see:
https://code.launchpad.net/~gtalent/openlp/easyworship6/+merge/321504
I removed the empty lines and added an option for choosing a directory at any level of the EW6 database directory.
lp:~gtalent/openlp/openlp (revision 2704)
[SUCCESS] https://ci.openlp.io/job/Branch-01-Pull/1960/
[SUCCESS] https://ci.openlp.io/job/Branch-02-Functional-Tests/1871/
[SUCCESS] https://ci.openlp.io/job/Branch-03-Interface-Tests/1812/
[SUCCESS] https://ci.openlp.io/job/Branch-04a-Windows_Functional_Tests/1538/
[SUCCESS] https://ci.openlp.io/job/Branch-04b-Windows_Interface_Tests/1128/
[SUCCESS] https://ci.openlp.io/job/Branch-05a-Code_Analysis/1196/
[SUCCESS] https://ci.openlp.io/job/Branch-05b-Test_Coverage/1064/
[SUCCESS] https://ci.openlp.io/job/Branch-05c-Code_Analysis2/195/
----
This adds support for importing EasyWorship 6 databases. It's been a while since I made the changes though, so I had to merge in the trunk changes beforehand to get it to pass the Jenkins tests.
lp:~gtalent/openlp/openlp (revision 2701)
[SUCCESS] https://ci.openlp.io/job/Branch-01-Pull/1955/
[SUCCESS] https://ci.openlp.io/job/Branch-02-Functional-Tests/1866/
[SUCCESS] https://ci.openlp.io/job/Branch-03-Interface-Tests/1807/
[SUCCESS] https://ci.openlp.io/job/Branch-04a-Windows_Functional_Tests/1533/
[SUCCESS] https://ci.openlp.io/job/Branch-04b-Windows_Interface_Tests/1123/
[SUCCESS] https://ci.openlp.io/job/Branch-05a-Code_Analysis/1191/
[SUCCESS] https://ci.openlp.io/job/Branch-05b-Test_Coverage/1059/
[SUCCESS] https://ci.openlp.io/job/Branch-05c-Code_Analysis2/190/
--
Your team OpenLP Core is subscribed to branch lp:openlp.
=== modified file 'openlp/plugins/songs/lib/__init__.py'
--- openlp/plugins/songs/lib/__init__.py 2017-01-15 21:12:55 +0000
+++ openlp/plugins/songs/lib/__init__.py 2017-03-31 01:18:28 +0000
@@ -512,10 +512,13 @@
elif not ignorable:
ebytes.append(int(hex_, 16))
elif tchar:
- if curskip > 0:
- curskip -= 1
- elif not ignorable:
+ if not ignorable:
ebytes += tchar.encode()
+ if len(ebytes) >= curskip:
+ ebytes = ebytes[curskip:]
+ else:
+ curskip -= len(ebytes)
+ ebytes = ""
text = ''.join(out)
return text, default_encoding
=== modified file 'openlp/plugins/songs/lib/importer.py'
--- openlp/plugins/songs/lib/importer.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importer.py 2017-03-31 01:18:28 +0000
@@ -158,26 +158,27 @@
DreamBeam = 4
EasySlides = 5
EasyWorshipDB = 6
- EasyWorshipService = 7
- FoilPresenter = 8
- Lyrix = 9
- MediaShout = 10
- OpenSong = 11
- OPSPro = 12
- PowerPraise = 13
- PowerSong = 14
- PresentationManager = 15
- ProPresenter = 16
- SongBeamer = 17
- SongPro = 18
- SongShowPlus = 19
- SongsOfFellowship = 20
- SundayPlus = 21
- VideoPsalm = 22
- WordsOfWorship = 23
- WorshipAssistant = 24
- WorshipCenterPro = 25
- ZionWorx = 26
+ EasyWorshipSqliteDB = 7
+ EasyWorshipService = 8
+ FoilPresenter = 9
+ Lyrix = 10
+ MediaShout = 11
+ OpenSong = 12
+ OPSPro = 13
+ PowerPraise = 14
+ PowerSong = 15
+ PresentationManager = 16
+ ProPresenter = 17
+ SongBeamer = 18
+ SongPro = 19
+ SongShowPlus = 20
+ SongsOfFellowship = 21
+ SundayPlus = 22
+ VideoPsalm = 23
+ WordsOfWorship = 24
+ WorshipAssistant = 25
+ WorshipCenterPro = 26
+ ZionWorx = 27
# Set optional attribute defaults
__defaults__ = {
@@ -242,8 +243,16 @@
'name': 'EasyWorship Song Database',
'prefix': 'ew',
'selectMode': SongFormatSelect.SingleFile,
+ 'filter': '{text} (*.DB)'.format(text=translate('SongsPlugin.ImportWizardForm',
+ 'EasyWorship Song Database'))
+ },
+ EasyWorshipSqliteDB: {
+ 'class': EasyWorshipSongImport,
+ 'name': 'EasyWorship 6 Song Database',
+ 'prefix': 'ew',
+ 'selectMode': SongFormatSelect.SingleFolder,
'filter': '{text} (*.db)'.format(text=translate('SongsPlugin.ImportWizardForm',
- 'EasyWorship Song Database'))
+ 'EasyWorship 6 Song Data Directory'))
},
EasyWorshipService: {
'class': EasyWorshipSongImport,
@@ -430,6 +439,7 @@
SongFormat.DreamBeam,
SongFormat.EasySlides,
SongFormat.EasyWorshipDB,
+ SongFormat.EasyWorshipSqliteDB,
SongFormat.EasyWorshipService,
SongFormat.FoilPresenter,
SongFormat.Lyrix,
=== modified file 'openlp/plugins/songs/lib/importers/easyworship.py'
--- openlp/plugins/songs/lib/importers/easyworship.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/easyworship.py 2017-03-31 01:18:28 +0000
@@ -28,6 +28,7 @@
import re
import zlib
import logging
+import sqlite3
from openlp.core.lib import translate
from openlp.plugins.songs.lib import VerseType
@@ -77,8 +78,10 @@
"""
if self.import_source.lower().endswith('ews'):
self.import_ews()
- else:
+ elif self.import_source.endswith('DB'):
self.import_db()
+ else:
+ self.import_sqlite_db()
def import_ews(self):
"""
@@ -125,8 +128,8 @@
else:
log.debug('Given ews file is of unknown version.')
return
- entry_count = self.get_i32(file_pos)
- entry_length = self.get_i16(file_pos + 4)
+ entry_count = self.ews_get_i32(file_pos)
+ entry_length = self.ews_get_i16(file_pos + 4)
file_pos += 6
self.import_wizard.progress_bar.setMaximum(entry_count)
# Loop over songs
@@ -144,13 +147,13 @@
# 0x08 = Audio, 0x09 = Web
# 1410 Song number cstring 10
self.set_defaults()
- self.title = self.get_string(file_pos + 0, 50)
- authors = self.get_string(file_pos + 307, 50)
- copyright = self.get_string(file_pos + 358, 100)
- admin = self.get_string(file_pos + 459, 50)
- cont_ptr = self.get_i32(file_pos + 800)
- cont_type = self.get_i32(file_pos + 820)
- self.ccli_number = self.get_string(file_pos + 1410, 10)
+ self.title = self.ews_get_string(file_pos + 0, 50)
+ authors = self.ews_get_string(file_pos + 307, 50)
+ copyright = self.ews_get_string(file_pos + 358, 100)
+ admin = self.ews_get_string(file_pos + 459, 50)
+ cont_ptr = self.ews_get_i32(file_pos + 800)
+ cont_type = self.ews_get_i32(file_pos + 820)
+ self.ccli_number = self.ews_get_string(file_pos + 1410, 10)
# Only handle content type 1 (songs)
if cont_type != 1:
file_pos += entry_length
@@ -164,9 +167,9 @@
# Checksum int32be 4 Alder-32 checksum.
# (unknown) 4 0x51 0x4b 0x03 0x04
# Content length int32le 4 Length of content after decompression
- content_length = self.get_i32(cont_ptr)
- deflated_content = self.get_bytes(cont_ptr + 4, content_length - 10)
- deflated_length = self.get_i32(cont_ptr + 4 + content_length - 6)
+ content_length = self.ews_get_i32(cont_ptr)
+ deflated_content = self.ews_get_bytes(cont_ptr + 4, content_length - 10)
+ deflated_length = self.ews_get_i32(cont_ptr + 4 + content_length - 6)
inflated_content = zlib.decompress(deflated_content, 15, deflated_length)
if copyright:
self.copyright = copyright
@@ -196,7 +199,7 @@
Import the songs from the database
"""
# Open the DB and MB files if they exist
- import_source_mb = self.import_source.replace('.DB', '.MB').replace('.db', '.mb')
+ import_source_mb = self.import_source.replace('.DB', '.MB')
if not os.path.isfile(self.import_source):
self.log_error(self.import_source, translate('SongsPlugin.EasyWorshipSongImport',
'This file does not exist.'))
@@ -260,16 +263,16 @@
for i, field_name in enumerate(field_names):
field_type, field_size = struct.unpack_from('BB', field_info, i * 2)
field_descriptions.append(FieldDescEntry(field_name, field_type, field_size))
- self.set_record_struct(field_descriptions)
+ self.db_set_record_struct(field_descriptions)
# Pick out the field description indexes we will need
try:
success = True
- fi_title = self.find_field(b'Title')
- fi_author = self.find_field(b'Author')
- fi_copy = self.find_field(b'Copyright')
- fi_admin = self.find_field(b'Administrator')
- fi_words = self.find_field(b'Words')
- fi_ccli = self.find_field(b'Song Number')
+ fi_title = self.db_find_field(b'Title')
+ fi_author = self.db_find_field(b'Author')
+ fi_copy = self.db_find_field(b'Copyright')
+ fi_admin = self.db_find_field(b'Administrator')
+ fi_words = self.db_find_field(b'Words')
+ fi_ccli = self.db_find_field(b'Song Number')
except IndexError:
# This is the wrong table
success = False
@@ -297,13 +300,13 @@
raw_record = db_file.read(record_size)
self.fields = self.record_structure.unpack(raw_record)
self.set_defaults()
- self.title = self.get_field(fi_title).decode(self.encoding)
+ self.title = self.db_get_field(fi_title).decode(self.encoding)
# Get remaining fields.
- copy = self.get_field(fi_copy)
- admin = self.get_field(fi_admin)
- ccli = self.get_field(fi_ccli)
- authors = self.get_field(fi_author)
- words = self.get_field(fi_words)
+ copy = self.db_get_field(fi_copy)
+ admin = self.db_get_field(fi_admin)
+ ccli = self.db_get_field(fi_ccli)
+ authors = self.db_get_field(fi_author)
+ words = self.db_get_field(fi_words)
if copy:
self.copyright = copy.decode(self.encoding)
if admin:
@@ -337,6 +340,84 @@
db_file.close()
self.memo_file.close()
+ def _find_file(self, base_path, path_list):
+ """
+ Find the specified file, with the option of the file being at any level in the specified directory structure.
+
+ :param base_path: the location search in
+ :param path_list: the targeted file, preceded by directories that may be their parents relative to the base_path
+ :return: path for targeted file
+ """
+ target_file = ''
+ while len(path_list) > 0:
+ target_file = os.path.join(path_list[len(path_list) - 1], target_file)
+ path_list = path_list[:len(path_list) - 1]
+ full_path = os.path.join(base_path, target_file)
+ full_path = full_path[:len(full_path) - 1]
+ if os.path.isfile(full_path):
+ return full_path
+ return ''
+
+ def import_sqlite_db(self):
+ """
+ Import the songs from an EasyWorship 6 SQLite database
+ """
+ songs_db_path = self._find_file(self.import_source, ["Databases", "Data", "Songs.db"])
+ song_words_db_path = self._find_file(self.import_source, ["Databases", "Data", "SongWords.db"])
+ # check to see if needed files are there
+ if not os.path.isfile(songs_db_path):
+ self.log_error(songs_db_path, translate('SongsPlugin.EasyWorshipSongImport',
+ 'This file does not exist.'))
+ return
+ if not os.path.isfile(song_words_db_path):
+ self.log_error(song_words_db_path, translate('SongsPlugin.EasyWorshipSongImport',
+ 'Could not find the "Songs.MB" file. It must be in the same '
+ 'folder as the "Songs.DB" file.'))
+ return
+ # get database handles
+ songs_conn = sqlite3.connect(songs_db_path)
+ words_conn = sqlite3.connect(song_words_db_path)
+ if songs_conn is None or words_conn is None:
+ self.log_error(self.import_source, translate('SongsPlugin.EasyWorshipSongImport',
+ 'This is not a valid Easy Worship 6 database.'))
+ songs_conn.close()
+ words_conn.close()
+ return
+ songs_db = songs_conn.cursor()
+ words_db = words_conn.cursor()
+ if songs_conn is None or words_conn is None:
+ self.log_error(self.import_source, translate('SongsPlugin.EasyWorshipSongImport',
+ 'This is not a valid Easy Worship 6 database.'))
+ songs_conn.close()
+ words_conn.close()
+ return
+ # Take a stab at how text is encoded
+ self.encoding = 'cp1252'
+ self.encoding = retrieve_windows_encoding(self.encoding)
+ if not self.encoding:
+ log.debug('No encoding set.')
+ return
+ # import songs
+ songs = songs_db.execute('SELECT rowid,title,author,copyright,vendor_id FROM song;')
+ for song in songs:
+ song_id = song[0]
+ # keep extra copy of title for error message because error check clears it
+ self.title = title = song[1]
+ self.author = song[2]
+ self.copyright = song[3]
+ self.ccli_number = song[4]
+ words = words_db.execute('SELECT words FROM word WHERE song_id = ?;', (song_id,))
+ self.set_song_import_object(self.author, words.fetchone()[0].encode())
+ if not self.finish():
+ self.log_error(self.import_source,
+ translate('SongsPlugin.EasyWorshipSongImport',
+ '"{title}" could not be imported. {entry}').
+ format(title=title, entry=self.entry_error_log))
+ # close database handles
+ songs_conn.close()
+ words_conn.close()
+ return
+
def set_song_import_object(self, authors, words):
"""
Set the SongImport object members.
@@ -409,7 +490,7 @@
self.comments += str(translate('SongsPlugin.EasyWorshipSongImport',
'\n[above are Song Tags with notes imported from EasyWorship]'))
- def find_field(self, field_name):
+ def db_find_field(self, field_name):
"""
Find a field in the descriptions
@@ -417,7 +498,7 @@
"""
return [i for i, x in enumerate(self.field_descriptions) if x.name == field_name][0]
- def set_record_struct(self, field_descriptions):
+ def db_set_record_struct(self, field_descriptions):
"""
Save the record structure
@@ -445,7 +526,7 @@
self.record_structure = struct.Struct(''.join(fsl))
self.field_descriptions = field_descriptions
- def get_field(self, field_desc_index):
+ def db_get_field(self, field_desc_index):
"""
Extract the field
@@ -489,7 +570,7 @@
else:
return 0
- def get_bytes(self, pos, length):
+ def ews_get_bytes(self, pos, length):
"""
Get bytes from ews_file
@@ -500,7 +581,7 @@
self.ews_file.seek(pos)
return self.ews_file.read(length)
- def get_string(self, pos, length):
+ def ews_get_string(self, pos, length):
"""
Get string from ews_file
@@ -508,12 +589,12 @@
:param length: Characters to read
:return: String read
"""
- bytes = self.get_bytes(pos, length)
+ bytes = self.ews_get_bytes(pos, length)
mask = '<' + str(length) + 's'
byte_str, = struct.unpack(mask, bytes)
return byte_str.decode(self.encoding).replace('\0', '').strip()
- def get_i16(self, pos):
+ def ews_get_i16(self, pos):
"""
Get short int from ews_file
@@ -521,19 +602,19 @@
:return: Short integer read
"""
- bytes = self.get_bytes(pos, 2)
+ bytes = self.ews_get_bytes(pos, 2)
mask = '<h'
number, = struct.unpack(mask, bytes)
return number
- def get_i32(self, pos):
+ def ews_get_i32(self, pos):
"""
Get long int from ews_file
:param pos: Position to read from
:return: Long integer read
"""
- bytes = self.get_bytes(pos, 4)
+ bytes = self.ews_get_bytes(pos, 4)
mask = '<i'
number, = struct.unpack(mask, bytes)
return number
=== modified file 'tests/functional/openlp_plugins/songs/test_ewimport.py'
--- tests/functional/openlp_plugins/songs/test_ewimport.py 2016-12-31 11:01:36 +0000
+++ tests/functional/openlp_plugins/songs/test_ewimport.py 2017-03-31 01:18:28 +0000
@@ -188,7 +188,7 @@
def test_find_field_exists(self):
"""
- Test finding an existing field in a given list using the :mod:`find_field`
+ Test finding an existing field in a given list using the :mod:`db_find_field`
"""
# GIVEN: A mocked out SongImport class, a mocked out "manager" and a list of field descriptions.
with patch('openlp.plugins.songs.lib.importers.easyworship.SongImport'):
@@ -202,11 +202,11 @@
for field_name in existing_fields:
# THEN: The item corresponding the index returned should have the same name attribute
- self.assertEqual(importer.field_descriptions[importer.find_field(field_name)].name, field_name)
+ self.assertEqual(importer.field_descriptions[importer.db_find_field(field_name)].name, field_name)
def test_find_non_existing_field(self):
"""
- Test finding an non-existing field in a given list using the :mod:`find_field`
+ Test finding an non-existing field in a given list using the :mod:`db_find_field`
"""
# GIVEN: A mocked out SongImport class, a mocked out "manager" and a list of field descriptions
with patch('openlp.plugins.songs.lib.importers.easyworship.SongImport'):
@@ -219,11 +219,11 @@
for field_name in non_existing_fields:
# THEN: The importer object should not be None
- self.assertRaises(IndexError, importer.find_field, field_name)
+ self.assertRaises(IndexError, importer.db_find_field, field_name)
def test_set_record_struct(self):
"""
- Test the :mod:`set_record_struct` module
+ Test the :mod:`db_set_record_struct` module
"""
# GIVEN: A mocked out SongImport class, a mocked out struct class, and a mocked out "manager" and a list of
# field descriptions
@@ -232,17 +232,17 @@
mocked_manager = MagicMock()
importer = EasyWorshipSongImport(mocked_manager, filenames=[])
- # WHEN: set_record_struct is called with a list of field descriptions
- return_value = importer.set_record_struct(TEST_FIELD_DESCS)
+ # WHEN: db_set_record_struct is called with a list of field descriptions
+ return_value = importer.db_set_record_struct(TEST_FIELD_DESCS)
- # THEN: set_record_struct should return None and Struct should be called with a value representing
+ # THEN: db_set_record_struct should return None and Struct should be called with a value representing
# the list of field descriptions
- self.assertIsNone(return_value, 'set_record_struct should return None')
+ self.assertIsNone(return_value, 'db_set_record_struct should return None')
mocked_struct.Struct.assert_called_with('>50sHIB250s250s10sQ')
def test_get_field(self):
"""
- Test the :mod:`get_field` module
+ Test the :mod:`db_get_field` module
"""
# GIVEN: A mocked out SongImport class, a mocked out "manager", an encoding and some test data and known results
with patch('openlp.plugins.songs.lib.importers.easyworship.SongImport'):
@@ -255,16 +255,16 @@
# WHEN: Called with test data
for field_index, result in field_results:
- return_value = importer.get_field(field_index)
+ return_value = importer.db_get_field(field_index)
- # THEN: get_field should return the known results
+ # THEN: db_get_field should return the known results
self.assertEqual(return_value, result,
- 'get_field should return "%s" when called with "%s"' %
+ 'db_get_field should return "%s" when called with "%s"' %
(result, TEST_FIELDS[field_index]))
def test_get_memo_field(self):
"""
- Test the :mod:`get_field` module
+ Test the :mod:`db_get_field` module
"""
for test_results in GET_MEMO_FIELD_TEST_RESULTS:
# GIVEN: A mocked out SongImport class, a mocked out "manager", a mocked out memo_file and an encoding
@@ -284,8 +284,9 @@
get_field_read_calls = test_results[2]['read']
get_field_seek_calls = test_results[2]['seek']
- # THEN: get_field should return the appropriate value with the appropriate mocked objects being called
- self.assertEqual(importer.get_field(field_index), get_field_result)
+ # THEN: db_get_field should return the appropriate value with the appropriate mocked objects being
+ # called
+ self.assertEqual(importer.db_get_field(field_index), get_field_result)
for call in get_field_read_calls:
mocked_memo_file.read.assert_any_call(call)
for call in get_field_seek_calls:
@@ -406,6 +407,12 @@
mocked_retrieve_windows_encoding.assert_any_call(encoding)
def test_db_file_import(self):
+ return self._test_db_file_import(os.path.join(TEST_PATH, 'Songs.DB'))
+
+ def test_sqlite_db_file_import(self):
+ return self._test_db_file_import(os.path.join(TEST_PATH, 'ew6'))
+
+ def _test_db_file_import(self, source_path):
"""
Test the actual import of real song database files and check that the imported data is correct.
"""
@@ -433,7 +440,7 @@
importer.topics = []
# WHEN: Importing each file
- importer.import_source = os.path.join(TEST_PATH, 'Songs.DB')
+ importer.import_source = source_path
import_result = importer.do_import()
# THEN: do_import should return none, the song data should be as expected, and finish should have been
=== added directory 'tests/resources/easyworshipsongs/ew6'
=== added directory 'tests/resources/easyworshipsongs/ew6/Databases'
=== added directory 'tests/resources/easyworshipsongs/ew6/Databases/Data'
=== added file 'tests/resources/easyworshipsongs/ew6/Databases/Data/SongWords.db'
Binary files tests/resources/easyworshipsongs/ew6/Databases/Data/SongWords.db 1970-01-01 00:00:00 +0000 and tests/resources/easyworshipsongs/ew6/Databases/Data/SongWords.db 2017-03-31 01:18:28 +0000 differ
=== added file 'tests/resources/easyworshipsongs/ew6/Databases/Data/Songs.db'
Binary files tests/resources/easyworshipsongs/ew6/Databases/Data/Songs.db 1970-01-01 00:00:00 +0000 and tests/resources/easyworshipsongs/ew6/Databases/Data/Songs.db 2017-03-31 01:18:28 +0000 differ
Follow ups
-
[Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: noreply, 2017-05-22
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Raoul Snyman, 2017-05-22
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Tomas Groth, 2017-05-18
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Gary Talent, 2017-04-27
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Raoul Snyman, 2017-04-27
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Gary Talent, 2017-04-01
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Gary Talent, 2017-04-01
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Gary Talent, 2017-04-01
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Gary Talent, 2017-04-01
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Raoul Snyman, 2017-04-01
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Gary Talent, 2017-03-31
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Raoul Snyman, 2017-03-31
-
Re: [Merge] lp:~gtalent/openlp/easyworship6 into lp:openlp
From: Raoul Snyman, 2017-03-31
