openlp-core team mailing list archive
-
openlp-core team
-
Mailing list archive
-
Message #27716
[Merge] lp:~tomasgroth/openlp/song-import-fixes into lp:openlp
Tomas Groth has proposed merging lp:~tomasgroth/openlp/song-import-fixes into lp:openlp.
Requested reviews:
OpenLP Core (openlp-core)
Related bugs:
Bug #1225760 in OpenLP: "Unsupported fields in WorshipCenter Pro importer"
https://bugs.launchpad.net/openlp/+bug/1225760
Bug #1510282 in OpenLP: "ZionWorks import fails in OpenLP 2.2"
https://bugs.launchpad.net/openlp/+bug/1510282
For more details, see:
https://code.launchpad.net/~tomasgroth/openlp/song-import-fixes/+merge/276032
Made ZionWorx importer work under python3, fixes bug 1510282.
Improved WorshipCenter Pro import, support more fields and verse types.
--
Your team OpenLP Core is requested to review the proposed merge of lp:~tomasgroth/openlp/song-import-fixes into lp:openlp.
=== modified file 'openlp/plugins/songs/lib/importers/worshipcenterpro.py'
--- openlp/plugins/songs/lib/importers/worshipcenterpro.py 2015-01-19 08:34:29 +0000
+++ openlp/plugins/songs/lib/importers/worshipcenterpro.py 2015-10-28 17:07:14 +0000
@@ -24,7 +24,7 @@
a WorshipCenter Pro database into the OpenLP database.
"""
import logging
-
+import re
import pyodbc
from openlp.core.common import translate
@@ -71,8 +71,41 @@
break
self.set_defaults()
self.title = songs[song]['TITLE']
+ if 'AUTHOR' in songs[song]:
+ self.parse_author(songs[song]['AUTHOR'])
+ if 'CCLISONGID' in songs[song]:
+ self.ccli_number = songs[song]['CCLISONGID']
+ if 'COMMENTS' in songs[song]:
+ self.add_comment(songs[song]['COMMENTS'])
+ if 'COPY' in songs[song]:
+ self.add_copyright(songs[song]['COPY'])
+ if 'SUBJECT' in songs[song]:
+ self.topics.append(songs[song]['SUBJECT'])
lyrics = songs[song]['LYRICS'].strip('&crlf;&crlf;')
for verse in lyrics.split('&crlf;&crlf;'):
verse = verse.replace('&crlf;', '\n')
- self.add_verse(verse)
+ marker_type = 'v'
+ # Find verse markers if any
+ marker_start = verse.find('<')
+ if marker_start > -1:
+ marker_end = verse.find('>')
+ marker = verse[marker_start + 1:marker_end]
+ # Identify the marker type
+ if 'REFRAIN' in marker or 'CHORUS' in marker:
+ marker_type = 'c'
+ elif 'BRIDGE' in marker:
+ marker_type = 'b'
+ elif 'PRECHORUS' in marker:
+ marker_type = 'p'
+ elif 'END' in marker:
+ marker_type = 'e'
+ elif 'INTRO' in marker:
+ marker_type = 'i'
+ elif 'TAG' in marker:
+ marker_type = 'o'
+ else:
+ marker_type = 'v'
+ # Strip tags from text
+ verse = re.sub('<[^<]+?>', '', verse)
+ self.add_verse(verse.strip(), marker_type)
self.finish()
=== modified file 'openlp/plugins/songs/lib/importers/zionworx.py'
--- openlp/plugins/songs/lib/importers/zionworx.py 2015-03-09 20:57:39 +0000
+++ openlp/plugins/songs/lib/importers/zionworx.py 2015-10-28 17:07:14 +0000
@@ -75,7 +75,8 @@
"""
Receive a CSV file (from a ZionWorx database dump) to import.
"""
- with open(self.import_source, 'rb') as songs_file:
+ # Encoding should always be ISO-8859-1
+ with open(self.import_source, 'rt', encoding='ISO-8859-1') as songs_file:
field_names = ['SongNum', 'Title1', 'Title2', 'Lyrics', 'Writer', 'Copyright', 'Keywords',
'DefaultStyle']
songs_reader = csv.DictReader(songs_file, field_names)
@@ -112,10 +113,10 @@
if line and not line.isspace():
verse += line + '\n'
elif verse:
- self.add_verse(verse)
+ self.add_verse(verse, 'v')
verse = ''
if verse:
- self.add_verse(verse)
+ self.add_verse(verse, 'v')
title = self.title
if not self.finish():
self.log_error(translate('SongsPlugin.ZionWorxImport', 'Record %d') % index +
@@ -123,8 +124,7 @@
def _decode(self, str):
"""
- Decodes CSV input to unicode, stripping all control characters (except new lines).
+ Strips all control characters (except new lines).
"""
- # This encoding choice seems OK. ZionWorx has no option for setting the
- # encoding for its songs, so we assume encoding is always the same.
- return str(str, 'cp1252').translate(CONTROL_CHARS_MAP)
+ # ZionWorx has no option for setting the encoding for its songs, so we assume encoding is always the same.
+ return str.translate(CONTROL_CHARS_MAP)
=== modified file 'tests/functional/openlp_plugins/songs/test_worshipcenterproimport.py'
--- tests/functional/openlp_plugins/songs/test_worshipcenterproimport.py 2015-01-19 08:34:29 +0000
+++ tests/functional/openlp_plugins/songs/test_worshipcenterproimport.py 2015-10-28 17:07:14 +0000
@@ -67,6 +67,10 @@
RECORDSET_TEST_DATA = [TestRecord(1, 'TITLE', 'Amazing Grace'),
+ TestRecord(1, 'AUTHOR', 'John Newton'),
+ TestRecord(1, 'CCLISONGID', '12345'),
+ TestRecord(1, 'COMMENTS', 'The original version'),
+ TestRecord(1, 'COPY', 'Public Domain'),
TestRecord(
1, 'LYRICS',
'Amazing grace! How&crlf;sweet the sound&crlf;That saved a wretch like me!&crlf;'
@@ -113,7 +117,10 @@
('The earth shall soon\ndissolve like snow,\nThe sun forbear to shine;\nBut God, Who called\n'
'me here below,\nShall be forever mine.'),
('When we\'ve been there\nten thousand years,\nBright shining as the sun,\n'
- 'We\'ve no less days to\nsing God\'s praise\nThan when we\'d first begun.')]},
+ 'We\'ve no less days to\nsing God\'s praise\nThan when we\'d first begun.')],
+ 'author': 'John Newton',
+ 'comments': 'The original version',
+ 'copyright': 'Public Domain'},
{'title': 'Beautiful Garden Of Prayer, The',
'verses': [
('There\'s a garden where\nJesus is waiting,\nThere\'s a place that\nis wondrously fair,\n'
@@ -191,6 +198,9 @@
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
mocked_add_verse = MagicMock()
+ mocked_parse_author = MagicMock()
+ mocked_add_comment = MagicMock()
+ mocked_add_copyright = MagicMock()
mocked_finish = MagicMock()
mocked_pyodbc.connect().cursor().fetchall.return_value = RECORDSET_TEST_DATA
mocked_translate.return_value = 'Translated Text'
@@ -198,6 +208,9 @@
importer.import_source = 'import_source'
importer.import_wizard = mocked_import_wizard
importer.add_verse = mocked_add_verse
+ importer.parse_author = mocked_parse_author
+ importer.add_comment = mocked_add_comment
+ importer.add_copyright = mocked_add_copyright
importer.stop_import_flag = False
importer.finish = mocked_finish
@@ -220,6 +233,12 @@
verse_calls = song_data['verses']
add_verse_call_count += len(verse_calls)
for call in verse_calls:
- mocked_add_verse.assert_any_call(call)
+ mocked_add_verse.assert_any_call(call, 'v')
+ if 'author' in song_data:
+ mocked_parse_author.assert_any_call(song_data['author'])
+ if 'comments' in song_data:
+ mocked_add_comment.assert_any_call(song_data['comments'])
+ if 'copyright' in song_data:
+ mocked_add_copyright.assert_any_call(song_data['copyright'])
self.assertEqual(mocked_add_verse.call_count, add_verse_call_count,
'Incorrect number of calls made to add_verse')
=== modified file 'tests/functional/openlp_plugins/songs/test_zionworximport.py'
--- tests/functional/openlp_plugins/songs/test_zionworximport.py 2015-01-18 13:39:21 +0000
+++ tests/functional/openlp_plugins/songs/test_zionworximport.py 2015-10-28 17:07:14 +0000
@@ -22,14 +22,19 @@
"""
This module contains tests for the ZionWorx song importer.
"""
+import os
from unittest import TestCase
from tests.functional import MagicMock, patch
+from tests.helpers.songfileimport import SongImportTestHelper
from openlp.plugins.songs.lib.importers.zionworx import ZionWorxImport
from openlp.plugins.songs.lib.importers.songimport import SongImport
from openlp.core.common import Registry
+TEST_PATH = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'zionworxsongs'))
+
class TestZionWorxImport(TestCase):
"""
@@ -54,3 +59,18 @@
# THEN: The importer should be an instance of SongImport
self.assertIsInstance(importer, SongImport)
+
+
+class TestZionWorxFileImport(SongImportTestHelper):
+
+ def __init__(self, *args, **kwargs):
+ self.importer_class_name = 'ZionWorxImport'
+ self.importer_module_name = 'zionworx'
+ super(TestZionWorxFileImport, self).__init__(*args, **kwargs)
+
+ def test_song_import(self):
+ """
+ Test that loading an ZionWorx file works correctly on various files
+ """
+ self.file_import(os.path.join(TEST_PATH, 'zionworx.csv'),
+ self.load_external_result_data(os.path.join(TEST_PATH, 'zionworx.json')))
=== added directory 'tests/resources/zionworxsongs'
=== added file 'tests/resources/zionworxsongs/zionworx.csv'
--- tests/resources/zionworxsongs/zionworx.csv 1970-01-01 00:00:00 +0000
+++ tests/resources/zionworxsongs/zionworx.csv 2015-10-28 17:07:14 +0000
@@ -0,0 +1,45 @@
+"1","Crown Him With Many Crowns",,"Crown him with many crowns,
+The Lamb upon His throne;
+Hark, how the heavenly anthem drowns
+All music but its own!
+Awake, my soul, and sing
+Of Him who died for thee,
+And hail Him as thy matchless King
+Through all eternity.
+
+Crown Him the Lord of life,
+Who triumphed o'er the grave
+And rose victorious in the strife
+For those He came to save:
+His glories now we sing,
+Who died and rose on high,
+Who died eternal life to bring
+And lives that death may die.
+
+Crown Him the Lord of love;
+Behold His hands and side,
+Those wounds yet visible above
+In beauty glorified:
+No angel in the sky
+Can fully bear that sight,
+But downward bends His burning eye
+At mysteries so bright.
+
+Crown Him the Lord of peace,
+Whose power a sceptre sways
+From pole to pole, that wars may cease,
+And all be prayer and praise:
+His reign shall know no end,
+And round His pierc�feet
+Fair flowers of paradise extend
+Their fragrance ever sweet.
+
+Crown Him the Lord of years,
+The Potentate of time,
+Creator of the rolling spheres,
+Ineffably sublime!
+All hail, Redeemer, hail!
+For Thou hast died for me;
+Thy praise shall never, never fail
+Throughout eternity.
+","Matthew Bridges","Public Domain",,
=== added file 'tests/resources/zionworxsongs/zionworx.json'
--- tests/resources/zionworxsongs/zionworx.json 1970-01-01 00:00:00 +0000
+++ tests/resources/zionworxsongs/zionworx.json 2015-10-28 17:07:14 +0000
@@ -0,0 +1,30 @@
+{
+ "authors": [
+ "Matthew Bridges"
+ ],
+ "copyright": "Public Domain",
+ "title": "Crown Him With Many Crowns",
+ "verse_order_list": [],
+ "verses": [
+ [
+ "Crown him with many crowns,\nThe Lamb upon His throne;\nHark, how the heavenly anthem drowns\nAll music but its own!\nAwake, my soul, and sing\nOf Him who died for thee,\nAnd hail Him as thy matchless King\nThrough all eternity.\n",
+ "v"
+ ],
+ [
+ "Crown Him the Lord of life,\nWho triumphed o'er the grave\nAnd rose victorious in the strife\nFor those He came to save:\nHis glories now we sing,\nWho died and rose on high,\nWho died eternal life to bring\nAnd lives that death may die.\n",
+ "v"
+ ],
+ [
+ "Crown Him the Lord of love;\nBehold His hands and side,\nThose wounds yet visible above\nIn beauty glorified:\nNo angel in the sky\nCan fully bear that sight,\nBut downward bends His burning eye\nAt mysteries so bright.\n",
+ "v"
+ ],
+ [
+ "Crown Him the Lord of peace,\nWhose power a sceptre sways\nFrom pole to pole, that wars may cease,\nAnd all be prayer and praise:\nHis reign shall know no end,\nAnd round His piercèd feet\nFair flowers of paradise extend\nTheir fragrance ever sweet.\n",
+ "v"
+ ],
+ [
+ "Crown Him the Lord of years,\nThe Potentate of time,\nCreator of the rolling spheres,\nIneffably sublime!\nAll hail, Redeemer, hail!\nFor Thou hast died for me;\nThy praise shall never, never fail\nThroughout eternity.\n",
+ "v"
+ ]
+ ]
+}
Follow ups
