← Back to team overview

openlp-core team mailing list archive

[Merge] lp:~tomasgroth/openlp/ews-import into lp:openlp

 

Tomas Groth has proposed merging lp:~tomasgroth/openlp/ews-import into lp:openlp.

Requested reviews:
  Tim Bentley (trb143)
Related bugs:
  Bug #1290248 in OpenLP: "Support for importing songs from EWS files"
  https://bugs.launchpad.net/openlp/+bug/1290248

For more details, see:
https://code.launchpad.net/~tomasgroth/openlp/ews-import/+merge/214370

Support for importing ews files (EasyWorship Service files).
-- 
https://code.launchpad.net/~tomasgroth/openlp/ews-import/+merge/214370
Your team OpenLP Core is subscribed to branch lp:openlp.
=== modified file 'openlp/plugins/songs/lib/ewimport.py'
--- openlp/plugins/songs/lib/ewimport.py	2014-03-29 13:31:28 +0000
+++ openlp/plugins/songs/lib/ewimport.py	2014-04-05 07:12:35 +0000
@@ -34,13 +34,13 @@
 import os
 import struct
 import re
+import zlib
 
 from openlp.core.lib import translate
 from openlp.plugins.songs.lib import VerseType
 from openlp.plugins.songs.lib import retrieve_windows_encoding, strip_rtf
 from .songimport import SongImport
 
-RTF_STRIPPING_REGEX = re.compile(r'\{\\tx[^}]*\}')
 # regex: at least two newlines, can have spaces between them
 SLIDE_BREAK_REGEX = re.compile(r'\n *?\n[\n ]*')
 NUMBER_REGEX = re.compile(r'[0-9]+')
@@ -77,7 +77,125 @@
 
     def do_import(self):
         """
-        Import the songs
+        Determines the type of file to import and calls the appropiate method
+
+        :return:
+        """
+        if self.import_source.lower().endswith('ews'):
+            self.import_ews()
+        else:
+            self.import_db()
+
+    def import_ews(self):
+        """
+        Import the songs from service file
+        The full spec of the ews files can be found here:
+        https://github.com/meinders/lithium-ews/blob/master/docs/ews%20file%20format.md
+        or here: http://wiki.openlp.org/Development:EasyWorship_EWS_Format
+
+        :return:
+        """
+        # Open ews file if it exists
+        if not os.path.isfile(self.import_source):
+            log.debug('Given ews file does not exists.')
+            return
+        # Make sure there is room for at least a header and one entry
+        if os.path.getsize(self.import_source) < 892:
+            log.debug('Given ews file is to small to contain valid data.')
+            return
+        # Take a stab at how text is encoded
+        self.encoding = 'cp1252'
+        self.encoding = retrieve_windows_encoding(self.encoding)
+        if not self.encoding:
+            log.debug('No encoding set.')
+            return
+        self.ews_file = open(self.import_source, 'rb')
+        # EWS header, version '1.6'/'  3'/'  5':
+        # Offset   Field             Data type    Length    Details
+        # --------------------------------------------------------------------------------------------------
+        #       0  Filetype           string           38    Specifies the file type and version.
+        #                                                   "EasyWorship Schedule File Version  1.6" or
+        #                                                   "EasyWorship Schedule File Version    3" or
+        #                                                   "EasyWorship Schedule File Version    5"
+        # 40/48/56 Entry count        int32le           4    Number of items in the schedule
+        # 44/52/60 Entry length       int16le           2    Length of schedule entries: 0x0718 = 1816
+        # Get file version
+        type, = struct.unpack('<38s', self.ews_file.read(38))
+        version = type.decode()[-3:]
+        # Set fileposition based on filetype/version
+        file_pos = 0
+        if version == '  5':
+            file_pos = 56
+        elif version == '  3':
+            file_pos = 48
+        elif version == '1.6':
+            file_pos = 40
+        else:
+            log.debug('Given ews file is of unknown version.')
+            return
+        entry_count = self.get_i32(file_pos)
+        entry_length = self.get_i16(file_pos+4)
+        file_pos += 6
+        self.import_wizard.progress_bar.setMaximum(entry_count)
+        # Loop over songs
+        for i in range(entry_count):
+            # Load EWS entry metadata:
+            # Offset  Field                  Data type    Length    Details
+            # ------------------------------------------------------------------------------------------------
+            #      0  Title                  cstring          50
+            #    307  Author                 cstring          50    
+            #    358  Copyright              cstring         100
+            #    459  Administrator          cstring          50
+            #    800  Content pointer        int32le           4    Position of the content for this entry.
+            #    820  Content type           int32le           4    0x01 = Song, 0x02 = Scripture, 0x03 = Presentation,
+            #                                                       0x04 = Video, 0x05 = Live video, 0x07 = Image,
+            #                                                       0x08 = Audio, 0x09 = Web
+            #   1410  Song number            cstring          10
+            self.set_defaults()
+            self.title = self.get_string(file_pos + 0, 50)
+            authors = self.get_string(file_pos + 307, 50)
+            copyright = self.get_string(file_pos + 358, 100)
+            admin = self.get_string(file_pos + 459, 50)
+            cont_ptr = self.get_i32(file_pos + 800)
+            cont_type = self.get_i32(file_pos + 820)
+            self.ccli_number = self.get_string(file_pos + 1410, 10)
+            # Only handle content type 1 (songs)
+            if cont_type != 1:
+                file_pos += entry_length
+                continue
+            # Load song content
+            # Offset  Field              Data type    Length    Details
+            # ------------------------------------------------------------------------------------------------
+            #      0  Length             int32le           4    Length (L) of the content, including the compressed content
+            #                                                   and the following fields (14 bytes total).
+            #      4  Content            string         L-14    Content compressed with deflate.
+            #         Checksum           int32be           4    Alder-32 checksum.
+            #         (unknown)                            4    0x51 0x4b 0x03 0x04
+            #         Content length     int32le           4    Length of content after decompression
+            content_length = self.get_i32(cont_ptr)
+            deflated_content = self.get_bytes(cont_ptr + 4, content_length - 10)
+            deflated_length = self.get_i32(cont_ptr + 4 + content_length - 6)
+            inflated_content = zlib.decompress(deflated_content, 15, deflated_length)
+            if copyright:
+                self.copyright = copyright
+            if admin:
+                if copyright:
+                    self.copyright += ', '
+                self.copyright += translate('SongsPlugin.EasyWorshipSongImport',
+                                            'Administered by %s') % admin
+            # Set the SongImport object members.
+            self.set_song_import_object(authors, inflated_content)
+            if self.stop_import_flag:
+                break
+            if not self.finish():
+                self.log_error(self.import_source)
+            # Set file_pos for next entry
+            file_pos += entry_length
+        self.ews_file.close()
+
+    def import_db(self):
+        """
+        Import the songs from the database
 
         :return:
         """
@@ -176,7 +294,6 @@
                 ccli = self.get_field(fi_ccli)
                 authors = self.get_field(fi_author)
                 words = self.get_field(fi_words)
-                # Set the SongImport object members.
                 if copy:
                     self.copyright = copy.decode()
                 if admin:
@@ -187,55 +304,11 @@
                 if ccli:
                     self.ccli_number = ccli.decode()
                 if authors:
-                    # Split up the authors
-                    author_list = authors.split(b'/')
-                    if len(author_list) < 2:
-                        author_list = authors.split(b';')
-                    if len(author_list) < 2:
-                        author_list = authors.split(b',')
-                    for author_name in author_list:
-                        self.add_author(author_name.decode().strip())
-                if words:
-                    # Format the lyrics
-                    result = strip_rtf(words.decode(), self.encoding)
-                    if result is None:
-                        return
-                    words, self.encoding = result
-                    verse_type = VerseType.tags[VerseType.Verse]
-                    for verse in SLIDE_BREAK_REGEX.split(words):
-                        verse = verse.strip()
-                        if not verse:
-                            continue
-                        verse_split = verse.split('\n', 1)
-                        first_line_is_tag = False
-                        # EW tags: verse, chorus, pre-chorus, bridge, tag,
-                        # intro, ending, slide
-                        for tag in VerseType.tags + ['tag', 'slide']:
-                            tag = tag.lower()
-                            ew_tag = verse_split[0].strip().lower()
-                            if ew_tag.startswith(tag):
-                                verse_type = tag[0]
-                                if tag == 'tag' or tag == 'slide':
-                                    verse_type = VerseType.tags[VerseType.Other]
-                                first_line_is_tag = True
-                                number_found = False
-                                # check if tag is followed by number and/or note
-                                if len(ew_tag) > len(tag):
-                                    match = NUMBER_REGEX.search(ew_tag)
-                                    if match:
-                                        number = match.group()
-                                        verse_type += number
-                                        number_found = True
-                                    match = NOTE_REGEX.search(ew_tag)
-                                    if match:
-                                        self.comments += ew_tag + '\n'
-                                if not number_found:
-                                    verse_type += '1'
-                                break
-                        self.add_verse(verse_split[-1].strip() if first_line_is_tag else verse, verse_type)
-                if len(self.comments) > 5:
-                    self.comments += str(translate('SongsPlugin.EasyWorshipSongImport',
-                                                   '\n[above are Song Tags with notes imported from EasyWorship]'))
+                    authors = authors.decode()
+                else:
+                    authors = ''
+                # Set the SongImport object members.
+                self.set_song_import_object(authors, words)
                 if self.stop_import_flag:
                     break
                 if not self.finish():
@@ -243,6 +316,65 @@
         db_file.close()
         self.memo_file.close()
 
+    def set_song_import_object(self, authors, words):
+        """
+        Set the SongImport object members.
+        
+        :param authors: String with authons
+        :param words: Bytes with rtf-encoding
+        :return:
+        """
+        if authors:
+            # Split up the authors
+            author_list = authors.split('/')
+            if len(author_list) < 2:
+                author_list = authors.split(';')
+            if len(author_list) < 2:
+                author_list = authors.split(',')
+            for author_name in author_list:
+                self.add_author(author_name.strip())
+        if words:
+            # Format the lyrics
+            result = strip_rtf(words.decode(), self.encoding)
+            if result is None:
+                return
+            words, self.encoding = result
+            verse_type = VerseType.tags[VerseType.Verse]
+            for verse in SLIDE_BREAK_REGEX.split(words):
+                verse = verse.strip()
+                if not verse:
+                    continue
+                verse_split = verse.split('\n', 1)
+                first_line_is_tag = False
+                # EW tags: verse, chorus, pre-chorus, bridge, tag,
+                # intro, ending, slide
+                for tag in VerseType.tags + ['tag', 'slide']:
+                    tag = tag.lower()
+                    ew_tag = verse_split[0].strip().lower()
+                    if ew_tag.startswith(tag):
+                        verse_type = tag[0]
+                        if tag == 'tag' or tag == 'slide':
+                            verse_type = VerseType.tags[VerseType.Other]
+                        first_line_is_tag = True
+                        number_found = False
+                        # check if tag is followed by number and/or note
+                        if len(ew_tag) > len(tag):
+                            match = NUMBER_REGEX.search(ew_tag)
+                            if match:
+                                number = match.group()
+                                verse_type += number
+                                number_found = True
+                            match = NOTE_REGEX.search(ew_tag)
+                            if match:
+                                self.comments += ew_tag + '\n'
+                        if not number_found:
+                            verse_type += '1'
+                        break
+                self.add_verse(verse_split[-1].strip() if first_line_is_tag else verse, verse_type)
+        if len(self.comments) > 5:
+            self.comments += str(translate('SongsPlugin.EasyWorshipSongImport',
+                                           '\n[above are Song Tags with notes imported from EasyWorship]'))
+
     def find_field(self, field_name):
         """
         Find a field in the descriptions
@@ -323,3 +455,52 @@
             return self.memo_file.read(blob_size)
         else:
             return 0
+
+    def get_bytes(self, pos, length):
+        """
+        Get bytes from ews_file
+        
+        :param pos: Position to read from
+        :param length: Bytes to read
+        :return: Bytes read
+        """
+        self.ews_file.seek(pos)
+        return self.ews_file.read(length)
+
+    def get_string(self, pos, length):
+        """
+        Get string from ews_file
+        
+        :param pos: Position to read from
+        :param length: Characters to read
+        :return: String read
+        """
+        bytes = self.get_bytes(pos, length)
+        mask = '<' + str(length) + 's'
+        byte_str, = struct.unpack(mask, bytes)
+        return byte_str.decode('unicode-escape').replace('\0', '').strip()
+
+    def get_i16(self, pos):
+        """
+        Get short int from ews_file
+        
+        :param pos: Position to read from
+        :return: Short integer read
+        """
+
+        bytes = self.get_bytes(pos, 2)
+        mask = '<h'
+        number, = struct.unpack(mask, bytes)
+        return number
+
+    def get_i32(self, pos):
+        """
+        Get long int from ews_file
+        
+        :param pos: Position to read from
+        :return: Long integer read
+        """
+        bytes = self.get_bytes(pos, 4)
+        mask = '<i'
+        number, = struct.unpack(mask, bytes)
+        return number

=== modified file 'openlp/plugins/songs/lib/importer.py'
--- openlp/plugins/songs/lib/importer.py	2014-03-17 19:05:55 +0000
+++ openlp/plugins/songs/lib/importer.py	2014-04-05 07:12:35 +0000
@@ -229,7 +229,10 @@
             'name': 'EasyWorship',
             'prefix': 'ew',
             'selectMode': SongFormatSelect.SingleFile,
-            'filter': '%s (*.db)' % translate('SongsPlugin.ImportWizardForm', 'EasyWorship Song Database')
+            'filter': '%s (*.db);; %s (*.ews)' % (translate('SongsPlugin.ImportWizardForm',
+                                                            'EasyWorship Song Database'),
+                                                  translate('SongsPlugin.ImportWizardForm',
+                                                            'EasyWorship Service File'))
         },
         FoilPresenter: {
             'class': FoilPresenterImport,

=== modified file 'tests/functional/openlp_plugins/songs/test_ewimport.py'
--- tests/functional/openlp_plugins/songs/test_ewimport.py	2014-04-02 19:35:09 +0000
+++ tests/functional/openlp_plugins/songs/test_ewimport.py	2014-04-05 07:12:35 +0000
@@ -69,6 +69,21 @@
        'Just to bow and receive a new blessing,\nIn the beautiful garden of prayer.', 'v3')],
      'verse_order_list': []}]
 
+EWS_SONG_TEST_DATA =\
+    { 'title' : 'Vi pløjed og vi så\'de',
+    'authors' : ['Matthias Claudius'],
+    'verses' : 
+        [('Vi pløjed og vi så\'de\nvor sæd i sorten jord,\nså bad vi ham os hjælpe,\nsom højt i Himlen bor,\n'
+          'og han lod snefald hegne\nmod frosten barsk og hård,\nhan lod det tø og regne\nog varme mildt i vår.', 
+          'v1'),
+         ('Alle gode gaver\nde kommer ovenned,\nså tak da Gud, ja, pris dog Gud\nfor al hans kærlighed!', 'c1'),
+         ('Han er jo den, hvis vilje\nopholder alle ting,\nhan klæder markens lilje\nog runder himlens ring,\n'
+          'ham lyder vind og vove,\nham rører ravnes nød,\nhvi skulle ej hans småbørn\nda og få dagligt brød?', 'v2'),
+         ('Ja, tak, du kære Fader,\nså mild, så rig, så rund,\nfor korn i hæs og lader,\nfor godt i allen stund!\n'
+          'Vi kan jo intet give,\nsom nogen ting er værd,\nmen tag vort stakkels hjerte,\nså ringe som det er!',
+          'v3')],
+    }
+
 
 class EasyWorshipSongImportLogger(EasyWorshipSongImport):
     """
@@ -357,9 +372,9 @@
                 self.assertIsNone(importer.do_import(), 'do_import should return None when db_size is less than 0x800')
                 mocked_retrieve_windows_encoding.assert_call(encoding)
 
-    def file_import_test(self):
+    def db_file_import_test(self):
         """
-        Test the actual import of real song files and check that the imported data is correct.
+        Test the actual import of real song database files and check that the imported data is correct.
         """
 
         # GIVEN: Test files with a mocked out SongImport class, a mocked out "manager", a mocked out "import_wizard",
@@ -411,3 +426,42 @@
                     self.assertEquals(importer.verse_order_list, verse_order_list,
                                       'verse_order_list for %s should be %s' % (title, verse_order_list))
                 mocked_finish.assert_called_with()
+
+    def ews_file_import_test(self):
+        """
+        Test the actual import of song from ews file and check that the imported data is correct.
+        """
+
+        # GIVEN: Test files with a mocked out SongImport class, a mocked out "manager", a mocked out "import_wizard",
+        #       and mocked out "author", "add_copyright", "add_verse", "finish" methods.
+        with patch('openlp.plugins.songs.lib.ewimport.SongImport'), \
+            patch('openlp.plugins.songs.lib.ewimport.retrieve_windows_encoding') as mocked_retrieve_windows_encoding:
+            mocked_retrieve_windows_encoding.return_value = 'cp1252'
+            mocked_manager = MagicMock()
+            mocked_import_wizard = MagicMock()
+            mocked_add_author = MagicMock()
+            mocked_add_verse = MagicMock()
+            mocked_finish = MagicMock()
+            mocked_title = MagicMock()
+            mocked_finish.return_value = True
+            importer = EasyWorshipSongImportLogger(mocked_manager)
+            importer.import_wizard = mocked_import_wizard
+            importer.stop_import_flag = False
+            importer.add_author = mocked_add_author
+            importer.add_verse = mocked_add_verse
+            importer.title = mocked_title
+            importer.finish = mocked_finish
+            importer.topics = []
+
+            # WHEN: Importing ews file
+            importer.import_source = os.path.join(TEST_PATH, 'test1.ews')
+
+            # THEN: do_import should return none, the song data should be as expected, and finish should have been
+            #       called.
+            title = EWS_SONG_TEST_DATA['title']
+            self.assertIsNone(importer.do_import(), 'do_import should return None when it has completed')
+            self.assertIn(title, importer._title_assignment_list, 'title for should be "%s"' % title)
+            mocked_add_author.assert_any_call(EWS_SONG_TEST_DATA['authors'][0])
+            for verse_text, verse_tag in EWS_SONG_TEST_DATA['verses']:
+                mocked_add_verse.assert_any_call(verse_text, verse_tag)
+            mocked_finish.assert_called_with()

=== added file 'tests/resources/easyworshipsongs/test1.ews'
Binary files tests/resources/easyworshipsongs/test1.ews	1970-01-01 00:00:00 +0000 and tests/resources/easyworshipsongs/test1.ews	2014-04-05 07:12:35 +0000 differ