openlp-core team mailing list archive

[Martin Thompson <martin@xxxxxxxxxxxxxxxxxx>]

Martin Thompson has proposed merging lp:~mjthompson/openlp/opensong_import into lp:openlp.

Requested reviews:
  OpenLP Core (openlp-core)


For review only - not ready for merging yet!

Makes use of existing SOF import classes to import Opensong format files.  Successfully imports all the Songs Of Fellowship files and all bar one of the "default" set fo opensong files (the one which doesn't has an error in the source file!)

Still todo 
 - make it work on unicode files with non-ASCII chars in.
 - import direct from zipfiles (code to move from "test_import_lots")
 - check the content of database post import (I've done a little bit, but it could do with mroe eyes at some point)
 - integrate with GUI

-- 
https://code.launchpad.net/~mjthompson/openlp/opensong_import/+merge/28466
Your team OpenLP Core is requested to review the proposed merge of lp:~mjthompson/openlp/opensong_import into lp:openlp.

=== modified file 'openlp/core/lib/songxmlhandler.py'
--- openlp/core/lib/songxmlhandler.py	2010-06-10 21:30:50 +0000
+++ openlp/core/lib/songxmlhandler.py	2010-06-24 21:04:25 +0000
@@ -39,9 +39,9 @@
 
 import logging
 
+from lxml.etree import ElementTree, XML, dump
+from xml.parsers.expat import ExpatError
 from xml.dom.minidom import Document
-from xml.etree.ElementTree import ElementTree, XML, dump
-from xml.parsers.expat import ExpatError
 
 log = logging.getLogger(__name__)
 

=== modified file 'openlp/core/lib/xmlrootclass.py'
--- openlp/core/lib/xmlrootclass.py	2010-06-12 20:22:58 +0000
+++ openlp/core/lib/xmlrootclass.py	2010-06-24 21:04:25 +0000
@@ -26,7 +26,7 @@
 import os
 import sys
 
-from xml.etree.ElementTree import ElementTree, XML
+from lxml.etree import ElementTree, XML
 
 sys.path.append(os.path.abspath(os.path.join(u'.', u'..', u'..')))
 

=== modified file 'openlp/plugins/songs/lib/__init__.py'
--- openlp/plugins/songs/lib/__init__.py	2010-06-21 16:43:59 +0000
+++ openlp/plugins/songs/lib/__init__.py	2010-06-24 21:04:25 +0000
@@ -96,4 +96,8 @@
 from mediaitem import SongMediaItem
 from sofimport import SofImport
 from oooimport import OooImport
-from songimport import SongImport
\ No newline at end of file
+<<<<<<< TREE
+from songimport import SongImport=======
+from songimport import SongImport
+from opensongimport import OpenSongImport
+>>>>>>> MERGE-SOURCE

=== renamed file 'openlp/plugins/songs/lib/xml.py' => 'openlp/plugins/songs/lib/lyrics_xml.py'
=== added file 'openlp/plugins/songs/lib/opensongimport.py'
--- openlp/plugins/songs/lib/opensongimport.py	1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/opensongimport.py	2010-06-24 21:04:25 +0000
@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection                                      #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2010 Raoul Snyman                                        #
+# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael      #
+# Gorven, Scott Guerrieri, Christian Richter, Maikel Stuivenberg, Martin      #
+# Thompson, Jon Tibble, Carsten Tinggaard                                     #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; version 2 of the License.                              #
+#                                                                             #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with this program; if not, write to the Free Software Foundation, Inc., 59  #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
+###############################################################################
+
+import os
+import re
+
+from songimport import SongImport
+from lxml.etree import Element
+from lxml import objectify
+
+class OpenSongImportError(Exception):
+    pass
+
+class OpenSongImport:
+    """
+    Import songs exported from OpenSong - the format is described loosly here:
+    http://www.opensong.org/d/manual/song_file_format_specification
+
+    However, it doesn't describe the <lyrics> section, so here's an attempt:
+
+    Verses can be expressed in one of 2 ways:
+    <lyrics>
+    [v1]List of words
+    Another Line
+
+    [v2]Some words for the 2nd verse
+    etc...
+    </lyrics>
+
+    The 'v' can be left out - it is implied
+    or:
+    <lyrics>
+    [V]
+    1List of words
+    2Some words for the 2nd Verse
+
+    1Another Line
+    2etc...
+    </lyrics>
+
+    Either or both forms can be used in one song.  The Number does not necessarily appear at the start of the line
+
+    The [v1] labels can have either upper or lower case Vs
+    Other labels can be used also:
+      C - Chorus
+      B - Bridge
+
+    Guitar chords can be provided 'above' the lyrics (the line is preceeded by a'.') and _s can be used to signify long-drawn-out words:
+
+    . A7        Bm
+    1 Some____ Words
+
+    Chords and _s are removed by this importer.
+
+    The verses etc. are imported and tagged appropriately.
+
+    The <presentation> tag is used to populate the OpenLP verse
+    display order field.  The Author and Copyright tags are also
+    imported to the appropriate places.
+
+    """
+    def __init__(self, songmanager):
+        """
+        Initialise the class. Requires a songmanager class which is passed
+        to SongImport for writing song to disk
+        """
+        self.songmanager=songmanager
+        self.song = None
+
+    def do_import(self, filename):
+        file=open(filename)
+        self.do_import_file(file)
+        
+    def do_import_file(self, file):
+        """
+        Process the OpenSong file
+        """            
+        self.song = SongImport(self.songmanager)
+        tree=objectify.parse(file)
+        root=tree.getroot()
+        fields=dir(root)
+        decode={u'copyright':self.song.add_copyright,
+                u'author':self.song.parse_author,
+                u'title':self.song.set_title,
+                u'aka':self.song.set_alternate_title,
+                u'hymn_number':self.song.set_song_number}
+        for (attr, fn) in decode.items():
+            if attr in fields:
+                fn(unicode(root.__getattr__(attr)))
+        
+        # data storage while importing
+        verses={}
+        lyrics=unicode(root.lyrics)
+        # keep track of a "default" verse order, in case none is specified
+        our_verse_order=[]
+        verses_seen={}
+        # in the absence of any other indication, verses are the default, erm, versetype!
+        versetype=u'V'
+        for l in lyrics.split(u'\n'):
+            # remove comments
+            semicolon = l.find(u';')
+            if semicolon >= 0:
+                l=l[:semicolon]
+            l=l.strip()
+            if l==u'':
+                continue
+            # skip inline guitar chords
+            if l[0] == u'.':
+                continue
+            
+            # verse/chorus/etc. marker
+            if l[0] == u'[':
+                versetype=l[1].upper()
+                if versetype.isdigit():
+                    versenum=versetype
+                    versetype=u'V'
+                elif l[2] != u']':
+                    # there's a number to go with it - extract that as well
+                    right_bracket=l.find(u']')
+                    versenum=l[2:right_bracket]
+                else:
+                    versenum = u''
+                continue
+            words=None
+
+            # number at start of line.. it's verse number
+            if l[0].isdigit():
+                versenum=l[0]
+                words=l[1:].strip()
+            if words is None and \
+                   versenum is not None and \
+                   versetype is not None:
+                words=l
+            if versenum is not None:
+                versetag=u'%s%s'%(versetype,versenum)
+                if not verses.has_key(versetype):
+                    verses[versetype]={}
+                if not verses[versetype].has_key(versenum):
+                    verses[versetype][versenum]=[] # storage for lines in this verse
+                if not verses_seen.has_key(versetag):
+                    verses_seen[versetag] = 1
+                    our_verse_order.append(versetag)
+            if words:
+                # Tidy text and remove the ____s from extended words
+                # words=self.song.tidy_text(words)
+                words=words.replace('_', '')
+                verses[versetype][versenum].append(words)
+        # done parsing
+        versetypes=verses.keys()
+        versetypes.sort()
+        versetags={}
+        for v in versetypes:
+            versenums=verses[v].keys()
+            versenums.sort()
+            for n in versenums:
+                versetag= u'%s%s' %(v,n)
+                lines=u'\n'.join(verses[v][n])
+                self.song.verses.append([versetag, lines])
+                versetags[versetag]=1 # keep track of what we have for error checking later
+        # now figure out the presentation order
+        if u'presentation' in fields and root.presentation != u'':
+            order=unicode(root.presentation)
+            order=order.split()
+        else:
+            assert len(our_verse_order)>0
+            order=our_verse_order
+        for tag in order:
+            if not versetags.has_key(tag):
+                print u'Got order', tag, u'but not in versetags, skipping'
+                raise OpenSongImportError
+            else:
+                self.song.verse_order_list.append(tag)
+    def finish(self):
+        """ Separate function, allows test suite to not pollute database"""
+        self.song.finish()

=== added file 'openlp/plugins/songs/lib/test.opensong'
--- openlp/plugins/songs/lib/test.opensong	1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/test.opensong	2010-06-24 21:04:25 +0000
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<song>
+  <title>Martins Test</title>
+  <author>Martin Thompson</author>
+  <copyright>2010 Martin Thompson</copyright>
+  <hymn_number>1</hymn_number>
+  <presentation>V1 C V2 C2 V3 B1 V1</presentation>
+  <ccli>Blah</ccli>
+  <capo print="false"></capo>
+  <key></key>
+  <aka></aka>
+  <key_line></key_line>
+  <user1></user1>
+  <user2></user2>
+  <user3></user3>
+  <theme></theme>
+  <tempo></tempo>
+  <time_sig></time_sig>
+  <lyrics>;Comment
+. A   B C
+1 v1 Line 1___
+2 v2 Line 1___
+. A B C7
+1 V1 Line 2
+2 V2 Line 2
+ 
+[3]
+V3 Line 1
+V3 Line 2
+
+[b1]
+ Bridge 1
+ Bridge 1 line 2
+[C]
+. A     B
+ Chorus 1
+ 
+[C2]
+.  A    B
+ Chorus 2
+ </lyrics>
+  <style index="default_style">
+  <title enabled="true" valign="bottom" align="center" include_verse="false" margin-left="0" margin-right="0" margin-top="0" margin-bottom="0" font="Helvetica" size="26" bold="true" italic="true" underline="false" color="#FFFFFF" border="true" border_color="#000000" shadow="true" shadow_color="#000000" fill="false" fill_color="#000000"/>
+  <subtitle enabled="true" valign="bottom" align="center" descriptive="false" margin-left="0" margin-right="0" margin-top="0" margin-bottom="0" font="Helvetica" size="18" bold="true" italic="true" underline="false" color="#FFFFFF" border="true" border_color="#000000" shadow="true" shadow_color="#000000" fill="false" fill_color="#000000"/>
+  <song_subtitle>author</song_subtitle>
+  <body enabled="true" auto_scale="false" valign="middle" align="center" highlight_chorus="true" margin-left="0" margin-right="0" margin-top="0" margin-bottom="0" font="Helvetica" size="34" bold="true" italic="false" underline="false" color="#FFFFFF" border="true" border_color="#000000" shadow="true" shadow_color="#000000" fill="false" fill_color="#FF0000">
+  <tabs/>
+</body>
+  <background strip_footer="0" color="#408080" position="1"/>
+</style></song>

=== added file 'openlp/plugins/songs/lib/test2.opensong'
--- openlp/plugins/songs/lib/test2.opensong	1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/test2.opensong	2010-06-24 21:04:25 +0000
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<song>
+  <title>Martins 2nd Test</title>
+  <author>Martin Thompson</author>
+  <copyright>2010 Martin Thompson</copyright>
+  <hymn_number>2</hymn_number>
+  <presentation></presentation>
+  <ccli>Blah</ccli>
+  <capo print="false"></capo>
+  <key></key>
+  <aka></aka>
+  <key_line></key_line>
+  <user1></user1>
+  <user2></user2>
+  <user3></user3>
+  <theme></theme>
+  <tempo></tempo>
+  <time_sig></time_sig>
+  <lyrics>;Comment
+[V]
+. A   B C
+1 v1 Line 1___
+2 v2 Line 1___
+. A B C7
+1 V1 Line 2
+2 V2 Line 2
+ 
+[b1]
+ Bridge 1
+ Bridge 1 line 2
+[C1]
+ Chorus 1
+ 
+[C2]
+ Chorus 2
+ </lyrics>
+  <style index="default_style">
+  <title enabled="true" valign="bottom" align="center" include_verse="false" margin-left="0" margin-right="0" margin-top="0" margin-bottom="0" font="Helvetica" size="26" bold="true" italic="true" underline="false" color="#FFFFFF" border="true" border_color="#000000" shadow="true" shadow_color="#000000" fill="false" fill_color="#000000"/>
+  <subtitle enabled="true" valign="bottom" align="center" descriptive="false" margin-left="0" margin-right="0" margin-top="0" margin-bottom="0" font="Helvetica" size="18" bold="true" italic="true" underline="false" color="#FFFFFF" border="true" border_color="#000000" shadow="true" shadow_color="#000000" fill="false" fill_color="#000000"/>
+  <song_subtitle>author</song_subtitle>
+  <body enabled="true" auto_scale="false" valign="middle" align="center" highlight_chorus="true" margin-left="0" margin-right="0" margin-top="0" margin-bottom="0" font="Helvetica" size="34" bold="true" italic="false" underline="false" color="#FFFFFF" border="true" border_color="#000000" shadow="true" shadow_color="#000000" fill="false" fill_color="#FF0000">
+  <tabs/>
+</body>
+  <background strip_footer="0" color="#408080" position="1"/>
+</style></song>

=== added file 'openlp/plugins/songs/lib/test_importing_lots.py'
--- openlp/plugins/songs/lib/test_importing_lots.py	1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/test_importing_lots.py	2010-06-24 21:04:25 +0000
@@ -0,0 +1,49 @@
+from openlp.plugins.songs.lib.opensongimport import OpenSongImport
+from openlp.plugins.songs.lib.manager import SongManager
+from glob import glob
+from zipfile import ZipFile
+import os
+from traceback import print_exc
+import sys
+import codecs
+def opensong_import_lots():
+    ziploc=u'/home/mjt/openlp/OpenSong_Data/'
+    files=[]
+    files.extend(glob(ziploc+u'Songs.zip'))
+    files.extend(glob(ziploc+u'SOF.zip'))
+#    files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip'))
+#    files.extend(glob(ziploc+u'opensong_*.zip'))
+    errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8')
+    manager=SongManager()
+    for file in files:
+        print u'Importing', file
+        z=ZipFile(file, u'r')
+        for song in z.infolist():
+            filename=song.filename.decode('cp852')
+            parts=os.path.split(filename)
+            if parts[-1] == u'':
+                #No final part => directory
+                continue
+            # xxx need to handle unicode filenames (CP437?? Winzip does this)
+            print "  ", file, ":",filename,
+            songfile=z.open(song)
+
+            o=OpenSongImport(manager)
+            try:
+                o.do_import_file(songfile)
+            except:
+                print "Failure",
+                
+                errfile.write(u'Failure: %s:%s\n' %(file, filename))
+                songfile=z.open(song)
+                for l in songfile.readlines():
+                    l=l.decode('utf8')
+                    print(u'  |%s\n'%l.strip())
+                    errfile.write(u'  |%s\n'%l.strip())   
+                print_exc(3, file=errfile)
+                continue
+            # o.finish()
+            print "OK"
+            # o.song.print_song()
+if __name__=="__main__":
+    opensong_import_lots()

=== added file 'openlp/plugins/songs/lib/test_opensongimport.py'
--- openlp/plugins/songs/lib/test_opensongimport.py	1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/test_opensongimport.py	2010-06-24 21:04:25 +0000
@@ -0,0 +1,46 @@
+from openlp.plugins.songs.lib.opensongimport import OpenSongImport
+from openlp.plugins.songs.lib.manager import SongManager
+
+def test():
+    manager=SongManager()
+    o=OpenSongImport(manager)
+    o.do_import(u'test.opensong')
+    # o.finish()
+    o.song.print_song()
+    assert o.song.copyright == u'2010 Martin Thompson'
+    assert o.song.authors == [u'Martin Thompson']
+    assert o.song.title == u'Martins Test'
+    assert o.song.alternate_title == u''
+    assert o.song.song_number == u'1'
+    assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses 
+    assert [u'C', u'Chorus 1'] in o.song.verses 
+    assert [u'C2', u'Chorus 2'] in o.song.verses 
+    assert not [u'C3', u'Chorus 3'] in o.song.verses 
+    assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses 
+    assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses
+    assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1']
+
+    o=OpenSongImport(manager)
+    o.do_import(u'test2.opensong')
+    # o.finish()
+    o.song.print_song()
+    assert o.song.copyright == u'2010 Martin Thompson'
+    assert o.song.authors == [u'Martin Thompson']
+    assert o.song.title == u'Martins 2nd Test'
+    assert o.song.alternate_title == u''
+    assert o.song.song_number == u'2'
+    print o.song.verses
+    assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses 
+    assert [u'C1', u'Chorus 1'] in o.song.verses 
+    assert [u'C2', u'Chorus 2'] in o.song.verses 
+    assert not [u'C3', u'Chorus 3'] in o.song.verses 
+    assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses 
+    assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses
+    print o.song.verse_order_list
+    assert o.song.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2']
+
+    print "Tests passed"
+    pass
+
+if __name__=="__main__":
+    test()