openlp-core team mailing list archive
-
openlp-core team
-
Mailing list archive
-
Message #02302
[Merge] lp:~mjthompson/openlp/opensong_import into lp:openlp
Martin Thompson has proposed merging lp:~mjthompson/openlp/opensong_import into lp:openlp.
Requested reviews:
OpenLP Core (openlp-core)
For review only - not ready for merging yet!
Makes use of existing SOF import classes to import Opensong format files. Successfully imports all the Songs Of Fellowship files and all bar one of the "default" set fo opensong files (the one which doesn't has an error in the source file!)
Still todo
- make it work on unicode files with non-ASCII chars in.
- import direct from zipfiles (code to move from "test_import_lots")
- check the content of database post import (I've done a little bit, but it could do with mroe eyes at some point)
- integrate with GUI
--
https://code.launchpad.net/~mjthompson/openlp/opensong_import/+merge/29148
Your team OpenLP Core is requested to review the proposed merge of lp:~mjthompson/openlp/opensong_import into lp:openlp.
=== modified file 'openlp/core/lib/songxmlhandler.py'
--- openlp/core/lib/songxmlhandler.py 2010-06-10 21:30:50 +0000
+++ openlp/core/lib/songxmlhandler.py 2010-07-03 14:01:27 +0000
@@ -39,9 +39,9 @@
import logging
+from lxml.etree import ElementTree, XML, dump
+from xml.parsers.expat import ExpatError
from xml.dom.minidom import Document
-from xml.etree.ElementTree import ElementTree, XML, dump
-from xml.parsers.expat import ExpatError
log = logging.getLogger(__name__)
=== modified file 'openlp/core/lib/xmlrootclass.py'
--- openlp/core/lib/xmlrootclass.py 2010-06-12 20:22:58 +0000
+++ openlp/core/lib/xmlrootclass.py 2010-07-03 14:01:27 +0000
@@ -26,7 +26,7 @@
import os
import sys
-from xml.etree.ElementTree import ElementTree, XML
+from lxml.etree import ElementTree, XML
sys.path.append(os.path.abspath(os.path.join(u'.', u'..', u'..')))
=== modified file 'openlp/plugins/songs/lib/__init__.py'
--- openlp/plugins/songs/lib/__init__.py 2010-06-30 22:05:51 +0000
+++ openlp/plugins/songs/lib/__init__.py 2010-07-03 14:01:27 +0000
@@ -140,6 +140,7 @@
from songstab import SongsTab
from mediaitem import SongMediaItem
from songimport import SongImport
+from opensongimport import OpenSongImport
try:
from sofimport import SofImport
from oooimport import OooImport
=== renamed file 'openlp/plugins/songs/lib/xml.py' => 'openlp/plugins/songs/lib/lyrics_xml.py'
=== added file 'openlp/plugins/songs/lib/opensongimport.py'
--- openlp/plugins/songs/lib/opensongimport.py 1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/opensongimport.py 2010-07-03 14:01:27 +0000
@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2010 Raoul Snyman #
+# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael #
+# Gorven, Scott Guerrieri, Christian Richter, Maikel Stuivenberg, Martin #
+# Thompson, Jon Tibble, Carsten Tinggaard #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it #
+# under the terms of the GNU General Public License as published by the Free #
+# Software Foundation; version 2 of the License. #
+# #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
+# more details. #
+# #
+# You should have received a copy of the GNU General Public License along #
+# with this program; if not, write to the Free Software Foundation, Inc., 59 #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
+###############################################################################
+
+import os
+import re
+
+from songimport import SongImport
+from lxml.etree import Element
+from lxml import objectify
+
+from zipfile import ZipFile
+
+import logging
+log = logging.getLogger(__name__)
+
+class OpenSongImportError(Exception):
+ pass
+
+class OpenSongImport:
+ """
+ Import songs exported from OpenSong - the format is described loosly here:
+ http://www.opensong.org/d/manual/song_file_format_specification
+
+ However, it doesn't describe the <lyrics> section, so here's an attempt:
+
+ Verses can be expressed in one of 2 ways:
+ <lyrics>
+ [v1]List of words
+ Another Line
+
+ [v2]Some words for the 2nd verse
+ etc...
+ </lyrics>
+
+ The 'v' can be left out - it is implied
+ or:
+ <lyrics>
+ [V]
+ 1List of words
+ 2Some words for the 2nd Verse
+
+ 1Another Line
+ 2etc...
+ </lyrics>
+
+ Either or both forms can be used in one song. The Number does not necessarily appear at the start of the line
+
+ The [v1] labels can have either upper or lower case Vs
+ Other labels can be used also:
+ C - Chorus
+ B - Bridge
+
+ Guitar chords can be provided 'above' the lyrics (the line is preceeded by a'.') and _s can be used to signify long-drawn-out words:
+
+ . A7 Bm
+ 1 Some____ Words
+
+ Chords and _s are removed by this importer.
+
+ The verses etc. are imported and tagged appropriately.
+
+ The <presentation> tag is used to populate the OpenLP verse
+ display order field. The Author and Copyright tags are also
+ imported to the appropriate places.
+
+ """
+ def __init__(self, songmanager):
+ """
+ Initialise the class. Requires a songmanager class which is passed
+ to SongImport for writing song to disk
+ """
+ self.songmanager = songmanager
+ self.song = None
+
+ def do_import(self, filename, commit=True):
+ """
+ Import either a single opensong file, or a zipfile containing multiple opensong files
+ If the commit parameter is set False, the import will not be committed to the database (useful for test scripts)
+ """
+ ext=os.path.splitext(filename)[1]
+ if ext.lower() == ".zip":
+ log.info('Zipfile found %s', filename)
+ z=ZipFile(filename, u'r')
+ for song in z.infolist():
+ parts=os.path.split(song.filename)
+ if parts[-1] == u'':
+ #No final part => directory
+ continue
+ songfile=z.open(song)
+ self.do_import_file(songfile)
+ if commit:
+ self.finish()
+ else:
+ log.info('Direct import %s', filename)
+ file = open(filename)
+ self.do_import_file(file)
+ if commit:
+ self.finish()
+ def do_import_file(self, file):
+ """
+ Process the OpenSong file - pass in a file-like object, not a filename
+ """
+ self.song = SongImport(self.songmanager)
+ tree = objectify.parse(file)
+ root = tree.getroot()
+ fields = dir(root)
+ decode = {u'copyright':self.song.add_copyright,
+ u'author':self.song.parse_author,
+ u'title':self.song.set_title,
+ u'aka':self.song.set_alternate_title,
+ u'hymn_number':self.song.set_song_number}
+ for (attr, fn) in decode.items():
+ if attr in fields:
+ fn(unicode(root.__getattr__(attr)))
+
+ # data storage while importing
+ verses = {}
+ lyrics = unicode(root.lyrics)
+ # keep track of a "default" verse order, in case none is specified
+ our_verse_order = []
+ verses_seen = {}
+ # in the absence of any other indication, verses are the default, erm, versetype!
+ versetype = u'V'
+ for l in lyrics.split(u'\n'):
+ # remove comments
+ semicolon = l.find(u';')
+ if semicolon >= 0:
+ l = l[:semicolon]
+ l = l.strip()
+ if len(l) == 0:
+ continue
+ # skip inline guitar chords
+ if l[0] == u'.':
+ continue
+
+ # verse/chorus/etc. marker
+ if l[0] == u'[':
+ versetype = l[1].upper()
+ if versetype.isdigit():
+ versenum = versetype
+ versetype = u'V'
+ elif l[2] != u']':
+ # there's a number to go with it - extract that as well
+ right_bracket = l.find(u']')
+ versenum = l[2:right_bracket]
+ else:
+ versenum = u''
+ continue
+ words=None
+
+ # number at start of line.. it's verse number
+ if l[0].isdigit():
+ versenum = l[0]
+ words = l[1:].strip()
+ if words is None and \
+ versenum is not None and \
+ versetype is not None:
+ words=l
+ if versenum is not None:
+ versetag = u'%s%s'%(versetype,versenum)
+ if not verses.has_key(versetype):
+ verses[versetype] = {}
+ if not verses[versetype].has_key(versenum):
+ verses[versetype][versenum] = [] # storage for lines in this verse
+ if not verses_seen.has_key(versetag):
+ verses_seen[versetag] = 1
+ our_verse_order.append(versetag)
+ if words:
+ # Tidy text and remove the ____s from extended words
+ # words=self.song.tidy_text(words)
+ words=words.replace('_', '')
+ verses[versetype][versenum].append(words)
+ # done parsing
+ versetypes = verses.keys()
+ versetypes.sort()
+ versetags = {}
+ for v in versetypes:
+ versenums = verses[v].keys()
+ versenums.sort()
+ for n in versenums:
+ versetag = u'%s%s' %(v,n)
+ lines = u'\n'.join(verses[v][n])
+ self.song.verses.append([versetag, lines])
+ versetags[versetag] = 1 # keep track of what we have for error checking later
+ # now figure out the presentation order
+ if u'presentation' in fields and root.presentation != u'':
+ order = unicode(root.presentation)
+ order = order.split()
+ else:
+ assert len(our_verse_order)>0
+ order = our_verse_order
+ for tag in order:
+ if not versetags.has_key(tag):
+ log.warn(u'Got order %s but not in versetags, skipping', tag)
+ else:
+ self.song.verse_order_list.append(tag)
+ def finish(self):
+ """ Separate function, allows test suite to not pollute database"""
+ self.song.finish()
=== modified file 'openlp/plugins/songs/lib/songimport.py'
--- openlp/plugins/songs/lib/songimport.py 2010-06-30 22:05:51 +0000
+++ openlp/plugins/songs/lib/songimport.py 2010-07-03 14:01:27 +0000
@@ -302,8 +302,12 @@
song.theme_name = self.theme_name
song.ccli_number = self.ccli_number
for authortext in self.authors:
+<<<<<<< TREE
author = self.manager.get_object_filtered(Author,
Author.display_name == authortext)
+=======
+ author = self.manager.get_object_filtered(Author, Author.display_name == authortext)
+>>>>>>> MERGE-SOURCE
if author is None:
author = Author()
author.display_name = authortext
@@ -312,8 +316,12 @@
self.manager.save_object(author)
song.authors.append(author)
if self.song_book_name:
+<<<<<<< TREE
song_book = self.manager.get_object_filtered(Book,
Book.name == self.song_book_name)
+=======
+ song_book = self.manager.get_object_filtered(Book, Book.name == self.song_book_name)
+>>>>>>> MERGE-SOURCE
if song_book is None:
song_book = Book()
song_book.name = self.song_book_name
@@ -321,8 +329,12 @@
self.manager.save_object(song_book)
song.song_book_id = song_book.id
for topictext in self.topics:
+<<<<<<< TREE
topic = self.manager.get_object_filtered(Topic,
Topic.name == topictext)
+=======
+ topic = self.manager.get_object_filtered(Topic.name == topictext)
+>>>>>>> MERGE-SOURCE
if topic is None:
topic = Topic()
topic.name = topictext
=== added directory 'openlp/plugins/songs/lib/test'
=== added file 'openlp/plugins/songs/lib/test/test2.opensong'
--- openlp/plugins/songs/lib/test/test2.opensong 1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/test/test2.opensong 2010-07-03 14:01:27 +0000
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<song>
+ <title>Martins 2nd Test</title>
+ <author>Martin Thompson</author>
+ <copyright>2010 Martin Thompson</copyright>
+ <hymn_number>2</hymn_number>
+ <presentation></presentation>
+ <ccli>Blah</ccli>
+ <capo print="false"></capo>
+ <key></key>
+ <aka></aka>
+ <key_line></key_line>
+ <user1></user1>
+ <user2></user2>
+ <user3></user3>
+ <theme></theme>
+ <tempo></tempo>
+ <time_sig></time_sig>
+ <lyrics>;Comment
+[V]
+. A B C
+1 v1 Line 1___
+2 v2 Line 1___
+. A B C7
+1 V1 Line 2
+2 V2 Line 2
+
+[b1]
+ Bridge 1
+ Bridge 1 line 2
+[C1]
+ Chorus 1
+
+[C2]
+ Chorus 2
+ </lyrics>
+ <style index="default_style">
+ <title enabled="true" valign="bottom" align="center" include_verse="false" margin-left="0" margin-right="0" margin-top="0" margin-bottom="0" font="Helvetica" size="26" bold="true" italic="true" underline="false" color="#FFFFFF" border="true" border_color="#000000" shadow="true" shadow_color="#000000" fill="false" fill_color="#000000"/>
+ <subtitle enabled="true" valign="bottom" align="center" descriptive="false" margin-left="0" margin-right="0" margin-top="0" margin-bottom="0" font="Helvetica" size="18" bold="true" italic="true" underline="false" color="#FFFFFF" border="true" border_color="#000000" shadow="true" shadow_color="#000000" fill="false" fill_color="#000000"/>
+ <song_subtitle>author</song_subtitle>
+ <body enabled="true" auto_scale="false" valign="middle" align="center" highlight_chorus="true" margin-left="0" margin-right="0" margin-top="0" margin-bottom="0" font="Helvetica" size="34" bold="true" italic="false" underline="false" color="#FFFFFF" border="true" border_color="#000000" shadow="true" shadow_color="#000000" fill="false" fill_color="#FF0000">
+ <tabs/>
+</body>
+ <background strip_footer="0" color="#408080" position="1"/>
+</style></song>
=== added file 'openlp/plugins/songs/lib/test/test_importing_lots.py'
--- openlp/plugins/songs/lib/test/test_importing_lots.py 1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/test/test_importing_lots.py 2010-07-03 14:01:27 +0000
@@ -0,0 +1,55 @@
+from openlp.plugins.songs.lib.opensongimport import OpenSongImport
+from openlp.plugins.songs.lib.manager import SongManager
+from glob import glob
+from zipfile import ZipFile
+import os
+from traceback import print_exc
+import sys
+import codecs
+def opensong_import_lots():
+ ziploc=u'/home/mjt/openlp/OpenSong_Data/'
+ files=[]
+ files=['test.opensong.zip']
+ files.extend(glob(ziploc+u'Songs.zip'))
+ files.extend(glob(ziploc+u'SOF.zip'))
+ # files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip'))
+# files.extend(glob(ziploc+u'opensong_*.zip'))
+ errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8')
+ manager=SongManager()
+ for file in files:
+ print u'Importing', file
+ z=ZipFile(file, u'r')
+ for song in z.infolist():
+ # need to handle unicode filenames (CP437 - Winzip does this)
+ filename=song.filename#.decode('cp852')
+ parts=os.path.split(filename)
+ if parts[-1] == u'':
+ #No final part => directory
+ continue
+ print " ", file, ":",filename,
+ songfile=z.open(song)
+ #z.extract(song)
+ #songfile=open(filename, u'r')
+ o=OpenSongImport(manager)
+ try:
+ o.do_import_file(songfile)
+ o.song.print_song()
+ except:
+ print "Failure",
+
+ errfile.write(u'Failure: %s:%s\n' %(file, filename.decode('cp437')))
+ songfile=z.open(song)
+ for l in songfile.readlines():
+ l=l.decode('utf8')
+ print(u' |%s\n'%l.strip())
+ errfile.write(u' |%s\n'%l.strip())
+ print_exc(3, file=errfile)
+ print_exc(3)
+ sys.exit(1)
+ # continue
+ o.finish()
+ print "OK"
+ #os.unlink(filename)
+ # o.song.print_song()
+if __name__=="__main__":
+ opensong_import_lots()
=== added file 'openlp/plugins/songs/lib/test/test_opensongimport.py'
--- openlp/plugins/songs/lib/test/test_opensongimport.py 1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/test/test_opensongimport.py 2010-07-03 14:01:27 +0000
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2010 Raoul Snyman #
+# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael #
+# Gorven, Scott Guerrieri, Christian Richter, Maikel Stuivenberg, Martin #
+# Thompson, Jon Tibble, Carsten Tinggaard #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it #
+# under the terms of the GNU General Public License as published by the Free #
+# Software Foundation; version 2 of the License. #
+# #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
+# more details. #
+# #
+# You should have received a copy of the GNU General Public License along #
+# with this program; if not, write to the Free Software Foundation, Inc., 59 #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
+###############################################################################
+from openlp.plugins.songs.lib.opensongimport import OpenSongImport
+from openlp.plugins.songs.lib.manager import SongManager
+
+def test():
+ manager = SongManager()
+ o = OpenSongImport(manager)
+ o.do_import(u'test.opensong', commit=False)
+ # o.finish()
+ o.song.print_song()
+ assert o.song.copyright == u'2010 Martin Thompson'
+ assert o.song.authors == [u'MartiÑ Thómpson']
+ assert o.song.title == u'Martins Test'
+ assert o.song.alternate_title == u''
+ assert o.song.song_number == u'1'
+ assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses
+ assert [u'C', u'Chorus 1'] in o.song.verses
+ assert [u'C2', u'Chorus 2'] in o.song.verses
+ assert not [u'C3', u'Chorus 3'] in o.song.verses
+ assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses
+ assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses
+ assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1']
+
+ o.do_import(u'test.opensong.zip', commit=False)
+ # o.finish()
+ o.song.print_song()
+ assert o.song.copyright == u'2010 Martin Thompson'
+ assert o.song.authors == [u'MartiÑ Thómpson']
+ assert o.song.title == u'Martins Test'
+ assert o.song.alternate_title == u''
+ assert o.song.song_number == u'1'
+ assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses
+ assert [u'C', u'Chorus 1'] in o.song.verses
+ assert [u'C2', u'Chorus 2'] in o.song.verses
+ assert not [u'C3', u'Chorus 3'] in o.song.verses
+ assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses
+ assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses
+ assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1']
+
+ o = OpenSongImport(manager)
+ o.do_import(u'test2.opensong', commit=False)
+ # o.finish()
+ o.song.print_song()
+ assert o.song.copyright == u'2010 Martin Thompson'
+ assert o.song.authors == [u'Martin Thompson']
+ assert o.song.title == u'Martins 2nd Test'
+ assert o.song.alternate_title == u''
+ assert o.song.song_number == u'2'
+ print o.song.verses
+ assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses
+ assert [u'C1', u'Chorus 1'] in o.song.verses
+ assert [u'C2', u'Chorus 2'] in o.song.verses
+ assert not [u'C3', u'Chorus 3'] in o.song.verses
+ assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses
+ assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses
+ print o.song.verse_order_list
+ assert o.song.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2']
+
+ print "Tests passed"
+ pass
+
+if __name__=="__main__":
+ test()
=== modified file 'openlp/plugins/songs/songsplugin.py'
--- openlp/plugins/songs/songsplugin.py 2010-07-02 18:21:45 +0000
+++ openlp/plugins/songs/songsplugin.py 2010-07-03 14:01:27 +0000
@@ -39,6 +39,8 @@
except ImportError:
OOo_available = False
+from openlp.plugins.songs.lib import OpenSongImport
+
log = logging.getLogger(__name__)
class SongsPlugin(Plugin):
@@ -136,6 +138,25 @@
QtCore.SIGNAL(u'triggered()'), self.onImportSofItemClick)
QtCore.QObject.connect(self.ImportOooItem,
QtCore.SIGNAL(u'triggered()'), self.onImportOooItemClick)
+ # OpenSong import menu item - will be removed and the
+ # functionality will be contained within the import wizard
+ self.ImportOpenSongItem = QtGui.QAction(import_menu)
+ self.ImportOpenSongItem.setObjectName(u'ImportOpenSongItem')
+ self.ImportOpenSongItem.setText(
+ translate('SongsPlugin',
+ 'OpenSong (temp menu item)'))
+ self.ImportOpenSongItem.setToolTip(
+ translate('SongsPlugin',
+ 'Import songs from OpenSong files' +
+ '(either raw text or ZIPfiles)'))
+ self.ImportOpenSongItem.setStatusTip(
+ translate('SongsPlugin',
+ 'Import songs from OpenSong files' +
+ '(either raw text or ZIPfiles)'))
+ import_menu.addAction(self.ImportOpenSongItem)
+ QtCore.QObject.connect(self.ImportOpenSongItem,
+ QtCore.SIGNAL(u'triggered()'), self.onImportOpenSongItemClick)
+
def add_export_menu_item(self, export_menu):
"""
@@ -176,6 +197,26 @@
QtGui.QMessageBox.Ok)
Receiver.send_message(u'songs_load_list')
+ def onImportOpenSongItemClick(self):
+ filenames = QtGui.QFileDialog.getOpenFileNames(
+ None, translate('SongsPlugin',
+ 'Open OpenSong file'),
+ u'', u'OpenSong file (*. *.zip *.ZIP)')
+ try:
+ for filename in filenames:
+ importer = OpenSongImport(self.manager)
+ importer.do_import(unicode(filename))
+ except:
+ log.exception('Could not import OpenSong file')
+ QtGui.QMessageBox.critical(None,
+ translate('SongsPlugin',
+ 'Import Error'),
+ translate('SongsPlugin',
+ 'Error importing OpenSong file'),
+ QtGui.QMessageBox.StandardButtons(QtGui.QMessageBox.Ok),
+ QtGui.QMessageBox.Ok)
+ Receiver.send_message(u'songs_load_list')
+
def onImportOooItemClick(self):
filenames = QtGui.QFileDialog.getOpenFileNames(
None, translate('SongsPlugin',
Follow ups
