openlp-core team mailing list archive
-
openlp-core team
-
Mailing list archive
-
Message #03314
[Merge] lp:~raoul-snyman/openlp/olp1-import into lp:openlp
Raoul Snyman has proposed merging lp:~raoul-snyman/openlp/olp1-import into lp:openlp.
Requested reviews:
OpenLP Core (openlp-core)
Can has openlp.org 1.x import? Fixes bug #608427 as well.
--
https://code.launchpad.net/~raoul-snyman/openlp/olp1-import/+merge/34718
Your team OpenLP Core is requested to review the proposed merge of lp:~raoul-snyman/openlp/olp1-import into lp:openlp.
=== modified file 'openlp/plugins/songs/forms/songimportform.py'
--- openlp/plugins/songs/forms/songimportform.py 2010-09-05 14:18:38 +0000
+++ openlp/plugins/songs/forms/songimportform.py 2010-09-07 05:36:56 +0000
@@ -31,7 +31,6 @@
from songimportwizard import Ui_SongImportWizard
from openlp.core.lib import Receiver, SettingsManager, translate
-#from openlp.core.utils import AppLocation
from openlp.plugins.songs.lib.importer import SongFormat
log = logging.getLogger(__name__)
@@ -136,7 +135,7 @@
self.openLP2BrowseButton.setFocus()
return False
elif source_format == SongFormat.OpenLP1:
- if self.openSongFilenameEdit.text().isEmpty():
+ if self.openLP1FilenameEdit.text().isEmpty():
QtGui.QMessageBox.critical(self,
translate('SongsPlugin.ImportWizardForm',
'No openlp.org 1.x Song Database Selected'),
@@ -292,7 +291,7 @@
def onCCLIRemoveButtonClicked(self):
self.removeSelectedItems(self.ccliFileListWidget)
-
+
def onSongsOfFellowshipAddButtonClicked(self):
self.getFiles(
translate('SongsPlugin.ImportWizardForm',
@@ -374,11 +373,11 @@
importer = self.plugin.importSongs(SongFormat.OpenLP2,
filename=unicode(self.openLP2FilenameEdit.text())
)
- #elif source_format == SongFormat.OpenLP1:
- # # Import an openlp.org database
- # importer = self.plugin.importSongs(SongFormat.OpenLP1,
- # filename=unicode(self.field(u'openlp1_filename').toString())
- # )
+ elif source_format == SongFormat.OpenLP1:
+ # Import an openlp.org database
+ importer = self.plugin.importSongs(SongFormat.OpenLP1,
+ filename=unicode(self.openLP1FilenameEdit.text())
+ )
elif source_format == SongFormat.OpenLyrics:
# Import OpenLyrics songs
importer = self.plugin.importSongs(SongFormat.OpenLyrics,
=== modified file 'openlp/plugins/songs/lib/importer.py'
--- openlp/plugins/songs/lib/importer.py 2010-09-01 20:20:27 +0000
+++ openlp/plugins/songs/lib/importer.py 2010-09-07 05:36:56 +0000
@@ -26,6 +26,7 @@
from opensongimport import OpenSongImport
from olpimport import OpenLPSongImport
+from olp1import import OpenLP1SongImport
try:
from sofimport import SofImport
from oooimport import OooImport
@@ -61,6 +62,8 @@
"""
if format == SongFormat.OpenLP2:
return OpenLPSongImport
+ if format == SongFormat.OpenLP1:
+ return OpenLP1SongImport
elif format == SongFormat.OpenSong:
return OpenSongImport
elif format == SongFormat.SongsOfFellowship:
@@ -70,7 +73,7 @@
elif format == SongFormat.Generic:
return OooImport
elif format == SongFormat.CCLI:
- return CCLIFileImport
+ return CCLIFileImport
# else:
return None
=== added file 'openlp/plugins/songs/lib/olp1import.py'
--- openlp/plugins/songs/lib/olp1import.py 1970-01-01 00:00:00 +0000
+++ openlp/plugins/songs/lib/olp1import.py 2010-09-07 05:36:56 +0000
@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2010 Raoul Snyman #
+# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael #
+# Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian #
+# Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble, #
+# Carsten Tinggaard, Frode Woldsund #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it #
+# under the terms of the GNU General Public License as published by the Free #
+# Software Foundation; version 2 of the License. #
+# #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
+# more details. #
+# #
+# You should have received a copy of the GNU General Public License along #
+# with this program; if not, write to the Free Software Foundation, Inc., 59 #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
+###############################################################################
+"""
+The :mod:`olp1import` module provides the functionality for importing
+openlp.org 1.x song databases into the current installation database.
+"""
+import logging
+import sqlite
+
+from openlp.core.lib import translate
+from songimport import SongImport
+
+log = logging.getLogger(__name__)
+
+class OpenLP1SongImport(SongImport):
+ """
+ The :class:`OpenLP1SongImport` class provides OpenLP with the ability to
+ import song databases from installations of openlp.org 1.x.
+ """
+ def __init__(self, manager, **kwargs):
+ """
+ Initialise the import.
+
+ ``manager``
+ The song manager for the running OpenLP installation.
+
+ ``filename``
+ The database providing the data to import.
+ """
+ SongImport.__init__(self, manager)
+ self.import_source = kwargs[u'filename']
+
+ def do_import(self):
+ """
+ Run the import for an openlp.org 1.x song database.
+ """
+ # Connect to the database
+ connection = sqlite.connect(self.import_source)
+ cursor = connection.cursor()
+ # Count the number of records we need to import, for the progress bar
+ cursor.execute(u'SELECT COUNT(songid) FROM songs')
+ count = int(cursor.fetchone()[0])
+ success = True
+ self.import_wizard.importProgressBar.setMaximum(count)
+ # "cache" our list of authors
+ cursor.execute(u'SELECT authorid, authorname FROM authors')
+ authors = cursor.fetchall()
+ # "cache" our list of tracks
+ cursor.execute(u'SELECT trackid, fulltrackname FROM tracks')
+ tracks = cursor.fetchall()
+ # Import the songs
+ cursor.execute(u'SELECT songid, songtitle, lyrics || \'\' AS lyrics, '
+ u'copyrightinfo FROM songs')
+ songs = cursor.fetchall()
+ for song in songs:
+ self.set_defaults()
+ if self.stop_import_flag:
+ success = False
+ break
+ song_id = song[0]
+ title = unicode(song[1], u'cp1252')
+ lyrics = unicode(song[2], u'cp1252').replace(u'\r', u'')
+ copyright = unicode(song[3], u'cp1252')
+ self.import_wizard.incrementProgressBar(
+ unicode(translate('SongsPlugin.ImportWizardForm',
+ 'Importing "%s"...')) % title)
+ self.title = title
+ self.process_song_text(lyrics)
+ self.add_copyright(copyright)
+ cursor.execute(u'SELECT authorid FROM songauthors '
+ u'WHERE songid = %s' % song_id)
+ author_ids = cursor.fetchall()
+ for author_id in author_ids:
+ if self.stop_import_flag:
+ success = False
+ break
+ for author in authors:
+ if author[0] == author_id[0]:
+ self.parse_author(unicode(author[1], u'cp1252'))
+ break
+ if self.stop_import_flag:
+ success = False
+ break
+ cursor.execute(u'SELECT name FROM sqlite_master '
+ u'WHERE type = \'table\' AND name = \'tracks\'')
+ table_list = cursor.fetchall()
+ if len(table_list) > 0:
+ cursor.execute(u'SELECT trackid FROM songtracks '
+ u'WHERE songid = %s ORDER BY listindex' % song_id)
+ track_ids = cursor.fetchall()
+ for track_id in track_ids:
+ if self.stop_import_flag:
+ success = False
+ break
+ for track in tracks:
+ if track[0] == track_id[0]:
+ self.add_media_file(unicode(track[1], u'cp1252'))
+ break
+ if self.stop_import_flag:
+ success = False
+ break
+ self.finish()
+ return success
=== modified file 'openlp/plugins/songs/lib/opensongimport.py'
--- openlp/plugins/songs/lib/opensongimport.py 2010-07-27 09:32:52 +0000
+++ openlp/plugins/songs/lib/opensongimport.py 2010-09-07 05:36:56 +0000
@@ -28,6 +28,7 @@
import os
from zipfile import ZipFile
from lxml import objectify
+from lxml.etree import Error, LxmlError
from openlp.plugins.songs.lib.songimport import SongImport
@@ -36,119 +37,163 @@
class OpenSongImportError(Exception):
pass
-class OpenSongImport(object):
+class OpenSongImport(SongImport):
"""
- Import songs exported from OpenSong - the format is described loosly here:
- http://www.opensong.org/d/manual/song_file_format_specification
-
- However, it doesn't describe the <lyrics> section, so here's an attempt:
-
- Verses can be expressed in one of 2 ways:
- <lyrics>
- [v1]List of words
- Another Line
-
- [v2]Some words for the 2nd verse
- etc...
- </lyrics>
-
- The 'v' can be left out - it is implied
- or:
- <lyrics>
- [V]
- 1List of words
- 2Some words for the 2nd Verse
-
- 1Another Line
- 2etc...
- </lyrics>
-
- Either or both forms can be used in one song. The Number does not
- necessarily appear at the start of the line
-
- The [v1] labels can have either upper or lower case Vs
+ Import songs exported from OpenSong
+
+ The format is described loosly on the `OpenSong File Format Specification
+ <http://www.opensong.org/d/manual/song_file_format_specification>`_ page on
+ the OpenSong web site. However, it doesn't describe the <lyrics> section,
+ so here's an attempt:
+
+ Verses can be expressed in one of 2 ways, either in complete verses, or by
+ line grouping, i.e. grouping all line 1's of a verse together, all line 2's
+ of a verse together, and so on.
+
+ An example of complete verses::
+
+ <lyrics>
+ [v1]
+ List of words
+ Another Line
+
+ [v2]
+ Some words for the 2nd verse
+ etc...
+ </lyrics>
+
+ The 'v' in the verse specifiers above can be left out, it is implied.
+
+ An example of line grouping::
+
+ <lyrics>
+ [V]
+ 1List of words
+ 2Some words for the 2nd Verse
+
+ 1Another Line
+ 2etc...
+ </lyrics>
+
+ Either or both forms can be used in one song. The number does not
+ necessarily appear at the start of the line. Additionally, the [v1] labels
+ can have either upper or lower case Vs.
+
Other labels can be used also:
- C - Chorus
- B - Bridge
-
- Guitar chords can be provided 'above' the lyrics (the line is
- preceeded by a'.') and _s can be used to signify long-drawn-out
- words:
-
- . A7 Bm
- 1 Some____ Words
-
- Chords and _s are removed by this importer.
-
- The verses etc. are imported and tagged appropriately.
-
- The <presentation> tag is used to populate the OpenLP verse
- display order field. The Author and Copyright tags are also
- imported to the appropriate places.
+
+ C
+ Chorus
+
+ B
+ Bridge
+
+ All verses are imported and tagged appropriately.
+
+ Guitar chords can be provided "above" the lyrics (the line is preceeded by
+ a period "."), and one or more "_" can be used to signify long-drawn-out
+ words. Chords and "_" are removed by this importer. For example::
+
+ . A7 Bm
+ 1 Some____ Words
+
+ The <presentation> tag is used to populate the OpenLP verse display order
+ field. The Author and Copyright tags are also imported to the appropriate
+ places.
"""
- def __init__(self, songmanager):
- """
- Initialise the class. Requires a songmanager class which
- is passed to SongImport for writing song to disk
- """
- self.songmanager = songmanager
+ def __init__(self, manager, **kwargs):
+ """
+ Initialise the class.
+ """
+ SongImport.__init__(self, manager)
+ self.filenames = kwargs[u'filenames']
self.song = None
+ self.commit = True
- def do_import(self, filename, commit=True):
- """
- Import either a single opensong file, or a zipfile
- containing multiple opensong files If the commit parameter is
- set False, the import will not be committed to the database
- (useful for test scripts)
- """
- ext = os.path.splitext(filename)[1]
- if ext.lower() == ".zip":
- log.info('Zipfile found %s', filename)
- z = ZipFile(filename, u'r')
- for song in z.infolist():
- parts = os.path.split(song.filename)
- if parts[-1] == u'':
- #No final part => directory
- continue
- songfile = z.open(song)
- self.do_import_file(songfile)
- if commit:
+ def do_import(self):
+ """
+ Import either a single opensong file, or a zipfile containing multiple
+ opensong files. If `self.commit` is set False, the import will not be
+ committed to the database (useful for test scripts).
+ """
+ success = True
+ self.import_wizard.importProgressBar.setMaximum(len(self.filenames))
+ for filename in self.filenames:
+ if self.stop_import_flag:
+ success = False
+ break
+ ext = os.path.splitext(filename)[1]
+ if ext.lower() == u'.zip':
+ log.debug(u'Zipfile found %s', filename)
+ z = ZipFile(filename, u'r')
+ self.import_wizard.importProgressBar.setMaximum(
+ self.import_wizard.importProgressBar.maximum() +
+ len(z.infolist()))
+ for song in z.infolist():
+ if self.stop_import_flag:
+ success = False
+ break
+ parts = os.path.split(song.filename)
+ if parts[-1] == u'':
+ #No final part => directory
+ continue
+ self.import_wizard.incrementProgressBar(
+ unicode(translate('SongsPlugin.ImportWizardForm',
+ 'Importing %s...')) % parts[-1])
+ songfile = z.open(song)
+ self.do_import_file(songfile)
+ if self.commit:
+ self.finish()
+ self.set_defaults()
+ if self.stop_import_flag:
+ success = False
+ break
+ else:
+ log.info('Direct import %s', filename)
+ self.import_wizard.incrementProgressBar(
+ unicode(translate('SongsPlugin.ImportWizardForm',
+ 'Importing %s...')) % os.path.split(filename)[-1])
+ file = open(filename)
+ self.do_import_file(file)
+ if self.commit:
self.finish()
- else:
- log.info('Direct import %s', filename)
- file = open(filename)
- self.do_import_file(file)
- if commit:
- self.finish()
+ self.set_defaults()
+ if not self.commit:
+ self.finish()
+ return success
-
def do_import_file(self, file):
"""
Process the OpenSong file - pass in a file-like object,
not a filename
- """
- self.song_import = SongImport(self.songmanager)
- tree = objectify.parse(file)
+ """
+ self.authors = []
+ try:
+ tree = objectify.parse(file)
+ except Error, LxmlError:
+ log.exception(u'Error parsing XML')
+ return
root = tree.getroot()
fields = dir(root)
- decode = {u'copyright':self.song_import.add_copyright,
- u'ccli':u'ccli_number',
- u'author':self.song_import.parse_author,
- u'title':u'title',
- u'aka':u'alternate_title',
- u'hymn_number':u'song_number'}
- for (attr, fn_or_string) in decode.items():
+ decode = {
+ u'copyright': self.add_copyright,
+ u'ccli': u'ccli_number',
+ u'author': self.parse_author,
+ u'title': u'title',
+ u'aka': u'alternate_title',
+ u'hymn_number': u'song_number'
+ }
+ for attr, fn_or_string in decode.items():
if attr in fields:
ustring = unicode(root.__getattr__(attr))
- if type(fn_or_string) == type(u''):
- self.song_import.__setattr__(fn_or_string, ustring)
+ if isinstance(fn_or_string, basestring):
+ setattr(self, fn_or_string, ustring)
else:
fn_or_string(ustring)
- if u'theme' in fields:
- self.song_import.topics.append(unicode(root.theme))
- if u'alttheme' in fields:
- self.song_import.topics.append(unicode(root.alttheme))
+ if u'theme' in fields and unicode(root.theme) not in self.topics:
+ self.topics.append(unicode(root.theme))
+ if u'alttheme' in fields and unicode(root.alttheme) not in self.topics:
+ self.topics.append(unicode(root.alttheme))
# data storage while importing
verses = {}
lyrics = unicode(root.lyrics)
@@ -158,6 +203,7 @@
# in the absence of any other indication, verses are the default,
# erm, versetype!
versetype = u'V'
+ versenum = None
for thisline in lyrics.split(u'\n'):
# remove comments
semicolon = thisline.find(u';')
@@ -170,7 +216,6 @@
if thisline[0] == u'.' or thisline.startswith(u'---') \
or thisline.startswith(u'-!!'):
continue
-
# verse/chorus/etc. marker
if thisline[0] == u'[':
versetype = thisline[1].upper()
@@ -186,7 +231,6 @@
versenum = u'1'
continue
words = None
-
# number at start of line.. it's verse number
if thisline[0].isdigit():
versenum = thisline[0]
@@ -207,7 +251,7 @@
our_verse_order.append(versetag)
if words:
# Tidy text and remove the ____s from extended words
- words = self.song_import.tidy_text(words)
+ words = self.tidy_text(words)
words = words.replace('_', '')
verses[versetype][versenum].append(words)
# done parsing
@@ -220,24 +264,23 @@
for num in versenums:
versetag = u'%s%s' % (versetype, num)
lines = u'\n'.join(verses[versetype][num])
- self.song_import.verses.append([versetag, lines])
+ self.verses.append([versetag, lines])
# Keep track of what we have for error checking later
versetags[versetag] = 1
# now figure out the presentation order
+ order = []
if u'presentation' in fields and root.presentation != u'':
order = unicode(root.presentation)
order = order.split()
else:
- assert len(our_verse_order)>0
- order = our_verse_order
+ if len(our_verse_order) > 0:
+ order = our_verse_order
+ else:
+ log.warn(u'No verse order available for %s, skipping.', self.title)
for tag in order:
if len(tag) == 1:
tag = tag + u'1' # Assume it's no.1 if it's not there
if not versetags.has_key(tag):
log.warn(u'Got order %s but not in versetags, skipping', tag)
else:
- self.song_import.verse_order_list.append(tag)
-
- def finish(self):
- """ Separate function, allows test suite to not pollute database"""
- self.song_import.finish()
+ self.verse_order_list.append(tag)
=== modified file 'openlp/plugins/songs/lib/songimport.py'
--- openlp/plugins/songs/lib/songimport.py 2010-09-05 15:16:48 +0000
+++ openlp/plugins/songs/lib/songimport.py 2010-09-07 05:36:56 +0000
@@ -30,7 +30,7 @@
from openlp.core.lib import Receiver, translate
from openlp.plugins.songs.lib import VerseType
-from openlp.plugins.songs.lib.db import Song, Author, Topic, Book
+from openlp.plugins.songs.lib.db import Song, Author, Topic, Book, MediaFile
from openlp.plugins.songs.lib.xml import SongXMLBuilder
log = logging.getLogger(__name__)
@@ -66,6 +66,7 @@
self.ccli_number = u''
self.authors = []
self.topics = []
+ self.media_files = []
self.song_book_name = u''
self.song_book_pub = u''
self.verse_order_list = []
@@ -76,7 +77,7 @@
'SongsPlugin.SongImport', 'copyright'))
self.copyright_symbol = unicode(translate(
'SongsPlugin.SongImport', '\xa9'))
-
+
def stop_import(self):
"""
Sets the flag for importers to stop their import
@@ -184,6 +185,14 @@
return
self.authors.append(author)
+ def add_media_file(self, filename):
+ """
+ Add a media file to the list
+ """
+ if filename in self.media_files:
+ return
+ self.media_files.append(filename)
+
def add_verse(self, verse, versetag=None):
"""
Add a verse. This is the whole verse, lines split by \n
@@ -279,11 +288,16 @@
for authortext in self.authors:
author = self.manager.get_object_filtered(Author,
Author.display_name == authortext)
- if author is None:
+ if not author:
author = Author.populate(display_name = authortext,
last_name=authortext.split(u' ')[-1],
first_name=u' '.join(authortext.split(u' ')[:-1]))
song.authors.append(author)
+ for filename in self.media_files:
+ media_file = self.manager.get_object_filtered(MediaFile,
+ MediaFile.file_name == filename)
+ if not media_file:
+ song.media_files.append(MediaFile.populate(file_name=filename))
if self.song_book_name:
song_book = self.manager.get_object_filtered(Book,
Book.name == self.song_book_name)
Follow ups
