openlp-core team mailing list archive
-
openlp-core team
-
Mailing list archive
-
Message #30989
[Merge] lp:~tomasgroth/openlp/song-import-fixes25 into lp:openlp
Tomas Groth has proposed merging lp:~tomasgroth/openlp/song-import-fixes25 into lp:openlp.
Requested reviews:
OpenLP Core (openlp-core)
Related bugs:
Bug #1530597 in OpenLP: "Importing Songbeamer songs using latin1 encoding doesn't get decoded correctly"
https://bugs.launchpad.net/openlp/+bug/1530597
Bug #1652851 in OpenLP: "VideoPsalm import fails due to unexpected format"
https://bugs.launchpad.net/openlp/+bug/1652851
Bug #1655985 in OpenLP: "EasySlide importer gets the verse order wrong "
https://bugs.launchpad.net/openlp/+bug/1655985
Bug #1655988 in OpenLP: "Formatting tags gets included in searchable lyric text"
https://bugs.launchpad.net/openlp/+bug/1655988
For more details, see:
https://code.launchpad.net/~tomasgroth/openlp/song-import-fixes25/+merge/315270
Clean search lyrics for formatting tags. Fixes bug #1655988.
Fix an issue with easyslide import not handling verse order correctly. Fixes bug #1655985.
Improve the songbeamer encoding detection. Fixes bug #1530597.
Handle a few videopsalm quirks. Fixes bug #1652851.
--
Your team OpenLP Core is requested to review the proposed merge of lp:~tomasgroth/openlp/song-import-fixes25 into lp:openlp.
=== modified file 'openlp/plugins/songs/lib/__init__.py'
--- openlp/plugins/songs/lib/__init__.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/__init__.py 2017-01-20 20:51:35 +0000
@@ -30,7 +30,7 @@
from PyQt5 import QtWidgets
from openlp.core.common import AppLocation, CONTROL_CHARS
-from openlp.core.lib import translate
+from openlp.core.lib import translate, clean_tags
from openlp.plugins.songs.lib.db import Author, MediaFile, Song, Topic
from openlp.plugins.songs.lib.ui import SongStrings
@@ -380,7 +380,7 @@
if isinstance(song.lyrics, bytes):
song.lyrics = str(song.lyrics, encoding='utf8')
verses = SongXML().get_verses(song.lyrics)
- song.search_lyrics = ' '.join([clean_string(verse[1]) for verse in verses])
+ song.search_lyrics = ' '.join([clean_string(clean_tags(verse[1])) for verse in verses])
# The song does not have any author, add one.
if not song.authors_songs:
name = SongStrings.AuthorUnknown
=== modified file 'openlp/plugins/songs/lib/importers/easyslides.py'
--- openlp/plugins/songs/lib/importers/easyslides.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/easyslides.py 2017-01-20 20:51:35 +0000
@@ -180,7 +180,7 @@
reg = default_region
verses[reg] = {}
# instance differentiates occurrences of same verse tag
- vt = 'V'
+ vt = 'v'
vn = '1'
inst = 1
for line in lines:
@@ -193,14 +193,14 @@
inst += 1
else:
# separators are not used, so empty line starts a new verse
- vt = 'V'
+ vt = 'v'
vn = len(verses[reg].get(vt, {})) + 1
inst = 1
elif line[0:7] == '[region':
reg = self._extract_region(line)
verses.setdefault(reg, {})
if not regions_in_verses:
- vt = 'V'
+ vt = 'v'
vn = '1'
inst = 1
elif line[0] == '[':
@@ -213,7 +213,7 @@
if match:
marker = match.group(1).strip()
vn = match.group(2)
- vt = MarkTypes.get(marker, 'O') if marker else 'V'
+ vt = MarkTypes.get(marker, 'o') if marker else 'v'
if regions_in_verses:
region = default_region
inst = 1
@@ -238,13 +238,13 @@
lines = '\n'.join(verses[reg][vt][vn][inst])
self.add_verse(lines, versetag)
SeqTypes = {
- 'p': 'P1',
- 'q': 'P2',
- 'c': 'C1',
- 't': 'C2',
- 'b': 'B1',
- 'w': 'B2',
- 'e': 'E1'}
+ 'p': 'p1',
+ 'q': 'p2',
+ 'c': 'c1',
+ 't': 'c2',
+ 'b': 'b1',
+ 'w': 'b2',
+ 'e': 'e1'}
# Make use of Sequence data, determining the order of verses
try:
order = str(song.Sequence).strip().split(',')
@@ -252,7 +252,7 @@
if not tag:
continue
elif tag[0].isdigit():
- tag = 'V' + tag
+ tag = 'v' + tag
elif tag.lower() in SeqTypes:
tag = SeqTypes[tag.lower()]
else:
=== modified file 'openlp/plugins/songs/lib/importers/songbeamer.py'
--- openlp/plugins/songs/lib/importers/songbeamer.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/songbeamer.py 2017-01-20 20:51:35 +0000
@@ -28,6 +28,7 @@
import os
import re
+from openlp.core.common import get_file_encoding
from openlp.plugins.songs.lib import VerseType
from openlp.plugins.songs.lib.importers.songimport import SongImport
@@ -113,13 +114,15 @@
read_verses = False
file_name = os.path.split(import_file)[1]
if os.path.isfile(import_file):
- # First open in binary mode to detect the encoding
- detect_file = open(import_file, 'rb')
- details = chardet.detect(detect_file.read())
- detect_file.close()
- infile = codecs.open(import_file, 'r', details['encoding'])
+ # Detect the encoding
+ self.input_file_encoding = get_file_encoding(import_file)['encoding']
+ # The encoding should only be ANSI (cp1252), UTF-8, Unicode, Big-Endian-Unicode.
+ # So if it doesn't start with 'u' we default to cp1252. See:
+ # https://forum.songbeamer.com/viewtopic.php?p=419&sid=ca4814924e37c11e4438b7272a98b6f2
+ if self.input_file_encoding.lower().startswith('u'):
+ self.input_file_encoding = 'cp1252'
+ infile = open(import_file, 'rt', encoding=self.input_file_encoding)
song_data = infile.readlines()
- infile.close()
else:
continue
self.title = file_name.split('.sng')[0]
=== modified file 'openlp/plugins/songs/lib/importers/videopsalm.py'
--- openlp/plugins/songs/lib/importers/videopsalm.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/videopsalm.py 2017-01-20 20:51:35 +0000
@@ -65,8 +65,8 @@
if c == '\n':
if inside_quotes:
processed_content += '\\n'
- # Put keys in quotes
- elif c.isalnum() and not inside_quotes:
+ # Put keys in quotes. The '-' is for handling nagative numbers
+ elif (c.isalnum() or c == '-') and not inside_quotes:
processed_content += '"' + c
c = next(file_content_it)
while c.isalnum():
@@ -121,6 +121,8 @@
if 'Memo3' in song:
self.add_comment(song['Memo3'])
for verse in song['Verses']:
+ if 'Text' not in verse:
+ continue
self.add_verse(verse['Text'], 'v')
if not self.finish():
self.log_error('Could not import {title}'.format(title=self.title))
=== modified file 'tests/functional/openlp_plugins/songs/test_easyslidesimport.py'
--- tests/functional/openlp_plugins/songs/test_easyslidesimport.py 2016-12-31 11:01:36 +0000
+++ tests/functional/openlp_plugins/songs/test_easyslidesimport.py 2017-01-20 20:51:35 +0000
@@ -43,3 +43,5 @@
"""
self.file_import(os.path.join(TEST_PATH, 'amazing-grace.xml'),
self.load_external_result_data(os.path.join(TEST_PATH, 'Amazing Grace.json')))
+ self.file_import(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.xml'),
+ self.load_external_result_data(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.json')))
=== modified file 'tests/resources/easyslidessongs/Amazing Grace.json'
--- tests/resources/easyslidessongs/Amazing Grace.json 2016-01-08 21:42:36 +0000
+++ tests/resources/easyslidessongs/Amazing Grace.json 2017-01-20 20:51:35 +0000
@@ -6,27 +6,27 @@
"verses": [
[
"Amazing grace! How sweet the sound\nThat saved a wretch like me;\nI once was lost, but now am found,\nWas blind, but now I see.",
- "V1"
+ "v1"
],
[
"'Twas grace that taught my heart to fear,\nAnd grace my fears relieved;\nHow precious did that grace appear,\nThe hour I first believed!",
- "V2"
+ "v2"
],
[
"Through many dangers, toils and snares\nI have already come;\n'Tis grace that brought me safe thus far,\nAnd grace will lead me home.",
- "V3"
+ "v3"
],
[
"The Lord has promised good to me,\nHis word my hope secures;\nHe will my shield and portion be\nAs long as life endures.",
- "V4"
+ "v4"
],
[
"Yes, when this heart and flesh shall fail,\nAnd mortal life shall cease,\nI shall possess within the veil\nA life of joy and peace.",
- "V5"
+ "v5"
],
[
"When we've been there a thousand years,\nBright shining as the sun,\nWe've no less days to sing God's praise\nThan when we first begun.",
- "V6"
+ "v6"
]
]
}
=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.json'
--- tests/resources/easyslidessongs/Export_2017-01-12_BB.json 1970-01-01 00:00:00 +0000
+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.json 2017-01-20 20:51:35 +0000
@@ -0,0 +1,44 @@
+{
+ "title": "BBBBBBBBB",
+ "authors": [
+ "John Newton (1725-1807)"
+ ],
+ "verses": [
+ [
+ "V1V1V1V1V1V1\nV1V1V1V1V1V1",
+ "v1"
+ ],
+ [
+ "V2V2V2V2V2V2\nV2V2V2V2V2V2",
+ "v2"
+ ],
+ [
+ "C1C1C1C1C1C1\nC1C1C1C1C1C1",
+ "c1"
+ ],
+ [
+ "C2C2C2C2C2C2\nC2C2C2C2C2C2",
+ "c2"
+ ],
+ [
+ "B1B1B1B1B1B1\nB1B1B1B1B1B1",
+ "b1"
+ ],
+ [
+ "B2B2B2B2B2B2\nB2B2B2B2B2B2",
+ "b2"
+ ],
+ [
+ "PRE1PRE1PRE1\nPRE1PRE1PRE1",
+ "p1"
+ ],
+ [
+ "PRE2PRE2PRE2\nPRE2PRE2PRE2",
+ "p2"
+ ],
+ [
+ "ENDENDENDEND\nENDENDENDEND",
+ "e1"
+ ]
+ ]
+}
=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.xml'
--- tests/resources/easyslidessongs/Export_2017-01-12_BB.xml 1970-01-01 00:00:00 +0000
+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.xml 2017-01-20 20:51:35 +0000
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EasiSlides>
+ <Item>
+ <Title1>BBBBBBBBB</Title1>
+ <Title2 />
+ <Folder>NAGY</Folder>
+ <SongNumber>0</SongNumber>
+ <Contents>[1]
+V1V1V1V1V1V1
+V1V1V1V1V1V1
+[2]
+V2V2V2V2V2V2
+V2V2V2V2V2V2
+[chorus]
+C1C1C1C1C1C1
+C1C1C1C1C1C1
+[chorus 2]
+C2C2C2C2C2C2
+C2C2C2C2C2C2
+[bridge]
+B1B1B1B1B1B1
+B1B1B1B1B1B1
+[bridge 2]
+B2B2B2B2B2B2
+B2B2B2B2B2B2
+[prechorus]
+PRE1PRE1PRE1
+PRE1PRE1PRE1
+[prechorus 2]
+PRE2PRE2PRE2
+PRE2PRE2PRE2
+[ending]
+ENDENDENDEND
+ENDENDENDEND</Contents>
+ <Notations />
+ <Sequence>1,2,c,t,b,w,p,q,e</Sequence>
+ <Writer />
+ <Copyright />
+ <Category />
+ <Timing />
+ <MusicKey />
+ <Capo>-1</Capo>
+ <LicenceAdmin1 />
+ <LicenceAdmin2 />
+ <BookReference />
+ <UserReference />
+ <FormatData />
+ <Settings>10=></Settings>
+ </Item>
+</EasiSlides>
\ No newline at end of file
=== modified file 'tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json'
--- tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json 2015-12-17 21:39:52 +0000
+++ tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json 2017-01-20 20:51:35 +0000
@@ -1,4 +1,4 @@
-{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Copyright:"Public
+{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Capo:-1,Copyright:"Public
Domain",Theme:"tema1
tema2",CCLI:"12345",Alias:"A safe stronghold",Memo1:"This is
the first comment
References
