openlp-core team mailing list archive

[Raoul Snyman <raoul.snyman@xxxxxxxxxxxxxxxxxxxxxxxx>]

Raoul Snyman has proposed merging lp:~raoul-snyman/openlp/olp1-import into lp:openlp.

Requested reviews:
  OpenLP Core (openlp-core)
Related bugs:
  #608427 Song Converter from Version 1 does not support new version 2 schema
  https://bugs.launchpad.net/bugs/608427
  #633354 Song Importer Broke
  https://bugs.launchpad.net/bugs/633354
  #635330 Support openlp.org 1.0 database imports
  https://bugs.launchpad.net/bugs/635330
  #635338 openlp.org v1 import gets encoding wrong
  https://bugs.launchpad.net/bugs/635338


Attempting to solve bug #635338
-- 
https://code.launchpad.net/~raoul-snyman/openlp/olp1-import/+merge/35781
Your team OpenLP Core is requested to review the proposed merge of lp:~raoul-snyman/openlp/olp1-import into lp:openlp.

=== modified file 'openlp/plugins/songs/lib/olp1import.py'
--- openlp/plugins/songs/lib/olp1import.py	2010-09-14 20:34:44 +0000
+++ openlp/plugins/songs/lib/olp1import.py	2010-09-17 06:48:45 +0000
@@ -41,6 +41,8 @@
     The :class:`OpenLP1SongImport` class provides OpenLP with the ability to
     import song databases from installations of openlp.org 1.x.
     """
+    last_encoding = u'windows-1252'
+
     def __init__(self, manager, **kwargs):
         """
         Initialise the import.
@@ -54,20 +56,33 @@
         SongImport.__init__(self, manager)
         self.import_source = kwargs[u'filename']
 
-    def decode_string(self, raw):
+    def decode_string(self, raw, guess):
         """
         Use chardet to detect the encoding of the raw string, and convert it
         to unicode.
 
         ``raw``
             The raw bytestring to decode.
+        ``guess``
+            What chardet guessed the encoding to be.
         """
-        detection = chardet.detect(raw)
-        if detection[u'confidence'] < 0.8:
+        if guess[u'confidence'] < 0.8:
             codec = u'windows-1252'
         else:
-            codec = detection[u'encoding']
-        return unicode(raw, codec)
+            codec = guess[u'encoding']
+        try:
+            decoded = unicode(raw, codec)
+            self.last_encoding = codec
+        except UnicodeDecodeError:
+            log.exception(u'Error in detecting openlp.org 1.x database encoding.')
+            try:
+                decoded = unicode(raw, self.last_encoding)
+            except UnicodeDecodeError:
+                # possibly show an error form
+                #self.import_wizard.showError(u'There was a problem '
+                #    u'detecting the encoding of a string')
+                decoded = raw
+        return decoded
 
     def do_import(self):
         """
@@ -103,9 +118,10 @@
                 success = False
                 break
             song_id = song[0]
-            title = self.decode_string(song[1])
-            lyrics = self.decode_string(song[2]).replace(u'\r', u'')
-            copyright = self.decode_string(song[3])
+            guess = chardet.detect(song[2])
+            title = self.decode_string(song[1], guess)
+            lyrics = self.decode_string(song[2], guess).replace(u'\r', u'')
+            copyright = self.decode_string(song[3], guess)
             self.import_wizard.incrementProgressBar(
                 unicode(translate('SongsPlugin.ImportWizardForm',
                     'Importing "%s"...')) % title)
@@ -121,7 +137,7 @@
                     break
                 for author in authors:
                     if author[0] == author_id[0]:
-                        self.parse_author(self.decode_string(author[1]))
+                        self.parse_author(self.decode_string(author[1], guess))
                         break
             if self.stop_import_flag:
                 success = False
@@ -136,7 +152,7 @@
                         break
                     for track in tracks:
                         if track[0] == track_id[0]:
-                            self.add_media_file(self.decode_string(track[1]))
+                            self.add_media_file(self.decode_string(track[1], guess))
                             break
             if self.stop_import_flag:
                 success = False