openlp-core team mailing list archive

[Raoul Snyman <raoul.snyman@xxxxxxxxxxxxxxxxxxxxxxxx>]

Raoul Snyman has proposed merging lp:~raoul-snyman/openlp/olp1-import into lp:openlp.

Requested reviews:
  OpenLP Core (openlp-core)
Related bugs:
  #608427 Song Converter from Version 1 does not support new version 2 schema
  https://bugs.launchpad.net/bugs/608427
  #635330 Support openlp.org 1.0 database imports
  https://bugs.launchpad.net/bugs/635330
  #635338 openlp.org v1 import gets encoding wrong
  https://bugs.launchpad.net/bugs/635338


Fixes bug #635330 and attempts to fix bug #635338
-- 
https://code.launchpad.net/~raoul-snyman/openlp/olp1-import/+merge/35201
Your team OpenLP Core is requested to review the proposed merge of lp:~raoul-snyman/openlp/olp1-import into lp:openlp.

=== modified file 'openlp/plugins/songs/lib/olp1import.py'
--- openlp/plugins/songs/lib/olp1import.py	2010-09-08 18:00:48 +0000
+++ openlp/plugins/songs/lib/olp1import.py	2010-09-11 21:18:39 +0000
@@ -28,6 +28,7 @@
 openlp.org 1.x song databases into the current installation database.
 """
 import logging
+import chardet
 try:
     import sqlite
 except:
@@ -56,6 +57,21 @@
         SongImport.__init__(self, manager)
         self.import_source = kwargs[u'filename']
 
+    def decode_string(self, raw):
+        """
+        Use chardet to detect the encoding of the raw string, and convert it
+        to unicode.
+
+        ``raw``
+            The raw bytestring to decode.
+        """
+        detection = chardet.detect(raw)
+        if detection[u'confidence'] < 0.8:
+            codec = u'windows-1252'
+        else:
+            codec = detection[u'encoding']
+        return unicode(raw, codec)
+
     def do_import(self):
         """
         Run the import for an openlp.org 1.x song database.
@@ -63,6 +79,11 @@
         # Connect to the database
         connection = sqlite.connect(self.import_source)
         cursor = connection.cursor()
+        # Determine if we're using a new or an old DB
+        cursor.execute(u'SELECT name FROM sqlite_master '
+            u'WHERE type = \'table\' AND name = \'tracks\'')
+        table_list = cursor.fetchall()
+        new_db = len(table_list) > 0
         # Count the number of records we need to import, for the progress bar
         cursor.execute(u'SELECT COUNT(songid) FROM songs')
         count = int(cursor.fetchone()[0])
@@ -71,9 +92,10 @@
         # "cache" our list of authors
         cursor.execute(u'SELECT authorid, authorname FROM authors')
         authors = cursor.fetchall()
-        # "cache" our list of tracks
-        cursor.execute(u'SELECT trackid, fulltrackname FROM tracks')
-        tracks = cursor.fetchall()
+        if new_db:
+          # "cache" our list of tracks
+          cursor.execute(u'SELECT trackid, fulltrackname FROM tracks')
+          tracks = cursor.fetchall()
         # Import the songs
         cursor.execute(u'SELECT songid, songtitle, lyrics || \'\' AS lyrics, '
             u'copyrightinfo FROM songs')
@@ -84,9 +106,9 @@
                 success = False
                 break
             song_id = song[0]
-            title = unicode(song[1], u'cp1252')
-            lyrics = unicode(song[2], u'cp1252').replace(u'\r', u'')
-            copyright = unicode(song[3], u'cp1252')
+            title = self.decode_string(song[1])
+            lyrics = self.decode_string(song[2]).replace(u'\r', u'')
+            copyright = self.decode_string(song[3])
             self.import_wizard.incrementProgressBar(
                 unicode(translate('SongsPlugin.ImportWizardForm',
                     'Importing "%s"...')) % title)
@@ -102,15 +124,12 @@
                     break
                 for author in authors:
                     if author[0] == author_id[0]:
-                        self.parse_author(unicode(author[1], u'cp1252'))
+                        self.parse_author(self.decode_string(author[1]))
                         break
             if self.stop_import_flag:
                 success = False
                 break
-            cursor.execute(u'SELECT name FROM sqlite_master '
-                u'WHERE type = \'table\' AND name = \'tracks\'')
-            table_list = cursor.fetchall()
-            if len(table_list) > 0:
+            if new_db:
                 cursor.execute(u'SELECT trackid FROM songtracks '
                     u'WHERE songid = %s ORDER BY listindex' % song_id)
                 track_ids = cursor.fetchall()
@@ -120,10 +139,11 @@
                         break
                     for track in tracks:
                         if track[0] == track_id[0]:
-                            self.add_media_file(unicode(track[1], u'cp1252'))
+                            self.add_media_file(self.decode_string(track[1]))
                             break
             if self.stop_import_flag:
                 success = False
                 break
             self.finish()
         return success
+