openlp-core team mailing list archive
-
openlp-core team
-
Mailing list archive
-
Message #32319
[Merge] lp:~phill-ridout/openlp/fixes-mkII into lp:openlp
Phill has proposed merging lp:~phill-ridout/openlp/fixes-mkII into lp:openlp.
Requested reviews:
OpenLP Core (openlp-core)
Related bugs:
Bug #1400415 in OpenLP: "Multiple exceptions merged into OSError"
https://bugs.launchpad.net/openlp/+bug/1400415
Bug #1532193 in OpenLP: "Typos in songusageplugin.py"
https://bugs.launchpad.net/openlp/+bug/1532193
Bug #1660473 in OpenLP: "OSZL is ignored on save (inconsistent gui)"
https://bugs.launchpad.net/openlp/+bug/1660473
Bug #1660478 in OpenLP: "Opening recent file does not prompt to save changes"
https://bugs.launchpad.net/openlp/+bug/1660478
Bug #1660486 in OpenLP: "Dragging item in service manager without changes triggeres "unsaved""
https://bugs.launchpad.net/openlp/+bug/1660486
Bug #1661416 in OpenLP: "Initial "extract song usage data" produces a traceback"
https://bugs.launchpad.net/openlp/+bug/1661416
Bug #1672229 in OpenLP: "Media Library duplicates on boot and when a new item is added"
https://bugs.launchpad.net/openlp/+bug/1672229
Bug #1698021 in OpenLP: "Copying and Pasting from Word inserts invalid characters"
https://bugs.launchpad.net/openlp/+bug/1698021
Bug #1715125 in OpenLP: "Missing .osz file extension on save service"
https://bugs.launchpad.net/openlp/+bug/1715125
Bug #1727517 in OpenLP: "Unicode control chars causes song importer to crash"
https://bugs.launchpad.net/openlp/+bug/1727517
For more details, see:
https://code.launchpad.net/~phill-ridout/openlp/fixes-mkII/+merge/333491
Fixed a number of bugs, and tests.
Failing on Code Analysis2, but this looks like fallout from the refactors (it hasn't passed since the beginning of october)
Add this to your merge proposal:
--------------------------------
lp:~phill-ridout/openlp/fixes-mkII (revision 2792)
[SUCCESS] https://ci.openlp.io/job/Branch-01-Pull/2263/
[SUCCESS] https://ci.openlp.io/job/Branch-02-Functional-Tests/2165/
[SUCCESS] https://ci.openlp.io/job/Branch-03-Interface-Tests/2046/
[SUCCESS] https://ci.openlp.io/job/Branch-04a-Code_Analysis/1387/
[SUCCESS] https://ci.openlp.io/job/Branch-04b-Test_Coverage/1212/
[FAILURE] https://ci.openlp.io/job/Branch-04c-Code_Analysis2/342/
Stopping after failure
--
Your team OpenLP Core is requested to review the proposed merge of lp:~phill-ridout/openlp/fixes-mkII into lp:openlp.
=== modified file 'openlp/core/common/__init__.py'
--- openlp/core/common/__init__.py 2017-10-07 07:05:07 +0000
+++ openlp/core/common/__init__.py 2017-11-09 21:09:17 +0000
@@ -43,9 +43,13 @@
FIRST_CAMEL_REGEX = re.compile('(.)([A-Z][a-z]+)')
SECOND_CAMEL_REGEX = re.compile('([a-z0-9])([A-Z])')
-CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]', re.UNICODE)
-INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]', re.UNICODE)
+CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]')
+INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]')
IMAGES_FILTER = None
+REPLACMENT_CHARS_MAP = str.maketrans({'\u2018': '\'', '\u2019': '\'', '\u201c': '"', '\u201d': '"', '\u2026': '...',
+ '\u2013': '-', '\u2014': '-', '\v': '\n\n', '\f': '\n\n'})
+NEW_LINE_REGEX = re.compile(r' ?(\r\n?|\n) ?')
+WHITESPACE_REGEX = re.compile(r'[ \t]+')
def trace_error_handler(logger):
@@ -339,7 +343,7 @@
if file_path.exists():
file_path.unlink()
return True
- except (IOError, OSError):
+ except OSError:
log.exception('Unable to delete file {file_path}'.format(file_path=file_path))
return False
@@ -436,3 +440,17 @@
return detector.result
except OSError:
log.exception('Error detecting file encoding')
+
+
+def normalize_str(irreg_str):
+ """
+ Normalize the supplied string. Remove unicode control chars and tidy up white space.
+
+ :param str irreg_str: The string to normalize.
+ :return: The normalized string
+ :rtype: str
+ """
+ irreg_str = irreg_str.translate(REPLACMENT_CHARS_MAP)
+ irreg_str = CONTROL_CHARS.sub('', irreg_str)
+ irreg_str = NEW_LINE_REGEX.sub('\n', irreg_str)
+ return WHITESPACE_REGEX.sub(' ', irreg_str)
=== modified file 'openlp/core/common/applocation.py'
--- openlp/core/common/applocation.py 2017-10-07 07:05:07 +0000
+++ openlp/core/common/applocation.py 2017-11-09 21:09:17 +0000
@@ -83,7 +83,7 @@
"""
# Check if we have a different data location.
if Settings().contains('advanced/data path'):
- path = Settings().value('advanced/data path')
+ path = Path(Settings().value('advanced/data path'))
else:
path = AppLocation.get_directory(AppLocation.DataDir)
create_paths(path)
=== modified file 'openlp/core/common/httputils.py'
--- openlp/core/common/httputils.py 2017-10-07 07:05:07 +0000
+++ openlp/core/common/httputils.py 2017-11-09 21:09:17 +0000
@@ -97,8 +97,8 @@
response = requests.get(url, headers=headers, proxies=proxies, timeout=float(CONNECTION_TIMEOUT))
log.debug('Downloaded page {url}'.format(url=response.url))
break
- except IOError:
- # For now, catch IOError. All requests errors inherit from IOError
+ except OSError:
+ # For now, catch OSError. All requests errors inherit from OSError
log.exception('Unable to connect to {url}'.format(url=url))
response = None
if retries >= CONNECTION_RETRIES:
@@ -127,7 +127,7 @@
try:
response = requests.head(url, timeout=float(CONNECTION_TIMEOUT), allow_redirects=True)
return int(response.headers['Content-Length'])
- except IOError:
+ except OSError:
if retries > CONNECTION_RETRIES:
raise ConnectionError('Unable to download {url}'.format(url=url))
else:
@@ -173,7 +173,7 @@
file_path.unlink()
return False
break
- except IOError:
+ except OSError:
trace_error_handler(log)
if retries > CONNECTION_RETRIES:
if file_path.exists():
=== modified file 'openlp/core/common/i18n.py'
--- openlp/core/common/i18n.py 2017-10-07 07:05:07 +0000
+++ openlp/core/common/i18n.py 2017-11-09 21:09:17 +0000
@@ -53,7 +53,7 @@
Language = namedtuple('Language', ['id', 'name', 'code'])
ICU_COLLATOR = None
-DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+', re.UNICODE)
+DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+')
LANGUAGES = sorted([
Language(1, translate('common.languages', '(Afan) Oromo', 'Language code: om'), 'om'),
Language(2, translate('common.languages', 'Abkhazian', 'Language code: ab'), 'ab'),
=== modified file 'openlp/core/common/path.py'
--- openlp/core/common/path.py 2017-10-07 07:05:07 +0000
+++ openlp/core/common/path.py 2017-11-09 21:09:17 +0000
@@ -233,7 +233,7 @@
try:
if not path.exists():
path.mkdir(parents=True)
- except IOError:
+ except OSError:
if not kwargs.get('do_not_log', False):
log.exception('failed to check if directory exists or create directory')
=== modified file 'openlp/core/common/settings.py'
--- openlp/core/common/settings.py 2017-10-07 07:05:07 +0000
+++ openlp/core/common/settings.py 2017-11-09 21:09:17 +0000
@@ -258,6 +258,12 @@
('media/last directory', 'media/last directory', [(str_to_path, None)])
]
+ __setting_upgrade_3__ = [
+ ('songuasge/db password', 'songusage/db password', []),
+ ('songuasge/db hostname', 'songusage/db hostname', []),
+ ('songuasge/db database', 'songusage/db database', [])
+ ]
+
@staticmethod
def extend_default_settings(default_values):
"""
=== modified file 'openlp/core/lib/__init__.py'
--- openlp/core/lib/__init__.py 2017-10-10 07:08:44 +0000
+++ openlp/core/lib/__init__.py 2017-11-09 21:09:17 +0000
@@ -104,7 +104,7 @@
# no BOM was found
file_handle.seek(0)
content = file_handle.read()
- except (IOError, UnicodeError):
+ except (OSError, UnicodeError):
log.exception('Failed to open text file {text}'.format(text=text_file_path))
return content
=== modified file 'openlp/core/lib/mediamanageritem.py'
--- openlp/core/lib/mediamanageritem.py 2017-10-23 22:09:57 +0000
+++ openlp/core/lib/mediamanageritem.py 2017-11-09 21:09:17 +0000
@@ -92,7 +92,7 @@
Run some initial setup. This method is separate from __init__ in order to mock it out in tests.
"""
self.hide()
- self.whitespace = re.compile(r'[\W_]+', re.UNICODE)
+ self.whitespace = re.compile(r'[\W_]+')
visible_title = self.plugin.get_string(StringContent.VisibleName)
self.title = str(visible_title['title'])
Registry().register(self.plugin.name, self)
@@ -344,7 +344,9 @@
else:
new_files.append(file_name)
if new_files:
- self.validate_and_load(new_files, data['target'])
+ if 'target' in data:
+ self.validate_and_load(new_files, data['target'])
+ self.validate_and_load(new_files)
def dnd_move_internal(self, target):
"""
=== modified file 'openlp/core/ui/exceptionform.py'
--- openlp/core/ui/exceptionform.py 2017-10-23 22:09:57 +0000
+++ openlp/core/ui/exceptionform.py 2017-11-09 21:09:17 +0000
@@ -155,7 +155,7 @@
try:
with file_path.open('w') as report_file:
report_file.write(report_text)
- except IOError:
+ except OSError:
log.exception('Failed to write crash report')
def on_send_report_button_clicked(self):
=== modified file 'openlp/core/ui/formattingtagcontroller.py'
--- openlp/core/ui/formattingtagcontroller.py 2017-10-07 07:05:07 +0000
+++ openlp/core/ui/formattingtagcontroller.py 2017-11-09 21:09:17 +0000
@@ -43,7 +43,7 @@
r'(?P<tag>[^\s/!\?>]+)(?:\s+[^\s=]+="[^"]*")*\s*(?P<empty>/)?'
r'|(?P<cdata>!\[CDATA\[(?:(?!\]\]>).)*\]\])'
r'|(?P<procinst>\?(?:(?!\?>).)*\?)'
- r'|(?P<comment>!--(?:(?!-->).)*--))>', re.UNICODE)
+ r'|(?P<comment>!--(?:(?!-->).)*--))>')
self.html_regex = re.compile(r'^(?:[^<>]*%s)*[^<>]*$' % self.html_tag_regex.pattern)
def pre_save(self):
=== added directory 'openlp/core/ui/lib'
=== modified file 'openlp/core/ui/mainwindow.py'
--- openlp/core/ui/mainwindow.py 2017-10-23 22:09:57 +0000
+++ openlp/core/ui/mainwindow.py 2017-11-09 21:09:17 +0000
@@ -180,7 +180,7 @@
triggers=self.service_manager_contents.on_load_service_clicked)
self.file_save_item = create_action(main_window, 'fileSaveItem', icon=':/general/general_save.png',
can_shortcuts=True, category=UiStrings().File,
- triggers=self.service_manager_contents.save_file)
+ triggers=self.service_manager_contents.decide_save_method)
self.file_save_as_item = create_action(main_window, 'fileSaveAsItem', can_shortcuts=True,
category=UiStrings().File,
triggers=self.service_manager_contents.save_file_as)
@@ -1367,7 +1367,7 @@
'- Please wait for copy to finish').format(path=self.new_data_path))
dir_util.copy_tree(str(old_data_path), str(self.new_data_path))
log.info('Copy successful')
- except (IOError, os.error, DistutilsFileError) as why:
+ except (OSError, DistutilsFileError) as why:
self.application.set_normal_cursor()
log.exception('Data copy failed {err}'.format(err=str(why)))
err_text = translate('OpenLP.MainWindow',
=== modified file 'openlp/core/ui/servicemanager.py'
--- openlp/core/ui/servicemanager.py 2017-10-23 22:09:57 +0000
+++ openlp/core/ui/servicemanager.py 2017-11-09 21:09:17 +0000
@@ -193,18 +193,6 @@
text=translate('OpenLP.ServiceManager', 'Move to &bottom'), icon=':/services/service_bottom.png',
tooltip=translate('OpenLP.ServiceManager', 'Move item to the end of the service.'),
can_shortcuts=True, category=UiStrings().Service, triggers=self.on_service_end)
- self.down_action = self.order_toolbar.add_toolbar_action(
- 'down',
- text=translate('OpenLP.ServiceManager', 'Move &down'), can_shortcuts=True,
- tooltip=translate('OpenLP.ServiceManager', 'Moves the selection down the window.'), visible=False,
- triggers=self.on_move_selection_down)
- action_list.add_action(self.down_action)
- self.up_action = self.order_toolbar.add_toolbar_action(
- 'up',
- text=translate('OpenLP.ServiceManager', 'Move up'), can_shortcuts=True,
- tooltip=translate('OpenLP.ServiceManager', 'Moves the selection up the window.'), visible=False,
- triggers=self.on_move_selection_up)
- action_list.add_action(self.up_action)
self.order_toolbar.addSeparator()
self.delete_action = self.order_toolbar.add_toolbar_action(
'delete', can_shortcuts=True,
@@ -300,8 +288,8 @@
self.theme_menu = QtWidgets.QMenu(translate('OpenLP.ServiceManager', '&Change Item Theme'))
self.menu.addMenu(self.theme_menu)
self.service_manager_list.addActions([self.move_down_action, self.move_up_action, self.make_live_action,
- self.move_top_action, self.move_bottom_action, self.up_action,
- self.down_action, self.expand_action, self.collapse_action])
+ self.move_top_action, self.move_bottom_action, self.expand_action,
+ self.collapse_action])
Registry().register_function('theme_update_list', self.update_theme_list)
Registry().register_function('config_screen_changed', self.regenerate_service_items)
Registry().register_function('theme_update_global', self.theme_change)
@@ -474,6 +462,12 @@
Load a recent file as the service triggered by mainwindow recent service list.
:param field:
"""
+ if self.is_modified():
+ result = self.save_modified_service()
+ if result == QtWidgets.QMessageBox.Cancel:
+ return False
+ elif result == QtWidgets.QMessageBox.Save:
+ self.decide_save_method()
sender = self.sender()
self.load_file(sender.data())
@@ -603,7 +597,7 @@
if not os.path.exists(save_file):
shutil.copy(audio_from, save_file)
zip_file.write(audio_from, audio_to)
- except IOError:
+ except OSError:
self.log_exception('Failed to save service to disk: {name}'.format(name=temp_file_name))
self.main_window.error_message(translate('OpenLP.ServiceManager', 'Error Saving File'),
translate('OpenLP.ServiceManager', 'There was an error saving your file.'))
@@ -664,7 +658,7 @@
zip_file = zipfile.ZipFile(temp_file_name, 'w', zipfile.ZIP_STORED, True)
# First we add service contents.
zip_file.writestr(service_file_name, service_content)
- except IOError:
+ except OSError:
self.log_exception('Failed to save service to disk: {name}'.format(name=temp_file_name))
self.main_window.error_message(translate('OpenLP.ServiceManager', 'Error Saving File'),
translate('OpenLP.ServiceManager', 'There was an error saving your file.'))
@@ -712,18 +706,23 @@
default_file_path = directory_path / default_file_path
# SaveAs from osz to oszl is not valid as the files will be deleted on exit which is not sensible or usable in
# the long term.
+ lite_filter = translate('OpenLP.ServiceManager', 'OpenLP Service Files - lite (*.oszl)')
+ packaged_filter = translate('OpenLP.ServiceManager', 'OpenLP Service Files (*.osz)')
+
if self._file_name.endswith('oszl') or self.service_has_all_original_files:
file_path, filter_used = FileDialog.getSaveFileName(
self.main_window, UiStrings().SaveService, default_file_path,
- translate('OpenLP.ServiceManager',
- 'OpenLP Service Files (*.osz);; OpenLP Service Files - lite (*.oszl)'))
+ '{packaged};; {lite}'.format(packaged=packaged_filter, lite=lite_filter))
else:
file_path, filter_used = FileDialog.getSaveFileName(
- self.main_window, UiStrings().SaveService, file_path,
- translate('OpenLP.ServiceManager', 'OpenLP Service Files (*.osz);;'))
+ self.main_window, UiStrings().SaveService, default_file_path,
+ '{packaged};;'.format(packaged=packaged_filter))
if not file_path:
return False
- file_path.with_suffix('.osz')
+ if filter_used == lite_filter:
+ file_path = file_path.with_suffix('.oszl')
+ else:
+ file_path = file_path.with_suffix('.osz')
self.set_file_name(file_path)
self.decide_save_method()
@@ -791,11 +790,11 @@
else:
critical_error_message_box(message=translate('OpenLP.ServiceManager', 'File is not a valid service.'))
self.log_error('File contains no service data')
- except (IOError, NameError):
+ except (OSError, NameError):
self.log_exception('Problem loading service file {name}'.format(name=file_name))
critical_error_message_box(message=translate('OpenLP.ServiceManager',
'File could not be opened because it is corrupt.'))
- except zipfile.BadZipfile:
+ except zipfile.BadZipFile:
if os.path.getsize(file_name) == 0:
self.log_exception('Service file is zero sized: {name}'.format(name=file_name))
QtWidgets.QMessageBox.information(self, translate('OpenLP.ServiceManager', 'Empty File'),
@@ -1657,14 +1656,15 @@
if start_pos == -1:
return
if item is None:
- end_pos = len(self.service_items)
+ end_pos = len(self.service_items) - 1
else:
end_pos = get_parent_item_data(item) - 1
service_item = self.service_items[start_pos]
- self.service_items.remove(service_item)
- self.service_items.insert(end_pos, service_item)
- self.repaint_service_list(end_pos, child)
- self.set_modified()
+ if start_pos != end_pos:
+ self.service_items.remove(service_item)
+ self.service_items.insert(end_pos, service_item)
+ self.repaint_service_list(end_pos, child)
+ self.set_modified()
else:
# we are not over anything so drop
replace = False
=== modified file 'openlp/core/ui/thememanager.py'
--- openlp/core/ui/thememanager.py 2017-10-23 22:09:57 +0000
+++ openlp/core/ui/thememanager.py 2017-11-09 21:09:17 +0000
@@ -604,7 +604,7 @@
else:
with full_name.open('wb') as out_file:
out_file.write(theme_zip.read(zipped_file))
- except (IOError, zipfile.BadZipfile):
+ except (OSError, zipfile.BadZipFile):
self.log_exception('Importing theme from zip failed {name}'.format(name=file_path))
raise ValidationError
except ValidationError:
@@ -667,7 +667,7 @@
theme_path = theme_dir / '{file_name}.json'.format(file_name=name)
try:
theme_path.write_text(theme_pretty)
- except IOError:
+ except OSError:
self.log_exception('Saving theme to file failed')
if image_source_path and image_destination_path:
if self.old_background_image_path and image_destination_path != self.old_background_image_path:
@@ -675,7 +675,7 @@
if image_source_path != image_destination_path:
try:
copyfile(image_source_path, image_destination_path)
- except IOError:
+ except OSError:
self.log_exception('Failed to save theme image')
self.generate_and_save_image(name, theme)
=== modified file 'openlp/core/version.py'
--- openlp/core/version.py 2017-10-07 07:05:07 +0000
+++ openlp/core/version.py 2017-11-09 21:09:17 +0000
@@ -96,7 +96,7 @@
remote_version = response.text
log.debug('New version found: %s', remote_version)
break
- except IOError:
+ except OSError:
log.exception('Unable to connect to OpenLP server to download version file')
retries += 1
else:
@@ -182,7 +182,7 @@
try:
version_file = open(file_path, 'r')
full_version = str(version_file.read()).rstrip()
- except IOError:
+ except OSError:
log.exception('Error in version file.')
full_version = '0.0.0-bzr000'
finally:
=== modified file 'openlp/core/widgets/edits.py'
--- openlp/core/widgets/edits.py 2017-10-23 22:09:57 +0000
+++ openlp/core/widgets/edits.py 2017-11-09 21:09:17 +0000
@@ -27,6 +27,7 @@
from PyQt5 import QtCore, QtGui, QtWidgets
+from openlp.core.common import CONTROL_CHARS
from openlp.core.common.i18n import UiStrings, translate
from openlp.core.common.path import Path, path_to_str, str_to_path
from openlp.core.common.settings import Settings
@@ -241,7 +242,7 @@
self.line_edit.editingFinished.connect(self.on_line_edit_editing_finished)
self.update_button_tool_tips()
- @property
+ @QtCore.pyqtProperty('QVariant')
def path(self):
"""
A property getter method to return the selected path.
@@ -349,7 +350,7 @@
:rtype: None
"""
if self._path != path:
- self.path = path
+ self._path = path
self.pathChanged.emit(path)
@@ -470,12 +471,21 @@
cursor.insertText(html['start tag'])
cursor.insertText(html['end tag'])
+ def insertFromMimeData(self, source):
+ """
+ Reimplement `insertFromMimeData` so that we can remove any control characters
+
+ :param QtCore.QMimeData source: The mime data to insert
+ :rtype: None
+ """
+ self.insertPlainText(CONTROL_CHARS.sub('', source.text()))
+
class Highlighter(QtGui.QSyntaxHighlighter):
"""
Provides a text highlighter for pointing out spelling errors in text.
"""
- WORDS = r'(?iu)[\w\']+'
+ WORDS = r'(?i)[\w\']+'
def __init__(self, *args):
"""
=== modified file 'openlp/core/widgets/views.py'
--- openlp/core/widgets/views.py 2017-10-23 22:09:57 +0000
+++ openlp/core/widgets/views.py 2017-11-09 21:09:17 +0000
@@ -336,7 +336,7 @@
for file in listing:
files.append(os.path.join(local_file, file))
Registry().execute('{mime_data}_dnd'.format(mime_data=self.mime_data_text),
- {'files': files, 'target': self.itemAt(event.pos())})
+ {'files': files})
else:
event.ignore()
=== modified file 'openlp/plugins/bibles/forms/booknameform.py'
--- openlp/plugins/bibles/forms/booknameform.py 2017-10-07 07:05:07 +0000
+++ openlp/plugins/bibles/forms/booknameform.py 2017-11-09 21:09:17 +0000
@@ -113,8 +113,7 @@
cor_book = self.corresponding_combo_box.currentText()
for character in '\\.^$*+?{}[]()':
cor_book = cor_book.replace(character, '\\' + character)
- books = [key for key in list(self.book_names.keys()) if re.match(cor_book, str(self.book_names[key]),
- re.UNICODE)]
+ books = [key for key in list(self.book_names.keys()) if re.match(cor_book, str(self.book_names[key]))]
books = [_f for _f in map(BiblesResourcesDB.get_book, books) if _f]
if books:
self.book_id = books[0]['id']
=== modified file 'openlp/plugins/bibles/lib/__init__.py'
--- openlp/plugins/bibles/lib/__init__.py 2017-10-07 07:05:07 +0000
+++ openlp/plugins/bibles/lib/__init__.py 2017-11-09 21:09:17 +0000
@@ -224,13 +224,13 @@
range_regex = '(?:(?P<from_chapter>[0-9]+){sep_v})?' \
'(?P<from_verse>[0-9]+)(?P<range_to>{sep_r}(?:(?:(?P<to_chapter>' \
'[0-9]+){sep_v})?(?P<to_verse>[0-9]+)|{sep_e})?)?'.format_map(REFERENCE_SEPARATORS)
- REFERENCE_MATCHES['range'] = re.compile(r'^\s*{range}\s*$'.format(range=range_regex), re.UNICODE)
- REFERENCE_MATCHES['range_separator'] = re.compile(REFERENCE_SEPARATORS['sep_l'], re.UNICODE)
+ REFERENCE_MATCHES['range'] = re.compile(r'^\s*{range}\s*$'.format(range=range_regex))
+ REFERENCE_MATCHES['range_separator'] = re.compile(REFERENCE_SEPARATORS['sep_l'])
# full reference match: <book>(<range>(,(?!$)|(?=$)))+
REFERENCE_MATCHES['full'] = \
re.compile(r'^\s*(?!\s)(?P<book>[\d]*[.]?[^\d\.]+)\.*(?<!\s)\s*'
r'(?P<ranges>(?:{range_regex}(?:{sep_l}(?!\s*$)|(?=\s*$)))+)\s*$'.format(
- range_regex=range_regex, sep_l=REFERENCE_SEPARATORS['sep_l']), re.UNICODE)
+ range_regex=range_regex, sep_l=REFERENCE_SEPARATORS['sep_l']))
def get_reference_separator(separator_type):
=== modified file 'openlp/plugins/bibles/lib/db.py'
--- openlp/plugins/bibles/lib/db.py 2017-10-23 22:09:57 +0000
+++ openlp/plugins/bibles/lib/db.py 2017-11-09 21:09:17 +0000
@@ -307,8 +307,7 @@
book_escaped = book
for character in RESERVED_CHARACTERS:
book_escaped = book_escaped.replace(character, '\\' + character)
- regex_book = re.compile('\\s*{book}\\s*'.format(book='\\s*'.join(book_escaped.split())),
- re.UNICODE | re.IGNORECASE)
+ regex_book = re.compile('\\s*{book}\\s*'.format(book='\\s*'.join(book_escaped.split())), re.IGNORECASE)
if language_selection == LanguageSelection.Bible:
db_book = self.get_book(book)
if db_book:
=== modified file 'openlp/plugins/images/lib/mediaitem.py'
--- openlp/plugins/images/lib/mediaitem.py 2017-10-23 22:09:57 +0000
+++ openlp/plugins/images/lib/mediaitem.py 2017-11-09 21:09:17 +0000
@@ -366,7 +366,7 @@
if validate_thumb(image.file_path, thumbnail_path):
icon = build_icon(thumbnail_path)
else:
- icon = create_thumb(image.file_path, thumbnail_path)
+ icon = create_thumb(str(image.file_path), str(thumbnail_path))
item_name = QtWidgets.QTreeWidgetItem([file_name])
item_name.setText(0, file_name)
item_name.setIcon(0, icon)
@@ -390,6 +390,7 @@
:param files: A List of strings containing the filenames of the files to be loaded
:param target_group: The QTreeWidgetItem of the group that will be the parent of the added files
"""
+ file_paths = [Path(file) for file in file_paths]
self.application.set_normal_cursor()
self.load_list(file_paths, target_group)
last_dir = file_paths[0].parent
=== modified file 'openlp/plugins/presentations/lib/pptviewcontroller.py'
--- openlp/plugins/presentations/lib/pptviewcontroller.py 2017-10-10 07:08:44 +0000
+++ openlp/plugins/presentations/lib/pptviewcontroller.py 2017-11-09 21:09:17 +0000
@@ -70,7 +70,7 @@
try:
self.start_process()
return self.process.CheckInstalled()
- except WindowsError:
+ except OSError:
return False
def start_process(self):
=== modified file 'openlp/plugins/songs/forms/editsongform.py'
--- openlp/plugins/songs/forms/editsongform.py 2017-10-23 22:09:57 +0000
+++ openlp/plugins/songs/forms/editsongform.py 2017-11-09 21:09:17 +0000
@@ -105,9 +105,9 @@
self.topics_list_view.setSortingEnabled(False)
self.topics_list_view.setAlternatingRowColors(True)
self.audio_list_widget.setAlternatingRowColors(True)
- self.find_verse_split = re.compile('---\[\]---\n', re.UNICODE)
- self.whitespace = re.compile(r'\W+', re.UNICODE)
- self.find_tags = re.compile(u'\{/?\w+\}', re.UNICODE)
+ self.find_verse_split = re.compile('---\[\]---\n')
+ self.whitespace = re.compile(r'\W+')
+ self.find_tags = re.compile(r'\{/?\w+\}')
def _load_objects(self, cls, combo, cache):
"""
=== modified file 'openlp/plugins/songs/lib/__init__.py'
--- openlp/plugins/songs/lib/__init__.py 2017-10-10 02:29:56 +0000
+++ openlp/plugins/songs/lib/__init__.py 2017-11-09 21:09:17 +0000
@@ -24,7 +24,6 @@
"""
import logging
-import os
import re
from PyQt5 import QtWidgets
@@ -39,8 +38,8 @@
log = logging.getLogger(__name__)
-WHITESPACE = re.compile(r'[\W_]+', re.UNICODE)
-APOSTROPHE = re.compile('[\'`’ʻ′]', re.UNICODE)
+WHITESPACE = re.compile(r'[\W_]+')
+APOSTROPHE = re.compile(r'[\'`’ʻ′]')
# PATTERN will look for the next occurence of one of these symbols:
# \controlword - optionally preceded by \*, optionally followed by a number
# \'## - where ## is a pair of hex digits, representing a single character
=== modified file 'openlp/plugins/songs/lib/importers/easyslides.py'
--- openlp/plugins/songs/lib/importers/easyslides.py 2017-09-30 20:16:30 +0000
+++ openlp/plugins/songs/lib/importers/easyslides.py 2017-11-09 21:09:17 +0000
@@ -25,6 +25,7 @@
from lxml import etree, objectify
+from openlp.core.common import normalize_str
from openlp.plugins.songs.lib import VerseType
from openlp.plugins.songs.lib.importers.songimport import SongImport
@@ -225,7 +226,7 @@
verses[reg].setdefault(vt, {})
verses[reg][vt].setdefault(vn, {})
verses[reg][vt][vn].setdefault(inst, [])
- verses[reg][vt][vn][inst].append(self.tidy_text(line))
+ verses[reg][vt][vn][inst].append(normalize_str(line))
# done parsing
versetags = []
# we use our_verse_order to ensure, we insert lyrics in the same order
=== modified file 'openlp/plugins/songs/lib/importers/mediashout.py'
--- openlp/plugins/songs/lib/importers/mediashout.py 2017-10-07 07:05:07 +0000
+++ openlp/plugins/songs/lib/importers/mediashout.py 2017-11-09 21:09:17 +0000
@@ -101,7 +101,7 @@
self.song_book_name = song.SongID
for verse in verses:
tag = VERSE_TAGS[verse.Type] + str(verse.Number) if verse.Type < len(VERSE_TAGS) else 'O'
- self.add_verse(self.tidy_text(verse.Text), tag)
+ self.add_verse(verse.Text, tag)
for order in verse_order:
if order.Type < len(VERSE_TAGS):
self.verse_order_list.append(VERSE_TAGS[order.Type] + str(order.Number))
=== modified file 'openlp/plugins/songs/lib/importers/openoffice.py'
--- openlp/plugins/songs/lib/importers/openoffice.py 2017-10-10 02:29:56 +0000
+++ openlp/plugins/songs/lib/importers/openoffice.py 2017-11-09 21:09:17 +0000
@@ -24,7 +24,7 @@
from PyQt5 import QtCore
-from openlp.core.common import is_win, get_uno_command, get_uno_instance
+from openlp.core.common import get_uno_command, get_uno_instance, is_win, normalize_str
from openlp.core.common.i18n import translate
from .songimport import SongImport
@@ -241,7 +241,7 @@
:param text: The text.
"""
- song_texts = self.tidy_text(text).split('\f')
+ song_texts = normalize_str(text).split('\f')
self.set_defaults()
for song_text in song_texts:
if song_text.strip():
=== modified file 'openlp/plugins/songs/lib/importers/opensong.py'
--- openlp/plugins/songs/lib/importers/opensong.py 2017-10-10 02:29:56 +0000
+++ openlp/plugins/songs/lib/importers/opensong.py 2017-11-09 21:09:17 +0000
@@ -25,6 +25,7 @@
from lxml import objectify
from lxml.etree import Error, LxmlError
+from openlp.core.common import normalize_str
from openlp.core.common.i18n import translate
from openlp.core.common.settings import Settings
from openlp.plugins.songs.lib import VerseType
@@ -262,7 +263,7 @@
post=this_line[offset + column:])
offset += len(chord) + 2
# Tidy text and remove the ____s from extended words
- this_line = self.tidy_text(this_line)
+ this_line = normalize_str(this_line)
this_line = this_line.replace('_', '')
this_line = this_line.replace('||', '\n[---]\n')
this_line = this_line.strip()
=== modified file 'openlp/plugins/songs/lib/importers/songimport.py'
--- openlp/plugins/songs/lib/importers/songimport.py 2017-10-23 22:09:57 +0000
+++ openlp/plugins/songs/lib/importers/songimport.py 2017-11-09 21:09:17 +0000
@@ -25,6 +25,7 @@
from PyQt5 import QtCore
+from openlp.core.common import normalize_str
from openlp.core.common.applocation import AppLocation
from openlp.core.common.i18n import translate
from openlp.core.common.path import copyfile, create_paths
@@ -130,26 +131,6 @@
def register(self, import_wizard):
self.import_wizard = import_wizard
- def tidy_text(self, text):
- """
- Get rid of some dodgy unicode and formatting characters we're not interested in. Some can be converted to ascii.
- """
- text = text.replace('\u2018', '\'')
- text = text.replace('\u2019', '\'')
- text = text.replace('\u201c', '"')
- text = text.replace('\u201d', '"')
- text = text.replace('\u2026', '...')
- text = text.replace('\u2013', '-')
- text = text.replace('\u2014', '-')
- # Replace vertical tab with 2 linebreaks
- text = text.replace('\v', '\n\n')
- # Replace form feed (page break) with 2 linebreaks
- text = text.replace('\f', '\n\n')
- # Remove surplus blank lines, spaces, trailing/leading spaces
- text = re.sub(r'[ \t]+', ' ', text)
- text = re.sub(r' ?(\r\n?|\n) ?', '\n', text)
- return text
-
def process_song_text(self, text):
"""
Process the song text from import
@@ -368,7 +349,7 @@
verse_tag = VerseType.tags[VerseType.Other]
log.info('Versetype {old} changing to {new}'.format(old=verse_def, new=new_verse_def))
verse_def = new_verse_def
- sxml.add_verse_to_lyrics(verse_tag, verse_def[1:], verse_text, lang)
+ sxml.add_verse_to_lyrics(verse_tag, verse_def[1:], normalize_str(verse_text), lang)
song.lyrics = str(sxml.extract_xml(), 'utf-8')
if not self.verse_order_list and self.verse_order_list_generated_useful:
self.verse_order_list = self.verse_order_list_generated
=== modified file 'openlp/plugins/songs/lib/importers/songsoffellowship.py'
--- openlp/plugins/songs/lib/importers/songsoffellowship.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/songsoffellowship.py 2017-11-09 21:09:17 +0000
@@ -194,7 +194,6 @@
:param text_portion: A Piece of text
"""
text = text_portion.getString()
- text = self.tidy_text(text)
if text.strip() == '':
return text
if text_portion.CharWeight == BOLD:
=== modified file 'openlp/plugins/songs/lib/importers/zionworx.py'
--- openlp/plugins/songs/lib/importers/zionworx.py 2017-10-10 02:29:56 +0000
+++ openlp/plugins/songs/lib/importers/zionworx.py 2017-11-09 21:09:17 +0000
@@ -30,9 +30,6 @@
log = logging.getLogger(__name__)
-# Used to strip control chars (except 10=LF, 13=CR)
-CONTROL_CHARS_MAP = dict.fromkeys(list(range(10)) + [11, 12] + list(range(14, 32)) + [127])
-
class ZionWorxImport(SongImport):
"""
@@ -95,12 +92,12 @@
return
self.set_defaults()
try:
- self.title = self._decode(record['Title1'])
+ self.title = record['Title1']
if record['Title2']:
- self.alternate_title = self._decode(record['Title2'])
- self.parse_author(self._decode(record['Writer']))
- self.add_copyright(self._decode(record['Copyright']))
- lyrics = self._decode(record['Lyrics'])
+ self.alternate_title = record['Title2']
+ self.parse_author(record['Writer'])
+ self.add_copyright(record['Copyright'])
+ lyrics = record['Lyrics']
except UnicodeDecodeError as e:
self.log_error(translate('SongsPlugin.ZionWorxImport', 'Record {index}').format(index=index),
translate('SongsPlugin.ZionWorxImport', 'Decoding error: {error}').format(error=e))
@@ -122,10 +119,3 @@
if not self.finish():
self.log_error(translate('SongsPlugin.ZionWorxImport', 'Record %d') % index +
(': "' + title + '"' if title else ''))
-
- def _decode(self, str):
- """
- Strips all control characters (except new lines).
- """
- # ZionWorx has no option for setting the encoding for its songs, so we assume encoding is always the same.
- return str.translate(CONTROL_CHARS_MAP)
=== modified file 'openlp/plugins/songs/lib/openlyricsxml.py'
--- openlp/plugins/songs/lib/openlyricsxml.py 2017-10-10 02:29:56 +0000
+++ openlp/plugins/songs/lib/openlyricsxml.py 2017-11-09 21:09:17 +0000
@@ -281,7 +281,7 @@
# Process the formatting tags.
# Have we any tags in song lyrics?
tags_element = None
- match = re.search('\{/?\w+\}', song.lyrics, re.UNICODE)
+ match = re.search(r'\{/?\w+\}', song.lyrics)
if match:
# Named 'format_' - 'format' is built-in function in Python.
format_ = etree.SubElement(song_xml, 'format')
=== modified file 'openlp/plugins/songusage/forms/songusagedetailform.py'
--- openlp/plugins/songusage/forms/songusagedetailform.py 2017-10-23 22:09:57 +0000
+++ openlp/plugins/songusage/forms/songusagedetailform.py 2017-11-09 21:09:17 +0000
@@ -54,8 +54,14 @@
"""
We need to set up the screen
"""
- self.from_date_calendar.setSelectedDate(Settings().value(self.plugin.settings_section + '/from date'))
- self.to_date_calendar.setSelectedDate(Settings().value(self.plugin.settings_section + '/to date'))
+ to_date = Settings().value(self.plugin.settings_section + '/to date')
+ if not (isinstance(to_date, QtCore.QDate) and to_date.isValid()):
+ to_date = QtCore.QDate.currentDate()
+ from_date = Settings().value(self.plugin.settings_section + '/from date')
+ if not (isinstance(from_date, QtCore.QDate) and from_date.isValid()):
+ from_date = to_date.addYears(-1)
+ self.from_date_calendar.setSelectedDate(from_date)
+ self.to_date_calendar.setSelectedDate(to_date)
self.report_path_edit.path = Settings().value(self.plugin.settings_section + '/last directory export')
def on_report_path_edit_path_changed(self, file_path):
=== modified file 'openlp/plugins/songusage/songusageplugin.py'
--- openlp/plugins/songusage/songusageplugin.py 2017-10-07 07:05:07 +0000
+++ openlp/plugins/songusage/songusageplugin.py 2017-11-09 21:09:17 +0000
@@ -38,20 +38,17 @@
log = logging.getLogger(__name__)
-YEAR = QtCore.QDate().currentDate().year()
-if QtCore.QDate().currentDate().month() < 9:
- YEAR -= 1
-
+TODAY = QtCore.QDate.currentDate()
__default_settings__ = {
'songusage/db type': 'sqlite',
'songusage/db username': '',
- 'songuasge/db password': '',
- 'songuasge/db hostname': '',
- 'songuasge/db database': '',
+ 'songusage/db password': '',
+ 'songusage/db hostname': '',
+ 'songusage/db database': '',
'songusage/active': False,
- 'songusage/to date': QtCore.QDate(YEAR, 8, 31),
- 'songusage/from date': QtCore.QDate(YEAR - 1, 9, 1),
+ 'songusage/to date': TODAY,
+ 'songusage/from date': TODAY.addYears(-1),
'songusage/last directory export': None
}
=== modified file 'tests/functional/openlp_core/common/test_actions.py'
--- tests/functional/openlp_core/common/test_actions.py 2017-10-07 07:05:07 +0000
+++ tests/functional/openlp_core/common/test_actions.py 2017-11-09 21:09:17 +0000
@@ -153,6 +153,7 @@
"""
Prepare the tests
"""
+ self.setup_application()
self.action_list = ActionList.get_instance()
self.build_settings()
self.settings = Settings()
=== modified file 'tests/functional/openlp_core/common/test_httputils.py'
--- tests/functional/openlp_core/common/test_httputils.py 2017-09-25 20:34:05 +0000
+++ tests/functional/openlp_core/common/test_httputils.py 2017-11-09 21:09:17 +0000
@@ -233,7 +233,7 @@
Test socket timeout gets caught
"""
# GIVEN: Mocked urlopen to fake a network disconnect in the middle of a download
- mocked_requests.get.side_effect = IOError
+ mocked_requests.get.side_effect = OSError
# WHEN: Attempt to retrieve a file
url_get_file(MagicMock(), url='http://localhost/test', file_path=Path(self.tempfile))
=== modified file 'tests/functional/openlp_core/common/test_i18n.py'
--- tests/functional/openlp_core/common/test_i18n.py 2017-10-07 07:05:07 +0000
+++ tests/functional/openlp_core/common/test_i18n.py 2017-11-09 21:09:17 +0000
@@ -155,7 +155,7 @@
assert first_instance is second_instance, 'Two UiStrings objects should be the same instance'
-def test_translate(self):
+def test_translate():
"""
Test the translate() function
"""
=== modified file 'tests/functional/openlp_core/common/test_path.py'
--- tests/functional/openlp_core/common/test_path.py 2017-10-07 07:05:07 +0000
+++ tests/functional/openlp_core/common/test_path.py 2017-11-09 21:09:17 +0000
@@ -371,13 +371,13 @@
@patch('openlp.core.common.path.log')
def test_create_paths_dir_io_error(self, mocked_logger):
"""
- Test the create_paths() when an IOError is raised
+ Test the create_paths() when an OSError is raised
"""
# GIVEN: A `Path` to check with patched out mkdir and exists methods
mocked_path = MagicMock()
- mocked_path.exists.side_effect = IOError('Cannot make directory')
+ mocked_path.exists.side_effect = OSError('Cannot make directory')
- # WHEN: An IOError is raised when checking the if the path exists.
+ # WHEN: An OSError is raised when checking the if the path exists.
create_paths(mocked_path)
# THEN: The Error should have been logged
@@ -385,7 +385,7 @@
def test_create_paths_dir_value_error(self):
"""
- Test the create_paths() when an error other than IOError is raised
+ Test the create_paths() when an error other than OSError is raised
"""
# GIVEN: A `Path` to check with patched out mkdir and exists methods
mocked_path = MagicMock()
=== modified file 'tests/functional/openlp_core/lib/test_lib.py'
--- tests/functional/openlp_core/lib/test_lib.py 2017-10-10 07:08:44 +0000
+++ tests/functional/openlp_core/lib/test_lib.py 2017-11-09 21:09:17 +0000
@@ -168,7 +168,7 @@
patch.object(Path, 'open'):
file_path = Path('testfile.txt')
file_path.is_file.return_value = True
- file_path.open.side_effect = IOError()
+ file_path.open.side_effect = OSError()
# WHEN: get_text_file_string is called
result = get_text_file_string(file_path)
=== modified file 'tests/functional/openlp_core/ui/test_first_time.py'
--- tests/functional/openlp_core/ui/test_first_time.py 2017-09-20 16:55:21 +0000
+++ tests/functional/openlp_core/ui/test_first_time.py 2017-11-09 21:09:17 +0000
@@ -40,7 +40,7 @@
Test get_web_page will attempt CONNECTION_RETRIES+1 connections - bug 1409031
"""
# GIVEN: Initial settings and mocks
- mocked_requests.get.side_effect = IOError('Unable to connect')
+ mocked_requests.get.side_effect = OSError('Unable to connect')
# WHEN: A webpage is requested
try:
=== modified file 'tests/functional/openlp_core/widgets/test_views.py'
--- tests/functional/openlp_core/widgets/test_views.py 2017-10-23 22:09:57 +0000
+++ tests/functional/openlp_core/widgets/test_views.py 2017-11-09 21:09:17 +0000
@@ -627,4 +627,3 @@
assert widget.allow_internal_dnd is False
assert widget.indentation() == 0
assert widget.isAnimated() is True
-
=== modified file 'tests/functional/openlp_plugins/presentations/test_presentationcontroller.py'
--- tests/functional/openlp_plugins/presentations/test_presentationcontroller.py 2017-10-07 07:05:07 +0000
+++ tests/functional/openlp_plugins/presentations/test_presentationcontroller.py 2017-11-09 21:09:17 +0000
@@ -144,7 +144,7 @@
# GIVEN: A mocked open, get_thumbnail_folder and exists
with patch('openlp.plugins.presentations.lib.presentationcontroller.Path.read_text') as mocked_read_text, \
patch(FOLDER_TO_PATCH) as mocked_get_thumbnail_folder:
- mocked_read_text.side_effect = IOError()
+ mocked_read_text.side_effect = OSError()
mocked_get_thumbnail_folder.return_value = Path('test')
# WHEN: calling get_titles_and_notes
=== modified file 'tests/interfaces/openlp_core/ui/test_projectormanager.py'
--- tests/interfaces/openlp_core/ui/test_projectormanager.py 2017-10-07 07:05:07 +0000
+++ tests/interfaces/openlp_core/ui/test_projectormanager.py 2017-11-09 21:09:17 +0000
@@ -42,8 +42,8 @@
"""
Create the UI and setup necessary options
"""
+ self.setup_application()
self.build_settings()
- self.setup_application()
Registry.create()
with patch('openlp.core.lib.projector.db.init_url') as mocked_init_url:
if os.path.exists(TEST_DB):
=== modified file 'tests/interfaces/openlp_core/ui/test_projectorsourceform.py'
--- tests/interfaces/openlp_core/ui/test_projectorsourceform.py 2017-10-07 07:05:07 +0000
+++ tests/interfaces/openlp_core/ui/test_projectorsourceform.py 2017-11-09 21:09:17 +0000
@@ -64,8 +64,8 @@
Set up anything necessary for all tests
"""
mocked_init_url.return_value = 'sqlite:///{}'.format(TEST_DB)
+ self.setup_application()
self.build_settings()
- self.setup_application()
Registry.create()
# Do not try to recreate if we've already been created from a previous test
if not hasattr(self, 'projectordb'):
=== modified file 'tests/interfaces/openlp_core/ui/test_thememanager.py'
--- tests/interfaces/openlp_core/ui/test_thememanager.py 2017-10-10 01:08:09 +0000
+++ tests/interfaces/openlp_core/ui/test_thememanager.py 2017-11-09 21:09:17 +0000
@@ -41,8 +41,8 @@
"""
Create the UI
"""
+ self.setup_application()
self.build_settings()
- self.setup_application()
Registry.create()
self.theme_manager = ThemeManager()
=== modified file 'tests/utils/__init__.py'
--- tests/utils/__init__.py 2016-12-31 11:01:36 +0000
+++ tests/utils/__init__.py 2017-11-09 21:09:17 +0000
@@ -36,7 +36,7 @@
try:
items = json.load(open_file)
first_line = items[row]
- except IOError:
+ except OSError:
first_line = ''
finally:
open_file.close()
Follow ups
