launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #25349
[Merge] ~cjwatson/launchpad:py3-services-unicode into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:py3-services-unicode into launchpad:master.
Commit message:
Port unicode() calls in lp.services to Python 3
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/391189
This unfortunately requires some contextual clues, because `six.text_type(b'foo')` returns `u'foo'` on Python 2 but `"b'foo'"` on Python 3, while `six.ensure_text` works on bytes or text but not on other types. Use single-argument `six.text_type` in cases where we know that the argument is not bytes, `six.ensure_text` where we know the argument is either bytes or text, and something more elaborate otherwise.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:py3-services-unicode into launchpad:master.
diff --git a/lib/lp/services/apachelogparser/base.py b/lib/lp/services/apachelogparser/base.py
index f2e556a..e5fbaa7 100644
--- a/lib/lp/services/apachelogparser/base.py
+++ b/lib/lp/services/apachelogparser/base.py
@@ -11,6 +11,7 @@ from lazr.uri import (
URI,
)
import pytz
+import six
from zope.component import getUtility
from lp.services.apachelogparser.model.parsedapachelog import ParsedApacheLog
@@ -34,7 +35,7 @@ def get_files_to_parse(file_paths):
file_paths = sorted(file_paths, key=lambda path: os.stat(path).st_mtime)
for file_path in file_paths:
fd, file_size = get_fd_and_file_size(file_path)
- first_line = unicode(fd.readline())
+ first_line = six.ensure_text(fd.readline())
parsed_file = store.find(ParsedApacheLog, first_line=first_line).one()
position = 0
if parsed_file is not None:
@@ -169,7 +170,7 @@ def parse_file(fd, start_position, logger, get_download_key, parsed_lines=0):
def create_or_update_parsedlog_entry(first_line, parsed_bytes):
"""Create or update the ParsedApacheLog with the given first_line."""
- first_line = unicode(first_line)
+ first_line = six.ensure_text(first_line)
parsed_file = IStore(ParsedApacheLog).find(
ParsedApacheLog, first_line=first_line).one()
if parsed_file is None:
diff --git a/lib/lp/services/apachelogparser/model/parsedapachelog.py b/lib/lp/services/apachelogparser/model/parsedapachelog.py
index 3bab7f6..128a46c 100644
--- a/lib/lp/services/apachelogparser/model/parsedapachelog.py
+++ b/lib/lp/services/apachelogparser/model/parsedapachelog.py
@@ -4,6 +4,7 @@
__metaclass__ = type
__all__ = ['ParsedApacheLog']
+import six
from storm.locals import (
Int,
Storm,
@@ -31,6 +32,6 @@ class ParsedApacheLog(Storm):
def __init__(self, first_line, bytes_read):
super(ParsedApacheLog, self).__init__()
- self.first_line = unicode(first_line)
+ self.first_line = six.ensure_text(first_line)
self.bytes_read = bytes_read
IStore(self.__class__).add(self)
diff --git a/lib/lp/services/features/model.py b/lib/lp/services/features/model.py
index 3a6b070..2eeb63f 100644
--- a/lib/lp/services/features/model.py
+++ b/lib/lp/services/features/model.py
@@ -12,6 +12,7 @@ __metaclass__ = type
from datetime import datetime
import pytz
+import six
from storm.locals import (
DateTime,
Int,
@@ -58,9 +59,9 @@ class FeatureFlagChangelogEntry(Storm):
def __init__(self, diff, comment, person):
super(FeatureFlagChangelogEntry, self).__init__()
- self.diff = unicode(diff)
+ self.diff = six.ensure_text(diff)
self.date_changed = datetime.now(pytz.timezone('UTC'))
- self.comment = unicode(comment)
+ self.comment = six.ensure_text(comment)
self.person = person
diff --git a/lib/lp/services/features/rulesource.py b/lib/lp/services/features/rulesource.py
index 65e3037..6f8310f 100644
--- a/lib/lp/services/features/rulesource.py
+++ b/lib/lp/services/features/rulesource.py
@@ -19,6 +19,7 @@ from collections import (
)
import re
+import six
from storm.locals import Desc
from lp.services.features.model import (
@@ -100,7 +101,7 @@ class FeatureRuleSource(object):
continue
flag, scope, priority_str, value = re.split('[ \t]+', line, 3)
priority = int(priority_str)
- r.append((flag, scope, priority, unicode(value)))
+ r.append((flag, scope, priority, six.ensure_text(value)))
if priority in seen_priorities[flag]:
raise DuplicatePriorityError(flag, priority)
seen_priorities[flag].add(priority)
@@ -146,8 +147,8 @@ class StormFeatureRuleSource(FeatureRuleSource):
store.execute('DELETE FROM FeatureFlag')
for (flag, scope, priority, value) in new_rules:
store.add(FeatureFlag(
- scope=unicode(scope),
- flag=unicode(flag),
+ scope=six.ensure_text(scope),
+ flag=six.ensure_text(flag),
value=value,
priority=priority))
store.flush()
diff --git a/lib/lp/services/features/testing.py b/lib/lp/services/features/testing.py
index 3a2d69b..e4ed8be 100644
--- a/lib/lp/services/features/testing.py
+++ b/lib/lp/services/features/testing.py
@@ -91,7 +91,7 @@ class FeatureFixtureMixin:
flag=flag_name,
scope='default',
priority=999,
- value=unicode(value))
+ value=six.text_type(value))
for flag_name, value in six.iteritems(self.desired_features)
if value is not None]
diff --git a/lib/lp/services/features/xmlrpc.py b/lib/lp/services/features/xmlrpc.py
index d7d681b..e2c6e54 100644
--- a/lib/lp/services/features/xmlrpc.py
+++ b/lib/lp/services/features/xmlrpc.py
@@ -9,6 +9,7 @@ __all__ = [
'FeatureFlagApplication',
]
+import six
from zope.component import getUtility
from zope.interface import implementer
@@ -52,7 +53,7 @@ class FeatureFlagApplication:
scopes.append(TeamScope(lambda: person))
else:
scopes.append(FixedScope(scope_name))
- flag_name = unicode(flag_name)
+ flag_name = six.ensure_text(flag_name)
controller = FeatureController(
MultiScopeHandler(scopes).lookup, StormFeatureRuleSource())
return controller.getFlag(flag_name)
diff --git a/lib/lp/services/feeds/feed.py b/lib/lp/services/feeds/feed.py
index 76ff15b..352e540 100644
--- a/lib/lp/services/feeds/feed.py
+++ b/lib/lp/services/feeds/feed.py
@@ -21,6 +21,7 @@ import operator
import os
import time
+import six
from six.moves.urllib.parse import urljoin
from zope.browserpage import ViewPageTemplateFile
from zope.component import getUtility
@@ -295,7 +296,7 @@ class FeedTypedData:
for a_tag in a_tags:
if a_tag['href'].startswith('/'):
a_tag['href'] = urljoin(self.root_url, a_tag['href'])
- altered_content = unicode(soup)
+ altered_content = six.text_type(soup)
else:
altered_content = self._content
@@ -303,7 +304,7 @@ class FeedTypedData:
altered_content = html_escape(altered_content)
elif self.content_type == 'xhtml':
soup = BeautifulSoup(altered_content)
- altered_content = unicode(soup)
+ altered_content = six.text_type(soup)
return altered_content
diff --git a/lib/lp/services/feeds/tests/helper.py b/lib/lp/services/feeds/tests/helper.py
index ca7c741..af37ade 100644
--- a/lib/lp/services/feeds/tests/helper.py
+++ b/lib/lp/services/feeds/tests/helper.py
@@ -26,6 +26,7 @@ if socket.getdefaulttimeout() != original_timeout:
from cStringIO import StringIO
from textwrap import wrap
+import six
from zope.interface import (
Attribute,
implementer,
@@ -125,7 +126,7 @@ def validate_feed(content, content_type, base_uri):
max(error_line_number - 2, 1),
min(error_line_number + 3, len(lines))))
for line_number in line_number_range:
- unicode_line = unicode(
+ unicode_line = six.ensure_text(
lines[line_number - 1], 'ascii', 'replace')
ascii_line = unicode_line.encode('ascii', 'replace')
wrapped_lines = wrap(ascii_line, max_line_length)
diff --git a/lib/lp/services/fields/__init__.py b/lib/lp/services/fields/__init__.py
index 5675366..9ad33f6 100644
--- a/lib/lp/services/fields/__init__.py
+++ b/lib/lp/services/fields/__init__.py
@@ -64,6 +64,7 @@ from lazr.uri import (
InvalidURIError,
URI,
)
+import six
from zope.component import getUtility
from zope.interface import (
implementer,
@@ -614,7 +615,7 @@ class URIField(TextLine):
uri = uri.ensureSlash()
else:
uri = uri.ensureNoSlash()
- input = unicode(uri)
+ input = six.text_type(uri)
return input
def _validate(self, value):
diff --git a/lib/lp/services/gpg/doc/gpg-encryption.txt b/lib/lp/services/gpg/doc/gpg-encryption.txt
index 1c79391..d26a497 100644
--- a/lib/lp/services/gpg/doc/gpg-encryption.txt
+++ b/lib/lp/services/gpg/doc/gpg-encryption.txt
@@ -5,6 +5,7 @@ This document describes the current procedure to encrypt and
decrypt contents in Launchpad, and demonstrates that the methods
can support Unicode contents.
+ >>> import six
>>> from lp.testing.gpgkeys import (
... import_public_test_keys, import_secret_test_key, decrypt_content)
>>> import transaction
@@ -83,7 +84,7 @@ Decrypt a unicode content:
>>> content = u'a\xe7ucar'
>>> cipher = gpghandler.encryptContent(content.encode('iso-8859-1'), key)
- >>> cipher = unicode(cipher)
+ >>> cipher = six.ensure_text(cipher)
>>> plain = decrypt_content(cipher, 'test')
Traceback (most recent call last):
...
diff --git a/lib/lp/services/mail/sendmail.py b/lib/lp/services/mail/sendmail.py
index 6f894e3..86d5ed1 100644
--- a/lib/lp/services/mail/sendmail.py
+++ b/lib/lp/services/mail/sendmail.py
@@ -452,7 +452,7 @@ def sendmail(message, to_addrs=None, bulk=True):
message_detail = message['Subject']
if not isinstance(message_detail, six.string_types):
# Might be a Header object; can be squashed.
- message_detail = unicode(message_detail)
+ message_detail = six.text_type(message_detail)
if _immediate_mail_delivery:
# Immediate email delivery is not unit tested, and won't be.
# The immediate-specific stuff is pretty simple though so this
diff --git a/lib/lp/services/messages/model/message.py b/lib/lp/services/messages/model/message.py
index 10a62ee..98c1f2a 100644
--- a/lib/lp/services/messages/model/message.py
+++ b/lib/lp/services/messages/model/message.py
@@ -153,7 +153,7 @@ class Message(SQLBase):
@classmethod
def chunks_text(cls, chunks):
- bits = [unicode(chunk) for chunk in chunks if chunk.content]
+ bits = [six.text_type(chunk) for chunk in chunks if chunk.content]
return '\n\n'.join(bits)
# XXX flacoste 2006-09-08: Bogus attribute only present so that
@@ -270,7 +270,7 @@ class MessageSet:
re_encoded_bits.append(
(self.decode(bytes, charset).encode('utf-8'), 'utf-8'))
- return unicode(email.header.make_header(re_encoded_bits))
+ return six.text_type(email.header.make_header(re_encoded_bits))
def fromEmail(self, email_message, owner=None, filealias=None,
parsed_message=None, create_missing_persons=False,
@@ -571,8 +571,8 @@ class UserToUserEmail(Storm):
# Initialize.
self.sender = sender
self.recipient = recipient
- self.message_id = unicode(message_id, 'ascii')
- self.subject = unicode(make_header(decode_header(subject)))
+ self.message_id = six.ensure_text(message_id, 'ascii')
+ self.subject = six.text_type(make_header(decode_header(subject)))
# Add the object to the store of the sender. Our StormMigrationGuide
# recommends against this saying "Note that the constructor should not
# usually add the object to a store -- leave that for a FooSet.new()
diff --git a/lib/lp/services/oauth/browser/__init__.py b/lib/lp/services/oauth/browser/__init__.py
index adb72dc..61e8845 100644
--- a/lib/lp/services/oauth/browser/__init__.py
+++ b/lib/lp/services/oauth/browser/__init__.py
@@ -16,6 +16,7 @@ from datetime import (
from lazr.restful import HTTPResource
import pytz
import simplejson
+import six
from zope.component import getUtility
from zope.formlib.form import (
Action,
@@ -415,7 +416,7 @@ class OAuthAccessTokenView(LaunchpadView):
def _set_status_and_error(self, error):
self.request.response.setStatus(403)
- return unicode(error)
+ return six.text_type(error)
def __call__(self):
"""Create an access token and respond with its key/secret/context.
diff --git a/lib/lp/services/tokens.py b/lib/lp/services/tokens.py
index 6e88cfa..35f7f42 100644
--- a/lib/lp/services/tokens.py
+++ b/lib/lp/services/tokens.py
@@ -11,6 +11,8 @@ __all__ = [
import random
+import six
+
def create_token(token_length):
"""Create a random token string.
@@ -23,4 +25,4 @@ def create_token(token_length):
token = ''.join(
random.SystemRandom().choice(characters)
for count in range(token_length))
- return unicode(token)
+ return six.ensure_text(token)
diff --git a/lib/lp/services/utils.py b/lib/lp/services/utils.py
index d3def35..dc15176 100644
--- a/lib/lp/services/utils.py
+++ b/lib/lp/services/utils.py
@@ -165,8 +165,10 @@ def value_string(item):
return '(not set)'
elif zope_isinstance(item, BaseItem):
return item.title
+ elif zope_isinstance(item, bytes):
+ return six.ensure_text(item)
else:
- return unicode(item)
+ return six.text_type(item)
def text_delta(instance_delta, delta_names, state_names, interface):
diff --git a/lib/lp/services/webapp/authentication.py b/lib/lp/services/webapp/authentication.py
index c48bb33..acab459 100644
--- a/lib/lp/services/webapp/authentication.py
+++ b/lib/lp/services/webapp/authentication.py
@@ -257,7 +257,7 @@ class LaunchpadPrincipal:
def __init__(self, id, title, description, account,
access_level=AccessLevel.WRITE_PRIVATE, scope_url=None):
- self.id = unicode(id)
+ self.id = six.text_type(id)
self.title = title
self.description = description
self.access_level = access_level
diff --git a/lib/lp/services/webapp/error.py b/lib/lp/services/webapp/error.py
index a883698..6b15896 100644
--- a/lib/lp/services/webapp/error.py
+++ b/lib/lp/services/webapp/error.py
@@ -17,6 +17,7 @@ __all__ = [
import sys
import traceback
+import six
from six.moves import http_client
from zope.browser.interfaces import ISystemErrorView
from zope.browserpage import ViewPageTemplateFile
@@ -205,7 +206,7 @@ class NotFoundView(SystemErrorView):
# to show a link back to the referring site, so we can't use
# replace or ignore. Best to just pretent it doesn't exist.
try:
- return unicode(referrer)
+ return six.ensure_text(referrer)
except UnicodeDecodeError:
return None
else:
diff --git a/lib/lp/services/webapp/errorlog.py b/lib/lp/services/webapp/errorlog.py
index df20c1d..4c3ed01 100644
--- a/lib/lp/services/webapp/errorlog.py
+++ b/lib/lp/services/webapp/errorlog.py
@@ -193,10 +193,10 @@ def attach_http_request(report, context):
if principal is not None and principal is not missing:
username = ', '.join([
- unicode(login),
- unicode(request.principal.id),
- unicode(request.principal.title),
- unicode(request.principal.description)])
+ six.ensure_text(login),
+ six.text_type(request.principal.id),
+ six.ensure_text(request.principal.title),
+ six.ensure_text(request.principal.description)])
report['username'] = username
if getattr(request, '_orig_env', None):
@@ -217,7 +217,7 @@ def attach_http_request(report, context):
args = request.getPositionalArguments()
# Request variables are strings: this could move to its own key and be
# raw.
- report['req_vars']['xmlrpc args'] = unicode(args)
+ report['req_vars']['xmlrpc args'] = six.text_type(args)
def attach_ignore_from_exception(report, context):
diff --git a/lib/lp/services/webapp/escaping.py b/lib/lp/services/webapp/escaping.py
index bce499e..e271477 100644
--- a/lib/lp/services/webapp/escaping.py
+++ b/lib/lp/services/webapp/escaping.py
@@ -48,7 +48,11 @@ def html_escape(message):
# It is possible that the message is wrapped in an
# internationalized object, so we need to translate it
# first. See bug #54987.
- raw = unicode(translate_if_i18n(message))
+ raw = translate_if_i18n(message)
+ if isinstance(raw, bytes):
+ raw = six.ensure_text(raw)
+ else:
+ raw = six.text_type(raw)
for needle, replacement in HTML_REPLACEMENTS:
raw = raw.replace(needle, replacement)
return raw
@@ -86,7 +90,11 @@ def translate_if_i18n(obj_or_msgid):
class structured:
def __init__(self, text, *reps, **kwreps):
- text = unicode(translate_if_i18n(text))
+ text = translate_if_i18n(text)
+ if isinstance(text, bytes):
+ text = six.ensure_text(text)
+ else:
+ text = six.text_type(text)
self.text = text
if reps and kwreps:
raise TypeError(
diff --git a/lib/lp/services/webapp/publication.py b/lib/lp/services/webapp/publication.py
index a289f44..eb98ae1 100644
--- a/lib/lp/services/webapp/publication.py
+++ b/lib/lp/services/webapp/publication.py
@@ -19,6 +19,7 @@ from lazr.uri import (
URI,
)
from psycopg2.extensions import TransactionRollbackError
+import six
from six.moves.urllib.parse import quote
from storm.database import STATE_DISCONNECTED
from storm.exceptions import (
@@ -248,7 +249,7 @@ class LaunchpadBrowserPublication(
threadrequestfile = open_for_writing(
'logs/thread-%s.request' % threadid, 'w')
try:
- request_txt = unicode(request).encode('UTF-8')
+ request_txt = six.text_type(request).encode('UTF-8')
except Exception:
request_txt = 'Exception converting request to string\n\n'
try:
diff --git a/lib/lp/services/webapp/publisher.py b/lib/lp/services/webapp/publisher.py
index 1c19bd8..199be4b 100644
--- a/lib/lp/services/webapp/publisher.py
+++ b/lib/lp/services/webapp/publisher.py
@@ -775,8 +775,8 @@ def canonical_url(
request is not None and
root_url.startswith(request.getApplicationURL()))
or force_local_path):
- return unicode('/' + path)
- return unicode(root_url + path)
+ return u'/' + path
+ return six.ensure_text(root_url + path)
def canonical_name(name):
diff --git a/lib/lp/services/webapp/servers.py b/lib/lp/services/webapp/servers.py
index e23198e..f9a01b4 100644
--- a/lib/lp/services/webapp/servers.py
+++ b/lib/lp/services/webapp/servers.py
@@ -861,7 +861,8 @@ class LaunchpadBrowserResponse(NotificationResponse, BrowserResponse):
else:
status = 303
super(LaunchpadBrowserResponse, self).redirect(
- unicode(location).encode('UTF-8'), status=status, trusted=trusted)
+ six.ensure_str(six.text_type(location)),
+ status=status, trusted=trusted)
def adaptResponseToSession(response):
diff --git a/lib/lp/services/webapp/sorting.py b/lib/lp/services/webapp/sorting.py
index efc4ee2..72d04dc 100644
--- a/lib/lp/services/webapp/sorting.py
+++ b/lib/lp/services/webapp/sorting.py
@@ -84,7 +84,7 @@ def sorted_version_numbers(sequence, key=_identity):
>>> class series:
... def __init__(self, name):
- ... self.name = unicode(name)
+ ... self.name = six.ensure_text(name)
>>> bzr_versions = [series('0.9'), series('0.10'), series('0.11'),
... series('bzr-0.9'), series('bzr-0.10'),
... series('bzr-0.11'), series('foo')]
@@ -130,7 +130,7 @@ def sorted_dotted_numbers(sequence, key=_identity):
>>> class series:
... def __init__(self, name):
- ... self.name = unicode(name)
+ ... self.name = six.ensure_text(name)
>>> bzr_versions = [series('0.9'), series('0.10'), series('0.11'),
... series('bzr-0.9'), series('bzr-0.10'),
... series('bzr-0.11'), series('foo')]
diff --git a/lib/lp/services/webapp/tests/test_errorlog.py b/lib/lp/services/webapp/tests/test_errorlog.py
index 20e4b52..3a04f8c 100644
--- a/lib/lp/services/webapp/tests/test_errorlog.py
+++ b/lib/lp/services/webapp/tests/test_errorlog.py
@@ -14,6 +14,7 @@ from lazr.batchnavigator.interfaces import InvalidBatchSizeError
from lazr.restful.declarations import error_status
import oops_amqp
import pytz
+import six
from six.moves import http_client
import testtools
from timeline.timeline import Timeline
@@ -655,5 +656,5 @@ class TestHooks(testtools.TestCase):
'http_request': {'SIMPLE': 'string', 'COMPLEX': complexthing}}
attach_http_request(report, context)
self.assertEqual(
- {'SIMPLE': 'string', 'COMPLEX': unicode(complexthing)},
+ {'SIMPLE': 'string', 'COMPLEX': six.text_type(complexthing)},
report['req_vars'])
diff --git a/lib/lp/services/webapp/tests/test_launchpad_login_source.txt b/lib/lp/services/webapp/tests/test_launchpad_login_source.txt
index ff9e54d..27a520c 100644
--- a/lib/lp/services/webapp/tests/test_launchpad_login_source.txt
+++ b/lib/lp/services/webapp/tests/test_launchpad_login_source.txt
@@ -4,6 +4,7 @@ LaunchpadLoginSource is used to create principals, from login
information, passing the email address to getPrincipalByLogin. If no
person is found with the given email address, None is returned
+ >>> import six
>>> from lp.services.webapp.authentication import (
... LaunchpadLoginSource)
>>> login_source = LaunchpadLoginSource()
@@ -21,7 +22,7 @@ account id.
>>> from lp.services.webapp.interfaces import ILaunchpadPrincipal
>>> ILaunchpadPrincipal.providedBy(principal)
True
- >>> principal.id == unicode(account.id)
+ >>> principal.id == six.text_type(account.id)
True
The corresponding Account and Person records are also in the
