launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24913
[Merge] ~cjwatson/launchpad:py3-librarianserver into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:py3-librarianserver into launchpad:master with ~cjwatson/launchpad:py3-fakeswift as a prerequisite.
Commit message:
Make the librarian server work on Python 3
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/386485
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:py3-librarianserver into launchpad:master.
diff --git a/lib/lp/services/librarian/client.py b/lib/lp/services/librarian/client.py
index 9550d56..9ef82f5 100644
--- a/lib/lp/services/librarian/client.py
+++ b/lib/lp/services/librarian/client.py
@@ -241,7 +241,6 @@ class FileUploadClient:
raise TypeError('No data')
if size <= 0:
raise UploadFailed('No data')
- name = six.ensure_binary(name)
self._connect()
try:
database_name = ConnectionString(dbconfig.main_master).dbname
diff --git a/lib/lp/services/librarian/tests/test_smoketest.py b/lib/lp/services/librarian/tests/test_smoketest.py
index 9cd0e91..9e6e046 100644
--- a/lib/lp/services/librarian/tests/test_smoketest.py
+++ b/lib/lp/services/librarian/tests/test_smoketest.py
@@ -3,6 +3,8 @@
"""Test the script that does a smoke-test of the librarian."""
+from __future__ import absolute_import, print_function, unicode_literals
+
__metaclass__ = type
from cStringIO import StringIO
diff --git a/lib/lp/services/librarianserver/librariangc.py b/lib/lp/services/librarianserver/librariangc.py
index ac2b630..68b9aba 100644
--- a/lib/lp/services/librarianserver/librariangc.py
+++ b/lib/lp/services/librarianserver/librariangc.py
@@ -14,13 +14,13 @@ import hashlib
import multiprocessing.pool
import os
import re
-import six
import sys
from time import time
import iso8601
import pytz
import scandir
+import six
from swiftclient import client as swiftclient
from zope.interface import implementer
@@ -226,8 +226,6 @@ def merge_duplicates(con):
for sha1, filesize in rows:
cur = con.cursor()
- sha1 = sha1.encode('US-ASCII') # Can't pass Unicode to execute (yet)
-
# Get a list of our dupes. Where multiple files exist, we return
# the most recently added one first, because this is the version
# most likely to exist on the staging server (it should be
@@ -757,7 +755,8 @@ def swift_files(max_lfc_id):
swift.quiet_swiftclient(
swift_connection.get_container,
container, full_listing=True)[1],
- key=lambda x: map(int, x['name'].split('/')))
+ key=lambda x: [
+ int(segment) for segment in x['name'].split('/')])
for name in names:
yield (container, name)
except swiftclient.ClientException as x:
diff --git a/lib/lp/services/librarianserver/libraryprotocol.py b/lib/lp/services/librarianserver/libraryprotocol.py
index 5eb7879..385d15e 100644
--- a/lib/lp/services/librarianserver/libraryprotocol.py
+++ b/lib/lp/services/librarianserver/libraryprotocol.py
@@ -62,7 +62,7 @@ class FileUploadProtocol(basic.LineReceiver):
were just established to start a new upload.
"""
- delimiter = '\r\n' # same as HTTP
+ delimiter = b'\r\n' # same as HTTP
state = 'command'
def lineReceived(self, line):
@@ -76,7 +76,7 @@ class FileUploadProtocol(basic.LineReceiver):
def sendError(self, msg, code='400'):
"""Sends a correctly formatted error to the client, and closes the
connection."""
- self.sendLine(code + ' ' + msg)
+ self.sendLine((code + ' ' + msg).encode('UTF-8'))
self.transport.loseConnection()
def unknownError(self, failure=None):
@@ -92,7 +92,7 @@ class FileUploadProtocol(basic.LineReceiver):
failure.trap(WrongDatabaseError)
exc = failure.value
raise ProtocolViolation(
- "Wrong database %r, should be %r"
+ "Wrong database '%s', should be '%s'"
% (exc.clientDatabaseName, exc.serverDatabaseName))
def protocolErrors(self, failure):
@@ -105,6 +105,7 @@ class FileUploadProtocol(basic.LineReceiver):
def line_command(self, line):
try:
command, args = line.split(None, 1)
+ command = command.decode('UTF-8')
except ValueError:
raise ProtocolViolation('Bad command: ' + line)
@@ -113,7 +114,7 @@ class FileUploadProtocol(basic.LineReceiver):
def line_header(self, line):
# Blank line signals the end of the headers
- if line == '':
+ if line == b'':
# If File-Content-ID was specified, File-Alias-ID must be too, and
# vice-versa.
contentID = self.newFile.contentID
@@ -134,13 +135,13 @@ class FileUploadProtocol(basic.LineReceiver):
# Make sure rawDataReceived is *always* called, so that zero-byte
# uploads don't hang. It's harmless the rest of the time.
- self.rawDataReceived('')
+ self.rawDataReceived(b'')
return
# Simple RFC 822-ish header parsing
try:
- name, value = line.split(':', 2)
+ name, value = line.decode('UTF-8').split(':', 2)
except ValueError:
raise ProtocolViolation('Invalid header: ' + line)
@@ -216,9 +217,10 @@ class FileUploadProtocol(basic.LineReceiver):
fileID, aliasID = ids
if self.newFile.contentID is None:
# Respond with deprecated server-generated IDs.
- self.sendLine('200 %s/%s' % (fileID, aliasID))
+ self.sendLine(
+ ('200 %s/%s' % (fileID, aliasID)).encode('UTF-8'))
else:
- self.sendLine('200')
+ self.sendLine(b'200')
deferred.addBoth(self.logDebugging)
deferred.addCallback(_sendID)
deferred.addErrback(self.translateErrors)
diff --git a/lib/lp/services/librarianserver/storage.py b/lib/lp/services/librarianserver/storage.py
index 5f86023..253bc41 100644
--- a/lib/lp/services/librarianserver/storage.py
+++ b/lib/lp/services/librarianserver/storage.py
@@ -162,10 +162,10 @@ class TxSwiftStream(swift.SwiftStream):
raise ValueError('I/O operation on closed file')
if self._swift_connection is None:
- defer.returnValue('') # EOF already reached, connection returned.
+ defer.returnValue(b'') # EOF already reached, connection returned.
if size == 0:
- defer.returnValue('')
+ defer.returnValue(b'')
if not self._chunk:
self._chunk = yield deferToThread(self._next_chunk)
@@ -177,7 +177,7 @@ class TxSwiftStream(swift.SwiftStream):
swift.connection_pool.put(self._swift_connection)
self._swift_connection = None
self._chunks = None
- defer.returnValue('')
+ defer.returnValue(b'')
return_chunk = self._chunk[:size]
self._chunk = self._chunk[size:]
@@ -203,7 +203,7 @@ class LibraryFileUpload(object):
# Create temporary file
tmpfile, tmpfilepath = tempfile.mkstemp(dir=self.storage.incoming)
- self.tmpfile = os.fdopen(tmpfile, 'w')
+ self.tmpfile = os.fdopen(tmpfile, 'wb')
self.tmpfilepath = tmpfilepath
self.md5_digester = hashlib.md5()
self.sha1_digester = hashlib.sha1()
diff --git a/lib/lp/services/librarianserver/swift.py b/lib/lp/services/librarianserver/swift.py
index 2c95092..4407ffd 100644
--- a/lib/lp/services/librarianserver/swift.py
+++ b/lib/lp/services/librarianserver/swift.py
@@ -237,7 +237,7 @@ def _put(log, swift_connection, lfc_id, container, obj_name, fs_path):
manifest = '{0}/{1}/'.format(quote(container), quote(obj_name))
manifest_headers = {'X-Object-Manifest': manifest}
swift_connection.put_object(
- container, obj_name, '', 0, headers=manifest_headers)
+ container, obj_name, b'', 0, headers=manifest_headers)
def swift_location(lfc_id):
@@ -281,10 +281,10 @@ class SwiftStream:
raise ValueError('I/O operation on closed file')
if self._swift_connection is None:
- return ''
+ return b''
if size == 0:
- return ''
+ return b''
return_chunks = []
return_size = 0
@@ -306,7 +306,7 @@ class SwiftStream:
return_size += len(return_chunks[-1])
self._offset += return_size
- return ''.join(return_chunks)
+ return b''.join(return_chunks)
def _next_chunk(self):
try:
@@ -341,7 +341,7 @@ class HashStream:
def read(self, size=-1):
if self._remaining is not None:
if self._remaining <= 0:
- return ''
+ return b''
size = min(size, self._remaining)
chunk = self._stream.read(size)
if self._remaining is not None:
diff --git a/lib/lp/services/librarianserver/testing/fake.py b/lib/lp/services/librarianserver/testing/fake.py
index e1e1f5e..c482abe 100644
--- a/lib/lp/services/librarianserver/testing/fake.py
+++ b/lib/lp/services/librarianserver/testing/fake.py
@@ -16,7 +16,7 @@ __all__ = [
]
import hashlib
-from StringIO import StringIO
+import io
from fixtures import Fixture
import six
@@ -54,7 +54,7 @@ class InstrumentedLibraryFileAlias(LibraryFileAlias):
def open(self, timeout=LIBRARIAN_SERVER_DEFAULT_TIMEOUT):
self.checkCommitted()
- self._datafile = StringIO(self.content_string)
+ self._datafile = io.BytesIO(self.content_bytes)
def read(self, chunksize=None, timeout=LIBRARIAN_SERVER_DEFAULT_TIMEOUT):
return self._datafile.read(chunksize)
@@ -127,7 +127,7 @@ class FakeLibrarian(Fixture):
"""See `IFileDownloadClient`."""
alias = self[aliasID]
alias.checkCommitted()
- return StringIO(alias.content_string)
+ return io.BytesIO(alias.content_bytes)
def pretendCommit(self):
"""Pretend that there's been a commit.
@@ -146,7 +146,7 @@ class FakeLibrarian(Fixture):
"""Create a `LibraryFileAlias`."""
alias = InstrumentedLibraryFileAlias(
contentID=file_id, filename=name, mimetype=content_type)
- alias.content_string = content
+ alias.content_bytes = content
return alias
def _makeLibraryFileContent(self, content):
diff --git a/lib/lp/services/librarianserver/testing/server.py b/lib/lp/services/librarianserver/testing/server.py
index 0e4ae88..ee11175 100644
--- a/lib/lp/services/librarianserver/testing/server.py
+++ b/lib/lp/services/librarianserver/testing/server.py
@@ -227,7 +227,8 @@ class LibrarianServerFixture(TacTestSetup):
def getLogChunks(self):
"""Get a list with the contents of the librarian log in it."""
- return open(self.logfile, 'rb').readlines()
+ with open(self.logfile, 'rb') as f:
+ return f.readlines()
def reset(self):
"""Reset the librarian to a consistent initial state."""
@@ -238,7 +239,7 @@ class LibrarianServerFixture(TacTestSetup):
def fillLibrarianFile(fileid, content=None):
"""Write contents in disk for a librarian sampledata."""
if content is None:
- content = 'x' * LibraryFileContent.get(fileid).filesize
+ content = b'x' * LibraryFileContent.get(fileid).filesize
filepath = os.path.join(
config.librarian_server.root, _relFileLocation(fileid))
diff --git a/lib/lp/services/librarianserver/testing/tests/test_fakelibrarian.py b/lib/lp/services/librarianserver/testing/tests/test_fakelibrarian.py
index c5dd2a3..ac849c5 100644
--- a/lib/lp/services/librarianserver/testing/tests/test_fakelibrarian.py
+++ b/lib/lp/services/librarianserver/testing/tests/test_fakelibrarian.py
@@ -5,7 +5,7 @@
__metaclass__ = type
-from StringIO import StringIO
+import io
import six
import transaction
@@ -45,10 +45,10 @@ class LibraryAccessScenarioMixin:
:return: Tuple of filename, file contents, alias id.
"""
name = self.factory.getUniqueString() + '.txt'
- text = self.factory.getUniqueString()
+ content = self.factory.getUniqueBytes()
alias = getUtility(ILibraryFileAliasSet).create(
- name, len(text), StringIO(text), 'text/plain')
- return name, text, alias
+ name, len(content), io.BytesIO(content), 'text/plain')
+ return name, content, alias
def test_baseline(self):
self.assertTrue(
@@ -59,47 +59,47 @@ class LibraryAccessScenarioMixin:
ILibraryFileAliasSet, getUtility(ILibraryFileAliasSet)))
def test_insert_retrieve(self):
- name, text, alias = self._storeFile()
+ name, content, alias = self._storeFile()
self.assertIsInstance(alias.id, six.integer_types)
transaction.commit()
library_file = getUtility(ILibrarianClient).getFileByAlias(alias.id)
- self.assertEqual(text, library_file.read())
+ self.assertEqual(content, library_file.read())
def test_alias_set(self):
- name, text, alias = self._storeFile()
+ name, content, alias = self._storeFile()
retrieved_alias = getUtility(ILibraryFileAliasSet)[alias.id]
self.assertEqual(alias, retrieved_alias)
def test_read(self):
- name, text, alias = self._storeFile()
+ name, content, alias = self._storeFile()
transaction.commit()
alias.open()
- self.assertEqual(text, alias.read())
+ self.assertEqual(content, alias.read())
def test_uncommitted_file(self):
- name, text, alias = self._storeFile()
+ name, content, alias = self._storeFile()
retrieved_alias = getUtility(ILibraryFileAliasSet)[alias.id]
self.assertRaises(LookupError, retrieved_alias.open)
def test_incorrect_upload_size(self):
name = self.factory.getUniqueString()
- text = self.factory.getUniqueString()
- wrong_length = len(text) + 1
+ content = self.factory.getUniqueBytes()
+ wrong_length = len(content) + 1
self.assertRaises(
AssertionError,
getUtility(ILibrarianClient).addFile,
- name, wrong_length, StringIO(text), 'text/plain')
+ name, wrong_length, io.BytesIO(content), 'text/plain')
def test_create_returns_alias(self):
alias = getUtility(ILibraryFileAliasSet).create(
- 'foo.txt', 3, StringIO('foo'), 'text/plain')
+ 'foo.txt', 3, io.BytesIO(b'foo'), 'text/plain')
self.assertIsInstance(alias, LibraryFileAlias)
def test_addFile_returns_alias_id(self):
alias_id = getUtility(ILibrarianClient).addFile(
- 'bar.txt', 3, StringIO('bar'), 'text/plain')
+ 'bar.txt', 3, io.BytesIO(b'bar'), 'text/plain')
self.assertIsInstance(alias_id, six.integer_types)
self.assertIsInstance(
getUtility(ILibraryFileAliasSet)[alias_id],
@@ -109,29 +109,29 @@ class LibraryAccessScenarioMixin:
# addFile takes an argument debugID that doesn't do anything
# observable. We get a LibraryFileAlias regardless.
alias = getUtility(ILibraryFileAliasSet).create(
- 'txt.txt', 3, StringIO('txt'), 'text/plain', debugID='txt')
+ 'txt.txt', 3, io.BytesIO(b'txt'), 'text/plain', debugID='txt')
self.assertNotEqual(None, alias)
def test_getURLForAlias(self):
- name, text, alias = self._storeFile()
+ name, content, alias = self._storeFile()
librarian = getUtility(ILibrarianClient)
self.assertIn(
librarian.getURLForAlias(alias.id),
(alias.http_url, alias.https_url))
def test_getURLForAliasObject(self):
- name, text, alias = self._storeFile()
+ name, content, alias = self._storeFile()
librarian = getUtility(ILibrarianClient)
self.assertEqual(
librarian.getURLForAlias(alias.id),
librarian.getURLForAliasObject(alias))
def test_getURL(self):
- name, text, alias = self._storeFile()
+ name, content, alias = self._storeFile()
self.assertIn(alias.getURL(), (alias.http_url, alias.https_url))
def test_deleted_alias_has_no_url(self):
- name, text, alias = self._storeFile()
+ name, content, alias = self._storeFile()
self.assertNotEqual(None, alias.getURL())
removeSecurityProxy(alias).content = None
@@ -152,13 +152,13 @@ class TestFakeLibrarian(LibraryAccessScenarioMixin, TestCaseWithFactory):
self.assertIsInstance(self.fake_librarian, FakeLibrarian)
def test_pretend_commit(self):
- name, text, alias = self._storeFile()
+ name, content, alias = self._storeFile()
self.fake_librarian.pretendCommit()
retrieved_alias = getUtility(ILibraryFileAliasSet)[alias.id]
retrieved_alias.open()
- self.assertEqual(text, retrieved_alias.read())
+ self.assertEqual(content, retrieved_alias.read())
class TestRealLibrarian(LibraryAccessScenarioMixin, TestCaseWithFactory):
diff --git a/lib/lp/services/librarianserver/testing/tests/test_server_fixture.py b/lib/lp/services/librarianserver/testing/tests/test_server_fixture.py
index 2d9315b..421e4a4 100644
--- a/lib/lp/services/librarianserver/testing/tests/test_server_fixture.py
+++ b/lib/lp/services/librarianserver/testing/tests/test_server_fixture.py
@@ -96,7 +96,7 @@ class TestLibrarianServerFixture(TestCase):
self.assertIsInstance(chunks, list)
found_started = False
for chunk in chunks:
- if 'daemon ready' in chunk:
+ if b'daemon ready' in chunk:
found_started = True
self.assertTrue(found_started)
@@ -113,9 +113,9 @@ class TestLibrarianServerFixture(TestCase):
config.librarian.restricted_download_host,
fixture.restricted_download_port)
# Both download ports work:
- self.assertIn('Copyright', urlopen(librarian_url).read())
+ self.assertIn(b'Copyright', urlopen(librarian_url).read())
self.assertIn(
- 'Copyright', urlopen(restricted_librarian_url).read())
+ b'Copyright', urlopen(restricted_librarian_url).read())
os.path.isdir(fixture.root)
# Ports are closed on cleanUp.
self.assertRaises(IOError, urlopen, librarian_url)
@@ -126,4 +126,4 @@ class TestLibrarianServerFixture(TestCase):
librarian_url = "http://%s:%d"; % (
config.librarian.download_host,
fixture.download_port)
- self.assertIn('Copyright', urlopen(librarian_url).read())
+ self.assertIn(b'Copyright', urlopen(librarian_url).read())
diff --git a/lib/lp/services/librarianserver/tests/test_db_outage.py b/lib/lp/services/librarianserver/tests/test_db_outage.py
index 9f28dfe..d6fa401 100644
--- a/lib/lp/services/librarianserver/tests/test_db_outage.py
+++ b/lib/lp/services/librarianserver/tests/test_db_outage.py
@@ -7,7 +7,7 @@ Database outages happen by accident and during fastdowntime deployments."""
__metaclass__ = type
-from cStringIO import StringIO
+import io
from fixtures import Fixture
from six.moves.urllib.error import HTTPError
@@ -74,9 +74,9 @@ class TestLibrarianDBOutage(TestCase):
self.url = self._makeLibraryFileUrl()
def _makeLibraryFileUrl(self):
- data = 'whatever'
+ data = b'whatever'
return self.client.remoteAddFile(
- 'foo.txt', len(data), StringIO(data), 'text/plain')
+ 'foo.txt', len(data), io.BytesIO(data), 'text/plain')
def getErrorCode(self):
# We need to talk to every Librarian thread to ensure all the
diff --git a/lib/lp/services/librarianserver/tests/test_gc.py b/lib/lp/services/librarianserver/tests/test_gc.py
index 8e1a74f..479c51c 100644
--- a/lib/lp/services/librarianserver/tests/test_gc.py
+++ b/lib/lp/services/librarianserver/tests/test_gc.py
@@ -7,12 +7,12 @@ __metaclass__ = type
import calendar
from contextlib import contextmanager
-from cStringIO import StringIO
from datetime import (
datetime,
timedelta,
)
import hashlib
+import io
import os
import shutil
from subprocess import (
@@ -95,13 +95,15 @@ class TestLibrarianGarbageCollectionBase:
if not os.path.exists(path):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
- content_text = '{0} content'.format(content.id)
- open(path, 'w').write(content_text)
+ content_bytes = '{0} content'.format(content.id).encode(
+ 'UTF-8')
+ with open(path, 'wb') as f:
+ f.write(content_bytes)
os.utime(path, (0, 0)) # Ancient past, never considered new.
- content.md5 = hashlib.md5(content_text).hexdigest()
- content.sha1 = hashlib.sha1(content_text).hexdigest()
- content.sha256 = hashlib.sha256(content_text).hexdigest()
- content.filesize = len(content_text)
+ content.md5 = hashlib.md5(content_bytes).hexdigest()
+ content.sha1 = hashlib.sha1(content_bytes).hexdigest()
+ content.sha256 = hashlib.sha256(content_bytes).hexdigest()
+ content.filesize = len(content_bytes)
transaction.commit()
self.con = connect(
@@ -127,13 +129,13 @@ class TestLibrarianGarbageCollectionBase:
ztm.begin()
# Add some duplicate files
- content = 'This is some content'
+ content = b'This is some content'
f1_id = self.client.addFile(
- 'foo.txt', len(content), StringIO(content), 'text/plain',
+ 'foo.txt', len(content), io.BytesIO(content), 'text/plain',
)
f1 = LibraryFileAlias.get(f1_id)
f2_id = self.client.addFile(
- 'foo.txt', len(content), StringIO(content), 'text/plain',
+ 'foo.txt', len(content), io.BytesIO(content), 'text/plain',
)
f2 = LibraryFileAlias.get(f2_id)
@@ -516,9 +518,9 @@ class TestLibrarianGarbageCollectionBase:
# There was a bug where delete_unwanted_files() would die
# if the last file found on disk was unwanted.
switch_dbuser('testadmin')
- content = 'foo'
+ content = b'foo'
self.client.addFile(
- 'foo.txt', len(content), StringIO(content), 'text/plain')
+ 'foo.txt', len(content), io.BytesIO(content), 'text/plain')
# Roll back the database changes, leaving the file on disk.
transaction.abort()
@@ -532,9 +534,9 @@ class TestLibrarianGarbageCollectionBase:
# to cope.
# First, let's make sure we have some trash.
switch_dbuser('testadmin')
- content = 'foo'
+ content = b'foo'
self.client.addFile(
- 'foo.txt', len(content), StringIO(content), 'text/plain')
+ 'foo.txt', len(content), io.BytesIO(content), 'text/plain')
# Roll back the database changes, leaving the file on disk.
transaction.abort()
@@ -611,15 +613,15 @@ class TestDiskLibrarianGarbageCollection(
# appended to their names. These are treated just like the
# original file, ignoring the extension.
switch_dbuser('testadmin')
- content = 'foo'
+ content = b'foo'
lfa = LibraryFileAlias.get(self.client.addFile(
- 'foo.txt', len(content), StringIO(content), 'text/plain'))
+ 'foo.txt', len(content), io.BytesIO(content), 'text/plain'))
id_aborted = lfa.contentID
# Roll back the database changes, leaving the file on disk.
transaction.abort()
lfa = LibraryFileAlias.get(self.client.addFile(
- 'bar.txt', len(content), StringIO(content), 'text/plain'))
+ 'bar.txt', len(content), io.BytesIO(content), 'text/plain'))
transaction.commit()
id_committed = lfa.contentID
@@ -746,13 +748,13 @@ class TestSwiftLibrarianGarbageCollection(
# Large files are handled by Swift as multiple segments joined
# by a manifest. GC treats the segments like the original file.
switch_dbuser('testadmin')
- content = 'uploading to swift bigly'
+ content = b'uploading to swift bigly'
big1_lfa = LibraryFileAlias.get(self.client.addFile(
- 'foo.txt', len(content), StringIO(content), 'text/plain'))
+ 'foo.txt', len(content), io.BytesIO(content), 'text/plain'))
big1_id = big1_lfa.contentID
big2_lfa = LibraryFileAlias.get(self.client.addFile(
- 'bar.txt', len(content), StringIO(content), 'text/plain'))
+ 'bar.txt', len(content), io.BytesIO(content), 'text/plain'))
big2_id = big2_lfa.contentID
transaction.commit()
@@ -917,8 +919,9 @@ class TestBlobCollection(TestCase):
if not os.path.exists(path):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
- data = sha1
- open(path, 'w').write(data)
+ data = sha1.encode('UTF-8')
+ with open(path, 'wb') as f:
+ f.write(data)
cur.execute(
"UPDATE LibraryFileContent "
"SET md5 = %s, sha1 = %s, sha256 = %s, filesize = %s "
diff --git a/lib/lp/services/librarianserver/tests/test_storage.py b/lib/lp/services/librarianserver/tests/test_storage.py
index 88844cf..c71a056 100644
--- a/lib/lp/services/librarianserver/tests/test_storage.py
+++ b/lib/lp/services/librarianserver/tests/test_storage.py
@@ -74,7 +74,7 @@ class LibrarianStorageTestCase(unittest.TestCase):
# correctly -- i.e that creating a file works both if the directory
# already exists, and if the directory doesn't already exist.
self.storage.library = StubLibrary()
- data = 'data ' * 50
+ data = b'data ' * 50
newfile = self.storage.startAddFile('file', len(data))
newfile.contentID = 0x11111111
newfile.append(data)
@@ -82,7 +82,7 @@ class LibrarianStorageTestCase(unittest.TestCase):
# First id from stub library should be 0x11111111
self.assertEqual(0x11111111, fileid1)
- data += 'more data'
+ data += b'more data'
newfile = self.storage.startAddFile('file', len(data))
newfile.contentID = 0x11111112
newfile.append(data)
@@ -96,7 +96,7 @@ class LibrarianStorageTestCase(unittest.TestCase):
def test_hashes(self):
# Check that the MD5, SHA1 and SHA256 hashes are correct.
- data = 'i am some data'
+ data = b'i am some data'
md5 = hashlib.md5(data).hexdigest()
sha1 = hashlib.sha1(data).hexdigest()
sha256 = hashlib.sha256(data).hexdigest()
diff --git a/lib/lp/services/librarianserver/tests/test_storage_db.py b/lib/lp/services/librarianserver/tests/test_storage_db.py
index 2dd1e2f..47fe847 100644
--- a/lib/lp/services/librarianserver/tests/test_storage_db.py
+++ b/lib/lp/services/librarianserver/tests/test_storage_db.py
@@ -1,6 +1,8 @@
# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
+from __future__ import absolute_import, print_function, unicode_literals
+
import hashlib
import os.path
import time
@@ -41,7 +43,7 @@ class LibrarianStorageDBTests(TestCase):
self.storage = LibrarianStorage(self.directory, db.Library())
def test_addFile(self):
- data = 'data ' * 50
+ data = b'data ' * 50
digest = hashlib.sha1(data).hexdigest()
newfile = self.storage.startAddFile('file1', len(data))
newfile.srcDigest = digest
@@ -51,7 +53,7 @@ class LibrarianStorageDBTests(TestCase):
def test_addFiles_identical(self):
# Start adding two files with identical data
- data = 'data ' * 5000
+ data = b'data ' * 5000
newfile1 = self.storage.startAddFile('file1', len(data))
newfile2 = self.storage.startAddFile('file2', len(data))
newfile1.append(data)
@@ -68,7 +70,7 @@ class LibrarianStorageDBTests(TestCase):
self.assertNotEqual(id1, id2)
def test_badDigest(self):
- data = 'data ' * 50
+ data = b'data ' * 50
digest = 'crud'
newfile = self.storage.startAddFile('file', len(data))
newfile.srcDigest = digest
@@ -77,7 +79,7 @@ class LibrarianStorageDBTests(TestCase):
def test_alias(self):
# Add a file (and so also add an alias)
- data = 'data ' * 50
+ data = b'data ' * 50
newfile = self.storage.startAddFile('file1', len(data))
newfile.mimetype = 'text/unknown'
newfile.append(data)
@@ -117,7 +119,7 @@ class LibrarianStorageDBTests(TestCase):
# Check the new behaviour specified by LibrarianTransactions
# spec: allow duplicate content with distinct IDs.
- content = 'some content'
+ content = b'some content'
# Store a file with id 6661
newfile1 = LibraryFileUpload(self.storage, 'filename', 0)
diff --git a/lib/lp/services/librarianserver/tests/test_swift.py b/lib/lp/services/librarianserver/tests/test_swift.py
index f9ade6d..6fe7ffd 100644
--- a/lib/lp/services/librarianserver/tests/test_swift.py
+++ b/lib/lp/services/librarianserver/tests/test_swift.py
@@ -5,12 +5,13 @@
__metaclass__ = type
-from cStringIO import StringIO
import hashlib
+import io
import os.path
import time
from mock import patch
+import six
from swiftclient import client as swiftclient
import transaction
@@ -52,7 +53,7 @@ class TestFeedSwift(TestCase):
# considered potential in-progress uploads.
the_past = time.time() - 25 * 60 * 60
self.librarian_client = LibrarianClient()
- self.contents = [str(i) * i for i in range(1, 5)]
+ self.contents = [str(i).encode('ASCII') * i for i in range(1, 5)]
self.lfa_ids = [
self.add_file('file_{0}'.format(i), content, when=the_past)
for i, content in enumerate(self.contents)]
@@ -71,7 +72,7 @@ class TestFeedSwift(TestCase):
@write_transaction
def add_file(self, name, content, when=None, content_type='text/plain'):
lfa_id = self.librarian_client.addFile(
- name=name, size=len(content), file=StringIO(content),
+ name=name, size=len(content), file=io.BytesIO(content),
contentType=content_type)
if when is None:
when = 0 # Very very old
@@ -190,7 +191,8 @@ class TestFeedSwift(TestCase):
# to be done in multiple chunks, but small enough that it is
# stored in Swift as a single object.
size = 512 * 1024 # 512KB
- expected_content = ''.join(chr(i % 256) for i in range(0, size))
+ expected_content = b''.join(
+ six.int2byte(i % 256) for i in range(0, size))
lfa_id = self.add_file('hello_bigboy.xls', expected_content)
# Data round trips when served from disk.
@@ -205,7 +207,8 @@ class TestFeedSwift(TestCase):
# stored in Swift as a single object.
size = LibrarianStorage.CHUNK_SIZE * 50
self.assertTrue(size > 1024 * 1024)
- expected_content = ''.join(chr(i % 256) for i in range(0, size))
+ expected_content = b''.join(
+ six.int2byte(i % 256) for i in range(0, size))
lfa_id = self.add_file('hello_bigboy.xls', expected_content)
lfc = IStore(LibraryFileAlias).get(LibraryFileAlias, lfa_id).content
@@ -227,7 +230,8 @@ class TestFeedSwift(TestCase):
# stored in Swift as a single object.
size = LibrarianStorage.CHUNK_SIZE * 50 + 1
self.assertTrue(size > 1024 * 1024)
- expected_content = ''.join(chr(i % 256) for i in range(0, size))
+ expected_content = b''.join(
+ six.int2byte(i % 256) for i in range(0, size))
lfa_id = self.add_file('hello_bigboy.xls', expected_content)
lfc = IStore(LibraryFileAlias).get(LibraryFileAlias, lfa_id).content
@@ -246,7 +250,8 @@ class TestFeedSwift(TestCase):
# it as multiple objects plus a manifest.
size = LibrarianStorage.CHUNK_SIZE * 50
self.assertTrue(size > 1024 * 1024)
- expected_content = ''.join(chr(i % 256) for i in range(0, size))
+ expected_content = b''.join(
+ six.int2byte(i % 256) for i in range(0, size))
lfa_id = self.add_file('hello_bigboy.xls', expected_content)
lfa = IStore(LibraryFileAlias).get(LibraryFileAlias, lfa_id)
lfc = lfa.content
@@ -269,7 +274,7 @@ class TestFeedSwift(TestCase):
# magic manifest header is set correctly.
container, name = swift.swift_location(lfc.id)
headers, obj = swift_client.get_object(container, name)
- self.assertEqual(obj, '')
+ self.assertEqual(obj, b'')
# The segments we expect are all in their expected locations.
_, obj1 = swift_client.get_object(container, '{0}/0000'.format(name))
@@ -288,62 +293,62 @@ class TestHashStream(TestCase):
def test_read(self):
empty_md5 = 'd41d8cd98f00b204e9800998ecf8427e'
- s = swift.HashStream(StringIO('make me a coffee'))
+ s = swift.HashStream(io.BytesIO(b'make me a coffee'))
self.assertEqual(s.hash.hexdigest(), empty_md5)
data = s.read()
- self.assertEqual(data, 'make me a coffee')
+ self.assertEqual(data, b'make me a coffee')
self.assertEqual(s.hash.hexdigest(),
'17dfd3e9f99a2260552e898406c696e9')
def test_partial_read(self):
empty_sha1 = 'da39a3ee5e6b4b0d3255bfef95601890afd80709'
s = swift.HashStream(
- StringIO('make me another coffee'), hash_factory=hashlib.sha1)
+ io.BytesIO(b'make me another coffee'), hash_factory=hashlib.sha1)
self.assertEqual(s.hash.hexdigest(), empty_sha1)
chunk = s.read(4)
- self.assertEqual(chunk, 'make')
+ self.assertEqual(chunk, b'make')
self.assertEqual(s.hash.hexdigest(),
'5821eb27d7b71c9078000da31a5a654c97e401b9')
chunk = s.read()
- self.assertEqual(chunk, ' me another coffee')
+ self.assertEqual(chunk, b' me another coffee')
self.assertEqual(s.hash.hexdigest(),
'8c826e573016ce05f3968044f82507b46fd2aa93')
def test_limited_length(self):
- base_stream = StringIO('make me a coffee')
+ base_stream = io.BytesIO(b'make me a coffee')
s = swift.HashStream(base_stream, length=8)
chunk = s.read(4)
- self.assertEqual(chunk, 'make')
+ self.assertEqual(chunk, b'make')
self.assertEqual(s.hash.hexdigest(),
'099dafc678df7d266c25f95ccf6cde22')
chunk = s.read(8)
- self.assertEqual(chunk, ' me ')
+ self.assertEqual(chunk, b' me ')
self.assertEqual(s.hash.hexdigest(),
'10a0334e435b75f35b1923842bd87f81')
- self.assertEqual(s.read(), '')
+ self.assertEqual(s.read(), b'')
self.assertEqual(s.tell(), 8)
self.assertEqual(base_stream.tell(), 8)
def test_tell(self):
- s = swift.HashStream(StringIO('hurry up with that coffee'))
+ s = swift.HashStream(io.BytesIO(b'hurry up with that coffee'))
self.assertEqual(s.tell(), 0)
s.read(4)
self.assertEqual(s.tell(), 4)
def test_seek(self):
- s = swift.HashStream(StringIO('hurry up with that coffee'))
+ s = swift.HashStream(io.BytesIO(b'hurry up with that coffee'))
s.seek(0)
self.assertEqual(s.tell(), 0)
s.seek(6)
self.assertEqual(s.tell(), 6)
chunk = s.read()
- self.assertEqual(chunk, 'up with that coffee')
+ self.assertEqual(chunk, b'up with that coffee')
self.assertEqual(s.hash.hexdigest(),
'0687b12af46824e3584530c5262fed36')
# Seek also must reset the hash.
s.seek(2)
chunk = s.read(3)
- self.assertEqual(chunk, 'rry')
+ self.assertEqual(chunk, b'rry')
self.assertEqual(s.hash.hexdigest(),
'35cd51ccd493b67542201d20b6ed7db9')
diff --git a/lib/lp/services/librarianserver/web.py b/lib/lp/services/librarianserver/web.py
index 90001dc..44ec313 100644
--- a/lib/lp/services/librarianserver/web.py
+++ b/lib/lp/services/librarianserver/web.py
@@ -7,6 +7,7 @@ from datetime import datetime
import time
from pymacaroons import Macaroon
+import six
from six.moves.urllib.parse import urlparse
from storm.exceptions import DisconnectionError
from twisted.internet import (
@@ -17,10 +18,7 @@ from twisted.internet import (
from twisted.internet.interfaces import IPushProducer
from twisted.internet.threads import deferToThread
from twisted.python import log
-from twisted.python.compat import (
- intToBytes,
- networkString,
- )
+from twisted.python.compat import intToBytes
from twisted.web import (
http,
proxy,
@@ -110,7 +108,7 @@ class LibraryFileAliasResource(resource.Resource):
# self.aliasIDd, And that the host precisely matches what we generate
# (specifically to stop people putting a good prefix to the left of an
# attacking one).
- hostname = request.getRequestHostname()
+ hostname = six.ensure_str(request.getRequestHostname())
if '.restricted.' in hostname:
# Configs can change without warning: evaluate every time.
download_url = config.librarian.download_url
@@ -126,7 +124,7 @@ class LibraryFileAliasResource(resource.Resource):
(expected_hostname, hostname))
return fourOhFour
- token = request.args.get('token', [None])[0]
+ token = request.args.get(b'token', [None])[0]
if token is None:
if not request.getUser() and request.getPassword():
try:
@@ -135,7 +133,7 @@ class LibraryFileAliasResource(resource.Resource):
# https://github.com/ecordell/pymacaroons/issues/50 is fixed.
except Exception:
pass
- path = request.path
+ path = six.ensure_text(request.path)
deferred = deferToThread(
self._getFileAlias, self.aliasID, token, path)
deferred.addCallback(
@@ -182,7 +180,7 @@ class LibraryFileAliasResource(resource.Resource):
if stream is not None:
# XXX: Brad Crittenden 2007-12-05 bug=174204: When encodings are
# stored as part of a file's metadata this logic will be replaced.
- encoding, mimetype = guess_librarian_encoding(filename, mimetype)
+ encoding, mimetype = guess_librarian_encoding(dbfilename, mimetype)
file = File(mimetype, encoding, date_created, stream, size)
# Set our caching headers. Public Librarian files can be
# cached forever, while private ones mustn't be at all.
@@ -221,10 +219,11 @@ class File(resource.Resource):
def _setContentHeaders(self, request):
request.setHeader(b'content-length', intToBytes(self.size))
if self.type:
- request.setHeader(b'content-type', networkString(self.type))
+ request.setHeader(
+ b'content-type', six.ensure_binary(self.type, 'ASCII'))
if self.encoding:
request.setHeader(
- b'content-encoding', networkString(self.encoding))
+ b'content-encoding', six.ensure_binary(self.encoding, 'ASCII'))
def render_GET(self, request):
"""See `Resource`."""
