launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #26367
[Merge] ~cjwatson/launchpad:py3-tagfiles-bytes into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:py3-tagfiles-bytes into launchpad:master.
Commit message:
Always parse tagfiles as bytes
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/398367
Most field values are restricted to ASCII anyway, but dealing with everything as bytes makes our data types more consistent, allowing them to work on Python 3.
Apologies for the size of this MP; I couldn't manage to reduce it while making it still reasonably obvious what was going on. It ended up being simplest to convert pretty much all the tagfile parsing logic at once.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:py3-tagfiles-bytes into launchpad:master.
diff --git a/lib/lp/archiveuploader/changesfile.py b/lib/lp/archiveuploader/changesfile.py
index e3d5c5c..09cb65e 100644
--- a/lib/lp/archiveuploader/changesfile.py
+++ b/lib/lp/archiveuploader/changesfile.py
@@ -17,6 +17,8 @@ __all__ = [
import os
+import six
+
from lp.archiveuploader.buildinfofile import BuildInfoFile
from lp.archiveuploader.dscfile import (
DSCFile,
@@ -47,6 +49,7 @@ from lp.registry.interfaces.sourcepackage import (
SourcePackageFileType,
SourcePackageUrgency,
)
+from lp.services.encoding import guess as guess_encoding
from lp.soyuz.enums import BinaryPackageFileType
@@ -241,14 +244,14 @@ class ChangesFile(SignableTagFile):
if 'Urgency' not in self._dict:
# Urgency is recommended but not mandatory. Default to 'low'
- self._dict['Urgency'] = "low"
+ self._dict['Urgency'] = b"low"
- raw_urgency = self._dict['Urgency'].lower()
+ raw_urgency = six.ensure_text(self._dict['Urgency']).lower()
if raw_urgency not in self.urgency_map:
yield UploadWarning(
"Unable to grok urgency %s, overriding with 'low'"
% (raw_urgency))
- self._dict['Urgency'] = "low"
+ self._dict['Urgency'] = b"low"
if not self.policy.unsigned_changes_ok:
assert self.signer is not None, (
@@ -297,7 +300,7 @@ class ChangesFile(SignableTagFile):
For example, 'hoary' or 'hoary-security'.
"""
- return self._dict['Distribution']
+ return six.ensure_text(self._dict['Distribution'])
@property
def architectures(self):
@@ -306,22 +309,23 @@ class ChangesFile(SignableTagFile):
For instance ['source', 'all'] or ['source', 'i386', 'amd64']
or ['source'].
"""
- return set(self._dict['Architecture'].split())
+ return set(six.ensure_text(self._dict['Architecture']).split())
@property
def binaries(self):
"""Return set of binary package names listed."""
- return set(self._dict.get('Binary', '').strip().split())
+ return set(
+ six.ensure_text(self._dict.get('Binary', '')).strip().split())
@property
def converted_urgency(self):
"""Return the appropriate SourcePackageUrgency item."""
- return self.urgency_map[self._dict['Urgency'].lower()]
+ return self.urgency_map[six.ensure_text(self._dict['Urgency']).lower()]
@property
def version(self):
"""Return changesfile claimed version."""
- return self._dict['Version']
+ return six.ensure_text(self._dict['Version'])
@classmethod
def formatChangesComment(cls, comment):
@@ -338,24 +342,24 @@ class ChangesFile(SignableTagFile):
@property
def changes_comment(self):
"""Return changesfile 'change' comment."""
- comment = self._dict['Changes']
+ comment = guess_encoding(self._dict['Changes'])
return self.formatChangesComment(comment)
@property
def date(self):
"""Return changesfile date."""
- return self._dict['Date']
+ return six.ensure_text(self._dict['Date'])
@property
def source(self):
"""Return changesfile claimed source name."""
- return self._dict['Source']
+ return six.ensure_text(self._dict['Source'])
@property
def architecture_line(self):
"""Return changesfile archicteture line."""
- return self._dict['Architecture']
+ return six.ensure_text(self._dict['Architecture'])
@property
def simulated_changelog(self):
@@ -369,8 +373,8 @@ class ChangesFile(SignableTagFile):
"""
changes_author = rfc822_encode_address(
self.changed_by['name'], self.changed_by['email'])
- return '%s\n\n -- %s %s' % (
- self.changes_comment, changes_author.encode('utf-8'), self.date)
+ return (u'%s\n\n -- %s %s' % (
+ self.changes_comment, changes_author, self.date)).encode('UTF-8')
def determine_file_class_and_name(filename):
diff --git a/lib/lp/archiveuploader/dscfile.py b/lib/lp/archiveuploader/dscfile.py
index 3ae5496..9140ef4 100644
--- a/lib/lp/archiveuploader/dscfile.py
+++ b/lib/lp/archiveuploader/dscfile.py
@@ -127,7 +127,7 @@ class SignableTagFile:
if self.signingkey is not None:
return self.signingkey.owner
- def parse(self, verify_signature=True, as_bytes=False):
+ def parse(self, verify_signature=True):
"""Parse the tag file, optionally verifying the signature.
If verify_signature is True, signingkey will be set to the signing
@@ -169,7 +169,7 @@ class SignableTagFile:
self.parsed_content = self.raw_content
try:
self._dict = parse_tagfile_content(
- self.parsed_content, filename=self.filepath, as_bytes=as_bytes)
+ self.parsed_content, filename=self.filepath)
except TagFileParseError as error:
raise UploadError(
"Unable to parse %s: %s" % (self.filename, error))
@@ -223,8 +223,8 @@ class SignableTagFile:
raise UploadError("Invalid Maintainer.")
if person is None and self.policy.create_people:
- package = self._dict['Source']
- version = self._dict['Version']
+ package = six.ensure_text(self._dict['Source'])
+ version = six.ensure_text(self._dict['Version'])
if self.policy.distroseries and self.policy.pocket:
policy_suite = ('%s/%s' % (self.policy.distroseries.name,
self.policy.pocket.name))
@@ -295,7 +295,7 @@ class DSCFile(SourceUploadFile, SignableTagFile):
SourceUploadFile.__init__(
self, filepath, checksums, size, component_and_section, priority,
package, version, changes, policy, logger)
- self.parse(verify_signature=not policy.unsigned_dsc_ok, as_bytes=True)
+ self.parse(verify_signature=not policy.unsigned_dsc_ok)
self.logger.debug("Performing DSC verification.")
for mandatory_field in self.mandatory_fields:
@@ -310,7 +310,7 @@ class DSCFile(SourceUploadFile, SignableTagFile):
# the wild generates dsc files with format missing, and we need
# to accept them.
if 'Format' not in self._dict:
- self._dict['Format'] = "1.0"
+ self._dict['Format'] = b"1.0"
if self.format is None:
raise EarlyReturnUploadError(
@@ -323,12 +323,12 @@ class DSCFile(SourceUploadFile, SignableTagFile):
@property
def source(self):
"""Return the DSC source name."""
- return self._dict['Source']
+ return six.ensure_text(self._dict['Source'])
@property
def dsc_version(self):
"""Return the DSC source version."""
- return self._dict['Version']
+ return six.ensure_text(self._dict['Version'])
@property
def format(self):
@@ -342,12 +342,12 @@ class DSCFile(SourceUploadFile, SignableTagFile):
@property
def architecture(self):
"""Return the DSC source architecture."""
- return self._dict['Architecture']
+ return six.ensure_text(self._dict['Architecture'])
@property
def binary(self):
"""Return the DSC claimed binary line."""
- return self._dict['Binary']
+ return six.ensure_text(self._dict['Binary'])
#
# DSC file checks.
@@ -411,6 +411,7 @@ class DSCFile(SourceUploadFile, SignableTagFile):
for field_name in ['Build-Depends', 'Build-Depends-Indep']:
field = self._dict.get(field_name, None)
if field is not None:
+ field = six.ensure_text(field)
if field.startswith("ARRAY"):
yield UploadError(
"%s: invalid %s field produced by a broken version "
diff --git a/lib/lp/archiveuploader/nascentuploadfile.py b/lib/lp/archiveuploader/nascentuploadfile.py
index dc5a8ed..48f079d 100644
--- a/lib/lp/archiveuploader/nascentuploadfile.py
+++ b/lib/lp/archiveuploader/nascentuploadfile.py
@@ -344,12 +344,12 @@ class PackageUploadFile(NascentUploadFile):
if self.section_name not in valid_sections:
raise UploadError(
- "%s: Unknown section %r" % (
+ "%s: Unknown section '%s'" % (
self.filename, self.section_name))
if self.component_name not in valid_components:
raise UploadError(
- "%s: Unknown component %r" % (
+ "%s: Unknown component '%s'" % (
self.filename, self.component_name))
@property
@@ -585,6 +585,7 @@ class BaseBinaryUploadFile(PackageUploadFile):
control_source = self.control.get("Source", None)
if control_source is not None:
+ control_source = six.ensure_text(control_source)
if "(" in control_source:
src_match = re_extract_src_version.match(control_source)
self.source_name = src_match.group(1)
@@ -595,14 +596,20 @@ class BaseBinaryUploadFile(PackageUploadFile):
else:
self.source_name = self.control.get("Package")
self.source_version = self.control.get("Version")
+ if self.source_name is not None:
+ self.source_name = six.ensure_text(self.source_name)
+ if self.source_version is not None:
+ self.source_version = six.ensure_text(self.source_version)
# Store control_version for external use (archive version consistency
# checks in nascentupload.py)
self.control_version = self.control.get("Version")
+ if self.control_version is not None:
+ self.control_version = six.ensure_text(self.control_version)
def verifyPackage(self):
"""Check if the binary is in changesfile and its name is valid."""
- control_package = self.control.get("Package", '')
+ control_package = six.ensure_text(self.control.get("Package", b''))
# Since DDEBs are generated after the original DEBs are processed
# and considered by `dpkg-genchanges` they are only half-incorporated
@@ -651,36 +658,37 @@ class BaseBinaryUploadFile(PackageUploadFile):
Also check if it is a valid architecture in LP context.
"""
- control_arch = self.control.get("Architecture", '')
+ control_arch = six.ensure_text(self.control.get("Architecture", b''))
valid_archs = [a.architecturetag
for a in self.policy.distroseries.architectures]
if control_arch not in valid_archs and control_arch != "all":
yield UploadError(
- "%s: Unknown architecture: %r" % (
+ "%s: Unknown architecture: '%s'" % (
self.filename, control_arch))
if control_arch not in self.changes.architectures:
yield UploadError(
- "%s: control file lists arch as %r which isn't "
+ "%s: control file lists arch as '%s' which isn't "
"in the changes file." % (self.filename, control_arch))
if control_arch != self.architecture:
yield UploadError(
- "%s: control file lists arch as %r which doesn't "
- "agree with version %r in the filename."
+ "%s: control file lists arch as '%s' which doesn't "
+ "agree with version '%s' in the filename."
% (self.filename, control_arch, self.architecture))
def verifyDepends(self):
"""Check if control depends field is present and not empty."""
- control_depends = self.control.get('Depends', "--unset-marker--")
+ control_depends = self.control.get('Depends', b"--unset-marker--")
if not control_depends:
yield UploadError(
"%s: Depends field present and empty." % self.filename)
def verifySection(self):
"""Check the section & priority match those in changesfile."""
- control_section_and_component = self.control.get('Section', '')
+ control_section_and_component = six.ensure_text(
+ self.control.get('Section', b''))
control_component, control_section = splitComponentAndSection(
control_section_and_component)
if ((control_component, control_section) !=
@@ -693,7 +701,7 @@ class BaseBinaryUploadFile(PackageUploadFile):
def verifyPriority(self):
"""Check if priority matches changesfile."""
- control_priority = self.control.get('Priority', '')
+ control_priority = six.ensure_text(self.control.get('Priority', b''))
if control_priority and self.priority_name != control_priority:
yield UploadError(
"%s control file lists priority as %s but changes file has "
@@ -896,7 +904,7 @@ class BaseBinaryUploadFile(PackageUploadFile):
is_essential = encoded.get('Essential', '').lower() == 'yes'
architecturespecific = not self.is_archindep
- installedsize = int(self.control.get('Installed-Size', '0'))
+ installedsize = int(self.control.get('Installed-Size', b'0'))
binary_name = getUtility(
IBinaryPackageNameSet).getOrCreateByName(self.package)
diff --git a/lib/lp/archiveuploader/tagfiles.py b/lib/lp/archiveuploader/tagfiles.py
index 5d3adc0..666c9c1 100644
--- a/lib/lp/archiveuploader/tagfiles.py
+++ b/lib/lp/archiveuploader/tagfiles.py
@@ -22,7 +22,7 @@ class TagFileParseError(Exception):
pass
-def parse_tagfile_content(content, filename=None, as_bytes=False):
+def parse_tagfile_content(content, filename=None):
"""Parses a tag file and returns a dictionary where each field is a key.
The mandatory first argument is the contents of the tag file as a
@@ -30,13 +30,15 @@ def parse_tagfile_content(content, filename=None, as_bytes=False):
An OpenPGP cleartext signature will be stripped before parsing if
one is present.
+
+ Header values are always returned as bytes.
"""
with tempfile.TemporaryFile() as f:
f.write(strip_pgp_signature(content))
f.seek(0)
try:
- stanzas = list(apt_pkg.TagFile(f, bytes=as_bytes))
+ stanzas = list(apt_pkg.TagFile(f, bytes=True))
except SystemError as e:
raise TagFileParseError("%s: %s" % (filename, e))
if len(stanzas) != 1:
@@ -61,8 +63,10 @@ def parse_tagfile(filename):
The mandatory first argument is the filename of the tag file, and
the contents of that file is passed on to parse_tagfile_content.
+
+ Header values are always returned as bytes.
"""
- with open(filename, "r") as changes_in:
+ with open(filename, "rb") as changes_in:
content = changes_in.read()
if not content:
raise TagFileParseError("%s: empty file" % filename)
diff --git a/lib/lp/archiveuploader/tests/nascentupload-closing-bugs.txt b/lib/lp/archiveuploader/tests/nascentupload-closing-bugs.txt
index dc39b37..91db6b5 100644
--- a/lib/lp/archiveuploader/tests/nascentupload-closing-bugs.txt
+++ b/lib/lp/archiveuploader/tests/nascentupload-closing-bugs.txt
@@ -78,8 +78,8 @@ This new version fixes bug #6 according its changesfiles:
>>> print(bar2_src.changes.changed_by['person'].name)
kinnison
- >>> bar2_src.changes._dict['Launchpad-bugs-fixed']
- '6'
+ >>> print(six.ensure_str(bar2_src.changes._dict['Launchpad-bugs-fixed']))
+ 6
>>> print bar2_src.changes.changes_comment
bar (1.0-2) breezy; urgency=low
diff --git a/lib/lp/archiveuploader/tests/nascentupload-epoch-handling.txt b/lib/lp/archiveuploader/tests/nascentupload-epoch-handling.txt
index 5acbc91..1ba7b85 100644
--- a/lib/lp/archiveuploader/tests/nascentupload-epoch-handling.txt
+++ b/lib/lp/archiveuploader/tests/nascentupload-epoch-handling.txt
@@ -204,11 +204,11 @@ SourcePackageRelease record store a proper 'version':
For source uploads, Changes.version == DSC.version == SPR.version:
- >>> bar_src_upload.changes.version
- '1:1.0-9'
+ >>> print(bar_src_upload.changes.version)
+ 1:1.0-9
- >>> bar_src_upload.changes.dsc.dsc_version
- '1:1.0-9'
+ >>> print(bar_src_upload.changes.dsc.dsc_version)
+ 1:1.0-9
>>> bar_src_queue = bar_src_upload.queue_root
>>> bar_spr = bar_src_queue.sources[0].sourcepackagerelease
@@ -247,21 +247,21 @@ The Changesfile version always refers to the source version and the
binary versions included in the upload can diverge between themselves
and from the source version.
- >>> bar_bin_upload.changes.version
- '1:1.0-9'
+ >>> print(bar_bin_upload.changes.version)
+ 1:1.0-9
>>> deb_file = bar_bin_upload.changes.files[0]
- >>> deb_file.filename
- 'bar_6.6.6_i386.deb'
+ >>> print(deb_file.filename)
+ bar_6.6.6_i386.deb
- >>> deb_file.version
- '1:1.0-9'
+ >>> print(deb_file.version)
+ 1:1.0-9
- >>> deb_file.source_version
- '1:1.0-9'
+ >>> print(deb_file.source_version)
+ 1:1.0-9
- >>> deb_file.control_version
- '1:6.6.6'
+ >>> print(deb_file.control_version)
+ 1:6.6.6
Anyway, the proper value for BinaryPackageRelease.version is the
version stored in the binary control file:
diff --git a/lib/lp/archiveuploader/tests/nascentupload.txt b/lib/lp/archiveuploader/tests/nascentupload.txt
index 6bdda3d..37f6836 100644
--- a/lib/lp/archiveuploader/tests/nascentupload.txt
+++ b/lib/lp/archiveuploader/tests/nascentupload.txt
@@ -231,8 +231,9 @@ should match the files target architectures:
... print f.filename, f
ed_0.2-20_i386.deb <...DebBinaryUploadFile...>
- >>> [a for a in ed_mismatched_upload.changes.architectures]
- ['amd64']
+ >>> for a in ed_mismatched_upload.changes.architectures:
+ ... print(a)
+ amd64
Since the changesfile specify that only 'amd64' will be used and
there is a file that depends on 'i386' the upload is rejected:
@@ -280,8 +281,10 @@ And because of that it's not considered native.
But if we check the DSC we will find the reference to the already
known ORIG file:
- >>> [f.filename for f in ed_upload.changes.dsc.files]
- ['ed_0.2.orig.tar.gz', 'ed_0.2-21.diff.gz']
+ >>> for f in ed_upload.changes.dsc.files:
+ ... print(f.filename)
+ ed_0.2.orig.tar.gz
+ ed_0.2-21.diff.gz
>>> success = ed_upload.do_accept()
>>> success
diff --git a/lib/lp/archiveuploader/tests/nascentuploadfile.txt b/lib/lp/archiveuploader/tests/nascentuploadfile.txt
index 0dbd48b..85a2ac7 100644
--- a/lib/lp/archiveuploader/tests/nascentuploadfile.txt
+++ b/lib/lp/archiveuploader/tests/nascentuploadfile.txt
@@ -90,18 +90,18 @@ file name.
At this point the changesfile content is already parsed:
- >>> ed_binary_changes.source
- 'ed'
+ >>> print(ed_binary_changes.source)
+ ed
- >>> ed_binary_changes.version
- '0.2-20'
+ >>> print(ed_binary_changes.version)
+ 0.2-20
>>> for item in ed_binary_changes.architectures:
... print(item)
i386
- >>> ed_binary_changes.suite_name
- 'unstable'
+ >>> print(ed_binary_changes.suite_name)
+ unstable
Push upload targeted suite into policy before the checks, nomally done
by NascentUpload object:
@@ -129,15 +129,19 @@ At this point we can inspect the list of files contained in the upload.
... print uploaded_file.filename
ed_0.2-20_i386.deb
- >>> [f.filename for f in ed_binary_changes.binary_package_files]
- ['ed_0.2-20_i386.deb']
- >>> [f.filename for f in ed_binary_changes.source_package_files]
- []
-
- >>> [f.filename for f in ed_source_changes.binary_package_files]
- []
- >>> [f.filename for f in ed_source_changes.source_package_files]
- ['ed_0.2-20.dsc', 'ed_0.2-20.diff.gz', 'ed_0.2.orig.tar.gz']
+ >>> for f in ed_binary_changes.binary_package_files:
+ ... print(f.filename)
+ ed_0.2-20_i386.deb
+ >>> for f in ed_binary_changes.source_package_files:
+ ... print(f.filename)
+
+ >>> for f in ed_source_changes.binary_package_files:
+ ... print(f.filename)
+ >>> for f in ed_source_changes.source_package_files:
+ ... print(f.filename)
+ ed_0.2-20.dsc
+ ed_0.2-20.diff.gz
+ ed_0.2.orig.tar.gz
Similar to what we have in 'processFiles' ChangesFile.verify is also
a error generator
@@ -364,14 +368,14 @@ in the ChangesFile instance.
The DSCFile also presents a similar behaviour to access its parsed
contents:
- >>> ed_source_dsc.source
- 'ed'
- >>> ed_source_dsc.version
- '0.2-20'
- >>> ed_source_dsc.architecture
- 'any'
- >>> ed_source_dsc.binary
- 'ed'
+ >>> print(ed_source_dsc.source)
+ ed
+ >>> print(ed_source_dsc.version)
+ 0.2-20
+ >>> print(ed_source_dsc.architecture)
+ any
+ >>> print(ed_source_dsc.binary)
+ ed
The DSC is GPG-signed most of the time, so we can guarantee who was
the author. The DSCFile class implements the same address parsing
@@ -482,7 +486,7 @@ changes file:
... 'main/net', 'important', 'foo', '1.2', ed_binary_changes,
... modified_insecure_policy, DevNullLogger())
>>> list(ed_binary_deb.verify())
- [UploadError('ed_0.2-20_i386.deb
+ [UploadError(...'ed_0.2-20_i386.deb
control file lists section as main/editors but changes file has
main/net.',)]
@@ -493,7 +497,7 @@ It also checks the priority against the changes file:
... 'main/editors', 'extra', 'foo', '1.2', ed_binary_changes,
... modified_insecure_policy, DevNullLogger())
>>> list(ed_binary_deb.verify())
- [UploadError('ed_0.2-20_i386.deb
+ [UploadError(...'ed_0.2-20_i386.deb
control file lists priority as important but changes file has extra.',)]
The timestamp of the files in the .deb are tested against the policy for
diff --git a/lib/lp/archiveuploader/tests/test_changesfile.py b/lib/lp/archiveuploader/tests/test_changesfile.py
index 60c8646..8f76d23 100644
--- a/lib/lp/archiveuploader/tests/test_changesfile.py
+++ b/lib/lp/archiveuploader/tests/test_changesfile.py
@@ -10,6 +10,7 @@ __metaclass__ = type
import os
from debian.deb822 import Changes
+import six
from testtools.matchers import (
Equals,
MatchesDict,
@@ -281,9 +282,9 @@ class ChangesFileTests(TestCase):
"mypkg_0.1_i386.changes", contents)
self.assertEqual([], list(changes.processAddresses()))
self.assertEqual(
- "Something changed\n\n"
- " -- Somebody <somebody@xxxxxxxxxx> "
- "Fri, 25 Jun 2010 11:20:22 -0600",
+ b"Something changed\n\n"
+ b" -- Somebody <somebody@xxxxxxxxxx> "
+ b"Fri, 25 Jun 2010 11:20:22 -0600",
changes.simulated_changelog)
def test_requires_changed_by(self):
@@ -377,7 +378,7 @@ class TestSignatureVerification(TestCase):
expected = "\\AFormat: 1.7\n.*foo_1.0-1.diff.gz\\Z"
self.assertTextMatchesExpressionIgnoreWhitespace(
expected,
- changesfile.parsed_content)
+ six.ensure_text(changesfile.parsed_content))
def test_no_signature_rejected(self):
# An unsigned changes file is rejected.
@@ -399,6 +400,6 @@ class TestSignatureVerification(TestCase):
expected = "\\AFormat: 1.7\n.*foo_1.0-1.diff.gz\\Z"
self.assertTextMatchesExpressionIgnoreWhitespace(
expected,
- changesfile.parsed_content)
+ six.ensure_text(changesfile.parsed_content))
self.assertEqual("breezy", changesfile.suite_name)
self.assertNotIn("evil", changesfile.changes_comment)
diff --git a/lib/lp/archiveuploader/tests/test_tagfiles.py b/lib/lp/archiveuploader/tests/test_tagfiles.py
index 3deff06..f5bf552 100755
--- a/lib/lp/archiveuploader/tests/test_tagfiles.py
+++ b/lib/lp/archiveuploader/tests/test_tagfiles.py
@@ -31,7 +31,7 @@ class Testtagfiles(unittest.TestCase):
reject them if it can't understand.
"""
parsed = parse_tagfile(datadir("bad-multiline-changes"))
- self.assertEqual('unstable', parsed['Distribution'])
+ self.assertEqual(b'unstable', parsed['Distribution'])
def testCheckParseMalformedMultiline(self):
"""Malformed but somewhat readable files do not raise an exception.
@@ -40,7 +40,7 @@ class Testtagfiles(unittest.TestCase):
reject them if it can't understand.
"""
parsed = parse_tagfile(datadir("bad-multiline-changes"))
- self.assertEqual('unstable', parsed['Distribution'])
+ self.assertEqual(b'unstable', parsed['Distribution'])
self.assertRaises(KeyError, parsed.__getitem__, 'Fish')
def testCheckParseEmptyChangesRaises(self):
@@ -82,7 +82,7 @@ class TestTagFileDebianPolicyCompat(unittest.TestCase):
tagfile_path = datadir("test436182_0.1_source.changes")
tagfile = open(tagfile_path)
- self.apt_pkg_parsed_version = apt_pkg.TagFile(tagfile)
+ self.apt_pkg_parsed_version = apt_pkg.TagFile(tagfile, bytes=True)
self.apt_pkg_parsed_version.step()
self.parse_tagfile_version = parse_tagfile(tagfile_path)
@@ -97,17 +97,17 @@ class TestTagFileDebianPolicyCompat(unittest.TestCase):
2. appended a trailing '\n' to the end of the value.
"""
- expected_text = (
- 'test75874 anotherbinary\n'
- ' andanother andonemore\n'
- '\tlastone')
+ expected_bytes = (
+ b'test75874 anotherbinary\n'
+ b' andanother andonemore\n'
+ b'\tlastone')
self.assertEqual(
- expected_text,
+ expected_bytes,
self.apt_pkg_parsed_version.section['Binary'])
self.assertEqual(
- expected_text,
+ expected_bytes,
self.parse_tagfile_version['Binary'])
def test_parse_tagfile_with_newline_delimited_field(self):
@@ -123,19 +123,19 @@ class TestTagFileDebianPolicyCompat(unittest.TestCase):
see http://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Files
"""
- expected_text = (
- 'f26bb9b29b1108e53139da3584a4dc92 1511 test75874_0.1.tar.gz\n '
- '29c955ff520cea32ab3e0316306d0ac1 393742 '
- 'pmount_0.9.7.orig.tar.gz\n'
- ' 91a8f46d372c406fadcb57c6ff7016f3 5302 '
- 'pmount_0.9.7-2ubuntu2.diff.gz')
+ expected_bytes = (
+ b'f26bb9b29b1108e53139da3584a4dc92 1511 test75874_0.1.tar.gz\n '
+ b'29c955ff520cea32ab3e0316306d0ac1 393742 '
+ b'pmount_0.9.7.orig.tar.gz\n'
+ b' 91a8f46d372c406fadcb57c6ff7016f3 5302 '
+ b'pmount_0.9.7-2ubuntu2.diff.gz')
self.assertEqual(
- expected_text,
+ expected_bytes,
self.apt_pkg_parsed_version.section['Files'])
self.assertEqual(
- expected_text,
+ expected_bytes,
self.parse_tagfile_version['Files'])
def test_parse_description_field(self):
@@ -144,19 +144,19 @@ class TestTagFileDebianPolicyCompat(unittest.TestCase):
See http://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Description
"""
- expected_text = (
- "Here's the single-line synopsis.\n"
- " Then there is the extended description which can\n"
- " span multiple lines, and even include blank-lines like this:\n"
- " .\n"
- " There you go. If a line starts with two or more spaces,\n"
- " it will be displayed verbatim. Like this one:\n"
- " Example verbatim line.\n"
- " Another verbatim line.\n"
- " OK, back to normal.")
+ expected_bytes = (
+ b"Here's the single-line synopsis.\n"
+ b" Then there is the extended description which can\n"
+ b" span multiple lines, and even include blank-lines like this:\n"
+ b" .\n"
+ b" There you go. If a line starts with two or more spaces,\n"
+ b" it will be displayed verbatim. Like this one:\n"
+ b" Example verbatim line.\n"
+ b" Another verbatim line.\n"
+ b" OK, back to normal.")
self.assertEqual(
- expected_text,
+ expected_bytes,
self.apt_pkg_parsed_version.section['Description'])
# In the past our parse_tagfile function replaced blank-line
@@ -164,5 +164,5 @@ class TestTagFileDebianPolicyCompat(unittest.TestCase):
# but it is now compatible with ParseTagFiles (and ready to be
# replaced by ParseTagFiles).
self.assertEqual(
- expected_text,
+ expected_bytes,
self.parse_tagfile_version['Description'])
diff --git a/lib/lp/archiveuploader/utils.py b/lib/lp/archiveuploader/utils.py
index 334a2aa..bc5cf80 100644
--- a/lib/lp/archiveuploader/utils.py
+++ b/lib/lp/archiveuploader/utils.py
@@ -270,7 +270,7 @@ def parse_file_list(s, field_name, count):
if s is None:
return None
processed = []
- for line in s.strip().split('\n'):
+ for line in six.ensure_text(s).strip().split('\n'):
split = line.strip().split()
if len(split) != count:
raise UploadError(
diff --git a/lib/lp/soyuz/doc/gina.txt b/lib/lp/soyuz/doc/gina.txt
index 2e2e253..f342cf3 100644
--- a/lib/lp/soyuz/doc/gina.txt
+++ b/lib/lp/soyuz/doc/gina.txt
@@ -134,38 +134,38 @@ Check STDERR for the errors we expected:
>>> print(proc.stderr.read())
ERROR Error processing package files for clearlooks
...
- ExecutionError: Error 2 unpacking source
+ ...ExecutionError: Error 2 unpacking source
WARNING Invalid format in db1-compat, assumed '1.0'
WARNING Source package ed lacks section, assumed 'misc'
ERROR Unable to create SourcePackageData for mkvmlinuz
...
- InvalidVersionError: mkvmlinuz has an invalid version: None
+ ...InvalidVersionError: mkvmlinuz has an invalid version: None
WARNING Invalid urgency in python-pam, None, assumed 'low'
ERROR Error processing package files for util-linux
...
- PoolFileNotFound: File util-linux_2.12p-2ubuntu2.2.dsc not in archive
+ ...PoolFileNotFound: File util-linux_2.12p-2ubuntu2.2.dsc not in archive
ERROR Error processing package files for bsdutils
...
- PoolFileNotFound: .../bsdutils_2.12p-2ubuntu2_i386.deb not found
+ ...PoolFileNotFound: .../bsdutils_2.12p-2ubuntu2_i386.deb not found
WARNING Binary package ed lacks valid priority, assumed 'extra'
ERROR Unable to create BinaryPackageData for mount
...
- InvalidVersionError: mount has an invalid version: -ewePP2.12p-2ubuntu2
+ ...InvalidVersionError: mount has an invalid version: -ewePP2.12p-2ubuntu2
WARNING Binary package python-pam lacks a section, assumed 'misc'
ERROR Error processing package files for python2.4-pam
...
- PoolFileNotFound: .../python2.4-pam_0.4.2-10.1ubuntu3_i386.deb not found
+ ...PoolFileNotFound: .../python2.4-pam_0.4.2-10.1ubuntu3_i386.deb not found
ERROR Error processing package files for python2.4-sqlite
...
- PoolFileNotFound: .../python2.4-sqlite_1.0.1-1ubuntu1_i386.deb not found
+ ...PoolFileNotFound: .../python2.4-sqlite_1.0.1-1ubuntu1_i386.deb not found
WARNING No source package rioutil (1.4.4-1.0.1) listed for rioutil (1.4.4-1.0.1), scrubbing archive...
WARNING Nope, couldn't find it. Could it be a bin-only-NMU? Checking...
ERROR Error processing package files for util-linux
...
- PoolFileNotFound: .../util-linux_2.12p-2ubuntu2_i386.deb not found
+ ...PoolFileNotFound: .../util-linux_2.12p-2ubuntu2_i386.deb not found
ERROR Unable to create BinaryPackageData for util-linux-locales
...
- MissingRequiredArguments: ['installed_size']
+ ...MissingRequiredArguments: ['installed_size']
ERROR Invalid Sources stanza in /tmp/tmp...
...
WARNING No changelog file found for mkvmlinuz in mkvmlinuz-14ubuntu1
@@ -173,13 +173,13 @@ Check STDERR for the errors we expected:
WARNING Invalid urgency in mkvmlinuz, None, assumed 'low'
ERROR Error processing package files for python-sqlite
...
- PoolFileNotFound: File python-sqlite_1.0.1-2ubuntu1.dsc not in archive
+ ...PoolFileNotFound: File python-sqlite_1.0.1-2ubuntu1.dsc not in archive
ERROR Error processing package files for util-linux
...
- PoolFileNotFound: File util-linux_2.12p-6ubuntu5.dsc not in archive
+ ...PoolFileNotFound: File util-linux_2.12p-6ubuntu5.dsc not in archive
ERROR Error processing package files for python-sqlite
...
- PoolFileNotFound: .../python-sqlite_1.0.1-2ubuntu1_all.deb not found
+ ...PoolFileNotFound: .../python-sqlite_1.0.1-2ubuntu1_all.deb not found
WARNING No source package ubuntu-meta (0.80) listed for ubuntu-base (0.80), scrubbing archive...
<BLANKLINE>
@@ -230,7 +230,7 @@ Check if the changelog message was stored correcly:
Check that the changelog was uploaded to the librarian correctly:
- >>> print(x11p.changelog.read())
+ >>> print(six.ensure_text(x11p.changelog.read()))
x11proto-damage (6.8.99.7-2) breezy; urgency=low
<BLANKLINE>
* Add dependency on x11proto-fixes-dev.
@@ -521,45 +521,45 @@ run.
>>> print(proc.stderr.read())
ERROR Error processing package files for clearlooks
...
- ExecutionError: Error 2 unpacking source
+ ...ExecutionError: Error 2 unpacking source
WARNING Source package ed lacks section, assumed 'misc'
ERROR Unable to create SourcePackageData for mkvmlinuz
...
- InvalidVersionError: mkvmlinuz has an invalid version: None
+ ...InvalidVersionError: mkvmlinuz has an invalid version: None
ERROR Error processing package files for util-linux
...
- PoolFileNotFound: File util-linux_2.12p-2ubuntu2.2.dsc not in archive
+ ...PoolFileNotFound: File util-linux_2.12p-2ubuntu2.2.dsc not in archive
ERROR Error processing package files for bsdutils
...
- PoolFileNotFound: .../bsdutils_2.12p-2ubuntu2_i386.deb not found
+ ...PoolFileNotFound: .../bsdutils_2.12p-2ubuntu2_i386.deb not found
WARNING Binary package ed lacks valid priority, assumed 'extra'
ERROR Unable to create BinaryPackageData for mount
...
- InvalidVersionError: mount has an invalid version: -ewePP2.12p-2ubuntu2
+ ...InvalidVersionError: mount has an invalid version: -ewePP2.12p-2ubuntu2
WARNING Binary package python-pam lacks a section, assumed 'misc'
ERROR Error processing package files for python2.4-pam
...
- PoolFileNotFound: .../python2.4-pam_0.4.2-10.1ubuntu3_i386.deb not found
+ ...PoolFileNotFound: .../python2.4-pam_0.4.2-10.1ubuntu3_i386.deb not found
ERROR Error processing package files for python2.4-sqlite
...
- PoolFileNotFound: .../python2.4-sqlite_1.0.1-1ubuntu1_i386.deb not found
+ ...PoolFileNotFound: .../python2.4-sqlite_1.0.1-1ubuntu1_i386.deb not found
ERROR Error processing package files for util-linux
...
- PoolFileNotFound: .../util-linux_2.12p-2ubuntu2_i386.deb not found
+ ...PoolFileNotFound: .../util-linux_2.12p-2ubuntu2_i386.deb not found
ERROR Unable to create BinaryPackageData for util-linux-locales
...
- MissingRequiredArguments: ['installed_size']
+ ...MissingRequiredArguments: ['installed_size']
ERROR Invalid Sources stanza in /tmp/tmp...
...
ERROR Error processing package files for python-sqlite
...
- PoolFileNotFound: File python-sqlite_1.0.1-2ubuntu1.dsc not in archive
+ ...PoolFileNotFound: File python-sqlite_1.0.1-2ubuntu1.dsc not in archive
ERROR Error processing package files for util-linux
...
- PoolFileNotFound: File util-linux_2.12p-6ubuntu5.dsc not in archive
+ ...PoolFileNotFound: File util-linux_2.12p-6ubuntu5.dsc not in archive
ERROR Error processing package files for python-sqlite
...
- PoolFileNotFound: .../python-sqlite_1.0.1-2ubuntu1_all.deb not found
+ ...PoolFileNotFound: .../python-sqlite_1.0.1-2ubuntu1_all.deb not found
<BLANKLINE>
>>> proc.wait()
0
@@ -815,7 +815,7 @@ For kicks, finally, run gina on a configured but incomplete archive:
>>> print(proc.stderr.read())
ERROR Failed to analyze archive for bogoland
...
- MangledArchiveError: No archive directory for bogoland/main
+ ...MangledArchiveError: No archive directory for bogoland/main
<BLANKLINE>
>>> proc.wait()
1
diff --git a/lib/lp/soyuz/doc/soyuz-upload.txt b/lib/lp/soyuz/doc/soyuz-upload.txt
index 5030c6a..7962cb2 100644
--- a/lib/lp/soyuz/doc/soyuz-upload.txt
+++ b/lib/lp/soyuz/doc/soyuz-upload.txt
@@ -61,13 +61,14 @@ been uploaded over FTP.
... tf = {}
...
... if "Source" in tf:
- ... package_names.append(tf["Source"])
+ ... package_names.append(six.ensure_text(tf["Source"]))
...
... send_filepaths = [changes_filepath]
... if "Files" in tf:
... send_filepaths.extend(
... [os.path.join(test_files_dir, line.split()[-1])
- ... for line in tf["Files"].splitlines() if line])
+ ... for line in six.ensure_text(tf["Files"]).splitlines()
+ ... if line])
...
... sent_filenames.extend(
... os.path.basename(filepath) for filepath in send_filepaths)
@@ -87,7 +88,7 @@ been uploaded over FTP.
Check that what we've just uploaded (everything in test_files_dir) is
what we were expecting to have uploaded.
- >>> package_names
+ >>> print(pretty(package_names))
['drdsl', 'etherwake']
At that point we must have a bunch of directories in the upload base
diff --git a/lib/lp/soyuz/mail/packageupload.py b/lib/lp/soyuz/mail/packageupload.py
index f194061..a5686a1 100644
--- a/lib/lp/soyuz/mail/packageupload.py
+++ b/lib/lp/soyuz/mail/packageupload.py
@@ -140,6 +140,8 @@ def fetch_information(spr, bprs, changes, previous_version=None):
changelog = ChangesFile.formatChangesComment(
sanitize_string(changes.get('Changes')))
date = changes.get('Date')
+ if date is not None:
+ date = six.ensure_text(date)
try:
changedby = parse_maintainer_bytes(
changes.get('Changed-By'), 'Changed-By')
@@ -151,7 +153,7 @@ def fetch_information(spr, bprs, changes, previous_version=None):
except ParseMaintError:
pass
notify_changed_by = changes.get(
- 'Launchpad-Notify-Changed-By', '') == 'yes'
+ 'Launchpad-Notify-Changed-By', b'') == b'yes'
elif spr or bprs:
if not spr and bprs:
spr = bprs[0].build.source_package_release
diff --git a/lib/lp/soyuz/mail/tests/test_packageupload.py b/lib/lp/soyuz/mail/tests/test_packageupload.py
index 7881462..d39d9c3 100644
--- a/lib/lp/soyuz/mail/tests/test_packageupload.py
+++ b/lib/lp/soyuz/mail/tests/test_packageupload.py
@@ -254,10 +254,10 @@ class TestNotification(TestCaseWithFactory):
def test_fetch_information_changes(self):
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Foo Bar <foo.bar@xxxxxxxxxxx>',
- 'Maintainer': 'Foo Bar <foo.bar@xxxxxxxxxxx>',
- 'Changes': ' * Foo!',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Foo Bar <foo.bar@xxxxxxxxxxx>',
+ 'Maintainer': b'Foo Bar <foo.bar@xxxxxxxxxxx>',
+ 'Changes': b' * Foo!',
}
info = fetch_information(None, None, changes)
self.assertEqual('2001-01-01', info['date'])
@@ -272,11 +272,11 @@ class TestNotification(TestCaseWithFactory):
def test_fetch_information_changes_notify_changed_by(self):
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Foo Bar <foo.bar@xxxxxxxxxxx>',
- 'Maintainer': 'Foo Bar <foo.bar@xxxxxxxxxxx>',
- 'Changes': ' * Foo!',
- 'Launchpad-Notify-Changed-By': 'yes',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Foo Bar <foo.bar@xxxxxxxxxxx>',
+ 'Maintainer': b'Foo Bar <foo.bar@xxxxxxxxxxx>',
+ 'Changes': b' * Foo!',
+ 'Launchpad-Notify-Changed-By': b'yes',
}
info = fetch_information(None, None, changes)
self.assertEqual('2001-01-01', info['date'])
@@ -410,10 +410,10 @@ class TestNotification(TestCaseWithFactory):
# Test getRecipientsForAction with good email addresses..
blamer, maintainer, changer = self._setup_recipients()
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Changer <changer@xxxxxxxxxxx>',
- 'Maintainer': 'Maintainer <maintainer@xxxxxxxxxxx>',
- 'Changes': ' * Foo!',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Changer <changer@xxxxxxxxxxx>',
+ 'Maintainer': b'Maintainer <maintainer@xxxxxxxxxxx>',
+ 'Changes': b' * Foo!',
}
self.assertRecipientsEqual(
[blamer, maintainer, changer],
@@ -422,10 +422,10 @@ class TestNotification(TestCaseWithFactory):
def test_getRecipientsForAction_bad_maintainer_email(self):
blamer, maintainer, changer = self._setup_recipients()
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Changer <changer@xxxxxxxxxxx>',
- 'Maintainer': 'Maintainer <maintainer at example.com>',
- 'Changes': ' * Foo!',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Changer <changer@xxxxxxxxxxx>',
+ 'Maintainer': b'Maintainer <maintainer at example.com>',
+ 'Changes': b' * Foo!',
}
self.assertRecipientsEqual(
[blamer, changer], changes, blamer, maintainer, changer)
@@ -434,10 +434,10 @@ class TestNotification(TestCaseWithFactory):
# Test getRecipientsForAction with invalid changedby email address.
blamer, maintainer, changer = self._setup_recipients()
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Changer <changer at example.com>',
- 'Maintainer': 'Maintainer <maintainer@xxxxxxxxxxx>',
- 'Changes': ' * Foo!',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Changer <changer at example.com>',
+ 'Maintainer': b'Maintainer <maintainer@xxxxxxxxxxx>',
+ 'Changes': b' * Foo!',
}
self.assertRecipientsEqual(
[blamer, maintainer], changes, blamer, maintainer, changer)
@@ -447,10 +447,10 @@ class TestNotification(TestCaseWithFactory):
# to the archive owner.
_, maintainer, changer = self._setup_recipients()
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Changer <changer@xxxxxxxxxxx>',
- 'Maintainer': 'Maintainer <maintainer@xxxxxxxxxxx>',
- 'Changes': ' * Foo!',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Changer <changer@xxxxxxxxxxx>',
+ 'Maintainer': b'Maintainer <maintainer@xxxxxxxxxxx>',
+ 'Changes': b' * Foo!',
}
self.assertRecipientsEqual(
[], changes, None, maintainer, changer,
@@ -461,10 +461,10 @@ class TestNotification(TestCaseWithFactory):
# signed the upload.
blamer, maintainer, changer = self._setup_recipients()
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Changer <changer@xxxxxxxxxxx>',
- 'Maintainer': 'Maintainer <maintainer@xxxxxxxxxxx>',
- 'Changes': ' * Foo!',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Changer <changer@xxxxxxxxxxx>',
+ 'Maintainer': b'Maintainer <maintainer@xxxxxxxxxxx>',
+ 'Changes': b' * Foo!',
}
self.assertRecipientsEqual(
[blamer], changes, blamer, maintainer, changer,
@@ -475,11 +475,11 @@ class TestNotification(TestCaseWithFactory):
# notifications go to the changer even for PPA uploads.
blamer, maintainer, changer = self._setup_recipients()
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Changer <changer@xxxxxxxxxxx>',
- 'Maintainer': 'Maintainer <maintainer@xxxxxxxxxxx>',
- 'Changes': ' * Foo!',
- 'Launchpad-Notify-Changed-By': 'yes',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Changer <changer@xxxxxxxxxxx>',
+ 'Maintainer': b'Maintainer <maintainer@xxxxxxxxxxx>',
+ 'Changes': b' * Foo!',
+ 'Launchpad-Notify-Changed-By': b'yes',
}
self.assertRecipientsEqual(
[blamer, changer], changes, blamer, maintainer, changer,
@@ -500,10 +500,10 @@ class TestNotification(TestCaseWithFactory):
spr = self.factory.makeSourcePackageRelease(
component=component, section_name="libs")
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Changer <changer@xxxxxxxxxxx>',
- 'Maintainer': 'Maintainer <maintainer@xxxxxxxxxxx>',
- 'Changes': ' * Foo!',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Changer <changer@xxxxxxxxxxx>',
+ 'Maintainer': b'Maintainer <maintainer@xxxxxxxxxxx>',
+ 'Changes': b' * Foo!',
}
mailer = PackageUploadMailer.forAction(
"accepted", blamer, spr, [], [], archive, distroseries,
@@ -545,10 +545,10 @@ class TestNotification(TestCaseWithFactory):
spr = self.factory.makeSourcePackageRelease(
component=component, section_name="libs")
changes = {
- 'Date': '2001-01-01',
- 'Changed-By': 'Changer <changer@xxxxxxxxxxx>',
- 'Maintainer': 'Maintainer <maintainer@xxxxxxxxxxx>',
- 'Changes': ' * Foo!',
+ 'Date': b'2001-01-01',
+ 'Changed-By': b'Changer <changer@xxxxxxxxxxx>',
+ 'Maintainer': b'Maintainer <maintainer@xxxxxxxxxxx>',
+ 'Changes': b' * Foo!',
}
mailer = PackageUploadMailer.forAction(
"accepted", blamer, spr, [], [], archive, distroseries,
diff --git a/lib/lp/soyuz/scripts/gina/archive.py b/lib/lp/soyuz/scripts/gina/archive.py
index 6ecb30a..e2f524c 100644
--- a/lib/lp/soyuz/scripts/gina/archive.py
+++ b/lib/lp/soyuz/scripts/gina/archive.py
@@ -22,6 +22,7 @@ import shutil
import tempfile
import apt_pkg
+import six
from lp.services.scripts import log
from lp.soyuz.scripts.gina import call
@@ -202,13 +203,14 @@ class PackagesMap:
# because most of them are the same for all architectures,
# but we go over it to also cover source packages that only
# compile for one architecture.
- sources = apt_pkg.TagFile(info_set.srcfile)
+ sources = apt_pkg.TagFile(info_set.srcfile, bytes=True)
try:
for section in sources:
try:
src_tmp = dict(section)
- src_tmp['Component'] = info_set.component
- src_name = src_tmp['Package']
+ src_tmp['Component'] = six.ensure_binary(
+ info_set.component)
+ src_name = six.ensure_text(src_tmp['Package'])
except KeyError:
log.exception(
"Invalid Sources stanza in %s",
@@ -229,13 +231,14 @@ class PackagesMap:
tmpbin_map = self.bin_map[info_set.arch]
- binaries = apt_pkg.TagFile(info_set.binfile)
+ binaries = apt_pkg.TagFile(info_set.binfile, bytes=True)
for section in binaries:
try:
bin_tmp = dict(section)
# The component isn't listed in the tagfile.
- bin_tmp['Component'] = info_set.component
- bin_name = bin_tmp['Package']
+ bin_tmp['Component'] = six.ensure_binary(
+ info_set.component)
+ bin_name = six.ensure_text(bin_tmp['Package'])
except KeyError:
log.exception(
"Invalid Releases stanza in %s",
@@ -244,12 +247,13 @@ class PackagesMap:
tmpbin_map[bin_name] = bin_tmp
# Run over the D-I stanzas and store info in tmp_bin_map.
- dibinaries = apt_pkg.TagFile(info_set.difile)
+ dibinaries = apt_pkg.TagFile(info_set.difile, bytes=True)
for section in dibinaries:
try:
dibin_tmp = dict(section)
- dibin_tmp['Component'] = info_set.component
- dibin_name = dibin_tmp['Package']
+ dibin_tmp['Component'] = six.ensure_binary(
+ info_set.component)
+ dibin_name = six.ensure_text(dibin_tmp['Package'])
except KeyError:
log.exception("Invalid D-I Releases stanza in %s" %
info_set.difile)
diff --git a/lib/lp/soyuz/scripts/gina/changelog.py b/lib/lp/soyuz/scripts/gina/changelog.py
index bd637da..ec20556 100644
--- a/lib/lp/soyuz/scripts/gina/changelog.py
+++ b/lib/lp/soyuz/scripts/gina/changelog.py
@@ -8,13 +8,15 @@ __metaclass__ = type
import re
import sys
-
-first_re = re.compile(r"^([a-z0-9][a-z0-9\\+\\.\\-]+)\s+\(([^ ]+)\)")
-urgency_re = re.compile(r'(?:urgency|priority)=([^ ,;:.]+)')
+import six
from lp.archivepublisher.debversion import Version
+first_re = re.compile(br"^([a-z0-9][a-z0-9\\+\\.\\-]+)\s+\(([^ ]+)\)")
+urgency_re = re.compile(br'(?:urgency|priority)=([^ ,;:.]+)')
+
+
def parse_first_line(line):
# SRCPKGNAME (VERSION).*((urgency|priority)=\S+)?
match = first_re.match(line)
@@ -33,8 +35,8 @@ def parse_first_line(line):
def parse_last_line(line):
- maint = line[:line.find(">") + 1].strip()
- date = line[line.find(">") + 1:].strip()
+ maint = line[:line.find(b">") + 1].strip()
+ date = line[line.find(b">") + 1:].strip()
return (maint, date)
@@ -48,40 +50,40 @@ def parse_changelog_stanza(firstline, stanza, lastline):
"urgency": urgency,
"maintainer": maint,
"date": date,
- "changes": "".join(stanza).strip("\n")
+ "changes": b"".join(stanza).strip(b"\n")
}
def parse_changelog(changelines):
state = 0
- firstline = ""
+ firstline = b""
stanza = []
rets = []
for line in changelines:
#print line[:-1]
if state == 0:
- if (line.startswith(" ") or line.startswith("\t") or
+ if (line.startswith(b" ") or line.startswith(b"\t") or
not line.rstrip()):
#print "State0 skip"
continue
try:
(source, version, urgency) = parse_first_line(line.strip())
- Version(version)
+ Version(six.ensure_text(version))
except:
stanza.append(line)
#print "state0 Exception skip"
continue
firstline = line.strip()
- stanza = [line, '\n']
+ stanza = [line, b'\n']
state = 1
continue
if state == 1:
stanza.append(line)
- stanza.append('\n')
+ stanza.append(b'\n')
- if line.startswith(" --") and "@" in line:
+ if line.startswith(b" --") and b"@" in line:
#print "state1 accept"
# Last line of stanza
rets.append(parse_changelog_stanza(firstline,
@@ -93,12 +95,12 @@ def parse_changelog(changelines):
if state == 1:
rets[-1]["changes"] += firstline
if len(rets):
- rets[-1]["changes"] += "".join(stanza).strip("\n")
+ rets[-1]["changes"] += b"".join(stanza).strip(b"\n")
return rets
if __name__ == '__main__':
import pprint
- with open(sys.argv[1]) as f:
+ with open(sys.argv[1], "rb") as f:
pprint.pprint(parse_changelog(f))
diff --git a/lib/lp/soyuz/scripts/gina/dominate.py b/lib/lp/soyuz/scripts/gina/dominate.py
index 6b27a18..0745f92 100644
--- a/lib/lp/soyuz/scripts/gina/dominate.py
+++ b/lib/lp/soyuz/scripts/gina/dominate.py
@@ -8,6 +8,7 @@ __all__ = [
'dominate_imported_source_packages',
]
+import six
from zope.component import getUtility
from lp.archivepublisher.domination import Dominator
@@ -27,7 +28,8 @@ def dominate_imported_source_packages(txn, logger, distro_name, series_name,
for package_name, pub_count in package_counts:
entries = packages_map.src_map.get(package_name, [])
live_versions = [
- entry['Version'] for entry in entries if 'Version' in entry]
+ six.ensure_text(entry['Version'])
+ for entry in entries if 'Version' in entry]
# Gina import just ensured that any live version in the Sources
# file has a Published publication. So there should be at least
diff --git a/lib/lp/soyuz/scripts/gina/handlers.py b/lib/lp/soyuz/scripts/gina/handlers.py
index d3d4333..85f7fce 100644
--- a/lib/lp/soyuz/scripts/gina/handlers.py
+++ b/lib/lp/soyuz/scripts/gina/handlers.py
@@ -513,21 +513,21 @@ class SourcePackageHandler:
# Since the dsc doesn't know, we add in the directory, package
# component and section
dsc_contents['directory'] = os.path.join("pool",
- poolify(sp_name, sp_component))
- dsc_contents['package'] = sp_name
- dsc_contents['component'] = sp_component
- dsc_contents['section'] = sp_section
+ poolify(sp_name, sp_component)).encode("ASCII")
+ dsc_contents['package'] = sp_name.encode("ASCII")
+ dsc_contents['component'] = sp_component.encode("ASCII")
+ dsc_contents['section'] = sp_section.encode("ASCII")
# the dsc doesn't list itself so add it ourselves
if 'files' not in dsc_contents:
log.error('DSC for %s didn\'t contain a files entry: %r' %
(dsc_name, dsc_contents))
return None
- if not dsc_contents['files'].endswith("\n"):
- dsc_contents['files'] += "\n"
+ if not dsc_contents['files'].endswith(b"\n"):
+ dsc_contents['files'] += b"\n"
# XXX kiko 2005-10-21: Why do we hack the md5sum and size of the DSC?
# Should probably calculate it properly.
- dsc_contents['files'] += "xxx 000 %s" % dsc_name
+ dsc_contents['files'] += ("xxx 000 %s" % dsc_name).encode("ASCII")
# SourcePackageData requires capitals
capitalized_dsc = {}
diff --git a/lib/lp/soyuz/scripts/gina/packages.py b/lib/lp/soyuz/scripts/gina/packages.py
index 6f294d2..504e907 100644
--- a/lib/lp/soyuz/scripts/gina/packages.py
+++ b/lib/lp/soyuz/scripts/gina/packages.py
@@ -29,6 +29,7 @@ import shutil
import tempfile
import scandir
+import six
from lp.app.validators.version import valid_debian_version
from lp.archivepublisher.diskpool import poolify
@@ -123,7 +124,7 @@ def read_dsc(package, version, component, distro_name, archive_root):
distro_name, archive_root)
try:
- with open(dsc_path) as f:
+ with open(dsc_path, "rb") as f:
dsc = f.read().strip()
fullpath = os.path.join(source_dir, "debian", "changelog")
@@ -331,11 +332,11 @@ class SourcePackageData(AbstractPackageData):
def __init__(self, **args):
for k, v in args.items():
if k == 'Binary':
- self.binaries = stripseq(v.split(","))
+ self.binaries = stripseq(six.ensure_text(v).split(","))
elif k == 'Section':
- self.section = parse_section(v)
+ self.section = parse_section(six.ensure_text(v))
elif k == 'Urgency':
- urgency = v
+ urgency = six.ensure_text(v)
# This is to handle cases like:
# - debget: 'high (actually works)
# - lxtools: 'low, closes=90239'
@@ -345,23 +346,20 @@ class SourcePackageData(AbstractPackageData):
urgency = urgency.split(",")[0]
self.urgency = urgency
elif k == 'Maintainer':
- displayname, emailaddress = parse_person(v)
try:
- self.maintainer = (
- encoding.guess(displayname),
- emailaddress,
- )
+ maintainer = encoding.guess(v)
except UnicodeDecodeError:
raise DisplayNameDecodingError(
- "Could not decode name %s" % displayname)
+ "Could not decode Maintainer field %r" % v)
+ self.maintainer = parse_person(maintainer)
elif k == 'Files' or k.startswith('Checksums-'):
if not hasattr(self, 'files'):
self.files = []
- files = v.split("\n")
+ files = six.ensure_text(v).split("\n")
for f in files:
self.files.append(stripseq(f.split(" "))[-1])
else:
- self.set_field(k, v)
+ self.set_field(k, encoding.guess(v))
if self.section is None:
self.section = 'misc'
@@ -390,7 +388,7 @@ class SourcePackageData(AbstractPackageData):
self.copyright = encoding.guess(copyright)
parsed_changelog = None
if changelog:
- parsed_changelog = parse_changelog(changelog.split('\n'))
+ parsed_changelog = parse_changelog(changelog.split(b'\n'))
self.urgency = None
self.changelog = None
@@ -398,17 +396,21 @@ class SourcePackageData(AbstractPackageData):
if parsed_changelog and parsed_changelog[0]:
cldata = parsed_changelog[0]
if 'changes' in cldata:
- if cldata["package"] != self.package:
+ cldata_package = six.ensure_text(cldata["package"])
+ cldata_version = six.ensure_text(cldata["version"])
+ if cldata_package != self.package:
log.warning(
"Changelog package %s differs from %s" %
- (cldata["package"], self.package))
- if cldata["version"] != self.version:
+ (cldata_package, self.package))
+ if cldata_version != self.version:
log.warning(
"Changelog version %s differs from %s" %
- (cldata["version"], self.version))
+ (cldata_version, self.version))
self.changelog_entry = encoding.guess(cldata["changes"])
self.changelog = changelog
self.urgency = cldata["urgency"]
+ if self.urgency is not None:
+ self.urgency = six.ensure_text(self.urgency)
else:
log.warning(
"Changelog empty for source %s (%s)" %
@@ -483,11 +485,11 @@ class BinaryPackageData(AbstractPackageData):
def __init__(self, **args):
for k, v in args.items():
if k == "Maintainer":
- self.maintainer = parse_person(v)
+ self.maintainer = parse_person(encoding.guess(v))
elif k == "Essential":
- self.essential = (v == "yes")
+ self.essential = (v == b"yes")
elif k == 'Section':
- self.section = parse_section(v)
+ self.section = parse_section(six.ensure_text(v))
elif k == "Description":
self.description = encoding.guess(v)
summary = self.description.split("\n")[0].strip()
@@ -495,21 +497,22 @@ class BinaryPackageData(AbstractPackageData):
summary = summary + '.'
self.summary = summary
elif k == "Installed-Size":
+ installed_size = six.ensure_text(v)
try:
- self.installed_size = int(v)
+ self.installed_size = int(installed_size)
except ValueError:
raise MissingRequiredArguments("Installed-Size is "
- "not a valid integer: %r" % v)
+ "not a valid integer: %r" % installed_size)
elif k == "Built-Using":
- self.built_using = v
+ self.built_using = six.ensure_text(v)
# Preserve the original form of Built-Using to avoid
# possible unfortunate apt behaviour. This is most easily
# done by adding it to _user_defined as well.
if self._user_defined is None:
self._user_defined = []
- self._user_defined.append([k, v])
+ self._user_defined.append([k, self.built_using])
else:
- self.set_field(k, v)
+ self.set_field(k, encoding.guess(v))
if self.source:
# We need to handle cases like "Source: myspell
diff --git a/lib/lp/soyuz/scripts/gina/runner.py b/lib/lp/soyuz/scripts/gina/runner.py
index f5056c6..c956744 100644
--- a/lib/lp/soyuz/scripts/gina/runner.py
+++ b/lib/lp/soyuz/scripts/gina/runner.py
@@ -10,6 +10,7 @@ import sys
import time
import psycopg2
+import six
from zope.component import getUtility
from lp.registry.interfaces.pocket import PackagePublishingPocket
@@ -120,7 +121,7 @@ def run_gina(options, ztm, target_section):
def attempt_source_package_import(distro, source, package_root,
importer_handler):
"""Attempt to import a source package, and handle typical errors."""
- package_name = source.get("Package", "unknown")
+ package_name = six.ensure_text(source.get("Package", "unknown"))
try:
try:
do_one_sourcepackage(
diff --git a/lib/lp/soyuz/scripts/tests/test_gina.py b/lib/lp/soyuz/scripts/tests/test_gina.py
index e36bc6a..0fbd765 100644
--- a/lib/lp/soyuz/scripts/tests/test_gina.py
+++ b/lib/lp/soyuz/scripts/tests/test_gina.py
@@ -12,6 +12,7 @@ from unittest import TestLoader
import apt_pkg
from fixtures import EnvironmentVariableFixture
+import six
from testtools.matchers import (
MatchesSetwise,
MatchesStructure,
@@ -274,10 +275,10 @@ class TestSourcePackageData(TestCaseWithFactory):
len(debian_tar_contents))))
dsc_contents = parse_tagfile(dsc_path)
- dsc_contents["Directory"] = pool_dir
- dsc_contents["Package"] = "foo"
- dsc_contents["Component"] = "main"
- dsc_contents["Section"] = "misc"
+ dsc_contents["Directory"] = six.ensure_binary(pool_dir)
+ dsc_contents["Package"] = b"foo"
+ dsc_contents["Component"] = b"main"
+ dsc_contents["Section"] = b"misc"
sp_data = SourcePackageData(**dsc_contents)
# Unpacking this in an Ubuntu context fails.
@@ -329,10 +330,10 @@ class TestSourcePackageData(TestCaseWithFactory):
len(debian_tar_contents))))
dsc_contents = parse_tagfile(dsc_path)
- dsc_contents["Directory"] = pool_dir
- dsc_contents["Package"] = "foo"
- dsc_contents["Component"] = "main"
- dsc_contents["Section"] = "misc"
+ dsc_contents["Directory"] = six.ensure_binary(pool_dir)
+ dsc_contents["Package"] = b"foo"
+ dsc_contents["Component"] = b"main"
+ dsc_contents["Section"] = b"misc"
sp_data = SourcePackageData(**dsc_contents)
unpack_tmpdir = self.makeTemporaryDirectory()
@@ -351,20 +352,20 @@ class TestSourcePackageData(TestCaseWithFactory):
def test_checksum_fields(self):
# We only need one of Files or Checksums-*.
base_dsc_contents = {
- "Package": "foo",
- "Binary": "foo",
- "Version": "1.0-1",
- "Maintainer": "Foo Bar <foo@xxxxxxxxxxxxx>",
- "Section": "misc",
- "Architecture": "all",
- "Directory": "pool/main/f/foo",
- "Component": "main",
+ "Package": b"foo",
+ "Binary": b"foo",
+ "Version": b"1.0-1",
+ "Maintainer": b"Foo Bar <foo@xxxxxxxxxxxxx>",
+ "Section": b"misc",
+ "Architecture": b"all",
+ "Directory": b"pool/main/f/foo",
+ "Component": b"main",
}
for field in (
"Files", "Checksums-Sha1", "Checksums-Sha256",
"Checksums-Sha512"):
dsc_contents = dict(base_dsc_contents)
- dsc_contents[field] = "xxx 000 foo_1.0-1.dsc"
+ dsc_contents[field] = b"xxx 000 foo_1.0-1.dsc"
sp_data = SourcePackageData(**dsc_contents)
self.assertEqual(["foo_1.0-1.dsc"], sp_data.files)
self.assertRaises(
@@ -382,18 +383,18 @@ class TestSourcePackageHandler(TestCaseWithFactory):
series.distribution.name, archive_root,
PackagePublishingPocket.RELEASE, None)
dsc_contents = {
- "Format": "3.0 (quilt)",
- "Source": "foo",
- "Binary": "foo",
- "Architecture": "all arm64",
- "Version": "1.0-1",
- "Maintainer": "Foo Bar <foo@xxxxxxxxxxxxx>",
- "Files": "xxx 000 foo_1.0-1.dsc",
- "Build-Indep-Architecture": "amd64",
- "Directory": "pool/main/f/foo",
- "Package": "foo",
- "Component": "main",
- "Section": "misc",
+ "Format": b"3.0 (quilt)",
+ "Source": b"foo",
+ "Binary": b"foo",
+ "Architecture": b"all arm64",
+ "Version": b"1.0-1",
+ "Maintainer": b"Foo Bar <foo@xxxxxxxxxxxxx>",
+ "Files": b"xxx 000 foo_1.0-1.dsc",
+ "Build-Indep-Architecture": b"amd64",
+ "Directory": b"pool/main/f/foo",
+ "Package": b"foo",
+ "Component": b"main",
+ "Section": b"misc",
}
sp_data = SourcePackageData(**dsc_contents)
self.assertEqual(
@@ -427,9 +428,10 @@ class TestSourcePackagePublisher(TestCaseWithFactory):
publisher = SourcePackagePublisher(series, pocket, None)
publisher.publish(spr, SourcePackageData(
- component='main', section=section.name, version='1.0',
- maintainer=maintainer.preferredemail, architecture='all',
- files='foo.py', binaries='foo.py'))
+ component=b'main', section=section.name.encode('ASCII'),
+ version=b'1.0',
+ maintainer=maintainer.preferredemail.email.encode('ASCII'),
+ architecture=b'all', files=b'foo.py', binaries=b'foo.py'))
[spph] = series.main_archive.getPublishedSources()
self.assertEqual(PackagePublishingStatus.PUBLISHED, spph.status)
@@ -442,21 +444,21 @@ class TestBinaryPackageData(TestCaseWithFactory):
def test_checksum_fields(self):
# We only need one of MD5sum or SHA*.
base_deb_contents = {
- "Package": "foo",
- "Installed-Size": "0",
- "Maintainer": "Foo Bar <foo@xxxxxxxxxxxxx>",
- "Section": "misc",
- "Architecture": "all",
- "Version": "1.0-1",
- "Filename": "pool/main/f/foo/foo_1.0-1_all.deb",
- "Component": "main",
- "Size": "0",
- "Description": "",
- "Priority": "extra",
+ "Package": b"foo",
+ "Installed-Size": b"0",
+ "Maintainer": b"Foo Bar <foo@xxxxxxxxxxxxx>",
+ "Section": b"misc",
+ "Architecture": b"all",
+ "Version": b"1.0-1",
+ "Filename": b"pool/main/f/foo/foo_1.0-1_all.deb",
+ "Component": b"main",
+ "Size": b"0",
+ "Description": b"",
+ "Priority": b"extra",
}
for field in ("MD5sum", "SHA1", "SHA256", "SHA512"):
deb_contents = dict(base_deb_contents)
- deb_contents[field] = "0"
+ deb_contents[field] = b"0"
BinaryPackageData(**deb_contents)
self.assertRaises(
MissingRequiredArguments, BinaryPackageData, **base_deb_contents)
@@ -477,21 +479,21 @@ class TestBinaryPackageHandler(TestCaseWithFactory):
spr = self.factory.makeSourcePackageRelease(
distroseries=das.distroseries)
deb_contents = {
- "Package": "foo",
- "Installed-Size": "0",
- "Maintainer": "Foo Bar <foo@xxxxxxxxxxxxx>",
- "Section": "misc",
- "Architecture": "amd64",
- "Version": "1.0-1",
- "Filename": "pool/main/f/foo/foo_1.0-1_amd64.deb",
- "Component": "main",
- "Size": "0",
- "MD5sum": "0" * 32,
- "Description": "",
- "Summary": "",
- "Priority": "extra",
- "Python-Version": "2.7",
- "Built-Using": "nonexistent (= 0.1)",
+ "Package": b"foo",
+ "Installed-Size": b"0",
+ "Maintainer": b"Foo Bar <foo@xxxxxxxxxxxxx>",
+ "Section": b"misc",
+ "Architecture": b"amd64",
+ "Version": b"1.0-1",
+ "Filename": b"pool/main/f/foo/foo_1.0-1_amd64.deb",
+ "Component": b"main",
+ "Size": b"0",
+ "MD5sum": b"0" * 32,
+ "Description": b"",
+ "Summary": b"",
+ "Priority": b"extra",
+ "Python-Version": b"2.7",
+ "Built-Using": b"nonexistent (= 0.1)",
}
bp_data = BinaryPackageData(**deb_contents)
self.assertContentEqual(
@@ -529,20 +531,20 @@ class TestBinaryPackageHandler(TestCaseWithFactory):
built_using_relationship = "%s (= %s)" % (
built_using_spr.name, built_using_spr.version)
deb_contents = {
- "Package": "foo",
- "Installed-Size": "0",
- "Maintainer": "Foo Bar <foo@xxxxxxxxxxxxx>",
- "Section": "misc",
- "Architecture": "amd64",
- "Version": "1.0-1",
- "Filename": "pool/main/f/foo/foo_1.0-1_amd64.deb",
- "Component": "main",
- "Size": "0",
- "MD5sum": "0" * 32,
- "Description": "",
- "Summary": "",
- "Priority": "extra",
- "Built-Using": built_using_relationship,
+ "Package": b"foo",
+ "Installed-Size": b"0",
+ "Maintainer": b"Foo Bar <foo@xxxxxxxxxxxxx>",
+ "Section": b"misc",
+ "Architecture": b"amd64",
+ "Version": b"1.0-1",
+ "Filename": b"pool/main/f/foo/foo_1.0-1_amd64.deb",
+ "Component": b"main",
+ "Size": b"0",
+ "MD5sum": b"0" * 32,
+ "Description": b"",
+ "Summary": b"",
+ "Priority": b"extra",
+ "Built-Using": built_using_relationship.encode("ASCII"),
}
bp_data = BinaryPackageData(**deb_contents)
self.assertContentEqual(
@@ -579,11 +581,13 @@ class TestBinaryPackagePublisher(TestCaseWithFactory):
publisher = BinaryPackagePublisher(series, pocket, None)
publisher.publish(bpr, BinaryPackageData(
- component='main', section=section.name, version='1.0',
- maintainer=maintainer.preferredemail, architecture='all',
- files='foo.py', binaries='foo.py', size=128, installed_size=1024,
- md5sum='e83b5dd68079d727a494a469d40dc8db', description='test',
- summary='Test!'))
+ component=b'main', section=section.name.encode('ASCII'),
+ version=b'1.0',
+ maintainer=maintainer.preferredemail.email.encode('ASCII'),
+ architecture=b'all', files=b'foo.py', binaries=b'foo.py',
+ size=128, installed_size=1024,
+ md5sum=b'e83b5dd68079d727a494a469d40dc8db', description=b'test',
+ summary=b'Test!'))
[bpph] = series.main_archive.getAllPublishedBinaries()
self.assertEqual(PackagePublishingStatus.PUBLISHED, bpph.status)
