launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #20545
[Merge] lp:~cjwatson/launchpad/orig-tarball-signatures into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/orig-tarball-signatures into lp:launchpad.
Commit message:
Handle original tarball signatures in source packages.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1587667 in Launchpad itself: "Import from Debian fails for source packages with included tarball .asc"
https://bugs.launchpad.net/launchpad/+bug/1587667
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/orig-tarball-signatures/+merge/296155
Handle original tarball signatures in source packages.
Some more upload processor tests should exist, but can't quite yet because we first need a dpkg with the necessary cherry-picks to be able to unpack these packages.
Component tarball signature checks could be more accurate (e.g. test that all signatures have a matching tarball), but that was a bit fiddly to wedge into the file checker abstraction and doesn't seem vitally important.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/orig-tarball-signatures into lp:launchpad.
=== modified file 'lib/lp/archiveuploader/dscfile.py'
--- lib/lp/archiveuploader/dscfile.py 2016-03-18 14:12:07 +0000
+++ lib/lp/archiveuploader/dscfile.py 2016-06-01 02:29:15 +0000
@@ -50,6 +50,7 @@
parse_maintainer_bytes,
ParseMaintError,
re_is_component_orig_tar_ext,
+ re_is_component_orig_tar_ext_sig,
re_issource,
re_valid_pkg_name,
re_valid_version,
@@ -441,7 +442,10 @@
if (self.policy.archive.purpose == ArchivePurpose.PPA and
determine_source_file_type(filename) in (
SourcePackageFileType.ORIG_TARBALL,
- SourcePackageFileType.COMPONENT_ORIG_TARBALL)):
+ SourcePackageFileType.COMPONENT_ORIG_TARBALL,
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE,
+ SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE,
+ )):
archives = [self.policy.archive, self.policy.distro.main_archive]
else:
archives = [self.policy.archive]
@@ -470,10 +474,12 @@
file_type_counts = {
SourcePackageFileType.DIFF: 0,
SourcePackageFileType.ORIG_TARBALL: 0,
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
SourcePackageFileType.DEBIAN_TARBALL: 0,
SourcePackageFileType.NATIVE_TARBALL: 0,
}
component_orig_tar_counts = {}
+ component_orig_tar_signature_counts = {}
bzip2_count = 0
xz_count = 0
files_missing = False
@@ -492,6 +498,15 @@
if component not in component_orig_tar_counts:
component_orig_tar_counts[component] = 0
component_orig_tar_counts[component] += 1
+ elif (
+ file_type ==
+ SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE):
+ # Split the count by component name.
+ component = re_is_component_orig_tar_ext_sig.match(
+ get_source_file_extension(sub_dsc_file.filename)).group(1)
+ if component not in component_orig_tar_signature_counts:
+ component_orig_tar_signature_counts[component] = 0
+ component_orig_tar_signature_counts[component] += 1
else:
file_type_counts[file_type] += 1
@@ -552,7 +567,7 @@
for error in file_checker(
self.filename, file_type_counts, component_orig_tar_counts,
- bzip2_count, xz_count):
+ component_orig_tar_signature_counts, bzip2_count, xz_count):
yield error
if files_missing:
@@ -791,12 +806,14 @@
return open(changelog_file, 'r').read()
-def check_format_1_0_files(filename, file_type_counts, component_counts,
+def check_format_1_0_files(filename, file_type_counts,
+ component_counts, component_signature_counts,
bzip2_count, xz_count):
"""Check that the given counts of each file type suit format 1.0.
A 1.0 source must be native (with only one tar.gz), or have an orig.tar.gz
- and a diff.gz. It cannot use bzip2 or xz compression.
+ and a diff.gz. It cannot use bzip2 or xz compression. If it has an
+ orig.tar.gz, it may have an orig.tar.gz.asc signature.
"""
if bzip2_count > 0:
yield UploadError(
@@ -811,11 +828,20 @@
{
SourcePackageFileType.NATIVE_TARBALL: 1,
SourcePackageFileType.ORIG_TARBALL: 0,
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
SourcePackageFileType.DEBIAN_TARBALL: 0,
SourcePackageFileType.DIFF: 0,
},
{
SourcePackageFileType.ORIG_TARBALL: 1,
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
+ SourcePackageFileType.DIFF: 1,
+ SourcePackageFileType.NATIVE_TARBALL: 0,
+ SourcePackageFileType.DEBIAN_TARBALL: 0,
+ },
+ {
+ SourcePackageFileType.ORIG_TARBALL: 1,
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 1,
SourcePackageFileType.DIFF: 1,
SourcePackageFileType.NATIVE_TARBALL: 0,
SourcePackageFileType.DEBIAN_TARBALL: 0,
@@ -823,14 +849,15 @@
]
if (file_type_counts not in valid_file_type_counts or
- len(component_counts) > 0):
+ len(component_counts) > 0 or len(component_signature_counts) > 0):
yield UploadError(
"%s: must have exactly one tar.gz, or an orig.tar.gz and diff.gz"
% filename)
def check_format_3_0_native_files(filename, file_type_counts,
- component_counts, bzip2_count, xz_count):
+ component_counts, component_signature_counts,
+ bzip2_count, xz_count):
"""Check that the given counts of each file type suit format 3.0 (native).
A 3.0 (native) source must have only one tar.*. Any of gzip, bzip2, and
@@ -841,28 +868,39 @@
{
SourcePackageFileType.NATIVE_TARBALL: 1,
SourcePackageFileType.ORIG_TARBALL: 0,
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
SourcePackageFileType.DEBIAN_TARBALL: 0,
SourcePackageFileType.DIFF: 0,
},
]
if (file_type_counts not in valid_file_type_counts or
- len(component_counts) > 0):
+ len(component_counts) > 0 or len(component_signature_counts) > 0):
yield UploadError("%s: must have only a tar.*." % filename)
def check_format_3_0_quilt_files(filename, file_type_counts,
- component_counts, bzip2_count, xz_count):
+ component_counts, component_signature_counts,
+ bzip2_count, xz_count):
"""Check that the given counts of each file type suit format 3.0 (native).
A 3.0 (quilt) source must have exactly one orig.tar.*, one debian.tar.*,
and at most one orig-COMPONENT.tar.* for each COMPONENT. Any of gzip,
- bzip2, and xz compression are permissible.
+ bzip2, and xz compression are permissible. It may have orig.tar.*.asc
+ or orig-COMPONENT.tar.* signatures.
"""
valid_file_type_counts = [
{
SourcePackageFileType.ORIG_TARBALL: 1,
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
+ SourcePackageFileType.DEBIAN_TARBALL: 1,
+ SourcePackageFileType.NATIVE_TARBALL: 0,
+ SourcePackageFileType.DIFF: 0,
+ },
+ {
+ SourcePackageFileType.ORIG_TARBALL: 1,
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 1,
SourcePackageFileType.DEBIAN_TARBALL: 1,
SourcePackageFileType.NATIVE_TARBALL: 0,
SourcePackageFileType.DIFF: 0,
@@ -879,6 +917,11 @@
yield UploadError(
"%s: has more than one orig-%s.tar.*."
% (filename, component))
+ for component in component_signature_counts:
+ if component_signature_counts[component] > 1:
+ yield UploadError(
+ "%s: has more than one orig-%s.tar.*.asc."
+ % (filename, component))
format_to_file_checker_map = {
=== modified file 'lib/lp/archiveuploader/tests/test_dscfile.py'
--- lib/lp/archiveuploader/tests/test_dscfile.py 2016-02-05 16:51:12 +0000
+++ lib/lp/archiveuploader/tests/test_dscfile.py 2016-06-01 02:29:15 +0000
@@ -41,6 +41,7 @@
ORIG_TARBALL = SourcePackageFileType.ORIG_TARBALL
+ORIG_TARBALL_SIGNATURE = SourcePackageFileType.ORIG_TARBALL_SIGNATURE
DEBIAN_TARBALL = SourcePackageFileType.DEBIAN_TARBALL
NATIVE_TARBALL = SourcePackageFileType.NATIVE_TARBALL
DIFF = SourcePackageFileType.DIFF
@@ -229,21 +230,24 @@
class BaseTestSourceFileVerification(TestCase):
- def assertErrorsForFiles(self, expected, files, components={},
+ def assertErrorsForFiles(self, expected, files,
+ components={}, component_signatures={},
bzip2_count=0, xz_count=0):
"""Check problems with the given set of files for the given format.
:param expected: a list of expected errors, as strings.
:param format: the `SourcePackageFormat` to check against.
:param files: a dict mapping `SourcePackageFileType`s to counts.
- :param components: a dict mapping orig component tarball components
- to counts.
+ :param components: a dict mapping orig component tarballs to counts.
+ :param component_signatures: a dict mapping orig component tarball
+ signatures to counts.
:param bzip2_count: number of files using bzip2 compression.
:param xz_count: number of files using xz compression.
"""
full_files = {
NATIVE_TARBALL: 0,
ORIG_TARBALL: 0,
+ ORIG_TARBALL_SIGNATURE: 0,
DIFF: 0,
DEBIAN_TARBALL: 0,
}
@@ -251,20 +255,23 @@
self.assertEquals(
expected,
[str(e) for e in format_to_file_checker_map[self.format](
- 'foo_1.dsc', full_files, components, bzip2_count, xz_count)])
+ 'foo_1.dsc', full_files, components, component_signatures,
+ bzip2_count, xz_count)])
- def assertFilesOK(self, files, components={}, bzip2_count=0, xz_count=0):
+ def assertFilesOK(self, files, components={}, component_signatures={},
+ bzip2_count=0, xz_count=0):
"""Check that the given set of files is OK for the given format.
:param format: the `SourcePackageFormat` to check against.
:param files: a dict mapping `SourcePackageFileType`s to counts.
- :param components: a dict mapping orig component tarball components
- to counts.
+ :param components: a dict mapping orig component tarball to counts.
+ :param component_signatures: a dict mapping orig component tarball
+ signatures to counts.
:param bzip2_count: number of files using bzip2 compression.
:param xz_count: number of files using xz compression.
"""
self.assertErrorsForFiles(
- [], files, components, bzip2_count, xz_count)
+ [], files, components, component_signatures, bzip2_count, xz_count)
class Test10SourceFormatVerification(BaseTestSourceFileVerification):
@@ -280,6 +287,11 @@
# A 1.0 source can contain an original tarball and a Debian diff
self.assertFilesOK({ORIG_TARBALL: 1, DIFF: 1})
+ def testFormat10DebianWithOrigSignature(self):
+ # A 1.0 source can contain an original tarball signature.
+ self.assertFilesOK(
+ {ORIG_TARBALL: 1, ORIG_TARBALL_SIGNATURE: 1, DIFF: 1})
+
def testFormat10Native(self):
# A 1.0 source can contain a native tarball.
self.assertFilesOK({NATIVE_TARBALL: 1})
@@ -287,25 +299,33 @@
def testFormat10CannotHaveWrongFiles(self):
# A 1.0 source cannot have a combination of native and
# non-native files, and cannot have just one of the non-native
- # files.
+ # files or an original tarball signature without an original
+ # tarball.
for combination in (
- {DIFF: 1}, {ORIG_TARBALL: 1}, {ORIG_TARBALL: 1, DIFF: 1,
- NATIVE_TARBALL: 1}):
+ {DIFF: 1}, {ORIG_TARBALL: 1}, {ORIG_TARBALL_SIGNATURE: 1},
+ {ORIG_TARBALL_SIGNATURE: 1, DIFF: 1}):
+ {ORIG_TARBALL: 1, DIFF: 1, NATIVE_TARBALL: 1},
self.assertErrorsForFiles([self.wrong_files_error], combination)
# A 1.0 source with component tarballs is invalid.
self.assertErrorsForFiles(
- [self.wrong_files_error], {ORIG_TARBALL: 1, DIFF: 1}, {'foo': 1})
+ [self.wrong_files_error], {ORIG_TARBALL: 1, DIFF: 1},
+ components={'foo': 1})
+
+ # A 1.0 source with component tarball signatures is invalid.
+ self.assertErrorsForFiles(
+ [self.wrong_files_error], {ORIG_TARBALL: 1, DIFF: 1},
+ component_signatures={'foo': 1})
def testFormat10CannotUseBzip2(self):
# 1.0 sources cannot use bzip2 compression.
self.assertErrorsForFiles(
- [self.bzip2_error], {NATIVE_TARBALL: 1}, {}, 1, 0)
+ [self.bzip2_error], {NATIVE_TARBALL: 1}, {}, bzip2_count=1)
def testFormat10CannotUseXz(self):
# 1.0 sources cannot use xz compression.
self.assertErrorsForFiles(
- [self.xz_error], {NATIVE_TARBALL: 1}, {}, 0, 1)
+ [self.xz_error], {NATIVE_TARBALL: 1}, {}, xz_count=1)
class Test30QuiltSourceFormatVerification(BaseTestSourceFileVerification):
@@ -318,14 +338,21 @@
def testFormat30Quilt(self):
# A 3.0 (quilt) source must contain an orig tarball and a debian
- # tarball. It may also contain at most one component tarball for
- # each component, and can use gzip, bzip2, or xz compression.
+ # tarball. It may also contain at most one component tarball (with
+ # optional signature) for each component, and can use gzip, bzip2,
+ # or xz compression.
for components in ({}, {'foo': 1}, {'foo': 1, 'bar': 1}):
- for bzip2_count in (0, 1):
- for xz_count in (0, 1):
- self.assertFilesOK(
- {ORIG_TARBALL: 1, DEBIAN_TARBALL: 1}, components,
- bzip2_count, xz_count)
+ for component_signatures in ({}, components):
+ for orig_signature in (0, 1):
+ for bzip2_count in (0, 1):
+ for xz_count in (0, 1):
+ self.assertFilesOK(
+ {ORIG_TARBALL: 1,
+ ORIG_TARBALL_SIGNATURE: orig_signature,
+ DEBIAN_TARBALL: 1},
+ components=components,
+ component_signatures=component_signatures,
+ bzip2_count=bzip2_count, xz_count=xz_count)
def testFormat30QuiltCannotHaveConflictingComponentTarballs(self):
# Multiple conflicting tarballs for a single component are
@@ -352,19 +379,24 @@
# 3.0 (native) sources must contain just a native tarball. They
# may use gzip, bzip2, or xz compression.
for bzip2_count in (0, 1):
- self.assertFilesOK({NATIVE_TARBALL: 1}, {},
- bzip2_count, 0)
- self.assertFilesOK({NATIVE_TARBALL: 1}, {}, 0, 1)
+ self.assertFilesOK({NATIVE_TARBALL: 1}, bzip2_count=bzip2_count)
+ self.assertFilesOK({NATIVE_TARBALL: 1}, xz_count=1)
def testFormat30NativeCannotHaveWrongFiles(self):
# 3.0 (quilt) sources may not have a diff, Debian tarball, orig
- # tarball, or any component tarballs.
- for filetype in (DIFF, DEBIAN_TARBALL, ORIG_TARBALL):
+ # tarball, orig tarball signature, or any component tarballs.
+ for filetype in (
+ DIFF, DEBIAN_TARBALL, ORIG_TARBALL, ORIG_TARBALL_SIGNATURE):
self.assertErrorsForFiles(
[self.wrong_files_error], {NATIVE_TARBALL: 1, filetype: 1})
# A 3.0 (native) source with component tarballs is invalid.
self.assertErrorsForFiles(
- [self.wrong_files_error], {NATIVE_TARBALL: 1}, {'foo': 1})
+ [self.wrong_files_error], {NATIVE_TARBALL: 1},
+ components={'foo': 1})
+ # A 3.0 (native) source with component tarball signatures is invalid.
+ self.assertErrorsForFiles(
+ [self.wrong_files_error], {NATIVE_TARBALL: 1},
+ component_signatures={'foo': 1})
class UnpackedDirTests(TestCase):
=== modified file 'lib/lp/archiveuploader/tests/test_utils.py'
--- lib/lp/archiveuploader/tests/test_utils.py 2015-07-29 07:11:41 +0000
+++ lib/lp/archiveuploader/tests/test_utils.py 2016-06-01 02:29:15 +0000
@@ -1,6 +1,6 @@
#!/usr/bin/python
#
-# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
# arch-tag: 90e6eb79-83a2-47e8-9f8b-3c687079c923
@@ -87,6 +87,27 @@
determine_source_file_type('foo_1.0.tar.xz'),
SourcePackageFileType.NATIVE_TARBALL)
+ # (Component) original tarball signatures are detected for any
+ # supported compression method.
+ self.assertEquals(
+ determine_source_file_type('foo_1.0.orig.tar.gz.asc'),
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE)
+ self.assertEquals(
+ determine_source_file_type('foo_1.0.orig.tar.bz2.asc'),
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE)
+ self.assertEquals(
+ determine_source_file_type('foo_1.0.orig.tar.xz.asc'),
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE)
+ self.assertEquals(
+ determine_source_file_type('foo_1.0.orig-foo.tar.gz.asc'),
+ SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE)
+ self.assertEquals(
+ determine_source_file_type('foo_1.0.orig-bar.tar.bz2.asc'),
+ SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE)
+ self.assertEquals(
+ determine_source_file_type('foo_1.0.orig-bar.tar.xz.asc'),
+ SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE)
+
self.assertEquals(None, determine_source_file_type('foo_1.0'))
self.assertEquals(None, determine_source_file_type('foo_1.0.blah.gz'))
=== modified file 'lib/lp/archiveuploader/utils.py'
--- lib/lp/archiveuploader/utils.py 2015-07-29 07:01:04 +0000
+++ lib/lp/archiveuploader/utils.py 2016-06-01 02:29:15 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Archive uploader utilities."""
@@ -19,6 +19,7 @@
're_isadeb',
're_issource',
're_is_component_orig_tar_ext',
+ 're_is_component_orig_tar_ext_sig',
're_no_epoch',
're_no_revision',
're_valid_version',
@@ -67,12 +68,15 @@
re_isadeb = re.compile(r"(.+?)_(.+?)_(.+)\.(u?d?deb)$")
source_file_exts = [
- 'orig(?:-.+)?\.tar\.(?:gz|bz2|xz)', 'diff.gz',
+ 'orig(?:-.+)?\.tar\.(?:gz|bz2|xz)(?:\.asc)?', 'diff.gz',
'(?:debian\.)?tar\.(?:gz|bz2|xz)', 'dsc']
re_issource = re.compile(
r"([^_]+)_(.+?)\.(%s)" % "|".join(ext for ext in source_file_exts))
re_is_component_orig_tar_ext = re.compile(r"^orig-(.+).tar.(?:gz|bz2|xz)$")
+re_is_component_orig_tar_ext_sig = re.compile(
+ r"^orig-(.+).tar.(?:gz|bz2|xz)\.asc$")
re_is_orig_tar_ext = re.compile(r"^orig.tar.(?:gz|bz2|xz)$")
+re_is_orig_tar_ext_sig = re.compile(r"^orig.tar.(?:gz|bz2|xz)\.asc$")
re_is_debian_tar_ext = re.compile(r"^debian.tar.(?:gz|bz2|xz)$")
re_is_native_tar_ext = re.compile(r"^tar.(?:gz|bz2|xz)$")
@@ -115,6 +119,10 @@
return SourcePackageFileType.DEBIAN_TARBALL
elif re_is_native_tar_ext.match(extension):
return SourcePackageFileType.NATIVE_TARBALL
+ elif re_is_orig_tar_ext_sig.match(extension):
+ return SourcePackageFileType.ORIG_TARBALL_SIGNATURE
+ elif re_is_component_orig_tar_ext_sig.match(extension):
+ return SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE
else:
return None
=== modified file 'lib/lp/registry/interfaces/sourcepackage.py'
--- lib/lp/registry/interfaces/sourcepackage.py 2015-11-24 01:44:28 +0000
+++ lib/lp/registry/interfaces/sourcepackage.py 2016-06-01 02:29:15 +0000
@@ -411,6 +411,17 @@
This is only part of the 3.0 (quilt) source package format.""")
+ ORIG_TARBALL_SIGNATURE = DBItem(9, """
+ Orig Tarball Signature
+
+ This file is a detached signature for an Ubuntu "orig" file.""")
+
+ COMPONENT_ORIG_TARBALL_SIGNATURE = DBItem(10, """
+ Component Orig Tarball Signature
+
+ This file is a detached signature for an Ubuntu component "orig"
+ file.""")
+
class SourcePackageType(DBEnumeratedType):
"""Source Package Format
=== modified file 'lib/lp/soyuz/model/files.py'
--- lib/lp/soyuz/model/files.py 2015-07-08 16:05:11 +0000
+++ lib/lp/soyuz/model/files.py 2016-06-01 02:29:15 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2013 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -42,7 +42,9 @@
def is_orig(self):
return self.filetype in (
SourcePackageFileType.ORIG_TARBALL,
- SourcePackageFileType.COMPONENT_ORIG_TARBALL
+ SourcePackageFileType.COMPONENT_ORIG_TARBALL,
+ SourcePackageFileType.ORIG_TARBALL_SIGNATURE,
+ SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE,
)
Follow ups
