launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #15688
[Merge] lp:~wgrant/launchpad/sha256-indices into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/sha256-indices into lp:launchpad.
Commit message:
Add SHA-256 hashes to Sources and Packages generated by NMAF.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #351620 in Launchpad itself: "Use SHA256 in PPA repository indexes"
https://bugs.launchpad.net/launchpad/+bug/351620
Bug #1190879 in Launchpad itself: "SPPH.buildIndexStanzaFields doesn't include SHA-1/SHA-256 hashes unless in user_defined_fields"
https://bugs.launchpad.net/launchpad/+bug/1190879
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/sha256-indices/+merge/169995
Add SHA-256 hashes to Sources and Packages generated by NMAF. This can't be deployed until the columns are fully populated on production.
--
https://code.launchpad.net/~wgrant/launchpad/sha256-indices/+merge/169995
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/sha256-indices into lp:launchpad.
=== modified file 'lib/lp/archivepublisher/tests/test_publisher.py'
--- lib/lp/archivepublisher/tests/test_publisher.py 2013-05-15 10:40:18 +0000
+++ lib/lp/archivepublisher/tests/test_publisher.py 2013-06-18 06:12:17 +0000
@@ -732,6 +732,8 @@
'Size: 18',
'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
+ 'SHA256: 006ca0f356f54b1916c24c282e6fd19961f4356441401f4b0966f2a'
+ '00bb3e945',
'Description: Foo app is great',
' My leading spaces are normalised to a single space but not '
'trailing. ',
@@ -765,6 +767,8 @@
'Size: 18',
'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
+ 'SHA256: 006ca0f356f54b1916c24c282e6fd19961f4356441401f4b0966f2a'
+ '00bb3e945',
'Description: Foo app is great',
' nice udeb',
''],
@@ -794,6 +798,8 @@
'Size: 18',
'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
+ 'SHA256: 006ca0f356f54b1916c24c282e6fd19961f4356441401f4b0966f2a'
+ '00bb3e945',
'Description: Foo app is great',
' My leading spaces are normalised to a single space but not '
'trailing. ',
=== modified file 'lib/lp/archiveuploader/dscfile.py'
--- lib/lp/archiveuploader/dscfile.py 2012-12-26 01:04:05 +0000
+++ lib/lp/archiveuploader/dscfile.py 2013-06-18 06:12:17 +0000
@@ -250,9 +250,11 @@
"Build-Depends-Indep",
"Build-Conflicts",
"Build-Conflicts-Indep",
+ "Checksums-Sha1",
+ "Checksums-Sha256",
"Format",
+ "Homepage",
"Standards-Version",
- "homepage",
]))
# Note that files is actually only set inside verify().
@@ -674,7 +676,7 @@
architecturehintlist=encoded.get('Architecture', ''),
creator=self.changes.changed_by['person'],
urgency=self.changes.converted_urgency,
- homepage=encoded.get('homepage'),
+ homepage=encoded.get('Homepage'),
dsc=encoded_raw_content,
dscsigningkey=self.signingkey,
dsc_maintainer_rfc822=encoded['Maintainer'],
=== modified file 'lib/lp/soyuz/model/publishing.py'
--- lib/lp/soyuz/model/publishing.py 2013-05-31 04:27:26 +0000
+++ lib/lp/soyuz/model/publishing.py 2013-06-18 06:12:17 +0000
@@ -366,6 +366,7 @@
"""Store and format ordered Index Stanza fields."""
def __init__(self):
+ self._names_lower = set()
self.fields = []
def append(self, name, value):
@@ -373,12 +374,16 @@
Then we can use the FIFO-like behaviour in makeOutput().
"""
+ if name.lower() in self._names_lower:
+ return
+ self._names_lower.add(name.lower())
self.fields.append((name, value))
def extend(self, entries):
"""Extend the internal list with the key-value pairs in entries.
"""
- self.fields.extend(entries)
+ for name, value in entries:
+ self.append(name, value)
def makeOutput(self):
"""Return a line-by-line aggregation of appended fields.
@@ -391,8 +396,8 @@
if not value:
continue
- # do not add separation space for the special field 'Files'
- if name != 'Files':
+ # do not add separation space for the special file list fields.
+ if name not in ('Files', 'Checksums-Sha1', 'Checksums-Sha256'):
value = ' %s' % value
# XXX Michael Nelson 20090930 bug=436182. We have an issue
@@ -721,16 +726,23 @@
name = release.sourcepackagename.name
return "%s %s in %s" % (name, release.version, self.distroseries.name)
+ def _formatFileList(self, l):
+ return ''.join('\n %s %s %s' % ((h,) + f) for (h, f) in l)
+
def buildIndexStanzaFields(self):
"""See `IPublishing`."""
# Special fields preparation.
spr = self.sourcepackagerelease
pool_path = makePoolPath(spr.name, self.component.name)
- files_subsection = ''.join(
- ['\n %s %s %s' % (spf.libraryfile.content.md5,
- spf.libraryfile.content.filesize,
- spf.libraryfile.filename)
- for spf in spr.files])
+ files_list = []
+ sha1_list = []
+ sha256_list = []
+ for spf in spr.files:
+ common = (
+ spf.libraryfile.content.filesize, spf.libraryfile.filename)
+ files_list.append((spf.libraryfile.content.md5, common))
+ sha1_list.append((spf.libraryfile.content.sha1, common))
+ sha256_list.append((spf.libraryfile.content.sha256, common))
# Filling stanza options.
fields = IndexStanzaFields()
fields.append('Package', spr.name)
@@ -746,7 +758,10 @@
fields.append('Standards-Version', spr.dsc_standards_version)
fields.append('Format', spr.dsc_format)
fields.append('Directory', pool_path)
- fields.append('Files', files_subsection)
+ fields.append('Files', self._formatFileList(files_list))
+ fields.append('Checksums-Sha1', self._formatFileList(sha1_list))
+ fields.append('Checksums-Sha256', self._formatFileList(sha256_list))
+ fields.append('Homepage', spr.homepage)
if spr.user_defined_fields:
fields.extend(spr.user_defined_fields)
@@ -1013,6 +1028,7 @@
bin_size = bin_file.libraryfile.content.filesize
bin_md5 = bin_file.libraryfile.content.md5
bin_sha1 = bin_file.libraryfile.content.sha1
+ bin_sha256 = bin_file.libraryfile.content.sha256
bin_filepath = os.path.join(
makePoolPath(spr.name, self.component.name), bin_filename)
# description field in index is an association of summary and
@@ -1066,6 +1082,7 @@
fields.append('Size', bin_size)
fields.append('MD5sum', bin_md5)
fields.append('SHA1', bin_sha1)
+ fields.append('SHA256', bin_sha256)
fields.append(
'Phased-Update-Percentage', self.phased_update_percentage)
fields.append('Description', bin_description)
=== modified file 'lib/lp/soyuz/tests/test_publish_archive_indexes.py'
--- lib/lp/soyuz/tests/test_publish_archive_indexes.py 2013-01-21 16:28:14 +0000
+++ lib/lp/soyuz/tests/test_publish_archive_indexes.py 2013-06-18 06:12:17 +0000
@@ -19,6 +19,15 @@
class TestNativeArchiveIndexes(TestNativePublishingBase):
+ deb_md5 = '008409e7feb1c24a6ccab9f6a62d24c5'
+ deb_sha1 = '30b7b4e583fa380772c5a40e428434628faef8cf'
+ deb_sha256 = (
+ '006ca0f356f54b1916c24c282e6fd19961f4356441401f4b0966f2a00bb3e945')
+ dsc_md5 = '5913c3ad52c14a62e6ae7eef51f9ef42'
+ dsc_sha1 = 'e35e29b2ea94bbaa831882e11d1f456690f04e69'
+ dsc_sha256 = (
+ 'ac512102db9724bee18f26945efeeb82fdab89819e64e120fbfda755ca50c2c6')
+
def setUp(self):
"""Setup global attributes."""
TestNativePublishingBase.setUp(self)
@@ -55,17 +64,28 @@
u'Format: 1.0',
u'Directory: pool/main/f/foo',
u'Files:',
- u' 5913c3ad52c14a62e6ae7eef51f9ef42 28 foo_666.dsc'],
+ u' %s 28 foo_666.dsc' % self.dsc_md5,
+ u'Checksums-Sha1:',
+ u' %s 28 foo_666.dsc' % self.dsc_sha1,
+ u'Checksums-Sha256:',
+ u' %s 28 foo_666.dsc' % self.dsc_sha256,
+ ],
pub_source.getIndexStanza().splitlines())
def testSourceStanzaCustomFields(self):
"""Check just-created source publication Index stanza
with custom fields (Python-Version).
+
+ A field is excluded if its key case-insensitively matches one that's
+ already there. This mostly affects sources that were uploaded before
+ Homepage, Checksums-Sha1 or Checksums-Sha256 were excluded.
"""
pub_source = self.getPubSource(
builddepends='fooish', builddependsindep='pyfoo',
build_conflicts='bar', build_conflicts_indep='pybar',
- user_defined_fields=[("Python-Version", "< 1.5")])
+ user_defined_fields=[
+ ("Python-Version", "< 1.5"),
+ ("CHECKSUMS-SHA1", "BLAH")])
self.assertEqual(
[u'Package: foo',
@@ -82,7 +102,11 @@
u'Format: 1.0',
u'Directory: pool/main/f/foo',
u'Files:',
- u' 5913c3ad52c14a62e6ae7eef51f9ef42 28 foo_666.dsc',
+ u' %s 28 foo_666.dsc' % self.dsc_md5,
+ u'Checksums-Sha1:',
+ u' %s 28 foo_666.dsc' % self.dsc_sha1,
+ u'Checksums-Sha256:',
+ u' %s 28 foo_666.dsc' % self.dsc_sha256,
u'Python-Version: < 1.5'],
pub_source.getIndexStanza().splitlines())
@@ -118,8 +142,9 @@
u'Breaks: old-foo',
u'Filename: pool/main/f/foo/foo-bin_666_all.deb',
u'Size: 18',
- u'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
- u'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
+ u'MD5sum: ' + self.deb_md5,
+ u'SHA1: ' + self.deb_sha1,
+ u'SHA256: ' + self.deb_sha256,
u'Phased-Update-Percentage: 50',
u'Description: Foo app is great',
u' Well ...',
@@ -157,8 +182,9 @@
u'Breaks: old-foo',
u'Filename: pool/main/f/foo/foo-bin_666_all.deb',
u'Size: 18',
- u'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
- u'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
+ u'MD5sum: ' + self.deb_md5,
+ u'SHA1: ' + self.deb_sha1,
+ u'SHA256: ' + self.deb_sha256,
u'Description: Foo app is great',
u' Well ...',
u' it does nothing, though',
@@ -203,8 +229,9 @@
u'Version: 666',
u'Filename: pool/main/f/foo/foo-bin_666_all.deb',
u'Size: 18',
- u'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
- u'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
+ u'MD5sum: ' + self.deb_md5,
+ u'SHA1: ' + self.deb_sha1,
+ u'SHA256: ' + self.deb_sha256,
u'Description: Foo app is great',
u' Normal',
u' Normal',
@@ -239,8 +266,9 @@
u'Version: 666',
u'Filename: pool/main/f/foo/foo-bin_666_all.deb',
u'Size: 18',
- u'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
- u'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
+ u'MD5sum: ' + self.deb_md5,
+ u'SHA1: ' + self.deb_sha1,
+ u'SHA256: ' + self.deb_sha256,
u'Description: Foo app is great',
u' Using non-ascii as: \xe7\xe3\xe9\xf3',
],
@@ -424,7 +452,6 @@
fields.append('one', 'um')
fields.append('three', 'tres')
fields.append('two', 'dois')
- fields.append(None, None)
self.assertEqual(
['one: um', 'three: tres', 'two: dois',
