launchpad-reviewers team mailing list archive

[Andy Whitcroft <apw@xxxxxxxxxxxxx>]

Andy Whitcroft has proposed merging lp:~apw/launchpad/signing-add-sha256-checksums into lp:launchpad.

Commit message:
Add Signing custom upload (raw-signing/raw-uefi) result checksumming.  This is the first step in providing a trust chain for the signing custom uploads (Bug #1285919).

Once the Signing Custom upload is unpacked and processed we make a pass over the results producing a SHA256 checksum for each file.  These are accumulated in a SHA256SUMS file which is added to the custom upload result directory.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~apw/launchpad/signing-add-sha256-checksums/+merge/295615

Add Signing custom upload (raw-signing/raw-uefi) result checksumming.  This is the first step in providing a trust chain for the signing custom uploads (Bug #1285919).

Once the Signing Custom upload is unpacked and processed we make a pass over the results producing a SHA256 checksum for each file.  These are accumulated in a SHA256SUMS file which is added to the custom upload result directory.

NOTE: this branch carries a missing options test which we rely on when testing checksumming.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~apw/launchpad/signing-add-sha256-checksums into lp:launchpad.

=== modified file 'lib/lp/archivepublisher/signing.py'
--- lib/lp/archivepublisher/signing.py	2016-05-23 10:18:21 +0000
+++ lib/lp/archivepublisher/signing.py	2016-05-24 17:29:48 +0000
@@ -17,6 +17,7 @@
     "SigningUpload",
     ]
 
+import hashlib
 import os
 import shutil
 import subprocess
@@ -296,6 +297,27 @@
         except OSError as exc:
             raise SigningUploadPackError(tarfilename, exc)
 
+    def checksumSha256(self, datafd):
+        """Calculate the SHA256 checksum for the passed file descriptor."""
+        file_hash = hashlib.sha256()
+        for chunk in iter(lambda: datafd.read(256 * 1024), ""):
+            file_hash.update(chunk)
+        return file_hash.hexdigest()
+
+    def generateChecksums(self):
+        """Generate SHA256 checksums for the custom upload."""
+        versiondir = os.path.join(self.tmpdir, self.version)
+        checksum_file = os.path.join(self.tmpdir, "SHA256SUMS.tmp")
+        prefix_len = len(versiondir) + 1
+        with open(checksum_file, "w") as sfd:
+            for dirpath, dirnames, filenames in os.walk(versiondir):
+                for filename in filenames:
+                    disk_name = os.path.join(dirpath, filename)
+                    with open(disk_name) as dfd:
+                        checksum = self.checksumSha256(dfd)
+                    print(checksum, disk_name[prefix_len:], file=sfd)
+        os.rename(checksum_file, os.path.join(versiondir, "SHA256SUMS"))
+
     def extract(self):
         """Copy the custom upload to a temporary directory, and sign it.
 
@@ -313,5 +335,7 @@
         if 'tarball' in self.signing_options:
             self.convertToTarball()
 
+        self.generateChecksums()
+
     def shouldInstall(self, filename):
         return filename.startswith("%s/" % self.version)

=== modified file 'lib/lp/archivepublisher/tests/test_signing.py'
--- lib/lp/archivepublisher/tests/test_signing.py	2016-05-23 11:59:17 +0000
+++ lib/lp/archivepublisher/tests/test_signing.py	2016-05-24 17:29:48 +0000
@@ -7,6 +7,7 @@
 
 import os
 import tarfile
+from cStringIO import StringIO
 
 from fixtures import MonkeyPatch
 
@@ -253,6 +254,23 @@
         self.assertContentEqual(['first', 'second'],
             upload.signing_options.keys())
 
+    def test_options_none(self):
+        # Specifying no options should leave us with an open tree.
+        self.setUpUefiKeys()
+        self.setUpKmodKeys()
+        self.openArchive("test", "1.0", "amd64")
+        self.archive.add_file("1.0/empty.efi", "")
+        self.archive.add_file("1.0/empty.ko", "")
+        self.process_emulate()
+        self.assertTrue(os.path.exists(os.path.join(
+            self.getSignedPath("test", "amd64"), "1.0", "empty.efi")))
+        self.assertTrue(os.path.exists(os.path.join(
+            self.getSignedPath("test", "amd64"), "1.0", "empty.efi.signed")))
+        self.assertTrue(os.path.exists(os.path.join(
+            self.getSignedPath("test", "amd64"), "1.0", "empty.ko")))
+        self.assertTrue(os.path.exists(os.path.join(
+            self.getSignedPath("test", "amd64"), "1.0", "empty.ko.sig")))
+
     def test_options_tarball(self):
         # Specifying the "tarball" option should create an tarball in
         # the tmpdir.
@@ -606,3 +624,45 @@
         self.assertEqual(1, upload.callLog.caller_count('Kmod keygen cert'))
         self.assertTrue(os.path.exists(self.kmod_pem))
         self.assertTrue(os.path.exists(self.kmod_x509))
+
+    def test_sha256_basic_summing(self):
+        upload = SigningUpload()
+
+        data = StringIO("Hello World!\n")
+        self.assertEqual(
+            '03ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab340',
+            upload.checksumSha256(data))
+
+        data = StringIO("Something somewhat longer?\n" * 100)
+        self.assertEqual(
+            '8bdc49d049287ac937e524afe6c7ba014169b7aff11c78f05467487cc50f6f67',
+            upload.checksumSha256(data))
+
+    def test_sha256_multiblock_summing(self):
+        upload = SigningUpload()
+
+        line = "A" * 1024 + "\n"
+        data = StringIO(line * 300)
+        self.assertEqual(
+            'e4d4650544a18a1dd837ee70c68137b553e3d9455e941ec71465cd5557ef0e20',
+            upload.checksumSha256(data))
+
+    def test_checksumming_tree(self):
+        # Specifying no options should leave us with an open tree,
+        # confirm it is checksummed.
+        self.setUpUefiKeys()
+        self.setUpKmodKeys()
+        self.openArchive("test", "1.0", "amd64")
+        self.archive.add_file("1.0/empty.efi", "")
+        self.archive.add_file("1.0/empty.ko", "")
+        self.process_emulate()
+        sha256file = os.path.join(self.getSignedPath("test", "amd64"),
+             "1.0", "SHA256SUMS")
+        self.assertTrue(os.path.exists(sha256file))
+        H = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+        expected = [[H, 'empty.efi'], [H, 'empty.efi.signed'],
+            [H, 'empty.ko'], [H, 'empty.ko.sig', ],
+            ]
+        with open(sha256file) as sfd:
+            content = [line.split() for line in sfd]
+        self.assertContentEqual(expected, content)