launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging lp:~cjwatson/launchpad/snap-uploader into lp:launchpad with lp:~cjwatson/launchpad/snap-build-behaviour as a prerequisite.

Commit message:
Add build upload processing for snap packages.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)
Related bugs:
  Bug #1476405 in Launchpad itself: "Add support for building snaps"
  https://bugs.launchpad.net/launchpad/+bug/1476405

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/snap-uploader/+merge/266739

Add build upload processing for snap packages.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/snap-uploader into lp:launchpad.

=== added file 'lib/lp/archiveuploader/snapupload.py'
--- lib/lp/archiveuploader/snapupload.py	1970-01-01 00:00:00 +0000
+++ lib/lp/archiveuploader/snapupload.py	2015-08-03 15:10:39 +0000
@@ -0,0 +1,62 @@
+# Copyright 2015 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Process a snap package upload."""
+
+__metaclass__ = type
+
+import os
+
+from zope.component import getUtility
+
+from lp.archiveuploader.utils import UploadError
+from lp.buildmaster.enums import BuildStatus
+from lp.services.helpers import filenameToContentType
+from lp.services.librarian.interfaces import ILibraryFileAliasSet
+
+
+class SnapUpload:
+    """A snap package upload.
+
+    Unlike package uploads, these have no .changes files.  We simply attach
+    all the files in the upload directory to the appropriate `ISnapBuild`.
+    """
+
+    def __init__(self, upload_path, logger):
+        """Create a `SnapUpload`.
+
+        :param upload_path: A directory containing files to upload.
+        :param logger: The logger to be used.
+        """
+        self.upload_path = upload_path
+        self.logger = logger
+
+        self.librarian = getUtility(ILibraryFileAliasSet)
+
+    def process(self, build):
+        """Process this upload, loading it into the database."""
+        self.logger.debug("Beginning processing.")
+
+        found_snap = False
+        for dirpath, _, filenames in os.walk(self.upload_path):
+            if dirpath == self.upload_path:
+                # All relevant files will be in a subdirectory.
+                continue
+            for snap_file in sorted(filenames):
+                snap_path = os.path.join(dirpath, snap_file)
+                libraryfile = self.librarian.create(
+                    snap_file, os.stat(snap_path).st_size,
+                    open(snap_path, "rb"),
+                    filenameToContentType(snap_path),
+                    restricted=build.is_private)
+                found_snap = True
+                build.addFile(libraryfile)
+
+        if not found_snap:
+            raise UploadError("Build did not produce any snap packages.")
+
+        # The master verifies the status to confirm successful upload.
+        self.logger.debug("Updating %s" % build.title)
+        build.updateStatus(BuildStatus.FULLYBUILT)
+
+        self.logger.debug("Finished upload.")

=== added file 'lib/lp/archiveuploader/tests/test_snapupload.py'
--- lib/lp/archiveuploader/tests/test_snapupload.py	1970-01-01 00:00:00 +0000
+++ lib/lp/archiveuploader/tests/test_snapupload.py	2015-08-03 15:10:39 +0000
@@ -0,0 +1,66 @@
+# Copyright 2015 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test uploads of SnapBuilds."""
+
+__metaclass__ = type
+
+import os
+
+from storm.store import Store
+from zope.component import getUtility
+
+from lp.archiveuploader.tests.test_uploadprocessor import (
+    TestUploadProcessorBase,
+    )
+from lp.archiveuploader.uploadprocessor import (
+    UploadHandler,
+    UploadStatusEnum,
+    )
+from lp.buildmaster.enums import BuildStatus
+from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.services.features.testing import FeatureFixture
+from lp.services.osutils import write_file
+from lp.snappy.interfaces.snap import SNAP_FEATURE_FLAG
+from lp.snappy.interfaces.snapbuild import ISnapBuildSet
+
+
+class TestSnapBuildUploads(TestUploadProcessorBase):
+    """End-to-end tests of Snap build uploads."""
+
+    def setUp(self):
+        super(TestSnapBuildUploads, self).setUp()
+
+        self.useFixture(FeatureFixture({SNAP_FEATURE_FLAG: u"on"}))
+        self.setupBreezy()
+
+        self.switchToAdmin()
+        self.snap = self.factory.makeSnap()
+        self.build = getUtility(ISnapBuildSet).new(
+            requester=self.snap.owner, snap=self.snap,
+            archive=self.factory.makeArchive(
+                distribution=self.ubuntu, owner=self.snap.owner),
+            distro_arch_series=self.breezy["i386"],
+            pocket=PackagePublishingPocket.RELEASE)
+        self.build.updateStatus(BuildStatus.UPLOADING)
+        Store.of(self.build).flush()
+        self.switchToUploader()
+        self.options.context = "buildd"
+
+        self.uploadprocessor = self.getUploadProcessor(
+            self.layer.txn, builds=True)
+
+    def test_sets_build_and_state(self):
+        # The upload processor uploads files and sets the correct status.
+        self.assertFalse(self.build.verifySuccessfulUpload())
+        upload_dir = os.path.join(
+            self.incoming_folder, "test", str(self.build.id), "ubuntu")
+        write_file(os.path.join(upload_dir, "wget_0_all.snap"), "snap")
+        handler = UploadHandler.forProcessor(
+            self.uploadprocessor, self.incoming_folder, "test", self.build)
+        result = handler.processSnap(self.log)
+        self.assertEqual(
+            UploadStatusEnum.ACCEPTED, result,
+            "Snap upload failed\nGot: %s" % self.log.getLogBuffer())
+        self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
+        self.assertTrue(self.build.verifySuccessfulUpload())

=== modified file 'lib/lp/archiveuploader/uploadprocessor.py'
--- lib/lp/archiveuploader/uploadprocessor.py	2015-02-17 11:26:14 +0000
+++ lib/lp/archiveuploader/uploadprocessor.py	2015-08-03 15:10:39 +0000
@@ -60,6 +60,7 @@
     EarlyReturnUploadError,
     NascentUpload,
     )
+from lp.archiveuploader.snapupload import SnapUpload
 from lp.archiveuploader.uploadpolicy import (
     BuildDaemonUploadPolicy,
     UploadPolicyError,
@@ -77,6 +78,7 @@
     ErrorReportingUtility,
     ScriptRequest,
     )
+from lp.snappy.interfaces.snapbuild import ISnapBuild
 from lp.soyuz.interfaces.archive import (
     IArchiveSet,
     NoSuchPPA,
@@ -611,6 +613,31 @@
             self.processor.ztm.abort()
             raise
 
+    def processSnap(self, logger=None):
+        """Process a snap package upload."""
+        assert ISnapBuild.providedBy(self.build)
+        if logger is None:
+            logger = self.processor.log
+        try:
+            logger.info("Processing Snap upload %s" % self.upload_path)
+            SnapUpload(self.upload_path, logger).process(self.build)
+
+            if self.processor.dry_run:
+                logger.info("Dry run, aborting transaction.")
+                self.processor.ztm.abort()
+            else:
+                logger.info(
+                    "Committing the transaction and any mails associated "
+                    "with this upload.")
+                self.processor.ztm.commit()
+            return UploadStatusEnum.ACCEPTED
+        except UploadError as e:
+            logger.error(str(e))
+            return UploadStatusEnum.REJECTED
+        except:
+            self.processor.ztm.abort()
+            raise
+
     def process(self):
         """Process an upload that is the result of a build.
 
@@ -641,11 +668,12 @@
             # because we want the standard cleanup to occur.
             recipe_deleted = (ISourcePackageRecipeBuild.providedBy(self.build)
                 and self.build.recipe is None)
-            is_livefs = ILiveFSBuild.providedBy(self.build)
             if recipe_deleted:
                 result = UploadStatusEnum.FAILED
-            elif is_livefs:
+            elif ILiveFSBuild.providedBy(self.build):
                 result = self.processLiveFS(logger)
+            elif ISnapBuild.providedBy(self.build):
+                result = self.processSnap(logger)
             else:
                 self.processor.log.debug("Build %s found" % self.build.id)
                 [changes_file] = self.locateChangesFiles()