launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging lp:~cjwatson/launchpad-buildd/git-recipes into lp:launchpad-buildd.

Commit message:
Use git-build-recipe to run git-based recipe builds (LP: #1453022).

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)
Related bugs:
  Bug #1453022 in launchpad-buildd: "Please support daily builds via git"
  https://bugs.launchpad.net/launchpad-buildd/+bug/1453022

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/git-recipes/+merge/281680

Use git-build-recipe to run git-based recipe builds.

git-build-recipe is in ppa:cjwatson/launchpad, and will need to be copied into the appropriate PPAs and installed via the launchpad-buildd-image-modifier charm before this is deployed.  It still needs various bits of work, but it works well enough for simple recipes.

The Launchpad webapp will also need to be taught about git-based recipes.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad-buildd/git-recipes into lp:launchpad-buildd.

=== modified file 'buildrecipe'
--- buildrecipe	2015-07-04 14:29:24 +0000
+++ buildrecipe	2016-01-05 17:45:56 +0000
@@ -7,6 +7,7 @@
 __metaclass__ = type
 
 
+from optparse import OptionParser
 import os
 import pwd
 import socket
@@ -47,13 +48,16 @@
     """Builds a package from a recipe."""
 
     def __init__(self, build_id, author_name, author_email,
-                 suite, distroseries_name, component, archive_purpose):
+                 suite, distroseries_name, component, archive_purpose,
+                 git=False):
         """Constructor.
 
         :param build_id: The id of the build (a str).
         :param author_name: The name of the author (a str).
         :param author_email: The email address of the author (a str).
         :param suite: The suite the package should be built for (a str).
+        :param git: If True, build a git-based recipe; if False, build a
+            bzr-based recipe.
         """
         self.build_id = build_id
         self.author_name = author_name.decode('utf-8')
@@ -62,7 +66,7 @@
         self.component = component
         self.distroseries_name = distroseries_name
         self.suite = suite
-        self.base_branch = None
+        self.git = git
         self.chroot_path = get_build_path(build_id, 'chroot-autobuild')
         self.work_dir_relative = os.environ['HOME'] + '/work'
         self.work_dir = os.path.join(self.chroot_path,
@@ -86,7 +90,7 @@
         """Build the recipe into a source tree.
 
         As a side-effect, sets self.source_dir_relative.
-        :return: a retcode from `bzr dailydeb`.
+        :return: a retcode from `bzr dailydeb` or `git-build-recipe`.
         """
         assert not os.path.exists(self.tree_path)
         recipe_path = os.path.join(self.work_dir, 'recipe')
@@ -99,16 +103,20 @@
         # As of bzr 2.2, a defined identity is needed.  In this case, we're
         # using buildd@<hostname>.
         hostname = socket.gethostname()
-        bzr_email = 'buildd@%s' % hostname
+        email = 'buildd@%s' % hostname
         lsb_release = Popen(['/usr/bin/sudo',
             '/usr/sbin/chroot', self.chroot_path, 'lsb_release',
             '-r', '-s'], stdout=PIPE)
         distroseries_version = lsb_release.communicate()[0].rstrip()
         assert lsb_release.returncode == 0
 
-        print 'Bazaar versions:'
-        check_call(['bzr', 'version'])
-        check_call(['bzr', 'plugins'])
+        if self.git:
+            print 'Git version:'
+            check_call(['git', '--version'])
+        else:
+            print 'Bazaar versions:'
+            check_call(['bzr', 'version'])
+            check_call(['bzr', 'plugins'])
 
         print 'Building recipe:'
         print recipe
@@ -116,17 +124,22 @@
         env = {
             'DEBEMAIL': self.author_email,
             'DEBFULLNAME': self.author_name.encode('utf-8'),
-            'BZR_EMAIL': bzr_email,
+            'EMAIL': email,
             'LANG': 'C.UTF-8',
             }
-        retcode = call_report_rusage([
-            'bzr',
-            '-Derror',
-            'dailydeb', '--safe', '--no-build', recipe_path,
-            self.tree_path, '--manifest', manifest_path,
+        if self.git:
+            cmd = ['git-build-recipe']
+        else:
+            cmd = ['bzr', '-Derror', 'dailydeb']
+        cmd.extend([
+            '--safe', '--no-build',
+            '--manifest', manifest_path,
             '--distribution', self.distroseries_name,
-            '--allow-fallback-to-native', '--append-version',
-            '~ubuntu%s.1' % distroseries_version], env=env)
+            '--allow-fallback-to-native',
+            '--append-version', '~ubuntu%s.1' % distroseries_version,
+            recipe_path, self.tree_path,
+            ])
+        retcode = call_report_rusage(cmd, env=env)
         if retcode != 0:
             return retcode
         (source,) = [name for name in os.listdir(self.tree_path)
@@ -295,14 +308,26 @@
         os.environ["HOME"], "build-" + build_id, *extra)
 
 
-if __name__ == '__main__':
-    builder = RecipeBuilder(*sys.argv[1:])
+def main():
+    parser = OptionParser(usage=(
+        "usage: %prog BUILD-ID AUTHOR-NAME AUTHOR-EMAIL SUITE "
+        "DISTROSERIES-NAME COMPONENT ARCHIVE-PURPOSE"))
+    parser.add_option(
+        "--git", default=False, action="store_true",
+        help="build a git recipe (default: bzr)")
+    options, args = parser.parse_args()
+
+    builder = RecipeBuilder(*args, git=options.git)
     if builder.install() != 0:
-        sys.exit(RETCODE_FAILURE_INSTALL)
+        return RETCODE_FAILURE_INSTALL
     if builder.buildTree() != 0:
-        sys.exit(RETCODE_FAILURE_BUILD_TREE)
+        return RETCODE_FAILURE_BUILD_TREE
     if builder.installBuildDeps() != 0:
-        sys.exit(RETCODE_FAILURE_INSTALL_BUILD_DEPS)
+        return RETCODE_FAILURE_INSTALL_BUILD_DEPS
     if builder.buildSourcePackage() != 0:
-        sys.exit(RETCODE_FAILURE_BUILD_SOURCE_PACKAGE)
-    sys.exit(RETCODE_SUCCESS)
+        return RETCODE_FAILURE_BUILD_SOURCE_PACKAGE
+    return RETCODE_SUCCESS
+
+
+if __name__ == '__main__':
+    sys.exit(main())

=== modified file 'debian/changelog'
--- debian/changelog	2015-11-25 15:00:43 +0000
+++ debian/changelog	2016-01-05 17:45:56 +0000
@@ -6,6 +6,8 @@
     that now.
   * Configure systemd-timesyncd to use the NTP server configured in
     /etc/launchpad-buildd/default.
+  * buildrecipe: Add option parsing framework.
+  * Use git-build-recipe to run git-based recipe builds (LP: #1453022).
 
  -- Colin Watson <cjwatson@xxxxxxxxxx>  Mon, 16 Nov 2015 18:12:29 +0000
 

=== modified file 'lpbuildd/sourcepackagerecipe.py'
--- lpbuildd/sourcepackagerecipe.py	2015-05-11 05:39:25 +0000
+++ lpbuildd/sourcepackagerecipe.py	2016-01-05 17:45:56 +0000
@@ -76,6 +76,7 @@
         self.author_email = extra_args['author_email']
         self.archive_purpose = extra_args['archive_purpose']
         self.distroseries_name = extra_args['distroseries_name']
+        self.git = extra_args.get('git', False)
 
         super(SourcePackageRecipeBuildManager, self).initiate(
             files, chroot, extra_args)
@@ -85,10 +86,13 @@
         os.makedirs(get_chroot_path(self.home, self._buildid, 'work'))
         recipe_path = get_chroot_path(self.home, self._buildid, 'work/recipe')
         splat_file(recipe_path, self.recipe_text)
-        args = [
-            "buildrecipe", self._buildid, self.author_name.encode('utf-8'),
+        args = ["buildrecipe"]
+        if self.git:
+            args.append("--git")
+        args.extend([
+            self._buildid, self.author_name.encode('utf-8'),
             self.author_email, self.suite, self.distroseries_name,
-            self.component, self.archive_purpose]
+            self.component, self.archive_purpose])
         self.runSubProcess(self.build_recipe_path, args)
 
     def iterate_BUILD_RECIPE(self, retcode):

=== modified file 'lpbuildd/tests/test_sourcepackagerecipe.py'
--- lpbuildd/tests/test_sourcepackagerecipe.py	2015-05-11 05:55:24 +0000
+++ lpbuildd/tests/test_sourcepackagerecipe.py	2016-01-05 17:45:56 +0000
@@ -54,7 +54,7 @@
         """Retrieve build manager's state."""
         return self.buildmanager._state
 
-    def startBuild(self):
+    def startBuild(self, git=False):
         # The build manager's iterate() kicks off the consecutive states
         # after INIT.
         extra_args = {
@@ -73,6 +73,8 @@
                     'ubuntu main',
                 ],
             }
+        if git:
+            extra_args['git'] = True
         self.buildmanager.initiate({}, 'chroot.tar.gz', extra_args)
 
         # Skip states that are done in DebianBuildManager to the state
@@ -84,10 +86,14 @@
         self.assertEqual(
             SourcePackageRecipeBuildState.BUILD_RECIPE, self.getState())
         expected_command = [
-            'sharepath/slavebin/buildrecipe', 'buildrecipe', self.buildid,
+            'sharepath/slavebin/buildrecipe', 'buildrecipe']
+        if git:
+            expected_command.append('--git')
+        expected_command.extend([
+            self.buildid,
             'Steve\u1234'.encode('utf-8'), 'stevea@xxxxxxxxxxx',
             'maverick', 'maverick', 'universe', 'puppies',
-            ]
+            ])
         self.assertEqual(expected_command, self.buildmanager.commands[-1])
         self.assertEqual(
             self.buildmanager.iterate, self.buildmanager.iterators[-1])
@@ -213,3 +219,9 @@
         self.assertEqual(expected_command, self.buildmanager.commands[-1])
         self.assertEqual(
             self.buildmanager.iterate, self.buildmanager.iterators[-1])
+
+    def test_iterate_git(self):
+        # Starting a git-based recipe build passes the correct option.  (The
+        # rest of the build is identical to bzr-based recipe builds from the
+        # build manager's point of view.)
+        self.startBuild(git=True)