launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #00426
[Merge] lp:~wgrant/launchpad/multi-arch-builders into lp:launchpad/devel
William Grant has proposed merging lp:~wgrant/launchpad/multi-arch-builders into lp:launchpad/devel.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
#491418 build architecture should not be hard-coded at ppa install time
https://bugs.launchpad.net/bugs/491418
This branch mostly fixes bug #491418, adjusting launchpad-buildd to take an optional arch_tag extra argument, which is then used when dealing with the chroot and performing the actual build. This lets buildd-manager decide which architecture it needs a slave to be for a particular build.
The master support for this is incomplete: in this branch, builders must have their architecture tweaked manually through the UI. The master's slave architecture check has also been removed, which means that an incorrect architecture setting could result in chroot setup failures rather than the current builder failure.
Eventually the master will be able to dynamically assign builds of different architectures to the same builder, but that requires more work.
--
https://code.launchpad.net/~wgrant/launchpad/multi-arch-builders/+merge/31740
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/multi-arch-builders into lp:launchpad/devel.
=== modified file 'lib/canonical/buildd/binarypackage.py'
--- lib/canonical/buildd/binarypackage.py 2010-06-12 03:48:34 +0000
+++ lib/canonical/buildd/binarypackage.py 2010-08-04 11:43:45 +0000
@@ -67,7 +67,7 @@
def doRunBuild(self):
"""Run the sbuild process to build the package."""
- args = ["sbuild-package", self._buildid ]
+ args = ["sbuild-package", self._buildid, self.arch_tag]
if self.suite:
args.extend([self.suite])
args.extend(self._sbuildargs)
@@ -82,6 +82,7 @@
args.extend(["--purpose=" + self.archive_purpose])
if self.build_debug_symbols:
args.extend(["--build-debug-symbols"])
+ args.extend(["--architecture=" + self.arch_tag])
args.extend(["--comp=" + self.component])
args.extend([self._dscfile])
self.runSubProcess( self._sbuildpath, args )
=== modified file 'lib/canonical/buildd/debian.py'
--- lib/canonical/buildd/debian.py 2010-06-12 03:48:34 +0000
+++ lib/canonical/buildd/debian.py 2010-08-04 11:43:45 +0000
@@ -47,6 +47,7 @@
def initiate(self, files, chroot, extra_args):
"""Initiate a build with a given set of files and chroot."""
+ self.arch_tag = extra_args.get('arch_tag', self._slave.getArch())
self.sources_list = extra_args.get('archives')
BuildManager.initiate(self, files, chroot, extra_args)
@@ -62,8 +63,9 @@
def doUpdateChroot(self):
"""Perform the chroot upgrade."""
- self.runSubProcess(self._updatepath,
- ["update-debian-chroot", self._buildid])
+ self.runSubProcess(
+ self._updatepath,
+ ["update-debian-chroot", self._buildid, self.arch_tag])
def doRunBuild(self):
"""Run the main build process.
@@ -95,8 +97,7 @@
yield filename
def getChangesFilename(self):
- changes = (
- self._dscfile[:-4] + "_" + self._slave.getArch() + ".changes")
+ changes = self._dscfile[:-4] + "_" + self.arch_tag + ".changes"
return get_build_path(self._buildid, changes)
def gatherResults(self):
=== modified file 'lib/canonical/buildd/debian/changelog'
--- lib/canonical/buildd/debian/changelog 2010-07-23 20:23:04 +0000
+++ lib/canonical/buildd/debian/changelog 2010-08-04 11:43:45 +0000
@@ -1,3 +1,10 @@
+launchpad-buildd (68) UNRELEASED; urgency=low
+
+ * Take an 'arch_tag' argument, so the master can override the slave
+ architecture.
+
+ -- William Grant <wgrant@xxxxxxxxxxxxxxxxxxx> Sun, 01 Aug 2010 22:00:32 +1000
+
launchpad-buildd (67) hardy-cat; urgency=low
* Force aptitude installation for recipe builds on maverick
=== modified file 'lib/canonical/buildd/sbuild'
--- lib/canonical/buildd/sbuild 2010-07-19 18:14:54 +0000
+++ lib/canonical/buildd/sbuild 2010-08-04 11:43:45 +0000
@@ -564,6 +564,9 @@
$main::build_source++;
$conf::purge_build_directory = "never";
}
+ elsif (/^--architecture=(.)/) {
+ $conf::arch=$1.$';
+ }
elsif (/^--archive=(.)/) {
$main::archive=$1.$';
}
=== modified file 'lib/canonical/buildd/sbuild-package'
--- lib/canonical/buildd/sbuild-package 2010-01-11 23:03:50 +0000
+++ lib/canonical/buildd/sbuild-package 2010-08-04 11:43:45 +0000
@@ -20,8 +20,9 @@
SBUILD=/usr/bin/sbuild
BUILDID=$1
SUITE=$2
+ARCHITECTURETAG=$3
-shift; shift
+shift 3
export NR_PROCESSORS=$(grep ^processor /proc/cpuinfo | wc -l)
@@ -34,11 +35,6 @@
cd "$HOME/build-$BUILDID"
hostarch=$(dpkg --print-architecture)
-architecturetag=$hostarch
-
-if [ -f /etc/launchpad-buildd/default ]; then
- eval `grep architecturetag /etc/launchpad-buildd/default | sed 's/ //g'`
-fi
WARN=""
case $hostarch in
@@ -51,7 +47,7 @@
WARN="--warnonly"
;;
amd64)
- if [ "$hostarch" != "$architecturetag" ]; then
+ if [ "$hostarch" != "$ARCHITECTURETAG" ]; then
SBUILD="linux32 $SBUILD"
WARN="--warnonly"
fi
=== modified file 'lib/canonical/buildd/sbuildrc'
--- lib/canonical/buildd/sbuildrc 2009-12-15 00:19:56 +0000
+++ lib/canonical/buildd/sbuildrc 2010-08-04 11:43:45 +0000
@@ -9,7 +9,7 @@
$arch="@ARCHTAG@";
# Maintainer name to use in .changes files (mandatory, no default!)
-$maintainer_name="Ubuntu/$arch Build Daemon <buildd\@@FQDN@>";
+$maintainer_name="Ubuntu Build Daemon <buildd\@@FQDN@>";
# When to purge the build directory afterwards; possible values are
# "never", "successful", and "always"
=== modified file 'lib/canonical/buildd/tests/test_translationtemplatesbuildmanager.py'
--- lib/canonical/buildd/tests/test_translationtemplatesbuildmanager.py 2010-04-01 09:31:13 +0000
+++ lib/canonical/buildd/tests/test_translationtemplatesbuildmanager.py 2010-08-04 11:43:45 +0000
@@ -43,6 +43,9 @@
def wasCalled(self, name):
return name in self._was_called
+ def getArch(self):
+ return 'i386'
+
addWaitingFile = FakeMethod()
=== modified file 'lib/canonical/buildd/update-debian-chroot'
--- lib/canonical/buildd/update-debian-chroot 2009-12-15 18:42:11 +0000
+++ lib/canonical/buildd/update-debian-chroot 2010-08-04 11:43:45 +0000
@@ -15,6 +15,7 @@
CHROOT=/usr/sbin/chroot
APTGET=/usr/bin/apt-get
BUILDID="$1"
+ARCHITECTURETAG="$2"
ROOT=$HOME/build-$BUILDID/chroot-autobuild
OPTIONS="-o DPkg::Options::=--force-confold"
@@ -25,18 +26,13 @@
echo "Updating debian chroot for build $BUILDID"
hostarch=$(dpkg --print-architecture)
-architecturetag=$hostarch
-
-if [ -f /etc/launchpad-buildd/default ]; then
- eval `grep architecturetag /etc/launchpad-buildd/default | sed 's/ //g'`
-fi
case $hostarch in
hppa|powerpc|sparc)
CHROOT="linux32 $CHROOT"
;;
amd64)
- if [ "$hostarch" != "$architecturetag" ]; then
+ if [ "$hostarch" != "$ARCHITECTURETAG" ]; then
CHROOT="linux32 $CHROOT"
fi
;;
=== modified file 'lib/lp/buildmaster/model/builder.py'
--- lib/lp/buildmaster/model/builder.py 2010-08-03 14:48:22 +0000
+++ lib/lp/buildmaster/model/builder.py 2010-08-04 11:43:45 +0000
@@ -25,7 +25,6 @@
from sqlobject import (
BoolCol, ForeignKey, IntCol, SQLObjectNotFound, StringCol)
from storm.expr import Coalesce, Count, Sum
-from storm.store import Store
from zope.component import getUtility
from zope.interface import implements
@@ -42,8 +41,7 @@
from lp.app.errors import NotFoundError
from lp.buildmaster.interfaces.builder import (
BuildDaemonError, BuildSlaveFailure, CannotBuild, CannotFetchFile,
- CannotResumeHost, CorruptBuildCookie, IBuilder, IBuilderSet,
- ProtocolVersionMismatch)
+ CannotResumeHost, CorruptBuildCookie, IBuilder, IBuilderSet)
from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSet
from lp.buildmaster.interfaces.buildfarmjobbehavior import (
BuildBehaviorMismatch)
@@ -204,7 +202,6 @@
try:
builder.checkSlaveAlive()
- builder.checkSlaveArchitecture()
builder.rescueIfLost(logger)
# Catch only known exceptions.
# XXX cprov 2007-06-15 bug=120571: ValueError & TypeError catching is
@@ -284,38 +281,6 @@
current_build_behavior = property(
_getCurrentBuildBehavior, _setCurrentBuildBehavior)
- def checkSlaveArchitecture(self):
- """See `IBuilder`."""
- # XXX cprov 2007-06-15 bug=545839:
- # This function currently depends on the operating system specific
- # details of the build slave to return a processor-family-name (the
- # architecturetag) which matches the distro_arch_series. In reality,
- # we should be checking the processor itself (e.g. amd64) as that is
- # what the distro policy is set from, the architecture tag is both
- # distro specific and potentially different for radically different
- # distributions - its not the right thing to be comparing.
-
- from lp.soyuz.model.distroarchseries import DistroArchSeries
-
- # query the slave for its active details.
- # XXX cprov 2007-06-15: Why is 'mechanisms' ignored?
- builder_vers, builder_arch, mechanisms = self.slave.info()
- # we can only understand one version of slave today:
- if builder_vers != '1.0':
- raise ProtocolVersionMismatch("Protocol version mismatch")
-
- # Find a distroarchseries with the returned arch tag.
- # This is ugly, sick and wrong, but so is the whole concept. See the
- # XXX above and its bug for details.
- das = Store.of(self).find(
- DistroArchSeries, architecturetag=builder_arch,
- processorfamily=self.processor.family).any()
-
- if das is None:
- raise BuildDaemonError(
- "Bad slave architecture tag: %s (registered family: %s)" %
- (builder_arch, self.processor.family.name))
-
def checkSlaveAlive(self):
"""See IBuilder."""
if self.slave.echo("Test")[0] != "Test":
=== modified file 'lib/lp/buildmaster/tests/test_manager.py'
--- lib/lp/buildmaster/tests/test_manager.py 2010-07-28 09:56:24 +0000
+++ lib/lp/buildmaster/tests/test_manager.py 2010-08-04 11:43:45 +0000
@@ -644,6 +644,7 @@
{'alsa-utils_1.0.9a-4ubuntu1.dsc':
'4e3961baf4f56fdbc95d0dd47f3c5bc275da8a33'},
{'arch_indep': True,
+ 'arch_tag': 'i386',
'archive_private': False,
'archive_purpose': 'PRIMARY',
'archives':
=== modified file 'lib/lp/code/model/recipebuilder.py'
--- lib/lp/code/model/recipebuilder.py 2010-05-27 17:55:16 +0000
+++ lib/lp/code/model/recipebuilder.py 2010-08-04 11:43:45 +0000
@@ -62,6 +62,7 @@
if self.build.pocket != PackagePublishingPocket.RELEASE:
suite += "-%s" % (self.build.pocket.name.lower())
args['suite'] = suite
+ args['arch_tag'] = distroarchseries.architecturetag
args["author_name"] = self.build.requester.displayname
args["author_email"] = self.build.requester.preferredemail.email
args["recipe_text"] = str(self.build.recipe.builder_recipe)
=== modified file 'lib/lp/code/model/tests/test_recipebuilder.py'
--- lib/lp/code/model/tests/test_recipebuilder.py 2010-06-11 04:45:18 +0000
+++ lib/lp/code/model/tests/test_recipebuilder.py 2010-08-04 11:43:45 +0000
@@ -140,6 +140,7 @@
expected_archives.append(
"deb http://foo %s main" % job.build.distroseries.name)
self.assertEqual({
+ 'arch_tag': 'i386',
'author_email': u'requester@xxxxxxxxxx',
'suite': u'mydistro',
'author_name': u'Joe User',
@@ -169,6 +170,7 @@
job.build, distroarchseries, None)
logger = BufferLogger()
self.assertEqual({
+ 'arch_tag': 'i386',
'author_email': u'requester@xxxxxxxxxx',
'suite': u'mydistro',
'author_name': u'Joe User',
=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehavior.py'
--- lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-05-20 13:24:01 +0000
+++ lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-08-04 11:43:45 +0000
@@ -192,6 +192,8 @@
suite += "-%s" % (build.pocket.name.lower())
args['suite'] = suite
+ args['arch_tag'] = build.distro_arch_series.architecturetag
+
archive_purpose = build.archive.purpose
if (archive_purpose == ArchivePurpose.PPA and
not build.archive.require_virtualized):
=== modified file 'lib/lp/translations/model/translationtemplatesbuildbehavior.py'
--- lib/lp/translations/model/translationtemplatesbuildbehavior.py 2010-05-24 10:14:41 +0000
+++ lib/lp/translations/model/translationtemplatesbuildbehavior.py 2010-08-04 11:43:45 +0000
@@ -44,15 +44,20 @@
self._builder.slave.cacheFile(logger, chroot)
cookie = self.buildfarmjob.generateSlaveBuildCookie()
- args = self.buildfarmjob.metadata
+ args = {'arch_tag': self._getDistroArchSeries().architecturetag}
+ args.update(self.buildfarmjob.metadata)
+
filemap = {}
self._builder.slave.build(
cookie, self.build_type, chroot_sha1, filemap, args)
def _getChroot(self):
+ return self._getDistroArchSeries().getChroot()
+
+ def _getDistroArchSeries(self):
ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
- return ubuntu.currentseries.nominatedarchindep.getChroot()
+ return ubuntu.currentseries.nominatedarchindep
def logStartBuild(self, logger):
"""See `IBuildFarmJobBehavior`."""
