launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #01149
[Merge] lp:~jml/launchpad/buildd-slavescanner-bustage into lp:launchpad/devel
Jonathan Lange has proposed merging lp:~jml/launchpad/buildd-slavescanner-bustage into lp:launchpad/devel.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
This branch takes the tests in buildd-slavescanner.txt that were exercising the BinaryPackageBuildBehavior class and turns them into more-focused Python tests.
Some of the tests were moved to archive-dependencies.txt, since they were really testing something at a much lower level.
To do this we had to update the mocks that we use and tweak the factory a little.
There are a few incidental cleanups & comments in the actual code, but nothing that puts the massive test refactoring at risk.
--
https://code.launchpad.net/~jml/launchpad/buildd-slavescanner-bustage/+merge/36187
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~jml/launchpad/buildd-slavescanner-bustage into lp:launchpad/devel.
=== modified file 'lib/lp/buildmaster/model/builder.py'
--- lib/lp/buildmaster/model/builder.py 2010-09-21 09:05:34 +0000
+++ lib/lp/buildmaster/model/builder.py 2010-09-21 18:41:18 +0000
@@ -115,6 +115,11 @@
class BuilderSlave(object):
"""Add in a few useful methods for the XMLRPC slave."""
+ # WARNING: If you change the API for this, you should also change the APIs
+ # of the mocks in soyuzbuilderhelpers to match. Otherwise, you will have
+ # many false positives in your test run and will most likely break
+ # production.
+
# XXX: This (BuilderSlave) should use composition, rather than
# inheritance.
=== modified file 'lib/lp/soyuz/doc/archive-dependencies.txt'
--- lib/lp/soyuz/doc/archive-dependencies.txt 2010-08-24 15:29:01 +0000
+++ lib/lp/soyuz/doc/archive-dependencies.txt 2010-09-21 18:41:18 +0000
@@ -250,18 +250,28 @@
deb http://ftpmaster.internal/ubuntu hoary-updates
main restricted universe multiverse
-Similarly, populated PPA dependencies are listed in the building
+Similarly, unpopulated PPA dependencies are *not* listed in the building
'sources_list'.
>>> mark = getUtility(IPersonSet).getByName('mark')
+ >>> archive_dependency = cprov.archive.addArchiveDependency(
+ ... mark.archive, PackagePublishingPocket.RELEASE,
+ ... getUtility(IComponentSet)['main'])
+ >>> print_building_sources_list(a_build)
+ deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
+ deb http://ftpmaster.internal/ubuntu hoary
+ main restricted universe multiverse
+ deb http://ftpmaster.internal/ubuntu hoary-security
+ main restricted universe multiverse
+ deb http://ftpmaster.internal/ubuntu hoary-updates
+ main restricted universe multiverse
+
+Similarly, but *populated* PPA dependencies *are* listed in the building
+'sources_list'.
+
>>> pub_binaries = test_publisher.getPubBinaries(
... binaryname='dep-bin', archive=mark.archive,
... status=PackagePublishingStatus.PUBLISHED)
-
- >>> archive_dependency = cprov.archive.addArchiveDependency(
- ... mark.archive, PackagePublishingPocket.RELEASE,
- ... getUtility(IComponentSet)['main'])
-
>>> print_building_sources_list(a_build)
deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
deb http://ppa.launchpad.dev/mark/ppa/ubuntu hoary main
=== modified file 'lib/lp/soyuz/doc/buildd-slave.txt'
--- lib/lp/soyuz/doc/buildd-slave.txt 2010-04-30 10:00:34 +0000
+++ lib/lp/soyuz/doc/buildd-slave.txt 2010-09-21 18:41:18 +0000
@@ -13,43 +13,6 @@
>>> from canonical.buildd.tests import BuilddSlaveTestSetup
>>> BuilddSlaveTestSetup().setUp()
-Use simple xmlrpclib client to certify the BuildSlave is running
-
- >>> import xmlrpclib
- >>> slave = xmlrpclib.Server('http://localhost:8221/rpc/')
- >>> slave.echo('Hello World')
- ['Hello World']
-
-With slave protocol v1.0new, the only way to get files to the slave is to
-put them in the librarian first...
-
- >>> from canonical.librarian.client import LibrarianClient
- >>> from StringIO import StringIO
- >>> from canonical.launchpad.database import LibraryFileAlias
- >>> import transaction
- >>> lc = LibrarianClient()
- >>> helloworld = "Hello World"
- >>> hw_sio = StringIO(helloworld)
- >>> alias = lc.addFile("HelloWorld.txt", len(helloworld),
- ... hw_sio, "text/plain")
- >>> transaction.commit()
- >>> lf = LibraryFileAlias.get(alias)
- >>> present, info = slave.ensurepresent(
- ... lf.content.sha1, lf.http_url, "", "")
- >>> present, info
- (True, 'Download')
-
-As of slave protocol v1.0new, /filecache/SHA1SUM is *THE* way
-to retrieve files from the slave. Verify it works...
-
- >>> from urllib2 import urlopen
- >>> f = urlopen("http://localhost:8221/filecache/"; + lf.content.sha1)
- >>> hw_str = f.read()
- >>> f.close()
- >>> hw_str == helloworld
- True
-
-
== BuilderSet polling operations ==
>>> import logging
@@ -90,6 +53,7 @@
At this point the buildd-slave is not accessible anymore.
+ >>> import xmlrpclib
>>> s = xmlrpclib.Server('http://localhost:8221/rpc/')
>>> s.info()
Traceback (most recent call last):
=== modified file 'lib/lp/soyuz/doc/buildd-slavescanner.txt'
--- lib/lp/soyuz/doc/buildd-slavescanner.txt 2010-09-16 14:36:47 +0000
+++ lib/lp/soyuz/doc/buildd-slavescanner.txt 2010-09-21 18:41:18 +0000
@@ -112,12 +112,10 @@
Make sure that a_builder has no active builds:
- >>> from canonical.launchpad.ftests import syncUpdate
>>> if a_builder.currentjob is not None:
... currentjob = a_builder.currentjob
... currentjob.setDateStarted(None)
... currentjob.builder = None
- ... syncUpdate(currentjob)
Force the test builder to be 'ok' as the code required to do this
automatically is not yet factored into the content class.
@@ -595,11 +593,9 @@
>>> resurrect_build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(
... current_job)
>>> resurrect_build.status = BuildStatus.NEEDSBUILD
- >>> syncUpdate(resurrect_build)
>>> current_job.builder = None
>>> current_job.setDateStarted(None)
>>> current_job.lastscore = 0
- >>> syncUpdate(current_job)
IBuilder.findCandidate also identifies if there are builds for
superseded source package releases in the queue and marks the
@@ -740,66 +736,6 @@
>>> commit()
>>> LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser)
-For building a candidate in the release pocket for the main component
-and the primary archive It will pass an 'archives' argument to the
-slave that contains sources.list entries for each pocket required in
-the primary archive dependency tree.
-
-We also pass arguments called 'suite' which is the current distroseries and
-pocket, (e.g. edgy-updates) and 'archive_purpose' which contains the build's
-archive.purpose (e.g. PRIMARY or PPA). These latter two arguments are
-used in the chroot to determine whether it needs to turn on some features
-or not (like pkgstriptranslations and pkgmaintainermangler).
-
-Please note also that the 'archive_private' flag is passed to the slave
-builder. It is True for private archives and False otherwise.
-
- >>> a_builder.setSlaveForTesting(OkSlave())
- >>> a_builder.is_available
- True
- >>> candidate = a_build.queueBuild()
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary main
- Suite: hoary
- Ogre-component: main
- Archive Purpose: PRIMARY
- Archive Private: False
-
- >>> candidate.destroySelf()
-
-Currently we can theoretically dispatch a build candidate for a
-builder in 'manual' mode.
-
-Although this will not be optimal, because we can only
-do it once the manual builder has been collected (due to the
-BuildQueue.builder constraint). Also because we don't yet provide a
-API/UI method to request the dispatch in advance.
-
- >>> a_builder.manual = True
- >>> commit()
- >>> a_builder.setSlaveForTesting(OkSlave())
- >>> a_builder.is_available
- True
- >>> candidate = a_build.queueBuild()
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary main
- Suite: hoary
- Ogre-component: main
- Archive Purpose: PRIMARY
- Archive Private: False
-
- >>> candidate.destroySelf()
-
Partner archive builds will set up the 'archives' argument such that it
references all the required pockets/components in the primary archive, in
addition to a reference to the release pocket in the partner archive itself.
@@ -812,10 +748,6 @@
>>> a_builder.is_available
True
- >>> candidate = a_build.queueBuild()
- >>> setupBuildQueue(candidate, a_builder)
- >>> last_stub_mail_count = len(stub.test_emails)
-
The partner archive won't be passed to the builder unless it has at
least one published binary availble in the target distroarchseries.
This feature fixes bug #196782, when archive/suites got passed to
@@ -823,229 +755,12 @@
any PPA/suite will fail during the first 20 minutes because no empty
indexes are published.
-Note that only a published binary in the right context will make the
-archive relevant, anything PENDING or published in another context
-wouldn't work.
-
- >>> warty = getUtility(IDistributionSet)['ubuntu']['warty']
- >>> create_binary_publication_for(
- ... partner_archive, warty, PackagePublishingStatus.PUBLISHED)
-
- >>> hoary = getUtility(IDistributionSet)['ubuntu']['hoary']
- >>> create_binary_publication_for(
- ... partner_archive, hoary, PackagePublishingStatus.PENDING)
-
-So, at moment, partner archive is still not relevant for builds in
-hoary/i386. It's not passed to the builder.
-
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
- Suite: hoary
- Ogre-component: main
- Archive Purpose: PARTNER
- Archive Private: False
-
-Let's try it again.
-
- >>> candidate.destroySelf()
- >>> a_builder.setSlaveForTesting(OkSlave())
- >>> a_builder.is_available
- True
-
- >>> candidate = a_build.queueBuild()
- >>> setupBuildQueue(candidate, a_builder)
- >>> last_stub_mail_count = len(stub.test_emails)
-
- >>> removeSecurityProxy(a_build).archive = ubuntu.main_archive
- >>> candidate.destroySelf()
-
-But this time We will create a valid publication on partner hoary/i386.
-
- >>> from lp.soyuz.interfaces.component import IComponentSet
- >>> commit()
- >>> LaunchpadZopelessLayer.switchDbUser('launchpad')
- >>> login('foo.bar@xxxxxxxxxxxxx')
- >>> pub_source = test_publisher.getPubSource(
- ... archive=partner_archive, distroseries=hoary,
- ... status=PackagePublishingStatus.PUBLISHED,
- ... component='partner')
- >>> pub_binaries = test_publisher.getPubBinaries(
- ... archive=partner_archive, pub_source=pub_source,
- ... distroseries=hoary, status=PackagePublishingStatus.PUBLISHED)
- >>> partner_build = pub_binaries[0].binarypackagerelease.build
- >>> partner_candidate = partner_build.buildqueue_record
- >>> commit()
- >>> LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser)
-
-Now when we dispatch the partner build, since it has one published
-binary in hoary/i386, the partner archive gets included in the builder
-sources_list.
-
- >>> removeSecurityProxy(
- ... a_builder)._dispatchBuildCandidate(partner_candidate)
- ensurepresent called, url=...
- ensurepresent called, url=http://localhost:58000/.../foo_666.dsc
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary
- main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-security
- main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-updates
- main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu-partner hoary partner
- Suite: hoary
- Ogre-component: partner
- Archive Purpose: PARTNER
- Archive Private: False
-
- >>> partner_candidate.destroySelf()
-
-Similarly, PPA builds pass the 'archives' arguments:
-
- >>> from canonical.launchpad.interfaces import IPersonSet
- >>> cprov_archive = getUtility(IPersonSet).getByName('cprov').archive
- >>> removeSecurityProxy(a_build).archive = cprov_archive
- >>> a_builder.virtualized = True
- >>> a_builder.vm_host = 'localhost.ppa'
- >>> commit()
- >>> a_builder.setSlaveForTesting(OkSlave())
- >>> a_builder.is_available
- True
-
- >>> candidate = a_build.queueBuild()
- >>> setupBuildQueue(candidate, a_builder)
- >>> last_stub_mail_count = len(stub.test_emails)
-
-Exactly as Partner, Celso's PPA won't be included if it doesn't
-contain any published binary in hoary/i386. We will create it before
-dispatching.
-
- >>> create_binary_publication_for(
- ... cprov_archive, hoary, PackagePublishingStatus.PUBLISHED)
-
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
- deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
- Suite: hoary
- Ogre-component: main
- Archive Purpose: PPA
- Archive Private: False
-
-If the build is for a private PPA, the slave scanner will pass a
-sources.list entry that contains a password to access the archive.
-
- >>> from canonical.testing import LaunchpadZopelessLayer
- >>> commit()
- >>> LaunchpadZopelessLayer.switchDbUser('launchpad')
- >>> login('foo.bar@xxxxxxxxxxxxx')
- >>> build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(
- ... candidate)
- >>> for build_file in build.source_package_release.files:
- ... removeSecurityProxy(build_file).libraryfile.restricted = True
- >>> private_ppa = factory.makeArchive(
- ... owner=cprov_archive.owner, name='pppa', private=True,
- ... virtualized=False, distribution=ubuntu)
-
-It's necessary to publish some binaries into the private PPA, otherwise
-the PPA won't be included as a dependency in the sources list below.
-
- >>> binaries = test_publisher.getPubBinaries(
- ... distroseries=ubuntu['hoary'], archive=private_ppa,
- ... status=PackagePublishingStatus.PUBLISHED)
- >>> removeSecurityProxy(build).archive = private_ppa
- >>> commit()
- >>> LaunchpadZopelessLayer.switchDbUser(test_dbuser)
- >>> login(ANONYMOUS)
-
-Dispatch the build again. Celso's archive sources.list entry now has the
-buildd:secret@ part in the URL.
-
-Also note that when ensurepresent() is called, it receives a URL that
-points to the private archive rather than the librarian for the private
-firefox file. This is because the build slaves are not allowed to
-access the restricted librarian as it cannot provide access via
-credentials, unlike the archive itself.
-
-Finally, the archive purpose is overridden to PRIMARY instead of PPA
-for any archives that have require_virtualized as False.
-
-In this circumstance, it also uses the component override from the PRIMARY
-archive and not the one from the PPA, which on the absence of ancestry
-defaults to 'universe'.
-
- >>> build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(candidate)
- >>> print build.current_component.name
- main
-
-This is so that the mangling tools will run over the built packages.
-
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://private-ppa.../cprov/pppa/.../firefox_0.9.2.orig.tar.gz
- URL authorisation with buildd/sekrit
- OkSlave BUILDING
- Archives:
- deb http://buildd:sekrit@private-ppa.../cprov/pppa/ubuntu hoary main
- deb http://ftpmaster.internal/ubuntu hoary
- main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-security
- main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-updates
- main restricted universe multiverse
- Suite: hoary
- Ogre-component: universe
- Archive Purpose: PRIMARY
- Archive Private: True
-
-We will create an ancestry in the primary archive target to the 'main'
-component and this time the dispatching will follow that component.
-
- >>> sourcename = build.source_package_release.name
-
- >>> LaunchpadZopelessLayer.switchDbUser('launchpad')
- >>> login('foo.bar@xxxxxxxxxxxxx')
-
- >>> ancestry = test_publisher.getPubSource(
- ... sourcename=sourcename, version='0.1', distroseries=hoary)
-
- >>> print ancestry.displayname
- mozilla-firefox 0.1 in hoary
-
- >>> print ancestry.component.name
- main
-
- >>> commit()
- >>> LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser)
- >>> login(ANONYMOUS)
-
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
- ensurepresent called, ...
- ...
- Ogre-component: main
- ...
-
- >>> candidate.destroySelf()
-
Since this is a build in a private archive, the log was uploaded to
the restricted librarian.
- >>> candidate = a_build.queueBuild()
+ >>> removeSecurityProxy(build).archive = private_ppa
+ >>> commit()
+ >>> candidate = build.queueBuild()
>>> setupBuildQueue(candidate, a_builder)
>>> build.upload_log = None
>>> candidate.builder.setSlaveForTesting(WaitingSlave('BuildStatus.OK'))
@@ -1059,7 +774,7 @@
>>> lfa.restricted
True
>>> print lfa.filename
- buildlog_ubuntu-hoary-i386.mozilla-firefox_0.9_BUILDING.txt.gz
+ buildlog_ubuntu-warty-i386.mozilla-firefox_0.9_BUILDING.txt.gz
The attempt to fetch the buildlog from the common librarian will fail
since this is a build in a private archive and the buildlog was thus
@@ -1079,7 +794,7 @@
... getUtility(IRestrictedLibrarianClient).getFileByAlias(lfa.id))
>>> url_parts = urlparse.urlsplit(lfa2.file.geturl())
>>> print os.path.basename(url_parts[2])
- buildlog_ubuntu-hoary-i386.mozilla-firefox_0.9_BUILDING.txt.gz
+ buildlog_ubuntu-warty-i386.mozilla-firefox_0.9_BUILDING.txt.gz
A PPA can depend on another PPA. We can make Celso's PPA depend on
Mark's PPA:
@@ -1088,218 +803,13 @@
>>> LaunchpadZopelessLayer.switchDbUser('launchpad')
>>> login('foo.bar@xxxxxxxxxxxxx')
-We'll switch the build's archive back to Celso's PPA and set the PPA to
-virtualized before adding the dependency on Mark's PPA.
-
- >>> removeSecurityProxy(build).archive = cprov_archive
- >>> cprov_archive.require_virtualized = True
- >>> for build_file in a_build.source_package_release.files:
- ... removeSecurityProxy(build_file).libraryfile.restricted = False
- >>> mark_archive = getUtility(IPersonSet).getByName('mark').archive
-
- >>> unused_dep = cprov_archive.addArchiveDependency(
- ... mark_archive, PackagePublishingPocket.RELEASE,
- ... getUtility(IComponentSet)['main'])
-
- >>> commit()
- >>> LaunchpadZopelessLayer.switchDbUser(test_dbuser)
- >>> login(ANONYMOUS)
-
-Now we can see that a build from Celso's PPA will be able to install
-dependencies from Mark's PPA, if Mark's PPA has at least one binary
-published in hoary/i386, which is not the case.
-
- >>> a_builder.setSlaveForTesting(OkSlave())
- >>> a_builder.is_available
- True
-
- >>> candidate = a_build.queueBuild()
- >>> setupBuildQueue(candidate, a_builder)
- >>> last_stub_mail_count = len(stub.test_emails)
-
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
- deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
- Suite: hoary
- Ogre-component: main
- Archive Purpose: PPA
- Archive Private: False
-
-We will create the required publication in Mark's PPA and try again.
-
- >>> candidate.destroySelf()
- >>> a_builder.setSlaveForTesting(OkSlave())
- >>> a_builder.is_available
- True
-
- >>> candidate = a_build.queueBuild()
- >>> setupBuildQueue(candidate, a_builder)
- >>> last_stub_mail_count = len(stub.test_emails)
-
- >>> create_binary_publication_for(
- ... mark_archive, hoary, PackagePublishingStatus.PUBLISHED)
-
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
- deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
- deb http://ppa.launchpad.dev/mark/ppa/ubuntu hoary main
- Suite: hoary
- Ogre-component: main
- Archive Purpose: PPA
- Archive Private: False
Clean up before continuing:
- >>> candidate.destroySelf()
>>> a_builder.virtualized = False
>>> removeSecurityProxy(a_build).archive = ubuntu.main_archive
>>> commit()
-Builddmaster stops before starting to build a denied build.
-Since hoary is in development, we are not able to dispatch
-builds for post-release pockets:
-
- >>> candidate = a_build.queueBuild()
- >>> setupBuildQueue(candidate, a_builder)
- >>> last_stub_mail_count = len(stub.test_emails)
-
-Make a build in the updates pocket:
-
- >>> hoary = hoary_i386.distroseries
- >>> hoary_evo = hoary.getSourcePackage(
- ... 'evolution').currentrelease.sourcepackagerelease
- >>> updates_build = hoary_evo.createBuild(
- ... distro_arch_series=hoary_i386,
- ... pocket=PackagePublishingPocket.UPDATES,
- ... processor=hoary_i386.default_processor,
- ... archive=hoary_i386.main_archive)
- >>> updates_bqItem = updates_build.queueBuild()
-
- >>> hoary_i386.distroseries.status.name
- 'DEVELOPMENT'
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(updates_bqItem)
- Traceback (most recent call last):
- ...
- AssertionError: i386 build of evolution 1.0 in ubuntu hoary UPDATES (...) can not be built for pocket UPDATES: invalid pocket due to the series status of hoary.
-
-== Pocket dependencies ==
-
-Change the distroseries status for testing. FROZEN allows building in
-all pockets:
-
- >>> from canonical.launchpad.interfaces import SeriesStatus
- >>> hoary_i386.distroseries.status = SeriesStatus.FROZEN
-
-Now we can start a build in other pockets, and see what archives are
-passed to the slave.
-
-A build in the updates pocket:
-
- >>> a_builder.currentjob.destroySelf()
-
- >>> bqItem3 = a_build.queueBuild()
- >>> build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(bqItem3)
- >>> removeSecurityProxy(build).pocket = (
- ... PackagePublishingPocket.UPDATES)
- >>> last_stub_mail_count = len(stub.test_emails)
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary main
- deb http://ftpmaster.internal/ubuntu hoary-security main
- deb http://ftpmaster.internal/ubuntu hoary-updates main
- Suite: hoary-updates
- Ogre-component: main
- Archive Purpose: PRIMARY
- Archive Private: False
-
-A build in the proposed pocket:
-
- >>> a_builder.currentjob.destroySelf()
-
- >>> bqItem3 = a_build.queueBuild()
- >>> removeSecurityProxy(build).pocket = (
- ... PackagePublishingPocket.PROPOSED)
- >>> last_stub_mail_count = len(stub.test_emails)
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary main
- deb http://ftpmaster.internal/ubuntu hoary-proposed main
- deb http://ftpmaster.internal/ubuntu hoary-security main
- deb http://ftpmaster.internal/ubuntu hoary-updates main
- Suite: hoary-proposed
- Ogre-component: main
- Archive Purpose: PRIMARY
- Archive Private: False
-
-A build in the backports pocket:
-
- >>> a_builder.currentjob.destroySelf()
-
- >>> bqItem3 = a_build.queueBuild()
- >>> removeSecurityProxy(build).pocket = (
- ... PackagePublishingPocket.BACKPORTS)
- >>> last_stub_mail_count = len(stub.test_emails)
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
- ensurepresent called, url=...
- ensurepresent called,
- url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
- OkSlave BUILDING
- Archives:
- deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-backports main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
- deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
- Suite: hoary-backports
- Ogre-component: main
- Archive Purpose: PRIMARY
- Archive Private: False
-
-A build in the security pocket:
-
- >>> a_builder.currentjob.destroySelf()
-
- >>> bqItem3 = a_build.queueBuild()
- >>> removeSecurityProxy(build).status = (
- ... BuildStatus.NEEDSBUILD)
- >>> removeSecurityProxy(build).pocket = (
- ... PackagePublishingPocket.SECURITY)
- >>> last_stub_mail_count = len(stub.test_emails)
-
-The pocket-dependency infrastructure is ready to deal with SECURITY
-pocket, however we explicitly skip security builds when dispatching
-because Embargoed-Archives and Restricted-UI implementations are not
-yet ready.
-
- >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
- Traceback (most recent call last):
- ...
- AssertionError: Soyuz is not yet capable of building SECURITY uploads.
-
-Builds for security pocket are marked as FAILEDTOBUILD inside the
-_findBuildCandidate() method, see doc/buildd-dispatching.txt
-
== Builder Status Handler ==
=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehavior.py'
--- lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-08-23 16:51:11 +0000
+++ lib/lp/soyuz/model/binarypackagebuildbehavior.py 2010-09-21 18:41:18 +0000
@@ -99,8 +99,9 @@
distroseries state.
"""
build = self.build
- assert not (not self._builder.virtualized and build.is_virtualized), (
- "Attempt to build non-virtual item on a virtual builder.")
+ if not self._builder.virtualized and build.is_virtualized:
+ raise AssertionError(
+ "Attempt to build non-virtual item on a virtual builder.")
# Assert that we are not silently building SECURITY jobs.
# See findBuildCandidates. Once we start building SECURITY
=== modified file 'lib/lp/soyuz/scripts/buildd.py'
--- lib/lp/soyuz/scripts/buildd.py 2010-08-27 11:19:54 +0000
+++ lib/lp/soyuz/scripts/buildd.py 2010-09-21 18:41:18 +0000
@@ -276,6 +276,8 @@
self.txn.commit()
+# XXX: This is the old slave scanner. Julian says it's not running on
+# production. We should either delete it or update it to use the async apis.
class SlaveScanner(LaunchpadCronScript):
def main(self):
=== modified file 'lib/lp/soyuz/tests/soyuzbuilddhelpers.py'
--- lib/lp/soyuz/tests/soyuzbuilddhelpers.py 2010-09-20 11:52:51 +0000
+++ lib/lp/soyuz/tests/soyuzbuilddhelpers.py 2010-09-21 18:41:18 +0000
@@ -17,10 +17,8 @@
]
from StringIO import StringIO
-import subprocess
import xmlrpclib
-from canonical.config import config
from lp.buildmaster.interfaces.builder import CannotFetchFile
from lp.buildmaster.model.builder import (
rescueBuilderIfLost,
@@ -31,6 +29,8 @@
)
from lp.testing.sampledata import I386_ARCHITECTURE_NAME
+# XXX: Almost everything in this module will need to be revisited & perhaps
+# deleted now that we have new APIs for BuilderSlave.
class MockBuilder:
"""Emulates a IBuilder class."""
@@ -85,68 +85,53 @@
updateBuilderStatus(self, logger)
+# XXX: It would be *really* nice to run some set of tests against the real
+# BuilderSlave and this one to prevent interface skew.
class OkSlave:
"""An idle mock slave that prints information about itself.
The architecture tag can be customised during initialisation."""
def __init__(self, arch_tag=I386_ARCHITECTURE_NAME):
+ self.call_log = []
self.arch_tag = arch_tag
def status(self):
return ('BuilderStatus.IDLE', '')
def ensurepresent(self, sha1, url, user=None, password=None):
- print "ensurepresent called, url=%s" % url
- if user is not None and user != "":
- print "URL authorisation with %s/%s" % (user, password)
+ self.call_log.append(('ensurepresent', url, user, password))
return True, None
def build(self, buildid, buildtype, chroot, filemap, args):
+ self.call_log.append(
+ ('build', buildid, buildtype, chroot, filemap.keys(), args))
info = 'OkSlave BUILDING'
- print info
- if 'archives' in args:
- print "Archives:"
- for archive_line in sorted(args['archives']):
- print " %s" % archive_line
- else:
- print "No archives set."
- print "Suite: %s" % args['suite']
- print "Ogre-component: %s" % args['ogrecomponent']
- print "Archive Purpose: %s" % args['archive_purpose']
- print "Archive Private: %s" % args['archive_private']
return ('BuildStatus.Building', info)
- def fetchlogtail(self, size):
- return 'BOGUS'
-
def echo(self, *args):
+ self.call_log.append(('echo',) + args)
return args
def clean(self):
- pass
+ self.call_log.append('clean')
def abort(self):
- pass
+ self.call_log.append('abort')
def info(self):
+ self.call_log.append('info')
return ('1.0', self.arch_tag, 'debian')
- def resume(self):
- resume_argv = config.builddmaster.vm_resume_command.split()
- resume_process = subprocess.Popen(
- resume_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = resume_process.communicate()
-
- return (stdout, stderr, resume_process.returncode)
-
def sendFileToSlave(self, sha1, url, username="", password=""):
+ self.call_log.append('sendFileToSlave')
present, info = self.ensurepresent(sha1, url, username, password)
if not present:
raise CannotFetchFile(url, info)
def cacheFile(self, logger, libraryfilealias):
- self.sendFileToSlave(
+ self.call_log.append('cacheFile')
+ return self.sendFileToSlave(
libraryfilealias.content.sha1, libraryfilealias.http_url)
@@ -158,10 +143,12 @@
self.build_id = build_id
def status(self):
+ self.call_log.append('status')
buildlog = xmlrpclib.Binary("This is a build log")
return ('BuilderStatus.BUILDING', self.build_id, buildlog)
def getFile(self, sum):
+ self.call_log.append('getFile')
if sum == "buildlog":
s = StringIO("This is a build log")
s.headers = {'content-length': 19}
@@ -183,10 +170,12 @@
self.valid_file_hashes = ['buildlog']
def status(self):
+ self.call_log.append('status')
return ('BuilderStatus.WAITING', self.state, self.build_id, {},
self.dependencies)
def getFile(self, hash):
+ self.call_log.append('getFile')
if hash in self.valid_file_hashes:
content = "This is a %s" % hash
s = StringIO(content)
@@ -198,6 +187,7 @@
"""A mock slave that looks like it's in the process of aborting."""
def status(self):
+ self.call_log.append('status')
return ('BuilderStatus.ABORTING', '1-1')
@@ -205,6 +195,7 @@
"""A mock slave that looks like it's aborted."""
def status(self):
+ self.call_log.append('status')
return ('BuilderStatus.ABORTED', '1-1')
@@ -214,10 +205,15 @@
When 'aborted' it raises an xmlrpclib.Fault(8002, 'Could not abort')
"""
+ def __init__(self):
+ self.call_log = []
+
def status(self):
+ self.call_log.append('status')
return ('BuilderStatus.BUILDING', '1000-10000')
def abort(self):
+ self.call_log.append('abort')
raise xmlrpclib.Fault(8002, "Could not abort")
@@ -225,4 +221,5 @@
"""A mock slave that reports that it is broken."""
def status(self):
+ self.call_log.append('status')
raise xmlrpclib.Fault(8001, "Broken slave")
=== added file 'lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py'
--- lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py 1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py 2010-09-21 18:41:18 +0000
@@ -0,0 +1,205 @@
+# Copyright 2010 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import with_statement
+
+"""Tests for BinaryPackageBuildBehavior."""
+
+__metaclass__ = type
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from zope.security.proxy import removeSecurityProxy
+
+from canonical.launchpad.scripts.logger import QuietFakeLogger
+from canonical.testing import TwistedLaunchpadZopelessLayer
+
+from lp.registry.interfaces.pocket import (
+ PackagePublishingPocket,
+ pocketsuffix,
+ )
+from lp.registry.interfaces.series import SeriesStatus
+from lp.soyuz.adapters.archivedependencies import (
+ get_sources_list_for_building,
+ )
+from lp.soyuz.enums import (
+ ArchivePurpose,
+ )
+from lp.soyuz.tests.soyuzbuilddhelpers import OkSlave
+from lp.testing import (
+ ANONYMOUS,
+ login_as,
+ logout,
+ )
+from lp.testing.factory import LaunchpadObjectFactory
+
+
+class TestBinaryBuildPackageBehavior(unittest.TestCase):
+ """Tests for the BinaryPackageBuildBehavior.
+
+ In particular, these tests are about how the BinaryPackageBuildBehavior
+ interacts with the build slave. We test this by using a test double that
+ implements the same interface as `BuilderSlave` but instead of actually
+ making XML-RPC calls, just records any method invocations along with
+ interesting parameters.
+ """
+
+ layer = TwistedLaunchpadZopelessLayer
+
+ def setUp(self):
+ super(TestBinaryBuildPackageBehavior, self).setUp()
+ self.factory = LaunchpadObjectFactory()
+ login_as(ANONYMOUS)
+ self.addCleanup(logout)
+ self.layer.switchDbUser('testadmin')
+
+ def assertSlaveInteraction(self, ignored, call_log, builder, build,
+ chroot, archive, archive_purpose, component,
+ extra_urls=None, filemap_names=None):
+ """Assert that 'call_log' matches our expectations of interaction.
+
+ 'call_log' is expected to be a recording from a test double slave like
+ OkSlave or one its subclasses. We assert that the calls in call_log
+ match our expectations, thus showing that the binary package behaviour
+ interacts with the slave in the way we expect.
+
+ :param ignored: Ignored. This parameter here only to make it easier
+ to use the assertion as a Twisted callback.
+ :param call_log: A list of calls to a `BuilderSlave`-like object.
+ :param builder: The builder we are using to build the binary package.
+ :param build: The build being done on the builder.
+ :param chroot: The `LibraryFileAlias` for the chroot in which we are
+ building.
+ :param archive: The `IArchive` into which we are building.
+ :param archive_purpose: The ArchivePurpose we are sending to the
+ builder. We specify this separately from the archive because
+ sometimes the behavior object has to give a different purpose
+ in order to trick the slave into building correctly.
+ """
+ job = removeSecurityProxy(builder.current_build_behavior).buildfarmjob
+ build_id = job.generateSlaveBuildCookie()
+ ds_name = build.distro_arch_series.distroseries.name
+ suite = ds_name + pocketsuffix[build.pocket]
+ archives = get_sources_list_for_building(
+ build, build.distro_arch_series,
+ build.source_package_release.name)
+ arch_indep = build.distro_arch_series.isNominatedArchIndep
+ if filemap_names is None:
+ filemap_names = []
+ if extra_urls is None:
+ extra_urls = []
+
+ def make_expected_upload(url):
+ return [
+ 'cacheFile',
+ 'sendFileToSlave',
+ ('ensurepresent', url, '', '')]
+
+ expected = []
+ for url in [chroot.http_url] + extra_urls:
+ expected.extend(make_expected_upload(url))
+ expected.extend([
+ ('build', build_id, 'binarypackage', chroot.content.sha1,
+ filemap_names,
+ {'arch_indep': arch_indep,
+ 'arch_tag': build.distro_arch_series.architecturetag,
+ 'archive_private': archive.private,
+ 'archive_purpose': archive_purpose.name,
+ 'archives': archives,
+ 'build_debug_symbols': archive.build_debug_symbols,
+ 'ogrecomponent': component,
+ 'suite': suite})])
+ self.assertEqual(call_log, expected)
+
+ def startBuild(self, builder, candidate):
+ builder = removeSecurityProxy(builder)
+ candidate = removeSecurityProxy(candidate)
+ return defer.maybeDeferred(
+ builder.startBuild, candidate, QuietFakeLogger())
+
+ def test_non_virtual_ppa_dispatch(self):
+ # When the BinaryPackageBuildBehavior dispatches PPA builds to
+ # non-virtual builders, it stores the chroot on the server and
+ # requests a binary package build, lying to say that the archive
+ # purpose is "PRIMARY" because this ensures that the package mangling
+ # tools will run over the built packages.
+ archive = self.factory.makeArchive(virtualized=False)
+ slave = OkSlave()
+ builder = self.factory.makeBuilder(virtualized=False)
+ builder.setSlaveForTesting(slave)
+ build = self.factory.makeBinaryPackageBuild(
+ builder=builder, archive=archive)
+ lf = self.factory.makeLibraryFileAlias()
+ self.layer.txn.commit()
+ build.distro_arch_series.addOrUpdateChroot(lf)
+ candidate = build.queueBuild()
+ d = self.startBuild(builder, candidate)
+ d.addCallback(
+ self.assertSlaveInteraction,
+ slave.call_log, builder, build, lf, archive,
+ ArchivePurpose.PRIMARY, 'universe')
+ return d
+
+ def test_partner_dispatch_no_publishing_history(self):
+ archive = self.factory.makeArchive(
+ virtualized=False, purpose=ArchivePurpose.PARTNER)
+ slave = OkSlave()
+ builder = self.factory.makeBuilder(virtualized=False)
+ builder.setSlaveForTesting(slave)
+ build = self.factory.makeBinaryPackageBuild(
+ builder=builder, archive=archive)
+ lf = self.factory.makeLibraryFileAlias()
+ self.layer.txn.commit()
+ build.distro_arch_series.addOrUpdateChroot(lf)
+ candidate = build.queueBuild()
+ d = self.startBuild(builder, candidate)
+ d.addCallback(
+ self.assertSlaveInteraction,
+ slave.call_log, builder, build, lf, archive,
+ ArchivePurpose.PARTNER, build.current_component.name)
+ return d
+
+ def test_dont_dispatch_release_builds(self):
+ archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY)
+ builder = self.factory.makeBuilder()
+ distroseries = self.factory.makeDistroSeries(
+ status=SeriesStatus.CURRENT, distribution=archive.distribution)
+ distro_arch_series = self.factory.makeDistroArchSeries(
+ distroseries=distroseries)
+ build = self.factory.makeBinaryPackageBuild(
+ builder=builder, archive=archive,
+ distroarchseries=distro_arch_series,
+ pocket=PackagePublishingPocket.RELEASE)
+ lf = self.factory.makeLibraryFileAlias()
+ self.layer.txn.commit()
+ build.distro_arch_series.addOrUpdateChroot(lf)
+ candidate = build.queueBuild()
+ behavior = candidate.required_build_behavior
+ behavior.setBuilder(build)
+ e = self.assertRaises(
+ AssertionError, behavior.verifyBuildRequest, QuietFakeLogger())
+ self.assertEqual(
+ "%s (%s) can not be built for pocket %s: invalid pocket due "
+ "to the series status of %s." % (
+ build.title, build.id, build.pocket.name,
+ build.distro_series.name),
+ str(e))
+
+ def test_dont_dispatch_security_builds(self):
+ archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY)
+ builder = self.factory.makeBuilder()
+ build = self.factory.makeBinaryPackageBuild(
+ builder=builder, archive=archive,
+ pocket=PackagePublishingPocket.SECURITY)
+ lf = self.factory.makeLibraryFileAlias()
+ self.layer.txn.commit()
+ build.distro_arch_series.addOrUpdateChroot(lf)
+ candidate = build.queueBuild()
+ behavior = candidate.required_build_behavior
+ behavior.setBuilder(build)
+ e = self.assertRaises(
+ AssertionError, behavior.verifyBuildRequest, QuietFakeLogger())
+ self.assertEqual(
+ 'Soyuz is not yet capable of building SECURITY uploads.',
+ str(e))
=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py 2010-09-21 11:45:15 +0000
+++ lib/lp/testing/factory.py 2010-09-21 18:41:18 +0000
@@ -1944,6 +1944,9 @@
processorfamily = ProcessorFamilySet().getByName('powerpc')
if owner is None:
owner = self.makePerson()
+ # XXX: architecturetag & processerfamily are tightly coupled. It's
+ # wrong to just make a fresh architecture tag without also making a
+ # processor family to go with it (ideally with processors!)
if architecturetag is None:
architecturetag = self.getUniqueString('arch')
return distroseries.newArch(
@@ -2625,7 +2628,7 @@
def makeBinaryPackageBuild(self, source_package_release=None,
distroarchseries=None, archive=None, builder=None,
- status=None):
+ status=None, pocket=None):
"""Create a BinaryPackageBuild.
If archive is not supplied, the source_package_release is used
@@ -2656,13 +2659,15 @@
processorfamily=processor.family)
if status is None:
status = BuildStatus.NEEDSBUILD
+ if pocket is None:
+ pocket = PackagePublishingPocket.RELEASE
binary_package_build = getUtility(IBinaryPackageBuildSet).new(
source_package_release=source_package_release,
processor=processor,
distro_arch_series=distroarchseries,
status=status,
archive=archive,
- pocket=PackagePublishingPocket.RELEASE,
+ pocket=pocket,
date_created=self.getUniqueDate())
naked_build = removeSecurityProxy(binary_package_build)
naked_build.builder = builder
=== modified file 'lib/lp/testing/tests/test_factory.py'
--- lib/lp/testing/tests/test_factory.py 2010-09-21 11:08:26 +0000
+++ lib/lp/testing/tests/test_factory.py 2010-09-21 18:41:18 +0000
@@ -113,12 +113,26 @@
status=BuildStatus.FULLYBUILT)
self.assertEqual(BuildStatus.FULLYBUILT, bpb.status)
- def test_makeBinaryPackageBuild_can_be_queued(self):
- build = self.factory.makeBinaryPackageBuild()
- # Just check that makeBinaryPackageBuild returns a build that can be
- # queued.
- build.queueBuild()
-
+<<<<<<< TREE
+ def test_makeBinaryPackageBuild_can_be_queued(self):
+ build = self.factory.makeBinaryPackageBuild()
+ # Just check that makeBinaryPackageBuild returns a build that can be
+ # queued.
+ build.queueBuild()
+
+=======
+ def test_makeBinaryPackageBuild_uses_pocket(self):
+ bpb = self.factory.makeBinaryPackageBuild(
+ pocket=PackagePublishingPocket.UPDATES)
+ self.assertEqual(PackagePublishingPocket.UPDATES, bpb.pocket)
+
+ def test_makeBinaryPackageBuild_can_be_queued(self):
+ build = self.factory.makeBinaryPackageBuild()
+ # Just check that makeBinaryPackageBuild returns a build that can be
+ # queued.
+ build.queueBuild()
+
+>>>>>>> MERGE-SOURCE
# makeBinaryPackageName
def test_makeBinaryPackageName_returns_proxied_IBinaryPackageName(self):
binarypackagename = self.factory.makeBinaryPackageName()
