launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #08882
[Merge] lp:~allenap/maas/effing-download-cache into lp:maas
Gavin Panella has proposed merging lp:~allenap/maas/effing-download-cache into lp:maas.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~allenap/maas/effing-download-cache/+merge/110534
This removes the need for the faulty zc.buildout bootstrap.py script. Instead it creates a virtualenv and uses pip to install zc.buildout from source.
Part of this required removing the allowed-eggs-from-site-packages setting in buildout.cfg. This isn't really needed - I think we always want to use the system package if it's available - and I believe this option won't be available in buildout 2. If we need to test using a package version different to the system package it can be installed using pip.
--
https://code.launchpad.net/~allenap/maas/effing-download-cache/+merge/110534
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~allenap/maas/effing-download-cache into lp:maas.
=== modified file '.bzrignore'
--- .bzrignore 2012-05-18 13:11:58 +0000
+++ .bzrignore 2012-06-15 14:08:20 +0000
@@ -12,15 +12,17 @@
./dist
./docs/_build
./docs/api.rst
-./download-cache
./eggs
+./include
+./lib
+./local
./logs/*
./media/demo/*
./media/development
./parts
./run/*
./services/*/supervise
-src/maasserver/static/js/enums.js
+./src/maasserver/static/js/enums.js
./TAGS
./tags
./twisted/plugins/dropin.cache
=== modified file 'HACKING.txt'
--- HACKING.txt 2012-06-15 09:27:14 +0000
+++ HACKING.txt 2012-06-15 14:08:20 +0000
@@ -49,7 +49,8 @@
Additionally, you need to install the following python libraries
for development convenience::
- $ sudo apt-get install python-sphinx python-lxml python-pocket-lint
+ $ sudo apt-get install python-sphinx python-lxml python-pocket-lint \
+ python-pip python-virtualenv
If you intend to run the test suite, you also need a few other things:
=== modified file 'Makefile'
--- Makefile 2012-06-13 15:04:27 +0000
+++ Makefile 2012-06-15 14:08:20 +0000
@@ -1,4 +1,6 @@
python := python2.7
+buildout := bin/buildout $(shell tty -s && echo -D)
+virtualenv := virtualenv
# Python enum modules.
py_enums := $(wildcard src/*/enum.py)
@@ -23,52 +25,61 @@
bin/py bin/ipy \
$(js_enums)
+# Note: the following target may not be needed. It remains as an
+# experiment, to see if it helps the situation with building MAAS in
+# the QA environment, which is isolated from the Internet at large.
+build-offline: buildout := $(buildout) buildout:offline=true
+build-offline: virtualenv := $(virtualenv) --never-download
+build-offline: build
+
all: build doc
-bin/buildout: bootstrap/bootstrap.py bootstrap/distribute_setup.py
- $(python) bootstrap/bootstrap.py --distribute \
- --setup-source bootstrap/distribute_setup.py \
- --download-base $(PWD)/bootstrap
+bin/python bin/pip:
+ $(virtualenv) --python=$(python) --system-site-packages $(PWD)
+
+bin/buildout: bin/pip bootstrap/zc.buildout-1.5.2.tar.gz
+ bin/pip --quiet install bootstrap/zc.buildout-1.5.2.tar.gz
+ $(RM) -f README.txt # zc.buildout installs an annoying README.txt.
@touch --no-create $@ # Ensure it's newer than its dependencies.
bin/database: bin/buildout buildout.cfg versions.cfg setup.py
- bin/buildout install database
+ $(buildout) install database
@touch --no-create $@
bin/maas: bin/buildout buildout.cfg versions.cfg setup.py $(js_enums)
- bin/buildout install maas
+ $(buildout) install maas
@touch --no-create $@
bin/test.maas: bin/buildout buildout.cfg versions.cfg setup.py $(js_enums)
- bin/buildout install maas-test
+ $(buildout) install maas-test
@touch --no-create $@
bin/test.maastesting: bin/buildout buildout.cfg versions.cfg setup.py
- bin/buildout install maastesting-test
+ $(buildout) install maastesting-test
@touch --no-create $@
bin/twistd.pserv: bin/buildout buildout.cfg versions.cfg setup.py
- bin/buildout install pserv
+ $(buildout) install pserv
@touch --no-create $@
bin/test.pserv: bin/buildout buildout.cfg versions.cfg setup.py
- bin/buildout install pserv-test
+ $(buildout) install pserv-test
@touch --no-create $@
bin/twistd.txlongpoll: bin/buildout buildout.cfg versions.cfg setup.py
- bin/buildout install txlongpoll
+ $(buildout) install txlongpoll
@touch --no-create $@
bin/flake8: bin/buildout buildout.cfg versions.cfg setup.py
- bin/buildout install flake8
+ $(buildout) install flake8
@touch --no-create $@
bin/sphinx: bin/buildout buildout.cfg versions.cfg setup.py
- bin/buildout install sphinx
+ $(buildout) install sphinx
@touch --no-create $@
bin/py bin/ipy: bin/buildout buildout.cfg versions.cfg setup.py
- bin/buildout install repl
+ $(buildout) install repl
@touch --no-create bin/py bin/ipy
test: bin/test.maas bin/test.maastesting bin/test.pserv $(js_enums)
@@ -117,8 +128,9 @@
$(RM) celerybeat-schedule
distclean: clean stop
+ $(RM) -r bin include lib local
$(RM) -r eggs develop-eggs
- $(RM) -r bin build dist logs/* parts
+ $(RM) -r build dist logs/* parts
$(RM) tags TAGS .installed.cfg
$(RM) -r *.egg *.egg-info src/*.egg-info
$(RM) docs/api.rst
@@ -139,6 +151,7 @@
define phony_targets
build
+ build-offline
check
clean
dbharness
=== renamed file 'README.txt' => 'README'
=== removed file 'bootstrap/bootstrap.py'
--- bootstrap/bootstrap.py 2012-06-13 15:04:27 +0000
+++ bootstrap/bootstrap.py 1970-01-01 00:00:00 +0000
@@ -1,262 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2006 Zope Foundation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-"""Bootstrap a buildout-based project
-
-Simply run this script in a directory containing a buildout.cfg.
-The script accepts buildout command-line options, so you can
-use the -c option to specify an alternate configuration file.
-"""
-
-import os, shutil, sys, tempfile, urllib, urllib2, subprocess
-from optparse import OptionParser
-
-if sys.platform == 'win32':
- def quote(c):
- if ' ' in c:
- return '"%s"' % c # work around spawn lamosity on windows
- else:
- return c
-else:
- quote = str
-
-# See zc.buildout.easy_install._has_broken_dash_S for motivation and comments.
-stdout, stderr = subprocess.Popen(
- [sys.executable, '-Sc',
- 'try:\n'
- ' import ConfigParser\n'
- 'except ImportError:\n'
- ' print 1\n'
- 'else:\n'
- ' print 0\n'],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-has_broken_dash_S = bool(int(stdout.strip()))
-
-# In order to be more robust in the face of system Pythons, we want to
-# run without site-packages loaded. This is somewhat tricky, in
-# particular because Python 2.6's distutils imports site, so starting
-# with the -S flag is not sufficient. However, we'll start with that:
-if not has_broken_dash_S and 'site' in sys.modules:
- # We will restart with python -S.
- args = sys.argv[:]
- args[0:0] = [sys.executable, '-S']
- args = map(quote, args)
- os.execv(sys.executable, args)
-# Now we are running with -S. We'll get the clean sys.path, import site
-# because distutils will do it later, and then reset the path and clean
-# out any namespace packages from site-packages that might have been
-# loaded by .pth files.
-clean_path = sys.path[:]
-import site # imported because of its side effects
-sys.path[:] = clean_path
-for k, v in sys.modules.items():
- if k in ('setuptools', 'pkg_resources') or (
- hasattr(v, '__path__') and
- len(v.__path__) == 1 and
- not os.path.exists(os.path.join(v.__path__[0], '__init__.py'))):
- # This is a namespace package. Remove it.
- sys.modules.pop(k)
-
-is_jython = sys.platform.startswith('java')
-
-setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py'
-distribute_source = 'http://python-distribute.org/distribute_setup.py'
-
-
-# parsing arguments
-def normalize_to_url(option, opt_str, value, parser):
- if value:
- if '://' not in value: # It doesn't smell like a URL.
- value = 'file://%s' % (
- urllib.pathname2url(
- os.path.abspath(os.path.expanduser(value))),)
- if opt_str == '--download-base' and not value.endswith('/'):
- # Download base needs a trailing slash to make the world happy.
- value += '/'
- else:
- value = None
- name = opt_str[2:].replace('-', '_')
- setattr(parser.values, name, value)
-
-usage = '''\
-[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
-
-Bootstraps a buildout-based project.
-
-Simply run this script in a directory containing a buildout.cfg, using the
-Python that you want bin/buildout to use.
-
-Note that by using --setup-source and --download-base to point to
-local resources, you can keep this script from going over the network.
-'''
-
-parser = OptionParser(usage=usage)
-parser.add_option("-v", "--version", dest="version",
- help="use a specific zc.buildout version")
-parser.add_option("-d", "--distribute",
- action="store_true", dest="use_distribute", default=False,
- help="Use Distribute rather than Setuptools.")
-parser.add_option("--setup-source", action="callback", dest="setup_source",
- callback=normalize_to_url, nargs=1, type="string",
- help=("Specify a URL or file location for the setup file. "
- "If you use Setuptools, this will default to " +
- setuptools_source + "; if you use Distribute, this "
- "will default to " + distribute_source + "."))
-parser.add_option("--download-base", action="callback", dest="download_base",
- callback=normalize_to_url, nargs=1, type="string",
- help=("Specify a URL or directory for downloading "
- "zc.buildout and either Setuptools or Distribute. "
- "Defaults to PyPI."))
-parser.add_option("--eggs",
- help=("Specify a directory for storing eggs. Defaults to "
- "a temporary directory that is deleted when the "
- "bootstrap script completes."))
-parser.add_option("-t", "--accept-buildout-test-releases",
- dest='accept_buildout_test_releases',
- action="store_true", default=False,
- help=("Normally, if you do not specify a --version, the "
- "bootstrap script and buildout gets the newest "
- "*final* versions of zc.buildout and its recipes and "
- "extensions for you. If you use this flag, "
- "bootstrap and buildout will get the newest releases "
- "even if they are alphas or betas."))
-parser.add_option("-c", None, action="store", dest="config_file",
- help=("Specify the path to the buildout configuration "
- "file to be used."))
-
-options, args = parser.parse_args()
-
-# if -c was provided, we push it back into args for buildout's main function
-if options.config_file is not None:
- args += ['-c', options.config_file]
-
-if options.eggs:
- eggs_dir = os.path.abspath(os.path.expanduser(options.eggs))
-else:
- eggs_dir = tempfile.mkdtemp()
-
-if options.setup_source is None:
- if options.use_distribute:
- options.setup_source = distribute_source
- else:
- options.setup_source = setuptools_source
-
-if options.accept_buildout_test_releases:
- args.append('buildout:accept-buildout-test-releases=true')
-args.append('bootstrap')
-
-try:
- import pkg_resources
- import setuptools # A flag. Sometimes pkg_resources is installed alone.
- if not hasattr(pkg_resources, '_distribute'):
- raise ImportError
-except ImportError:
- ez_code = urllib2.urlopen(
- options.setup_source).read().replace('\r\n', '\n')
- ez = {}
- exec ez_code in ez
- setup_args = dict(to_dir=eggs_dir, download_delay=0)
- if options.download_base:
- setup_args['download_base'] = options.download_base
- if options.use_distribute:
- setup_args['no_fake'] = True
- ez['use_setuptools'](**setup_args)
- if 'pkg_resources' in sys.modules:
- reload(sys.modules['pkg_resources'])
- import pkg_resources
- # This does not (always?) update the default working set. We will
- # do it.
- for path in sys.path:
- if path not in pkg_resources.working_set.entries:
- pkg_resources.working_set.add_entry(path)
-
-cmd = [quote(sys.executable),
- '-c',
- quote('from setuptools.command.easy_install import main; main()'),
- '-mqNxd',
- quote(eggs_dir)]
-
-if not has_broken_dash_S:
- cmd.insert(1, '-S')
-
-find_links = options.download_base
-if not find_links:
- find_links = os.environ.get('bootstrap-testing-find-links')
-if find_links:
- cmd.extend(['-f', quote(find_links)])
-
-if options.use_distribute:
- setup_requirement = 'distribute'
-else:
- setup_requirement = 'setuptools'
-ws = pkg_resources.working_set
-setup_requirement_path = ws.find(
- pkg_resources.Requirement.parse(setup_requirement)).location
-env = dict(
- os.environ,
- PYTHONPATH=setup_requirement_path)
-
-requirement = 'zc.buildout'
-version = options.version
-if version is None and not options.accept_buildout_test_releases:
- # Figure out the most recent final version of zc.buildout.
- import setuptools.package_index
- _final_parts = '*final-', '*final'
-
- def _final_version(parsed_version):
- for part in parsed_version:
- if (part[:1] == '*') and (part not in _final_parts):
- return False
- return True
- index = setuptools.package_index.PackageIndex(
- search_path=[setup_requirement_path])
- if find_links:
- index.add_find_links((find_links,))
- req = pkg_resources.Requirement.parse(requirement)
- if index.obtain(req) is not None:
- best = []
- bestv = None
- for dist in index[req.project_name]:
- distv = dist.parsed_version
- if _final_version(distv):
- if bestv is None or distv > bestv:
- best = [dist]
- bestv = distv
- elif distv == bestv:
- best.append(dist)
- if best:
- best.sort()
- version = best[-1].version
-if version:
- requirement = '=='.join((requirement, version))
-cmd.append(requirement)
-
-if is_jython:
- import subprocess
- exitcode = subprocess.Popen(cmd, env=env).wait()
-else: # Windows prefers this, apparently; otherwise we would prefer subprocess
- exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env]))
-if exitcode != 0:
- sys.stdout.flush()
- sys.stderr.flush()
- print ("An error occurred when trying to install zc.buildout. "
- "Look above this message for any errors that "
- "were output by easy_install.")
- sys.exit(exitcode)
-
-ws.add_entry(eggs_dir)
-ws.require(requirement)
-import zc.buildout.buildout
-zc.buildout.buildout.main(args)
-if not options.eggs: # clean up temporary egg directory
- shutil.rmtree(eggs_dir)
=== removed file 'bootstrap/distribute-0.6.24.tar.gz'
Binary files bootstrap/distribute-0.6.24.tar.gz 2012-06-13 15:04:27 +0000 and bootstrap/distribute-0.6.24.tar.gz 1970-01-01 00:00:00 +0000 differ
=== added file 'bootstrap/zc.buildout-1.5.2.tar.gz'
Binary files bootstrap/zc.buildout-1.5.2.tar.gz 1970-01-01 00:00:00 +0000 and bootstrap/zc.buildout-1.5.2.tar.gz 2012-06-15 14:08:20 +0000 differ
=== modified file 'buildout.cfg'
--- buildout.cfg 2012-06-14 15:20:52 +0000
+++ buildout.cfg 2012-06-15 14:08:20 +0000
@@ -13,44 +13,12 @@
buildout_versions_file = versions.cfg
versions = versions
extends = versions.cfg
-install-from-cache = false
+offline = false
-# Since MAAS's main deployment target is Ubuntu, all
-# runtime dependencies should come from python packages.
-# Only development-time dependencies should come from eggs.
-# For convenience, we allow some of those to come from site-packages,
-# mainly those which contains C extensions like lxml
+# Since MAAS's main deployment target is Ubuntu, all runtime
+# dependencies should come from python packages. However, it's okay
+# for development-time dependencies to come from eggs.
include-site-packages = true
-allowed-eggs-from-site-packages =
- anyjson
- avahi
- celery
- convoy
- dbus
- Django
- django-picklefield
- South
- amqplib
- django-piston
- FormEncode
- kombu
- oauth
- oops
- oops-datedir-repo
- oops-twisted
- oops-wsgi
- psycopg2
- python-dateutil
- PyYAML
- Tempita
- Twisted
- txAMQP
- # Convenient developer dependencies
- Jinja2
- Pygments
- Sphinx
- docutils
- lxml
prefer-final = true
allow-picked-versions = false
@@ -109,6 +77,7 @@
pyasn1
rabbitfixture
South
+ Tempita
twisted
entry-points =
maas=django.core.management:execute_from_command_line
=== renamed file 'bootstrap/distribute_setup.py' => 'distribute_setup.py'
=== modified symlink 'docs/readme.rst'
=== target changed u'../README.txt' => u'../README'
=== modified file 'setup.py'
--- setup.py 2012-05-15 08:13:43 +0000
+++ setup.py 2012-06-15 14:08:20 +0000
@@ -43,7 +43,7 @@
url="https://launchpad.net/maas";,
license="AGPLv3",
description="Metal as as Service",
- long_description=read('README.txt'),
+ long_description=read('README'),
author="MAAS Developers",
author_email="juju@xxxxxxxxxxxxxxxx",
