cf-charmers team mailing list archive
-
cf-charmers team
-
Mailing list archive
-
Message #00160
[Merge] lp:~johnsca/charm-helpers/charm-helpers into lp:~cf-charmers/charm-helpers/cloud-foundry
Cory Johns has proposed merging lp:~johnsca/charm-helpers/charm-helpers into lp:~cf-charmers/charm-helpers/cloud-foundry.
Requested reviews:
Cloud Foundry Charmers (cf-charmers)
For more details, see:
https://code.launchpad.net/~johnsca/charm-helpers/charm-helpers/+merge/219910
Refactor services & templating framework to core
--
https://code.launchpad.net/~johnsca/charm-helpers/charm-helpers/+merge/219910
Your team Cloud Foundry Charmers is requested to review the proposed merge of lp:~johnsca/charm-helpers/charm-helpers into lp:~cf-charmers/charm-helpers/cloud-foundry.
=== modified file 'charmhelpers/contrib/cloudfoundry/azure/add-cf-ports.py'
--- charmhelpers/contrib/cloudfoundry/azure/add-cf-ports.py 2014-03-26 17:38:14 +0000
+++ charmhelpers/contrib/cloudfoundry/azure/add-cf-ports.py 2014-05-16 22:13:17 +0000
@@ -7,7 +7,7 @@
# Then authenticate :
# azure account download # this will open browser and download cert file
# azure import <cert-file-path>
-#
+#
import os
import yaml
@@ -21,9 +21,9 @@
for vm in azure_vm_list:
name = vm['VMName']
for p in ports:
- try:
- command = 'azure vm endpoint create %s %s' % (name, p)
- print command
- os.system(command)
- except:
- print "Port can't be created %s:%s" % (name, p)
\ No newline at end of file
+ try:
+ command = 'azure vm endpoint create %s %s' % (name, p)
+ print command
+ os.system(command)
+ except:
+ print "Port can't be created %s:%s" % (name, p)
=== modified file 'charmhelpers/contrib/cloudfoundry/common.py'
--- charmhelpers/contrib/cloudfoundry/common.py 2014-04-01 06:50:54 +0000
+++ charmhelpers/contrib/cloudfoundry/common.py 2014-05-16 22:13:17 +0000
@@ -1,11 +1,3 @@
-import sys
-import os
-import pwd
-import grp
-import subprocess
-
-from contextlib import contextmanager
-from charmhelpers.core.hookenv import log, ERROR, DEBUG
from charmhelpers.core import host
from charmhelpers.fetch import (
@@ -13,55 +5,6 @@
)
-def run(command, exit_on_error=True, quiet=False):
- '''Run a command and return the output.'''
- if not quiet:
- log("Running {!r}".format(command), DEBUG)
- p = subprocess.Popen(
- command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- shell=isinstance(command, basestring))
- p.stdin.close()
- lines = []
- for line in p.stdout:
- if line:
- if not quiet:
- print line
- lines.append(line)
- elif p.poll() is not None:
- break
-
- p.wait()
-
- if p.returncode == 0:
- return '\n'.join(lines)
-
- if p.returncode != 0 and exit_on_error:
- log("ERROR: {}".format(p.returncode), ERROR)
- sys.exit(p.returncode)
-
- raise subprocess.CalledProcessError(
- p.returncode, command, '\n'.join(lines))
-
-
-def chownr(path, owner, group):
- uid = pwd.getpwnam(owner).pw_uid
- gid = grp.getgrnam(group).gr_gid
- for root, dirs, files in os.walk(path):
- for momo in dirs:
- os.chown(os.path.join(root, momo), uid, gid)
- for momo in files:
- os.chown(os.path.join(root, momo), uid, gid)
-
-
-@contextmanager
-def chdir(d):
- cur = os.getcwd()
- try:
- yield os.chdir(d)
- finally:
- os.chdir(cur)
-
-
def prepare_cloudfoundry_environment(config_data, packages):
add_source(config_data['source'], config_data.get('key'))
apt_update(fatal=True)
=== removed file 'charmhelpers/contrib/cloudfoundry/config_helper.py'
--- charmhelpers/contrib/cloudfoundry/config_helper.py 2014-03-30 16:40:45 +0000
+++ charmhelpers/contrib/cloudfoundry/config_helper.py 1970-01-01 00:00:00 +0000
@@ -1,11 +0,0 @@
-import jinja2
-
-TEMPLATES_DIR = 'templates'
-
-def render_template(template_name, context, template_dir=TEMPLATES_DIR):
- templates = jinja2.Environment(
- loader=jinja2.FileSystemLoader(template_dir))
- template = templates.get_template(template_name)
- return template.render(context)
-
-
=== modified file 'charmhelpers/contrib/cloudfoundry/contexts.py'
--- charmhelpers/contrib/cloudfoundry/contexts.py 2014-04-17 12:19:21 +0000
+++ charmhelpers/contrib/cloudfoundry/contexts.py 2014-05-16 22:13:17 +0000
@@ -1,61 +1,26 @@
import os
-import yaml
-
-from charmhelpers.core import hookenv
-from charmhelpers.contrib.openstack.context import OSContextGenerator
-
-
-class RelationContext(OSContextGenerator):
- def __call__(self):
- if not hookenv.relation_ids(self.interface):
- return {}
-
- ctx = {}
- for rid in hookenv.relation_ids(self.interface):
- for unit in hookenv.related_units(rid):
- reldata = hookenv.relation_get(rid=rid, unit=unit)
- required = set(self.required_keys)
- if set(reldata.keys()).issuperset(required):
- ns = ctx.setdefault(self.interface, {})
- for k, v in reldata.items():
- ns[k] = v
- return ctx
-
- return {}
-
-
-class ConfigContext(OSContextGenerator):
- def __call__(self):
- return hookenv.config()
+
+from charmhelpers.core.templating import (
+ ContextGenerator,
+ RelationContext,
+ StorableContext,
+)
# Stores `config_data` hash into yaml file with `file_name` as a name
# if `file_name` already exists, then it loads data from `file_name`.
-class StoredContext(OSContextGenerator):
+class StoredContext(ContextGenerator, StorableContext):
def __init__(self, file_name, config_data):
- self.data = config_data
if os.path.exists(file_name):
- with open(file_name, 'r') as file_stream:
- self.data = yaml.load(file_stream)
- if not self.data:
- raise OSError("%s is empty" % file_name)
+ self.data = self.read_context(file_name)
else:
- with open(file_name, 'w') as file_stream:
- yaml.dump(config_data, file_stream)
+ self.store_context(file_name, config_data)
self.data = config_data
def __call__(self):
return self.data
-class StaticContext(OSContextGenerator):
- def __init__(self, data):
- self.data = data
-
- def __call__(self):
- return self.data
-
-
class NatsContext(RelationContext):
interface = 'nats'
required_keys = ['nats_port', 'nats_address', 'nats_user', 'nats_password']
@@ -65,6 +30,12 @@
interface = 'router'
required_keys = ['domain']
+
class LogRouterContext(RelationContext):
interface = 'logrouter'
required_keys = ['shared-secret', 'logrouter-address']
+
+
+class LoggregatorContext(RelationContext):
+ interface = 'loggregator'
+ required_keys = ['shared_secret', 'loggregator_address']
=== removed file 'charmhelpers/contrib/cloudfoundry/install.py'
--- charmhelpers/contrib/cloudfoundry/install.py 2014-03-30 16:50:30 +0000
+++ charmhelpers/contrib/cloudfoundry/install.py 1970-01-01 00:00:00 +0000
@@ -1,35 +0,0 @@
-import os
-import subprocess
-
-
-def install(src, dest, fileprops=None, sudo=False):
- """Install a file from src to dest. Dest can be a complete filename
- or a target directory. fileprops is a dict with 'owner' (username of owner)
- and mode (octal string) as keys, the defaults are 'ubuntu' and '400'
-
- When owner is passed or when access requires it sudo can be set to True and
- sudo will be used to install the file.
- """
- if not fileprops:
- fileprops = {}
- mode = fileprops.get('mode', '400')
- owner = fileprops.get('owner')
- cmd = ['install']
-
- if not os.path.exists(src):
- raise OSError(src)
-
- if not os.path.exists(dest) and not os.path.exists(os.path.dirname(dest)):
- # create all but the last component as path
- cmd.append('-D')
-
- if mode:
- cmd.extend(['-m', mode])
-
- if owner:
- cmd.extend(['-o', owner])
-
- if sudo:
- cmd.insert(0, 'sudo')
- cmd.extend([src, dest])
- subprocess.check_call(cmd)
=== removed file 'charmhelpers/contrib/cloudfoundry/services.py'
--- charmhelpers/contrib/cloudfoundry/services.py 2014-04-04 16:41:29 +0000
+++ charmhelpers/contrib/cloudfoundry/services.py 1970-01-01 00:00:00 +0000
@@ -1,118 +0,0 @@
-import os
-import tempfile
-from charmhelpers.core import host
-
-from charmhelpers.contrib.cloudfoundry.install import install
-from charmhelpers.core.hookenv import log
-from jinja2 import Environment, FileSystemLoader
-
-SERVICE_CONFIG = []
-TEMPLATE_LOADER = None
-
-
-def render_template(template_name, context):
- """Render template to a tempfile returning the name"""
- _, fn = tempfile.mkstemp()
- template = load_template(template_name)
- output = template.render(context)
- with open(fn, "w") as fp:
- fp.write(output)
- return fn
-
-
-def collect_contexts(context_providers):
- ctx = {}
- for provider in context_providers:
- c = provider()
- if not c:
- return {}
- ctx.update(c)
- return ctx
-
-
-def load_template(name):
- return TEMPLATE_LOADER.get_template(name)
-
-
-def configure_templates(template_dir):
- global TEMPLATE_LOADER
- TEMPLATE_LOADER = Environment(loader=FileSystemLoader(template_dir))
-
-
-def register(service_configs, template_dir):
- """Register a list of service configs.
-
- Service Configs are dicts in the following formats:
-
- {
- "service": <service name>,
- "templates": [ {
- 'target': <render target of template>,
- 'source': <optional name of template in passed in template_dir>
- 'file_properties': <optional dict taking owner and octal mode>
- 'contexts': [ context generators, see contexts.py ]
- }
- ] }
-
- If 'source' is not provided for a template the template_dir will
- be consulted for ``basename(target).j2``.
- """
- global SERVICE_CONFIG
- if template_dir:
- configure_templates(template_dir)
- SERVICE_CONFIG.extend(service_configs)
-
-
-def reset():
- global SERVICE_CONFIG
- SERVICE_CONFIG = []
-
-
-# def service_context(name):
-# contexts = collect_contexts(template['contexts'])
-
-def reconfigure_service(service_name, restart=True):
- global SERVICE_CONFIG
- service = None
- for service in SERVICE_CONFIG:
- if service['service'] == service_name:
- break
- if not service or service['service'] != service_name:
- raise KeyError('Service not registered: %s' % service_name)
-
- templates = service['templates']
- for template in templates:
- contexts = collect_contexts(template['contexts'])
- if contexts:
- template_target = template['target']
- default_template = "%s.j2" % os.path.basename(template_target)
- template_name = template.get('source', default_template)
- output_file = render_template(template_name, contexts)
- file_properties = template.get('file_properties')
- install(output_file, template_target, file_properties)
- os.unlink(output_file)
- else:
- restart = False
-
- if restart:
- host.service_restart(service_name)
-
-
-def stop_services():
- global SERVICE_CONFIG
- for service in SERVICE_CONFIG:
- if host.service_running(service['service']):
- host.service_stop(service['service'])
-
-
-def get_service(service_name):
- global SERVICE_CONFIG
- for service in SERVICE_CONFIG:
- if service_name == service['service']:
- return service
- return None
-
-
-def reconfigure_services(restart=True):
- for service in SERVICE_CONFIG:
- reconfigure_service(service['service'], restart=restart)
=== removed file 'charmhelpers/contrib/cloudfoundry/upstart_helper.py'
--- charmhelpers/contrib/cloudfoundry/upstart_helper.py 2014-04-14 09:50:23 +0000
+++ charmhelpers/contrib/cloudfoundry/upstart_helper.py 1970-01-01 00:00:00 +0000
@@ -1,14 +0,0 @@
-import os
-import glob
-from charmhelpers.core import hookenv
-from charmhelpers.core.hookenv import charm_dir
-from charmhelpers.contrib.cloudfoundry.install import install
-
-
-def install_upstart_scripts(dirname=os.path.join(hookenv.charm_dir(),
- 'files/upstart'),
- pattern='*.conf'):
- for script in glob.glob("%s/%s" % (dirname, pattern)):
- filename = os.path.join(dirname, script)
- hookenv.log('Installing upstart job:' + filename, hookenv.DEBUG)
- install(filename, '/etc/init')
=== modified file 'charmhelpers/core/host.py'
--- charmhelpers/core/host.py 2014-02-26 15:53:02 +0000
+++ charmhelpers/core/host.py 2014-05-16 22:13:17 +0000
@@ -12,6 +12,8 @@
import string
import subprocess
import hashlib
+import shutil
+from contextlib import contextmanager
from collections import OrderedDict
@@ -143,6 +145,16 @@
target.write(content)
+def copy_file(src, dst, owner='root', group='root', perms=0444):
+ """Create or overwrite a file with the contents of another file"""
+ log("Writing file {} {}:{} {:o} from {}".format(dst, owner, group, perms, src))
+ uid = pwd.getpwnam(owner).pw_uid
+ gid = grp.getgrnam(group).gr_gid
+ shutil.copyfile(src, dst)
+ os.chown(dst, uid, gid)
+ os.chmod(dst, perms)
+
+
def mount(device, mountpoint, options=None, persist=False):
"""Mount a filesystem at a particular mountpoint"""
cmd_args = ['mount']
@@ -295,3 +307,28 @@
if 'link/ether' in words:
hwaddr = words[words.index('link/ether') + 1]
return hwaddr
+
+
+@contextmanager
+def chdir(d):
+ cur = os.getcwd()
+ try:
+ yield os.chdir(d)
+ finally:
+ os.chdir(cur)
+
+
+def chownr(path, owner, group, fatal=False):
+ uid = pwd.getpwnam(owner).pw_uid
+ gid = grp.getgrnam(group).gr_gid
+
+ def _raise(e):
+ raise e
+
+ for root, dirs, files in os.walk(path, onerror=_raise if fatal else None):
+ for name in dirs + files:
+ try:
+ os.chown(os.path.join(root, name), uid, gid)
+ except Exception:
+ if fatal:
+ raise
=== added file 'charmhelpers/core/services.py'
--- charmhelpers/core/services.py 1970-01-01 00:00:00 +0000
+++ charmhelpers/core/services.py 2014-05-16 22:13:17 +0000
@@ -0,0 +1,84 @@
+from charmhelpers.core import templating
+from charmhelpers.core import host
+
+
+SERVICES = {}
+
+
+def register(services, templates_dir=None):
+ """
+ Register a list of service configs.
+
+ Service Configs are dicts in the following formats:
+
+ {
+ "service": <service name>,
+ "templates": [ {
+ 'target': <render target of template>,
+ 'source': <optional name of template in passed in templates_dir>
+ 'file_properties': <optional dict taking owner and octal mode>
+ 'contexts': [ context generators, see contexts.py ]
+ }
+ ] }
+
+ Either `source` or `target` must be provided.
+
+ If 'source' is not provided for a template the templates_dir will
+ be consulted for ``basename(target).j2``.
+
+ If `target` is not provided, it will be assumed to be
+ ``/etc/init/<service name>.conf``.
+ """
+ for service in services:
+ service.setdefault('templates_dir', templates_dir)
+ SERVICES[service['service']] = service
+
+
+def reconfigure_services(restart=True):
+ """
+ Update all files for all services and optionally restart them, if ready.
+ """
+ for service_name in SERVICES.keys():
+ reconfigure_service(service_name, restart=restart)
+
+
+def reconfigure_service(service_name, restart=True):
+ """
+ Update all files for a single service and optionally restart it, if ready.
+ """
+ service = SERVICES.get(service_name)
+ if not service or service['service'] != service_name:
+ raise KeyError('Service not registered: %s' % service_name)
+
+ manager_type = service.get('type', UpstartService)
+ manager_type(service).reconfigure(restart)
+
+
+def stop_services():
+ for service_name in SERVICES.keys():
+ if host.service_running(service_name):
+ host.service_stop(service_name)
+
+
+class ServiceTypeManager(object):
+ def __init__(self, service_definition):
+ self.service_name = service_definition['service']
+ self.templates = service_definition['templates']
+ self.templates_dir = service_definition['templates_dir']
+
+ def reconfigure(self, restart=True):
+ raise NotImplementedError()
+
+
+class UpstartService(ServiceTypeManager):
+ def __init__(self, service_definition):
+ super(UpstartService, self).__init__(service_definition)
+ for tmpl in self.templates:
+ if 'target' not in tmpl:
+ tmpl['target'] = '/etc/init/%s.conf' % self.service_name
+
+ def reconfigure(self, restart):
+ complete = templating.render(self.templates, self.templates_dir)
+
+ if restart and complete:
+ host.service_restart(self.service_name)
=== added file 'charmhelpers/core/templating.py'
--- charmhelpers/core/templating.py 1970-01-01 00:00:00 +0000
+++ charmhelpers/core/templating.py 2014-05-16 22:13:17 +0000
@@ -0,0 +1,159 @@
+import os
+import yaml
+
+from charmhelpers.core import host
+from charmhelpers.core import hookenv
+
+LOADER = None
+
+
+class ContextGenerator(object):
+ """
+ Base interface for template context container generators.
+
+ A template context is a dictionary that contains data needed to populate
+ the template. The generator instance should produce the context when
+ called (without arguments) by collecting information from juju (config-get,
+ relation-get, etc), the system, or whatever other sources are appropriate.
+
+ A context generator should only return any values if it has enough information
+ to provide all of its values. Any context that is missing data is considered
+ incomplete and will cause that template to not render until it has all of its
+ necessary data.
+
+ The template may receive several contexts, which will be merged together,
+ so care should be taken in the key names.
+ """
+ def __call__(self):
+ raise NotImplementedError
+
+
+class StorableContext(object):
+ """
+ A mixin for persisting a context to disk.
+ """
+ def store_context(self, file_name, config_data):
+ with open(file_name, 'w') as file_stream:
+ yaml.dump(config_data, file_stream)
+
+ def read_context(self, file_name):
+ with open(file_name, 'r') as file_stream:
+ data = yaml.load(file_stream)
+ if not data:
+ raise OSError("%s is empty" % file_name)
+ return data
+
+
+class ConfigContext(ContextGenerator):
+ """
+ A context generator that generates a context containing all of the
+ juju config values.
+ """
+ def __call__(self):
+ return hookenv.config()
+
+
+class RelationContext(ContextGenerator):
+ """
+ Base class for a context generator that gets relation data from juju.
+
+ Subclasses must provide `interface`, which is the interface type of interest,
+ and `required_keys`, which is the set of keys required for the relation to
+ be considered complete. The first relation for the interface that is complete
+ will be used to populate the data for template.
+
+ The generated context will be namespaced under the interface type, to prevent
+ potential naming conflicts.
+ """
+ interface = None
+ required_keys = []
+
+ def __call__(self):
+ if not hookenv.relation_ids(self.interface):
+ return {}
+
+ ctx = {}
+ for rid in hookenv.relation_ids(self.interface):
+ for unit in hookenv.related_units(rid):
+ reldata = hookenv.relation_get(rid=rid, unit=unit)
+ required = set(self.required_keys)
+ if set(reldata.keys()).issuperset(required):
+ ns = ctx.setdefault(self.interface, {})
+ for k, v in reldata.items():
+ ns[k] = v
+ return ctx
+
+ return {}
+
+
+class StaticContext(ContextGenerator):
+ def __init__(self, data):
+ self.data = data
+
+ def __call__(self):
+ return self.data
+
+
+def _collect_contexts(context_providers):
+ """
+ Helper function to collect and merge contexts from a list of providers.
+
+ If any of the contexts are incomplete (i.e., they return an empty dict),
+ the template is considered incomplete and will not render.
+ """
+ ctx = {}
+ for provider in context_providers:
+ c = provider()
+ if not c:
+ return False
+ ctx.update(c)
+ return ctx
+
+
+def render(template_definitions, templates_dir=None):
+ """
+ Render one or more templates, given a list of template definitions.
+
+ The template definitions should be dicts with the keys: `source`, `target`,
+ `file_properties`, and `contexts`.
+
+ The `source` path, if not absolute, is relative to the `templates_dir`
+ given when the rendered was created. If `source` is not provided
+ for a template the `template_dir` will be consulted for
+ ``basename(target).j2``.
+
+ The `target` path should be absolute.
+
+ The `file_properties` should be a dict optionally containing
+ `owner`, `group`, or `perms` options, to be passed to `write_file`.
+
+ The `contexts` should be a list containing zero or more ContextGenerators.
+
+ The `template_dir` defaults to `$CHARM_DIR/templates`
+
+ Returns True if all of the templates were "complete" (i.e., the context
+ generators were able to collect the information needed to render the
+ template) and were rendered.
+ """
+ from jinja2 import FileSystemLoader, Environment, exceptions
+ all_complete = True
+ if templates_dir is None:
+ templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+ loader = Environment(loader=FileSystemLoader(templates_dir))
+ for tmpl in template_definitions:
+ ctx = _collect_contexts(tmpl.get('contexts', []))
+ if ctx is False:
+ all_complete = False
+ continue
+ try:
+ source = tmpl.get('source', os.path.basename(tmpl['target'])+'.j2')
+ template = loader.get_template(source)
+ except exceptions.TemplateNotFound as e:
+ hookenv.log('Could not load template %s from %s.' %
+ (tmpl['source'], templates_dir),
+ level=hookenv.ERROR)
+ raise e
+ content = template.render(ctx)
+ host.mkdir(os.path.dirname(tmpl['target']))
+ host.write_file(tmpl['target'], content, **tmpl.get('file_properties', {}))
+ return all_complete
=== removed directory 'tests/contrib/cloudfoundry/templates'
=== removed file 'tests/contrib/cloudfoundry/templates/cloud_controller_ng.yml'
--- tests/contrib/cloudfoundry/templates/cloud_controller_ng.yml 2014-03-26 17:38:14 +0000
+++ tests/contrib/cloudfoundry/templates/cloud_controller_ng.yml 1970-01-01 00:00:00 +0000
@@ -1,173 +0,0 @@
----
-# TODO cc_ip cc public ip
-local_route: {{ domain }}
-port: {{ cc_port }}
-pid_filename: /var/vcap/sys/run/cloud_controller_ng/cloud_controller_ng.pid
-development_mode: false
-
-message_bus_servers:
- - nats://{{ nats_user }}:{{ nats_password }}@{{ nats_address }}:{{ nats_port }}
-
-external_domain:
- - api.{{ domain }}
-
-system_domain_organization: {{ default_organization }}
-system_domain: {{ domain }}
-app_domains: [ {{ domain }} ]
-srv_api_uri: http://api.{{ domain }}
-
-default_app_memory: 1024
-
-cc_partition: default
-
-bootstrap_admin_email: admin@{{ default_organization }}
-
-bulk_api:
- auth_user: bulk_api
- auth_password: "Password"
-
-nginx:
- use_nginx: false
- instance_socket: "/var/vcap/sys/run/cloud_controller_ng/cloud_controller.sock"
-
-index: 1
-name: cloud_controller_ng
-
-info:
- name: vcap
- build: "2222"
- version: 2
- support_address: http://support.cloudfoundry.com
- description: Cloud Foundry sponsored by Pivotal
- api_version: 2.0.0
-
-
-directories:
- tmpdir: /var/vcap/data/cloud_controller_ng/tmp
-
-
-logging:
- file: /var/vcap/sys/log/cloud_controller_ng/cloud_controller_ng.log
-
- syslog: vcap.cloud_controller_ng
-
- level: debug2
- max_retries: 1
-
-
-
-
-
-db: &db
- database: sqlite:///var/lib/cloudfoundry/cfcloudcontroller/db/cc.db
- max_connections: 25
- pool_timeout: 10
- log_level: debug2
-
-
-login:
- url: http://uaa.{{ domain }}
-
-uaa:
- url: http://uaa.{{ domain }}
- resource_id: cloud_controller
- #symmetric_secret: cc-secret
- verification_key: |
- -----BEGIN PUBLIC KEY-----
- MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDHFr+KICms+tuT1OXJwhCUmR2d
- KVy7psa8xzElSyzqx7oJyfJ1JZyOzToj9T5SfTIq396agbHJWVfYphNahvZ/7uMX
- qHxf+ZH9BL1gk9Y6kCnbM5R60gfwjyW1/dQPjOzn9N394zd2FJoFHwdq9Qs0wBug
- spULZVNRxq7veq/fzwIDAQAB
- -----END PUBLIC KEY-----
-
-# App staging parameters
-staging:
- max_staging_runtime: 900
- auth:
- user:
- password: "Password"
-
-maximum_health_check_timeout: 180
-
-runtimes_file: /var/lib/cloudfoundry/cfcloudcontroller/jobs/config/runtimes.yml
-stacks_file: /var/lib/cloudfoundry/cfcloudcontroller/jobs/config/stacks.yml
-
-quota_definitions:
- free:
- non_basic_services_allowed: false
- total_services: 2
- total_routes: 1000
- memory_limit: 1024
- paid:
- non_basic_services_allowed: true
- total_services: 32
- total_routes: 1000
- memory_limit: 204800
- runaway:
- non_basic_services_allowed: true
- total_services: 500
- total_routes: 1000
- memory_limit: 204800
- trial:
- non_basic_services_allowed: false
- total_services: 10
- memory_limit: 2048
- total_routes: 1000
- trial_db_allowed: true
-
-default_quota_definition: free
-
-resource_pool:
- minimum_size: 65536
- maximum_size: 536870912
- resource_directory_key: cc-resources
-
- cdn:
- uri:
- key_pair_id:
- private_key: ""
-
- fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"}
-
-packages:
- app_package_directory_key: cc-packages
-
- cdn:
- uri:
- key_pair_id:
- private_key: ""
-
- fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"}
-
-droplets:
- droplet_directory_key: cc-droplets
-
- cdn:
- uri:
- key_pair_id:
- private_key: ""
-
- fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"}
-
-buildpacks:
- buildpack_directory_key: cc-buildpacks
-
- cdn:
- uri:
- key_pair_id:
- private_key: ""
-
- fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"}
-
-db_encryption_key: Password
-
-trial_db:
- guid: "78ad16cf-3c22-4427-a982-b9d35d746914"
-
-tasks_disabled: false
-hm9000_noop: true
-flapping_crash_count_threshold: 3
-
-disable_custom_buildpacks: false
-
-broker_client_timeout_seconds: 60
=== removed file 'tests/contrib/cloudfoundry/templates/fake_cc.yml'
--- tests/contrib/cloudfoundry/templates/fake_cc.yml 2014-03-28 22:28:36 +0000
+++ tests/contrib/cloudfoundry/templates/fake_cc.yml 1970-01-01 00:00:00 +0000
@@ -1,3 +0,0 @@
-host: {{nats['nats_host']}}
-port: {{nats['nats_port']}}
-domain: {{router['domain']}}
=== removed file 'tests/contrib/cloudfoundry/templates/nginx.conf'
--- tests/contrib/cloudfoundry/templates/nginx.conf 2014-03-26 17:38:14 +0000
+++ tests/contrib/cloudfoundry/templates/nginx.conf 1970-01-01 00:00:00 +0000
@@ -1,154 +0,0 @@
-# deployment cloudcontroller nginx.conf
-#user vcap vcap;
-
-error_log /var/vcap/sys/log/nginx_ccng/nginx.error.log;
-pid /var/vcap/sys/run/nginx_ccng/nginx.pid;
-
-events {
- worker_connections 8192;
- use epoll;
-}
-
-http {
- include mime.types;
- default_type text/html;
- server_tokens off;
- variables_hash_max_size 1024;
-
- log_format main '$host - [$time_local] '
- '"$request" $status $bytes_sent '
- '"$http_referer" "$http_#user_agent" '
- '$proxy_add_x_forwarded_for response_time:$upstream_response_time';
-
- access_log /var/vcap/sys/log/nginx_ccng/nginx.access.log main;
-
- sendfile on; #enable use of sendfile()
- tcp_nopush on;
- tcp_nodelay on; #disable nagel's algorithm
-
- keepalive_timeout 75 20; #inherited from router
-
- client_max_body_size 256M; #already enforced upstream/but doesn't hurt.
-
- upstream cloud_controller {
- server unix:/var/vcap/sys/run/cloud_controller_ng/cloud_controller.sock;
- }
-
- server {
- listen {{ nginx_port }};
- server_name _;
- server_name_in_redirect off;
- proxy_send_timeout 300;
- proxy_read_timeout 300;
-
- # proxy and log all CC traffic
- location / {
- access_log /var/vcap/sys/log/nginx_ccng/nginx.access.log main;
- proxy_buffering off;
- proxy_set_header Host $host;
- proxy_set_header X-Real_IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_redirect off;
- proxy_connect_timeout 10;
- proxy_pass http://cloud_controller;
- }
-
-
- # used for x-accel-redirect uri://location/foo.txt
- # nginx will serve the file root || location || foo.txt
- location /droplets/ {
- internal;
- root /var/vcap/nfs/store;
- }
-
-
-
- # used for x-accel-redirect uri://location/foo.txt
- # nginx will serve the file root || location || foo.txt
- location /cc-packages/ {
- internal;
- root /var/vcap/nfs/store;
- }
-
-
- # used for x-accel-redirect uri://location/foo.txt
- # nginx will serve the file root || location || foo.txt
- location /cc-droplets/ {
- internal;
- root /var/vcap/nfs/store;
- }
-
-
- location ~ (/apps/.*/application|/v2/apps/.*/bits|/services/v\d+/configurations/.*/serialized/data|/v2/buildpacks/.*/bits) {
- # Pass altered request body to this location
- upload_pass @cc_uploads;
- upload_pass_args on;
-
- # Store files to this directory
- upload_store /var/vcap/data/cloud_controller_ng/tmp/uploads;
-
- # No limit for output body forwarded to CC
- upload_max_output_body_len 0;
-
- # Allow uploaded files to be read only by #user
- #upload_store_access #user:r;
-
- # Set specified fields in request body
- upload_set_form_field "${upload_field_name}_name" $upload_file_name;
- upload_set_form_field "${upload_field_name}_path" $upload_tmp_path;
-
- #forward the following fields from existing body
- upload_pass_form_field "^resources$";
- upload_pass_form_field "^_method$";
-
- #on any error, delete uploaded files.
- upload_cleanup 400-505;
- }
-
- location ~ /staging/(buildpack_cache|droplets)/.*/upload {
-
- # Allow download the droplets and buildpacks
- if ($request_method = GET){
- proxy_pass http://cloud_controller;
- }
-
- # Pass along auth header
- set $auth_header $upstream_http_x_auth;
- proxy_set_header Authorization $auth_header;
-
- # Pass altered request body to this location
- upload_pass @cc_uploads;
-
- # Store files to this directory
- upload_store /var/vcap/data/cloud_controller_ng/tmp/staged_droplet_uploads;
-
- # Allow uploaded files to be read only by #user
- upload_store_access user:r;
-
- # Set specified fields in request body
- upload_set_form_field "droplet_path" $upload_tmp_path;
-
- #on any error, delete uploaded files.
- upload_cleanup 400-505;
- }
-
- # Pass altered request body to a backend
- location @cc_uploads {
- proxy_pass http://unix:/var/vcap/sys/run/cloud_controller_ng/cloud_controller.sock;
- }
-
- location ~ ^/internal_redirect/(.*){
- # only allow internal redirects
- internal;
-
- set $download_url $1;
-
- #have to manualy pass along auth header
- set $auth_header $upstream_http_x_auth;
- proxy_set_header Authorization $auth_header;
-
- # Download the file and send it to client
- proxy_pass $download_url;
- }
- }
-}
=== removed file 'tests/contrib/cloudfoundry/templates/test.conf'
--- tests/contrib/cloudfoundry/templates/test.conf 2014-03-26 17:38:14 +0000
+++ tests/contrib/cloudfoundry/templates/test.conf 1970-01-01 00:00:00 +0000
@@ -1,3 +0,0 @@
-something
-listen {{nginx_port}}
-something else
=== removed file 'tests/contrib/cloudfoundry/test_config_helper.py'
--- tests/contrib/cloudfoundry/test_config_helper.py 2014-03-30 16:40:45 +0000
+++ tests/contrib/cloudfoundry/test_config_helper.py 1970-01-01 00:00:00 +0000
@@ -1,33 +0,0 @@
-import unittest
-
-from pkg_resources import resource_filename
-
-from charmhelpers.contrib.cloudfoundry.config_helper import render_template
-
-TEMPLATE_DIR = "templates"
-
-
-def template_dir():
- return resource_filename(__name__, TEMPLATE_DIR)
-
-
-class TestConfigHelper(unittest.TestCase):
- def test_render_ngnix(self):
- output = render_template('nginx.conf',
- dict(nginx_port=12345),
- template_dir=template_dir())
-
- self.assertRegexpMatches(output, 'listen\s+12345')
-
- def test_render_cloudcontroller_conf(self):
- output = render_template('cloud_controller_ng.yml',
- dict(cc_port="8080", domain="example.net"),
- template_dir=template_dir())
-
- self.assertRegexpMatches(output, 'port:\s*8080')
- self.assertRegexpMatches(output,
- 'srv_api_uri:\s*http://api.example.net')
-
-
-if __name__ == '__main__':
- unittest.main()
=== removed file 'tests/contrib/cloudfoundry/test_install.py'
--- tests/contrib/cloudfoundry/test_install.py 2014-03-30 16:50:30 +0000
+++ tests/contrib/cloudfoundry/test_install.py 1970-01-01 00:00:00 +0000
@@ -1,47 +0,0 @@
-import mock
-import os
-import shutil
-import tempfile
-import unittest
-import pkg_resources
-
-from charmhelpers.contrib.cloudfoundry.install import install
-
-FILES_DIR = pkg_resources.resource_filename(__name__, 'files')
-
-
-class TestInstall(unittest.TestCase):
- def setUp(self):
- self.testdir = tempfile.mkdtemp()
-
- def tearDown(self):
- shutil.rmtree(self.testdir)
-
- def test_install_no_src(self):
- self.assertRaises(OSError, install, 'missing', '/tmp')
-
- @mock.patch('subprocess.check_call')
- def test_install_use_sudo(self, mcall):
- src = os.path.join(FILES_DIR, 'test.conf')
- dest = os.path.join(self.testdir, 'bar')
- install(src, dest, {'owner': 'root'}, sudo=True)
- self.assertEqual(mcall.mock_calls[0],
- mock.call(['sudo', 'install', '-m', '400',
- '-o', 'root', src, dest]))
-
- @mock.patch('subprocess.check_call')
- def test_install_create_dir(self, mcall):
- src = os.path.join(FILES_DIR, 'test.conf')
- dest = os.path.join(self.testdir, '1/2/3/file')
- install(src, dest)
- self.assertEqual(mcall.call_args,
- mock.call(['install', '-D', '-m', '400',
- src, dest]))
-
- def test_install_missing_file(self):
- self.assertRaises(OSError, install,
- 'files/missing.conf', self.testdir)
-
-
-if __name__ == '__main__':
- unittest.main()
=== modified file 'tests/contrib/cloudfoundry/test_render_context.py'
--- tests/contrib/cloudfoundry/test_render_context.py 2014-04-11 06:49:27 +0000
+++ tests/contrib/cloudfoundry/test_render_context.py 2014-05-16 22:13:17 +0000
@@ -57,13 +57,6 @@
self.assertEqual(n(), expected)
-class TestConfigContext(unittest.TestCase):
- @mock.patch('charmhelpers.core.hookenv.config')
- def test_config_context(self, mconfig):
- contexts.ConfigContext()()
- self.assertTrue(mconfig.called)
-
-
class TestStoredContext(unittest.TestCase):
def test_context_saving(self):
@@ -88,10 +81,5 @@
contexts.StoredContext(file_name, {'key': 'initial_value'})
os.unlink(file_name)
- def test_static_context(self):
- a = contexts.StaticContext('a')
- self.assertEqual(a.data, 'a')
- self.assertEqual(a(), 'a')
-
if __name__ == '__main__':
unittest.main()
=== removed file 'tests/contrib/cloudfoundry/test_services.py'
--- tests/contrib/cloudfoundry/test_services.py 2014-04-11 06:49:27 +0000
+++ tests/contrib/cloudfoundry/test_services.py 1970-01-01 00:00:00 +0000
@@ -1,141 +0,0 @@
-import pkg_resources
-import mock
-import unittest
-from charmhelpers.contrib.cloudfoundry import services
-
-CF = 'charmhelpers.contrib.cloudfoundry'
-SERVICES = CF + '.services'
-TEMPLATES_DIR = pkg_resources.resource_filename(__name__, 'templates')
-
-# Test methods for context collection
-def noop():
- return {}
-
-
-def found():
- return {'foo': 'bar'}
-
-
-def static_content(data):
- def context():
- return data
- return context
-
-
-# Sample render context used to test rendering paths
-default_context = {
- 'nats': {
- 'nats_port': '1234',
- 'nats_host': 'example.com',
- },
- 'router': {
- 'domain': 'api.foo.com'
- }
-}
-
-
-# Method returning a mutable copy of a default config
-# for services.register()
-def default_service_config():
- return [{
- 'service': 'cf-cloudcontroller',
- 'templates': [
- {'target': 'cc.yml',
- 'source': 'fake_cc.yml',
- 'file_properties': {},
- 'contexts': [static_content(default_context)]
- }]
- }]
-
-
-class TestService(unittest.TestCase):
- def setUp(self):
- services.reset()
-
- @mock.patch(SERVICES + '.reconfigure_service')
- def test_reconfigure_services(self, mreconfig):
- services.register(default_service_config(), TEMPLATES_DIR)
- services.reconfigure_services()
- # Verify that we called reconfigure_service(name, restart=True)
- self.assertEqual(mreconfig.call_args[0][0], 'cf-cloudcontroller')
- self.assertTrue(mreconfig.call_args[1]['restart'])
-
- @mock.patch(SERVICES + '.install')
- @mock.patch('charmhelpers.core.host.service_restart')
- def test_reconfigure_service_restart(self, mrestart, minstall):
- services.register(default_service_config(), TEMPLATES_DIR)
- services.reconfigure_service('cf-cloudcontroller', restart=True)
- # Verify that we called service_restart(name)
- self.assertEqual(mrestart.call_args[0][0], 'cf-cloudcontroller')
- self.assertTrue(mrestart.called)
-
- @mock.patch('os.unlink')
- @mock.patch(SERVICES + '.render_template')
- @mock.patch(SERVICES + '.install')
- def test_reconfigure_service_no_source(self, minstall, mrender, munlink):
- config = default_service_config()
- # erase the contexts
- config[0]['templates'][0]['contexts'] = {}
- services.register(config, TEMPLATES_DIR)
- services.reconfigure_service('cf-cloudcontroller', restart=True)
- # verify that we not called render template
- self.assertFalse(mrender.called)
- self.assertFalse(minstall.called)
- self.assertFalse(munlink.called)
-
- @mock.patch('os.unlink')
- @mock.patch(SERVICES + '.render_template')
- @mock.patch(SERVICES + '.install')
- def test_reconfigure_service_no_context(self, minstall, mrender, munlink):
- config = default_service_config()
- # delete the source so the lookup pattern is used
- del config[0]['templates'][0]['source']
- services.register(config, TEMPLATES_DIR)
- services.reconfigure_service('cf-cloudcontroller', restart=False)
- # verify that we called render template with the expected name
- self.assertEqual(mrender.call_args[0][0], 'cc.yml.j2')
- self.assertTrue(minstall.called)
- self.assertTrue(munlink.called)
- self.assertRaises(KeyError, services.reconfigure_service, 'unknownservice', restart=False)
-
-
- def test_render_template(self):
- services.configure_templates(TEMPLATES_DIR)
- output = services.render_template('fake_cc.yml', default_context)
- contents = open(output).read()
- self.assertRegexpMatches(contents, 'port: 1234')
- self.assertRegexpMatches(contents, 'host: example.com')
- self.assertRegexpMatches(contents, 'domain: api.foo.com')
-
- def test_collect_contexts_fail(self):
- cc = services.collect_contexts
- self.assertEqual(cc([]), {})
- self.assertEqual(cc([noop]), {})
- self.assertEqual(cc([noop, noop]), {})
- self.assertEqual(cc([noop, found]), {})
-
- def test_collect_contexts_found(self):
- cc = services.collect_contexts
- expected = {'foo': 'bar'}
- self.assertEqual(cc([found]), expected)
- self.assertEqual(cc([found, found]), expected)
-
- @mock.patch('charmhelpers.core.host.service_stop')
- @mock.patch('charmhelpers.core.host.service_running')
- def test_stop_services(self, mrunning, mstop):
- services.register(default_service_config(), TEMPLATES_DIR)
- services.stop_services()
- self.assertTrue(mrunning.call_args[0], 'cf-cloudcontroller')
- self.assertTrue(mstop.call_args[0], 'cf-cloudcontroller')
-
- def test_get_service(self):
- services.register(default_service_config(), TEMPLATES_DIR)
- service = services.get_service('cf-cloudcontroller')
- self.assertEqual(service['service'], 'cf-cloudcontroller')
-
- def test_get_service_none(self):
- services.register(default_service_config(), TEMPLATES_DIR)
- self.assertIsNone(services.get_service('non_existing_one'))
-
-if __name__ == '__main__':
- unittest.main()
=== added directory 'tests/core/templates'
=== added file 'tests/core/templates/cloud_controller_ng.yml'
--- tests/core/templates/cloud_controller_ng.yml 1970-01-01 00:00:00 +0000
+++ tests/core/templates/cloud_controller_ng.yml 2014-05-16 22:13:17 +0000
@@ -0,0 +1,173 @@
+---
+# TODO cc_ip cc public ip
+local_route: {{ domain }}
+port: {{ cc_port }}
+pid_filename: /var/vcap/sys/run/cloud_controller_ng/cloud_controller_ng.pid
+development_mode: false
+
+message_bus_servers:
+ - nats://{{ nats_user }}:{{ nats_password }}@{{ nats_address }}:{{ nats_port }}
+
+external_domain:
+ - api.{{ domain }}
+
+system_domain_organization: {{ default_organization }}
+system_domain: {{ domain }}
+app_domains: [ {{ domain }} ]
+srv_api_uri: http://api.{{ domain }}
+
+default_app_memory: 1024
+
+cc_partition: default
+
+bootstrap_admin_email: admin@{{ default_organization }}
+
+bulk_api:
+ auth_user: bulk_api
+ auth_password: "Password"
+
+nginx:
+ use_nginx: false
+ instance_socket: "/var/vcap/sys/run/cloud_controller_ng/cloud_controller.sock"
+
+index: 1
+name: cloud_controller_ng
+
+info:
+ name: vcap
+ build: "2222"
+ version: 2
+ support_address: http://support.cloudfoundry.com
+ description: Cloud Foundry sponsored by Pivotal
+ api_version: 2.0.0
+
+
+directories:
+ tmpdir: /var/vcap/data/cloud_controller_ng/tmp
+
+
+logging:
+ file: /var/vcap/sys/log/cloud_controller_ng/cloud_controller_ng.log
+
+ syslog: vcap.cloud_controller_ng
+
+ level: debug2
+ max_retries: 1
+
+
+
+
+
+db: &db
+ database: sqlite:///var/lib/cloudfoundry/cfcloudcontroller/db/cc.db
+ max_connections: 25
+ pool_timeout: 10
+ log_level: debug2
+
+
+login:
+ url: http://uaa.{{ domain }}
+
+uaa:
+ url: http://uaa.{{ domain }}
+ resource_id: cloud_controller
+ #symmetric_secret: cc-secret
+ verification_key: |
+ -----BEGIN PUBLIC KEY-----
+ MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDHFr+KICms+tuT1OXJwhCUmR2d
+ KVy7psa8xzElSyzqx7oJyfJ1JZyOzToj9T5SfTIq396agbHJWVfYphNahvZ/7uMX
+ qHxf+ZH9BL1gk9Y6kCnbM5R60gfwjyW1/dQPjOzn9N394zd2FJoFHwdq9Qs0wBug
+ spULZVNRxq7veq/fzwIDAQAB
+ -----END PUBLIC KEY-----
+
+# App staging parameters
+staging:
+ max_staging_runtime: 900
+ auth:
+ user:
+ password: "Password"
+
+maximum_health_check_timeout: 180
+
+runtimes_file: /var/lib/cloudfoundry/cfcloudcontroller/jobs/config/runtimes.yml
+stacks_file: /var/lib/cloudfoundry/cfcloudcontroller/jobs/config/stacks.yml
+
+quota_definitions:
+ free:
+ non_basic_services_allowed: false
+ total_services: 2
+ total_routes: 1000
+ memory_limit: 1024
+ paid:
+ non_basic_services_allowed: true
+ total_services: 32
+ total_routes: 1000
+ memory_limit: 204800
+ runaway:
+ non_basic_services_allowed: true
+ total_services: 500
+ total_routes: 1000
+ memory_limit: 204800
+ trial:
+ non_basic_services_allowed: false
+ total_services: 10
+ memory_limit: 2048
+ total_routes: 1000
+ trial_db_allowed: true
+
+default_quota_definition: free
+
+resource_pool:
+ minimum_size: 65536
+ maximum_size: 536870912
+ resource_directory_key: cc-resources
+
+ cdn:
+ uri:
+ key_pair_id:
+ private_key: ""
+
+ fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"}
+
+packages:
+ app_package_directory_key: cc-packages
+
+ cdn:
+ uri:
+ key_pair_id:
+ private_key: ""
+
+ fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"}
+
+droplets:
+ droplet_directory_key: cc-droplets
+
+ cdn:
+ uri:
+ key_pair_id:
+ private_key: ""
+
+ fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"}
+
+buildpacks:
+ buildpack_directory_key: cc-buildpacks
+
+ cdn:
+ uri:
+ key_pair_id:
+ private_key: ""
+
+ fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"}
+
+db_encryption_key: Password
+
+trial_db:
+ guid: "78ad16cf-3c22-4427-a982-b9d35d746914"
+
+tasks_disabled: false
+hm9000_noop: true
+flapping_crash_count_threshold: 3
+
+disable_custom_buildpacks: false
+
+broker_client_timeout_seconds: 60
=== added file 'tests/core/templates/fake_cc.yml'
--- tests/core/templates/fake_cc.yml 1970-01-01 00:00:00 +0000
+++ tests/core/templates/fake_cc.yml 2014-05-16 22:13:17 +0000
@@ -0,0 +1,3 @@
+host: {{nats['nats_host']}}
+port: {{nats['nats_port']}}
+domain: {{router['domain']}}
=== added file 'tests/core/templates/nginx.conf'
--- tests/core/templates/nginx.conf 1970-01-01 00:00:00 +0000
+++ tests/core/templates/nginx.conf 2014-05-16 22:13:17 +0000
@@ -0,0 +1,154 @@
+# deployment cloudcontroller nginx.conf
+#user vcap vcap;
+
+error_log /var/vcap/sys/log/nginx_ccng/nginx.error.log;
+pid /var/vcap/sys/run/nginx_ccng/nginx.pid;
+
+events {
+ worker_connections 8192;
+ use epoll;
+}
+
+http {
+ include mime.types;
+ default_type text/html;
+ server_tokens off;
+ variables_hash_max_size 1024;
+
+ log_format main '$host - [$time_local] '
+ '"$request" $status $bytes_sent '
+ '"$http_referer" "$http_#user_agent" '
+ '$proxy_add_x_forwarded_for response_time:$upstream_response_time';
+
+ access_log /var/vcap/sys/log/nginx_ccng/nginx.access.log main;
+
+ sendfile on; #enable use of sendfile()
+ tcp_nopush on;
+ tcp_nodelay on; #disable nagel's algorithm
+
+ keepalive_timeout 75 20; #inherited from router
+
+ client_max_body_size 256M; #already enforced upstream/but doesn't hurt.
+
+ upstream cloud_controller {
+ server unix:/var/vcap/sys/run/cloud_controller_ng/cloud_controller.sock;
+ }
+
+ server {
+ listen {{ nginx_port }};
+ server_name _;
+ server_name_in_redirect off;
+ proxy_send_timeout 300;
+ proxy_read_timeout 300;
+
+ # proxy and log all CC traffic
+ location / {
+ access_log /var/vcap/sys/log/nginx_ccng/nginx.access.log main;
+ proxy_buffering off;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real_IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_redirect off;
+ proxy_connect_timeout 10;
+ proxy_pass http://cloud_controller;
+ }
+
+
+ # used for x-accel-redirect uri://location/foo.txt
+ # nginx will serve the file root || location || foo.txt
+ location /droplets/ {
+ internal;
+ root /var/vcap/nfs/store;
+ }
+
+
+
+ # used for x-accel-redirect uri://location/foo.txt
+ # nginx will serve the file root || location || foo.txt
+ location /cc-packages/ {
+ internal;
+ root /var/vcap/nfs/store;
+ }
+
+
+ # used for x-accel-redirect uri://location/foo.txt
+ # nginx will serve the file root || location || foo.txt
+ location /cc-droplets/ {
+ internal;
+ root /var/vcap/nfs/store;
+ }
+
+
+ location ~ (/apps/.*/application|/v2/apps/.*/bits|/services/v\d+/configurations/.*/serialized/data|/v2/buildpacks/.*/bits) {
+ # Pass altered request body to this location
+ upload_pass @cc_uploads;
+ upload_pass_args on;
+
+ # Store files to this directory
+ upload_store /var/vcap/data/cloud_controller_ng/tmp/uploads;
+
+ # No limit for output body forwarded to CC
+ upload_max_output_body_len 0;
+
+ # Allow uploaded files to be read only by #user
+ #upload_store_access #user:r;
+
+ # Set specified fields in request body
+ upload_set_form_field "${upload_field_name}_name" $upload_file_name;
+ upload_set_form_field "${upload_field_name}_path" $upload_tmp_path;
+
+ #forward the following fields from existing body
+ upload_pass_form_field "^resources$";
+ upload_pass_form_field "^_method$";
+
+ #on any error, delete uploaded files.
+ upload_cleanup 400-505;
+ }
+
+ location ~ /staging/(buildpack_cache|droplets)/.*/upload {
+
+ # Allow download the droplets and buildpacks
+ if ($request_method = GET){
+ proxy_pass http://cloud_controller;
+ }
+
+ # Pass along auth header
+ set $auth_header $upstream_http_x_auth;
+ proxy_set_header Authorization $auth_header;
+
+ # Pass altered request body to this location
+ upload_pass @cc_uploads;
+
+ # Store files to this directory
+ upload_store /var/vcap/data/cloud_controller_ng/tmp/staged_droplet_uploads;
+
+ # Allow uploaded files to be read only by #user
+ upload_store_access user:r;
+
+ # Set specified fields in request body
+ upload_set_form_field "droplet_path" $upload_tmp_path;
+
+ #on any error, delete uploaded files.
+ upload_cleanup 400-505;
+ }
+
+ # Pass altered request body to a backend
+ location @cc_uploads {
+ proxy_pass http://unix:/var/vcap/sys/run/cloud_controller_ng/cloud_controller.sock;
+ }
+
+ location ~ ^/internal_redirect/(.*){
+ # only allow internal redirects
+ internal;
+
+ set $download_url $1;
+
+ #have to manualy pass along auth header
+ set $auth_header $upstream_http_x_auth;
+ proxy_set_header Authorization $auth_header;
+
+ # Download the file and send it to client
+ proxy_pass $download_url;
+ }
+ }
+}
=== added file 'tests/core/templates/test.conf'
--- tests/core/templates/test.conf 1970-01-01 00:00:00 +0000
+++ tests/core/templates/test.conf 2014-05-16 22:13:17 +0000
@@ -0,0 +1,3 @@
+something
+listen {{nginx_port}}
+something else
=== modified file 'tests/core/test_host.py'
--- tests/core/test_host.py 2013-11-29 11:24:48 +0000
+++ tests/core/test_host.py 2014-05-16 22:13:17 +0000
@@ -435,6 +435,50 @@
os_.fchmod.assert_called_with(fileno, perms)
mock_file.write.assert_called_with('what is {juju}')
+ @patch('pwd.getpwnam')
+ @patch('grp.getgrnam')
+ @patch.object(host, 'log')
+ @patch.object(host, 'shutil')
+ @patch.object(host, 'os')
+ def test_copies_content_to_a_file(self, os_, shutil_, log, getgrnam, getpwnam):
+ # Curly brackets here demonstrate that we are *not* rendering
+ # these strings with Python's string formatting. This is a
+ # change from the original behavior per Bug #1195634.
+ uid = 123
+ gid = 234
+ owner = 'some-user-{foo}'
+ group = 'some-group-{bar}'
+ src = '/some/path/{baz}'
+ dst = '/some/other/path/{qux}'
+ perms = 0644
+
+ getpwnam.return_value.pw_uid = uid
+ getgrnam.return_value.gr_gid = gid
+
+ host.copy_file(src, dst, owner=owner, group=group, perms=perms)
+
+ getpwnam.assert_called_with('some-user-{foo}')
+ getgrnam.assert_called_with('some-group-{bar}')
+ shutil_.copyfile.assert_called_with(src, dst)
+ os_.chown.assert_called_with(dst, uid, gid)
+ os_.chmod.assert_called_with(dst, perms)
+
+ @patch.object(host, 'log')
+ @patch.object(host, 'shutil')
+ @patch.object(host, 'os')
+ def test_copies_content_with_default(self, os_, shutil_, log):
+ uid = 0
+ gid = 0
+ src = '/some/path/{baz}'
+ dst = '/some/other/path/{qux}'
+ perms = 0444
+
+ host.copy_file(src, dst)
+
+ shutil_.copyfile.assert_called_with(src, dst)
+ os_.chown.assert_called_with(dst, uid, gid)
+ os_.chmod.assert_called_with(dst, perms)
+
@patch('subprocess.check_output')
@patch.object(host, 'log')
def test_mounts_a_device(self, log, check_output):
@@ -690,3 +734,66 @@
nic = "eth0"
hwaddr = host.get_nic_hwaddr(nic)
self.assertEqual(hwaddr, 'e4:11:5b:ab:a7:3c')
+
+ @patch('os.getcwd')
+ @patch('os.chdir')
+ def test_chdir(self, chdir, getcwd):
+ getcwd.side_effect = ['quw', 'qar']
+ try:
+ with host.chdir('foo'):
+ pass
+ with host.chdir('bar'):
+ raise ValueError()
+ except ValueError:
+ pass
+ else:
+ assert False, 'Expected ValueError'
+ self.assertEqual(chdir.call_args_list, [
+ call('foo'), call('quw'),
+ call('bar'), call('qar'),
+ ])
+
+ @patch('grp.getgrnam')
+ @patch('pwd.getpwnam')
+ @patch('os.chown')
+ @patch('os.walk')
+ def test_chownr(self, walk, chown, getpwnam, getgrnam):
+ getpwnam.return_value.pw_uid = 1
+ getgrnam.return_value.gr_gid = 2
+ walk.return_value = [
+ ('root1', ['dir1', 'dir2'], ['file1', 'file2']),
+ ('root2', ['dir3'], ['file3']),
+ ]
+ host.chownr('path', 'owner', 'group')
+ getpwnam.assert_called_once_with('owner')
+ getgrnam.assert_called_once_with('group')
+ self.assertEqual(chown.call_args_list, [
+ call('root1/dir1', 1, 2),
+ call('root1/dir2', 1, 2),
+ call('root1/file1', 1, 2),
+ call('root1/file2', 1, 2),
+ call('root2/dir3', 1, 2),
+ call('root2/file3', 1, 2),
+ ])
+
+ @patch('grp.getgrnam')
+ @patch('pwd.getpwnam')
+ @patch('os.chown')
+ @patch('os.listdir')
+ def test_chownr_fatal(self, listdir, chown, getpwnam, getgrnam):
+ nonfatal = lambda: host.chownr('path', 'owner', 'group', fatal=False)
+ fatal = lambda: host.chownr('path', 'owner', 'group', fatal=True)
+ listdir.side_effect = OSError('not found')
+ nonfatal()
+ self.assertRaises(OSError, fatal)
+ self.assertFalse(chown.called)
+
+ listdir.side_effect = None
+ listdir.return_value = ['foo']
+ chown.side_effect = OSError('not found')
+ self.assertRaises(OSError, fatal)
+ chown.assert_called_once()
+
+ chown.reset_mock()
+ nonfatal()
+ chown.assert_called_once()
=== added file 'tests/core/test_services.py'
--- tests/core/test_services.py 1970-01-01 00:00:00 +0000
+++ tests/core/test_services.py 2014-05-16 22:13:17 +0000
@@ -0,0 +1,126 @@
+import pkg_resources
+import mock
+import unittest
+from charmhelpers.core import services
+
+TEMPLATES_DIR = pkg_resources.resource_filename(__name__, 'templates')
+
+
+def static_content(data):
+ def context():
+ return data
+ return context
+
+
+# Sample render context used to test rendering paths
+default_context = {
+ 'nats': {
+ 'nats_port': '1234',
+ 'nats_host': 'example.com',
+ },
+ 'router': {
+ 'domain': 'api.foo.com'
+ }
+}
+
+
+# Method returning a mutable copy of a default config
+# for services.register()
+def default_service_config():
+ return [{
+ 'service': 'cf-cloudcontroller',
+ 'templates': [
+ {'target': 'cc.yml',
+ 'source': 'fake_cc.yml',
+ 'file_properties': {},
+ 'contexts': [static_content(default_context)]
+ }]
+ }]
+
+
+class TestService(unittest.TestCase):
+ def setUp(self):
+ services.SERVICES = {}
+
+ @mock.patch('charmhelpers.core.host.service_restart')
+ @mock.patch('charmhelpers.core.services.templating')
+ def test_register_no_target(self, mtemplating, mservice_restart):
+ config = default_service_config()
+ del config[0]['templates'][0]['target']
+ services.register(config, TEMPLATES_DIR)
+ services.reconfigure_services()
+ self.assertEqual(mtemplating.render.call_args[0][0][0]['target'],
+ '/etc/init/cf-cloudcontroller.conf')
+ self.assertEqual(mtemplating.render.call_args[0][1], TEMPLATES_DIR)
+
+ @mock.patch('charmhelpers.core.host.service_restart')
+ @mock.patch('charmhelpers.core.services.templating')
+ def test_register_default_tmpl_dir(self, mtemplating, mservice_restart):
+ config = default_service_config()
+ services.register(config)
+ services.reconfigure_services()
+ self.assertEqual(mtemplating.render.call_args[0][1], None)
+
+ @mock.patch('charmhelpers.core.services.templating')
+ @mock.patch('charmhelpers.core.services.reconfigure_service')
+ def test_reconfigure_services(self, mreconfig, mtemplating):
+ services.register(default_service_config(), TEMPLATES_DIR)
+ services.reconfigure_services()
+ mreconfig.assert_called_once_with('cf-cloudcontroller', restart=True)
+
+ @mock.patch('charmhelpers.core.services.templating')
+ @mock.patch('charmhelpers.core.host.service_restart')
+ def test_reconfigure_service_restart(self, mrestart, mtemplating):
+ services.register(default_service_config(), TEMPLATES_DIR)
+ services.reconfigure_service('cf-cloudcontroller', restart=True)
+ mrestart.assert_called_once_with('cf-cloudcontroller')
+
+ @mock.patch('charmhelpers.core.services.templating')
+ @mock.patch('charmhelpers.core.host.service_restart')
+ def test_reconfigure_service_no_restart(self, mrestart, mtemplating):
+ services.register(default_service_config(), TEMPLATES_DIR)
+ services.reconfigure_service('cf-cloudcontroller', restart=False)
+ self.assertFalse(mrestart.called)
+
+ @mock.patch('charmhelpers.core.services.templating')
+ @mock.patch('charmhelpers.core.host.service_restart')
+ def test_reconfigure_service_incomplete(self, mrestart, mtemplating):
+ config = default_service_config()
+ mtemplating.render.return_value = False # render fails when incomplete
+ services.register(config, TEMPLATES_DIR)
+ services.reconfigure_service('cf-cloudcontroller', restart=True)
+ # verify that we did not restart the service
+ self.assertFalse(mrestart.called)
+
+ @mock.patch('charmhelpers.core.services.templating')
+ @mock.patch('charmhelpers.core.host.service_restart')
+ def test_reconfigure_service_no_context(self, mrestart, mtemplating):
+ config = default_service_config()
+ services.register(config, 'foo')
+ services.reconfigure_service('cf-cloudcontroller', restart=False)
+ # verify that we called render template with the expected name
+ mtemplating.render.assert_called_once_with(config[0]['templates'], 'foo')
+ self.assertFalse(mrestart.called)
+ self.assertRaises(KeyError, services.reconfigure_service, 'unknownservice', restart=False)
+
+ @mock.patch('charmhelpers.core.services.templating')
+ @mock.patch('charmhelpers.core.host.service_restart')
+ def test_custom_service_type(self, mrestart, mtemplating):
+ config = default_service_config()
+ config[0]['type'] = mock.Mock(name='CustomService')
+ services.register(config, 'foo')
+ services.reconfigure_service('cf-cloudcontroller', restart=False)
+ config[0]['type'].assert_called_once_with(config[0])
+ config[0]['type'].return_value.reconfigure.assert_called_once_with(False)
+
+ @mock.patch('charmhelpers.core.services.templating')
+ @mock.patch('charmhelpers.core.host.service_stop')
+ @mock.patch('charmhelpers.core.host.service_running')
+ def test_stop_services(self, mrunning, mstop, mtemplating):
+ services.register(default_service_config(), TEMPLATES_DIR)
+ services.stop_services()
+ mrunning.assert_called_once_with('cf-cloudcontroller')
+ mstop.assert_called_once_with('cf-cloudcontroller')
+
+if __name__ == '__main__':
+ unittest.main()
=== added file 'tests/core/test_templating.py'
--- tests/core/test_templating.py 1970-01-01 00:00:00 +0000
+++ tests/core/test_templating.py 2014-05-16 22:13:17 +0000
@@ -0,0 +1,213 @@
+import os
+import pkg_resources
+import tempfile
+import unittest
+
+import mock
+from charmhelpers.core import templating
+
+
+TEMPLATES_DIR = pkg_resources.resource_filename(__name__, 'templates')
+
+
+def noop():
+ return {}
+
+
+def found():
+ return {'foo': 'bar'}
+
+
+DEFAULT_CONTEXT = {
+ 'nats': {
+ 'nats_port': '1234',
+ 'nats_host': 'example.com',
+ },
+ 'router': {
+ 'domain': 'api.foo.com'
+ },
+ 'nginx_port': 80,
+}
+
+
+def default_context_provider():
+ return DEFAULT_CONTEXT
+
+
+class TestTemplating(unittest.TestCase):
+ def setUp(self):
+ self.charm_dir = pkg_resources.resource_filename(__name__, '')
+ self._charm_dir_patch = mock.patch.object(templating.hookenv, 'charm_dir')
+ self._charm_dir_mock = self._charm_dir_patch.start()
+ self._charm_dir_mock.side_effect = lambda: self.charm_dir
+
+ def tearDown(self):
+ self._charm_dir_patch.stop()
+
+ @mock.patch.object(templating.host.os, 'fchown')
+ @mock.patch.object(templating.host, 'mkdir')
+ @mock.patch.object(templating.host, 'log')
+ def test_render(self, log, mkdir, fchown):
+ _, fn1 = tempfile.mkstemp()
+ _, fn2 = tempfile.mkstemp()
+ template_definitions = [
+ {'source': 'fake_cc.yml',
+ 'target': fn1,
+ 'contexts': [default_context_provider]},
+ {'source': 'test.conf',
+ 'target': fn2,
+ 'contexts': [default_context_provider]},
+ ]
+ self.assertTrue(templating.render(template_definitions))
+ contents = open(fn1).read()
+ self.assertRegexpMatches(contents, 'port: 1234')
+ self.assertRegexpMatches(contents, 'host: example.com')
+ self.assertRegexpMatches(contents, 'domain: api.foo.com')
+ contents = open(fn2).read()
+ self.assertRegexpMatches(contents, 'listen 80')
+ self.assertEqual(fchown.call_count, 2)
+ self.assertEqual(mkdir.call_count, 2)
+
+ @mock.patch.object(templating.host.os, 'fchown')
+ @mock.patch.object(templating.host, 'mkdir')
+ @mock.patch.object(templating.host, 'log')
+ def test_render_incomplete(self, log, mkdir, fchown):
+ _, fn1 = tempfile.mkstemp()
+ _, fn2 = tempfile.mkstemp()
+ os.remove(fn1)
+ os.remove(fn2)
+ template_definitions = [
+ {'source': 'fake_cc.yml',
+ 'target': fn1,
+ 'contexts': [lambda: {}]},
+ {'source': 'test.conf',
+ 'target': fn2,
+ 'contexts': [default_context_provider]},
+ ]
+ self.assertFalse(templating.render(template_definitions))
+ self.assertFalse(os.path.exists(fn1))
+ contents = open(fn2).read()
+ self.assertRegexpMatches(contents, 'listen 80')
+ self.assertEqual(fchown.call_count, 1)
+ self.assertEqual(mkdir.call_count, 1)
+
+ @mock.patch('jinja2.Environment')
+ @mock.patch.object(templating.host, 'mkdir')
+ @mock.patch.object(templating.host, 'write_file')
+ def test_render_no_source(self, write_file, mkdir, Env):
+ template_definitions = [{
+ 'target': 'fake_cc.yml',
+ 'contexts': [default_context_provider],
+ }]
+ self.assertTrue(templating.render(template_definitions))
+ Env().get_template.assert_called_once_with('fake_cc.yml.j2')
+ Env().get_template.return_value.render.assert_called_once()
+ write_file.assert_called_once()
+
+ @mock.patch('jinja2.Environment')
+ @mock.patch.object(templating.host, 'mkdir')
+ @mock.patch.object(templating.host, 'write_file')
+ def test_render_no_contexts(self, write_file, mkdir, Env):
+ template_definitions = [{
+ 'target': 'fake_cc.yml',
+ 'contexts': [],
+ }]
+ self.assertTrue(templating.render(template_definitions))
+ Env().get_template.assert_called_once_with('fake_cc.yml.j2')
+ Env().get_template.return_value.render.assert_called_once_with({})
+ write_file.assert_called_once()
+
+ @mock.patch('jinja2.FileSystemLoader')
+ @mock.patch('jinja2.Environment')
+ @mock.patch.object(templating.host, 'mkdir')
+ @mock.patch.object(templating.host, 'write_file')
+ def test_render_implicit_dir(self, write_file, mkdir, Env, FSL):
+ self.charm_dir = 'foo'
+ template_definitions = [{
+ 'target': 'fake_cc.yml',
+ 'contexts': [default_context_provider],
+ }]
+ self.assertTrue(templating.render(template_definitions))
+ FSL.assert_called_once_with('foo/templates')
+
+ @mock.patch('jinja2.FileSystemLoader')
+ @mock.patch('jinja2.Environment')
+ @mock.patch.object(templating.host, 'mkdir')
+ @mock.patch.object(templating.host, 'write_file')
+ def test_render_explicit_dir(self, write_file, mkdir, Env, FSL):
+ self.charm_dir = 'foo'
+ template_definitions = [{
+ 'target': 'fake_cc.yml',
+ 'contexts': [default_context_provider],
+ }]
+ self.assertTrue(templating.render(template_definitions, 'bar'))
+ FSL.assert_called_once_with('bar')
+
+ def test_collect_contexts_fail(self):
+ cc = templating._collect_contexts
+ self.assertEqual(cc([]), {})
+ self.assertEqual(cc([noop]), False)
+ self.assertEqual(cc([noop, noop]), False)
+ self.assertEqual(cc([noop, found]), False)
+
+ def test_collect_contexts_found(self):
+ cc = templating._collect_contexts
+ expected = {'foo': 'bar'}
+ self.assertEqual(cc([found]), expected)
+ self.assertEqual(cc([found, found]), expected)
+
+
+class TestConfigContext(unittest.TestCase):
+ @mock.patch('charmhelpers.core.hookenv.config')
+ def test_config_context(self, mconfig):
+ templating.ConfigContext()()
+ self.assertTrue(mconfig.called)
+
+
+class TestStaticContext(unittest.TestCase):
+ def test_static_context(self):
+ a = templating.StaticContext('a')
+ self.assertEqual(a.data, 'a')
+ self.assertEqual(a(), 'a')
+
+
+class TestRelationContext(unittest.TestCase):
+ def setUp(self):
+ self.context_provider = templating.RelationContext()
+ self.context_provider.interface = 'http'
+ self.context_provider.required_keys = ['foo', 'bar']
+
+ @mock.patch.object(templating, 'hookenv')
+ def test_no_relations(self, mhookenv):
+ mhookenv.relation_ids.return_value = []
+ self.assertEqual(self.context_provider(), {})
+ mhookenv.relation_ids.assert_called_once_with('http')
+
+ @mock.patch.object(templating, 'hookenv')
+ def test_no_units(self, mhookenv):
+ mhookenv.relation_ids.return_value = ['nginx']
+ mhookenv.related_units.return_value = []
+ self.assertEqual(self.context_provider(), {})
+
+ @mock.patch.object(templating, 'hookenv')
+ def test_incomplete(self, mhookenv):
+ mhookenv.relation_ids.return_value = ['nginx', 'apache']
+ mhookenv.related_units.side_effect = lambda i: [i+'/0']
+ mhookenv.relation_get.side_effect = [{}, {'foo': '1'}]
+ self.assertEqual(self.context_provider(), {})
+ self.assertEqual(mhookenv.relation_get.call_args_list, [
+ mock.call(rid='nginx', unit='nginx/0'),
+ mock.call(rid='apache', unit='apache/0'),
+ ])
+
+ @mock.patch.object(templating, 'hookenv')
+ def test_complete(self, mhookenv):
+ mhookenv.relation_ids.return_value = ['nginx', 'apache', 'tomcat']
+ mhookenv.related_units.side_effect = lambda i: [i+'/0']
+ mhookenv.relation_get.side_effect = [{'foo': '1'}, {'foo': '2', 'bar': '3'}, {}]
+ self.assertEqual(self.context_provider(), {'http': {'foo': '2', 'bar': '3'}})
+ mhookenv.relation_ids.assert_called_with('http')
+ self.assertEqual(mhookenv.relation_get.call_args_list, [
+ mock.call(rid='nginx', unit='nginx/0'),
+ mock.call(rid='apache', unit='apache/0'),
+ ])
Follow ups
-
[Merge] lp:~johnsca/charm-helpers/charm-helpers into lp:~cf-charmers/charm-helpers/cloud-foundry
From: noreply, 2014-05-30
-
[Merge] lp:~johnsca/charm-helpers/charm-helpers into lp:~cf-charmers/charm-helpers/cloud-foundry
From: Cory Johns, 2014-05-30
-
Re: Refactor services & templating framework to core (issue 92420046)
From: Cory Johns, 2014-05-20
-
[Merge] lp:~johnsca/charm-helpers/charm-helpers into lp:~cf-charmers/charm-helpers/cloud-foundry
From: noreply, 2014-05-20
-
Re: Refactor services & templating framework to core (issue 92420046)
From: Benjamin Saller, 2014-05-19
-
Re: Refactor services & templating framework to core (issue 92420046)
From: Cory Johns, 2014-05-16
-
Re: Refactor services & templating framework to core (issue 92420046)
From: Cory Johns, 2014-05-16
-
Re: Refactor services & templating framework to core (issue 92420046)
From: Cory Johns, 2014-05-16
-
Refactor services & templating framework to core (issue 92420046)
From: Cory Johns, 2014-05-16
-
[Merge] lp:~johnsca/charm-helpers/charm-helpers into lp:~cf-charmers/charm-helpers/cloud-foundry
From: Cory Johns, 2014-05-16
