cloud-init-dev team mailing list archive
-
cloud-init-dev team
-
Mailing list archive
-
Message #00243
[Merge] lp:~avishai-ish-shalom/cloud-init/chef-refactor into lp:cloud-init
Avishai Ish-Shalom has proposed merging lp:~avishai-ish-shalom/cloud-init/chef-refactor into lp:cloud-init.
Requested reviews:
cloud init development team (cloud-init-dev)
Related bugs:
Bug #847358 in cloud-init: "chef integration should support chef-solo"
https://bugs.launchpad.net/cloud-init/+bug/847358
Bug #1027188 in cloud-init: "Cloud-init chef plugin should support more distros and installation methods"
https://bugs.launchpad.net/cloud-init/+bug/1027188
For more details, see:
https://code.launchpad.net/~avishai-ish-shalom/cloud-init/chef-refactor/+merge/164009
Refactored cc_chef, added omnibus and chef-solo support
--
https://code.launchpad.net/~avishai-ish-shalom/cloud-init/chef-refactor/+merge/164009
Your team cloud init development team is requested to review the proposed merge of lp:~avishai-ish-shalom/cloud-init/chef-refactor into lp:cloud-init.
=== modified file 'cloudinit/config/cc_chef.py'
--- cloudinit/config/cc_chef.py 2012-12-12 15:39:43 +0000
+++ cloudinit/config/cc_chef.py 2013-05-15 17:20:33 +0000
@@ -20,10 +20,15 @@
import json
import os
+import urllib
from cloudinit import templater
from cloudinit import url_helper
from cloudinit import util
+from cloudinit.settings import PER_INSTANCE
+from urlparse import urlparse
+
+frequency = PER_INSTANCE
RUBY_VERSION_DEFAULT = "1.8"
@@ -38,20 +43,51 @@
OMNIBUS_URL = "https://www.opscode.com/chef/install.sh";
+BIN_PATHS = (
+ "/usr/bin", "/usr/local/bin",
+ "/var/lib/gems/2.0.0/bin",
+ "/var/lib/gems/1.9.1/bin",
+ "/var/lib/gems/1.9/bin",
+ "/var/lib/gems/1.8/bin"
+)
+DEFAULT_ARGS = {
+ 'interval': ('-i', 1800),
+ 'splay': ('-s', 300),
+ 'daemonize': ('-d', True),
+ 'fork': ('-f', True)
+}
+DEFAULT_CFG = {
+ 'repo_path': '/var/lib/cloud/chef_repo',
+ 'repo_source_type': 'tarball'
+}
+JSON_ATTRIB_FILE = '/etc/chef/firstboot.json'
+
def handle(name, cfg, cloud, log, _args):
-
+ log.info("Starting cc_chef")
# If there isn't a chef key in the configuration don't do anything
if 'chef' not in cfg:
log.debug(("Skipping module named %s,"
" no 'chef' key in configuration"), name)
return
- chef_cfg = cfg['chef']
+ chef_cfg = dict(DEFAULT_CFG.items() + cfg['chef'].items())
+ log.debug("Chef config: %r", chef_cfg)
# Ensure the chef directories we use exist
for d in CHEF_DIRS:
util.ensure_dir(d)
+ if 'mode' in chef_cfg:
+ chef_mode = chef_cfg['mode']
+ else:
+ if 'server_url' in chef_cfg and \
+ ('validation_key' in chef_cfg or 'validation_cert' in chef_cfg):
+ chef_mode = "client"
+ else:
+ chef_mode = "solo"
+
+ log.debug("Chef mode is %s", chef_mode)
+
# Set the validation key based on the presence of either 'validation_key'
# or 'validation_cert'. In the case where both exist, 'validation_key'
# takes precedence
@@ -61,80 +97,191 @@
break
# Create the chef config from template
- template_fn = cloud.get_template_filename('chef_client.rb')
- if template_fn:
- iid = str(cloud.datasource.get_instance_id())
- params = {
- 'server_url': chef_cfg['server_url'],
- 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid),
- 'environment': util.get_cfg_option_str(chef_cfg, 'environment',
- '_default'),
- 'validation_name': chef_cfg['validation_name']
- }
- templater.render_to_file(template_fn, '/etc/chef/client.rb', params)
- else:
- log.warn("No template found, not rendering to /etc/chef/client.rb")
+ if not write_chef_config(chef_cfg, cloud, chef_mode, log):
+ return False
# set the firstboot json
- initial_json = {}
- if 'run_list' in chef_cfg:
- initial_json['run_list'] = chef_cfg['run_list']
- if 'initial_attributes' in chef_cfg:
- initial_attributes = chef_cfg['initial_attributes']
- for k in list(initial_attributes.keys()):
- initial_json[k] = initial_attributes[k]
- util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json))
+ write_json_attrib_file(chef_cfg)
# If chef is not installed, we install chef based on 'install_type'
+ install_chef(chef_cfg, cloud)
+
+ if chef_mode == 'solo':
+ get_cookbooks(chef_cfg, cloud)
+
+ chef_args = chef_args_from_cfg(chef_cfg, ['-j', JSON_ATTRIB_FILE])
+ if util.get_cfg_option_bool(chef_cfg, 'autostart', default=True):
+ run_chef(log, chef_mode, chef_args)
+
+
+def install_chef(chef_cfg, cloud):
if (not os.path.isfile('/usr/bin/chef-client') or
- util.get_cfg_option_bool(chef_cfg, 'force_install', default=False)):
+ util.get_cfg_option_bool(chef_cfg, 'force_install', default=False)):
install_type = util.get_cfg_option_str(chef_cfg, 'install_type',
'packages')
+ chef_version = util.get_cfg_option_str(chef_cfg, 'version', None)
+
if install_type == "gems":
# this will install and run the chef-client from gems
- chef_version = util.get_cfg_option_str(chef_cfg, 'version', None)
ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version',
RUBY_VERSION_DEFAULT)
- install_chef_from_gems(cloud.distro, ruby_version, chef_version)
- # and finally, run chef-client
- log.debug('Running chef-client')
- util.subp(['/usr/bin/chef-client',
- '-d', '-i', '1800', '-s', '20'], capture=False)
+ ohai_version = util.get_cfg_option_str(chef_cfg, 'ohai_version', None)
+ install_chef_from_gems(cloud.distro, ruby_version, chef_version, ohai_version)
elif install_type == 'packages':
# this will install and run the chef-client from packages
cloud.distro.install_packages(('chef',))
elif install_type == 'omnibus':
url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL)
- content = url_helper.readurl(url=url, retries=5)
+ content = url_helper.readurl(url=url, retries=5).contents
with util.tempdir() as tmpd:
# use tmpd over tmpfile to avoid 'Text file busy' on execute
tmpf = "%s/chef-omnibus-install" % tmpd
util.write_file(tmpf, content, mode=0700)
- util.subp([tmpf], capture=False)
+ args = []
+ if chef_version:
+ args.append("-v")
+ args.append(chef_version)
+ util.subp([tmpf] + args, capture=False)
else:
- log.warn("Unknown chef install type %s", install_type)
+ raise RuntimeError("Unknown chef install type %s" % install_type)
+
+
+def write_chef_config(chef_cfg, cloud, chef_mode, log):
+ "Write chef config file from template"
+ template_fn = cloud.get_template_filename('chef_client.rb')
+ cfg_filename = "/etc/chef/" + ("client.rb" if chef_mode == "client" else "solo.rb")
+ if template_fn:
+ iid = str(cloud.datasource.get_instance_id())
+ params = {
+ 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid),
+ 'environment': util.get_cfg_option_str(chef_cfg, 'environment',
+ '_default'),
+ 'mode': chef_mode
+ }
+
+ if chef_mode == "client":
+ # server_url is required
+ params['server_url'] = chef_cfg['server_url']
+ params['validation_name'] = chef_cfg.get('validation_name', None)
+ elif chef_mode == "solo":
+ params['repo_path'] = chef_cfg['repo_path']
+
+ templater.render_to_file(template_fn, cfg_filename, params)
+ return True
+ else:
+ log.warn("No template found, not rendering to %s", cfg_filename)
+ return False
+
+
+def write_json_attrib_file(chef_cfg):
+ initial_json = {}
+ if 'run_list' in chef_cfg:
+ initial_json['run_list'] = chef_cfg['run_list']
+ if 'initial_attributes' in chef_cfg:
+ initial_attributes = chef_cfg['initial_attributes']
+ for k in list(initial_attributes.keys()):
+ initial_json[k] = initial_attributes[k]
+ util.write_file(JSON_ATTRIB_FILE, json.dumps(initial_json))
+
+
+def run_chef(log, chef_type, chef_args):
+ chef_bin = "chef-%s" % chef_type
+ chef_exec = None
+ for path in BIN_PATHS:
+ f = os.path.join(path, chef_bin)
+ if os.path.isfile(f) and os.access(f, os.X_OK):
+ chef_exec = f
+ break
+ if chef_exec is None:
+ raise RuntimeError("Couldn't find chef executable for %s" % chef_bin)
+ log.debug("Running %s", chef_exec)
+ util.subp([chef_exec] + chef_args, capture=False)
def get_ruby_packages(version):
# return a list of packages needed to install ruby at version
- pkgs = ['ruby%s' % version, 'ruby%s-dev' % version]
+ pkgs = ['ruby%s' % version, 'ruby%s-dev' % version, 'build-essential']
if version == "1.8":
pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8'))
return pkgs
-def install_chef_from_gems(ruby_version, chef_version, distro):
+def install_chef_from_gems(distro, ruby_version, chef_version, ohai_version=None):
distro.install_packages(get_ruby_packages(ruby_version))
+
+ def gem_install(gem, version=None):
+ cmd_args = ['/usr/bin/gem', 'install', gem]
+ if version is not None:
+ cmd_args.append('-v')
+ cmd_args.append(version)
+
+ cmd_args += ['--no-rdoc', '--bindir', '/usr/bin', '-q']
+
+ return util.subp(cmd_args, capture=False)
+
if not os.path.exists('/usr/bin/gem'):
util.sym_link('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem')
if not os.path.exists('/usr/bin/ruby'):
util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby')
- if chef_version:
- util.subp(['/usr/bin/gem', 'install', 'chef',
- '-v %s' % chef_version, '--no-ri',
- '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False)
+ if ohai_version:
+ gem_install('ohai', ohai_version)
+ gem_install('chef', chef_version)
+
+
+def chef_args_from_cfg(cfg, extra_args=[]):
+ merged_args = {}
+ for (k, v) in DEFAULT_ARGS.iteritems():
+ merged_args[k] = (v[0], cfg.get(k, v[1]))
+
+ if merged_args['daemonize'][1] is False:
+ del merged_args['interval']
+ del merged_args['splay']
+
+ args = []
+ for (k, (arg, v)) in merged_args.iteritems():
+ if type(v) == bool:
+ if util.get_cfg_option_bool(cfg, k, v):
+ args.append(arg)
+ else:
+ args.append(arg)
+ args.append(str(cfg.get(k, v)))
+
+ return args + extra_args
+
+
+def get_cookbooks(cfg, cloud):
+ repo_source_type = util.get_cfg_option_str(
+ cfg, 'repo_source_type', default='tarball')
+ if repo_source_type == 'git':
+ get_cookbooks_from_git(cfg, cloud)
+ elif repo_source_type == 'tarball':
+ get_cookbooks_from_tarball(cfg)
else:
- util.subp(['/usr/bin/gem', 'install', 'chef',
- '--no-ri', '--no-rdoc', '--bindir',
- '/usr/bin', '-q'], capture=False)
+ raise RuntimeError("Unknown cookbooks source type %s" % repo_source_type)
+
+
+def get_cookbooks_from_git(cfg, cloud):
+ cloud.distro.install_packages('git')
+ enclosing_dir = os.path.dirname(cfg['repo_path'])
+ if not os.path.isdir(enclosing_dir):
+ os.makedirs(enclosing_dir)
+ util.subp(['git', 'clone', '--recurse-submodules', cfg['repo_source'], cfg['repo_path']])
+
+
+def get_cookbooks_from_tarball(cfg):
+ with util.tempdir() as tmpd:
+ filename = os.path.basename(urlparse(cfg['repo_source']).path)
+ tmpfile = os.path.join(tmpd, filename)
+ urllib.urlretrieve(cfg['repo_source'], tmpfile)
+ if not os.path.isdir(cfg['repo_path']):
+ os.makedirs(cfg['repo_path'])
+ util.subp(['tar', '-xf', tmpfile, '-C', cfg['repo_path']])
+
+
+def get_cookbooks_from_berkshelf(cfg):
+ raise NotImplementedError()
+
+
+def get_cookbooks_from_librarian(cfg):
+ raise NotImplementedError()
=== modified file 'doc/examples/cloud-config-chef.txt'
--- doc/examples/cloud-config-chef.txt 2012-12-12 15:39:43 +0000
+++ doc/examples/cloud-config-chef.txt 2013-05-15 17:20:33 +0000
@@ -84,8 +84,23 @@
maxclients: 100
keepalive: "off"
- # if install_type is 'omnibus', change the url to download
- omnibus_url: "https://www.opscode.com/chef/install.sh";
+ # For chef-solo, we want to download cookbooks. Currently git and tarball can be used
+ # tarball/git should contain cookbooks, roles and data_bags folders
+ #repo_source_type: git
+ #repo_source: https://github.com/some-org/cookbooks-repo.git
+ #repo_path: /var/lib/cloud/chef_repo
+
+ # if install_type is 'omnibus', change the url to the download script
+ #omnibus_url: "https://www.opscode.com/chef/install.sh";
+
+ # Daemonize and run every 'interval'
+ #daemonize: true
+ #interval: 1800
+ # Random wait before starting the run
+ #splay: 300
+
+ # fork off a worker for every chef run, effective against memory leaks
+ fork: true
# Capture all subprocess output into a logfile
=== modified file 'templates/chef_client.rb.tmpl'
--- templates/chef_client.rb.tmpl 2012-07-09 20:45:26 +0000
+++ templates/chef_client.rb.tmpl 2013-05-15 17:20:33 +0000
@@ -11,13 +11,18 @@
log_level :info
log_location "/var/log/chef/client.log"
ssl_verify_mode :verify_none
+#if $mode == 'client'
validation_client_name "$validation_name"
validation_key "/etc/chef/validation.pem"
client_key "/etc/chef/client.pem"
chef_server_url "$server_url"
environment "$environment"
+#else if $mode == 'solo'
+cookbook_path "$repo_path/cookbooks"
+data_bag_path "$repo_path/data_bags"
+role_path "$repo_path/roles"
+#end if
node_name "$node_name"
-json_attribs "/etc/chef/firstboot.json"
file_cache_path "/var/cache/chef"
file_backup_path "/var/backups/chef"
pid_file "/var/run/chef/client.pid"
