← Back to team overview

bigdata-dev team mailing list archive

[Merge] lp:~johnsca/charm-helpers/test-fixes into lp:~bigdata-dev/charm-helpers/framework

 

Cory Johns has proposed merging lp:~johnsca/charm-helpers/test-fixes into lp:~bigdata-dev/charm-helpers/framework.

Requested reviews:
  Juju Big Data Development (bigdata-dev)

For more details, see:
https://code.launchpad.net/~johnsca/charm-helpers/test-fixes/+merge/279937

Fixed or deleted failing tests
-- 
Your team Juju Big Data Development is requested to review the proposed merge of lp:~johnsca/charm-helpers/test-fixes into lp:~bigdata-dev/charm-helpers/framework.
=== removed directory 'charmhelpers/contrib/bigdata'
=== removed file 'charmhelpers/contrib/bigdata/__init__.py'
--- charmhelpers/contrib/bigdata/__init__.py	2015-01-23 21:35:56 +0000
+++ charmhelpers/contrib/bigdata/__init__.py	1970-01-01 00:00:00 +0000
@@ -1,19 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-from . import utils  # noqa
-from . import relations  # noqa
-from . import handlers  # noqa

=== removed directory 'charmhelpers/contrib/bigdata/handlers'
=== removed file 'charmhelpers/contrib/bigdata/handlers/__init__.py'
--- charmhelpers/contrib/bigdata/handlers/__init__.py	2015-01-23 21:35:56 +0000
+++ charmhelpers/contrib/bigdata/handlers/__init__.py	1970-01-01 00:00:00 +0000
@@ -1,17 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-from . import apache  # noqa

=== removed file 'charmhelpers/contrib/bigdata/handlers/apache.py'
--- charmhelpers/contrib/bigdata/handlers/apache.py	2015-12-07 20:40:11 +0000
+++ charmhelpers/contrib/bigdata/handlers/apache.py	1970-01-01 00:00:00 +0000
@@ -1,479 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-import re
-from subprocess import check_call, check_output
-import time
-
-from path import Path
-
-import jujuresources
-
-from charmhelpers.core import host
-from charmhelpers.core import hookenv
-from charmhelpers.core import unitdata
-from charmhelpers.core.charmframework import helpers
-from charmhelpers.contrib.bigdata import utils
-
-
-class HadoopBase(object):
-    def __init__(self, dist_config):
-        self.dist_config = dist_config
-        self.charm_config = hookenv.config()
-        self.cpu_arch = host.cpu_arch()
-
-        # dist_config will have simple validation done on primary keys in the
-        # dist.yaml, but we need to ensure deeper values are present.
-        required_dirs = ['hadoop', 'hadoop_conf', 'hdfs_log_dir',
-                         'yarn_log_dir']
-        missing_dirs = set(required_dirs) - set(self.dist_config.dirs.keys())
-        if missing_dirs:
-            raise ValueError('dirs option in {} is missing required entr{}: {}'.format(
-                self.dist_config.yaml_file,
-                'ies' if len(missing_dirs) > 1 else 'y',
-                ', '.join(missing_dirs)))
-
-        self.client_spec = {
-            'hadoop': self.dist_config.hadoop_version,
-        }
-        self.verify_conditional_resources = utils.verify_resources('hadoop-%s' % self.cpu_arch)
-
-    def spec(self):
-        """
-        Generate the full spec for keeping charms in sync.
-
-        NB: This has to be a callback instead of a plain property because it is
-        passed to the relations during construction of the Manager but needs to
-        properly reflect the Java version in the same hook invocation that installs
-        Java.
-        """
-        java_version = unitdata.kv().get('java.version')
-        if java_version:
-            return {
-                'vendor': self.dist_config.vendor,
-                'hadoop': self.dist_config.hadoop_version,
-                'java': java_version,
-                'arch': self.cpu_arch,
-            }
-        else:
-            return None
-
-    def is_installed(self):
-        return unitdata.kv().get('hadoop.base.installed')
-
-    def install(self, force=False):
-        if not force and self.is_installed():
-            return
-        self.configure_hosts_file()
-        self.dist_config.add_users()
-        self.dist_config.add_dirs()
-        self.dist_config.add_packages()
-        self.install_base_packages()
-        self.setup_hadoop_config()
-        self.configure_hadoop()
-        unitdata.kv().set('hadoop.base.installed', True)
-        unitdata.kv().flush(True)
-
-    def configure_hosts_file(self):
-        """
-        Add the unit's private-address to /etc/hosts to ensure that Java
-        can resolve the hostname of the server to its real IP address.
-        We derive our hostname from the unit_id, replacing / with -.
-        """
-        private_address = hookenv.unit_get('private-address')
-        hostname = hookenv.local_unit().replace('/', '-')
-
-        etc_hostname = Path('/etc/hostname')
-        etc_hostname.write_text(hostname)
-        check_call(["hostname", "-F", "/etc/hostname"])
-
-        etc_hosts = Path('/etc/hosts')
-        hosts = etc_hosts.lines()
-        line = '%s %s' % (private_address, hostname)
-        IP_pat = r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}'
-        if not re.match(IP_pat, private_address):
-            line = '# %s  # private-address did not return an IP' % line
-        if hosts[0] != line:
-            hosts.insert(0, line)
-            etc_hosts.write_lines(hosts)
-
-    def install_base_packages(self):
-        with utils.disable_firewall():
-            self.install_java()
-            self.install_hadoop()
-
-    def install_java(self):
-        """
-        Run the java-installer resource to install Java and determine
-        the JAVA_HOME and Java version.
-
-        The java-installer must be idempotent and its only output (on stdout)
-        should be two lines: the JAVA_HOME path, and the Java version, respectively.
-
-        If there is an error installing Java, the installer should exit
-        with a non-zero exit code.
-        """
-        env = utils.read_etc_env()
-        java_installer = Path(jujuresources.resource_path('java-installer'))
-        java_installer.chmod(0o755)
-        output = check_output([java_installer], env=env)
-        java_home, java_version = map(str.strip, output.strip().split('\n'))
-        unitdata.kv().set('java.home', java_home)
-        unitdata.kv().set('java.version', java_version)
-
-    def install_hadoop(self):
-        jujuresources.install('hadoop-%s' %
-                              self.cpu_arch,
-                              destination=self.dist_config.path('hadoop'),
-                              skip_top_level=True)
-
-    def setup_hadoop_config(self):
-        # copy default config into alternate dir
-        conf_dir = self.dist_config.path('hadoop') / 'etc/hadoop'
-        self.dist_config.path('hadoop_conf').rmtree_p()
-        conf_dir.copytree(self.dist_config.path('hadoop_conf'))
-        (self.dist_config.path('hadoop_conf') / 'slaves').remove_p()
-        mapred_site = self.dist_config.path('hadoop_conf') / 'mapred-site.xml'
-        if not mapred_site.exists():
-            (self.dist_config.path('hadoop_conf') / 'mapred-site.xml.template').copy(mapred_site)
-
-    def configure_hadoop(self):
-        java_home = Path(unitdata.kv().get('java.home'))
-        java_bin = java_home / 'bin'
-        hadoop_bin = self.dist_config.path('hadoop') / 'bin'
-        hadoop_sbin = self.dist_config.path('hadoop') / 'sbin'
-        with utils.environment_edit_in_place('/etc/environment') as env:
-            env['JAVA_HOME'] = java_home
-            if java_bin not in env['PATH']:
-                env['PATH'] = ':'.join([env['PATH'], java_bin])
-            if hadoop_bin not in env['PATH']:
-                env['PATH'] = ':'.join([env['PATH'], hadoop_bin])
-            if hadoop_sbin not in env['PATH']:
-                env['PATH'] = ':'.join([env['PATH'], hadoop_sbin])
-            env['HADOOP_LIBEXEC_DIR'] = self.dist_config.path('hadoop') / 'libexec'
-            env['HADOOP_INSTALL'] = self.dist_config.path('hadoop')
-            env['HADOOP_HOME'] = self.dist_config.path('hadoop')
-            env['HADOOP_COMMON_HOME'] = self.dist_config.path('hadoop')
-            env['HADOOP_HDFS_HOME'] = self.dist_config.path('hadoop')
-            env['HADOOP_MAPRED_HOME'] = self.dist_config.path('hadoop')
-            env['HADOOP_YARN_HOME'] = self.dist_config.path('hadoop')
-            env['YARN_HOME'] = self.dist_config.path('hadoop')
-            env['HADOOP_CONF_DIR'] = self.dist_config.path('hadoop_conf')
-            env['YARN_CONF_DIR'] = self.dist_config.path('hadoop_conf')
-            env['YARN_LOG_DIR'] = self.dist_config.path('yarn_log_dir')
-            env['HDFS_LOG_DIR'] = self.dist_config.path('hdfs_log_dir')
-            env['HADOOP_LOG_DIR'] = self.dist_config.path('hdfs_log_dir')  # for hadoop 2.2.0 only
-            env['MAPRED_LOG_DIR'] = '/var/log/hadoop/mapred'  # should be moved to config, but could
-            env['MAPRED_PID_DIR'] = '/var/run/hadoop/mapred'  # be destructive for mapreduce operation
-
-        hadoop_env = self.dist_config.path('hadoop_conf') / 'hadoop-env.sh'
-        utils.re_edit_in_place(hadoop_env, {
-            r'export JAVA_HOME *=.*': 'export JAVA_HOME=%s' % java_home,
-        })
-
-    def register_slaves(self, relation):
-        """
-        Add slaves to a hdfs or yarn master, determined by the relation name.
-
-        :param str relation: 'datanode' for registering HDFS slaves;
-                             'nodemanager' for registering YARN slaves.
-        """
-        slaves = helpers.all_ready_units(relation)
-        slaves_file = self.dist_config.path('hadoop_conf') / 'slaves'
-        slaves_file.write_lines(
-            [
-                '# DO NOT EDIT',
-                '# This file is automatically managed by Juju',
-            ] + [
-                data['hostname'] for slave, data in slaves
-            ]
-        )
-        slaves_file.chown('ubuntu', 'hadoop')
-
-    def run(self, user, command, *args, **kwargs):
-        """
-        Run a Hadoop command as the `hdfs` user.
-
-        :param str command: Command to run, prefixed with `bin/` or `sbin/`
-        :param list args: Additional args to pass to the command
-        """
-        return utils.run_as(user,
-                            self.dist_config.path('hadoop') / command,
-                            *args, **kwargs)
-
-
-class HDFS(object):
-    def __init__(self, hadoop_base):
-        self.hadoop_base = hadoop_base
-
-    def stop_namenode(self):
-        self._hadoop_daemon('stop', 'namenode')
-
-    def start_namenode(self):
-        if not utils.jps('NameNode'):
-            self._hadoop_daemon('start', 'namenode')
-            # Some hadoop processes take a bit of time to start
-            # we need to let them get to a point where they are
-            # ready to accept connections - increase the value for hadoop 2.4.1
-            time.sleep(30)
-
-    def stop_secondarynamenode(self):
-        self._hadoop_daemon('stop', 'secondarynamenode')
-
-    def start_secondarynamenode(self):
-        if not utils.jps('SecondaryNameNode'):
-            self._hadoop_daemon('start', 'secondarynamenode')
-            # Some hadoop processes take a bit of time to start
-            # we need to let them get to a point where they are
-            # ready to accept connections - increase the value for hadoop 2.4.1
-            time.sleep(30)
-
-    def stop_datanode(self):
-        self._hadoop_daemon('stop', 'datanode')
-
-    def start_datanode(self):
-        if not utils.jps('DataNode'):
-            self._hadoop_daemon('start', 'datanode')
-
-    def _remote(self, relation):
-        """
-        Return the hostname of the unit on the other end of the given
-        relation (derived from that unit's name) and the port used to talk
-        to that unit.
-        :param str relation: Name of the relation, e.g. "datanode" or "namenode"
-        """
-        unit, data = helpers.any_ready_unit(relation)
-        host = unit.replace('/', '-')
-        return host, data['port']
-
-    def _local(self):
-        """
-        Return the local hostname (which we derive from our unit name),
-        and namenode port from our dist.yaml
-        """
-        host = hookenv.local_unit().replace('/', '-')
-        port = self.hadoop_base.dist_config.port('namenode')
-        return host, port
-
-    def configure_namenode(self):
-        self.configure_hdfs_base(*self._local())
-        cfg = self.hadoop_base.charm_config
-        dc = self.hadoop_base.dist_config
-        hdfs_site = dc.path('hadoop_conf') / 'hdfs-site.xml'
-        with utils.xmlpropmap_edit_in_place(hdfs_site) as props:
-            props['dfs.replication'] = cfg['dfs_replication']
-            props['dfs.blocksize'] = int(cfg['dfs_blocksize'])
-            props['dfs.namenode.datanode.registration.ip-hostname-check'] = 'true'
-            props['dfs.namenode.http-address'] = '0.0.0.0:{}'.format(dc.port('nn_webapp_http'))
-            # TODO: support SSL
-            # props['dfs.namenode.https-address'] = '0.0.0.0:{}'.format(dc.port('nn_webapp_https'))
-
-    def configure_secondarynamenode(self):
-        """
-        Configure the Secondary Namenode when the apache-hadoop-hdfs-secondary
-        charm is deployed and related to apache-hadoop-hdfs-master.
-
-        The only purpose of the secondary namenode is to perform periodic
-        checkpoints. The secondary name-node periodically downloads current
-        namenode image and edits log files, joins them into new image and
-        uploads the new image back to the (primary and the only) namenode.
-        """
-        self.configure_hdfs_base(*self._remote("namenode"))
-
-    def configure_datanode(self):
-            self.configure_hdfs_base(*self._remote("datanode"))
-            dc = self.hadoop_base.dist_config
-            hdfs_site = dc.path('hadoop_conf') / 'hdfs-site.xml'
-            with utils.xmlpropmap_edit_in_place(hdfs_site) as props:
-                props['dfs.datanode.http.address'] = '0.0.0.0:{}'.format(dc.port('dn_webapp_http'))
-                # TODO: support SSL
-                # props['dfs.datanode.https.address'] = '0.0.0.0:{}'.format(dc.port('dn_webapp_https'))
-
-    def configure_client(self):
-        self.configure_hdfs_base(*self._remote("namenode"))
-
-    def configure_hdfs_base(self, host, port):
-        dc = self.hadoop_base.dist_config
-        core_site = dc.path('hadoop_conf') / 'core-site.xml'
-        with utils.xmlpropmap_edit_in_place(core_site) as props:
-            props['fs.defaultFS'] = "hdfs://{host}:{port}".format(host=host, port=port)
-            props['hadoop.proxyuser.hue.hosts'] = "*"
-            props['hadoop.proxyuser.hue.groups'] = "*"
-            props['hadoop.proxyuser.oozie.groups'] = '*'
-            props['hadoop.proxyuser.oozie.hosts'] = '*'
-        hdfs_site = dc.path('hadoop_conf') / 'hdfs-site.xml'
-        with utils.xmlpropmap_edit_in_place(hdfs_site) as props:
-            props['dfs.webhdfs.enabled'] = "true"
-            props['dfs.namenode.name.dir'] = dc.path('hdfs_dir_base') / 'cache/hadoop/dfs/name'
-            props['dfs.datanode.data.dir'] = dc.path('hdfs_dir_base') / 'cache/hadoop/dfs/name'
-            props['dfs.permissions'] = 'false'  # TODO - secure this hadoop installation!
-
-    def format_namenode(self):
-        if unitdata.kv().get('hdfs.namenode.formatted'):
-            return
-        self.stop_namenode()
-        # Run without prompting; this will fail if the namenode has already
-        # been formatted -- we do not want to reformat existing data!
-        self._hdfs('namenode', '-format', '-noninteractive')
-        unitdata.kv().set('hdfs.namenode.formatted', True)
-        unitdata.kv().flush(True)
-
-    def create_hdfs_dirs(self):
-        if unitdata.kv().get('hdfs.namenode.dirs.created'):
-            return
-        self._hdfs('dfs', '-mkdir', '-p', '/tmp/hadoop/mapred/staging')
-        self._hdfs('dfs', '-chmod', '-R', '1777', '/tmp/hadoop/mapred/staging')
-        self._hdfs('dfs', '-mkdir', '-p', '/tmp/hadoop-yarn/staging')
-        self._hdfs('dfs', '-chmod', '-R', '1777', '/tmp/hadoop-yarn')
-        self._hdfs('dfs', '-mkdir', '-p', '/user/ubuntu')
-        self._hdfs('dfs', '-chown', '-R', 'ubuntu', '/user/ubuntu')
-        # for JobHistory
-        self._hdfs('dfs', '-mkdir', '-p', '/mr-history/tmp')
-        self._hdfs('dfs', '-chmod', '-R', '1777', '/mr-history/tmp')
-        self._hdfs('dfs', '-mkdir', '-p', '/mr-history/done')
-        self._hdfs('dfs', '-chmod', '-R', '1777', '/mr-history/done')
-        self._hdfs('dfs', '-chown', '-R', 'mapred:hdfs', '/mr-history')
-        self._hdfs('dfs', '-mkdir', '-p', '/app-logs')
-        self._hdfs('dfs', '-chmod', '-R', '1777', '/app-logs')
-        self._hdfs('dfs', '-chown', 'yarn', '/app-logs')
-        unitdata.kv().set('hdfs.namenode.dirs.created', True)
-        unitdata.kv().flush(True)
-
-    def register_slaves(self):
-        self.hadoop_base.register_slaves('datanode')
-
-    def _hadoop_daemon(self, command, service):
-        self.hadoop_base.run('hdfs', 'sbin/hadoop-daemon.sh',
-                             '--config',
-                             self.hadoop_base.dist_config.path('hadoop_conf'),
-                             command, service)
-
-    def _hdfs(self, command, *args):
-        self.hadoop_base.run('hdfs', 'bin/hdfs', command, *args)
-
-
-class YARN(object):
-    def __init__(self, hadoop_base):
-        self.hadoop_base = hadoop_base
-
-    def stop_resourcemanager(self):
-        self._yarn_daemon('stop', 'resourcemanager')
-
-    def start_resourcemanager(self):
-        if not utils.jps('ResourceManager'):
-            self._yarn_daemon('start', 'resourcemanager')
-
-    def stop_jobhistory(self):
-        self._jobhistory_daemon('stop', 'historyserver')
-
-    def start_jobhistory(self):
-        if utils.jps('JobHistoryServer'):
-            self._jobhistory_daemon('stop', 'historyserver')
-        self._jobhistory_daemon('start', 'historyserver')
-
-    def stop_nodemanager(self):
-        self._yarn_daemon('stop', 'nodemanager')
-
-    def start_nodemanager(self):
-        if not utils.jps('NodeManager'):
-            self._yarn_daemon('start', 'nodemanager')
-
-    def _remote(self, relation):
-        """
-        Return the hostname of the unit on the other end of the given
-        relation (derived from that unit's name) and the port used to talk
-        to that unit.
-        :param str relation: Name of the relation, e.g. "resourcemanager" or "nodemanager"
-        """
-        unit, data = helpers.any_ready_unit(relation)
-        host = unit.replace('/', '-')
-        return host, data['port']
-
-    def _local(self):
-        """
-        Return the local hostname (which we derive from our unit name),
-        and resourcemanager port from our dist.yaml
-        """
-        host = hookenv.local_unit().replace('/', '-')
-        port = self.hadoop_base.dist_config.port('resourcemanager')
-        return host, port
-
-    def configure_resourcemanager(self):
-        self.configure_yarn_base(*self._local())
-        dc = self.hadoop_base.dist_config
-        yarn_site = dc.path('hadoop_conf') / 'yarn-site.xml'
-        with utils.xmlpropmap_edit_in_place(yarn_site) as props:
-            # 0.0.0.0 will listen on all interfaces, which is what we want on the server
-            props['yarn.resourcemanager.webapp.address'] = '0.0.0.0:{}'.format(dc.port('rm_webapp_http'))
-            # TODO: support SSL
-            # props['yarn.resourcemanager.webapp.https.address'] = '0.0.0.0:{}'.format(dc.port('rm_webapp_https'))
-
-    def configure_jobhistory(self):
-        self.configure_yarn_base(*self._local())
-        dc = self.hadoop_base.dist_config
-        mapred_site = dc.path('hadoop_conf') / 'mapred-site.xml'
-        with utils.xmlpropmap_edit_in_place(mapred_site) as props:
-            # 0.0.0.0 will listen on all interfaces, which is what we want on the server
-            props["mapreduce.jobhistory.address"] = "0.0.0.0:{}".format(dc.port('jobhistory'))
-            props["mapreduce.jobhistory.webapp.address"] = "0.0.0.0:{}".format(dc.port('jh_webapp_http'))
-
-    def configure_nodemanager(self):
-        self.configure_yarn_base(*self._remote("nodemanager"))
-
-    def configure_client(self):
-        self.configure_yarn_base(*self._remote("resourcemanager"))
-
-    def configure_yarn_base(self, host, port):
-        dc = self.hadoop_base.dist_config
-        yarn_site = dc.path('hadoop_conf') / 'yarn-site.xml'
-        with utils.xmlpropmap_edit_in_place(yarn_site) as props:
-            props['yarn.nodemanager.aux-services'] = 'mapreduce_shuffle'
-            props['yarn.resourcemanager.hostname'] = '{}'.format(host)
-            props['yarn.resourcemanager.address'] = '{}:{}'.format(host, port)
-            props["yarn.log.server.url"] = "{}:{}/jobhistory/logs/".format(host, dc.port('rm_log'))
-        mapred_site = dc.path('hadoop_conf') / 'mapred-site.xml'
-        with utils.xmlpropmap_edit_in_place(mapred_site) as props:
-            props["mapreduce.jobhistory.address"] = "{}:{}".format(host, dc.port('jobhistory'))
-            props["mapreduce.framework.name"] = 'yarn'
-
-    def install_demo(self):
-        if unitdata.kv().get('yarn.client.demo.installed'):
-            return
-        # Copy our demo (TeraSort) to the target location and set mode/owner
-        demo_source = 'scripts/terasort.sh'
-        demo_target = '/home/ubuntu/terasort.sh'
-
-        Path(demo_source).copy(demo_target)
-        Path(demo_target).chmod(0o755)
-        Path(demo_target).chown('ubuntu', 'hadoop')
-        unitdata.kv().set('yarn.client.demo.installed', True)
-        unitdata.kv().flush(True)
-
-    def register_slaves(self):
-        self.hadoop_base.register_slaves('nodemanager')
-
-    def _yarn_daemon(self, command, service):
-        self.hadoop_base.run('yarn', 'sbin/yarn-daemon.sh',
-                             '--config',
-                             self.hadoop_base.dist_config.path('hadoop_conf'),
-                             command, service)
-
-    def _jobhistory_daemon(self, command, service):
-        # TODO refactor job history to separate class
-        self.hadoop_base.run('mapred', 'sbin/mr-jobhistory-daemon.sh',
-                             '--config',
-                             self.hadoop_base.dist_config.path('hadoop_conf'),
-                             command, service)

=== removed file 'charmhelpers/contrib/bigdata/relations.py'
--- charmhelpers/contrib/bigdata/relations.py	2015-12-07 20:40:11 +0000
+++ charmhelpers/contrib/bigdata/relations.py	1970-01-01 00:00:00 +0000
@@ -1,269 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-import json
-
-from charmhelpers.core import hookenv
-from charmhelpers.core.charmframework.helpers import Relation
-from charmhelpers.contrib.bigdata import utils
-
-
-class SpecMatchingRelation(Relation):
-    """
-    Relation base class that validates that a version and environment
-    between two related charms match, to prevent interoperability issues.
-
-    This class adds a ``spec`` key to the ``required_keys`` and populates it
-    in :meth:`provide`.  The ``spec`` value must be passed in to :meth:`__init__`.
-
-    The ``spec`` should be a mapping (or a callback that returns a mapping)
-    which describes all aspects of the charm's environment or configuration
-    that might affect its interoperability with the remote charm.  The charm
-    on the requires side of the relation will verify that all of the keys in
-    its ``spec`` are present and exactly equal on the provides side of the
-    relation.  This does mean that the requires side can be a subset of the
-    provides side, but not the other way around.
-
-    An example spec string might be::
-
-        {
-            'arch': 'x86_64',
-            'vendor': 'apache',
-            'version': '2.4',
-        }
-    """
-    def __init__(self, spec=None, *args, **kwargs):
-        """
-        Create a new relation handler instance.
-
-        :param str spec: Spec string that should capture version or environment
-            particulars which can cause issues if mismatched.
-        """
-        super(SpecMatchingRelation, self).__init__(*args, **kwargs)
-        self._spec = spec
-
-    @property
-    def spec(self):
-        if callable(self._spec):
-            return self._spec()
-        return self._spec
-
-    def provide(self, remote_service, all_ready):
-        """
-        Provide the ``spec`` data to the remote service.
-
-        Subclasses *must* either delegate to this method (e.g., via `super()`)
-        or include ``'spec': json.dumps(self.spec)`` in the provided data themselves.
-        """
-        data = super(SpecMatchingRelation, self).provide(remote_service, all_ready)
-        if self.spec:
-            data['spec'] = json.dumps(self.spec)
-        return data
-
-    def is_ready(self):
-        """
-        Validate the ``spec`` data from the connected units to ensure that
-        it matches the local ``spec``.
-        """
-        if not super(SpecMatchingRelation, self).is_ready():
-            return False
-        if not self.spec:
-            return True
-        for unit, data in self.filtered_data().iteritems():
-            remote_spec = json.loads(data.get('spec', '{}'))
-            for k, v in self.spec.items():
-                if v != remote_spec.get(k):
-                    # TODO XXX Once extended status reporting is available,
-                    #          we should use that instead of erroring.
-                    raise ValueError(
-                        'Spec mismatch with related unit %s: '
-                        '%r != %r' % (unit, data.get('spec'), json.dumps(self.spec)))
-        return True
-
-
-class NameNode(SpecMatchingRelation):
-    """
-    Relation which communicates the NameNode (HDFS) connection & status info.
-
-    This is the relation that clients should use.
-    """
-    relation_name = 'namenode'
-    required_keys = ['private-address', 'etc_hosts', 'port', 'ready']
-
-    def __init__(self, spec=None, port=None):
-        self.port = port  # only needed for provides
-        utils.initialize_kv_host()
-        super(NameNode, self).__init__(spec)
-
-    def provide(self, remote_service, all_ready):
-        data = super(NameNode, self).provide(remote_service, all_ready)
-        if all_ready and DataNode().is_ready():
-            data.update({
-                'ready': 'true',
-                'etc_hosts': utils.get_kv_hosts(),
-                'port': self.port,
-            })
-        return data
-
-
-class NameNodeMaster(NameNode):
-    """
-    Alternate NameNode relation for DataNodes.
-    """
-    relation_name = 'datanode'
-    required_keys = ['private-address', 'etc_hosts', 'port', 'ready', 'ssh-key']
-
-    def provide(self, remote_service, all_ready):
-        data = super(NameNodeMaster, self).provide(remote_service, all_ready)
-        data.update({
-            'ssh-key': utils.get_ssh_key('hdfs'),
-        })
-        return data
-
-    def install_ssh_keys(self):
-        relation_values = self.filtered_data().values()[0]
-        ssh_key = relation_values.get('ssh-key')
-        utils.install_ssh_key('hdfs', ssh_key)
-
-
-class ResourceManager(SpecMatchingRelation):
-    """
-    Relation which communicates the ResourceManager (YARN) connection & status info.
-
-    This is the relation that clients should use.
-    """
-    relation_name = 'resourcemanager'
-    required_keys = ['private-address', 'etc_hosts', 'port', 'ready']
-
-    def __init__(self, spec=None, port=None):
-        self.port = port  # only needed for provides
-        utils.initialize_kv_host()
-        super(ResourceManager, self).__init__(spec)
-
-    def provide(self, remote_service, all_ready):
-        data = super(ResourceManager, self).provide(remote_service, all_ready)
-        if all_ready:
-            data.update({
-                'ready': 'true',
-                'etc_hosts': utils.get_kv_hosts(),
-                'port': self.port,
-            })
-        return data
-
-
-class ResourceManagerMaster(ResourceManager):
-    """
-    Alternate ResourceManager relation for NodeManagers.
-    """
-    relation_name = 'nodemanager'
-    required_keys = ['private-address', 'etc_hosts', 'port', 'ready', 'ssh-key']
-
-    def provide(self, remote_service, all_ready):
-        data = super(ResourceManagerMaster, self).provide(remote_service, all_ready)
-        data.update({
-            'ssh-key': utils.get_ssh_key('yarn'),
-        })
-        return data
-
-    def install_ssh_keys(self):
-        relation_values = self.filtered_data().values()[0]
-        ssh_key = relation_values.get('ssh-key')
-        utils.install_ssh_key('yarn', ssh_key)
-
-
-class DataNode(Relation):
-    """
-    Relation which communicates DataNode info back to NameNodes.
-    """
-    relation_name = 'datanode'
-    required_keys = ['private-address', 'hostname']
-
-    def provide(self, remote_service, all_ready):
-        data = super(DataNode, self).provide(remote_service, all_ready)
-        hostname = hookenv.local_unit().replace('/', '-')
-        data.update({
-            'hostname': hostname,
-        })
-        return data
-
-
-class NodeManager(Relation):
-    """
-    Relation which communicates NodeManager info back to ResourceManagers.
-    """
-    relation_name = 'nodemanager'
-    required_keys = ['private-address', 'hostname']
-
-    def provide(self, remote_service, all_ready):
-        data = super(NodeManager, self).provide(remote_service, all_ready)
-        hostname = hookenv.local_unit().replace('/', '-')
-        data.update({
-            'hostname': hostname,
-        })
-        return data
-
-
-class HadoopPlugin(Relation):
-    relation_name = 'hadoop-plugin'
-    required_keys = ['private-address', 'hdfs-ready']
-
-    def __init__(self, *args, **kwargs):
-        super(HadoopPlugin, self).__init__(*args, **kwargs)
-
-    def provide(self, remote_service, all_ready):
-        if not all_ready:
-            return {}
-        utils.wait_for_hdfs(400)  # will error if timeout
-        return {'hdfs-ready': True}
-
-
-class MySQL(Relation):
-    relation_name = 'db'
-    required_keys = ['host', 'database', 'user', 'password']
-
-
-class FlumeAgent(Relation):
-    relation_name = 'flume-agent'
-    required_keys = ['private-address', 'port']
-
-    def provide(self, remote_service, all_ready):
-        data = super(FlumeAgent, self).provide(remote_service, all_ready)
-        flume_protocol = hookenv.config('protocol')
-        if (flume_protocol not in ['avro']):
-            hookenv.log('Invalid flume protocol {}'.format(flume_protocol), hookenv.ERROR)
-            return data
-        data.update({
-            'protocol': hookenv.config('protocol'),
-        })
-        return data
-
-
-class Hive(Relation):
-    relation_name = 'hive'
-    required_keys = ['private-address', 'port', 'ready']
-
-    def __init__(self, port=None):
-        self.port = port  # only needed for provides
-        super(Hive, self).__init__()
-
-    def provide(self, remote_service, all_ready):
-        data = super(Hive, self).provide(remote_service, all_ready)
-        if all_ready:
-            data.update({
-                'ready': 'true',
-                'port': self.port,
-            })
-        return data

=== removed file 'charmhelpers/contrib/bigdata/utils.py'
--- charmhelpers/contrib/bigdata/utils.py	2015-05-11 02:51:38 +0000
+++ charmhelpers/contrib/bigdata/utils.py	1970-01-01 00:00:00 +0000
@@ -1,438 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-import ast
-import re
-import time
-import yaml
-from contextlib import contextmanager
-from subprocess import check_call, check_output, CalledProcessError
-from xml.etree import ElementTree as ET
-from xml.dom import minidom
-from distutils.util import strtobool
-from path import Path
-from json import dumps
-
-from charmhelpers.core import unitdata
-from charmhelpers.core import hookenv
-from charmhelpers.core import host
-from charmhelpers import fetch
-
-
-class DistConfig(object):
-    """
-    This class processes distribution-specific configuration options.
-
-    Some configuration options are specific to the Hadoop distribution,
-    (e.g. Apache, Hortonworks, MapR, etc). These options are immutable and
-    must not change throughout the charm deployment lifecycle.
-
-    Helper methods are provided for keys that require action. Presently, this
-    includes adding/removing directories, dependent packages, and groups/users.
-    Other required keys may be listed when instantiating this class, but this
-    will only validate these keys exist in the yaml; it will not provide any
-    helper functionality for unkown keys.
-
-    :param str filename: File to process (default dist.yaml)
-    :param list required_keys: A list of keys required to be present in the yaml
-
-    Example dist.yaml with supported keys:
-        vendor: '<name>'
-        hadoop_version: '<version>'
-        packages:
-            - '<package 1>'
-            - '<package 2>'
-        groups:
-            - '<name>'
-        users:
-            <user 1>:
-                groups: ['<primary>', '<group>', '<group>']
-            <user 2>:
-                groups: ['<primary>']
-        dirs:
-            <dir 1>:
-                path: '</path/to/dir>'
-                perms: 0777
-            <dir 2>:
-                path: '{config[<option>]}'  # value comes from config option
-                owner: '<user>'
-                group: '<group>'
-                perms: 0755
-        ports:
-            <name1>:
-                port: <port>
-                exposed_on: <service>  # optional
-            <name2>:
-                port: <port>
-                exposed_on: <service>  # optional
-    """
-    def __init__(self, filename='dist.yaml', required_keys=None):
-        self.yaml_file = filename
-        self.dist_config = yaml.load(Path(self.yaml_file).text())
-
-        # validate dist.yaml
-        missing_keys = set(required_keys or []) - set(self.dist_config.keys())
-        if missing_keys:
-            raise ValueError('{} is missing required option{}: {}'.format(
-                filename,
-                's' if len(missing_keys) > 1 else '',
-                ', '.join(missing_keys)))
-
-        for opt in required_keys:
-            setattr(self, opt, self.dist_config[opt])
-
-    def path(self, key):
-        config = hookenv.config()
-        dirs = {name: self.dirs[name]['path'] for name in self.dirs.keys()}
-        levels = 0
-        old_path = None
-        path = self.dirs[key]['path']
-        while '{' in path and path != old_path:
-            levels += 1
-            if levels > 100:
-                raise ValueError('Maximum level of nested dirs references exceeded for: {}'.format(key))
-            old_path = path
-            path = path.format(config=config, dirs=dirs)
-        return Path(path)
-
-    def port(self, key):
-        return self.ports.get(key, {}).get('port')
-
-    def exposed_ports(self, service):
-        exposed = []
-        for port in self.ports.values():
-            if port.get('exposed_on') == service:
-                exposed.append(port['port'])
-        return exposed
-
-    def add_dirs(self):
-        for name, details in self.dirs.items():
-            host.mkdir(
-                self.path(name),
-                owner=details.get('owner', 'root'),
-                group=details.get('group', 'root'),
-                perms=details.get('perms', 0o755))
-
-    def add_packages(self):
-        with disable_firewall():
-            fetch.apt_update()
-            fetch.apt_install(self.packages)
-
-    def add_users(self):
-        for group in self.groups:
-            host.add_group(group)
-        for username, details in self.users.items():
-            primary_group = None
-            groups = details.get('groups', [])
-            if groups:
-                primary_group = groups[0]
-            host.adduser(username, group=primary_group)
-            for group in groups:
-                host.add_user_to_group(username, group)
-
-    def remove_dirs(self):
-        # TODO: no removal function exists in CH, just log what we would do.
-        for name in self.dirs.items():
-            hookenv.log('noop: remove directory {0}'.format(name))
-
-    def remove_packages(self):
-        # TODO: no removal function exists in CH, just log what we would do.
-        for name in self.packages.items():
-            hookenv.log('noop: remove package {0}'.format(name))
-
-    def remove_users(self):
-        # TODO: no removal function exists in CH, just log what we would do.
-        for user in self.users.items():
-            hookenv.log('noop: remove user {0}'.format(user))
-        for group in self.groups:
-            hookenv.log('noop: remove group {0}'.format(group))
-
-
-@contextmanager
-def disable_firewall():
-    """
-    Temporarily disable the firewall, via ufw.
-    """
-    status = check_output(['ufw', 'status'])
-    already_disabled = 'inactive' in status
-    if not already_disabled:
-        check_call(['ufw', 'disable'])
-    try:
-        yield
-    finally:
-        if not already_disabled:
-            check_call(['ufw', 'enable'])
-
-
-def re_edit_in_place(filename, subs):
-    """
-    Perform a set of in-place edits to a file.
-
-    :param str filename: Name of file to edit
-    :param dict subs: Mapping of patterns to replacement strings
-    """
-    with Path(filename).in_place() as (reader, writer):
-        for line in reader:
-            for pat, repl in subs.iteritems():
-                line = re.sub(pat, repl, line)
-            writer.write(line)
-
-
-@contextmanager
-def xmlpropmap_edit_in_place(filename):
-    """
-    Edit an XML property map (configuration) file in-place.
-
-    This helper acts as a context manager which edits an XML file of the form:
-
-        <configuration>
-            <property>
-                <name>property-name</name>
-                <value>property-value</value>
-                <description>Optional property description</description>
-            </property>
-            ...
-        </configuration>
-
-    This context manager yields a dict containing the existing name/value
-    mappings.  Properties can then be modified, added, or removed, and the
-    changes will be reflected in the file.
-
-    Example usage:
-
-        with xmlpropmap_edit_in_place('my.xml') as props:
-            props['foo'] = 'bar'
-            del props['removed']
-
-    Note that the file is not locked during the edits.
-    """
-    tree = ET.parse(filename)
-    root = tree.getroot()
-    props = {}
-    for prop in root.findall('property'):
-        props[prop.find('name').text] = prop.find('value').text
-    old_props = set(props.keys())
-    yield props
-    new_props = set(props.keys())
-    added = new_props - old_props
-    modified = new_props & old_props
-    removed = old_props - new_props
-    for prop in root.findall('property'):
-        name = prop.find('name').text
-        if name in modified and props[name] is not None:
-            prop.find('value').text = str(props[name])
-        elif name in removed:
-            root.remove(prop)
-    for name in added:
-        prop = ET.SubElement(root, 'property')
-        ET.SubElement(prop, 'name').text = name
-        ET.SubElement(prop, 'value').text = str(props[name])
-    for node in tree.iter():
-        node.tail = None
-        node.text = (node.text or '').strip() or None
-    prettied = minidom.parseString(ET.tostring(root)).toprettyxml(indent='    ')
-    Path(filename).write_text(prettied)
-
-
-@contextmanager
-def environment_edit_in_place(filename='/etc/environment'):
-    """
-    Edit the `/etc/environment` file in-place.
-
-    There is no standard definition for the format of `/etc/environment`,
-    but the convention, which this helper supports, is simple key-value
-    pairs, separated by `=`, with optionally quoted values.
-
-    Note that this helper will implicitly quote all values.
-
-    Also note that the file is not locked during the edits.
-    """
-    etc_env = Path(filename)
-    lines = [l.strip().split('=') for l in etc_env.lines()]
-    data = {k.strip(): v.strip(' \'"') for k, v in lines}
-    yield data
-    etc_env.write_lines('{}="{}"'.format(k, v) for k, v in data.items())
-
-
-def normalize_strbool(value):
-    intbool = strtobool(value)
-    return str(bool(intbool)).lower()
-
-
-def jps(name):
-    """
-    Get PIDs for named Java processes, for any user.
-    """
-    pat = re.sub(r'^(.)', r'^[^ ]*java .*[\1]', name)
-    try:
-        output = check_output(['sudo', 'pgrep', '-f', pat])
-    except CalledProcessError:
-        return []
-    return filter(None, map(str.strip, output.split('\n')))
-
-
-class TimeoutError(Exception):
-    pass
-
-
-def read_etc_env():
-    """
-    Read /etc/environment and return it as a dict.
-    """
-    etc_env = Path('/etc/environment')
-    env = {}
-    if etc_env.exists():
-        for line in etc_env.lines():
-            var, value = line.split('=')
-            env[var.strip()] = value.strip().strip('"')
-    return env
-
-
-def run_as(user, command, *args, **kwargs):
-    """
-    Run a command as a particular user, using ``/etc/environment`` and optionally
-    capturing and returning the output.
-
-    Raises subprocess.CalledProcessError if command fails.
-
-    :param str user: Username to run command as
-    :param str command: Command to run
-    :param list args: Additional args to pass to command
-    :param dict env: Additional env variables (will be merged with ``/etc/environment``)
-    :param bool capture_output: Capture and return output (default: False)
-    """
-    parts = [command] + list(args)
-    quoted = ' '.join("'%s'" % p for p in parts)
-    env = read_etc_env()
-    if 'env' in kwargs:
-        env.update(kwargs['env'])
-    run = check_output if kwargs.get('capture_output') else check_call
-    return run(['su', user, '-c', quoted], env=env)
-
-
-def update_etc_hosts(hosts):
-    '''
-    Update /etc/hosts on the unit
-
-    :param str hosts: json string of host dictionaries
-    '''
-    etc_hosts = Path('/etc/hosts')
-    hosts_contents = etc_hosts.lines()
-
-    for key, data in ast.literal_eval(hosts).items():
-        line = '%s %s' % (data['private-address'], data['hostname'])
-        IP_pat = r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}'
-        if not re.match(IP_pat, data['private-address']):
-            line = '# %s  # INVALID IP' % line
-        for l, hosts_line in enumerate(hosts_contents):
-            if re.match(r'(# )?%s\s' % data['private-address'], hosts_line):
-                # update existing host
-                hosts_contents[l] = line
-                break
-        else:
-            # add new host
-            hosts_contents.append(line)
-
-        # write new /etc/hosts
-        etc_hosts.write_lines(hosts_contents, append=False)
-
-
-def initialize_kv_host():
-    # get the hostname attrs from our local unit and update the kv store
-    local_ip = hookenv.unit_private_ip()
-    local_host = hookenv.local_unit().replace('/', '-')
-    update_kv_host(local_ip, local_host)
-
-
-def get_kv_hosts():
-    unit_kv = unitdata.kv()
-    # all our hosts in the kv are prefixed with 'etc_host.'; they'll come
-    # out of the kv as a unicode object, but convert them to a json string
-    # for ease of use later -- this string of hosts is what we set on
-    # various relations so units can update their local /etc/hosts file.
-    kv_hosts = dumps(unit_kv.getrange('etc_host'))
-    return kv_hosts
-
-
-def update_kv_host(ip, host):
-    unit_kv = unitdata.kv()
-
-    # store attrs in the kv as 'etc_host.<ip>'; kv.update will insert
-    # a new record or update any existing key with current data.
-    unit_kv.update({ip: {'private-address': ip,
-                         'hostname': host}},
-                   prefix="etc_host.")
-    unit_kv.flush(True)
-
-
-def get_ssh_key(user):
-    sshdir = Path('/home/%s/.ssh' % user)
-    if not sshdir.exists():
-        host.mkdir(sshdir, owner=user, group='hadoop', perms=0o755)
-    keyfile = sshdir / 'id_rsa'
-    pubfile = sshdir / 'id_rsa.pub'
-    authfile = sshdir / 'authorized_keys'
-    if not pubfile.exists():
-        (sshdir / 'config').write_lines([
-            'Host *',
-            '    StrictHostKeyChecking no'
-        ], append=True)
-        check_call(['ssh-keygen', '-t', 'rsa', '-P', '', '-f', keyfile])
-        host.chownr(sshdir, user, 'hadoop')
-    # allow ssh'ing to localhost; useful for things like start_dfs.sh
-    if not authfile.exists():
-        Path.copy(pubfile, authfile)
-    return pubfile.text()
-
-
-def install_ssh_key(user, ssh_key):
-    sshdir = Path('/home/%s/.ssh' % user)
-    if not sshdir.exists():
-        host.mkdir(sshdir, owner=user, group='hadoop', perms=0o755)
-    Path(sshdir / 'authorized_keys').write_text(ssh_key, append=True)
-    host.chownr(sshdir, user, 'hadoop')
-
-
-def wait_for_hdfs(timeout):
-    start = time.time()
-    while time.time() - start < timeout:
-        try:
-            output = run_as('hdfs', 'hdfs', 'dfsadmin', '-report', capture_output=True)
-            if 'Datanodes available' in output:
-                return True
-        except CalledProcessError as e:
-            output = e.output  # probably a "connection refused"; wait and try again
-        time.sleep(2)
-    raise TimeoutError('Timed-out waiting for HDFS:\n%s' % output)
-
-
-class verify_resources(object):
-    """
-    Predicate for specific named resources, with useful rendering in the logs.
-
-    :param str *which: One or more resource names to fetch & verify.  Defaults to
-        all non-optional resources.
-    """
-    def __init__(self, *which):
-        self.which = list(which)
-
-    def __str__(self):
-        return '<resources %s>' % ', '.join(map(repr, self.which))
-
-    def __call__(self):
-        import jujuresources
-        mirror_url = hookenv.config('resources_mirror')
-        return jujuresources.fetch(self.which, mirror_url=mirror_url)

=== removed directory 'tests/contrib/bigdata'
=== removed file 'tests/contrib/bigdata/__init__.py'
--- tests/contrib/bigdata/__init__.py	2015-01-22 14:30:19 +0000
+++ tests/contrib/bigdata/__init__.py	1970-01-01 00:00:00 +0000
@@ -1,15 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

=== removed file 'tests/contrib/bigdata/test_relations.py'
--- tests/contrib/bigdata/test_relations.py	2015-04-15 12:12:19 +0000
+++ tests/contrib/bigdata/test_relations.py	1970-01-01 00:00:00 +0000
@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-
-import unittest
-import mock
-
-try:
-    from path import Path
-except ImportError:
-    Path = None
-    relations = None
-else:
-    from charmhelpers.contrib.bigdata import relations
-
-
-@unittest.skipUnless(Path, 'charmhelpers.contrib.bigdata requires path.py')
-class TestSpecMatchingRelation(unittest.TestCase):
-    def setUp(self):
-        self.data = None
-        self.cache = {}
-        self.relation = relations.SpecMatchingRelation(
-            spec='valid',
-            relation_name='test',
-            required_keys=['foo'],
-            datastore=mock.MagicMock(),
-            cache=self.cache)
-        self.relation.unfiltered_data = lambda: self.data
-
-    def test_ready(self):
-        self.data = {'unit/0': {'spec': 'valid', 'foo': 'bar'}}
-        self.assertTrue(self.relation.is_ready())
-
-    def test_not_ready(self):
-        self.data = {}
-        self.assertFalse(self.relation.is_ready())
-        self.cache.clear()
-        self.data = {'unit/0': {}}
-        self.assertFalse(self.relation.is_ready())
-        self.cache.clear()
-        self.data = {'unit/0': {'no-spec': 'valid', 'foo': 'bar'}}
-        self.assertFalse(self.relation.is_ready())
-        self.cache.clear()
-        self.data = {'unit/0': {'spec': 'valid', 'no-foo': 'bar'}}
-        self.assertFalse(self.relation.is_ready())
-        self.cache.clear()
-        self.data = {'unit/0': {'spec': 'invalid', 'no-foo': 'bar'}}
-        self.assertFalse(self.relation.is_ready())
-
-    def test_invalid(self):
-        self.data = {'unit/0': {'spec': 'invalid', 'foo': 'bar'}}
-        self.assertRaises(ValueError, self.relation.is_ready)
-
-
-if __name__ == '__main__':
-    unittest.main()

=== removed file 'tests/contrib/bigdata/test_utils.py'
--- tests/contrib/bigdata/test_utils.py	2015-01-22 14:30:19 +0000
+++ tests/contrib/bigdata/test_utils.py	1970-01-01 00:00:00 +0000
@@ -1,125 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-
-import os
-import tempfile
-import unittest
-import mock
-
-try:
-    from path import Path
-except ImportError:
-    Path = None
-    utils = None
-else:
-    from charmhelpers.contrib.bigdata import utils
-
-
-class TestError(RuntimeError):
-    pass
-
-
-@unittest.skipUnless(Path, 'charmhelpers.contrib.bigdata requires path.py')
-class TestUtils(unittest.TestCase):
-    def test_disable_firewall(self):
-        with mock.patch.object(utils, 'check_call') as check_call:
-            with utils.disable_firewall():
-                check_call.assert_called_once_with(['ufw', 'disable'])
-            check_call.assert_called_with(['ufw', 'enable'])
-
-    def test_disable_firewall_on_error(self):
-        with mock.patch.object(utils, 'check_call') as check_call:
-            try:
-                with utils.disable_firewall():
-                    check_call.assert_called_once_with(['ufw', 'disable'])
-                    raise TestError()
-            except TestError:
-                check_call.assert_called_with(['ufw', 'enable'])
-
-    def test_re_edit_in_place(self):
-        fd, filename = tempfile.mkstemp()
-        os.close(fd)
-        tmp_file = Path(filename)
-        try:
-            tmp_file.write_text('foo\nbar\nqux')
-            utils.re_edit_in_place(tmp_file, {
-                r'oo$': 'OO',
-                r'a': 'A',
-                r'^qux$': 'QUX',
-            })
-            self.assertEqual(tmp_file.text(), 'fOO\nbAr\nQUX')
-        finally:
-            tmp_file.remove()
-
-    def test_xmlpropmap_edit_in_place(self):
-        fd, filename = tempfile.mkstemp()
-        os.close(fd)
-        tmp_file = Path(filename)
-        try:
-            tmp_file.write_text(
-                '<?xml version="1.0"?>\n'
-                '<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>\n'
-                '\n'
-                '<!-- Put site-specific property overrides in this file. -->\n'
-                '\n'
-                '<configuration>\n'
-                '   <property>\n'
-                '       <name>modify.me</name>\n'
-                '       <value>1</value>\n'
-                '       <description>Property to be modified</description>\n'
-                '   </property>\n'
-                '   <property>\n'
-                '       <name>delete.me</name>\n'
-                '       <value>None</value>\n'
-                '       <description>Property to be removed</description>\n'
-                '   </property>\n'
-                '   <property>\n'
-                '       <name>do.not.modify.me</name>\n'
-                '       <value>0</value>\n'
-                '       <description>Property to *not* be modified</description>\n'
-                '   </property>\n'
-                '</configuration>')
-            with utils.xmlpropmap_edit_in_place(tmp_file) as props:
-                del props['delete.me']
-                props['modify.me'] = 'one'
-                props['add.me'] = 'NEW'
-            self.assertEqual(
-                tmp_file.text(),
-                '<?xml version="1.0" ?>\n'
-                '<configuration>\n'
-                '    <property>\n'
-                '        <name>modify.me</name>\n'
-                '        <value>one</value>\n'
-                '        <description>Property to be modified</description>\n'
-                '    </property>\n'
-                '    <property>\n'
-                '        <name>do.not.modify.me</name>\n'
-                '        <value>0</value>\n'
-                '        <description>Property to *not* be modified</description>\n'
-                '    </property>\n'
-                '    <property>\n'
-                '        <name>add.me</name>\n'
-                '        <value>NEW</value>\n'
-                '    </property>\n'
-                '</configuration>\n')
-        finally:
-            tmp_file.remove()
-
-
-if __name__ == '__main__':
-    unittest.main()

=== modified file 'tests/core/test_host.py'
--- tests/core/test_host.py	2015-04-29 12:52:18 +0000
+++ tests/core/test_host.py	2015-12-08 19:13:38 +0000
@@ -272,19 +272,34 @@
         getpwnam.assert_called_with(username)
 
     @patch('subprocess.check_call')
+    @patch('grp.getgrnam')
+    @patch('pwd.getpwnam')
     @patch.object(host, 'log')
-    def test_adds_a_user_to_a_group(self, log, check_call):
+    def test_adds_a_user_to_a_group(self, log, getpwnam, getgrnam, check_call):
         username = 'foo'
         group = 'bar'
+        getpwnam.return_value.pw_gid = 'ogid'
+        getgrnam.return_value.gr_gid = 'gid'
+        getgrnam.return_value.gr_mem = ['user']
 
         host.add_user_to_group(username, group)
-
         check_call.assert_called_with([
             'gpasswd', '-a',
             username,
             group
         ])
 
+        check_call.reset_mock()
+        getpwnam.return_value.pw_gid = 'gid'
+        host.add_user_to_group(username, group)
+        assert not check_call.called
+
+        check_call.reset_mock()
+        getpwnam.return_value.pw_gid = 'ogid'
+        getgrnam.return_value.gr_mem = ['user', 'foo']
+        host.add_user_to_group(username, group)
+        assert not check_call.called
+
     @patch('grp.getgrnam')
     @patch('subprocess.check_call')
     @patch.object(host, 'log')

=== removed file 'tests/core/test_services.py'
--- tests/core/test_services.py	2015-04-03 14:53:10 +0000
+++ tests/core/test_services.py	1970-01-01 00:00:00 +0000
@@ -1,786 +0,0 @@
-import mock
-import unittest
-from charmhelpers.core import hookenv
-from charmhelpers.core import services
-
-
-class TestServiceManager(unittest.TestCase):
-    def setUp(self):
-        self.pcharm_dir = mock.patch.object(hookenv, 'charm_dir')
-        self.mcharm_dir = self.pcharm_dir.start()
-        self.mcharm_dir.return_value = 'charm_dir'
-
-    def tearDown(self):
-        self.pcharm_dir.stop()
-
-    def test_register(self):
-        manager = services.ServiceManager([
-            {'service': 'service1',
-             'foo': 'bar'},
-            {'service': 'service2',
-             'qux': 'baz'},
-        ])
-        self.assertEqual(manager.services, {
-            'service1': {'service': 'service1',
-                         'foo': 'bar'},
-            'service2': {'service': 'service2',
-                         'qux': 'baz'},
-        })
-
-    def test_register_preserves_order(self):
-        service_list = [dict(service='a'), dict(service='b')]
-
-        # Test that the services list order is preserved by checking
-        # both forwards and backwards - only one of these will be
-        # dictionary order, and if both work we know order is being
-        # preserved.
-        manager = services.ServiceManager(service_list)
-        self.assertEqual(list(manager.services.keys()), ['a', 'b'])
-        manager = services.ServiceManager(reversed(service_list))
-        self.assertEqual(list(manager.services.keys()), ['b', 'a'])
-
-    @mock.patch.object(services.ServiceManager, 'reconfigure_services')
-    @mock.patch.object(services.ServiceManager, 'stop_services')
-    @mock.patch.object(hookenv, 'hook_name')
-    @mock.patch.object(hookenv, 'config')
-    def test_manage_stop(self, config, hook_name, stop_services, reconfigure_services):
-        manager = services.ServiceManager()
-        hook_name.return_value = 'stop'
-        manager.manage()
-        stop_services.assert_called_once_with()
-        assert not reconfigure_services.called
-
-    @mock.patch.object(services.ServiceManager, 'provide_data')
-    @mock.patch.object(services.ServiceManager, 'reconfigure_services')
-    @mock.patch.object(services.ServiceManager, 'stop_services')
-    @mock.patch.object(hookenv, 'hook_name')
-    @mock.patch.object(hookenv, 'config')
-    def test_manage_other(self, config, hook_name, stop_services, reconfigure_services, provide_data):
-        manager = services.ServiceManager()
-        hook_name.return_value = 'config-changed'
-        manager.manage()
-        assert not stop_services.called
-        reconfigure_services.assert_called_once_with()
-        provide_data.assert_called_once_with()
-
-    @mock.patch.object(hookenv, 'config')
-    def test_manage_config_saved(self, config):
-        config = config.return_value
-        config.implicit_save = True
-        manager = services.ServiceManager()
-        manager.manage()
-        self.assertTrue(config.save.called)
-
-    @mock.patch.object(hookenv, 'config')
-    def test_manage_config_not_saved(self, config):
-        config = config.return_value
-        config.implicit_save = False
-        manager = services.ServiceManager()
-        manager.manage()
-        self.assertFalse(config.save.called)
-
-    @mock.patch.object(services.ServiceManager, 'save_ready')
-    @mock.patch.object(services.ServiceManager, 'fire_event')
-    @mock.patch.object(services.ServiceManager, 'is_ready')
-    def test_reconfigure_ready(self, is_ready, fire_event, save_ready):
-        manager = services.ServiceManager([
-            {'service': 'service1'}, {'service': 'service2'}])
-        is_ready.return_value = True
-        manager.reconfigure_services()
-        is_ready.assert_has_calls([
-            mock.call('service1'),
-            mock.call('service2'),
-        ], any_order=True)
-        fire_event.assert_has_calls([
-            mock.call('data_ready', 'service1'),
-            mock.call('start', 'service1', default=[
-                services.service_restart,
-                services.manage_ports]),
-        ], any_order=False)
-        fire_event.assert_has_calls([
-            mock.call('data_ready', 'service2'),
-            mock.call('start', 'service2', default=[
-                services.service_restart,
-                services.manage_ports]),
-        ], any_order=False)
-        save_ready.assert_has_calls([
-            mock.call('service1'),
-            mock.call('service2'),
-        ], any_order=True)
-
-    @mock.patch.object(services.ServiceManager, 'save_ready')
-    @mock.patch.object(services.ServiceManager, 'fire_event')
-    @mock.patch.object(services.ServiceManager, 'is_ready')
-    def test_reconfigure_ready_list(self, is_ready, fire_event, save_ready):
-        manager = services.ServiceManager([
-            {'service': 'service1'}, {'service': 'service2'}])
-        is_ready.return_value = True
-        manager.reconfigure_services('service3', 'service4')
-        self.assertEqual(is_ready.call_args_list, [
-            mock.call('service3'),
-            mock.call('service4'),
-        ])
-        self.assertEqual(fire_event.call_args_list, [
-            mock.call('data_ready', 'service3'),
-            mock.call('start', 'service3', default=[
-                services.service_restart,
-                services.open_ports]),
-            mock.call('data_ready', 'service4'),
-            mock.call('start', 'service4', default=[
-                services.service_restart,
-                services.open_ports]),
-        ])
-        self.assertEqual(save_ready.call_args_list, [
-            mock.call('service3'),
-            mock.call('service4'),
-        ])
-
-    @mock.patch.object(services.ServiceManager, 'save_lost')
-    @mock.patch.object(services.ServiceManager, 'fire_event')
-    @mock.patch.object(services.ServiceManager, 'was_ready')
-    @mock.patch.object(services.ServiceManager, 'is_ready')
-    def test_reconfigure_not_ready(self, is_ready, was_ready, fire_event, save_lost):
-        manager = services.ServiceManager([
-            {'service': 'service1'}, {'service': 'service2'}])
-        is_ready.return_value = False
-        was_ready.return_value = False
-        manager.reconfigure_services()
-        is_ready.assert_has_calls([
-            mock.call('service1'),
-            mock.call('service2'),
-        ], any_order=True)
-        fire_event.assert_has_calls([
-            mock.call('stop', 'service1', default=[
-                services.close_ports,
-                services.service_stop]),
-            mock.call('stop', 'service2', default=[
-                services.close_ports,
-                services.service_stop]),
-        ], any_order=True)
-        save_lost.assert_has_calls([
-            mock.call('service1'),
-            mock.call('service2'),
-        ], any_order=True)
-
-    @mock.patch.object(services.ServiceManager, 'save_lost')
-    @mock.patch.object(services.ServiceManager, 'fire_event')
-    @mock.patch.object(services.ServiceManager, 'was_ready')
-    @mock.patch.object(services.ServiceManager, 'is_ready')
-    def test_reconfigure_no_longer_ready(self, is_ready, was_ready, fire_event, save_lost):
-        manager = services.ServiceManager([
-            {'service': 'service1'}, {'service': 'service2'}])
-        is_ready.return_value = False
-        was_ready.return_value = True
-        manager.reconfigure_services()
-        is_ready.assert_has_calls([
-            mock.call('service1'),
-            mock.call('service2'),
-        ], any_order=True)
-        fire_event.assert_has_calls([
-            mock.call('data_lost', 'service1'),
-            mock.call('stop', 'service1', default=[
-                services.close_ports,
-                services.service_stop]),
-        ], any_order=False)
-        fire_event.assert_has_calls([
-            mock.call('data_lost', 'service2'),
-            mock.call('stop', 'service2', default=[
-                services.close_ports,
-                services.service_stop]),
-        ], any_order=False)
-        save_lost.assert_has_calls([
-            mock.call('service1'),
-            mock.call('service2'),
-        ], any_order=True)
-
-    @mock.patch.object(services.ServiceManager, 'fire_event')
-    def test_stop_services(self, fire_event):
-        manager = services.ServiceManager([
-            {'service': 'service1'}, {'service': 'service2'}])
-        manager.stop_services()
-        fire_event.assert_has_calls([
-            mock.call('stop', 'service1', default=[
-                services.close_ports,
-                services.service_stop]),
-            mock.call('stop', 'service2', default=[
-                services.close_ports,
-                services.service_stop]),
-        ], any_order=True)
-
-    @mock.patch.object(services.ServiceManager, 'fire_event')
-    def test_stop_services_list(self, fire_event):
-        manager = services.ServiceManager([
-            {'service': 'service1'}, {'service': 'service2'}])
-        manager.stop_services('service3', 'service4')
-        self.assertEqual(fire_event.call_args_list, [
-            mock.call('stop', 'service3', default=[
-                services.close_ports,
-                services.service_stop]),
-            mock.call('stop', 'service4', default=[
-                services.close_ports,
-                services.service_stop]),
-        ])
-
-    def test_get_service(self):
-        service = {'service': 'test', 'test': 'test_service'}
-        manager = services.ServiceManager([service])
-        self.assertEqual(manager.get_service('test'), service)
-
-    def test_get_service_not_registered(self):
-        service = {'service': 'test', 'test': 'test_service'}
-        manager = services.ServiceManager([service])
-        self.assertRaises(KeyError, manager.get_service, 'foo')
-
-    @mock.patch.object(services.ServiceManager, 'get_service')
-    def test_fire_event_default(self, get_service):
-        get_service.return_value = {}
-        cb = mock.Mock()
-        manager = services.ServiceManager()
-        manager.fire_event('event', 'service', cb)
-        cb.assert_called_once_with('service')
-
-    @mock.patch.object(services.ServiceManager, 'get_service')
-    def test_fire_event_default_list(self, get_service):
-        get_service.return_value = {}
-        cb = mock.Mock()
-        manager = services.ServiceManager()
-        manager.fire_event('event', 'service', [cb])
-        cb.assert_called_once_with('service')
-
-    @mock.patch.object(services.ServiceManager, 'get_service')
-    def test_fire_event_simple_callback(self, get_service):
-        cb = mock.Mock()
-        dcb = mock.Mock()
-        get_service.return_value = {'event': cb}
-        manager = services.ServiceManager()
-        manager.fire_event('event', 'service', dcb)
-        assert not dcb.called
-        cb.assert_called_once_with('service')
-
-    @mock.patch.object(services.ServiceManager, 'get_service')
-    def test_fire_event_simple_callback_list(self, get_service):
-        cb = mock.Mock()
-        dcb = mock.Mock()
-        get_service.return_value = {'event': [cb]}
-        manager = services.ServiceManager()
-        manager.fire_event('event', 'service', dcb)
-        assert not dcb.called
-        cb.assert_called_once_with('service')
-
-    @mock.patch.object(services.ManagerCallback, '__call__')
-    @mock.patch.object(services.ServiceManager, 'get_service')
-    def test_fire_event_manager_callback(self, get_service, mcall):
-        cb = services.ManagerCallback()
-        dcb = mock.Mock()
-        get_service.return_value = {'event': cb}
-        manager = services.ServiceManager()
-        manager.fire_event('event', 'service', dcb)
-        assert not dcb.called
-        mcall.assert_called_once_with(manager, 'service', 'event')
-
-    @mock.patch.object(services.ManagerCallback, '__call__')
-    @mock.patch.object(services.ServiceManager, 'get_service')
-    def test_fire_event_manager_callback_list(self, get_service, mcall):
-        cb = services.ManagerCallback()
-        dcb = mock.Mock()
-        get_service.return_value = {'event': [cb]}
-        manager = services.ServiceManager()
-        manager.fire_event('event', 'service', dcb)
-        assert not dcb.called
-        mcall.assert_called_once_with(manager, 'service', 'event')
-
-    @mock.patch.object(services.ServiceManager, 'get_service')
-    def test_is_ready(self, get_service):
-        get_service.side_effect = [
-            {},
-            {'required_data': [True]},
-            {'required_data': [False]},
-            {'required_data': [True, False]},
-        ]
-        manager = services.ServiceManager()
-        assert manager.is_ready('foo')
-        assert manager.is_ready('bar')
-        assert not manager.is_ready('foo')
-        assert not manager.is_ready('foo')
-        get_service.assert_has_calls([mock.call('foo'), mock.call('bar')])
-
-    def test_load_ready_file_short_circuit(self):
-        manager = services.ServiceManager()
-        manager._ready = 'foo'
-        manager._load_ready_file()
-        self.assertEqual(manager._ready, 'foo')
-
-    @mock.patch('os.path.exists')
-    @mock.patch.object(services.base, 'open', create=True)
-    def test_load_ready_file_new(self, mopen, exists):
-        manager = services.ServiceManager()
-        exists.return_value = False
-        manager._load_ready_file()
-        self.assertEqual(manager._ready, set())
-        assert not mopen.called
-
-    @mock.patch('json.load')
-    @mock.patch('os.path.exists')
-    @mock.patch.object(services.base, 'open', create=True)
-    def test_load_ready_file(self, mopen, exists, jload):
-        manager = services.ServiceManager()
-        exists.return_value = True
-        jload.return_value = ['bar']
-        manager._load_ready_file()
-        self.assertEqual(manager._ready, set(['bar']))
-        exists.assert_called_once_with('charm_dir/READY-SERVICES.json')
-        mopen.assert_called_once_with('charm_dir/READY-SERVICES.json')
-
-    @mock.patch('json.dump')
-    @mock.patch.object(services.base, 'open', create=True)
-    def test_save_ready_file(self, mopen, jdump):
-        manager = services.ServiceManager()
-        manager._save_ready_file()
-        assert not mopen.called
-        manager._ready = set(['foo'])
-        manager._save_ready_file()
-        mopen.assert_called_once_with('charm_dir/READY-SERVICES.json', 'w')
-        jdump.assert_called_once_with(['foo'], mopen.return_value.__enter__())
-
-    @mock.patch.object(services.base.ServiceManager, '_save_ready_file')
-    @mock.patch.object(services.base.ServiceManager, '_load_ready_file')
-    def test_save_ready(self, _lrf, _srf):
-        manager = services.ServiceManager()
-        manager._ready = set(['foo'])
-        manager.save_ready('bar')
-        _lrf.assert_called_once_with()
-        self.assertEqual(manager._ready, set(['foo', 'bar']))
-        _srf.assert_called_once_with()
-
-    @mock.patch.object(services.base.ServiceManager, '_save_ready_file')
-    @mock.patch.object(services.base.ServiceManager, '_load_ready_file')
-    def test_save_lost(self, _lrf, _srf):
-        manager = services.ServiceManager()
-        manager._ready = set(['foo', 'bar'])
-        manager.save_lost('bar')
-        _lrf.assert_called_once_with()
-        self.assertEqual(manager._ready, set(['foo']))
-        _srf.assert_called_once_with()
-        manager.save_lost('bar')
-        self.assertEqual(manager._ready, set(['foo']))
-
-    @mock.patch.object(services.base.ServiceManager, '_save_ready_file')
-    @mock.patch.object(services.base.ServiceManager, '_load_ready_file')
-    def test_was_ready(self, _lrf, _srf):
-        manager = services.ServiceManager()
-        manager._ready = set()
-        manager.save_ready('foo')
-        manager.save_ready('bar')
-        assert manager.was_ready('foo')
-        assert manager.was_ready('bar')
-        manager.save_lost('bar')
-        assert manager.was_ready('foo')
-        assert not manager.was_ready('bar')
-
-    @mock.patch.object(services.base.hookenv, 'relation_set')
-    @mock.patch.object(services.base.hookenv, 'hook_name')
-    def test_provide_data_no_match(self, hook_name, relation_set):
-        provider = mock.Mock()
-        provider.name = 'provided'
-        manager = services.ServiceManager([
-            {'service': 'service', 'provided_data': [provider]}
-        ])
-        hook_name.return_value = 'not-provided-relation-joined'
-        manager.provide_data()
-        assert not provider.provide_data.called
-
-        hook_name.return_value = 'provided-relation-broken'
-        manager.provide_data()
-        assert not provider.provide_data.called
-
-    @mock.patch.object(services.base.hookenv, 'relation_set')
-    @mock.patch.object(services.base.hookenv, 'hook_name')
-    def test_provide_data_not_ready(self, hook_name, relation_set):
-        provider = mock.Mock()
-        provider.name = 'provided'
-        data = provider.provide_data.return_value = {'data': True}
-        provider._is_ready.return_value = False
-        manager = services.ServiceManager([
-            {'service': 'service', 'provided_data': [provider]}
-        ])
-        hook_name.return_value = 'provided-relation-joined'
-        manager.provide_data()
-        assert not relation_set.called
-        provider._is_ready.assert_called_once_with(data)
-
-    @mock.patch.object(services.base.hookenv, 'relation_set')
-    @mock.patch.object(services.base.hookenv, 'hook_name')
-    def test_provide_data_ready(self, hook_name, relation_set):
-        provider = mock.Mock()
-        provider.name = 'provided'
-        data = provider.provide_data.return_value = {'data': True}
-        provider._is_ready.return_value = True
-        manager = services.ServiceManager([
-            {'service': 'service', 'provided_data': [provider]}
-        ])
-        hook_name.return_value = 'provided-relation-changed'
-        manager.provide_data()
-        relation_set.assert_called_once_with(None, data)
-
-
-class TestRelationContext(unittest.TestCase):
-    def setUp(self):
-        self.phookenv = mock.patch.object(services.helpers, 'hookenv')
-        self.mhookenv = self.phookenv.start()
-        self.mhookenv.relation_ids.return_value = []
-        self.context = services.RelationContext()
-        self.context.name = 'http'
-        self.context.interface = 'http'
-        self.context.required_keys = ['foo', 'bar']
-        self.mhookenv.reset_mock()
-
-    def tearDown(self):
-        self.phookenv.stop()
-
-    def test_no_relations(self):
-        self.context.get_data()
-        self.assertFalse(self.context.is_ready())
-        self.assertEqual(self.context, {})
-        self.mhookenv.relation_ids.assert_called_once_with('http')
-
-    def test_no_units(self):
-        self.mhookenv.relation_ids.return_value = ['nginx']
-        self.mhookenv.related_units.return_value = []
-        self.context.get_data()
-        self.assertFalse(self.context.is_ready())
-        self.assertEqual(self.context, {'http': []})
-
-    def test_incomplete(self):
-        self.mhookenv.relation_ids.return_value = ['nginx', 'apache']
-        self.mhookenv.related_units.side_effect = lambda i: [i + '/0']
-        self.mhookenv.relation_get.side_effect = [{}, {'foo': '1'}]
-        self.context.get_data()
-        self.assertFalse(bool(self.context))
-        self.assertEqual(self.mhookenv.relation_get.call_args_list, [
-            mock.call(rid='apache', unit='apache/0'),
-            mock.call(rid='nginx', unit='nginx/0'),
-        ])
-
-    def test_complete(self):
-        self.mhookenv.relation_ids.return_value = ['nginx', 'apache', 'tomcat']
-        self.mhookenv.related_units.side_effect = lambda i: [i + '/0']
-        self.mhookenv.relation_get.side_effect = [{'foo': '1'}, {'foo': '2', 'bar': '3'}, {}]
-        self.context.get_data()
-        self.assertTrue(self.context.is_ready())
-        self.assertEqual(self.context, {'http': [
-            {
-                'foo': '2',
-                'bar': '3',
-            },
-        ]})
-        self.mhookenv.relation_ids.assert_called_with('http')
-        self.assertEqual(self.mhookenv.relation_get.call_args_list, [
-            mock.call(rid='apache', unit='apache/0'),
-            mock.call(rid='nginx', unit='nginx/0'),
-            mock.call(rid='tomcat', unit='tomcat/0'),
-        ])
-
-    def test_provide(self):
-        self.assertEqual(self.context.provide_data(), {})
-
-
-class TestHttpRelation(unittest.TestCase):
-    def setUp(self):
-        self.phookenv = mock.patch.object(services.helpers, 'hookenv')
-        self.mhookenv = self.phookenv.start()
-
-        self.context = services.helpers.HttpRelation()
-
-    def tearDown(self):
-        self.phookenv.stop()
-
-    def test_provide_data(self):
-        self.mhookenv.unit_get.return_value = "127.0.0.1"
-        self.assertEqual(self.context.provide_data(), {
-            'host': "127.0.0.1",
-            'port': 80,
-        })
-
-    def test_complete(self):
-        self.mhookenv.relation_ids.return_value = ['website']
-        self.mhookenv.related_units.side_effect = lambda i: [i + '/0']
-        self.mhookenv.relation_get.side_effect = [{'host': '127.0.0.2',
-                                                   'port': 8080}]
-        self.context.get_data()
-        self.assertTrue(self.context.is_ready())
-        self.assertEqual(self.context, {'website': [
-            {
-                'host': '127.0.0.2',
-                'port': 8080,
-            },
-        ]})
-
-        self.mhookenv.relation_ids.assert_called_with('website')
-        self.assertEqual(self.mhookenv.relation_get.call_args_list, [
-            mock.call(rid='website', unit='website/0'),
-        ])
-
-
-class TestMysqlRelation(unittest.TestCase):
-
-    def setUp(self):
-        self.phookenv = mock.patch.object(services.helpers, 'hookenv')
-        self.mhookenv = self.phookenv.start()
-
-        self.context = services.helpers.MysqlRelation()
-
-    def tearDown(self):
-        self.phookenv.stop()
-
-    def test_complete(self):
-        self.mhookenv.relation_ids.return_value = ['db']
-        self.mhookenv.related_units.side_effect = lambda i: [i + '/0']
-        self.mhookenv.relation_get.side_effect = [{'host': '127.0.0.2',
-                                                   'user': 'mysql',
-                                                   'password': 'mysql',
-                                                   'database': 'mysql',
-                                                   }]
-        self.context.get_data()
-        self.assertTrue(self.context.is_ready())
-        self.assertEqual(self.context, {'db': [
-            {
-                'host': '127.0.0.2',
-                'user': 'mysql',
-                'password': 'mysql',
-                'database': 'mysql',
-            },
-        ]})
-
-        self.mhookenv.relation_ids.assert_called_with('db')
-        self.assertEqual(self.mhookenv.relation_get.call_args_list, [
-            mock.call(rid='db', unit='db/0'),
-        ])
-
-
-class TestRequiredConfig(unittest.TestCase):
-    def setUp(self):
-        self.options = {
-            'options': {
-                'option1': {
-                    'type': 'string',
-                    'description': 'First option',
-                },
-                'option2': {
-                    'type': 'int',
-                    'default': 0,
-                    'description': 'Second option',
-                },
-            },
-        }
-        self.config = {
-            'option1': None,
-            'option2': 0,
-        }
-        self._pyaml = mock.patch.object(services.helpers, 'yaml')
-        self.myaml = self._pyaml.start()
-        self.myaml.load.side_effect = lambda fp: self.options
-        self._pconfig = mock.patch.object(hookenv, 'config')
-        self.mconfig = self._pconfig.start()
-        self.mconfig.side_effect = lambda: self.config
-        self._pcharm_dir = mock.patch.object(hookenv, 'charm_dir')
-        self.mcharm_dir = self._pcharm_dir.start()
-        self.mcharm_dir.return_value = 'charm_dir'
-
-    def tearDown(self):
-        self._pyaml.stop()
-        self._pconfig.stop()
-        self._pcharm_dir.stop()
-
-    def test_none_changed(self):
-        with mock.patch.object(services.helpers, 'open', mock.mock_open(), create=True):
-            context = services.helpers.RequiredConfig('option1', 'option2')
-        self.assertFalse(bool(context))
-        self.assertEqual(context['config']['option1'], None)
-        self.assertEqual(context['config']['option2'], 0)
-
-    def test_partial(self):
-        self.config['option1'] = 'value'
-        with mock.patch.object(services.helpers, 'open', mock.mock_open(), create=True):
-            context = services.helpers.RequiredConfig('option1', 'option2')
-        self.assertFalse(bool(context))
-        self.assertEqual(context['config']['option1'], 'value')
-        self.assertEqual(context['config']['option2'], 0)
-
-    def test_ready(self):
-        self.config['option1'] = 'value'
-        self.config['option2'] = 1
-        with mock.patch.object(services.helpers, 'open', mock.mock_open(), create=True):
-            context = services.helpers.RequiredConfig('option1', 'option2')
-        self.assertTrue(bool(context))
-        self.assertEqual(context['config']['option1'], 'value')
-        self.assertEqual(context['config']['option2'], 1)
-
-    def test_none_empty(self):
-        self.config['option1'] = ''
-        self.config['option2'] = 1
-        with mock.patch.object(services.helpers, 'open', mock.mock_open(), create=True):
-            context = services.helpers.RequiredConfig('option1', 'option2')
-        self.assertFalse(bool(context))
-        self.assertEqual(context['config']['option1'], '')
-        self.assertEqual(context['config']['option2'], 1)
-
-
-class TestStoredContext(unittest.TestCase):
-    @mock.patch.object(services.helpers.StoredContext, 'read_context')
-    @mock.patch.object(services.helpers.StoredContext, 'store_context')
-    @mock.patch('os.path.exists')
-    def test_new(self, exists, store_context, read_context):
-        exists.return_value = False
-        context = services.helpers.StoredContext('foo.yaml', {'key': 'val'})
-        assert not read_context.called
-        store_context.assert_called_once_with('foo.yaml', {'key': 'val'})
-        self.assertEqual(context, {'key': 'val'})
-
-    @mock.patch.object(services.helpers.StoredContext, 'read_context')
-    @mock.patch.object(services.helpers.StoredContext, 'store_context')
-    @mock.patch('os.path.exists')
-    def test_existing(self, exists, store_context, read_context):
-        exists.return_value = True
-        read_context.return_value = {'key': 'other'}
-        context = services.helpers.StoredContext('foo.yaml', {'key': 'val'})
-        read_context.assert_called_once_with('foo.yaml')
-        assert not store_context.called
-        self.assertEqual(context, {'key': 'other'})
-
-    @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir')
-    @mock.patch.object(services.helpers.StoredContext, 'read_context')
-    @mock.patch.object(services.helpers, 'yaml')
-    @mock.patch('os.fchmod')
-    @mock.patch('os.path.exists')
-    def test_store_context(self, exists, fchmod, yaml, read_context):
-        exists.return_value = False
-        mopen = mock.mock_open()
-        with mock.patch.object(services.helpers, 'open', mopen, create=True):
-            services.helpers.StoredContext('foo.yaml', {'key': 'val'})
-        mopen.assert_called_once_with('charm_dir/foo.yaml', 'w')
-        fchmod.assert_called_once_with(mopen.return_value.fileno(), 0o600)
-        yaml.dump.assert_called_once_with({'key': 'val'}, mopen.return_value)
-
-    @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir')
-    @mock.patch.object(services.helpers.StoredContext, 'read_context')
-    @mock.patch.object(services.helpers, 'yaml')
-    @mock.patch('os.fchmod')
-    @mock.patch('os.path.exists')
-    def test_store_context_abs(self, exists, fchmod, yaml, read_context):
-        exists.return_value = False
-        mopen = mock.mock_open()
-        with mock.patch.object(services.helpers, 'open', mopen, create=True):
-            services.helpers.StoredContext('/foo.yaml', {'key': 'val'})
-        mopen.assert_called_once_with('/foo.yaml', 'w')
-
-    @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir')
-    @mock.patch.object(services.helpers, 'yaml')
-    @mock.patch('os.path.exists')
-    def test_read_context(self, exists, yaml):
-        exists.return_value = True
-        yaml.load.return_value = {'key': 'other'}
-        mopen = mock.mock_open()
-        with mock.patch.object(services.helpers, 'open', mopen, create=True):
-            context = services.helpers.StoredContext('foo.yaml', {'key': 'val'})
-        mopen.assert_called_once_with('charm_dir/foo.yaml', 'r')
-        yaml.load.assert_called_once_with(mopen.return_value)
-        self.assertEqual(context, {'key': 'other'})
-
-    @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir')
-    @mock.patch.object(services.helpers, 'yaml')
-    @mock.patch('os.path.exists')
-    def test_read_context_abs(self, exists, yaml):
-        exists.return_value = True
-        yaml.load.return_value = {'key': 'other'}
-        mopen = mock.mock_open()
-        with mock.patch.object(services.helpers, 'open', mopen, create=True):
-            context = services.helpers.StoredContext('/foo.yaml', {'key': 'val'})
-        mopen.assert_called_once_with('/foo.yaml', 'r')
-        yaml.load.assert_called_once_with(mopen.return_value)
-        self.assertEqual(context, {'key': 'other'})
-
-    @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir')
-    @mock.patch.object(services.helpers, 'yaml')
-    @mock.patch('os.path.exists')
-    def test_read_context_empty(self, exists, yaml):
-        exists.return_value = True
-        yaml.load.return_value = None
-        mopen = mock.mock_open()
-        with mock.patch.object(services.helpers, 'open', mopen, create=True):
-            self.assertRaises(OSError, services.helpers.StoredContext, '/foo.yaml', {})
-
-
-class TestTemplateCallback(unittest.TestCase):
-    @mock.patch.object(services.helpers, 'templating')
-    def test_template_defaults(self, mtemplating):
-        manager = mock.Mock(**{'get_service.return_value': {
-            'required_data': [{'foo': 'bar'}]}})
-        self.assertRaises(TypeError, services.template, source='foo.yml')
-        callback = services.template(source='foo.yml', target='bar.yml')
-        assert isinstance(callback, services.ManagerCallback)
-        assert not mtemplating.render.called
-        callback(manager, 'test', 'event')
-        mtemplating.render.assert_called_once_with(
-            'foo.yml', 'bar.yml', {'foo': 'bar'},
-            'root', 'root', 0o444)
-
-    @mock.patch.object(services.helpers, 'templating')
-    def test_template_explicit(self, mtemplating):
-        manager = mock.Mock(**{'get_service.return_value': {
-            'required_data': [{'foo': 'bar'}]}})
-        callback = services.template(
-            source='foo.yml', target='bar.yml',
-            owner='user', group='group', perms=0o555
-        )
-        assert isinstance(callback, services.ManagerCallback)
-        assert not mtemplating.render.called
-        callback(manager, 'test', 'event')
-        mtemplating.render.assert_called_once_with(
-            'foo.yml', 'bar.yml', {'foo': 'bar'},
-            'user', 'group', 0o555)
-
-
-class TestPortsCallback(unittest.TestCase):
-    def setUp(self):
-        self.phookenv = mock.patch.object(services.base, 'hookenv')
-        self.mhookenv = self.phookenv.start()
-        self.mhookenv.relation_ids.return_value = []
-        self.mhookenv.charm_dir.return_value = 'charm_dir'
-        self.popen = mock.patch.object(services.base, 'open', create=True)
-        self.mopen = self.popen.start()
-
-    def tearDown(self):
-        self.phookenv.stop()
-        self.popen.stop()
-
-    def test_no_ports(self):
-        manager = mock.Mock(**{'get_service.return_value': {}})
-        services.PortManagerCallback()(manager, 'service', 'event')
-        assert not self.mhookenv.open_port.called
-        assert not self.mhookenv.close_port.called
-
-    def test_open_ports(self):
-        manager = mock.Mock(**{'get_service.return_value': {'ports': [1, 2]}})
-        services.open_ports(manager, 'service', 'start')
-        self.mhookenv.open_port.has_calls([mock.call(1), mock.call(2)])
-        assert not self.mhookenv.close_port.called
-
-    def test_close_ports(self):
-        manager = mock.Mock(**{'get_service.return_value': {'ports': [1, 2]}})
-        services.close_ports(manager, 'service', 'stop')
-        assert not self.mhookenv.open_port.called
-        self.mhookenv.close_port.has_calls([mock.call(1), mock.call(2)])
-
-    def test_close_old_ports(self):
-        self.mopen.return_value.read.return_value = '10,20'
-        manager = mock.Mock(**{'get_service.return_value': {'ports': [1, 2]}})
-        services.close_ports(manager, 'service', 'stop')
-        assert not self.mhookenv.open_port.called
-        self.mhookenv.close_port.has_calls([
-            mock.call(10),
-            mock.call(20),
-            mock.call(1),
-            mock.call(2)])
-
-if __name__ == '__main__':
-    unittest.main()

=== modified file 'tests/fetch/test_giturl.py'
--- tests/fetch/test_giturl.py	2014-12-02 18:04:16 +0000
+++ tests/fetch/test_giturl.py	2015-12-08 19:13:38 +0000
@@ -50,16 +50,26 @@
             self.assertNotEqual(result, True, url)
 
     @unittest.skipIf(six.PY3, 'git does not support Python 3')
-    @patch('git.Repo.clone_from')
-    def test_branch(self, _clone_from):
+    @patch.object(giturl, 'Repo')
+    def test_branch(self, Repo):
         dest_path = "/destination/path"
         branch = "master"
         for url in self.valid_urls:
             self.fh.remote_branch = MagicMock()
             self.fh.load_plugins = MagicMock()
-            self.fh.clone(url, dest_path, branch)
-
-            _clone_from.assert_called_with(url, dest_path)
+            repo = MagicMock()
+            Repo.side_effect = [giturl.InvalidGitRepositoryError, repo]
+            Repo.clone_from.return_value = repo
+
+            self.fh.clone(url, dest_path, branch)
+            Repo.clone_from.assert_called_once_with(url, dest_path)
+            repo.git.checkout.assert_called_once_with(branch)
+
+            Repo.clone_from.reset_mock()
+            repo.git.checkout.reset_mock()
+            self.fh.clone(url, dest_path, branch)
+            assert not Repo.clone_from.called
+            repo.git.checkout.assert_called_once_with(branch)
 
         for url in self.invalid_urls:
             with patch.dict('os.environ', {'CHARM_DIR': 'foo'}):


Follow ups