bigdata-dev team mailing list archive
-
bigdata-dev team
-
Mailing list archive
-
Message #00274
[Merge] lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk into lp:charms/trusty/apache-hadoop-compute-slave
Kevin W Monroe has proposed merging lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk into lp:charms/trusty/apache-hadoop-compute-slave.
Requested reviews:
Juju Big Data Development (bigdata-dev)
For more details, see:
https://code.launchpad.net/~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk/+merge/271162
--
Your team Juju Big Data Development is requested to review the proposed merge of lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk into lp:charms/trusty/apache-hadoop-compute-slave.
=== modified file 'README.md'
--- README.md 2015-08-24 23:09:31 +0000
+++ README.md 2015-09-15 17:34:38 +0000
@@ -37,6 +37,26 @@
juju add-unit compute-slave -n 3
+## Monitoring
+
+This charm supports monitoring via Ganglia. To enable monitoring, you must
+do **both** of the following (the order does not matter):
+
+ * Add a relation to the [Ganglia charm][] via the `:master` relation
+ * Enable the `ganglia_metrics` config option
+
+You must **also** enable metrics on [yarn-master][] and / or [hdfs-master][]
+to initiate the restart of the NodeManager and / or DataNode components for
+them to begin collecting metrics.
+
+For example:
+
+ juju add-relation compute-slave ganglia:master
+ juju add-relation yarn-master ganglia:master
+ juju set compute-slave ganglia_metrics=true
+ juju set yarn-master ganglia_metrics=true
+
+
## Deploying in Network-Restricted Environments
The Apache Hadoop charms can be deployed in environments with limited network
@@ -85,3 +105,8 @@
- [Apache Hadoop bug trackers](http://hadoop.apache.org/issue_tracking.html)
- [Apache Hadoop mailing lists](http://hadoop.apache.org/mailing_lists.html)
- [Apache Hadoop Juju Charm](http://jujucharms.com/?text=hadoop)
+
+
+[Ganglia charm]: http://jujucharms.com/ganglia/
+[yarn-master]: http://jujucharms.com/apache-hadoop-yarn-master/
+[hdfs-master]: http://jujucharms.com/apache-hadoop-hdfs-master/
=== modified file 'config.yaml'
--- config.yaml 2015-04-03 16:49:16 +0000
+++ config.yaml 2015-09-15 17:34:38 +0000
@@ -4,3 +4,13 @@
default: ''
description: |
URL from which to fetch resources (e.g., Hadoop binaries) instead of Launchpad.
+ ganglia_metrics:
+ type: boolean
+ default: false
+ description: |
+ Enable metrics using Ganglia. Note that enabling this option will
+ have no effect if the service is not related to a ganglia service
+ via the ganglia:master relation. Enabling this option also will
+ *not* restart the DataNode nor NodeManager components, so it will
+ also be necessary to enable metrics on one or more of the hdfs-master
+ or yarn-master services. See the README for more information.
=== modified file 'hooks/callbacks.py'
--- hooks/callbacks.py 2015-08-13 21:19:50 +0000
+++ hooks/callbacks.py 2015-09-15 17:34:38 +0000
@@ -18,7 +18,10 @@
from charmhelpers.core import hookenv
from charmhelpers.core import unitdata
-from jujubigdata.relations import NameNodeMaster, ResourceManagerMaster
+from jujubigdata.relations import NameNodeMaster, ResourceManagerMaster, Ganglia
+from charmhelpers.core.templating import render
+from functools import partial
+from subprocess import check_call
def update_blocked_status():
@@ -42,35 +45,108 @@
' and '.join(unready_rels),
' and '.join(unready_ress),
))
- elif missing_hosts:
- hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % (
- ' and '.join(missing_hosts),
- ))
-
-
-def update_working_status():
- if unitdata.kv().get('charm.active', False):
- hookenv.status_set('maintenance', 'Updating configuration')
- return
- yarn_connected = ResourceManagerMaster().connected_units()
- hookenv.status_set('maintenance', 'Setting up DataNode%s' % (
- ' and NodeManager' if yarn_connected else '',
- ))
+<<<<<<< TREE
+ elif missing_hosts:
+ hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % (
+ ' and '.join(missing_hosts),
+ ))
+
+
+def update_working_status():
+ if unitdata.kv().get('charm.active', False):
+ hookenv.status_set('maintenance', 'Updating configuration')
+ return
+ yarn_connected = ResourceManagerMaster().connected_units()
+ hookenv.status_set('maintenance', 'Setting up DataNode%s' % (
+ ' and NodeManager' if yarn_connected else '',
+ ))
+=======
+ elif missing_hosts:
+ hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % (
+ ' and '.join(missing_hosts),
+ ))
+
+
+def update_working_status():
+ if unitdata.kv().get('charm.active', False):
+ hookenv.status_set('maintenance', 'Updating configuration')
+ return
+ yarn_connected = ResourceManagerMaster().connected_units()
+ hookenv.status_set('maintenance', 'Setting up DataNode%s' % (
+ ' and NodeManager' if yarn_connected else '',
+ ))
+
+
+def update_working_status():
+ if unitdata.kv().get('charm.active', False):
+ hookenv.status_set('maintenance', 'Updating configuration')
+ return
+ yarn_connected = ResourceManagerMaster().connected_units()
+ hookenv.status_set('maintenance', 'Setting up DataNode%s' % (
+ ' and NodeManager' if yarn_connected else '',
+ ))
+>>>>>>> MERGE-SOURCE
def update_active_status():
- hdfs_ready = NameNodeMaster().is_ready()
- yarn_connected = ResourceManagerMaster().connected_units()
- yarn_ready = ResourceManagerMaster().is_ready()
- if hdfs_ready and (not yarn_connected or yarn_ready):
- unitdata.kv().set('charm.active', True)
- hookenv.status_set('active', 'Ready%s' % (
- '' if yarn_ready else ' (HDFS only)'
- ))
- else:
- clear_active_flag()
- update_blocked_status()
-
-
-def clear_active_flag():
- unitdata.kv().set('charm.active', False)
+<<<<<<< TREE
+ hdfs_ready = NameNodeMaster().is_ready()
+ yarn_connected = ResourceManagerMaster().connected_units()
+ yarn_ready = ResourceManagerMaster().is_ready()
+ if hdfs_ready and (not yarn_connected or yarn_ready):
+ unitdata.kv().set('charm.active', True)
+ hookenv.status_set('active', 'Ready%s' % (
+ '' if yarn_ready else ' (HDFS only)'
+ ))
+ else:
+ clear_active_flag()
+ update_blocked_status()
+
+
+def clear_active_flag():
+ unitdata.kv().set('charm.active', False)
+=======
+ hdfs_ready = NameNodeMaster().is_ready()
+ yarn_connected = ResourceManagerMaster().connected_units()
+ yarn_ready = ResourceManagerMaster().is_ready()
+ if hdfs_ready and (not yarn_connected or yarn_ready):
+ unitdata.kv().set('charm.active', True)
+ hookenv.status_set('active', 'Ready%s' % (
+ '' if yarn_ready else ' (HDFS only)'
+ ))
+ else:
+ clear_active_flag()
+ update_blocked_status()
+
+
+def clear_active_flag():
+ unitdata.kv().set('charm.active', False)
+
+
+def conf_ganglia_metrics(purgeConf=False):
+ """
+ Send hadoop specific metrics to a ganglia server
+ """
+ config = hookenv.config()
+ ganglia_metrics = config['ganglia_metrics'] and not purgeConf
+ ganglia_metrics_changed = ganglia_metrics != unitdata.kv().get('ganglia_metrics', False)
+ unitdata.kv().set('ganglia_metrics', ganglia_metrics)
+ comment = '#' if not ganglia_metrics else ''
+ ganglia_host = 'UNSET_BY_JUJU' if not ganglia_metrics else Ganglia().host()
+ ganglia_sink_str = comment + '*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31'
+ hookenv.log("Configuring ganglia sink in /etc/hadoop/conf/hadoop-metrics2.properties", level=None)
+ render(
+ source='hadoop-metrics2.properties.j2',
+ target='/etc/hadoop/conf/hadoop-metrics2.properties',
+ context={
+ 'ganglia_host': ganglia_host,
+ 'ganglia_sink_str': ganglia_sink_str,
+ },
+ ),
+ if ganglia_metrics_changed:
+ #check_call(['actions/restart-hdfs'])
+ # IMPLEMENT RESTART COMPUTE SLAVE?
+ hookenv.log("please manually restart compute slave hadoop components", level=None)
+
+purge_ganglia_metrics = partial(conf_ganglia_metrics, purgeConf=True)
+>>>>>>> MERGE-SOURCE
=== modified file 'hooks/common.py' (properties changed: +x to -x)
--- hooks/common.py 2015-08-18 15:18:07 +0000
+++ hooks/common.py 2015-09-15 17:34:38 +0000
@@ -125,9 +125,29 @@
charmframework.helpers.close_ports(
dist_config.exposed_ports('compute-slave-yarn')),
yarn.stop_nodemanager,
- callbacks.update_active_status, # might still be active if HDFS-only
- ],
- },
+<<<<<<< TREE
+ callbacks.update_active_status, # might still be active if HDFS-only
+ ],
+ },
+=======
+ callbacks.update_active_status, # might still be active if HDFS-only
+ ],
+ },
+ {
+ 'name': 'ganglia',
+ 'requires': [
+ hadoop.is_installed,
+ jujubigdata.relations.Ganglia,
+ ],
+ 'callbacks': [
+ callbacks.conf_ganglia_metrics,
+ ],
+ 'cleanup': [
+ callbacks.purge_ganglia_metrics
+ ],
+ },
+
+>>>>>>> MERGE-SOURCE
])
manager.manage()
=== added file 'hooks/ganglia-relation-broken'
--- hooks/ganglia-relation-broken 1970-01-01 00:00:00 +0000
+++ hooks/ganglia-relation-broken 2015-09-15 17:34:38 +0000
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful. As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
+import common
+common.manage()
=== added file 'hooks/ganglia-relation-changed'
--- hooks/ganglia-relation-changed 1970-01-01 00:00:00 +0000
+++ hooks/ganglia-relation-changed 2015-09-15 17:34:38 +0000
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful. As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
+import common
+common.manage()
=== modified file 'metadata.yaml'
--- metadata.yaml 2015-03-06 22:28:48 +0000
+++ metadata.yaml 2015-09-15 17:34:38 +0000
@@ -12,3 +12,5 @@
interface: dfs-slave
nodemanager:
interface: mapred-slave
+ ganglia:
+ interface: monitor
=== added file 'resources/python/jujuresources-0.2.11.tar.gz'
Binary files resources/python/jujuresources-0.2.11.tar.gz 1970-01-01 00:00:00 +0000 and resources/python/jujuresources-0.2.11.tar.gz 2015-09-15 17:34:38 +0000 differ
=== added directory 'templates'
=== added file 'templates/hadoop-metrics2.properties.j2'
--- templates/hadoop-metrics2.properties.j2 1970-01-01 00:00:00 +0000
+++ templates/hadoop-metrics2.properties.j2 2015-09-15 17:34:38 +0000
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# syntax: [prefix].[source|sink].[instance].[options]
+# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
+
+*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink
+# default sampling period, in seconds
+*.period=10
+
+# Defining sink for Ganglia 3.1
+{{ ganglia_sink_str }}
+
+# Default polling period for GangliaSink
+*.sink.ganglia.period=10
+
+# default for supportsparse is false
+*.sink.ganglia.supportsparse=true
+
+# Directing output to ganglia servers
+
+*.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
+*.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
+
+namenode.sink.ganglia.servers={{ ganglia_host }}:8649
+datanode.sink.ganglia.servers={{ ganglia_host }}:8649
+jobtracker.sink.ganglia.servers={{ ganglia_host }}:8649
+tasktracker.sink.ganglia.servers={{ ganglia_host }}:8649
+maptask.sink.ganglia.servers={{ ganglia_host }}:8649
+reducetask.sink.ganglia.servers={{ ganglia_host }}:8649
+resourcemanager.sink.ganglia.servers={{ ganglia_host }}:8649
+nodemanager.sink.ganglia.servers={{ ganglia_host }}:8649
+historyserver.sink.ganglia.servers={{ ganglia_host }}:8649
+journalnode.sink.ganglia.servers={{ ganglia_host }}:8649
+resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
+
+# The namen de-metrics. ut will contain metrics from all context
+#namenode.sink.file.filename=namenode-metrics.out
+# Specifying a special sampling period for namenode:
+#namenode.sink.*.period=8
+
+#datanode.sink.file.filename=datanode-metrics.out
+
+# the following example split metrics of different
+# context to different sinks (in this case files)
+#jobtracker.sink.file_jvm.context=jvm
+#jobtracker.sink.file_jvm.filename=jobtracker-jvm-metrics.out
+#jobtracker.sink.file_mapred.context=mapred
+#jobtracker.sink.file_mapred.filename=jobtracker-mapred-metrics.out
+
+#tasktracker.sink.file.filename=tasktracker-metrics.out
+
+#maptask.sink.file.filename=maptask-metrics.out
+
+#reducetask.sink.file.filename=reducetask-metrics.out
=== modified file 'tests/01-basic-deployment.py'
--- tests/01-basic-deployment.py 2015-03-04 00:56:45 +0000
+++ tests/01-basic-deployment.py 2015-09-15 17:34:38 +0000
@@ -16,8 +16,8 @@
def setUpClass(cls):
cls.d = amulet.Deployment(series='trusty')
cls.d.add('apache-hadoop-compute-slave')
- cls.d.setup(timeout=9000)
- cls.d.sentry.wait()
+ cls.d.setup(timeout=900)
+ cls.d.sentry.wait(timeout=1800)
cls.unit = cls.d.sentry.unit['apache-hadoop-compute-slave/0']
def test_deploy(self):
Follow ups