bigdata-dev team mailing list archive
-
bigdata-dev team
-
Mailing list archive
-
Message #00106
[Merge] lp:~bigdata-dev/charms/trusty/apache-hadoop-yarn-master/status into lp:~bigdata-dev/charms/trusty/apache-hadoop-yarn-master/trunk
Cory Johns has proposed merging lp:~bigdata-dev/charms/trusty/apache-hadoop-yarn-master/status into lp:~bigdata-dev/charms/trusty/apache-hadoop-yarn-master/trunk.
Requested reviews:
Juju Big Data Development (bigdata-dev)
For more details, see:
https://code.launchpad.net/~bigdata-dev/charms/trusty/apache-hadoop-yarn-master/status/+merge/262923
Added extended status support.
Example status (tabular) outputs:
http://pastebin.ubuntu.com/11769979/ (in progress)
http://pastebin.ubuntu.com/11770196/ (complete)
Example status-history outputs:
http://pastebin.ubuntu.com/11770192/ (hdfs-master)
http://pastebin.ubuntu.com/11770195/ (compute-slave)
--
Your team Juju Big Data Development is requested to review the proposed merge of lp:~bigdata-dev/charms/trusty/apache-hadoop-yarn-master/status into lp:~bigdata-dev/charms/trusty/apache-hadoop-yarn-master/trunk.
=== modified file 'hooks/callbacks.py'
--- hooks/callbacks.py 2015-05-12 21:51:52 +0000
+++ hooks/callbacks.py 2015-06-24 22:19:52 +0000
@@ -15,3 +15,34 @@
Add any additional tasks / setup here. If a callback is used by mutliple
charms, consider refactoring it up to the jujubigdata library.
"""
+
+from charmhelpers.core import hookenv
+from charmhelpers.core import unitdata
+from jujubigdata.relations import NameNode, NodeManager
+
+
+def update_blocked_status():
+ if unitdata.kv().get('charm.active', False):
+ return
+ if NameNode().connected_units():
+ hookenv.status_set('waiting', 'Waiting for HDFS master to provide NameNode'),
+ else:
+ hookenv.status_set('blocked', 'Waiting for relation to HDFS master'),
+
+
+def update_working_status():
+ if unitdata.kv().get('charm.active', False):
+ hookenv.status_set('maintenance', 'Updating configuration')
+ return
+ hookenv.status_set('maintenance', 'Setting up Yarn master')
+
+
+def update_active_status():
+ nodemanager = NodeManager()
+ if nodemanager.is_ready():
+ hookenv.status_set('active', 'Ready (%s NodeManagers)' % len(nodemanager.filtered_data()))
+ unitdata.kv().set('charm.active', True)
+ elif nodemanager.connected_units():
+ hookenv.status_set('waiting', 'Waiting for compute slaves to provide NodeManagers')
+ else:
+ hookenv.status_set('blocked', 'Waiting for relation to compute slaves')
=== modified file 'hooks/common.py'
--- hooks/common.py 2015-06-17 17:30:49 +0000
+++ hooks/common.py 2015-06-24 22:19:52 +0000
@@ -15,29 +15,39 @@
"""
import jujuresources
+from charmhelpers.core import hookenv
+from charmhelpers.core import unitdata
+from charmhelpers.core import charmframework
def bootstrap_resources():
"""
Attempt to load and install resources defined in resources.yaml
"""
+ if unitdata.kv().get('charm.bootstrapped', False):
+ return True
+ hookenv.status_set('maintenance', 'Installing base resources')
mirror_url = jujuresources.config_get('resources_mirror')
if not jujuresources.fetch(mirror_url=mirror_url):
- jujuresources.juju_log('Required resources unavailable; changing resources_mirror option is required', 'ERROR')
+ missing = jujuresources.invalid()
+ hookenv.status_set('blocked', 'Unable to fetch required resource%s: %s' % (
+ 's' if len(missing) > 1 else '',
+ ', '.join(missing),
+ ))
return False
- jujuresources.install(['pathlib', 'pyaml', 'six', 'charmhelpers', 'jujubigdata'])
+ jujuresources.install(['pathlib', 'jujubigdata'])
+ unitdata.kv().set('charm.bootstrapped', True)
return True
def manage():
if not bootstrap_resources():
- # defer until resources are available, since charmhelpers, and thus
- # the framework, are required (will require manual intervention)
+ # defer until resources are available, since jujubigdata, and thus the
+ # classes needed for the requires blocks, (will be) managed by jujuresources
return
- from charmhelpers.core import charmframework
import jujubigdata
- import callbacks # noqa (ignore when linting)
+ import callbacks
# list of keys required to be in the dist.yaml
yarn_reqs = ['vendor', 'hadoop_version', 'packages', 'groups',
@@ -60,6 +70,7 @@
],
'callbacks': [
hadoop.install,
+ callbacks.update_blocked_status,
],
},
{
@@ -74,6 +85,11 @@
jujubigdata.relations.NodeManager(optional=True),
],
'callbacks': [
+ # These callbacks will be executed once the Hadoop base packages
+ # are installed and HDFS is available. New items can be added
+ # to the end of this list and to hooks/callbacks.py to extend
+ # the functionality of this charm.
+ callbacks.update_working_status,
nodemanagers.register_connected_hosts,
namenode.register_provided_hosts,
jujubigdata.utils.manage_etc_hosts,
@@ -84,6 +100,7 @@
yarn.start_resourcemanager,
yarn.start_jobhistory,
charmframework.helpers.open_ports(dist_config.exposed_ports('yarn-master')),
+ callbacks.update_active_status,
],
'cleanup': [
charmframework.helpers.close_ports(dist_config.exposed_ports('yarn-master')),
=== modified file 'hooks/config-changed'
--- hooks/config-changed 2015-02-09 18:17:08 +0000
+++ hooks/config-changed 2015-06-24 22:19:52 +0000
@@ -11,5 +11,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful. As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
import common
common.manage()
=== modified file 'hooks/install'
--- hooks/install 2015-02-09 18:17:08 +0000
+++ hooks/install 2015-06-24 22:19:52 +0000
@@ -13,5 +13,16 @@
import setup
setup.pre_install()
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful. As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
import common
common.manage()
=== modified file 'hooks/namenode-relation-changed'
--- hooks/namenode-relation-changed 2015-05-12 21:51:52 +0000
+++ hooks/namenode-relation-changed 2015-06-24 22:19:52 +0000
@@ -11,5 +11,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful. As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
import common
common.manage()
=== modified file 'hooks/nodemanager-relation-changed'
--- hooks/nodemanager-relation-changed 2015-05-12 21:51:52 +0000
+++ hooks/nodemanager-relation-changed 2015-06-24 22:19:52 +0000
@@ -11,5 +11,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful. As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
import common
common.manage()
=== modified file 'hooks/resourcemanager-relation-changed'
--- hooks/resourcemanager-relation-changed 2015-05-19 16:28:14 +0000
+++ hooks/resourcemanager-relation-changed 2015-06-24 22:19:52 +0000
@@ -11,5 +11,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful. As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
import common
common.manage()
=== modified file 'hooks/setup.py'
--- hooks/setup.py 2015-03-03 21:31:05 +0000
+++ hooks/setup.py 2015-06-24 22:19:52 +0000
@@ -12,24 +12,22 @@
import subprocess
from glob import glob
+
def pre_install():
"""
Do any setup required before the install hook.
"""
install_pip()
- install_jujuresources()
+ install_bundled_resources()
def install_pip():
subprocess.check_call(['apt-get', 'install', '-yq', 'python-pip', 'bzr'])
-def install_jujuresources():
- """
- Install the bundled jujuresources library, if not present.
- """
- try:
- import jujuresources # noqa
- except ImportError:
- jr_archive = glob('resources/jujuresources-*.tar.gz')[0]
- subprocess.check_call(['pip', 'install', jr_archive])
+def install_bundled_resources():
+ """
+ Install the bundled resources libraries.
+ """
+ archives = glob('resources/*')
+ subprocess.check_call(['pip', 'install'] + archives)
=== modified file 'hooks/start'
--- hooks/start 2015-02-09 18:17:08 +0000
+++ hooks/start 2015-06-24 22:19:52 +0000
@@ -11,5 +11,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful. As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
import common
common.manage()
=== modified file 'hooks/stop'
--- hooks/stop 2015-02-09 18:17:08 +0000
+++ hooks/stop 2015-06-24 22:19:52 +0000
@@ -11,5 +11,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful. As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
import common
common.manage()
=== modified file 'resources.yaml'
--- resources.yaml 2015-06-19 16:48:37 +0000
+++ resources.yaml 2015-06-24 22:19:52 +0000
@@ -3,16 +3,8 @@
resources:
pathlib:
pypi: path.py>=7.0
- pyaml:
- pypi: pyaml
- six:
- pypi: six
jujubigdata:
- pypi: jujubigdata>=2.0.0,<3.0.0
- charmhelpers:
- pypi: http://bazaar.launchpad.net/~bigdata-dev/bigdata-data/trunk/download/kevin.monroe%40canonical.com-20150610215108-gq4ud2rriy1sp1h6/charmhelpers0.3.1.ta-20150610215050-vxrqhizdsrqwnmcm-1/charmhelpers-0.3.1.tar.gz
- hash: 061fe7204289b96fab5d3ca02883040ea3026526ebf0ad38e21457527a18d139
- hash_type: sha256
+ pypi: jujubigdata>=2.0.2,<3.0.0
java-installer:
# This points to a script which manages installing Java.
# If replaced with an alternate implementation, it must output *only* two
=== added file 'resources/PyYAML-3.11.tar.gz'
Binary files resources/PyYAML-3.11.tar.gz 1970-01-01 00:00:00 +0000 and resources/PyYAML-3.11.tar.gz 2015-06-24 22:19:52 +0000 differ
=== added file 'resources/charmhelpers-0.3.1.tar.gz'
Binary files resources/charmhelpers-0.3.1.tar.gz 1970-01-01 00:00:00 +0000 and resources/charmhelpers-0.3.1.tar.gz 2015-06-24 22:19:52 +0000 differ
=== removed file 'resources/jujuresources-0.2.5.tar.gz'
Binary files resources/jujuresources-0.2.5.tar.gz 2015-03-03 19:58:27 +0000 and resources/jujuresources-0.2.5.tar.gz 1970-01-01 00:00:00 +0000 differ
=== added file 'resources/jujuresources-0.2.8.tar.gz'
Binary files resources/jujuresources-0.2.8.tar.gz 1970-01-01 00:00:00 +0000 and resources/jujuresources-0.2.8.tar.gz 2015-06-24 22:19:52 +0000 differ
=== added file 'resources/pyaml-15.5.7.tar.gz'
Binary files resources/pyaml-15.5.7.tar.gz 1970-01-01 00:00:00 +0000 and resources/pyaml-15.5.7.tar.gz 2015-06-24 22:19:52 +0000 differ
=== added file 'resources/six-1.9.0-py2.py3-none-any.whl'
Binary files resources/six-1.9.0-py2.py3-none-any.whl 1970-01-01 00:00:00 +0000 and resources/six-1.9.0-py2.py3-none-any.whl 2015-06-24 22:19:52 +0000 differ
Follow ups