bigdata-dev team mailing list archive

[Kevin W Monroe <kevin.monroe@xxxxxxxxxxxxx>]

Kevin W Monroe has proposed merging lp:~bigdata-dev/charms/trusty/apache-hive/trunk into lp:charms/trusty/apache-hive.

Requested reviews:
  Juju Big Data Development (bigdata-dev)

For more details, see:
https://code.launchpad.net/~bigdata-dev/charms/trusty/apache-hive/trunk/+merge/269660

This is a hotfix that cannot wait for the next bd-dev -> bd-charmers merge window. apache-hive is currently broken in the store because we were trying to initialize the hive schema prior to a db unit being connected.

Move 'schematool -initSchema' to configure_hive and guard it with 'hive.schema.initialized' so it only runs once.
-- 
Your team Juju Big Data Development is requested to review the proposed merge of lp:~bigdata-dev/charms/trusty/apache-hive/trunk into lp:charms/trusty/apache-hive.

=== modified file 'hooks/callbacks.py'
--- hooks/callbacks.py	2015-08-24 23:23:36 +0000
+++ hooks/callbacks.py	2015-08-31 17:29:31 +0000
@@ -50,6 +50,10 @@
     hookenv.status_set('active', 'Ready')
 
 
+def clear_active_flag():
+    unitdata.kv().set('charm.active', False)
+
+
 # Main Hive class for callbacks
 class Hive(object):
     HIVE_VERSION = {'x86_64': '1.0.0', 'ppc64le': '0.13.0'}
@@ -76,9 +80,8 @@
         self.dist_config.add_packages()
         self.setup_hive_config()
 
-        # create hdfs storage space and init our schema
+        # create hdfs storage space
         utils.run_as('hive', 'hdfs', 'dfs', '-mkdir', '-p', '/user/hive/warehouse')
-        utils.run_as('hive', 'schematool', '-initSchema', '-dbType', 'mysql')
 
         unitdata.kv().set('hive.installed', True)
         unitdata.kv().flush(True)
@@ -96,33 +99,37 @@
         default_conf.rmtree_p()
         hive_conf.symlink(default_conf)
 
+        # Configure immutable bits
+        hive_bin = self.dist_config.path('hive') / 'bin'
+        with utils.environment_edit_in_place('/etc/environment') as env:
+            if hive_bin not in env['PATH']:
+                env['PATH'] = ':'.join([env['PATH'], hive_bin])
+            env['HIVE_CONF_DIR'] = self.dist_config.path('hive_conf')
+
         hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh'
         if not hive_env.exists():
             (self.dist_config.path('hive_conf') / 'hive-env.sh.template').copy(hive_env)
+
         hive_site = self.dist_config.path('hive_conf') / 'hive-site.xml'
         if not hive_site.exists():
             (self.dist_config.path('hive_conf') / 'hive-default.xml.template').copy(hive_site)
+        with utils.xmlpropmap_edit_in_place(hive_site) as props:
+            # TODO (kwm): we should be able to export java.io.tmpdir so these 4 arent needed
+            props['hive.exec.local.scratchdir'] = "/tmp/hive"
+            props['hive.downloaded.resources.dir'] = "/tmp/hive_resources"
+            props['hive.querylog.location'] = "/tmp/hive"
+            props['hive.server2.logging.operation.log.location'] = "/tmp/hive"
+            ####
 
     # called during config-changed events
     def configure_hive(self):
         config = hookenv.config()
-        hive_bin = self.dist_config.path('hive') / 'bin'
-        with utils.environment_edit_in_place('/etc/environment') as env:
-            if hive_bin not in env['PATH']:
-                env['PATH'] = ':'.join([env['PATH'], hive_bin])
-            env['HIVE_CONF_DIR'] = self.dist_config.path('hive_conf')
         hive_site = self.dist_config.path('hive_conf') / 'hive-site.xml'
         with utils.xmlpropmap_edit_in_place(hive_site) as props:
             mysql = unitdata.kv().get('relations.ready')['db'].values()[0]
             props['javax.jdo.option.ConnectionURL'] = "jdbc:mysql://{}/{}".format(
                 mysql['host'], mysql['database']
             )
-            # TODO (kwm): we should be able to export java.io.tmpdir so these 4 arent needed
-            props['hive.exec.local.scratchdir'] = "/tmp/hive"
-            props['hive.downloaded.resources.dir'] = "/tmp/hive_resources"
-            props['hive.querylog.location'] = "/tmp/hive"
-            props['hive.server2.logging.operation.log.location'] = "/tmp/hive"
-            ####
             props['javax.jdo.option.ConnectionUserName'] = mysql['user']
             props['javax.jdo.option.ConnectionPassword'] = mysql['password']
             props['javax.jdo.option.ConnectionDriverName'] = "com.mysql.jdbc.Driver"
@@ -134,6 +141,11 @@
             r'.*export HIVE_AUX_JARS_PATH *=.*': 'export HIVE_AUX_JARS_PATH=/usr/share/java/mysql-connector-java.jar',
         })
 
+        # Now that we have db connection info, init our schema (only once)
+        if not unitdata.kv().get('hive.schema.initialized'):
+            utils.run_as('hive', 'schematool', '-initSchema', '-dbType', 'mysql')
+            unitdata.kv().set('hive.schema.initialized', True)
+
     def run_bg(self, user, command, *args):
         """
         Run a Hive command as the `hive` user in the background.
@@ -147,12 +159,12 @@
         Popen(['su', user, '-c', quoted], env=e)
 
     def start(self):
-        if not utils.jps('HiveServer2'):
-            self.run_bg(
-                'hive', 'hive',
-                '--config', self.dist_config.path('hive_conf'),
-                '--service', 'hiveserver2')
-            time.sleep(5)
+        self.stop()
+        self.run_bg(
+            'hive', 'hive',
+            '--config', self.dist_config.path('hive_conf'),
+            '--service', 'hiveserver2')
+        time.sleep(5)
 
     def stop(self):
         hive_pids = utils.jps('HiveServer2')

=== modified file 'hooks/common.py'
--- hooks/common.py	2015-08-24 23:23:36 +0000
+++ hooks/common.py	2015-08-31 17:29:31 +0000
@@ -74,8 +74,10 @@
                 callbacks.update_active_status,
             ],
             'cleanup': [
+                callbacks.clear_active_flag,
                 hive.stop,
                 hive.cleanup,
+                callbacks.update_blocked_status,
             ],
         },
     ])