From 9eff90c1f2973ea15ee3535f887a6e6642eb2870 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Mon, 6 Jun 2016 15:06:41 +0000 Subject: BIGTOP-2779: new hive charm Closes #217 --- .../src/charm/hive/layer-hive/README.md | 237 +++++++++++++++++++++ .../src/charm/hive/layer-hive/actions.yaml | 4 + .../src/charm/hive/layer-hive/actions/restart | 36 ++++ .../src/charm/hive/layer-hive/actions/smoke-test | 49 +++++ .../src/charm/hive/layer-hive/config.yaml | 6 + .../src/charm/hive/layer-hive/copyright | 16 ++ bigtop-packages/src/charm/hive/layer-hive/icon.svg | 75 +++++++ .../src/charm/hive/layer-hive/layer.yaml | 31 +++ .../layer-hive/lib/charms/layer/bigtop_hive.py | 138 ++++++++++++ .../src/charm/hive/layer-hive/metadata.yaml | 19 ++ .../src/charm/hive/layer-hive/reactive/hive.py | 184 ++++++++++++++++ .../hive/layer-hive/tests/01-basic-deployment.py | 39 ++++ .../charm/hive/layer-hive/tests/02-smoke-test.py | 60 ++++++ .../src/charm/hive/layer-hive/tests/tests.yaml | 3 + 14 files changed, 897 insertions(+) create mode 100644 bigtop-packages/src/charm/hive/layer-hive/README.md create mode 100644 bigtop-packages/src/charm/hive/layer-hive/actions.yaml create mode 100755 bigtop-packages/src/charm/hive/layer-hive/actions/restart create mode 100755 bigtop-packages/src/charm/hive/layer-hive/actions/smoke-test create mode 100644 bigtop-packages/src/charm/hive/layer-hive/config.yaml create mode 100644 bigtop-packages/src/charm/hive/layer-hive/copyright create mode 100644 bigtop-packages/src/charm/hive/layer-hive/icon.svg create mode 100644 bigtop-packages/src/charm/hive/layer-hive/layer.yaml create mode 100755 bigtop-packages/src/charm/hive/layer-hive/lib/charms/layer/bigtop_hive.py create mode 100644 bigtop-packages/src/charm/hive/layer-hive/metadata.yaml create mode 100644 bigtop-packages/src/charm/hive/layer-hive/reactive/hive.py create mode 100755 bigtop-packages/src/charm/hive/layer-hive/tests/01-basic-deployment.py create mode 100755 bigtop-packages/src/charm/hive/layer-hive/tests/02-smoke-test.py create mode 100644 bigtop-packages/src/charm/hive/layer-hive/tests/tests.yaml diff --git a/bigtop-packages/src/charm/hive/layer-hive/README.md b/bigtop-packages/src/charm/hive/layer-hive/README.md new file mode 100644 index 00000000..ae2ddd8e --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/README.md @@ -0,0 +1,237 @@ + +# Overview + +Apache Hive is a data warehouse infrastructure built on top of Hadoop that +supports data summarization, query, and analysis. Hive provides an SQL-like +language called HiveQL that transparently converts queries to MapReduce for +execution on large datasets stored in Hadoop's HDFS. Learn more at +[hive.apache.org][]. + +This charm deploys version 1.2.1 of the Hive component from [Apache Bigtop][]. + +[hive.apache.org]: http://hive.apache.org/ +[Apache Bigtop]: http://bigtop.apache.org/ + + +# Deploying + +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. + +This charm is intended to be deployed via one of the [bigtop hadoop bundles][]. +For example: + + juju deploy hadoop-processing + +This will deploy an Apache Bigtop Hadoop cluster. More information about this +deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). + +Now add Hive and relate it to the cluster via the hadoop-plugin: + + juju deploy hive + juju add-relation hive plugin + +## Metastore +This charm will start the Hive Metastore service using a local Apache Derby +metastore database by default. This is suitable for unit or smoke testing Hive, +but this configuration should not be used in production. Deploying an external +database for the Hive metastore is recommended: + + juju deploy mariadb + juju add-relation hive mariadb + +## HBase Integration +This charm supports interacting with HBase using Hive. Enable this by relating +Hive to a deployment that includes HBase. For example: + + juju deploy hadoop-hbase + juju add-relation hive hbase + +See the [hadoop-hbase][] bundle for more information about this HBase +deployment. + +> **Note:** Applications that are duplicated in multiple bundles will be +reused. This means when deploying both `hadoop-processing` and `hadoop-hbase`, +Juju will reuse (and not duplicate) common applications like the NameNode, +ResourceManager, Slaves, etc. + +## Network-Restricted Environments +Charms can be deployed in environments with limited network access. To deploy +in this environment, configure a Juju model with appropriate proxy and/or +mirror options. See [Configuring Models][] for more information. + +[getting-started]: https://jujucharms.com/docs/stable/getting-started +[bigtop hadoop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles +[Configuring Models]: https://jujucharms.com/docs/stable/models-config +[hadoop-hbase]: https://jujucharms.com/hadoop-hbase/ + + +# Verifying + +## Status +Apache Bigtop charms provide extended status reporting to indicate when they +are ready: + + juju status + +This is particularly useful when combined with `watch` to track the on-going +progress of the deployment: + + watch -n 2 juju status + +The message column will provide information about a given unit's state. +This charm is ready for use once the status message indicates that it is +ready. + +## Smoke Test +This charm provides a `smoke-test` action that can be used to verify the +application is functioning as expected. Run the action as follows: + + juju run-action hive/0 smoke-test + +Watch the progress of the smoke test actions with: + + watch -n 2 juju show-action-status + +Eventually, the action should settle to `status: completed`. If it +reports `status: failed`, the application is not working as expected. Get +more information about a specific smoke test with: + + juju show-action-output + + +# Using + +This charm provides a variety of actions and interfaces that can be used +to interact with Hive. + +## Actions +Run a smoke test (as described in the **Verifying** section): + + juju run-action hive/0 smoke-test + juju show-action-output # <-- id from above command + +Restart all Hive services on a unit: + + juju run-action hive/0 restart + juju show-action-output # <-- id from above command + +## Command Line Interface + + $ juju ssh hive/0 + $ hive + ... + hive> create table foo(col1 int, col2 string); + OK + Time taken: 0.381 seconds + hive> show tables; + OK + foo + hivesmoke + Time taken: 0.202 seconds, Fetched: 2 row(s) + hive> exit; + +### HBase +As mentioned in the **Deploying** section, this charm supports integration +with HBase. When HBase is deployed and related to Hive, use the Hive CLI to +interact with HBase: + + $ juju ssh hive/0 + $ hive + ... + hive> CREATE TABLE myhivetable(key STRING, mycol STRING) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ('hbase.columns.mapping' = ':key,cf:mycol') + TBLPROPERTIES ('hbase.table.name' = 'myhbasetable'); + OK + Time taken: 2.497 seconds + hive> DESCRIBE myhivetable; + OK + key string from deserializer + mycol string from deserializer + Time taken: 0.174 seconds, Fetched: 2 row(s) + +## Thrift Interface +The HiveServer2 service provides a thrift server that can be used by Hive +clients. To access this interface from external clients (i.e. applications +that are not part of the Juju deployment), find the `Public address` of the +hive unit and expose the application: + + juju status hive + juju expose hive + +External clients will be able to access Hive using: + + thrift://HIVE_PUBLIC_IP:10000 + + +# Configuring + +Charm configuration can be changed at runtime with `juju config`. This charm +supports the following config parameters. + +## Heap +The default heap size for the the Hive shell JVM is 1024MB. Set a different +value (in MB) with the following: + + juju config hbase heap=4096 + + +# Limitations + +## Restarting Hive +Restarting Hive is potentially disruptive when queries are running. Be aware +that the following events will cause a restart of all Hive services: + +- Adding or removing an HBase relation +- Changing charm configuration with `juju config` +- Upgrading this charm + +## Hive Web Interface +The Hive Web Interface (HWI) has been removed upstream ([HIVE-15622][]). This +charm does not provide HWI. Use the command line or thrift interfaces for +interacting with Hive. + +[HIVE-15622]: https://issues.apache.org/jira/browse/HIVE-15622 + +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. + + +# Contact Information + +- + + +# Resources + +- [Apache Hive home page](http://hive.apache.org/) +- [Apache Bigtop home page](http://bigtop.apache.org/) +- [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) +- [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) +- [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) diff --git a/bigtop-packages/src/charm/hive/layer-hive/actions.yaml b/bigtop-packages/src/charm/hive/layer-hive/actions.yaml new file mode 100644 index 00000000..7870f30b --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/actions.yaml @@ -0,0 +1,4 @@ +restart: + description: Retart Hive Metastore and HiveServer2. +smoke-test: + description: Verify that Hive is working. diff --git a/bigtop-packages/src/charm/hive/layer-hive/actions/restart b/bigtop-packages/src/charm/hive/layer-hive/actions/restart new file mode 100755 index 00000000..904614fe --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/actions/restart @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.append('lib') + +from charmhelpers.core import hookenv # noqa: E402 +from charms.layer.bigtop_hive import Hive # noqa: E402 +from charms.reactive import is_state # noqa: E402 + + +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) + hookenv.action_fail(msg) + sys.exit() + + +if not is_state('hive.installed'): + fail('Hive is not yet ready') + +hive = Hive() +hive.restart() diff --git a/bigtop-packages/src/charm/hive/layer-hive/actions/smoke-test b/bigtop-packages/src/charm/hive/layer-hive/actions/smoke-test new file mode 100755 index 00000000..d6ac2444 --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/actions/smoke-test @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.append('lib') + +from charmhelpers.core import hookenv # noqa: E402 +from charms.layer.apache_bigtop_base import Bigtop # noqa: E402 +from charms.reactive import is_state # noqa: E402 + + +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) + hookenv.action_fail(msg) + sys.exit() + + +if not is_state('hive.installed'): + fail('Charm is not yet ready to run the Bigtop smoke test(s)') + +# Bigtop smoke test components +smoke_components = ['hive'] + +# Env required by test components +smoke_env = { + 'HIVE_HOME': '/usr/lib/hive', + 'HIVE_CONF_DIR': '/etc/hive/conf', +} + +bigtop = Bigtop() +result = bigtop.run_smoke_tests(smoke_components, smoke_env) +if result == 'success': + hookenv.action_set({'outcome': 'success'}) +else: + fail('{} smoke tests failed with: {}'.format(smoke_components, result)) diff --git a/bigtop-packages/src/charm/hive/layer-hive/config.yaml b/bigtop-packages/src/charm/hive/layer-hive/config.yaml new file mode 100644 index 00000000..90ce3375 --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/config.yaml @@ -0,0 +1,6 @@ +options: + heap: + type: int + default: 1024 + description: | + The maximum heap size (in MB) used by the Hive shell JVM. diff --git a/bigtop-packages/src/charm/hive/layer-hive/copyright b/bigtop-packages/src/charm/hive/layer-hive/copyright new file mode 100644 index 00000000..e900b97c --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/copyright @@ -0,0 +1,16 @@ +Format: http://dep.debian.net/deps/dep5/ + +Files: * +Copyright: Copyright 2015, Canonical Ltd., All Rights Reserved. +License: Apache License 2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bigtop-packages/src/charm/hive/layer-hive/icon.svg b/bigtop-packages/src/charm/hive/layer-hive/icon.svg new file mode 100644 index 00000000..12771d3d --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/icon.svg @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bigtop-packages/src/charm/hive/layer-hive/layer.yaml b/bigtop-packages/src/charm/hive/layer-hive/layer.yaml new file mode 100644 index 00000000..5be7ac9f --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/layer.yaml @@ -0,0 +1,31 @@ +repo: https://github.com/apache/bigtop/tree/master/bigtop-packages/src/charm/hive/layer-hive +includes: + - 'layer:apache-bigtop-base' + - 'layer:hadoop-client' + - 'interface:hbase' + - 'interface:hive' + - 'interface:mysql' +options: + basic: + packages: + - 'libmysql-java' + - 'mysql-client' + apache-bigtop-base: + groups: + - 'hadoop' + - 'hive' + users: + ubuntu: + groups: ['hadoop', 'hive'] + dirs: + hive: + path: '/usr/lib/hive' + hive_conf: + path: '/etc/hive/conf' + ports: + hive-thrift: + port: 10000 + exposed_on: 'hive' + hive-thrift-web: + port: 10001 + exposed_on: 'hive' diff --git a/bigtop-packages/src/charm/hive/layer-hive/lib/charms/layer/bigtop_hive.py b/bigtop-packages/src/charm/hive/layer-hive/lib/charms/layer/bigtop_hive.py new file mode 100755 index 00000000..c7d9cc6f --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/lib/charms/layer/bigtop_hive.py @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.core import hookenv, host, unitdata +from charms import layer +from charms.layer.apache_bigtop_base import Bigtop +from jujubigdata import utils + + +class Hive(object): + '''This class manages Hive.''' + def __init__(self): + self.dist_config = utils.DistConfig( + data=layer.options('apache-bigtop-base')) + + def install(self, hbase=None): + ''' + Trigger the Bigtop puppet recipe that handles the Hive service. + ''' + # Dirs are handled by the bigtop deb. No need to call out to + # dist_config to do that. We do want 'ubuntu' in the hive group though. + self.dist_config.add_users() + + # Prep config + roles = ['hive-client', 'hive-metastore', 'hive-server2'] + metastore = "thrift://{}:9083".format(hookenv.unit_private_ip()) + if hbase: + roles.append('hive-hbase') + hb_connect = "{}:{}".format(hbase['host'], hbase['master_port']) + zk_connect = hbase['zk_connect'] + else: + hb_connect = "" + zk_connect = "" + + override = { + 'hadoop_hive::common_config::hbase_master': hb_connect, + 'hadoop_hive::common_config::hbase_zookeeper_quorum': zk_connect, + 'hadoop_hive::common_config::metastore_uris': metastore, + 'hadoop_hive::common_config::server2_thrift_port': + self.dist_config.port('hive-thrift'), + 'hadoop_hive::common_config::server2_thrift_http_port': + self.dist_config.port('hive-thrift-web'), + } + bigtop = Bigtop() + bigtop.render_site_yaml(roles=roles, overrides=override) + bigtop.trigger_puppet() + + # Bigtop doesn't create a hive-env.sh, but we need it for heap config + hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh' + if not hive_env.exists(): + (self.dist_config.path('hive_conf') / 'hive-env.sh.template').copy(hive_env) + + def configure_hive(self): + ''' + Called during config-changed events + ''' + config = hookenv.config() + hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh' + utils.re_edit_in_place(hive_env, { + r'.*export HADOOP_HEAPSIZE *=.*': 'export HADOOP_HEAPSIZE=%s' % config['heap'], + }) + + def configure_remote_db(self, mysql): + hive_site = self.dist_config.path('hive_conf') / 'hive-site.xml' + jdbc_url = "jdbc:mysql://{}:{}/{}?createDatabaseIfNotExist=true".format( + mysql.host(), mysql.port(), mysql.database() + ) + with utils.xmlpropmap_edit_in_place(hive_site) as props: + props['javax.jdo.option.ConnectionURL'] = jdbc_url + props['javax.jdo.option.ConnectionUserName'] = mysql.user() + props['javax.jdo.option.ConnectionPassword'] = mysql.password() + props['javax.jdo.option.ConnectionDriverName'] = "com.mysql.jdbc.Driver" + + hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh' + utils.re_edit_in_place(hive_env, { + r'.*export HIVE_AUX_JARS_PATH *=.*': 'export HIVE_AUX_JARS_PATH=/usr/share/java/mysql-connector-java.jar', + }) + + # Now that we have db connection info, init our schema (only once) + remote_db = hookenv.remote_service_name() + if not unitdata.kv().get('hive.schema.initialized.%s' % remote_db): + tool_path = "{}/bin/schematool".format(self.dist_config.path('hive')) + utils.run_as('ubuntu', tool_path, '-initSchema', '-dbType', 'mysql') + unitdata.kv().set('hive.schema.initialized.%s' % remote_db, True) + unitdata.kv().flush(True) + + def configure_local_db(self): + local_url = 'jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true' + local_driver = 'org.apache.derby.jdbc.EmbeddedDriver' + hive_site = self.dist_config.path('hive_conf') / 'hive-site.xml' + with utils.xmlpropmap_edit_in_place(hive_site) as props: + props['javax.jdo.option.ConnectionURL'] = local_url + props['javax.jdo.option.ConnectionUserName'] = 'APP' + props['javax.jdo.option.ConnectionPassword'] = 'mine' + props['javax.jdo.option.ConnectionDriverName'] = local_driver + + hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh' + utils.re_edit_in_place(hive_env, { + r'.*export HIVE_AUX_JARS_PATH *=.*': '# export HIVE_AUX_JARS_PATH=', + }) + + def restart(self): + self.stop() + self.start() + + def start(self): + # order is important; metastore must start first. + hookenv.log('Starting Hive services') + host.service_start('hive-metastore') + host.service_start('hive-server2') + hookenv.log('Hive services have been started') + + def stop(self): + # order is important; metastore must stop last. + hookenv.log('Stopping Hive services') + host.service_stop('hive-server2') + host.service_stop('hive-metastore') + hookenv.log('Hive services have been stopped') + + def open_ports(self): + for port in self.dist_config.exposed_ports('hive'): + hookenv.open_port(port) + + def close_ports(self): + for port in self.dist_config.exposed_ports('hive'): + hookenv.close_port(port) diff --git a/bigtop-packages/src/charm/hive/layer-hive/metadata.yaml b/bigtop-packages/src/charm/hive/layer-hive/metadata.yaml new file mode 100644 index 00000000..998dd72c --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/metadata.yaml @@ -0,0 +1,19 @@ +name: hive +summary: Hive from Apache Bigtop +maintainer: Juju Big Data +description: > + Apache Hive is a data warehouse infrastructure built on top of Hadoop that + supports data summarization, query, and analysis. Hive provides an SQL-like + language called HiveQL that transparently converts queries to MapReduce for + execution on large datasets stored in Hadoop's HDFS. + + This charm provides version 1.2.1 of the Hive application from Apache Bigtop. +tags: [] +provides: + client: + interface: hive +requires: + database: + interface: mysql + hbase: + interface: hbase diff --git a/bigtop-packages/src/charm/hive/layer-hive/reactive/hive.py b/bigtop-packages/src/charm/hive/layer-hive/reactive/hive.py new file mode 100644 index 00000000..0deebd47 --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/reactive/hive.py @@ -0,0 +1,184 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.core import hookenv +from charms.layer.apache_bigtop_base import get_layer_opts, get_package_version +from charms.layer.bigtop_hive import Hive +from charms.reactive import ( + RelationBase, + is_state, + remove_state, + set_state, + when, + when_not, +) +from charms.reactive.helpers import data_changed + + +@when('bigtop.available') +def report_status(): + hadoop_joined = is_state('hadoop.joined') + hadoop_ready = is_state('hadoop.ready') + hbase_joined = is_state('hbase.joined') + hbase_ready = is_state('hbase.ready') + database_joined = is_state('database.connected') + database_ready = is_state('database.available') + hive_installed = is_state('hive.installed') + if not hadoop_joined: + hookenv.status_set('blocked', + 'waiting for relation to hadoop plugin') + elif not hadoop_ready: + hookenv.status_set('waiting', + 'waiting for hadoop to become ready') + elif database_joined and not database_ready: + hookenv.status_set('waiting', + 'waiting for database to become ready') + elif hbase_joined and not hbase_ready: + hookenv.status_set('waiting', + 'waiting for hbase to become ready') + elif hive_installed and not database_ready: + hookenv.status_set('active', + 'ready (local metastore)') + elif hive_installed and database_ready: + hookenv.status_set('active', + 'ready (remote metastore)') + + +@when('bigtop.available', 'hadoop.ready') +def install_hive(hadoop): + ''' + Anytime our dependencies are available, check to see if we have a valid + reason to (re)install. These include: + - initial install + - HBase has joined/departed + ''' + # Hive cannot handle - in the metastore db name and + # mysql uses the service name to name the db + if "-" in hookenv.service_name(): + hookenv.status_set('blocked', "application name may not contain '-'; " + "redeploy with a different name") + return + + # Get hbase connection dict if it's available + if is_state('hbase.ready'): + hbase = RelationBase.from_state('hbase.ready') + hbserver = hbase.hbase_servers()[0] + else: + hbserver = None + + # Use this to determine if we need to reinstall + deployment_matrix = { + 'hbase': hbserver, + } + + # Handle nuances when installing versus re-installing + if not is_state('hive.installed'): + prefix = "installing" + + # On initial install, prime our kv with the current deployment matrix. + # Subsequent calls will use this to determine if a reinstall is needed. + data_changed('deployment_matrix', deployment_matrix) + else: + prefix = "configuring" + + # Return if our matrix has not changed + if not data_changed('deployment_matrix', deployment_matrix): + return + + hookenv.status_set('maintenance', '{} hive'.format(prefix)) + hookenv.log("{} hive with: {}".format(prefix, deployment_matrix)) + hive = Hive() + hive.install(hbase=hbserver) + hive.restart() + hive.open_ports() + set_state('hive.installed') + report_status() + + # set app version string for juju status output + hive_version = get_package_version('hive') or 'unknown' + hookenv.application_version_set(hive_version) + + +@when('hive.installed', 'config.changed.heap') +def config_changed(): + hookenv.status_set('maintenance', 'configuring with new options') + hive = Hive() + hive.configure_hive() + hive.restart() + report_status() + + +@when('hive.installed', 'database.available') +@when_not('hive.db.configured') +def configure_with_remote_db(db): + hookenv.status_set('maintenance', 'configuring external database') + hive = Hive() + hive.configure_remote_db(db) + hive.restart() + set_state('hive.db.configured') + report_status() + + +@when('hive.installed', 'hive.db.configured') +@when_not('database.available') +def configure_with_local_db(): + ''' + Reconfigure Hive using a local metastore db. + + The initial installation will configure Hive with a local metastore_db. + Once an external db becomes available, we reconfigure Hive to use it. If + that external db goes away, we'll use this method to set Hive back into + local mode. + ''' + hookenv.status_set('maintenance', 'configuring local database') + hive = Hive() + hive.configure_local_db() + hive.restart() + remove_state('hive.db.configured') + report_status() + + +@when('hive.installed') +@when_not('hadoop.ready') +def stop_hive(): + ''' + Hive depends on Hadoop. If we are installed and hadoop goes away, shut down + services and remove our installed state. + ''' + hive = Hive() + hive.close_ports() + hive.stop() + remove_state('hive.installed') + report_status() + + +@when('hive.installed', 'client.joined') +def serve_client(client): + ''' + Inform clients when hive is ready to serve. + ''' + port = get_layer_opts().port('hive-thrift') + client.send_port(port) + client.set_ready() + + +@when('client.joined') +@when_not('hive.installed') +def stop_serving_client(client): + ''' + Inform connected clients that Hive is no longer ready. This can happen + if Hadoop goes away (the 'installed' state will be removed). + ''' + client.clear_ready() diff --git a/bigtop-packages/src/charm/hive/layer-hive/tests/01-basic-deployment.py b/bigtop-packages/src/charm/hive/layer-hive/tests/01-basic-deployment.py new file mode 100755 index 00000000..04a614db --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/tests/01-basic-deployment.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import amulet +import re +import unittest + + +class TestDeploy(unittest.TestCase): + """ + Trivial deployment test for Apache Bigtop Hive. + """ + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='xenial') + cls.d.add('hive') + + cls.d.setup(timeout=1800) + cls.d.sentry.wait_for_messages({'hive': re.compile('ready|waiting')}, + timeout=1800) + cls.unit = cls.d.sentry['hive'][0] + + +if __name__ == '__main__': + unittest.main() diff --git a/bigtop-packages/src/charm/hive/layer-hive/tests/02-smoke-test.py b/bigtop-packages/src/charm/hive/layer-hive/tests/02-smoke-test.py new file mode 100755 index 00000000..86930328 --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/tests/02-smoke-test.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import amulet +import re +import unittest + + +class TestDeploy(unittest.TestCase): + """ + Hadoop/Hive deployment and smoke test for the Apache Bigtop Hive service. + """ + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='xenial') + cls.d.add('hive') + cls.d.add('namenode', 'hadoop-namenode') + cls.d.add('plugin', 'hadoop-plugin') + cls.d.add('resourcemanager', 'hadoop-resourcemanager') + cls.d.add('slave', 'hadoop-slave') + + cls.d.relate('hive:hadoop', 'plugin:hadoop-plugin') + cls.d.relate('plugin:namenode', 'namenode:namenode') + cls.d.relate('plugin:resourcemanager', 'resourcemanager:resourcemanager') + cls.d.relate('resourcemanager:namenode', 'namenode:namenode') + cls.d.relate('slave:namenode', 'namenode:datanode') + cls.d.relate('slave:resourcemanager', 'resourcemanager:nodemanager') + + cls.d.setup(timeout=3600) + cls.d.sentry.wait_for_messages({'hive': re.compile('ready')}, + timeout=3600) + cls.hive = cls.d.sentry['hive'][0] + + def test_hive(self): + """ + Validate Hive by running the smoke-test action. + """ + uuid = self.hive.run_action('smoke-test') + result = self.d.action_fetch(uuid, full_output=True) + # action status=completed on success + if (result['status'] != "completed"): + self.fail('Hive smoke-test failed: %s' % result) + + +if __name__ == '__main__': + unittest.main() diff --git a/bigtop-packages/src/charm/hive/layer-hive/tests/tests.yaml b/bigtop-packages/src/charm/hive/layer-hive/tests/tests.yaml new file mode 100644 index 00000000..3b6ce3e5 --- /dev/null +++ b/bigtop-packages/src/charm/hive/layer-hive/tests/tests.yaml @@ -0,0 +1,3 @@ +reset: false +packages: + - amulet -- cgit v1.2.3