BIGTOP-2123. Add zeppelin Puppet recipes

6 files changed, 235 insertions, 1 deletions

diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index e86af39e..de98502c 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -169,3 +169,8 @@ hadoop::common::tez_jars: "/usr/lib/tez"
 
 # to enable tez in hive, uncomment the lines below
 # hadoop_hive::client::hive_execution_engine: "tez"
+
+zeppelin::server::spark_master_url: "yarn-client"
+zeppelin::server::hiveserver2_url: "jdbc:hive2://%{hiera('hadoop-hive::common::hiveserver2_host')}:%{hiera('hadoop-hive::common::hiveserver2_port')}"
+zeppelin::server::hiveserver2_user: "%{hiera('bigtop::hiveserver2_user')}"
+zeppelin::server::hiveserver2_password: "%{hiera('bigtop::hiveserver2_password')}"
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 14d078eb..a0be5678 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -102,6 +102,9 @@ $roles_map = {
   ycsb => {
     client => ["ycsb-client"],
   },
+  zeppelin => {
+    master => ["zeppelin-server"],
+  },
 }
 
 class hadoop_cluster_node (
@@ -166,7 +169,8 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_
     "tachyon",
     "tez",
     "ycsb",
-    "kerberos"
+    "kerberos",
+    "zeppelin"
   ]
 
   deploy_module { $modules:
diff --git a/bigtop-deploy/puppet/modules/zeppelin/manifests/init.pp b/bigtop-deploy/puppet/modules/zeppelin/manifests/init.pp
new file mode 100644
index 00000000..98c6e8f3
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/zeppelin/manifests/init.pp
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class zeppelin {
+
+  class deploy ($roles) {
+    if ('zeppelin-server' in $roles) {
+      include server
+    }
+  }
+
+  class server(
+      $spark_master_url = 'yarn-client',
+      $server_port = 9080,
+      $web_socket_port = 9081,
+      $hiveserver2_url = 'jdbc:hive2://localhost:10000',
+      $hiveserver2_user = 'hive',
+      $hiveserver2_password = '') {
+    package { 'zeppelin':
+      ensure => latest,
+    }
+
+    file { '/etc/zeppelin/conf/zeppelin-env.sh':
+      content   => template('zeppelin/zeppelin-env.sh'),
+      require   => Package['zeppelin'],
+    }
+
+    file { '/etc/zeppelin/conf/interpreter.json':
+      content => template('zeppelin/interpreter.json'),
+      require => Package['zeppelin'],
+      owner   => 'zeppelin',
+      group   => 'zeppelin',
+    }
+
+    service { 'zeppelin':
+      ensure     => running,
+      subscribe  => [
+          Package['zeppelin'],
+          File['/etc/zeppelin/conf/zeppelin-env.sh'],
+          File['/etc/zeppelin/conf/interpreter.json'],
+      ],
+      hasrestart => true,
+      hasstatus  => true,
+    }
+  }
+}
diff --git a/bigtop-deploy/puppet/modules/zeppelin/templates/interpreter.json b/bigtop-deploy/puppet/modules/zeppelin/templates/interpreter.json
new file mode 100644
index 00000000..a5e7c7ae
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/zeppelin/templates/interpreter.json
@@ -0,0 +1,124 @@
+{
+  "interpreterSettings": {
+    "2AJXGMUUJ": {
+      "id": "2AJXGMUUJ",
+      "name": "md",
+      "group": "md",
+      "properties": {},
+      "interpreterGroup": [
+        {
+          "class": "org.apache.zeppelin.markdown.Markdown",
+          "name": "md"
+        }
+      ],
+      "option": {
+        "remote": true
+      }
+    },
+    "2AM1YV5CU": {
+      "id": "2AM1YV5CU",
+      "name": "angular",
+      "group": "angular",
+      "properties": {},
+      "interpreterGroup": [
+        {
+          "class": "org.apache.zeppelin.angular.AngularInterpreter",
+          "name": "angular"
+        }
+      ],
+      "option": {
+        "remote": true
+      }
+    },
+    "2AK8P7CPX": {
+      "id": "2AK8P7CPX",
+      "name": "hive",
+      "group": "hive",
+      "properties": {
+        "hive.hiveserver2.url": "<%= @hiveserver2_url %>",
+        "hive.hiveserver2.password": "<%= @hiveserver2_password %>",
+        "hive.hiveserver2.user": "<%= @hiveserver2_user %>"
+      },
+      "interpreterGroup": [
+        {
+          "class": "org.apache.zeppelin.hive.HiveInterpreter",
+          "name": "hive"
+        }
+      ],
+      "option": {
+        "remote": true
+      }
+    },
+    "2AKK3QQXU": {
+      "id": "2AKK3QQXU",
+      "name": "sh",
+      "group": "sh",
+      "properties": {},
+      "interpreterGroup": [
+        {
+          "class": "org.apache.zeppelin.shell.ShellInterpreter",
+          "name": "sh"
+        }
+      ],
+      "option": {
+        "remote": true
+      }
+    },
+    "2ANGGHHMQ": {
+      "id": "2ANGGHHMQ",
+      "name": "spark",
+      "group": "spark",
+      "properties": {
+        "spark.cores.max": "",
+        "spark.yarn.jar": "",
+        "master": "<%= @spark_master_url %>",
+        "zeppelin.spark.maxResult": "1000",
+        "zeppelin.dep.localrepo": "local-repo",
+        "spark.app.name": "Zeppelin",
+        "spark.executor.memory": "512m",
+        "zeppelin.spark.useHiveContext": "true",
+        "args": "",
+        "spark.home": "/usr/lib/spark",
+        "zeppelin.spark.concurrentSQL": "false",
+        "zeppelin.pyspark.python": "python"
+      },
+      "interpreterGroup": [
+        {
+          "class": "org.apache.zeppelin.spark.SparkInterpreter",
+          "name": "spark"
+        },
+        {
+          "class": "org.apache.zeppelin.spark.PySparkInterpreter",
+          "name": "pyspark"
+        },
+        {
+          "class": "org.apache.zeppelin.spark.SparkSqlInterpreter",
+          "name": "sql"
+        },
+        {
+          "class": "org.apache.zeppelin.spark.DepInterpreter",
+          "name": "dep"
+        }
+      ],
+      "option": {
+        "remote": true
+      }
+    }
+  },
+  "interpreterBindings": {
+    "2A94M5J1Z": [
+      "2ANGGHHMQ",
+      "2AJXGMUUJ",
+      "2AM1YV5CU",
+      "2AKK3QQXU",
+      "2AK8P7CPX"
+    ],
+    "2AN1VR2PQ": [
+      "2ANGGHHMQ",
+      "2AJXGMUUJ",
+      "2AM1YV5CU",
+      "2AKK3QQXU",
+      "2AK8P7CPX"
+    ]
+  }
+}
diff --git a/bigtop-deploy/puppet/modules/zeppelin/templates/zeppelin-env.sh b/bigtop-deploy/puppet/modules/zeppelin/templates/zeppelin-env.sh
new file mode 100644
index 00000000..8095f010
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/zeppelin/templates/zeppelin-env.sh
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export ZEPPELIN_PORT=<%= @server_port %>
+export ZEPPELIN_WEBSOCKET_PORT=<%= @web_socket_port %>
+export ZEPPELIN_CONF_DIR=/etc/zeppelin/conf
+export ZEPPELIN_LOG_DIR=/var/log/zeppelin
+export ZEPPELIN_PID_DIR=/var/run/zeppelin
+export ZEPPELIN_NOTEBOOK_DIR=/var/lib/zeppelin/notebook
+export MASTER=<%= @spark_master_url %>
+export SPARK_HOME=/usr/lib/spark
+export HADOOP_CONF_DIR=/etc/hadoop/conf
diff --git a/bigtop-deploy/puppet/modules/zeppelin/tests/init.pp b/bigtop-deploy/puppet/modules/zeppelin/tests/init.pp
new file mode 100644
index 00000000..46e8fc55
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/zeppelin/tests/init.pp
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+zeppelin::server { 'zeppelin server':
+  server_port     => '8080',
+  web_socket_port => '8081',
+}