BIGTOP-2766. [Puppet] Spark worker startup failed due to default master_url is yarn

5 files changed, 28 insertions, 4 deletions

diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml
index f18aa9c7..02f8e45c 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -13,11 +13,13 @@ hadoop::hadoop_storage_dirs:
 
 #hadoop_cluster_node::cluster_components:
 #  - alluxio
+#  - ambari
 #  - apex
 #  - crunch
 #  - flink
 #  - flume
 #  - giraph
+#  - gpdb
 #  - ignite_hadoop
 #  - hbase
 #  - hcat
@@ -31,14 +33,13 @@ hadoop::hadoop_storage_dirs:
 #  - qfs
 #  - solrcloud
 #  - spark
+#  - spark-standalone
 #  - sqoop
 #  - sqoop2
 #  - tez
 #  - yarn
-#  - zookeeper
 #  - ycsb
-#  - gpdb
-#  - ambari
+#  - zookeeper
 
 #bigtop::roles_enabled: false
 
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index e4e22b20..7579606c 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -69,6 +69,11 @@ $roles_map = {
     worker => ["solr-server"],
   },
   spark => {
+    worker => ["spark-on-yarn"],
+    client => ["spark-client"],
+    library => ["spark-yarn-slave"],
+  },
+  spark-standalone => {
     master => ["spark-master"],
     worker => ["spark-worker"],
   },
@@ -278,4 +283,6 @@ class node_with_components inherits hadoop_cluster_node {
   class { 'node_with_roles':
     roles => $roles,
   }
+
+  notice("Roles to deploy: ${roles}")
 }
diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
index 4c72d20d..ebb25d07 100644
--- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
@@ -135,7 +135,7 @@ class spark {
   }
 
   class common(
-      $master_url = 'yarn',
+      $master_url = undef,
       $master_host = $fqdn,
       $zookeeper_connection_string = undef,
       $master_port = 7077,
diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
index 5978becc..a99d9bd1 100644
--- a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
+++ b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
@@ -13,7 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+<% if @master_url -%>
 spark.master <%= @master_url %>
+<% else -%>
+<% if (scope['deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
+spark.master spark://<%= @master_host %>:<%= @master_port %>
+<% else -%>
+spark.master yarn
+<% end -%>
+<% end -%>
 spark.eventLog.enabled true
 spark.eventLog.dir <%= @event_log_dir %>
 spark.history.fs.logDirectory <%= @history_log_dir %>
diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
index 0aabc658..2f3e406e 100755
--- a/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
+++ b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
@@ -24,7 +24,15 @@ export HIVE_CONF_DIR=${HIVE_CONF_DIR:-/etc/hive/conf}
 export STANDALONE_SPARK_MASTER_HOST=<%= @master_host %>
 export SPARK_MASTER_PORT=<%= @master_port %>
 export SPARK_MASTER_IP=$STANDALONE_SPARK_MASTER_HOST
+<% if @master_url -%>
 export SPARK_MASTER_URL=<%= @master_url %>
+<% else -%>
+<% if (scope['deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
+export SPARK_MASTER_URL=spark://<%= @master_host %>:<%= @master_port %>
+<% else -%>
+export SPARK_MASTER_URL=yarn
+<% end -%>
+<% end -%>
 export SPARK_MASTER_WEBUI_PORT=<%= @master_ui_port %>
 
 export SPARK_WORKER_DIR=${SPARK_WORKER_DIR:-/var/run/spark/work}