diff options
| author | Kevin W Monroe <kevin.monroe@canonical.com> | 2017-06-23 11:41:54 -0500 |
|---|---|---|
| committer | Kevin W Monroe <kevin.monroe@canonical.com> | 2017-06-28 09:45:18 -0500 |
| commit | e14693842b0ccadcbed37a08f6a1c4077ee8a406 (patch) | |
| tree | c8ad78462e7ae8f3560bf6823d784492f99c38cb /bigtop-deploy | |
| parent | 5ea264beee555c55816dc775981900e747025cca (diff) | |
BIGTOP-2821: expose extra config options for spark
Closes #238
Diffstat (limited to 'bigtop-deploy')
| -rw-r--r-- | bigtop-deploy/puppet/modules/spark/manifests/init.pp | 3 | ||||
| -rw-r--r-- | bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf | 7 |
2 files changed, 9 insertions, 1 deletions
diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp index ebb25d07..6fe23dc1 100644 --- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp @@ -146,6 +146,9 @@ class spark { $use_yarn_shuffle_service = false, $event_log_dir = "hdfs:///var/log/spark/apps", $history_log_dir = "hdfs:///var/log/spark/apps", + $extra_lib_dirs = "/usr/lib/hadoop/lib/native", + $driver_mem = "1g", + $executor_mem = "1g", ) { ### This is an ungodly hack to deal with the consequence of adding diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf index a99d9bd1..2ef4a041 100644 --- a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf +++ b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf @@ -30,4 +30,9 @@ spark.yarn.historyServer.address <%= @master_host %>:<%= @history_ui_port %> <% end -%> spark.history.ui.port <%= @history_ui_port %> spark.shuffle.service.enabled <%= @use_yarn_shuffle_service %> -spark.driver.extraJavaOptions -Djava.library.path=/usr/lib/hadoop/lib/native +<% if @extra_lib_dirs -%> +spark.driver.extraLibraryPath <%= @extra_lib_dirs %> +spark.executor.extraLibraryPath <%= @extra_lib_dirs %> +<% end -%> +spark.driver.memory <%= @driver_mem %> +spark.executor.memory <%= @executor_mem %> |