BIGTOP-2663: puppet hadoop module: Consolidate memory resource settings

3 files changed, 39 insertions, 4 deletions

diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index a8a8152e..30a68e2b 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -382,6 +382,10 @@ class hadoop ($hadoop_security_authentication = "simple",
       $mapreduce_output_fileoutputformat_compress_type = "BLOCK", # "RECORD" default
       $mapreduce_map_output_compress = undef,
       $mapreduce_job_reduce_slowstart_completedmaps = undef,
+      $mapreduce_map_memory_mb = undef,
+      $mapreduce_reduce_memory_mb = undef,
+      $mapreduce_map_java_opts = "-Xmx1024m",
+      $mapreduce_reduce_java_opts = "-Xmx1024m",
       $hadoop_security_authentication = $hadoop::hadoop_security_authentication,
       $kerberos_realm = $hadoop::kerberos_realm,
   ) inherits hadoop {
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
index ea499261..a25f66b8 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
@@ -111,20 +111,44 @@
     <name>yarn.app.mapreduce.am.staging-dir</name>
     <value><%= @yarn_app_mapreduce_am_staging_dir %></value>
   </property>
+<% end -%>
+
+<!-- Memory settings -->
+<% if @mapreduce_map_memory_mb -%>
+  <property>
+    <name>mapreduce.map.memory.mb</name>
+    <value><%= @mapreduce_map_memory_mb %></value>
+  </property>
+<% end -%>
+
+<% if @mapreduce_reduce_memory_mb -%>
+  <property>
+    <name>mapreduce.reduce.memory.mb</name>
+    <value><%= @mapreduce_reduce_memory_mb %></value>
+  </property>
+<% end -%>
 
+<% if @mapreduce_map_java_opts -%>
+  <property>
+    <name>mapreduce.map.java.opts</name>
+    <value><%= @mapreduce_map_java_opts %></value>
+  </property>
 <% end -%>
+
+<% if @mapreduce_reduce_java_opts -%>
   <property>
-    <name>mapred.child.java.opts</name>
-    <value>-Xmx1024m</value>
+    <name>mapreduce.reduce.java.opts</name>
+    <value><%= @mapreduce_reduce_java_opts %></value>
   </property>
+<% end -%>
 
 <% if @mapreduce_task_io_sort_mb -%>
   <property>
     <name>mapreduce.task.io.sort.mb</name>
     <value><%= @mapreduce_task_io_sort_mb %></value>
   </property>
-
 <% end -%>
+
 <% if @mapreduce_task_io_sort_factor -%>
   <property>
     <name>mapreduce.task.io.sort.factor</name>
@@ -189,6 +213,5 @@
     <name>mapreduce.job.reduce.slowstart.completedmaps</name>
     <value><%= @mapreduce_job_reduce_slowstart_completedmaps %></value>
   </property>
-
 <% end -%>
 </configuration>
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
index 56c60152..e9e6a054 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
@@ -234,4 +234,12 @@
     <value><%= @yarn_resourcemanager_scheduler_class %></value>
   </property>
 <% end -%>
+
+<!-- The defaut setting (2.1) is silly. The virtual memory is not 
+     a limiting factor on 64Bit systems, at least not a limiting  
+     resource, so make it large, very large. -->
+  <property>
+    <name>yarn.nodemanager.vmem-pmem-ratio</name>
+    <value>100</value>
+  </property>
 </configuration>