AMBARI-895. Need name consistency for metrics. (Tom Beerbower via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/AMBARI-666@1401565 13f79535-47bb-0310-9956-ffa450edef68

7 files changed, 138 insertions, 83 deletions

diff --git a/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp b/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
index 9b07dcdb1f..af8dbbeabc 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
@@ -28,47 +28,27 @@ class hdp-hbase(
   $config_dir = $hdp-hbase::params::conf_dir

   

   $hdp::params::component_exists['hdp-hbase'] = true

+
 

-  #Configs generation  

-  configgenerator::configfile{'hbase-site.xml.erb': 

-    module => 'hdp-hbase',

-    properties => {'hbase.rootdir' => 'hdfs://<%=scope.function_hdp_host("namenode_host")%>:8020<%=scope.function_hdp_template_var("hbase_hdfs_root_dir")%>',

-      'hbase.cluster.distributed' => 'true',

-      'hbase.tmp.dir' => '<%=scope.function_hdp_template_var("hbase_tmp_dir")%>',

-      'hbase.master.info.bindAddress' => '<%=scope.function_hdp_host("hbase_master_host")%>',

-      'hbase.regionserver.global.memstore.upperLimit' => '<%=scope.function_hdp_template_var("regionserver_memstore_upperlimit")%>',

-      'hbase.regionserver.handler.count' => '<%=scope.function_hdp_template_var("regionserver_handlers")%>',

-      'hbase.hregion.majorcompaction' => '<%=scope.function_hdp_template_var("hregion_majorcompaction")%>',

-      'hbase.regionserver.global.memstore.lowerLimit' => '<%=scope.function_hdp_template_var("regionserver_memstore_lowerlimit")%>',

-      'hbase.hregion.memstore.block.multiplier' => '<%=scope.function_hdp_template_var("hregion_blockmultiplier")%>',

-      'hbase.hregion.memstore.flush.size' => '<%=scope.function_hdp_template_var("hregion_memstoreflushsize")%>',

-      'hbase.hregion.memstore.mslab.enabled' => '<%=scope.function_hdp_template_var("regionserver_memstore_lab")%>',

-      'hbase.hregion.max.filesize' => '<%=scope.function_hdp_template_var("hstorefile_maxsize")%>',

-      'hbase.client.scanner.caching' => '<%=scope.function_hdp_template_var("client_scannercaching")%>',

-      'zookeeper.session.timeout' => '<%=scope.function_hdp_template_var("zookeeper_sessiontimeout")%>',

-      'hbase.client.keyvalue.maxsize' => '<%=scope.function_hdp_template_var("hfile_max_keyvalue_size")%>',

-      'hbase.hstore.compactionThreshold' => '<%=scope.function_hdp_template_var("hstore_compactionthreshold")%>',

-      'hbase.hstore.blockingStoreFiles' => '<%=scope.function_hdp_template_var("hstore_blockingstorefiles")%>',

-      'hfile.block.cache.size' => '<%=scope.function_hdp_template_var("hfile_blockcache_size")%>',

-      'hbase.master.keytab.file' => '<%=scope.function_hdp_template_var("keytab_path")%>/hm.service.keytab',

-      'hbase.master.kerberos.principal' => 'hm/_HOST@<%=scope.function_hdp_template_var("kerberos_domain")%>',

-      'hbase.regionserver.keytab.file' => '<%=scope.function_hdp_template_var("keytab_path")%>/rs.service.keytab',

-      'hbase.regionserver.kerberos.principal' => 'rs/_HOST@<%=scope.function_hdp_template_var("kerberos_domain")%>',

-      'hbase.superuser' => 'hbase',

-      'hbase.coprocessor.region.classes' => '<%=scope.function_hdp_template_var("preloaded_regioncoprocessor_classes")%>',

-      'hbase.coprocessor.master.classes' => '<%=scope.function_hdp_template_var("preloaded_mastercoprocessor_classes")%>',

-      'hbase.zookeeper.quorum' => '<%=zkh=scope.function_hdp_host("zookeeper_hosts");scope.function_hdp_is_empty(zkh) ? "" : [zkh].flatten.join(",")%>',

-      'dfs.support.append' => '<%=scope.function_hdp_template_var("hdfs_support_append")%>',

-      'dfs.client.read.shortcircuit' => '<%=scope.function_hdp_template_var("hdfs_enable_shortcircuit_read")%>',

-      'dfs.client.read.shortcircuit.skip.checksum' => '<%=scope.function_hdp_template_var("hdfs_enable_shortcircuit_skipchecksum")%>',}

+  #Configs generation  
+
+  if has_key($configuration, 'hbase_site') {

+    configgenerator::configfile{'hbase_site': 

+      modulespath => $hdp-hbase::params::conf_dir,

+      filename => 'hbase-site.xml',

+      module => 'hdp-hbase',

+      configuration => $configuration['hbase_site']

       }

-

-  configgenerator::configfile{'hbase-policy.xml.erb': 

-    module => 'hdp-hbase',

-    properties => {'security.client.protocol.acl' => '*',

-      'security.admin.protocol.acl' => '*',

-      'security.masterregion.protocol.acl' => '*',}

+    }
+
+  if has_key($configuration, 'hbase_policy') {

+    configgenerator::configfile{'hbase_policy': 

+      modulespath => $hdp-hbase::params::conf_dir,

+      filename => 'hbase-policy.xml',

+      module => 'hdp-hbase',

+      configuration => $configuration['hbase_policy']

       }

+    }

 

   anchor{'hdp-hbase::begin':}

   anchor{'hdp-hbase::end':}

@@ -94,7 +74,7 @@ class hdp-hbase(
       force => true

     }

 

-   hdp-hbase::configfile { ['hbase-env.sh','hbase-site.xml','hbase-policy.xml','log4j.properties','hadoop-metrics.properties']: 

+   hdp-hbase::configfile { ['hbase-env.sh','log4j.properties','hadoop-metrics.properties']: 

       type => $type

     }

     hdp-hbase::configfile { 'regionservers':}

diff --git a/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/init.pp b/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/init.pp
index 9e0d232b41..454522d7e6 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/init.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/init.pp
@@ -30,11 +30,12 @@ class hdp-hive(
 
 # Configs generation  
 
-  if has_key($configuration, 'hdp_hive__hive_site') {
-    configgenerator::configfile{'hive_site_xml': 
+  if has_key($configuration, 'hive_site') {
+    configgenerator::configfile{'hive_site':
+      modulespath => $hdp-hive::params::hive_conf_dir, 
       filename => 'hive-site.xml',
       module => 'hdp-hive',
-      configuration => $configuration['hdp_hive__hive_site']
+      configuration => $configuration['hive_site']
     }
   }
 
@@ -66,7 +67,7 @@ class hdp-hive(
       force => true
     }
 
-    hdp-hive::configfile { ['hive-env.sh','hive-site.xml']: }
+    hdp-hive::configfile { ['hive-env.sh']: }
   
     Anchor['hdp-hive::begin'] -> Hdp::Package['hive'] -> Hdp::User[$hive_user] ->  
      Hdp::Directory[$hive_config_dir] -> Hdp-hive::Configfile<||> ->  Anchor['hdp-hive::end']
diff --git a/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/init.pp b/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/init.pp
index 5c68fc79d1..d76a0a073f 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/init.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/init.pp
@@ -28,11 +28,12 @@ class hdp-oozie(
 
 # Configs generation  
 
-  if has_key($configuration, 'hdp_oozie__oozie_site') {
-    configgenerator::configfile{'oozie_site_xml': 
+  if has_key($configuration, 'oozie_site') {
+    configgenerator::configfile{'oozie_site':
+      modulespath => $hdp-oozie::params::conf_dir, 
       filename => 'oozie-site.xml',
       module => 'hdp-oozie',
-      configuration => $configuration['hdp_oozie__oozie_site']
+      configuration => $configuration['oozie_site']
     }
   }
 
@@ -72,7 +73,7 @@ class hdp-oozie(
       force => true
     }
 
-     hdp-oozie::configfile { ['oozie-site.xml','oozie-env.sh','oozie-log4j.properties']: }
+     hdp-oozie::configfile { ['oozie-env.sh','oozie-log4j.properties']: }
 
     anchor { 'hdp-oozie::begin': } -> Hdp::Package['oozie-client'] -> Hdp::User[$oozie_user] -> Hdp::Directory[$oozie_config_dir] -> Hdp-oozie::Configfile<||> -> anchor { 'hdp-oozie::end': }
 
diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
index 887939cf6a..7dcf644395 100644
--- a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
@@ -327,7 +327,7 @@ class hdp::params()
     $hadoop_bin = "/usr/lib/hadoop/bin"
     $hadoop_conf_dir = "/etc/hadoop/conf"
     $zk_conf_dir = "/etc/zookeeper/conf"
-    $hbase_conf_dir = "/etc/hbase/conf/"
+    $hbase_conf_dir = "/etc/hbase/conf"
     $sqoop_conf_dir = "/usr/lib/sqoop/conf"
     $pig_conf_dir = "/etc/pig/conf"
     $oozie_conf_dir = "/etc/oozie/conf"
diff --git a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
index 87af868b46..cdb18b0eea 100644
--- a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
+++ b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
@@ -41,6 +41,7 @@ def generateManifest(parsedJson, fileName, modulesdir):
   clusterHostInfo = parsedJson['clusterHostInfo']

   params = parsedJson['params']

   configurations = parsedJson['configurations']

+  xmlConfigurationsKeys = parsedJson['xml_configurations_keys']

   #hostAttributes = parsedJson['hostAttributes']

   roles = parsedJson['roleCommands']

   

@@ -56,8 +57,19 @@ def generateManifest(parsedJson, fileName, modulesdir):
   #writing params from map

   writeParams(manifest, params)

   

+  

+  xmlConfigurations = {}

+  flatConfigurations = {}

+

+  for configKey in configurations.iterkeys():

+    if configKey in xmlConfigurationsKeys:

+      xmlConfigurations[configKey] = configurations[configKey]

+    else:

+      flatConfigurations[configKey] = configurations[configKey]

+      

   #writing config maps

-  writeConfigurations(manifest, configurations)

+  writeXmlConfigurations(manifest, xmlConfigurations)

+  writeFlatConfigurations(manifest, flatConfigurations)

 

   #writing host attributes

   #writeHostAttributes(manifest, hostAttributes)

@@ -92,23 +104,23 @@ def writeNodes(outputFile, clusterHostInfo):
     outputFile.write(']\n')

 

 #write params

-def writeParams(outputFile, params):
-
+def writeParams(outputFile, params):

+

   for paramName in params.iterkeys():

-
-    param = params[paramName]
-    if type(param) is dict:
-
+

+    param = params[paramName]

+    if type(param) is dict:

+

       outputFile.write('$' + paramName + '= {\n')

 

-      coma = ''
+      coma = ''

 

       for subParam in param.iterkeys():

         outputFile.write(coma + '"' + subParam + '" => "' + param[subParam] + '"')

-        coma = ',\n'
-
-      outputFile.write('\n}\n')
-    else:
+        coma = ',\n'

+

+      outputFile.write('\n}\n')

+    else:

       outputFile.write('$' +  paramName + '="' + param + '"\n')

     

 

@@ -123,14 +135,21 @@ def writeHostAttributes(outputFile, hostAttributes):
 

   outputFile.write('}\n')

 

-#write configurations

-def writeConfigurations(outputFile, configs):

+#write flat configurations

+def writeFlatConfigurations(outputFile, flatConfigs):

+  for flatConfigName in flatConfigs.iterkeys():

+    for flatConfig in flatConfigs[flatConfigName].iterkeys():

+      outputFile.write('$' + flatConfig + ' = "' + flatConfigs[flatConfigName][flatConfig] + '"' + os.linesep)

+

+#write xml configurations

+def writeXmlConfigurations(outputFile, xmlConfigs):

   outputFile.write('$configuration =  {\n')

 

-  for configName in configs.iterkeys():

-    outputFile.write(configName + '=> {\n')

-    config = configs[configName]

+  for configName in xmlConfigs.iterkeys():

+

+    config = xmlConfigs[configName]

     

+    outputFile.write(configName + '=> {\n')

     coma = ''

     for configParam in config.iterkeys():

       outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')

@@ -192,7 +211,7 @@ def writeStages(outputFile, numStages):
   outputFile.write('\n')

     

 

-def main():
+def main():

   logging.basicConfig(level=logging.DEBUG)    

   #test code

   jsonFile = open('test.json', 'r')

@@ -200,9 +219,10 @@ def main():
   modulesdir = os.path.abspath(os.getcwd() + ".." + os.sep + ".." + 

                                os.sep + ".." + os.sep + "puppet" + 

                                os.sep + "modules" + os.sep)

+  inputJsonStr = jsonStr

   parsedJson = json.loads(inputJsonStr)

   generateManifest(parsedJson, 'site.pp', modulesdir)

 

 if __name__ == '__main__':

-  main()
-
+  main()

+

diff --git a/ambari-agent/src/main/python/ambari_agent/site.pp b/ambari-agent/src/main/python/ambari_agent/site.pp
index bcd3dd9287..ba41b9338c 100644
--- a/ambari-agent/src/main/python/ambari_agent/site.pp
+++ b/ambari-agent/src/main/python/ambari_agent/site.pp
@@ -25,8 +25,12 @@ capacity_scheduler=> {
 "mapred.capacity-scheduler.queue.default.capacity" => "100",
 "mapred.capacity-scheduler.queue.default.supports-priorit" => "false"
 },
-core_site=> {
-"fs.default.name" => "hrt8n36.cc1.ygridcore.net"
+hdfs_site=> {
+"dfs.block.size" => "256000000",
+"dfs.replication" => "1"
+},
+hbase_policy=> {
+"security.client.protocol.acl" => "*"
 },
 hadoop_policy=> {
 "security.client.datanode.protocol.acl" => "*",
@@ -36,9 +40,17 @@ mapred_queue_acls=> {
 "mapred.queue.default.acl-submit-job" => "*",
 "mapred.queue.default.acl-administer-jobs" => "*"
 },
-hdfs_site=> {
-"dfs.block.size" => "256000000",
-"dfs.replication" => "1"
+hbase_site=> {
+"hbase.cluster.distributed" => "true"
+},
+core_site=> {
+"fs.default.name" => "hrt8n36.cc1.ygridcore.net"
+},
+hive_site=> {
+"hive.exec.scratchdir" => "/tmp"
+},
+oozie_site=> {
+"oozie.service.ActionService.executor.ext.classes" => "org.apache.oozie.action.hadoop.HiveActionExecutor, org.apache.oozie.action.hadoop.SqoopActionExecutor,org.apache.oozie.action.email.EmailActionExecutor,"
 },
 mapred_site=> {
 "mapred.queue.names" => "hive,pig,default",
@@ -46,8 +58,16 @@ mapred_site=> {
 },
 
 }
+$security_enabled = "true"
+$task_bin_exe = "ls"
+$hadoop_piddirprefix = "/tmp"
+$ganglia_server_host = "localhost"
 node /default/ {
- stage{1 :} -> stage{2 :}
-class {'hdp-hadoop::namenode': stage => 1, service_state => running}
+ stage{1 :} -> stage{2 :} -> stage{3 :} -> stage{4 :} -> stage{5 :} -> stage{6 :}
+class {'hdp-hadoop::namenode': stage => 1, service_state => installed_and_configured}
 class {'hdp-hadoop::datanode': stage => 2, service_state => installed_and_configured}
+class {'hdp-hbase::master': stage => 3, service_state => installed_and_configured}
+class {'hdp-hive::server': stage => 4, service_state => installed_and_configured}
+class {'hdp-hive::client': stage => 5, service_state => installed_and_configured}
+class {'hdp-oozie::server': stage => 6, service_state => installed_and_configured}
 }
diff --git a/ambari-agent/src/main/python/ambari_agent/test.json b/ambari-agent/src/main/python/ambari_agent/test.json
index 24e16db43e..c968dc0ff1 100644
--- a/ambari-agent/src/main/python/ambari_agent/test.json
+++ b/ambari-agent/src/main/python/ambari_agent/test.json
@@ -16,6 +16,11 @@
 "java64_home" : "/usr/jdk64/jdk1.6.0_31",
 "jdk_bins" :  { "32" : "jdk-6u31-linux-x64.bin", "64" : "jdk-6u31-linux-x64.bin" }
 },
+
+"xml_configurations_keys" : ["hdfs_site", "hdfs_site", "core_site", "mapred_queue_acls", 
+                             "hadoop_policy", "mapred_site", "capacity_scheduler", "hbase_site",
+                             "hbase_policy", "hive_site", "oozie_site"],
+
 "configurations" : {
 "hdfs_site" : { "dfs.block.size" : "256000000", "dfs.replication" : "1" } ,
 "core_site" :  { "fs.default.name" : "hrt8n36.cc1.ygridcore.net" } ,
@@ -26,19 +31,47 @@
 "mapred_site" : {"mapred.jobtracker.taskScheduler" : "org.apache.hadoop.mapred.CapacityTaskScheduler",
 		 "mapred.queue.names" : "hive,pig,default"},
 "capacity_scheduler" : {"mapred.capacity-scheduler.queue.default.capacity" : "100",
-			"mapred.capacity-scheduler.queue.default.supports-priorit" : "false"}
+			"mapred.capacity-scheduler.queue.default.supports-priorit" : "false"},
+"health_check" : {"security_enabled" : "true",
+                  "task_bin_exe" : "ls"},
+"hadoop_env" : {"hadoop_piddirprefix" : "/tmp"},
+
+"hbase_site" : {"hbase.cluster.distributed" : "true"},
+"hbase_policy" : {"security.client.protocol.acl" : "*"},
+"hadoop_metrics" : {"ganglia_server_host" : "localhost"},
+"hive_site" : {"hive.exec.scratchdir" : "/tmp"},
+"oozie_site" : {"oozie.service.ActionService.executor.ext.classes" : "org.apache.oozie.action.hadoop.HiveActionExecutor, org.apache.oozie.action.hadoop.SqoopActionExecutor,org.apache.oozie.action.email.EmailActionExecutor,"}
 },
 "roleCommands": [
 {
-"role" : "NAMENODE", 
-"cmd": "START",
-"roleParams" : {
-}
+  "role": "NAMENODE",
+  "cmd": "INSTALL",
+  "roleParams" : {}
+},
+{
+  "role": "DATANODE",
+  "cmd": "INSTALL",
+  "roleParams" : {}
+},
+{
+  "role": "HBASE_MASTER",
+  "cmd": "INSTALL",
+  "roleParams" : {}
+},
+{
+  "role": "HIVE_SERVER",
+  "cmd": "INSTALL",
+  "roleParams" : {}
+},
+{
+  "role": "HIVE_CLIENT",
+  "cmd": "INSTALL",
+  "roleParams" : {}
 },
 {
-"role": "DATANODE",
-"cmd": "INSTALL",
-"roleParams" : {}
+  "role": "OOZIE_SERVER",
+  "cmd": "INSTALL",
+  "roleParams" : {}
 }
 ]
 }