AMBARI-828. Manifest generation for various actions from the server. (mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/AMBARI-666@1395833 13f79535-47bb-0310-9956-ffa450edef68

4 files changed, 220 insertions, 98 deletions

diff --git a/ambari-agent/src/main/python/manifestGenerator/manifestGenerator.py b/ambari-agent/src/main/python/manifestGenerator/manifestGenerator.py
index 5442d6f9ac..2e67bd4fb6 100644
--- a/ambari-agent/src/main/python/manifestGenerator/manifestGenerator.py
+++ b/ambari-agent/src/main/python/manifestGenerator/manifestGenerator.py
@@ -1,98 +1,157 @@
-import json
-
-def generateManifest(inputJsonStr):
-#reading json
-  parsedJson = json.loads(inputJsonStr)
-  hostname = parsedJson['hostname']
-  clusterHostInfo = parsedJson['clusterHostInfo']
-  params = parsedJson['params']
-  configurations = parsedJson['configurations']
-  hostAttributes = parsedJson['hostAttributes']
-  roles = parsedJson['roles']
-  
-#writing manifest
-  manifest = open('site.pp', 'w')
-
-  #writing imports from external static file
-  writeImports(manifest)
-  
-  #writing nodes
-  writeNodes(manifest, clusterHostInfo)
-  
-  #writing params from map
-  writeParams(manifest, params)
-  
-  #writing config maps
-  writeConfigurations(manifest, configurations)
-
-  #cycle here - writing host attributes
-  writeHostAttributes(manifest, hostAttributes)
-
-  #writing task definitions 
-  writeTasks(manifest, roles)
-     
-  manifest.close()
-    
-  
-  #read static imports from file and write them to manifest
-  def writeImports(outputFile, inputFileName='imports.txt'):
-    inputFile = open(inputFileName, 'r')
-    
-    for line in inputFile:
-      outputFile.write(line)
-      
-    inputFile.close()
-
-  #write nodes
-  def writeNodes(outputFile, clusterHostInfo):
-    for node in clusterHostInfo.iterkeys():
-      outputFile.write('$' + node + '= ['
-    
-      coma = ''
-      for host in node:
-        outputFile.write(coma + '\'' + host + '\'')
-        coma = ', '
-
-      outputFile.write(']\n'
-
-  #write params
-  def writeParams(outputFile, params):
-    for param in params.iterkeys():
-      outputFile.write('$' +  param + '="' + params[param] + '"\n')
-
-  #write host attributes
-  def writeHostAttributes(outputFile, hostAttributes):
-    outputFile.write('$hostAttributes={\n')
-  
-    coma = ''
-    for attribute in hostAttributes.iterkeys():
-      outputFile.write(coma + '"' +  attribute + '" => "{' + hostAttributes[attribute] + '"}')
-      coma = ',\n'
-
-    outputFile.write('}\n')
-
-  #write configurations
-  def writeConfigurations(outputFile, configs):
-    outputFile.write('$configuration =  {\n'
-  
-    for configName in configs.iterkeys():
-      outputFile.write('$' + configName + '=> {\n')
-      config = configs[configName]
-      
-      coma = ''
-      for configParam in config.iterkeys():
-        outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')
-        coma = ',\n'
-
-      outputFile.write('\n}\n')
-      
-    outputFile.write('\n}\n'
-
-  #write node tasks
-  def writeTasks(outputFile, tasks):
-    for task in tasks :
-      nodename = task['role']
-      command = task['roleCommand']
-      taskParams = task['params']
-    #TODO: write node task to file
-      
\ No newline at end of file
+import json

+

+  #read static imports from file and write them to manifest

+def writeImports(outputFile, inputFileName='imports.txt'):

+  inputFile = open(inputFileName, 'r')

+  

+  for line in inputFile:

+    outputFile.write(line)

+    

+  inputFile.close()

+

+def generateManifest(inputJsonStr):

+#reading json

+  parsedJson = json.loads(inputJsonStr)

+  hostname = parsedJson['hostname']

+  clusterHostInfo = parsedJson['clusterHostInfo']

+  params = parsedJson['params']

+  configurations = parsedJson['configurations']

+  #hostAttributes = parsedJson['hostAttributes']

+  roles = parsedJson['roles']

+  

+#writing manifest

+  manifest = open('site.pp', 'w')

+

+  #writing imports from external static file

+  writeImports(manifest)

+  

+  #writing nodes

+  writeNodes(manifest, clusterHostInfo)

+  

+  #writing params from map

+  writeParams(manifest, params)

+  

+  #writing config maps

+  writeConfigurations(manifest, configurations)

+

+  #writing host attributes

+  #writeHostAttributes(manifest, hostAttributes)

+

+  #writing task definitions 

+  writeTasks(manifest, roles)

+     

+  manifest.close()

+    

+  

+  #read dictionary

+def readDict(file, separator='='):

+  result = dict()

+  

+  for line in file :

+    dictTuple = line.partition(separator)

+    result[dictTuple[0].strip()] = dictTuple[2].strip()

+  

+  return result

+  

+

+  #write nodes

+def writeNodes(outputFile, clusterHostInfo):

+  for node in clusterHostInfo.iterkeys():

+    outputFile.write('$' + node + '= [')

+    coma = ''

+    

+    for host in node:

+      outputFile.write(coma + '\'' + host + '\'')

+      coma = ', '

+

+    outputFile.write(']\n')

+

+#write params

+def writeParams(outputFile, params):

+  for param in params.iterkeys():

+    outputFile.write('$' +  param + '="' + params[param] + '"\n')

+

+#write host attributes

+def writeHostAttributes(outputFile, hostAttributes):

+  outputFile.write('$hostAttributes={\n')

+

+  coma = ''

+  for attribute in hostAttributes.iterkeys():

+    outputFile.write(coma + '"' +  attribute + '" => "{' + hostAttributes[attribute] + '"}')

+    coma = ',\n'

+

+  outputFile.write('}\n')

+

+#write configurations

+def writeConfigurations(outputFile, configs):

+  outputFile.write('$configuration =  {\n')

+

+  for configName in configs.iterkeys():

+    outputFile.write('$' + configName + '=> {\n')

+    config = configs[configName]

+    

+    coma = ''

+    for configParam in config.iterkeys():

+      outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')

+      coma = ',\n'

+

+    outputFile.write('\n}\n')

+    

+  outputFile.write('\n}\n')

+

+#write node tasks

+def writeTasks(outputFile, roles):

+  #reading dictionaries

+  rolesToClassFile = open('rolesToClass.dict', 'r')

+  rolesToClass = readDict(rolesToClassFile)

+  rolesToClassFile.close()

+

+  serviceStatesFile =  open('serviceStates.dict', 'r')

+  serviceStates = readDict(serviceStatesFile)

+  serviceStatesFile.close()

+

+  outputFile.write('node /<toBeDefined>/ {\n ')

+  writeStages(outputFile, len(roles))

+  stageNum = 1

+

+  for role in roles :

+    rolename = role['role']

+    command = role['roleCommand']

+    taskParams = role['params']

+    taskParamsNormalized = normalizeTaskParams(taskParams)

+    taskParamsPostfix = ''

+    

+    if len(taskParamsNormalized) > 0 :

+      taskParamsPostfix = ', ' + taskParamsNormalized

+    

+    className = rolesToClass[rolename]

+    serviceState = serviceStates[command]

+    

+    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) + ', service_state => ' + serviceState + taskParamsPostfix + '}\n')

+    stageNum = stageNum + 1

+    

+def normalizeTaskParams(taskParams):

+  result = ''

+  coma = ''

+  

+  for paramName in taskParams.iterkeys():

+    result = coma + result + paramName + ' => ' + taskParams[paramName]

+    coma = ','

+    

+  return result

+  

+def writeStages(outputFile, numStages):

+  arrow = ''

+  

+  for i in range(numStages):

+    outputFile.write(arrow + 'stage{' + str(i) + ' :}')

+    arrow = ' -> '

+  

+  outputFile.write('\n')

+    

+    

+#test code

+jsonFile = open('test.json', 'r')

+jsonStr = jsonFile.read() 

+print '>>>JSON STRING: ' + jsonStr

+generateManifest(jsonStr)

diff --git a/ambari-agent/src/main/python/manifestGenerator/rolesToClass.dict b/ambari-agent/src/main/python/manifestGenerator/rolesToClass.dict
new file mode 100644
index 0000000000..13949ace85
--- /dev/null
+++ b/ambari-agent/src/main/python/manifestGenerator/rolesToClass.dict
@@ -0,0 +1,28 @@
+NAMENODE = hdp-hadoop::namenode
+DATANODE = hdp-hadoop::datanode
+SNAMENODE = hdp-hadoop::snamenode
+JOBTRACKER = hdp-hadoop::jobtracker
+TASKTRACKER = hdp-hadoop::tasktracker
+HDFS_CLIENT = hdp-hadoop::client
+MAPREDUCE_CLIENT = hdp-hadoop::client
+ZOOKEEPER_SERVER = hdp-zookeeper
+ZOOKEEPER_CLIENT = hdp-zookeeper::client
+HBASE_MASTER = hdp-hbase::master
+HBASE_REGIONSERVER = hdp-hbase::regionserver
+HBASE_CLIENT = hdp-hbase::client
+PIG_CLIENT = hdp-pig
+SQOOP_CLIENT = hdp-sqoop
+OOZIE_SERVER = hdp-oozie::server
+OOZIE_CLIENT = hdp-oozie::client
+HIVE_CLIENT = hdp-hive::client
+HCATALOG_CLIENT = hdp-hcat
+HCATALOG_SERVER = hdp-hcat::server
+HIVE_SERVER = hdp-hive::server
+HIVE_MYSQL = hdp-mysql::server
+TEMPLETON_SERVER = hdp-templeton::server
+TEMPLETON_CLIENT = hdp-templeton::client
+DASHBOARD = hdp-dashboard
+NAGIOS_SERVER = hdp-nagios::server
+GANGLIA_MONITOR_SERVER = hdp-ganglia::server
+GANGLIA_MONITOR = hdp-ganglia::monitor
+HTTPD = hdp-monitor-webserver
\ No newline at end of file
diff --git a/ambari-agent/src/main/python/manifestGenerator/serviceStates.dict b/ambari-agent/src/main/python/manifestGenerator/serviceStates.dict
new file mode 100644
index 0000000000..a4e4236495
--- /dev/null
+++ b/ambari-agent/src/main/python/manifestGenerator/serviceStates.dict
@@ -0,0 +1,2 @@
+START = running
+INSTALL = installed_and_configured
\ No newline at end of file
diff --git a/ambari-agent/src/main/python/manifestGenerator/test.json b/ambari-agent/src/main/python/manifestGenerator/test.json
new file mode 100644
index 0000000000..c221797ffa
--- /dev/null
+++ b/ambari-agent/src/main/python/manifestGenerator/test.json
@@ -0,0 +1,33 @@
+{
+"commandId": "234",
+"hostname":  "h1.hortonworks.com",
+
+"clusterHostInfo" : 
+{
+"NAMENODE": ["h2.hortonworks.com"],
+"DATANODE": ["h1.hortonworks.com", "h2.hortonworks.com"]
+},
+"params": 
+{
+"hdfs_user" : "hdfs",
+"jdk_location": "lah/blah"
+},
+"configurations" : {
+"hdfs_site" : { "dfs.block.size" : "256000000", "dfs.replication" : "1" } ,
+"core_site" :  { "fs.default.name" : "hrt8n36.cc1.ygridcore.net" }
+},
+"roles": [
+{
+"role" : "DATANODE", 
+"roleCommand": "START",
+"params" : {
+"myid" : "23"
+}
+},
+{
+"role": "NAMENODE",
+"roleCommand": "INSTALL",
+"params" : {}
+}
+]
+}