AMBARI-868. Clean up site.pp generation on the agent and remove the imports in the sample site.pp. (mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/AMBARI-666@1398674 13f79535-47bb-0310-9956-ffa450edef68

2 files changed, 188 insertions, 200 deletions

diff --git a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
index 1007a36547..0464834a78 100644
--- a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
+++ b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
@@ -1,184 +1,184 @@
-#!/usr/bin/env python2.6

-

-'''

-Licensed to the Apache Software Foundation (ASF) under one

-or more contributor license agreements.  See the NOTICE file

-distributed with this work for additional information

-regarding copyright ownership.  The ASF licenses this file

-to you under the Apache License, Version 2.0 (the

-"License"); you may not use this file except in compliance

-with the License.  You may obtain a copy of the License at

-

-    http://www.apache.org/licenses/LICENSE-2.0

-

-Unless required by applicable law or agreed to in writing, software

-distributed under the License is distributed on an "AS IS" BASIS,

-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-See the License for the specific language governing permissions and

-limitations under the License.

-'''

-

-import json

-import os.path

-import logging

-

-logger = logging.getLogger()

-

-  #read static imports from file and write them to manifest

-def writeImports(outputFile, inputFileName='imports.txt'):

-  inputFile = open(inputFileName, 'r')

-  modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")

-  logger.info("Modules dir is " + modulesdir)

-  for line in inputFile:

-    modulename = line.rstrip('\n')

-    line = "import " + "\"" + modulesdir + "/" + modulename + "\"\n"

-    outputFile.write(line)

-    

-  inputFile.close()

-

-def generateManifest(inputJsonStr):

-#reading json

-  parsedJson = json.loads(inputJsonStr)

-  hostname = parsedJson['hostname']

-  clusterHostInfo = parsedJson['clusterHostInfo']

-  params = parsedJson['params']

-  configurations = parsedJson['configurations']

-  #hostAttributes = parsedJson['hostAttributes']

-  roles = parsedJson['roleCommands']

-  

-#writing manifest

-  manifest = open('site.pp', 'w')

-

-  #writing imports from external static file

-  writeImports(manifest)

-  

-  #writing nodes

-  writeNodes(manifest, clusterHostInfo)

-  

-  #writing params from map

-  writeParams(manifest, params)

-  

-  #writing config maps

-  writeConfigurations(manifest, configurations)

-

-  #writing host attributes

-  #writeHostAttributes(manifest, hostAttributes)

-

-  #writing task definitions 

-  writeTasks(manifest, roles)

-     

-  manifest.close()

-    

-  

-  #read dictionary

-def readDict(file, separator='='):

-  result = dict()

-  

-  for line in file :

-    dictTuple = line.partition(separator)

-    result[dictTuple[0].strip()] = dictTuple[2].strip()

-  

-  return result

-  

-

-  #write nodes

-def writeNodes(outputFile, clusterHostInfo):

-  for node in clusterHostInfo.iterkeys():

-    outputFile.write('$' + node + '= [')

-    coma = ''

-    

-    for value in clusterHostInfo[node]:

-      outputFile.write(coma + '\'' + value + '\'')

-      coma = ', '

-

-    outputFile.write(']\n')

-

-#write params

-def writeParams(outputFile, params):

-  for param in params.iterkeys():

-    outputFile.write('$' +  param + '="' + params[param] + '"\n')

-

-#write host attributes

-def writeHostAttributes(outputFile, hostAttributes):

-  outputFile.write('$hostAttributes={\n')

-

-  coma = ''

-  for attribute in hostAttributes.iterkeys():

-    outputFile.write(coma + '"' +  attribute + '" => "{' + hostAttributes[attribute] + '"}')

-    coma = ',\n'

-

-  outputFile.write('}\n')

-

-#write configurations

-def writeConfigurations(outputFile, configs):

-  outputFile.write('$configuration =  {\n')

-

-  for configName in configs.iterkeys():

-    outputFile.write(configName + '=> {\n')

-    config = configs[configName]

-    

-    coma = ''

-    for configParam in config.iterkeys():

-      outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')

-      coma = ',\n'

-

-    outputFile.write('\n},\n')

-    

-  outputFile.write('\n}\n')

-

-#write node tasks

-def writeTasks(outputFile, roles):

-  #reading dictionaries

-  rolesToClassFile = open('rolesToClass.dict', 'r')

-  rolesToClass = readDict(rolesToClassFile)

-  rolesToClassFile.close()

-

-  serviceStatesFile =  open('serviceStates.dict', 'r')

-  serviceStates = readDict(serviceStatesFile)

-  serviceStatesFile.close()

-

-  outputFile.write('node /default/ {\n ')

-  writeStages(outputFile, len(roles))

-  stageNum = 1

-

-  for role in roles :

-    rolename = role['role']

-    command = role['cmd']

-    taskParams = role['roleParams']

-    taskParamsNormalized = normalizeTaskParams(taskParams)

-    taskParamsPostfix = ''

-    

-    if len(taskParamsNormalized) > 0 :

-      taskParamsPostfix = ', ' + taskParamsNormalized

-    

-    className = rolesToClass[rolename]

-    serviceState = serviceStates[command]

-    

-    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) + 

-                     ', service_state => ' + serviceState + taskParamsPostfix + '}\n')

-    stageNum = stageNum + 1

-  outputFile.write('}\n')

-def normalizeTaskParams(taskParams):

-  result = ''

-  coma = ''

-  

-  for paramName in taskParams.iterkeys():

-    result = coma + result + paramName + ' => ' + taskParams[paramName]

-    coma = ','

-    

-  return result

-  

-def writeStages(outputFile, numStages):

-  arrow = ''

-  

-  for i in range(numStages):

-    outputFile.write(arrow + 'stage{' + str(i) + ' :}')

-    arrow = ' -> '

-  

-  outputFile.write('\n')

-    

-logging.basicConfig(level=logging.DEBUG)    

-#test code

-jsonFile = open('test.json', 'r')

-jsonStr = jsonFile.read() 

-generateManifest(jsonStr)

+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import json
+import os.path
+import logging
+
+logger = logging.getLogger()
+
+  #read static imports from file and write them to manifest
+def writeImports(outputFile, inputFileName='imports.txt'):
+  inputFile = open(inputFileName, 'r')
+  modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")
+  logger.info("Modules dir is " + modulesdir)
+  for line in inputFile:
+    modulename = line.rstrip('\n')
+    line = "import '" + modulesdir + "/" + modulename + "'\n"
+    outputFile.write(line)
+    
+  inputFile.close()
+
+def generateManifest(inputJsonStr):
+#reading json
+  parsedJson = json.loads(inputJsonStr)
+  hostname = parsedJson['hostname']
+  clusterHostInfo = parsedJson['clusterHostInfo']
+  params = parsedJson['params']
+  configurations = parsedJson['configurations']
+  #hostAttributes = parsedJson['hostAttributes']
+  roles = parsedJson['roleCommands']
+  
+#writing manifest
+  manifest = open('site.pp', 'w')
+
+  #writing imports from external static file
+  writeImports(manifest)
+  
+  #writing nodes
+  writeNodes(manifest, clusterHostInfo)
+  
+  #writing params from map
+  writeParams(manifest, params)
+  
+  #writing config maps
+  writeConfigurations(manifest, configurations)
+
+  #writing host attributes
+  #writeHostAttributes(manifest, hostAttributes)
+
+  #writing task definitions 
+  writeTasks(manifest, roles)
+     
+  manifest.close()
+    
+  
+  #read dictionary
+def readDict(file, separator='='):
+  result = dict()
+  
+  for line in file :
+    dictTuple = line.partition(separator)
+    result[dictTuple[0].strip()] = dictTuple[2].strip()
+  
+  return result
+  
+
+  #write nodes
+def writeNodes(outputFile, clusterHostInfo):
+  for node in clusterHostInfo.iterkeys():
+    outputFile.write('$' + node + '= [')
+    coma = ''
+    
+    for value in clusterHostInfo[node]:
+      outputFile.write(coma + '\'' + value + '\'')
+      coma = ', '
+
+    outputFile.write(']\n')
+
+#write params
+def writeParams(outputFile, params):
+  for param in params.iterkeys():
+    outputFile.write('$' +  param + '="' + params[param] + '"\n')
+
+#write host attributes
+def writeHostAttributes(outputFile, hostAttributes):
+  outputFile.write('$hostAttributes={\n')
+
+  coma = ''
+  for attribute in hostAttributes.iterkeys():
+    outputFile.write(coma + '"' +  attribute + '" => "{' + hostAttributes[attribute] + '"}')
+    coma = ',\n'
+
+  outputFile.write('}\n')
+
+#write configurations
+def writeConfigurations(outputFile, configs):
+  outputFile.write('$configuration =  {\n')
+
+  for configName in configs.iterkeys():
+    outputFile.write(configName + '=> {\n')
+    config = configs[configName]
+    
+    coma = ''
+    for configParam in config.iterkeys():
+      outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')
+      coma = ',\n'
+
+    outputFile.write('\n},\n')
+    
+  outputFile.write('\n}\n')
+
+#write node tasks
+def writeTasks(outputFile, roles):
+  #reading dictionaries
+  rolesToClassFile = open('rolesToClass.dict', 'r')
+  rolesToClass = readDict(rolesToClassFile)
+  rolesToClassFile.close()
+
+  serviceStatesFile =  open('serviceStates.dict', 'r')
+  serviceStates = readDict(serviceStatesFile)
+  serviceStatesFile.close()
+
+  outputFile.write('node /default/ {\n ')
+  writeStages(outputFile, len(roles))
+  stageNum = 1
+
+  for role in roles :
+    rolename = role['role']
+    command = role['cmd']
+    taskParams = role['roleParams']
+    taskParamsNormalized = normalizeTaskParams(taskParams)
+    taskParamsPostfix = ''
+    
+    if len(taskParamsNormalized) > 0 :
+      taskParamsPostfix = ', ' + taskParamsNormalized
+    
+    className = rolesToClass[rolename]
+    serviceState = serviceStates[command]
+    
+    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) + 
+                     ', service_state => ' + serviceState + taskParamsPostfix + '}\n')
+    stageNum = stageNum + 1
+  outputFile.write('}\n')
+def normalizeTaskParams(taskParams):
+  result = ''
+  coma = ''
+  
+  for paramName in taskParams.iterkeys():
+    result = coma + result + paramName + ' => ' + taskParams[paramName]
+    coma = ','
+    
+  return result
+  
+def writeStages(outputFile, numStages):
+  arrow = ''
+  
+  for i in range(numStages):
+    outputFile.write(arrow + 'stage{' + str(i) + ' :}')
+    arrow = ' -> '
+  
+  outputFile.write('\n')
+    
+logging.basicConfig(level=logging.DEBUG)    
+#test code
+jsonFile = open('test.json', 'r')
+jsonStr = jsonFile.read() 
+generateManifest(jsonStr)
diff --git a/ambari-agent/src/main/python/ambari_agent/site.pp b/ambari-agent/src/main/python/ambari_agent/site.pp
index 3a96e708d4..4fe44c6311 100644
--- a/ambari-agent/src/main/python/ambari_agent/site.pp
+++ b/ambari-agent/src/main/python/ambari_agent/site.pp
@@ -1,27 +1,15 @@
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hadoop/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hbase/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-zookeeper/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-oozie/manifests/*.pp""
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-pig/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-sqoop/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-templeton/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hive/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hcat/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-mysql/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-monitor-webserver/manifests/*.pp"
 $NAMENODE= ['h2.hortonworks.com']
 $DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
 $hdfs_user="hdfs"
 $jdk_location="lah/blah"
 $configuration =  {
-$hdfs_site=> {
+hdfs_site=> {
 "dfs.block.size" => "256000000",
 "dfs.replication" => "1"
-}
-$core_site=> {
+},
+core_site=> {
 "fs.default.name" => "hrt8n36.cc1.ygridcore.net"
-}
+},
 
 }
 node /default/ {