AMBARI-831. Move manifest generation into the ambari agent directory. (mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/AMBARI-666@1395859 13f79535-47bb-0310-9956-ffa450edef68

7 files changed, 88 insertions, 31 deletions

diff --git a/ambari-agent/src/main/python/ambari_agent/imports.txt b/ambari-agent/src/main/python/ambari_agent/imports.txt
new file mode 100644
index 0000000000..ea49b991a2
--- /dev/null
+++ b/ambari-agent/src/main/python/ambari_agent/imports.txt
@@ -0,0 +1,12 @@
+hdp/manifests/*.pp
+hdp-hadoop/manifests/*.pp
+hdp-hbase/manifests/*.pp
+hdp-zookeeper/manifests/*.pp
+hdp-oozie/manifests/*.pp
+hdp-pig/manifests/*.pp
+hdp-sqoop/manifests/*.pp
+hdp-templeton/manifests/*.pp
+hdp-hive/manifests/*.pp
+hdp-hcat/manifests/*.pp
+hdp-mysql/manifests/*.pp
+hdp-monitor-webserver/manifests/*.pp
diff --git a/ambari-agent/src/main/python/manifestGenerator/manifestGenerator.py b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
index 2e67bd4fb6..1007a36547 100644
--- a/ambari-agent/src/main/python/manifestGenerator/manifestGenerator.py
+++ b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
@@ -1,10 +1,37 @@
+#!/usr/bin/env python2.6

+

+'''

+Licensed to the Apache Software Foundation (ASF) under one

+or more contributor license agreements.  See the NOTICE file

+distributed with this work for additional information

+regarding copyright ownership.  The ASF licenses this file

+to you under the Apache License, Version 2.0 (the

+"License"); you may not use this file except in compliance

+with the License.  You may obtain a copy of the License at

+

+    http://www.apache.org/licenses/LICENSE-2.0

+

+Unless required by applicable law or agreed to in writing, software

+distributed under the License is distributed on an "AS IS" BASIS,

+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+See the License for the specific language governing permissions and

+limitations under the License.

+'''

+

 import json

+import os.path

+import logging

+

+logger = logging.getLogger()

 

   #read static imports from file and write them to manifest

 def writeImports(outputFile, inputFileName='imports.txt'):

   inputFile = open(inputFileName, 'r')

-  

+  modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")

+  logger.info("Modules dir is " + modulesdir)

   for line in inputFile:

+    modulename = line.rstrip('\n')

+    line = "import " + "\"" + modulesdir + "/" + modulename + "\"\n"

     outputFile.write(line)

     

   inputFile.close()

@@ -17,7 +44,7 @@ def generateManifest(inputJsonStr):
   params = parsedJson['params']

   configurations = parsedJson['configurations']

   #hostAttributes = parsedJson['hostAttributes']

-  roles = parsedJson['roles']

+  roles = parsedJson['roleCommands']

   

 #writing manifest

   manifest = open('site.pp', 'w')

@@ -60,8 +87,8 @@ def writeNodes(outputFile, clusterHostInfo):
     outputFile.write('$' + node + '= [')

     coma = ''

     

-    for host in node:

-      outputFile.write(coma + '\'' + host + '\'')

+    for value in clusterHostInfo[node]:

+      outputFile.write(coma + '\'' + value + '\'')

       coma = ', '

 

     outputFile.write(']\n')

@@ -87,7 +114,7 @@ def writeConfigurations(outputFile, configs):
   outputFile.write('$configuration =  {\n')

 

   for configName in configs.iterkeys():

-    outputFile.write('$' + configName + '=> {\n')

+    outputFile.write(configName + '=> {\n')

     config = configs[configName]

     

     coma = ''

@@ -95,7 +122,7 @@ def writeConfigurations(outputFile, configs):
       outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')

       coma = ',\n'

 

-    outputFile.write('\n}\n')

+    outputFile.write('\n},\n')

     

   outputFile.write('\n}\n')

 

@@ -110,14 +137,14 @@ def writeTasks(outputFile, roles):
   serviceStates = readDict(serviceStatesFile)

   serviceStatesFile.close()

 

-  outputFile.write('node /<toBeDefined>/ {\n ')

+  outputFile.write('node /default/ {\n ')

   writeStages(outputFile, len(roles))

   stageNum = 1

 

   for role in roles :

     rolename = role['role']

-    command = role['roleCommand']

-    taskParams = role['params']

+    command = role['cmd']

+    taskParams = role['roleParams']

     taskParamsNormalized = normalizeTaskParams(taskParams)

     taskParamsPostfix = ''

     

@@ -127,9 +154,10 @@ def writeTasks(outputFile, roles):
     className = rolesToClass[rolename]

     serviceState = serviceStates[command]

     

-    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) + ', service_state => ' + serviceState + taskParamsPostfix + '}\n')

+    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) + 

+                     ', service_state => ' + serviceState + taskParamsPostfix + '}\n')

     stageNum = stageNum + 1

-    

+  outputFile.write('}\n')

 def normalizeTaskParams(taskParams):

   result = ''

   coma = ''

@@ -149,9 +177,8 @@ def writeStages(outputFile, numStages):
   

   outputFile.write('\n')

     

-    

+logging.basicConfig(level=logging.DEBUG)    

 #test code

 jsonFile = open('test.json', 'r')

 jsonStr = jsonFile.read() 

-print '>>>JSON STRING: ' + jsonStr

 generateManifest(jsonStr)

diff --git a/ambari-agent/src/main/python/manifestGenerator/rolesToClass.dict b/ambari-agent/src/main/python/ambari_agent/rolesToClass.dict
index 13949ace85..13949ace85 100644
--- a/ambari-agent/src/main/python/manifestGenerator/rolesToClass.dict
+++ b/ambari-agent/src/main/python/ambari_agent/rolesToClass.dict
diff --git a/ambari-agent/src/main/python/manifestGenerator/serviceStates.dict b/ambari-agent/src/main/python/ambari_agent/serviceStates.dict
index a4e4236495..a4e4236495 100644
--- a/ambari-agent/src/main/python/manifestGenerator/serviceStates.dict
+++ b/ambari-agent/src/main/python/ambari_agent/serviceStates.dict
diff --git a/ambari-agent/src/main/python/ambari_agent/site.pp b/ambari-agent/src/main/python/ambari_agent/site.pp
new file mode 100644
index 0000000000..3a96e708d4
--- /dev/null
+++ b/ambari-agent/src/main/python/ambari_agent/site.pp
@@ -0,0 +1,31 @@
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hadoop/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hbase/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-zookeeper/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-oozie/manifests/*.pp""
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-pig/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-sqoop/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-templeton/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hive/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hcat/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-mysql/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-monitor-webserver/manifests/*.pp"
+$NAMENODE= ['h2.hortonworks.com']
+$DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
+$hdfs_user="hdfs"
+$jdk_location="lah/blah"
+$configuration =  {
+$hdfs_site=> {
+"dfs.block.size" => "256000000",
+"dfs.replication" => "1"
+}
+$core_site=> {
+"fs.default.name" => "hrt8n36.cc1.ygridcore.net"
+}
+
+}
+node /default/ {
+ stage{0 :} -> stage{1 :}
+class {'hdp-hadoop::datanode': stage => 1, service_state => running}
+class {'hdp-hadoop::namenode': stage => 2, service_state => installed_and_configured}
+}
diff --git a/ambari-agent/src/main/python/manifestGenerator/test.json b/ambari-agent/src/main/python/ambari_agent/test.json
index c221797ffa..2c1a5fbe79 100644
--- a/ambari-agent/src/main/python/manifestGenerator/test.json
+++ b/ambari-agent/src/main/python/ambari_agent/test.json
@@ -16,18 +16,17 @@
 "hdfs_site" : { "dfs.block.size" : "256000000", "dfs.replication" : "1" } ,
 "core_site" :  { "fs.default.name" : "hrt8n36.cc1.ygridcore.net" }
 },
-"roles": [
+"roleCommands": [
 {
 "role" : "DATANODE", 
-"roleCommand": "START",
-"params" : {
-"myid" : "23"
+"cmd": "START",
+"roleParams" : {
 }
 },
 {
 "role": "NAMENODE",
-"roleCommand": "INSTALL",
-"params" : {}
+"cmd": "INSTALL",
+"roleParams" : {}
 }
 ]
 }
diff --git a/ambari-agent/src/main/python/manifestGenerator/imports.txt b/ambari-agent/src/main/python/manifestGenerator/imports.txt
deleted file mode 100644
index 7a1f7de44a..0000000000
--- a/ambari-agent/src/main/python/manifestGenerator/imports.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-import "/etc/puppet/agent/modules/hdp/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-hadoop/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-hbase/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-zookeeper/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-oozie/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-pig/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-sqoop/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-templeton/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-hive/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-hcat/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-mysql/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-monitor-webserver/manifests/*.pp"