AMBARI-971. Add api support for creating multiple resources in a single request. (John Speidel via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/AMBARI-666@1406492 13f79535-47bb-0310-9956-ffa450edef68

10 files changed, 172 insertions, 85 deletions

diff --git a/ambari-agent/conf/unix/ambari-agent b/ambari-agent/conf/unix/ambari-agent
index 49f28af9e3..a88de0fedc 100644
--- a/ambari-agent/conf/unix/ambari-agent
+++ b/ambari-agent/conf/unix/ambari-agent
@@ -6,7 +6,6 @@
 case "$1" in
   start)
         echo -e "Starting ambari-agent"
-        export AMBARI_PASSPHRASE=pass_phrase
         python /usr/lib/python2.6/site-packages/ambari_agent/main.py
         ;;
   stop)
diff --git a/ambari-agent/conf/unix/ambari.ini b/ambari-agent/conf/unix/ambari.ini
index 075301f441..b1ce7e810b 100644
--- a/ambari-agent/conf/unix/ambari.ini
+++ b/ambari-agent/conf/unix/ambari.ini
@@ -10,7 +10,7 @@ prefix=/tmp/ambari-agent
 installprefix=/var/ambari/
 
 [puppet]
-puppetmodules=/etc/ambari-agent/puppet
+puppetmodules=/var/lib/ambari-agent/puppet
 puppet_home=/usr/bin/puppet
 facter_home=/usr/bin/facter
 
@@ -19,7 +19,7 @@ maxretries=2
 sleepBetweenRetries=1
 
 [security]
-keysdir=/etc/ambari-agent/keys
+keysdir=/var/lib/ambari-agent/keys
 server_crt=ca.crt
 passphrase_env_var_name=AMBARI_PASSPHRASE
 
diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml
index 43a6229b56..ee7191ad68 100644
--- a/ambari-agent/pom.xml
+++ b/ambari-agent/pom.xml
@@ -113,6 +113,9 @@
           <copyright>2012, Apache Software Foundation</copyright>
           <group>Development</group>
           <description>Maven Recipe: RPM Package.</description>
+          <requires>
+            <require>puppet = 2.7.9</require>
+          </requires>
           <mappings>
 
             <mapping>
@@ -125,7 +128,7 @@
             </mapping>
 
             <mapping>
-              <directory>/etc/${project.artifactId}/puppet</directory>
+              <directory>/var/lib/${project.artifactId}/puppet</directory>
               <sources>
                 <source>
                   <location>src/main/puppet</location>
@@ -155,9 +158,14 @@
             <mapping>
               <directory>/var/run/ambari</directory>
             </mapping>
-
             <mapping>
-              <directory>/etc/ambari-agent/keys</directory>
+              <directory>/var/lib/${project.artifactId}/keys</directory>
+            </mapping>
+            <mapping>
+              <directory>/var/log/ambari</directory>
+            </mapping>
+            <mapping>
+              <directory>/var/ambari</directory>
             </mapping>
 
             <!-- -->
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp b/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
index 5dcbcba403..3646aab026 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
@@ -40,9 +40,6 @@ class hdp-hbase(
       configuration => $configuration['hbase-site']
       }
     }
-    hdp-hbase::configfile { 'regionservers':}
-    Anchor['hdp-hbase::begin'] -> Hdp::Package['hbase'] -> Hdp::User[$hbase_user] -> Hdp::Directory[$config_dir] -> 
-    Hdp-hbase::Configfile<||> ->  Anchor['hdp-hbase::end']
 
   if has_key($configuration, 'hbase-policy') {
     configgenerator::configfile{'hbase-policy': 
@@ -91,7 +88,7 @@ define hdp-hbase::configfile(
   $mode = undef,
   $hbase_master_host = undef,
   $template_tag = undef,
-  $type = undef,
+  $type = undef
 ) 
 {
   if ($name == 'hadoop-metrics.properties') {
diff --git a/ambari-agent/src/main/puppet/modules/hdp-repos/templates/repo.erb b/ambari-agent/src/main/puppet/modules/hdp-repos/templates/repo.erb
index b8f9b9fcda..ea5a33b94e 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-repos/templates/repo.erb
+++ b/ambari-agent/src/main/puppet/modules/hdp-repos/templates/repo.erb
@@ -22,6 +22,5 @@
 [<%=repo_id%>]
 name=<%=repo_name %>
 baseurl=<%=base_url %>
+path=/
 enabled=1
-
-
diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
index 8b4c9c4253..b9fd437628 100644
--- a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
@@ -67,6 +67,9 @@ class hdp::params()
         /^6\..+$/: { $hdp_os_type = "rhel6" }
       }
     }
+    suse: {
+      $hdp_os_type = "suse"
+    }
     default: {
       hdp_fail("No support for os  ${hdp_os} ${hdp_os_version}")
     }
@@ -354,7 +357,8 @@ class hdp::params()
 
   $repos_paths = 
   {
-    centos6 => '/etc/yum.repos.d'
+    centos6 => '/etc/yum.repos.d',
+    suse => '/etc/zypp/repos.d'
   }
 
   }
diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
index 74ec908cf5..6cb0d1e158 100644
--- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
+++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
@@ -133,6 +133,10 @@ class ActionQueue(threading.Thread):
                   'exitCode' : commandresult['exitcode'],
                   'serviceName' : serviceName,
                   'status' : status}
+    if roleResult['stdout'] == '':
+      roleResult['stdout'] = 'None'
+    if roleResult['stderr'] == '':
+      roleResult['stderr'] = 'None'
     result.append(roleResult)
     pass
     return result
diff --git a/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py b/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py
new file mode 100644
index 0000000000..7b77ee076a
--- /dev/null
+++ b/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import logging
+import os
+import json
+from shell import shellRunner
+from manifestGenerator import writeImports
+
+
+PUPPET_EXT=".pp"
+
+logger = logging.getLogger()
+
+class RepoInstaller:
+  def __init__(self, parsedJson, path, modulesdir, taskId):
+    self.parsedJson = parsedJson
+    self.path = path
+    self.modulesdir = modulesdir
+    self.taskId = taskId
+    self.sh = shellRunner()
+
+  def prepareReposInfo(self):
+    params = {}
+    self.repoInfoList = []
+    if self.parsedJson.has_key('hostLevelParams'):
+      params = self.parsedJson['hostLevelParams']
+    if params.has_key('repo_info'):
+      self.repoInfoList = params['repo_info']
+
+  def generateFiles(self):
+    repoPuppetFiles = []
+    for repo in self.repoInfoList:
+      repoFile = open(self.path + os.sep + repo['repo_id'] + '-' + str(self.taskId) + PUPPET_EXT, 'w+')
+      writeImports(repoFile, self.modulesdir, inputFileName='imports.txt')
+      repoFile.write('node /default/ {')
+      repoFile.write('class{ "hdp-repos::process_repo" : ' + ' os_type => "' + repo['os_type'] +
+      '", repo_id => "' + repo['repo_id'] + '", base_url => "' + repo['base_url'] +
+      '", repo_name => "' + repo['repo_name'] + '" }' )
+      repoFile.write('}')
+      repoFile.close()
+      repoPuppetFiles.append(repoFile.name)
+
+    return repoPuppetFiles
+
+  def installRepos(self):
+    self.prepareReposInfo()
+    repoPuppetFiles = self.generateFiles()
+    return repoPuppetFiles
+
+def main():
+  #Test code
+  jsonFile = open('test.json', 'r')
+  jsonStr = jsonFile.read() 
+  parsedJson = json.loads(jsonStr)
+  repoInstaller = RepoInstaller(parsedJson, '/tmp', '/home/centos/ambari_repo_info/ambari-agent/src/main/puppet/modules')
+  repoInstaller.installRepos()
+  
+if __name__ == '__main__':
+  main()
+
+
diff --git a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
index 3c05cd217b..f3526bffd9 100644
--- a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
+++ b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
@@ -24,8 +24,6 @@ import logging
 from uuid import getnode as get_mac
 from shell import shellRunner
 
-REPO_INFO_DIR="repos_info"
-PUPPET_EXT=".pp"
 
 logger = logging.getLogger()
 
@@ -138,9 +136,7 @@ def writeNodes(outputFile, clusterHostInfo):
 def writeParams(outputFile, params, modulesdir):
 
   for paramName in params.iterkeys():
-    # todo handle repo information properly
-    if paramName == 'repo_info':
-      processRepo(params[paramName],modulesdir)      
+    if paramName == 'repo_info':     
       continue
       
 
@@ -257,39 +253,7 @@ def writeStages(outputFile, numStages):
   
   outputFile.write('\n')
 
-def processRepo(repoInfoList, modulesdir):
 
-  if not os.path.exists(REPO_INFO_DIR):
-    os.makedirs(REPO_INFO_DIR)
-
-  for repo in repoInfoList:
-
-    repoFile = open(REPO_INFO_DIR + os.sep + repo['repo_id'] + PUPPET_EXT, 'w+')
-    writeImports(repoFile, modulesdir, inputFileName='imports.txt')
-    repoFile.write('node /default/ {')
-    repoFile.write('class{ "hdp-repos::process_repo" : ' + ' os_type => "' + repo['os_type'] +
-    '", repo_id => "' + repo['repo_id'] + '", base_url => "' + repo['base_url'] +
-    '", repo_name => "' + repo['repo_name'] + '" }' )
-    repoFile.write('}')
-    repoFile.close()
-
-
-def installRepos():
-  sh = shellRunner()
-  agentdir = os.getcwd()
-  confdir = os.path.abspath(agentdir + ".." + os.sep + ".." + 
-                               os.sep + ".." + os.sep + "puppet")
-
-  for repo in os.listdir(REPO_INFO_DIR):
-    if not repo.endswith(PUPPET_EXT):
-      continue
-    logfile = repo + '_log.log'
-    res = sh.run(['puppet apply', '--confdir=' + confdir, REPO_INFO_DIR + os.sep + repo,
-    '--logdest=' + agentdir + os.sep + REPO_INFO_DIR + os.sep + logfile])
-    if res['exitCode'] == 0:
-      logger.info('Repository ' + repo + ' was installed')
-    else:
-      logger.error('Repository ' + repo + ' wasn''t installed. Please find detailed info in logfile:' + logfile)
   
 def main():
   logging.basicConfig(level=logging.DEBUG)    
diff --git a/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py b/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py
index b9a84b4391..81c002ddb1 100644
--- a/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py
+++ b/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py
@@ -22,6 +22,7 @@ import os.path
 import logging
 import subprocess
 from manifestGenerator import generateManifest
+from RepoInstaller import RepoInstaller
 import pprint
 from Grep import Grep
 
@@ -29,6 +30,9 @@ logger = logging.getLogger()
 
 class puppetExecutor:
 
+  """ Class that executes the commands that come from the server using puppet.
+  This is the class that provides the pluggable point for executing the puppet"""
+
   # How many lines from command output send to server
   OUTPUT_LAST_LINES = 10
   # How many lines from command error output send to server (before Err phrase)
@@ -38,9 +42,6 @@ class puppetExecutor:
 
   NO_ERROR = "none"
 
-  """ Class that executes the commands that come from the server using puppet.
-  This is the class that provides the pluggable point for executing the puppet"""
-  
   def __init__(self, puppetModule, puppetInstall, facterInstall, tmpDir):
     self.puppetModule = puppetModule
     self.puppetInstall = puppetInstall
@@ -76,39 +77,70 @@ class puppetExecutor:
       taskId = command['taskId']
       
     puppetEnv = os.environ
+    #Install repos
+    modulesdir = self.puppetModule + "/modules"
+    repoInstaller = RepoInstaller(command, self.tmpDir, modulesdir, taskId)
+
+    puppetFiles = repoInstaller.installRepos()
     siteppFileName = os.path.join(self.tmpDir, "site-" + str(taskId) + ".pp") 
-    generateManifest(command, siteppFileName, self.puppetModule + "/modules")
-    puppetcommand = self.puppetCommand(siteppFileName)
-    """ Run the command and make sure the output gets propagated"""
-    rubyLib = ""
-    if os.environ.has_key("RUBYLIB"):
-      rubyLib = os.environ["RUBYLIB"]
-      logger.info("Ruby Lib env from Env " + rubyLib)
-    rubyLib = rubyLib + ":" + self.facterLib() + ":" + self.puppetLib()
-    puppetEnv["RUBYLIB"] = rubyLib
-    logger.info("Setting RUBYLIB as: " + rubyLib)
-    logger.info("Running command " + pprint.pformat(puppetcommand))
-    puppet = subprocess.Popen(puppetcommand,
-                                  stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE,
-                                  env=puppetEnv)
-    stderr_out = puppet.communicate()
-    error = self.NO_ERROR
-    returncode = 0
-    if puppet.returncode != 0 and puppet.returncode != 2:
-      returncode = puppet.returncode
-      error = stderr_out[1]
-      logging.error("Error running puppet: \n" + stderr_out[1])
-      pass
-    result["stderr"] = error
-    puppetOutput = stderr_out[0]
-    logger.info("Output from puppet :\n" + puppetOutput)
-    result["exitcode"] = returncode
+    puppetFiles.append(siteppFileName)
+    generateManifest(command, siteppFileName, modulesdir)
+    #Run all puppet commands, from manifest generator and for repos installation
+    #Appending outputs and errors, exitcode - maximal from all
+    for puppetFile in puppetFiles:
+      puppetcommand = self.puppetCommand(puppetFile)
+      """ Run the command and make sure the output gets propagated"""
+      rubyLib = ""
+      if os.environ.has_key("RUBYLIB"):
+        rubyLib = os.environ["RUBYLIB"]
+        logger.info("Ruby Lib env from Env " + rubyLib)
+      rubyLib = rubyLib + ":" + self.facterLib() + ":" + self.puppetLib()
+      puppetEnv["RUBYLIB"] = rubyLib
+      logger.info("Setting RUBYLIB as: " + rubyLib)
+      logger.info("Running command " + pprint.pformat(puppetcommand))
+      puppet = subprocess.Popen(puppetcommand,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE,
+                                    env=puppetEnv)
+      stderr_out = puppet.communicate()
+      error = "none"
+      returncode = 0
+      if (puppet.returncode != 0 and puppet.returncode != 2) :
+        returncode = puppet.returncode
+        error = stderr_out[1]
+        logging.error("Error running puppet: \n" + stderr_out[1])
+        pass
+		
+      if result.has_key("stderr"):
+        result["stderr"] = result["stderr"] + os.linesep + error
+      else:
+        result["stderr"] = error
+      puppetOutput = stderr_out[0]
+      logger.info("Output from puppet :\n" + puppetOutput)
+      if result.has_key("exitcode"):
+        result["exitcode"] = max(returncode, result["exitcode"])
+      else:
+        result["exitcode"] = returncode
+        
+
+      if result.has_key("stdout"):
+        result["stdout"] = result["stdout"] + os.linesep + puppetOutput
+      else:
+        result["stdout"] = puppetOutput
+
     if error == self.NO_ERROR:
-      result["stdout"] = grep.tail(puppetOutput, self.OUTPUT_LAST_LINES)
+      if result.has_key("stdout"):
+        result["stdout"] = result["stdout"] + os.linesep + str(grep.tail(puppetOutput, self.OUTPUT_LAST_LINES))
+      else:
+        result["stdout"] = grep.tail(puppetOutput, self.OUTPUT_LAST_LINES)
     else:
-      result["stdout"] = grep.grep(puppetOutput, "err", self.ERROR_LAST_LINES_BEFORE, self.ERROR_LAST_LINES_AFTER)
+      if result.has_key("stdout"):
+        result["stdout"] = result["stdout"] + os.linesep + str(grep.grep(puppetOutput, "err", self.ERROR_LAST_LINES_BEFORE, self.ERROR_LAST_LINES_AFTER))
+      else:
+        result["stdout"] = str(grep.grep(puppetOutput, "err", self.ERROR_LAST_LINES_BEFORE, self.ERROR_LAST_LINES_AFTER))
+	
     logger.info("ExitCode : "  + str(result["exitcode"]))
+
     return result
  
 def main():
@@ -118,14 +150,15 @@ def main():
   jsonStr = jsonFile.read() 
   # Below is for testing only.
   
-  puppetInstance = puppetExecutor("/root/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/",
-                                  "/root/workspace/puppet-install/puppet-2.7.9",
+  puppetInstance = puppetExecutor("/home/centos/ambari_repo_info/ambari-agent/src/main/puppet/",
+                                  "/usr/",
                                   "/root/workspace/puppet-install/facter-1.6.10/",
                                   "/tmp")
   jsonFile = open('test.json', 'r')
   jsonStr = jsonFile.read() 
   parsedJson = json.loads(jsonStr)
-  puppetInstance.runCommand(parsedJson)
+  result = puppetInstance.runCommand(parsedJson)
+  logger.debug(result)
   
 if __name__ == '__main__':
   main()