BIGTOP-1416. Refactor vagrant-puppet's Vagrantfile to make instance number configurable

Signed-off-by: jay@apache.org <jayunit100>

3 files changed, 59 insertions, 164 deletions

diff --git a/bigtop-deploy/vm/vagrant-puppet/README.md b/bigtop-deploy/vm/vagrant-puppet/README.md
index 70afb365..91acadd0 100644
--- a/bigtop-deploy/vm/vagrant-puppet/README.md
+++ b/bigtop-deploy/vm/vagrant-puppet/README.md
@@ -19,8 +19,8 @@
 
 ## Overview
 
-The startup.sh script runs one of 3 vagrant templates, and creates a bigtop virtual hadoop cluster for you, by
-pulling from existing publishing bigtop repositories.  This cluster can be used:
+The Vagrantfile definition creates a bigtop virtual hadoop cluster for you, by pulling from existing publishing bigtop repositories.
+This cluster can be used:
 
 - to test bigtop smoke tests
 - to test bigtop puppet recipes
@@ -45,21 +45,13 @@ vagrant plugin install vagrant-cachier
 3) To provision a 3 node Apache Hadoop cluster on top of vagrant boxes
 
 ```
-./startup.sh --cluster
+vagrant up
 ```
 
-4) See options with -h specified
+4) You can specify number of nodes you'd like to provision by modifying `num_instances` in Vagrantfile
 
 ```
-$ ./startup.sh -h
-
-     usage: startup.sh [options]
-
-       -s, --standalone        deploy a standalone hadoop vm
-
-       -c, --cluster           deploy a 3 node hadoop cluster
-
-       -h, --help
+num_instances = 5
 ```
 
 ##Example:
@@ -67,11 +59,15 @@ $ ./startup.sh -h
 5) Run hbase-test.sh to evaluate the deployment.
 
 ##Configure Apache Hadoop ecosystem components
-* Choose the ecosystem you want to be deployed by modify components in provision.sh.
+* Choose the ecosystem you want to be deployed by modifying components in provision.sh.
 
 ```
-     components,hadoop,hbase,...
+     components,hadoop,hbase,yarn,mapred-app,...
 ```
 
-By default, Apache Hadoop and Apache HBase will be installed.
+By default, Apache Hadoop, YARN, and Apache HBase will be installed.
 See `bigtop-deploy/puppet/config/site.csv.example` for more details.
+
+##Note:
+
+For bigtop 0.7.0 code base, you must change the value of the [yarn-site.xml](https://github.com/apache/bigtop/blob/master/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml) yarn.nodemanager.aux.services from "mapreduce_shuffle" to "mapreduce.shuffle" before `vagrant up`
diff --git a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
index 15a3dec5..c6825ad8 100755
--- a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
@@ -20,127 +20,66 @@ VAGRANTFILE_API_VERSION = "2"
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   vmbox = "puppetlab-centos-64-nocm"
 
-  # Virtual Box implementation
+  # Virtual box implementation
   vmboxurl = "http://puppet-vagrant-boxes.puppetlabs.com/centos-64-x64-vbox4210-nocm.box"
 
-  # Network environment settings
-  bigtop1_hostname = "bigtop1.vagrant"
-  bigtop1_ip = "10.10.10.12"
-  bigtop2_hostname = "bigtop2.vagrant"
-  bigtop2_ip = "10.10.10.13"
-  bigtop3_hostname = "bigtop3.vagrant"
-  bigtop3_ip = "10.10.10.14"
+  # number of instances
+  num_instances = 3
 
-  # The bigtop puppet recipes
+  # the bigtop puppet recipes
   bigtop_puppet = "../../puppet"
 
   # manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager)
-  config.hostmanager.enabled = true
+  # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier)
+  if Vagrant.has_plugin?("vagrant-hostmanager")
+    config.hostmanager.enabled = true
+  end
 
   # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier)
   if Vagrant.has_plugin?("vagrant-cachier")
     config.cache.scope = :box
   end
 
-  # First node in the cluster
-  config.vm.define :bigtop1 do |bigtop1|
-
-    bigtop1.vm.box = vmbox
-    bigtop1.vm.box_url = vmboxurl
-
-    bigtop1.vm.provider :virtualbox do |vb|
-      vb.customize ["modifyvm", :id, "--memory", "1024"]
-    end
-
-    bigtop1.vm.network :private_network, ip: bigtop1_ip
-    bigtop1.vm.hostname = bigtop1_hostname
-
-    bigtop1.vm.synced_folder bigtop_puppet, "/bigtop-puppet"
-
-    # Shell provisioner to setup basic vm environment
-    bigtop1.vm.provision :shell do |shell|
-      shell.path = "provision.sh"
-      shell.args = bigtop1_hostname + " " + bigtop1_ip + " " + bigtop1_hostname + " " + bigtop1_ip
-    end
-
-    bigtop1.vm.provision :hostmanager
-
-    # Deploy Apache Hadoop and it's ecosystem
-    bigtop1.vm.provision :puppet do |puppet|
-      puppet.module_path = bigtop_puppet + "/modules"
-      puppet.manifests_path = bigtop_puppet + "/manifests"
-      puppet.manifest_file  = "site.pp"
-      puppet.options = "--debug --confdir=/bigtop-puppet"
-    end
-
-  end
-
-
-  # Second node in the cluster
-  config.vm.define :bigtop2 do |bigtop2|
-
-    bigtop2.vm.box = vmbox
-    bigtop2.vm.box_url = vmboxurl
-
-    bigtop2.vm.provider :virtualbox do |vb|
-      vb.customize ["modifyvm", :id, "--memory", "1024"]
-    end
-
-    bigtop2.vm.network :private_network, ip: bigtop2_ip
-    bigtop2.vm.hostname = bigtop2_hostname
-
-    bigtop2.vm.synced_folder bigtop_puppet, "/bigtop-puppet"
-
-    # Shell provisioner to setup basic vm environment
-    bigtop2.vm.provision :shell do |shell|
-      shell.path = "provision.sh"
-      shell.args = bigtop1_hostname + " " + bigtop1_ip + " " + bigtop2_hostname + " " + bigtop2_ip
-    end
-
-    bigtop2.vm.provision :hostmanager
-
-    # Deploy Apache Hadoop and it's ecosystem
-    bigtop2.vm.provision :puppet do |puppet|
-      puppet.module_path = bigtop_puppet + "/modules"
-      puppet.manifests_path = bigtop_puppet + "/manifests"
-      puppet.manifest_file  = "site.pp"
-      puppet.options = "--debug --confdir=/bigtop-puppet"
-    end
-
-  end
-
-
-  # Third node in the cluster
-  config.vm.define :bigtop3 do |bigtop3|
-
-    bigtop3.vm.box = vmbox
-    bigtop3.vm.box_url = vmboxurl
-
-    bigtop3.vm.provider :virtualbox do |vb|
-      vb.customize ["modifyvm", :id, "--memory", "1024"]
-    end
-
-    bigtop3.vm.network :private_network, ip: bigtop3_ip
-    bigtop3.vm.hostname = bigtop3_hostname
-
-    bigtop3.vm.synced_folder bigtop_puppet, "/bigtop-puppet"
-
-    # Shell provisioner to setup basic vm environment
-    bigtop3.vm.provision :shell do |shell|
-      shell.path = "provision.sh"
-      shell.args = bigtop1_hostname + " " + bigtop1_ip + " " + bigtop3_hostname + " " + bigtop3_ip
+  # declare the variable first, will be override in following loop
+  bigtop_master = ""
+
+  # nodes definition
+  (1..num_instances).each do |i|
+    config.vm.define "bigtop#{i}" do |bigtop|
+
+      bigtop.vm.box = vmbox
+      bigtop.vm.box_url = vmboxurl
+      bigtop_hostname="bigtop#{i}.vagrant"
+      bigtop_ip="10.10.10.1#{i}"
+      if "#{i}" == "1"
+        bigtop_master = bigtop_hostname
+      end
+
+      bigtop.vm.provider :virtualbox do |vb|
+        vb.customize ["modifyvm", :id, "--memory", "1024"]
+      end
+
+      bigtop.vm.network :private_network, ip: bigtop_ip
+      bigtop.vm.hostname = bigtop_hostname
+
+      bigtop.vm.synced_folder bigtop_puppet, "/bigtop-puppet"
+
+      # shell provisioner to setup basic vm environment
+      bigtop.vm.provision :shell do |shell|
+        shell.path = "provision.sh"
+        shell.args = bigtop_master
+      end
+
+      bigtop.vm.provision :hostmanager
+
+      # deploy Apache Hadoop and it's ecosystem
+      bigtop.vm.provision :puppet do |puppet|
+        puppet.module_path = bigtop_puppet + "/modules"
+        puppet.manifests_path = bigtop_puppet + "/manifests"
+        puppet.manifest_file  = "site.pp"
+        puppet.options = "--debug --confdir=/bigtop-puppet"
+      end
     end
-
-    bigtop3.vm.provision :hostmanager
-
-    # Deploy Apache Hadoop and it's ecosystem
-    bigtop3.vm.provision :puppet do |puppet|
-      puppet.module_path = bigtop_puppet + "/modules"
-      puppet.manifests_path = bigtop_puppet + "/manifests"
-      puppet.manifest_file  = "site.pp"
-      puppet.options = "--debug --confdir=/bigtop-puppet"
-    end
-
   end
 
 end
diff --git a/bigtop-deploy/vm/vagrant-puppet/startup.sh b/bigtop-deploy/vm/vagrant-puppet/startup.sh
deleted file mode 100755
index 34b52ad2..00000000
--- a/bigtop-deploy/vm/vagrant-puppet/startup.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-usage() {
-    echo "usage: `basename $0` [options]"
-    echo "       -s, --standalone        deploy a standalone hadoop VM"
-    echo "       -c, --cluster           deploy a 3 node hadoop cluster"
-    echo "       -h, --help"
-    exit 1
-}
-
-case "$1" in
--s|--standalone)
-    vagrant up bigtop1
-    shift;;
--c|--cluster)
-    vagrant up --provision-with shell && vagrant provision --provision-with hostmanager,puppet
-    shift;;
--h|--help)
-    usage
-    shift;;
-"")
-    usage
-    shift
-    break;;
-esac