BIGTOP-2646. Move Vagrant Provisioner from bigtop-deploy/vm to provisioner directory

9 files changed, 518 insertions, 0 deletions

diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh
new file mode 100755
index 00000000..86b6a3c2
--- /dev/null
+++ b/provisioner/utils/setup-env-centos.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+yum -y install rng-tools
+if [ -x /usr/bin/systemctl ] ; then
+    sed -i 's@ExecStart=/sbin/rngd -f@ExecStart=/sbin/rngd -f -r /dev/urandom@' /usr/lib/systemd/system/rngd.service
+    systemctl daemon-reload
+    systemctl start rngd
+else
+    sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
+    service rngd start
+fi
+
+if [ $enable_local_repo == "true" ]; then
+    echo "Enabling local yum."
+    yum -y install yum-utils
+    sudo echo "gpgcheck=0" >> /etc/yum.conf
+    sudo yum-config-manager --add-repo file:///bigtop-home/output
+else
+    echo "local yum = $enable_local_repo ; NOT Enabling local yum.  Packages will be pulled from remote..."
+fi
+
diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh
new file mode 100755
index 00000000..098155b5
--- /dev/null
+++ b/provisioner/utils/setup-env-debian.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+apt-get -y install rng-tools
+sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools
+service rng-tools start
+
+if [ $enable_local_repo == "true" ]; then
+    echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list
+    apt-get update
+else
+    apt-get install -y apt-transport-https
+    echo "local apt = $enable_local_repo ; NOT Enabling local apt. Packages will be pulled from remote..."
+fi
+
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
new file mode 100755
index 00000000..93f795e6
--- /dev/null
+++ b/provisioner/utils/smoke-tests.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HCFS_USER="hdfs"
+SMOKE_TESTS=${1:-mapreduce,pig}
+
+# Autodetect JAVA_HOME
+if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+else
+  >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n"
+  exit 2
+fi
+
+echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
+
+export HADOOP_CONF_DIR=/etc/hadoop/conf/
+export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
+export HIVE_HOME=/usr/lib/hive/
+export PIG_HOME=/usr/lib/pig/
+export FLUME_HOME=/usr/lib/flume/
+export SQOOP_HOME=/usr/lib/sqoop/
+export HIVE_CONF_DIR=/etc/hive/conf/
+export MAHOUT_HOME="/usr/lib/mahout"
+
+prep() {
+    HADOOP_COMMAND=$1
+    su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -mkdir /user/vagrant /user/root"
+    su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/vagrant"
+    su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/root"
+}
+
+prep hadoop
+if [[ $SMOKE_TESTS == *"qfs"* ]]; then
+    prep hadoop-qfs
+fi
+
+if [ -f /etc/debian_version ] ; then
+    apt-get -y install pig hive flume mahout sqoop
+else
+    yum install -y pig hive flume mahout sqoop
+fi
+ALL_SMOKE_TASKS=""
+for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do
+  ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
+done
+cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info
+# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same
+# permissions as used for smoke-tests execution
+rm -rf buildSrc/build/test-results/binary
+
diff --git a/provisioner/vagrant/.gitignore b/provisioner/vagrant/.gitignore
new file mode 100644
index 00000000..04204c7c
--- /dev/null
+++ b/provisioner/vagrant/.gitignore
@@ -0,0 +1 @@
+config
diff --git a/provisioner/vagrant/README.md b/provisioner/vagrant/README.md
new file mode 100644
index 00000000..0b1ff20f
--- /dev/null
+++ b/provisioner/vagrant/README.md
@@ -0,0 +1,99 @@
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements. See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+------------------------------------------------------------------------------------------------------------------------------------------------------
+
+#BigTop VM provisioner
+
+## Overview
+
+The Vagrantfile definition creates a bigtop virtual hadoop cluster for you, by pulling from existing publishing bigtop repositories.
+This cluster can be used:
+
+- to test bigtop smoke tests
+- to test bigtop puppet recipes
+
+Eventually, we may also add ability to build AND provision bigtop in a vagrant recipe, which would essentially
+give full validation of the BigTop stack.
+
+## USAGE
+
+1) Install [Vagrant](https://www.vagrantup.com/downloads.html)(DO NOT install 1.8.5 because of a critical [bug](https://github.com/mitchellh/vagrant/issues/7631))
+
+2) Install [vagrant-hostmanager plugin](https://github.com/smdahlen/vagrant-hostmanager) to manage `/etc/hosts`
+
+```
+vagrant plugin install vagrant-hostmanager
+```
+
+3) (Optional) Install [vagrant-cachier plugin](https://github.com/fgrehm/vagrant-cachier) to cache packages at local
+
+```
+vagrant plugin install vagrant-cachier
+```
+
+4) To provision a one node Apache Hadoop cluster on top of vagrant boxes
+
+```
+vagrant up
+```
+
+5) You can specify number of nodes you'd like to provision by modifying `num_instances` in vagrantconfig.yaml
+
+```
+num_instances: 5
+```
+
+6) Test on local built packages is available by:
+
+first, build up local yum repo
+
+```
+cd bigtop; ./gradlew alluxio-yum
+```
+
+and then enable local yum in vagrantconfig.yaml
+
+```
+enable_local_repo: true
+```
+
+finally, spin up the cluster
+```
+vagrant up
+```
+
+## Override Vagrant configurations
+You can override vagrant configurations from environment variables:
+```
+$ export REPO=http://repo.example.com/path/to
+$ vagrant up
+```
+
+or
+
+```
+$ MEMORY_SIZE=8000 vagrant up
+```
+
+##Configure Apache Hadoop ecosystem components
+* Choose the ecosystem you want to be deployed by modifying components in vagrantconfig.yaml
+
+```
+components: "hadoop, hbase, yarn,..."
+```
+
+By default, Apache Hadoop and YARN will be installed.
+See `bigtop-deploy/puppet/hieradata/site.yaml` for more details.
diff --git a/provisioner/vagrant/Vagrantfile b/provisioner/vagrant/Vagrantfile
new file mode 100755
index 00000000..fc6bfc19
--- /dev/null
+++ b/provisioner/vagrant/Vagrantfile
@@ -0,0 +1,166 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "yaml"
+
+_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "vagrantconfig.yaml"), File::RDONLY).read)
+CONF = _config
+
+# Override vagrant configurations using environment variables
+keys = CONF.keys
+keys.each do |k|
+  if ENV[k.upcase] != nil then
+    puts "Overide from environment variable: " + k.upcase + " = " + ENV[k.upcase]
+    if /^\d+/.match(ENV[k.upcase])
+      CONF[k] = Integer(ENV[k.upcase])
+    else
+      CONF[k] = ENV[k.upcase]
+    end
+  end
+end
+
+# Repository
+# Example for testing a Release candidate.
+repo = CONF['repo']
+# repo = "http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64"
+
+# Which Linux Distribution to use. Right now only centos is tested
+distro = CONF['distro']
+
+# number of instances
+num_instances = CONF['num_instances']
+
+# hadoop ecosystem components
+components = CONF['components']
+
+# Whether to run smoke tests
+run_smoke_tests = CONF['run_smoke_tests']
+
+# Smoke test Components to run
+smoke_test_components = CONF['smoke_test_components'].join(',')
+
+# This is a update to allow dev packages
+# Force success - not worried if this step fails, since we generally only use it for development.
+enable_local_repo = CONF['enable_local_repo']
+puts "vagrant conf local repo enabled:  #{enable_local_repo}"
+
+# JDK package name
+jdk = CONF['jdk']
+
+# master node hostname
+bigtop_master = "bigtop1.vagrant"
+
+$script = <<SCRIPT
+service iptables stop
+service firewalld stop
+chkconfig iptables off
+# Remove 127.0.0.1 entry since vagrant's hostname setting will map it to FQDN,
+# which miss leads some daemons to bind on 127.0.0.1 instead of public or private IP
+sed -i "/127.0.0.1/d" /etc/hosts
+echo "Bigtop yum repo = #{repo}"
+# Prepare host manipulation files needed in standalone box
+cp /bigtop-home/provisioner/vagrant/config_hosts /etc/init.d
+cp /bigtop-home/provisioner/vagrant/gen_hosts.sh /usr/bin
+chkconfig --add config_hosts
+# Prepare puppet configuration file
+mkdir -p /etc/puppet/hieradata
+cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet
+cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/
+cat > /etc/puppet/hieradata/site.yaml << EOF
+bigtop::hadoop_head_node: #{bigtop_master}
+hadoop::hadoop_storage_dirs: [/data/1, /data/2]
+bigtop::bigtop_repo_uri: #{repo}
+hadoop_cluster_node::cluster_components: #{components}
+bigtop::jdk_package_name: #{jdk}
+hadoop::common_hdfs::testonly_hdfs_sshkeys: "yes"
+EOF
+SCRIPT
+
+VAGRANTFILE_API_VERSION = "2"
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  # manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager)
+  # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier)
+  config.hostmanager.enabled = true
+
+  # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier)
+  if Vagrant.has_plugin?("vagrant-cachier")
+    config.cache.scope = :box
+  end
+
+  # nodes definition
+  (1..num_instances).each do |i|
+    config.vm.define "bigtop#{i}" do |bigtop|
+
+      bigtop.vm.box = CONF['box']
+      bigtop_hostname="bigtop#{i}"
+      bigtop_fqdn="#{bigtop_hostname}.vagrant"
+      bigtop_ip="10.10.10.1#{i}"
+
+      bigtop.vm.provider :virtualbox do |vb|
+        vb.customize ["modifyvm", :id, "--memory", CONF['memory_size']]
+	vb.customize ['modifyvm', :id, '--cpus', CONF['number_cpus']]
+      end
+
+      bigtop.vm.network :private_network, ip: bigtop_ip
+      bigtop.vm.hostname = bigtop_fqdn
+
+      # two levels up is the bigtop "home" directory.
+      # the current directory has puppet recipes which we need for provisioning.
+      bigtop.vm.synced_folder "../../", "/bigtop-home"
+
+      # We also add the bigtop-home output/ dir, so that locally built rpms will be available.
+      puts "Adding output/ repo ? #{enable_local_repo}"
+
+      # Install puppet
+      bigtop.vm.provision :shell do |shell|
+        shell.path = "../../bigtop_toolchain/bin/puppetize.sh"
+      end
+
+      # carry on w/ installation
+      bigtop.vm.provision :shell do |shell|
+        shell.path = "../utils/setup-env-" + distro + ".sh"
+        shell.args = ["#{enable_local_repo}"]
+      end
+      bigtop.vm.provision "shell", inline: $script
+      # Add the ip to FQDN and hostname mapping in /etc/hosts
+      bigtop.hostmanager.aliases = "#{bigtop_fqdn} #{bigtop_hostname}"
+
+      # deploy Apache Hadoop and it's ecosystem
+      bigtop.vm.provision :puppet do |puppet|
+        puppet.module_path = "../../bigtop-deploy/puppet/modules/"
+        puppet.manifests_path =  "../../bigtop-deploy/puppet/manifests/"
+        puppet.manifest_file = "."
+        puppet.options = "--parser future"
+      end
+
+      if run_smoke_tests then
+        if i==num_instances then
+          puts "creating provisioner directive for running tests"
+          bigtop.vm.provision :shell do |shell|
+            shell.path = "../utils/smoke-tests.sh"
+            shell.args = ["#{smoke_test_components}"]
+          end
+        else
+          puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}"
+        end
+      end
+
+    end
+  end
+
+end
diff --git a/provisioner/vagrant/config_hosts b/provisioner/vagrant/config_hosts
new file mode 100755
index 00000000..07e1d5ff
--- /dev/null
+++ b/provisioner/vagrant/config_hosts
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Hosts reconfiguration script
+# chkconfig: 345 70 1
+# description: config_hosts
+
+# Licensed to Hortonworks, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Hortonworks, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+NAME="config_hosts"
+LOG="/var/log/config_hosts.log"
+
+source /etc/rc.d/init.d/functions
+
+case "$1" in
+start)
+    printf "%-50s\n" "Starting $NAME..." | tee -a $LOG
+    printf "%-50s\n" "Updating IP..." | tee -a $LOG
+    bash /usr/bin/gen_hosts.sh
+;;
+
+stop)
+    date >> $LOG
+;;
+
+restart)
+;;
+
+*)
+        echo "Usage: $0 {start|stop|restart}"
+        exit 1
+esac
diff --git a/provisioner/vagrant/gen_hosts.sh b/provisioner/vagrant/gen_hosts.sh
new file mode 100755
index 00000000..c629c85a
--- /dev/null
+++ b/provisioner/vagrant/gen_hosts.sh
@@ -0,0 +1,44 @@
+# Licensed to Hortonworks, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Hortonworks, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "# Do not remove the following line, or various programs" > /etc/hosts
+echo "# that require network functionality will fail." >> /etc/hosts
+echo "127.0.0.1		localhost.localdomain localhost" >> /etc/hosts
+
+function get_inet_iface(){
+  route | grep default | awk '{print $8}'
+}
+
+function get_ip() {
+  ip addr | grep 'inet ' | grep -E "($(get_inet_iface))" | awk '{ print $2 }' | awk -F'/' '{print $1}'
+}
+
+HOST=$(get_ip)
+NUM=5
+while [ -z "$HOST" ]; do
+  HOST=$(get_ip)
+  sleep 5
+  NUM=$(($NUM-1))
+  if [ $NUM -le 0 ]; then
+    HOST="127.0.0.1"
+    echo "Failed to update IP"
+    break
+  fi
+done
+echo "$HOST	`hostname`" >> /etc/hosts
+
+echo 0 > /proc/sys/kernel/hung_task_timeout_secs
+ethtool -K $(get_inet_iface) tso off
diff --git a/provisioner/vagrant/vagrantconfig.yaml b/provisioner/vagrant/vagrantconfig.yaml
new file mode 100644
index 00000000..4ad1aac2
--- /dev/null
+++ b/provisioner/vagrant/vagrantconfig.yaml
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+memory_size: 4096
+number_cpus: 1
+box: "puppetlabs/centos-7.2-64-nocm"
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"
+num_instances: 1
+distro: centos
+components: [hadoop, yarn, hbase]
+enable_local_repo: false
+run_smoke_tests: false
+smoke_test_components: [mapreduce, pig]
+jdk: "java-1.7.0-openjdk-devel.x86_64"