diff options
Diffstat (limited to 'provisioner/vagrant')
-rw-r--r-- | provisioner/vagrant/.gitignore | 1 | ||||
-rw-r--r-- | provisioner/vagrant/README.md | 99 | ||||
-rwxr-xr-x | provisioner/vagrant/Vagrantfile | 166 | ||||
-rwxr-xr-x | provisioner/vagrant/config_hosts | 45 | ||||
-rwxr-xr-x | provisioner/vagrant/gen_hosts.sh | 44 | ||||
-rw-r--r-- | provisioner/vagrant/vagrantconfig.yaml | 26 |
6 files changed, 381 insertions, 0 deletions
diff --git a/provisioner/vagrant/.gitignore b/provisioner/vagrant/.gitignore new file mode 100644 index 00000000..04204c7c --- /dev/null +++ b/provisioner/vagrant/.gitignore @@ -0,0 +1 @@ +config diff --git a/provisioner/vagrant/README.md b/provisioner/vagrant/README.md new file mode 100644 index 00000000..0b1ff20f --- /dev/null +++ b/provisioner/vagrant/README.md @@ -0,0 +1,99 @@ + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------------------------------------------------------------ + +#BigTop VM provisioner + +## Overview + +The Vagrantfile definition creates a bigtop virtual hadoop cluster for you, by pulling from existing publishing bigtop repositories. +This cluster can be used: + +- to test bigtop smoke tests +- to test bigtop puppet recipes + +Eventually, we may also add ability to build AND provision bigtop in a vagrant recipe, which would essentially +give full validation of the BigTop stack. + +## USAGE + +1) Install [Vagrant](https://www.vagrantup.com/downloads.html)(DO NOT install 1.8.5 because of a critical [bug](https://github.com/mitchellh/vagrant/issues/7631)) + +2) Install [vagrant-hostmanager plugin](https://github.com/smdahlen/vagrant-hostmanager) to manage `/etc/hosts` + +``` +vagrant plugin install vagrant-hostmanager +``` + +3) (Optional) Install [vagrant-cachier plugin](https://github.com/fgrehm/vagrant-cachier) to cache packages at local + +``` +vagrant plugin install vagrant-cachier +``` + +4) To provision a one node Apache Hadoop cluster on top of vagrant boxes + +``` +vagrant up +``` + +5) You can specify number of nodes you'd like to provision by modifying `num_instances` in vagrantconfig.yaml + +``` +num_instances: 5 +``` + +6) Test on local built packages is available by: + +first, build up local yum repo + +``` +cd bigtop; ./gradlew alluxio-yum +``` + +and then enable local yum in vagrantconfig.yaml + +``` +enable_local_repo: true +``` + +finally, spin up the cluster +``` +vagrant up +``` + +## Override Vagrant configurations +You can override vagrant configurations from environment variables: +``` +$ export REPO=http://repo.example.com/path/to +$ vagrant up +``` + +or + +``` +$ MEMORY_SIZE=8000 vagrant up +``` + +##Configure Apache Hadoop ecosystem components +* Choose the ecosystem you want to be deployed by modifying components in vagrantconfig.yaml + +``` +components: "hadoop, hbase, yarn,..." +``` + +By default, Apache Hadoop and YARN will be installed. +See `bigtop-deploy/puppet/hieradata/site.yaml` for more details. diff --git a/provisioner/vagrant/Vagrantfile b/provisioner/vagrant/Vagrantfile new file mode 100755 index 00000000..fc6bfc19 --- /dev/null +++ b/provisioner/vagrant/Vagrantfile @@ -0,0 +1,166 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "yaml" + +_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "vagrantconfig.yaml"), File::RDONLY).read) +CONF = _config + +# Override vagrant configurations using environment variables +keys = CONF.keys +keys.each do |k| + if ENV[k.upcase] != nil then + puts "Overide from environment variable: " + k.upcase + " = " + ENV[k.upcase] + if /^\d+/.match(ENV[k.upcase]) + CONF[k] = Integer(ENV[k.upcase]) + else + CONF[k] = ENV[k.upcase] + end + end +end + +# Repository +# Example for testing a Release candidate. +repo = CONF['repo'] +# repo = "http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64" + +# Which Linux Distribution to use. Right now only centos is tested +distro = CONF['distro'] + +# number of instances +num_instances = CONF['num_instances'] + +# hadoop ecosystem components +components = CONF['components'] + +# Whether to run smoke tests +run_smoke_tests = CONF['run_smoke_tests'] + +# Smoke test Components to run +smoke_test_components = CONF['smoke_test_components'].join(',') + +# This is a update to allow dev packages +# Force success - not worried if this step fails, since we generally only use it for development. +enable_local_repo = CONF['enable_local_repo'] +puts "vagrant conf local repo enabled: #{enable_local_repo}" + +# JDK package name +jdk = CONF['jdk'] + +# master node hostname +bigtop_master = "bigtop1.vagrant" + +$script = <<SCRIPT +service iptables stop +service firewalld stop +chkconfig iptables off +# Remove 127.0.0.1 entry since vagrant's hostname setting will map it to FQDN, +# which miss leads some daemons to bind on 127.0.0.1 instead of public or private IP +sed -i "/127.0.0.1/d" /etc/hosts +echo "Bigtop yum repo = #{repo}" +# Prepare host manipulation files needed in standalone box +cp /bigtop-home/provisioner/vagrant/config_hosts /etc/init.d +cp /bigtop-home/provisioner/vagrant/gen_hosts.sh /usr/bin +chkconfig --add config_hosts +# Prepare puppet configuration file +mkdir -p /etc/puppet/hieradata +cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet +cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/ +cat > /etc/puppet/hieradata/site.yaml << EOF +bigtop::hadoop_head_node: #{bigtop_master} +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +bigtop::bigtop_repo_uri: #{repo} +hadoop_cluster_node::cluster_components: #{components} +bigtop::jdk_package_name: #{jdk} +hadoop::common_hdfs::testonly_hdfs_sshkeys: "yes" +EOF +SCRIPT + +VAGRANTFILE_API_VERSION = "2" +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager) + # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier) + config.hostmanager.enabled = true + + # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier) + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.scope = :box + end + + # nodes definition + (1..num_instances).each do |i| + config.vm.define "bigtop#{i}" do |bigtop| + + bigtop.vm.box = CONF['box'] + bigtop_hostname="bigtop#{i}" + bigtop_fqdn="#{bigtop_hostname}.vagrant" + bigtop_ip="10.10.10.1#{i}" + + bigtop.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", CONF['memory_size']] + vb.customize ['modifyvm', :id, '--cpus', CONF['number_cpus']] + end + + bigtop.vm.network :private_network, ip: bigtop_ip + bigtop.vm.hostname = bigtop_fqdn + + # two levels up is the bigtop "home" directory. + # the current directory has puppet recipes which we need for provisioning. + bigtop.vm.synced_folder "../../", "/bigtop-home" + + # We also add the bigtop-home output/ dir, so that locally built rpms will be available. + puts "Adding output/ repo ? #{enable_local_repo}" + + # Install puppet + bigtop.vm.provision :shell do |shell| + shell.path = "../../bigtop_toolchain/bin/puppetize.sh" + end + + # carry on w/ installation + bigtop.vm.provision :shell do |shell| + shell.path = "../utils/setup-env-" + distro + ".sh" + shell.args = ["#{enable_local_repo}"] + end + bigtop.vm.provision "shell", inline: $script + # Add the ip to FQDN and hostname mapping in /etc/hosts + bigtop.hostmanager.aliases = "#{bigtop_fqdn} #{bigtop_hostname}" + + # deploy Apache Hadoop and it's ecosystem + bigtop.vm.provision :puppet do |puppet| + puppet.module_path = "../../bigtop-deploy/puppet/modules/" + puppet.manifests_path = "../../bigtop-deploy/puppet/manifests/" + puppet.manifest_file = "." + puppet.options = "--parser future" + end + + if run_smoke_tests then + if i==num_instances then + puts "creating provisioner directive for running tests" + bigtop.vm.provision :shell do |shell| + shell.path = "../utils/smoke-tests.sh" + shell.args = ["#{smoke_test_components}"] + end + else + puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}" + end + end + + end + end + +end diff --git a/provisioner/vagrant/config_hosts b/provisioner/vagrant/config_hosts new file mode 100755 index 00000000..07e1d5ff --- /dev/null +++ b/provisioner/vagrant/config_hosts @@ -0,0 +1,45 @@ +#!/bin/bash + +# Hosts reconfiguration script +# chkconfig: 345 70 1 +# description: config_hosts + +# Licensed to Hortonworks, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Hortonworks, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +NAME="config_hosts" +LOG="/var/log/config_hosts.log" + +source /etc/rc.d/init.d/functions + +case "$1" in +start) + printf "%-50s\n" "Starting $NAME..." | tee -a $LOG + printf "%-50s\n" "Updating IP..." | tee -a $LOG + bash /usr/bin/gen_hosts.sh +;; + +stop) + date >> $LOG +;; + +restart) +;; + +*) + echo "Usage: $0 {start|stop|restart}" + exit 1 +esac diff --git a/provisioner/vagrant/gen_hosts.sh b/provisioner/vagrant/gen_hosts.sh new file mode 100755 index 00000000..c629c85a --- /dev/null +++ b/provisioner/vagrant/gen_hosts.sh @@ -0,0 +1,44 @@ +# Licensed to Hortonworks, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Hortonworks, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "# Do not remove the following line, or various programs" > /etc/hosts +echo "# that require network functionality will fail." >> /etc/hosts +echo "127.0.0.1 localhost.localdomain localhost" >> /etc/hosts + +function get_inet_iface(){ + route | grep default | awk '{print $8}' +} + +function get_ip() { + ip addr | grep 'inet ' | grep -E "($(get_inet_iface))" | awk '{ print $2 }' | awk -F'/' '{print $1}' +} + +HOST=$(get_ip) +NUM=5 +while [ -z "$HOST" ]; do + HOST=$(get_ip) + sleep 5 + NUM=$(($NUM-1)) + if [ $NUM -le 0 ]; then + HOST="127.0.0.1" + echo "Failed to update IP" + break + fi +done +echo "$HOST `hostname`" >> /etc/hosts + +echo 0 > /proc/sys/kernel/hung_task_timeout_secs +ethtool -K $(get_inet_iface) tso off diff --git a/provisioner/vagrant/vagrantconfig.yaml b/provisioner/vagrant/vagrantconfig.yaml new file mode 100644 index 00000000..4ad1aac2 --- /dev/null +++ b/provisioner/vagrant/vagrantconfig.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +memory_size: 4096 +number_cpus: 1 +box: "puppetlabs/centos-7.2-64-nocm" +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64" +num_instances: 1 +distro: centos +components: [hadoop, yarn, hbase] +enable_local_repo: false +run_smoke_tests: false +smoke_test_components: [mapreduce, pig] +jdk: "java-1.7.0-openjdk-devel.x86_64" |