From 3d4ed8c1ee278e553ae052a82bcecb3f189dd2f1 Mon Sep 17 00:00:00 2001 From: Evans Ye Date: Wed, 28 Dec 2016 01:31:54 +0800 Subject: BIGTOP-2646. Move Vagrant Provisioner from bigtop-deploy/vm to provisioner directory --- provisioner/utils/setup-env-centos.sh | 40 ++++++++ provisioner/utils/setup-env-debian.sh | 33 +++++++ provisioner/utils/smoke-tests.sh | 64 +++++++++++++ provisioner/vagrant/.gitignore | 1 + provisioner/vagrant/README.md | 99 ++++++++++++++++++++ provisioner/vagrant/Vagrantfile | 166 +++++++++++++++++++++++++++++++++ provisioner/vagrant/config_hosts | 45 +++++++++ provisioner/vagrant/gen_hosts.sh | 44 +++++++++ provisioner/vagrant/vagrantconfig.yaml | 26 ++++++ 9 files changed, 518 insertions(+) create mode 100755 provisioner/utils/setup-env-centos.sh create mode 100755 provisioner/utils/setup-env-debian.sh create mode 100755 provisioner/utils/smoke-tests.sh create mode 100644 provisioner/vagrant/.gitignore create mode 100644 provisioner/vagrant/README.md create mode 100755 provisioner/vagrant/Vagrantfile create mode 100755 provisioner/vagrant/config_hosts create mode 100755 provisioner/vagrant/gen_hosts.sh create mode 100644 provisioner/vagrant/vagrantconfig.yaml (limited to 'provisioner') diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh new file mode 100755 index 00000000..86b6a3c2 --- /dev/null +++ b/provisioner/utils/setup-env-centos.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable_local_repo=${1:-false} + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +yum -y install rng-tools +if [ -x /usr/bin/systemctl ] ; then + sed -i 's@ExecStart=/sbin/rngd -f@ExecStart=/sbin/rngd -f -r /dev/urandom@' /usr/lib/systemd/system/rngd.service + systemctl daemon-reload + systemctl start rngd +else + sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd + service rngd start +fi + +if [ $enable_local_repo == "true" ]; then + echo "Enabling local yum." + yum -y install yum-utils + sudo echo "gpgcheck=0" >> /etc/yum.conf + sudo yum-config-manager --add-repo file:///bigtop-home/output +else + echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." +fi + diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh new file mode 100755 index 00000000..098155b5 --- /dev/null +++ b/provisioner/utils/setup-env-debian.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable_local_repo=${1:-false} + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +apt-get -y install rng-tools +sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools +service rng-tools start + +if [ $enable_local_repo == "true" ]; then + echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list + apt-get update +else + apt-get install -y apt-transport-https + echo "local apt = $enable_local_repo ; NOT Enabling local apt. Packages will be pulled from remote..." +fi + diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh new file mode 100755 index 00000000..93f795e6 --- /dev/null +++ b/provisioner/utils/smoke-tests.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HCFS_USER="hdfs" +SMOKE_TESTS=${1:-mapreduce,pig} + +# Autodetect JAVA_HOME +if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then + . /usr/lib/bigtop-utils/bigtop-detect-javahome +else + >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n" + exit 2 +fi + +echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n" + +export HADOOP_CONF_DIR=/etc/hadoop/conf/ +export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ +export HIVE_HOME=/usr/lib/hive/ +export PIG_HOME=/usr/lib/pig/ +export FLUME_HOME=/usr/lib/flume/ +export SQOOP_HOME=/usr/lib/sqoop/ +export HIVE_CONF_DIR=/etc/hive/conf/ +export MAHOUT_HOME="/usr/lib/mahout" + +prep() { + HADOOP_COMMAND=$1 + su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -mkdir /user/vagrant /user/root" + su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/vagrant" + su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/root" +} + +prep hadoop +if [[ $SMOKE_TESTS == *"qfs"* ]]; then + prep hadoop-qfs +fi + +if [ -f /etc/debian_version ] ; then + apt-get -y install pig hive flume mahout sqoop +else + yum install -y pig hive flume mahout sqoop +fi +ALL_SMOKE_TASKS="" +for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do + ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test" +done +cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info +# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same +# permissions as used for smoke-tests execution +rm -rf buildSrc/build/test-results/binary + diff --git a/provisioner/vagrant/.gitignore b/provisioner/vagrant/.gitignore new file mode 100644 index 00000000..04204c7c --- /dev/null +++ b/provisioner/vagrant/.gitignore @@ -0,0 +1 @@ +config diff --git a/provisioner/vagrant/README.md b/provisioner/vagrant/README.md new file mode 100644 index 00000000..0b1ff20f --- /dev/null +++ b/provisioner/vagrant/README.md @@ -0,0 +1,99 @@ + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------------------------------------------------------------ + +#BigTop VM provisioner + +## Overview + +The Vagrantfile definition creates a bigtop virtual hadoop cluster for you, by pulling from existing publishing bigtop repositories. +This cluster can be used: + +- to test bigtop smoke tests +- to test bigtop puppet recipes + +Eventually, we may also add ability to build AND provision bigtop in a vagrant recipe, which would essentially +give full validation of the BigTop stack. + +## USAGE + +1) Install [Vagrant](https://www.vagrantup.com/downloads.html)(DO NOT install 1.8.5 because of a critical [bug](https://github.com/mitchellh/vagrant/issues/7631)) + +2) Install [vagrant-hostmanager plugin](https://github.com/smdahlen/vagrant-hostmanager) to manage `/etc/hosts` + +``` +vagrant plugin install vagrant-hostmanager +``` + +3) (Optional) Install [vagrant-cachier plugin](https://github.com/fgrehm/vagrant-cachier) to cache packages at local + +``` +vagrant plugin install vagrant-cachier +``` + +4) To provision a one node Apache Hadoop cluster on top of vagrant boxes + +``` +vagrant up +``` + +5) You can specify number of nodes you'd like to provision by modifying `num_instances` in vagrantconfig.yaml + +``` +num_instances: 5 +``` + +6) Test on local built packages is available by: + +first, build up local yum repo + +``` +cd bigtop; ./gradlew alluxio-yum +``` + +and then enable local yum in vagrantconfig.yaml + +``` +enable_local_repo: true +``` + +finally, spin up the cluster +``` +vagrant up +``` + +## Override Vagrant configurations +You can override vagrant configurations from environment variables: +``` +$ export REPO=http://repo.example.com/path/to +$ vagrant up +``` + +or + +``` +$ MEMORY_SIZE=8000 vagrant up +``` + +##Configure Apache Hadoop ecosystem components +* Choose the ecosystem you want to be deployed by modifying components in vagrantconfig.yaml + +``` +components: "hadoop, hbase, yarn,..." +``` + +By default, Apache Hadoop and YARN will be installed. +See `bigtop-deploy/puppet/hieradata/site.yaml` for more details. diff --git a/provisioner/vagrant/Vagrantfile b/provisioner/vagrant/Vagrantfile new file mode 100755 index 00000000..fc6bfc19 --- /dev/null +++ b/provisioner/vagrant/Vagrantfile @@ -0,0 +1,166 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "yaml" + +_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "vagrantconfig.yaml"), File::RDONLY).read) +CONF = _config + +# Override vagrant configurations using environment variables +keys = CONF.keys +keys.each do |k| + if ENV[k.upcase] != nil then + puts "Overide from environment variable: " + k.upcase + " = " + ENV[k.upcase] + if /^\d+/.match(ENV[k.upcase]) + CONF[k] = Integer(ENV[k.upcase]) + else + CONF[k] = ENV[k.upcase] + end + end +end + +# Repository +# Example for testing a Release candidate. +repo = CONF['repo'] +# repo = "http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64" + +# Which Linux Distribution to use. Right now only centos is tested +distro = CONF['distro'] + +# number of instances +num_instances = CONF['num_instances'] + +# hadoop ecosystem components +components = CONF['components'] + +# Whether to run smoke tests +run_smoke_tests = CONF['run_smoke_tests'] + +# Smoke test Components to run +smoke_test_components = CONF['smoke_test_components'].join(',') + +# This is a update to allow dev packages +# Force success - not worried if this step fails, since we generally only use it for development. +enable_local_repo = CONF['enable_local_repo'] +puts "vagrant conf local repo enabled: #{enable_local_repo}" + +# JDK package name +jdk = CONF['jdk'] + +# master node hostname +bigtop_master = "bigtop1.vagrant" + +$script = <