diff options
Diffstat (limited to 'provisioner')
-rw-r--r-- | provisioner/docker/.gitignore | 1 | ||||
-rw-r--r-- | provisioner/docker/README.md | 131 | ||||
-rw-r--r-- | provisioner/docker/config.yaml | 32 | ||||
-rw-r--r-- | provisioner/docker/config_debian.yaml | 32 | ||||
-rw-r--r-- | provisioner/docker/config_ubuntu.yaml | 32 | ||||
-rw-r--r-- | provisioner/docker/docker-compose.yml | 10 | ||||
-rwxr-xr-x | provisioner/docker/docker-hadoop.sh | 179 | ||||
-rwxr-xr-x | provisioner/utils/setup-env-centos.sh | 36 | ||||
-rwxr-xr-x | provisioner/utils/setup-env-debian.sh | 34 | ||||
-rwxr-xr-x | provisioner/utils/smoke-tests.sh | 56 |
10 files changed, 543 insertions, 0 deletions
diff --git a/provisioner/docker/.gitignore b/provisioner/docker/.gitignore new file mode 100644 index 00000000..f733c4b5 --- /dev/null +++ b/provisioner/docker/.gitignore @@ -0,0 +1 @@ +config/ diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md new file mode 100644 index 00000000..51bac857 --- /dev/null +++ b/provisioner/docker/README.md @@ -0,0 +1,131 @@ + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------------------------------------------------------------ + +#BigTop Docker provisioner + +## Overview + +The Docker Compose definition and wrapper script that creates Bigtop virtual Hadoop cluster on top of Docker containers for you, by pulling from existing publishing bigtop repositories. +This cluster can be used: + +- to test bigtop smoke tests +- to test bigtop puppet recipes +- to run integration test with your application + +This has been verified on Docker Engine 1.9.1, with api version 1.15, and Docker Compose 1.5.2 on Amazon Linux 2015.09 release. + +## Prerequisites + +### OS X and Windows + +* Install [Docker Toolbox](https://www.docker.com/docker-toolbox) + +### Linux + +* Install [Docker](https://docs.docker.com/installation/) + +* Install [Docker Compose](https://docs.docker.com/compose/install/) + +* Start the Docker daemon + +``` +service docker start +``` + +## USAGE + +1) Create a Bigtop Hadoop cluster by given # of node. + +``` +./docker-hadoop.sh --create 3 +``` + +2) Destroy the cluster. + +``` +./docker-hadoop.sh --destroy +``` + +3) Update your cluster after doing configuration changes on ./config. (re-run puppet apply) + +``` +./docker-hadoop.sh --provision +``` + +4) Run Bigtop smoke tests + +``` +./docker-hadoop.sh --smoke-tests +``` + +5) Chain your operations with-in one command. + +``` +./docker-hadoop.sh --create 5 --smoke-tests --destroy +``` + +Commands will be executed by following order: + +``` +create 5 node cluster => run smoke tests => destroy the cluster +``` + +6) See helper message: + +``` +./docker-hadoop.sh -h +usage: docker-hadoop.sh [-C file ] args + -C file Use alternate file for config.yaml + commands: + -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster + -p, --provision Deploy configuration changes + -s, --smoke-tests Run Bigtop smoke tests + -d, --destroy Destroy the cluster + -h, --help +``` + +##Configurations + +* There are several parameters can be configured in config.yaml: + +1) Modify memory limit for Docker containers + +``` +docker: + memory_size: "2048" + +``` + +2) Use different host ports mapping for web UIs + +``` +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" + +``` +Note: If running on OS X or Windows, the boot2docker VM should be reloaded after ports changed + + +##Configure Apache Hadoop ecosystem components +* Choose the ecosystem you want to be deployed by modifying components in config.yaml + +``` +components: "hadoop, hbase, yarn,..." +``` + +By default, Apache Hadoop and YARN will be installed. diff --git a/provisioner/docker/config.yaml b/provisioner/docker/config.yaml new file mode 100644 index 00000000..86f35e75 --- /dev/null +++ b/provisioner/docker/config.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_size: "4096" + image: "bigtop/deploy:centos-6" + +boot2docker: + memory_size: "4096" + number_cpus: "1" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/6/x86_64" +distro: centos +components: [hadoop, yarn] +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "java-1.7.0-openjdk-devel.x86_64" diff --git a/provisioner/docker/config_debian.yaml b/provisioner/docker/config_debian.yaml new file mode 100644 index 00000000..f6b11aea --- /dev/null +++ b/provisioner/docker/config_debian.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_size: "4096" + image: "bigtop/deploy:debian-8" + +boot2docker: + memory_size: "4096" + number_cpus: "1" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/debian/8/x86_64" +distro: debian +components: [hadoop, yarn] +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "openjdk-7-jdk" diff --git a/provisioner/docker/config_ubuntu.yaml b/provisioner/docker/config_ubuntu.yaml new file mode 100644 index 00000000..e4ea6f3b --- /dev/null +++ b/provisioner/docker/config_ubuntu.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_size: "4096" + image: "bigtop/deploy:ubuntu-14.04" + +boot2docker: + memory_size: "4096" + number_cpus: "1" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/ubuntu/trusty/x86_64" +distro: debian +components: [hadoop, yarn] +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "openjdk-7-jdk" diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml new file mode 100644 index 00000000..95831d10 --- /dev/null +++ b/provisioner/docker/docker-compose.yml @@ -0,0 +1,10 @@ +bigtop: + image: ${DOCKER_IMAGE} + command: sleep infinity + domainname: bigtop.apache.org + privileged: true + volumes: + - ../../:/bigtop-home + - ./config/hiera.yaml:/etc/puppet/hiera.yaml + - ./config/hieradata:/etc/puppet/hieradata + - ./config/hosts:/etc/hosts diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh new file mode 100755 index 00000000..e8bd767e --- /dev/null +++ b/provisioner/docker/docker-hadoop.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BIGTOP_PUPPET_DIR=../../bigtop-deploy/puppet + +usage() { + echo "usage: $PROG [-C file ] args" + echo " -C file Use alternate file for config.yaml" + echo " commands:" + echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster" + echo " -p, --provision Deploy configuration changes" + echo " -s, --smoke-tests Run Bigtop smoke tests" + echo " -d, --destroy Destroy the cluster" + echo " -h, --help" + exit 1 +} + +create() { + # Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon + mkdir config 2> /dev/null + cat /dev/null > ./config/hiera.yaml + cat /dev/null > ./config/hosts + export DOCKER_IMAGE=$(get-yaml-config docker image) + + # Startup instances + docker-compose scale bigtop=$1 + if [ $? -ne 0 ]; then + echo "Docker container(s) startup failed!"; + exit 1; + fi + + # Get the headnode FQDN + nodes=(`docker-compose ps -q`) + hadoop_head_node=`docker inspect --format {{.Config.Hostname}}.{{.Config.Domainname}} ${nodes[0]}` + + # Fetch configurations form specificed yaml config file + repo=$(get-yaml-config repo) + components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]" + jdk=$(get-yaml-config jdk) + distro=$(get-yaml-config distro) + enable_local_repo=$(get-yaml-config enable_local_repo) + generate-config "$hadoop_head_node" "$repo" "$components" "$jdk" + + # Start provisioning + generate-hosts + bootstrap $distro $enable_local_repo + provision +} + +generate-hosts() { + nodes=(`docker-compose ps -q`) + for node in ${nodes[*]}; do + echo `docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node` >> ./config/hosts + done + wait + +} + +generate-config() { + echo "Bigtop Puppet configurations are shared between instances, and can be modified under config/hieradata" + cat $BIGTOP_PUPPET_DIR/hiera.yaml > ./config/hiera.yaml + yes | cp -vr $BIGTOP_PUPPET_DIR/hieradata ./config/ + cat > ./config/hieradata/site.yaml << EOF +bigtop::hadoop_head_node: $1 +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +bigtop::bigtop_repo_uri: $2 +hadoop_cluster_node::cluster_components: $3 +bigtop::jdk_package_name: $4 +EOF +} + +copy-to-instances() { + nodes=(`docker-compose ps -q`) + for node in ${nodes[*]}; do + docker cp $1 $node:$2 & + done + wait +} + +bootstrap() { + nodes=(`docker-compose ps -q`) + for node in ${nodes[*]}; do + docker exec $node bash -c "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$1.sh $2" & + done + wait +} + +provision() { + nodes=(`docker-compose ps -q`) + for node in ${nodes[*]}; do + bigtop-puppet $node & + done + wait +} + +smoke-tests() { + nodes=(`docker-compose ps -q`) + hadoop_head_node=${nodes:0:12} + smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`" + docker exec $hadoop_head_node bash -c "bash -x /bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh $smoke_test_components" +} + +destroy() { + docker-compose stop + docker-compose rm -f + rm -rvf ./config +} + +bigtop-puppet() { + docker exec $1 bash -c 'puppet apply -d --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp' +} + +get-yaml-config() { + RUBY_EXE=ruby + if [ $# -eq 1 ]; then + RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1'];" + elif [ $# -eq 2 ]; then + RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1']['$2'];" + else + echo "The yaml config retrieval function can only take 1 or 2 parameters."; + exit 1; + fi + cat ${yamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r' +} + +PROG=`basename $0` + +if [ $# -eq 0 ]; then + usage +fi + +yamlconf="config.yaml" +while [ $# -gt 0 ]; do + case "$1" in + -c|--create) + if [ $# -lt 2 ]; then + echo "Create requires a number" 1>&2 + usage + fi + create $2 + shift 2;; + -C|--conf) + if [ $# -lt 2 ]; then + echo "Alternative config file for config.yaml" 1>&2 + usage + fi + yamlconf=$2 + shift 2;; + -p|--provision) + provision + shift;; + -s|--smoke-tests) + smoke-tests + shift;; + -d|--destroy) + destroy + shift;; + -h|--help) + usage + shift;; + *) + echo "Unknown argument: '$1'" 1>&2 + usage;; + esac +done diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh new file mode 100755 index 00000000..bb25d286 --- /dev/null +++ b/provisioner/utils/setup-env-centos.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable_local_repo=${1:-false} + +bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +yum -y install rng-tools +sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd +service rngd start + +if [ $enable_local_repo == "true" ]; then + echo "Enabling local yum." + yum -y install yum-utils + sudo echo "gpgcheck=0" >> /etc/yum.conf + sudo yum-config-manager --add-repo file:///bigtop-home/output +else + echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." +fi + diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh new file mode 100755 index 00000000..c5768d9a --- /dev/null +++ b/provisioner/utils/setup-env-debian.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable_local_repo=${1:-false} + +bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +apt-get -y install rng-tools +sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools +service rng-tools start + +if [ $enable_local_repo == "true" ]; then + echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list + apt-get update +else + echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." +fi + diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh new file mode 100755 index 00000000..8dac31c0 --- /dev/null +++ b/provisioner/utils/smoke-tests.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HCFS_USER="hdfs" +SMOKE_TESTS=${1:-mapreduce,pig} + +# Autodetect JAVA_HOME +if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then + . /usr/lib/bigtop-utils/bigtop-detect-javahome +else + >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n" + exit 2 +fi + +echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n" + +export HADOOP_CONF_DIR=/etc/hadoop/conf/ +export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ +export HIVE_HOME=/usr/lib/hive/ +export PIG_HOME=/usr/lib/pig/ +export FLUME_HOME=/usr/lib/flume/ +export SQOOP_HOME=/usr/lib/sqoop/ +export HIVE_CONF_DIR=/etc/hive/conf/ +export MAHOUT_HOME="/usr/lib/mahout" + +su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' + +if [ -f /etc/debian_version ] ; then + apt-get -y install pig hive flume mahout sqoop +else + yum install -y pig hive flume mahout sqoop +fi +ALL_SMOKE_TASKS="" +for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do + ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test" +done +cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info +# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same +# permissions as used for smoke-tests execution +rm -rf buildSrc/build/test-results/binary + |