BIGTOP-2254. Replace the Docker orchestration tool from Vagrant to Docker Compose

10 files changed, 543 insertions, 0 deletions

diff --git a/provisioner/docker/.gitignore b/provisioner/docker/.gitignore
new file mode 100644
index 00000000..f733c4b5
--- /dev/null
+++ b/provisioner/docker/.gitignore
@@ -0,0 +1 @@
+config/
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
new file mode 100644
index 00000000..51bac857
--- /dev/null
+++ b/provisioner/docker/README.md
@@ -0,0 +1,131 @@
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements. See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+------------------------------------------------------------------------------------------------------------------------------------------------------
+
+#BigTop Docker provisioner
+
+## Overview
+
+The Docker Compose definition and wrapper script that creates Bigtop virtual Hadoop cluster on top of Docker containers for you, by pulling from existing publishing bigtop repositories.
+This cluster can be used:
+
+- to test bigtop smoke tests
+- to test bigtop puppet recipes
+- to run integration test with your application
+
+This has been verified on Docker Engine 1.9.1, with api version 1.15, and Docker Compose 1.5.2 on Amazon Linux 2015.09 release.
+
+## Prerequisites
+
+### OS X and Windows
+
+* Install [Docker Toolbox](https://www.docker.com/docker-toolbox)
+
+### Linux
+
+* Install [Docker](https://docs.docker.com/installation/)
+
+* Install [Docker Compose](https://docs.docker.com/compose/install/)
+
+* Start the Docker daemon
+
+```
+service docker start
+```
+
+## USAGE
+
+1) Create a Bigtop Hadoop cluster by given # of node.
+
+```
+./docker-hadoop.sh --create 3
+```
+
+2) Destroy the cluster.
+
+```
+./docker-hadoop.sh --destroy
+```
+
+3) Update your cluster after doing configuration changes on ./config. (re-run puppet apply)
+
+```
+./docker-hadoop.sh --provision
+```
+
+4) Run Bigtop smoke tests
+
+```
+./docker-hadoop.sh --smoke-tests
+```
+
+5) Chain your operations with-in one command.
+
+```
+./docker-hadoop.sh --create 5 --smoke-tests --destroy
+```
+
+Commands will be executed by following order:
+
+```
+create 5 node cluster => run smoke tests => destroy the cluster
+```
+
+6) See helper message:
+
+```
+./docker-hadoop.sh -h
+usage: docker-hadoop.sh [-C file ] args
+       -C file                                   Use alternate file for config.yaml
+  commands:
+       -c NUM_INSTANCES, --create=NUM_INSTANCES  Create a Docker based Bigtop Hadoop cluster
+       -p, --provision                           Deploy configuration changes
+       -s, --smoke-tests                         Run Bigtop smoke tests
+       -d, --destroy                             Destroy the cluster
+       -h, --help
+```
+
+##Configurations
+
+* There are several parameters can be configured in config.yaml:
+
+1) Modify memory limit for Docker containers
+
+```
+docker:
+        memory_size: "2048"
+
+```
+
+2) Use different host ports mapping for web UIs
+
+```
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+
+```
+Note: If running on OS X or Windows, the boot2docker VM should be reloaded after ports changed
+
+
+##Configure Apache Hadoop ecosystem components
+* Choose the ecosystem you want to be deployed by modifying components in config.yaml
+
+```
+components: "hadoop, hbase, yarn,..."
+```
+
+By default, Apache Hadoop and YARN will be installed.
diff --git a/provisioner/docker/config.yaml b/provisioner/docker/config.yaml
new file mode 100644
index 00000000..86f35e75
--- /dev/null
+++ b/provisioner/docker/config.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+        memory_size: "4096"
+        image: "bigtop/deploy:centos-6"
+
+boot2docker:
+        memory_size: "4096"
+        number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/6/x86_64"
+distro: centos
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "java-1.7.0-openjdk-devel.x86_64"
diff --git a/provisioner/docker/config_debian.yaml b/provisioner/docker/config_debian.yaml
new file mode 100644
index 00000000..f6b11aea
--- /dev/null
+++ b/provisioner/docker/config_debian.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+        memory_size: "4096"
+        image:  "bigtop/deploy:debian-8"
+
+boot2docker:
+        memory_size: "4096"
+        number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/debian/8/x86_64"
+distro: debian
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "openjdk-7-jdk"
diff --git a/provisioner/docker/config_ubuntu.yaml b/provisioner/docker/config_ubuntu.yaml
new file mode 100644
index 00000000..e4ea6f3b
--- /dev/null
+++ b/provisioner/docker/config_ubuntu.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+        memory_size: "4096"
+        image:  "bigtop/deploy:ubuntu-14.04"
+
+boot2docker:
+        memory_size: "4096"
+        number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/ubuntu/trusty/x86_64"
+distro: debian
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "openjdk-7-jdk"
diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml
new file mode 100644
index 00000000..95831d10
--- /dev/null
+++ b/provisioner/docker/docker-compose.yml
@@ -0,0 +1,10 @@
+bigtop:
+    image: ${DOCKER_IMAGE}
+    command: sleep infinity
+    domainname: bigtop.apache.org
+    privileged: true
+    volumes:
+    - ../../:/bigtop-home
+    - ./config/hiera.yaml:/etc/puppet/hiera.yaml
+    - ./config/hieradata:/etc/puppet/hieradata
+    - ./config/hosts:/etc/hosts
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
new file mode 100755
index 00000000..e8bd767e
--- /dev/null
+++ b/provisioner/docker/docker-hadoop.sh
@@ -0,0 +1,179 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BIGTOP_PUPPET_DIR=../../bigtop-deploy/puppet
+
+usage() {
+    echo "usage: $PROG [-C file ] args"
+    echo "       -C file                                   Use alternate file for config.yaml"
+    echo "  commands:"
+    echo "       -c NUM_INSTANCES, --create=NUM_INSTANCES  Create a Docker based Bigtop Hadoop cluster"
+    echo "       -p, --provision                           Deploy configuration changes"
+    echo "       -s, --smoke-tests                         Run Bigtop smoke tests"
+    echo "       -d, --destroy                             Destroy the cluster"
+    echo "       -h, --help"
+    exit 1
+}
+
+create() {
+    # Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
+    mkdir config 2> /dev/null
+    cat /dev/null > ./config/hiera.yaml
+    cat /dev/null > ./config/hosts
+    export DOCKER_IMAGE=$(get-yaml-config docker image)
+
+    # Startup instances
+    docker-compose scale bigtop=$1
+    if [ $? -ne 0 ]; then
+        echo "Docker container(s) startup failed!";
+        exit 1;
+    fi
+
+    # Get the headnode FQDN
+    nodes=(`docker-compose ps -q`)
+    hadoop_head_node=`docker inspect --format {{.Config.Hostname}}.{{.Config.Domainname}} ${nodes[0]}`
+
+    # Fetch configurations form specificed yaml config file
+    repo=$(get-yaml-config repo)
+    components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]"
+    jdk=$(get-yaml-config jdk)
+    distro=$(get-yaml-config distro)
+    enable_local_repo=$(get-yaml-config enable_local_repo)
+    generate-config "$hadoop_head_node" "$repo" "$components" "$jdk"
+
+    # Start provisioning
+    generate-hosts
+    bootstrap $distro $enable_local_repo
+    provision
+}
+
+generate-hosts() {
+    nodes=(`docker-compose ps -q`)
+    for node in ${nodes[*]}; do
+        echo `docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node` >> ./config/hosts
+    done
+    wait
+
+}
+
+generate-config() {
+    echo "Bigtop Puppet configurations are shared between instances, and can be modified under config/hieradata"
+    cat $BIGTOP_PUPPET_DIR/hiera.yaml > ./config/hiera.yaml
+    yes | cp -vr $BIGTOP_PUPPET_DIR/hieradata ./config/
+    cat > ./config/hieradata/site.yaml << EOF
+bigtop::hadoop_head_node: $1
+hadoop::hadoop_storage_dirs: [/data/1, /data/2]
+bigtop::bigtop_repo_uri: $2
+hadoop_cluster_node::cluster_components: $3
+bigtop::jdk_package_name: $4
+EOF
+}
+
+copy-to-instances() {
+    nodes=(`docker-compose ps -q`)
+    for node in ${nodes[*]}; do
+        docker cp  $1 $node:$2 &
+    done
+    wait
+}
+
+bootstrap() {
+    nodes=(`docker-compose ps -q`)
+    for node in ${nodes[*]}; do
+        docker exec $node bash -c "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$1.sh $2" &
+    done
+    wait
+}
+
+provision() {
+    nodes=(`docker-compose ps -q`)
+    for node in ${nodes[*]}; do
+        bigtop-puppet $node &
+    done
+    wait
+}
+
+smoke-tests() {
+    nodes=(`docker-compose ps -q`)
+    hadoop_head_node=${nodes:0:12}
+    smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`"
+    docker exec $hadoop_head_node bash -c "bash -x /bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh $smoke_test_components"
+}
+
+destroy() {
+    docker-compose stop
+    docker-compose rm -f
+    rm -rvf ./config
+}
+
+bigtop-puppet() {
+    docker exec $1 bash -c 'puppet apply -d --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp'
+}
+
+get-yaml-config() {
+    RUBY_EXE=ruby
+    if [ $# -eq 1 ]; then
+        RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1'];"
+    elif [ $# -eq 2 ]; then
+        RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1']['$2'];"
+    else
+        echo "The yaml config retrieval function can only take 1 or 2 parameters.";
+        exit 1;
+    fi
+    cat ${yamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r'
+}
+
+PROG=`basename $0`
+
+if [ $# -eq 0 ]; then
+    usage
+fi
+
+yamlconf="config.yaml"
+while [ $# -gt 0 ]; do
+    case "$1" in
+    -c|--create)
+        if [ $# -lt 2 ]; then
+          echo "Create requires a number" 1>&2
+          usage
+        fi
+        create $2
+        shift 2;;
+    -C|--conf)
+        if [ $# -lt 2 ]; then
+          echo "Alternative config file for config.yaml" 1>&2
+          usage
+        fi
+	yamlconf=$2
+        shift 2;;
+    -p|--provision)
+        provision
+        shift;;
+    -s|--smoke-tests)
+        smoke-tests
+        shift;;
+    -d|--destroy)
+        destroy
+        shift;;
+    -h|--help)
+        usage
+        shift;;
+    *)
+        echo "Unknown argument: '$1'" 1>&2
+        usage;;
+    esac
+done
diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh
new file mode 100755
index 00000000..bb25d286
--- /dev/null
+++ b/provisioner/utils/setup-env-centos.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+yum -y install rng-tools
+sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
+service rngd start
+
+if [ $enable_local_repo == "true" ]; then
+    echo "Enabling local yum."
+    yum -y install yum-utils
+    sudo echo "gpgcheck=0" >> /etc/yum.conf
+    sudo yum-config-manager --add-repo file:///bigtop-home/output
+else
+    echo "local yum = $enable_local_repo ; NOT Enabling local yum.  Packages will be pulled from remote..."
+fi
+
diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh
new file mode 100755
index 00000000..c5768d9a
--- /dev/null
+++ b/provisioner/utils/setup-env-debian.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+apt-get -y install rng-tools
+sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools
+service rng-tools start
+
+if [ $enable_local_repo == "true" ]; then
+    echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list
+    apt-get update
+else
+    echo "local yum = $enable_local_repo ; NOT Enabling local yum.  Packages will be pulled from remote..."
+fi
+
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
new file mode 100755
index 00000000..8dac31c0
--- /dev/null
+++ b/provisioner/utils/smoke-tests.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HCFS_USER="hdfs"
+SMOKE_TESTS=${1:-mapreduce,pig}
+
+# Autodetect JAVA_HOME
+if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+else
+  >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n"
+  exit 2
+fi
+
+echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
+
+export HADOOP_CONF_DIR=/etc/hadoop/conf/
+export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
+export HIVE_HOME=/usr/lib/hive/
+export PIG_HOME=/usr/lib/pig/
+export FLUME_HOME=/usr/lib/flume/
+export SQOOP_HOME=/usr/lib/sqoop/
+export HIVE_CONF_DIR=/etc/hive/conf/
+export MAHOUT_HOME="/usr/lib/mahout"
+
+su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root'
+su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant'
+su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root'
+
+if [ -f /etc/debian_version ] ; then
+    apt-get -y install pig hive flume mahout sqoop
+else
+    yum install -y pig hive flume mahout sqoop
+fi
+ALL_SMOKE_TASKS=""
+for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do
+  ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
+done
+cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info
+# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same
+# permissions as used for smoke-tests execution
+rm -rf buildSrc/build/test-results/binary
+