From 40a52870b5a4694ae3af9cf59734a8b2d35675ec Mon Sep 17 00:00:00 2001 From: Olaf Flebbe Date: Sat, 23 Jul 2016 12:34:12 +0200 Subject: BIGTOP-2505: Support systemd containers, fix exec (-e) flag --- provisioner/docker/config_centos7.yaml | 32 +++++++++++++++++++ provisioner/docker/docker-compose.yml | 2 +- provisioner/docker/docker-hadoop.sh | 10 +++--- provisioner/utils/setup-env-centos.sh | 36 ---------------------- provisioner/utils/setup-env-debian.sh | 34 --------------------- provisioner/utils/smoke-tests.sh | 56 ---------------------------------- 6 files changed, 38 insertions(+), 132 deletions(-) create mode 100644 provisioner/docker/config_centos7.yaml delete mode 100755 provisioner/utils/setup-env-centos.sh delete mode 100755 provisioner/utils/setup-env-debian.sh delete mode 100755 provisioner/utils/smoke-tests.sh diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml new file mode 100644 index 00000000..3610a8f1 --- /dev/null +++ b/provisioner/docker/config_centos7.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_size: "4096" + image: "bigtop/deploy:centos-7" + +boot2docker: + memory_size: "4096" + number_cpus: "1" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/7/x86_64" +distro: centos +components: [hadoop, yarn] +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "java-1.7.0-openjdk-devel.x86_64" diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml index 95831d10..d71b2477 100644 --- a/provisioner/docker/docker-compose.yml +++ b/provisioner/docker/docker-compose.yml @@ -1,6 +1,6 @@ bigtop: image: ${DOCKER_IMAGE} - command: sleep infinity + command: /sbin/init domainname: bigtop.apache.org privileged: true volumes: diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh index 89ac6bbc..4b1c0df5 100755 --- a/provisioner/docker/docker-hadoop.sh +++ b/provisioner/docker/docker-hadoop.sh @@ -41,8 +41,8 @@ create() { PROVISION_ID=`cat .provision_id` # Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon mkdir -p config/hieradata 2> /dev/null - cat /dev/null > ./config/hiera.yaml - cat /dev/null > ./config/hosts + echo > ./config/hiera.yaml + echo > ./config/hosts export DOCKER_IMAGE=$(get-yaml-config docker image) # Startup instances @@ -81,8 +81,8 @@ generate-hosts() { generate-config() { echo "Bigtop Puppet configurations are shared between instances, and can be modified under config/hieradata" - cat $BIGTOP_PUPPET_DIR/hiera.yaml > ./config/hiera.yaml - yes | cp -vr $BIGTOP_PUPPET_DIR/hieradata ./config/ + cat $BIGTOP_PUPPET_DIR/hiera.yaml >> ./config/hiera.yaml + cp -vfr $BIGTOP_PUPPET_DIR/hieradata ./config/ cat > ./config/hieradata/site.yaml << EOF bigtop::hadoop_head_node: $1 hadoop::hadoop_storage_dirs: [/data/1, /data/2] @@ -185,7 +185,7 @@ if [ -e .provision_id ]; then PROVISION_ID=`cat .provision_id` fi if [ -n "$PROVISION_ID" ]; then - NODES=`docker-compose -p $PROVISION_ID ps -q` + NODES=(`docker-compose -p $PROVISION_ID ps -q`) fi while [ $# -gt 0 ]; do diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh deleted file mode 100755 index bb25d286..00000000 --- a/provisioner/utils/setup-env-centos.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -enable_local_repo=${1:-false} - -bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh - -# Setup rng-tools to improve virtual machine entropy performance. -# The poor entropy performance will cause kerberos provisioning failed. -yum -y install rng-tools -sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd -service rngd start - -if [ $enable_local_repo == "true" ]; then - echo "Enabling local yum." - yum -y install yum-utils - sudo echo "gpgcheck=0" >> /etc/yum.conf - sudo yum-config-manager --add-repo file:///bigtop-home/output -else - echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." -fi - diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh deleted file mode 100755 index c5768d9a..00000000 --- a/provisioner/utils/setup-env-debian.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -enable_local_repo=${1:-false} - -bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh - -# Setup rng-tools to improve virtual machine entropy performance. -# The poor entropy performance will cause kerberos provisioning failed. -apt-get -y install rng-tools -sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools -service rng-tools start - -if [ $enable_local_repo == "true" ]; then - echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list - apt-get update -else - echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." -fi - diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh deleted file mode 100755 index 8dac31c0..00000000 --- a/provisioner/utils/smoke-tests.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HCFS_USER="hdfs" -SMOKE_TESTS=${1:-mapreduce,pig} - -# Autodetect JAVA_HOME -if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then - . /usr/lib/bigtop-utils/bigtop-detect-javahome -else - >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n" - exit 2 -fi - -echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n" - -export HADOOP_CONF_DIR=/etc/hadoop/conf/ -export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ -export HIVE_HOME=/usr/lib/hive/ -export PIG_HOME=/usr/lib/pig/ -export FLUME_HOME=/usr/lib/flume/ -export SQOOP_HOME=/usr/lib/sqoop/ -export HIVE_CONF_DIR=/etc/hive/conf/ -export MAHOUT_HOME="/usr/lib/mahout" - -su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' -su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' -su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' - -if [ -f /etc/debian_version ] ; then - apt-get -y install pig hive flume mahout sqoop -else - yum install -y pig hive flume mahout sqoop -fi -ALL_SMOKE_TASKS="" -for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do - ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test" -done -cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info -# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same -# permissions as used for smoke-tests execution -rm -rf buildSrc/build/test-results/binary - -- cgit v1.2.3