diff options
| -rw-r--r-- | README.md | 5 | ||||
| -rwxr-xr-x | bigtop-deploy/vm/smoke-tests.sh | 3 | ||||
| -rwxr-xr-x | bigtop-deploy/vm/vagrant-puppet/provision.sh | 3 | ||||
| -rw-r--r-- | bigtop-tests/smoke-tests/README | 5 | ||||
| -rw-r--r-- | bigtop-tests/smoke-tests/build.gradle | 5 | ||||
| -rw-r--r-- | bigtop-tests/smoke-tests/gradle/wrapper/gradle-wrapper.properties | 18 | ||||
| -rwxr-xr-x | bigtop-tests/smoke-tests/gradlew | 197 | ||||
| -rw-r--r-- | build.gradle | 1 | ||||
| -rw-r--r-- | gradle/wrapper/gradle-wrapper.properties | 18 | ||||
| -rwxr-xr-x | gradlew | 197 |
10 files changed, 438 insertions, 14 deletions
@@ -75,9 +75,8 @@ For Users: Running the smoke tests. The simplest way to test bigtop is to: -* Step 1: Install gradle. -* Step 2: cd bigtop-tests/smoke-tests/ -* Step 3: Follow the instructions in the smoke-tests/README file. +* Step 1: cd bigtop-tests/smoke-tests/ +* Step 2: Follow the instructions in the smoke-tests/README file. For integration (API level) testing with maven, read on. diff --git a/bigtop-deploy/vm/smoke-tests.sh b/bigtop-deploy/vm/smoke-tests.sh index 9ee111ab..cfb40505 100755 --- a/bigtop-deploy/vm/smoke-tests.sh +++ b/bigtop-deploy/vm/smoke-tests.sh @@ -1,6 +1,5 @@ #!/bin/bash HCFS_USER="hdfs" -gradle="/usr/local/gradle/bin/gradle" export HADOOP_CONF_DIR=/etc/hadoop/conf/ export BIGTOP_HOME=/bigtop-home/ @@ -18,4 +17,4 @@ su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' yum install -y pig hive flume mahout -cd /bigtop-home/bigtop-tests/smoke-tests && $gradle compileGroovy test -Dsmoke.tests=pig --info +cd /bigtop-home/bigtop-tests/smoke-tests && ./gradlew compileGroovy test -Dsmoke.tests=pig --info diff --git a/bigtop-deploy/vm/vagrant-puppet/provision.sh b/bigtop-deploy/vm/vagrant-puppet/provision.sh index 2b660c2e..4ab42d94 100755 --- a/bigtop-deploy/vm/vagrant-puppet/provision.sh +++ b/bigtop-deploy/vm/vagrant-puppet/provision.sh @@ -41,6 +41,3 @@ mkdir -p /data/{1,2} yum -y install rng-tools sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd service rngd start - -echo "Now installing gradle" -cd /bigtop-home && puppet apply --modulepath=./ -e "include bigtop_toolchain::gradle" # alias gradle=/usr/local/gradle/bin/gradle diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README index f682b616..2f7ec41d 100644 --- a/bigtop-tests/smoke-tests/README +++ b/bigtop-tests/smoke-tests/README @@ -9,8 +9,7 @@ customize and test Apache BigTop's hadoop distribution. - Debug tests easily and make logging extremely transparent. - Geneate easy to read test reports. - Run bigtop tests without needing a jar file. - - (todo) Run bigtop tests easily, ideally without gradle/maven - (this can be done with gradle wrapper) + - Run bigtop tests easily, ideally without gradle/maven So overall, this framework makes testing with apache bigtop easy as well as fun! @@ -34,7 +33,7 @@ Each directory is a gradle "subproject" mapping to an ecosystem component. If you specify, for example, "flume", then the flume test runs. To test hive as well, you would specify "flume,hive", and so on. An example is below: ``` - gradle compileGroovy test -Dsmoke.tests=flume,hive --info + ./gradlew compileGroovy test -Dsmoke.tests=flume,hive --info ``` The --info option is a nice feature to have here: You can easily debug your tests this way, diff --git a/bigtop-tests/smoke-tests/build.gradle b/bigtop-tests/smoke-tests/build.gradle index 44a52e8e..66ce596e 100644 --- a/bigtop-tests/smoke-tests/build.gradle +++ b/bigtop-tests/smoke-tests/build.gradle @@ -14,7 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */apply plugin: 'groovy' + */ +apply plugin: 'groovy' repositories { mavenCentral() @@ -67,7 +68,7 @@ subprojects { /** * Two important environment variables. * java_home should be declared in all cases. - * itest can equal = 0.7.0, 0.8.0-SNAPSHOT, + * itest can be greater than or equal to = 0.7.0, 0.8.0 */ checkEnv(["JAVA_HOME","ITEST"]) ext.itestVersion = System.getenv("ITEST") diff --git a/bigtop-tests/smoke-tests/gradle/wrapper/gradle-wrapper.properties b/bigtop-tests/smoke-tests/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..7dcaad82 --- /dev/null +++ b/bigtop-tests/smoke-tests/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +distributionPath=wrapper/dists +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.1-bin.zip diff --git a/bigtop-tests/smoke-tests/gradlew b/bigtop-tests/smoke-tests/gradlew new file mode 100755 index 00000000..2ad3dae8 --- /dev/null +++ b/bigtop-tests/smoke-tests/gradlew @@ -0,0 +1,197 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +## Tries to recreate Gradle's gradlew command in pure bash. +## This way you don't have to worry about binaries in your build. +## +## Depdencies +## unzip +## + +set -e +set -o pipefail + + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="-Dorg.gradle.appname=$APP_BASE_NAME" + +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` + +if [ -e "$bin/gradle/wrapper/gradle-wrapper.properties" ]; then + . "$bin/gradle/wrapper/gradle-wrapper.properties" +else + # the location that the wrapper is at doesn't have a properties + # check PWD, gradlew may be shared + if [ -e "$PWD/gradle/wrapper/gradle-wrapper.properties" ]; then + . "$PWD/gradle/wrapper/gradle-wrapper.properties" + else + echo "Unable to locate gradle-wrapper.properties. Not at $PWD/gradle/wrapper/gradle-wrapper.properties or $bin/gradle/wrapper/gradle-wrapper.properties" 1>&2 + exit 1 + fi +fi + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +darwin=false +case "`uname`" in + Darwin* ) + darwin=true + ;; +esac + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVA="$JAVA_HOME/jre/sh/java" + else + JAVA="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVA" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVA="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$bin/media/gradle.icns\"" +fi + +# does not match gradle's hash +# waiting for http://stackoverflow.com/questions/26642077/java-biginteger-in-bash-rewrite-gradlew +hash() { + local input="$1" + if $darwin; then + md5 -q -s "$1" + else + echo -n "$1" | md5sum | cut -d" " -f1 + fi +} + +dist_path() { + local dir=$(basename $distributionUrl | sed 's;.zip;;g') + local id=$(hash "$distributionUrl") + + echo "$HOME/.gradle/${distributionPath:-wrapper/dists}/$dir/$id" +} + +zip_path() { + local dir=$(basename $distributionUrl | sed 's;.zip;;g') + local id=$(hash "$distributionUrl") + + echo "$HOME/.gradle/${zipStorePath:-wrapper/dists}/$dir/$id" +} + +download() { + local base_path=$(dist_path) + local file_name=$(basename $distributionUrl) + local dir_name=$(echo "$file_name" | sed 's;-bin.zip;;g' | sed 's;-src.zip;;g' |sed 's;-all.zip;;g') + + if [ ! -d "$base_path" ]; then + mkdir -p "$base_path" + else + # if data already exists, it means we failed to do this before + # so cleanup last run and try again + rm -rf $base_path/* + fi + + # download dist. curl on mac doesn't like the cert provided... + local zip_path=$(zip_path) + curl --insecure -L -o "$zip_path/$file_name" "$distributionUrl" + + pushd "$base_path" + touch "$file_name.lck" + unzip "$zip_path/$file_name" 1> /dev/null + touch "$file_name.ok" + popd +} + +is_cached() { + local file_name=$(basename $distributionUrl) + + [ -e "$(dist_path)/$file_name.ok" ] +} + +lib_path() { + local base_path=$(dist_path) + local file_name=$(basename $distributionUrl | sed 's;-bin.zip;;g' | sed 's;-src.zip;;g' |sed 's;-all.zip;;g') + + echo "$base_path/$file_name/lib" +} + +classpath() { + local dir=$(lib_path) + local cp=$(ls -1 $dir/*.jar | tr '\n' ':') + echo "$dir:$cp" +} + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +splitJvmOpts() { + JVM_OPTS=("$@") +} + +main() { + if ! is_cached; then + download + fi + + eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS + + $JAVA "${JVM_OPTS[@]}" -cp $(classpath) org.gradle.launcher.GradleMain "$@" +} + +main "$@" diff --git a/build.gradle b/build.gradle index 4a740f29..cca237d5 100644 --- a/build.gradle +++ b/build.gradle @@ -25,7 +25,6 @@ version = '0.9.0-SNAPSHOT' description = """Bigtop""" - sourceCompatibility = 1.6 targetCompatibility = 1.6 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..7dcaad82 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +distributionPath=wrapper/dists +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.1-bin.zip diff --git a/gradlew b/gradlew new file mode 100755 index 00000000..2ad3dae8 --- /dev/null +++ b/gradlew @@ -0,0 +1,197 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +## Tries to recreate Gradle's gradlew command in pure bash. +## This way you don't have to worry about binaries in your build. +## +## Depdencies +## unzip +## + +set -e +set -o pipefail + + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="-Dorg.gradle.appname=$APP_BASE_NAME" + +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` + +if [ -e "$bin/gradle/wrapper/gradle-wrapper.properties" ]; then + . "$bin/gradle/wrapper/gradle-wrapper.properties" +else + # the location that the wrapper is at doesn't have a properties + # check PWD, gradlew may be shared + if [ -e "$PWD/gradle/wrapper/gradle-wrapper.properties" ]; then + . "$PWD/gradle/wrapper/gradle-wrapper.properties" + else + echo "Unable to locate gradle-wrapper.properties. Not at $PWD/gradle/wrapper/gradle-wrapper.properties or $bin/gradle/wrapper/gradle-wrapper.properties" 1>&2 + exit 1 + fi +fi + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +darwin=false +case "`uname`" in + Darwin* ) + darwin=true + ;; +esac + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVA="$JAVA_HOME/jre/sh/java" + else + JAVA="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVA" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVA="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$bin/media/gradle.icns\"" +fi + +# does not match gradle's hash +# waiting for http://stackoverflow.com/questions/26642077/java-biginteger-in-bash-rewrite-gradlew +hash() { + local input="$1" + if $darwin; then + md5 -q -s "$1" + else + echo -n "$1" | md5sum | cut -d" " -f1 + fi +} + +dist_path() { + local dir=$(basename $distributionUrl | sed 's;.zip;;g') + local id=$(hash "$distributionUrl") + + echo "$HOME/.gradle/${distributionPath:-wrapper/dists}/$dir/$id" +} + +zip_path() { + local dir=$(basename $distributionUrl | sed 's;.zip;;g') + local id=$(hash "$distributionUrl") + + echo "$HOME/.gradle/${zipStorePath:-wrapper/dists}/$dir/$id" +} + +download() { + local base_path=$(dist_path) + local file_name=$(basename $distributionUrl) + local dir_name=$(echo "$file_name" | sed 's;-bin.zip;;g' | sed 's;-src.zip;;g' |sed 's;-all.zip;;g') + + if [ ! -d "$base_path" ]; then + mkdir -p "$base_path" + else + # if data already exists, it means we failed to do this before + # so cleanup last run and try again + rm -rf $base_path/* + fi + + # download dist. curl on mac doesn't like the cert provided... + local zip_path=$(zip_path) + curl --insecure -L -o "$zip_path/$file_name" "$distributionUrl" + + pushd "$base_path" + touch "$file_name.lck" + unzip "$zip_path/$file_name" 1> /dev/null + touch "$file_name.ok" + popd +} + +is_cached() { + local file_name=$(basename $distributionUrl) + + [ -e "$(dist_path)/$file_name.ok" ] +} + +lib_path() { + local base_path=$(dist_path) + local file_name=$(basename $distributionUrl | sed 's;-bin.zip;;g' | sed 's;-src.zip;;g' |sed 's;-all.zip;;g') + + echo "$base_path/$file_name/lib" +} + +classpath() { + local dir=$(lib_path) + local cp=$(ls -1 $dir/*.jar | tr '\n' ':') + echo "$dir:$cp" +} + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +splitJvmOpts() { + JVM_OPTS=("$@") +} + +main() { + if ! is_cached; then + download + fi + + eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS + + $JAVA "${JVM_OPTS[@]}" -cp $(classpath) org.gradle.launcher.GradleMain "$@" +} + +main "$@" |