Merge branch 'erp17.08' of ssh://git.linaro.org/leg/bigdata/bigtop-trunkerp17.08-master

10 files changed, 178 insertions, 2 deletions

diff --git a/bigtop-packages/src/common/hbase/do-component-build b/bigtop-packages/src/common/hbase/do-component-build
index 353fb38d..c69ac4ef 100644
--- a/bigtop-packages/src/common/hbase/do-component-build
+++ b/bigtop-packages/src/common/hbase/do-component-build
@@ -29,6 +29,10 @@ if [ $HOSTTYPE = "powerpc64le" ] ; then
   sed -i 's|<jruby.version>.*</jruby.version>|<jruby.version>1.7.23</jruby.version>|' pom.xml
   sed  -i "s|<version>1.5.0-alpha.6</version>|<version>1.5.0-alpha.11</version>|" pom.xml
 fi
+if [ $HOSTTYPE = "aarch64" ] ; then
+  sed -i '/<version>1.5.0-alpha.6<\/version>/,+1d' pom.xml
+  sed -i '/<artifactId>asciidoctorj-pdf<\/artifactId>/a\<version>1.5.0-alpha.6<\/version>\n</dependency>\n<dependency>\n<groupId>org.jruby<\/groupId>\n<artifactId>jruby-complete<\/artifactId>\n<version>9.1.8.0<\/version>\n<\/dependency>' pom.xml
+fi
 mvn -DskipTests -Dslf4j.version=1.6.1         \
     -Dhadoop-two.version=$HADOOP_VERSION      \
     -Dzookeeper.version=$ZOOKEEPER_VERSION    \
diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
index 77af19c1..177d8ebd 100644
--- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
+++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
@@ -130,7 +130,7 @@ class TestHadoopExamples {
       aggregatewordhist: "$EXAMPLES/text $EXAMPLES_OUT/aggregatewordhist 2 textinputformat",
       grep: "$EXAMPLES/text $EXAMPLES_OUT/grep '[Cc]uriouser'",
       secondarysort: "$EXAMPLES/ints $EXAMPLES_OUT/secondarysort",
-      randomtextwriter: "-D $RANDOMTEXTWRITER_TOTALBYTES=1073741824 $EXAMPLES_OUT/randomtextwriter"
+      randomtextwriter: "-D $RANDOMTEXTWRITER_TOTALBYTES=10737418 $EXAMPLES_OUT/randomtextwriter"
     ];
 
   private String testName;
diff --git a/bigtop-tests/test-artifacts/spark/pom.xml b/bigtop-tests/test-artifacts/spark/pom.xml
index 83bfdc67..4dffebdd 100644
--- a/bigtop-tests/test-artifacts/spark/pom.xml
+++ b/bigtop-tests/test-artifacts/spark/pom.xml
@@ -28,7 +28,6 @@
   </parent>
 
   <groupId>org.apache.bigtop.itest</groupId>
-  <artifactId>spark-smoke</artifactId>
   <version>1.3.0-SNAPSHOT</version>
   <name>sparksmoke</name>
 
diff --git a/docker/bigtop-puppet/debian-8-aarch64/Dockerfile b/docker/bigtop-puppet/debian-8-aarch64/Dockerfile
new file mode 100644
index 00000000..f31781d9
--- /dev/null
+++ b/docker/bigtop-puppet/debian-8-aarch64/Dockerfile
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM aarch64/debian:jessie
+MAINTAINER naresh.bhat@linaro.org
+
+COPY puppetize.sh /tmp/puppetize.sh
+
+RUN bash /tmp/puppetize.sh
diff --git a/docker/bigtop-puppet/debian-8-aarch64/build.sh b/docker/bigtop-puppet/debian-8-aarch64/build.sh
new file mode 100755
index 00000000..67ed45cf
--- /dev/null
+++ b/docker/bigtop-puppet/debian-8-aarch64/build.sh
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+cp ../../../bigtop_toolchain/bin/puppetize.sh .
+docker build --pull=true -t bigtop/puppet:debian-8-aarch64 .
diff --git a/docker/bigtop-puppet/debian-8-aarch64/puppetize.sh b/docker/bigtop-puppet/debian-8-aarch64/puppetize.sh
new file mode 100755
index 00000000..6147a581
--- /dev/null
+++ b/docker/bigtop-puppet/debian-8-aarch64/puppetize.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use /etc/os-release to determine Linux Distro
+
+if [ -f /etc/os-release ]; then
+    . /etc/os-release
+else
+    if [ -f /etc/redhat-release ]; then
+	if grep "CentOS release 6" /etc/redhat-release >/dev/null ; then
+	    ID=centos
+	    VERSION_ID=6
+	fi
+    else
+	echo "Unknown Linux Distribution."
+	exit 1
+    fi
+fi
+
+case ${ID}-${VERSION_ID} in
+    fedora-25*)
+        dnf -y install yum-utils
+        dnf -y update 
+        dnf -y install hostname findutils curl sudo unzip wget puppet
+        ;;
+    ubuntu-14.04)
+	apt-get update
+	apt-get -y install wget
+	if [ $HOSTTYPE = "x86_64" ] ; then
+	  # BIGTOP-2003. A workaround to install newer hiera to get rid of hiera 1.3.0 bug.
+	  wget -O /tmp/puppetlabs-release-trusty.deb https://apt.puppetlabs.com/puppetlabs-release-trusty.deb && dpkg -i /tmp/puppetlabs-release-trusty.deb
+	  rm -f /tmp/puppetlabs-release-trusty.deb
+	  apt-get update
+        fi
+	apt-get -y install curl sudo unzip puppet software-properties-common
+	;;
+    ubuntu-*)
+	apt-get update
+	apt-get -y install curl sudo unzip wget puppet software-properties-common
+	;;
+    debian-8*)
+	apt-get update
+	apt-get -y install wget
+	# BIGTOP-2523. in order to install puppet 3.8 we need to get it from puppet repo
+	wget -O /tmp/puppetlabs-release-trusty.deb https://apt.puppetlabs.com/puppetlabs-release-trusty.deb && dpkg -i /tmp/puppetlabs-release-trusty.deb
+	rm -f /tmp/puppetlabs-release-trusty.deb
+	apt-get update
+	apt-get -y install curl sudo unzip puppet
+	;;
+    opensuse-*)
+	zypper --gpg-auto-import-keys install -y curl sudo unzip wget puppet suse-release ca-certificates-mozilla net-tools tar
+	;;
+    centos-6*)
+        rpm -ivh http://yum.puppetlabs.com/puppetlabs-release-el-6.noarch.rpm
+	yum -y install curl sudo unzip wget puppet tar
+	;;
+    centos-7*)
+        rpm -ivh http://yum.puppetlabs.com/puppetlabs-release-el-7.noarch.rpm
+	yum -y install hostname curl sudo unzip wget puppet
+	;;
+    *)
+	echo "Unsupported OS ${ID}-${VERSION_ID}."
+	exit 1
+esac
+
+puppet module install puppetlabs-stdlib
+
+case ${ID} in
+   debian|ubuntu)
+       version=""
+       if [ `puppet --version | cut -c1` -lt "4" ]; then
+           version="--version 2.4.0"
+       fi
+      puppet module install puppetlabs-apt $version;;
+esac
diff --git a/docker/bigtop-slaves/debian-8-aarch64/Dockerfile b/docker/bigtop-slaves/debian-8-aarch64/Dockerfile
new file mode 100644
index 00000000..e21e06ed
--- /dev/null
+++ b/docker/bigtop-slaves/debian-8-aarch64/Dockerfile
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM bigtop/puppet:debian-8-aarch64
+MAINTAINER naresh.bhat@linaro.org
+
+COPY bigtop_toolchain /etc/puppet/modules/bigtop_toolchain
+
+RUN  apt-get clean && apt-get update && puppet apply -e "include bigtop_toolchain::installer"
+
+COPY . /tmp/bigtop
+RUN cd /tmp/bigtop && ./gradlew && cd && rm -rf /tmp/bigtop
diff --git a/docker/pseudo-cluster/config/hieradata/site.yaml b/docker/pseudo-cluster/config/hieradata/site.yaml
index eb338406..d68849e5 100644
--- a/docker/pseudo-cluster/config/hieradata/site.yaml
+++ b/docker/pseudo-cluster/config/hieradata/site.yaml
@@ -8,3 +8,4 @@ hadoop_cluster_node::cluster_components:
   - mapreduce
 bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.2.0/ubuntu/16.04/x86_64
 
+
diff --git a/docker/sandbox/site.yaml.template.debian-8_hadoop b/docker/sandbox/site.yaml.template.debian-8_hadoop
index 1bbae6c6..c5f1018e 100644
--- a/docker/sandbox/site.yaml.template.debian-8_hadoop
+++ b/docker/sandbox/site.yaml.template.debian-8_hadoop
@@ -17,3 +17,4 @@ bigtop::hadoop_head_node: "head.node.fqdn"
 hadoop::hadoop_storage_dirs: [/data/1, /data/2]
 bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.2.0/debian/8/x86_64
 hadoop_cluster_node::cluster_components: [hdfs, yarn, mapred-app]
+
diff --git a/provisioner/docker/config_debian-8-aarch64.yaml b/provisioner/docker/config_debian-8-aarch64.yaml
new file mode 100644
index 00000000..fa67996f
--- /dev/null
+++ b/provisioner/docker/config_debian-8-aarch64.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+        memory_limit: "4g"
+        image: "bigtop/puppet:debian-8-aarch64"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.2.0/debian/8-aarch64/aarch64"
+distro: debian
+components: [hdfs, yarn, mapreduce]
+enable_local_repo: false
+smoke_test_components: [hdfs, yarn, mapreduce]