BIGTOP-612. Add Crunch to Bigtop

16 files changed, 375 insertions, 0 deletions

diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 57d73b32..99d5af38 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -286,6 +286,8 @@ class hadoop_gateway_node inherits hadoop_cluster_node {
   giraph::client { "giraph client":
      zookeeper_quorum => $giraph_zookeeper_quorum,
   }
+  crunch::client { "crunch client":
+  }
   hadoop-pig::client { "pig client":
   }
   hadoop-hive::client { "hive client":
diff --git a/bigtop-deploy/puppet/modules/crunch/manifests/init.pp b/bigtop-deploy/puppet/modules/crunch/manifests/init.pp
new file mode 100644
index 00000000..d4466670
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/crunch/manifests/init.pp
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class crunch {
+  define client {
+    package { ["crunch", "crunch-doc"]:
+      ensure => latest,
+    } 
+  }
+}
diff --git a/bigtop-deploy/puppet/modules/crunch/tests/init.pp b/bigtop-deploy/puppet/modules/crunch/tests/init.pp
new file mode 100644
index 00000000..abef3713
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/crunch/tests/init.pp
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include crunch 
+crunch::client { "test-crunch": } 
diff --git a/bigtop-packages/src/common/crunch/do-component-build b/bigtop-packages/src/common/crunch/do-component-build
new file mode 100644
index 00000000..4285f6f4
--- /dev/null
+++ b/bigtop-packages/src/common/crunch/do-component-build
@@ -0,0 +1,30 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+. `dirname $0`/bigtop.bom
+
+mvn -DskipTests -Dcrunch.platform=2           \
+    -Dslf4j.version=1.6.1                     \
+    -Dhadoop.version=$HADOOP_VERSION          \
+    -Dhbase.version=$HBASE_VERSION            \
+    -Papache-release clean site:site package "$@"
+
+rm -rf build
+mkdir build
+tar --strip-component=1 -C build -xzvf crunch-dist/target/apache-crunch-*-bin.tar.gz
+cp -r target/site build
diff --git a/bigtop-packages/src/common/crunch/install_crunch.sh b/bigtop-packages/src/common/crunch/install_crunch.sh
new file mode 100644
index 00000000..d7598824
--- /dev/null
+++ b/bigtop-packages/src/common/crunch/install_crunch.sh
@@ -0,0 +1,99 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+  echo "
+usage: $0 <options>
+  Required not-so-options:
+     --build-dir=DIR             path to crunch build dir
+     --prefix=PREFIX             path to install into
+
+  Optional options:
+     --lib-dir=DIR               path to install crunch home [/usr/lib/crunch]
+     --doc-dir=DIR               path to install crunch docs [/usr/share/doc/crunch]
+     ... [ see source for more similar options ]
+  "
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'prefix:' \
+  -l 'lib-dir:' \
+  -l 'doc-dir:' \
+  -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+    usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+    case "$1" in
+        --prefix)
+        PREFIX=$2 ; shift 2
+        ;;
+        --build-dir)
+        BUILD_DIR=$2 ; shift 2
+        ;;
+        --lib-dir)
+        LIB_DIR=$2 ; shift 2
+        ;;
+        --doc-dir)
+        DOC_DIR=$2 ; shift 2
+        ;;
+        --)
+        shift ; break
+        ;;
+        *)
+        echo "Unknown option: $1"
+        usage
+        exit 1
+        ;;
+    esac
+done
+
+for var in PREFIX BUILD_DIR ; do
+  if [ -z "$(eval "echo \$$var")" ]; then
+    echo Missing param: $var
+    usage
+  fi
+done
+
+LIB_DIR=${LIB_DIR:-/usr/lib/crunch}
+DOC_DIR=${DOC_DIR:-/usr/share/doc/crunch}
+
+# First we'll move everything into lib
+install -d -m 0755 $PREFIX/$LIB_DIR
+cp $BUILD_DIR/*.jar $PREFIX/$LIB_DIR
+install -d -m 0755 $PREFIX/$LIB_DIR/lib
+cp $BUILD_DIR/lib/*.jar $PREFIX/$LIB_DIR/lib
+
+# Now take care of the docs
+install -d -m 0755 $PREFIX/$DOC_DIR
+cp $BUILD_DIR/DISCLAIMER $PREFIX/$DOC_DIR
+cp $BUILD_DIR/LICENSE $PREFIX/$DOC_DIR
+cp $BUILD_DIR/NOTICE $PREFIX/$DOC_DIR
+cp $BUILD_DIR/README $PREFIX/$DOC_DIR
+cp -r $BUILD_DIR/site/* $PREFIX/$DOC_DIR
+
+# Finally move some file from lib to docs
+mv $PREFIX/$LIB_DIR/crunch-examples-*.jar $PREFIX/$DOC_DIR
+mv $PREFIX/$LIB_DIR/crunch-test-*.jar $PREFIX/$DOC_DIR
diff --git a/bigtop-packages/src/deb/crunch/changelog b/bigtop-packages/src/deb/crunch/changelog
new file mode 100644
index 00000000..547ed021
--- /dev/null
+++ b/bigtop-packages/src/deb/crunch/changelog
@@ -0,0 +1 @@
+--- This is auto-generated 
diff --git a/bigtop-packages/src/deb/crunch/compat b/bigtop-packages/src/deb/crunch/compat
new file mode 100644
index 00000000..7f8f011e
--- /dev/null
+++ b/bigtop-packages/src/deb/crunch/compat
@@ -0,0 +1 @@
+7
diff --git a/bigtop-packages/src/deb/crunch/control b/bigtop-packages/src/deb/crunch/control
new file mode 100644
index 00000000..a05f621d
--- /dev/null
+++ b/bigtop-packages/src/deb/crunch/control
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Source: crunch
+Section: misc
+Priority: extra
+Maintainer: Bigtop <bigtop-dev@incubator.apache.org>
+Build-Depends: debhelper (>= 7.0.50~)
+Standards-Version: 3.8.0
+Homepage: http://incubator.apache.org/crunch/
+
+Package: crunch
+Architecture: all
+Description: Simple and Efficient MapReduce Pipelines.
+ Apache Crunch (incubating) is a Java library for writing, testing, and running
+ MapReduce pipelines, based on Google's FlumeJava. Its goal is to make 
+ pipelines that are composed of many user-defined functions simple to write, 
+ easy to test, and efficient to run.
+
+Package: crunch-doc
+Architecture: all
+Description: Apache Crunch (incubating) documentation
+ Apache Crunch (incubating) documentation
+
diff --git a/bigtop-packages/src/deb/crunch/copyright b/bigtop-packages/src/deb/crunch/copyright
new file mode 100644
index 00000000..be638e0d
--- /dev/null
+++ b/bigtop-packages/src/deb/crunch/copyright
@@ -0,0 +1,15 @@
+Format: http://dep.debian.net/deps/dep5
+Source: http://incubator.apache.org/crunch/
+Upstream-Name: Apache Crunch (incubating)
+
+Files: *
+Copyright: 2012, The Apache Software Foundation
+License: Apache-2.0
+
+Files debian/*
+Copyright: 2011, The Apache Software Foundation
+License: Apache-2.0
+
+License: Apache-2.0
+ On Debian systems, the complete text of the Apache 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".
diff --git a/bigtop-packages/src/deb/crunch/crunch-doc.install b/bigtop-packages/src/deb/crunch/crunch-doc.install
new file mode 100644
index 00000000..77b99dd9
--- /dev/null
+++ b/bigtop-packages/src/deb/crunch/crunch-doc.install
@@ -0,0 +1 @@
+/usr/share/doc/crunch
diff --git a/bigtop-packages/src/deb/crunch/crunch.install b/bigtop-packages/src/deb/crunch/crunch.install
new file mode 100644
index 00000000..a955cc93
--- /dev/null
+++ b/bigtop-packages/src/deb/crunch/crunch.install
@@ -0,0 +1 @@
+/usr/lib/crunch
diff --git a/bigtop-packages/src/deb/crunch/rules b/bigtop-packages/src/deb/crunch/rules
new file mode 100644
index 00000000..d3f973fa
--- /dev/null
+++ b/bigtop-packages/src/deb/crunch/rules
@@ -0,0 +1,41 @@
+#!/usr/bin/make -f
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+export DH_VERBOSE=1
+
+# This has to be exported to make some magic below work.
+export DH_OPTIONS
+
+
+%:
+	dh $@
+
+override_dh_auto_build:
+	# we'll just use the build from the tarball.
+	# I could not find a way to add debian/ to RAT exclude list through cmd line
+	# or to unbind rat:check goal
+	# So I am redirecting its attention with a decoy
+	mkdir -p debian/tmp/bigtop-empty
+	bash debian/do-component-build -Drat.basedir=$${PWD}/debian/tmp/bigtop-empty -Dmaven.repo.local=${HOME}/.m2/repository
+
+override_dh_auto_install:
+	sh -x debian/install_crunch.sh \
+	  --build-dir=$${PWD}/build     \
+	  --prefix=debian/tmp
diff --git a/bigtop-packages/src/deb/crunch/source/format b/bigtop-packages/src/deb/crunch/source/format
new file mode 100644
index 00000000..163aaf8d
--- /dev/null
+++ b/bigtop-packages/src/deb/crunch/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
diff --git a/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec b/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec
new file mode 100644
index 00000000..ece47755
--- /dev/null
+++ b/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+%define crunch_name crunch
+%define lib_crunch /usr/lib/crunch
+%define crunch_folder apache-%{crunch_name}-%{crunch_base_version}-src
+
+%if  %{?suse_version:1}0
+%define doc_crunch %{_docdir}/crunch-doc
+%else
+%define doc_crunch %{_docdir}/crunch-doc-%{crunch_version}
+%endif
+
+# disable repacking jars
+%define __os_install_post %{nil}
+
+Name: crunch
+Version: %{crunch_version}
+Release: %{crunch_release}
+Summary: Simple and Efficient MapReduce Pipelines.
+URL: http://incubator.apache.org/crunch/
+Group: Development/Libraries
+BuildArch: noarch
+Buildroot: %(mktemp -ud %{_tmppath}/%{crunch_name}-%{version}-%{release}-XXXXXX)
+License: ASL 2.0 
+Source0: %{crunch_folder}.tar.gz
+Source1: do-component-build 
+Source2: install_%{crunch_name}.sh
+
+
+%description 
+Apache Crunch (incubating) is a Java library for writing, testing, and running
+MapReduce pipelines, based on Google's FlumeJava. Its goal is to make 
+pipelines that are composed of many user-defined functions simple to write, 
+easy to test, and efficient to run.
+
+%package doc
+Summary: Apache Crunch (incubating) documentation
+Group: Documentation
+%description doc
+Apache Crunch (incubating) documentation
+
+%prep
+%setup -n %{crunch_folder}
+
+%build
+bash $RPM_SOURCE_DIR/do-component-build
+
+%install
+%__rm -rf $RPM_BUILD_ROOT
+sh $RPM_SOURCE_DIR/install_crunch.sh \
+          --build-dir=${PWD}/build   \
+          --doc-dir=%{doc_crunch}    \
+          --prefix=$RPM_BUILD_ROOT
+
+#######################
+#### FILES SECTION ####
+#######################
+%files 
+%defattr(-,root,root,755)
+%{lib_crunch}
+
+%files doc
+%defattr(-,root,root,755)
+%{doc_crunch}
diff --git a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
index 17b0baa0..a1920567 100644
--- a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
+++ b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
@@ -343,6 +343,23 @@ server.</description>
       <zookeeper>/self</zookeeper>
     </deps>
   </zookeeper-server>
+  <crunch>
+    <metadata>
+      <summary>Simple and Efficient MapReduce Pipelines.</summary>
+      <description>Apache Crunch (incubating) is a Java library for writing, testing, and running
+MapReduce pipelines, based on Google's FlumeJava. Its goal is to make 
+pipelines that are composed of many user-defined functions simple to write, 
+easy to test, and efficient to run.</description>
+      <url></url>
+    </metadata>
+  </crunch>
+  <crunch-doc>
+    <metadata>
+      <summary>Apache Crunch (incubating) documentation</summary>
+      <description>Apache Crunch (incubating) documentation</description>
+      <url></url>
+    </metadata>
+  </crunch-doc>
   <pig>
     <metadata>
       <summary>Pig is a platform for analyzing large data sets</summary>
diff --git a/bigtop.mk b/bigtop.mk
index f59437ee..0af8b1d6 100644
--- a/bigtop.mk
+++ b/bigtop.mk
@@ -211,6 +211,20 @@ SOLR_SITE=$(APACHE_MIRROR)$(SOLR_DOWNLOAD_PATH)
 SOLR_ARCHIVE=$(APACHE_ARCHIVE)$(SOLR_DOWNLOAD_PATH)
 $(eval $(call PACKAGE,solr,SOLR))
 
+# Crunch
+CRUNCH_NAME=crunch
+CRUNCH_RELNOTES_NAME=Java library for MapReduce pipelines
+CRUNCH_PKG_NAME=crunch
+CRUNCH_BASE_VERSION=0.4.0-incubating
+CRUNCH_PKG_VERSION=0.4.0
+CRUNCH_RELEASE_VERSION=1
+CRUNCH_TARBALL_DST=apache-crunch-$(CRUNCH_BASE_VERSION)-src.tar.gz
+CRUNCH_TARBALL_SRC=$(CRUNCH_TARBALL_DST)
+CRUNCH_DOWNLOAD_PATH=/incubator/crunch/crunch-$(CRUNCH_BASE_VERSION)
+CRUNCH_SITE=$(APACHE_MIRROR)$(CRUNCH_DOWNLOAD_PATH)
+CRUNCH_ARCHIVE=$(CRUNCH_SITE)
+$(eval $(call PACKAGE,crunch,CRUNCH))
+
 # Bigtop-utils
 BIGTOP_UTILS_NAME=bigtop-utils
 BIGTOP_UTILS__RELNOTES_NAME=Bigtop-utils