Merge branch 'master' of ssh://git.linaro.org/leg/bigdata/bigtop-odpi

8 files changed, 51 insertions, 41 deletions

diff --git a/bigtop-packages/src/common/spark/do-component-build b/bigtop-packages/src/common/spark/do-component-build
index 63db81ac..7e26a82f 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -34,10 +34,14 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${
 #        http://maven.40175.n5.nabble.com/Not-finding-artifact-in-local-repo-td3727753.html
 export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -XX:PermSize=1024m -XX:MaxPermSize=1024m"
 
-#Patch Spark-2.0 with Debian package creation support.
-patch -p1 < $RPM_SOURCE_DIR/0001-branch-2.0-Add-deb-profile-to-assembly-pom.xml.patch
-
 #BIGTOP-2607
 sed -i -e /useZincServer/d -e /recompileMode/d pom.xml
 
-mvn $BUILD_OPTS -DskipTests package -Pdeb
+mvn $BUILD_OPTS -DskipTests install
+
+# Tests must be run after Spark has already been packaged.
+# See http://spark.apache.org/docs/latest/building-spark.html#spark-tests-in-maven
+if [ "$SPARK_RUN_TESTS" = "true" ]; then
+    mvn $BUILD_OPTS test
+fi
+
diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh
index 4248be09..4e3e1817 100644
--- a/bigtop-packages/src/common/spark/install_spark.sh
+++ b/bigtop-packages/src/common/spark/install_spark.sh
@@ -36,6 +36,7 @@ usage: $0 <options>
   exit 1
 }
 
+
 OPTS=$(getopt \
   -n $0 \
   -o '' \
@@ -148,6 +149,7 @@ cp ${BUILD_DIR}/sql/*/target/spark-*${SPARK_VERSION}.jar $PREFIX/$LIB_DIR/jars/
 cp ${BUILD_DIR}/{core,graphx,launcher,mllib,mllib-local,streaming,repl,yarn}/target/spark-*${SPARK_VERSION}.jar $PREFIX/$LIB_DIR/jars/
 cp ${BUILD_DIR}/common/network-yarn/target/scala-*/spark-${SPARK_VERSION}-yarn-shuffle.jar $PREFIX/$LIB_DIR/yarn
 cp ${BUILD_DIR}/assembly/target/scala-*/jars/avro-*.jar $PREFIX/$LIB_DIR/jars/
+cp ${BUILD_DIR}/assembly/target/scala-*/jars/datanucleus-*.jar $PREFIX/$LIB_DIR/jars/
 
 # Spark license files
 cp ${BUILD_DIR}/licenses/* $PREFIX/$LIB_DIR/licenses/
@@ -198,6 +200,9 @@ cp ${BUILD_DIR}/{LICENSE,NOTICE} ${PREFIX}/${LIB_DIR}/
 # Version-less symlinks
 pushd $PREFIX/$LIB_DIR/yarn
 ln -s ./spark-*yarn-shuffle*.jar spark-yarn-shuffle.jar
+ln -s ../../jars/datanucleus-api-jdo*.jar datanucleus-api-jdo.jar
+ln -s ../../jars/datanucleus-core*.jar datanucleus-core.jar
+ln -s ../../jars/datanucleus-rdbms*.jar datanucleus-rdbms.jar
 popd
 pushd $PREFIX/$LIB_DIR/extras/lib
 for j in $(ls *.jar); do
diff --git a/bigtop-packages/src/common/spark/0001-branch-2.0-Add-deb-profile-to-assembly-pom.xml.patch b/bigtop-packages/src/deb/spark/0001-branch-2.0-Add-deb-profile-to-assembly-pom.xml.patch
index 49ea2a2c..49ea2a2c 100644
--- a/bigtop-packages/src/common/spark/0001-branch-2.0-Add-deb-profile-to-assembly-pom.xml.patch
+++ b/bigtop-packages/src/deb/spark/0001-branch-2.0-Add-deb-profile-to-assembly-pom.xml.patch
diff --git a/bigtop-packages/src/deb/spark/control b/bigtop-packages/src/deb/spark/control
index c4f37194..ed17df01 100644
--- a/bigtop-packages/src/deb/spark/control
+++ b/bigtop-packages/src/deb/spark/control
@@ -66,13 +66,14 @@ Architecture: all
 Description: DataNucleus libraries for Apache Spark
  DataNucleus libraries used by Spark SQL with Hive Support
 
-Package: spark-external
+Package: spark-extras
 Architecture: all
-Description: External libraries for Apache Spark
- External libraries built for Apache Spark but not included in the main distribution (e.g., external streaming libraries)
+Description: Extras libraries for Apache Spark
+ Extras libraries built for Apache Spark but not included in the main
+ distribution (e.g., extras streaming libraries)
 
 
-Package: spark-network-shuffle
+Package: spark-yarn-shuffle
 Architecture: all
-Description: Spark Network Shuffle Service
+Description: Spark Yarn Shuffle Service
  Spark YARN Shuffle Service
diff --git a/bigtop-packages/src/deb/spark/rules b/bigtop-packages/src/deb/spark/rules
index 58815189..c0a6b0f2 100644
--- a/bigtop-packages/src/deb/spark/rules
+++ b/bigtop-packages/src/deb/spark/rules
@@ -26,6 +26,7 @@ export DH_OPTIONS
 	dh $@
 
 override_dh_auto_build:
+	patch -p1 < ./debian/0001-branch-2.0-Add-deb-profile-to-assembly-pom.xml.patch
 	bash debian/do-component-build
 
 svcs=spark-master spark-worker spark-history-server spark-thriftserver
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index 6b6a1ba8..69cf0ea6 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -126,20 +126,20 @@ Group: Development/Libraries
 %description -n spark-datanucleus
 DataNucleus libraries used by Spark SQL with Hive Support
 
-%package -n spark-external
-Summary: External libraries for Apache Spark
+%package -n spark-extras
+Summary: Extras libraries for Apache Spark
 Group: Development/Libraries
 
-%description -n spark-external
-Summary: External libraries for Apache Spark but not included in the main
-assembly JAR (e.g., external streaming libraries)
+%description -n spark-extras
+Summary: Extras libraries for Apache Spark but not included in the main
+assembly JAR (e.g., extras streaming libraries)
 
-%package -n spark-network-shuffle
-Summary: Spark Network Shuffle Service
+%package -n spark-yarn-shuffle
+Summary: Spark Yarn Shuffle Service
 Group: Development/Libraries
 
-%description -n spark-network-shuffle
-Spark Network Shuffle Service
+%description -n spark-yarn-shuffle
+Spark Yarn Shuffle Service
 
 %prep
 %setup -n %{spark_name}-%{spark_base_version}
@@ -191,25 +191,20 @@ done
 %defattr(-,root,root,755)
 %config(noreplace) %{config_spark}.dist
 %doc %{doc_spark}
-%{lib_spark}/conf
 %{lib_spark}/LICENSE
-%{lib_spark}/RELEASE
-%{lib_spark}/README
 %{lib_spark}/NOTICE
-%{lib_spark}/bin
-%{lib_spark}/lib
-%{lib_spark}/spark*.jar
+%{lib_spark}/README.md
+%{lib_spark}/RELEASE
+%{bin_spark}
+%exclude %{bin_spark}/pyspark
+%{lib_spark}/conf
+%{lib_spark}/data
+%{lib_spark}/examples
 %{lib_spark}/jars
 %exclude %{lib_spark}/jars/datanucleus-*.jar
-%exclude %{lib_spark}/spark-network-shuffle*.jar
+%{lib_spark}/licenses
 %{lib_spark}/sbin
-%{lib_spark}/data
-%{lib_spark}/examples
 %{lib_spark}/work
-%{lib_spark}/ui-resources
-%{bin_spark}
-%exclude %{bin_spark}/pyspark
-%exclude %{lib_spark}/python
 %{etc_spark}
 %attr(0755,spark,spark) %{var_lib_spark}
 %attr(0755,spark,spark) %{var_run_spark}
@@ -228,13 +223,12 @@ done
 %{lib_spark}/jars/datanucleus-*.jar
 %{lib_spark}/yarn/lib/datanucleus-*.jar
 
-%files -n spark-external
+%files -n spark-extras
 %defattr(-,root,root,755)
-%{lib_spark}/external
+%{lib_spark}/extras
 
-%files -n spark-network-shuffle
+%files -n spark-yarn-shuffle
 %defattr(-,root,root,755)
-%{lib_spark}/spark-network-shuffle*.jar
 %{lib_spark}/yarn/spark-*-yarn-shuffle.jar
 %{lib_spark}/yarn/lib/spark-yarn-shuffle.jar
 
diff --git a/bigtop.bom b/bigtop.bom
index b5da6e22..72147a5d 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -89,7 +89,7 @@ bigtop {
   dependencies = [
     zookeeper:['hadoop', 'hbase'],
     hadoop:['ignite-hadoop', 'hbase', 'crunch', 'pig', 'hive', 'tez', 'sqoop', 'sqoop2',
-      'oozie', 'mahout', 'flume', 'solr', 'spark',
+      'oozie', 'mahout', 'flume', 'giraph', 'solr', 'crunch', 'spark',
       'phoenix', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin',
       'tajo', 'apex'
     ],
@@ -271,6 +271,16 @@ bigtop {
                 site = "${apache.APACHE_MIRROR}/${download_path}"
                 archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
     }
+    'crunch' {
+      name    = 'crunch'
+      relNotes = 'Apache Crunch'
+      version { base = '0.13.0'; pkg = base; release = 1 }
+      tarball { destination = "apache-$name-${version.base}-src.tar.gz"
+                source      = destination }
+      url     { download_path = "/$name/$name-${version.base}/"
+                site = "${apache.APACHE_MIRROR}/${download_path}"
+                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
+    }
     'spark' {
       name    = 'spark'
       pkg     = 'spark-core'
diff --git a/pom.xml b/pom.xml
index 1c26a142..6b32445c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -336,11 +336,6 @@
               <exclude>bigtop-packages/src/common/*/*.json</exclude>
               <exclude>bigtop-packages/src/common/**/*.default</exclude>
               <exclude>bigtop-repos/apt/distributions</exclude>
-              <!-- Juju charm files with rigid structure -->
-              <exclude>bigtop-packages/src/charm/**/wheelhouse.txt</exclude>
-              <exclude>bigtop-packages/src/charm/**/*.yaml</exclude>
-              <exclude>bigtop-deploy/juju/**/*.yaml</exclude>
-              <exclude>bigtop-tests/cloud-weather-report/**/*.yaml</exclude>
               <!-- Miscelaneous individual files -->
               <exclude>src/site/resources/bigtop.rdf</exclude>
               <exclude>src/site/resources/images/bigtop-logo.ai</exclude>