diff options
| -rw-r--r-- | bigtop-packages/src/common/spark/install_spark.sh | 6 | ||||
| -rw-r--r-- | bigtop-packages/src/common/spark/run | 148 | ||||
| -rw-r--r-- | bigtop-packages/src/deb/spark/control | 2 | ||||
| -rw-r--r-- | bigtop-packages/src/deb/spark/rules | 3 | ||||
| -rw-r--r-- | bigtop-packages/src/rpm/spark/SPECS/spark.spec | 4 |
5 files changed, 5 insertions, 158 deletions
diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh index d29dfd83..45996dbd 100644 --- a/bigtop-packages/src/common/spark/install_spark.sh +++ b/bigtop-packages/src/common/spark/install_spark.sh @@ -170,11 +170,7 @@ for wrap in spark-executor spark-shell ; do #!/bin/bash # Autodetect JAVA_HOME if not defined -if [ -e /usr/libexec/bigtop-detect-javahome ]; then - . /usr/libexec/bigtop-detect-javahome -elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then - . /usr/lib/bigtop-utils/bigtop-detect-javahome -fi +. /usr/lib/bigtop-utils/bigtop-detect-javahome exec $INSTALLED_LIB_DIR/$wrap "\$@" EOF diff --git a/bigtop-packages/src/common/spark/run b/bigtop-packages/src/common/spark/run deleted file mode 100644 index 50b0e824..00000000 --- a/bigtop-packages/src/common/spark/run +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -SCALA_VERSION=2.9.3 - -# Figure out where the Scala framework is installed -FWDIR="$(cd `dirname $0`; pwd)" - -# Export this as SPARK_HOME -export SPARK_HOME="$FWDIR" - -# Load environment variables from conf/spark-env.sh, if it exists -if [ -e $FWDIR/conf/spark-env.sh ] ; then - . $FWDIR/conf/spark-env.sh -fi - -if [ -z "$1" ]; then - echo "Usage: run <spark-class> [<args>]" >&2 - exit 1 -fi - -# If this is a standalone cluster daemon, reset SPARK_JAVA_OPTS and SPARK_MEM to reasonable -# values for that; it doesn't need a lot -if [ "$1" = "spark.deploy.master.Master" -o "$1" = "spark.deploy.worker.Worker" ]; then - SPARK_MEM=${SPARK_DAEMON_MEMORY:-512m} - SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.akka.logLifecycleEvents=true" - # Do not overwrite SPARK_JAVA_OPTS environment variable in this script - OUR_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS" # Empty by default -else - OUR_JAVA_OPTS="$SPARK_JAVA_OPTS" -fi - - -# Add java opts for master, worker, executor. The opts maybe null -case "$1" in - 'spark.deploy.master.Master') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_MASTER_OPTS" - ;; - 'spark.deploy.worker.Worker') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_WORKER_OPTS" - ;; - 'spark.executor.StandaloneExecutorBackend') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_EXECUTOR_OPTS" - ;; - 'spark.executor.MesosExecutorBackend') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_EXECUTOR_OPTS" - ;; - 'spark.repl.Main') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_REPL_OPTS" - ;; -esac - -# Figure out whether to run our class with java or with the scala launcher. -# In most cases, we'd prefer to execute our process with java because scala -# creates a shell script as the parent of its Java process, which makes it -# hard to kill the child with stuff like Process.destroy(). However, for -# the Spark shell, the wrapper is necessary to properly reset the terminal -# when we exit, so we allow it to set a variable to launch with scala. -if [ "$SPARK_LAUNCH_WITH_SCALA" == "1" ]; then - if [ "$SCALA_HOME" ]; then - RUNNER="${SCALA_HOME}/bin/scala" - else - if [ `command -v scala` ]; then - RUNNER="scala" - else - echo "SCALA_HOME is not set and scala is not in PATH" >&2 - exit 1 - fi - fi -else - if [ -n "${JAVA_HOME}" ]; then - RUNNER="${JAVA_HOME}/bin/java" - else - if [ `command -v java` ]; then - RUNNER="java" - else - echo "JAVA_HOME is not set" >&2 - exit 1 - fi - fi - if [ -z "$SCALA_LIBRARY_PATH" ]; then - if [ -z "$SCALA_HOME" ]; then - echo "SCALA_HOME is not set" >&2 - exit 1 - fi - SCALA_LIBRARY_PATH="$SCALA_HOME/lib" - fi -fi - -# Figure out how much memory to use per executor and set it as an environment -# variable so that our process sees it and can report it to Mesos -if [ -z "$SPARK_MEM" ] ; then - SPARK_MEM="512m" -fi -export SPARK_MEM - -# Set JAVA_OPTS to be able to load native libraries and to set heap size -JAVA_OPTS="$OUR_JAVA_OPTS" -JAVA_OPTS="$JAVA_OPTS -Djava.library.path=$SPARK_LIBRARY_PATH" -JAVA_OPTS="$JAVA_OPTS -Xms$SPARK_MEM -Xmx$SPARK_MEM" -# Load extra JAVA_OPTS from conf/java-opts, if it exists -if [ -e $FWDIR/conf/java-opts ] ; then - JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`" -fi -export JAVA_OPTS -# Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in ExecutorRunner.scala! - -CORE_DIR="$FWDIR/core" -EXAMPLES_DIR="$FWDIR/examples" -REPL_DIR="$FWDIR/repl" - -# Compute classpath using external script -CLASSPATH=`$FWDIR/bin/compute-classpath.sh` -export CLASSPATH - -# Figure out the JAR file that our examples were packaged into. This includes a bit of a hack -# to avoid the -sources and -doc packages that are built by publish-local. -if [ -e "$EXAMPLES_DIR/lib/spark-examples"*[0-9T].jar ]; then - # Use the JAR from the SBT build - export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/lib/spark-examples"*[0-9T].jar` -fi -if [ -e "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar ]; then - # Use the JAR from the Maven build - export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar` -fi - -if [ "$SPARK_LAUNCH_WITH_SCALA" == "1" ]; then - EXTRA_ARGS="" # Java options will be passed to scala as JAVA_OPTS -else - # The JVM doesn't read JAVA_OPTS by default so we need to pass it in - EXTRA_ARGS="$JAVA_OPTS" -fi - -exec "$RUNNER" -cp "$CLASSPATH" $EXTRA_ARGS "$@" diff --git a/bigtop-packages/src/deb/spark/control b/bigtop-packages/src/deb/spark/control index e57994ad..4e36ba83 100644 --- a/bigtop-packages/src/deb/spark/control +++ b/bigtop-packages/src/deb/spark/control @@ -23,7 +23,7 @@ Homepage: http://spark.incubator.apache.org/ Package: spark-core Architecture: all -Depends: bigtop-utils +Depends: bigtop-utils (>= 0.7) Description: Lightning-Fast Cluster Computing Spark is a MapReduce-like cluster computing framework designed to support low-latency iterative jobs and interactive use from an interpreter. It is diff --git a/bigtop-packages/src/deb/spark/rules b/bigtop-packages/src/deb/spark/rules index f559f495..a21ba820 100644 --- a/bigtop-packages/src/deb/spark/rules +++ b/bigtop-packages/src/deb/spark/rules @@ -26,8 +26,7 @@ export DH_OPTIONS dh $@ override_dh_auto_build: - # we'll just use the build from the tarball. - env FULL_VERSION=${SPARK_PATCHED_VERSION} bash debian/do-component-build -Divy.home=${HOME}/.ivy2 + bash debian/do-component-build svcs=spark-master spark-worker diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec index e9d26b13..41f1743f 100644 --- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec +++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec @@ -52,7 +52,7 @@ Source1: do-component-build Source2: install_%{spark_name}.sh Source3: spark-master.svc Source4: spark-worker.svc -Requires: bigtop-utils +Requires: bigtop-utils >= 0.7 Requires(preun): /sbin/service %global initd_dir %{_sysconfdir}/init.d @@ -111,7 +111,7 @@ bash $RPM_SOURCE_DIR/do-component-build %__rm -rf $RPM_BUILD_ROOT %__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/ -sh $RPM_SOURCE_DIR/install_spark.sh \ +bash $RPM_SOURCE_DIR/install_spark.sh \ --build-dir=`pwd` \ --source-dir=$RPM_SOURCE_DIR \ --prefix=$RPM_BUILD_ROOT \ |