diff options
| author | Sean Mackrory <mackrorysd@gmail.com> | 2013-03-15 14:21:32 -0700 |
|---|---|---|
| committer | Roman Shaposhnik <rvs@cloudera.com> | 2013-03-21 11:36:37 -0700 |
| commit | f73ed4346349fc791f65e20ad7c81426f707273f (patch) | |
| tree | 49626448787c621c5fffaf000cdd463337da0224 | |
| parent | 552f9afcd6c02e384094160b4bdb54f8b7f2bd36 (diff) | |
BIGTOP-805. add Sqoop 2 to Bigtop
27 files changed, 903 insertions, 555 deletions
diff --git a/bigtop-packages/src/common/sqoop/catalina.properties b/bigtop-packages/src/common/sqoop/catalina.properties new file mode 100644 index 00000000..b5490604 --- /dev/null +++ b/bigtop-packages/src/common/sqoop/catalina.properties @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# List of comma-separated packages that start with or equal this string +# will cause a security exception to be thrown when +# passed to checkPackageAccess unless the +# corresponding RuntimePermission ("accessClassInPackage."+package) has +# been granted. +package.access=sun.,org.apache.catalina.,org.apache.coyote.,org.apache.tomcat.,org.apache.jasper.,sun.beans. +# +# List of comma-separated packages that start with or equal this string +# will cause a security exception to be thrown when +# passed to checkPackageDefinition unless the +# corresponding RuntimePermission ("defineClassInPackage."+package) has +# been granted. +# +# by default, no packages are restricted for definition, and none of +# the class loaders supplied with the JDK call checkPackageDefinition. +# +package.definition=sun.,java.,org.apache.catalina.,org.apache.coyote.,org.apache.tomcat.,org.apache.jasper. + +# +# +# List of comma-separated paths defining the contents of the "common" +# classloader. Prefixes should be used to define what is the repository type. +# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute. +# If left as blank,the JVM system loader will be used as Catalina's "common" +# loader. +# Examples: +# "foo": Add this folder as a class repository +# "foo/*.jar": Add all the JARs of the specified folder as class +# repositories +# "foo/bar.jar": Add bar.jar as a class repository +common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,/usr/lib/sqoop/server-lib/*.jar,/usr/lib/hadoop/client/*.jar +# +# List of comma-separated paths defining the contents of the "server" +# classloader. Prefixes should be used to define what is the repository type. +# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute. +# If left as blank, the "common" loader will be used as Catalina's "server" +# loader. +# Examples: +# "foo": Add this folder as a class repository +# "foo/*.jar": Add all the JARs of the specified folder as class +# repositories +# "foo/bar.jar": Add bar.jar as a class repository +server.loader= + +# +# List of comma-separated paths defining the contents of the "shared" +# classloader. Prefixes should be used to define what is the repository type. +# Path may be relative to the CATALINA_BASE path or absolute. If left as blank, +# the "common" loader will be used as Catalina's "shared" loader. +# Examples: +# "foo": Add this folder as a class repository +# "foo/*.jar": Add all the JARs of the specified folder as class +# repositories +# "foo/bar.jar": Add bar.jar as a class repository +# Please note that for single jars, e.g. bar.jar, you need the URL form +# starting with file:. +shared.loader= + +# +# String cache configuration. +tomcat.util.buf.StringCache.byte.enabled=true +#tomcat.util.buf.StringCache.char.enabled=true +#tomcat.util.buf.StringCache.trainThreshold=500000 +#tomcat.util.buf.StringCache.cacheSize=5000 diff --git a/bigtop-packages/src/common/sqoop/do-component-build b/bigtop-packages/src/common/sqoop/do-component-build index 42a24423..dc7200fe 100644 --- a/bigtop-packages/src/common/sqoop/do-component-build +++ b/bigtop-packages/src/common/sqoop/do-component-build @@ -15,5 +15,19 @@ # limitations under the License. set -ex + . `dirname ${0}`/bigtop.bom -ant -f build.xml -Dhadoopversion=200 -Dhadoop.version.full=$HADOOP_VERSION -Dhadoop.version=$HADOOP_VERSION -Dhbase.version=$HBASE_VERSION -Dzookeeper.version=$ZOOKEEPER_VERSION package jar "$@" + +export MAVEN_OPTS='-Xmx512m -XX:MaxPermSize=512m' +mvn -DskipTests=true -Dhadoop.profile=200 -Dhadoop.2.version=$HADOOP_VERSION install package -P=binary $@ + +cd dist/target +for file in `find ./ -name 'sqoop-*-bin-hadoop*' -type d`; do + mv $file sqoop-$SQOOP_VERSION +done +tar czf sqoop-$SQOOP_VERSION.tar.gz sqoop-$SQOOP_VERSION +cd ../.. + +mkdir build +mv dist/target/sqoop-$SQOOP_VERSION.tar.gz build/ + diff --git a/bigtop-packages/src/common/sqoop/install_sqoop.sh b/bigtop-packages/src/common/sqoop/install_sqoop.sh index 771931a3..15f07b21 100755 --- a/bigtop-packages/src/common/sqoop/install_sqoop.sh +++ b/bigtop-packages/src/common/sqoop/install_sqoop.sh @@ -23,6 +23,7 @@ usage: $0 <options> Required not-so-options: --build-dir=DIR path to sqoopdist.dir --prefix=PREFIX path to install into + --extra-dir=DIR path to Bigtop distribution files Optional options: --doc-dir=DIR path to install docs into [/usr/share/doc/sqoop] @@ -31,6 +32,7 @@ usage: $0 <options> --bin-dir=DIR path to install bins [/usr/bin] --conf-dir=DIR path to configuration files provided by the package [/etc/sqoop/conf.dist] --examples-dir=DIR path to install examples [doc-dir/examples] + --initd-dir=DIR path to install init scripts [/etc/init.d] ... [ see source for more similar options ] " exit 1 @@ -46,7 +48,10 @@ OPTS=$(getopt \ -l 'installed-lib-dir:' \ -l 'bin-dir:' \ -l 'examples-dir:' \ - -l 'build-dir:' -- "$@") + -l 'build-dir:' \ + -l 'extra-dir:' \ + -l 'initd-dir:' \ + -l 'dist-dir:' -- "$@") if [ $? != 0 ] ; then usage @@ -80,6 +85,15 @@ while true ; do --examples-dir) EXAMPLES_DIR=$2 ; shift 2 ;; + --extra-dir) + EXTRA_DIR=$2 ; shift 2 + ;; + --initd-dir) + INITD_DIR=$2 ; shift 2 + ;; + --dist-dir) + DIST_DIR=$2 ; shift 2 + ;; --) shift ; break ;; @@ -104,51 +118,47 @@ BIN_DIR=${BIN_DIR:-/usr/lib/sqoop/bin} ETC_DIR=${ETC_DIR:-/etc/sqoop} MAN_DIR=${MAN_DIR:-/usr/share/man/man1} CONF_DIR=${CONF_DIR:-${ETC_DIR}/conf.dist} +WEB_DIR=${WEB_DIR:-/usr/lib/sqoop/sqoop-server} +INITD_DIR=${INITD_DIR:-/etc/init.d} +DIST_DIR=${DIST_DIR:-dist/target/sqoop-*} install -d -m 0755 ${PREFIX}/${LIB_DIR} - -install -d -m 0755 ${PREFIX}/${LIB_DIR} -cp $BUILD_DIR/sqoop*.jar ${PREFIX}/${LIB_DIR} - +install -d -m 0755 ${PREFIX}/${LIB_DIR}/client-lib +install -d -m 0755 ${PREFIX}/${LIB_DIR}/server-lib install -d -m 0755 ${PREFIX}/${LIB_DIR}/lib -cp -a $BUILD_DIR/lib/*.jar ${PREFIX}/${LIB_DIR}/lib - -#install -d -m 0755 ${PREFIX}/${LIB_DIR}/shims -#cp -a shims/*.jar ${PREFIX}/${LIB_DIR}/shims - -install -d -m 0755 $PREFIX/usr/bin - -install -d -m 0755 $PREFIX/${BIN_DIR} -cp $BUILD_DIR/bin/* $PREFIX/${BIN_DIR} - -install -d -m 0755 $PREFIX/${DOC_DIR} -cp $BUILD_DIR/docs/*.html $PREFIX/${DOC_DIR} -cp $BUILD_DIR/docs/*.css $PREFIX/${DOC_DIR} -cp -r $BUILD_DIR/docs/api $PREFIX/${DOC_DIR} -cp -r $BUILD_DIR/docs/images $PREFIX/${DOC_DIR} - - -install -d -m 0755 $PREFIX/$MAN_DIR -for i in sqoop sqoop-codegen sqoop-export sqoop-import-all-tables sqoop-version sqoop-create-hive-table sqoop-help sqoop-list-databases sqoop-eval sqoop-import sqoop-list-tables sqoop-job sqoop-metastore sqoop-merge - do echo "Copying manpage $i" - cp ${BUILD_DIR}/docs/man/$i* $PREFIX/$MAN_DIR - echo "Creating wrapper for $i" - wrapper=$PREFIX/usr/bin/$i - mkdir -p `dirname $wrapper` - cat > $wrapper <<EOF -#!/bin/sh - -# Autodetect JAVA_HOME if not defined -. /usr/lib/bigtop-utils/bigtop-detect-javahome - -export SQOOP_HOME=$LIB_DIR -exec $BIN_DIR/$i "\$@" -EOF - chmod 0755 $wrapper +install -d -m 0755 ${PREFIX}/${BIN_DIR} +install -d -m 0755 ${PREFIX}/${CONF_DIR} +install -d -m 0755 ${PREFIX}/etc/default + +install -m 0644 ${DIST_DIR}/client/lib/*.jar ${PREFIX}/${LIB_DIR}/client-lib/ +install -m 0755 ${DIST_DIR}/bin/sqoop.sh ${PREFIX}/${BIN_DIR}/ +install -m 0755 ${EXTRA_DIR}/setenv.sh ${PREFIX}/${BIN_DIR}/ + +install -m 0644 ${DIST_DIR}/server/conf/sqoop_bootstrap.properties ${PREFIX}/${CONF_DIR} +install -m 0644 ${EXTRA_DIR}/sqoop.properties ${PREFIX}/${CONF_DIR} +install -m 0644 ${EXTRA_DIR}/sqoop.default ${PREFIX}/etc/default/sqoop-server + +# Explode the WAR +SQOOP_WEBAPPS=${PREFIX}/${LIB_DIR}/webapps +cp -r ${DIST_DIR}/server/webapps $SQOOP_WEBAPPS +unzip -d $SQOOP_WEBAPPS/sqoop $SQOOP_WEBAPPS/sqoop.war + +# Create MR2 configuration +install -d -m 0755 ${PREFIX}/${LIB_DIR}/sqoop-server/conf +for conf in web.xml tomcat-users.xml server.xml logging.properties context.xml catalina.policy +do + install -m 0644 ${DIST_DIR}/server/conf/$conf ${PREFIX}/${LIB_DIR}/sqoop-server/conf/ done +sed -i -e "s|<Host |<Host workDir=\"/var/tmp/sqoop\" |" ${PREFIX}/${LIB_DIR}/sqoop-server/conf/server.xml +sed -i -e "s|\${catalina\.base}/logs|/var/log/sqoop|" ${PREFIX}/${LIB_DIR}/sqoop-server/conf/logging.properties +cp -f ${EXTRA_DIR}/catalina.properties ${PREFIX}/${LIB_DIR}/sqoop-server/conf/catalina.properties +ln -s ../webapps ${PREFIX}/${LIB_DIR}/sqoop-server/webapps +ln -s ../bin ${PREFIX}/${LIB_DIR}/sqoop-server/bin + +# Create wrapper scripts for the client and server +client_wrapper=$PREFIX/usr/bin/sqoop +server_wrapper=$PREFIX/usr/bin/sqoop-server +mkdir -p $PREFIX/usr/bin +install -m 0755 $EXTRA_DIR/sqoop.sh $client_wrapper +install -m 0755 $EXTRA_DIR/sqoop-server.sh $server_wrapper -install -d -m 0755 $PREFIX/$CONF_DIR -(cd ${BUILD_DIR}/conf && tar cf - .) | (cd $PREFIX/$CONF_DIR && tar xf -) - -unlink $PREFIX/$LIB_DIR/conf || /bin/true -ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/conf diff --git a/bigtop-packages/src/common/sqoop/setenv.sh b/bigtop-packages/src/common/sqoop/setenv.sh new file mode 100644 index 00000000..fec8ae0a --- /dev/null +++ b/bigtop-packages/src/common/sqoop/setenv.sh @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +JAVA_OPTS=-Dsqoop.config.dir=/etc/sqoop/conf + diff --git a/bigtop-packages/src/common/sqoop/sqoop-metastore.sh b/bigtop-packages/src/common/sqoop/sqoop-metastore.sh deleted file mode 100644 index ab22ef30..00000000 --- a/bigtop-packages/src/common/sqoop/sqoop-metastore.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# chkconfig: 2345 90 10 -# description: Sqoop allows easy imports and exports of data sets between \ -# databases and the Hadoop Distributed File System (HDFS). The Sqoop \ -# metastore allows users to define saved jobs for repeated execution and \ -# share them with other users of the cluster. -# processname: java -# pidfile: /var/run/sqoop/sqoop-metastore.pid -### BEGIN INIT INFO -# Provides: Sqoop -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $remote_fs -# Should-Start: $named -# Should-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Sqoop allows easy imports and exports of data sets between databases and the Hadoop Distributed File System (HDFS). -### END INIT INFO -set -e - -# Autodetect JAVA_HOME if not defined -if [ -e /usr/libexec/bigtop-detect-javahome ]; then - . /usr/libexec/bigtop-detect-javahome -elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then - . /usr/lib/bigtop-utils/bigtop-detect-javahome -fi - -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -NAME=sqoop-metastore -DESC="Sqoop metastore" -PID_FILE=/var/run/sqoop/sqoop-metastore.pid -LOGDIR=/var/log/sqoop - -DODTIME=3 - -# Returns 0 if pid is alive, 1 if not. -hadoop_is_process_alive() { - local pid="$1" - ps -fp $pid | grep $pid | grep sqoop > /dev/null 2>&1 -} - -hadoop_check_pidfile() { - local pidfile="$1" # IN - local pid - - pid=`cat "$pidfile" 2>/dev/null` - if [ "$pid" = '' ]; then - # The file probably does not exist or is empty. - return 1 - fi - - set -- $pid - pid="$1" - - hadoop_is_process_alive $pid -} - -hadoop_process_kill() { - local pid="$1" # IN - local signal="$2" # IN - local second - - kill -$signal $pid 2>/dev/null - - # Wait a bit to see if the dirty job has really been done - for second in {0..10}; do - if hadoop_is_process_alive "$pid"; then - # Success - return 0 - fi - - sleep 1 - done - - # Timeout - return 1 -} -hadoop_stop_pidfile() { - local pidfile="$1" # IN - local pid - - pid=`cat "$pidfile" 2>/dev/null` - if [ "$pid" = '' ]; then - # The file probably does not exist or is empty. Success - return 0 - fi - - set -- $pid - pid="$1" - - # First try the easy way - if hadoop_process_kill "$pid" 15; then - return 0 - fi - - # Otherwise try the hard way - if hadoop_process_kill "$pid" 9; then - return 0 - fi - - return 1 -} - - -start() { - # Pid files created in sqoop-specific directory under /var/run. - # The dir should be recreated first. - local piddir=`dirname "$PID_FILE"` - install -d -m 0755 -o sqoop -g sqoop "$piddir" - su -s /bin/sh sqoop -c \ - "/usr/lib/sqoop/bin/start-metastore.sh -p $PID_FILE -l $LOGDIR" -} -stop() { - su -s /bin/sh sqoop -c \ - "/usr/lib/sqoop/bin/stop-metastore.sh -p $PID_FILE" -} - -case "$1" in - start) - start - ;; - stop) - stop - ;; - force-stop) - echo -n "Forcefully stopping $DESC: " - hadoop_stop_pidfile $PID_FILE - if ! hadoop_check_pidfile $PID_FILE ; then - echo "$NAME." - else - echo "ERROR." - fi - rm $PID_FILE - ;; - force-reload|condrestart|try-restart) - # check whether $DAEMON is running. If so, restart - hadoop_check_pidfile $PID_FILE && $0 restart - ;; - restart|reload) - echo -n "Restarting $DESC: " - stop - [ -n "$DODTIME" ] && sleep $DODTIME - $0 start - ;; - status) - echo -n "$NAME is " - if hadoop_check_pidfile $PID_FILE ; then - echo "running" - else - echo "not running." - exit 1 - fi - ;; - - *) - N=/etc/init.d/$NAME - echo "Usage: $N {start|stop|restart|force-reload|status|force-stop|condrestart|try-restart}" >&2 - exit 1 - ;; -esac - -exit 0 diff --git a/bigtop-packages/src/common/sqoop/sqoop-metastore.sh.suse b/bigtop-packages/src/common/sqoop/sqoop-metastore.sh.suse deleted file mode 100644 index bcc1fef8..00000000 --- a/bigtop-packages/src/common/sqoop/sqoop-metastore.sh.suse +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Starts a Sqoop metastore -# -# chkconfig: 2345 90 10 -# description: Sqoop allows easy imports and exports of data sets between \ -# databases and the Hadoop Distributed File System (HDFS). The Sqoop \ -# metastore allows users to define saved jobs for repeated execution and \ -# share them with other users of the cluster. -# processname: java -# -### BEGIN INIT INFO -# Provides: Sqoop -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $remote_fs -# Should-Start: $named -# Should-Stop: -# Default-Start: 3 4 5 -# Default-Stop: 0 1 2 6 -# Short-Description: Sqoop allows easy imports and exports of data sets between databases and the Hadoop Distributed File System (HDFS). -### END INIT INFO - -. /lib/lsb/init-functions - -# Autodetect JAVA_HOME if not defined -if [ -e /usr/libexec/bigtop-detect-javahome ]; then - . /usr/libexec/bigtop-detect-javahome -elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then - . /usr/lib/bigtop-utils/bigtop-detect-javahome -fi - -STATUS_RUNNING=0 -STATUS_DEAD=1 -STATUS_DEAD_AND_LOCK=2 -STATUS_NOT_RUNNING=3 - -ERROR_PROGRAM_NOT_INSTALLED=5 -ERROR_PROGRAM_NOT_CONFIGURED=6 - - -RETVAL=0 -NAME=sqoop-metastore -DESC="Sqoop metastore" -PID_FILE=/var/run/sqoop/sqoop-metastore.pid -LOCKFILE="/var/lock/subsys/sqoop-metastore" -LOGDIR=/var/log/sqoop -SQOOP_BIN_PATH="/usr/lib/sqoop/bin" -USER="sqoop" -GROUP="sqoop" - -start() { - [ -x ${SQOOP_BIN_PATH}/start-metastore.sh ] || exit $ERROR_PROGRAM_NOT_INSTALLED - - # Pid files created in sqoop-specific directory under /var/run. - # The dir should be recreated first. - local piddir=`dirname "$PID_FILE"` - install -d -m 0755 -o $USER -g $GROUP "$piddir" - - log_success_msg "Starting $DESC: " - start_daemon -u $USER ${SQOOP_BIN_PATH}/start-metastore.sh -p $PID_FILE -l $LOGDIR - RETVAL=$? - echo - [ $RETVAL -eq 0 ] && touch $LOCKFILE - return $RETVAL -} - -stop() { - [ -x ${SQOOP_BIN_PATH}/stop-metastore.sh ] || exit $ERROR_PROGRAM_NOT_INSTALLED - log_success_msg "Stopping $DESC: " - start_daemon -u $USER ${SQOOP_BIN_PATH}/stop-metastore.sh -p $PID_FILE - RETVAL=$? - sleep 5 - echo - [ $RETVAL -eq 0 ] && rm -f $LOCKFILE $PIDFILE -} - -restart() { - stop - start -} - -checkstatus(){ - pidofproc -p $PID_FILE java > /dev/null - status=$? - - case "$status" in - $STATUS_RUNNING) - log_success_msg "$DESC is running" - ;; - $STATUS_DEAD) - log_failure_msg "$DESC is dead and pid file exists" - ;; - $STATUS_DEAD_AND_LOCK) - log_failure_msg "$DESC is dead and lock file exists" - ;; - $STATUS_NOT_RUNNING) - log_failure_msg "$DESC is not running" - ;; - *) - log_failure_msg "$DESC status is unknown" - ;; - esac - return $status -} - -condrestart(){ - [ -e $LOCKFILE ] && restart || : -} - -case "$1" in - start) - start - ;; - stop) - stop - ;; - status) - checkstatus - ;; - restart) - restart - ;; - condrestart|try-restart) - condrestart - ;; - *) - echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart}" - exit 1 -esac - -exit $RETVAL diff --git a/bigtop-packages/src/common/sqoop/sqoop-server.sh b/bigtop-packages/src/common/sqoop/sqoop-server.sh new file mode 100644 index 00000000..b1de3010 --- /dev/null +++ b/bigtop-packages/src/common/sqoop/sqoop-server.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autodetect JAVA_HOME if not defined +if [ -e /usr/libexec/bigtop-detect-javahome ]; then + . /usr/libexec/bigtop-detect-javahome +elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then + . /usr/lib/bigtop-utils/bigtop-detect-javahome +fi + +LIB_DIR=${LIB_DIR:-/usr/lib} + +SQOOP_HOME=${LIB_DIR}/sqoop +TOMCAT_HOME=${LIB_DIR}/bigtop-tomcat + +export CATALINA_BIN=${CATALINA_BIN:-${TOMCAT_HOME}/bin} +export CATALINA_BASE=${CATALINA_BASE:-${SQOOP_HOME}/sqoop-server} +export CATALINA_OPTS=${CATALINA_OPTS:--Xmx1024m} +export CATALINA_OUT=${CATALINE_OUT:-/var/log/sqoop/sqoop-tomcat.log} + +env CLASSPATH=$CLASSPATH $SQOOP_HOME/bin/sqoop.sh server $@ + diff --git a/bigtop-packages/src/common/sqoop/sqoop-server.svc b/bigtop-packages/src/common/sqoop/sqoop-server.svc new file mode 100644 index 00000000..dba8970c --- /dev/null +++ b/bigtop-packages/src/common/sqoop/sqoop-server.svc @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DAEMON="sqoop-server" +DESC="Sqoop Server" +EXEC_PATH="/usr/bin/sqoop-server" +SVC_USER="${SQOOP_USER:-sqoop}" +DAEMON_FLAGS="" +CONF_DIR="${SQOOP_CONFIG:-/etc/sqoop/conf}" +PIDFILE="${SQOOP_PID:-/var/run/sqoop/sqoop-server-$SVC_USER.pid}" +LOCKDIR="/var/lock/subsys" +LOCKFILE="sqoop-server" + +generate_functions() { + +cat <<'__EOT__' +sqoop_env() { + export SQOOP_CONFIG + export CATALINA_BASE + export CATALINA_BIN + export CATALINA_PID=$SQOOP_PID + export CATALINA_OPTS + export CATALINA_OUT + export CATALINA_TMPDIR +} +__EOT__ + +} + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED + [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED + log_success_msg "Starting ${DESC}: " + + sqoop_env + + su -s /bin/bash -c "cd ~/ && ${EXEC_PATH} start $DAEMON_FLAGS" $SVC_USER + + for second in {5..0} + do + checkstatusofproc + RETVAL=$? + if [ "$RETVAL" -eq $RETVAL_SUCCESS ] ; then + break + fi + sleep 1 + done + + [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + log_success_msg "Stopping ${DESC}: " + # FIXME: workaround for BIGTOP-537 + checkstatusofproc + if [ "$?" = "$STATUS_RUNNING" ] ; then + + sqoop_env + + su -s /bin/bash -c "cd ~/ && ${EXEC_PATH} stop" $SVC_USER + + SQOOP_SHUTDOWN_TIMEOUT=${SQOOP_SHUTDOWN_TIMEOUT:-60} + + if [ -f $PIDFILE ]; then + PID=`cat $PIDFILE` + if [ -n $PID ]; then + kill -TERM $PID &>/dev/null + for i in `seq 1 ${SQOOP_SHUTDOWN_TIMEOUT}` ; do + kill -0 ${SQOOP_PID} &>/dev/null || break + sleep 1 + done + kill -KILL ${SQOOP_PID} &>/dev/null + fi + fi + + RETVAL=$? + else + RETVAL=$RETVAL_SUCCESS + fi + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE +} +__EOT__ + +} diff --git a/bigtop-packages/src/common/sqoop/sqoop.default b/bigtop-packages/src/common/sqoop/sqoop.default new file mode 100644 index 00000000..830687e8 --- /dev/null +++ b/bigtop-packages/src/common/sqoop/sqoop.default @@ -0,0 +1,11 @@ +CATALINA_BASE=/usr/lib/sqoop/sqoop-server +SQOOP_USER=sqoop +SQOOP_CONFIG=/etc/conf/sqoop +SQOOP_LOG=/var/log/sqoop +SQOOP_TEMP=/var/run/sqoop +SQOOP_PID=/var/run/sqoop/sqoop-server-sqoop.pid +CATALINA_BIN=/usr/lib/bigtop-tomcat/bin +CATALINA_TMPDIR=/var/tmp/sqoop +CATALINA_OPTS=-Xmx1024m +CATALINA_OUT=/var/log/sqoop/sqoop-tomcat.log + diff --git a/bigtop-packages/src/common/sqoop/sqoop.properties b/bigtop-packages/src/common/sqoop/sqoop.properties new file mode 100644 index 00000000..a98ab0a2 --- /dev/null +++ b/bigtop-packages/src/common/sqoop/sqoop.properties @@ -0,0 +1,101 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Sqoop configuration file used by the built in configuration +# provider: org.apache.sqoop.core.PropertiesConfigurationProvider. +# This file must reside in the system configuration directory +# which is specified by the system property "sqoop.config.dir" +# and must be called sqoop.properties. +# + +# +# Logging Configuration +# Any property that starts with the prefix +# org.apache.sqoop.log4j is parsed out by the configuration +# system and passed to the log4j subsystem. This allows you +# to specify log4j configuration properties from within the +# Sqoop configuration. +# +org.apache.sqoop.log4j.appender.file=org.apache.log4j.RollingFileAppender +org.apache.sqoop.log4j.appender.file.File=/var/log/sqoop/sqoop.log +org.apache.sqoop.log4j.appender.file.MaxFileSize=25MB +org.apache.sqoop.log4j.appender.file.MaxBackupIndex=5 +org.apache.sqoop.log4j.appender.file.layout=org.apache.log4j.PatternLayout +org.apache.sqoop.log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} [%l] %m%n +org.apache.sqoop.log4j.debug=true +org.apache.sqoop.log4j.rootCategory=WARN, file +org.apache.sqoop.log4j.category.org.apache.sqoop=DEBUG +org.apache.sqoop.log4j.category.org.apache.derby=INFO + +# +# Repository configuration +# The Repository subsystem provides the special prefix which +# is "org.apache.sqoop.repository.sysprop". Any property that +# is specified with this prefix is parsed out and set as a +# system property. For example, if the built in Derby repository +# is being used, the sysprop prefixed proeprties can be used +# to affect Derby configuration at startup time by setting +# the appropriate system properties. +# + +# Repository provider +org.apache.sqoop.repository.provider=org.apache.sqoop.repository.JdbcRepositoryProvider + +# JDBC repository provider configuration +org.apache.sqoop.repository.jdbc.handler=org.apache.sqoop.repository.derby.DerbyRepositoryHandler +org.apache.sqoop.repository.jdbc.transaction.isolation=READ_COMMITTED +org.apache.sqoop.repository.jdbc.maximum.connections=10 +org.apache.sqoop.repository.jdbc.url=jdbc:derby:/var/lib/sqoop/repository/db;create=true +org.apache.sqoop.repository.jdbc.create.schema=true +org.apache.sqoop.repository.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver +org.apache.sqoop.repository.jdbc.user=sa +org.apache.sqoop.repository.jdbc.password= + +# System properties for embedded Derby configuration +org.apache.sqoop.repository.sysprop.derby.stream.error.file=/var/log/sqoop/derbyrepo.log + +# Sleeping period for reloading configuration file (once a minute) +org.apache.sqoop.core.configuration.provider.properties.sleep=60000 + +# +# Submission engine configuration +# + +# Submission engine class +org.apache.sqoop.submission.engine=org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine + +# Number of milliseconds, submissions created before this limit will be removed, default is one day +#org.apache.sqoop.submission.purge.threshold= + +# Number of milliseconds for purge thread to sleep, by default one day +#org.apache.sqoop.submission.purge.sleep= + +# Number of milliseconds for update thread to sleep, by default 5 minutes +#org.apache.sqoop.submission.update.sleep= + +# +# Configuration for Mapreduce submission engine (applicable if it's configured) +# + +# Hadoop configuration directory +org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/etc/hadoop/conf/ + +# +# Execution engine configuration +# +org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.MapreduceExecutionEngine diff --git a/bigtop-packages/src/common/sqoop/sqoop.sh b/bigtop-packages/src/common/sqoop/sqoop.sh new file mode 100644 index 00000000..afe1ac6a --- /dev/null +++ b/bigtop-packages/src/common/sqoop/sqoop.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autodetect JAVA_HOME if not defined +if [ -e /usr/libexec/bigtop-detect-javahome ]; then + . /usr/libexec/bigtop-detect-javahome +elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then + . /usr/lib/bigtop-utils/bigtop-detect-javahome +fi + +LIB_DIR=/usr/lib/sqoop +BIN_DIR=${LIB_DIR}/bin + +CLASSPATH=$CLASSPATH:$(echo "$LIB_DIR"/client-lib/*.jar | tr ' ' ':') + +env CLASSPATH=$CLASSPATH $BIN_DIR/sqoop.sh client $@ + diff --git a/bigtop-packages/src/deb/sqoop/changelog b/bigtop-packages/src/deb/sqoop/changelog index 547ed021..d4858cd4 100644 --- a/bigtop-packages/src/deb/sqoop/changelog +++ b/bigtop-packages/src/deb/sqoop/changelog @@ -1 +1 @@ ---- This is auto-generated +--- This is auto-generated diff --git a/bigtop-packages/src/deb/sqoop/compat b/bigtop-packages/src/deb/sqoop/compat index 7f8f011e..4d446d8d 100644 --- a/bigtop-packages/src/deb/sqoop/compat +++ b/bigtop-packages/src/deb/sqoop/compat @@ -1 +1,2 @@ 7 + diff --git a/bigtop-packages/src/deb/sqoop/control b/bigtop-packages/src/deb/sqoop/control index dcc61069..39d911b8 100644 --- a/bigtop-packages/src/deb/sqoop/control +++ b/bigtop-packages/src/deb/sqoop/control @@ -16,20 +16,21 @@ Source: sqoop Section: misc Priority: extra Maintainer: Bigtop <dev@bigtop.apache.org> -Build-Depends: debhelper (>= 7.0.50~), asciidoc, xmlto Standards-Version: 3.8.0 -Homepage: http://incubator.apache.org/sqoop/ +Homepage: http://sqoop.apache.org/ Package: sqoop Architecture: all -Depends: hadoop-client, bigtop-utils -Description: Sqoop allows easy imports and exports of data sets between databases and the Hadoop Distributed File System (HDFS). - Sqoop allows easy imports and exports of data sets between databases and the Hadoop Distributed File System (HDFS). +Depends: bigtop-utils, hadoop-client, bigtop-tomcat, sqoop-client (= ${source:Version}) +Description: Tool for easy imports and exports of data sets between databases and the Hadoop ecosystem + Sqoop is a tool that provides the ability to import and export data sets between + the Hadoop Distributed File System (HDFS) and relational databases. -Package: sqoop-metastore +Package: sqoop-client Architecture: all -Depends: sqoop (= ${source:Version}), adduser -Description: Shared metadata repository for Sqoop. - Shared metadata repository for Sqoop. This optional package hosts a metadata - server for Sqoop clients across a network to use. +Description: Lightweight client for Sqoop. +Package: sqoop-server +Architecture: all +Depends: sqoop (= ${source:Version}) +Description: Server for Sqoop. diff --git a/bigtop-packages/src/deb/sqoop/copyright b/bigtop-packages/src/deb/sqoop/copyright index ec176a91..fa3369a4 100644 --- a/bigtop-packages/src/deb/sqoop/copyright +++ b/bigtop-packages/src/deb/sqoop/copyright @@ -1,9 +1,9 @@ Format: http://dep.debian.net/deps/dep5 -Source: http://incubator.apache.org/projects/sqoop.html +Source: http://sqoop.apache.org Upstream-Name: Sqoop -Files * -Copyright: 2011, The Apache Software Foundation +Files debian/* +Copyright: 2013, The Apache Software Foundation License: Apache-2.0 License: Apache-2.0 diff --git a/bigtop-packages/src/deb/sqoop/rules b/bigtop-packages/src/deb/sqoop/rules index 7992dcde..3ee27f98 100755 --- a/bigtop-packages/src/deb/sqoop/rules +++ b/bigtop-packages/src/deb/sqoop/rules @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# # -*- makefile -*- # Uncomment this to turn on verbose mode. @@ -27,11 +27,18 @@ export DH_OPTIONS dh $@ override_dh_auto_build: - bash debian/do-component-build -Divy.home=${HOME}/.ivy2 + # There's no good way to get the RAT plugin to ignore the debian/ directory, so bigtop-empty is used as a decoy + rm -rf bigtop-empty + mkdir -p bigtop-empty + env FULL_VERSION=${SQOOP_VERSION} bash debian/do-component-build \ + -Drat.basedir=$${PWD}/bigtop-empty -Dmaven.repo.local=${HOME}/.m2/repository override_dh_auto_install: sh -x debian/install_sqoop.sh \ - --build-dir=build/sqoop-${SQOOP_BASE_VERSION}.bin__hadoop-* \ - --conf-dir=/etc/sqoop/conf.dist \ - --prefix=debian/sqoop - cp debian/sqoop-metastore.sh debian/sqoop-metastore.init + --build-dir=build/sqoop-${SQOOP_VERSION} \ + --doc-dir=/usr/share/doc/sqoop \ + --prefix=debian/tmp \ + --extra-dir=debian + bash debian/init.d.tmpl debian/sqoop-server.svc deb debian/tmp/etc/init.d/sqoop-server + dh_install --sourcedir=debian/tmp + diff --git a/bigtop-packages/src/deb/sqoop/sqoop-client.install b/bigtop-packages/src/deb/sqoop/sqoop-client.install new file mode 100644 index 00000000..669a2069 --- /dev/null +++ b/bigtop-packages/src/deb/sqoop/sqoop-client.install @@ -0,0 +1,3 @@ +/usr/bin/sqoop +/usr/lib/sqoop/bin/sqoop.sh +/usr/lib/sqoop/client-lib diff --git a/bigtop-packages/src/deb/sqoop/sqoop-server.install b/bigtop-packages/src/deb/sqoop/sqoop-server.install new file mode 100644 index 00000000..49505ce4 --- /dev/null +++ b/bigtop-packages/src/deb/sqoop/sqoop-server.install @@ -0,0 +1 @@ +/etc/init.d/sqoop-server diff --git a/bigtop-packages/src/deb/sqoop/sqoop-metastore.postinst b/bigtop-packages/src/deb/sqoop/sqoop-server.postinst index b8b6635c..02613830 100644 --- a/bigtop-packages/src/deb/sqoop/sqoop-metastore.postinst +++ b/bigtop-packages/src/deb/sqoop/sqoop-server.postinst @@ -1,5 +1,4 @@ #!/bin/sh -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,21 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# postinst script for sqoop - set -e - -case "$1" in - configure) - ;; - - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -#DEBHELPER# +if [ -x "/etc/init.d/sqoop-server" ]; then + update-rc.d sqoop-server defaults > /dev/null || exit 1 + invoke-rc.d sqoop-server start || : +fi diff --git a/bigtop-packages/src/deb/sqoop/sqoop.install b/bigtop-packages/src/deb/sqoop/sqoop.install new file mode 100644 index 00000000..aac3cc4a --- /dev/null +++ b/bigtop-packages/src/deb/sqoop/sqoop.install @@ -0,0 +1,7 @@ +/usr/bin/sqoop-server +/etc/sqoop/conf.dist +/etc/default/sqoop-server +/usr/lib/sqoop/bin/setenv.sh +/usr/lib/sqoop/sqoop-server +/usr/lib/sqoop/webapps +/usr/lib/sqoop/server-lib diff --git a/bigtop-packages/src/deb/sqoop/sqoop.postinst b/bigtop-packages/src/deb/sqoop/sqoop.postinst index 37cae7e3..3155330a 100644 --- a/bigtop-packages/src/deb/sqoop/sqoop.postinst +++ b/bigtop-packages/src/deb/sqoop/sqoop.postinst @@ -1,5 +1,4 @@ #!/bin/sh -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,24 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# postinst script for sqoop - set -e -case "$1" in - configure) - # Install config alternatives - update-alternatives --install /etc/sqoop/conf sqoop-conf /etc/sqoop/conf.dist 30 - - ;; - - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac +update-alternatives --install /etc/sqoop/conf sqoop-conf /etc/sqoop/conf.dist 30 -#DEBHELPER# diff --git a/bigtop-packages/src/deb/sqoop/sqoop-metastore.preinst b/bigtop-packages/src/deb/sqoop/sqoop.preinst index 76cfb265..289519a6 100644 --- a/bigtop-packages/src/deb/sqoop/sqoop-metastore.preinst +++ b/bigtop-packages/src/deb/sqoop/sqoop.preinst @@ -1,21 +1,12 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 +#!/bin/bash # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -# preinst script for sqoop-metastore +# +# preinst script for sqoop-server # # see: dh_installdeb(1) @@ -37,13 +28,14 @@ case "$1" in adduser \ --system \ --group \ - --home /var/lib/sqoop \ + --home /var/run/sqoop \ --gecos "Sqoop User" \ --shell /bin/false \ sqoop >/dev/null fi - install -d -m 0755 -o sqoop -g sqoop /var/lib/sqoop - install -d -m 0755 -o sqoop -g sqoop /var/log/sqoop + install -d -m 0755 -o sqoop -g sqoop /var/log/sqoop + install -d -m 0755 -o sqoop -g sqoop /var/lib/sqoop + install -d -m 0755 -o sqoop -g sqoop /var/tmp/sqoop ;; abort-upgrade) diff --git a/bigtop-packages/src/deb/sqoop/sqoop.prerm b/bigtop-packages/src/deb/sqoop/sqoop.prerm index b9a25a53..16cc2965 100644 --- a/bigtop-packages/src/deb/sqoop/sqoop.prerm +++ b/bigtop-packages/src/deb/sqoop/sqoop.prerm @@ -15,24 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# prerm script for sqoop +# prerm script for sqoop # # see: dh_installdeb(1) set -e -# summary of how this script can be called: -# * <prerm> `remove' -# * <old-prerm> `upgrade' <new-version> -# * <new-prerm> `failed-upgrade' <old-version> -# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version> -# * <deconfigured's-prerm> `deconfigure' `in-favour' -# <package-being-installed> <version> `removing' -# <conflicting-package> <version> -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - case "$1" in remove|upgrade|deconfigure) update-alternatives --remove sqoop-conf /etc/sqoop/conf.dist || : diff --git a/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec b/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec index 9774326e..55e80f24 100644 --- a/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec +++ b/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec @@ -12,20 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + %define lib_sqoop /usr/lib/sqoop -%define conf_sqoop %{_sysconfdir}/%{name}/conf +%define conf_sqoop %{_sysconfdir}/sqoop/conf %define conf_sqoop_dist %{conf_sqoop}.dist - +%define run_sqoop /var/run/sqoop %if %{?suse_version:1}0 -# Only tested on openSUSE 11.4. le'ts update it for previous release when confirmed +# Only tested on openSUSE 11.4. let's update it for previous release when confirmed %if 0%{suse_version} > 1130 %define suse_check \# Define an empty suse_check for compatibility with older sles %endif # SLES is more strict anc check all symlinks point to valid path -# But we do point to a hadoop jar which is not there at build time +# But we do point to a conf which is not there at build time # (but would be at install time). # Since our package build system does not handle dependencies, # these symlink checks are deactivated @@ -35,22 +36,21 @@ %{nil} %define doc_sqoop %{_docdir}/sqoop -%global initd_dir %{_sysconfdir}/rc.d +%define initd_dir %{_sysconfdir}/rc.d %define alternatives_cmd update-alternatives %else %define doc_sqoop %{_docdir}/sqoop-%{sqoop_version} -%global initd_dir %{_sysconfdir}/rc.d/init.d +%define initd_dir %{_sysconfdir}/rc.d/init.d %define alternatives_cmd alternatives %endif - Name: sqoop Version: %{sqoop_version} Release: %{sqoop_release} -Summary: Sqoop allows easy imports and exports of data sets between databases and the Hadoop Distributed File System (HDFS). +Summary: Tool for easy imports and exports of data sets between databases and the Hadoop ecosystem URL: http://incubator.apache.org/sqoop/ Group: Development/Libraries Buildroot: %{_topdir}/INSTALL/%{name}-%{version} @@ -58,21 +58,31 @@ License: APL2 Source0: %{name}-%{sqoop_base_version}.tar.gz Source1: do-component-build Source2: install_%{name}.sh -Source3: sqoop-metastore.sh -Source4: sqoop-metastore.sh.suse +Source3: sqoop.sh +Source4: sqoop.properties +Source5: catalina.properties +Source6: setenv.sh +Source7: sqoop.default +Source8: init.d.tmpl +Source9: sqoop-server.svc +Source10: sqoop-server.sh Buildarch: noarch -BuildRequires: asciidoc, xmlto -Requires: hadoop-client, bigtop-utils +BuildRequires: asciidoc +Requires: hadoop-client, bigtop-utils, bigtop-tomcat, %{name}-client = %{version}-%{release} -%description -Sqoop allows easy imports and exports of data sets between databases and the Hadoop Distributed File System (HDFS). +%description +Sqoop is a tool that provides the ability to import and export data sets between the Hadoop Distributed File System (HDFS) and relational databases. -%package metastore -Summary: Shared metadata repository for Sqoop. -URL: http://incubator.apache.org/sqoop/ +%package client +Summary: Client for Sqoop. +URL: http://sqoop.apache.org Group: System/Daemons -Requires: sqoop = %{version}-%{release} -Requires(pre): %{name} = %{version}-%{release} + +%package server +Summary: Server for Sqoop. +URL: http://sqoop.apache.org +Group: System/Daemons +Requires: sqoop = %{version}-%{release} %if %{?suse_version:1}0 # Required for init scripts @@ -91,110 +101,86 @@ Requires: initscripts Requires: redhat-lsb %endif +%description client +Lightweight client for Sqoop. -%description metastore -Shared metadata repository for Sqoop. This optional package hosts a metadata -server for Sqoop clients across a network to use. +%description server +Centralized server for Sqoop. %prep -%setup -n %{name}-%{sqoop_base_version} +%setup -n sqoop-%{sqoop_base_version} %build -bash %{SOURCE1} -Dversion=%{sqoop_base_version} +# No easy way to disable the default RAT run which fails the build because of some fails in the debian/ directory +rm -rf bigtop-empty +mkdir -p bigtop-empty +# I could not find a way to add debian/ to RAT exclude list through cmd line +# or to unbind rat:check goal +# So I am redirecting its attention with a decoy +env FULL_VERSION=%{sqoop_base_version} bash %{SOURCE1} -Drat.basedir=${PWD}/bigtop-empty %install %__rm -rf $RPM_BUILD_ROOT sh %{SOURCE2} \ - --build-dir=build/sqoop-%{sqoop_base_version}.bin__hadoop-* \ + --build-dir=build/sqoop-%{sqoop_version} \ --conf-dir=%{conf_sqoop_dist} \ --doc-dir=%{doc_sqoop} \ - --prefix=$RPM_BUILD_ROOT - -%__install -d -m 0755 $RPM_BUILD_ROOT/usr/bin -%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/ - -%__rm -f $RPM_BUILD_ROOT/%{lib_sqoop}/lib/hadoop-mrunit*.jar + --prefix=$RPM_BUILD_ROOT \ + --extra-dir=$RPM_SOURCE_DIR \ + --initd-dir=%{initd_dir} -%if %{?suse_version:1}0 -orig_init_file=$RPM_SOURCE_DIR/sqoop-metastore.sh.suse -%else -orig_init_file=$RPM_SOURCE_DIR/sqoop-metastore.sh -%endif +# Install init script +init_file=$RPM_BUILD_ROOT/%{initd_dir}/sqoop-server +bash $RPM_SOURCE_DIR/init.d.tmpl $RPM_SOURCE_DIR/sqoop-server.svc rpm $init_file -init_file=$RPM_BUILD_ROOT/%{initd_dir}/sqoop-metastore -%__cp $orig_init_file $init_file -chmod 0755 $init_file - -%__install -d -m 0755 $RPM_BUILD_ROOT/var/lib/sqoop -%__install -d -m 0755 $RPM_BUILD_ROOT/var/log/sqoop +%__install -d -m 0755 $RPM_BUILD_ROOT/usr/bin %pre getent group sqoop >/dev/null || groupadd -r sqoop -getent passwd sqoop > /dev/null || useradd -c "Sqoop" -s /sbin/nologin \ - -g sqoop -r -d /var/lib/sqoop sqoop 2> /dev/null || : +getent passwd sqoop >/dev/null || useradd -c "Sqoop User" -s /sbin/nologin -g sqoop -r -d %{run_sqoop} sqoop 2> /dev/null || : +%__install -d -o sqoop -g sqoop -m 0755 /var/lib/sqoop +%__install -d -o sqoop -g sqoop -m 0755 /var/log/sqoop +%__install -d -o sqoop -g sqoop -m 0755 /var/tmp/sqoop +%__install -d -o sqoop -g sqoop -m 0755 /var/run/sqoop %post -%{alternatives_cmd} --install %{conf_sqoop} %{name}-conf %{conf_sqoop_dist} 30 +%{alternatives_cmd} --install %{conf_sqoop} sqoop-conf %{conf_sqoop_dist} 30 + +%post server +chkconfig --add sqoop-server %preun -if [ "$1" = 0 ]; then - %{alternatives_cmd} --remove %{name}-conf %{conf_sqoop_dist} || : +if [ "$1" = "0" ] ; then + %{alternatives_cmd} --remove sqoop-conf %{conf_sqoop_dist} || : fi -%post metastore -chkconfig --add sqoop-metastore - -%preun metastore -if [ $1 = 0 ] ; then - service sqoop-metastore stop > /dev/null 2>&1 - chkconfig --del sqoop-metastore +%preun server +if [ "$1" = "0" ] ; then + service sqoop-server stop > /dev/null 2>&1 + chkconfig --del sqoop-server fi -%postun metastore +%postun server if [ $1 -ge 1 ]; then - service sqoop-metastore condrestart > /dev/null 2>&1 + service sqoop-server condrestart > /dev/null 2>&1 fi -%files metastore -%attr(0755,root,root) %{initd_dir}/sqoop-metastore -%attr(0755,sqoop,sqoop) /var/lib/sqoop -%attr(0755,sqoop,sqoop) /var/log/sqoop - -# Files for main package -%files +%files %defattr(0755,root,root) -%{lib_sqoop} -%config(noreplace) %{conf_sqoop_dist} -%{_bindir}/sqoop -%{_bindir}/sqoop-codegen -%{_bindir}/sqoop-create-hive-table -%{_bindir}/sqoop-eval -%{_bindir}/sqoop-export -%{_bindir}/sqoop-help -%{_bindir}/sqoop-import -%{_bindir}/sqoop-import-all-tables -%{_bindir}/sqoop-job -%{_bindir}/sqoop-list-databases -%{_bindir}/sqoop-list-tables -%{_bindir}/sqoop-metastore -%{_bindir}/sqoop-version -%{_bindir}/sqoop-merge - -%defattr(0644,root,root,0755) -%{lib_sqoop}/lib/ -%{lib_sqoop}/*.jar -%{_mandir}/man1/sqoop.1.* -%{_mandir}/man1/sqoop-codegen.1.* -%{_mandir}/man1/sqoop-create-hive-table.1.* -%{_mandir}/man1/sqoop-eval.1.* -%{_mandir}/man1/sqoop-export.1.* -%{_mandir}/man1/sqoop-help.1.* -%{_mandir}/man1/sqoop-import-all-tables.1.* -%{_mandir}/man1/sqoop-import.1.* -%{_mandir}/man1/sqoop-job.1.* -%{_mandir}/man1/sqoop-list-databases.1.* -%{_mandir}/man1/sqoop-list-tables.1.* -%{_mandir}/man1/sqoop-metastore.1.* -%{_mandir}/man1/sqoop-version.1.* -%{_mandir}/man1/sqoop-merge.1.* -%doc %{doc_sqoop} +/usr/bin/sqoop-server +%config(noreplace) /etc/sqoop/conf.dist +%config(noreplace) /etc/default/sqoop-server +%{lib_sqoop}/sqoop-server +%{lib_sqoop}/webapps +%{lib_sqoop}/bin/setenv.sh +%{lib_sqoop}/server-lib + +%files client +%attr(0755,root,root) +/usr/bin/sqoop +%{lib_sqoop}/bin/sqoop.sh +%{lib_sqoop}/client-lib + +%files server +%attr(0755,root,root) %{initd_dir}/sqoop-server + diff --git a/bigtop-packages/src/templates/init.d.tmpl b/bigtop-packages/src/templates/init.d.tmpl new file mode 100644 index 00000000..61db5d56 --- /dev/null +++ b/bigtop-packages/src/templates/init.d.tmpl @@ -0,0 +1,309 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a poor man's templating engine for generating init.d scripts to +# support all the Apache services that Bigtop distro has. An actual init.d +# script gets generate via running this script under bash and giving it +# a mandatory argument of a file containing the configuration for the service. +# The argument file should be a valid piece of bash code since it gets directly +# source into this template. E.g. +# $ bash ./init.d.tmpl hadoop-hdfs-namenode.svc > hadoop-hdfs-namenode +# +# You must declare the following in your .svc configuration file: +# DAEMON="name of the resulting init.d script" +# DESC="Free form human readable description of the service" +# EXEC_PATH="path to the upstream daemon management script" +# SVC_USER="user to run this service as" +# DAEMON_FLAGS="flags to be passed to the $EXEC_PATH" +# CONF_DIR="path to the configuration directory" +# PIDFILE="file holding a PID of the running daemon" +# LOCKFILE="file signifying the service lock" +# +# CHKCONFIG="chkconfig(8) registration signature" +# INIT_DEFAULT_START="run levels to use" +# INIT_DEFAULT_STOP="run levels not to use ;-)" +# +# You can, also, override parts of the generated init.d script by providing +# function definitions for: generate_start, generate_stop, generate_functions, and generate_extra_commands. +# See the default implemenations below and feel free to customize. Also look +# for exising .svc files in common to see how different services are tweaking +# the defaults. +# +# Of course, if this whole templating thing grows too big we might need to +# consider a real templating engine (I have played with m4, but it seems +# qutie brittle when used for manipulating pieces of the shell code -- think +# $0 the like). + +if [ $# -lt 3 ] ; then + echo "Usage: ${BASH_SOURCE-0} service.definition.svc rpm|deb init.d.script" + exit 1 +fi + +# Override anything necessary for supported systems +case "$2" in + "deb" ) + # The defaults will work well on Debian-based systems + ;; + "rpm" ) + # On RedHat, SuSE and Mageia run-level 2 is networkless, hence excluding it + CHKCONFIG="345 85 15" + INIT_DEFAULT_START="3 4 5" + INIT_DEFAULT_STOP="0 1 2 6" + ;; + "*" ) + ;; +esac + +# Create the init script, make it executable, and send stdout to it +mkdir -p `dirname $3` +touch $3 +chmod 0755 $3 +exec > $3 + +generate_start() { + +cat <<'__EOT__' + +start() { + [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED + [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED + log_success_msg "Starting ${DESC}: " + + su -s /bin/bash $SVC_USER -c "$EXEC_PATH --config '$CONF_DIR' start $DAEMON_FLAGS" + + # Some processes are slow to start + sleep $SLEEP_TIME + checkstatusofproc + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE + return $RETVAL +} + +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' + +stop() { + log_success_msg "Stopping ${DESC}: " + start_daemon $EXEC_PATH --config "$CONF_DIR" stop $DAEMON_FLAGS + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE +} + +__EOT__ + +} + +generate_functions() { + : +} + +generate_extra_commands() { + +cat <<'__EOT__' + *) + echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart}" + exit 1 +__EOT__ + +} + +################################################################### +# Some reasonable defaults for the run-level settings, these are +# tweaked on a per-OS basis from the SPEC/rules files that call us +# and can be further tweaked inside of individual .svc templates +CHKCONFIG=${CHKCONFIG:-"2345 85 15"} +INIT_DEFAULT_START=${INIT_DEFAULT_START:-"2 3 4 5"} +INIT_DEFAULT_STOP=${INIT_DEFAULT_STOP:-"0 1 6"} + +################################################################### +# NOTE how we are sourcing the argument here so that a user-defined +# settings have a chance to override the default values for +# generate_start, generate_stop, generate_functions and generate_extra_commands. If you +# ever want to make this template even more flexible -- define the +# default values above this line +. $1 + +cat <<__EOT__ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Starts a $DESC +# +# chkconfig: $CHKCONFIG +# description: $DESC +# +### BEGIN INIT INFO +# Provides: $DAEMON +# Short-Description: $DESC +# Default-Start: $INIT_DEFAULT_START +# Default-Stop: $INIT_DEFAULT_STOP +# Required-Start: \$syslog \$remote_fs +# Required-Stop: \$syslog \$remote_fs +# Should-Start: +# Should-Stop: +### END INIT INFO + +. /lib/lsb/init-functions +. /etc/default/hadoop + +if [ -f /etc/default/$DAEMON ] ; then + . /etc/default/$DAEMON +fi + +# Autodetect JAVA_HOME if not defined +if [ -e /usr/libexec/bigtop-detect-javahome ]; then + . /usr/libexec/bigtop-detect-javahome +elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then + . /usr/lib/bigtop-utils/bigtop-detect-javahome +fi + +RETVAL_SUCCESS=0 + +STATUS_RUNNING=0 +STATUS_DEAD=1 +STATUS_DEAD_AND_LOCK=2 +STATUS_NOT_RUNNING=3 +STATUS_OTHER_ERROR=102 + + +ERROR_PROGRAM_NOT_INSTALLED=5 +ERROR_PROGRAM_NOT_CONFIGURED=6 + + +RETVAL=0 +SLEEP_TIME=5 +PROC_NAME="java" + +DAEMON="$DAEMON" +DESC="$DESC" +EXEC_PATH="$EXEC_PATH" +SVC_USER="$SVC_USER" +DAEMON_FLAGS="$DAEMON_FLAGS" +CONF_DIR="$CONF_DIR" +PIDFILE="$PIDFILE" +LOCKDIR="$LOCKDIR" +LOCKFILE="\$LOCKDIR/$LOCKFILE" + +install -d -m 0755 -o $SVC_USER -g $SVC_USER $(dirname $PIDFILE) 1>/dev/null 2>&1 || : +[ -d "\$LOCKDIR" ] || install -d -m 0755 \$LOCKDIR 1>/dev/null 2>&1 || : +__EOT__ + +generate_functions +generate_start +generate_stop + +cat <<'__EOT__' +restart() { + stop + start +} + +checkstatusofproc(){ + pidofproc -p $PIDFILE $PROC_NAME > /dev/null +} + +checkstatus(){ + checkstatusofproc + status=$? + + case "$status" in + $STATUS_RUNNING) + log_success_msg "${DESC} is running" + ;; + $STATUS_DEAD) + log_failure_msg "${DESC} is dead and pid file exists" + ;; + $STATUS_DEAD_AND_LOCK) + log_failure_msg "${DESC} is dead and lock file exists" + ;; + $STATUS_NOT_RUNNING) + log_failure_msg "${DESC} is not running" + ;; + *) + log_failure_msg "${DESC} status is unknown" + ;; + esac + return $status +} + +condrestart(){ + [ -e $LOCKFILE ] && restart || : +} + +check_for_root() { + if [ $(id -ur) -ne 0 ]; then + echo 'Error: root user required' + echo + exit 1 + fi +} + +service() { + case "$1" in + start) + check_for_root + start + ;; + stop) + check_for_root + stop + ;; + status) + checkstatus + RETVAL=$? + ;; + restart) + check_for_root + restart + ;; + condrestart|try-restart) + check_for_root + condrestart + ;; +__EOT__ + +generate_extra_commands + +cat <<'__EOT__' + esac +} + +service "$1" + +exit $RETVAL +__EOT__ @@ -88,8 +88,8 @@ $(eval $(call PACKAGE,hive,HIVE)) SQOOP_NAME=sqoop SQOOP_RELNOTES_NAME=Sqoop SQOOP_PKG_NAME=sqoop -SQOOP_BASE_VERSION=1.4.2 -SQOOP_PKG_VERSION=1.4.2 +SQOOP_BASE_VERSION=1.99.1 +SQOOP_PKG_VERSION=1.99.1 SQOOP_RELEASE_VERSION=1 SQOOP_TARBALL_DST=$(SQOOP_NAME)-$(SQOOP_BASE_VERSION).tar.gz SQOOP_TARBALL_SRC=$(SQOOP_TARBALL_DST) @@ -51,6 +51,7 @@ $(BUILD_DIR)/%/.srpm: cp -r $(BASE_DIR)/bigtop-packages/src/rpm/$($(PKG)_NAME)/* $(PKG_BUILD_DIR)/rpm/ mkdir -p $(PKG_BUILD_DIR)/rpm/{INSTALL,SOURCES,BUILD,SRPMS} [ -z "$($(PKG)_TARBALL_SRC)" ] || cp $($(PKG)_DOWNLOAD_DST) $(PKG_BUILD_DIR)/rpm/SOURCES + cp $(BASE_DIR)/bigtop-packages/src/templates/init.d.tmpl $(PKG_BUILD_DIR)/rpm/SOURCES [ -d $(BASE_DIR)/bigtop-packages/src/common/$($(PKG)_NAME) ] && cp -r $(BASE_DIR)/bigtop-packages/src/common/$($(PKG)_NAME)/* $(PKG_BUILD_DIR)/rpm/SOURCES echo -e "$(BIGTOP_BOM)" | tr ' ' '\012' >> $(PKG_BUILD_DIR)/rpm/SOURCES/bigtop.bom PKG_NAME_FOR_PKG=$(subst -,_,$($(PKG)_NAME)); \ @@ -99,6 +100,7 @@ $(BUILD_DIR)/%/.sdeb: fi cd $(PKG_BUILD_DIR)/deb/$($(PKG)_NAME)-$(PKG_PKG_VERSION)$(BIGTOP_BUILD_STAMP) && \ cp -r $(BASE_DIR)/bigtop-packages/src/deb/$($(PKG)_NAME) debian && \ + cp $(BASE_DIR)/bigtop-packages/src/templates/init.d.tmpl debian && \ cp -r $(BASE_DIR)/bigtop-packages/src/common/$($(PKG)_NAME)/* debian && \ echo -e "$(BIGTOP_BOM)" | tr ' ' '\012' >> debian/bigtop.bom && \ (echo -e "$($(PKG)_PKG_NAME) ($(PKG_PKG_VERSION)$(BIGTOP_BUILD_STAMP)-$($(PKG)_RELEASE)) stable; urgency=low\n" && \ |