diff options
Diffstat (limited to 'bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs')
-rw-r--r-- | bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs | 79 |
1 files changed, 79 insertions, 0 deletions
diff --git a/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs b/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs new file mode 100644 index 00000000..7041daf2 --- /dev/null +++ b/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A simple wrapper script around hadoop which enables the use of qfs easily. +# Using the hadoop command with qfs requires users to specify overrides for +# various parameters (e.g. fs.defaultFS) which can become cumbersome to do on +# the command line every time. The goal here is to provide a clean user +# experience in using hadoop and qfs. + +export JAVA_LIBRARY_PATH=/usr/lib/qfs + +QFS_OPTS=" -Dfs.qfs.impl=com.quantcast.qfs.hadoop.QuantcastFileSystem" +QFS_OPTS="$QFS_OPTS -Dfs.defaultFS=qfs://<%= scope['qfs::common::metaserver_host'] %>:<%= scope['qfs::common::metaserver_client_port'] %>" +QFS_OPTS="$QFS_OPTS -Dfs.qfs.metaServerHost=<%= scope['qfs::common::metaserver_host'] %>" +QFS_OPTS="$QFS_OPTS -Dfs.qfs.metaServerPort=<%= scope['qfs::common::metaserver_port'] %>" + +usage() { + echo " +usage: $0 <options> + $0 is a simple wrapper around the standard hadoop command that allows for an + easy interface with qfs. A subset of the standard hadoop commands are + supported (where it makes sense). For all other commands, use the hadoop + command directly. + + Commands: + fs query the qfs filesystem + jar <jar> <main class> submit a job to hadoop that uses qfs + Note: the main class is required + distcp <srcurl> <desturl> copy file or directories recursively + Note: supports copy into qfs from hdfs + " + exit 1 +} + +if [ $# == 0 ]; then + usage +fi + +COMMAND=$1 +shift + +case $COMMAND in + # usage flags + --help|-h) + usage + ;; + fs|distcp) + hadoop $COMMAND $QFS_OPTS "$@" + ;; + jar) + if [ $# -lt 2 ]; then + echo "$COMMAND: jar file and main class are required" + usage + fi + + JAR_FILE=$1 + MAIN_CLASS=$2 + shift 2 + hadoop $COMMAND $JAR_FILE $MAIN_CLASS $QFS_OPTS "$@" + ;; + *) + echo "$COMMAND: unsupported -- try using the hadoop command directly" + usage + ;; +esac |