1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
usage() {
echo "usage: $PROG [-C file ] args"
echo " -C file Use alternate file for vagrantconfig.yaml"
echo " commands:"
echo " -b, --build-image Build base Docker image for Bigtop Hadoop"
echo " (must be exectued at least once before creating cluster)"
echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
echo " -d, --destroy Destroy the cluster"
echo " -h, --help"
exit 1
}
build-image() {
echo "\$vagrantyamlconf = \"$vagrantyamlconf\"" > config.rb
vagrant up image --provider docker
{
echo "echo -e '\nBUILD IMAGE SUCCESS.\n'" |vagrant ssh image
} || {
>&2 echo -e "\nBUILD IMAGE FAILED!\n"
exit 2
}
}
create() {
echo "\$num_instances = $1" > config.rb
echo "\$vagrantyamlconf = \"$vagrantyamlconf\"" >> config.rb
vagrant up --no-parallel
if [ $? -ne 0 ]; then
echo "Docker container(s) startup failed!";
exit 1;
fi
nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`)
hadoop_head_node=(`echo "hostname -f" |vagrant ssh ${nodes[0]} |tail -n 1`)
repo=$(get-yaml-config repo)
components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]"
jdk=$(get-yaml-config jdk)
distro=$(get-yaml-config distro)
enable_local_repo=$(get-yaml-config enable_local_repo)
# setup environment before running bigtop puppet deployment
for node in ${nodes[*]}; do
(
echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$distro.sh $enable_local_repo" |vagrant ssh $node
echo "/vagrant/provision.sh $hadoop_head_node $repo \"$components\" $jdk" |vagrant ssh $node
) &
done
wait
# run bigtop puppet (master node need to be provisioned before slave nodes)
bigtop-puppet ${nodes[0]}
for ((i=1 ; i<${#nodes[*]} ; i++)); do
bigtop-puppet ${nodes[$i]} &
done
wait
}
provision() {
nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`)
for node in ${nodes[*]}; do
bigtop-puppet $node &
done
wait
}
smoke-tests() {
nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`)
smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`"
echo "/bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh \"$smoke_test_components\"" |vagrant ssh ${nodes[0]}
}
destroy() {
nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`)
rm -rvf ./hosts ./config.rb
for node in ${nodes[*]}; do
vagrant destroy -f $node
done
wait
}
bigtop-puppet() {
echo "puppet apply -d --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules:/usr/share/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp" |vagrant ssh $1
}
get-yaml-config() {
RUBY_EXE=ruby
which ruby > /dev/null 2>&1
if [ $? -ne 0 ]; then
# use vagrant embedded ruby on Windows
RUBY_EXE=$(dirname $(which vagrant))/../embedded/bin/ruby
fi
RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1'];"
cat ${vagrantyamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r'
}
PROG=`basename $0`
if [ $# -eq 0 ]; then
usage
fi
vagrantyamlconf="vagrantconfig.yaml"
while [ $# -gt 0 ]; do
case "$1" in
-b|--build-image)
build-image
shift;;
-c|--create)
if [ $# -lt 2 ]; then
echo "Create requires a number" 1>&2
usage
fi
create $2
shift 2;;
-C|--conf)
if [ $# -lt 2 ]; then
echo "Alternative config file for vagrantconfig.yaml" 1>&2
usage
fi
vagrantyamlconf=$2
shift 2;;
-p|--provision)
provision
shift;;
-s|--smoke-tests)
smoke-tests
shift;;
-d|--destroy)
destroy
shift;;
-h|--help)
usage
shift;;
*)
echo "Unknown argument: '$1'" 1>&2
usage;;
esac
done
|
