diff options
| -rwxr-xr-x | bigtop-deploy/vm/vagrant-puppet/Vagrantfile | 32 | ||||
| -rwxr-xr-x | bigtop-deploy/vm/vagrant-puppet/provision.sh | 9 | ||||
| -rwxr-xr-x | bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh | 21 | ||||
| -rw-r--r-- | bigtop-tests/smoke-tests/README | 18 | ||||
| -rw-r--r-- | bigtop-tests/smoke-tests/build.gradle | 14 |
5 files changed, 74 insertions, 20 deletions
diff --git a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile index c6825ad8..ffe23a92 100755 --- a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile +++ b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile @@ -20,14 +20,16 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| vmbox = "puppetlab-centos-64-nocm" + # Repository + # Example for testing a Release candidate. + repo = "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/" + # repo = "http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64" + # Virtual box implementation vmboxurl = "http://puppet-vagrant-boxes.puppetlabs.com/centos-64-x64-vbox4210-nocm.box" # number of instances - num_instances = 3 - - # the bigtop puppet recipes - bigtop_puppet = "../../puppet" + num_instances = 1 # manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager) # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier) @@ -56,29 +58,39 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end bigtop.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--memory", "1024"] + vb.customize ["modifyvm", :id, "--memory", "10024"] end bigtop.vm.network :private_network, ip: bigtop_ip bigtop.vm.hostname = bigtop_hostname - bigtop.vm.synced_folder bigtop_puppet, "/bigtop-puppet" + # three levels up is the bigtop "home" directory. + # the current directory has puppet recipes which we need for provisioning. + bigtop.vm.synced_folder "../../../", "/bigtop-home" # shell provisioner to setup basic vm environment bigtop.vm.provision :shell do |shell| shell.path = "provision.sh" - shell.args = bigtop_master + shell.args = [ bigtop_master , repo ] end bigtop.vm.provision :hostmanager # deploy Apache Hadoop and it's ecosystem bigtop.vm.provision :puppet do |puppet| - puppet.module_path = bigtop_puppet + "/modules" - puppet.manifests_path = bigtop_puppet + "/manifests" + puppet.module_path = "../../puppet/modules/" + puppet.manifests_path = "../../puppet/manifests/" puppet.manifest_file = "site.pp" - puppet.options = "--debug --confdir=/bigtop-puppet" + puppet.options = "--debug --confdir=/bigtop-home/bigtop-deploy/puppet/" end + + if i==num_instances then + puts "creating provisioner directive for running tests" + bigtop.vm.provision "shell", path: "smoke-tests.sh" + else + puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}" + end + end end diff --git a/bigtop-deploy/vm/vagrant-puppet/provision.sh b/bigtop-deploy/vm/vagrant-puppet/provision.sh index 1cf37246..fa0571b3 100755 --- a/bigtop-deploy/vm/vagrant-puppet/provision.sh +++ b/bigtop-deploy/vm/vagrant-puppet/provision.sh @@ -23,11 +23,13 @@ service iptables stop chkconfig iptables off cat /dev/null > /etc/hosts +echo "Bigtop yum repo = $2" + # Prepare puppet configuration file -cat > /bigtop-puppet/config/site.csv << EOF +cat > /bigtop-home/bigtop-deploy/puppet/config/site.csv << EOF hadoop_head_node,$1 hadoop_storage_dirs,/data/1,/data/2 -bigtop_yumrepo_uri,http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64 +bigtop_yumrepo_uri,$2 jdk_package_name,java-1.7.0-openjdk-devel.x86_64 components,hadoop,hbase,yarn,mapred-app EOF @@ -39,3 +41,6 @@ mkdir -p /data/{1,2} yum -y install rng-tools sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd service rngd start + +echo "Now installing gradle" +cd /bigtop-home && puppet apply --modulepath=./ -e "include bigtop_toolchain::gradle" # alias gradle=/usr/local/gradle/bin/gradle diff --git a/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh b/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh new file mode 100755 index 00000000..d02c128d --- /dev/null +++ b/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh @@ -0,0 +1,21 @@ +#!/bin/bash +HCFS_USER="hdfs" +gradle="/usr/local/gradle/bin/gradle" + +export HADOOP_CONF_DIR=/etc/hadoop/conf/ +export BIGTOP_HOME=/bigtop-home/ +export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ +export HIVE_HOME=/usr/lib/hive/ +export PIG_HOME=/usr/lib/pig/ +export FLUME_HOME=/usr/lib/flume/ +export HIVE_CONF_DIR=/etc/hive/conf/ +export JAVA_HOME="/usr/lib/jvm/java-1.7.0-openjdk-1.7.0.65.x86_64/" +export MAHOUT_HOME="/usr/lib/mahout" +export ITEST="0.7.0" + +su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' + +yum install -y pig hive flume mahout +cd /bigtop-home/bigtop-tests/smoke-tests && $gradle compileGroovy test -Dsmoke.tests=pig --info diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README index c8ac345b..f682b616 100644 --- a/bigtop-tests/smoke-tests/README +++ b/bigtop-tests/smoke-tests/README @@ -16,14 +16,28 @@ So overall, this framework makes testing with apache bigtop easy as well as fun! ## Running +You need to export several environment variables. For example, +``` +export HADOOP_CONF_DIR=/etc/hadoop/conf/ +export BIGTOP_HOME=/bigtop-home/ +export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapred/ +export HIVE_HOME=/usr/lib/hive/ +export HIVE_CONF_DIR=/etc/hive/conf/ +export JAVA_HOME="/usr/lib/jvm/java-1.7.0-openjdk-1.7.0.65.x86_64/" +export MAHOUT_HOME="/usr/lib/mahout" +export ITEST="0.7.0" +``` + +Then, simply invoke the tests including the smoke.tests you want to run, as described below. + Each directory is a gradle "subproject" mapping to an ecosystem component. If you specify, for example, "flume", then the flume test runs. To test hive as well, you would specify "flume,hive", and so on. An example is below: ``` - gradle compileGroovy test -Dsmoke-tests=flume,hive --info + gradle compileGroovy test -Dsmoke.tests=flume,hive --info ``` -The --info option is a critical feature here: You can easily debug your tests this way, +The --info option is a nice feature to have here: You can easily debug your tests this way, just make sure you have log4j.properties in your conf/ directory locally. ## Adding more tests diff --git a/bigtop-tests/smoke-tests/build.gradle b/bigtop-tests/smoke-tests/build.gradle index 7907600b..44a52e8e 100644 --- a/bigtop-tests/smoke-tests/build.gradle +++ b/bigtop-tests/smoke-tests/build.gradle @@ -27,8 +27,6 @@ subprojects { ext.hadoopVersion = '2.3.0' //Note you need a snapshot itest to run this. - //if you dont have it, change to 0.7.0. - ext.itestVersion = '0.8.0-SNAPSHOT' ext.doExclude = { String filename -> //print("Exclude? ${filename} ... ") def keep_this_test = true; @@ -67,9 +65,13 @@ subprojects { } /** - * In general, java_home should be declared in all cases. + * Two important environment variables. + * java_home should be declared in all cases. + * itest can equal = 0.7.0, 0.8.0-SNAPSHOT, */ - checkEnv(["JAVA_HOME"]) -test.dependsOn compileGroovy -compileGroovy.dependsOn clean + checkEnv(["JAVA_HOME","ITEST"]) + ext.itestVersion = System.getenv("ITEST") + + test.dependsOn compileGroovy + compileGroovy.dependsOn clean } |