Added tests into TestSpecsRuntime and testRuntimeSpecConf so they get run. Tested against an HDP cluster.

(cherry picked from commit a911703ccacc70b738720c560574fc8a9aee8b2b)

5 files changed, 130 insertions, 77 deletions

diff --git a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java
index 2ae97a29..d95c010d 100644
--- a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java
+++ b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java
@@ -1,11 +1,7 @@
 /**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
  * <p>
@@ -62,15 +58,6 @@ public class ApiExaminer {
     unloadableClasses.add("org.apache.hadoop.io.compress.lz4.Lz4Compressor");
     unloadableClasses.add("org.apache.hadoop.record.compiler.ant.RccTask");
 
-    /*
-    jarsWeCareAbout = new ArrayList<>();
-    jarsWeCareAbout.add("hadoop-common");
-    jarsWeCareAbout.add("hadoop-hdfs");
-    jarsWeCareAbout.add("hadoop-yarn-common");
-    jarsWeCareAbout.add("hadoop-yarn-client");
-    jarsWeCareAbout.add("hadoop-yarn-api");
-    jarsWeCareAbout.add("hadoop-mapreduce-client-core");
-    */
   }
 
   public static void main(String[] args) {
diff --git a/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-checker.sh b/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-checker.sh
deleted file mode 100755
index 1cb95832..00000000
--- a/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-checker.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# <p>
-# http://www.apache.org/licenses/LICENSE-2.0
-# <p>
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-############################################################################
-
-function usage() {
-    echo "You must set the following variables:  HADOOP_COMMON_HOME HADOOP_COMMON_DIR HADOOP_COMMON_LIB_JARS_DIR "
-    echo "HADOOP_HDFS_HOME HDFS_DIR HDFS_LIB_JARS_DIR HADOOP_YARN_HOME YARN_DIR YARN_LIB_JARS_DIR "
-    echo "HADOOP_MAPRED_HOME MAPRED_DIR MAPRED_LIB_JARS_DIR BIGTOP_HOME (location of bigtop source)"
-    echo "You can get the Hadoop environment variables by using hadoop envvars, hdfs envvars, yarn envvars, and mapred envvars"
-}
-
-for envar in x$HADOOP_COMMON_HOME x$HADOOP_COMMON_DIR x$HADOOP_COMMON_LIB_JARS_DIR x$HADOOP_HDFS_HOME x$HDFS_DIR \
-             x$HDFS_LIB_JARS_DIR x$HADOOP_YARN_HOME x$YARN_DIR x$YARN_LIB_JARS_DIR x$HADOOP_MAPRED_HOME x$MAPRED_DIR \
-             x$MAPRED_LIB_JARS_DIR
-do
-    if [ "${envar}" = "x" ]
-    then
-        usage
-        exit 1
-    fi
-done
-
-
-for dir in $BIGTOP_HOME/bigtop-tests/spec-tests/runtime/build/libs/ $HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR \
-            $HADOOP_COMMON_HOME/$HADOOP_COMMON_LIB_JARS_DIR $HADOOP_HDFS_HOME/$HDFS_DIR \
-            $HADOOP_HDFS_HOME/$HDFS_LIB_JARS_DIR $HADOOP_YARN_HOME/$YARN_DIR $HADOOP_YARN_HOME/$YARN_LIB_JARS_DIR \
-            $HADOOP_MAPRED_HOME/$MAPRED_DIR $HADOOP_MAPRED_HOME/$MAPRED_LIB_JARS_DIR
-do
-    for jar in `find $dir -name \*.jar`
-    do
-        CLASSPATH=$CLASSPATH:$jar
-    done
-done
-
-java -cp $CLASSPATH org.odpi.specs.runtime.hadoop.ApiExaminer $@
-
diff --git a/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-prep.sh b/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-prep.sh
index 0bdfe05e..8c9ab5e6 100755
--- a/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-prep.sh
+++ b/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-prep.sh
@@ -1,16 +1,12 @@
 #!/usr/bin/env bash
 
 ############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# <p>
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# <p>
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,6 +14,30 @@
 # limitations under the License.
 ############################################################################
 
+############################################################################
+# This script is used to generate the hadoop-*-api.report.json files in the
+# test/resources directory.  To use it, you will first need to download an
+# Apache binary distribution of Hadoop and set APACHE_HADOOP_DIR to the
+# directory where you untar that distribution.  You will then need to set
+# BIGTTOP_HOME to the directory where your bigtop source is located.  Then
+# run this script for each of the jars you want to generate a report for.
+# The arguments passed to this script should be -p <outputdir> -j <jarfile>
+# where outputdir is the directory you'd like to write the report to and
+# jarfile is the full path of the jar to generate the report for.  Reports
+# should be generated for the following jars: hadoop-common, hadoop-hdfs,
+# hadoop-yarn-common, hadoop-yarn-client, hadoop-yarn-api, and
+# hadoop-mapreduce-client-core
+#
+# Example usage:
+# export APACHE_HADOOP_DIR=/tmp/hadoop-2.7.3
+# export BIGTOP_HOME=/home/me/git/bigtop
+# $BIGTOP_HOME/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner.sh -j $HADOOP_HOME/share/hadoop/common/hadoop-common-2.7.3.jar -p $BIGTOP_HOME/bigtop-tests/spec-tests/runtime/src/test/resources
+#
+# The resulting reports should be committed to git.  This script only needs
+# to be run once per ODPi release.
+############################################################################
+
+
 if [ "x${APACHE_HADOOP_DIR}" = "x" ]
 then
     echo "You must set APACHE_HADOOP_DIR to the directory you have placed the Apache Hadoop binary distribution in"
diff --git a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy
index 2426278b..bc2a3b20 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy
+++ b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy
@@ -25,6 +25,9 @@ import org.junit.runner.RunWith
 import org.junit.runners.Parameterized
 import org.junit.runners.Parameterized.Parameters
 
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
 /**
  * Check all expected environment
  * Tests are constructed dynamically, using external DSL to define
@@ -236,6 +239,35 @@ public class TestSpecsRuntime {
           }
         )
         break
+      case 'api_examination':
+        def basedir = getEnv(arguments['baseDirEnv'], arguments['envcmd'])
+        def libdir = getEnv(arguments['libDir'], arguments['envcmd'])
+
+        def dir = new File(basedir + "/" + libdir)
+        Assert.assertTrue("Expected " + dir.getPath() + " to be a directory", dir.isDirectory())
+        def pattern = Pattern.compile(arguments['jar'] + "-[0-9]+.*\\.jar")
+        def String[] jars = dir.list(new FilenameFilter() {
+          @Override
+          boolean accept(File d, String name) {
+            Matcher matcher = pattern.matcher(name)
+            return (matcher.matches() && !name.contains("test"))
+          }
+        })
+        Assert.assertEquals("Expected only one jar, but got " + jars.join(", "), 1, jars.length)
+        def jar = dir.getAbsolutePath() + "/" + jars[0]
+
+        def examinerJar = System.properties['odpi.test.hive.hcat.job.jar']
+        def resourceFile = System.properties['test.resources.dir']+ "/" + arguments['resourceFile']
+        Shell sh = new Shell()
+        def results = sh.exec("hadoop jar " + examinerJar + " org.odpi.specs.runtime.hadoop.ApiExaminer -c " + resourceFile + " -j " + jar).getErr()
+        int rc = sh.getRet()
+        Assert.assertEquals("Expected command to succeed, but got return code " + rc, 0, rc)
+        if (results.size() > 0) {
+          System.out.println("Received report for jar " + arguments['jar'] + results.join("\n"))
+        }
+        break;
+
+
       default:
         break
     }
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy b/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy
index f5521c5d..339de4cb 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy
@@ -360,5 +360,71 @@ specs {
       arguments {
       }
     }
+    'HADOOP_API1' {
+      name = "HADOOP_API1"
+      type = 'api_examination'
+      arguments {
+        baseDirEnv = 'HADOOP_COMMON_HOME'
+        libDir = 'HADOOP_COMMON_DIR'
+        envcmd = 'hadoop envvars'
+        jar = 'hadoop-common'
+        resourceFile = 'hadoop-common-2.7.3-api-report.json'
+      }
+    }
+    'HADOOP_API2' {
+      name = "HADOOP_API2"
+      type = 'api_examination'
+      arguments {
+        baseDirEnv = 'HADOOP_HDFS_HOME'
+        libDir = 'HDFS_DIR'
+        envcmd = 'hdfs envvars'
+        jar = 'hadoop-hdfs'
+        resourceFile = 'hadoop-hdfs-2.7.3-api-report.json'
+      }
+    }
+    'HADOOP_API3' {
+      name = "HADOOP_API3"
+      type = 'api_examination'
+      arguments {
+        baseDirEnv = 'HADOOP_YARN_HOME'
+        libDir = 'YARN_DIR'
+        envcmd = 'yarn envvars'
+        jar = 'hadoop-yarn-common'
+        resourceFile = 'hadoop-yarn-common-2.7.3-api-report.json'
+      }
+    }
+    'HADOOP_API4' {
+      name = "HADOOP_API4"
+      type = 'api_examination'
+      arguments {
+        baseDirEnv = 'HADOOP_YARN_HOME'
+        libDir = 'YARN_DIR'
+        envcmd = 'yarn envvars'
+        jar = 'hadoop-yarn-client'
+        resourceFile = 'hadoop-yarn-client-2.7.3-api-report.json'
+      }
+    }
+    'HADOOP_API5' {
+      name = "HADOOP_API5"
+      type = 'api_examination'
+      arguments {
+        baseDirEnv = 'HADOOP_YARN_HOME'
+        libDir = 'YARN_DIR'
+        envcmd = 'yarn envvars'
+        jar = 'hadoop-yarn-api'
+        resourceFile = 'hadoop-yarn-api-2.7.3-api-report.json'
+      }
+    }
+    'HADOOP_API6' {
+      name = "HADOOP_API6"
+      type = 'api_examination'
+      arguments {
+        baseDirEnv = 'HADOOP_MAPRED_HOME'
+        libDir = 'MAPRED_DIR'
+        envcmd = 'mapred envvars'
+        jar = 'hadoop-mapreduce-client-core'
+        resourceFile = 'hadoop-mapreduce-client-core-2.7.3-api-report.json'
+      }
+    }
   }
 }