1 files changed, 485 insertions, 0 deletions
diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java b/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java
new file mode 100644
index 00000000..d95c010d
--- /dev/null
+++ b/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java
@@ -0,0 +1,485 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.odpi.specs.runtime.hadoop;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A tool that generates API conformance tests for Hadoop libraries
+ */
+public class ApiExaminer {
+
+  private static final Log LOG = LogFactory.getLog(ApiExaminer.class.getName());
+
+  static private Set<String> unloadableClasses;
+
+  private List<String> errors;
+  private List<String> warnings;
+
+  static {
+    unloadableClasses = new HashSet<>();
+    unloadableClasses.add("org.apache.hadoop.security.JniBasedUnixGroupsMapping");
+    unloadableClasses.add("org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping");
+    unloadableClasses.add("org.apache.hadoop.io.compress.lz4.Lz4Compressor");
+    unloadableClasses.add("org.apache.hadoop.record.compiler.ant.RccTask");
+
+  }
+
+  public static void main(String[] args) {
+    Options options = new Options();
+
+    options.addOption("c", "compare", true,
+        "Compare against a spec, argument is the json file containing spec");
+    options.addOption("h", "help", false, "You're looking at it");
+    options.addOption("j", "jar", true, "Jar to examine");
+    options.addOption("p", "prepare-spec", true,
+        "Prepare the spec, argument is the directory to write the spec to");
+
+    try {
+      CommandLine cli = new GnuParser().parse(options, args);
+
+      if (cli.hasOption('h')) {
+        usage(options);
+        return;
+      }
+
+      if ((!cli.hasOption('c') && !cli.hasOption('p')) ||
+          (cli.hasOption('c') && cli.hasOption('p'))) {
+        System.err.println("You must choose either -c or -p");
+        usage(options);
+        return;
+      }
+
+      if (!cli.hasOption('j')) {
+        System.err.println("You must specify the jar to prepare or compare");
+        usage(options);
+        return;
+      }
+
+      String jar = cli.getOptionValue('j');
+      ApiExaminer examiner = new ApiExaminer();
+
+      if (cli.hasOption('c')) {
+        examiner.compareAgainstStandard(cli.getOptionValue('c'), jar);
+      } else if (cli.hasOption('p')) {
+        examiner.prepareExpected(jar, cli.getOptionValue('p'));
+      }
+    } catch (Exception e) {
+      System.err.println("Received exception while processing");
+      e.printStackTrace();
+    }
+  }
+
+  private static void usage(Options options) {
+    HelpFormatter help = new HelpFormatter();
+    help.printHelp("api-examiner", options);
+
+  }
+
+  private ApiExaminer() {
+  }
+
+  private void prepareExpected(String jarFile, String outputDir) throws IOException,
+      ClassNotFoundException {
+    JarInfo jarInfo = new JarInfo(jarFile, this);
+    jarInfo.dumpToFile(new File(outputDir));
+  }
+
+  private void compareAgainstStandard(String json, String jarFile) throws IOException,
+      ClassNotFoundException {
+    errors = new ArrayList<>();
+    warnings = new ArrayList<>();
+    JarInfo underTest = new JarInfo(jarFile, this);
+    JarInfo standard = jarInfoFromFile(new File(json));
+    standard.compareAndReport(underTest);
+
+    if (errors.size() > 0) {
+      System.err.println("Found " + errors.size() + " incompatibilities:");
+      for (String error : errors) {
+        System.err.println(error);
+      }
+    }
+
+    if (warnings.size() > 0) {
+      System.err.println("Found " + warnings.size() + " possible issues: ");
+      for (String warning : warnings) {
+        System.err.println(warning);
+      }
+    }
+
+
+  }
+
+  private JarInfo jarInfoFromFile(File inputFile) throws IOException {
+    ObjectMapper mapper = new ObjectMapper();
+    JarInfo jarInfo = mapper.readValue(inputFile, JarInfo.class);
+    jarInfo.patchUpClassBackPointers(this);
+    return jarInfo;
+  }
+
+  private static class JarInfo {
+    String name;
+    String version;
+    ApiExaminer container;
+    Map<String, ClassInfo> classes;
+
+    // For use by Jackson
+    public JarInfo() {
+
+    }
+
+    JarInfo(String jarFile, ApiExaminer container) throws IOException, ClassNotFoundException {
+      this.container = container;
+      LOG.info("Processing jar " + jarFile);
+      File f = new File(jarFile);
+      Pattern pattern = Pattern.compile("(hadoop-[a-z\\-]+)-([0-9]\\.[0-9]\\.[0-9]).*");
+      Matcher matcher = pattern.matcher(f.getName());
+      if (!matcher.matches()) {
+        String msg = "Unable to determine name and version from " + f.getName();
+        LOG.error(msg);
+        throw new RuntimeException(msg);
+      }
+      name = matcher.group(1);
+      version = matcher.group(2);
+      classes = new HashMap<>();
+
+      JarFile jar = new JarFile(jarFile);
+      Enumeration<JarEntry> entries = jar.entries();
+      while (entries.hasMoreElements()) {
+        String name = entries.nextElement().getName();
+        if (name.endsWith(".class")) {
+          name = name.substring(0, name.length() - 6);
+          name = name.replace('/', '.');
+          if (!unloadableClasses.contains(name)) {
+            LOG.debug("Processing class " + name);
+            Class<?> clazz = Class.forName(name);
+            if (clazz.getAnnotation(InterfaceAudience.Public.class) != null &&
+                clazz.getAnnotation(InterfaceStability.Stable.class) != null) {
+              classes.put(name, new ClassInfo(this, clazz));
+            }
+          }
+        }
+      }
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public void setName(String name) {
+      this.name = name;
+    }
+
+    public String getVersion() {
+      return version;
+    }
+
+    public void setVersion(String version) {
+      this.version = version;
+    }
+
+    public Map<String, ClassInfo> getClasses() {
+      return classes;
+    }
+
+    public void setClasses(Map<String, ClassInfo> classes) {
+      this.classes = classes;
+    }
+
+    void compareAndReport(JarInfo underTest) {
+      Set<ClassInfo> underTestClasses = new HashSet<>(underTest.classes.values());
+      for (ClassInfo classInfo : classes.values()) {
+        if (underTestClasses.contains(classInfo)) {
+          classInfo.compareAndReport(underTest.classes.get(classInfo.name));
+          underTestClasses.remove(classInfo);
+        } else {
+          container.errors.add(underTest + " does not contain class " + classInfo);
+        }
+      }
+
+      if (underTestClasses.size() > 0) {
+        for (ClassInfo extra : underTestClasses) {
+          container.warnings.add(underTest + " contains extra class " + extra);
+        }
+      }
+    }
+
+    void dumpToFile(File outputDir) throws IOException {
+      File output = new File(outputDir, name + "-" + version + "-api-report.json");
+      ObjectMapper mapper = new ObjectMapper();
+      mapper.writeValue(output, this);
+    }
+
+    void patchUpClassBackPointers(ApiExaminer container) {
+      this.container = container;
+      for (ClassInfo classInfo : classes.values()) {
+        classInfo.setJar(this);
+        classInfo.patchUpBackMethodBackPointers();
+      }
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      if (!(other instanceof JarInfo)) return false;
+      JarInfo that = (JarInfo)other;
+      return name.equals(that.name) && version.equals(that.version);
+    }
+
+    @Override
+    public String toString() {
+      return name + "-" + version;
+    }
+  }
+
+  private static class ClassInfo {
+    @JsonIgnore JarInfo jar;
+    String name;
+    Map<String, MethodInfo> methods;
+
+    // For use by Jackson
+    public ClassInfo() {
+
+    }
+
+    ClassInfo(JarInfo jar, Class<?> clazz) {
+      this.jar = jar;
+      this.name = clazz.getName();
+      methods = new HashMap<>();
+
+      for (Method method : clazz.getMethods()) {
+        if (method.getDeclaringClass().equals(clazz)) {
+          LOG.debug("Processing method " + method.getName());
+          MethodInfo mi = new MethodInfo(this, method);
+          methods.put(mi.toString(), mi);
+        }
+      }
+    }
+
+    public JarInfo getJar() {
+      return jar;
+    }
+
+    public void setJar(JarInfo jar) {
+      this.jar = jar;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public void setName(String name) {
+      this.name = name;
+    }
+
+    public Map<String, MethodInfo> getMethods() {
+      return methods;
+    }
+
+    public void setMethods(Map<String, MethodInfo> methods) {
+      this.methods = methods;
+    }
+
+    void compareAndReport(ClassInfo underTest) {
+      // Make a copy so we can remove them as we match them, making it easy to find additional ones
+      Set<MethodInfo> underTestMethods = new HashSet<>(underTest.methods.values());
+      for (MethodInfo methodInfo : methods.values()) {
+        if (underTestMethods.contains(methodInfo)) {
+          methodInfo.compareAndReport(underTest.methods.get(methodInfo.toString()));
+          underTestMethods.remove(methodInfo);
+        } else {
+          jar.container.errors.add(underTest + " does not contain method " + methodInfo);
+        }
+      }
+
+      if (underTestMethods.size() > 0) {
+        for (MethodInfo extra : underTestMethods) {
+          jar.container.warnings.add(underTest + " contains extra method " + extra);
+        }
+      }
+    }
+
+    void patchUpBackMethodBackPointers() {
+      for (MethodInfo methodInfo : methods.values()) methodInfo.setContainingClass(this);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      if (!(other instanceof ClassInfo)) return false;
+      ClassInfo that = (ClassInfo)other;
+      return name.equals(that.name);  // Classes can be compared just on names
+    }
+
+    @Override
+    public int hashCode() {
+      return name.hashCode();
+    }
+
+    @Override
+    public String toString() {
+      return jar + " " + name;
+    }
+  }
+
+  private static class MethodInfo {
+    @JsonIgnore ClassInfo containingClass;
+    String name;
+    String returnType;
+    List<String> args;
+    Set<String> exceptions;
+
+    // For use by Jackson
+    public MethodInfo() {
+
+    }
+
+    MethodInfo(ClassInfo containingClass, Method method) {
+      this.containingClass = containingClass;
+      this.name = method.getName();
+      args = new ArrayList<>();
+      for (Class<?> argClass : method.getParameterTypes()) {
+        args.add(argClass.getName());
+      }
+      returnType = method.getReturnType().getName();
+      exceptions = new HashSet<>();
+      for (Class<?> exception : method.getExceptionTypes()) {
+        exceptions.add(exception.getName());
+      }
+    }
+
+    public ClassInfo getContainingClass() {
+      return containingClass;
+    }
+
+    public void setContainingClass(ClassInfo containingClass) {
+      this.containingClass = containingClass;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public void setName(String name) {
+      this.name = name;
+    }
+
+    public String getReturnType() {
+      return returnType;
+    }
+
+    public void setReturnType(String returnType) {
+      this.returnType = returnType;
+    }
+
+    public List<String> getArgs() {
+      return args;
+    }
+
+    public void setArgs(List<String> args) {
+      this.args = args;
+    }
+
+    public Set<String> getExceptions() {
+      return exceptions;
+    }
+
+    public void setExceptions(Set<String> exceptions) {
+      this.exceptions = exceptions;
+    }
+
+    void compareAndReport(MethodInfo underTest) {
+      // Check to see if they've added or removed exceptions
+      // Make a copy so I can remove them as I check them off and easily find any that have been
+      // added.
+      Set<String> underTestExceptions = new HashSet<>(underTest.exceptions);
+      for (String exception : exceptions) {
+        if (underTest.exceptions.contains(exception)) {
+          underTestExceptions.remove(exception);
+        } else {
+          containingClass.jar.container.warnings.add(underTest.containingClass.jar + " " +
+              underTest.containingClass + "." + name + " removes exception " + exception);
+        }
+      }
+      if (underTestExceptions.size() > 0) {
+        for (String underTestException : underTest.exceptions) {
+          containingClass.jar.container.warnings.add(underTest.containingClass.jar + " " +
+              underTest.containingClass + "." + name + " adds exception " + underTestException);
+        }
+      }
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      if (!(other instanceof MethodInfo)) return false;
+      MethodInfo that = (MethodInfo)other;
+
+      return containingClass.equals(that.containingClass) && name.equals(that.name) &&
+          returnType.equals(that.returnType) && args.equals(that.args);
+    }
+
+    @Override
+    public int hashCode() {
+      return ((containingClass.hashCode() * 31 + name.hashCode()) * 31 + returnType.hashCode()) * 31 +
+          args.hashCode();
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder buf = new StringBuilder(returnType)
+          .append(" ")
+          .append(name)
+          .append('(');
+      boolean first = true;
+      for (String arg : args) {
+        if (first) first = false;
+        else buf.append(", ");
+        buf.append(arg);
+      }
+      buf.append(")");
+      if (exceptions.size() > 0) {
+        buf.append(" throws ");
+        first = true;
+        for (String exception : exceptions) {
+          if (first) first = false;
+          else buf.append(", ");
+          buf.append(exception);
+        }
+      }
+      return buf.toString();
+    }
+  }
+}