ODPI-117: Improvements in checking directory content

This change does the following:
  - Directory contents can now contain regex (to allow for hadoop-*-2.7.[0-9])
  - The tests now display both missing files and extra files that the
    distribution includes
  - Added some allowance for variance in a few key jars

(cherry picked from commit db2a75d5a1ed52a43f13ef7f487008e171b6ccfe)

6 files changed, 180 insertions, 153 deletions

diff --git a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy
index 5133b486..2426278b 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy
+++ b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy
@@ -162,7 +162,7 @@ public class TestSpecsRuntime {
       case 'dircontent':
         def expectedFiles = []
         new File("${testsList}", "${arguments['referenceList']}").eachLine { line ->
-          expectedFiles << line
+          expectedFiles << ~line
         }
 
         def baseDir = getEnv(arguments['baseDirEnv'], arguments['envcmd'])
@@ -187,9 +187,37 @@ public class TestSpecsRuntime {
           }
         }
 
+        def missingList = []
+        for (def wantFile : expectedFiles) {
+          def ok = false
+          for (def haveFile : actualFiles) {
+            if (haveFile =~ wantFile) {
+              ok = true
+              break
+            }
+          }
+          if (! ok) {
+            missingList << wantFile
+          }
+        }
+
+        def extraList = []
+        for (def haveFile : actualFiles) {
+          def ok = false
+          for (def wantFile : expectedFiles) {
+            if (haveFile =~ wantFile) {
+              ok = true
+              break
+            }
+          }
+          if (! ok) {
+            extraList << haveFile
+          }
+        }
+
         def commonFiles = actualFiles.intersect(expectedFiles)
-        Assert.assertTrue("${testName} fail: Directory content for ${dir.path} does not match reference. ",
-           commonFiles.size()==actualFiles.size() && commonFiles.size()==expectedFiles.size())
+        Assert.assertTrue("${testName} fail: Directory content for ${dir.path} does not match reference. Missing files: ${missingList}. Extra files: ${extraList}",
+           missingList.size() == 0 && extraList.size() == 0)
         break
       case 'hadoop_tools':
         def toolsPathStr = getEnv("HADOOP_TOOLS_PATH", "hadoop envvars")
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list
index 2a543932..17edeaeb 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list
@@ -1,61 +1,60 @@
-api-util-1.0.0-M20.jar
-curator-recipes-2.7.1.jar
-curator-framework-2.7.1.jar
-netty-3.6.2.Final.jar
-gson-2.2.4.jar
-paranamer-2.3.jar
-jackson-core-asl-1.9.13.jar
-jackson-xc-1.9.13.jar
-jersey-server-1.9.jar
-stax-api-1.0-2.jar
-zookeeper-3.4.6.jar
-htrace-core-3.1.0-incubating.jar
-slf4j-api-1.7.10.jar
-avro-1.7.4.jar
-slf4j-log4j12-1.7.10.jar
-curator-client-2.7.1.jar
-jets3t-0.9.0.jar
-commons-net-3.1.jar
-jaxb-impl-2.2.3-1.jar
-httpclient-4.2.5.jar
-apacheds-kerberos-codec-2.0.0-M15.jar
-commons-cli-1.2.jar
-log4j-1.2.17.jar
-jackson-mapper-asl-1.9.13.jar
-java-xmlbuilder-0.4.jar
-jsp-api-2.1.jar
-guava-11.0.2.jar
-jetty-6.1.26.jar
-commons-logging-1.1.3.jar
-snappy-java-1.0.4.1.jar
-commons-httpclient-3.1.jar
-jsch-0.1.42.jar
-jersey-core-1.9.jar
-commons-compress-1.4.1.jar
-jettison-1.1.jar
-junit-4.11.jar
-commons-collections-3.2.1.jar
-xz-1.0.jar
-asm-3.2.jar
-commons-codec-1.4.jar
-commons-digester-1.8.jar
-api-asn1-api-1.0.0-M20.jar
-xmlenc-0.52.jar
-commons-configuration-1.6.jar
-mockito-all-1.8.5.jar
-commons-lang-2.6.jar
-jetty-util-6.1.26.jar
-jsr305-3.0.0.jar
-protobuf-java-2.5.0.jar
-httpcore-4.2.5.jar
-commons-io-2.4.jar
-activation-1.1.jar
-jersey-json-1.9.jar
-jaxb-api-2.2.2.jar
-commons-math3-3.1.1.jar
-hamcrest-core-1.3.jar
-commons-beanutils-core-1.8.0.jar
-apacheds-i18n-2.0.0-M15.jar
-servlet-api-2.5.jar
-commons-beanutils-1.7.0.jar
-jackson-jaxrs-1.9.13.jar
+api-util-1\.0\.0-M20([^.0-9].*)?\.jar
+curator-recipes-2\.7\.1([^.0-9].*)?\.jar
+curator-framework-2\.7\.1([^.0-9].*)?\.jar
+netty-3\.6\.2\.Final([^.0-9].*)?\.jar
+gson-2\.2\.4([^.0-9].*)?\.jar
+paranamer-2\.3([^.0-9].*)?\.jar
+jackson-core-asl-1\.9\.13([^.0-9].*)?\.jar
+jackson-xc-1\.9\.13([^.0-9].*)?\.jar
+jersey-server-1\.9([^.0-9].*)?\.jar
+stax-api-1\.0-2([^.0-9].*)?\.jar
+zookeeper-3\.4\.6([^.0-9].*)?\.jar
+htrace-core-3\.1\.0-incubating([^.0-9].*)?\.jar
+slf4j-api-1\.7\.10([^.0-9].*)?\.jar
+avro-1\.7\.[4-7]([^.0-9].*)?\.jar
+slf4j-log4j12-1\.7\.10([^.0-9].*)?\.jar
+curator-client-2\.7\.1([^.0-9].*)?\.jar
+jets3t-0\.9\.0([^.0-9].*)?\.jar
+commons-net-3\.1([^.0-9].*)?\.jar
+jaxb-impl-2\.2\.3-1([^.0-9].*)?\.jar
+httpclient-4\.2\.5([^.0-9].*)?\.jar
+apacheds-kerberos-codec-2\.0\.0-M15([^.0-9].*)?\.jar
+commons-cli-1\.2([^.0-9].*)?\.jar
+log4j-1\.2\.17([^.0-9].*)?\.jar
+jackson-mapper-asl-1\.9\.13([^.0-9].*)?\.jar
+java-xmlbuilder-0\.4([^.0-9].*)?\.jar
+jsp-api-2\.1([^.0-9].*)?\.jar
+guava-11\.0\.2([^.0-9].*)?\.jar
+jetty-6\.1\.26([^.0-9].*)?\.jar
+commons-logging-1\.1\.3([^.0-9].*)?\.jar
+snappy-java-1\.0\.[45](\.[0-9])?([^.0-9].*)?\.jar
+commons-httpclient-3\.1([^.0-9].*)?\.jar
+jsch-0\.1\.(4[2-9]|[5-9]\d)([^.0-9].*)?\.jar
+jersey-core-1\.9([^.0-9].*)?\.jar
+commons-compress-1\.4\.1([^.0-9].*)?\.jar
+jettison-1\.1([^.0-9].*)?\.jar
+junit-4\.11([^.0-9].*)?\.jar
+commons-collections-3\.2\.[12]([^.0-9].*)?\.jar
+xz-1\.0([^.0-9].*)?\.jar
+asm-3\.2([^.0-9].*)?\.jar
+commons-codec-1\.4([^.0-9].*)?\.jar
+commons-digester-1\.8([^.0-9].*)?\.jar
+api-asn1-api-1\.0\.0-M20([^.0-9].*)?\.jar
+xmlenc-0\.52([^.0-9].*)?\.jar
+commons-configuration-1\.6([^.0-9].*)?\.jar
+mockito-all-1\.8\.5([^.0-9].*)?\.jar
+commons-lang-2\.6([^.0-9].*)?\.jar
+jetty-util-6\.1\.26([^.0-9].*)?\.jar
+jsr305-3\.0\.0([^.0-9].*)?\.jar
+protobuf-java-2\.5\.0([^.0-9].*)?\.jar
+httpcore-4\.2\.5([^.0-9].*)?\.jar
+commons-io-2\.4([^.0-9].*)?\.jar
+activation-1\.1([^.0-9].*)?\.jar
+jersey-json-1\.9([^.0-9].*)?\.jar
+jaxb-api-2\.2\.2([^.0-9].*)?\.jar
+commons-math3-3\.1\.1([^.0-9].*)?\.jar
+hamcrest-core-1\.3([^.0-9].*)?\.jar
+commons-beanutils(-core)?-1\.8\.0([^.0-9].*)?\.jar
+apacheds-i18n-2\.0\.0-M15([^.0-9].*)?\.jar
+servlet-api-2\.5([^.0-9].*)?\.jar
+jackson-jaxrs-1\.9\.13([^.0-9].*)?\.jar
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list
index d24d7640..41c171bf 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list
@@ -1,25 +1,25 @@
-netty-3.6.2.Final.jar
-leveldbjni-all-1.8.jar
-jackson-core-asl-1.9.13.jar
-jersey-server-1.9.jar
-htrace-core-3.1.0-incubating.jar
-commons-daemon-1.0.13.jar
-commons-cli-1.2.jar
-log4j-1.2.17.jar
-jackson-mapper-asl-1.9.13.jar
-guava-11.0.2.jar
-jetty-6.1.26.jar
-commons-logging-1.1.3.jar
-jersey-core-1.9.jar
-asm-3.2.jar
-commons-codec-1.4.jar
-xml-apis-1.3.04.jar
-xercesImpl-2.9.1.jar
-xmlenc-0.52.jar
-commons-lang-2.6.jar
-netty-all-4.0.23.Final.jar
-jetty-util-6.1.26.jar
-jsr305-3.0.0.jar
-protobuf-java-2.5.0.jar
-commons-io-2.4.jar
-servlet-api-2.5.jar
+netty-3\.6\.2\.Final([^.0-9].*)?\.jar
+leveldbjni-all-1\.8([^.0-9].*)?\.jar
+jackson-core-asl-1\.9\.13([^.0-9].*)?\.jar
+jersey-server-1\.9([^.0-9].*)?\.jar
+htrace-core-3\.1\.0-incubating([^.0-9].*)?\.jar
+commons-daemon-1\.0\.13([^.0-9].*)?\.jar
+commons-cli-1\.2([^.0-9].*)?\.jar
+log4j-1\.2\.17([^.0-9].*)?\.jar
+jackson-mapper-asl-1\.9\.13([^.0-9].*)?\.jar
+guava-11\.0\.2([^.0-9].*)?\.jar
+jetty-6\.1\.26([^.0-9].*)?\.jar
+commons-logging-1\.1\.3([^.0-9].*)?\.jar
+jersey-core-1\.9([^.0-9].*)?\.jar
+asm-3\.2([^.0-9].*)?\.jar
+commons-codec-1\.4([^.0-9].*)?\.jar
+xml-apis-1\.3\.04([^.0-9].*)?\.jar
+xercesImpl-2\.9\.1([^.0-9].*)?\.jar
+xmlenc-0\.52([^.0-9].*)?\.jar
+commons-lang-2\.6([^.0-9].*)?\.jar
+netty-all-4\.0\.23\.Final([^.0-9].*)?\.jar
+jetty-util-6\.1\.26([^.0-9].*)?\.jar
+jsr305-3\.0\.0([^.0-9].*)?\.jar
+protobuf-java-2\.5\.0([^.0-9].*)?\.jar
+commons-io-2\.4([^.0-9].*)?\.jar
+servlet-api-2\.5([^.0-9].*)?\.jar
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list
index 7677cd16..ec23b7c2 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list
@@ -1,22 +1,22 @@
-netty-3.6.2.Final.jar
-leveldbjni-all-1.8.jar
-paranamer-2.3.jar
-jackson-core-asl-1.9.13.jar
-jersey-server-1.9.jar
-guice-3.0.jar
-avro-1.7.4.jar
-log4j-1.2.17.jar
-jackson-mapper-asl-1.9.13.jar
-snappy-java-1.0.4.1.jar
-jersey-core-1.9.jar
-jersey-guice-1.9.jar
-commons-compress-1.4.1.jar
-junit-4.11.jar
-xz-1.0.jar
-asm-3.2.jar
-aopalliance-1.0.jar
-javax.inject-1.jar
-protobuf-java-2.5.0.jar
-commons-io-2.4.jar
-hamcrest-core-1.3.jar
-guice-servlet-3.0.jar
+netty-3\.6\.2\.Final([^.0-9].*)?\.jar
+leveldbjni-all-1\.8([^.0-9].*)?\.jar
+paranamer-2\.3([^.0-9].*)?\.jar
+jackson-core-asl-1\.9\.13([^.0-9].*)?\.jar
+jersey-server-1\.9([^.0-9].*)?\.jar
+guice-3\.0([^.0-9].*)?\.jar
+avro-1\.7\.[4-7]([^.0-9].*)?\.jar
+log4j-1\.2\.17([^.0-9].*)?\.jar
+jackson-mapper-asl-1\.9\.13([^.0-9].*)?\.jar
+snappy-java-1\.0\.[45](\.[0-9])?([^.0-9].*)?\.jar
+jersey-core-1\.9([^.0-9].*)?\.jar
+jersey-guice-1\.9([^.0-9].*)?\.jar
+commons-compress-1\.4\.1([^.0-9].*)?\.jar
+junit-4\.11([^.0-9].*)?\.jar
+xz-1\.0([^.0-9].*)?\.jar
+asm-3\.2([^.0-9].*)?\.jar
+aopalliance-1\.0([^.0-9].*)?\.jar
+javax\.inject-1([^.0-9].*)?\.jar
+protobuf-java-2\.5\.0([^.0-9].*)?\.jar
+commons-io-2\.4([^.0-9].*)?\.jar
+hamcrest-core-1\.3([^.0-9].*)?\.jar
+guice-servlet-3\.0([^.0-9].*)?\.jar
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-subprojs.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-subprojs.list
index b4d3c2f6..b0a5654d 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-subprojs.list
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-subprojs.list
@@ -1,4 +1,4 @@
-hadoop-annotations.jar
-hadoop-auth.jar
-hadoop-common.jar
-hadoop-nfs.jar
+hadoop-annotations\.jar
+hadoop-auth\.jar
+hadoop-common\.jar
+hadoop-nfs\.jar
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list
index 68b7120a..983cec9e 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list
@@ -1,38 +1,38 @@
-netty-3.6.2.Final.jar
-leveldbjni-all-1.8.jar
-jackson-core-asl-1.9.13.jar
-jackson-xc-1.9.13.jar
-jersey-server-1.9.jar
-stax-api-1.0-2.jar
-zookeeper-3.4.6.jar
-guice-3.0.jar
-jaxb-impl-2.2.3-1.jar
-zookeeper-3.4.6-tests.jar
-jersey-client-1.9.jar
-commons-cli-1.2.jar
-log4j-1.2.17.jar
-jackson-mapper-asl-1.9.13.jar
-guava-11.0.2.jar
-jetty-6.1.26.jar
-commons-logging-1.1.3.jar
-jersey-core-1.9.jar
-jersey-guice-1.9.jar
-commons-compress-1.4.1.jar
-jettison-1.1.jar
-commons-collections-3.2.1.jar
-xz-1.0.jar
-asm-3.2.jar
-commons-codec-1.4.jar
-aopalliance-1.0.jar
-javax.inject-1.jar
-commons-lang-2.6.jar
-jetty-util-6.1.26.jar
-jsr305-3.0.0.jar
-protobuf-java-2.5.0.jar
-commons-io-2.4.jar
-activation-1.1.jar
-jersey-json-1.9.jar
-jaxb-api-2.2.2.jar
-guice-servlet-3.0.jar
-servlet-api-2.5.jar
-jackson-jaxrs-1.9.13.jar
+netty-3\.6\.2\.Final([^.0-9].*)?\.jar
+leveldbjni-all-1\.8([^.0-9].*)?\.jar
+jackson-core-asl-1\.9\.13([^.0-9].*)?\.jar
+jackson-xc-1\.9\.13([^.0-9].*)?\.jar
+jersey-server-1\.9([^.0-9].*)?\.jar
+stax-api-1\.0-2([^.0-9].*)?\.jar
+zookeeper-3\.4\.6([^.0-9].*)?\.jar
+guice-3\.0([^.0-9].*)?\.jar
+jaxb-impl-2\.2\.3-1([^.0-9].*)?\.jar
+zookeeper-3\.4\.6([^.0-9].*)?-tests\.jar
+jersey-client-1\.9([^.0-9].*)?\.jar
+commons-cli-1\.2([^.0-9].*)?\.jar
+log4j-1\.2\.17([^.0-9].*)?\.jar
+jackson-mapper-asl-1\.9\.13([^.0-9].*)?\.jar
+guava-11\.0\.2([^.0-9].*)?\.jar
+jetty-6\.1\.26([^.0-9].*)?\.jar
+commons-logging-1\.1\.3([^.0-9].*)?\.jar
+jersey-core-1\.9([^.0-9].*)?\.jar
+jersey-guice-1\.9([^.0-9].*)?\.jar
+commons-compress-1\.4\.1([^.0-9].*)?\.jar
+jettison-1\.1([^.0-9].*)?\.jar
+commons-collections-3\.2\.[12]([^.0-9].*)?\.jar
+xz-1\.0([^.0-9].*)?\.jar
+asm-3\.2([^.0-9].*)?\.jar
+commons-codec-1\.4([^.0-9].*)?\.jar
+aopalliance-1\.0([^.0-9].*)?\.jar
+javax\.inject-1([^.0-9].*)?\.jar
+commons-lang-2\.6([^.0-9].*)?\.jar
+jetty-util-6\.1\.26([^.0-9].*)?\.jar
+jsr305-3\.0\.0([^.0-9].*)?\.jar
+protobuf-java-2\.5\.0([^.0-9].*)?\.jar
+commons-io-2\.4([^.0-9].*)?\.jar
+activation-1\.1([^.0-9].*)?\.jar
+jersey-json-1\.9([^.0-9].*)?\.jar
+jaxb-api-2\.2\.2([^.0-9].*)?\.jar
+guice-servlet-3\.0([^.0-9].*)?\.jar
+servlet-api-2\.5([^.0-9].*)?\.jar
+jackson-jaxrs-1\.9\.13([^.0-9].*)?\.jar