abe.sh: Add support for --check <directive> and --excludecheck <directive>

The full syntax is: --check [{all|gdb|glibc|binutils|gcc}]
               and: --excludecheck {all|gdb|glibc|binutils|gcc}

This patch allows the selective inclusion and exclusion of running
unit-testing for the user designated packages.  The existing option
(--check) without a directive still initiates make check on all
toolchain component packages and is the equivalent of --check all.

Change-Id: Ia5e5aa65031bbd578dd3ce3fef20997fc05f16ef

7 files changed, 698 insertions, 58 deletions

diff --git a/abe.sh b/abe.sh
index 001d6112..2c962fde 100755
--- a/abe.sh
+++ b/abe.sh
@@ -23,8 +23,10 @@ usage()
   ${abe} [''| [--timeout <value>]
              [[--build [<package> --stage {1|2}]|all]
               |[--checkout <package>|all]]
-             [--ccache] [--check] [--enable {bootstrap|gerrit}]
+             [--check [{all|glibc|gcc|gdb|binutils}]]
+             [--ccache] [--enable {bootstrap|gerrit}]
              [--disable {install|update|make_docs|building}] [--dryrun] [--dump]
+             [--excludecheck {all|glibc|gcc|gdb|binutils}]
              [--fetch <url>] [--force] [--host <host_triple>] [--help]
              [--list] [--march <march>] [--manifest <manifest_file>]
              [--parallel] [--release <release_version_string>]
@@ -56,13 +58,14 @@ EOF
     cat << EOF
 KEY
 
-  [--foo]	  Optional switch
-  [<foo>]	  Optional user specified field 
-  <foo>		  Non-optional user specified field.
-  {foo|bar|bat}   Non-optional choice field.
-  [{foo|bar|bat}] Optional choice field.
-  [foo]		  Optional field 
-  ['']		  Optional Empty field 
+  [--foo]         Optional switch
+  [<foo>]         Optional user specified field
+  <foo>           Non-optional user specified field
+  {foo|bar|bat}   Non-optional choice field
+  [{foo|bar|bat}] Optional choice field
+  [foo]           Optional field
+  ['']            Optional Empty field
+  <>              Indicates when no directive is specified
 
 DESCRIPTION
 
@@ -100,9 +103,18 @@ OPTIONS
 
   --ccache	Use ccache when building packages.
 
-  --check
-		Run make check on packages.  For cross builds this will run
-		the tests on native hardware.
+  --check [{all|glibc|gcc|gdb|binutils}]
+
+                For cross builds this will run package unit-tests on native
+                hardware
+
+                glibc|gcc|gdb|binutils
+                        Run make check on the specified package only.
+                all
+                        Run make check on all supported packages.
+                <>
+                        If there is no directive it's the same as 'all' and
+                        make check will be run on all supported packages.
 
   --checkout <package>[~branch][@revision]|all
 
@@ -149,6 +161,35 @@ OPTIONS
                         Enable posting comments to Gerrit on the build
                         progress.
 
+  --excludecheck {all|glibc|gcc|gdb|binutils}
+
+                {glibc|gcc|gdb|binutils}
+                        When used with --check this will remove the
+                        specified package from having its unit-tests
+                        executed during make check.  When used without
+                        --check this will do nothing.
+
+                all
+                        When 'all' is specified no unit tests will be run
+                        regardless of what was specified with --check.
+
+                <>
+                        --excludecheck requires an input directive.
+                        Calling --excludecheck without a directive is an
+                        error that will cause ${abe} to abort.
+
+                Note: This may be called several times and all valid
+                packages will be removed from the list of packages to have
+                unit-test executed against, e.g., the following will only
+                leave glibc and gcc to have unit-tests executed:
+
+                --check all --excludecheck gdb --excludecheck binutils
+
+                Note: All --excludecheck packages are processed after all
+                --check packages, e.g., the following will NOT check gdb:
+
+                --check gdb --excludecheck gdb --check gdb
+
   --fetch <url>
 
   		Fetch the specified URL into the snapshots directory.
@@ -368,8 +409,8 @@ command_line_arguments=$*
 # designated target.
 crosscheck_clibrary_target()
 {
-    local test_clibrary=$1
-    local test_target=$2
+    local test_clibrary="$1"
+    local test_target="$2"
     case ${test_target} in
 	arm*-eabi|aarch64*-*elf|*-mingw32)
 	    # Bare metal targets only support newlib.
@@ -385,6 +426,29 @@ crosscheck_clibrary_target()
     return 0
 }
 
+
+# Returns '0' if $package ($1) is in the list of all_unit_tests.  Returns '1'
+# if not found.
+crosscheck_unit_test()
+{
+    local package="$1"
+
+    # 'all' is an acceptable equivalent to the full string of packages.
+    if test x"${package}" = x"all"; then
+	return 0
+    fi
+
+    # We have to search for exact matches.  We don't want to match on 'gd' or
+    # 'g', but rather 'gdb' and 'gcc' or the results will be unpredictable.
+    for i in ${all_unit_tests}; do
+        if test x"$i" = x"${package}"; then
+            return 0
+	fi
+    done
+
+    return 1
+}
+
 set_package()
 {
     local package="`echo $1 | cut -d '=' -f 1`"
@@ -490,10 +554,10 @@ test_success()
 # parse the -- of the following switch.
 check_directive()
 {
-    switch="$1"
-    long="$2"
-    short="$3"
-    directive="$4"
+    local switch="$1"
+    local long="$2"
+    local short="$3"
+    local directive="$4"
 
     if test `echo ${switch} | grep -c "\-${short}.*=" ` -gt 0; then
 	error "A '=' is invalid after --${long}.  A space is expected between the switch and the directive."
@@ -507,6 +571,43 @@ check_directive()
     build_failure
 }
 
+# Some switches allow an optional following directive. We need to make sure
+# they don't parse the -- of the following switch.  If there isnt a following
+# directive this function will echo the default ($5).  This function can't
+# distinguish whether --foo--bar is valid, so it will return 1 in this case
+# and consume the --bar as part of --foo.
+#
+# Return Value(s):
+#	stdout - caller provided directive or default
+#	0 - if $directive is provided by caller
+#	1 - if $directive is not provided by caller
+#	exit - Execution will abort if the input is invalid.
+check_optional_directive()
+{
+    local switch="$1"
+    local long="$2"
+    local short="$3"
+    local directive="$4"
+    local default="$5"
+
+    if test `echo ${switch} | grep -c "\-${short}.*=" ` -gt 0; then
+	error "A '=' is invalid after --${long}.  A space is expected between the switch and the directive."
+	build_failure
+    elif test x"$directive" = x; then
+	notice "There is no directive accompanying this switch.  Using --$long $default."
+	directive="$default"
+	echo "$directive"
+	return 1
+    elif test `echo ${directive} | egrep -c "^\-+"` -gt 0; then
+	notice "There is no directive accompanying this switch.  Using --$long $default."
+	directive="$default"
+	echo "$directive"
+	return 1
+    fi
+    echo "$directive"
+    return 0
+}
+
 # This gets a list from a remote server of the available tarballs. We use HTTP
 # instead of SSH so it's more accessible to those behind nasty firewalls.
 # base - already checkout out source trees
@@ -594,6 +695,22 @@ dump()
     if test x"${release}" != x; then
         echo "Release Name       ${release}"
     fi
+
+    if test x"${do_makecheck}" = x"all"; then
+        echo "check              ${do_makecheck} {$all_unit_tests}"
+    elif test ! -z "${do_makecheck}"; then
+        echo "check              ${do_makecheck}"
+    fi
+
+    if test x"${do_excludecheck}" != x; then
+        echo "excludecheck       ${do_excludecheck}"
+    fi
+
+    if test x"${runtests}" != x; then
+        echo "checking           ${runtests}"
+    else
+        echo "checking           {none}"
+    fi
 }
 
 export PATH="${local_builds}/destdir/${build}/bin:$PATH"
@@ -602,6 +719,8 @@ export PATH="${local_builds}/destdir/${build}/bin:$PATH"
 # other switches.
 do_dump=
 do_checkout=
+do_makecheck=
+do_excludecheck=
 do_build=
 do_build_stage=stage2
 
@@ -620,9 +739,6 @@ while test $# -gt 0; do
 	    # Shift off the 'all' or the package identifier.
 	    shift
 	    ;;
-	--check|-check)
-	    runtests=yes
-	    ;;
 	--checkout*|-checkout*)
 	    check_directive $1 checkout "checkout" $2
 	    # Save and process this after all other elements have been processed.
@@ -631,6 +747,50 @@ while test $# -gt 0; do
 	    # Shift off the 'all' or the package identifier.
 	    shift
 	    ;;
+	# This is after --checkout because we want to catch every other usage
+	# of check* but NOT 'checkout'.
+	--check*|-check*)
+	    tmp_do_makecheck=
+	    tmp_do_makecheck="`check_optional_directive $1 check "check" "$2" "all"`"
+	    ret=$?
+
+	    # do_makecheck already contains the directive or 'all'.  This
+	    # test determines whether we need to strip off an additional
+	    # parameter from the command line argument if directive was
+	    # provided.
+	    if test $ret -eq 0; then
+	      shift;
+            fi
+
+	    crosscheck_unit_test ${tmp_do_makecheck}
+	    ret=$?
+	    if test $ret -eq 1; then
+		error "${tmp_do_makecheck} is an invalid package name to pass to --check. The choices are {all $all_unit_tests}."
+		build_failure
+	    fi
+
+	    # Accumulate --check packages from consecutive --check calls.  Yes
+	    # there might be potential duplicates but we'll prune those later.
+	    # parse later.
+	    do_makecheck=${do_makecheck:+${do_makecheck} }${tmp_do_makecheck}
+	    ;;
+	# This will exclude an individual package from the list of packages
+	# to run make check (unit-test) against.
+        --excludecheck*|-excludecheck*)
+	    check_directive $1 excludecheck "excludecheck" $2
+
+	    # Verify that $2 is a valid option to exclude.
+	    crosscheck_unit_test $2
+	    if test $? -eq 1; then
+		error "${2} is an invalid package name to pass to --excludecheck. The choices are {all $all_unit_tests}."
+		build_failure
+	    fi
+
+	    # Concatenate this onto the list of packages to exclude from make check.
+            do_excludecheck="${do_excludecheck:+${do_excludecheck} }$2"
+
+	    shift
+	    ;;
 	--march*|-march*)
 	    check_directive $1 march "march" $2
 	    default_march=$2
@@ -955,6 +1115,49 @@ wget_timeout=10
 fetch md5sums
 wget_timeout=${timeout_save}
 
+if test ! -z "${do_makecheck}"; then
+    # If we encounter 'all' in ${do_makecheck} anywhere we just overwrite
+    # runtests with ${all_unit_tests} and ignore the rest.
+    test_all="${do_makecheck//all/}"
+
+    if test x"${test_all}" != x"${do_makecheck}"; then
+	runtests="${all_unit_tests}"
+    else
+	# Don't accumulate any duplicates.
+        for i in ${do_makecheck}; do
+	    # Remove it if it's already there
+	    runtests=${runtests//${i}/}
+	    # Remove any redundant whitespace
+	    runtests=${runtests//  /}
+	    # Reinsert it if it was already in the list.
+            runtests="${runtests:+${runtests} }${i}"
+        done
+    fi
+fi
+
+if test ! -z "${do_excludecheck}"; then
+
+    # If we encounter 'all' in ${do_excludecheck} anywhere we just
+    # empty out runtests because 'all' trumps everything.
+    exclude_all="${do_excludecheck//all/}"
+    if test x"${exclude_all}" != x"${do_excludecheck}"; then
+        runtests=
+    else
+	#Remove excluded packages (stored in do_excludecheck) from ${runtests}
+	for i in ${do_excludecheck}; do
+	    runtests="${runtests//$i/}"
+	    # Strip redundant white spaces
+	    runtests="${runtests//  / }"
+	done
+	# Strip white space from the beginning of the string
+	runtests=${runtests# }
+	# Strip white space from the end of the string
+	runtests=${runtests% }
+    fi
+fi
+
+# Process 'dump' after we process 'check' and 'excludecheck' so that the list
+# of tests to be evaluated is resolved before the dump.
 if test ! -z ${do_dump}; then
     dump
 fi
diff --git a/lib/common.sh b/lib/common.sh
index 996e1e89..ce4cde24 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -769,3 +769,23 @@ create_release_tag()
 #    echo ${revision}
 #    return 0
 #}
+
+# Search $runtests to see if $package should be unit tested.
+# List of tests to run
+# $1: ${runtests}
+# $2: ${package} to check
+# Return Value(s)
+# 0 = package found in runtests
+# 1 = package not found in runtests
+is_package_in_runtests()
+{
+    local unit_test="$1"
+    local package="$2"
+
+    for i in ${unit_test}; do
+        if test x"$i" = x"$package"; then
+            return 0
+	fi
+    done
+    return 1
+}
diff --git a/lib/globals.sh b/lib/globals.sh
index dbf8ae2a..71c1bb6b 100644
--- a/lib/globals.sh
+++ b/lib/globals.sh
@@ -84,7 +84,17 @@ interactive=no
 nodepends=no
 verbose=1
 network=""
-runtests=no
+
+# Don't modify this in this file unless you're adding to it.  This is the list
+# of packages that have make check run against them.  It will be queried for
+# content when the users passes --check <package> or --excludecheck <package>.
+all_unit_tests="glibc gcc gdb binutils"
+
+# Packages to run make check (unit-test) on.  This variable is composed from
+# all --check <package> and --excludecheck <package> switches.  Don't modify
+# this parameter manually.
+runtests=
+
 ccache=no
 #gerrit=no
 
diff --git a/lib/make.sh b/lib/make.sh
index 895388be..95ba9be4 100755
--- a/lib/make.sh
+++ b/lib/make.sh
@@ -146,7 +146,72 @@ build_all()
     # Notify that the build completed successfully
     build_success
 
-    if test x"${gerrit}" = xyes -a x"${runtests}" = xyes; then
+    # If we're building a full toolchain the binutils tests need to be built
+    # with the stage 2 compiler, and therefore we shouldn't run unit-test
+    # until the full toolchain is built.  Therefore we test all toolchain
+    # packages after the full toolchain is built.  If ${runtests} is empty
+    # the user has requested that no tests run.  Binary tarballs have
+    # testing executed on the installed libraries and executables, not on
+    # the source tree.
+    if test x"${runtests}" != x -a x"${tarbin}" != x"yes"; then
+	notice "Testing components ${runtests}..."
+	buildingall=no
+	local check_ret=
+	local check_failed=
+
+	is_package_in_runtests "${runtests}" binutils
+	if test $? -eq 0; then
+	    make_check ${binutils_version}
+	    check_ret=$?
+	    check_failed="${check_failed} binutils"
+	fi
+
+	is_package_in_runtests "${runtests}" gcc
+	if test $? -eq 0; then
+	    make_check ${gcc_version} stage2
+	    check_ret=$?
+	    check_failed="${check_failed} gcc-stage2"
+	fi
+
+	is_package_in_runtests "${runtests}" gdb
+	if test $? -eq 0; then
+	    make_check ${gdb_version}
+	    check_ret=$?
+	    check_failed="${check_failed} gdb"
+	fi
+
+	# Only perform unit tests on [e]glibc when we're building native.
+        if test x"${target}" = x"${build}"; then
+	    # TODO: Get glibc make check working 'native'
+	    is_package_in_runtests "${runtests}" glibc
+	    if test $? -eq 0; then
+		#make_check ${glibc_version}
+	        #check_ret=$?
+	        #check_failed="${check_failed} glibc"
+		notice "make check on native glibc is not yet implemented."
+	    fi
+
+	    is_package_in_runtests "${runtests}" eglibc
+	    if test $? -eq 0; then
+		#make_check ${eglibc_version}
+	        #check_ret=$?
+	        #check_failed="${check_failed} eglibc"
+		notice "make check on native eglibc is not yet implemented."
+	    fi
+	fi
+
+	if test ${check_ret} -ne 0; then
+	    error "Failed checking of ${check_failed}."
+	    return 1
+	fi
+    fi
+
+    # Notify that the test run completed successfully
+    test_success
+
+    # If any unit-tests have been run, then we should send a message to gerrit.
+    # TODO: Authentication from abe to jenkins does not yet work.
+    if test x"${gerrit}" = xyes -a x"${runtests}" != x; then
 	local sumsfile="/tmp/sums$$.txt"
 	local sums="`find ${local_builds}/${host}/${target} -name \*.sum`"
 	for i in ${sums}; do
@@ -165,28 +230,6 @@ build_all()
 	fi
     fi
     rm -f ${sumsfile}
-    
-    if test x"${runtests}" = xyes -a x"${tarbin}" != x"yes"; then
-	notice "Testing components"
-	buildingall=no
-	make_check ${binutils_version}
-	check_ret=$?
-	make_check ${gcc_version} stage2
-	if test $? -ne 0; then
-	    check_ret=1
-	fi
-	#make_check ${gdb_version}
-	#if test $? -ne 0; then
-	#    check_ret=1
-	#fi
-	if test ${check_ret} -ne 0; then
-	    error "Failed checking."
-	    return 1
-	fi
-    fi
-
-    # Notify that the test run completed successfully
-    test_success
 
     if test x"${tarsrc}" = x"yes"; then
         if test "`echo ${with_packages} | grep -c toolchain`" -gt 0; then
@@ -220,7 +263,9 @@ build_all()
 #            binary_gdb
 #        fi
         notice "Packaging took ${SECONDS} seconds"
-        if test x"${runtests}" = xyes; then
+	# If there aren't any tests specified to run then don't bother calling
+	# test_binary_toolchain.
+        if test x"${runtests}" != x; then
 	    test_binary_toolchain
             notice "Testing packaging took ${SECONDS} seconds"
 	fi
@@ -251,6 +296,11 @@ build()
 
     local srcdir="`get_srcdir ${gitinfo} ${2:+$2}`"
 
+    # We have to use get_toolname because binutils-gdb use the same
+    # repository and get_toolname needs to parse the branchname to
+    # determine the tool.
+    local tool="`get_toolname ${srcdir}`"
+
     local stamp=
     stamp="`get_stamp_name build ${gitinfo} ${2:+$2}`"
 
@@ -363,15 +413,28 @@ build()
 	# compiler, so any tests that get executed on the target can be fully linked.
     fi
 
-    if test x"${runtests}" = xyes -a x"${tool}" != x"eglibc" -a x"${tarbin}" != xyes; then
-        if test x"$2" != x"stage1" -a x"$2" != x"gdbserver"; then
-	    if test x"${buildingall}" = xno; then
-		notice "Starting test run for ${tag}${2:+ $2}"
-		make_check ${gitinfo}${2:+ $2}
-		if test $? -gt 0; then
-                    return 1
-		fi
+    # Only execute make_check in build() if build_all() isn't being invoked for
+    # this run of abe.sh.  This is because build_all() will invoke make_check()
+    # in sequence after all builds are executed if it's been directed to run
+    # unit-tests. If --tarbin was specified we're never going to run make check
+    # because it takes too long and testing should have been run with an
+    # earlier invocation of abe.
+    # TODO: eliminate buildingall as a global and make it a local check passed
+    # via a parameter to build().
+    if test x"${buildingall}" = xno -a x"${tarbin}" != xyes; then
+
+	# Skip make_check if it isn't designated to be executed in ${runtests}
+	is_package_in_runtests "${runtests}" ${tool}
+	if test $? -eq 0 -a x"$2" != x"stage1" -a x"$2" != x"gdbserver"; then
+	    # We don't run make check on gcc stage1 or on gdbserver because
+	    # it's unnecessary.
+	    notice "Starting test run for ${tag}${2:+ $2}"
+	    make_check ${gitinfo}${2:+ $2}
+	    if test $? -gt 0; then
+	        return 1
 	    fi
+	else
+	    notice "make check skipped for ${tag}${2:+ $2}"
 	fi
     fi
 
@@ -915,6 +978,7 @@ make_target_sysroot()
     echo $sysroot
 }
 
+# TODO: Should copy_gcc_libs_to_sysroot() use the input parameter in $1?
 # $1 - compiler (and any compiler flags) to query multilib information
 copy_gcc_libs_to_sysroot()
 {
diff --git a/lib/package.sh b/lib/package.sh
index 22c6f46d..e765c708 100644
--- a/lib/package.sh
+++ b/lib/package.sh
@@ -531,6 +531,29 @@ test_binary_toolchain()
 {
     local install="/tmp/install.$$"
 
+    local testgcc=
+    local testbin=
+    is_package_in_runtests "${runtests}" gcc
+    testgcc=$?
+
+    is_package_in_runtests "${runtests}" binutils
+    testbin=$?
+
+    # Check early and bail if binutils or gcc aren't to be tested.  In reality
+    # binary toolchains will probably be built with --check, or --check all
+    # (which is the same thing).
+    if test $testgcc -ne 0 -a $testbin -ne 0; then
+	notice "Nothing to test in ${runtests} for test_binary_toolchain()."
+	return 0
+    fi
+
+    # ${runtests} might contain something other than just gcc and binutils but
+    # we only test those for binary toolchains.
+    local testing="`if test ${testgcc} -eq 0; then echo -n ' gcc'; fi;`"
+    testing="${testing}`if test ${testbin} -eq 0; then echo -n ' binutils'; fi`"
+
+    notice "Testing the following binary toolchain packages:${testing}"
+
     if test ! -d ${install}; then
 	dryrun "mkdir -p ${install}"
     fi
@@ -545,12 +568,18 @@ test_binary_toolchain()
     local compiler="`dirname ${compiler}`"
     export PATH="${compiler}:$PATH"
 
-    # test GCC using the build we just completed, since we need access to the test cases.
-    make_clean ${binutils_version}
-    make_check ${binutils_version}
+    # Only test binutils if the user has requested it.
+    if test $testbin -eq 0; then
+	# test GCC using the build we just completed, since we need access to the test cases.
+	make_clean ${binutils_version}
+	make_check ${binutils_version}
+    fi
 
-    make_clean ${gcc_version} stage2
-    make_check ${gcc_version} stage2
+    # Only test gcc if the user has requested it.
+    if test $testgcc -eq 0; then
+	make_clean ${gcc_version} stage2
+	make_check ${gcc_version} stage2
+    fi
 
     rm -fr ${install}
 }
diff --git a/test.sh b/test.sh
index 1855b0bf..0d7309fb 100755
--- a/test.sh
+++ b/test.sh
@@ -626,6 +626,237 @@ cb_commands="--dry-run --target arm-none-linux-gnueabihf --march=armv8-a --dump"
 match="A space is expected"
 test_failure "${cb_commands}" "${match}"
 
+cb_commands="--dry-run --target arm-none-linux-gnueabihf --check=foo"
+match='is invalid after'
+test_failure "${cb_commands}" "${match}"
+
+cb_commands="--dry-run --target arm-none-linux-gnueabihf --dump --check"
+match='check              all'
+test_pass "${cb_commands}" "${match}"
+
+cb_commands="--dry-run --target arm-none-linux-gnueabihf --dump --check --dump"
+match='check              all'
+test_pass "${cb_commands}" "${match}"
+
+# Yes this won't work because we match on 'exact' package name only.
+cb_commands="--dry-run --target arm-none-linux-gnueabihf --dump --check gdb--dump"
+match='dump is an invalid package'
+test_failure "${cb_commands}" "${match}"
+
+cb_commands="--dry-run --target arm-none-linux-gnueabihf --check --dump"
+match='check              all'
+test_pass "${cb_commands}" "${match}"
+
+cb_commands="--dry-run --target arm-none-linux-gnueabihf --check gdb --dump"
+match='check              gdb'
+test_pass "${cb_commands}" "${match}"
+
+cb_commands="--dry-run --target arm-none-linux-gnueabihf --check all --dump"
+match='check              all'
+test_pass "${cb_commands}" "${match}"
+
+# Verify that --check without a directive doesn't strip the next switch from
+# the command line.
+cb_commands="--dry-run --check --target arm-none-linux-gnueabihf --dump"
+match='         arm-none-linux-gnueabihf'
+test_pass "${cb_commands}" "${match}"
+
+# test various combinations of --check and --excludecheck
+
+# This should explicitly add all tests to runtests but NOT include 'all' in the text
+cb_commands="--check all --dump"
+match='checking           glibc gcc gdb binutils'
+test_pass "${cb_commands}" "${match}"
+
+# Simply exclude 'gdb' from the list of all runtests.
+cb_commands="--check all --excludecheck gdb --dump"
+match='checking           glibc gcc binutils'
+test_pass "${cb_commands}" "${match}"
+
+# This should be the same as --check all --excludecheck gdb
+cb_commands="--check  --excludecheck gdb --dump"
+match='checking           glibc gcc binutils'
+test_pass "${cb_commands}" "${match}"
+
+# 'binutils' is on the end of the list which might have some whitespace issues.
+cb_commands="--check all --excludecheck binutils --dump"
+match='checking           glibc gcc gdb'
+test_pass "${cb_commands}" "${match}"
+
+# 'glibc' is at the beginning of the list which might have some whitespace issues.
+cb_commands="--check all --excludecheck glibc --dump"
+match='checking           gcc gdb binutils'
+test_pass "${cb_commands}" "${match}"
+
+# Make sure both are accounted for.
+cb_commands="--check all --excludecheck glibc --excludecheck binutils --dump"
+match='checking           gcc gdb'
+test_pass "${cb_commands}" "${match}"
+
+# Check a single test
+cb_commands="--check gdb --dump"
+match='checking           gdb'
+test_pass "${cb_commands}" "${match}"
+
+# Check binutils
+cb_commands="--check binutils --dump"
+match='checking           binutils'
+test_pass "${cb_commands}" "${match}"
+
+# Check glibc
+cb_commands="--check glibc --dump"
+match='checking           glibc'
+test_pass "${cb_commands}" "${match}"
+
+# Check gcc
+cb_commands="--check gcc --dump"
+match='checking           gcc'
+test_pass "${cb_commands}" "${match}"
+
+# Check that --dump is processed after --check.
+cb_commands="--dump --check gcc"
+match='checking           gcc'
+test_pass "${cb_commands}" "${match}"
+
+# What happens when you add several tests?
+cb_commands="--check gdb --check gcc --dump"
+match='checking           gdb gcc'
+test_pass "${cb_commands}" "${match}"
+
+# This should result in 'gdb gcc' in runtests because the order depends on when they were added with --check.
+cb_commands="--check gdb --check gcc --dump"
+match='checking           gdb gcc'
+test_pass "${cb_commands}" "${match}"
+
+# what if you mix 'all' and individual tests?  It should be all tests in all_unit_tests and no redundant tests.
+cb_commands="--check all --check gdb --check glibc --dump"
+match='checking           glibc gcc gdb binutils'
+test_pass "${cb_commands}" "${match}"
+
+# Make sure we get the same result with --check (without a directive) since this is the same as 'all'.
+# It should be all tests in all_unit_tests and no redundant tests.
+cb_commands="--check --check gdb --check glibc --dump"
+match='checking           glibc gcc gdb binutils'
+test_pass "${cb_commands}" "${match}"
+
+# Make sure we can exclude binutils when 'all' is mixed with individual tests.
+cb_commands="--check all --check gdb --check glibc --excludecheck binutils --dump"
+match='checking           glibc gcc gdb'
+test_pass "${cb_commands}" "${match}"
+
+# Make sure we can exclude several packages when 'all' is mixed with individual tests.
+cb_commands="--check all --check gdb --check glibc --excludecheck binutils --excludecheck gdb --dump"
+match='checking           glibc gcc'
+test_pass "${cb_commands}" "${match}"
+
+# Order of where --check all shows up shouldn't affect outcome.
+# Make sure we can exclude several packages when 'all' is mixed with individual tests.
+cb_commands="--check gdb --check glibc --excludecheck binutils --excludecheck gdb --check all --dump"
+match='checking           glibc gcc'
+test_pass "${cb_commands}" "${match}"
+
+# Order of where --check shows up shouldn't affect outcome.
+# Make sure we can exclude several packages when 'all' is mixed with individual tests.
+cb_commands="--check gdb --check glibc --excludecheck binutils --excludecheck gdb --check --dump"
+match='checking           glibc gcc'
+test_pass "${cb_commands}" "${match}"
+
+# Make sure we can exclude several packages when 'all' is implicitly mixed with individual tests.
+cb_commands="--check --check gdb --check glibc --excludecheck binutils --excludecheck gdb --dump"
+match='checking           glibc gcc'
+test_pass "${cb_commands}" "${match}"
+
+# Order of --check and --excludecheck doesn't matter.  We always 'exclude' after we process 'check'.
+# If we add --check gdb after we've already excluded it, it'll remain excluded.
+cb_commands="--check --check gdb --check glibc --excludecheck binutils --excludecheck gdb --check gdb --dump"
+match='checking           glibc gcc'
+test_pass "${cb_commands}" "${match}"
+
+# Removing everything that was added should result in no unit tests being run.
+cb_commands="--check gdb --check glibc --excludecheck gdb --excludecheck glibc --dump"
+match='checking           {none}'
+test_pass "${cb_commands}" "${match}"
+
+# Redundant check tests should have all instances overridden by excludecheck.
+cb_commands="--check gdb --check gdb --check glibc --excludecheck gdb --excludecheck glibc --dump"
+match='checking           {none}'
+test_pass "${cb_commands}" "${match}"
+
+# Redundant excludecheck tests shouldn't do anything unexpected.
+cb_commands="--check gdb --check glibc --excludecheck glibc --excludecheck gdb --excludecheck glibc --dump"
+match='checking           {none}'
+test_pass "${cb_commands}" "${match}"
+
+# Redundant excludecheck tests shouldn't accidentally remove an included test.
+cb_commands="--check gdb --check glibc --excludecheck glibc --excludecheck glibc --dump"
+match='checking           gdb'
+test_pass "${cb_commands}" "${match}"
+
+# Redundant check tests should only result in one instance of the test
+cb_commands="--check gdb --check gdb --check glibc --dump"
+match='checking           gdb glibc'
+test_pass "${cb_commands}" "${match}"
+
+# There should be nothing in runtests because nothing was specified with --check
+cb_commands="--excludecheck glibc --dump"
+match='checking           {none}'
+test_pass "${cb_commands}" "${match}"
+
+# This should error out because 'excludecheck' requires a directive
+cb_commands="--check gdb --check gdb --check glibc --excludecheck --dump"
+match='excludecheck requires a directive'
+test_pass "${cb_commands}" "${match}"
+
+# excluding a test that isn't being checked should work fine.
+cb_commands="--check gdb --check gdb --check glibc --excludecheck gcc --dump"
+match='checking           gdb glibc'
+test_pass "${cb_commands}" "${match}"
+
+# excluding this combination shouldn't leave extraneous spaces in runtests.
+cb_commands="--check --excludecheck gcc --excludecheck gdb --dump"
+match='checking           glibc binutils'
+test_pass "${cb_commands}" "${match}"
+
+
+# excluding all tests should work
+cb_commands="--check all --excludecheck all --dump"
+match='checking           {none}'
+test_pass "${cb_commands}" "${match}"
+
+# excluding all tests should work regardless of what other tests are included or excluded.
+cb_commands="--check all --excludecheck all --check gdb --excludecheck gcc --dump"
+match='checking           {none}'
+test_pass "${cb_commands}" "${match}"
+
+# excluding all tests should work even if no other tests have been included.
+cb_commands="--excludecheck all --dump"
+match='checking           {none}'
+test_pass "${cb_commands}" "${match}"
+
+# excluding all tests should work even if only one test has been included.
+cb_commands="--check glibc --excludecheck all --dump"
+match='checking           {none}'
+test_pass "${cb_commands}" "${match}"
+
+# checking a partial package name shoulderror
+cb_commands="--check gd --dump"
+match='gd is an invalid package name'
+test_failure "${cb_commands}" "${match}"
+
+# checking an invalid package should error.
+cb_commands="--check foo --dump"
+match='foo is an invalid package name'
+test_failure "${cb_commands}" "${match}"
+
+# excluding a partial package name should error
+cb_commands="--check --excludecheck gd --dump"
+match='gd is an invalid package name'
+test_failure "${cb_commands}" "${match}"
+
+# excluding an invalid package name should error
+cb_commands="--check --excludecheck foo --dump"
+match='foo is an invalid package name'
+test_failure "${cb_commands}" "${match}"
 
 # If the tests pass successfully clean up /tmp/<tmpdir> but only if the
 # directory name is conformant.  We don't want to accidentally remove /tmp.
diff --git a/testsuite/test.sh b/testsuite/test.sh
index 966323d7..a4f5e610 100755
--- a/testsuite/test.sh
+++ b/testsuite/test.sh
@@ -132,6 +132,89 @@ totals()
 
 # ----------------------------------------------------------------------------------
 
+echo "=========== is_package_in_runtests() tests ============="
+
+
+# test the package at the beginning of the list
+in_runtests="glibc gdb gcc binutils"
+testing="is_package_in_runtests \"${in_runtests}\" glibc"
+in_package="glibc"
+out="`is_package_in_runtests "${in_runtests}" ${in_package}`"
+ret=$?
+if test ${ret} -eq 0; then
+    pass "${testing}"
+else
+    fail "${testing}"
+    fixme "is_package_in_runtests \"${in_runtests}\" ${in_package} resulted in '${ret}'"
+fi
+
+# test the package at the end of the list
+in_runtests="glibc gdb gcc binutils"
+testing="is_package_in_runtests \"${in_runtests}\" binutils"
+in_package="binutils"
+out="`is_package_in_runtests "${in_runtests}" ${in_package}`"
+ret=$?
+if test ${ret} -eq 0; then
+    pass "${testing}"
+else
+    fail "${testing}"
+    fixme "is_package_in_runtests \"${in_runtests}\" ${in_package} resulted in '${ret}'"
+fi
+
+# test the package in the middle of the list
+in_runtests="glibc gdb gcc binutils"
+testing="is_package_in_runtests \"${in_runtests}\" gdb"
+in_package="gdb"
+out="`is_package_in_runtests "${in_runtests}" ${in_package}`"
+ret=$?
+if test ${ret} -eq 0; then
+    pass "${testing}"
+else
+    fail "${testing}"
+    fixme "is_package_in_runtests \"${in_runtests}\" ${in_package} resulted in '${ret}'"
+fi
+
+# test a package not in the list
+in_runtests="glibc gdb gcc binutils"
+testing="is_package_in_runtests \"${in_runtests}\" foo"
+in_package="foo"
+out="`is_package_in_runtests "${in_runtests}" ${in_package}`"
+ret=$?
+if test ${ret} -eq 1; then
+    pass "${testing}"
+else
+    fail "${testing}"
+    fixme "is_package_in_runtests \"${in_runtests}\" ${in_package} resulted in '${ret}' expected '1'"
+fi
+
+# test a partial package name
+in_runtests="glibc gdb gcc binutils"
+testing="is_package_in_runtests \"${in_runtests}\" gd"
+in_package="gd"
+out="`is_package_in_runtests "${in_runtests}" ${in_package}`"
+ret=$?
+if test ${ret} -eq 1; then
+    pass "${testing}"
+else
+    fail "${testing}"
+    fixme "is_package_in_runtests \"${in_runtests}\" ${in_package} resulted in '${ret}' expected '1'"
+fi
+
+# test that unquoted $runtests fails
+in_runtests="glibc gdb gcc binutils"
+testing="is_package_in_runtests ${in_runtests} glibc (unquoted \${in_runtests})"
+in_package="glibc"
+out="`is_package_in_runtests ${in_runtests} ${in_package}`"
+ret=$?
+if test ${ret} -eq 1; then
+    pass "${testing}"
+else
+    fail "${testing}"
+    fixme "is_package_in_runtests ${in_runtests} ${in_package} resulted in '${ret}'"
+fi
+
+
+
 echo "============= get_toolname() tests ================"
 
 testing="get_toolname: uncompressed tarball"