aboutsummaryrefslogtreecommitdiff
path: root/contrib/test-tool
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/test-tool')
-rwxr-xr-xcontrib/test-tool378
1 files changed, 378 insertions, 0 deletions
diff --git a/contrib/test-tool b/contrib/test-tool
new file mode 100755
index 0000000..32ab887
--- /dev/null
+++ b/contrib/test-tool
@@ -0,0 +1,378 @@
+#!/bin/sh
+# This script automatically test the given tool with the tool's test cases,
+# reporting anything of interest.
+
+# exits with 1 if there is nothing of interest
+# exits with 0 if there is something interesting
+# exits with 2 if an error occurred
+
+# Syntax: test-tool [-expectedpass] [-keepoutput] [-noupdate] g++|gcc|gdb|...
+#
+# -expectedpass: Turn XFAIL into "pass", XPASS into "fail".
+# The default is XFAIL->fail, XPASS->pass.
+# -keepoutput: Save "make check" output in test-$tool.log.
+# -noupdate: Don't update log files.
+
+# Limitations, don't run this multiple times in one day, unless the -noupdate
+# flag is given.
+
+# Written by Mike Stump <mrs@cygnus.com>
+
+expectedpass=no
+keepoutput=no
+update=yes
+tool=""
+
+# See if cp -p works.
+pwd=`pwd`
+cd /tmp
+rm -f test-tool-$$-1 test-tool-$$-2
+touch test-tool-$$-1
+cp -p test-tool-$$-1 test-tool-$$-2 2>/dev/null
+if [ $? = 0 -a -f test-tool-$$-2 ] ; then
+ CP="cp -p"
+else
+ CP=cp
+fi
+rm -f test-tool-$$-1 test-tool-$$-2
+cd $pwd
+
+for arg in $*
+do
+ case $arg in
+ -expectedpass) expectedpass=yes ;;
+ -keepoutput) keepoutput=yes ;;
+ -noupdate) update=no ;;
+ -*)
+ echo "Usage: [-expectedpass] [-keepoutput] [-noupdate] tool_name" >&2
+ exit 2
+ ;;
+ *)
+ if [ "$tool" != "" ]; then
+ echo "Usage: [-expectedpass] [-keepoutput] [-noupdate] tool_name" >&2
+ exit 2
+ fi
+ tool=$arg
+ ;;
+ esac
+done
+
+# FIXME: It sure would be nice if `testdir' wasn't necessary. :-(
+
+case $tool in
+ g++)
+ devoname=gcc
+ checktarget=check-g++
+ testdir=testsuite
+ ;;
+ gcc)
+ devoname=gcc
+ checktarget=check-gcc
+ testdir=testsuite
+ ;;
+ ld|gld)
+ devoname=ld
+ checktarget=check
+ testdir=.
+ ;;
+ binutils)
+ devoname=binutils
+ checktarget=check
+ testdir=.
+ ;;
+ *)
+ devoname=$tool
+ checktarget=check
+ testdir=testsuite
+ ;;
+esac
+
+# Default DEVOSRCDIR
+if [ "$DEVOSRCDIR" = "" ]; then
+ DEVOSRCDIR=$HOME/devo ; export DEVOSRCDIR
+fi
+
+# Check DEVOSRCDIR
+if [ ! -d "$DEVOSRCDIR" ]; then
+ echo "$0: no directory $DEVOSRCDIR" >&2
+ exit 2
+fi
+
+# Default DEVOBINDIR
+if [ "$DEVOBINDIR" = "" ]; then
+ CPU=`$DEVOSRCDIR/config.guess`
+ if [ $? != 0 ]; then
+ echo "$0: cannot run config.guess" >&2
+ exit 2
+ fi
+ DEVOBINDIR=$HOME/$CPU ; export DEVOBINDIR
+fi
+
+# Check DEVOBINDIR
+if [ ! -d "$DEVOBINDIR" ]; then
+ echo "$0: no directory $DEVOBINDIR" >&2
+ exit 2
+fi
+
+# Specialize DEVOSRCDIR
+if [ -d "$DEVOSRCDIR/$devoname" ]; then
+ DEVOSRCDIR=$DEVOSRCDIR/$devoname
+else
+ echo "$0: Cannot find source directory." >&2
+ exit 2
+fi
+
+# Default LOGDIR
+if [ "$LOGDIR" = "" ]; then
+ LOGDIR=$HOME/logs ; export LOGDIR
+fi
+
+# Check LOGDIR
+if [ ! -d "$LOGDIR" ]; then
+ echo "$0: no directory $LOGDIR" >&2
+ exit 2
+fi
+
+# Specialize DEVOBINDIR
+if [ -d "$DEVOBINDIR/$devoname" ]; then
+ DEVOBINDIR=$DEVOBINDIR/$devoname
+else
+ echo "$0: Cannot find binary directory." >&2
+ exit 2
+fi
+
+# Binary directory
+cd $DEVOBINDIR || exit 2
+
+TMPDIR=${TMPDIR-/tmp}
+
+tmp=$TMPDIR/$tool-testing.$$a
+tmp1=$TMPDIR/$tool-testing.$$b
+tmp2=$TMPDIR/$tool-testing.$$c
+now_s=$TMPDIR/$tool-testing.$$d
+before_s=$TMPDIR/$tool-testing.$$e
+
+if [ "$keepoutput" = yes ]; then
+ rm -f test-$tool.log
+ make RUNTESTFLAGS="-v -v" $checktarget >test-$tool.log 2>&1
+else
+ make RUNTESTFLAGS="-v -v" $checktarget >/dev/null 2>&1
+fi
+
+# Check for DEJAGNU errors that prevented any output at all.
+if [ ! -f $testdir/$tool.sum ]; then
+ echo "Tests didn't run, probably because of a framework error."
+ if [ "$keepoutput" = yes ]; then
+ echo
+ tail -20 test-$tool.log
+ else
+ echo "Unable to determine why. Rerun with -keepoutput."
+ fi
+ exit 2
+fi
+
+# Canonicalize XFAIL and XPASS so the rest of the script can ignore them.
+if [ "$expectedpass" = yes ]; then
+ sed 's/^XFAIL/PASS(XFAIL)/; s/^XPASS/FAIL(XPASS)/' <$testdir/$tool.sum >$testdir/$tool.1.sum || exit 2
+else
+ sed 's/^XFAIL/FAIL(XFAIL)/; s/^XPASS/PASS(XPASS)/' <$testdir/$tool.sum >$testdir/$tool.1.sum || exit 2
+fi
+mv $testdir/$tool.1.sum $testdir/$tool.sum
+
+patterns="$LOGDIR/$tool-??????.sum $LOGDIR/$tool-??????.sum.gz $LOGDIR/$tool-??????????.sum $LOGDIR/$tool-??????????.sum.gz"
+before=`ls -1t $patterns 2>/dev/null | sed 1q`
+
+todayname=`date '+%y%m%d'`
+if [ "$update" = no ]; then
+ now=$testdir/$tool.sum
+else
+ mv -f $testdir/$tool.sum $LOGDIR/$tool-$todayname.sum || exit 2
+ mv -f $testdir/$tool.log $LOGDIR/$tool-$todayname.log || exit 2
+
+ # Say where the logs are stored so they appear in email messages.
+ echo
+ echo "Log files: $LOGDIR/$tool-$todayname.*"
+ echo
+
+ now="$LOGDIR/$tool-$todayname.sum";
+fi
+
+trap "rm -f $tmp $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
+
+case $before in
+ "") before="cat /dev/null" ;;
+ *.gz) before="gunzip -c $before" ;;
+ *) before="cat $before" ;;
+esac
+
+# First, the test summary.
+egrep '^# of |===.*Summary.*===' "$now" || echo "(No test summary?)"
+echo
+
+# Pick out the PASS/FAIL/Ufoo messages.
+# We grep for them again later but that's for robustness' sake.
+grep '^[PFU][A-Z()]*:' "$now" | sort -t ':' +1 > "$now_s"
+$before | grep '^[PFU][A-Z()]*:' | sort -t ':' +1 > "$before_s"
+
+grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1
+grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -12 $tmp1 - >$tmp2
+
+grep -s . $tmp2 >/dev/null
+if [ $? = 0 ]; then
+ echo "Tests that now unexpectedly fail, but worked before:"
+ echo
+ cat $tmp2
+ showchangelog=1
+ echo
+fi
+
+grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1
+grep '^[PFU][A-Z()]*:' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -23 $tmp1 - >$tmp2
+
+grep -s . $tmp2 >/dev/null
+if [ $? = 0 ]; then
+ echo "New tests that unexpectedly FAIL:"
+ echo
+ cat $tmp2
+ echo
+fi
+
+grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1
+grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -12 $tmp1 - >$tmp2
+
+grep -s . $tmp2 >/dev/null
+if [ $? = 0 ]; then
+ echo "Tests that still don't work:"
+ echo
+ cat $tmp2
+ echo
+fi
+
+grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1
+grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -12 $tmp1 - >$tmp2
+
+grep -s . $tmp2 >/dev/null
+if [ $? = 0 ]; then
+ echo "Tests that now work, but didn't before:"
+ echo
+ cat $tmp2
+ echo
+fi
+
+grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1
+grep '^[PFU][A-Z()]*:' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -23 $tmp1 - >$tmp2
+
+grep -s . $tmp2 >/dev/null
+if [ $? = 0 ]; then
+ echo "New tests that PASS:"
+ echo
+ cat $tmp2
+ echo
+fi
+
+grep '^[PFU][A-Z()]*:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1
+grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -13 $tmp1 - >$tmp2
+
+grep -s . $tmp2 >/dev/null
+if [ $? = 0 ]; then
+ echo "Old tests that passed, that have disappeared: (Eeek!)"
+ echo
+ cat $tmp2
+ echo
+fi
+
+grep '^[PFU][A-Z()]*:' "$now_s" | sed 's/^[^:]*:[ ]*//' >$tmp1
+grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]*//' | comm -13 $tmp1 - >$tmp2
+
+grep -s . $tmp2 >/dev/null
+if [ $? = 0 ]; then
+ echo "Old tests that failed, that have disappeared: (Eeek!)"
+ echo
+ cat $tmp2
+ echo
+fi
+
+egrep '^(ERROR|WARNING):' "$now" >$tmp1
+
+if grep -s . $tmp1 > /dev/null; then
+ echo "Errors and warnings:"
+ echo
+ cat $tmp1
+ echo
+fi
+
+if [ "$tool" = g++ ]; then
+ if [ -f $DEVOBINDIR/libio/run-make-check ]; then
+ cd $DEVOBINDIR/libio
+ make check >$TMPDIR/clgpp$$ 2>&1
+ if [ $? != 0 ]; then
+ echo
+ echo "libio fails to make check:"
+ tail -20 $TMPDIR/clgpp$$
+ fi
+ fi
+ if [ -f $DEVOBINDIR/libstdc++/run-make-check ]; then
+ cd $DEVOBINDIR/libstdc++
+ make check >$TMPDIR/clgpp$$ 2>&1
+ if [ $? != 0 ]; then
+ echo
+ echo "libstdc++ fails to make check:"
+ tail -20 $TMPDIR/clgpp$$
+ fi
+ fi
+ if [ -f $DEVOBINDIR/libg++/run-make-check ]; then
+ cd $DEVOBINDIR/libg++
+ make check >$TMPDIR/clgpp$$ 2>&1
+ if [ $? != 0 ]; then
+ echo
+ echo "libg++ fails to make check:"
+ tail -20 $TMPDIR/clgpp$$
+ fi
+ fi
+ rm -f $TMPDIR/clgpp$$
+ cd $DEVOBINDIR
+fi
+
+if [ "$devoname" != "" ]; then
+ if [ "$showchangelog" = 1 ]; then
+ echo "Here is what's new in the ChangeLog:"
+ echo
+ diff -c $LOGDIR/$devoname.ChangeLog $DEVOSRCDIR/ChangeLog
+ echo
+ if [ "$tool" = g++ ]; then
+ echo
+ echo "Here is what's new in the ChangeLog.egcs:"
+ echo
+ diff -c $LOGDIR/gcc.ChangeLog.egcs $DEVOSRCDIR/ChangeLog.egcs
+
+ echo
+ echo "Here is what's new in the cp/ChangeLog:"
+ echo
+ diff -c $LOGDIR/g++.ChangeLog $DEVOSRCDIR/cp/ChangeLog
+ fi
+ echo
+ fi
+ if [ "$update" != no ]; then
+ # save the old ChangeLog as a reference for next time
+ rm -f $LOGDIR/$devoname.ChangeLog.BAK
+ mv $LOGDIR/$devoname.ChangeLog $LOGDIR/$devoname.ChangeLog.BAK 2>/dev/null
+ $CP $DEVOSRCDIR/ChangeLog $LOGDIR/$devoname.ChangeLog
+ if [ "$tool" = g++ ]; then
+ rm -f $LOGDIR/gcc.ChangeLog.egcs.BAK
+ mv $LOGDIR/gcc.ChangeLog.egcs $LOGDIR/gcc.ChangeLog.egcs.BAK 2>/dev/null
+ $CP $DEVOSRCDIR/ChangeLog.egcs $LOGDIR/gcc.ChangeLog.egcs
+
+ rm -f $LOGDIR/g++.ChangeLog.BAK
+ mv $LOGDIR/g++.ChangeLog $LOGDIR/g++.ChangeLog.BAK 2>/dev/null
+ $CP $DEVOSRCDIR/cp/ChangeLog $LOGDIR/g++.ChangeLog
+ fi
+ fi
+fi
+
+$before | diff - $now | grep -s . >/dev/null
+if [ $? = 0 ]; then
+ echo "Details:"
+ echo
+ $before | diff - $now
+ echo
+fi
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-12 10:59:13 +0000