aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJacob Bachmeyer <jcb62281@gmail.com>2019-01-02 22:48:20 +1100
committerBen Elliston <bje@gnu.org>2019-01-02 22:48:20 +1100
commitdc67c894db920e8f145de970ac2e61396d8db9db (patch)
tree64c09e5a1d95bf68b5737f057f716db3dad0ef45
parentb51a38c4410bbff17d4296fcb1fda13313b251ac (diff)
* NEWS: Document report card.
* Makefile.am (clean-local): Add target. (clean-local-check): Add target; mark as PHONY. (commands_DATA): Add "report-card" scripts. (dist_man_MANS): Add dejagnu-report-card.1 and split. (DEJATOOL): Add "report-card" tool. (TESTSUITE_FILES): Add testsuite for "report-card" tool. * commands/report-card.awk: New command script. * doc/dejagnu.texi (Invoking dejagnu report card): New node. * doc/dejagnu-report-card.1: New man page. * testsuite/lib/bohman_ssd.exp: New file. * testsuite/lib/report-card.exp: New file. * testsuite/report-card.all/onetest.exp: New file. * testsuite/report-card.all/passes.exp: New file. Signed-off-by: Ben Elliston <bje@gnu.org>
-rw-r--r--ChangeLog21
-rw-r--r--Makefile.am24
-rw-r--r--Makefile.in65
-rw-r--r--NEWS3
-rw-r--r--commands/report-card.awk238
-rw-r--r--doc/dejagnu-report-card.1146
-rw-r--r--doc/dejagnu.texi45
-rw-r--r--doc/version.texi4
-rw-r--r--testsuite/lib/bohman_ssd.exp225
-rw-r--r--testsuite/lib/report-card.exp39
-rw-r--r--testsuite/report-card.all/onetest.exp209
-rw-r--r--testsuite/report-card.all/passes.exp276
12 files changed, 1262 insertions, 33 deletions
diff --git a/ChangeLog b/ChangeLog
index 455168a..68c1cce 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,24 @@
+2019-01-02 Jacob Bachmeyer <jcb62281@gmail.com>
+
+ * NEWS: Document report card.
+
+ * Makefile.am (clean-local): Add target.
+ (clean-local-check): Add target; mark as PHONY.
+ (commands_DATA): Add "report-card" scripts.
+ (dist_man_MANS): Add dejagnu-report-card.1 and split.
+ (DEJATOOL): Add "report-card" tool.
+ (TESTSUITE_FILES): Add testsuite for "report-card" tool.
+
+ * commands/report-card.awk: New command script.
+
+ * doc/dejagnu.texi (Invoking dejagnu report card): New node.
+ * doc/dejagnu-report-card.1: New man page.
+
+ * testsuite/lib/bohman_ssd.exp: New file.
+ * testsuite/lib/report-card.exp: New file.
+ * testsuite/report-card.all/onetest.exp: New file.
+ * testsuite/report-card.all/passes.exp: New file.
+
2019-01-02 Ben Elliston <bje@gnu.org>
* Makefile.am (DISTCLEANFILES): Don't use this.
diff --git a/Makefile.am b/Makefile.am
index 99a7eec..d78993c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,4 +1,4 @@
-# Copyright (C) 1992-2016, 2018 Free Software Foundation, Inc.
+# Copyright (C) 1992-2016, 2018, 2019 Free Software Foundation, Inc.
#
# This file is part of DejaGnu.
#
@@ -28,6 +28,14 @@ EXTRA_DIST = ChangeLog-1992 MAINTAINERS dejagnu runtest \
CLEANFILES = options-init.exp stats-init.exp
+clean-local: clean-local-check
+.PHONY: clean-local-check
+clean-local-check:
+ -rm -rf testsuite/launcher.all/command/bin
+ -rm -rf testsuite/launcher.all/command/share
+ -rm -rf testsuite/report-card.all/onetest
+ -rm -rf testsuite/report-card.all/passes
+
# Give a reassuring message so that users know the "build" worked.
all-local:
@echo "Done. Now run 'make install'."
@@ -60,7 +68,8 @@ pkgdata_DATA = \
commandsdir = $(pkgdatadir)/commands
commands_DATA = \
- commands/help.sh
+ commands/help.sh \
+ commands/report-card.awk
configdir = $(pkgdatadir)/config
config_DATA = \
@@ -157,6 +166,8 @@ TESTSUITE_FILES = \
testsuite/launcher.all/help.exp \
testsuite/launcher.all/interp.exp \
testsuite/launcher.all/verbose.exp \
+ testsuite/report-card.all/onetest.exp \
+ testsuite/report-card.all/passes.exp \
testsuite/runtest.libs/topdir/subdir1/subsubdir1/subsubfile1 \
testsuite/runtest.libs/topdir/subdir1/subfile1 \
testsuite/runtest.libs/topdir/subdir1/subfile2 \
@@ -173,14 +184,16 @@ TESTSUITE_FILES = \
testsuite/runtest.main/options/testsuite/null.test/null.exp \
testsuite/runtest.main/stats.exp \
testsuite/runtest.main/stats/testsuite/stat.test/stats-sub.exp \
+ testsuite/lib/bohman_ssd.exp \
testsuite/lib/launcher.exp \
testsuite/lib/libdejagnu.exp \
testsuite/lib/libsup.exp \
+ testsuite/lib/report-card.exp \
testsuite/lib/runtest.exp \
testsuite/lib/util-defs.exp \
testsuite/libdejagnu/tunit.exp
-DEJATOOL = launcher libdejagnu runtest
+DEJATOOL = launcher libdejagnu report-card runtest
RUNTEST = ${top_srcdir}/runtest
@@ -191,5 +204,8 @@ unit_SOURCES = testsuite/libdejagnu/unit.cc
# Documentation.
TEXINFO_TEX = doc/texinfo.tex
-dist_man_MANS = doc/dejagnu.1 doc/dejagnu-help.1 doc/runtest.1
+dist_man_MANS = doc/dejagnu.1 \
+ doc/dejagnu-help.1 \
+ doc/dejagnu-report-card.1 \
+ doc/runtest.1
info_TEXINFOS = doc/dejagnu.texi
diff --git a/Makefile.in b/Makefile.in
index 1a6d1ef..1cd211c 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -14,7 +14,7 @@
@SET_MAKE@
-# Copyright (C) 1992-2016, 2018 Free Software Foundation, Inc.
+# Copyright (C) 1992-2016, 2018, 2019 Free Software Foundation, Inc.
#
# This file is part of DejaGnu.
#
@@ -409,7 +409,8 @@ pkgdata_DATA = \
commandsdir = $(pkgdatadir)/commands
commands_DATA = \
- commands/help.sh
+ commands/help.sh \
+ commands/report-card.awk
configdir = $(pkgdatadir)/config
config_DATA = \
@@ -505,6 +506,8 @@ TESTSUITE_FILES = \
testsuite/launcher.all/help.exp \
testsuite/launcher.all/interp.exp \
testsuite/launcher.all/verbose.exp \
+ testsuite/report-card.all/onetest.exp \
+ testsuite/report-card.all/passes.exp \
testsuite/runtest.libs/topdir/subdir1/subsubdir1/subsubfile1 \
testsuite/runtest.libs/topdir/subdir1/subfile1 \
testsuite/runtest.libs/topdir/subdir1/subfile2 \
@@ -521,21 +524,27 @@ TESTSUITE_FILES = \
testsuite/runtest.main/options/testsuite/null.test/null.exp \
testsuite/runtest.main/stats.exp \
testsuite/runtest.main/stats/testsuite/stat.test/stats-sub.exp \
+ testsuite/lib/bohman_ssd.exp \
testsuite/lib/launcher.exp \
testsuite/lib/libdejagnu.exp \
testsuite/lib/libsup.exp \
+ testsuite/lib/report-card.exp \
testsuite/lib/runtest.exp \
testsuite/lib/util-defs.exp \
testsuite/libdejagnu/tunit.exp
-DEJATOOL = launcher libdejagnu runtest
+DEJATOOL = launcher libdejagnu report-card runtest
RUNTEST = ${top_srcdir}/runtest
AM_CXXFLAGS = -I$(top_srcdir) -g
unit_SOURCES = testsuite/libdejagnu/unit.cc
# Documentation.
TEXINFO_TEX = doc/texinfo.tex
-dist_man_MANS = doc/dejagnu.1 doc/dejagnu-help.1 doc/runtest.1
+dist_man_MANS = doc/dejagnu.1 \
+ doc/dejagnu-help.1 \
+ doc/dejagnu-report-card.1 \
+ doc/runtest.1
+
info_TEXINFOS = doc/dejagnu.texi
all: all-am
@@ -1304,7 +1313,7 @@ maintainer-clean-generic:
@echo "it deletes files that may require special tools to rebuild."
clean: clean-am
-clean-am: clean-aminfo clean-checkPROGRAMS clean-generic \
+clean-am: clean-aminfo clean-checkPROGRAMS clean-generic clean-local \
mostlyclean-am
distclean: distclean-am
@@ -1480,25 +1489,25 @@ uninstall-man: uninstall-man1
.PHONY: CTAGS GTAGS TAGS all all-am all-local am--refresh check \
check-DEJAGNU check-am clean clean-aminfo clean-checkPROGRAMS \
- clean-cscope clean-generic cscope cscopelist-am ctags ctags-am \
- dist dist-all dist-bzip2 dist-gzip dist-info dist-lzip \
- dist-shar dist-tarZ dist-xz dist-zip distcheck distclean \
- distclean-DEJAGNU distclean-compile distclean-generic \
- distclean-tags distcleancheck distdir distuninstallcheck dvi \
- dvi-am html html-am info info-am install install-am \
- install-baseboardDATA install-binSCRIPTS install-commandsDATA \
- install-configDATA install-data install-data-am \
- install-djlibexecSCRIPTS install-dvi install-dvi-am \
- install-exec install-exec-am install-html install-html-am \
- install-includeHEADERS install-info install-info-am \
- install-man install-man1 install-pdf install-pdf-am \
- install-pkgdataDATA install-ps install-ps-am install-strip \
- installcheck installcheck-am installdirs maintainer-clean \
- maintainer-clean-aminfo maintainer-clean-generic \
- maintainer-clean-vti mostlyclean mostlyclean-aminfo \
- mostlyclean-compile mostlyclean-generic mostlyclean-vti pdf \
- pdf-am ps ps-am tags tags-am uninstall uninstall-am \
- uninstall-baseboardDATA uninstall-binSCRIPTS \
+ clean-cscope clean-generic clean-local cscope cscopelist-am \
+ ctags ctags-am dist dist-all dist-bzip2 dist-gzip dist-info \
+ dist-lzip dist-shar dist-tarZ dist-xz dist-zip distcheck \
+ distclean distclean-DEJAGNU distclean-compile \
+ distclean-generic distclean-tags distcleancheck distdir \
+ distuninstallcheck dvi dvi-am html html-am info info-am \
+ install install-am install-baseboardDATA install-binSCRIPTS \
+ install-commandsDATA install-configDATA install-data \
+ install-data-am install-djlibexecSCRIPTS install-dvi \
+ install-dvi-am install-exec install-exec-am install-html \
+ install-html-am install-includeHEADERS install-info \
+ install-info-am install-man install-man1 install-pdf \
+ install-pdf-am install-pkgdataDATA install-ps install-ps-am \
+ install-strip installcheck installcheck-am installdirs \
+ maintainer-clean maintainer-clean-aminfo \
+ maintainer-clean-generic maintainer-clean-vti mostlyclean \
+ mostlyclean-aminfo mostlyclean-compile mostlyclean-generic \
+ mostlyclean-vti pdf pdf-am ps ps-am tags tags-am uninstall \
+ uninstall-am uninstall-baseboardDATA uninstall-binSCRIPTS \
uninstall-commandsDATA uninstall-configDATA \
uninstall-djlibexecSCRIPTS uninstall-dvi-am uninstall-html-am \
uninstall-includeHEADERS uninstall-info-am uninstall-man \
@@ -1509,6 +1518,14 @@ uninstall-man: uninstall-man1
export DEJAGNU
+clean-local: clean-local-check
+.PHONY: clean-local-check
+clean-local-check:
+ -rm -rf testsuite/launcher.all/command/bin
+ -rm -rf testsuite/launcher.all/command/share
+ -rm -rf testsuite/report-card.all/onetest
+ -rm -rf testsuite/report-card.all/passes
+
# Give a reassuring message so that users know the "build" worked.
all-local:
@echo "Done. Now run 'make install'."
diff --git a/NEWS b/NEWS
index b3669e6..081f1a4 100644
--- a/NEWS
+++ b/NEWS
@@ -19,6 +19,9 @@ Changes since 1.6.2:
auxiliary commands not directly involved with running tests. The
"runtest" command will remain for that purpose for the forseeable
future.
+8. The first auxiliary command is added: "report card". The "dejagnu
+ report card" command reads DejaGnu summary files and produces a
+ compact tabular summary across multiple tools.
Changes since 1.6.1:
diff --git a/commands/report-card.awk b/commands/report-card.awk
new file mode 100644
index 0000000..b04c0e9
--- /dev/null
+++ b/commands/report-card.awk
@@ -0,0 +1,238 @@
+# report-card.awk -- Test summary tool
+# Copyright (C) 2018 Free Software Foundation, Inc.
+#
+# This file is part of DejaGnu.
+#
+# DejaGnu is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# DejaGnu is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DejaGnu; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA.
+
+# This file was written by Jacob Bachmeyer.
+
+# ##help
+# #Usage: dejagnu report card [<option>|<tool>|<file>]...
+# #Usage: dejagnu report-card [<option>|<tool>|<file>]...
+# # --verbose, -v Emit additional messages
+# ##end
+
+# Arrays storing lists in this program store items in numbered keys, with a
+# count in the "C" key, similar to Awk's ARGV/ARGC.
+
+# The Tools array stores a list of tools in 1..N.
+
+# The Passes array stores a global list of passes seen, a per-tool list of
+# passes seen, and a global index of passes seen if DejaGnu's multipass
+# support is used.
+# Key prefixes:
+# "" -- global list: 1..N; "C"
+# "t", <tool> -- per-tool list: 1..N; "C"
+# Key patterns:
+# "p", <pass> -- count of tools using <pass>
+
+# The Totals array stores counts of test results, indexed by tool and pass.
+# A summarization step adds per-tool, per-pass, and grand totals.
+# Key patterns:
+# "tp", <Tool>, <Pass>, <result>
+# "t", <Tool>, <result>
+# "p", <Pass>, <result>
+# <result>
+
+##
+## Get list of files to scan
+
+BEGIN {
+ Tools["C"] = 1
+ Passes["", "C"] = 1
+ ToolWidth = 0
+ PassWidth = 0
+ Verbose = 0
+ # remove arguments from ARGV
+ for (i = 1; i < ARGC; i++) {
+ if (ARGV[i] ~ /^-/) {
+ if (ARGV[i] ~ /^--?v(erb.*)?$/)
+ Verbose++
+ else if (ARGV[i] == "--")
+ break
+ delete ARGV[i]
+ }
+ }
+ if (ARGV[i] == "--")
+ delete ARGV[i]
+ if (Verbose) print "Verbose level is "Verbose
+ # adjust filenames in ARGV
+ FileCount = 0
+ for (i = 1; i < ARGC; i++) {
+ if (i in ARGV) FileCount++
+ else continue
+ if (ARGV[i] ~ /\.sum$/) continue
+ else if (ARGV[i] ~ /\.log$/) sub(/\.log$/, ".sum", ARGV[i])
+ else if (ARGV[i] ~/\.$/) sub(/\.$/, ".sum", ARGV[i])
+ else ARGV[i] = (ARGV[i]".sum")
+ }
+ if (FileCount == 0) {
+ cmd_ls_files = "ls -1 *.sum"
+ while (cmd_ls_files | getline File) {
+ FileCount++
+ ARGV[ARGC++] = File
+ }
+ close(cmd_ls_files)
+ }
+ if (Verbose > 2) {
+ print "Reading "FileCount" file(s)"
+ for (i = 1; i < ARGC; i++)
+ if (i in ARGV)
+ print " "ARGV[i]
+ }
+}
+
+##
+## Read files and collect data
+
+FNR == 1 {
+ if (Verbose)
+ print "Reading `"FILENAME"' ..."
+ Pass = ""
+ Tool = File = FILENAME
+ sub(/\.sum$/, "", Tool)
+ if (length(Tool) > ToolWidth)
+ ToolWidth = length(Tool)
+ Tools[Tools["C"]++] = Tool
+ Passes["t", Tool, "C"] = 1
+ Passes["t", Tool, 1] = "" # will be overwritten if multipass is used
+}
+
+/^Running pass `[^']*' .../ {
+ Pass = $3
+ sub(/^`/, "", Pass)
+ sub(/'$/, "", Pass)
+ if (("p", Pass) in Passes)
+ Passes["p", Pass]++
+ else {
+ if (length(Pass) > PassWidth)
+ PassWidth = length(Pass)
+ Passes["", Passes["", "C"]++] = Pass
+ Passes["p", Pass] = 1
+ }
+ Passes["t", Tool, Passes["t", Tool, "C"]++] = Pass
+}
+
+$1 ~ /:$/ { sub(/:$/, "", $1); Totals["tp", Tool, Pass, $1]++ }
+
+##
+## Compute totals
+
+END {
+ $0 = ("PASS FAIL KPASS KFAIL XPASS XFAIL UNSUPPORTED UNRESOLVED UNTESTED")
+ for (i = 1; i in Tools; i++)
+ for (j = 1; ("t", Tools[i], j) in Passes; j++)
+ for (k = 1; k <= NF; k++) {
+ Totals[$k] \
+ += Totals["tp", Tools[i], Passes["t", Tools[i], j], $k]
+ Totals["t", Tools[i], $k] \
+ += Totals["tp", Tools[i], Passes["t", Tools[i], j], $k]
+ Totals["p", Passes["t", Tools[i], j], $k] \
+ += Totals["tp", Tools[i], Passes["t", Tools[i], j], $k]
+ }
+}
+
+##
+## Compute total name column width
+
+END {
+ if (Passes["", "C"] > 1)
+ NameWidth = ToolWidth + 3 + PassWidth
+ else
+ NameWidth = ToolWidth
+}
+
+##
+## Emit header
+
+END {
+ printf "%*s __________________________________________________\n", \
+ NameWidth, ""
+ printf "%*s / %6s %6s %6s %6s %6s %6s %6s\n", NameWidth, "", \
+ "PASS", "FAIL", "?PASS", "?FAIL", "UNSUP", "UNRES", "UNTEST"
+ printf "%*s |--------------------------------------------------\n", \
+ NameWidth, ""
+}
+
+##
+## Emit counts
+
+END {
+ for (i = 1; i in Tools; i++) {
+ Tool = Tools[i]
+ for (j = 1; ("t", Tool, j) in Passes; j++) {
+ Pass = Passes["t", Tool, j]
+ if (Passes["t", Tool, "C"] > 1)
+ printf "%*s / %-*s | ", ToolWidth, Tool, PassWidth, Pass
+ else if (Passes["", "C"] > 1)
+ printf "%*s %*s | ", ToolWidth, Tool, PassWidth, ""
+ else
+ printf "%*s | ", NameWidth, Tool
+ # Passes["t", <tool>, 1] is a pass name or a null string if
+ # <tool> did not use multipass.
+ printf " %6d %6d %6d %6d %6d %6d %6d%s%s\n", \
+ Totals["tp", Tool, Pass, "PASS"], \
+ Totals["tp", Tool, Pass, "FAIL"], \
+ Totals["tp", Tool, Pass, "KPASS"] \
+ + Totals["tp", Tool, Pass, "XPASS"], \
+ Totals["tp", Tool, Pass, "KFAIL"] \
+ + Totals["tp", Tool, Pass, "XFAIL"], \
+ Totals["tp", Tool, Pass, "UNSUPPORTED"], \
+ Totals["tp", Tool, Pass, "UNRESOLVED"], \
+ Totals["tp", Tool, Pass, "UNTESTED"], \
+ (Totals["tp", Tool, Pass, "ERROR" ] > 0 ? " !E!" : ""), \
+ (Totals["tp", Tool, Pass, "WARNING"] > 0 ? " !W!" : "")
+ }
+ }
+}
+
+##
+## Emit pass totals
+
+END {
+ if (Passes["", "C"] > 1) {
+ printf "%*s |--------------------------------------------------\n", \
+ NameWidth, ""
+ for (i = 1; ("", i) in Passes; i++)
+ printf "%*s %-*s | %6d %6d %6d %6d %6d %6d %6d\n", \
+ ToolWidth, "", PassWidth, Passes["", i], \
+ Totals["p", Passes["", i], "PASS"], \
+ Totals["p", Passes["", i], "FAIL"], \
+ Totals["p", Passes["", i], "KPASS"] \
+ + Totals["p", Passes["", i], "XPASS"], \
+ Totals["p", Passes["", i], "KFAIL"] \
+ + Totals["p", Passes["", i], "XFAIL"], \
+ Totals["p", Passes["", i], "UNSUPPORTED"], \
+ Totals["p", Passes["", i], "UNRESOLVED"], \
+ Totals["p", Passes["", i], "UNTESTED"]
+ }
+}
+
+##
+## Emit grand totals
+
+END {
+ printf "%*s |--------------------------------------------------\n", \
+ NameWidth, ""
+ printf "%*s | %6d %6d %6d %6d %6d %6d %6d\n", NameWidth, "", \
+ Totals["PASS"], Totals["FAIL"], \
+ Totals["KPASS"] + Totals["XPASS"], Totals["KFAIL"] + Totals["XFAIL"], \
+ Totals["UNSUPPORTED"], Totals["UNRESOLVED"], Totals["UNTESTED"]
+ printf "%*s \\__________________________________________________\n", \
+ NameWidth, ""
+}
+
+#EOF
diff --git a/doc/dejagnu-report-card.1 b/doc/dejagnu-report-card.1
new file mode 100644
index 0000000..2e69fd0
--- /dev/null
+++ b/doc/dejagnu-report-card.1
@@ -0,0 +1,146 @@
+.\" Copyright (C) 2018 Free Software Foundation, Inc.
+.\" You may distribute this file under the terms of the GNU Free
+.\" Documentation License.
+.Dd December 31, 2018
+.Os GNU
+.Dt DEJAGNU-REPORT-CARD 1 URM
+.Sh NAME
+.Nm dejagnu\ report\ card
+.Nd summarize results from testing multiple tools
+.Sh SYNOPSIS
+.Nm dejagnu\ report\ card
+.Oo Ao Ar option Ac \*(Ba Ao Ar tool Ac \*(Ba Ao Ar file Ac Oc ...
+.Sh DESCRIPTION
+The
+.Nm
+command displays results from testing multiple tools in a tabular format.
+The produced table lists, for each tool (and if multiple passes were run,
+each pass) the number of tests passed, failed, unsupported, unresolved, and
+untested. Tests that are expected to fail are counted in separate columns
+from tests expected to pass, but "known" failures and "expected" failures
+are summarized together. If a test generated warnings or errors, a tag
+.Ql !W!
+or
+.Ql !E!
+is appended at the end of the relevant line.
+.Pp
+Aside from options, the argument list may include tool or file names. The
+.Nm
+command prefers to read DejaGnu summary files and will translate names accordingly:
+.Bl -tag -width ".Pa *.sum"
+.It Pa *.sum
+Used as-is.
+.It Pa *.log
+Rewritten to
+.Pa *.sum
+with the same stem.
+.It Pa *.
+The string
+.Pa sum
+is appended to select a summary file. This processing is done for
+convenience when using Readline file name completion in a shell, which will
+complete to the dot.
+.It Pa *
+Taken as a tool name;
+.Pa .sum
+is appended.
+.El
+.Sh OPTIONS
+.Bl -tag -width ".Fl v , -verbose"
+.It Fl v , -verbose
+Emit additional output describing the operation of
+.Nm
+itself.
+.El
+.Sh FILES
+The
+.Nm
+command produces its output by reading the summary files produced by
+DejaGnu and counting "PASS", "FAIL", etc.
+.Pp
+If no names are given as arguments, all files matching
+.Pa *.sum
+in the current directory are read.
+.Sh EXAMPLES
+.Ss A simple example from DejaGnu's own testsuite
+.Bd -literal
+$ dejagnu report card
+\ __________________________________________________
+\ / PASS FAIL ?PASS ?FAIL UNSUP UNRES UNTEST
+\ |--------------------------------------------------
+\ launcher | 52 0 0 0 0 0 0
+libdejagnu | 5 0 0 0 0 0 0
+\ runtest | 135 0 0 0 0 0 0
+\ |--------------------------------------------------
+\ | 192 0 0 0 0 0 0
+\ \\__________________________________________________
+.Ed
+.Pp
+Three tools were tested, with a total of 192 tests, all expected to pass.
+In this example, all tests did pass, so all other columns are zero. The
+.Ql ?PASS
+and
+.Ql ?FAIL
+columns count tests known or expected to fail that either unexpectedly
+passed or failed as expected. The remaining three columns count the
+exceptional results for unsupported tests, unresolved tests and stub tests
+that simply declare themselves untested.
+.Pp
+.ne 16v
+.Ss The same example after tests were added for dejagnu-report-card
+.Bd -literal
+$ dejagnu report-card
+\ __________________________________________________
+\ / PASS FAIL ?PASS ?FAIL UNSUP UNRES UNTEST
+\ |--------------------------------------------------
+\ launcher | 52 0 0 0 0 0 0
+\ libdejagnu | 5 0 0 0 0 0 0
+report-card / awk | 36 0 0 0 0 0 0
+report-card / sh | 36 0 0 0 0 0 0
+report-card / tcl | 36 0 0 0 0 0 0
+\ runtest | 135 0 0 0 0 0 0
+\ |--------------------------------------------------
+\ awk | 36 0 0 0 0 0 0
+\ sh | 36 0 0 0 0 0 0
+\ tcl | 36 0 0 0 0 0 0
+\ |--------------------------------------------------
+\ | 300 0 0 0 0 0 0
+\ \\__________________________________________________
+.Ed
+.Pp
+The
+.Ql report-card
+tool has been added, with three passes, one for each implementation. (The
+shell and Tcl implementations were later dropped to reduce future
+maintenance burden.) As before, all tests passed as expected. The
+interesting difference from the previous example is the use of DejaGnu's
+multipass testing feature and the additional per-pass summary lines added.
+For this example, only the
+.Ql report-card
+tool uses multipass testing, so each pass total is simply the count of
+tests for
+.Ql report-card
+instead of a distinct total.
+.Pp
+Also note that the command used to invoke
+.Nm
+is slightly different here. The
+.Xr dejagnu 1
+launcher will also accept multiple words joined with dashes into a single
+argument. This allows individual words in a command name to be separated
+with either dashes or spaces on the command line interchangeably.
+.Sh SEE ALSO
+.Xr dejagnu 1
+.Xr runtest 1
+.Pp
+The full documentation for DejaGnu is maintained as a Texinfo manual. If the
+.Nm info
+program is properly installed at your site, the command
+.Li info dejagnu
+should give you access to the complete manual.
+.Sh AUTHORS
+.An Jacob Bachmeyer
+.\".Sh BUGS
+.\" LocalWords: Dt dejagnu URM Nm Ao Oo Oc DejaGnu Xr runtest DejaGnu's Bd Ql
+.\" LocalWords: testsuite UNSUP UNRES UNTEST libdejagnu Readline Ss tcl awk
+.\" LocalWords: ne multipass
diff --git a/doc/dejagnu.texi b/doc/dejagnu.texi
index e07a40a..11d433e 100644
--- a/doc/dejagnu.texi
+++ b/doc/dejagnu.texi
@@ -80,6 +80,7 @@ Running other DejaGnu commands
* Invoking dejagnu:: Command line options for the launcher itself.
* Invoking dejagnu help:: Reading man pages for dejagnu subcommands.
+* Invoking dejagnu report card:: Summarizing test results from many tools.
Customizing DejaGnu
@@ -1026,7 +1027,8 @@ then runs the requested command.
@menu
* Invoking dejagnu:: Command line options for the launcher itself.
-* Invoking dejagnu help:: Reading man pages for dejagnu subcommands.
+* Invoking dejagnu help:: Reading man pages for dejagnu subcommands.
+* Invoking dejagnu report card:: Summarizing test results from many tools.
@end menu
@node Invoking dejagnu, Invoking dejagnu help, Running other DejaGnu commands, Running other DejaGnu commands
@@ -1089,7 +1091,7 @@ invoked command.
All arguments after the command name are passed to the invoked command.
-@node Invoking dejagnu help, , Invoking dejagnu, Running other DejaGnu commands
+@node Invoking dejagnu help, Invoking dejagnu report card, Invoking dejagnu, Running other DejaGnu commands
@section Invoking @command{dejagnu help}
@cindex dejagnu help, invoking
@@ -1116,6 +1118,42 @@ inner workings of the @command{dejagnu help} command to be produced.
The @option{--path}, @option{-w}, and @option{-W} options are passed
to @command{man}.
+@node Invoking dejagnu report card, , Invoking dejagnu help, Running other DejaGnu commands
+@section Invoking @command{dejagnu report card}
+@cindex dejagnu report card, invoking
+@cindex dejagnu report-card, invoking
+
+The @command{dejagnu report card} tool produces a tabular summary of
+the results from test runs by reading the summary files that DejaGnu
+produces.
+
+@example
+@command{dejagnu report card} [<option>|<tool>|<file>]...
+@end example
+
+The @option{--verbose} option causes additional output describing the
+inner workings of the @command{dejagnu report card} command to be produced.
+
+Aside from options, the command may include a list of tools or files.
+Names ending in @samp{.sum} are used as-is. Names ending in
+@samp{.log} are changed to instead refer to the summary file. Names
+ending with a simple dot (@samp{.}) have @samp{sum} appended, for
+convenience when using Readline filename completion in a shell, which
+will complete to the dot, since there are both @samp{.sum} and
+@samp{.log} files produced for each tool tested. Lastly, all other
+names are taken as tool names and @samp{.sum} is appended to refer to
+the summary file produced by DejaGnu.
+
+The relevant summary files are read and an ASCII-art table is
+produced. The table has columns for counts of tests passed, failed,
+unsupported, unresolved, and untested. Tests that are expected to
+pass and tests that are expected to fail are counted in separate
+columns, but known failures (@samp{KFAIL} and @samp{KPASS}) are
+summarized together with expected failures (@samp{XFAIL} and
+@samp{XPASS}) in two additional columns: @samp{?PASS} and
+@samp{?FAIL}. Additionally, if a test produced any warnings or
+errors, tags @samp{!W!} or @samp{!E!} are added at the end of the row.
+
@node Customizing DejaGnu, Extending DejaGnu, Running other DejaGnu commands, Top
@chapter Customizing DejaGnu
@cindex customization
@@ -5610,4 +5648,5 @@ This makes @code{runtest} exit. Abbreviation: @kbd{q}.
@bye
@c LocalWords: subdirectory prepend prepended testsuite filename Expect's svn
-@c LocalWords: DejaGnu CVS RCS SCCS prepending subcommands
+@c LocalWords: DejaGnu CVS RCS SCCS prepending subcommands Tcl Awk Readline
+@c LocalWords: POSIX KFAIL KPASS XFAIL XPASS
diff --git a/doc/version.texi b/doc/version.texi
index e3c9a08..f2d1f35 100644
--- a/doc/version.texi
+++ b/doc/version.texi
@@ -1,4 +1,4 @@
-@set UPDATED 20 December 2018
-@set UPDATED-MONTH December 2018
+@set UPDATED 2 January 2019
+@set UPDATED-MONTH January 2019
@set EDITION 1.6.3-git
@set VERSION 1.6.3-git
diff --git a/testsuite/lib/bohman_ssd.exp b/testsuite/lib/bohman_ssd.exp
new file mode 100644
index 0000000..25b1072
--- /dev/null
+++ b/testsuite/lib/bohman_ssd.exp
@@ -0,0 +1,225 @@
+# Copyright (C) 2018 Free Software Foundation, Inc.
+#
+# This file is part of DejaGnu.
+#
+# DejaGnu is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# DejaGnu is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DejaGnu; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA.
+
+# This file was written by Jacob Bachmeyer.
+
+# This library provides functions for generating subset-sum-distinct sets
+# using a construction published by Tom Bohman in:
+# T. Bohman, A construction for sets of integers with distinct subset sums,
+# The Electronic. Journal of Combinatorics 5 (1998) /#R3
+# <URL:http://www.combinatorics.org/Volume_5/PDF/v5i1r3.pdf>,
+# retrieved 2018-12-28 SHA-1 1c35035427b3406a44f7290f13ec8fbc3d105041
+namespace eval ::math_utils::Bohman_SSD {
+
+ # b_n(i)
+ proc b { n i } {
+ if { $n <= 1 } { error "invalid parameter n: $n" }
+ if { $i <= 2*$n } { error "invalid parameter i: $i" }
+
+ if { $i >= 2*$n + 4 } {
+ return [expr { round(sqrt(2*($i + 2 - 2*$n))) }]
+ } elseif { $i == 2*$n + 3 } {
+ return [expr { $n + 2 }]
+ } else { # $i == 2*$n + 1 || $i == 2*$n + 2
+ return [expr { $n + 1 }]
+ }
+ }
+
+ variable d_memo
+ array unset d_memo
+ array set d_memo {}
+
+ # d_n(i)
+ proc d { n i } {
+ variable d_memo
+ if { [info exists d_memo($n,$i)] } { return $d_memo($n,$i) }
+
+ if { $n <= 1 } { error "invalid parameter n: $n" }
+ if { $i < 1 } { error "invalid parameter i: $i" }
+
+ if { $i == $n } {
+ return 1
+ } elseif { $i < $n } {
+ set j [expr { $n - $i }]
+ return [expr { 2 * round(pow(4,($j - 1))) }]
+ } elseif { $i <= 2*$n } {
+ set j [expr { $i - $n }]
+ return [expr { round(pow(4,($j - 1))) }]
+ } else { # $i > 2*$n
+ set sum 0
+ for { set j [expr { $i - [b $n $i] }] } { $j < $i } { incr j } {
+ incr sum [d $n $j]
+ }
+ set d_memo($n,$i) $sum
+ return $sum
+ }
+ }
+
+ # S_{n,m} returns list
+ proc S { n m } {
+ if { $n <= 1 } { error "invalid parameter n: $n" }
+ if { $m < 2*$n } { error "invalid parameter m: $m" }
+
+ set dv [list]
+ for { set i 1 } { $i <= $m } { incr i } { lappend dv [d $n $i] }
+ set sum 0
+ foreach d $dv { incr sum $d }
+ set result [list]
+ foreach d $dv {
+ lappend result $sum
+ incr sum -$d
+ }
+ return $result
+ }
+
+ # b'_n(i)
+ proc bp { n i } {
+ if { $n < 1 } { error "invalid parameter n: $n" }
+ if { $i <= 2*$n + 1 } { error "invalid parameter i: $i" }
+
+ if { $i >= 2*$n + 5 } {
+ return [expr { round(sqrt(2*($i + 1 - 2*$n))) }]
+ } elseif { $i == 2*$n + 2 } {
+ return [expr { $n + 1 }]
+ } else { # $i == 2*$n + 3 || $i == 2*$n + 4
+ return [expr { $n + 2 }]
+ }
+ }
+
+ variable dp_memo
+ array unset dp_memo
+ array set dp_memo {}
+
+ # d'_n(i)
+ proc dp { n i } {
+ variable dp_memo
+ if { [info exists dp_memo($n,$i)] } { return $dp_memo($n,$i) }
+
+ if { $n < 1 } { error "invalid parameter n: $n" }
+ if { $i < 1 } { error "invalid parameter i: $i" }
+
+ if { $i == $n + 1 } {
+ return 1
+ } elseif { $i < $n + 1 } {
+ set j [expr { $n + 1 - $i }]
+ return [expr { round(pow(4,($j - 1))) }]
+ } elseif { $i <= 2*$n + 1 } {
+ set j [expr { $i - $n - 1 }]
+ return [expr { 2 * round(pow(4,($j - 1))) }]
+ } else { # $i > 2*$n + 1
+ set sum 0
+ for { set j [expr { $i - [bp $n $i] }] } { $j < $i } { incr j } {
+ incr sum [dp $n $j]
+ }
+ set dp_memo($n,$i) $sum
+ return $sum
+ }
+ }
+ # The example for d'_3 in the paper is wrong starting at i=11. The
+ # paper says that it is 200, but it is actually 300.
+
+ # S'_{n,m} returns list
+ proc Sp { n m } {
+ if { $n < 1 } { error "invalid parameter n: $n" }
+ if { $m < 2*$n + 1 } { error "invalid parameter m: $m" }
+
+ set dv [list]
+ for { set i 1 } { $i <= $m } { incr i } { lappend dv [dp $n $i] }
+ set sum 0
+ foreach d $dv { incr sum $d }
+ set result [list]
+ foreach d $dv {
+ lappend result $sum
+ incr sum -$d
+ }
+ return $result
+ }
+
+ # Given a list of numbers, verify that all sums of all subsets are in
+ # fact unique.
+ #
+ # This is a brute force search and not based on Bohman's paper. This
+ # quickly becomes impractical for large lists, requiring inordinate
+ # amounts of both time and space.
+ proc check { base } {
+ set bound [expr { int(pow(2,[llength $base])) }]
+ for { set i 0 } { $i < $bound } { incr i } {
+ set R $i
+ set sum 0
+ foreach v $base {
+ if { $R & 1 } { incr sum $v }
+ set R [expr { $R >> 1 }]
+ }
+ if { [info exists output($sum)] } {
+ # emit counterexample
+ set cexl [list]
+ set R $i
+ foreach v $base {
+ if { $R & 1 } { lappend cexl $v }
+ set R [expr { $R >> 1 }]
+ }
+ set cex [join $cexl "+"]
+ append cex "=" $sum "="
+ set cexl [list]
+ set R $output($sum)
+ foreach v $base {
+ if { $R & 1 } { lappend cexl $v }
+ set R [expr { $R >> 1 }]
+ }
+ append cex [join $cexl "+"]
+ error "list is not subset-sum-distinct: $cex"
+ }
+ set output($sum) $i
+ }
+ return 1
+ }
+
+ # Given a list of numbers and a sum of a subset of that list, find a
+ # subset that produces the given sum. If the list of numbers is
+ # subset-sum-distinct, this will return the unique solution.
+ # Otherwise, an unspecified solution is returned. If the sum is not
+ # actually a sum of a subset of the list, an empty list is returned.
+ #
+ # This is a brute force search and not based on Bohman's paper. This
+ # requires constant space, but quickly becomes impractical for large
+ # lists, requiring inordinate time to complete.
+ proc summands { base goal } {
+ set bound [expr { int(pow(2,[llength $base])) }]
+ for { set i 0 } { $i < $bound } { incr i } {
+ set R $i
+ set sum 0
+ foreach v $base {
+ if { $R & 1 } { incr sum $v }
+ set R [expr { $R >> 1 }]
+ }
+ if { $sum == $goal } {
+ set resl [list]
+ set R $i
+ foreach v $base {
+ if { $R & 1 } { lappend resl $v }
+ set R [expr { $R >> 1 }]
+ }
+ return $resl
+ }
+ }
+ return [list]
+ }
+
+}
+
+#EOF
diff --git a/testsuite/lib/report-card.exp b/testsuite/lib/report-card.exp
new file mode 100644
index 0000000..7fa8838
--- /dev/null
+++ b/testsuite/lib/report-card.exp
@@ -0,0 +1,39 @@
+# Copyright (C) 2018 Free Software Foundation, Inc.
+#
+# This file is part of DejaGnu.
+#
+# DejaGnu is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# DejaGnu is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DejaGnu; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA.
+
+# This file was written by Jacob Bachmeyer.
+
+# Ensure that the dejagnu(1) launcher is available for testing.
+if { ![info exists LAUNCHER] } {
+ set LAUNCHER \
+ [file join [file dirname [testsuite file -source -top]] dejagnu]
+}
+verbose "Using LAUNCHER $LAUNCHER" 2
+
+if { [which $LAUNCHER] == 0 } {
+ perror "Can't find LAUNCHER = $LAUNCHER"
+ exit 2
+}
+
+# stub: dejagnu-report-card is non-interactive
+proc report-card_exit {} {}
+
+# stub: dejagnu-report-card does not have a separate version number
+proc report-card_version {} {}
+
+#EOF
diff --git a/testsuite/report-card.all/onetest.exp b/testsuite/report-card.all/onetest.exp
new file mode 100644
index 0000000..b2ae814
--- /dev/null
+++ b/testsuite/report-card.all/onetest.exp
@@ -0,0 +1,209 @@
+# Copyright (C) 2018 Free Software Foundation, Inc.
+#
+# This file is part of DejaGnu.
+#
+# DejaGnu is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# DejaGnu is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DejaGnu; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA.
+
+# This file was written by Jacob Bachmeyer.
+
+set header_column_names { PASS FAIL ?PASS ?FAIL UNSUP UNRES UNTEST }
+set separator_count 0
+set re_digit_columns {}
+for { set i 0 } { $i < 7 } { incr i } {
+ append re_digit_columns {[[:space:]]+([[:digit:]]+)}
+}
+
+set test_names { pass fail kpass kfail xpass xfail
+ unsupported unresolved untested
+ note warning error }
+set test_results { PASS FAIL KPASS KFAIL XPASS XFAIL
+ UNSUPPORTED UNRESOLVED UNTESTED
+ NOTE WARNING ERROR }
+
+foreach name $test_names result $test_results {
+ set fd [open [testsuite file -object -test onetest one-${name}.sum] w]
+ puts $fd "${result}: one test"
+ close $fd
+}
+
+set stty_init { -onlcr -onlret }
+
+spawn /bin/sh -c \
+ "cd [testsuite file -object -test onetest]\
+ && exec $LAUNCHER report-card"
+
+# check header
+expect {
+ -re {^[[:space:]]+_+[\r\n]+} {
+ # discard initial header line
+ exp_continue
+ }
+ -re {^[[:space:]]+/([^\r\n]*)[\r\n]+} {
+ # check column labels
+ foreach want $header_column_names have $expect_out(1,string) {
+ if { $have eq $want } {
+ pass "header item $want"
+ } else {
+ fail "header item $want"
+ }
+ }
+ exp_continue
+ }
+ -re {^[[:space:]]+\|-+[\r\n]+} {
+ incr separator_count
+ }
+}
+
+# check results
+array unset scoreboard
+array set scoreboard {
+ pass 0 fail 0 kpass 0 kfail 0 xpass 0 xfail 0
+ unsupported 0 unresolved 0 untested 0
+ note 0 warning 0 error 0
+}
+array unset column_subexp_map
+array set column_subexp_map {
+ pass 2 fail 3 kpass 4 kfail 5 xpass 4 xfail 5
+ unsupported 6 unresolved 7 untested 8
+ note 0 warning 9 error 9
+}
+set re_table_row {^[[:space:]]*one-([[:alpha:]]+)[[:space:]]+\|}
+append re_table_row $re_digit_columns
+append re_table_row {((?:[[:space:]]+![EW]!)*)[\r\n]+}
+expect {
+ -re $re_table_row {
+ for { set i 2 } { $i < 9 } { incr i } {
+ if { $expect_out($i,string)\
+ == ( $i == $column_subexp_map($expect_out(1,string))\
+ ? 1 : 0 ) } {
+ incr scoreboard($expect_out(1,string))
+ } else {
+ incr scoreboard($expect_out(1,string)) -1
+ }
+ }
+ set have_warning_tag [string match "*!W!*" $expect_out(9,string)]
+ set have_error_tag [string match "*!E!*" $expect_out(9,string)]
+ if { $column_subexp_map($expect_out(1,string)) == 9 } {
+ # testing an after-row tag
+ switch -- $expect_out(1,string) {
+ warning {
+ incr scoreboard(warning) \
+ [expr { $have_warning_tag ? 1 : -1 }]
+ incr scoreboard(error) \
+ [expr { $have_error_tag ? -1 : 1 }]
+ }
+ error {
+ incr scoreboard(warning) \
+ [expr { $have_warning_tag ? -1 : 1 }]
+ incr scoreboard(error) \
+ [expr { $have_error_tag ? 1 : -1 }]
+ }
+ default { error "unknown tag $expect_out(1,string)" }
+ }
+ } else {
+ incr scoreboard(warning) [expr { $have_warning_tag ? -1 : 1 }]
+ incr scoreboard(error) [expr { $have_error_tag ? -1 : 1 }]
+ }
+ exp_continue
+ }
+ -re {^[[:space:]]+\|-+[\r\n]+} {
+ incr separator_count
+ }
+}
+foreach result [lsort [array names scoreboard]] {
+ verbose -log "scoreboard($result) = $scoreboard($result)"
+}
+foreach result [array names scoreboard] {
+ if { $scoreboard($result) == ( 7 + ( $column_subexp_map($result) == 9\
+ ? [llength $test_names] : 0 ) ) } {
+ pass "count result $result"
+ } else {
+ fail "count result $result"
+ }
+}
+
+# check totals
+set column_totals { pad 1 1 2 2 1 1 1 }
+set re_totals_row {^[[:space:]]+\|}
+append re_totals_row $re_digit_columns
+append re_totals_row {[\r\n]+}
+set totals_matched 0
+expect {
+ -re $re_totals_row {
+ for { set i 1 } { $i < 8 } { incr i } {
+ if { [lindex $column_totals $i] == $expect_out($i,string) } {
+ incr totals_matched
+ }
+ }
+ exp_continue
+ }
+ -re {^[[:space:]]+\|-+[\r\n]+} {
+ incr separator_count
+ }
+ -re {^[[:space:]]+\\_+[\r\n]+} {
+ # all done
+ }
+}
+
+if { $totals_matched == 7 } {
+ pass "expected total count"
+} else {
+ fail "expected total count"
+}
+
+if { $separator_count == 2 } {
+ pass "expected separator lines"
+} else {
+ fail "expected separator lines"
+}
+
+# Ensure that totals map correctly by reading each file one at a time
+foreach name $test_names {
+ set separator_count 0
+ spawn /bin/sh -c \
+ "cd [testsuite file -object -test onetest]\
+ && exec $LAUNCHER report-card one-${name}.sum"
+ # skip header
+ expect {
+ -re {^[[:space:]]+_+[\r\n]+} { exp_continue }
+ -re {^[[:space:]]+/([^\r\n]*)[\r\n]+} { exp_continue }
+ -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count }
+ }
+ # capture the item line
+ expect -re {^one-[^|]+(\|[[:space:][:digit:]]*)[[:space:]!EW]*[\r\n]+} {
+ regsub {[[:space:]]*$} $expect_out(1,string) "" item_line
+ }
+ # skip the separator
+ expect -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count }
+ # capture the totals line
+ expect -re {^[[:space:]]+(\|[[:space:][:digit:]]*)[\r\n]+} {
+ regsub {[[:space:]]*$} $expect_out(1,string) "" totals_line
+ }
+ # skip the footer
+ expect -re {.+} { exp_continue }
+ # do the item and totals lines match?
+ if { $item_line eq $totals_line } {
+ pass "verify total for $name"
+ } else {
+ fail "verify total for $name"
+ }
+ if { $separator_count == 2 } {
+ pass "expected separator lines for $name"
+ } else {
+ fail "expected separator lines for $name"
+ }
+}
+
+#EOF
diff --git a/testsuite/report-card.all/passes.exp b/testsuite/report-card.all/passes.exp
new file mode 100644
index 0000000..012e9ac
--- /dev/null
+++ b/testsuite/report-card.all/passes.exp
@@ -0,0 +1,276 @@
+# Copyright (C) 2018 Free Software Foundation, Inc.
+#
+# This file is part of DejaGnu.
+#
+# DejaGnu is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# DejaGnu is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DejaGnu; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA.
+
+# This file was written by Jacob Bachmeyer.
+
+load_lib bohman_ssd.exp
+
+set header_column_names { PASS FAIL ?PASS ?FAIL UNSUP UNRES UNTEST }
+set result_column_map {
+ PASS FAIL { KPASS XPASS } { KFAIL XFAIL }
+ UNSUPPORTED UNRESOLVED UNTESTED
+}
+
+set test_results { PASS FAIL KPASS KFAIL XPASS XFAIL
+ UNSUPPORTED UNRESOLVED UNTESTED }
+
+# each entry: { {mode n} { suffix_tag... } { pass... } { { result name }... } }
+array unset tuplemap
+array set tuplemap {
+ basic { {S 3} { a b } { foo bar }
+ { { PASS pass } { FAIL fail } } }
+ kxpass { {S 2} { a b } { foo bar }
+ { { KPASS kpass } { XPASS xpass } } }
+ kxfail { {Sp 2} { a b } { foo bar }
+ { { KFAIL kfail } { XFAIL xfail } } }
+ unresult { {S 2} { a b } { foo bar }
+ { { UNSUPPORTED unsupported }
+ { UNRESOLVED unresolved } { UNTESTED untested } } }
+}
+
+# Given: TUPLES: { { result ... }... }, PASSES: { pass... }
+# Return: Cartesian product TUPLES x PASSES: { { result pass ... }... }
+proc build_tuple_list { tuples passes } {
+ set result [list]
+ foreach cell $tuples {
+ foreach pass $passes {
+ lappend result [linsert $cell 1 $pass]
+ }
+ }
+ return $result
+}
+
+# Given: TUPLES: { { result pass name }... }, MODE: S | Sp, N
+# Return: { { result pass name count }... } where COUNT is from an SSD-set
+proc annotate_tuple_list { tuples mode n } {
+ set m [llength $tuples]
+ set ssd [switch -- $mode {
+ S { ::math_utils::Bohman_SSD::S $n $m }
+ Sp { ::math_utils::Bohman_SSD::Sp $n $m }
+ }]
+ set result [list]
+ foreach cell $tuples ssdterm $ssd {
+ lappend result [linsert $cell end $ssdterm]
+ }
+ return $result
+}
+
+# Given: TUPLES: { { result pass name count }... }; (RESULT,PASS) not unique
+# Return: { { result pass expected_total }... } where (RESULT,PASS) is unique
+proc compute_expected_pass_totals { tuples } {
+ foreach cell $tuples { set count([lrange $cell 0 1]) 0 }
+ foreach cell $tuples { incr count([lrange $cell 0 1]) [lindex $cell 3] }
+ set result [list]
+ foreach name [lsort [array names count]] {
+ lappend result [concat $name $count($name)]
+ }
+ return $result
+}
+
+# Given: TUPLES: { { result pass name count }... }; (RESULT,PASS) not unique
+# Return: { { result expected_grand_total }... }
+proc compute_expected_grand_totals { tuples } {
+ foreach cell $tuples { set count([lindex $cell 0]) 0 }
+ foreach cell $tuples { incr count([lindex $cell 0]) [lindex $cell 3] }
+ set result [list]
+ foreach name [lsort [array names count]] {
+ lappend result [list $name $count($name)]
+ }
+ return $result
+}
+
+# Given: TUPLES: { { result pass ... }... } where (RESULT,PASS) repeats later
+# Return: { { { result pass ... }... }... }; (RESULT,PASS) unique per sublist
+proc split_tuple_list { tuples } {
+ set result [list]
+ set sublist [list]
+ foreach cell $tuples {
+ if { [info exists seen([lrange $cell 0 1])] } {
+ # split here
+ lappend result $sublist
+ set sublist [list]
+ array unset seen
+ }
+ lappend sublist $cell
+ set seen([lrange $cell 0 1]) 1
+ }
+ lappend result $sublist
+ return $result
+}
+
+# TUPLES is: { { result pass name count }... }
+proc write_file { basename tuples } {
+ set fd [open [testsuite file -object -test passes ${basename}.sum] w]
+ set pass {}
+ foreach cell [lsort -index 1 $tuples] {
+ if { $pass ne [lindex $cell 1] } {
+ puts $fd "Running pass `[lindex $cell 1]' ..."
+ set pass [lindex $cell 1]
+ }
+ for { set i 1 } { $i <= [lindex $cell 3] } { incr i } {
+ puts $fd "[lindex $cell 0]: [lindex $cell 1]:\
+ [lindex $cell 2] test ${i}/[lindex $cell 3]"
+ }
+ }
+ close $fd
+}
+
+proc run_multipass_output_test { filetag } {
+ global LAUNCHER
+ global header_column_names
+ global result_column_map
+ global test_results
+ global tuplemap
+
+ set ssdpar [lindex $tuplemap($filetag) 0]
+ set tags [lindex $tuplemap($filetag) 1]
+ set passes [lindex $tuplemap($filetag) 2]
+ set results {}
+ foreach dummy $tags { lappend results [lindex $tuplemap($filetag) 3] }
+ set results [join $results]
+
+ # initialize totals arrays to zero
+ foreach result $test_results { set have_grand_totals($result) 0 }
+ array set want_grand_totals [array get have_grand_totals]
+ foreach cell [build_tuple_list $test_results $passes] {
+ set have_pass_totals([join [lrange $cell 0 1] ","]) 0
+ }
+ array set want_pass_totals [array get have_pass_totals]
+
+ # get the test list
+ set list [build_tuple_list $results $passes]
+ set list [annotate_tuple_list $list [lindex $ssdpar 0] [lindex $ssdpar 1]]
+
+ # compute expected totals
+ # note that this only fills non-zero array positions
+ foreach cell [compute_expected_pass_totals $list] {
+ set want_pass_totals([join [lrange $cell 0 1] ","]) [lindex $cell 2]
+ }
+ array set want_grand_totals [join [compute_expected_grand_totals $list]]
+
+ # write the test data files and store expected per-file counts
+ foreach tag $tags fileset [split_tuple_list $list] {
+ # write test file
+ write_file "${filetag}-${tag}" $fileset
+ # initialize test results for this file
+ foreach result $test_results {
+ foreach pass $passes {
+ set want_file_counts(${filetag}-${tag},$result,$pass) 0
+ set have_file_counts(${filetag}-${tag},$result,$pass) 0
+ }
+ }
+ # store expected results for this file
+ foreach cell $fileset {
+ set want_file_counts(${filetag}-${tag},[join [lrange $cell 0 1] \
+ ","]) [lindex $cell 3]
+ }
+ }
+
+ # run the dejagnu-report-card tool
+ set separator_count 0
+ spawn /bin/sh -c \
+ "cd [testsuite file -object -test passes]\
+ && exec $LAUNCHER report-card ${filetag}-*.sum"
+
+ # skip header
+ expect {
+ -re {^[[:space:]]+_+[\r\n]+} { exp_continue }
+ -re {^[[:space:]]+/([^\r\n]*)[\r\n]+} { exp_continue }
+ -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count }
+ }
+
+ # read individual file lines
+ set re_file_row {^[[:space:]]*}
+ append re_file_row {(} $filetag {-[[:alpha:]]+)[[:space:]]+}
+ append re_file_row {/[[:space:]]+([[:alpha:]]+)[[:space:]]+\|}
+ append re_file_row {[[:space:]]*([[:digit:][:space:]]+)[\r\n]+}
+ expect {
+ -re $re_file_row {
+ foreach column $result_column_map colname $header_column_names \
+ have $expect_out(3,string) {
+ set want 0
+ foreach rs $column {
+ set tmp $expect_out(1,string),$rs,$expect_out(2,string)
+ incr want $want_file_counts($tmp)
+ }
+ if { $have == $want } {
+ pass "count $colname\
+ for pass $expect_out(2,string)\
+ in file $expect_out(1,string)"
+ } else {
+ fail "count $colname\
+ for pass $expect_out(2,string)\
+ in file $expect_out(1,string)"
+ }
+ }
+ exp_continue
+ }
+ -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count }
+ }
+
+ # read pass totals lines
+ set re_pass_row {^[[:space:]]+([[:alpha:]]+)[[:space:]]+\|}
+ append re_pass_row {[[:space:]]*([[:digit:][:space:]]+)[\r\n]+}
+ expect {
+ -re $re_pass_row {
+ foreach column $result_column_map colname $header_column_names \
+ have $expect_out(2,string) {
+ set want 0
+ foreach rs $column {
+ incr want $want_pass_totals($rs,$expect_out(1,string))
+ }
+ if { $have == $want } {
+ pass "total $colname for pass $expect_out(1,string)"
+ } else {
+ fail "total $colname for pass $expect_out(1,string)"
+ }
+ }
+ exp_continue
+ }
+ -re {^[[:space:]]+\|-+[\r\n]+} { incr separator_count }
+ }
+
+ # read grand totals line
+ expect -re {^[[:space:]]+\|[[:space:]]*([[:digit:][:space:]]+)[\r\n]+} {
+ foreach column $result_column_map colname $header_column_names \
+ have $expect_out(1,string) {
+ set want 0
+ foreach rs $column { incr want $want_grand_totals($rs) }
+ if { $have == $want } {
+ pass "grand total $colname"
+ } else {
+ fail "grand total $colname"
+ }
+ }
+ }
+
+ # skip the footer
+ expect -re {.+} { exp_continue }
+
+ if { $separator_count == 3 } {
+ pass "expected separator lines"
+ } else {
+ fail "expected separator lines"
+ }
+}
+
+foreach filetag [lsort [array names tuplemap]] {
+ run_multipass_output_test $filetag
+}
+
+#EOF