From: Eric S. Raymond Date: Wed, 13 Apr 2022 07:13:53 +0000 (-0400) Subject: Switch normal regression testing to TAP output. X-Git-Tag: 1.11~16 X-Git-Url: https://jxself.org/git/?p=open-adventure.git;a=commitdiff_plain;h=d235313e97a52a7f8d94ff4f3b186be71993e3a1 Switch normal regression testing to TAP output. --- diff --git a/tests/Makefile b/tests/Makefile index 4bf20e5..b3cc2c3 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -19,7 +19,7 @@ TESTLOADS := $(shell ls -1 *.log | sed '/.log/s///' | sort) .PHONY: check coverage clean testlist listcheck savegames buildregress .PHONY: savecheck regress -check: savecheck regress +check: savecheck tapcheck @echo "=== No diff output is good news." @-advent -x 2>/dev/null # Get usage message into coverage tests @-advent -l /dev/null /dev/null @@ -87,22 +87,6 @@ savecheck: savegames @advent -r thousand_saves.adv < pitfall.log > /tmp/coverage_advent_readfail 2>&1 || exit 1 @rm -f /tmp/coverage* -# General regression testing of commands and output; look at the *.log and -# corresponding *.chk files to see which tests this runs. -regress: - @for file in $(TESTLOADS); do \ - $(ECHO) -n " $${file} "; grep '##' $${file}.log || echo ' ## (no description)'; \ - OPTS=`sed -n /#options:/s///p <$${file}.log`; \ - if $(advent) $$OPTS < $${file}.log >/tmp/regress$$$$ 2>&1; \ - then diff --text -u $${file}.chk /tmp/regress$$$$ || exit 1; \ - else echo "*** Nonzero return status on $${file}!"; exit 1; fi \ - done; \ - $(ECHO) " multifile ## Test multiple file arguments" - @(echo "inven" | advent isofoo.log /dev/stdin | diff --text -u multifile.chk -) || exit 1 - rm -f scratch.tmp /tmp/regress$$$$ - -# Steps towards TAP. savecheck is not yet integrated. - # The TAP filter. Only affects presentation of the test suite messages TAPCONSUMER=tapview diff --git a/tests/tapdiffer b/tests/tapdiffer new file mode 100755 index 0000000..b529a39 --- /dev/null +++ b/tests/tapdiffer @@ -0,0 +1,31 @@ +#! /bin/sh +# +# tapdiffer - Render diff between input and checkfile as a TAP report +# +# Usage: tapdiffer LEGEND CHECKFILE +# +# Output is a TAP report, ok if the diff is empty and not ok otherwise. +# A nonempty diff is shipped as a TAP YAML block following "not ok" +# unless QUIET=1 in the environment. +# +legend=$1 +checkfile=$2 + +trap 'rm /tmp/tapdiff$$' EXIT HUP INT QUIT TERM + +if diff --text -u ${checkfile} - >/tmp/tapdiff$$ +then + echo "ok - ${legend}" + exit 0 +else + echo "not ok - ${checkfile}: ${legend}" + if [ ! "${QUIET}" = 1 ] + then + echo " --- |" + sed /dev/null + then + ship_line "$line" + status=2 + break + fi + if expr "$line" : '1\.\.[0-9][0-9]*' >/dev/null >/dev/null + then + if [ "$expect" != "" ] + then + if [ "${testcount}" -gt 0 ] + then + echo "" + fi + ship_line "Cannot have more than one plan line." + echo "${report}" + exit 1 + fi + if expr "$line" : ".* *SKIP" >/dev/null || expr "$line" : ".* *skip" >/dev/null + then + ship_line "$line" + echo "${report}" + exit 1 # Not specified in the standard whether this should exit 1 or 0 + fi + expect=$(expr "$line" : '1\.\.\([0-9][0-9]*\)') + continue + fi + if expr "$line" : "ok" >/dev/null + then + testcount=$((testcount + 1)) + if [ "$expect" = "" ] + then + test_before_plan=yes + else + test_after_plan=yes + fi + if expr "$line" : ".*# *TODO" >/dev/null || expr "$line" : ".*# *todo" >/dev/null + then + ship_char ${TODO_OK} + ship_line "$line" + todocount=$((todocount + 1)) + elif expr "$line" : ".*# *SKIP" >/dev/null || expr "$line" : ".*# *skip" >/dev/null + then + ship_char ${SKIP} + ship_line "$line" + skipcount=$((skipcount + 1)) + else + ship_char ${OK} + fi + state=ok + continue + fi + if expr "$line" : "not ok" >/dev/null + then + testcount=$((testcount + 1)) + if [ "$expect" = "" ] + then + test_before_plan=yes + else + test_after_plan=yes + fi + if expr "$line" : ".*# *SKIP" >/dev/null || expr "$line" : ".*# *skip" >/dev/null + then + ship_char "${SKIP}" + state=ok + skipcount=$((skipcount + 1)) + continue + fi + if expr "$line" : ".*# *TODO" >/dev/null || expr "$line" : ".*# *todo" >/dev/null + then + ship_char ${TODO_NOT_OK} + state=ok + todocount=$((todocount + 1)) + continue + fi + ship_char "${FAIL}" + ship_line "$line" + state=not_ok + failcount=$((failcount + 1)) + status=1 + continue + fi + # shellcheck disable=SC2166 + if [ "${state}" = "yaml" ] + then + ship_line "$line" + if expr "$line" : '[ ]*\.\.\.' >/dev/null + then + state=ok + fi + elif expr "$line" : "[ ]*---" >/dev/null + then + ship_line "$line" + state=yaml + fi +done + +/bin/echo "" + +if [ -z "$expect" ] +then + ship_line "Missing a plan." + status=1 +elif [ "$test_before_plan" = "yes" ] && [ "$test_after_plan" = "yes" ] +then + ship_line "A plan line may only be placed before or after all tests." + status=1 +elif [ "${expect}" -gt "${testcount}" ] +then + ship_line "Expected ${expect} tests but only ${testcount} ran." + status=1 +elif [ "${expect}" -lt "${testcount}" ] +then + ship_line "Expected ${expect} tests but ${testcount} ran." + status=1 +fi + +report="${report}${testcount} tests, ${failcount} failures" +if [ "$todocount" != 0 ] +then + report="${report}, ${todocount} TODOs" +fi +if [ "$skipcount" != 0 ] +then + report="${report}, ${skipcount} SKIPs" +fi + +echo "${report}." + +exit "${status}" + +# end