From 5e5c006eb713b785c6761c32b907e02abc277f9a Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Sun, 16 Dec 2012 18:28:09 +0000 Subject: tests: test number comes first in 'not ok $count - $message' The old output to say "not ok - 1 messsage" was working by accident only because the test numbers are optional in TAP. Signed-off-by: Adam Spiers Signed-off-by: Junio C Hamano --- t/t0000-basic.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 't/t0000-basic.sh') diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index ae6a3f0777..c6b42deef9 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -167,13 +167,13 @@ test_expect_success 'tests clean up even on failures' " ! test -s err && ! test -f \"trash directory.failing-cleanup/clean-after-failure\" && sed -e 's/Z$//' -e 's/^> //' >expect <<-\\EOF && - > not ok - 1 tests clean up even after a failure + > not ok 1 - tests clean up even after a failure > # Z > # touch clean-after-failure && > # test_when_finished rm clean-after-failure && > # (exit 1) > # Z - > not ok - 2 failure to clean up causes the test to fail + > not ok 2 - failure to clean up causes the test to fail > # Z > # test_when_finished \"(exit 2)\" > # Z -- cgit 1.2.3-korg From 565b6fa87bb0554eb7f1b8ec004df93e55d177a9 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Sun, 16 Dec 2012 18:28:13 +0000 Subject: tests: refactor mechanics of testing in a sub test-lib This will allow us to test the test framework more thoroughly without disrupting the top-level test metrics. Signed-off-by: Adam Spiers Signed-off-by: Junio C Hamano --- t/t0000-basic.sh | 85 ++++++++++++++++++++++++++------------------------------ 1 file changed, 40 insertions(+), 45 deletions(-) (limited to 't/t0000-basic.sh') diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index c6b42deef9..d0f46e8b25 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -55,39 +55,53 @@ test_expect_failure 'pretend we have a known breakage' ' false ' -test_expect_success 'pretend we have fixed a known breakage (run in sub test-lib)' " - mkdir passing-todo && - (cd passing-todo && - cat >passing-todo.sh <<-EOF && - #!$SHELL_PATH - - test_description='A passing TODO test - - This is run in a sub test-lib so that we do not get incorrect - passing metrics - ' - - # Point to the t/test-lib.sh, which isn't in ../ as usual - TEST_DIRECTORY=\"$TEST_DIRECTORY\" - . \"\$TEST_DIRECTORY\"/test-lib.sh +run_sub_test_lib_test () { + name="$1" descr="$2" # stdin is the body of the test code + mkdir "$name" && + ( + cd "$name" && + cat >"$name.sh" <<-EOF && + #!$SHELL_PATH + + test_description='$descr (run in sub test-lib) + + This is run in a sub test-lib so that we do not get incorrect + passing metrics + ' + + # Point to the t/test-lib.sh, which isn't in ../ as usual + . "\$TEST_DIRECTORY"/test-lib.sh + EOF + cat >>"$name.sh" && + chmod +x "$name.sh" && + export TEST_DIRECTORY && + ./"$name.sh" >out 2>err + ) +} - test_expect_failure 'pretend we have fixed a known breakage' ' - : - ' +check_sub_test_lib_test () { + name="$1" # stdin is the expected output from the test + ( + cd "$name" && + ! test -s err && + sed -e 's/^> //' -e 's/Z$//' >expect && + test_cmp expect out + ) +} +test_expect_success 'pretend we have fixed a known breakage' " + run_sub_test_lib_test passing-todo 'A passing TODO test' <<-\\EOF && + test_expect_failure 'pretend we have fixed a known breakage' 'true' test_done EOF - chmod +x passing-todo.sh && - ./passing-todo.sh >out 2>err && - ! test -s err && - sed -e 's/^> //' >expect <<-\\EOF && + check_sub_test_lib_test passing-todo <<-\\EOF > ok 1 - pretend we have fixed a known breakage # TODO known breakage > # fixed 1 known breakage(s) > # passed all 1 test(s) > 1..1 EOF - test_cmp expect out) " + test_set_prereq HAVEIT haveit=no test_expect_success HAVEIT 'test runs if prerequisite is satisfied' ' @@ -137,19 +151,8 @@ then fi test_expect_success 'tests clean up even on failures' " - mkdir failing-cleanup && - ( - cd failing-cleanup && - - cat >failing-cleanup.sh <<-EOF && - #!$SHELL_PATH - - test_description='Failing tests with cleanup commands' - - # Point to the t/test-lib.sh, which isn't in ../ as usual - TEST_DIRECTORY=\"$TEST_DIRECTORY\" - . \"\$TEST_DIRECTORY\"/test-lib.sh - + test_must_fail run_sub_test_lib_test \ + failing-cleanup 'Failing tests with cleanup commands' <<-\\EOF && test_expect_success 'tests clean up even after a failure' ' touch clean-after-failure && test_when_finished rm clean-after-failure && @@ -159,14 +162,8 @@ test_expect_success 'tests clean up even on failures' " test_when_finished \"(exit 2)\" ' test_done - EOF - - chmod +x failing-cleanup.sh && - test_must_fail ./failing-cleanup.sh >out 2>err && - ! test -s err && - ! test -f \"trash directory.failing-cleanup/clean-after-failure\" && - sed -e 's/Z$//' -e 's/^> //' >expect <<-\\EOF && + check_sub_test_lib_test failing-cleanup <<-\\EOF > not ok 1 - tests clean up even after a failure > # Z > # touch clean-after-failure && @@ -180,8 +177,6 @@ test_expect_success 'tests clean up even on failures' " > # failed 2 among 2 test(s) > 1..2 EOF - test_cmp expect out - ) " ################################################################ -- cgit 1.2.3-korg From 5ebf89e88637fa641ced15edc10be0cf8d5dcbc8 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Sun, 16 Dec 2012 18:28:14 +0000 Subject: tests: test the test framework more thoroughly Add 5 new full test suite runs each with a different number of passing/failing/broken/fixed tests, in order to ensure that the correct exit code and output are generated in each case. As before, these are run in a subdirectory to avoid disrupting the metrics for the parent tests. Signed-off-by: Adam Spiers Signed-off-by: Junio C Hamano --- t/t0000-basic.sh | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) (limited to 't/t0000-basic.sh') diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index d0f46e8b25..384b0aee47 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -89,6 +89,56 @@ check_sub_test_lib_test () { ) } +test_expect_success 'pretend we have a fully passing test suite' " + run_sub_test_lib_test full-pass '3 passing tests' <<-\\EOF && + for i in 1 2 3 + do + test_expect_success \"passing test #\$i\" 'true' + done + test_done + EOF + check_sub_test_lib_test full-pass <<-\\EOF + > ok 1 - passing test #1 + > ok 2 - passing test #2 + > ok 3 - passing test #3 + > # passed all 3 test(s) + > 1..3 + EOF +" + +test_expect_success 'pretend we have a partially passing test suite' " + test_must_fail run_sub_test_lib_test \ + partial-pass '2/3 tests passing' <<-\\EOF && + test_expect_success 'passing test #1' 'true' + test_expect_success 'failing test #2' 'false' + test_expect_success 'passing test #3' 'true' + test_done + EOF + check_sub_test_lib_test partial-pass <<-\\EOF + > ok 1 - passing test #1 + > not ok 2 - failing test #2 + # false + > ok 3 - passing test #3 + > # failed 1 among 3 test(s) + > 1..3 + EOF +" + +test_expect_success 'pretend we have a known breakage' " + run_sub_test_lib_test failing-todo 'A failing TODO test' <<-\\EOF && + test_expect_success 'passing test' 'true' + test_expect_failure 'pretend we have a known breakage' 'false' + test_done + EOF + check_sub_test_lib_test failing-todo <<-\\EOF + > ok 1 - passing test + > not ok 2 - pretend we have a known breakage # TODO known breakage + > # still have 1 known breakage(s) + > # passed all remaining 1 test(s) + > 1..2 + EOF +" + test_expect_success 'pretend we have fixed a known breakage' " run_sub_test_lib_test passing-todo 'A passing TODO test' <<-\\EOF && test_expect_failure 'pretend we have fixed a known breakage' 'true' @@ -102,6 +152,61 @@ test_expect_success 'pretend we have fixed a known breakage' " EOF " +test_expect_success 'pretend we have a pass, fail, and known breakage' " + test_must_fail run_sub_test_lib_test \ + mixed-results1 'mixed results #1' <<-\\EOF && + test_expect_success 'passing test' 'true' + test_expect_success 'failing test' 'false' + test_expect_failure 'pretend we have a known breakage' 'false' + test_done + EOF + check_sub_test_lib_test mixed-results1 <<-\\EOF + > ok 1 - passing test + > not ok 2 - failing test + > # false + > not ok 3 - pretend we have a known breakage # TODO known breakage + > # still have 1 known breakage(s) + > # failed 1 among remaining 2 test(s) + > 1..3 + EOF +" + +test_expect_success 'pretend we have a mix of all possible results' " + test_must_fail run_sub_test_lib_test \ + mixed-results2 'mixed results #2' <<-\\EOF && + test_expect_success 'passing test' 'true' + test_expect_success 'passing test' 'true' + test_expect_success 'passing test' 'true' + test_expect_success 'passing test' 'true' + test_expect_success 'failing test' 'false' + test_expect_success 'failing test' 'false' + test_expect_success 'failing test' 'false' + test_expect_failure 'pretend we have a known breakage' 'false' + test_expect_failure 'pretend we have a known breakage' 'false' + test_expect_failure 'pretend we have fixed a known breakage' 'true' + test_done + EOF + check_sub_test_lib_test mixed-results2 <<-\\EOF + > ok 1 - passing test + > ok 2 - passing test + > ok 3 - passing test + > ok 4 - passing test + > not ok 5 - failing test + > # false + > not ok 6 - failing test + > # false + > not ok 7 - failing test + > # false + > not ok 8 - pretend we have a known breakage # TODO known breakage + > not ok 9 - pretend we have a known breakage # TODO known breakage + > ok 10 - pretend we have fixed a known breakage # TODO known breakage + > # fixed 1 known breakage(s) + > # still have 2 known breakage(s) + > # failed 3 among remaining 8 test(s) + > 1..10 + EOF +" + test_set_prereq HAVEIT haveit=no test_expect_success HAVEIT 'test runs if prerequisite is satisfied' ' -- cgit 1.2.3-korg From b73d9a2363beedd59c212b5aa6b9db2977c38b2b Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Sun, 16 Dec 2012 18:28:15 +0000 Subject: tests: paint unexpectedly fixed known breakages in bold red Change color of unexpectedly fixed known breakages to bold red. An unexpectedly passing test indicates that the test code is somehow broken or out of sync with the code it is testing. Either way this is an error which is potentially as bad as a failing test, and as such is no longer portrayed as a pass in the output. Signed-off-by: Adam Spiers Signed-off-by: Junio C Hamano --- t/t0000-basic.sh | 30 ++++++++++++++++++++++++------ t/test-lib.sh | 13 +++++++++---- 2 files changed, 33 insertions(+), 10 deletions(-) (limited to 't/t0000-basic.sh') diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index 384b0aee47..8973d2ccc8 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -145,13 +145,31 @@ test_expect_success 'pretend we have fixed a known breakage' " test_done EOF check_sub_test_lib_test passing-todo <<-\\EOF - > ok 1 - pretend we have fixed a known breakage # TODO known breakage - > # fixed 1 known breakage(s) - > # passed all 1 test(s) + > ok 1 - pretend we have fixed a known breakage # TODO known breakage vanished + > # 1 known breakage(s) vanished; please update test(s) > 1..1 EOF " +test_expect_success 'pretend we have fixed one of two known breakages (run in sub test-lib)' " + run_sub_test_lib_test partially-passing-todos \ + '2 TODO tests, one passing' <<-\\EOF && + test_expect_failure 'pretend we have a known breakage' 'false' + test_expect_success 'pretend we have a passing test' 'true' + test_expect_failure 'pretend we have fixed another known breakage' 'true' + test_done + EOF + check_sub_test_lib_test partially-passing-todos <<-\\EOF + > not ok 1 - pretend we have a known breakage # TODO known breakage + > ok 2 - pretend we have a passing test + > ok 3 - pretend we have fixed another known breakage # TODO known breakage vanished + > # 1 known breakage(s) vanished; please update test(s) + > # still have 1 known breakage(s) + > # passed all remaining 1 test(s) + > 1..3 + EOF +" + test_expect_success 'pretend we have a pass, fail, and known breakage' " test_must_fail run_sub_test_lib_test \ mixed-results1 'mixed results #1' <<-\\EOF && @@ -199,10 +217,10 @@ test_expect_success 'pretend we have a mix of all possible results' " > # false > not ok 8 - pretend we have a known breakage # TODO known breakage > not ok 9 - pretend we have a known breakage # TODO known breakage - > ok 10 - pretend we have fixed a known breakage # TODO known breakage - > # fixed 1 known breakage(s) + > ok 10 - pretend we have fixed a known breakage # TODO known breakage vanished + > # 1 known breakage(s) vanished; please update test(s) > # still have 2 known breakage(s) - > # failed 3 among remaining 8 test(s) + > # failed 3 among remaining 7 test(s) > 1..10 EOF " diff --git a/t/test-lib.sh b/t/test-lib.sh index 8b75c9a275..30a0937dd7 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -278,7 +278,7 @@ test_failure_ () { test_known_broken_ok_ () { test_fixed=$(($test_fixed+1)) - say_color "" "ok $test_count - $@ # TODO known breakage" + say_color error "ok $test_count - $@ # TODO known breakage vanished" } test_known_broken_failure_ () { @@ -373,13 +373,18 @@ test_done () { if test "$test_fixed" != 0 then - say_color pass "# fixed $test_fixed known breakage(s)" + say_color error "# $test_fixed known breakage(s) vanished; please update test(s)" fi if test "$test_broken" != 0 then say_color warn "# still have $test_broken known breakage(s)" - msg="remaining $(($test_count-$test_broken)) test(s)" + fi + if test "$test_broken" != 0 || test "$test_fixed" != 0 + then + test_remaining=$(( $test_count - $test_broken - $test_fixed )) + msg="remaining $test_remaining test(s)" else + test_remaining=$test_count msg="$test_count test(s)" fi case "$test_failure" in @@ -393,7 +398,7 @@ test_done () { if test $test_external_has_tap -eq 0 then - if test $test_count -gt 0 + if test $test_remaining -gt 0 then say_color pass "# passed all $msg" fi -- cgit 1.2.3-korg