git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 5/6] Test the test framework more thoroughly

From
Adam Spiers <git@adamspiers.org>
Date
Sep 19, 2012, 17:15 UTC
Message-ID
<1348074915-19985-6-git-send-email-git@adamspiers.org>
In-Reply-To
<1348074915-19985-1-git-send-email-git@adamspiers.org>

Add 5 new full test suite runs each with a different number of passing/failing/broken/fixed tests, in order to ensure that the correct exit code and output are generated in each case. As before, these are run in a subdirectory in order to disrupt the metrics for the parent tests.

Signed-off-by: Adam Spiers <git@adamspiers.org>
---
 t/t0000-basic.sh | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh
index 662cd2f..644cc2c 100755
--- a/t/t0000-basic.sh
+++ b/t/t0000-basic.sh
@@ -77,6 +77,15 @@ run_sub_test_lib_test () {
 	./$name.sh >out 2>err)
 }
 
+run_sub_test_lib_test_expecting_failures () {
+	if run_sub_test_lib_test "$@"; then
+		echo 'sub test-lib run should have failed'
+		return 1
+	else
+		return 0
+	fi
+}
+
 check_sub_test_lib_test () {
 	name="$1" # stdin is test's expected stdout
 	(cd $name &&
@@ -85,6 +94,54 @@ check_sub_test_lib_test () {
 	test_cmp expect out)
 }
 
+test_expect_success 'pretend we have a fully passing test suite' "
+	run_sub_test_lib_test full-pass '3 passing tests' <<-EOF &&
+	for i in 1 2 3; do
+		test_expect_success \"passing test #\\\$i\" 'true'
+	done
+	test_done
+	EOF
+	check_sub_test_lib_test full-pass <<-EOF
+	> ok 1 - passing test #1
+	> ok 2 - passing test #2
+	> ok 3 - passing test #3
+	> # passed all 3 test(s)
+	> 1..3
+	EOF
+"
+
+test_expect_success 'pretend we have a partially passing test suite' "
+	run_sub_test_lib_test_expecting_failures partial-pass '2/3 tests passing' <<-EOF &&
+	test_expect_success 'passing test #1' 'true'
+	test_expect_success 'failing test #2' 'false'
+	test_expect_success 'passing test #3' 'true'
+	test_done
+	EOF
+	check_sub_test_lib_test partial-pass <<-EOF
+	> ok 1 - passing test #1
+	> not ok 2 - failing test #2
+	#	false
+	> ok 3 - passing test #3
+	> # failed 1 among 3 test(s)
+	> 1..3
+	EOF
+"
+
+test_expect_success 'pretend we have a known breakage' "
+	run_sub_test_lib_test failing-todo 'A failing TODO test' <<-EOF &&
+	test_expect_success 'passing test' 'true'
+	test_expect_failure 'pretend we have a known breakage' 'false'
+	test_done
+	EOF
+	check_sub_test_lib_test failing-todo <<-EOF
+	> ok 1 - passing test
+	> not ok 2 - pretend we have a known breakage # TODO known breakage
+	> # still have 1 known breakage(s)
+	> # passed all remaining 1 test(s)
+	> 1..2
+	EOF
+"
+
 test_expect_success 'pretend we have fixed a known breakage' "
 	run_sub_test_lib_test passing-todo 'A passing TODO test' <<-EOF &&
 	test_expect_failure 'pretend we have fixed a known breakage' 'true'
@@ -98,6 +155,59 @@ test_expect_success 'pretend we have fixed a known breakage' "
 	EOF
 "
 
+test_expect_success 'pretend we have a pass, fail, and known breakage' "
+	run_sub_test_lib_test_expecting_failures mixed-results1 'mixed results #1' <<-EOF &&
+	test_expect_success 'passing test' 'true'
+	test_expect_success 'failing test' 'false'
+	test_expect_failure 'pretend we have a known breakage' 'false'
+	test_done
+	EOF
+	check_sub_test_lib_test mixed-results1 <<-EOF
+	> ok 1 - passing test
+	> not ok 2 - failing test
+	> #	false
+	> not ok 3 - pretend we have a known breakage # TODO known breakage
+	> # still have 1 known breakage(s)
+	> # failed 1 among remaining 2 test(s)
+	> 1..3
+	EOF
+"
+
+test_expect_success 'pretend we have a mix of all possible results' "
+	run_sub_test_lib_test_expecting_failures mixed-results2 'mixed results #2' <<-EOF &&
+	test_expect_success 'passing test' 'true'
+	test_expect_success 'passing test' 'true'
+	test_expect_success 'passing test' 'true'
+	test_expect_success 'passing test' 'true'
+	test_expect_success 'failing test' 'false'
+	test_expect_success 'failing test' 'false'
+	test_expect_success 'failing test' 'false'
+	test_expect_failure 'pretend we have a known breakage' 'false'
+	test_expect_failure 'pretend we have a known breakage' 'false'
+	test_expect_failure 'pretend we have fixed a known breakage' 'true'
+	test_done
+	EOF
+	check_sub_test_lib_test mixed-results2 <<-EOF
+	> ok 1 - passing test
+	> ok 2 - passing test
+	> ok 3 - passing test
+	> ok 4 - passing test
+	> not ok 5 - failing test
+	> #	false
+	> not ok 6 - failing test
+	> #	false
+	> not ok 7 - failing test
+	> #	false
+	> not ok 8 - pretend we have a known breakage # TODO known breakage
+	> not ok 9 - pretend we have a known breakage # TODO known breakage
+	> ok 10 - pretend we have fixed a known breakage # TODO known breakage
+	> # fixed 1 known breakage(s)
+	> # still have 2 known breakage(s)
+	> # failed 3 among remaining 8 test(s)
+	> 1..10
+	EOF
+"
+
 test_set_prereq HAVEIT
 haveit=no
 test_expect_success HAVEIT 'test runs if prerequisite is satisfied' '

-- 
1.7.12.147.g6d168f4
Previous: Adam SpiersNext: Adam Spiers
Message 17 of 19 in “make test output coloring more intuitive”
  1. 0/6 make test output coloring more intuitiveAdam Spiers, Sep 19, 2012
  2. 1/6 Change the color of individual known breakagesAdam Spiers, Sep 19, 2012
  3. 2/6 Make 'not ok $count - $message' consistent with 'ok $count - $message'Adam Spiers, Sep 19, 2012
  4. Jeff KingSep 19, 2012
  5. Junio C HamanoSep 19, 2012
  6. Adam SpiersSep 19, 2012
  7. 3/6 Color skipped tests the same as informational messagesAdam Spiers, Sep 19, 2012
  8. 4/6 Refactor mechanics of testing in a sub test-libAdam Spiers, Sep 19, 2012
  9. Jeff KingSep 19, 2012
  10. Adam SpiersSep 19, 2012
  11. 4/6 Refactor mechanics of testing in a sub test-libAdam Spiers, Sep 19, 2012
  12. 5/6 Test the test framework more thoroughlyAdam Spiers, Sep 19, 2012
  13. 6/6 Treat unexpectedly fixed known breakages more seriouslyAdam Spiers, Sep 19, 2012
  14. Junio C HamanoSep 20, 2012
  15. Jeff KingSep 19, 2012
  16. Adam SpiersSep 19, 2012
  17. 5/6 Test the test framework more thoroughlyAdam Spiers, Sep 19, 2012
  18. 6/6 Treat unexpectedly fixed known breakages more seriouslyAdam Spiers, Sep 19, 2012
  19. Jeff KingSep 19, 2012

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.