mirror of
https://github.com/git/git
synced 2024-09-20 17:01:35 +00:00
test-lib: add a --invert-exit-code switch
Add the ability to have those tests that fail return 0, and those tests that succeed return 1. This is useful e.g. to run "--stress" tests on tests that fail 99% of the time on some setup, i.e. to smoke out the flaky run which yielded success. In a subsequent commit a new SANITIZE=leak mode will make use of this. Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
parent
25c2351d85
commit
46fb057aaa
|
@ -578,6 +578,78 @@ test_expect_success 'subtest: --run invalid range end' '
|
||||||
EOF_ERR
|
EOF_ERR
|
||||||
'
|
'
|
||||||
|
|
||||||
|
test_expect_success 'subtest: --invert-exit-code without --immediate' '
|
||||||
|
run_sub_test_lib_test_err full-pass \
|
||||||
|
--invert-exit-code &&
|
||||||
|
check_sub_test_lib_test_err full-pass \
|
||||||
|
<<-\EOF_OUT 3<<-EOF_ERR
|
||||||
|
ok 1 - passing test #1
|
||||||
|
ok 2 - passing test #2
|
||||||
|
ok 3 - passing test #3
|
||||||
|
# passed all 3 test(s)
|
||||||
|
1..3
|
||||||
|
# faking up non-zero exit with --invert-exit-code
|
||||||
|
EOF_OUT
|
||||||
|
EOF_ERR
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'subtest: --invert-exit-code with --immediate: all passed' '
|
||||||
|
run_sub_test_lib_test_err full-pass \
|
||||||
|
--invert-exit-code --immediate &&
|
||||||
|
check_sub_test_lib_test_err full-pass \
|
||||||
|
<<-\EOF_OUT 3<<-EOF_ERR
|
||||||
|
ok 1 - passing test #1
|
||||||
|
ok 2 - passing test #2
|
||||||
|
ok 3 - passing test #3
|
||||||
|
# passed all 3 test(s)
|
||||||
|
1..3
|
||||||
|
# faking up non-zero exit with --invert-exit-code
|
||||||
|
EOF_OUT
|
||||||
|
EOF_ERR
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'subtest: --invert-exit-code without --immediate: partial pass' '
|
||||||
|
run_sub_test_lib_test partial-pass \
|
||||||
|
--invert-exit-code &&
|
||||||
|
check_sub_test_lib_test partial-pass <<-\EOF
|
||||||
|
ok 1 - passing test #1
|
||||||
|
not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
|
||||||
|
# false
|
||||||
|
ok 3 - passing test #3
|
||||||
|
# failed 1 among 3 test(s)
|
||||||
|
1..3
|
||||||
|
# faked up failures as TODO & now exiting with 0 due to --invert-exit-code
|
||||||
|
EOF
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'subtest: --invert-exit-code with --immediate: partial pass' '
|
||||||
|
run_sub_test_lib_test partial-pass \
|
||||||
|
--invert-exit-code --immediate &&
|
||||||
|
check_sub_test_lib_test partial-pass \
|
||||||
|
<<-\EOF_OUT 3<<-EOF_ERR
|
||||||
|
ok 1 - passing test #1
|
||||||
|
not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
|
||||||
|
# false
|
||||||
|
1..2
|
||||||
|
# faked up failures as TODO & now exiting with 0 due to --invert-exit-code
|
||||||
|
EOF_OUT
|
||||||
|
EOF_ERR
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'subtest: --invert-exit-code --immediate: got a failure' '
|
||||||
|
run_sub_test_lib_test partial-pass \
|
||||||
|
--invert-exit-code --immediate &&
|
||||||
|
check_sub_test_lib_test_err partial-pass \
|
||||||
|
<<-\EOF_OUT 3<<-EOF_ERR
|
||||||
|
ok 1 - passing test #1
|
||||||
|
not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
|
||||||
|
# false
|
||||||
|
1..2
|
||||||
|
# faked up failures as TODO & now exiting with 0 due to --invert-exit-code
|
||||||
|
EOF_OUT
|
||||||
|
EOF_ERR
|
||||||
|
'
|
||||||
|
|
||||||
test_expect_success 'subtest: tests respect prerequisites' '
|
test_expect_success 'subtest: tests respect prerequisites' '
|
||||||
write_and_run_sub_test_lib_test prereqs <<-\EOF &&
|
write_and_run_sub_test_lib_test prereqs <<-\EOF &&
|
||||||
|
|
||||||
|
|
|
@ -238,6 +238,9 @@ parse_option () {
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
;;
|
;;
|
||||||
|
--invert-exit-code)
|
||||||
|
invert_exit_code=t
|
||||||
|
;;
|
||||||
*)
|
*)
|
||||||
echo "error: unknown test option '$opt'" >&2; exit 1 ;;
|
echo "error: unknown test option '$opt'" >&2; exit 1 ;;
|
||||||
esac
|
esac
|
||||||
|
@ -788,15 +791,31 @@ test_ok_ () {
|
||||||
finalize_test_case_output ok "$@"
|
finalize_test_case_output ok "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_invert_exit_code_failure_end_blurb () {
|
||||||
|
say_color warn "# faked up failures as TODO & now exiting with 0 due to --invert-exit-code"
|
||||||
|
}
|
||||||
|
|
||||||
test_failure_ () {
|
test_failure_ () {
|
||||||
failure_label=$1
|
failure_label=$1
|
||||||
test_failure=$(($test_failure + 1))
|
test_failure=$(($test_failure + 1))
|
||||||
say_color error "not ok $test_count - $1"
|
local pfx=""
|
||||||
|
if test -n "$invert_exit_code" # && test -n "$HARNESS_ACTIVE"
|
||||||
|
then
|
||||||
|
pfx="# TODO induced breakage (--invert-exit-code):"
|
||||||
|
fi
|
||||||
|
say_color error "not ok $test_count - ${pfx:+$pfx }$1"
|
||||||
shift
|
shift
|
||||||
printf '%s\n' "$*" | sed -e 's/^/# /'
|
printf '%s\n' "$*" | sed -e 's/^/# /'
|
||||||
if test -n "$immediate"
|
if test -n "$immediate"
|
||||||
then
|
then
|
||||||
say_color error "1..$test_count"
|
say_color error "1..$test_count"
|
||||||
|
if test -n "$invert_exit_code"
|
||||||
|
then
|
||||||
|
finalize_test_output
|
||||||
|
_invert_exit_code_failure_end_blurb
|
||||||
|
GIT_EXIT_OK=t
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
_error_exit
|
_error_exit
|
||||||
fi
|
fi
|
||||||
finalize_test_case_output failure "$failure_label" "$@"
|
finalize_test_case_output failure "$failure_label" "$@"
|
||||||
|
@ -1229,7 +1248,14 @@ test_done () {
|
||||||
esac
|
esac
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if test -z "$debug" && test -n "$remove_trash"
|
if test -n "$stress" && test -n "$invert_exit_code"
|
||||||
|
then
|
||||||
|
# We're about to move our "$TRASH_DIRECTORY"
|
||||||
|
# to "$TRASH_DIRECTORY.stress-failed" if
|
||||||
|
# --stress is combined with
|
||||||
|
# --invert-exit-code.
|
||||||
|
say "with --stress and --invert-exit-code we're not removing '$TRASH_DIRECTORY'"
|
||||||
|
elif test -z "$debug" && test -n "$remove_trash"
|
||||||
then
|
then
|
||||||
test -d "$TRASH_DIRECTORY" ||
|
test -d "$TRASH_DIRECTORY" ||
|
||||||
error "Tests passed but trash directory already removed before test cleanup; aborting"
|
error "Tests passed but trash directory already removed before test cleanup; aborting"
|
||||||
|
@ -1242,6 +1268,14 @@ test_done () {
|
||||||
} ||
|
} ||
|
||||||
error "Tests passed but test cleanup failed; aborting"
|
error "Tests passed but test cleanup failed; aborting"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if test -z "$skip_all" && test -n "$invert_exit_code"
|
||||||
|
then
|
||||||
|
say_color warn "# faking up non-zero exit with --invert-exit-code"
|
||||||
|
GIT_EXIT_OK=t
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
test_at_end_hook_
|
test_at_end_hook_
|
||||||
|
|
||||||
GIT_EXIT_OK=t
|
GIT_EXIT_OK=t
|
||||||
|
@ -1254,6 +1288,13 @@ test_done () {
|
||||||
say "1..$test_count"
|
say "1..$test_count"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if test -n "$invert_exit_code"
|
||||||
|
then
|
||||||
|
_invert_exit_code_failure_end_blurb
|
||||||
|
GIT_EXIT_OK=t
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
GIT_EXIT_OK=t
|
GIT_EXIT_OK=t
|
||||||
exit 1 ;;
|
exit 1 ;;
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue