diff --git a/t/test-lib-junit.sh b/t/test-lib-junit.sh new file mode 100644 index 0000000000..9d55d74d76 --- /dev/null +++ b/t/test-lib-junit.sh @@ -0,0 +1,126 @@ +# Library of functions to format test scripts' output in JUnit XML +# format, to support Git's test suite result to be presented in an +# easily digestible way on Azure Pipelines. +# +# Copyright (c) 2022 Johannes Schindelin +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/ . +# +# The idea is for `test-lib.sh` to source this file when the user asks +# for JUnit XML; these functions will then override (empty) functions +# that are are called at the appropriate times during the test runs. + +start_test_output () { + junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out" + mkdir -p "$junit_xml_dir" + junit_xml_base=${1##*/} + junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml" + junit_attrs="name=\"${junit_xml_base%.sh}\"" + junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \ + date +%Y-%m-%dT%H:%M:%S)\"" + write_junit_xml --truncate "" " " + junit_suite_start=$(test-tool date getnanos) + if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + GIT_TEST_TEE_OFFSET=0 + fi +} + +start_test_case_output () { + junit_start=$(test-tool date getnanos) +} + +finalize_test_case_output () { + test_case_result=$1 + shift + case "$test_case_result" in + ok) + set "$*" + ;; + failure) + junit_insert="" + junit_insert="$junit_insert $(xml_attr_encode \ + "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + test-tool path-utils skip-n-bytes \ + "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET + else + printf '%s\n' "$@" | sed 1d + fi)")" + junit_insert="$junit_insert" + if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + junit_insert="$junit_insert$(xml_attr_encode \ + "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")" + fi + set "$1" " $junit_insert" + ;; + fixed) + set "$* (breakage fixed)" + ;; + broken) + set "$* (known breakage)" + ;; + skip) + message="$(xml_attr_encode "$skipped_reason")" + set "$1" " " + ;; + esac + + junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\"" + shift + junit_attrs="$junit_attrs classname=\"$this_test\"" + junit_attrs="$junit_attrs time=\"$(test-tool \ + date getnanos $junit_start)\"" + write_junit_xml "$(printf '%s\n' \ + " " "$@" " ")" + junit_have_testcase=t +} + +finalize_test_output () { + if test -n "$junit_xml_path" + then + test -n "$junit_have_testcase" || { + junit_start=$(test-tool date getnanos) + write_junit_xml_testcase "all tests skipped" + } + + # adjust the overall time + junit_time=$(test-tool date getnanos $junit_suite_start) + sed -e "s/\(]*/& time=\"$junit_time\"/" \ + -e '/^ *<\/testsuite/d' \ + <"$junit_xml_path" >"$junit_xml_path.new" + mv "$junit_xml_path.new" "$junit_xml_path" + + write_junit_xml " " "" + write_junit_xml= + fi +} + +write_junit_xml () { + case "$1" in + --truncate) + >"$junit_xml_path" + junit_have_testcase= + shift + ;; + esac + printf '%s\n' "$@" >>"$junit_xml_path" +} + +xml_attr_encode () { + printf '%s\n' "$@" | test-tool xml-encode +} diff --git a/t/test-lib.sh b/t/test-lib.sh index f09e8f3efc..bdb11e28ee 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -137,6 +137,12 @@ mark_option_requires_arg () { store_arg_to=$2 } +# These functions can be overridden e.g. to output JUnit XML +start_test_output () { :; } +start_test_case_output () { :; } +finalize_test_case_output () { :; } +finalize_test_output () { :; } + parse_option () { local opt="$1" @@ -196,7 +202,7 @@ parse_option () { tee=t ;; --write-junit-xml) - write_junit_xml=t + . "$TEST_DIRECTORY/test-lib-junit.sh" ;; --stress) stress=t ;; @@ -664,7 +670,7 @@ exec 6<&0 exec 7>&2 _error_exit () { - finalize_junit_xml + finalize_test_output GIT_EXIT_OK=t exit 1 } @@ -774,35 +780,13 @@ trap '{ code=$?; set +x; } 2>/dev/null; exit $code' INT TERM HUP # the test_expect_* functions instead. test_ok_ () { - if test -n "$write_junit_xml" - then - write_junit_xml_testcase "$*" - fi + finalize_test_case_output ok "$@" test_success=$(($test_success + 1)) say_color "" "ok $test_count - $@" } test_failure_ () { - if test -n "$write_junit_xml" - then - junit_insert="" - junit_insert="$junit_insert $(xml_attr_encode \ - "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE" - then - test-tool path-utils skip-n-bytes \ - "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET - else - printf '%s\n' "$@" | sed 1d - fi)")" - junit_insert="$junit_insert" - if test -n "$GIT_TEST_TEE_OUTPUT_FILE" - then - junit_insert="$junit_insert$(xml_attr_encode \ - "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")" - fi - write_junit_xml_testcase "$1" " $junit_insert" - fi + finalize_test_case_output failure "$@" test_failure=$(($test_failure + 1)) say_color error "not ok $test_count - $1" shift @@ -815,19 +799,13 @@ test_failure_ () { } test_known_broken_ok_ () { - if test -n "$write_junit_xml" - then - write_junit_xml_testcase "$* (breakage fixed)" - fi + finalize_test_case_output fixed "$@" test_fixed=$(($test_fixed+1)) say_color error "ok $test_count - $@ # TODO known breakage vanished" } test_known_broken_failure_ () { - if test -n "$write_junit_xml" - then - write_junit_xml_testcase "$* (known breakage)" - fi + finalize_test_case_output broken "$@" test_broken=$(($test_broken+1)) say_color warn "not ok $test_count - $@ # TODO known breakage" } @@ -1104,10 +1082,7 @@ test_start_ () { test_count=$(($test_count+1)) maybe_setup_verbose maybe_setup_valgrind - if test -n "$write_junit_xml" - then - junit_start=$(test-tool date getnanos) - fi + start_test_case_output } test_finish_ () { @@ -1158,12 +1133,7 @@ test_skip () { case "$to_skip" in t) - if test -n "$write_junit_xml" - then - message="$(xml_attr_encode "$skipped_reason")" - write_junit_xml_testcase "$1" \ - " " - fi + finalize_test_case_output skip "$@" say_color skip "ok $test_count # skip $1 ($skipped_reason)" : true @@ -1179,53 +1149,6 @@ test_at_end_hook_ () { : } -write_junit_xml () { - case "$1" in - --truncate) - >"$junit_xml_path" - junit_have_testcase= - shift - ;; - esac - printf '%s\n' "$@" >>"$junit_xml_path" -} - -xml_attr_encode () { - printf '%s\n' "$@" | test-tool xml-encode -} - -write_junit_xml_testcase () { - junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\"" - shift - junit_attrs="$junit_attrs classname=\"$this_test\"" - junit_attrs="$junit_attrs time=\"$(test-tool \ - date getnanos $junit_start)\"" - write_junit_xml "$(printf '%s\n' \ - " " "$@" " ")" - junit_have_testcase=t -} - -finalize_junit_xml () { - if test -n "$write_junit_xml" && test -n "$junit_xml_path" - then - test -n "$junit_have_testcase" || { - junit_start=$(test-tool date getnanos) - write_junit_xml_testcase "all tests skipped" - } - - # adjust the overall time - junit_time=$(test-tool date getnanos $junit_suite_start) - sed -e "s/\(]*/& time=\"$junit_time\"/" \ - -e '/^ *<\/testsuite/d' \ - <"$junit_xml_path" >"$junit_xml_path.new" - mv "$junit_xml_path.new" "$junit_xml_path" - - write_junit_xml " " "" - write_junit_xml= - fi -} - test_atexit_cleanup=: test_atexit_handler () { # In a succeeding test script 'test_atexit_handler' is invoked @@ -1248,7 +1171,7 @@ test_done () { # removed, so the commands can access pidfiles and socket files. test_atexit_handler - finalize_junit_xml + finalize_test_output if test -z "$HARNESS_ACTIVE" then @@ -1539,22 +1462,7 @@ fi # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$TRASH_DIRECTORY" || exit 1 -if test -n "$write_junit_xml" -then - junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out" - mkdir -p "$junit_xml_dir" - junit_xml_base=${0##*/} - junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml" - junit_attrs="name=\"${junit_xml_base%.sh}\"" - junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \ - date +%Y-%m-%dT%H:%M:%S)\"" - write_junit_xml --truncate "" " " - junit_suite_start=$(test-tool date getnanos) - if test -n "$GIT_TEST_TEE_OUTPUT_FILE" - then - GIT_TEST_TEE_OFFSET=0 - fi -fi +start_test_output "$0" # Convenience # A regexp to match 5 and 35 hexdigits