2013-11-25 21:03:06 +00:00
|
|
|
# Test framework for git. See t/README for usage.
|
2005-05-14 05:50:32 +00:00
|
|
|
#
|
|
|
|
# Copyright (c) 2005 Junio C Hamano
|
|
|
|
#
|
2010-04-16 13:53:59 +00:00
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
|
|
|
# the Free Software Foundation, either version 2 of the License, or
|
|
|
|
# (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
# along with this program. If not, see http://www.gnu.org/licenses/ .
|
2005-05-14 05:50:32 +00:00
|
|
|
|
2012-06-25 05:01:35 +00:00
|
|
|
# Test the binaries we have just built. The tests are kept in
|
|
|
|
# t/ subdirectory and are run in 'trash directory' subdirectory.
|
|
|
|
if test -z "$TEST_DIRECTORY"
|
|
|
|
then
|
2013-11-18 04:12:43 +00:00
|
|
|
# ensure that TEST_DIRECTORY is an absolute path so that it
|
|
|
|
# is valid even if the current working directory is changed
|
2022-02-27 10:25:11 +00:00
|
|
|
TEST_DIRECTORY=$(pwd)
|
|
|
|
else
|
|
|
|
# The TEST_DIRECTORY will always be the path to the "t"
|
|
|
|
# directory in the git.git checkout. This is overridden by
|
|
|
|
# e.g. t/lib-subtest.sh, but only because its $(pwd) is
|
|
|
|
# different. Those tests still set "$TEST_DIRECTORY" to the
|
|
|
|
# same path.
|
|
|
|
#
|
|
|
|
# See use of "$GIT_BUILD_DIR" and "$TEST_DIRECTORY" below for
|
|
|
|
# hard assumptions about "$GIT_BUILD_DIR/t" existing and being
|
|
|
|
# the "$TEST_DIRECTORY", and e.g. "$TEST_DIRECTORY/helper"
|
|
|
|
# needing to exist.
|
2013-11-18 04:12:43 +00:00
|
|
|
TEST_DIRECTORY=$(cd "$TEST_DIRECTORY" && pwd) || exit 1
|
2012-06-25 05:01:35 +00:00
|
|
|
fi
|
|
|
|
if test -z "$TEST_OUTPUT_DIRECTORY"
|
|
|
|
then
|
|
|
|
# Similarly, override this to store the test-results subdir
|
|
|
|
# elsewhere
|
|
|
|
TEST_OUTPUT_DIRECTORY=$TEST_DIRECTORY
|
|
|
|
fi
|
2022-02-27 10:25:12 +00:00
|
|
|
GIT_BUILD_DIR="${TEST_DIRECTORY%/t}"
|
|
|
|
if test "$TEST_DIRECTORY" = "$GIT_BUILD_DIR"
|
2022-02-27 10:25:11 +00:00
|
|
|
then
|
|
|
|
echo "PANIC: Running in a $TEST_DIRECTORY that doesn't end in '/t'?" >&2
|
|
|
|
exit 1
|
|
|
|
fi
|
2022-10-18 10:59:04 +00:00
|
|
|
if test -f "$GIT_BUILD_DIR/GIT-BUILD-DIR"
|
|
|
|
then
|
|
|
|
GIT_BUILD_DIR="$(cat "$GIT_BUILD_DIR/GIT-BUILD-DIR")" || exit 1
|
|
|
|
# On Windows, we must convert Windows paths lest they contain a colon
|
|
|
|
case "$(uname -s)" in
|
|
|
|
*MINGW*)
|
|
|
|
GIT_BUILD_DIR="$(cygpath -au "$GIT_BUILD_DIR")"
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
fi
|
2012-06-25 05:01:35 +00:00
|
|
|
|
2022-02-27 10:25:10 +00:00
|
|
|
# Prepend a string to a VAR using an arbitrary ":" delimiter, not
|
|
|
|
# adding the delimiter if VAR or VALUE is empty. I.e. a generalized:
|
|
|
|
#
|
|
|
|
# VAR=$1${VAR:+${1:+$2}$VAR}
|
|
|
|
#
|
|
|
|
# Usage (using ":" as the $2 delimiter):
|
|
|
|
#
|
|
|
|
# prepend_var VAR : VALUE
|
|
|
|
prepend_var () {
|
2022-06-30 10:18:35 +00:00
|
|
|
eval "$1=\"$3\${$1:+${3:+$2}\$$1}\""
|
2022-02-27 10:25:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
# If [AL]SAN is in effect we want to abort so that we notice
|
|
|
|
# problems. The GIT_SAN_OPTIONS variable can be used to set common
|
|
|
|
# defaults shared between [AL]SAN_OPTIONS.
|
|
|
|
prepend_var GIT_SAN_OPTIONS : abort_on_error=1
|
2022-06-30 10:18:35 +00:00
|
|
|
prepend_var GIT_SAN_OPTIONS : strip_path_prefix="$GIT_BUILD_DIR/"
|
2022-02-27 10:25:10 +00:00
|
|
|
|
2017-07-10 13:24:35 +00:00
|
|
|
# If we were built with ASAN, it may complain about leaks
|
|
|
|
# of program-lifetime variables. Disable it by default to lower
|
|
|
|
# the noise level. This needs to happen at the start of the script,
|
|
|
|
# before we even do our "did we build git yet" check (since we don't
|
|
|
|
# want that one to complain to stderr).
|
2022-02-27 10:25:10 +00:00
|
|
|
prepend_var ASAN_OPTIONS : $GIT_SAN_OPTIONS
|
|
|
|
prepend_var ASAN_OPTIONS : detect_leaks=0
|
2017-07-10 13:24:35 +00:00
|
|
|
export ASAN_OPTIONS
|
|
|
|
|
2022-02-27 10:25:10 +00:00
|
|
|
prepend_var LSAN_OPTIONS : $GIT_SAN_OPTIONS
|
2022-02-27 10:25:13 +00:00
|
|
|
prepend_var LSAN_OPTIONS : fast_unwind_on_malloc=0
|
2017-09-05 13:04:04 +00:00
|
|
|
export LSAN_OPTIONS
|
|
|
|
|
tests: explicitly use `git.exe` on Windows
On Windows, when we refer to `/an/absolute/path/to/git`, it magically
resolves `git.exe` at that location. Except if something of the name
`git` exists next to that `git.exe`. So if we call `$BUILD_DIR/git`, it
will find `$BUILD_DIR/git.exe` *only* if there is not, say, a directory
called `$BUILD_DIR/git`.
Such a directory, however, exists in Git for Windows when building with
Visual Studio (our Visual Studio project generator defaults to putting
the build files into a directory whose name is the base name of the
corresponding `.exe`).
In the bin-wrappers/* scripts, we already take pains to use `git.exe`
rather than `git`, as this could pick up the wrong thing on Windows
(i.e. if there exists a `git` file or directory in the build directory).
Now we do the same in the tests' start-up code.
This also helps when testing an installed Git, as there might be even
more likely some stray file or directory in the way.
Note: the only way we can record whether the `.exe` suffix is by writing
it to the `GIT-BUILD-OPTIONS` file and sourcing it at the beginning of
`t/test-lib.sh`. This is not a requirement introduced by this patch, but
we move the call to be able to use the `$X` variable that holds the file
extension, if any.
Note also: the many, many calls to `git this` and `git that` are
unaffected, as the regular PATH search will find the `.exe` files on
Windows (and not be confused by a directory of the name `git` that is
in one of the directories listed in the `PATH` variable), while
`/path/to/git` would not, per se, know that it is looking for an
executable and happily prefer such a directory.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-11-14 16:32:11 +00:00
|
|
|
if test ! -f "$GIT_BUILD_DIR"/GIT-BUILD-OPTIONS
|
|
|
|
then
|
|
|
|
echo >&2 'error: GIT-BUILD-OPTIONS missing (has Git been built?).'
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
. "$GIT_BUILD_DIR"/GIT-BUILD-OPTIONS
|
|
|
|
export PERL_PATH SHELL_PATH
|
|
|
|
|
2021-07-20 06:32:26 +00:00
|
|
|
# In t0000, we need to override test directories of nested testcases. In case
|
|
|
|
# the developer has TEST_OUTPUT_DIRECTORY part of his build options, then we'd
|
|
|
|
# reset this value to instead contain what the developer has specified. We thus
|
|
|
|
# have this knob to allow overriding the directory.
|
|
|
|
if test -n "${TEST_OUTPUT_DIRECTORY_OVERRIDE}"
|
|
|
|
then
|
|
|
|
TEST_OUTPUT_DIRECTORY="${TEST_OUTPUT_DIRECTORY_OVERRIDE}"
|
|
|
|
fi
|
|
|
|
|
tests: disallow the use of abbreviated options (by default)
Git's command-line parsers support uniquely abbreviated options, e.g.
`git init --ba` would automatically expand `--ba` to `--bare`.
This is a very convenient feature in every day life for Git users, in
particular when tab completion is not available.
However, it is not a good idea to rely on that in Git's test suite, as
something that is a unique abbreviation of a command line option today
might no longer be a unique abbreviation tomorrow.
For example, if a future contribution added a new mode
`git init --babyproofing` and a previously-introduced test case used the
fact that `git init --ba` expanded to `git init --bare`, that future
contribution would now have to touch seemingly unrelated tests just to
keep the test suite from failing.
So let's disallow abbreviated options in the test suite by default.
Note: for ease of implementation, this patch really only touches the
`parse-options` machinery: more and more hand-rolled option parsers are
converted to use that internal API, and more and more scripts are
converted to built-ins (naturally using the parse-options API, too), so
in practice this catches most issues, and is definitely the biggest bang
for the buck.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-04-12 09:37:24 +00:00
|
|
|
# Disallow the use of abbreviated options in the test suite by default
|
|
|
|
if test -z "${GIT_TEST_DISALLOW_ABBREVIATED_OPTIONS}"
|
|
|
|
then
|
|
|
|
GIT_TEST_DISALLOW_ABBREVIATED_OPTIONS=true
|
|
|
|
export GIT_TEST_DISALLOW_ABBREVIATED_OPTIONS
|
|
|
|
fi
|
|
|
|
|
2021-05-10 14:19:09 +00:00
|
|
|
# Explicitly set the default branch name for testing, to avoid the
|
|
|
|
# transitory "git init" warning under --verbose.
|
|
|
|
: ${GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME:=master}
|
|
|
|
export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
|
|
|
|
|
2012-09-17 17:06:19 +00:00
|
|
|
################################################################
|
|
|
|
# It appears that people try to run tests without building...
|
tests: explicitly use `git.exe` on Windows
On Windows, when we refer to `/an/absolute/path/to/git`, it magically
resolves `git.exe` at that location. Except if something of the name
`git` exists next to that `git.exe`. So if we call `$BUILD_DIR/git`, it
will find `$BUILD_DIR/git.exe` *only* if there is not, say, a directory
called `$BUILD_DIR/git`.
Such a directory, however, exists in Git for Windows when building with
Visual Studio (our Visual Studio project generator defaults to putting
the build files into a directory whose name is the base name of the
corresponding `.exe`).
In the bin-wrappers/* scripts, we already take pains to use `git.exe`
rather than `git`, as this could pick up the wrong thing on Windows
(i.e. if there exists a `git` file or directory in the build directory).
Now we do the same in the tests' start-up code.
This also helps when testing an installed Git, as there might be even
more likely some stray file or directory in the way.
Note: the only way we can record whether the `.exe` suffix is by writing
it to the `GIT-BUILD-OPTIONS` file and sourcing it at the beginning of
`t/test-lib.sh`. This is not a requirement introduced by this patch, but
we move the call to be able to use the `$X` variable that holds the file
extension, if any.
Note also: the many, many calls to `git this` and `git that` are
unaffected, as the regular PATH search will find the `.exe` files on
Windows (and not be confused by a directory of the name `git` that is
in one of the directories listed in the `PATH` variable), while
`/path/to/git` would not, per se, know that it is looking for an
executable and happily prefer such a directory.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-11-14 16:32:11 +00:00
|
|
|
"${GIT_TEST_INSTALLED:-$GIT_BUILD_DIR}/git$X" >/dev/null
|
2012-09-17 17:06:19 +00:00
|
|
|
if test $? != 1
|
|
|
|
then
|
2018-11-14 16:32:10 +00:00
|
|
|
if test -n "$GIT_TEST_INSTALLED"
|
|
|
|
then
|
|
|
|
echo >&2 "error: there is no working Git at '$GIT_TEST_INSTALLED'"
|
|
|
|
else
|
|
|
|
echo >&2 'error: you do not seem to have built git yet.'
|
|
|
|
fi
|
2012-09-17 17:06:19 +00:00
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
store_arg_to=
|
test-lib: allow short options to be bundled
When debugging a test (or a set of tests), it's common to execute it
with some combination of short options, such as:
$ ./txxx-testname.sh -d -x -i
In cases like this, CLIs usually allow the short options to be bundled
in a single argument, for convenience and agility. Let's add this
feature to test-lib, allowing the above command to be run as:
$ ./txxx-testname.sh -dxi
(or any other permutation, e.g. '-ixd')
Note: Short options that require an argument can also be used in a
bundle, in any position. So, for example, '-r 5 -x', '-xr 5' and '-rx 5'
are all valid and equivalent. A special case would be having a bundle
with more than one of such options. To keep things simple, this case is
not allowed for now. This shouldn't be a major limitation, though, as
the only short option that requires an argument today is '-r'. And
concatenating '-r's as in '-rr 5 6' would probably not be very
practical: its unbundled format would be '-r 5 -r 6', for which test-lib
currently considers only the last argument. Therefore, if '-rr 5 6' were
to be allowed, it would have the same effect as just typing '-r 6'.
Note: the test-lib currently doesn't support '-r5', as an alternative
for '-r 5', so the former is not supported in bundles as well.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-22 15:58:57 +00:00
|
|
|
opt_required_arg=
|
|
|
|
# $1: option string
|
|
|
|
# $2: name of the var where the arg will be stored
|
|
|
|
mark_option_requires_arg () {
|
|
|
|
if test -n "$opt_required_arg"
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
then
|
test-lib: allow short options to be bundled
When debugging a test (or a set of tests), it's common to execute it
with some combination of short options, such as:
$ ./txxx-testname.sh -d -x -i
In cases like this, CLIs usually allow the short options to be bundled
in a single argument, for convenience and agility. Let's add this
feature to test-lib, allowing the above command to be run as:
$ ./txxx-testname.sh -dxi
(or any other permutation, e.g. '-ixd')
Note: Short options that require an argument can also be used in a
bundle, in any position. So, for example, '-r 5 -x', '-xr 5' and '-rx 5'
are all valid and equivalent. A special case would be having a bundle
with more than one of such options. To keep things simple, this case is
not allowed for now. This shouldn't be a major limitation, though, as
the only short option that requires an argument today is '-r'. And
concatenating '-r's as in '-rr 5 6' would probably not be very
practical: its unbundled format would be '-r 5 -r 6', for which test-lib
currently considers only the last argument. Therefore, if '-rr 5 6' were
to be allowed, it would have the same effect as just typing '-r 6'.
Note: the test-lib currently doesn't support '-r5', as an alternative
for '-r 5', so the former is not supported in bundles as well.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-22 15:58:57 +00:00
|
|
|
echo "error: options that require args cannot be bundled" \
|
|
|
|
"together: '$opt_required_arg' and '$1'" >&2
|
|
|
|
exit 1
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
fi
|
test-lib: allow short options to be bundled
When debugging a test (or a set of tests), it's common to execute it
with some combination of short options, such as:
$ ./txxx-testname.sh -d -x -i
In cases like this, CLIs usually allow the short options to be bundled
in a single argument, for convenience and agility. Let's add this
feature to test-lib, allowing the above command to be run as:
$ ./txxx-testname.sh -dxi
(or any other permutation, e.g. '-ixd')
Note: Short options that require an argument can also be used in a
bundle, in any position. So, for example, '-r 5 -x', '-xr 5' and '-rx 5'
are all valid and equivalent. A special case would be having a bundle
with more than one of such options. To keep things simple, this case is
not allowed for now. This shouldn't be a major limitation, though, as
the only short option that requires an argument today is '-r'. And
concatenating '-r's as in '-rr 5 6' would probably not be very
practical: its unbundled format would be '-r 5 -r 6', for which test-lib
currently considers only the last argument. Therefore, if '-rr 5 6' were
to be allowed, it would have the same effect as just typing '-r 6'.
Note: the test-lib currently doesn't support '-r5', as an alternative
for '-r 5', so the former is not supported in bundles as well.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-22 15:58:57 +00:00
|
|
|
opt_required_arg=$1
|
|
|
|
store_arg_to=$2
|
|
|
|
}
|
|
|
|
|
2022-05-21 22:18:46 +00:00
|
|
|
# These functions can be overridden e.g. to output JUnit XML
|
|
|
|
start_test_output () { :; }
|
|
|
|
start_test_case_output () { :; }
|
|
|
|
finalize_test_case_output () { :; }
|
|
|
|
finalize_test_output () { :; }
|
|
|
|
|
test-lib: allow short options to be bundled
When debugging a test (or a set of tests), it's common to execute it
with some combination of short options, such as:
$ ./txxx-testname.sh -d -x -i
In cases like this, CLIs usually allow the short options to be bundled
in a single argument, for convenience and agility. Let's add this
feature to test-lib, allowing the above command to be run as:
$ ./txxx-testname.sh -dxi
(or any other permutation, e.g. '-ixd')
Note: Short options that require an argument can also be used in a
bundle, in any position. So, for example, '-r 5 -x', '-xr 5' and '-rx 5'
are all valid and equivalent. A special case would be having a bundle
with more than one of such options. To keep things simple, this case is
not allowed for now. This shouldn't be a major limitation, though, as
the only short option that requires an argument today is '-r'. And
concatenating '-r's as in '-rr 5 6' would probably not be very
practical: its unbundled format would be '-r 5 -r 6', for which test-lib
currently considers only the last argument. Therefore, if '-rr 5 6' were
to be allowed, it would have the same effect as just typing '-r 6'.
Note: the test-lib currently doesn't support '-r5', as an alternative
for '-r 5', so the former is not supported in bundles as well.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-22 15:58:57 +00:00
|
|
|
parse_option () {
|
|
|
|
local opt="$1"
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
|
|
|
|
case "$opt" in
|
|
|
|
-d|--d|--de|--deb|--debu|--debug)
|
|
|
|
debug=t ;;
|
|
|
|
-i|--i|--im|--imm|--imme|--immed|--immedi|--immedia|--immediat|--immediate)
|
|
|
|
immediate=t ;;
|
|
|
|
-l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests)
|
|
|
|
GIT_TEST_LONG=t; export GIT_TEST_LONG ;;
|
|
|
|
-r)
|
test-lib: allow short options to be bundled
When debugging a test (or a set of tests), it's common to execute it
with some combination of short options, such as:
$ ./txxx-testname.sh -d -x -i
In cases like this, CLIs usually allow the short options to be bundled
in a single argument, for convenience and agility. Let's add this
feature to test-lib, allowing the above command to be run as:
$ ./txxx-testname.sh -dxi
(or any other permutation, e.g. '-ixd')
Note: Short options that require an argument can also be used in a
bundle, in any position. So, for example, '-r 5 -x', '-xr 5' and '-rx 5'
are all valid and equivalent. A special case would be having a bundle
with more than one of such options. To keep things simple, this case is
not allowed for now. This shouldn't be a major limitation, though, as
the only short option that requires an argument today is '-r'. And
concatenating '-r's as in '-rr 5 6' would probably not be very
practical: its unbundled format would be '-r 5 -r 6', for which test-lib
currently considers only the last argument. Therefore, if '-rr 5 6' were
to be allowed, it would have the same effect as just typing '-r 6'.
Note: the test-lib currently doesn't support '-r5', as an alternative
for '-r 5', so the former is not supported in bundles as well.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-22 15:58:57 +00:00
|
|
|
mark_option_requires_arg "$opt" run_list
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
;;
|
|
|
|
--run=*)
|
|
|
|
run_list=${opt#--*=} ;;
|
|
|
|
-h|--h|--he|--hel|--help)
|
|
|
|
help=t ;;
|
|
|
|
-v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose)
|
|
|
|
verbose=t ;;
|
|
|
|
--verbose-only=*)
|
|
|
|
verbose_only=${opt#--*=}
|
|
|
|
;;
|
|
|
|
-q|--q|--qu|--qui|--quie|--quiet)
|
|
|
|
# Ignore --quiet under a TAP::Harness. Saying how many tests
|
|
|
|
# passed without the ok/not ok details is always an error.
|
|
|
|
test -z "$HARNESS_ACTIVE" && quiet=t ;;
|
|
|
|
--with-dashes)
|
|
|
|
with_dashes=t ;;
|
2019-01-29 14:19:37 +00:00
|
|
|
--no-bin-wrappers)
|
|
|
|
no_bin_wrappers=t ;;
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
--no-color)
|
|
|
|
color= ;;
|
|
|
|
--va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind)
|
|
|
|
valgrind=memcheck
|
|
|
|
tee=t
|
|
|
|
;;
|
|
|
|
--valgrind=*)
|
|
|
|
valgrind=${opt#--*=}
|
|
|
|
tee=t
|
|
|
|
;;
|
|
|
|
--valgrind-only=*)
|
|
|
|
valgrind_only=${opt#--*=}
|
|
|
|
tee=t
|
|
|
|
;;
|
|
|
|
--tee)
|
|
|
|
tee=t ;;
|
|
|
|
--root=*)
|
|
|
|
root=${opt#--*=} ;;
|
|
|
|
--chain-lint)
|
|
|
|
GIT_TEST_CHAIN_LINT=1 ;;
|
|
|
|
--no-chain-lint)
|
|
|
|
GIT_TEST_CHAIN_LINT=0 ;;
|
|
|
|
-x)
|
|
|
|
trace=t ;;
|
|
|
|
-V|--verbose-log)
|
|
|
|
verbose_log=t
|
|
|
|
tee=t
|
|
|
|
;;
|
2019-01-29 14:19:27 +00:00
|
|
|
--write-junit-xml)
|
2022-05-21 22:18:46 +00:00
|
|
|
. "$TEST_DIRECTORY/test-lib-junit.sh"
|
2019-01-29 14:19:27 +00:00
|
|
|
;;
|
2022-05-21 22:18:51 +00:00
|
|
|
--github-workflow-markup)
|
|
|
|
. "$TEST_DIRECTORY/test-lib-github-workflow-markup.sh"
|
2019-01-29 14:19:27 +00:00
|
|
|
;;
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
--stress)
|
|
|
|
stress=t ;;
|
|
|
|
--stress=*)
|
2019-03-03 14:44:55 +00:00
|
|
|
echo "error: --stress does not accept an argument: '$opt'" >&2
|
|
|
|
echo "did you mean --stress-jobs=${opt#*=} or --stress-limit=${opt#*=}?" >&2
|
|
|
|
exit 1
|
|
|
|
;;
|
|
|
|
--stress-jobs=*)
|
|
|
|
stress=t;
|
test-lib: prevent '--stress-jobs=X' from being ignored
'./t1234-foo.sh --stress-jobs=X ...' is supposed to run that test
script in X parallel jobs, but the number of jobs specified on the
command line is entirely ignored if other '--stress'-related options
follow. I.e. both './t1234-foo.sh --stress-jobs=X --stress-limit=Y'
and './t1234-foo.sh --stress-jobs=X --stress' fall back to using twice
the number of CPUs parallel jobs instead.
The former has been broken since commit de69e6f6c9 (tests: let
--stress-limit=<N> imply --stress, 2019-03-03) [1], which started to
unconditionally overwrite the $stress variable holding the specified
number of jobs in its effort to imply '--stress'. The latter has been
broken since f545737144 (tests: introduce --stress-jobs=<N>,
2019-03-03), because it didn't consider that handling '--stress' will
overwrite that variable as well.
We could fix this by being more careful about (over)writing that
$stress variable and checking first whether it has already been set.
But I think it's cleaner to use a dedicated variable to hold the
number of specified parallel jobs, so let's do that instead.
[1] In de69e6f6c9 there was no '--stress-jobs=X' option yet, the
number of parallel jobs had to be specified via '--stress=X', so,
strictly speaking, de69e6f6c9 broke './t1234-foo.sh --stress=X
--stress-limit=Y'.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-01-26 22:05:33 +00:00
|
|
|
stress_jobs=${opt#--*=}
|
|
|
|
case "$stress_jobs" in
|
2019-02-11 19:58:03 +00:00
|
|
|
*[!0-9]*|0*|"")
|
2019-03-03 14:44:55 +00:00
|
|
|
echo "error: --stress-jobs=<N> requires the number of jobs to run" >&2
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
exit 1
|
|
|
|
;;
|
|
|
|
*) # Good.
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
;;
|
test-lib: make '--stress' more bisect-friendly
Let's suppose that a test somehow becomes flaky between 'master' and
'pu', and tends to fail within the first 50 repetitions when run with
'--stress'. In such a case we could use 'git bisect' to find the
culprit: if the test script fails with '--stress', then the commit is
definitely bad, but if it survives, say, 300 repetitions, then we could
consider it good with reasonable confidence.
Unfortunately, all this could only be done manually, because
'--stress' would run the test script repeatedly for all eternity on a
good commit, and it would exit with success even when it found a
failure on a bad commit.
So let's make '--stress' usable with 'git bisect run':
- Make it exit with failure if a failure is found.
- Add the '--stress-limit=<N>' option to repeat the test script
at most N times in each of the parallel jobs, and exit with
success when the limit is reached.
And then we could simply run something like:
$ git bisect start origin/pu master
$ git bisect run sh -c 'make && cd t &&
./t1234-foo.sh --stress --stress-limit=300'
Sure, as a brand new feature it won't be any useful right now, but in
a release or three most cooking topics will already contain this, so
we could automatically bisect at least newly introduced flakiness.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-02-08 11:50:45 +00:00
|
|
|
--stress-limit=*)
|
2019-03-03 14:44:54 +00:00
|
|
|
stress=t;
|
test-lib: make '--stress' more bisect-friendly
Let's suppose that a test somehow becomes flaky between 'master' and
'pu', and tends to fail within the first 50 repetitions when run with
'--stress'. In such a case we could use 'git bisect' to find the
culprit: if the test script fails with '--stress', then the commit is
definitely bad, but if it survives, say, 300 repetitions, then we could
consider it good with reasonable confidence.
Unfortunately, all this could only be done manually, because
'--stress' would run the test script repeatedly for all eternity on a
good commit, and it would exit with success even when it found a
failure on a bad commit.
So let's make '--stress' usable with 'git bisect run':
- Make it exit with failure if a failure is found.
- Add the '--stress-limit=<N>' option to repeat the test script
at most N times in each of the parallel jobs, and exit with
success when the limit is reached.
And then we could simply run something like:
$ git bisect start origin/pu master
$ git bisect run sh -c 'make && cd t &&
./t1234-foo.sh --stress --stress-limit=300'
Sure, as a brand new feature it won't be any useful right now, but in
a release or three most cooking topics will already contain this, so
we could automatically bisect at least newly introduced flakiness.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-02-08 11:50:45 +00:00
|
|
|
stress_limit=${opt#--*=}
|
|
|
|
case "$stress_limit" in
|
2019-02-11 19:58:03 +00:00
|
|
|
*[!0-9]*|0*|"")
|
test-lib: make '--stress' more bisect-friendly
Let's suppose that a test somehow becomes flaky between 'master' and
'pu', and tends to fail within the first 50 repetitions when run with
'--stress'. In such a case we could use 'git bisect' to find the
culprit: if the test script fails with '--stress', then the commit is
definitely bad, but if it survives, say, 300 repetitions, then we could
consider it good with reasonable confidence.
Unfortunately, all this could only be done manually, because
'--stress' would run the test script repeatedly for all eternity on a
good commit, and it would exit with success even when it found a
failure on a bad commit.
So let's make '--stress' usable with 'git bisect run':
- Make it exit with failure if a failure is found.
- Add the '--stress-limit=<N>' option to repeat the test script
at most N times in each of the parallel jobs, and exit with
success when the limit is reached.
And then we could simply run something like:
$ git bisect start origin/pu master
$ git bisect run sh -c 'make && cd t &&
./t1234-foo.sh --stress --stress-limit=300'
Sure, as a brand new feature it won't be any useful right now, but in
a release or three most cooking topics will already contain this, so
we could automatically bisect at least newly introduced flakiness.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-02-08 11:50:45 +00:00
|
|
|
echo "error: --stress-limit=<N> requires the number of repetitions" >&2
|
|
|
|
exit 1
|
|
|
|
;;
|
|
|
|
*) # Good.
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
;;
|
2022-07-27 23:13:32 +00:00
|
|
|
--invert-exit-code)
|
|
|
|
invert_exit_code=t
|
|
|
|
;;
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
*)
|
|
|
|
echo "error: unknown test option '$opt'" >&2; exit 1 ;;
|
|
|
|
esac
|
test-lib: allow short options to be bundled
When debugging a test (or a set of tests), it's common to execute it
with some combination of short options, such as:
$ ./txxx-testname.sh -d -x -i
In cases like this, CLIs usually allow the short options to be bundled
in a single argument, for convenience and agility. Let's add this
feature to test-lib, allowing the above command to be run as:
$ ./txxx-testname.sh -dxi
(or any other permutation, e.g. '-ixd')
Note: Short options that require an argument can also be used in a
bundle, in any position. So, for example, '-r 5 -x', '-xr 5' and '-rx 5'
are all valid and equivalent. A special case would be having a bundle
with more than one of such options. To keep things simple, this case is
not allowed for now. This shouldn't be a major limitation, though, as
the only short option that requires an argument today is '-r'. And
concatenating '-r's as in '-rr 5 6' would probably not be very
practical: its unbundled format would be '-r 5 -r 6', for which test-lib
currently considers only the last argument. Therefore, if '-rr 5 6' were
to be allowed, it would have the same effect as just typing '-r 6'.
Note: the test-lib currently doesn't support '-r5', as an alternative
for '-r 5', so the former is not supported in bundles as well.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-22 15:58:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
# Parse options while taking care to leave $@ intact, so we will still
|
|
|
|
# have all the original command line options when executing the test
|
|
|
|
# script again for '--tee' and '--verbose-log' later.
|
|
|
|
for opt
|
|
|
|
do
|
|
|
|
if test -n "$store_arg_to"
|
|
|
|
then
|
|
|
|
eval $store_arg_to=\$opt
|
|
|
|
store_arg_to=
|
|
|
|
opt_required_arg=
|
|
|
|
continue
|
|
|
|
fi
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
|
test-lib: allow short options to be bundled
When debugging a test (or a set of tests), it's common to execute it
with some combination of short options, such as:
$ ./txxx-testname.sh -d -x -i
In cases like this, CLIs usually allow the short options to be bundled
in a single argument, for convenience and agility. Let's add this
feature to test-lib, allowing the above command to be run as:
$ ./txxx-testname.sh -dxi
(or any other permutation, e.g. '-ixd')
Note: Short options that require an argument can also be used in a
bundle, in any position. So, for example, '-r 5 -x', '-xr 5' and '-rx 5'
are all valid and equivalent. A special case would be having a bundle
with more than one of such options. To keep things simple, this case is
not allowed for now. This shouldn't be a major limitation, though, as
the only short option that requires an argument today is '-r'. And
concatenating '-r's as in '-rr 5 6' would probably not be very
practical: its unbundled format would be '-r 5 -r 6', for which test-lib
currently considers only the last argument. Therefore, if '-rr 5 6' were
to be allowed, it would have the same effect as just typing '-r 6'.
Note: the test-lib currently doesn't support '-r5', as an alternative
for '-r 5', so the former is not supported in bundles as well.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-22 15:58:57 +00:00
|
|
|
case "$opt" in
|
|
|
|
--*|-?)
|
|
|
|
parse_option "$opt" ;;
|
|
|
|
-?*)
|
|
|
|
# bundled short options must be fed separately to parse_option
|
|
|
|
opt=${opt#-}
|
|
|
|
while test -n "$opt"
|
|
|
|
do
|
|
|
|
extra=${opt#?}
|
|
|
|
this=${opt%$extra}
|
|
|
|
opt=$extra
|
|
|
|
parse_option "-$this"
|
|
|
|
done
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
echo "error: unknown test option '$opt'" >&2; exit 1 ;;
|
|
|
|
esac
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
done
|
|
|
|
if test -n "$store_arg_to"
|
|
|
|
then
|
test-lib: allow short options to be bundled
When debugging a test (or a set of tests), it's common to execute it
with some combination of short options, such as:
$ ./txxx-testname.sh -d -x -i
In cases like this, CLIs usually allow the short options to be bundled
in a single argument, for convenience and agility. Let's add this
feature to test-lib, allowing the above command to be run as:
$ ./txxx-testname.sh -dxi
(or any other permutation, e.g. '-ixd')
Note: Short options that require an argument can also be used in a
bundle, in any position. So, for example, '-r 5 -x', '-xr 5' and '-rx 5'
are all valid and equivalent. A special case would be having a bundle
with more than one of such options. To keep things simple, this case is
not allowed for now. This shouldn't be a major limitation, though, as
the only short option that requires an argument today is '-r'. And
concatenating '-r's as in '-rr 5 6' would probably not be very
practical: its unbundled format would be '-r 5 -r 6', for which test-lib
currently considers only the last argument. Therefore, if '-rr 5 6' were
to be allowed, it would have the same effect as just typing '-r 6'.
Note: the test-lib currently doesn't support '-r5', as an alternative
for '-r 5', so the former is not supported in bundles as well.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-22 15:58:57 +00:00
|
|
|
echo "error: $opt_required_arg requires an argument" >&2
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
if test -n "$valgrind_only"
|
|
|
|
then
|
|
|
|
test -z "$valgrind" && valgrind=memcheck
|
|
|
|
test -z "$verbose" && verbose_only="$valgrind_only"
|
|
|
|
elif test -n "$valgrind"
|
|
|
|
then
|
|
|
|
test -z "$verbose_log" && verbose=t
|
|
|
|
fi
|
|
|
|
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
if test -n "$stress"
|
|
|
|
then
|
|
|
|
verbose=t
|
|
|
|
trace=t
|
|
|
|
immediate=t
|
|
|
|
fi
|
|
|
|
|
|
|
|
TEST_STRESS_JOB_SFX="${GIT_TEST_STRESS_JOB_NR:+.stress-$GIT_TEST_STRESS_JOB_NR}"
|
2019-01-05 01:08:56 +00:00
|
|
|
TEST_NAME="$(basename "$0" .sh)"
|
2019-08-05 21:04:47 +00:00
|
|
|
TEST_NUMBER="${TEST_NAME%%-*}"
|
|
|
|
TEST_NUMBER="${TEST_NUMBER#t}"
|
2019-01-05 01:08:56 +00:00
|
|
|
TEST_RESULTS_DIR="$TEST_OUTPUT_DIRECTORY/test-results"
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
TEST_RESULTS_BASE="$TEST_RESULTS_DIR/$TEST_NAME$TEST_STRESS_JOB_SFX"
|
2022-07-27 23:13:34 +00:00
|
|
|
TEST_RESULTS_SAN_FILE_PFX=trace
|
|
|
|
TEST_RESULTS_SAN_DIR_SFX=leak
|
|
|
|
TEST_RESULTS_SAN_FILE=
|
|
|
|
TEST_RESULTS_SAN_DIR="$TEST_RESULTS_DIR/$TEST_NAME.$TEST_RESULTS_SAN_DIR_SFX"
|
test-lib: have the "check" mode for SANITIZE=leak consider leak logs
As noted in previous on-list discussions[1] we have various tests that
will falsely report being leak-free because we're missing the relevant
exit code from LSAN as summarized below.
We should fix those issues, but in the meantime and as an additional
sanity check we can and should consider our own ASAN logs before
reporting that a test is leak-free.
Before this compiling with SANITIZE=leak and running:
./t6407-merge-binary.sh
Will exit successfully, now we'll get an error and an informative
message on:
GIT_TEST_SANITIZE_LEAK_LOG=true ./t6407-merge-binary.sh
Even better, as noted in the updated t/README we'll now error out when
combined with the "check" mode:
GIT_TEST_PASSING_SANITIZE_LEAK=check \
GIT_TEST_SANITIZE_LEAK_LOG=true \
./t4058-diff-duplicates.sh
Why do we miss these leaks? Because:
* We have leaks inside "test_expect_failure" blocks, which by design
will not distinguish a "normal" failure from an abort() or
segfault. See [1] for a discussion of it shortcomings.
* We have "git" invocations outside of "test_expect_success",
e.g. setup code in the main body of the test, or in test helper
functions that don't use &&-chaining.
* Our tests will otherwise catch segfaults and abort(), but if we
invoke a command that invokes another command it needs to ferry the
exit code up to us.
Notably a command that e.g. might invoke "git pack-objects" might
itself exit with status 128 if that "pack-objects" segfaults or
abort()'s. If the test invoking the parent command(s) is using
"test_must_fail" we'll consider it an expected "ok" failure.
* run-command.c doesn't (but probably should) ferry up such exit
codes, so for e.g. "git push" tests where we expect a failure and an
underlying "git" command fails we won't ferry up the segfault or
abort exit code.
* We have gitweb.perl and some other perl code ignoring return values
from close(), i.e. ignoring exit codes from "git rev-parse" et al.
* We have in-tree shellscripts like "git-merge-one-file.sh" invoking
git commands, they'll usually return their own exit codes on "git"
failure, rather then ferrying up segfault or abort() exit code.
E.g. these invocations in git-merge-one-file.sh leak, but aren't
reflected in the "git merge" exit code:
src1=$(git unpack-file $2)
src2=$(git unpack-file $3)
That case would be easily "fixed" by adding a line like this after
each assignment:
test $? -ne 0 && exit $?
But we'd then in e.g. "t6407-merge-binary.sh" run into
write_tree_trivial() in "builtin/merge.c" calling die() instead of
ferrying up the relevant exit code.
Let's remove "TEST_PASSES_SANITIZE_LEAK=true" from tests we
were falsely marking as leak-free.
In the case of t6407-merge-binary.sh it was marked as leak-free in
9081a421a6d (checkout: fix "branch info" memory leaks,
2021-11-16). I'd previously removed other bad
"TEST_PASSES_SANITIZE_LEAK=true" opt-ins in the series merged in
ea05fd5fbf7 (Merge branch 'ab/keep-git-exit-codes-in-tests',
2022-03-16). The case of t1060-object-corruption.sh is more subtle,
and will be discussed in a subsequent commit.
1. https://lore.kernel.org/git/cover-0.7-00000000000-20220318T002951Z-avarab@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:39 +00:00
|
|
|
TEST_RESULTS_SAN_DIR_NR_LEAKS_STARTUP=
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
TRASH_DIRECTORY="trash directory.$TEST_NAME$TEST_STRESS_JOB_SFX"
|
2019-01-05 01:08:57 +00:00
|
|
|
test -n "$root" && TRASH_DIRECTORY="$root/$TRASH_DIRECTORY"
|
|
|
|
case "$TRASH_DIRECTORY" in
|
|
|
|
/*) ;; # absolute path is good
|
|
|
|
*) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$TRASH_DIRECTORY" ;;
|
|
|
|
esac
|
2019-01-05 01:08:56 +00:00
|
|
|
|
test-lib: have the "check" mode for SANITIZE=leak consider leak logs
As noted in previous on-list discussions[1] we have various tests that
will falsely report being leak-free because we're missing the relevant
exit code from LSAN as summarized below.
We should fix those issues, but in the meantime and as an additional
sanity check we can and should consider our own ASAN logs before
reporting that a test is leak-free.
Before this compiling with SANITIZE=leak and running:
./t6407-merge-binary.sh
Will exit successfully, now we'll get an error and an informative
message on:
GIT_TEST_SANITIZE_LEAK_LOG=true ./t6407-merge-binary.sh
Even better, as noted in the updated t/README we'll now error out when
combined with the "check" mode:
GIT_TEST_PASSING_SANITIZE_LEAK=check \
GIT_TEST_SANITIZE_LEAK_LOG=true \
./t4058-diff-duplicates.sh
Why do we miss these leaks? Because:
* We have leaks inside "test_expect_failure" blocks, which by design
will not distinguish a "normal" failure from an abort() or
segfault. See [1] for a discussion of it shortcomings.
* We have "git" invocations outside of "test_expect_success",
e.g. setup code in the main body of the test, or in test helper
functions that don't use &&-chaining.
* Our tests will otherwise catch segfaults and abort(), but if we
invoke a command that invokes another command it needs to ferry the
exit code up to us.
Notably a command that e.g. might invoke "git pack-objects" might
itself exit with status 128 if that "pack-objects" segfaults or
abort()'s. If the test invoking the parent command(s) is using
"test_must_fail" we'll consider it an expected "ok" failure.
* run-command.c doesn't (but probably should) ferry up such exit
codes, so for e.g. "git push" tests where we expect a failure and an
underlying "git" command fails we won't ferry up the segfault or
abort exit code.
* We have gitweb.perl and some other perl code ignoring return values
from close(), i.e. ignoring exit codes from "git rev-parse" et al.
* We have in-tree shellscripts like "git-merge-one-file.sh" invoking
git commands, they'll usually return their own exit codes on "git"
failure, rather then ferrying up segfault or abort() exit code.
E.g. these invocations in git-merge-one-file.sh leak, but aren't
reflected in the "git merge" exit code:
src1=$(git unpack-file $2)
src2=$(git unpack-file $3)
That case would be easily "fixed" by adding a line like this after
each assignment:
test $? -ne 0 && exit $?
But we'd then in e.g. "t6407-merge-binary.sh" run into
write_tree_trivial() in "builtin/merge.c" calling die() instead of
ferrying up the relevant exit code.
Let's remove "TEST_PASSES_SANITIZE_LEAK=true" from tests we
were falsely marking as leak-free.
In the case of t6407-merge-binary.sh it was marked as leak-free in
9081a421a6d (checkout: fix "branch info" memory leaks,
2021-11-16). I'd previously removed other bad
"TEST_PASSES_SANITIZE_LEAK=true" opt-ins in the series merged in
ea05fd5fbf7 (Merge branch 'ab/keep-git-exit-codes-in-tests',
2022-03-16). The case of t1060-object-corruption.sh is more subtle,
and will be discussed in a subsequent commit.
1. https://lore.kernel.org/git/cover-0.7-00000000000-20220318T002951Z-avarab@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:39 +00:00
|
|
|
# Utility functions using $TEST_RESULTS_* variables
|
|
|
|
nr_san_dir_leaks_ () {
|
|
|
|
# stderr piped to /dev/null because the directory may have
|
|
|
|
# been "rmdir"'d already.
|
|
|
|
find "$TEST_RESULTS_SAN_DIR" \
|
|
|
|
-type f \
|
|
|
|
-name "$TEST_RESULTS_SAN_FILE_PFX.*" 2>/dev/null |
|
test-lib: ignore uninteresting LSan output
When I run the tests in leak-checking mode the same way our CI job does,
like:
make SANITIZE=leak \
GIT_TEST_PASSING_SANITIZE_LEAK=true \
GIT_TEST_SANITIZE_LEAK_LOG=true \
test
then LSan can racily produce useless entries in the log files that look
like this:
==git==3034393==Unable to get registers from thread 3034307.
I think they're mostly harmless based on the source here:
https://github.com/llvm/llvm-project/blob/7e0a52e8e9ef6394bb62e0b56e17fa23e7262411/compiler-rt/lib/lsan/lsan_common.cpp#L414
which reads:
PtraceRegistersStatus have_registers =
suspended_threads.GetRegistersAndSP(i, ®isters, &sp);
if (have_registers != REGISTERS_AVAILABLE) {
Report("Unable to get registers from thread %llu.\n", os_id);
// If unable to get SP, consider the entire stack to be reachable unless
// GetRegistersAndSP failed with ESRCH.
if (have_registers == REGISTERS_UNAVAILABLE_FATAL)
continue;
sp = stack_begin;
}
The program itself still runs fine and LSan doesn't cause us to abort.
But test-lib.sh looks for any non-empty LSan logs and marks the test as
a failure anyway, under the assumption that we simply missed the failing
exit code somehow.
I don't think I've ever seen this happen in the CI job, but running
locally using clang-14 on an 8-core machine, I can't seem to make it
through a full run of the test suite without having at least one
failure. And it's a different one every time (though they do seem to
often be related to packing tests, which makes sense, since that is one
of our biggest users of threaded code).
We can hack around this by only counting LSan log files that contain a
line that doesn't match our known-uninteresting pattern.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-08-28 18:37:35 +00:00
|
|
|
xargs grep -lv "Unable to get registers from thread" |
|
test-lib: have the "check" mode for SANITIZE=leak consider leak logs
As noted in previous on-list discussions[1] we have various tests that
will falsely report being leak-free because we're missing the relevant
exit code from LSAN as summarized below.
We should fix those issues, but in the meantime and as an additional
sanity check we can and should consider our own ASAN logs before
reporting that a test is leak-free.
Before this compiling with SANITIZE=leak and running:
./t6407-merge-binary.sh
Will exit successfully, now we'll get an error and an informative
message on:
GIT_TEST_SANITIZE_LEAK_LOG=true ./t6407-merge-binary.sh
Even better, as noted in the updated t/README we'll now error out when
combined with the "check" mode:
GIT_TEST_PASSING_SANITIZE_LEAK=check \
GIT_TEST_SANITIZE_LEAK_LOG=true \
./t4058-diff-duplicates.sh
Why do we miss these leaks? Because:
* We have leaks inside "test_expect_failure" blocks, which by design
will not distinguish a "normal" failure from an abort() or
segfault. See [1] for a discussion of it shortcomings.
* We have "git" invocations outside of "test_expect_success",
e.g. setup code in the main body of the test, or in test helper
functions that don't use &&-chaining.
* Our tests will otherwise catch segfaults and abort(), but if we
invoke a command that invokes another command it needs to ferry the
exit code up to us.
Notably a command that e.g. might invoke "git pack-objects" might
itself exit with status 128 if that "pack-objects" segfaults or
abort()'s. If the test invoking the parent command(s) is using
"test_must_fail" we'll consider it an expected "ok" failure.
* run-command.c doesn't (but probably should) ferry up such exit
codes, so for e.g. "git push" tests where we expect a failure and an
underlying "git" command fails we won't ferry up the segfault or
abort exit code.
* We have gitweb.perl and some other perl code ignoring return values
from close(), i.e. ignoring exit codes from "git rev-parse" et al.
* We have in-tree shellscripts like "git-merge-one-file.sh" invoking
git commands, they'll usually return their own exit codes on "git"
failure, rather then ferrying up segfault or abort() exit code.
E.g. these invocations in git-merge-one-file.sh leak, but aren't
reflected in the "git merge" exit code:
src1=$(git unpack-file $2)
src2=$(git unpack-file $3)
That case would be easily "fixed" by adding a line like this after
each assignment:
test $? -ne 0 && exit $?
But we'd then in e.g. "t6407-merge-binary.sh" run into
write_tree_trivial() in "builtin/merge.c" calling die() instead of
ferrying up the relevant exit code.
Let's remove "TEST_PASSES_SANITIZE_LEAK=true" from tests we
were falsely marking as leak-free.
In the case of t6407-merge-binary.sh it was marked as leak-free in
9081a421a6d (checkout: fix "branch info" memory leaks,
2021-11-16). I'd previously removed other bad
"TEST_PASSES_SANITIZE_LEAK=true" opt-ins in the series merged in
ea05fd5fbf7 (Merge branch 'ab/keep-git-exit-codes-in-tests',
2022-03-16). The case of t1060-object-corruption.sh is more subtle,
and will be discussed in a subsequent commit.
1. https://lore.kernel.org/git/cover-0.7-00000000000-20220318T002951Z-avarab@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:39 +00:00
|
|
|
wc -l
|
|
|
|
}
|
|
|
|
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
# If --stress was passed, run this test repeatedly in several parallel loops.
|
|
|
|
if test "$GIT_TEST_STRESS_STARTED" = "done"
|
|
|
|
then
|
|
|
|
: # Don't stress test again.
|
|
|
|
elif test -n "$stress"
|
|
|
|
then
|
test-lib: prevent '--stress-jobs=X' from being ignored
'./t1234-foo.sh --stress-jobs=X ...' is supposed to run that test
script in X parallel jobs, but the number of jobs specified on the
command line is entirely ignored if other '--stress'-related options
follow. I.e. both './t1234-foo.sh --stress-jobs=X --stress-limit=Y'
and './t1234-foo.sh --stress-jobs=X --stress' fall back to using twice
the number of CPUs parallel jobs instead.
The former has been broken since commit de69e6f6c9 (tests: let
--stress-limit=<N> imply --stress, 2019-03-03) [1], which started to
unconditionally overwrite the $stress variable holding the specified
number of jobs in its effort to imply '--stress'. The latter has been
broken since f545737144 (tests: introduce --stress-jobs=<N>,
2019-03-03), because it didn't consider that handling '--stress' will
overwrite that variable as well.
We could fix this by being more careful about (over)writing that
$stress variable and checking first whether it has already been set.
But I think it's cleaner to use a dedicated variable to hold the
number of specified parallel jobs, so let's do that instead.
[1] In de69e6f6c9 there was no '--stress-jobs=X' option yet, the
number of parallel jobs had to be specified via '--stress=X', so,
strictly speaking, de69e6f6c9 broke './t1234-foo.sh --stress=X
--stress-limit=Y'.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-01-26 22:05:33 +00:00
|
|
|
if test -n "$stress_jobs"
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
then
|
test-lib: prevent '--stress-jobs=X' from being ignored
'./t1234-foo.sh --stress-jobs=X ...' is supposed to run that test
script in X parallel jobs, but the number of jobs specified on the
command line is entirely ignored if other '--stress'-related options
follow. I.e. both './t1234-foo.sh --stress-jobs=X --stress-limit=Y'
and './t1234-foo.sh --stress-jobs=X --stress' fall back to using twice
the number of CPUs parallel jobs instead.
The former has been broken since commit de69e6f6c9 (tests: let
--stress-limit=<N> imply --stress, 2019-03-03) [1], which started to
unconditionally overwrite the $stress variable holding the specified
number of jobs in its effort to imply '--stress'. The latter has been
broken since f545737144 (tests: introduce --stress-jobs=<N>,
2019-03-03), because it didn't consider that handling '--stress' will
overwrite that variable as well.
We could fix this by being more careful about (over)writing that
$stress variable and checking first whether it has already been set.
But I think it's cleaner to use a dedicated variable to hold the
number of specified parallel jobs, so let's do that instead.
[1] In de69e6f6c9 there was no '--stress-jobs=X' option yet, the
number of parallel jobs had to be specified via '--stress=X', so,
strictly speaking, de69e6f6c9 broke './t1234-foo.sh --stress=X
--stress-limit=Y'.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-01-26 22:05:33 +00:00
|
|
|
job_count=$stress_jobs
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
elif test -n "$GIT_TEST_STRESS_LOAD"
|
|
|
|
then
|
|
|
|
job_count="$GIT_TEST_STRESS_LOAD"
|
|
|
|
elif job_count=$(getconf _NPROCESSORS_ONLN 2>/dev/null) &&
|
|
|
|
test -n "$job_count"
|
|
|
|
then
|
|
|
|
job_count=$((2 * $job_count))
|
|
|
|
else
|
|
|
|
job_count=8
|
|
|
|
fi
|
|
|
|
|
|
|
|
mkdir -p "$TEST_RESULTS_DIR"
|
|
|
|
stressfail="$TEST_RESULTS_BASE.stress-failed"
|
|
|
|
rm -f "$stressfail"
|
|
|
|
|
|
|
|
stress_exit=0
|
|
|
|
trap '
|
|
|
|
kill $job_pids 2>/dev/null
|
|
|
|
wait
|
|
|
|
stress_exit=1
|
|
|
|
' TERM INT HUP
|
|
|
|
|
|
|
|
job_pids=
|
|
|
|
job_nr=0
|
|
|
|
while test $job_nr -lt "$job_count"
|
|
|
|
do
|
|
|
|
(
|
|
|
|
GIT_TEST_STRESS_STARTED=done
|
|
|
|
GIT_TEST_STRESS_JOB_NR=$job_nr
|
|
|
|
export GIT_TEST_STRESS_STARTED GIT_TEST_STRESS_JOB_NR
|
|
|
|
|
|
|
|
trap '
|
|
|
|
kill $test_pid 2>/dev/null
|
|
|
|
wait
|
|
|
|
exit 1
|
|
|
|
' TERM INT
|
|
|
|
|
test-lib: make '--stress' more bisect-friendly
Let's suppose that a test somehow becomes flaky between 'master' and
'pu', and tends to fail within the first 50 repetitions when run with
'--stress'. In such a case we could use 'git bisect' to find the
culprit: if the test script fails with '--stress', then the commit is
definitely bad, but if it survives, say, 300 repetitions, then we could
consider it good with reasonable confidence.
Unfortunately, all this could only be done manually, because
'--stress' would run the test script repeatedly for all eternity on a
good commit, and it would exit with success even when it found a
failure on a bad commit.
So let's make '--stress' usable with 'git bisect run':
- Make it exit with failure if a failure is found.
- Add the '--stress-limit=<N>' option to repeat the test script
at most N times in each of the parallel jobs, and exit with
success when the limit is reached.
And then we could simply run something like:
$ git bisect start origin/pu master
$ git bisect run sh -c 'make && cd t &&
./t1234-foo.sh --stress --stress-limit=300'
Sure, as a brand new feature it won't be any useful right now, but in
a release or three most cooking topics will already contain this, so
we could automatically bisect at least newly introduced flakiness.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-02-08 11:50:45 +00:00
|
|
|
cnt=1
|
|
|
|
while ! test -e "$stressfail" &&
|
|
|
|
{ test -z "$stress_limit" ||
|
|
|
|
test $cnt -le $stress_limit ; }
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
do
|
|
|
|
$TEST_SHELL_PATH "$0" "$@" >"$TEST_RESULTS_BASE.stress-$job_nr.out" 2>&1 &
|
|
|
|
test_pid=$!
|
|
|
|
|
|
|
|
if wait $test_pid
|
|
|
|
then
|
|
|
|
printf "OK %2d.%d\n" $GIT_TEST_STRESS_JOB_NR $cnt
|
|
|
|
else
|
|
|
|
echo $GIT_TEST_STRESS_JOB_NR >>"$stressfail"
|
|
|
|
printf "FAIL %2d.%d\n" $GIT_TEST_STRESS_JOB_NR $cnt
|
|
|
|
fi
|
|
|
|
cnt=$(($cnt + 1))
|
|
|
|
done
|
|
|
|
) &
|
|
|
|
job_pids="$job_pids $!"
|
|
|
|
job_nr=$(($job_nr + 1))
|
|
|
|
done
|
|
|
|
|
|
|
|
wait
|
|
|
|
|
|
|
|
if test -f "$stressfail"
|
|
|
|
then
|
test-lib: make '--stress' more bisect-friendly
Let's suppose that a test somehow becomes flaky between 'master' and
'pu', and tends to fail within the first 50 repetitions when run with
'--stress'. In such a case we could use 'git bisect' to find the
culprit: if the test script fails with '--stress', then the commit is
definitely bad, but if it survives, say, 300 repetitions, then we could
consider it good with reasonable confidence.
Unfortunately, all this could only be done manually, because
'--stress' would run the test script repeatedly for all eternity on a
good commit, and it would exit with success even when it found a
failure on a bad commit.
So let's make '--stress' usable with 'git bisect run':
- Make it exit with failure if a failure is found.
- Add the '--stress-limit=<N>' option to repeat the test script
at most N times in each of the parallel jobs, and exit with
success when the limit is reached.
And then we could simply run something like:
$ git bisect start origin/pu master
$ git bisect run sh -c 'make && cd t &&
./t1234-foo.sh --stress --stress-limit=300'
Sure, as a brand new feature it won't be any useful right now, but in
a release or three most cooking topics will already contain this, so
we could automatically bisect at least newly introduced flakiness.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-02-08 11:50:45 +00:00
|
|
|
stress_exit=1
|
test-lib: add the '--stress' option to run a test repeatedly under load
Unfortunately, we have a few flaky tests, whose failures tend to be
hard to reproduce. We've found that the best we can do to reproduce
such a failure is to run the test script repeatedly while the machine
is under load, and wait in the hope that the load creates enough
variance in the timing of the test's commands that a failure is
evenually triggered. I have a command to do that, and I noticed that
two other contributors have rolled their own scripts to do the same,
all choosing slightly different approaches.
To help reproduce failures in flaky tests, introduce the '--stress'
option to run a test script repeatedly in multiple parallel jobs until
one of them fails, thereby using the test script itself to increase
the load on the machine.
The number of parallel jobs is determined by, in order of precedence:
the number specified as '--stress=<N>', or the value of the
GIT_TEST_STRESS_LOAD environment variable, or twice the number of
available processors (as reported by the 'getconf' utility), or 8.
Make '--stress' imply '--verbose -x --immediate' to get the most
information about rare failures; there is really no point in spending
all the extra effort to reproduce such a failure, and then not know
which command failed and why.
To prevent the several parallel invocations of the same test from
interfering with each other:
- Include the parallel job's number in the name of the trash
directory and the various output files under 't/test-results/' as
a '.stress-<Nr>' suffix.
- Add the parallel job's number to the port number specified by the
user or to the test number, so even tests involving daemons
listening on a TCP socket can be stressed.
- Redirect each parallel test run's verbose output to
't/test-results/$TEST_NAME.stress-<nr>.out', because dumping the
output of several parallel running tests to the terminal would
create a big ugly mess.
For convenience, print the output of the failed test job at the end,
and rename its trash directory to end with the '.stress-failed'
suffix, so it's easy to find in a predictable path (OTOH, all absolute
paths recorded in the trash directory become invalid; we'll see
whether this causes any issues in practice). If, in an unlikely case,
more than one jobs were to fail nearly at the same time, then print
the output of all failed jobs, and rename the trash directory of only
the last one (i.e. with the highest job number), as it is the trash
directory of the test whose output will be at the bottom of the user's
terminal.
Based on Jeff King's 'stress' script.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:59 +00:00
|
|
|
echo "Log(s) of failed test run(s):"
|
|
|
|
for failed_job_nr in $(sort -n "$stressfail")
|
|
|
|
do
|
|
|
|
echo "Contents of '$TEST_RESULTS_BASE.stress-$failed_job_nr.out':"
|
|
|
|
cat "$TEST_RESULTS_BASE.stress-$failed_job_nr.out"
|
|
|
|
done
|
|
|
|
rm -rf "$TRASH_DIRECTORY.stress-failed"
|
|
|
|
# Move the last one.
|
|
|
|
mv "$TRASH_DIRECTORY.stress-$failed_job_nr" "$TRASH_DIRECTORY.stress-failed"
|
|
|
|
fi
|
|
|
|
|
|
|
|
exit $stress_exit
|
|
|
|
fi
|
|
|
|
|
2012-09-22 04:55:10 +00:00
|
|
|
# if --tee was passed, write the output not only to the terminal, but
|
|
|
|
# additionally to the file test-results/$BASENAME.out, too.
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
if test "$GIT_TEST_TEE_STARTED" = "done"
|
|
|
|
then
|
|
|
|
: # do not redirect again
|
|
|
|
elif test -n "$tee"
|
|
|
|
then
|
2019-01-05 01:08:56 +00:00
|
|
|
mkdir -p "$TEST_RESULTS_DIR"
|
test-lib: add --verbose-log option
The "--verbose" option redirects output from arbitrary
test commands to stdout. This is useful for examining the
output manually, like:
./t5547-push-quarantine.sh -v | less
But it also means that the output is intermingled with the
TAP directives, which can confuse a TAP parser like "prove".
This has always been a potential problem, but became an
issue recently when one test happened to output the word
"ok" on a line by itself, which prove interprets as a test
success:
$ prove t5547-push-quarantine.sh :: -v
t5547-push-quarantine.sh .. 1/? To dest.git
* [new branch] HEAD -> master
To dest.git
! [remote rejected] reject -> reject (pre-receive hook declined)
error: failed to push some refs to 'dest.git'
fatal: git cat-file d08c8eba97f4e683ece08654c7c8d2ba0c03b129: bad file
t5547-push-quarantine.sh .. Failed -1/4 subtests
Test Summary Report
-------------------
t5547-push-quarantine.sh (Wstat: 0 Tests: 5 Failed: 0)
Parse errors: Tests out of sequence. Found (2) but expected (3)
Tests out of sequence. Found (3) but expected (4)
Tests out of sequence. Found (4) but expected (5)
Bad plan. You planned 4 tests but ran 5.
Files=1, Tests=5, 0 wallclock secs ( 0.01 usr + 0.01 sys = 0.02 CPU)
Result: FAIL
One answer is "if it hurts, don't do it", but that's not
quite the whole story. The Travis tests use "--verbose
--tee" so that they can get the benefit of prove's parallel
options, along with a verbose log in case there is a
failure. We just need the verbose output to go to the log,
but keep stdout clean.
Getting this right turns out to be surprisingly difficult.
Here's the progression of alternatives I considered:
1. Add an option to write verbose output to stderr. This is
hard to capture, though, because we want each test to
have its own log (because they're all run in parallel
and the jumbled output would be useless).
2. Add an option to write verbose output to a file in
test-results. This works, but the log is missing all of
the non-verbose output, which gives context.
3. Like (2), but teach say_color() to additionally output
to the log. This mostly works, but misses any output
that happens outside of the say() functions (which isn't
a lot, but is a potential maintenance headache).
4. Like (2), but make the log file the same as the "--tee"
file. That almost works, but now we have two processes
opening the same file. That gives us two separate
descriptors, each with their own idea of the current
position. They'll each start writing at offset 0, and
overwrite each other's data.
5. Like (4), but in each case open the file for appending.
That atomically positions each write at the end of the
file.
It's possible we may still get sheared writes between
the two processes, but this is already the case when
writing to stdout. It's not a problem in practice
because the test harness generally waits for snippets to
finish before writing the TAP output.
We can ignore buffering issues with tee, because POSIX
mandates that it does not buffer. Likewise, POSIX
specifies "tee -a", so it should be available
everywhere.
This patch implements option (5), which seems to work well
in practice.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-10-21 10:48:00 +00:00
|
|
|
|
|
|
|
# Make this filename available to the sub-process in case it is using
|
|
|
|
# --verbose-log.
|
2019-01-05 01:08:56 +00:00
|
|
|
GIT_TEST_TEE_OUTPUT_FILE=$TEST_RESULTS_BASE.out
|
test-lib: add --verbose-log option
The "--verbose" option redirects output from arbitrary
test commands to stdout. This is useful for examining the
output manually, like:
./t5547-push-quarantine.sh -v | less
But it also means that the output is intermingled with the
TAP directives, which can confuse a TAP parser like "prove".
This has always been a potential problem, but became an
issue recently when one test happened to output the word
"ok" on a line by itself, which prove interprets as a test
success:
$ prove t5547-push-quarantine.sh :: -v
t5547-push-quarantine.sh .. 1/? To dest.git
* [new branch] HEAD -> master
To dest.git
! [remote rejected] reject -> reject (pre-receive hook declined)
error: failed to push some refs to 'dest.git'
fatal: git cat-file d08c8eba97f4e683ece08654c7c8d2ba0c03b129: bad file
t5547-push-quarantine.sh .. Failed -1/4 subtests
Test Summary Report
-------------------
t5547-push-quarantine.sh (Wstat: 0 Tests: 5 Failed: 0)
Parse errors: Tests out of sequence. Found (2) but expected (3)
Tests out of sequence. Found (3) but expected (4)
Tests out of sequence. Found (4) but expected (5)
Bad plan. You planned 4 tests but ran 5.
Files=1, Tests=5, 0 wallclock secs ( 0.01 usr + 0.01 sys = 0.02 CPU)
Result: FAIL
One answer is "if it hurts, don't do it", but that's not
quite the whole story. The Travis tests use "--verbose
--tee" so that they can get the benefit of prove's parallel
options, along with a verbose log in case there is a
failure. We just need the verbose output to go to the log,
but keep stdout clean.
Getting this right turns out to be surprisingly difficult.
Here's the progression of alternatives I considered:
1. Add an option to write verbose output to stderr. This is
hard to capture, though, because we want each test to
have its own log (because they're all run in parallel
and the jumbled output would be useless).
2. Add an option to write verbose output to a file in
test-results. This works, but the log is missing all of
the non-verbose output, which gives context.
3. Like (2), but teach say_color() to additionally output
to the log. This mostly works, but misses any output
that happens outside of the say() functions (which isn't
a lot, but is a potential maintenance headache).
4. Like (2), but make the log file the same as the "--tee"
file. That almost works, but now we have two processes
opening the same file. That gives us two separate
descriptors, each with their own idea of the current
position. They'll each start writing at offset 0, and
overwrite each other's data.
5. Like (4), but in each case open the file for appending.
That atomically positions each write at the end of the
file.
It's possible we may still get sheared writes between
the two processes, but this is already the case when
writing to stdout. It's not a problem in practice
because the test harness generally waits for snippets to
finish before writing the TAP output.
We can ignore buffering issues with tee, because POSIX
mandates that it does not buffer. Likewise, POSIX
specifies "tee -a", so it should be available
everywhere.
This patch implements option (5), which seems to work well
in practice.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-10-21 10:48:00 +00:00
|
|
|
export GIT_TEST_TEE_OUTPUT_FILE
|
|
|
|
|
|
|
|
# Truncate before calling "tee -a" to get rid of the results
|
|
|
|
# from any previous runs.
|
|
|
|
>"$GIT_TEST_TEE_OUTPUT_FILE"
|
|
|
|
|
t/Makefile: introduce TEST_SHELL_PATH
You may want to run the test suite with a different shell
than you use to build Git. For instance, you may build with
SHELL_PATH=/bin/sh (because it's faster, or it's what you
expect to exist on systems where the build will be used) but
want to run the test suite with bash (e.g., since that
allows using "-x" reliably across the whole test suite).
There's currently no good way to do this.
You might think that doing two separate make invocations,
like:
make &&
make -C t SHELL_PATH=/bin/bash
would work. And it _almost_ does. The second make will see
our bash SHELL_PATH, and we'll use that to run the
individual test scripts (or tell prove to use it to do so).
So far so good.
But this breaks down when "--tee" or "--verbose-log" is
used. Those options cause the test script to actually
re-exec itself using $SHELL_PATH. But wait, wouldn't our
second make invocation have set SHELL_PATH correctly in the
environment?
Yes, but test-lib.sh sources GIT-BUILD-OPTIONS, which we
built during the first "make". And that overrides the
environment, giving us the original SHELL_PATH again.
Let's introduce a new variable that lets you specify a
specific shell to be run for the test scripts. Note that we
have to touch both the main and t/ Makefiles, since we have
to record it in GIT-BUILD-OPTIONS in one, and use it in the
latter.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-12-08 10:47:22 +00:00
|
|
|
(GIT_TEST_TEE_STARTED=done ${TEST_SHELL_PATH} "$0" "$@" 2>&1;
|
2019-01-05 01:08:56 +00:00
|
|
|
echo $? >"$TEST_RESULTS_BASE.exit") | tee -a "$GIT_TEST_TEE_OUTPUT_FILE"
|
|
|
|
test "$(cat "$TEST_RESULTS_BASE.exit")" = 0
|
2012-09-22 04:55:10 +00:00
|
|
|
exit
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
fi
|
|
|
|
|
|
|
|
if test -n "$trace" && test -n "$test_untraceable"
|
|
|
|
then
|
|
|
|
# '-x' tracing requested, but this test script can't be reliably
|
|
|
|
# traced, unless it is run with a Bash version supporting
|
|
|
|
# BASH_XTRACEFD (introduced in Bash v4.1).
|
|
|
|
#
|
|
|
|
# Perform this version check _after_ the test script was
|
|
|
|
# potentially re-executed with $TEST_SHELL_PATH for '--tee' or
|
|
|
|
# '--verbose-log', so the right shell is checked and the
|
|
|
|
# warning is issued only once.
|
|
|
|
if test -n "$BASH_VERSION" && eval '
|
|
|
|
test ${BASH_VERSINFO[0]} -gt 4 || {
|
|
|
|
test ${BASH_VERSINFO[0]} -eq 4 &&
|
|
|
|
test ${BASH_VERSINFO[1]} -ge 1
|
|
|
|
}
|
|
|
|
'
|
|
|
|
then
|
|
|
|
: Executed by a Bash version supporting BASH_XTRACEFD. Good.
|
|
|
|
else
|
|
|
|
echo >&2 "warning: ignoring -x; '$0' is untraceable without BASH_XTRACEFD"
|
|
|
|
trace=
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
if test -n "$trace" && test -z "$verbose_log"
|
|
|
|
then
|
|
|
|
verbose=t
|
|
|
|
fi
|
2012-09-22 04:55:10 +00:00
|
|
|
|
2021-08-05 19:48:25 +00:00
|
|
|
# Since bash 5.0, checkwinsize is enabled by default which does
|
|
|
|
# update the COLUMNS variable every time a non-builtin command
|
|
|
|
# completes, even for non-interactive shells.
|
|
|
|
# Disable that since we are aiming for repeatability.
|
|
|
|
test -n "$BASH_VERSION" && shopt -u checkwinsize 2>/dev/null
|
|
|
|
|
2005-05-14 05:50:32 +00:00
|
|
|
# For repeatability, reset the environment to known value.
|
2015-06-17 21:11:21 +00:00
|
|
|
# TERM is sanitized below, after saving color control sequences.
|
2005-05-14 05:50:32 +00:00
|
|
|
LANG=C
|
2005-10-10 04:58:02 +00:00
|
|
|
LC_ALL=C
|
2005-08-11 02:10:01 +00:00
|
|
|
PAGER=cat
|
2005-05-14 05:50:32 +00:00
|
|
|
TZ=UTC
|
test-lib.sh: set COLUMNS=80 for --verbose repeatability
Some tests will fail under --verbose because while we've unset COLUMNS
since b1d645b58ac (tests: unset COLUMNS inherited from environment,
2012-03-27), we also look for the columns with an ioctl(..,
TIOCGWINSZ, ...) on some platforms. By setting COLUMNS again we
preempt the TIOCGWINSZ lookup in pager.c's term_columns(), it'll take
COLUMNS over TIOCGWINSZ,
This fixes t0500-progress-display.sh., which broke because of a
combination of the this issue and the progress output reacting to the
column width since 545dc345ebd (progress: break too long progress bar
lines, 2019-04-12). The t5324-split-commit-graph.sh fails in a similar
manner due to progress output, see [1] for details.
The issue is not specific to progress.c, the diff code also checks
COLUMNS and some of its tests can be made to fail in a similar
manner[2], anything that invokes a pager is potentially affected.
See ea77e675e56 (Make "git help" react to window size correctly,
2005-12-18) and ad6c3739a33 (pager: find out the terminal width before
spawning the pager, 2012-02-12) for how the TIOCGWINSZ code ended up
in pager.c
1. http://lore.kernel.org/git/20210624051253.GG6312@szeder.dev
2. https://lore.kernel.org/git/20210627074419.GH6312@szeder.dev/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-06-29 11:29:36 +00:00
|
|
|
COLUMNS=80
|
|
|
|
export LANG LC_ALL PAGER TZ COLUMNS
|
2006-07-11 19:01:54 +00:00
|
|
|
EDITOR=:
|
i18n: make GETTEXT_POISON a runtime option
Change the GETTEXT_POISON compile-time + runtime GIT_GETTEXT_POISON
test parameter to only be a GIT_TEST_GETTEXT_POISON=<non-empty?>
runtime parameter, to be consistent with other parameters documented
in "Running tests with special setups" in t/README.
When I added GETTEXT_POISON in bb946bba76 ("i18n: add GETTEXT_POISON
to simulate unfriendly translator", 2011-02-22) I was concerned with
ensuring that the _() function would get constant folded if NO_GETTEXT
was defined, and likewise that GETTEXT_POISON would be compiled out
unless it was defined.
But as the benchmark in my [1] shows doing a one-off runtime
getenv("GIT_TEST_[...]") is trivial, and since GETTEXT_POISON was
originally added the GIT_TEST_* env variables have become the common
idiom for turning on special test setups.
So change GETTEXT_POISON to work the same way. Now the
GETTEXT_POISON=YesPlease compile-time option is gone, and running the
tests with GIT_TEST_GETTEXT_POISON=[YesPlease|] can be toggled on/off
without recompiling.
This allows for conditionally amending tests to test with/without
poison, similar to what 859fdc0c3c ("commit-graph: define
GIT_TEST_COMMIT_GRAPH", 2018-08-29) did for GIT_TEST_COMMIT_GRAPH. Do
some of that, now we e.g. always run the t0205-gettext-poison.sh test.
I did enough there to remove the GETTEXT_POISON prerequisite, but its
inverse C_LOCALE_OUTPUT is still around, and surely some tests using
it can be converted to e.g. always set GIT_TEST_GETTEXT_POISON=.
Notes on the implementation:
* We still compile a dedicated GETTEXT_POISON build in Travis
CI. Perhaps this should be revisited and integrated into the
"linux-gcc" build, see ae59a4e44f ("travis: run tests with
GIT_TEST_SPLIT_INDEX", 2018-01-07) for prior art in that area. Then
again maybe not, see [2].
* We now skip a test in t0000-basic.sh under
GIT_TEST_GETTEXT_POISON=YesPlease that wasn't skipped before. This
test relies on C locale output, but due to an edge case in how the
previous implementation of GETTEXT_POISON worked (reading it from
GIT-BUILD-OPTIONS) wasn't enabling poison correctly. Now it does,
and needs to be skipped.
* The getenv() function is not reentrant, so out of paranoia about
code of the form:
printf(_("%s"), getenv("some-env"));
call use_gettext_poison() in our early setup in git_setup_gettext()
so we populate the "poison_requested" variable in a codepath that's
won't suffer from that race condition.
* We error out in the Makefile if you're still saying
GETTEXT_POISON=YesPlease to prompt users to change their
invocation.
* We should not print out poisoned messages during the test
initialization itself to keep it more readable, so the test library
hides the variable if set in $GIT_TEST_GETTEXT_POISON_ORIG during
setup. See [3].
See also [4] for more on the motivation behind this patch, and the
history of the GETTEXT_POISON facility.
1. https://public-inbox.org/git/871s8gd32p.fsf@evledraar.gmail.com/
2. https://public-inbox.org/git/20181102163725.GY30222@szeder.dev/
3. https://public-inbox.org/git/20181022202241.18629-2-szeder.dev@gmail.com/
4. https://public-inbox.org/git/878t2pd6yu.fsf@evledraar.gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-11-08 21:15:29 +00:00
|
|
|
|
2012-03-02 18:48:36 +00:00
|
|
|
# A call to "unset" with no arguments causes at least Solaris 10
|
|
|
|
# /usr/xpg4/bin/sh and /bin/ksh to bail out. So keep the unsets
|
|
|
|
# deriving from the command substitution clustered with the other
|
|
|
|
# ones.
|
test-lib.sh: set COLUMNS=80 for --verbose repeatability
Some tests will fail under --verbose because while we've unset COLUMNS
since b1d645b58ac (tests: unset COLUMNS inherited from environment,
2012-03-27), we also look for the columns with an ioctl(..,
TIOCGWINSZ, ...) on some platforms. By setting COLUMNS again we
preempt the TIOCGWINSZ lookup in pager.c's term_columns(), it'll take
COLUMNS over TIOCGWINSZ,
This fixes t0500-progress-display.sh., which broke because of a
combination of the this issue and the progress output reacting to the
column width since 545dc345ebd (progress: break too long progress bar
lines, 2019-04-12). The t5324-split-commit-graph.sh fails in a similar
manner due to progress output, see [1] for details.
The issue is not specific to progress.c, the diff code also checks
COLUMNS and some of its tests can be made to fail in a similar
manner[2], anything that invokes a pager is potentially affected.
See ea77e675e56 (Make "git help" react to window size correctly,
2005-12-18) and ad6c3739a33 (pager: find out the terminal width before
spawning the pager, 2012-02-12) for how the TIOCGWINSZ code ended up
in pager.c
1. http://lore.kernel.org/git/20210624051253.GG6312@szeder.dev
2. https://lore.kernel.org/git/20210627074419.GH6312@szeder.dev/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-06-29 11:29:36 +00:00
|
|
|
unset VISUAL EMAIL LANGUAGE $("$PERL_PATH" -e '
|
2011-03-15 10:10:45 +00:00
|
|
|
my @env = keys %ENV;
|
2011-03-28 19:16:09 +00:00
|
|
|
my $ok = join("|", qw(
|
|
|
|
TRACE
|
|
|
|
DEBUG
|
|
|
|
TEST
|
|
|
|
.*_TEST
|
|
|
|
PROVE
|
|
|
|
VALGRIND
|
2013-01-06 17:47:57 +00:00
|
|
|
UNZIP
|
2013-01-15 13:50:56 +00:00
|
|
|
PERF_
|
2014-05-22 09:28:50 +00:00
|
|
|
CURL_VERBOSE
|
2016-09-05 10:24:42 +00:00
|
|
|
TRACE_CURL
|
2011-03-28 19:16:09 +00:00
|
|
|
));
|
|
|
|
my @vars = grep(/^GIT_/ && !/^GIT_($ok)/o, @env);
|
2011-03-15 10:10:45 +00:00
|
|
|
print join("\n", @vars);
|
|
|
|
')
|
2018-02-16 02:46:04 +00:00
|
|
|
unset XDG_CACHE_HOME
|
2012-07-24 11:53:05 +00:00
|
|
|
unset XDG_CONFIG_HOME
|
2013-07-04 20:38:55 +00:00
|
|
|
unset GITPERLLIB
|
2022-01-14 03:33:36 +00:00
|
|
|
unset GIT_TRACE2_PARENT_NAME
|
|
|
|
unset GIT_TRACE2_PARENT_SID
|
2019-10-29 12:09:14 +00:00
|
|
|
TEST_AUTHOR_LOCALNAME=author
|
|
|
|
TEST_AUTHOR_DOMAIN=example.com
|
|
|
|
GIT_AUTHOR_EMAIL=${TEST_AUTHOR_LOCALNAME}@${TEST_AUTHOR_DOMAIN}
|
2006-02-11 03:11:23 +00:00
|
|
|
GIT_AUTHOR_NAME='A U Thor'
|
test-lib: set deterministic default author/committer date
We always set the name and email for committer and author idents to make
the test suite more deterministic, but not timestamps. Many scripts use
test_tick to get consistent and sensibly incrementing timestamps as they
create commits. But other scripts don't particularly care about the
timestamp, and are happy to use whatever the current system time is.
This non-determinism can be annoying:
- when debugging a test, comparing results between two runs can be
difficult, because the commit ids change
- this can sometimes cause tests to be racy. E.g., traversal order
depends on timestamp order. Even in a well-ordered set of commands,
because our timestamp granularity is one second, two commits might
sometimes have the same timestamp and sometimes differ.
Let's set a default timestamp for all scripts to use. Any that use
test_tick already will be unaffected (because their first test_tick call
will overwrite our default), but it will make things a bit more
deterministic for those that don't.
We should be able to choose any time we want here. I picked this one
because:
- it differs from the initial test_tick default, which may make it
easier to distinguish when debugging tests. I picked "April 1st
13:14:15" in the hope that it might stand out.
- it's slightly before the test_tick default. Some tests create some
commits before the first call to test_tick, so using an older
timestamps for those makes sense chronologically. Note that this
isn't how things currently work (where system times are usually more
recent than test_tick), but that also allows us to flush out a few
hidden timestamp dependencies (like the one recently fixed in
t5539).
- we could likewise pick any timezone we want. Choosing +0000 would
have required fixing up fewer tests, but we're more likely to turn
up interesting cases by not matching $TZ exactly. And since
test_tick already checks "-0700", let's try something in the "+"
zone range for variety.
It's possible that the non-deterministic times could help flush out bugs
(e.g., if something broke when the clock flipped over to 2021, our test
suite would let us know). But historically that hasn't been the case;
all time-dependent outcomes we've seen turned out to be accidentally
flaky tests (which we fixed by using test_tick). If we do want to cover
handling the current time, we should dedicate one script to doing so,
and have it unset GIT_COMMITTER_DATE explicitly.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-07-09 20:42:04 +00:00
|
|
|
GIT_AUTHOR_DATE='1112354055 +0200'
|
2019-10-29 12:09:14 +00:00
|
|
|
TEST_COMMITTER_LOCALNAME=committer
|
|
|
|
TEST_COMMITTER_DOMAIN=example.com
|
|
|
|
GIT_COMMITTER_EMAIL=${TEST_COMMITTER_LOCALNAME}@${TEST_COMMITTER_DOMAIN}
|
2006-02-11 03:11:23 +00:00
|
|
|
GIT_COMMITTER_NAME='C O Mitter'
|
test-lib: set deterministic default author/committer date
We always set the name and email for committer and author idents to make
the test suite more deterministic, but not timestamps. Many scripts use
test_tick to get consistent and sensibly incrementing timestamps as they
create commits. But other scripts don't particularly care about the
timestamp, and are happy to use whatever the current system time is.
This non-determinism can be annoying:
- when debugging a test, comparing results between two runs can be
difficult, because the commit ids change
- this can sometimes cause tests to be racy. E.g., traversal order
depends on timestamp order. Even in a well-ordered set of commands,
because our timestamp granularity is one second, two commits might
sometimes have the same timestamp and sometimes differ.
Let's set a default timestamp for all scripts to use. Any that use
test_tick already will be unaffected (because their first test_tick call
will overwrite our default), but it will make things a bit more
deterministic for those that don't.
We should be able to choose any time we want here. I picked this one
because:
- it differs from the initial test_tick default, which may make it
easier to distinguish when debugging tests. I picked "April 1st
13:14:15" in the hope that it might stand out.
- it's slightly before the test_tick default. Some tests create some
commits before the first call to test_tick, so using an older
timestamps for those makes sense chronologically. Note that this
isn't how things currently work (where system times are usually more
recent than test_tick), but that also allows us to flush out a few
hidden timestamp dependencies (like the one recently fixed in
t5539).
- we could likewise pick any timezone we want. Choosing +0000 would
have required fixing up fewer tests, but we're more likely to turn
up interesting cases by not matching $TZ exactly. And since
test_tick already checks "-0700", let's try something in the "+"
zone range for variety.
It's possible that the non-deterministic times could help flush out bugs
(e.g., if something broke when the clock flipped over to 2021, our test
suite would let us know). But historically that hasn't been the case;
all time-dependent outcomes we've seen turned out to be accidentally
flaky tests (which we fixed by using test_tick). If we do want to cover
handling the current time, we should dedicate one script to doing so,
and have it unset GIT_COMMITTER_DATE explicitly.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-07-09 20:42:04 +00:00
|
|
|
GIT_COMMITTER_DATE='1112354055 +0200'
|
2007-02-04 05:45:47 +00:00
|
|
|
GIT_MERGE_VERBOSITY=5
|
2012-01-11 06:44:45 +00:00
|
|
|
GIT_MERGE_AUTOEDIT=no
|
|
|
|
export GIT_MERGE_VERBOSITY GIT_MERGE_AUTOEDIT
|
2006-02-11 03:11:23 +00:00
|
|
|
export GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME
|
|
|
|
export GIT_COMMITTER_EMAIL GIT_COMMITTER_NAME
|
test-lib: set deterministic default author/committer date
We always set the name and email for committer and author idents to make
the test suite more deterministic, but not timestamps. Many scripts use
test_tick to get consistent and sensibly incrementing timestamps as they
create commits. But other scripts don't particularly care about the
timestamp, and are happy to use whatever the current system time is.
This non-determinism can be annoying:
- when debugging a test, comparing results between two runs can be
difficult, because the commit ids change
- this can sometimes cause tests to be racy. E.g., traversal order
depends on timestamp order. Even in a well-ordered set of commands,
because our timestamp granularity is one second, two commits might
sometimes have the same timestamp and sometimes differ.
Let's set a default timestamp for all scripts to use. Any that use
test_tick already will be unaffected (because their first test_tick call
will overwrite our default), but it will make things a bit more
deterministic for those that don't.
We should be able to choose any time we want here. I picked this one
because:
- it differs from the initial test_tick default, which may make it
easier to distinguish when debugging tests. I picked "April 1st
13:14:15" in the hope that it might stand out.
- it's slightly before the test_tick default. Some tests create some
commits before the first call to test_tick, so using an older
timestamps for those makes sense chronologically. Note that this
isn't how things currently work (where system times are usually more
recent than test_tick), but that also allows us to flush out a few
hidden timestamp dependencies (like the one recently fixed in
t5539).
- we could likewise pick any timezone we want. Choosing +0000 would
have required fixing up fewer tests, but we're more likely to turn
up interesting cases by not matching $TZ exactly. And since
test_tick already checks "-0700", let's try something in the "+"
zone range for variety.
It's possible that the non-deterministic times could help flush out bugs
(e.g., if something broke when the clock flipped over to 2021, our test
suite would let us know). But historically that hasn't been the case;
all time-dependent outcomes we've seen turned out to be accidentally
flaky tests (which we fixed by using test_tick). If we do want to cover
handling the current time, we should dedicate one script to doing so,
and have it unset GIT_COMMITTER_DATE explicitly.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-07-09 20:42:04 +00:00
|
|
|
export GIT_COMMITTER_DATE GIT_AUTHOR_DATE
|
Do not use VISUAL editor on dumb terminals
Refuse to use $VISUAL and fall back to $EDITOR if TERM is unset
or set to "dumb". Traditionally, VISUAL is set to a screen
editor and EDITOR to a line-based editor, which should be more
useful in that situation.
vim, for example, is happy to assume a terminal supports ANSI
sequences even if TERM is dumb (e.g., when running from a text
editor like Acme). git already refuses to fall back to vi on a
dumb terminal if GIT_EDITOR, core.editor, VISUAL, and EDITOR are
unset, but without this patch, that check is suppressed by
VISUAL=vi.
Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-11-11 23:56:07 +00:00
|
|
|
export EDITOR
|
2005-05-14 05:50:32 +00:00
|
|
|
|
2020-07-29 23:14:24 +00:00
|
|
|
GIT_DEFAULT_HASH="${GIT_TEST_DEFAULT_HASH:-sha1}"
|
|
|
|
export GIT_DEFAULT_HASH
|
2021-03-20 00:03:56 +00:00
|
|
|
GIT_TEST_MERGE_ALGORITHM="${GIT_TEST_MERGE_ALGORITHM:-ort}"
|
|
|
|
export GIT_TEST_MERGE_ALGORITHM
|
2020-07-29 23:14:24 +00:00
|
|
|
|
2014-07-12 00:03:01 +00:00
|
|
|
# Tests using GIT_TRACE typically don't want <timestamp> <file>:<line> output
|
|
|
|
GIT_TRACE_BARE=1
|
|
|
|
export GIT_TRACE_BARE
|
|
|
|
|
2021-11-29 13:47:45 +00:00
|
|
|
# Some tests scan the GIT_TRACE2_EVENT feed for events, but the
|
|
|
|
# default depth is 2, which frequently causes issues when the
|
|
|
|
# events are wrapped in new regions. Set it to a sufficiently
|
|
|
|
# large depth to avoid custom changes in the test suite.
|
|
|
|
GIT_TRACE2_EVENT_NESTING=100
|
|
|
|
export GIT_TRACE2_EVENT_NESTING
|
|
|
|
|
2018-09-18 23:29:36 +00:00
|
|
|
# Use specific version of the index file format
|
|
|
|
if test -n "${GIT_TEST_INDEX_VERSION:+isset}"
|
2014-02-23 20:49:58 +00:00
|
|
|
then
|
2018-09-18 23:29:36 +00:00
|
|
|
GIT_INDEX_VERSION="$GIT_TEST_INDEX_VERSION"
|
2014-02-23 20:49:58 +00:00
|
|
|
export GIT_INDEX_VERSION
|
|
|
|
fi
|
|
|
|
|
perl: check for perl warnings while running tests
We set "use warnings" in most of our perl code to catch problems. But as
the name implies, warnings just emit a message to stderr and don't
otherwise affect the program. So our tests are quite likely to miss that
warnings are being spewed, as most of them do not look at stderr.
We could ask perl to make all warnings fatal, but this is likely
annoying for non-developers, who would rather have a running program
with a warning than something that refuses to work at all.
So instead, let's teach the perl code to respect an environment variable
(GIT_PERL_FATAL_WARNINGS) to increase the severity of the warnings. This
can be set for day-to-day running if people want to be really pedantic,
but the primary use is to trigger it within the test suite.
We could also trigger that for every test run, but likewise even the
tests failing may be annoying to distro builders, etc (just as -Werror
would be for compiling C code). So we'll tie it to a special test-mode
variable (GIT_TEST_PERL_FATAL_WARNINGS) that can be set in the
environment or as a Makefile knob, and we'll automatically turn the knob
when DEVELOPER=1 is set. That should give developers and CI the more
careful view without disrupting normal users or packagers.
Note that the mapping from the GIT_TEST_* form to the GIT_* form in
test-lib.sh is necessary even if they had the same name: the perl
scripts need it to be normalized to a perl truth value, and we also have
to make sure it's exported (we might have gotten it from the
environment, but we might also have gotten it from GIT-BUILD-OPTIONS
directly).
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-10-22 03:24:00 +00:00
|
|
|
if test -n "$GIT_TEST_PERL_FATAL_WARNINGS"
|
|
|
|
then
|
|
|
|
GIT_PERL_FATAL_WARNINGS=1
|
|
|
|
export GIT_PERL_FATAL_WARNINGS
|
|
|
|
fi
|
|
|
|
|
2021-10-29 00:15:52 +00:00
|
|
|
case $GIT_TEST_FSYNC in
|
|
|
|
'')
|
|
|
|
GIT_TEST_FSYNC=0
|
|
|
|
export GIT_TEST_FSYNC
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
2022-04-09 12:28:37 +00:00
|
|
|
# Add libc MALLOC and MALLOC_PERTURB test only if we are not executing
|
2022-09-28 10:01:54 +00:00
|
|
|
# the test with valgrind and have not compiled with conflict SANITIZE
|
|
|
|
# options.
|
test-lib: parse command line options earlier
'test-lib.sh' looks for the presence of certain options like '--tee'
and '--verbose-log', so it can execute the test script again to save
its standard output and error. It looks for '--valgrind' as well, to
set up some Valgrind-specific stuff. These all happen before the
actual option parsing loop, and the conditions looking for these
options look a bit odd, too. They are not completely correct, either,
because in a bogus invocation like './t1234-foo.sh -r --tee' they
recognize '--tee', although it should be handled as the required
argument of the '-r' option. This patch series will add two more
options to look out for early, and, in addition, will have to extract
these options' stuck arguments (i.e. '--opt=arg') as well.
So let's move the option parsing loop and the couple of related
conditions following it earlier in 'test-lib.sh', before the place
where the test script is executed again for '--tee' and its friends.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-05 01:08:55 +00:00
|
|
|
if test -n "$valgrind" ||
|
2022-04-09 12:28:37 +00:00
|
|
|
test -n "$SANITIZE_ADDRESS" ||
|
2022-09-28 10:01:54 +00:00
|
|
|
test -n "$SANITIZE_LEAK" ||
|
2012-09-26 20:16:39 +00:00
|
|
|
test -n "$TEST_NO_MALLOC_CHECK"
|
2012-09-15 03:38:24 +00:00
|
|
|
then
|
|
|
|
setup_malloc_check () {
|
|
|
|
: nothing
|
|
|
|
}
|
|
|
|
teardown_malloc_check () {
|
|
|
|
: nothing
|
|
|
|
}
|
|
|
|
else
|
2022-08-04 13:38:25 +00:00
|
|
|
_USE_GLIBC_TUNABLES=
|
|
|
|
if _GLIBC_VERSION=$(getconf GNU_LIBC_VERSION 2>/dev/null) &&
|
|
|
|
_GLIBC_VERSION=${_GLIBC_VERSION#"glibc "} &&
|
|
|
|
expr 2.34 \<= "$_GLIBC_VERSION" >/dev/null
|
|
|
|
then
|
|
|
|
_USE_GLIBC_TUNABLES=YesPlease
|
|
|
|
fi
|
2012-09-15 03:38:24 +00:00
|
|
|
setup_malloc_check () {
|
2022-03-09 21:41:43 +00:00
|
|
|
local g
|
|
|
|
local t
|
2012-09-15 03:38:24 +00:00
|
|
|
MALLOC_CHECK_=3 MALLOC_PERTURB_=165
|
|
|
|
export MALLOC_CHECK_ MALLOC_PERTURB_
|
2022-08-04 13:38:25 +00:00
|
|
|
if test -n "$_USE_GLIBC_TUNABLES"
|
2022-03-04 13:37:02 +00:00
|
|
|
then
|
|
|
|
g=
|
|
|
|
LD_PRELOAD="libc_malloc_debug.so.0"
|
|
|
|
for t in \
|
|
|
|
glibc.malloc.check=1 \
|
|
|
|
glibc.malloc.perturb=165
|
|
|
|
do
|
|
|
|
g="${g#:}:$t"
|
|
|
|
done
|
|
|
|
GLIBC_TUNABLES=$g
|
|
|
|
export LD_PRELOAD GLIBC_TUNABLES
|
|
|
|
fi
|
2012-09-15 03:38:24 +00:00
|
|
|
}
|
|
|
|
teardown_malloc_check () {
|
|
|
|
unset MALLOC_CHECK_ MALLOC_PERTURB_
|
2022-03-04 13:37:02 +00:00
|
|
|
unset LD_PRELOAD GLIBC_TUNABLES
|
2012-09-15 03:38:24 +00:00
|
|
|
}
|
|
|
|
fi
|
Add MALLOC_CHECK_ and MALLOC_PERTURB_ libc env to the test suite for detecting heap corruption
Recent versions of Linux libc (later than 5.4.23) and glibc (2.x)
include a malloc() implementation which is tunable via environment
variables. When MALLOC_CHECK_ is set, a special (less efficient)
implementation is used which is designed to be tolerant against
simple errors, such as double calls of free() with the same argument,
or overruns of a single byte (off-by-one bugs). When MALLOC_CHECK_
is set to 3, a diagnostic message is printed on stderr
and the program is aborted.
Setting the MALLOC_PERTURB_ environment variable causes the malloc
functions in libc to return memory which has been wiped and clear
memory when it is returned.
Of course this does not affect calloc which always does clear the memory.
The reason for this exercise is, of course, to find code which uses
memory returned by malloc without initializing it and code which uses
code after it is freed. valgrind can do this but it's costly to run.
The MALLOC_PERTURB_ exchanges the ability to detect problems in 100%
of the cases with speed.
The byte value used to initialize values returned by malloc is the byte
value of the environment value. The value used to clear memory is the
bitwise inverse. Setting MALLOC_PERTURB_ to zero disables the feature.
This technique can find hard to detect bugs.
It is therefore suggested to always use this flag (at least temporarily)
when testing out code or a new distribution.
But the test suite can use also valgrind(memcheck) via 'make valgrind'
or 'make GIT_TEST_OPTS="--valgrind"'.
Memcheck wraps client calls to malloc(), and puts a "red zone" on
each end of each block in order to detect access overruns.
Memcheck already detects double free() (up to the limit of the buffer
which remembers pending free()). Thus memcheck subsumes all the
documented coverage of MALLOC_CHECK_.
If MALLOC_CHECK_ is set non-zero when running memcheck, then the
overruns that might be detected by MALLOC_CHECK_ would be overruns
on the wrapped blocks which include the red zones. Thus MALLOC_CHECK_
would be checking memcheck, and not the client. This is not useful,
and actually is wasteful. The only possible [documented] advantage
of using MALLOC_CHECK_ and memcheck together, would be if MALLOC_CHECK_
detected duplicate free() in more cases than memcheck because memcheck's
buffer is too small.
Therefore we don't use MALLOC_CHECK_ and valgrind(memcheck) at the
same time.
Signed-off-by: Elia Pinto <gitter.spiros@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-09-14 16:54:22 +00:00
|
|
|
|
2007-04-24 18:21:47 +00:00
|
|
|
# Protect ourselves from common misconfiguration to export
|
|
|
|
# CDPATH into the environment
|
|
|
|
unset CDPATH
|
|
|
|
|
2009-11-18 16:15:19 +00:00
|
|
|
unset GREP_OPTIONS
|
2013-01-06 17:47:57 +00:00
|
|
|
unset UNZIP
|
2009-11-18 16:15:19 +00:00
|
|
|
|
2006-09-22 22:35:20 +00:00
|
|
|
case $(echo $GIT_TRACE |tr "[A-Z]" "[a-z]") in
|
2012-09-01 18:11:49 +00:00
|
|
|
1|2|true)
|
2015-03-13 04:50:56 +00:00
|
|
|
GIT_TRACE=4
|
2012-09-01 18:11:49 +00:00
|
|
|
;;
|
2006-09-02 16:23:48 +00:00
|
|
|
esac
|
|
|
|
|
2011-08-08 18:51:00 +00:00
|
|
|
# Line feed
|
|
|
|
LF='
|
|
|
|
'
|
|
|
|
|
2019-09-05 22:10:05 +00:00
|
|
|
# Single quote
|
|
|
|
SQ=\'
|
|
|
|
|
2014-12-15 23:15:20 +00:00
|
|
|
# UTF-8 ZERO WIDTH NON-JOINER, which HFS+ ignores
|
|
|
|
# when case-folding filenames
|
|
|
|
u200c=$(printf '\342\200\214')
|
|
|
|
|
2021-09-11 11:17:51 +00:00
|
|
|
export _x05 _x35 LF u200c EMPTY_TREE EMPTY_BLOB ZERO_OID OID_REGEX
|
2012-02-17 10:25:09 +00:00
|
|
|
|
2015-06-17 21:11:21 +00:00
|
|
|
test "x$TERM" != "xdumb" && (
|
2015-06-17 19:06:25 +00:00
|
|
|
test -t 1 &&
|
|
|
|
tput bold >/dev/null 2>&1 &&
|
|
|
|
tput setaf 1 >/dev/null 2>&1 &&
|
|
|
|
tput sgr0 >/dev/null 2>&1
|
|
|
|
) &&
|
|
|
|
color=t
|
2005-05-14 05:50:32 +00:00
|
|
|
|
2015-06-17 19:06:25 +00:00
|
|
|
if test -n "$color"
|
|
|
|
then
|
2015-06-17 21:11:21 +00:00
|
|
|
# Save the color control sequences now rather than run tput
|
|
|
|
# each time say_color() is called. This is done for two
|
|
|
|
# reasons:
|
|
|
|
# * TERM will be changed to dumb
|
|
|
|
# * HOME will be changed to a temporary directory and tput
|
|
|
|
# might need to read ~/.terminfo from the original HOME
|
|
|
|
# directory to get the control sequences
|
|
|
|
# Note: This approach assumes the control sequences don't end
|
|
|
|
# in a newline for any terminal of interest (command
|
|
|
|
# substitutions strip trailing newlines). Given that most
|
|
|
|
# (all?) terminals in common use are related to ECMA-48, this
|
|
|
|
# shouldn't be a problem.
|
|
|
|
say_color_error=$(tput bold; tput setaf 1) # bold red
|
|
|
|
say_color_skip=$(tput setaf 4) # blue
|
|
|
|
say_color_warn=$(tput setaf 3) # brown/yellow
|
|
|
|
say_color_pass=$(tput setaf 2) # green
|
|
|
|
say_color_info=$(tput setaf 6) # cyan
|
|
|
|
say_color_reset=$(tput sgr0)
|
|
|
|
say_color_="" # no formatting for normal text
|
2015-06-17 19:06:25 +00:00
|
|
|
say_color () {
|
2015-06-17 21:11:21 +00:00
|
|
|
test -z "$1" && test -n "$quiet" && return
|
|
|
|
eval "say_color_color=\$say_color_$1"
|
2015-06-17 19:06:25 +00:00
|
|
|
shift
|
2015-06-17 21:11:21 +00:00
|
|
|
printf "%s\\n" "$say_color_color$*$say_color_reset"
|
2015-06-17 19:06:25 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
say_color() {
|
|
|
|
test -z "$1" && test -n "$quiet" && return
|
|
|
|
shift
|
|
|
|
printf "%s\n" "$*"
|
|
|
|
}
|
|
|
|
fi
|
|
|
|
|
test-lib-functions: optionally keep HOME, TERM and SHELL in 'test_pause'
The 'test_pause' function, which is designed to help interactive
debugging and exploration of tests, currently inherits the value of HOME
and TERM set by 'test-lib.sh': HOME="$TRASH_DIRECTORY" and TERM=dumb. It
also invokes the shell defined by TEST_SHELL_PATH, which defaults to
/bin/sh (through SHELL_PATH).
Changing the value of HOME means that any customization configured in a
developers' shell startup files and any Git aliases defined in their
global Git configuration file are not available in the shell invoked by
'test_pause'.
Changing the value of TERM to 'dumb' means that colored output
is disabled for all commands in that shell.
Using /bin/sh as the shell invoked by 'test_pause' is not ideal since
some platforms (i.e. Debian and derivatives) use Dash as /bin/sh, and
this shell is usually compiled without readline support, which makes for
a poor interactive command line experience.
To make the interactive command line experience in the shell invoked by
'test_pause' more pleasant, save the values of HOME and TERM in
USER_HOME and USER_TERM before changing them in test-lib.sh, and add
options to 'test_pause' to optionally use these variables to invoke the
shell. Also add an option to invoke SHELL instead of TEST_SHELL_PATH, so
that developer's interactive shell is used.
We use options instead of changing the behaviour unconditionally since
these three variables can slightly change command behaviour. Moreover,
using the original HOME means commands could overwrite files in a user's
home directory. Be explicit about these caveats in the new 'Usage'
section in test-lib-functions.sh.
Finally, add '[options]' to the test_pause synopsys in t/README, and
mention that the full list of helper functions and their options can be
found in test-lib-functions.sh.
Helped-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Philippe Blain <levraiphilippeblain@gmail.com>
Acked-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-06 04:38:59 +00:00
|
|
|
USER_TERM="$TERM"
|
2015-06-17 21:11:21 +00:00
|
|
|
TERM=dumb
|
test-lib-functions: optionally keep HOME, TERM and SHELL in 'test_pause'
The 'test_pause' function, which is designed to help interactive
debugging and exploration of tests, currently inherits the value of HOME
and TERM set by 'test-lib.sh': HOME="$TRASH_DIRECTORY" and TERM=dumb. It
also invokes the shell defined by TEST_SHELL_PATH, which defaults to
/bin/sh (through SHELL_PATH).
Changing the value of HOME means that any customization configured in a
developers' shell startup files and any Git aliases defined in their
global Git configuration file are not available in the shell invoked by
'test_pause'.
Changing the value of TERM to 'dumb' means that colored output
is disabled for all commands in that shell.
Using /bin/sh as the shell invoked by 'test_pause' is not ideal since
some platforms (i.e. Debian and derivatives) use Dash as /bin/sh, and
this shell is usually compiled without readline support, which makes for
a poor interactive command line experience.
To make the interactive command line experience in the shell invoked by
'test_pause' more pleasant, save the values of HOME and TERM in
USER_HOME and USER_TERM before changing them in test-lib.sh, and add
options to 'test_pause' to optionally use these variables to invoke the
shell. Also add an option to invoke SHELL instead of TEST_SHELL_PATH, so
that developer's interactive shell is used.
We use options instead of changing the behaviour unconditionally since
these three variables can slightly change command behaviour. Moreover,
using the original HOME means commands could overwrite files in a user's
home directory. Be explicit about these caveats in the new 'Usage'
section in test-lib-functions.sh.
Finally, add '[options]' to the test_pause synopsys in t/README, and
mention that the full list of helper functions and their options can be
found in test-lib-functions.sh.
Helped-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Philippe Blain <levraiphilippeblain@gmail.com>
Acked-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-06 04:38:59 +00:00
|
|
|
export TERM USER_TERM
|
2015-06-17 21:11:21 +00:00
|
|
|
|
2021-12-01 08:53:15 +00:00
|
|
|
# What is written by tests to stdout and stderr is sent to different places
|
|
|
|
# depending on the test mode (e.g. /dev/null in non-verbose mode, piped to tee
|
|
|
|
# with --tee option, etc.). We save the original stdin to FD #6 and stdout and
|
|
|
|
# stderr to #5 and #7, so that the test framework can use them (e.g. for
|
|
|
|
# printing errors within the test framework) independently of the test mode.
|
|
|
|
exec 5>&1
|
|
|
|
exec 6<&0
|
|
|
|
exec 7>&2
|
|
|
|
|
2021-10-14 00:47:28 +00:00
|
|
|
_error_exit () {
|
2022-05-21 22:18:46 +00:00
|
|
|
finalize_test_output
|
2009-06-01 12:14:41 +00:00
|
|
|
GIT_EXIT_OK=t
|
2007-10-24 20:03:38 +00:00
|
|
|
exit 1
|
|
|
|
}
|
|
|
|
|
2021-10-14 00:47:28 +00:00
|
|
|
error () {
|
|
|
|
say_color error "error: $*"
|
|
|
|
_error_exit
|
|
|
|
}
|
|
|
|
|
tests: send "bug in the test script" errors to the script's stderr
Some of the functions in our test library check that they were invoked
properly with conditions like this:
test "$#" = 2 ||
error "bug in the test script: not 2 parameters to test-expect-success"
If this particular condition is triggered, then 'error' will abort the
whole test script with a bold red error message [1] right away.
However, under certain circumstances the test script will be aborted
completely silently, namely if:
- a similar condition in a test helper function like
'test_line_count' is triggered,
- which is invoked from the test script's "main" shell [2],
- and the test script is run manually (i.e. './t1234-foo.sh' as
opposed to 'make t1234-foo.sh' or 'make test') [3]
- and without the '--verbose' option,
because the error message is printed from within 'test_eval_', where
standard output is redirected either to /dev/null or to a log file.
The only indication that something is wrong is that not all tests in
the script are executed and at the end of the test script's output
there is no "# passed all N tests" message, which are subtle and can
easily go unnoticed, as I had to experience myself.
Send these "bug in the test script" error messages directly to the
test scripts standard error and thus to the terminal, so those bugs
will be much harder to overlook. Instead of updating all ~20 such
'error' calls with a redirection, let's add a BUG() function to
'test-lib.sh', wrapping an 'error' call with the proper redirection
and also including the common prefix of those error messages, and
convert all those call sites [4] to use this new BUG() function
instead.
[1] That particular error message from 'test_expect_success' is
printed in color only when running with or without '--verbose';
with '--tee' or '--verbose-log' the error is printed without
color, but it is printed to the terminal nonetheless.
[2] If such a condition is triggered in a subshell of a test, then
'error' won't be able to abort the whole test script, but only the
subshell, which in turn causes the test to fail in the usual way,
indicating loudly and clearly that something is wrong.
[3] Well, 'error' aborts the test script the same way when run
manually or by 'make' or 'prove', but both 'make' and 'prove' pay
attention to the test script's exit status, and even a silently
aborted test script would then trigger those tools' usual
noticable error messages.
[4] Strictly speaking, not all those 'error' calls need that
redirection to send their output to the terminal, see e.g.
'test_expect_success' in the opening example, but I think it's
better to be consistent.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-11-19 13:13:26 +00:00
|
|
|
BUG () {
|
|
|
|
error >&7 "bug in the test script: $*"
|
|
|
|
}
|
|
|
|
|
test-lib.sh: use "Bail out!" syntax on bad SANITIZE=leak use
Improve the "GIT_TEST_PASSING_SANITIZE_LEAK=true" test mode added in
956d2e4639b (tests: add a test mode for SANITIZE=leak, run it in CI,
2021-09-23) to use a TAP "Bail out!" message when exiting. This will
cause the test run to exit immediately under a TAP consumer like
"prove(1)".
See 614fe015212 (test-lib: bail out when "-v" used under "prove",
2016-10-22) for the initial introduction of "Bail out!" to the
--verbose being amended here.
Before this compiling with "SANITIZE=" and running the tests with
"prove(1)" would cause all the tests to be run to the end (output
trimmed for fewer columns):
$ GIT_TEST_PASSING_SANITIZE_LEAK=true make
rm -f -r 'test-results'
*** prove ***
t0000-basic.sh ......... Dubious, test returned 1 (wstat 256, 0x100)
No subtests run
t0001-init.sh .......... Dubious, test returned 1 (wstat 256, 0x100)
No subtests run
[...output where we list every single t[0-9]*.sh file as failing snipped]
Whereas now we'll fail early, like this ("->" line wrapping added):
$ GIT_TEST_PASSING_SANITIZE_LEAK=true make
[...]
t0000-basic.sh ..................................... Bailout called. Further testing stopped:
-> GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except when compiled with SANITIZE=leak
FAILED--Further testing stopped: GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except
-> when compiled with SANITIZE=leak
make: *** [Makefile:53: prove] Error 1
This change also adds a red color to the "Bailout called" line, as
we're now using "say_color error". That improves existing output in
the case of e.g.:
$ prove -j8 t[0-9]*.sh :: -v
Bailout called. Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
FAILED--Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
We don't need to have a "Bail out! " prefix when we're not running
under a TAP consumer (i.e. if test -n "$HARNESS_ACTIVE"), but let's
not make the output conditional on that. Showing it under e.g.:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0095-bloom.sh
Bail out! GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except when compiled with SANITIZE=leak
Doesn't harm anything, and I don't think the (small) complexity of
only adding this if we're under "$HARNESS_ACTIVE" is worth it.
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-14 00:47:29 +00:00
|
|
|
BAIL_OUT () {
|
|
|
|
test $# -ne 1 && BUG "1 param"
|
|
|
|
|
|
|
|
# Do not change "Bail out! " string. It's part of TAP syntax:
|
|
|
|
# https://testanything.org/tap-specification.html
|
|
|
|
local bail_out="Bail out! "
|
|
|
|
local message="$1"
|
|
|
|
|
2021-12-01 08:53:15 +00:00
|
|
|
say_color >&5 error $bail_out "$message"
|
test-lib.sh: use "Bail out!" syntax on bad SANITIZE=leak use
Improve the "GIT_TEST_PASSING_SANITIZE_LEAK=true" test mode added in
956d2e4639b (tests: add a test mode for SANITIZE=leak, run it in CI,
2021-09-23) to use a TAP "Bail out!" message when exiting. This will
cause the test run to exit immediately under a TAP consumer like
"prove(1)".
See 614fe015212 (test-lib: bail out when "-v" used under "prove",
2016-10-22) for the initial introduction of "Bail out!" to the
--verbose being amended here.
Before this compiling with "SANITIZE=" and running the tests with
"prove(1)" would cause all the tests to be run to the end (output
trimmed for fewer columns):
$ GIT_TEST_PASSING_SANITIZE_LEAK=true make
rm -f -r 'test-results'
*** prove ***
t0000-basic.sh ......... Dubious, test returned 1 (wstat 256, 0x100)
No subtests run
t0001-init.sh .......... Dubious, test returned 1 (wstat 256, 0x100)
No subtests run
[...output where we list every single t[0-9]*.sh file as failing snipped]
Whereas now we'll fail early, like this ("->" line wrapping added):
$ GIT_TEST_PASSING_SANITIZE_LEAK=true make
[...]
t0000-basic.sh ..................................... Bailout called. Further testing stopped:
-> GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except when compiled with SANITIZE=leak
FAILED--Further testing stopped: GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except
-> when compiled with SANITIZE=leak
make: *** [Makefile:53: prove] Error 1
This change also adds a red color to the "Bailout called" line, as
we're now using "say_color error". That improves existing output in
the case of e.g.:
$ prove -j8 t[0-9]*.sh :: -v
Bailout called. Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
FAILED--Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
We don't need to have a "Bail out! " prefix when we're not running
under a TAP consumer (i.e. if test -n "$HARNESS_ACTIVE"), but let's
not make the output conditional on that. Showing it under e.g.:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0095-bloom.sh
Bail out! GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except when compiled with SANITIZE=leak
Doesn't harm anything, and I don't think the (small) complexity of
only adding this if we're under "$HARNESS_ACTIVE" is worth it.
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-14 00:47:29 +00:00
|
|
|
_error_exit
|
|
|
|
}
|
|
|
|
|
2007-10-24 20:03:38 +00:00
|
|
|
say () {
|
|
|
|
say_color info "$*"
|
|
|
|
}
|
|
|
|
|
test-lib: bail out when "-v" used under "prove"
When there is a TAP harness consuming the output of our test
scripts, the "--verbose" breaks the output by mingling
test command output with TAP. Because the TAP::Harness
module used by "prove" is fairly lenient, this _usually_
works, but it violates the spec, and things get very
confusing if the commands happen to output a line that looks
like TAP (e.g., the word "ok" on its own line).
Let's detect this situation and complain. Just calling
error() isn't great, though; prove will tell us that the
script failed, but the message doesn't make it through to
the user. Instead, we can use the special TAP signal "Bail
out!". This not only shows the message to the user, but
instructs the harness to stop running the tests entirely.
This is exactly what we want here, as the problem is in the
command-line options, and every test script would produce
the same error.
The result looks like this (the first "Bailout called" line
is in red if prove uses color on your terminal):
$ make GIT_TEST_OPTS='--verbose --tee'
rm -f -r 'test-results'
*** prove ***
Bailout called. Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
FAILED--Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
Makefile:39: recipe for target 'prove' failed
make: *** [prove] Error 255
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-10-22 04:45:06 +00:00
|
|
|
if test -n "$HARNESS_ACTIVE"
|
|
|
|
then
|
|
|
|
if test "$verbose" = t || test -n "$verbose_only"
|
|
|
|
then
|
test-lib.sh: use "Bail out!" syntax on bad SANITIZE=leak use
Improve the "GIT_TEST_PASSING_SANITIZE_LEAK=true" test mode added in
956d2e4639b (tests: add a test mode for SANITIZE=leak, run it in CI,
2021-09-23) to use a TAP "Bail out!" message when exiting. This will
cause the test run to exit immediately under a TAP consumer like
"prove(1)".
See 614fe015212 (test-lib: bail out when "-v" used under "prove",
2016-10-22) for the initial introduction of "Bail out!" to the
--verbose being amended here.
Before this compiling with "SANITIZE=" and running the tests with
"prove(1)" would cause all the tests to be run to the end (output
trimmed for fewer columns):
$ GIT_TEST_PASSING_SANITIZE_LEAK=true make
rm -f -r 'test-results'
*** prove ***
t0000-basic.sh ......... Dubious, test returned 1 (wstat 256, 0x100)
No subtests run
t0001-init.sh .......... Dubious, test returned 1 (wstat 256, 0x100)
No subtests run
[...output where we list every single t[0-9]*.sh file as failing snipped]
Whereas now we'll fail early, like this ("->" line wrapping added):
$ GIT_TEST_PASSING_SANITIZE_LEAK=true make
[...]
t0000-basic.sh ..................................... Bailout called. Further testing stopped:
-> GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except when compiled with SANITIZE=leak
FAILED--Further testing stopped: GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except
-> when compiled with SANITIZE=leak
make: *** [Makefile:53: prove] Error 1
This change also adds a red color to the "Bailout called" line, as
we're now using "say_color error". That improves existing output in
the case of e.g.:
$ prove -j8 t[0-9]*.sh :: -v
Bailout called. Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
FAILED--Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
We don't need to have a "Bail out! " prefix when we're not running
under a TAP consumer (i.e. if test -n "$HARNESS_ACTIVE"), but let's
not make the output conditional on that. Showing it under e.g.:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0095-bloom.sh
Bail out! GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except when compiled with SANITIZE=leak
Doesn't harm anything, and I don't think the (small) complexity of
only adding this if we're under "$HARNESS_ACTIVE" is worth it.
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-14 00:47:29 +00:00
|
|
|
BAIL_OUT 'verbose mode forbidden under TAP harness; try --verbose-log'
|
test-lib: bail out when "-v" used under "prove"
When there is a TAP harness consuming the output of our test
scripts, the "--verbose" breaks the output by mingling
test command output with TAP. Because the TAP::Harness
module used by "prove" is fairly lenient, this _usually_
works, but it violates the spec, and things get very
confusing if the commands happen to output a line that looks
like TAP (e.g., the word "ok" on its own line).
Let's detect this situation and complain. Just calling
error() isn't great, though; prove will tell us that the
script failed, but the message doesn't make it through to
the user. Instead, we can use the special TAP signal "Bail
out!". This not only shows the message to the user, but
instructs the harness to stop running the tests entirely.
This is exactly what we want here, as the problem is in the
command-line options, and every test script would produce
the same error.
The result looks like this (the first "Bailout called" line
is in red if prove uses color on your terminal):
$ make GIT_TEST_OPTS='--verbose --tee'
rm -f -r 'test-results'
*** prove ***
Bailout called. Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
FAILED--Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
Makefile:39: recipe for target 'prove' failed
make: *** [prove] Error 255
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-10-22 04:45:06 +00:00
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
2007-11-10 14:17:25 +00:00
|
|
|
test "${test_description}" != "" ||
|
|
|
|
error "Test script did not set test_description."
|
|
|
|
|
|
|
|
if test "$help" = "t"
|
|
|
|
then
|
2014-03-18 00:14:11 +00:00
|
|
|
printf '%s\n' "$test_description"
|
2007-11-10 14:17:25 +00:00
|
|
|
exit 0
|
|
|
|
fi
|
|
|
|
|
test-lib: add --verbose-log option
The "--verbose" option redirects output from arbitrary
test commands to stdout. This is useful for examining the
output manually, like:
./t5547-push-quarantine.sh -v | less
But it also means that the output is intermingled with the
TAP directives, which can confuse a TAP parser like "prove".
This has always been a potential problem, but became an
issue recently when one test happened to output the word
"ok" on a line by itself, which prove interprets as a test
success:
$ prove t5547-push-quarantine.sh :: -v
t5547-push-quarantine.sh .. 1/? To dest.git
* [new branch] HEAD -> master
To dest.git
! [remote rejected] reject -> reject (pre-receive hook declined)
error: failed to push some refs to 'dest.git'
fatal: git cat-file d08c8eba97f4e683ece08654c7c8d2ba0c03b129: bad file
t5547-push-quarantine.sh .. Failed -1/4 subtests
Test Summary Report
-------------------
t5547-push-quarantine.sh (Wstat: 0 Tests: 5 Failed: 0)
Parse errors: Tests out of sequence. Found (2) but expected (3)
Tests out of sequence. Found (3) but expected (4)
Tests out of sequence. Found (4) but expected (5)
Bad plan. You planned 4 tests but ran 5.
Files=1, Tests=5, 0 wallclock secs ( 0.01 usr + 0.01 sys = 0.02 CPU)
Result: FAIL
One answer is "if it hurts, don't do it", but that's not
quite the whole story. The Travis tests use "--verbose
--tee" so that they can get the benefit of prove's parallel
options, along with a verbose log in case there is a
failure. We just need the verbose output to go to the log,
but keep stdout clean.
Getting this right turns out to be surprisingly difficult.
Here's the progression of alternatives I considered:
1. Add an option to write verbose output to stderr. This is
hard to capture, though, because we want each test to
have its own log (because they're all run in parallel
and the jumbled output would be useless).
2. Add an option to write verbose output to a file in
test-results. This works, but the log is missing all of
the non-verbose output, which gives context.
3. Like (2), but teach say_color() to additionally output
to the log. This mostly works, but misses any output
that happens outside of the say() functions (which isn't
a lot, but is a potential maintenance headache).
4. Like (2), but make the log file the same as the "--tee"
file. That almost works, but now we have two processes
opening the same file. That gives us two separate
descriptors, each with their own idea of the current
position. They'll each start writing at offset 0, and
overwrite each other's data.
5. Like (4), but in each case open the file for appending.
That atomically positions each write at the end of the
file.
It's possible we may still get sheared writes between
the two processes, but this is already the case when
writing to stdout. It's not a problem in practice
because the test harness generally waits for snippets to
finish before writing the TAP output.
We can ignore buffering issues with tee, because POSIX
mandates that it does not buffer. Likewise, POSIX
specifies "tee -a", so it should be available
everywhere.
This patch implements option (5), which seems to work well
in practice.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-10-21 10:48:00 +00:00
|
|
|
if test "$verbose_log" = "t"
|
|
|
|
then
|
|
|
|
exec 3>>"$GIT_TEST_TEE_OUTPUT_FILE" 4>&3
|
|
|
|
elif test "$verbose" = "t"
|
2005-05-14 05:50:32 +00:00
|
|
|
then
|
|
|
|
exec 4>&2 3>&1
|
|
|
|
else
|
|
|
|
exec 4>/dev/null 3>/dev/null
|
|
|
|
fi
|
|
|
|
|
test-lib: set BASH_XTRACEFD automatically
Passing "-x" to a test script enables the shell's "set -x"
tracing, which can help with tracking down the command that
is causing a failure. Unfortunately, it can also _cause_
failures in some tests that redirect the stderr of a shell
function. Inside the function the shell continues to
respect "set -x", and the trace output is collected along
with whatever stderr is generated normally by the function.
You can see an example of this by running:
./t0040-parse-options.sh -x -i
which will fail immediately in the first test, as it
expects:
test_must_fail some-cmd 2>output.err
to leave output.err empty (but with "-x" it has our trace
output).
Unfortunately there isn't a portable or scalable solution to
this. We could teach test_must_fail to disable "set -x", but
that doesn't help any of the other functions or subshells.
However, we can work around it by pointing the "set -x"
output to our descriptor 4, which always points to the
original stderr of the test script. Unfortunately this only
works for bash, but it's better than nothing (and other
shells will just ignore the BASH_XTRACEFD variable).
The patch itself is a simple one-liner, but note the caveats
in the accompanying comments.
Automatic tests for our "-x" option may be a bit too meta
(and a pain, because they are bash-specific), but I did
confirm that it works correctly both with regular "-x" and
with "--verbose-only=1". This works because the latter flips
"set -x" off and on for particular tests (if it didn't, we
would get tracing for all tests, as going to descriptor 4
effectively circumvents the verbose flag).
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-05-11 13:44:04 +00:00
|
|
|
# Send any "-x" output directly to stderr to avoid polluting tests
|
|
|
|
# which capture stderr. We can do this unconditionally since it
|
|
|
|
# has no effect if tracing isn't turned on.
|
|
|
|
#
|
|
|
|
# Note that this sets up the trace fd as soon as we assign the variable, so it
|
|
|
|
# must come after the creation of descriptor 4 above. Likewise, we must never
|
|
|
|
# unset this, as it has the side effect of closing descriptor 4, which we
|
|
|
|
# use to show verbose tests to the user.
|
|
|
|
#
|
|
|
|
# Note also that we don't need or want to export it. The tracing is local to
|
|
|
|
# this shell, and we would not want to influence any shells we exec.
|
|
|
|
BASH_XTRACEFD=4
|
|
|
|
|
2005-05-14 05:50:32 +00:00
|
|
|
test_failure=0
|
|
|
|
test_count=0
|
2008-02-01 09:50:53 +00:00
|
|
|
test_fixed=0
|
|
|
|
test_broken=0
|
2008-06-08 14:04:33 +00:00
|
|
|
test_success=0
|
2005-05-14 05:50:32 +00:00
|
|
|
|
2021-11-20 15:03:59 +00:00
|
|
|
test_missing_prereq=
|
|
|
|
|
2010-06-24 17:44:46 +00:00
|
|
|
test_external_has_tap=0
|
|
|
|
|
2008-02-27 19:28:45 +00:00
|
|
|
die () {
|
2009-06-01 12:14:41 +00:00
|
|
|
code=$?
|
test-lib: introduce 'test_atexit'
When running Apache, 'git daemon', or p4d, we want to kill them at the
end of the test script, otherwise a leftover daemon process will keep
its port open indefinitely, and thus will interfere with subsequent
executions of the same test script.
So far, we stop these daemon processes "manually", i.e.:
- by registering functions or commands in the trap on EXIT to stop
the daemon while preserving the last seen exit code before the
trap (to deal with a failure when run with '--immediate' or with
interrupts by ctrl-C),
- and by invoking these functions/commands last thing before
'test_done' (and sometimes restoring the test framework's default
trap on EXIT, to prevent the daemons from being killed twice).
On one hand, we do this inconsistently, e.g. 'git p4' tests invoke
different functions in the trap on EXIT and in the last test before
'test_done', and they neither restore the test framework's default trap
on EXIT nor preserve the last seen exit code. On the other hand, this
is error prone, because, as shown in a previous patch in this series,
any output from the cleanup commands in the trap on EXIT can prevent a
proper cleanup when a test script run with '--verbose-log' and certain
shells, notably 'dash', is interrupted.
Let's introduce 'test_atexit', which is loosely modeled after
'test_when_finished', but has a broader scope: rather than running the
commands after the current test case, run them when the test script
finishes, and also run them when the test is interrupted, or exits
early in case of a failure while the '--immediate' option is in
effect.
When running the cleanup commands at the end of a successful test,
then they will be run in 'test_done' before it removes the trash
directory, i.e. the cleanup commands will still be able to access any
pidfiles or socket files in there. When running the cleanup commands
after an interrupt or failure with '--immediate', then they will be
run in the trap on EXIT. In both cases they will be run in
'test_eval_', i.e. both standard error and output of all cleanup
commands will go where they should according to the '-v' or
'--verbose-log' options, and thus won't cause any troubles when
interrupting a test script run with '--verbose-log'.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-03-13 12:24:11 +00:00
|
|
|
# This is responsible for running the atexit commands even when a
|
|
|
|
# test script run with '--immediate' fails, or when the user hits
|
|
|
|
# ctrl-C, i.e. when 'test_done' is not invoked at all.
|
|
|
|
test_atexit_handler || code=$?
|
2009-06-01 12:14:41 +00:00
|
|
|
if test -n "$GIT_EXIT_OK"
|
|
|
|
then
|
|
|
|
exit $code
|
|
|
|
else
|
|
|
|
echo >&5 "FATAL: Unexpected exit with code $code"
|
|
|
|
exit 1
|
|
|
|
fi
|
2008-02-27 19:28:45 +00:00
|
|
|
}
|
|
|
|
|
2009-06-01 12:14:41 +00:00
|
|
|
GIT_EXIT_OK=
|
2009-01-19 23:43:26 +00:00
|
|
|
trap 'die' EXIT
|
test-lib: fix interrupt handling with 'dash' and '--verbose-log -x'
When a test script run with 'dash' and '--verbose-log -x' is
interrupted by ctrl-C, SIGTERM, or closing the terminal window, then
most of the time the registered EXIT trap actions are not executed.
This is an annoying issue with tests involving daemons, because they
should run cleanup commands to kill those daemon processes in the trap
on EXIT, but since these cleanup commands are not executed, the
daemons are left alive and keep their port open, thus interfering with
subsequent execution of the same test script.
The cause of this issue is the subtle combination of several factors
(bear with me, or skip over the indented part):
- Even when the test script is interrupted, the cleanup commands are
not run in the trap on INT, TERM, or HUP, but in the trap on EXIT
after the trap on the signals invokes 'exit' [1].
- According to POSIX [2]:
"The environment in which the shell executes a trap on EXIT
shall be identical to the environment immediately after the last
command executed before the trap on EXIT was taken."
Pertinent to the issue at hand is that all open file descriptors
and the state of '-x' tracing should be preserved. All shells
I've tried [3] preserve '-x'. Unfortunately, however:
- 'dash' doesn't conform to this when it comes to open file
descriptors: even when standard output and/or error are
redirected somewhere when 'exit' is invoked, anything written
to them in the trap on EXIT goes to the script's original
stdout and stderr [4].
We can't dismiss this with a simple "it doesn't conform to
POSIX, so we don't care", because 'dash' is the default
/bin/sh in some of the more popular Linux distros.
- As far as I can tell, POSIX doesn't explicitly say anything
about the environment of trap actions for various signals.
In practice it seems that most shells behave sensibly and
preserve both open file descriptors and the state of '-x'
tracing for the traps on INT, TERM, and HUP, including even
'dash'. The exceptions are 'mksh' and 'lksh': they do
preserve '-x', but not the open file descriptors.
- When a test script run with '-x' tracing enabled is interrupted,
then it's very likely that the signal arrives mid-test, i.e.:
- while '-x' tracing is enabled, and, consequently, our trap
actions on INT, TERM, HUP, and EXIT will produce trace output
as well.
- while standard output and error are redirected to a log file,
to the test script's original standard output and error, or to
/dev/null, depending on whether the test script was run with
'--verbose-log', '-v', or neither. According to the above, we
can't rely on these redirections still be in effect when
running the traps on INT, TERM, HUP, and/or EXIT.
- When a test script is run with '--verbose-log', then the test
script is re-executed with its standard output and error piped
into 'tee', in order to send the "regular" non-verbose test's
output both to the terminal and to the log file. When the test is
interrupted, then the signal interrupts the downstream 'tee' as
well.
Putting these together, when a test script run with 'dash' and
'--verbose-log -x' is interrupted, then 'dash' tries to write the
trace output from the EXIT trap to the script's original standard
error, but it very likely can't, because the 'tee' downstream of the
pipe is interrupted as well. This causes the shell running the test
script to die because of SIGPIPE, without running any of the commands
in the EXIT trap.
Disable '-x' tracing in the trap on INT, TERM, and HUP to avoid this
issue, as it disables tracing in the chained trap on EXIT as well.
Wrap it in a '{ ... } 2>/dev/null' block, so the trace of the command
disabling the tracing doesn't go to standard error either [5].
Note that it's not only '-x' tracing that can be problematic, but any
shell builtin, e.g. 'echo', that writes to standard output or error in
the trap on EXIT, while a test running with 'dash' and '--verbose-log'
(even without '-x') is interrupted. As far as I can tell, this is not
an issue at the moment:
- The cleanup commands to stop the credential-helper, Apache, or
'p4d' don't use any such shell builtins.
- stop_git_daemon() does use 'say' and 'error', both wrappers around
'echo', but it redirects 'say' to fd 3, i.e. to the log file, and
while 'error' does write to standard output, it comes only after
the daemon was killed.
- The non-builtin commands that actually stop the daemons ('kill',
'apache2 -k stop', 'git credential-cache exit') are silent, so they
won't get SIGPIPE before finishing their job.
[1] The trap on EXIT must run cleanup commands, because we want to
stop any daemons when a test script run with '--immediate' fails
and exits early with error. By chaining up the trap on signals to
the trap on EXIT we can deal with cleanup commands a bit simpler,
because the tests involving daemons only have to set a single
trap.
[2] http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#trap
[3] The shells I tried: dash, Bash, ksh, ksh93, mksh, lksh, yash,
BusyBox sh, FreeBSD /bin/sh, NetBSD /bin/sh.
[4] $ cat trap-output.sh
#!/bin/sh
trap "echo output; echo error >&2" EXIT
{ exit; } >OUT 2>ERR
$ dash ./trap-output.sh
output
error
$ wc -c OUT ERR
0 OUT
0 ERR
On a related note, 'ksh', 'ksh93', and BusyBox sh don't conform to
the specs in this respect, either.
[5] This '{ set +x; } 2>/dev/null' trick won't help those shells that
show trace output for any redirections and don't preserve open
file descriptors for the trap on INT, TERM and HUP. The only such
shells I'm aware of are 'mksh' and 'lksh'.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-03-13 12:24:09 +00:00
|
|
|
# Disable '-x' tracing, because with some shells, notably dash, it
|
|
|
|
# prevents running the cleanup commands when a test script run with
|
|
|
|
# '--verbose-log -x' is interrupted.
|
|
|
|
trap '{ code=$?; set +x; } 2>/dev/null; exit $code' INT TERM HUP
|
2005-08-11 16:00:40 +00:00
|
|
|
|
2012-02-17 10:25:08 +00:00
|
|
|
# The user-facing functions are loaded from a separate file so that
|
|
|
|
# test_perf subshells can have them too
|
2012-06-25 05:01:35 +00:00
|
|
|
. "$TEST_DIRECTORY/test-lib-functions.sh"
|
2010-10-16 18:36:58 +00:00
|
|
|
|
2005-05-14 07:24:27 +00:00
|
|
|
# You are not expected to call test_ok_ and test_failure_ directly, use
|
2013-10-27 09:56:33 +00:00
|
|
|
# the test_expect_* functions instead.
|
2005-05-14 07:24:27 +00:00
|
|
|
|
|
|
|
test_ok_ () {
|
2009-02-05 19:59:27 +00:00
|
|
|
test_success=$(($test_success + 1))
|
2013-10-19 21:06:08 +00:00
|
|
|
say_color "" "ok $test_count - $@"
|
2022-05-21 22:18:56 +00:00
|
|
|
finalize_test_case_output ok "$@"
|
2005-05-14 05:50:32 +00:00
|
|
|
}
|
|
|
|
|
2022-07-27 23:13:32 +00:00
|
|
|
_invert_exit_code_failure_end_blurb () {
|
|
|
|
say_color warn "# faked up failures as TODO & now exiting with 0 due to --invert-exit-code"
|
|
|
|
}
|
|
|
|
|
2005-05-14 07:24:27 +00:00
|
|
|
test_failure_ () {
|
2022-05-21 22:18:56 +00:00
|
|
|
failure_label=$1
|
2009-02-05 19:59:27 +00:00
|
|
|
test_failure=$(($test_failure + 1))
|
2022-07-27 23:13:32 +00:00
|
|
|
local pfx=""
|
|
|
|
if test -n "$invert_exit_code" # && test -n "$HARNESS_ACTIVE"
|
|
|
|
then
|
|
|
|
pfx="# TODO induced breakage (--invert-exit-code):"
|
|
|
|
fi
|
|
|
|
say_color error "not ok $test_count - ${pfx:+$pfx }$1"
|
2005-07-23 02:09:34 +00:00
|
|
|
shift
|
2014-03-18 00:14:11 +00:00
|
|
|
printf '%s\n' "$*" | sed -e 's/^/# /'
|
2022-03-23 20:51:31 +00:00
|
|
|
if test -n "$immediate"
|
|
|
|
then
|
|
|
|
say_color error "1..$test_count"
|
2022-07-27 23:13:32 +00:00
|
|
|
if test -n "$invert_exit_code"
|
|
|
|
then
|
|
|
|
finalize_test_output
|
|
|
|
_invert_exit_code_failure_end_blurb
|
|
|
|
GIT_EXIT_OK=t
|
|
|
|
exit 0
|
|
|
|
fi
|
2022-03-23 20:51:31 +00:00
|
|
|
_error_exit
|
|
|
|
fi
|
2022-05-21 22:18:56 +00:00
|
|
|
finalize_test_case_output failure "$failure_label" "$@"
|
2005-05-14 07:24:27 +00:00
|
|
|
}
|
|
|
|
|
2008-02-01 09:50:53 +00:00
|
|
|
test_known_broken_ok_ () {
|
|
|
|
test_fixed=$(($test_fixed+1))
|
2022-07-27 23:13:29 +00:00
|
|
|
say_color error "ok $test_count - $1 # TODO known breakage vanished"
|
|
|
|
finalize_test_case_output fixed "$1"
|
2008-02-01 09:50:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
test_known_broken_failure_ () {
|
|
|
|
test_broken=$(($test_broken+1))
|
2022-07-27 23:13:29 +00:00
|
|
|
say_color warn "not ok $test_count - $1 # TODO known breakage"
|
|
|
|
finalize_test_case_output broken "$1"
|
2008-02-01 09:50:53 +00:00
|
|
|
}
|
2005-05-14 07:24:27 +00:00
|
|
|
|
|
|
|
test_debug () {
|
2005-08-11 05:53:27 +00:00
|
|
|
test "$debug" = "" || eval "$1"
|
2005-05-14 05:50:32 +00:00
|
|
|
}
|
|
|
|
|
2013-06-18 12:25:58 +00:00
|
|
|
match_pattern_list () {
|
|
|
|
arg="$1"
|
|
|
|
shift
|
|
|
|
test -z "$*" && return 1
|
test-lib: avoid accidental globbing in match_pattern_list()
We have a custom match_pattern_list() function which we use for matching
test names (like "t1234") against glob-like patterns (like "t1???") for
$GIT_SKIP_TESTS, --verbose-only, etc.
Those patterns may have multiple whitespace-separated elements (e.g.,
"t0* t1234 t5?78"). The callers of match_pattern_list thus pass the
strings unquoted, so that the shell does the usual field-splitting into
separate arguments.
But this also means the shell will do the usual globbing for each
argument, which can result in us seeing an expansion based on what's in
the filesystem, rather than the real pattern. For example, if I have the
path "t5000" in the filesystem, and you feed the pattern "t?000", that
_should_ match the string "t0000", but it won't after the shell has
expanded it to "t5000".
This has been a bug ever since that function was introduced. But it
didn't usually trigger since we typically use the function inside the
trash directory, which has a very limited set of files that are unlikely
to match. It became a lot easier to trigger after edc23840b0 (test-lib:
bring $remove_trash out of retirement, 2021-05-10), because now we match
$GIT_SKIP_TESTS before even entering the trash directory. So the t5000
example above can be seen with:
GIT_SKIP_TESTS=t?000 ./t0000-basic.sh
which should skip all tests but doesn't.
We can fix this by using "set -f" to ask the shell not to glob (which is
in POSIX, so should hopefully be portable enough). We only want to do
this in a subshell (to avoid polluting the rest of the script), which
means we need to get the whole string intact into the match_pattern_list
function by quoting it. Arguably this is a good idea anyway, since it
makes it much more obvious that we intend to split, and it's not simply
sloppy scripting.
Diagnosed-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-06-16 10:23:07 +00:00
|
|
|
# We need to use "$*" to get field-splitting, but we want to
|
|
|
|
# disable globbing, since we are matching against an arbitrary
|
|
|
|
# $arg, not what's in the filesystem. Using "set -f" accomplishes
|
|
|
|
# that, but we must do it in a subshell to avoid impacting the
|
|
|
|
# rest of the script. The exit value of the subshell becomes
|
|
|
|
# the function's return value.
|
|
|
|
(
|
|
|
|
set -f
|
|
|
|
for pattern_ in $*
|
|
|
|
do
|
|
|
|
case "$arg" in
|
|
|
|
$pattern_)
|
|
|
|
exit 0
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
done
|
|
|
|
exit 1
|
|
|
|
)
|
2013-06-18 12:25:58 +00:00
|
|
|
}
|
|
|
|
|
2014-04-30 09:50:44 +00:00
|
|
|
match_test_selector_list () {
|
2020-10-18 00:23:45 +00:00
|
|
|
operation="$1"
|
|
|
|
shift
|
2014-04-30 09:50:44 +00:00
|
|
|
title="$1"
|
|
|
|
shift
|
|
|
|
arg="$1"
|
|
|
|
shift
|
|
|
|
test -z "$1" && return 0
|
|
|
|
|
2020-10-18 00:23:45 +00:00
|
|
|
# Commas are accepted as separators.
|
2014-04-30 09:50:44 +00:00
|
|
|
OLDIFS=$IFS
|
2020-10-18 00:23:45 +00:00
|
|
|
IFS=','
|
2014-04-30 09:50:44 +00:00
|
|
|
set -- $1
|
|
|
|
IFS=$OLDIFS
|
|
|
|
|
|
|
|
# If the first selector is negative we include by default.
|
|
|
|
include=
|
|
|
|
case "$1" in
|
|
|
|
!*) include=t ;;
|
|
|
|
esac
|
|
|
|
|
|
|
|
for selector
|
|
|
|
do
|
|
|
|
orig_selector=$selector
|
|
|
|
|
|
|
|
positive=t
|
|
|
|
case "$selector" in
|
|
|
|
!*)
|
|
|
|
positive=
|
|
|
|
selector=${selector##?}
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
|
|
|
test -z "$selector" && continue
|
|
|
|
|
|
|
|
case "$selector" in
|
|
|
|
*-*)
|
|
|
|
if expr "z${selector%%-*}" : "z[0-9]*[^0-9]" >/dev/null
|
|
|
|
then
|
2020-10-18 00:23:45 +00:00
|
|
|
echo "error: $operation: invalid non-numeric in range" \
|
2014-04-30 09:50:44 +00:00
|
|
|
"start: '$orig_selector'" >&2
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
if expr "z${selector#*-}" : "z[0-9]*[^0-9]" >/dev/null
|
|
|
|
then
|
2020-10-18 00:23:45 +00:00
|
|
|
echo "error: $operation: invalid non-numeric in range" \
|
2014-04-30 09:50:44 +00:00
|
|
|
"end: '$orig_selector'" >&2
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
if expr "z$selector" : "z[0-9]*[^0-9]" >/dev/null
|
|
|
|
then
|
2020-10-18 00:23:45 +00:00
|
|
|
case "$title" in *${selector}*)
|
|
|
|
include=$positive
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
continue
|
2014-04-30 09:50:44 +00:00
|
|
|
fi
|
|
|
|
esac
|
|
|
|
|
|
|
|
# Short cut for "obvious" cases
|
|
|
|
test -z "$include" && test -z "$positive" && continue
|
|
|
|
test -n "$include" && test -n "$positive" && continue
|
|
|
|
|
|
|
|
case "$selector" in
|
|
|
|
-*)
|
|
|
|
if test $arg -le ${selector#-}
|
|
|
|
then
|
|
|
|
include=$positive
|
|
|
|
fi
|
|
|
|
;;
|
|
|
|
*-)
|
|
|
|
if test $arg -ge ${selector%-}
|
|
|
|
then
|
|
|
|
include=$positive
|
|
|
|
fi
|
|
|
|
;;
|
|
|
|
*-*)
|
|
|
|
if test ${selector%%-*} -le $arg \
|
|
|
|
&& test $arg -le ${selector#*-}
|
|
|
|
then
|
|
|
|
include=$positive
|
|
|
|
fi
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
if test $arg -eq $selector
|
|
|
|
then
|
|
|
|
include=$positive
|
|
|
|
fi
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
done
|
|
|
|
|
|
|
|
test -n "$include"
|
|
|
|
}
|
|
|
|
|
2013-06-23 18:12:56 +00:00
|
|
|
maybe_teardown_verbose () {
|
|
|
|
test -z "$verbose_only" && return
|
|
|
|
exec 4>/dev/null 3>/dev/null
|
|
|
|
verbose=
|
|
|
|
}
|
|
|
|
|
|
|
|
last_verbose=t
|
|
|
|
maybe_setup_verbose () {
|
|
|
|
test -z "$verbose_only" && return
|
test-lib: avoid accidental globbing in match_pattern_list()
We have a custom match_pattern_list() function which we use for matching
test names (like "t1234") against glob-like patterns (like "t1???") for
$GIT_SKIP_TESTS, --verbose-only, etc.
Those patterns may have multiple whitespace-separated elements (e.g.,
"t0* t1234 t5?78"). The callers of match_pattern_list thus pass the
strings unquoted, so that the shell does the usual field-splitting into
separate arguments.
But this also means the shell will do the usual globbing for each
argument, which can result in us seeing an expansion based on what's in
the filesystem, rather than the real pattern. For example, if I have the
path "t5000" in the filesystem, and you feed the pattern "t?000", that
_should_ match the string "t0000", but it won't after the shell has
expanded it to "t5000".
This has been a bug ever since that function was introduced. But it
didn't usually trigger since we typically use the function inside the
trash directory, which has a very limited set of files that are unlikely
to match. It became a lot easier to trigger after edc23840b0 (test-lib:
bring $remove_trash out of retirement, 2021-05-10), because now we match
$GIT_SKIP_TESTS before even entering the trash directory. So the t5000
example above can be seen with:
GIT_SKIP_TESTS=t?000 ./t0000-basic.sh
which should skip all tests but doesn't.
We can fix this by using "set -f" to ask the shell not to glob (which is
in POSIX, so should hopefully be portable enough). We only want to do
this in a subshell (to avoid polluting the rest of the script), which
means we need to get the whole string intact into the match_pattern_list
function by quoting it. Arguably this is a good idea anyway, since it
makes it much more obvious that we intend to split, and it's not simply
sloppy scripting.
Diagnosed-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-06-16 10:23:07 +00:00
|
|
|
if match_pattern_list $test_count "$verbose_only"
|
2013-06-23 18:12:56 +00:00
|
|
|
then
|
|
|
|
exec 4>&2 3>&1
|
|
|
|
# Emit a delimiting blank line when going from
|
|
|
|
# non-verbose to verbose. Within verbose mode the
|
|
|
|
# delimiter is printed by test_expect_*. The choice
|
|
|
|
# of the initial $last_verbose is such that before
|
|
|
|
# test 1, we do not print it.
|
|
|
|
test -z "$last_verbose" && echo >&3 ""
|
|
|
|
verbose=t
|
|
|
|
else
|
|
|
|
exec 4>/dev/null 3>/dev/null
|
|
|
|
verbose=
|
|
|
|
fi
|
|
|
|
last_verbose=$verbose
|
|
|
|
}
|
|
|
|
|
2013-06-23 18:12:57 +00:00
|
|
|
maybe_teardown_valgrind () {
|
|
|
|
test -z "$GIT_VALGRIND" && return
|
|
|
|
GIT_VALGRIND_ENABLED=
|
|
|
|
}
|
|
|
|
|
|
|
|
maybe_setup_valgrind () {
|
|
|
|
test -z "$GIT_VALGRIND" && return
|
2013-10-19 21:06:07 +00:00
|
|
|
if test -z "$valgrind_only"
|
2013-06-23 18:12:57 +00:00
|
|
|
then
|
|
|
|
GIT_VALGRIND_ENABLED=t
|
|
|
|
return
|
|
|
|
fi
|
|
|
|
GIT_VALGRIND_ENABLED=
|
test-lib: avoid accidental globbing in match_pattern_list()
We have a custom match_pattern_list() function which we use for matching
test names (like "t1234") against glob-like patterns (like "t1???") for
$GIT_SKIP_TESTS, --verbose-only, etc.
Those patterns may have multiple whitespace-separated elements (e.g.,
"t0* t1234 t5?78"). The callers of match_pattern_list thus pass the
strings unquoted, so that the shell does the usual field-splitting into
separate arguments.
But this also means the shell will do the usual globbing for each
argument, which can result in us seeing an expansion based on what's in
the filesystem, rather than the real pattern. For example, if I have the
path "t5000" in the filesystem, and you feed the pattern "t?000", that
_should_ match the string "t0000", but it won't after the shell has
expanded it to "t5000".
This has been a bug ever since that function was introduced. But it
didn't usually trigger since we typically use the function inside the
trash directory, which has a very limited set of files that are unlikely
to match. It became a lot easier to trigger after edc23840b0 (test-lib:
bring $remove_trash out of retirement, 2021-05-10), because now we match
$GIT_SKIP_TESTS before even entering the trash directory. So the t5000
example above can be seen with:
GIT_SKIP_TESTS=t?000 ./t0000-basic.sh
which should skip all tests but doesn't.
We can fix this by using "set -f" to ask the shell not to glob (which is
in POSIX, so should hopefully be portable enough). We only want to do
this in a subshell (to avoid polluting the rest of the script), which
means we need to get the whole string intact into the match_pattern_list
function by quoting it. Arguably this is a good idea anyway, since it
makes it much more obvious that we intend to split, and it's not simply
sloppy scripting.
Diagnosed-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-06-16 10:23:07 +00:00
|
|
|
if match_pattern_list $test_count "$valgrind_only"
|
2013-06-23 18:12:57 +00:00
|
|
|
then
|
|
|
|
GIT_VALGRIND_ENABLED=t
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
tests: do not let lazy prereqs inside `test_expect_*` turn off tracing
The `test_expect_*` functions use `test_eval_` and so does
`test_run_lazy_prereq_`. If tracing is enabled via the `-x` option,
`test_eval_` turns on tracing while evaluating the code block, and turns
it off directly after it.
This is unwanted for nested invocations.
One somewhat surprising example of this is when running a test that
calls `test_i18ngrep`: that function requires the `C_LOCALE_OUTPUT`
prereq, and that prereq is a lazy one, so it is evaluated via
`test_eval_`, the command tracing is turned off, and the test case
continues to run _without tracing the commands_.
Another somewhat surprising example is when one lazy prereq depends on
another lazy prereq: the former will call `test_have_prereq` with the
latter one, which in turn calls `test_eval_` and -- you guessed it --
tracing (if enabled) will be turned off _before_ returning to evaluating
the other lazy prereq.
As we will introduce just such a scenario with the GPG, GPGSM and
RFC1991 prereqs, let's fix that by introducing a variable that keeps
track of the current trace level: nested `test_eval_` calls will
increment and then decrement the level, and only when it reaches 0, the
tracing will _actually_ be turned off.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-26 15:35:26 +00:00
|
|
|
trace_level_=0
|
test-lib: disable trace when test is not verbose
The "-x" test-script option turns on the shell's "-x"
tracing, which can help show why a particular test is
failing. Unfortunately, this can create false negatives in
some tests if they invoke a shell function with its stderr
redirected. t5512.10 is such a test, as it does:
test_must_fail git ls-remote refs*master >actual 2>&1 &&
test_cmp exp actual
The "actual" file gets the "-x" trace for the test_must_fail
function, which prevents it from matching the expected
output.
There's no way to avoid this without managing the
trace flag inside each sub-function, which isn't really a
workable solution. But unless you specifically care about
t5512.10, we can work around it by enabling tracing only for
the specific tests we want.
You can already do:
./t5512-ls-remote.sh -x --verbose-only=16
to see the trace only for a specific test. But that doesn't
_disable_ the tracing in the other tests; it just sends it
to /dev/null. However, there's no point in generating a
trace that the user won't see, so we can simply disable
tracing whenever it doesn't have a matching verbose flag.
The normal case of just "./t5512-ls-remote.sh -x" stays the
same, as "-x" already implies "--verbose" (and
"--verbose-only" overrides "--verbose", which is why this
works at all). And for our test, we need only check
$verbose, as maybe_setup_verbose will have already
set that flag based on the $verbose_only list).
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-08-06 05:33:57 +00:00
|
|
|
want_trace () {
|
2017-12-08 10:47:17 +00:00
|
|
|
test "$trace" = t && {
|
|
|
|
test "$verbose" = t || test "$verbose_log" = t
|
|
|
|
}
|
test-lib: disable trace when test is not verbose
The "-x" test-script option turns on the shell's "-x"
tracing, which can help show why a particular test is
failing. Unfortunately, this can create false negatives in
some tests if they invoke a shell function with its stderr
redirected. t5512.10 is such a test, as it does:
test_must_fail git ls-remote refs*master >actual 2>&1 &&
test_cmp exp actual
The "actual" file gets the "-x" trace for the test_must_fail
function, which prevents it from matching the expected
output.
There's no way to avoid this without managing the
trace flag inside each sub-function, which isn't really a
workable solution. But unless you specifically care about
t5512.10, we can work around it by enabling tracing only for
the specific tests we want.
You can already do:
./t5512-ls-remote.sh -x --verbose-only=16
to see the trace only for a specific test. But that doesn't
_disable_ the tracing in the other tests; it just sends it
to /dev/null. However, there's no point in generating a
trace that the user won't see, so we can simply disable
tracing whenever it doesn't have a matching verbose flag.
The normal case of just "./t5512-ls-remote.sh -x" stays the
same, as "-x" already implies "--verbose" (and
"--verbose-only" overrides "--verbose", which is why this
works at all). And for our test, we need only check
$verbose, as maybe_setup_verbose will have already
set that flag based on the $verbose_only list).
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-08-06 05:33:57 +00:00
|
|
|
}
|
|
|
|
|
test-lib.sh: support -x option for shell-tracing
Usually running a test under "-v" makes it clear which
command is failing. However, sometimes it can be useful to
also see a complete trace of the shell commands being run in
the test. You can do so without any support from the test
suite by running "sh -x tXXXX-foo.sh". However, this
produces quite a large bit of output, as we see a trace of
the entire test suite.
This patch instead introduces a "-x" option to the test
scripts (i.e., "./tXXXX-foo.sh -x"). When enabled, this
turns on "set -x" only for the tests themselves. This can
still be a bit verbose, but should keep things to a more
manageable level. You can even use "--verbose-only" to see
the trace only for a specific test.
The implementation is a little invasive. We turn on the "set
-x" inside the "eval" of the test code. This lets the eval
itself avoid being reported in the trace (which would be
long, and redundant with the verbose listing we already
showed). And then after the eval runs, we do some trickery
with stderr to avoid showing the "set +x" to the user.
We also show traces for test_cleanup functions (since they
can impact the test outcome, too). However, we do avoid
running the noop ":" cleanup (the default if the test does
not use test_cleanup at all), as it creates unnecessary
noise in the "set -x" output.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-10 06:47:27 +00:00
|
|
|
# This is a separate function because some tests use
|
|
|
|
# "return" to end a test_expect_success block early
|
|
|
|
# (and we want to make sure we run any cleanup like
|
|
|
|
# "set +x").
|
|
|
|
test_eval_inner_ () {
|
tests: skip test_eval_ in internal chain-lint
To check for broken &&-chains, we run "fail_117 && $1" as a test
snippet, and check the exit code. We use test_eval_ to do so, because
that's the way we run the actual test.
But we don't need any of its niceties, like "set -x" tracing. In fact,
they hinder us, because we have to explicitly disable them. So let's
skip that and use "eval" more directly, which is simpler. I had hoped it
would also be faster, but it doesn't seem to produce a measurable
improvement (probably because it's just running internal shell commands,
with no subshells or forks).
Note that there is one gotcha: even though we don't intend to run any of
the commands if the &&-chain is intact, an error like this:
test_expect_success 'broken' '
# this next line breaks the &&-chain
true
# and then this one is executed even by the linter
return 1
'
means we'll "return 1" from the eval, and thus from test_run_(). We
actually do notice this in test_expect_success, but only by saying "hey,
this test didn't say it was OK, so it must have failed", which is not
right (it should say "broken &&-chain").
We can handle this by calling test_eval_inner_() instead, which is our
trick for wrapping "return" in a test snippet. But to do that, we have
to push the trace code out of that inner function and into test_eval_().
This is arguably where it belonged in the first place, but it never
mattered because the "inner_" function had only one caller.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-03-30 19:30:56 +00:00
|
|
|
eval "$*"
|
test-lib.sh: support -x option for shell-tracing
Usually running a test under "-v" makes it clear which
command is failing. However, sometimes it can be useful to
also see a complete trace of the shell commands being run in
the test. You can do so without any support from the test
suite by running "sh -x tXXXX-foo.sh". However, this
produces quite a large bit of output, as we see a trace of
the entire test suite.
This patch instead introduces a "-x" option to the test
scripts (i.e., "./tXXXX-foo.sh -x"). When enabled, this
turns on "set -x" only for the tests themselves. This can
still be a bit verbose, but should keep things to a more
manageable level. You can even use "--verbose-only" to see
the trace only for a specific test.
The implementation is a little invasive. We turn on the "set
-x" inside the "eval" of the test code. This lets the eval
itself avoid being reported in the trace (which would be
long, and redundant with the verbose listing we already
showed). And then after the eval runs, we do some trickery
with stderr to avoid showing the "set +x" to the user.
We also show traces for test_cleanup functions (since they
can impact the test outcome, too). However, we do avoid
running the noop ":" cleanup (the default if the test does
not use test_cleanup at all), as it creates unnecessary
noise in the "set -x" output.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-10 06:47:27 +00:00
|
|
|
}
|
|
|
|
|
test: cope better with use of return for errors
In olden times, tests would quietly exit the script when they failed
at an inconvenient moment, which was a little disconcerting.
Therefore v0.99.5~24^2~4 (Trapping exit in tests, using return for
errors, 2005-08-10) switched to an idiom of using "return" instead,
wrapping evaluation of test code in a function to make that safe:
test_run_ () {
eval >&3 2>&4 "$1"
eval_ret="$?"
return 0
}
Years later, the implementation of test_when_finished (v1.7.1.1~95,
2010-05-02) and v1.7.2-rc2~1^2~13 (test-lib: output a newline before
"ok" under a TAP harness, 2010-06-24) took advantage of test_run_ as a
place to put code shared by all test assertion functions, without
paying attention to the function's former purpose:
test_run_ () {
...
eval >&3 2>&4 "$1"
eval_ret=$?
if should run cleanup
then
eval >&3 2>&4 "$test_cleanup"
fi
if TAP format requires a newline here
then
echo
fi
return 0
}
That means cleanup commands and the newline to put TAP output at
column 0 are skipped when tests use "return" to fail early. Fix it by
introducing a test_eval_ function to catch the "return", with a
comment explaining the new function's purpose for the next person who
might touch this code.
Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
Acked-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-08-08 01:17:09 +00:00
|
|
|
test_eval_ () {
|
test-lib: silence "-x" cleanup under bash
When the test suite's "-x" option is used with bash, we end
up seeing cleanup cruft in the output:
$ bash t0001-init.sh -x
[...]
++ diff -u expected actual
+ test_eval_ret_=0
+ want_trace
+ test t = t
+ test t = t
+ set +x
ok 42 - re-init from a linked worktree
This ranges from mildly annoying (for a successful test) to
downright confusing (when we say "last command exited with
error", but it's really 5 commands back).
We normally are able to suppress this cleanup. As the
in-code comment explains, we can't convince the shell not to
print it, but we can redirect its stderr elsewhere.
But since d88785e424 (test-lib: set BASH_XTRACEFD
automatically, 2016-05-11), that doesn't hold for bash. It
sends the "set -x" output directly to descriptor 4, not to
stderr.
We can fix this by also redirecting descriptor 4, and
paying close attention to which commands redirected and
which are not (see the updated comment).
Two alternatives I considered and rejected:
- unsetting and setting BASH_XTRACEFD; doing so closes the
descriptor, which we must avoid
- we could keep everything in a single block as before,
redirect 4>/dev/null there, but retain 5>&4 as a copy.
And then selectively restore 4>&5 for commands which
should be allowed to trace. This would work, but the
descriptor swapping seems unnecessarily confusing.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-12-08 10:47:08 +00:00
|
|
|
# If "-x" tracing is in effect, then we want to avoid polluting stderr
|
|
|
|
# with non-test commands. But once in "set -x" mode, we cannot prevent
|
test-lib.sh: support -x option for shell-tracing
Usually running a test under "-v" makes it clear which
command is failing. However, sometimes it can be useful to
also see a complete trace of the shell commands being run in
the test. You can do so without any support from the test
suite by running "sh -x tXXXX-foo.sh". However, this
produces quite a large bit of output, as we see a trace of
the entire test suite.
This patch instead introduces a "-x" option to the test
scripts (i.e., "./tXXXX-foo.sh -x"). When enabled, this
turns on "set -x" only for the tests themselves. This can
still be a bit verbose, but should keep things to a more
manageable level. You can even use "--verbose-only" to see
the trace only for a specific test.
The implementation is a little invasive. We turn on the "set
-x" inside the "eval" of the test code. This lets the eval
itself avoid being reported in the trace (which would be
long, and redundant with the verbose listing we already
showed). And then after the eval runs, we do some trickery
with stderr to avoid showing the "set +x" to the user.
We also show traces for test_cleanup functions (since they
can impact the test outcome, too). However, we do avoid
running the noop ":" cleanup (the default if the test does
not use test_cleanup at all), as it creates unnecessary
noise in the "set -x" output.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-10 06:47:27 +00:00
|
|
|
# the shell from printing the "set +x" to turn it off (nor the saving
|
|
|
|
# of $? before that). But we can make sure that the output goes to
|
|
|
|
# /dev/null.
|
|
|
|
#
|
test-lib: silence "-x" cleanup under bash
When the test suite's "-x" option is used with bash, we end
up seeing cleanup cruft in the output:
$ bash t0001-init.sh -x
[...]
++ diff -u expected actual
+ test_eval_ret_=0
+ want_trace
+ test t = t
+ test t = t
+ set +x
ok 42 - re-init from a linked worktree
This ranges from mildly annoying (for a successful test) to
downright confusing (when we say "last command exited with
error", but it's really 5 commands back).
We normally are able to suppress this cleanup. As the
in-code comment explains, we can't convince the shell not to
print it, but we can redirect its stderr elsewhere.
But since d88785e424 (test-lib: set BASH_XTRACEFD
automatically, 2016-05-11), that doesn't hold for bash. It
sends the "set -x" output directly to descriptor 4, not to
stderr.
We can fix this by also redirecting descriptor 4, and
paying close attention to which commands redirected and
which are not (see the updated comment).
Two alternatives I considered and rejected:
- unsetting and setting BASH_XTRACEFD; doing so closes the
descriptor, which we must avoid
- we could keep everything in a single block as before,
redirect 4>/dev/null there, but retain 5>&4 as a copy.
And then selectively restore 4>&5 for commands which
should be allowed to trace. This would work, but the
descriptor swapping seems unnecessarily confusing.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-12-08 10:47:08 +00:00
|
|
|
# There are a few subtleties here:
|
|
|
|
#
|
|
|
|
# - we have to redirect descriptor 4 in addition to 2, to cover
|
|
|
|
# BASH_XTRACEFD
|
|
|
|
#
|
|
|
|
# - the actual eval has to come before the redirection block (since
|
|
|
|
# it needs to see descriptor 4 to set up its stderr)
|
|
|
|
#
|
|
|
|
# - likewise, any error message we print must be outside the block to
|
|
|
|
# access descriptor 4
|
|
|
|
#
|
|
|
|
# - checking $? has to come immediately after the eval, but it must
|
|
|
|
# be _inside_ the block to avoid polluting the "set -x" output
|
|
|
|
#
|
|
|
|
|
tests: skip test_eval_ in internal chain-lint
To check for broken &&-chains, we run "fail_117 && $1" as a test
snippet, and check the exit code. We use test_eval_ to do so, because
that's the way we run the actual test.
But we don't need any of its niceties, like "set -x" tracing. In fact,
they hinder us, because we have to explicitly disable them. So let's
skip that and use "eval" more directly, which is simpler. I had hoped it
would also be faster, but it doesn't seem to produce a measurable
improvement (probably because it's just running internal shell commands,
with no subshells or forks).
Note that there is one gotcha: even though we don't intend to run any of
the commands if the &&-chain is intact, an error like this:
test_expect_success 'broken' '
# this next line breaks the &&-chain
true
# and then this one is executed even by the linter
return 1
'
means we'll "return 1" from the eval, and thus from test_run_(). We
actually do notice this in test_expect_success, but only by saying "hey,
this test didn't say it was OK, so it must have failed", which is not
right (it should say "broken &&-chain").
We can handle this by calling test_eval_inner_() instead, which is our
trick for wrapping "return" in a test snippet. But to do that, we have
to push the trace code out of that inner function and into test_eval_().
This is arguably where it belonged in the first place, but it never
mattered because the "inner_" function had only one caller.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-03-30 19:30:56 +00:00
|
|
|
# Do not add anything extra (including LF) after '$*'
|
|
|
|
test_eval_inner_ </dev/null >&3 2>&4 "
|
|
|
|
want_trace && trace_level_=$(($trace_level_+1)) && set -x
|
|
|
|
$*"
|
test-lib.sh: support -x option for shell-tracing
Usually running a test under "-v" makes it clear which
command is failing. However, sometimes it can be useful to
also see a complete trace of the shell commands being run in
the test. You can do so without any support from the test
suite by running "sh -x tXXXX-foo.sh". However, this
produces quite a large bit of output, as we see a trace of
the entire test suite.
This patch instead introduces a "-x" option to the test
scripts (i.e., "./tXXXX-foo.sh -x"). When enabled, this
turns on "set -x" only for the tests themselves. This can
still be a bit verbose, but should keep things to a more
manageable level. You can even use "--verbose-only" to see
the trace only for a specific test.
The implementation is a little invasive. We turn on the "set
-x" inside the "eval" of the test code. This lets the eval
itself avoid being reported in the trace (which would be
long, and redundant with the verbose listing we already
showed). And then after the eval runs, we do some trickery
with stderr to avoid showing the "set +x" to the user.
We also show traces for test_cleanup functions (since they
can impact the test outcome, too). However, we do avoid
running the noop ":" cleanup (the default if the test does
not use test_cleanup at all), as it creates unnecessary
noise in the "set -x" output.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-10 06:47:27 +00:00
|
|
|
{
|
|
|
|
test_eval_ret_=$?
|
test-lib: disable trace when test is not verbose
The "-x" test-script option turns on the shell's "-x"
tracing, which can help show why a particular test is
failing. Unfortunately, this can create false negatives in
some tests if they invoke a shell function with its stderr
redirected. t5512.10 is such a test, as it does:
test_must_fail git ls-remote refs*master >actual 2>&1 &&
test_cmp exp actual
The "actual" file gets the "-x" trace for the test_must_fail
function, which prevents it from matching the expected
output.
There's no way to avoid this without managing the
trace flag inside each sub-function, which isn't really a
workable solution. But unless you specifically care about
t5512.10, we can work around it by enabling tracing only for
the specific tests we want.
You can already do:
./t5512-ls-remote.sh -x --verbose-only=16
to see the trace only for a specific test. But that doesn't
_disable_ the tracing in the other tests; it just sends it
to /dev/null. However, there's no point in generating a
trace that the user won't see, so we can simply disable
tracing whenever it doesn't have a matching verbose flag.
The normal case of just "./t5512-ls-remote.sh -x" stays the
same, as "-x" already implies "--verbose" (and
"--verbose-only" overrides "--verbose", which is why this
works at all). And for our test, we need only check
$verbose, as maybe_setup_verbose will have already
set that flag based on the $verbose_only list).
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-08-06 05:33:57 +00:00
|
|
|
if want_trace
|
test-lib.sh: support -x option for shell-tracing
Usually running a test under "-v" makes it clear which
command is failing. However, sometimes it can be useful to
also see a complete trace of the shell commands being run in
the test. You can do so without any support from the test
suite by running "sh -x tXXXX-foo.sh". However, this
produces quite a large bit of output, as we see a trace of
the entire test suite.
This patch instead introduces a "-x" option to the test
scripts (i.e., "./tXXXX-foo.sh -x"). When enabled, this
turns on "set -x" only for the tests themselves. This can
still be a bit verbose, but should keep things to a more
manageable level. You can even use "--verbose-only" to see
the trace only for a specific test.
The implementation is a little invasive. We turn on the "set
-x" inside the "eval" of the test code. This lets the eval
itself avoid being reported in the trace (which would be
long, and redundant with the verbose listing we already
showed). And then after the eval runs, we do some trickery
with stderr to avoid showing the "set +x" to the user.
We also show traces for test_cleanup functions (since they
can impact the test outcome, too). However, we do avoid
running the noop ":" cleanup (the default if the test does
not use test_cleanup at all), as it creates unnecessary
noise in the "set -x" output.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-10 06:47:27 +00:00
|
|
|
then
|
tests: do not let lazy prereqs inside `test_expect_*` turn off tracing
The `test_expect_*` functions use `test_eval_` and so does
`test_run_lazy_prereq_`. If tracing is enabled via the `-x` option,
`test_eval_` turns on tracing while evaluating the code block, and turns
it off directly after it.
This is unwanted for nested invocations.
One somewhat surprising example of this is when running a test that
calls `test_i18ngrep`: that function requires the `C_LOCALE_OUTPUT`
prereq, and that prereq is a lazy one, so it is evaluated via
`test_eval_`, the command tracing is turned off, and the test case
continues to run _without tracing the commands_.
Another somewhat surprising example is when one lazy prereq depends on
another lazy prereq: the former will call `test_have_prereq` with the
latter one, which in turn calls `test_eval_` and -- you guessed it --
tracing (if enabled) will be turned off _before_ returning to evaluating
the other lazy prereq.
As we will introduce just such a scenario with the GPG, GPGSM and
RFC1991 prereqs, let's fix that by introducing a variable that keeps
track of the current trace level: nested `test_eval_` calls will
increment and then decrement the level, and only when it reaches 0, the
tracing will _actually_ be turned off.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-26 15:35:26 +00:00
|
|
|
test 1 = $trace_level_ && set +x
|
|
|
|
trace_level_=$(($trace_level_-1))
|
test-lib.sh: support -x option for shell-tracing
Usually running a test under "-v" makes it clear which
command is failing. However, sometimes it can be useful to
also see a complete trace of the shell commands being run in
the test. You can do so without any support from the test
suite by running "sh -x tXXXX-foo.sh". However, this
produces quite a large bit of output, as we see a trace of
the entire test suite.
This patch instead introduces a "-x" option to the test
scripts (i.e., "./tXXXX-foo.sh -x"). When enabled, this
turns on "set -x" only for the tests themselves. This can
still be a bit verbose, but should keep things to a more
manageable level. You can even use "--verbose-only" to see
the trace only for a specific test.
The implementation is a little invasive. We turn on the "set
-x" inside the "eval" of the test code. This lets the eval
itself avoid being reported in the trace (which would be
long, and redundant with the verbose listing we already
showed). And then after the eval runs, we do some trickery
with stderr to avoid showing the "set +x" to the user.
We also show traces for test_cleanup functions (since they
can impact the test outcome, too). However, we do avoid
running the noop ":" cleanup (the default if the test does
not use test_cleanup at all), as it creates unnecessary
noise in the "set -x" output.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-10 06:47:27 +00:00
|
|
|
fi
|
test-lib: silence "-x" cleanup under bash
When the test suite's "-x" option is used with bash, we end
up seeing cleanup cruft in the output:
$ bash t0001-init.sh -x
[...]
++ diff -u expected actual
+ test_eval_ret_=0
+ want_trace
+ test t = t
+ test t = t
+ set +x
ok 42 - re-init from a linked worktree
This ranges from mildly annoying (for a successful test) to
downright confusing (when we say "last command exited with
error", but it's really 5 commands back).
We normally are able to suppress this cleanup. As the
in-code comment explains, we can't convince the shell not to
print it, but we can redirect its stderr elsewhere.
But since d88785e424 (test-lib: set BASH_XTRACEFD
automatically, 2016-05-11), that doesn't hold for bash. It
sends the "set -x" output directly to descriptor 4, not to
stderr.
We can fix this by also redirecting descriptor 4, and
paying close attention to which commands redirected and
which are not (see the updated comment).
Two alternatives I considered and rejected:
- unsetting and setting BASH_XTRACEFD; doing so closes the
descriptor, which we must avoid
- we could keep everything in a single block as before,
redirect 4>/dev/null there, but retain 5>&4 as a copy.
And then selectively restore 4>&5 for commands which
should be allowed to trace. This would work, but the
descriptor swapping seems unnecessarily confusing.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-12-08 10:47:08 +00:00
|
|
|
} 2>/dev/null 4>&2
|
|
|
|
|
|
|
|
if test "$test_eval_ret_" != 0 && want_trace
|
|
|
|
then
|
|
|
|
say_color error >&4 "error: last command exited with \$?=$test_eval_ret_"
|
|
|
|
fi
|
test-lib.sh: support -x option for shell-tracing
Usually running a test under "-v" makes it clear which
command is failing. However, sometimes it can be useful to
also see a complete trace of the shell commands being run in
the test. You can do so without any support from the test
suite by running "sh -x tXXXX-foo.sh". However, this
produces quite a large bit of output, as we see a trace of
the entire test suite.
This patch instead introduces a "-x" option to the test
scripts (i.e., "./tXXXX-foo.sh -x"). When enabled, this
turns on "set -x" only for the tests themselves. This can
still be a bit verbose, but should keep things to a more
manageable level. You can even use "--verbose-only" to see
the trace only for a specific test.
The implementation is a little invasive. We turn on the "set
-x" inside the "eval" of the test code. This lets the eval
itself avoid being reported in the trace (which would be
long, and redundant with the verbose listing we already
showed). And then after the eval runs, we do some trickery
with stderr to avoid showing the "set +x" to the user.
We also show traces for test_cleanup functions (since they
can impact the test outcome, too). However, we do avoid
running the noop ":" cleanup (the default if the test does
not use test_cleanup at all), as it creates unnecessary
noise in the "set -x" output.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-10 06:47:27 +00:00
|
|
|
return $test_eval_ret_
|
test: cope better with use of return for errors
In olden times, tests would quietly exit the script when they failed
at an inconvenient moment, which was a little disconcerting.
Therefore v0.99.5~24^2~4 (Trapping exit in tests, using return for
errors, 2005-08-10) switched to an idiom of using "return" instead,
wrapping evaluation of test code in a function to make that safe:
test_run_ () {
eval >&3 2>&4 "$1"
eval_ret="$?"
return 0
}
Years later, the implementation of test_when_finished (v1.7.1.1~95,
2010-05-02) and v1.7.2-rc2~1^2~13 (test-lib: output a newline before
"ok" under a TAP harness, 2010-06-24) took advantage of test_run_ as a
place to put code shared by all test assertion functions, without
paying attention to the function's former purpose:
test_run_ () {
...
eval >&3 2>&4 "$1"
eval_ret=$?
if should run cleanup
then
eval >&3 2>&4 "$test_cleanup"
fi
if TAP format requires a newline here
then
echo
fi
return 0
}
That means cleanup commands and the newline to put TAP output at
column 0 are skipped when tests use "return" to fail early. Fix it by
introducing a test_eval_ function to catch the "return", with a
comment explaining the new function's purpose for the next person who
might touch this code.
Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
Acked-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-08-08 01:17:09 +00:00
|
|
|
}
|
|
|
|
|
2023-03-30 19:27:49 +00:00
|
|
|
fail_117 () {
|
|
|
|
return 117
|
|
|
|
}
|
|
|
|
|
2005-08-11 03:56:21 +00:00
|
|
|
test_run_ () {
|
2010-05-06 08:41:10 +00:00
|
|
|
test_cleanup=:
|
2011-06-27 18:02:22 +00:00
|
|
|
expecting_failure=$2
|
t/test-lib: introduce --chain-lint option
It's easy to miss an "&&"-chain in a test script, like:
test_expect_success 'check something important' '
cmd1 &&
cmd2
cmd3
'
The test harness will notice if cmd3 fails, but a failure of
cmd1 or cmd2 will go unnoticed, as their exit status is lost
after cmd3 runs.
The toy example above is easy to spot because the "cmds" are
all the same length, but real code is much more complicated.
It's also difficult to detect these situations by statically
analyzing the shell code with regexps (like the
check-non-portable-shell script does); there's too much
context required to know whether a &&-chain is appropriate
on a given line or not.
This patch instead lets the shell check each test by
sticking a command with a specific and unusual return code
at the top of each test, like:
(exit 117) &&
cmd1 &&
cmd2
cmd3
In a well-formed test, the non-zero exit from the first
command prevents any of the rest from being run, and the
test's exit code is 117. In a bad test (like the one above),
the 117 is lost, and cmd3 is run.
When we encounter a failure of this check, we abort the test
script entirely. For one thing, we have no clue which subset
of the commands in the test snippet were actually run.
Running further tests would be pointless, because we're now
in an unknown state. And two, this is not a "test failure"
in the traditional sense. The test script is buggy, not the
code it is testing. We should be able to fix these problems
in the script once, and not have them come back later as a
regression in git's code.
After checking a test snippet for --chain-lint, we do still
run the test itself. We could actually have a pure-lint
mode which just checks each test, but there are a few
reasons not to. One, because the tests are executing
arbitrary code, which could impact the later environment
(e.g., that could impact which set of tests we run at all).
And two, because a pure-lint mode would still be expensive
to run, because a significant amount of code runs outside of
the test_expect_* blocks. Instead, this option is designed
to be used as part of a normal test suite run, where it adds
very little overhead.
Turning on this option detects quite a few problems in
existing tests, which will be fixed in subsequent patches.
However, there are a number of places it cannot reach:
- it cannot find a failure to break out of loops on error,
like:
cmd1 &&
for i in a b c; do
cmd2 $i
done &&
cmd3
which will not notice failures of "cmd2 a" or "cmd b"
- it cannot find a missing &&-chain inside a block or
subfunction, like:
foo () {
cmd1
cmd2
}
foo &&
bar
which will not notice a failure of cmd1.
- it only checks tests that you run; every platform will
have some tests skipped due to missing prequisites,
so it's impossible to say from one run that the test
suite is free of broken &&-chains. However, all tests get
run by _somebody_, so eventually we will notice problems.
- it does not operate on test_when_finished or prerequisite
blocks. It could, but these tends to be much shorter and
less of a problem, so I punted on them in this patch.
This patch was inspired by an earlier patch by Jonathan
Nieder:
http://article.gmane.org/gmane.comp.version-control.git/235913
This implementation and all bugs are mine.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-03-20 10:05:48 +00:00
|
|
|
|
2015-04-22 20:09:57 +00:00
|
|
|
if test "${GIT_TEST_CHAIN_LINT:-1}" != 0; then
|
t/test-lib: introduce --chain-lint option
It's easy to miss an "&&"-chain in a test script, like:
test_expect_success 'check something important' '
cmd1 &&
cmd2
cmd3
'
The test harness will notice if cmd3 fails, but a failure of
cmd1 or cmd2 will go unnoticed, as their exit status is lost
after cmd3 runs.
The toy example above is easy to spot because the "cmds" are
all the same length, but real code is much more complicated.
It's also difficult to detect these situations by statically
analyzing the shell code with regexps (like the
check-non-portable-shell script does); there's too much
context required to know whether a &&-chain is appropriate
on a given line or not.
This patch instead lets the shell check each test by
sticking a command with a specific and unusual return code
at the top of each test, like:
(exit 117) &&
cmd1 &&
cmd2
cmd3
In a well-formed test, the non-zero exit from the first
command prevents any of the rest from being run, and the
test's exit code is 117. In a bad test (like the one above),
the 117 is lost, and cmd3 is run.
When we encounter a failure of this check, we abort the test
script entirely. For one thing, we have no clue which subset
of the commands in the test snippet were actually run.
Running further tests would be pointless, because we're now
in an unknown state. And two, this is not a "test failure"
in the traditional sense. The test script is buggy, not the
code it is testing. We should be able to fix these problems
in the script once, and not have them come back later as a
regression in git's code.
After checking a test snippet for --chain-lint, we do still
run the test itself. We could actually have a pure-lint
mode which just checks each test, but there are a few
reasons not to. One, because the tests are executing
arbitrary code, which could impact the later environment
(e.g., that could impact which set of tests we run at all).
And two, because a pure-lint mode would still be expensive
to run, because a significant amount of code runs outside of
the test_expect_* blocks. Instead, this option is designed
to be used as part of a normal test suite run, where it adds
very little overhead.
Turning on this option detects quite a few problems in
existing tests, which will be fixed in subsequent patches.
However, there are a number of places it cannot reach:
- it cannot find a failure to break out of loops on error,
like:
cmd1 &&
for i in a b c; do
cmd2 $i
done &&
cmd3
which will not notice failures of "cmd2 a" or "cmd b"
- it cannot find a missing &&-chain inside a block or
subfunction, like:
foo () {
cmd1
cmd2
}
foo &&
bar
which will not notice a failure of cmd1.
- it only checks tests that you run; every platform will
have some tests skipped due to missing prequisites,
so it's impossible to say from one run that the test
suite is free of broken &&-chains. However, all tests get
run by _somebody_, so eventually we will notice problems.
- it does not operate on test_when_finished or prerequisite
blocks. It could, but these tends to be much shorter and
less of a problem, so I punted on them in this patch.
This patch was inspired by an earlier patch by Jonathan
Nieder:
http://article.gmane.org/gmane.comp.version-control.git/235913
This implementation and all bugs are mine.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-03-20 10:05:48 +00:00
|
|
|
# 117 is magic because it is unlikely to match the exit
|
|
|
|
# code of other programs
|
tests: skip test_eval_ in internal chain-lint
To check for broken &&-chains, we run "fail_117 && $1" as a test
snippet, and check the exit code. We use test_eval_ to do so, because
that's the way we run the actual test.
But we don't need any of its niceties, like "set -x" tracing. In fact,
they hinder us, because we have to explicitly disable them. So let's
skip that and use "eval" more directly, which is simpler. I had hoped it
would also be faster, but it doesn't seem to produce a measurable
improvement (probably because it's just running internal shell commands,
with no subshells or forks).
Note that there is one gotcha: even though we don't intend to run any of
the commands if the &&-chain is intact, an error like this:
test_expect_success 'broken' '
# this next line breaks the &&-chain
true
# and then this one is executed even by the linter
return 1
'
means we'll "return 1" from the eval, and thus from test_run_(). We
actually do notice this in test_expect_success, but only by saying "hey,
this test didn't say it was OK, so it must have failed", which is not
right (it should say "broken &&-chain").
We can handle this by calling test_eval_inner_() instead, which is our
trick for wrapping "return" in a test snippet. But to do that, we have
to push the trace code out of that inner function and into test_eval_().
This is arguably where it belonged in the first place, but it never
mattered because the "inner_" function had only one caller.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-03-30 19:30:56 +00:00
|
|
|
test_eval_inner_ "fail_117 && $1" </dev/null >&3 2>&4
|
2023-03-30 19:30:47 +00:00
|
|
|
if test $? != 117
|
2017-03-23 05:43:18 +00:00
|
|
|
then
|
2023-03-30 19:30:47 +00:00
|
|
|
BUG "broken &&-chain: $1"
|
t/test-lib: introduce --chain-lint option
It's easy to miss an "&&"-chain in a test script, like:
test_expect_success 'check something important' '
cmd1 &&
cmd2
cmd3
'
The test harness will notice if cmd3 fails, but a failure of
cmd1 or cmd2 will go unnoticed, as their exit status is lost
after cmd3 runs.
The toy example above is easy to spot because the "cmds" are
all the same length, but real code is much more complicated.
It's also difficult to detect these situations by statically
analyzing the shell code with regexps (like the
check-non-portable-shell script does); there's too much
context required to know whether a &&-chain is appropriate
on a given line or not.
This patch instead lets the shell check each test by
sticking a command with a specific and unusual return code
at the top of each test, like:
(exit 117) &&
cmd1 &&
cmd2
cmd3
In a well-formed test, the non-zero exit from the first
command prevents any of the rest from being run, and the
test's exit code is 117. In a bad test (like the one above),
the 117 is lost, and cmd3 is run.
When we encounter a failure of this check, we abort the test
script entirely. For one thing, we have no clue which subset
of the commands in the test snippet were actually run.
Running further tests would be pointless, because we're now
in an unknown state. And two, this is not a "test failure"
in the traditional sense. The test script is buggy, not the
code it is testing. We should be able to fix these problems
in the script once, and not have them come back later as a
regression in git's code.
After checking a test snippet for --chain-lint, we do still
run the test itself. We could actually have a pure-lint
mode which just checks each test, but there are a few
reasons not to. One, because the tests are executing
arbitrary code, which could impact the later environment
(e.g., that could impact which set of tests we run at all).
And two, because a pure-lint mode would still be expensive
to run, because a significant amount of code runs outside of
the test_expect_* blocks. Instead, this option is designed
to be used as part of a normal test suite run, where it adds
very little overhead.
Turning on this option detects quite a few problems in
existing tests, which will be fixed in subsequent patches.
However, there are a number of places it cannot reach:
- it cannot find a failure to break out of loops on error,
like:
cmd1 &&
for i in a b c; do
cmd2 $i
done &&
cmd3
which will not notice failures of "cmd2 a" or "cmd b"
- it cannot find a missing &&-chain inside a block or
subfunction, like:
foo () {
cmd1
cmd2
}
foo &&
bar
which will not notice a failure of cmd1.
- it only checks tests that you run; every platform will
have some tests skipped due to missing prequisites,
so it's impossible to say from one run that the test
suite is free of broken &&-chains. However, all tests get
run by _somebody_, so eventually we will notice problems.
- it does not operate on test_when_finished or prerequisite
blocks. It could, but these tends to be much shorter and
less of a problem, so I punted on them in this patch.
This patch was inspired by an earlier patch by Jonathan
Nieder:
http://article.gmane.org/gmane.comp.version-control.git/235913
This implementation and all bugs are mine.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-03-20 10:05:48 +00:00
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
2013-06-17 09:18:46 +00:00
|
|
|
setup_malloc_check
|
test: cope better with use of return for errors
In olden times, tests would quietly exit the script when they failed
at an inconvenient moment, which was a little disconcerting.
Therefore v0.99.5~24^2~4 (Trapping exit in tests, using return for
errors, 2005-08-10) switched to an idiom of using "return" instead,
wrapping evaluation of test code in a function to make that safe:
test_run_ () {
eval >&3 2>&4 "$1"
eval_ret="$?"
return 0
}
Years later, the implementation of test_when_finished (v1.7.1.1~95,
2010-05-02) and v1.7.2-rc2~1^2~13 (test-lib: output a newline before
"ok" under a TAP harness, 2010-06-24) took advantage of test_run_ as a
place to put code shared by all test assertion functions, without
paying attention to the function's former purpose:
test_run_ () {
...
eval >&3 2>&4 "$1"
eval_ret=$?
if should run cleanup
then
eval >&3 2>&4 "$test_cleanup"
fi
if TAP format requires a newline here
then
echo
fi
return 0
}
That means cleanup commands and the newline to put TAP output at
column 0 are skipped when tests use "return" to fail early. Fix it by
introducing a test_eval_ function to catch the "return", with a
comment explaining the new function's purpose for the next person who
might touch this code.
Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
Acked-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-08-08 01:17:09 +00:00
|
|
|
test_eval_ "$1"
|
2010-05-06 08:41:10 +00:00
|
|
|
eval_ret=$?
|
2013-06-17 09:18:46 +00:00
|
|
|
teardown_malloc_check
|
2011-06-27 18:02:22 +00:00
|
|
|
|
test-lib.sh: support -x option for shell-tracing
Usually running a test under "-v" makes it clear which
command is failing. However, sometimes it can be useful to
also see a complete trace of the shell commands being run in
the test. You can do so without any support from the test
suite by running "sh -x tXXXX-foo.sh". However, this
produces quite a large bit of output, as we see a trace of
the entire test suite.
This patch instead introduces a "-x" option to the test
scripts (i.e., "./tXXXX-foo.sh -x"). When enabled, this
turns on "set -x" only for the tests themselves. This can
still be a bit verbose, but should keep things to a more
manageable level. You can even use "--verbose-only" to see
the trace only for a specific test.
The implementation is a little invasive. We turn on the "set
-x" inside the "eval" of the test code. This lets the eval
itself avoid being reported in the trace (which would be
long, and redundant with the verbose listing we already
showed). And then after the eval runs, we do some trickery
with stderr to avoid showing the "set +x" to the user.
We also show traces for test_cleanup functions (since they
can impact the test outcome, too). However, we do avoid
running the noop ":" cleanup (the default if the test does
not use test_cleanup at all), as it creates unnecessary
noise in the "set -x" output.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-10 06:47:27 +00:00
|
|
|
if test -z "$immediate" || test $eval_ret = 0 ||
|
|
|
|
test -n "$expecting_failure" && test "$test_cleanup" != ":"
|
2011-06-27 18:02:22 +00:00
|
|
|
then
|
2012-09-15 03:38:24 +00:00
|
|
|
setup_malloc_check
|
test: cope better with use of return for errors
In olden times, tests would quietly exit the script when they failed
at an inconvenient moment, which was a little disconcerting.
Therefore v0.99.5~24^2~4 (Trapping exit in tests, using return for
errors, 2005-08-10) switched to an idiom of using "return" instead,
wrapping evaluation of test code in a function to make that safe:
test_run_ () {
eval >&3 2>&4 "$1"
eval_ret="$?"
return 0
}
Years later, the implementation of test_when_finished (v1.7.1.1~95,
2010-05-02) and v1.7.2-rc2~1^2~13 (test-lib: output a newline before
"ok" under a TAP harness, 2010-06-24) took advantage of test_run_ as a
place to put code shared by all test assertion functions, without
paying attention to the function's former purpose:
test_run_ () {
...
eval >&3 2>&4 "$1"
eval_ret=$?
if should run cleanup
then
eval >&3 2>&4 "$test_cleanup"
fi
if TAP format requires a newline here
then
echo
fi
return 0
}
That means cleanup commands and the newline to put TAP output at
column 0 are skipped when tests use "return" to fail early. Fix it by
introducing a test_eval_ function to catch the "return", with a
comment explaining the new function's purpose for the next person who
might touch this code.
Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
Acked-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-08-08 01:17:09 +00:00
|
|
|
test_eval_ "$test_cleanup"
|
2012-09-15 03:38:24 +00:00
|
|
|
teardown_malloc_check
|
2011-06-27 18:02:22 +00:00
|
|
|
fi
|
2012-09-01 18:11:49 +00:00
|
|
|
if test "$verbose" = "t" && test -n "$HARNESS_ACTIVE"
|
|
|
|
then
|
test-lib: output a newline before "ok" under a TAP harness
Some tests in the testsuite will emit a line that doesn't end with a
newline, right before we're about to output "ok" or "not ok". This
breaks the TAP output with "Tests out of sequence" errors since a TAP
harness can't understand this:
ok 1 - A test
[some output here]ok 2 - Another test
ok 3 - Yet another test
Work around it by emitting an empty line before we're about to say
"ok" or "not ok", but only if we're running under --verbose and
HARNESS_ACTIVE=1 is set, which'll only be the case when running under
a harnesses like prove(1).
I think it's better to do this than fix each tests by adding `&& echo'
everywhere. More tests might be added that break TAP in the future,
and a human isn't going to look at the extra whitespace, since
HARNESS_ACTIVE=1 always means a harness is reading it.
The tests that had issues were:
t1007, t3410, t3413, t3409, t3414, t3415, t3416, t3412, t3404,
t5407, t7402, t7003, t9001
With this workaround the entire test suite runs without errors under:
prove -j 10 ./t[0-9]*.sh :: --verbose
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 17:44:47 +00:00
|
|
|
echo ""
|
|
|
|
fi
|
2011-08-08 01:15:34 +00:00
|
|
|
return "$eval_ret"
|
2005-08-11 03:56:21 +00:00
|
|
|
}
|
|
|
|
|
2013-06-18 12:25:59 +00:00
|
|
|
test_start_ () {
|
2009-02-05 20:20:56 +00:00
|
|
|
test_count=$(($test_count+1))
|
2013-06-23 18:12:56 +00:00
|
|
|
maybe_setup_verbose
|
2013-06-23 18:12:57 +00:00
|
|
|
maybe_setup_valgrind
|
2022-05-21 22:18:51 +00:00
|
|
|
start_test_case_output "$@"
|
2013-06-18 12:25:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
test_finish_ () {
|
|
|
|
echo >&3 ""
|
2013-06-23 18:12:57 +00:00
|
|
|
maybe_teardown_valgrind
|
2013-06-23 18:12:56 +00:00
|
|
|
maybe_teardown_verbose
|
tests: include detailed trace logs with --write-junit-xml upon failure
The JUnit XML format lends itself to be presented in a powerful UI,
where you can drill down to the information you are interested in very
quickly.
For test failures, this usually means that you want to see the detailed
trace of the failing tests.
With Travis CI, we passed the `--verbose-log` option to get those
traces. However, that seems excessive, as we do not need/use the logs in
almost all of those cases: only when a test fails do we have a way to
include the trace.
So let's do something different when using Azure DevOps: let's run all
the tests with `--quiet` first, and only if a failure is encountered,
try to trace the commands as they are executed.
Of course, we cannot turn on `--verbose-log` after the fact. So let's
just re-run the test with all the same options, adding `--verbose-log`.
And then munging the output file into the JUnit XML on the fly.
Note: there is an off chance that re-running the test in verbose mode
"fixes" the failures (and this does happen from time to time!). That is
a possibility we should be able to live with. Ideally, we would label
this as "Passed upon rerun", and Azure Pipelines even know about that
outcome, but it is not available when using the JUnit XML format for
now:
https://github.com/Microsoft/azure-pipelines-agent/blob/master/src/Agent.Worker/TestResults/JunitResultReader.cs
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-29 14:19:34 +00:00
|
|
|
if test -n "$GIT_TEST_TEE_OFFSET"
|
|
|
|
then
|
|
|
|
GIT_TEST_TEE_OFFSET=$(test-tool path-utils file-size \
|
|
|
|
"$GIT_TEST_TEE_OUTPUT_FILE")
|
|
|
|
fi
|
2013-06-18 12:25:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
test_skip () {
|
2006-12-29 01:58:00 +00:00
|
|
|
to_skip=
|
2014-04-30 09:50:43 +00:00
|
|
|
skipped_reason=
|
test-lib: avoid accidental globbing in match_pattern_list()
We have a custom match_pattern_list() function which we use for matching
test names (like "t1234") against glob-like patterns (like "t1???") for
$GIT_SKIP_TESTS, --verbose-only, etc.
Those patterns may have multiple whitespace-separated elements (e.g.,
"t0* t1234 t5?78"). The callers of match_pattern_list thus pass the
strings unquoted, so that the shell does the usual field-splitting into
separate arguments.
But this also means the shell will do the usual globbing for each
argument, which can result in us seeing an expansion based on what's in
the filesystem, rather than the real pattern. For example, if I have the
path "t5000" in the filesystem, and you feed the pattern "t?000", that
_should_ match the string "t0000", but it won't after the shell has
expanded it to "t5000".
This has been a bug ever since that function was introduced. But it
didn't usually trigger since we typically use the function inside the
trash directory, which has a very limited set of files that are unlikely
to match. It became a lot easier to trigger after edc23840b0 (test-lib:
bring $remove_trash out of retirement, 2021-05-10), because now we match
$GIT_SKIP_TESTS before even entering the trash directory. So the t5000
example above can be seen with:
GIT_SKIP_TESTS=t?000 ./t0000-basic.sh
which should skip all tests but doesn't.
We can fix this by using "set -f" to ask the shell not to glob (which is
in POSIX, so should hopefully be portable enough). We only want to do
this in a subshell (to avoid polluting the rest of the script), which
means we need to get the whole string intact into the match_pattern_list
function by quoting it. Arguably this is a good idea anyway, since it
makes it much more obvious that we intend to split, and it's not simply
sloppy scripting.
Diagnosed-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-06-16 10:23:07 +00:00
|
|
|
if match_pattern_list $this_test.$test_count "$GIT_SKIP_TESTS"
|
2013-06-18 12:25:58 +00:00
|
|
|
then
|
|
|
|
to_skip=t
|
2014-04-30 09:50:43 +00:00
|
|
|
skipped_reason="GIT_SKIP_TESTS"
|
2013-06-18 12:25:58 +00:00
|
|
|
fi
|
2019-11-12 12:24:38 +00:00
|
|
|
if test -z "$to_skip" && test -n "$run_list" &&
|
2020-10-18 00:23:45 +00:00
|
|
|
! match_test_selector_list '--run' "$1" $test_count "$run_list"
|
2019-11-12 12:24:38 +00:00
|
|
|
then
|
|
|
|
to_skip=t
|
|
|
|
skipped_reason="--run"
|
|
|
|
fi
|
2010-10-16 18:36:58 +00:00
|
|
|
if test -z "$to_skip" && test -n "$test_prereq" &&
|
|
|
|
! test_have_prereq "$test_prereq"
|
2009-03-01 20:04:46 +00:00
|
|
|
then
|
|
|
|
to_skip=t
|
2014-04-30 09:50:43 +00:00
|
|
|
|
2010-08-24 07:34:10 +00:00
|
|
|
of_prereq=
|
2010-10-16 18:36:58 +00:00
|
|
|
if test "$missing_prereq" != "$test_prereq"
|
2010-08-24 07:34:10 +00:00
|
|
|
then
|
2010-10-16 18:36:58 +00:00
|
|
|
of_prereq=" of $test_prereq"
|
2010-08-24 07:34:10 +00:00
|
|
|
fi
|
2014-04-30 09:50:43 +00:00
|
|
|
skipped_reason="missing $missing_prereq${of_prereq}"
|
2021-11-20 15:03:59 +00:00
|
|
|
|
|
|
|
# Keep a list of all the missing prereq for result aggregation
|
|
|
|
if test -z "$missing_prereq"
|
|
|
|
then
|
|
|
|
test_missing_prereq=$missing_prereq
|
|
|
|
else
|
|
|
|
test_missing_prereq="$test_missing_prereq,$missing_prereq"
|
|
|
|
fi
|
2014-04-30 09:50:43 +00:00
|
|
|
fi
|
2010-08-24 07:34:10 +00:00
|
|
|
|
2014-04-30 09:50:43 +00:00
|
|
|
case "$to_skip" in
|
|
|
|
t)
|
2019-01-29 14:19:27 +00:00
|
|
|
|
2014-04-30 09:50:43 +00:00
|
|
|
say_color skip "ok $test_count # skip $1 ($skipped_reason)"
|
2006-12-29 01:58:00 +00:00
|
|
|
: true
|
2022-05-21 22:18:56 +00:00
|
|
|
finalize_test_case_output skip "$@"
|
2006-12-29 01:58:00 +00:00
|
|
|
;;
|
|
|
|
*)
|
|
|
|
false
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
}
|
|
|
|
|
2012-02-17 10:25:09 +00:00
|
|
|
# stub; perf-lib overrides it
|
|
|
|
test_at_end_hook_ () {
|
|
|
|
:
|
|
|
|
}
|
|
|
|
|
test-lib: introduce 'test_atexit'
When running Apache, 'git daemon', or p4d, we want to kill them at the
end of the test script, otherwise a leftover daemon process will keep
its port open indefinitely, and thus will interfere with subsequent
executions of the same test script.
So far, we stop these daemon processes "manually", i.e.:
- by registering functions or commands in the trap on EXIT to stop
the daemon while preserving the last seen exit code before the
trap (to deal with a failure when run with '--immediate' or with
interrupts by ctrl-C),
- and by invoking these functions/commands last thing before
'test_done' (and sometimes restoring the test framework's default
trap on EXIT, to prevent the daemons from being killed twice).
On one hand, we do this inconsistently, e.g. 'git p4' tests invoke
different functions in the trap on EXIT and in the last test before
'test_done', and they neither restore the test framework's default trap
on EXIT nor preserve the last seen exit code. On the other hand, this
is error prone, because, as shown in a previous patch in this series,
any output from the cleanup commands in the trap on EXIT can prevent a
proper cleanup when a test script run with '--verbose-log' and certain
shells, notably 'dash', is interrupted.
Let's introduce 'test_atexit', which is loosely modeled after
'test_when_finished', but has a broader scope: rather than running the
commands after the current test case, run them when the test script
finishes, and also run them when the test is interrupted, or exits
early in case of a failure while the '--immediate' option is in
effect.
When running the cleanup commands at the end of a successful test,
then they will be run in 'test_done' before it removes the trash
directory, i.e. the cleanup commands will still be able to access any
pidfiles or socket files in there. When running the cleanup commands
after an interrupt or failure with '--immediate', then they will be
run in the trap on EXIT. In both cases they will be run in
'test_eval_', i.e. both standard error and output of all cleanup
commands will go where they should according to the '-v' or
'--verbose-log' options, and thus won't cause any troubles when
interrupting a test script run with '--verbose-log'.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-03-13 12:24:11 +00:00
|
|
|
test_atexit_cleanup=:
|
|
|
|
test_atexit_handler () {
|
|
|
|
# In a succeeding test script 'test_atexit_handler' is invoked
|
|
|
|
# twice: first from 'test_done', then from 'die' in the trap on
|
|
|
|
# EXIT.
|
|
|
|
# This condition and resetting 'test_atexit_cleanup' below makes
|
|
|
|
# sure that the registered cleanup commands are run only once.
|
|
|
|
test : != "$test_atexit_cleanup" || return 0
|
|
|
|
|
|
|
|
setup_malloc_check
|
|
|
|
test_eval_ "$test_atexit_cleanup"
|
|
|
|
test_atexit_cleanup=:
|
|
|
|
teardown_malloc_check
|
|
|
|
}
|
|
|
|
|
test-lib: have the "check" mode for SANITIZE=leak consider leak logs
As noted in previous on-list discussions[1] we have various tests that
will falsely report being leak-free because we're missing the relevant
exit code from LSAN as summarized below.
We should fix those issues, but in the meantime and as an additional
sanity check we can and should consider our own ASAN logs before
reporting that a test is leak-free.
Before this compiling with SANITIZE=leak and running:
./t6407-merge-binary.sh
Will exit successfully, now we'll get an error and an informative
message on:
GIT_TEST_SANITIZE_LEAK_LOG=true ./t6407-merge-binary.sh
Even better, as noted in the updated t/README we'll now error out when
combined with the "check" mode:
GIT_TEST_PASSING_SANITIZE_LEAK=check \
GIT_TEST_SANITIZE_LEAK_LOG=true \
./t4058-diff-duplicates.sh
Why do we miss these leaks? Because:
* We have leaks inside "test_expect_failure" blocks, which by design
will not distinguish a "normal" failure from an abort() or
segfault. See [1] for a discussion of it shortcomings.
* We have "git" invocations outside of "test_expect_success",
e.g. setup code in the main body of the test, or in test helper
functions that don't use &&-chaining.
* Our tests will otherwise catch segfaults and abort(), but if we
invoke a command that invokes another command it needs to ferry the
exit code up to us.
Notably a command that e.g. might invoke "git pack-objects" might
itself exit with status 128 if that "pack-objects" segfaults or
abort()'s. If the test invoking the parent command(s) is using
"test_must_fail" we'll consider it an expected "ok" failure.
* run-command.c doesn't (but probably should) ferry up such exit
codes, so for e.g. "git push" tests where we expect a failure and an
underlying "git" command fails we won't ferry up the segfault or
abort exit code.
* We have gitweb.perl and some other perl code ignoring return values
from close(), i.e. ignoring exit codes from "git rev-parse" et al.
* We have in-tree shellscripts like "git-merge-one-file.sh" invoking
git commands, they'll usually return their own exit codes on "git"
failure, rather then ferrying up segfault or abort() exit code.
E.g. these invocations in git-merge-one-file.sh leak, but aren't
reflected in the "git merge" exit code:
src1=$(git unpack-file $2)
src2=$(git unpack-file $3)
That case would be easily "fixed" by adding a line like this after
each assignment:
test $? -ne 0 && exit $?
But we'd then in e.g. "t6407-merge-binary.sh" run into
write_tree_trivial() in "builtin/merge.c" calling die() instead of
ferrying up the relevant exit code.
Let's remove "TEST_PASSES_SANITIZE_LEAK=true" from tests we
were falsely marking as leak-free.
In the case of t6407-merge-binary.sh it was marked as leak-free in
9081a421a6d (checkout: fix "branch info" memory leaks,
2021-11-16). I'd previously removed other bad
"TEST_PASSES_SANITIZE_LEAK=true" opt-ins in the series merged in
ea05fd5fbf7 (Merge branch 'ab/keep-git-exit-codes-in-tests',
2022-03-16). The case of t1060-object-corruption.sh is more subtle,
and will be discussed in a subsequent commit.
1. https://lore.kernel.org/git/cover-0.7-00000000000-20220318T002951Z-avarab@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:39 +00:00
|
|
|
sanitize_leak_log_message_ () {
|
|
|
|
local new="$1" &&
|
|
|
|
local old="$2" &&
|
|
|
|
local file="$3" &&
|
|
|
|
|
|
|
|
printf "With SANITIZE=leak at exit we have %d leak logs, but started with %d
|
|
|
|
|
|
|
|
This means that we have a blindspot where git is leaking but we're
|
|
|
|
losing the exit code somewhere, or not propagating it appropriately
|
|
|
|
upwards!
|
|
|
|
|
|
|
|
See the logs at \"%s.*\";
|
|
|
|
those logs are reproduced below." \
|
|
|
|
"$new" "$old" "$file"
|
|
|
|
}
|
|
|
|
|
|
|
|
check_test_results_san_file_ () {
|
|
|
|
if test -z "$TEST_RESULTS_SAN_FILE"
|
|
|
|
then
|
|
|
|
return
|
|
|
|
fi &&
|
|
|
|
local old="$TEST_RESULTS_SAN_DIR_NR_LEAKS_STARTUP" &&
|
|
|
|
local new="$(nr_san_dir_leaks_)" &&
|
|
|
|
|
|
|
|
if test $new -le $old
|
|
|
|
then
|
|
|
|
return
|
|
|
|
fi &&
|
|
|
|
local out="$(sanitize_leak_log_message_ "$new" "$old" "$TEST_RESULTS_SAN_FILE")" &&
|
|
|
|
say_color error "$out" &&
|
|
|
|
if test "$old" != 0
|
|
|
|
then
|
|
|
|
echo &&
|
|
|
|
say_color error "The logs include output from past runs to avoid" &&
|
|
|
|
say_color error "that remove 'test-results' between runs."
|
|
|
|
fi &&
|
|
|
|
say_color error "$(cat "$TEST_RESULTS_SAN_FILE".*)" &&
|
|
|
|
|
|
|
|
if test -n "$passes_sanitize_leak" && test "$test_failure" = 0
|
|
|
|
then
|
|
|
|
say "As TEST_PASSES_SANITIZE_LEAK=true and our logs show we're leaking, exit non-zero!" &&
|
|
|
|
invert_exit_code=t
|
|
|
|
elif test -n "$passes_sanitize_leak"
|
|
|
|
then
|
|
|
|
say "As TEST_PASSES_SANITIZE_LEAK=true and our logs show we're leaking, and we're failing for other reasons too..." &&
|
|
|
|
invert_exit_code=
|
|
|
|
elif test -n "$sanitize_leak_check" && test "$test_failure" = 0
|
|
|
|
then
|
|
|
|
say "As TEST_PASSES_SANITIZE_LEAK=true isn't set the above leak is 'ok' with GIT_TEST_PASSING_SANITIZE_LEAK=check" &&
|
|
|
|
invert_exit_code=
|
|
|
|
elif test -n "$sanitize_leak_check"
|
|
|
|
then
|
|
|
|
say "As TEST_PASSES_SANITIZE_LEAK=true isn't set the above leak is 'ok' with GIT_TEST_PASSING_SANITIZE_LEAK=check" &&
|
|
|
|
invert_exit_code=t
|
|
|
|
else
|
|
|
|
say "With GIT_TEST_SANITIZE_LEAK_LOG=true our logs revealed a memory leak, exit non-zero!" &&
|
|
|
|
invert_exit_code=t
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2005-05-14 05:50:32 +00:00
|
|
|
test_done () {
|
test-lib: introduce 'test_atexit'
When running Apache, 'git daemon', or p4d, we want to kill them at the
end of the test script, otherwise a leftover daemon process will keep
its port open indefinitely, and thus will interfere with subsequent
executions of the same test script.
So far, we stop these daemon processes "manually", i.e.:
- by registering functions or commands in the trap on EXIT to stop
the daemon while preserving the last seen exit code before the
trap (to deal with a failure when run with '--immediate' or with
interrupts by ctrl-C),
- and by invoking these functions/commands last thing before
'test_done' (and sometimes restoring the test framework's default
trap on EXIT, to prevent the daemons from being killed twice).
On one hand, we do this inconsistently, e.g. 'git p4' tests invoke
different functions in the trap on EXIT and in the last test before
'test_done', and they neither restore the test framework's default trap
on EXIT nor preserve the last seen exit code. On the other hand, this
is error prone, because, as shown in a previous patch in this series,
any output from the cleanup commands in the trap on EXIT can prevent a
proper cleanup when a test script run with '--verbose-log' and certain
shells, notably 'dash', is interrupted.
Let's introduce 'test_atexit', which is loosely modeled after
'test_when_finished', but has a broader scope: rather than running the
commands after the current test case, run them when the test script
finishes, and also run them when the test is interrupted, or exits
early in case of a failure while the '--immediate' option is in
effect.
When running the cleanup commands at the end of a successful test,
then they will be run in 'test_done' before it removes the trash
directory, i.e. the cleanup commands will still be able to access any
pidfiles or socket files in there. When running the cleanup commands
after an interrupt or failure with '--immediate', then they will be
run in the trap on EXIT. In both cases they will be run in
'test_eval_', i.e. both standard error and output of all cleanup
commands will go where they should according to the '-v' or
'--verbose-log' options, and thus won't cause any troubles when
interrupting a test script run with '--verbose-log'.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-03-13 12:24:11 +00:00
|
|
|
# Run the atexit commands _before_ the trash directory is
|
|
|
|
# removed, so the commands can access pidfiles and socket files.
|
|
|
|
test_atexit_handler
|
|
|
|
|
2022-05-21 22:18:46 +00:00
|
|
|
finalize_test_output
|
2019-01-29 14:19:27 +00:00
|
|
|
|
2012-09-01 18:11:49 +00:00
|
|
|
if test -z "$HARNESS_ACTIVE"
|
|
|
|
then
|
2019-01-05 01:08:56 +00:00
|
|
|
mkdir -p "$TEST_RESULTS_DIR"
|
2010-08-11 19:37:31 +00:00
|
|
|
|
2019-01-05 01:08:56 +00:00
|
|
|
cat >"$TEST_RESULTS_BASE.counts" <<-EOF
|
2011-04-29 12:30:30 +00:00
|
|
|
total $test_count
|
|
|
|
success $test_success
|
|
|
|
fixed $test_fixed
|
|
|
|
broken $test_broken
|
|
|
|
failed $test_failure
|
2021-11-20 15:03:59 +00:00
|
|
|
missing_prereq $test_missing_prereq
|
2011-04-29 12:30:30 +00:00
|
|
|
|
|
|
|
EOF
|
2010-08-11 19:37:31 +00:00
|
|
|
fi
|
2008-02-01 09:50:53 +00:00
|
|
|
|
|
|
|
if test "$test_fixed" != 0
|
|
|
|
then
|
2013-10-19 21:06:08 +00:00
|
|
|
say_color error "# $test_fixed known breakage(s) vanished; please update test(s)"
|
2008-02-01 09:50:53 +00:00
|
|
|
fi
|
|
|
|
if test "$test_broken" != 0
|
|
|
|
then
|
2013-10-19 21:06:08 +00:00
|
|
|
say_color warn "# still have $test_broken known breakage(s)"
|
2012-12-16 18:28:15 +00:00
|
|
|
fi
|
|
|
|
if test "$test_broken" != 0 || test "$test_fixed" != 0
|
|
|
|
then
|
|
|
|
test_remaining=$(( $test_count - $test_broken - $test_fixed ))
|
|
|
|
msg="remaining $test_remaining test(s)"
|
2008-02-03 08:23:02 +00:00
|
|
|
else
|
2012-12-16 18:28:15 +00:00
|
|
|
test_remaining=$test_count
|
2008-02-03 08:23:02 +00:00
|
|
|
msg="$test_count test(s)"
|
2008-02-01 09:50:53 +00:00
|
|
|
fi
|
2005-05-14 05:50:32 +00:00
|
|
|
case "$test_failure" in
|
2005-12-10 01:32:18 +00:00
|
|
|
0)
|
test-lib: simplify by removing test_external
Remove the "test_external" function added in [1]. This arguably makes
the output of t9700-perl-git.sh and friends worse. But as we'll argue
below the trade-off is worth it, since "chaining" to another TAP
emitter in test-lib.sh is more trouble than it's worth.
The new output of t9700-perl-git.sh is now:
$ ./t9700-perl-git.sh
ok 1 - set up test repository
ok 2 - use t9700/test.pl to test Git.pm
# passed all 2 test(s)
1..2
Whereas before this change it would be:
$ ./t9700-perl-git.sh
ok 1 - set up test repository
# run 1: Perl API (perl /home/avar/g/git/t/t9700/test.pl)
ok 2 - use Git;
[... omitting tests 3..46 from t/t9700/test.pl ...]
ok 47 - unquote escape sequences
1..47
# test_external test Perl API was ok
# test_external_without_stderr test no stderr: Perl API was ok
At the time of its addition supporting "test_external" was easy, but
when test-lib.sh itself started to emit TAP in [2] we needed to make
everything surrounding the emission of the plan consider
"test_external". I added that support in [2] so that we could run:
prove ./t9700-perl-git.sh :: -v
But since then in [3] the door has been closed on combining
$HARNESS_ACTIVE and -v, we'll now just die:
$ prove ./t9700-perl-git.sh :: -v
Bailout called. Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
FAILED--Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
So the only use of this has been that *if* we had failure in one of
these tests we could e.g. in CI see which test failed based on the
test number. Now we'll need to look at the full verbose logs to get
that same information.
I think this trade-off is acceptable given the reduction in
complexity, and it brings these tests in line with other similar
tests, e.g. the reftable tests added in [4] will be condensed down to
just one test, which invokes the C helper:
$ ./t0032-reftable-unittest.sh
ok 1 - unittests
# passed all 1 test(s)
1..1
It would still be nice to have that ":: -v" form work again, it
never *really* worked, but even though we've had edge cases test
output screwing up the TAP it mostly worked between d998bd4ab67 and
[3], so we may have been overzealous in forbidding it outright.
I have local patches which I'm planning to submit sooner than later
that get us to that goal, and in a way that isn't buggy. In the
meantime getting rid of this special case makes hacking on this area
of test-lib.sh easier, as we'll do in subsequent commits.
The switch from "perl" to "$PERL_PATH" here is because "perl" is
defined as a shell function in the test suite, see a5bf824f3b4 (t:
prevent '-x' tracing from interfering with test helpers' stderr,
2018-02-25). On e.g. the OSX CI the "command perl"... will be part of
the emitted stderr.
1. fb32c410087 (t/test-lib.sh: add test_external and
test_external_without_stderr, 2008-06-19)
2. d998bd4ab67 (test-lib: Make the test_external_* functions
TAP-aware, 2010-06-24)
3. 614fe015212 (test-lib: bail out when "-v" used under
"prove", 2016-10-22)
4. ef8a6c62687 (reftable: utility functions, 2021-10-07)
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:37 +00:00
|
|
|
if test $test_remaining -gt 0
|
2012-09-01 18:11:49 +00:00
|
|
|
then
|
test-lib: simplify by removing test_external
Remove the "test_external" function added in [1]. This arguably makes
the output of t9700-perl-git.sh and friends worse. But as we'll argue
below the trade-off is worth it, since "chaining" to another TAP
emitter in test-lib.sh is more trouble than it's worth.
The new output of t9700-perl-git.sh is now:
$ ./t9700-perl-git.sh
ok 1 - set up test repository
ok 2 - use t9700/test.pl to test Git.pm
# passed all 2 test(s)
1..2
Whereas before this change it would be:
$ ./t9700-perl-git.sh
ok 1 - set up test repository
# run 1: Perl API (perl /home/avar/g/git/t/t9700/test.pl)
ok 2 - use Git;
[... omitting tests 3..46 from t/t9700/test.pl ...]
ok 47 - unquote escape sequences
1..47
# test_external test Perl API was ok
# test_external_without_stderr test no stderr: Perl API was ok
At the time of its addition supporting "test_external" was easy, but
when test-lib.sh itself started to emit TAP in [2] we needed to make
everything surrounding the emission of the plan consider
"test_external". I added that support in [2] so that we could run:
prove ./t9700-perl-git.sh :: -v
But since then in [3] the door has been closed on combining
$HARNESS_ACTIVE and -v, we'll now just die:
$ prove ./t9700-perl-git.sh :: -v
Bailout called. Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
FAILED--Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
So the only use of this has been that *if* we had failure in one of
these tests we could e.g. in CI see which test failed based on the
test number. Now we'll need to look at the full verbose logs to get
that same information.
I think this trade-off is acceptable given the reduction in
complexity, and it brings these tests in line with other similar
tests, e.g. the reftable tests added in [4] will be condensed down to
just one test, which invokes the C helper:
$ ./t0032-reftable-unittest.sh
ok 1 - unittests
# passed all 1 test(s)
1..1
It would still be nice to have that ":: -v" form work again, it
never *really* worked, but even though we've had edge cases test
output screwing up the TAP it mostly worked between d998bd4ab67 and
[3], so we may have been overzealous in forbidding it outright.
I have local patches which I'm planning to submit sooner than later
that get us to that goal, and in a way that isn't buggy. In the
meantime getting rid of this special case makes hacking on this area
of test-lib.sh easier, as we'll do in subsequent commits.
The switch from "perl" to "$PERL_PATH" here is because "perl" is
defined as a shell function in the test suite, see a5bf824f3b4 (t:
prevent '-x' tracing from interfering with test helpers' stderr,
2018-02-25). On e.g. the OSX CI the "command perl"... will be part of
the emitted stderr.
1. fb32c410087 (t/test-lib.sh: add test_external and
test_external_without_stderr, 2008-06-19)
2. d998bd4ab67 (test-lib: Make the test_external_* functions
TAP-aware, 2010-06-24)
3. 614fe015212 (test-lib: bail out when "-v" used under
"prove", 2016-10-22)
4. ef8a6c62687 (reftable: utility functions, 2021-10-07)
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:37 +00:00
|
|
|
say_color pass "# passed all $msg"
|
2010-06-24 17:44:46 +00:00
|
|
|
fi
|
Enable parallel tests
On multiprocessor machines, or with I/O heavy tests (that leave the
CPU waiting a lot), it makes sense to parallelize the tests.
However, care has to be taken that the different jobs use different
trash directories.
This commit does so, by creating the trash directories with a suffix
that is unique with regard to the test, as it is the test's base name.
Further, the trash directory is removed in the test itself if
everything went fine, so that the trash directories do not
pile up only to be removed at the very end.
If a test failed, the trash directory is not removed. Chances are
that the exact error message is lost in the clutter, but you can still
see what test failed from the name of the trash directory, and repeat
the test (without -j).
If all was good, you will see the aggregated results.
Suggestions to simplify this commit came from Junio and René.
There still is an issue with tests that want to run a server process and
listen to a fixed port (http and svn) --- they cannot run in parallel but
this patch does not address this issue.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-08-08 11:08:37 +00:00
|
|
|
|
test-lib: simplify by removing test_external
Remove the "test_external" function added in [1]. This arguably makes
the output of t9700-perl-git.sh and friends worse. But as we'll argue
below the trade-off is worth it, since "chaining" to another TAP
emitter in test-lib.sh is more trouble than it's worth.
The new output of t9700-perl-git.sh is now:
$ ./t9700-perl-git.sh
ok 1 - set up test repository
ok 2 - use t9700/test.pl to test Git.pm
# passed all 2 test(s)
1..2
Whereas before this change it would be:
$ ./t9700-perl-git.sh
ok 1 - set up test repository
# run 1: Perl API (perl /home/avar/g/git/t/t9700/test.pl)
ok 2 - use Git;
[... omitting tests 3..46 from t/t9700/test.pl ...]
ok 47 - unquote escape sequences
1..47
# test_external test Perl API was ok
# test_external_without_stderr test no stderr: Perl API was ok
At the time of its addition supporting "test_external" was easy, but
when test-lib.sh itself started to emit TAP in [2] we needed to make
everything surrounding the emission of the plan consider
"test_external". I added that support in [2] so that we could run:
prove ./t9700-perl-git.sh :: -v
But since then in [3] the door has been closed on combining
$HARNESS_ACTIVE and -v, we'll now just die:
$ prove ./t9700-perl-git.sh :: -v
Bailout called. Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
FAILED--Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
So the only use of this has been that *if* we had failure in one of
these tests we could e.g. in CI see which test failed based on the
test number. Now we'll need to look at the full verbose logs to get
that same information.
I think this trade-off is acceptable given the reduction in
complexity, and it brings these tests in line with other similar
tests, e.g. the reftable tests added in [4] will be condensed down to
just one test, which invokes the C helper:
$ ./t0032-reftable-unittest.sh
ok 1 - unittests
# passed all 1 test(s)
1..1
It would still be nice to have that ":: -v" form work again, it
never *really* worked, but even though we've had edge cases test
output screwing up the TAP it mostly worked between d998bd4ab67 and
[3], so we may have been overzealous in forbidding it outright.
I have local patches which I'm planning to submit sooner than later
that get us to that goal, and in a way that isn't buggy. In the
meantime getting rid of this special case makes hacking on this area
of test-lib.sh easier, as we'll do in subsequent commits.
The switch from "perl" to "$PERL_PATH" here is because "perl" is
defined as a shell function in the test suite, see a5bf824f3b4 (t:
prevent '-x' tracing from interfering with test helpers' stderr,
2018-02-25). On e.g. the OSX CI the "command perl"... will be part of
the emitted stderr.
1. fb32c410087 (t/test-lib.sh: add test_external and
test_external_without_stderr, 2008-06-19)
2. d998bd4ab67 (test-lib: Make the test_external_* functions
TAP-aware, 2010-06-24)
3. 614fe015212 (test-lib: bail out when "-v" used under
"prove", 2016-10-22)
4. ef8a6c62687 (reftable: utility functions, 2021-10-07)
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:37 +00:00
|
|
|
# Maybe print SKIP message
|
|
|
|
test -z "$skip_all" || skip_all="# SKIP $skip_all"
|
|
|
|
case "$test_count" in
|
|
|
|
0)
|
|
|
|
say "1..$test_count${skip_all:+ $skip_all}"
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
test -z "$skip_all" ||
|
|
|
|
say_color warn "$skip_all"
|
|
|
|
say "1..$test_count"
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
2022-07-27 23:13:32 +00:00
|
|
|
if test -n "$stress" && test -n "$invert_exit_code"
|
|
|
|
then
|
|
|
|
# We're about to move our "$TRASH_DIRECTORY"
|
|
|
|
# to "$TRASH_DIRECTORY.stress-failed" if
|
|
|
|
# --stress is combined with
|
|
|
|
# --invert-exit-code.
|
|
|
|
say "with --stress and --invert-exit-code we're not removing '$TRASH_DIRECTORY'"
|
|
|
|
elif test -z "$debug" && test -n "$remove_trash"
|
2017-04-25 06:39:47 +00:00
|
|
|
then
|
2017-04-24 00:15:09 +00:00
|
|
|
test -d "$TRASH_DIRECTORY" ||
|
2017-04-25 06:39:47 +00:00
|
|
|
error "Tests passed but trash directory already removed before test cleanup; aborting"
|
Enable parallel tests
On multiprocessor machines, or with I/O heavy tests (that leave the
CPU waiting a lot), it makes sense to parallelize the tests.
However, care has to be taken that the different jobs use different
trash directories.
This commit does so, by creating the trash directories with a suffix
that is unique with regard to the test, as it is the test's base name.
Further, the trash directory is removed in the test itself if
everything went fine, so that the trash directories do not
pile up only to be removed at the very end.
If a test failed, the trash directory is not removed. Chances are
that the exact error message is lost in the clutter, but you can still
see what test failed from the name of the trash directory, and repeat
the test (without -j).
If all was good, you will see the aggregated results.
Suggestions to simplify this commit came from Junio and René.
There still is an issue with tests that want to run a server process and
listen to a fixed port (http and svn) --- they cannot run in parallel but
this patch does not address this issue.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-08-08 11:08:37 +00:00
|
|
|
|
2017-04-24 00:15:09 +00:00
|
|
|
cd "$TRASH_DIRECTORY/.." &&
|
2019-01-29 14:19:35 +00:00
|
|
|
rm -fr "$TRASH_DIRECTORY" || {
|
|
|
|
# try again in a bit
|
|
|
|
sleep 5;
|
|
|
|
rm -fr "$TRASH_DIRECTORY"
|
|
|
|
} ||
|
2017-04-25 06:39:47 +00:00
|
|
|
error "Tests passed but test cleanup failed; aborting"
|
|
|
|
fi
|
2022-07-27 23:13:32 +00:00
|
|
|
|
test-lib: have the "check" mode for SANITIZE=leak consider leak logs
As noted in previous on-list discussions[1] we have various tests that
will falsely report being leak-free because we're missing the relevant
exit code from LSAN as summarized below.
We should fix those issues, but in the meantime and as an additional
sanity check we can and should consider our own ASAN logs before
reporting that a test is leak-free.
Before this compiling with SANITIZE=leak and running:
./t6407-merge-binary.sh
Will exit successfully, now we'll get an error and an informative
message on:
GIT_TEST_SANITIZE_LEAK_LOG=true ./t6407-merge-binary.sh
Even better, as noted in the updated t/README we'll now error out when
combined with the "check" mode:
GIT_TEST_PASSING_SANITIZE_LEAK=check \
GIT_TEST_SANITIZE_LEAK_LOG=true \
./t4058-diff-duplicates.sh
Why do we miss these leaks? Because:
* We have leaks inside "test_expect_failure" blocks, which by design
will not distinguish a "normal" failure from an abort() or
segfault. See [1] for a discussion of it shortcomings.
* We have "git" invocations outside of "test_expect_success",
e.g. setup code in the main body of the test, or in test helper
functions that don't use &&-chaining.
* Our tests will otherwise catch segfaults and abort(), but if we
invoke a command that invokes another command it needs to ferry the
exit code up to us.
Notably a command that e.g. might invoke "git pack-objects" might
itself exit with status 128 if that "pack-objects" segfaults or
abort()'s. If the test invoking the parent command(s) is using
"test_must_fail" we'll consider it an expected "ok" failure.
* run-command.c doesn't (but probably should) ferry up such exit
codes, so for e.g. "git push" tests where we expect a failure and an
underlying "git" command fails we won't ferry up the segfault or
abort exit code.
* We have gitweb.perl and some other perl code ignoring return values
from close(), i.e. ignoring exit codes from "git rev-parse" et al.
* We have in-tree shellscripts like "git-merge-one-file.sh" invoking
git commands, they'll usually return their own exit codes on "git"
failure, rather then ferrying up segfault or abort() exit code.
E.g. these invocations in git-merge-one-file.sh leak, but aren't
reflected in the "git merge" exit code:
src1=$(git unpack-file $2)
src2=$(git unpack-file $3)
That case would be easily "fixed" by adding a line like this after
each assignment:
test $? -ne 0 && exit $?
But we'd then in e.g. "t6407-merge-binary.sh" run into
write_tree_trivial() in "builtin/merge.c" calling die() instead of
ferrying up the relevant exit code.
Let's remove "TEST_PASSES_SANITIZE_LEAK=true" from tests we
were falsely marking as leak-free.
In the case of t6407-merge-binary.sh it was marked as leak-free in
9081a421a6d (checkout: fix "branch info" memory leaks,
2021-11-16). I'd previously removed other bad
"TEST_PASSES_SANITIZE_LEAK=true" opt-ins in the series merged in
ea05fd5fbf7 (Merge branch 'ab/keep-git-exit-codes-in-tests',
2022-03-16). The case of t1060-object-corruption.sh is more subtle,
and will be discussed in a subsequent commit.
1. https://lore.kernel.org/git/cover-0.7-00000000000-20220318T002951Z-avarab@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:39 +00:00
|
|
|
check_test_results_san_file_ "$test_failure"
|
|
|
|
|
2022-07-27 23:13:32 +00:00
|
|
|
if test -z "$skip_all" && test -n "$invert_exit_code"
|
|
|
|
then
|
|
|
|
say_color warn "# faking up non-zero exit with --invert-exit-code"
|
|
|
|
GIT_EXIT_OK=t
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2012-02-17 10:25:09 +00:00
|
|
|
test_at_end_hook_
|
|
|
|
|
2022-07-27 23:13:31 +00:00
|
|
|
GIT_EXIT_OK=t
|
2005-05-14 05:50:32 +00:00
|
|
|
exit 0 ;;
|
|
|
|
|
|
|
|
*)
|
test-lib: simplify by removing test_external
Remove the "test_external" function added in [1]. This arguably makes
the output of t9700-perl-git.sh and friends worse. But as we'll argue
below the trade-off is worth it, since "chaining" to another TAP
emitter in test-lib.sh is more trouble than it's worth.
The new output of t9700-perl-git.sh is now:
$ ./t9700-perl-git.sh
ok 1 - set up test repository
ok 2 - use t9700/test.pl to test Git.pm
# passed all 2 test(s)
1..2
Whereas before this change it would be:
$ ./t9700-perl-git.sh
ok 1 - set up test repository
# run 1: Perl API (perl /home/avar/g/git/t/t9700/test.pl)
ok 2 - use Git;
[... omitting tests 3..46 from t/t9700/test.pl ...]
ok 47 - unquote escape sequences
1..47
# test_external test Perl API was ok
# test_external_without_stderr test no stderr: Perl API was ok
At the time of its addition supporting "test_external" was easy, but
when test-lib.sh itself started to emit TAP in [2] we needed to make
everything surrounding the emission of the plan consider
"test_external". I added that support in [2] so that we could run:
prove ./t9700-perl-git.sh :: -v
But since then in [3] the door has been closed on combining
$HARNESS_ACTIVE and -v, we'll now just die:
$ prove ./t9700-perl-git.sh :: -v
Bailout called. Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
FAILED--Further testing stopped: verbose mode forbidden under TAP harness; try --verbose-log
So the only use of this has been that *if* we had failure in one of
these tests we could e.g. in CI see which test failed based on the
test number. Now we'll need to look at the full verbose logs to get
that same information.
I think this trade-off is acceptable given the reduction in
complexity, and it brings these tests in line with other similar
tests, e.g. the reftable tests added in [4] will be condensed down to
just one test, which invokes the C helper:
$ ./t0032-reftable-unittest.sh
ok 1 - unittests
# passed all 1 test(s)
1..1
It would still be nice to have that ":: -v" form work again, it
never *really* worked, but even though we've had edge cases test
output screwing up the TAP it mostly worked between d998bd4ab67 and
[3], so we may have been overzealous in forbidding it outright.
I have local patches which I'm planning to submit sooner than later
that get us to that goal, and in a way that isn't buggy. In the
meantime getting rid of this special case makes hacking on this area
of test-lib.sh easier, as we'll do in subsequent commits.
The switch from "perl" to "$PERL_PATH" here is because "perl" is
defined as a shell function in the test suite, see a5bf824f3b4 (t:
prevent '-x' tracing from interfering with test helpers' stderr,
2018-02-25). On e.g. the OSX CI the "command perl"... will be part of
the emitted stderr.
1. fb32c410087 (t/test-lib.sh: add test_external and
test_external_without_stderr, 2008-06-19)
2. d998bd4ab67 (test-lib: Make the test_external_* functions
TAP-aware, 2010-06-24)
3. 614fe015212 (test-lib: bail out when "-v" used under
"prove", 2016-10-22)
4. ef8a6c62687 (reftable: utility functions, 2021-10-07)
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:37 +00:00
|
|
|
say_color error "# failed $test_failure among $msg"
|
|
|
|
say "1..$test_count"
|
test-lib: Adjust output to be valid TAP format
TAP, the Test Anything Protocol, is a simple text-based interface
between testing modules in a test harness. test-lib.sh's output was
already very close to being valid TAP. This change brings it all the
way there. Before:
$ ./t0005-signals.sh
* ok 1: sigchain works
* passed all 1 test(s)
And after:
$ ./t0005-signals.sh
ok 1 - sigchain works
# passed all 1 test(s)
1..1
The advantage of using TAP is that any program that reads the format
(a "test harness") can run the tests. The most popular of these is the
prove(1) utility that comes with Perl. It can run tests in parallel,
display colored output, format the output to console, file, HTML etc.,
and much more. An example:
$ prove ./t0005-signals.sh
./t0005-signals.sh .. ok
All tests successful.
Files=1, Tests=1, 0 wallclock secs ( 0.03 usr 0.00 sys + 0.01 cusr 0.02 csys = 0.06 CPU)
Result: PASS
prove(1) gives you human readable output without being too
verbose. Running the test suite in parallel with `make test -j15`
produces a flood of text. Running them with `prove -j 15 ./t[0-9]*.sh`
makes it easy to follow what's going on.
All this patch does is re-arrange the output a bit so that it conforms
with the TAP spec, everything that the test suite did before continues
to work. That includes aggregating results in t/test-results/, the
--verbose, --debug and other options for tests, and the test color
output.
TAP harnesses ignore everything that they don't know about, so running
the tests with --verbose works:
$ prove ./t0005-signals.sh :: --verbose --debug
./t0005-signals.sh .. Terminated
./t0005-signals.sh .. ok
All tests successful.
Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.01 sys + 0.01 cusr 0.01 csys = 0.05 CPU)
Result: PASS
Just supply the -v option to prove itself to get all the verbose
output that it suppresses:
$ prove -v ./t0005-signals.sh :: --verbose --debug
./t0005-signals.sh ..
Initialized empty Git repository in /home/avar/g/git/t/trash directory.t0005-signals/.git/
expecting success:
test-sigchain >actual
case "$?" in
143) true ;; # POSIX w/ SIGTERM=15
3) true ;; # Windows
*) false ;;
esac &&
test_cmp expect actual
Terminated
ok 1 - sigchain works
# passed all 1 test(s)
1..1
ok
All tests successful.
Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.01 cusr 0.01 csys = 0.04 CPU)
Result: PASS
As a further example, consider this test script that uses a lot of
test-lib.sh features by Jakub Narebski:
#!/bin/sh
test_description='this is a sample test.
This test is here to see various test outputs.'
. ./test-lib.sh
say 'diagnostic message'
test_expect_success 'true test' 'true'
test_expect_success 'false test' 'false'
test_expect_failure 'true test (todo)' 'true'
test_expect_failure 'false test (todo)' 'false'
test_debug 'echo "debug message"'
test_done
The output of that was previously:
* diagnostic message # yellow
* ok 1: true test
* FAIL 2: false test # bold red
false
* FIXED 3: true test (todo)
* still broken 4: false test (todo) # bold green
* fixed 1 known breakage(s) # green
* still have 1 known breakage(s) # bold red
* failed 1 among remaining 3 test(s) # bold red
But is now:
diagnostic message # yellow
ok 1 - true test
not ok - 2 false test # bold red
# false
ok 3 - true test (todo) # TODO known breakage
not ok 4 - false test (todo) # TODO known breakage # bold green
# fixed 1 known breakage(s) # green
# still have 1 known breakage(s) # bold red
# failed 1 among remaining 3 test(s) # bold red
1..4
All the coloring is preserved when the test is run manually. Under
prove(1) the test performs as expected, even with --debug and
--verbose options:
$ prove ./example.sh :: --debug --verbose
./example.sh .. Dubious, test returned 1 (wstat 256, 0x100)
Failed 1/4 subtests
(1 TODO test unexpectedly succeeded)
Test Summary Report
-------------------
./example.sh (Wstat: 256 Tests: 4 Failed: 1)
Failed test: 2
TODO passed: 3
Non-zero exit status: 1
Files=1, Tests=4, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.00 cusr 0.01 csys = 0.03 CPU)
Result: FAIL
The TAP harness itself doesn't get confused by the color output, they
aren't used by test-lib.sh stdout isn't open to a terminal (test -t 1).
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-06-24 21:52:12 +00:00
|
|
|
|
test-lib: have the "check" mode for SANITIZE=leak consider leak logs
As noted in previous on-list discussions[1] we have various tests that
will falsely report being leak-free because we're missing the relevant
exit code from LSAN as summarized below.
We should fix those issues, but in the meantime and as an additional
sanity check we can and should consider our own ASAN logs before
reporting that a test is leak-free.
Before this compiling with SANITIZE=leak and running:
./t6407-merge-binary.sh
Will exit successfully, now we'll get an error and an informative
message on:
GIT_TEST_SANITIZE_LEAK_LOG=true ./t6407-merge-binary.sh
Even better, as noted in the updated t/README we'll now error out when
combined with the "check" mode:
GIT_TEST_PASSING_SANITIZE_LEAK=check \
GIT_TEST_SANITIZE_LEAK_LOG=true \
./t4058-diff-duplicates.sh
Why do we miss these leaks? Because:
* We have leaks inside "test_expect_failure" blocks, which by design
will not distinguish a "normal" failure from an abort() or
segfault. See [1] for a discussion of it shortcomings.
* We have "git" invocations outside of "test_expect_success",
e.g. setup code in the main body of the test, or in test helper
functions that don't use &&-chaining.
* Our tests will otherwise catch segfaults and abort(), but if we
invoke a command that invokes another command it needs to ferry the
exit code up to us.
Notably a command that e.g. might invoke "git pack-objects" might
itself exit with status 128 if that "pack-objects" segfaults or
abort()'s. If the test invoking the parent command(s) is using
"test_must_fail" we'll consider it an expected "ok" failure.
* run-command.c doesn't (but probably should) ferry up such exit
codes, so for e.g. "git push" tests where we expect a failure and an
underlying "git" command fails we won't ferry up the segfault or
abort exit code.
* We have gitweb.perl and some other perl code ignoring return values
from close(), i.e. ignoring exit codes from "git rev-parse" et al.
* We have in-tree shellscripts like "git-merge-one-file.sh" invoking
git commands, they'll usually return their own exit codes on "git"
failure, rather then ferrying up segfault or abort() exit code.
E.g. these invocations in git-merge-one-file.sh leak, but aren't
reflected in the "git merge" exit code:
src1=$(git unpack-file $2)
src2=$(git unpack-file $3)
That case would be easily "fixed" by adding a line like this after
each assignment:
test $? -ne 0 && exit $?
But we'd then in e.g. "t6407-merge-binary.sh" run into
write_tree_trivial() in "builtin/merge.c" calling die() instead of
ferrying up the relevant exit code.
Let's remove "TEST_PASSES_SANITIZE_LEAK=true" from tests we
were falsely marking as leak-free.
In the case of t6407-merge-binary.sh it was marked as leak-free in
9081a421a6d (checkout: fix "branch info" memory leaks,
2021-11-16). I'd previously removed other bad
"TEST_PASSES_SANITIZE_LEAK=true" opt-ins in the series merged in
ea05fd5fbf7 (Merge branch 'ab/keep-git-exit-codes-in-tests',
2022-03-16). The case of t1060-object-corruption.sh is more subtle,
and will be discussed in a subsequent commit.
1. https://lore.kernel.org/git/cover-0.7-00000000000-20220318T002951Z-avarab@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:39 +00:00
|
|
|
check_test_results_san_file_ "$test_failure"
|
|
|
|
|
2022-07-27 23:13:32 +00:00
|
|
|
if test -n "$invert_exit_code"
|
|
|
|
then
|
|
|
|
_invert_exit_code_failure_end_blurb
|
|
|
|
GIT_EXIT_OK=t
|
|
|
|
exit 0
|
|
|
|
fi
|
|
|
|
|
2022-07-27 23:13:31 +00:00
|
|
|
GIT_EXIT_OK=t
|
2005-05-14 05:50:32 +00:00
|
|
|
exit 1 ;;
|
|
|
|
|
|
|
|
esac
|
|
|
|
}
|
|
|
|
|
2009-12-03 05:14:06 +00:00
|
|
|
if test -n "$valgrind"
|
2009-02-03 23:25:59 +00:00
|
|
|
then
|
|
|
|
make_symlink () {
|
|
|
|
test -h "$2" &&
|
|
|
|
test "$1" = "$(readlink "$2")" || {
|
|
|
|
# be super paranoid
|
|
|
|
if mkdir "$2".lock
|
|
|
|
then
|
|
|
|
rm -f "$2" &&
|
|
|
|
ln -s "$1" "$2" &&
|
|
|
|
rm -r "$2".lock
|
|
|
|
else
|
|
|
|
while test -d "$2".lock
|
|
|
|
do
|
|
|
|
say "Waiting for lock on $2."
|
|
|
|
sleep 1
|
|
|
|
done
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
make_valgrind_symlink () {
|
2011-06-17 20:36:32 +00:00
|
|
|
# handle only executables, unless they are shell libraries that
|
2013-11-25 21:03:52 +00:00
|
|
|
# need to be in the exec-path.
|
2011-06-17 20:36:32 +00:00
|
|
|
test -x "$1" ||
|
2018-08-24 15:20:11 +00:00
|
|
|
test "# " = "$(test_copy_bytes 2 <"$1")" ||
|
2011-06-17 20:36:32 +00:00
|
|
|
return;
|
2009-02-03 23:25:59 +00:00
|
|
|
|
|
|
|
base=$(basename "$1")
|
2016-10-27 22:14:00 +00:00
|
|
|
case "$base" in
|
|
|
|
test-*)
|
|
|
|
symlink_target="$GIT_BUILD_DIR/t/helper/$base"
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
symlink_target="$GIT_BUILD_DIR/$base"
|
|
|
|
;;
|
|
|
|
esac
|
2009-02-03 23:25:59 +00:00
|
|
|
# do not override scripts
|
|
|
|
if test -x "$symlink_target" &&
|
|
|
|
test ! -d "$symlink_target" &&
|
2018-08-24 15:20:11 +00:00
|
|
|
test "#!" != "$(test_copy_bytes 2 <"$symlink_target")"
|
2009-02-03 23:25:59 +00:00
|
|
|
then
|
|
|
|
symlink_target=../valgrind.sh
|
|
|
|
fi
|
2009-02-03 23:26:08 +00:00
|
|
|
case "$base" in
|
|
|
|
*.sh|*.perl)
|
|
|
|
symlink_target=../unprocessed-script
|
|
|
|
esac
|
2009-02-03 23:25:59 +00:00
|
|
|
# create the link, or replace it if it is out of date
|
|
|
|
make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit
|
|
|
|
}
|
|
|
|
|
2013-10-19 21:06:07 +00:00
|
|
|
# override all git executables in TEST_DIRECTORY/..
|
|
|
|
GIT_VALGRIND=$TEST_DIRECTORY/valgrind
|
|
|
|
mkdir -p "$GIT_VALGRIND"/bin
|
2016-07-11 11:45:08 +00:00
|
|
|
for file in $GIT_BUILD_DIR/git* $GIT_BUILD_DIR/t/helper/test-*
|
2013-10-19 21:06:07 +00:00
|
|
|
do
|
|
|
|
make_valgrind_symlink $file
|
|
|
|
done
|
|
|
|
# special-case the mergetools loadables
|
|
|
|
make_symlink "$GIT_BUILD_DIR"/mergetools "$GIT_VALGRIND/bin/mergetools"
|
|
|
|
OLDIFS=$IFS
|
|
|
|
IFS=:
|
|
|
|
for path in $PATH
|
|
|
|
do
|
|
|
|
ls "$path"/git-* 2> /dev/null |
|
|
|
|
while read file
|
test-lib: support running tests under valgrind in parallel
With the new --valgrind-parallel=<n> option, we support running the
tests in a single test script under valgrind in parallel using 'n'
processes.
This really follows the dumbest approach possible, as follows:
* We spawn the test script 'n' times, using a throw-away
TEST_OUTPUT_DIRECTORY. Each of the instances is given options that
ensures that it only runs every n-th test under valgrind, but
together they cover the entire range.
* We add up the numbers from the individual tests, and provide the
usual output.
This is really a gross hack at this point, and should be improved. In
particular we should keep the actual outputs somewhere more easily
discoverable, and summarize them to the user.
Nevertheless, this is already workable and gives a speedup of more
than 2 on a dual-core (hyperthreaded) machine, using n=4. This is
expected since the overhead of valgrind is so big (on the order of 20x
under good conditions, and a large startup overhead at every git
invocation) that redundantly running the non-valgrind tests in between
is not that expensive.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Thomas Rast <trast@inf.ethz.ch>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2013-06-23 18:12:59 +00:00
|
|
|
do
|
2013-10-19 21:06:07 +00:00
|
|
|
make_valgrind_symlink "$file"
|
test-lib: support running tests under valgrind in parallel
With the new --valgrind-parallel=<n> option, we support running the
tests in a single test script under valgrind in parallel using 'n'
processes.
This really follows the dumbest approach possible, as follows:
* We spawn the test script 'n' times, using a throw-away
TEST_OUTPUT_DIRECTORY. Each of the instances is given options that
ensures that it only runs every n-th test under valgrind, but
together they cover the entire range.
* We add up the numbers from the individual tests, and provide the
usual output.
This is really a gross hack at this point, and should be improved. In
particular we should keep the actual outputs somewhere more easily
discoverable, and summarize them to the user.
Nevertheless, this is already workable and gives a speedup of more
than 2 on a dual-core (hyperthreaded) machine, using n=4. This is
expected since the overhead of valgrind is so big (on the order of 20x
under good conditions, and a large startup overhead at every git
invocation) that redundantly running the non-valgrind tests in between
is not that expensive.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Thomas Rast <trast@inf.ethz.ch>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2013-06-23 18:12:59 +00:00
|
|
|
done
|
2013-10-19 21:06:07 +00:00
|
|
|
done
|
|
|
|
IFS=$OLDIFS
|
2009-02-03 23:25:59 +00:00
|
|
|
PATH=$GIT_VALGRIND/bin:$PATH
|
|
|
|
GIT_EXEC_PATH=$GIT_VALGRIND/bin
|
|
|
|
export GIT_VALGRIND
|
2013-03-31 08:00:16 +00:00
|
|
|
GIT_VALGRIND_MODE="$valgrind"
|
|
|
|
export GIT_VALGRIND_MODE
|
2013-06-23 18:12:57 +00:00
|
|
|
GIT_VALGRIND_ENABLED=t
|
2013-10-19 21:06:07 +00:00
|
|
|
test -n "$valgrind_only" && GIT_VALGRIND_ENABLED=
|
2013-06-23 18:12:57 +00:00
|
|
|
export GIT_VALGRIND_ENABLED
|
2012-09-01 18:11:49 +00:00
|
|
|
elif test -n "$GIT_TEST_INSTALLED"
|
|
|
|
then
|
2009-12-03 05:14:06 +00:00
|
|
|
GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path) ||
|
|
|
|
error "Cannot run git from $GIT_TEST_INSTALLED."
|
2018-11-12 13:48:33 +00:00
|
|
|
PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$PATH
|
2009-12-03 05:14:06 +00:00
|
|
|
GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH}
|
|
|
|
else # normal case, use ../bin-wrappers only unless $with_dashes:
|
2019-01-29 14:19:37 +00:00
|
|
|
if test -n "$no_bin_wrappers"
|
2012-09-01 18:11:49 +00:00
|
|
|
then
|
2019-01-29 14:19:37 +00:00
|
|
|
with_dashes=t
|
|
|
|
else
|
|
|
|
git_bin_dir="$GIT_BUILD_DIR/bin-wrappers"
|
|
|
|
if ! test -x "$git_bin_dir/git"
|
2012-09-01 18:11:49 +00:00
|
|
|
then
|
2019-01-29 14:19:37 +00:00
|
|
|
if test -z "$with_dashes"
|
|
|
|
then
|
|
|
|
say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH"
|
|
|
|
fi
|
|
|
|
with_dashes=t
|
2009-12-03 05:14:06 +00:00
|
|
|
fi
|
2019-01-29 14:19:37 +00:00
|
|
|
PATH="$git_bin_dir:$PATH"
|
2009-12-03 05:14:06 +00:00
|
|
|
fi
|
2010-08-19 16:08:10 +00:00
|
|
|
GIT_EXEC_PATH=$GIT_BUILD_DIR
|
2012-09-01 18:11:49 +00:00
|
|
|
if test -n "$with_dashes"
|
|
|
|
then
|
2019-01-29 14:19:35 +00:00
|
|
|
PATH="$GIT_BUILD_DIR:$GIT_BUILD_DIR/t/helper:$PATH"
|
2009-12-03 05:14:06 +00:00
|
|
|
fi
|
2009-02-03 23:25:59 +00:00
|
|
|
fi
|
2010-08-19 16:08:10 +00:00
|
|
|
GIT_TEMPLATE_DIR="$GIT_BUILD_DIR"/templates/blt
|
2008-02-06 10:11:53 +00:00
|
|
|
GIT_CONFIG_NOSYSTEM=1
|
2011-03-15 09:05:10 +00:00
|
|
|
GIT_ATTR_NOSYSTEM=1
|
test-lib: set GIT_CEILING_DIRECTORIES to protect the surrounding repository
Every once in a while a test somehow manages to escape from its trash
directory and modifies the surrounding repository, whether because of
a bug in git itself, a bug in a test [1], or e.g. when trying to run
tests with a shell that is, in general, unable to run our tests [2].
Set GIT_CEILING_DIRECTORIES="$TRASH_DIRECTORY/.." as an additional
safety measure to protect the surrounding repository at least from
modifications by git commands executed in the tests (assuming that
handling of ceiling directories during repository discovery is not
broken, and, of course, it won't save us from regular shell commands,
e.g. 'cd .. && rm -f ...').
[1] e.g. https://public-inbox.org/git/20210423051255.GD2947267@szeder.dev
[2] $ git symbolic-ref HEAD
refs/heads/master
$ ksh ./t2011-checkout-invalid-head.sh
[... a lot of "not ok" ...]
$ git symbolic-ref HEAD
refs/heads/other
(In short: 'ksh' doesn't support the 'local' builtin command,
which is used by 'test_oid', causing it to return with error
whenever it's called, leaving ZERO_OID set to empty, so when the
test 'checkout main from invalid HEAD' runs 'echo $ZERO_OID
>.git/HEAD' it writes a corrupt (not invalid) HEAD, and subsequent
git commands don't recognize the repository in the trash directory
anymore, but operate on the surrounding repo.)
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-08-29 09:25:36 +00:00
|
|
|
GIT_CEILING_DIRECTORIES="$TRASH_DIRECTORY/.."
|
|
|
|
export PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR GIT_CONFIG_NOSYSTEM GIT_ATTR_NOSYSTEM GIT_CEILING_DIRECTORIES
|
2005-12-08 05:52:28 +00:00
|
|
|
|
2010-06-11 16:40:25 +00:00
|
|
|
if test -z "$GIT_TEST_CMP"
|
|
|
|
then
|
|
|
|
if test -n "$GIT_TEST_CMP_USE_COPIED_CONTEXT"
|
|
|
|
then
|
|
|
|
GIT_TEST_CMP="$DIFF -c"
|
|
|
|
else
|
|
|
|
GIT_TEST_CMP="$DIFF -u"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
Makefile: replace perl/Makefile.PL with simple make rules
Replace the perl/Makefile.PL and the fallback perl/Makefile used under
NO_PERL_MAKEMAKER=NoThanks with a much simpler implementation heavily
inspired by how the i18n infrastructure's build process works[1].
The reason for having the Makefile.PL in the first place is that it
was initially[2] building a perl C binding to interface with libgit,
this functionality, that was removed[3] before Git.pm ever made it to
the master branch.
We've since since started maintaining a fallback perl/Makefile, as
MakeMaker wouldn't work on some platforms[4]. That's just the tip of
the iceberg. We have the PM.stamp hack in the top-level Makefile[5] to
detect whether we need to regenerate the perl/perl.mak, which I fixed
just recently to deal with issues like the perl version changing from
under us[6].
There is absolutely no reason for why this needs to be so complex
anymore. All we're getting out of this elaborate Rube Goldberg machine
was copying perl/* to perl/blib/* as we do a string-replacement on
the *.pm files to hardcode @@LOCALEDIR@@ in the source, as well as
pod2man-ing Git.pm & friends.
So replace the whole thing with something that's pretty much a copy of
how we generate po/build/**.mo from po/*.po, just with a small sed(1)
command instead of msgfmt. As that's being done rename the files
from *.pm to *.pmc just to indicate that they're generated (see
"perldoc -f require").
While I'm at it, change the fallback for Error.pm from being something
where we'll ship our own Error.pm if one doesn't exist at build time
to one where we just use a Git::Error wrapper that'll always prefer
the system-wide Error.pm, only falling back to our own copy if it
really doesn't exist at runtime. It's now shipped as
Git::FromCPAN::Error, making it easy to add other modules to
Git::FromCPAN::* in the future if that's needed.
Functional changes:
* This will not always install into perl's idea of its global
"installsitelib". This only potentially matters for packagers that
need to expose Git.pm for non-git use, and as explained in the
INSTALL file there's a trivial workaround.
* The scripts themselves will 'use lib' the target directory, but if
INSTLIBDIR is set it overrides it. It doesn't have to be this way,
it could be set in addition to INSTLIBDIR, but my reading of [7] is
that this is the desired behavior.
* We don't build man pages for all of the perl modules as we used to,
only Git(3pm). As discussed on-list[8] that we were building
installed manpages for purely internal APIs like Git::I18N or
private-Error.pm was always a bug anyway, and all the Git::SVN::*
ones say they're internal APIs.
There are apparently external users of Git.pm, but I don't expect
there to be any of the others.
As a side-effect of these general changes the perl documentation
now only installed by install-{doc,man}, not a mere "install" as
before.
1. 5e9637c629 ("i18n: add infrastructure for translating Git with
gettext", 2011-11-18)
2. b1edc53d06 ("Introduce Git.pm (v4)", 2006-06-24)
3. 18b0fc1ce1 ("Git.pm: Kill Git.xs for now", 2006-09-23)
4. f848718a69 ("Make perl/ build procedure ActiveState friendly.",
2006-12-04)
5. ee9be06770 ("perl: detect new files in MakeMaker builds",
2012-07-27)
6. c59c4939c2 ("perl: regenerate perl.mak if perl -V changes",
2017-03-29)
7. 0386dd37b1 ("Makefile: add PERLLIB_EXTRA variable that adds to
default perl path", 2013-11-15)
8. 87bmjjv1pu.fsf@evledraar.booking.com ("Re: [PATCH] Makefile:
replace perl/Makefile.PL with simple make rules"
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-12-10 21:13:33 +00:00
|
|
|
GITPERLLIB="$GIT_BUILD_DIR"/perl/build/lib
|
2006-07-03 21:16:32 +00:00
|
|
|
export GITPERLLIB
|
2010-08-19 16:08:10 +00:00
|
|
|
test -d "$GIT_BUILD_DIR"/templates/blt || {
|
2022-07-27 23:13:31 +00:00
|
|
|
BAIL_OUT "You haven't built things yet, have you?"
|
2005-12-11 04:55:32 +00:00
|
|
|
}
|
2005-05-14 05:50:32 +00:00
|
|
|
|
2019-01-21 15:12:19 +00:00
|
|
|
if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool$X
|
2012-09-01 18:11:49 +00:00
|
|
|
then
|
2022-07-27 23:13:31 +00:00
|
|
|
BAIL_OUT 'You need to build test-tool; Run "make t/helper/test-tool" in the source (toplevel) directory'
|
2007-02-25 00:59:52 +00:00
|
|
|
fi
|
|
|
|
|
test-lib: bring $remove_trash out of retirement
There's no point in creating a repository or directory only to decide
right afterwards that we're skipping all the tests. We can save
ourselves the redundant "git init" or "mkdir" and "rm -rf" in this
case.
We carry around the "$remove_trash" variable because if the directory
is unexpectedly gone at test_done time we'll still want to hit the
"trash directory already removed" error, but not if we never created
the trash directory. See df4c0d1a792 (test-lib: abort when can't
remove trash directory, 2017-04-20) for the addition of that error.
So let's partially revert 06478dab4c (test-lib: retire $remove_trash
variable, 2017-04-23) and move the decision about whether to skip all
tests earlier.
Let's also fix a bug that was with us since abc5d372ec (Enable
parallel tests, 2008-08-08): we would leak $remove_trash from the
environment. We don't want this to error out, so let's reset it to the
empty string first:
remove_trash=t GIT_SKIP_TESTS=t0001 ./t0001-init.sh
I tested this with --debug, see 4d0912a206 (test-lib.sh: do not barf
under --debug at the end of the test, 2017-04-24) for a bug we don't
want to re-introduce.
While I'm at it, let's move the HOME assignment to just before
test_create_repo, it could be lower, but it seems better to set it
before calling anything in test-lib-functions.sh
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-10 14:19:00 +00:00
|
|
|
# Are we running this test at all?
|
|
|
|
remove_trash=
|
|
|
|
this_test=${0##*/}
|
|
|
|
this_test=${this_test%%-*}
|
test-lib: avoid accidental globbing in match_pattern_list()
We have a custom match_pattern_list() function which we use for matching
test names (like "t1234") against glob-like patterns (like "t1???") for
$GIT_SKIP_TESTS, --verbose-only, etc.
Those patterns may have multiple whitespace-separated elements (e.g.,
"t0* t1234 t5?78"). The callers of match_pattern_list thus pass the
strings unquoted, so that the shell does the usual field-splitting into
separate arguments.
But this also means the shell will do the usual globbing for each
argument, which can result in us seeing an expansion based on what's in
the filesystem, rather than the real pattern. For example, if I have the
path "t5000" in the filesystem, and you feed the pattern "t?000", that
_should_ match the string "t0000", but it won't after the shell has
expanded it to "t5000".
This has been a bug ever since that function was introduced. But it
didn't usually trigger since we typically use the function inside the
trash directory, which has a very limited set of files that are unlikely
to match. It became a lot easier to trigger after edc23840b0 (test-lib:
bring $remove_trash out of retirement, 2021-05-10), because now we match
$GIT_SKIP_TESTS before even entering the trash directory. So the t5000
example above can be seen with:
GIT_SKIP_TESTS=t?000 ./t0000-basic.sh
which should skip all tests but doesn't.
We can fix this by using "set -f" to ask the shell not to glob (which is
in POSIX, so should hopefully be portable enough). We only want to do
this in a subshell (to avoid polluting the rest of the script), which
means we need to get the whole string intact into the match_pattern_list
function by quoting it. Arguably this is a good idea anyway, since it
makes it much more obvious that we intend to split, and it's not simply
sloppy scripting.
Diagnosed-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-06-16 10:23:07 +00:00
|
|
|
if match_pattern_list "$this_test" "$GIT_SKIP_TESTS"
|
test-lib: bring $remove_trash out of retirement
There's no point in creating a repository or directory only to decide
right afterwards that we're skipping all the tests. We can save
ourselves the redundant "git init" or "mkdir" and "rm -rf" in this
case.
We carry around the "$remove_trash" variable because if the directory
is unexpectedly gone at test_done time we'll still want to hit the
"trash directory already removed" error, but not if we never created
the trash directory. See df4c0d1a792 (test-lib: abort when can't
remove trash directory, 2017-04-20) for the addition of that error.
So let's partially revert 06478dab4c (test-lib: retire $remove_trash
variable, 2017-04-23) and move the decision about whether to skip all
tests earlier.
Let's also fix a bug that was with us since abc5d372ec (Enable
parallel tests, 2008-08-08): we would leak $remove_trash from the
environment. We don't want this to error out, so let's reset it to the
empty string first:
remove_trash=t GIT_SKIP_TESTS=t0001 ./t0001-init.sh
I tested this with --debug, see 4d0912a206 (test-lib.sh: do not barf
under --debug at the end of the test, 2017-04-24) for a bug we don't
want to re-introduce.
While I'm at it, let's move the HOME assignment to just before
test_create_repo, it could be lower, but it seems better to set it
before calling anything in test-lib-functions.sh
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-10 14:19:00 +00:00
|
|
|
then
|
|
|
|
say_color info >&3 "skipping test $this_test altogether"
|
|
|
|
skip_all="skip all tests in $this_test"
|
|
|
|
test_done
|
|
|
|
fi
|
|
|
|
|
2022-07-27 23:13:34 +00:00
|
|
|
BAIL_OUT_ENV_NEEDS_SANITIZE_LEAK () {
|
|
|
|
BAIL_OUT "$1 has no effect except when compiled with SANITIZE=leak"
|
|
|
|
}
|
|
|
|
|
tests: add a test mode for SANITIZE=leak, run it in CI
While git can be compiled with SANITIZE=leak, we have not run
regression tests under that mode. Memory leaks have only been fixed as
one-offs without structured regression testing.
This change adds CI testing for it. We'll now build and small set of
whitelisted t00*.sh tests under Linux with a new job called
"linux-leaks".
The CI target uses a new GIT_TEST_PASSING_SANITIZE_LEAK=true test
mode. When running in that mode, we'll assert that we were compiled
with SANITIZE=leak. We'll then skip all tests, except those that we've
opted-in by setting "TEST_PASSES_SANITIZE_LEAK=true".
A test setting "TEST_PASSES_SANITIZE_LEAK=true" setting can in turn
make use of the "SANITIZE_LEAK" prerequisite, should they wish to
selectively skip tests even under
"GIT_TEST_PASSING_SANITIZE_LEAK=true". In the preceding commit we
started doing this in "t0004-unwritable.sh" under SANITIZE=leak, now
it'll combine nicely with "GIT_TEST_PASSING_SANITIZE_LEAK=true".
This is how tests that don't set "TEST_PASSES_SANITIZE_LEAK=true" will
be skipped under GIT_TEST_PASSING_SANITIZE_LEAK=true:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0001-init.sh
1..0 # SKIP skip all tests in t0001 under SANITIZE=leak, TEST_PASSES_SANITIZE_LEAK not set
The intent is to add more TEST_PASSES_SANITIZE_LEAK=true annotations
as follow-up change, but let's start small to begin with.
In ci/run-build-and-tests.sh we make use of the default "*" case to
run "make test" without any GIT_TEST_* modes. SANITIZE=leak is known
to fail in combination with GIT_TEST_SPLIT_INDEX=true in
t0016-oidmap.sh, and we're likely to have other such failures in
various GIT_TEST_* modes. Let's focus on getting the base tests
passing, we can expand coverage to GIT_TEST_* modes later.
It would also be possible to implement a more lightweight version of
this by only relying on setting "LSAN_OPTIONS". See
<YS9OT/pn5rRK9cGB@coredump.intra.peff.net>[1] and
<YS9ZIDpANfsh7N+S@coredump.intra.peff.net>[2] for a discussion of
that. I've opted for this approach of adding a GIT_TEST_* mode instead
because it's consistent with how we handle other special test modes.
Being able to add a "!SANITIZE_LEAK" prerequisite and calling
"test_done" early if it isn't satisfied also means that we can more
incrementally add regression tests without being forced to fix
widespread and hard-to-fix leaks at the same time.
We have tests that do simple checking of some tool we're interested
in, but later on in the script might be stressing trace2, or common
sources of leaks like "git log" in combination with the tool (e.g. the
commit-graph tests). To be clear having a prerequisite could also be
accomplished by using "LSAN_OPTIONS" directly.
On the topic of "LSAN_OPTIONS": It would be nice to have a mode to
aggregate all failures in our various scripts, see [2] for a start at
doing that which sets "log_path" in "LSAN_OPTIONS". I've punted on
that for now, it can be added later.
As of writing this we've got major regressions between master..seen,
i.e. the t000*.sh tests and more fixed since 31f9acf9ce2 (Merge branch
'ah/plugleaks', 2021-08-04) have regressed recently.
See the discussion at <87czsv2idy.fsf@evledraar.gmail.com>[3] about
the lack of this sort of test mode, and 0e5bba53af (add UNLEAK
annotation for reducing leak false positives, 2017-09-08) for the
initial addition of SANITIZE=leak.
See also 09595ab381 (Merge branch 'jk/leak-checkers', 2017-09-19),
7782066f67 (Merge branch 'jk/apache-lsan', 2019-05-19) and the recent
936e58851a (Merge branch 'ah/plugleaks', 2021-05-07) for some of the
past history of "one-off" SANITIZE=leak (and more) fixes.
As noted in [5] we can't support this on OSX yet until Clang 14 is
released, at that point we'll probably want to resurrect that
"osx-leaks" job.
1. https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer
2. https://lore.kernel.org/git/YS9OT%2Fpn5rRK9cGB@coredump.intra.peff.net/
3. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/
4. https://lore.kernel.org/git/YS9ZIDpANfsh7N+S@coredump.intra.peff.net/
5. https://lore.kernel.org/git/20210916035603.76369-1-carenas@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Carlo Marcelo Arenas Belón <carenas@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-23 09:20:46 +00:00
|
|
|
if test -n "$SANITIZE_LEAK"
|
|
|
|
then
|
2022-07-27 23:13:38 +00:00
|
|
|
# Normalize with test_bool_env
|
|
|
|
passes_sanitize_leak=
|
|
|
|
|
2023-01-12 16:03:21 +00:00
|
|
|
# We need to see TEST_PASSES_SANITIZE_LEAK in "test-tool
|
|
|
|
# env-helper" (via test_bool_env)
|
2022-07-27 23:13:38 +00:00
|
|
|
export TEST_PASSES_SANITIZE_LEAK
|
|
|
|
if test_bool_env TEST_PASSES_SANITIZE_LEAK false
|
|
|
|
then
|
|
|
|
passes_sanitize_leak=t
|
|
|
|
fi
|
|
|
|
|
|
|
|
if test "$GIT_TEST_PASSING_SANITIZE_LEAK" = "check"
|
tests: add a test mode for SANITIZE=leak, run it in CI
While git can be compiled with SANITIZE=leak, we have not run
regression tests under that mode. Memory leaks have only been fixed as
one-offs without structured regression testing.
This change adds CI testing for it. We'll now build and small set of
whitelisted t00*.sh tests under Linux with a new job called
"linux-leaks".
The CI target uses a new GIT_TEST_PASSING_SANITIZE_LEAK=true test
mode. When running in that mode, we'll assert that we were compiled
with SANITIZE=leak. We'll then skip all tests, except those that we've
opted-in by setting "TEST_PASSES_SANITIZE_LEAK=true".
A test setting "TEST_PASSES_SANITIZE_LEAK=true" setting can in turn
make use of the "SANITIZE_LEAK" prerequisite, should they wish to
selectively skip tests even under
"GIT_TEST_PASSING_SANITIZE_LEAK=true". In the preceding commit we
started doing this in "t0004-unwritable.sh" under SANITIZE=leak, now
it'll combine nicely with "GIT_TEST_PASSING_SANITIZE_LEAK=true".
This is how tests that don't set "TEST_PASSES_SANITIZE_LEAK=true" will
be skipped under GIT_TEST_PASSING_SANITIZE_LEAK=true:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0001-init.sh
1..0 # SKIP skip all tests in t0001 under SANITIZE=leak, TEST_PASSES_SANITIZE_LEAK not set
The intent is to add more TEST_PASSES_SANITIZE_LEAK=true annotations
as follow-up change, but let's start small to begin with.
In ci/run-build-and-tests.sh we make use of the default "*" case to
run "make test" without any GIT_TEST_* modes. SANITIZE=leak is known
to fail in combination with GIT_TEST_SPLIT_INDEX=true in
t0016-oidmap.sh, and we're likely to have other such failures in
various GIT_TEST_* modes. Let's focus on getting the base tests
passing, we can expand coverage to GIT_TEST_* modes later.
It would also be possible to implement a more lightweight version of
this by only relying on setting "LSAN_OPTIONS". See
<YS9OT/pn5rRK9cGB@coredump.intra.peff.net>[1] and
<YS9ZIDpANfsh7N+S@coredump.intra.peff.net>[2] for a discussion of
that. I've opted for this approach of adding a GIT_TEST_* mode instead
because it's consistent with how we handle other special test modes.
Being able to add a "!SANITIZE_LEAK" prerequisite and calling
"test_done" early if it isn't satisfied also means that we can more
incrementally add regression tests without being forced to fix
widespread and hard-to-fix leaks at the same time.
We have tests that do simple checking of some tool we're interested
in, but later on in the script might be stressing trace2, or common
sources of leaks like "git log" in combination with the tool (e.g. the
commit-graph tests). To be clear having a prerequisite could also be
accomplished by using "LSAN_OPTIONS" directly.
On the topic of "LSAN_OPTIONS": It would be nice to have a mode to
aggregate all failures in our various scripts, see [2] for a start at
doing that which sets "log_path" in "LSAN_OPTIONS". I've punted on
that for now, it can be added later.
As of writing this we've got major regressions between master..seen,
i.e. the t000*.sh tests and more fixed since 31f9acf9ce2 (Merge branch
'ah/plugleaks', 2021-08-04) have regressed recently.
See the discussion at <87czsv2idy.fsf@evledraar.gmail.com>[3] about
the lack of this sort of test mode, and 0e5bba53af (add UNLEAK
annotation for reducing leak false positives, 2017-09-08) for the
initial addition of SANITIZE=leak.
See also 09595ab381 (Merge branch 'jk/leak-checkers', 2017-09-19),
7782066f67 (Merge branch 'jk/apache-lsan', 2019-05-19) and the recent
936e58851a (Merge branch 'ah/plugleaks', 2021-05-07) for some of the
past history of "one-off" SANITIZE=leak (and more) fixes.
As noted in [5] we can't support this on OSX yet until Clang 14 is
released, at that point we'll probably want to resurrect that
"osx-leaks" job.
1. https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer
2. https://lore.kernel.org/git/YS9OT%2Fpn5rRK9cGB@coredump.intra.peff.net/
3. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/
4. https://lore.kernel.org/git/YS9ZIDpANfsh7N+S@coredump.intra.peff.net/
5. https://lore.kernel.org/git/20210916035603.76369-1-carenas@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Carlo Marcelo Arenas Belón <carenas@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-23 09:20:46 +00:00
|
|
|
then
|
test-lib: have the "check" mode for SANITIZE=leak consider leak logs
As noted in previous on-list discussions[1] we have various tests that
will falsely report being leak-free because we're missing the relevant
exit code from LSAN as summarized below.
We should fix those issues, but in the meantime and as an additional
sanity check we can and should consider our own ASAN logs before
reporting that a test is leak-free.
Before this compiling with SANITIZE=leak and running:
./t6407-merge-binary.sh
Will exit successfully, now we'll get an error and an informative
message on:
GIT_TEST_SANITIZE_LEAK_LOG=true ./t6407-merge-binary.sh
Even better, as noted in the updated t/README we'll now error out when
combined with the "check" mode:
GIT_TEST_PASSING_SANITIZE_LEAK=check \
GIT_TEST_SANITIZE_LEAK_LOG=true \
./t4058-diff-duplicates.sh
Why do we miss these leaks? Because:
* We have leaks inside "test_expect_failure" blocks, which by design
will not distinguish a "normal" failure from an abort() or
segfault. See [1] for a discussion of it shortcomings.
* We have "git" invocations outside of "test_expect_success",
e.g. setup code in the main body of the test, or in test helper
functions that don't use &&-chaining.
* Our tests will otherwise catch segfaults and abort(), but if we
invoke a command that invokes another command it needs to ferry the
exit code up to us.
Notably a command that e.g. might invoke "git pack-objects" might
itself exit with status 128 if that "pack-objects" segfaults or
abort()'s. If the test invoking the parent command(s) is using
"test_must_fail" we'll consider it an expected "ok" failure.
* run-command.c doesn't (but probably should) ferry up such exit
codes, so for e.g. "git push" tests where we expect a failure and an
underlying "git" command fails we won't ferry up the segfault or
abort exit code.
* We have gitweb.perl and some other perl code ignoring return values
from close(), i.e. ignoring exit codes from "git rev-parse" et al.
* We have in-tree shellscripts like "git-merge-one-file.sh" invoking
git commands, they'll usually return their own exit codes on "git"
failure, rather then ferrying up segfault or abort() exit code.
E.g. these invocations in git-merge-one-file.sh leak, but aren't
reflected in the "git merge" exit code:
src1=$(git unpack-file $2)
src2=$(git unpack-file $3)
That case would be easily "fixed" by adding a line like this after
each assignment:
test $? -ne 0 && exit $?
But we'd then in e.g. "t6407-merge-binary.sh" run into
write_tree_trivial() in "builtin/merge.c" calling die() instead of
ferrying up the relevant exit code.
Let's remove "TEST_PASSES_SANITIZE_LEAK=true" from tests we
were falsely marking as leak-free.
In the case of t6407-merge-binary.sh it was marked as leak-free in
9081a421a6d (checkout: fix "branch info" memory leaks,
2021-11-16). I'd previously removed other bad
"TEST_PASSES_SANITIZE_LEAK=true" opt-ins in the series merged in
ea05fd5fbf7 (Merge branch 'ab/keep-git-exit-codes-in-tests',
2022-03-16). The case of t1060-object-corruption.sh is more subtle,
and will be discussed in a subsequent commit.
1. https://lore.kernel.org/git/cover-0.7-00000000000-20220318T002951Z-avarab@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:39 +00:00
|
|
|
sanitize_leak_check=t
|
2022-07-27 23:13:38 +00:00
|
|
|
if test -n "$invert_exit_code"
|
|
|
|
then
|
|
|
|
BAIL_OUT "cannot use --invert-exit-code under GIT_TEST_PASSING_SANITIZE_LEAK=check"
|
|
|
|
fi
|
tests: add a test mode for SANITIZE=leak, run it in CI
While git can be compiled with SANITIZE=leak, we have not run
regression tests under that mode. Memory leaks have only been fixed as
one-offs without structured regression testing.
This change adds CI testing for it. We'll now build and small set of
whitelisted t00*.sh tests under Linux with a new job called
"linux-leaks".
The CI target uses a new GIT_TEST_PASSING_SANITIZE_LEAK=true test
mode. When running in that mode, we'll assert that we were compiled
with SANITIZE=leak. We'll then skip all tests, except those that we've
opted-in by setting "TEST_PASSES_SANITIZE_LEAK=true".
A test setting "TEST_PASSES_SANITIZE_LEAK=true" setting can in turn
make use of the "SANITIZE_LEAK" prerequisite, should they wish to
selectively skip tests even under
"GIT_TEST_PASSING_SANITIZE_LEAK=true". In the preceding commit we
started doing this in "t0004-unwritable.sh" under SANITIZE=leak, now
it'll combine nicely with "GIT_TEST_PASSING_SANITIZE_LEAK=true".
This is how tests that don't set "TEST_PASSES_SANITIZE_LEAK=true" will
be skipped under GIT_TEST_PASSING_SANITIZE_LEAK=true:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0001-init.sh
1..0 # SKIP skip all tests in t0001 under SANITIZE=leak, TEST_PASSES_SANITIZE_LEAK not set
The intent is to add more TEST_PASSES_SANITIZE_LEAK=true annotations
as follow-up change, but let's start small to begin with.
In ci/run-build-and-tests.sh we make use of the default "*" case to
run "make test" without any GIT_TEST_* modes. SANITIZE=leak is known
to fail in combination with GIT_TEST_SPLIT_INDEX=true in
t0016-oidmap.sh, and we're likely to have other such failures in
various GIT_TEST_* modes. Let's focus on getting the base tests
passing, we can expand coverage to GIT_TEST_* modes later.
It would also be possible to implement a more lightweight version of
this by only relying on setting "LSAN_OPTIONS". See
<YS9OT/pn5rRK9cGB@coredump.intra.peff.net>[1] and
<YS9ZIDpANfsh7N+S@coredump.intra.peff.net>[2] for a discussion of
that. I've opted for this approach of adding a GIT_TEST_* mode instead
because it's consistent with how we handle other special test modes.
Being able to add a "!SANITIZE_LEAK" prerequisite and calling
"test_done" early if it isn't satisfied also means that we can more
incrementally add regression tests without being forced to fix
widespread and hard-to-fix leaks at the same time.
We have tests that do simple checking of some tool we're interested
in, but later on in the script might be stressing trace2, or common
sources of leaks like "git log" in combination with the tool (e.g. the
commit-graph tests). To be clear having a prerequisite could also be
accomplished by using "LSAN_OPTIONS" directly.
On the topic of "LSAN_OPTIONS": It would be nice to have a mode to
aggregate all failures in our various scripts, see [2] for a start at
doing that which sets "log_path" in "LSAN_OPTIONS". I've punted on
that for now, it can be added later.
As of writing this we've got major regressions between master..seen,
i.e. the t000*.sh tests and more fixed since 31f9acf9ce2 (Merge branch
'ah/plugleaks', 2021-08-04) have regressed recently.
See the discussion at <87czsv2idy.fsf@evledraar.gmail.com>[3] about
the lack of this sort of test mode, and 0e5bba53af (add UNLEAK
annotation for reducing leak false positives, 2017-09-08) for the
initial addition of SANITIZE=leak.
See also 09595ab381 (Merge branch 'jk/leak-checkers', 2017-09-19),
7782066f67 (Merge branch 'jk/apache-lsan', 2019-05-19) and the recent
936e58851a (Merge branch 'ah/plugleaks', 2021-05-07) for some of the
past history of "one-off" SANITIZE=leak (and more) fixes.
As noted in [5] we can't support this on OSX yet until Clang 14 is
released, at that point we'll probably want to resurrect that
"osx-leaks" job.
1. https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer
2. https://lore.kernel.org/git/YS9OT%2Fpn5rRK9cGB@coredump.intra.peff.net/
3. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/
4. https://lore.kernel.org/git/YS9ZIDpANfsh7N+S@coredump.intra.peff.net/
5. https://lore.kernel.org/git/20210916035603.76369-1-carenas@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Carlo Marcelo Arenas Belón <carenas@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-23 09:20:46 +00:00
|
|
|
|
2022-07-27 23:13:38 +00:00
|
|
|
if test -z "$passes_sanitize_leak"
|
tests: add a test mode for SANITIZE=leak, run it in CI
While git can be compiled with SANITIZE=leak, we have not run
regression tests under that mode. Memory leaks have only been fixed as
one-offs without structured regression testing.
This change adds CI testing for it. We'll now build and small set of
whitelisted t00*.sh tests under Linux with a new job called
"linux-leaks".
The CI target uses a new GIT_TEST_PASSING_SANITIZE_LEAK=true test
mode. When running in that mode, we'll assert that we were compiled
with SANITIZE=leak. We'll then skip all tests, except those that we've
opted-in by setting "TEST_PASSES_SANITIZE_LEAK=true".
A test setting "TEST_PASSES_SANITIZE_LEAK=true" setting can in turn
make use of the "SANITIZE_LEAK" prerequisite, should they wish to
selectively skip tests even under
"GIT_TEST_PASSING_SANITIZE_LEAK=true". In the preceding commit we
started doing this in "t0004-unwritable.sh" under SANITIZE=leak, now
it'll combine nicely with "GIT_TEST_PASSING_SANITIZE_LEAK=true".
This is how tests that don't set "TEST_PASSES_SANITIZE_LEAK=true" will
be skipped under GIT_TEST_PASSING_SANITIZE_LEAK=true:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0001-init.sh
1..0 # SKIP skip all tests in t0001 under SANITIZE=leak, TEST_PASSES_SANITIZE_LEAK not set
The intent is to add more TEST_PASSES_SANITIZE_LEAK=true annotations
as follow-up change, but let's start small to begin with.
In ci/run-build-and-tests.sh we make use of the default "*" case to
run "make test" without any GIT_TEST_* modes. SANITIZE=leak is known
to fail in combination with GIT_TEST_SPLIT_INDEX=true in
t0016-oidmap.sh, and we're likely to have other such failures in
various GIT_TEST_* modes. Let's focus on getting the base tests
passing, we can expand coverage to GIT_TEST_* modes later.
It would also be possible to implement a more lightweight version of
this by only relying on setting "LSAN_OPTIONS". See
<YS9OT/pn5rRK9cGB@coredump.intra.peff.net>[1] and
<YS9ZIDpANfsh7N+S@coredump.intra.peff.net>[2] for a discussion of
that. I've opted for this approach of adding a GIT_TEST_* mode instead
because it's consistent with how we handle other special test modes.
Being able to add a "!SANITIZE_LEAK" prerequisite and calling
"test_done" early if it isn't satisfied also means that we can more
incrementally add regression tests without being forced to fix
widespread and hard-to-fix leaks at the same time.
We have tests that do simple checking of some tool we're interested
in, but later on in the script might be stressing trace2, or common
sources of leaks like "git log" in combination with the tool (e.g. the
commit-graph tests). To be clear having a prerequisite could also be
accomplished by using "LSAN_OPTIONS" directly.
On the topic of "LSAN_OPTIONS": It would be nice to have a mode to
aggregate all failures in our various scripts, see [2] for a start at
doing that which sets "log_path" in "LSAN_OPTIONS". I've punted on
that for now, it can be added later.
As of writing this we've got major regressions between master..seen,
i.e. the t000*.sh tests and more fixed since 31f9acf9ce2 (Merge branch
'ah/plugleaks', 2021-08-04) have regressed recently.
See the discussion at <87czsv2idy.fsf@evledraar.gmail.com>[3] about
the lack of this sort of test mode, and 0e5bba53af (add UNLEAK
annotation for reducing leak false positives, 2017-09-08) for the
initial addition of SANITIZE=leak.
See also 09595ab381 (Merge branch 'jk/leak-checkers', 2017-09-19),
7782066f67 (Merge branch 'jk/apache-lsan', 2019-05-19) and the recent
936e58851a (Merge branch 'ah/plugleaks', 2021-05-07) for some of the
past history of "one-off" SANITIZE=leak (and more) fixes.
As noted in [5] we can't support this on OSX yet until Clang 14 is
released, at that point we'll probably want to resurrect that
"osx-leaks" job.
1. https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer
2. https://lore.kernel.org/git/YS9OT%2Fpn5rRK9cGB@coredump.intra.peff.net/
3. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/
4. https://lore.kernel.org/git/YS9ZIDpANfsh7N+S@coredump.intra.peff.net/
5. https://lore.kernel.org/git/20210916035603.76369-1-carenas@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Carlo Marcelo Arenas Belón <carenas@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-23 09:20:46 +00:00
|
|
|
then
|
2022-07-27 23:13:38 +00:00
|
|
|
say "in GIT_TEST_PASSING_SANITIZE_LEAK=check mode, setting --invert-exit-code for TEST_PASSES_SANITIZE_LEAK != true"
|
|
|
|
invert_exit_code=t
|
tests: add a test mode for SANITIZE=leak, run it in CI
While git can be compiled with SANITIZE=leak, we have not run
regression tests under that mode. Memory leaks have only been fixed as
one-offs without structured regression testing.
This change adds CI testing for it. We'll now build and small set of
whitelisted t00*.sh tests under Linux with a new job called
"linux-leaks".
The CI target uses a new GIT_TEST_PASSING_SANITIZE_LEAK=true test
mode. When running in that mode, we'll assert that we were compiled
with SANITIZE=leak. We'll then skip all tests, except those that we've
opted-in by setting "TEST_PASSES_SANITIZE_LEAK=true".
A test setting "TEST_PASSES_SANITIZE_LEAK=true" setting can in turn
make use of the "SANITIZE_LEAK" prerequisite, should they wish to
selectively skip tests even under
"GIT_TEST_PASSING_SANITIZE_LEAK=true". In the preceding commit we
started doing this in "t0004-unwritable.sh" under SANITIZE=leak, now
it'll combine nicely with "GIT_TEST_PASSING_SANITIZE_LEAK=true".
This is how tests that don't set "TEST_PASSES_SANITIZE_LEAK=true" will
be skipped under GIT_TEST_PASSING_SANITIZE_LEAK=true:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0001-init.sh
1..0 # SKIP skip all tests in t0001 under SANITIZE=leak, TEST_PASSES_SANITIZE_LEAK not set
The intent is to add more TEST_PASSES_SANITIZE_LEAK=true annotations
as follow-up change, but let's start small to begin with.
In ci/run-build-and-tests.sh we make use of the default "*" case to
run "make test" without any GIT_TEST_* modes. SANITIZE=leak is known
to fail in combination with GIT_TEST_SPLIT_INDEX=true in
t0016-oidmap.sh, and we're likely to have other such failures in
various GIT_TEST_* modes. Let's focus on getting the base tests
passing, we can expand coverage to GIT_TEST_* modes later.
It would also be possible to implement a more lightweight version of
this by only relying on setting "LSAN_OPTIONS". See
<YS9OT/pn5rRK9cGB@coredump.intra.peff.net>[1] and
<YS9ZIDpANfsh7N+S@coredump.intra.peff.net>[2] for a discussion of
that. I've opted for this approach of adding a GIT_TEST_* mode instead
because it's consistent with how we handle other special test modes.
Being able to add a "!SANITIZE_LEAK" prerequisite and calling
"test_done" early if it isn't satisfied also means that we can more
incrementally add regression tests without being forced to fix
widespread and hard-to-fix leaks at the same time.
We have tests that do simple checking of some tool we're interested
in, but later on in the script might be stressing trace2, or common
sources of leaks like "git log" in combination with the tool (e.g. the
commit-graph tests). To be clear having a prerequisite could also be
accomplished by using "LSAN_OPTIONS" directly.
On the topic of "LSAN_OPTIONS": It would be nice to have a mode to
aggregate all failures in our various scripts, see [2] for a start at
doing that which sets "log_path" in "LSAN_OPTIONS". I've punted on
that for now, it can be added later.
As of writing this we've got major regressions between master..seen,
i.e. the t000*.sh tests and more fixed since 31f9acf9ce2 (Merge branch
'ah/plugleaks', 2021-08-04) have regressed recently.
See the discussion at <87czsv2idy.fsf@evledraar.gmail.com>[3] about
the lack of this sort of test mode, and 0e5bba53af (add UNLEAK
annotation for reducing leak false positives, 2017-09-08) for the
initial addition of SANITIZE=leak.
See also 09595ab381 (Merge branch 'jk/leak-checkers', 2017-09-19),
7782066f67 (Merge branch 'jk/apache-lsan', 2019-05-19) and the recent
936e58851a (Merge branch 'ah/plugleaks', 2021-05-07) for some of the
past history of "one-off" SANITIZE=leak (and more) fixes.
As noted in [5] we can't support this on OSX yet until Clang 14 is
released, at that point we'll probably want to resurrect that
"osx-leaks" job.
1. https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer
2. https://lore.kernel.org/git/YS9OT%2Fpn5rRK9cGB@coredump.intra.peff.net/
3. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/
4. https://lore.kernel.org/git/YS9ZIDpANfsh7N+S@coredump.intra.peff.net/
5. https://lore.kernel.org/git/20210916035603.76369-1-carenas@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Carlo Marcelo Arenas Belón <carenas@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-23 09:20:46 +00:00
|
|
|
fi
|
2022-07-27 23:13:38 +00:00
|
|
|
elif test -z "$passes_sanitize_leak" &&
|
|
|
|
test_bool_env GIT_TEST_PASSING_SANITIZE_LEAK false
|
|
|
|
then
|
|
|
|
skip_all="skipping $this_test under GIT_TEST_PASSING_SANITIZE_LEAK=true"
|
|
|
|
test_done
|
tests: add a test mode for SANITIZE=leak, run it in CI
While git can be compiled with SANITIZE=leak, we have not run
regression tests under that mode. Memory leaks have only been fixed as
one-offs without structured regression testing.
This change adds CI testing for it. We'll now build and small set of
whitelisted t00*.sh tests under Linux with a new job called
"linux-leaks".
The CI target uses a new GIT_TEST_PASSING_SANITIZE_LEAK=true test
mode. When running in that mode, we'll assert that we were compiled
with SANITIZE=leak. We'll then skip all tests, except those that we've
opted-in by setting "TEST_PASSES_SANITIZE_LEAK=true".
A test setting "TEST_PASSES_SANITIZE_LEAK=true" setting can in turn
make use of the "SANITIZE_LEAK" prerequisite, should they wish to
selectively skip tests even under
"GIT_TEST_PASSING_SANITIZE_LEAK=true". In the preceding commit we
started doing this in "t0004-unwritable.sh" under SANITIZE=leak, now
it'll combine nicely with "GIT_TEST_PASSING_SANITIZE_LEAK=true".
This is how tests that don't set "TEST_PASSES_SANITIZE_LEAK=true" will
be skipped under GIT_TEST_PASSING_SANITIZE_LEAK=true:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0001-init.sh
1..0 # SKIP skip all tests in t0001 under SANITIZE=leak, TEST_PASSES_SANITIZE_LEAK not set
The intent is to add more TEST_PASSES_SANITIZE_LEAK=true annotations
as follow-up change, but let's start small to begin with.
In ci/run-build-and-tests.sh we make use of the default "*" case to
run "make test" without any GIT_TEST_* modes. SANITIZE=leak is known
to fail in combination with GIT_TEST_SPLIT_INDEX=true in
t0016-oidmap.sh, and we're likely to have other such failures in
various GIT_TEST_* modes. Let's focus on getting the base tests
passing, we can expand coverage to GIT_TEST_* modes later.
It would also be possible to implement a more lightweight version of
this by only relying on setting "LSAN_OPTIONS". See
<YS9OT/pn5rRK9cGB@coredump.intra.peff.net>[1] and
<YS9ZIDpANfsh7N+S@coredump.intra.peff.net>[2] for a discussion of
that. I've opted for this approach of adding a GIT_TEST_* mode instead
because it's consistent with how we handle other special test modes.
Being able to add a "!SANITIZE_LEAK" prerequisite and calling
"test_done" early if it isn't satisfied also means that we can more
incrementally add regression tests without being forced to fix
widespread and hard-to-fix leaks at the same time.
We have tests that do simple checking of some tool we're interested
in, but later on in the script might be stressing trace2, or common
sources of leaks like "git log" in combination with the tool (e.g. the
commit-graph tests). To be clear having a prerequisite could also be
accomplished by using "LSAN_OPTIONS" directly.
On the topic of "LSAN_OPTIONS": It would be nice to have a mode to
aggregate all failures in our various scripts, see [2] for a start at
doing that which sets "log_path" in "LSAN_OPTIONS". I've punted on
that for now, it can be added later.
As of writing this we've got major regressions between master..seen,
i.e. the t000*.sh tests and more fixed since 31f9acf9ce2 (Merge branch
'ah/plugleaks', 2021-08-04) have regressed recently.
See the discussion at <87czsv2idy.fsf@evledraar.gmail.com>[3] about
the lack of this sort of test mode, and 0e5bba53af (add UNLEAK
annotation for reducing leak false positives, 2017-09-08) for the
initial addition of SANITIZE=leak.
See also 09595ab381 (Merge branch 'jk/leak-checkers', 2017-09-19),
7782066f67 (Merge branch 'jk/apache-lsan', 2019-05-19) and the recent
936e58851a (Merge branch 'ah/plugleaks', 2021-05-07) for some of the
past history of "one-off" SANITIZE=leak (and more) fixes.
As noted in [5] we can't support this on OSX yet until Clang 14 is
released, at that point we'll probably want to resurrect that
"osx-leaks" job.
1. https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer
2. https://lore.kernel.org/git/YS9OT%2Fpn5rRK9cGB@coredump.intra.peff.net/
3. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/
4. https://lore.kernel.org/git/YS9ZIDpANfsh7N+S@coredump.intra.peff.net/
5. https://lore.kernel.org/git/20210916035603.76369-1-carenas@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Carlo Marcelo Arenas Belón <carenas@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-23 09:20:46 +00:00
|
|
|
fi
|
2022-07-27 23:13:34 +00:00
|
|
|
|
|
|
|
if test_bool_env GIT_TEST_SANITIZE_LEAK_LOG false
|
|
|
|
then
|
|
|
|
if ! mkdir -p "$TEST_RESULTS_SAN_DIR"
|
|
|
|
then
|
|
|
|
BAIL_OUT "cannot create $TEST_RESULTS_SAN_DIR"
|
|
|
|
fi &&
|
|
|
|
TEST_RESULTS_SAN_FILE="$TEST_RESULTS_SAN_DIR/$TEST_RESULTS_SAN_FILE_PFX"
|
|
|
|
|
test-lib: have the "check" mode for SANITIZE=leak consider leak logs
As noted in previous on-list discussions[1] we have various tests that
will falsely report being leak-free because we're missing the relevant
exit code from LSAN as summarized below.
We should fix those issues, but in the meantime and as an additional
sanity check we can and should consider our own ASAN logs before
reporting that a test is leak-free.
Before this compiling with SANITIZE=leak and running:
./t6407-merge-binary.sh
Will exit successfully, now we'll get an error and an informative
message on:
GIT_TEST_SANITIZE_LEAK_LOG=true ./t6407-merge-binary.sh
Even better, as noted in the updated t/README we'll now error out when
combined with the "check" mode:
GIT_TEST_PASSING_SANITIZE_LEAK=check \
GIT_TEST_SANITIZE_LEAK_LOG=true \
./t4058-diff-duplicates.sh
Why do we miss these leaks? Because:
* We have leaks inside "test_expect_failure" blocks, which by design
will not distinguish a "normal" failure from an abort() or
segfault. See [1] for a discussion of it shortcomings.
* We have "git" invocations outside of "test_expect_success",
e.g. setup code in the main body of the test, or in test helper
functions that don't use &&-chaining.
* Our tests will otherwise catch segfaults and abort(), but if we
invoke a command that invokes another command it needs to ferry the
exit code up to us.
Notably a command that e.g. might invoke "git pack-objects" might
itself exit with status 128 if that "pack-objects" segfaults or
abort()'s. If the test invoking the parent command(s) is using
"test_must_fail" we'll consider it an expected "ok" failure.
* run-command.c doesn't (but probably should) ferry up such exit
codes, so for e.g. "git push" tests where we expect a failure and an
underlying "git" command fails we won't ferry up the segfault or
abort exit code.
* We have gitweb.perl and some other perl code ignoring return values
from close(), i.e. ignoring exit codes from "git rev-parse" et al.
* We have in-tree shellscripts like "git-merge-one-file.sh" invoking
git commands, they'll usually return their own exit codes on "git"
failure, rather then ferrying up segfault or abort() exit code.
E.g. these invocations in git-merge-one-file.sh leak, but aren't
reflected in the "git merge" exit code:
src1=$(git unpack-file $2)
src2=$(git unpack-file $3)
That case would be easily "fixed" by adding a line like this after
each assignment:
test $? -ne 0 && exit $?
But we'd then in e.g. "t6407-merge-binary.sh" run into
write_tree_trivial() in "builtin/merge.c" calling die() instead of
ferrying up the relevant exit code.
Let's remove "TEST_PASSES_SANITIZE_LEAK=true" from tests we
were falsely marking as leak-free.
In the case of t6407-merge-binary.sh it was marked as leak-free in
9081a421a6d (checkout: fix "branch info" memory leaks,
2021-11-16). I'd previously removed other bad
"TEST_PASSES_SANITIZE_LEAK=true" opt-ins in the series merged in
ea05fd5fbf7 (Merge branch 'ab/keep-git-exit-codes-in-tests',
2022-03-16). The case of t1060-object-corruption.sh is more subtle,
and will be discussed in a subsequent commit.
1. https://lore.kernel.org/git/cover-0.7-00000000000-20220318T002951Z-avarab@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-07-27 23:13:39 +00:00
|
|
|
# In case "test-results" is left over from a previous
|
|
|
|
# run: Only report if new leaks show up.
|
|
|
|
TEST_RESULTS_SAN_DIR_NR_LEAKS_STARTUP=$(nr_san_dir_leaks_)
|
|
|
|
|
2022-07-27 23:13:34 +00:00
|
|
|
# Don't litter *.leak dirs if there was nothing to report
|
|
|
|
test_atexit "rmdir \"$TEST_RESULTS_SAN_DIR\" 2>/dev/null || :"
|
|
|
|
|
|
|
|
prepend_var LSAN_OPTIONS : dedup_token_length=9999
|
|
|
|
prepend_var LSAN_OPTIONS : log_exe_name=1
|
|
|
|
prepend_var LSAN_OPTIONS : log_path=\"$TEST_RESULTS_SAN_FILE\"
|
|
|
|
export LSAN_OPTIONS
|
|
|
|
fi
|
2022-07-27 23:13:38 +00:00
|
|
|
elif test "$GIT_TEST_PASSING_SANITIZE_LEAK" = "check" ||
|
|
|
|
test_bool_env GIT_TEST_PASSING_SANITIZE_LEAK false
|
tests: add a test mode for SANITIZE=leak, run it in CI
While git can be compiled with SANITIZE=leak, we have not run
regression tests under that mode. Memory leaks have only been fixed as
one-offs without structured regression testing.
This change adds CI testing for it. We'll now build and small set of
whitelisted t00*.sh tests under Linux with a new job called
"linux-leaks".
The CI target uses a new GIT_TEST_PASSING_SANITIZE_LEAK=true test
mode. When running in that mode, we'll assert that we were compiled
with SANITIZE=leak. We'll then skip all tests, except those that we've
opted-in by setting "TEST_PASSES_SANITIZE_LEAK=true".
A test setting "TEST_PASSES_SANITIZE_LEAK=true" setting can in turn
make use of the "SANITIZE_LEAK" prerequisite, should they wish to
selectively skip tests even under
"GIT_TEST_PASSING_SANITIZE_LEAK=true". In the preceding commit we
started doing this in "t0004-unwritable.sh" under SANITIZE=leak, now
it'll combine nicely with "GIT_TEST_PASSING_SANITIZE_LEAK=true".
This is how tests that don't set "TEST_PASSES_SANITIZE_LEAK=true" will
be skipped under GIT_TEST_PASSING_SANITIZE_LEAK=true:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0001-init.sh
1..0 # SKIP skip all tests in t0001 under SANITIZE=leak, TEST_PASSES_SANITIZE_LEAK not set
The intent is to add more TEST_PASSES_SANITIZE_LEAK=true annotations
as follow-up change, but let's start small to begin with.
In ci/run-build-and-tests.sh we make use of the default "*" case to
run "make test" without any GIT_TEST_* modes. SANITIZE=leak is known
to fail in combination with GIT_TEST_SPLIT_INDEX=true in
t0016-oidmap.sh, and we're likely to have other such failures in
various GIT_TEST_* modes. Let's focus on getting the base tests
passing, we can expand coverage to GIT_TEST_* modes later.
It would also be possible to implement a more lightweight version of
this by only relying on setting "LSAN_OPTIONS". See
<YS9OT/pn5rRK9cGB@coredump.intra.peff.net>[1] and
<YS9ZIDpANfsh7N+S@coredump.intra.peff.net>[2] for a discussion of
that. I've opted for this approach of adding a GIT_TEST_* mode instead
because it's consistent with how we handle other special test modes.
Being able to add a "!SANITIZE_LEAK" prerequisite and calling
"test_done" early if it isn't satisfied also means that we can more
incrementally add regression tests without being forced to fix
widespread and hard-to-fix leaks at the same time.
We have tests that do simple checking of some tool we're interested
in, but later on in the script might be stressing trace2, or common
sources of leaks like "git log" in combination with the tool (e.g. the
commit-graph tests). To be clear having a prerequisite could also be
accomplished by using "LSAN_OPTIONS" directly.
On the topic of "LSAN_OPTIONS": It would be nice to have a mode to
aggregate all failures in our various scripts, see [2] for a start at
doing that which sets "log_path" in "LSAN_OPTIONS". I've punted on
that for now, it can be added later.
As of writing this we've got major regressions between master..seen,
i.e. the t000*.sh tests and more fixed since 31f9acf9ce2 (Merge branch
'ah/plugleaks', 2021-08-04) have regressed recently.
See the discussion at <87czsv2idy.fsf@evledraar.gmail.com>[3] about
the lack of this sort of test mode, and 0e5bba53af (add UNLEAK
annotation for reducing leak false positives, 2017-09-08) for the
initial addition of SANITIZE=leak.
See also 09595ab381 (Merge branch 'jk/leak-checkers', 2017-09-19),
7782066f67 (Merge branch 'jk/apache-lsan', 2019-05-19) and the recent
936e58851a (Merge branch 'ah/plugleaks', 2021-05-07) for some of the
past history of "one-off" SANITIZE=leak (and more) fixes.
As noted in [5] we can't support this on OSX yet until Clang 14 is
released, at that point we'll probably want to resurrect that
"osx-leaks" job.
1. https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer
2. https://lore.kernel.org/git/YS9OT%2Fpn5rRK9cGB@coredump.intra.peff.net/
3. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/
4. https://lore.kernel.org/git/YS9ZIDpANfsh7N+S@coredump.intra.peff.net/
5. https://lore.kernel.org/git/20210916035603.76369-1-carenas@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Carlo Marcelo Arenas Belón <carenas@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-23 09:20:46 +00:00
|
|
|
then
|
2022-07-27 23:13:34 +00:00
|
|
|
BAIL_OUT_ENV_NEEDS_SANITIZE_LEAK "GIT_TEST_PASSING_SANITIZE_LEAK=true"
|
|
|
|
elif test_bool_env GIT_TEST_SANITIZE_LEAK_LOG false
|
|
|
|
then
|
|
|
|
BAIL_OUT_ENV_NEEDS_SANITIZE_LEAK "GIT_TEST_SANITIZE_LEAK_LOG=true"
|
tests: add a test mode for SANITIZE=leak, run it in CI
While git can be compiled with SANITIZE=leak, we have not run
regression tests under that mode. Memory leaks have only been fixed as
one-offs without structured regression testing.
This change adds CI testing for it. We'll now build and small set of
whitelisted t00*.sh tests under Linux with a new job called
"linux-leaks".
The CI target uses a new GIT_TEST_PASSING_SANITIZE_LEAK=true test
mode. When running in that mode, we'll assert that we were compiled
with SANITIZE=leak. We'll then skip all tests, except those that we've
opted-in by setting "TEST_PASSES_SANITIZE_LEAK=true".
A test setting "TEST_PASSES_SANITIZE_LEAK=true" setting can in turn
make use of the "SANITIZE_LEAK" prerequisite, should they wish to
selectively skip tests even under
"GIT_TEST_PASSING_SANITIZE_LEAK=true". In the preceding commit we
started doing this in "t0004-unwritable.sh" under SANITIZE=leak, now
it'll combine nicely with "GIT_TEST_PASSING_SANITIZE_LEAK=true".
This is how tests that don't set "TEST_PASSES_SANITIZE_LEAK=true" will
be skipped under GIT_TEST_PASSING_SANITIZE_LEAK=true:
$ GIT_TEST_PASSING_SANITIZE_LEAK=true ./t0001-init.sh
1..0 # SKIP skip all tests in t0001 under SANITIZE=leak, TEST_PASSES_SANITIZE_LEAK not set
The intent is to add more TEST_PASSES_SANITIZE_LEAK=true annotations
as follow-up change, but let's start small to begin with.
In ci/run-build-and-tests.sh we make use of the default "*" case to
run "make test" without any GIT_TEST_* modes. SANITIZE=leak is known
to fail in combination with GIT_TEST_SPLIT_INDEX=true in
t0016-oidmap.sh, and we're likely to have other such failures in
various GIT_TEST_* modes. Let's focus on getting the base tests
passing, we can expand coverage to GIT_TEST_* modes later.
It would also be possible to implement a more lightweight version of
this by only relying on setting "LSAN_OPTIONS". See
<YS9OT/pn5rRK9cGB@coredump.intra.peff.net>[1] and
<YS9ZIDpANfsh7N+S@coredump.intra.peff.net>[2] for a discussion of
that. I've opted for this approach of adding a GIT_TEST_* mode instead
because it's consistent with how we handle other special test modes.
Being able to add a "!SANITIZE_LEAK" prerequisite and calling
"test_done" early if it isn't satisfied also means that we can more
incrementally add regression tests without being forced to fix
widespread and hard-to-fix leaks at the same time.
We have tests that do simple checking of some tool we're interested
in, but later on in the script might be stressing trace2, or common
sources of leaks like "git log" in combination with the tool (e.g. the
commit-graph tests). To be clear having a prerequisite could also be
accomplished by using "LSAN_OPTIONS" directly.
On the topic of "LSAN_OPTIONS": It would be nice to have a mode to
aggregate all failures in our various scripts, see [2] for a start at
doing that which sets "log_path" in "LSAN_OPTIONS". I've punted on
that for now, it can be added later.
As of writing this we've got major regressions between master..seen,
i.e. the t000*.sh tests and more fixed since 31f9acf9ce2 (Merge branch
'ah/plugleaks', 2021-08-04) have regressed recently.
See the discussion at <87czsv2idy.fsf@evledraar.gmail.com>[3] about
the lack of this sort of test mode, and 0e5bba53af (add UNLEAK
annotation for reducing leak false positives, 2017-09-08) for the
initial addition of SANITIZE=leak.
See also 09595ab381 (Merge branch 'jk/leak-checkers', 2017-09-19),
7782066f67 (Merge branch 'jk/apache-lsan', 2019-05-19) and the recent
936e58851a (Merge branch 'ah/plugleaks', 2021-05-07) for some of the
past history of "one-off" SANITIZE=leak (and more) fixes.
As noted in [5] we can't support this on OSX yet until Clang 14 is
released, at that point we'll probably want to resurrect that
"osx-leaks" job.
1. https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer
2. https://lore.kernel.org/git/YS9OT%2Fpn5rRK9cGB@coredump.intra.peff.net/
3. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/
4. https://lore.kernel.org/git/YS9ZIDpANfsh7N+S@coredump.intra.peff.net/
5. https://lore.kernel.org/git/20210916035603.76369-1-carenas@gmail.com/
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Carlo Marcelo Arenas Belón <carenas@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-23 09:20:46 +00:00
|
|
|
fi
|
|
|
|
|
tests: run internal chain-linter under "make test"
Since 69b9924b875 (t/Makefile: teach `make test` and `make prove` to run
chainlint.pl, 2022-09-01), we run a single chainlint.pl process for all
scripts, and then instruct each individual script to run with the
equivalent of --no-chain-lint, which tells them not to redundantly run
the chainlint script themselves.
However, this also disables the internal linter run within the shell by
eval-ing "(exit 117) && $1" and confirming we get code 117. In theory
the external linter produces a superset of complaints, and we don't need
the internal one anymore. However, we know there is at least one case
where they differ. A test like:
test_expect_success 'should fail linter' '
false &&
sleep 2 &
pid=$! &&
kill $pid
'
is buggy (it ignores the failure from "false", because it is
backgrounded along with the sleep). The internal linter catches this,
but the external one doesn't (and teaching it to do so is complicated[1]).
So not only does "make test" miss this problem, but it's doubly
confusing because running the script standalone does complain.
Let's teach the suppression in the Makefile to only turn off the
external linter (which we know is redundant, as it was already run) and
leave the internal one intact.
I've used a new environment variable to do this here, and intentionally
did not add a "--no-ext-chain-lint" option. This is an internal
optimization used by the Makefile, and not something that ordinary users
would need to tweak.
[1] For discussion of chainlint.pl and this case, see:
https://lore.kernel.org/git/CAPig+cQtLFX4PgXyyK_AAkCvg4Aw2RAC5MmLbib-aHHgTBcDuw@mail.gmail.com/
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-03-30 19:27:38 +00:00
|
|
|
if test "${GIT_TEST_CHAIN_LINT:-1}" != 0 &&
|
|
|
|
test "${GIT_TEST_EXT_CHAIN_LINT:-1}" != 0
|
2022-09-01 00:29:54 +00:00
|
|
|
then
|
|
|
|
"$PERL_PATH" "$TEST_DIRECTORY/chainlint.pl" "$0" ||
|
|
|
|
BUG "lint error (see '?!...!? annotations above)"
|
|
|
|
fi
|
|
|
|
|
test-lib: bring $remove_trash out of retirement
There's no point in creating a repository or directory only to decide
right afterwards that we're skipping all the tests. We can save
ourselves the redundant "git init" or "mkdir" and "rm -rf" in this
case.
We carry around the "$remove_trash" variable because if the directory
is unexpectedly gone at test_done time we'll still want to hit the
"trash directory already removed" error, but not if we never created
the trash directory. See df4c0d1a792 (test-lib: abort when can't
remove trash directory, 2017-04-20) for the addition of that error.
So let's partially revert 06478dab4c (test-lib: retire $remove_trash
variable, 2017-04-23) and move the decision about whether to skip all
tests earlier.
Let's also fix a bug that was with us since abc5d372ec (Enable
parallel tests, 2008-08-08): we would leak $remove_trash from the
environment. We don't want this to error out, so let's reset it to the
empty string first:
remove_trash=t GIT_SKIP_TESTS=t0001 ./t0001-init.sh
I tested this with --debug, see 4d0912a206 (test-lib.sh: do not barf
under --debug at the end of the test, 2017-04-24) for a bug we don't
want to re-introduce.
While I'm at it, let's move the HOME assignment to just before
test_create_repo, it could be lower, but it seems better to set it
before calling anything in test-lib-functions.sh
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-10 14:19:00 +00:00
|
|
|
# Last-minute variable setup
|
test-lib-functions: optionally keep HOME, TERM and SHELL in 'test_pause'
The 'test_pause' function, which is designed to help interactive
debugging and exploration of tests, currently inherits the value of HOME
and TERM set by 'test-lib.sh': HOME="$TRASH_DIRECTORY" and TERM=dumb. It
also invokes the shell defined by TEST_SHELL_PATH, which defaults to
/bin/sh (through SHELL_PATH).
Changing the value of HOME means that any customization configured in a
developers' shell startup files and any Git aliases defined in their
global Git configuration file are not available in the shell invoked by
'test_pause'.
Changing the value of TERM to 'dumb' means that colored output
is disabled for all commands in that shell.
Using /bin/sh as the shell invoked by 'test_pause' is not ideal since
some platforms (i.e. Debian and derivatives) use Dash as /bin/sh, and
this shell is usually compiled without readline support, which makes for
a poor interactive command line experience.
To make the interactive command line experience in the shell invoked by
'test_pause' more pleasant, save the values of HOME and TERM in
USER_HOME and USER_TERM before changing them in test-lib.sh, and add
options to 'test_pause' to optionally use these variables to invoke the
shell. Also add an option to invoke SHELL instead of TEST_SHELL_PATH, so
that developer's interactive shell is used.
We use options instead of changing the behaviour unconditionally since
these three variables can slightly change command behaviour. Moreover,
using the original HOME means commands could overwrite files in a user's
home directory. Be explicit about these caveats in the new 'Usage'
section in test-lib-functions.sh.
Finally, add '[options]' to the test_pause synopsys in t/README, and
mention that the full list of helper functions and their options can be
found in test-lib-functions.sh.
Helped-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Philippe Blain <levraiphilippeblain@gmail.com>
Acked-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-06 04:38:59 +00:00
|
|
|
USER_HOME="$HOME"
|
test-lib: bring $remove_trash out of retirement
There's no point in creating a repository or directory only to decide
right afterwards that we're skipping all the tests. We can save
ourselves the redundant "git init" or "mkdir" and "rm -rf" in this
case.
We carry around the "$remove_trash" variable because if the directory
is unexpectedly gone at test_done time we'll still want to hit the
"trash directory already removed" error, but not if we never created
the trash directory. See df4c0d1a792 (test-lib: abort when can't
remove trash directory, 2017-04-20) for the addition of that error.
So let's partially revert 06478dab4c (test-lib: retire $remove_trash
variable, 2017-04-23) and move the decision about whether to skip all
tests earlier.
Let's also fix a bug that was with us since abc5d372ec (Enable
parallel tests, 2008-08-08): we would leak $remove_trash from the
environment. We don't want this to error out, so let's reset it to the
empty string first:
remove_trash=t GIT_SKIP_TESTS=t0001 ./t0001-init.sh
I tested this with --debug, see 4d0912a206 (test-lib.sh: do not barf
under --debug at the end of the test, 2017-04-24) for a bug we don't
want to re-introduce.
While I'm at it, let's move the HOME assignment to just before
test_create_repo, it could be lower, but it seems better to set it
before calling anything in test-lib-functions.sh
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-10 14:19:00 +00:00
|
|
|
HOME="$TRASH_DIRECTORY"
|
|
|
|
GNUPGHOME="$HOME/gnupg-home-not-used"
|
test-lib-functions: optionally keep HOME, TERM and SHELL in 'test_pause'
The 'test_pause' function, which is designed to help interactive
debugging and exploration of tests, currently inherits the value of HOME
and TERM set by 'test-lib.sh': HOME="$TRASH_DIRECTORY" and TERM=dumb. It
also invokes the shell defined by TEST_SHELL_PATH, which defaults to
/bin/sh (through SHELL_PATH).
Changing the value of HOME means that any customization configured in a
developers' shell startup files and any Git aliases defined in their
global Git configuration file are not available in the shell invoked by
'test_pause'.
Changing the value of TERM to 'dumb' means that colored output
is disabled for all commands in that shell.
Using /bin/sh as the shell invoked by 'test_pause' is not ideal since
some platforms (i.e. Debian and derivatives) use Dash as /bin/sh, and
this shell is usually compiled without readline support, which makes for
a poor interactive command line experience.
To make the interactive command line experience in the shell invoked by
'test_pause' more pleasant, save the values of HOME and TERM in
USER_HOME and USER_TERM before changing them in test-lib.sh, and add
options to 'test_pause' to optionally use these variables to invoke the
shell. Also add an option to invoke SHELL instead of TEST_SHELL_PATH, so
that developer's interactive shell is used.
We use options instead of changing the behaviour unconditionally since
these three variables can slightly change command behaviour. Moreover,
using the original HOME means commands could overwrite files in a user's
home directory. Be explicit about these caveats in the new 'Usage'
section in test-lib-functions.sh.
Finally, add '[options]' to the test_pause synopsys in t/README, and
mention that the full list of helper functions and their options can be
found in test-lib-functions.sh.
Helped-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Philippe Blain <levraiphilippeblain@gmail.com>
Acked-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-06 04:38:59 +00:00
|
|
|
export HOME GNUPGHOME USER_HOME
|
test-lib: bring $remove_trash out of retirement
There's no point in creating a repository or directory only to decide
right afterwards that we're skipping all the tests. We can save
ourselves the redundant "git init" or "mkdir" and "rm -rf" in this
case.
We carry around the "$remove_trash" variable because if the directory
is unexpectedly gone at test_done time we'll still want to hit the
"trash directory already removed" error, but not if we never created
the trash directory. See df4c0d1a792 (test-lib: abort when can't
remove trash directory, 2017-04-20) for the addition of that error.
So let's partially revert 06478dab4c (test-lib: retire $remove_trash
variable, 2017-04-23) and move the decision about whether to skip all
tests earlier.
Let's also fix a bug that was with us since abc5d372ec (Enable
parallel tests, 2008-08-08): we would leak $remove_trash from the
environment. We don't want this to error out, so let's reset it to the
empty string first:
remove_trash=t GIT_SKIP_TESTS=t0001 ./t0001-init.sh
I tested this with --debug, see 4d0912a206 (test-lib.sh: do not barf
under --debug at the end of the test, 2017-04-24) for a bug we don't
want to re-introduce.
While I'm at it, let's move the HOME assignment to just before
test_create_repo, it could be lower, but it seems better to set it
before calling anything in test-lib-functions.sh
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-10 14:19:00 +00:00
|
|
|
|
test-lib.sh: try to re-chmod & retry on failed trash removal
Try to re-chmod the trash directory on startup if we fail to "rm -rf"
it. This fixes problems where the test leaves the trash directory
behind in a bad permission state for whatever reason.
This fixes an interaction between [1] where t0004-unwritable.sh was
made to use "test_when_finished" for cleanup, and [2] which added the
"--immediate" mode. If a test in this file failed when running with
"--immediate" we wouldn't run the "test_when_finished" block, which
re-chmods the ".git/objects" directory (see [1]).
This can be demonstrated as e.g. (output snipped for less verbosity):
$ ./t0004-unwritable.sh --run=3 --immediate
ok 1 # skip setup (--run)
ok 2 # skip write-tree should notice unwritable repository (--run)
not ok 3 - commit should notice unwritable repository
[...]
$ ./t0004-unwritable.sh --run=3 --immediate
rm: cannot remove '[...]/trash directory.t0004-unwritable/.git/objects/info': Permission denied
FATAL: Cannot prepare test area
[...]
Instead of some version of reverting [1] let's make the test-lib.sh
resilient to this edge-case, it will happen due to [1], but also
e.g. if the relevant "test-lib.sh" process is kill -9'd during the
test run. We should try harder to recover in this case. If we fail to
remove the test directory let's retry after (re-)chmod-ing it.
This doesn't need to be guarded by something that's equivalent to
"POSIXPERM" since if we don't support "chmod" we were about to fail
anyway.
Let's also discard any error output from (a possibly nonexisting)
"chmod", we'll fail on the subsequent "rm -rf" anyway, likewise for
the first "rm -rf" invocation, we don't want to get the "cannot
remove" output if we can get around it with the "chmod", but we do
want any error output from the second "rm -rf", in case that doesn't
fix the issue.
The lack of &&-chaining between the "chmod" and "rm -rf" is
intentional, if we fail the first "rm -rf", can't chmod, but then
succeed the second time around that's what we were hoping for. We just
want to nuke the directory, not carry forward every possible error
code or error message.
1. dbda967684d (t0004 (unwritable files): simplify error handling,
2010-09-06)
2. b586744a864 (test: skip clean-up when running under --immediate
mode, 2011-06-27)
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-11 01:41:59 +00:00
|
|
|
# "rm -rf" existing trash directory, even if a previous run left it
|
|
|
|
# with bad permissions.
|
|
|
|
remove_trash_directory () {
|
|
|
|
dir="$1"
|
|
|
|
if ! rm -rf "$dir" 2>/dev/null
|
|
|
|
then
|
|
|
|
chmod -R u+rwx "$dir"
|
|
|
|
rm -rf "$dir"
|
|
|
|
fi
|
|
|
|
! test -d "$dir"
|
|
|
|
}
|
|
|
|
|
2005-05-14 05:50:32 +00:00
|
|
|
# Test repository
|
test-lib.sh: try to re-chmod & retry on failed trash removal
Try to re-chmod the trash directory on startup if we fail to "rm -rf"
it. This fixes problems where the test leaves the trash directory
behind in a bad permission state for whatever reason.
This fixes an interaction between [1] where t0004-unwritable.sh was
made to use "test_when_finished" for cleanup, and [2] which added the
"--immediate" mode. If a test in this file failed when running with
"--immediate" we wouldn't run the "test_when_finished" block, which
re-chmods the ".git/objects" directory (see [1]).
This can be demonstrated as e.g. (output snipped for less verbosity):
$ ./t0004-unwritable.sh --run=3 --immediate
ok 1 # skip setup (--run)
ok 2 # skip write-tree should notice unwritable repository (--run)
not ok 3 - commit should notice unwritable repository
[...]
$ ./t0004-unwritable.sh --run=3 --immediate
rm: cannot remove '[...]/trash directory.t0004-unwritable/.git/objects/info': Permission denied
FATAL: Cannot prepare test area
[...]
Instead of some version of reverting [1] let's make the test-lib.sh
resilient to this edge-case, it will happen due to [1], but also
e.g. if the relevant "test-lib.sh" process is kill -9'd during the
test run. We should try harder to recover in this case. If we fail to
remove the test directory let's retry after (re-)chmod-ing it.
This doesn't need to be guarded by something that's equivalent to
"POSIXPERM" since if we don't support "chmod" we were about to fail
anyway.
Let's also discard any error output from (a possibly nonexisting)
"chmod", we'll fail on the subsequent "rm -rf" anyway, likewise for
the first "rm -rf" invocation, we don't want to get the "cannot
remove" output if we can get around it with the "chmod", but we do
want any error output from the second "rm -rf", in case that doesn't
fix the issue.
The lack of &&-chaining between the "chmod" and "rm -rf" is
intentional, if we fail the first "rm -rf", can't chmod, but then
succeed the second time around that's what we were hoping for. We just
want to nuke the directory, not carry forward every possible error
code or error message.
1. dbda967684d (t0004 (unwritable files): simplify error handling,
2010-09-06)
2. b586744a864 (test: skip clean-up when running under --immediate
mode, 2011-06-27)
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-11 01:41:59 +00:00
|
|
|
remove_trash_directory "$TRASH_DIRECTORY" || {
|
2022-07-27 23:13:31 +00:00
|
|
|
BAIL_OUT 'cannot prepare test area'
|
2008-03-19 04:58:01 +00:00
|
|
|
}
|
|
|
|
|
test-lib: bring $remove_trash out of retirement
There's no point in creating a repository or directory only to decide
right afterwards that we're skipping all the tests. We can save
ourselves the redundant "git init" or "mkdir" and "rm -rf" in this
case.
We carry around the "$remove_trash" variable because if the directory
is unexpectedly gone at test_done time we'll still want to hit the
"trash directory already removed" error, but not if we never created
the trash directory. See df4c0d1a792 (test-lib: abort when can't
remove trash directory, 2017-04-20) for the addition of that error.
So let's partially revert 06478dab4c (test-lib: retire $remove_trash
variable, 2017-04-23) and move the decision about whether to skip all
tests earlier.
Let's also fix a bug that was with us since abc5d372ec (Enable
parallel tests, 2008-08-08): we would leak $remove_trash from the
environment. We don't want this to error out, so let's reset it to the
empty string first:
remove_trash=t GIT_SKIP_TESTS=t0001 ./t0001-init.sh
I tested this with --debug, see 4d0912a206 (test-lib.sh: do not barf
under --debug at the end of the test, 2017-04-24) for a bug we don't
want to re-introduce.
While I'm at it, let's move the HOME assignment to just before
test_create_repo, it could be lower, but it seems better to set it
before calling anything in test-lib-functions.sh
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-10 14:19:00 +00:00
|
|
|
remove_trash=t
|
2012-09-01 18:11:49 +00:00
|
|
|
if test -z "$TEST_NO_CREATE_REPO"
|
|
|
|
then
|
tests: don't depend on template-created .git/branches
As noted in c8a58ac5a52 (Revert "Don't create the $GIT_DIR/branches
directory on init", 2009-10-31) there was an attempt long ago in
0cc5691a8b0 (Don't create the $GIT_DIR/branches directory on init,
2009-10-30) to get rid of the legacy "branches" directory.
We should probably get rid of its creation by removing the
"templates/branches--" file. But whatever our default behavior, our
tests should be tightened up to explicitly create the .git/branches
directory if they rely on our default templates, to make the
dependency on those templates clear.
So let's amend the two tests that would fail if .git/branches wasn't
created. To do this introduce a new "TEST_CREATE_REPO_NO_TEMPLATE"
variable, which we'll set before sourcing test-lib.sh, and change the
"git clone" and "git init" commands in the tests themselves to
explicitly pass "--template=".
This way they won't get a .git/branches in either their top-level
.git, or in the ones they create. We can then amend the tests that
rely on the ".git/branches" directory existing to create it
explicitly, and to remove it after its creation.
This new "TEST_CREATE_REPO_NO_TEMPLATE" variable is a less
heavy-handed version of the "NO_SET_GIT_TEMPLATE_DIR" variable. See
a94d305bf80 (t/t0001-init.sh: add test for 'init with init.templatedir
set', 2010-02-26) for its implementation.
Unlike "TEST_CREATE_REPO_NO_TEMPLATE", this new
"TEST_CREATE_REPO_NO_TEMPLATE" variable is narrowly scoped to what the
"git init" in test-lib.sh does, as opposed to the global effect of
"NO_SET_GIT_TEMPLATE_DIR" and the setting of "GIT_TEMPLATE_DIR" in
wrap-for-bin.sh.
I experimented with adding a new "GIT_WRAP_FOR_BIN_VIA_TEST_LIB"
variable set in test-lib.sh, which would cause wrap-for-bin.sh to not
set GIT_TEMPLATE_DIR, GITPERLLIB etc, as we set those in
test-lib.sh. I think that's a viable approach, but it would interact
e.g. with the appending feature of GITPERLLIB added in
8bade1e12e2 (wrap-for-bin: make bin-wrappers chainable, 2013-07-04).
Doing so would allow us to convert the tests in t0001-init.sh that now
use "NO_SET_GIT_TEMPLATE_DIR" to simply unset "GIT_TEMPLATE_DIR" in a
sub-shell before invoking "git init" or "git clone". I think that
approach is worth pursuing, but let's table it for now. Some future
wrap-for-bin.sh refactoring can try to address it.
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-06-03 11:15:04 +00:00
|
|
|
git init \
|
|
|
|
${TEST_CREATE_REPO_NO_TEMPLATE:+--template=} \
|
|
|
|
"$TRASH_DIRECTORY" >&3 2>&4 ||
|
test-lib: split up and deprecate test_create_repo()
Remove various redundant or obsolete code from the test_create_repo()
function, and split up its use in test-lib.sh from what tests need
from it.
This leave us with a pass-through wrapper for "git init" in
test-lib-functions.sh, in test-lib.sh we have the same, except for
needing to redirect stdout/stderr, and emitting an error ourselves if
it fails. We don't need to error() ourselves when test_create_repo()
is invoked, as the invocation will be a part of a test's "&&"-chain.
Everything below this paragraph is a detailed summary of the history
of test_create_repo() explaining why it's safe to remove the various
things it was doing:
1. "mkdir -p" isn't needed because "git init" itself will create
leading directories if needed.
2. Since we're now a simple wrapper for "git init" we don't need to
check that we have only one argument. If someone wants to run
"test_create_repo --bare x" that's OK.
3. We won't ever hit that "Cannot setup test environment"
error.
Checking the test environment sanity when doing "git init" dates
back to eea420693be (t0000: catch trivial pilot errors.,
2005-12-10) and 2ccd2027b01 (trivial: check, if t/trash directory
was successfully created, 2006-01-05).
We can also see it in another form a bit later in my own
0d314ce834d (test-lib: use subshell instead of cd $new && .. && cd
$old, 2010-08-30).
But since 2006f0adaee (t/test-lib: make sure Git has already been
built, 2012-09-17) we already check if we have a built git
earlier.
The one thing this was testing after that 2012 change was that
we'd just built "git", but not "git-init", but since
3af4c7156c4 (tests: respect GIT_TEST_INSTALLED when initializing
repositories, 2018-11-12) we invoke "git", not "git-init".
So all of that's been checked already, and we don't need to
re-check it here.
4. We don't need to move .git/hooks out of the way.
That dates back to c09a69a83e3 (Disable hooks during tests.,
2005-10-16), since then hooks became disabled by default in
f98f8cbac01 (Ship sample hooks with .sample suffix, 2008-06-24).
So the hooks were already disabled by default, but as can be seen
from "mkdir .git/hooks" changes various tests needed to re-setup
that directory. Now they no longer do.
This makes us implicitly depend on the default hooks being
disabled, which is a good thing. If and when we'd have any
on-by-default hooks (I see no reason we ever would) we'd want to
see the subtle and not so subtle ways that would break the test
suite.
5. We don't need to "cd" to the "$repo" directory at all anymore.
In the code being removed here we both "cd"'d to the repository
before calling "init", and did so in a subshell.
It's not important to do either, so both of those can be
removed. We cd'd because this code grew from test-lib.sh code
where we'd have done so already, see eedf8f97e58 (Abstract
test_create_repo out for use in tests., 2006-02-17), and later
"cd"'d inside a subshell since 0d314ce834d to avoid having to keep
track of an "old pwd" variable to cd back after the setup.
Being in the repository directory made moving the hooks around
easier (we wouldn't have to fully qualify the path). Since we're
not moving the hooks per #4 above we don't need to "cd" for that
reason either.
6. We can drop the --template argument and instead rely on the
GIT_TEMPLATE_DIR set to the same path earlier in test-lib.sh. See
8683a45d669 (Introduce GIT_TEMPLATE_DIR, 2006-12-19)
7. We only needed that ">&3 2>&4" redirection when invoked from
test-lib.sh.
We could still invoke test_create_repo() there, but as the
invocation is now trivial and we don't have a good reason to use
test_create_repo() elsewhere let's call "git init" there
ourselves.
8. We didn't need to resolve "git" as
"${GIT_TEST_INSTALLED:-$GIT_EXEC_PATH}/git$X" in test_create_repo(),
even for the use of test-lib.sh
PATH is already set up in test-lib.sh to start with
GIT_TEST_INSTALLED and/or GIT_EXEC_PATH before
test_create_repo() (now "git init") is called.. So we can simply
run "git" and rely on the PATH lookup choosing the right
executable.
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-10 14:19:10 +00:00
|
|
|
error "cannot run git init"
|
2012-02-17 10:25:09 +00:00
|
|
|
else
|
2013-04-14 16:34:56 +00:00
|
|
|
mkdir -p "$TRASH_DIRECTORY"
|
2012-02-17 10:25:09 +00:00
|
|
|
fi
|
2019-01-29 14:19:27 +00:00
|
|
|
|
2008-05-31 21:11:21 +00:00
|
|
|
# Use -P to resolve symlinks in our working directory so that the cwd
|
|
|
|
# in subprocesses like git equals our $PWD (for pathname comparisons).
|
2022-07-27 23:13:31 +00:00
|
|
|
cd -P "$TRASH_DIRECTORY" || BAIL_OUT "cannot cd -P to \"$TRASH_DIRECTORY\""
|
2006-12-29 01:58:00 +00:00
|
|
|
|
2022-05-21 22:18:46 +00:00
|
|
|
start_test_output "$0"
|
2019-01-29 14:19:27 +00:00
|
|
|
|
2020-02-22 20:17:31 +00:00
|
|
|
# Convenience
|
2021-09-11 11:17:51 +00:00
|
|
|
# A regexp to match 5 and 35 hexdigits
|
2020-02-22 20:17:31 +00:00
|
|
|
_x05='[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]'
|
|
|
|
_x35="$_x05$_x05$_x05$_x05$_x05$_x05$_x05"
|
|
|
|
|
|
|
|
test_oid_init
|
|
|
|
|
|
|
|
ZERO_OID=$(test_oid zero)
|
|
|
|
OID_REGEX=$(echo $ZERO_OID | sed -e 's/0/[0-9a-f]/g')
|
2020-05-13 00:53:41 +00:00
|
|
|
OIDPATH_REGEX=$(test_oid_to_path $ZERO_OID | sed -e 's/0/[0-9a-f]/g')
|
2020-02-22 20:17:31 +00:00
|
|
|
EMPTY_TREE=$(test_oid empty_tree)
|
|
|
|
EMPTY_BLOB=$(test_oid empty_blob)
|
|
|
|
|
2019-02-09 18:25:26 +00:00
|
|
|
# Provide an implementation of the 'yes' utility; the upper bound
|
|
|
|
# limit is there to help Windows that cannot stop this loop from
|
|
|
|
# wasting cycles when the downstream stops reading, so do not be
|
|
|
|
# tempted to turn it into an infinite loop. cf. 6129c930 ("test-lib:
|
|
|
|
# limit the output of the yes utility", 2016-02-02)
|
2009-08-28 22:32:41 +00:00
|
|
|
yes () {
|
|
|
|
if test $# = 0
|
|
|
|
then
|
|
|
|
y=y
|
|
|
|
else
|
|
|
|
y="$*"
|
|
|
|
fi
|
|
|
|
|
2016-02-02 18:15:53 +00:00
|
|
|
i=0
|
|
|
|
while test $i -lt 99
|
2009-08-28 22:32:41 +00:00
|
|
|
do
|
2016-02-02 18:15:53 +00:00
|
|
|
echo "$y"
|
|
|
|
i=$(($i+1))
|
2009-08-28 22:32:41 +00:00
|
|
|
done
|
|
|
|
}
|
|
|
|
|
2019-06-21 10:18:12 +00:00
|
|
|
# The GIT_TEST_FAIL_PREREQS code hooks into test_set_prereq(), and
|
|
|
|
# thus needs to be set up really early, and set an internal variable
|
|
|
|
# for convenience so the hot test_set_prereq() codepath doesn't need
|
2023-01-12 16:03:21 +00:00
|
|
|
# to call "test-tool env-helper" (via test_bool_env). Only do that work
|
tests: add 'test_bool_env' to catch non-bool GIT_TEST_* values
Since 3b072c577b (tests: replace test_tristate with "git env--helper",
2019-06-21) we get the normalized bool values of various GIT_TEST_*
environment variables via 'git env--helper'. Now, while the 'git
env--helper' command itself does catch invalid values in the
environment variable or in the given --default and exits with error
(exit code 128 or 129, respectively), it's invoked in conditions like
'if ! git env--helper ...', which means that all invalid bool values
are interpreted the same as the ordinary 'false' (exit code 1). This
has led to inadvertently skipped httpd tests in our CI builds for a
couple of weeks, see 3960290675 (ci: restore running httpd tests,
2019-09-06).
Let's be more careful about what the test suite accepts as bool values
in GIT_TEST_* environment variables, and error out loud and clear on
invalid values instead of simply skipping tests. Add the
'test_bool_env' helper function to encapsulate the invocation of 'git
env--helper' and the verification of its exit code, and replace all
invocations of that command in our test framework and test suite with
a call to this new helper (except in 't0017-env-helper.sh', of
course).
$ GIT_TEST_GIT_DAEMON=YesPlease ./t5570-git-daemon.sh
fatal: bad numeric config value 'YesPlease' for 'GIT_TEST_GIT_DAEMON': invalid unit
error: test_bool_env requires bool values both for $GIT_TEST_GIT_DAEMON and for the default fallback
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-11-22 13:14:36 +00:00
|
|
|
# if needed by seeing if GIT_TEST_FAIL_PREREQS is set at all.
|
2019-06-21 10:18:12 +00:00
|
|
|
GIT_TEST_FAIL_PREREQS_INTERNAL=
|
|
|
|
if test -n "$GIT_TEST_FAIL_PREREQS"
|
|
|
|
then
|
tests: add 'test_bool_env' to catch non-bool GIT_TEST_* values
Since 3b072c577b (tests: replace test_tristate with "git env--helper",
2019-06-21) we get the normalized bool values of various GIT_TEST_*
environment variables via 'git env--helper'. Now, while the 'git
env--helper' command itself does catch invalid values in the
environment variable or in the given --default and exits with error
(exit code 128 or 129, respectively), it's invoked in conditions like
'if ! git env--helper ...', which means that all invalid bool values
are interpreted the same as the ordinary 'false' (exit code 1). This
has led to inadvertently skipped httpd tests in our CI builds for a
couple of weeks, see 3960290675 (ci: restore running httpd tests,
2019-09-06).
Let's be more careful about what the test suite accepts as bool values
in GIT_TEST_* environment variables, and error out loud and clear on
invalid values instead of simply skipping tests. Add the
'test_bool_env' helper function to encapsulate the invocation of 'git
env--helper' and the verification of its exit code, and replace all
invocations of that command in our test framework and test suite with
a call to this new helper (except in 't0017-env-helper.sh', of
course).
$ GIT_TEST_GIT_DAEMON=YesPlease ./t5570-git-daemon.sh
fatal: bad numeric config value 'YesPlease' for 'GIT_TEST_GIT_DAEMON': invalid unit
error: test_bool_env requires bool values both for $GIT_TEST_GIT_DAEMON and for the default fallback
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-11-22 13:14:36 +00:00
|
|
|
if test_bool_env GIT_TEST_FAIL_PREREQS false
|
2019-06-21 10:18:12 +00:00
|
|
|
then
|
|
|
|
GIT_TEST_FAIL_PREREQS_INTERNAL=true
|
|
|
|
test_set_prereq FAIL_PREREQS
|
|
|
|
fi
|
|
|
|
else
|
|
|
|
test_lazy_prereq FAIL_PREREQS '
|
tests: add 'test_bool_env' to catch non-bool GIT_TEST_* values
Since 3b072c577b (tests: replace test_tristate with "git env--helper",
2019-06-21) we get the normalized bool values of various GIT_TEST_*
environment variables via 'git env--helper'. Now, while the 'git
env--helper' command itself does catch invalid values in the
environment variable or in the given --default and exits with error
(exit code 128 or 129, respectively), it's invoked in conditions like
'if ! git env--helper ...', which means that all invalid bool values
are interpreted the same as the ordinary 'false' (exit code 1). This
has led to inadvertently skipped httpd tests in our CI builds for a
couple of weeks, see 3960290675 (ci: restore running httpd tests,
2019-09-06).
Let's be more careful about what the test suite accepts as bool values
in GIT_TEST_* environment variables, and error out loud and clear on
invalid values instead of simply skipping tests. Add the
'test_bool_env' helper function to encapsulate the invocation of 'git
env--helper' and the verification of its exit code, and replace all
invocations of that command in our test framework and test suite with
a call to this new helper (except in 't0017-env-helper.sh', of
course).
$ GIT_TEST_GIT_DAEMON=YesPlease ./t5570-git-daemon.sh
fatal: bad numeric config value 'YesPlease' for 'GIT_TEST_GIT_DAEMON': invalid unit
error: test_bool_env requires bool values both for $GIT_TEST_GIT_DAEMON and for the default fallback
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-11-22 13:14:36 +00:00
|
|
|
test_bool_env GIT_TEST_FAIL_PREREQS false
|
2019-06-21 10:18:12 +00:00
|
|
|
'
|
|
|
|
fi
|
|
|
|
|
2019-11-27 17:15:07 +00:00
|
|
|
# Fix some commands on Windows, and other OS-specific things
|
2016-07-21 16:02:54 +00:00
|
|
|
uname_s=$(uname -s)
|
|
|
|
case $uname_s in
|
2009-03-11 20:17:26 +00:00
|
|
|
*MINGW*)
|
|
|
|
# Windows has its own (incompatible) sort and find
|
|
|
|
sort () {
|
|
|
|
/usr/bin/sort "$@"
|
|
|
|
}
|
|
|
|
find () {
|
|
|
|
/usr/bin/find "$@"
|
|
|
|
}
|
2009-03-13 22:35:24 +00:00
|
|
|
# git sees Windows-style pwd
|
|
|
|
pwd () {
|
|
|
|
builtin pwd -W
|
|
|
|
}
|
2009-03-13 21:55:27 +00:00
|
|
|
# no POSIX permissions
|
2009-03-13 22:00:15 +00:00
|
|
|
# backslashes in pathspec are converted to '/'
|
2009-03-25 12:21:15 +00:00
|
|
|
# exec does not inherit the PID
|
2010-09-12 09:37:24 +00:00
|
|
|
test_set_prereq MINGW
|
2014-08-30 21:39:07 +00:00
|
|
|
test_set_prereq NATIVE_CRLF
|
2010-12-14 18:32:12 +00:00
|
|
|
test_set_prereq SED_STRIPS_CR
|
2013-07-18 21:44:57 +00:00
|
|
|
test_set_prereq GREP_STRIPS_CR
|
cygwin: disallow backslashes in file names
The backslash character is not a valid part of a file name on Windows.
If, in Windows, Git attempts to write a file that has a backslash
character in the filename, it will be incorrectly interpreted as a
directory separator.
This caused CVE-2019-1354 in MinGW, as this behaviour can be manipulated
to cause the checkout to write to files it ought not write to, such as
adding code to the .git/hooks directory. This was fixed by e1d911dd4c
(mingw: disallow backslash characters in tree objects' file names,
2019-09-12). However, the vulnerability also exists in Cygwin: while
Cygwin mostly provides a POSIX-like path system, it will still interpret
a backslash as a directory separator.
To avoid this vulnerability, CVE-2021-29468, extend the previous fix to
also apply to Cygwin.
Similarly, extend the test case added by the previous version of the
commit. The test suite doesn't have an easy way to say "run this test
if in MinGW or Cygwin", so add a new test prerequisite that covers both.
As well as checking behaviour in the presence of paths containing
backslashes, the existing test also checks behaviour in the presence of
paths that differ only by the presence of a trailing ".". MinGW follows
normal Windows application behaviour and treats them as the same path,
but Cygwin more closely emulates *nix systems (at the expense of
compatibility with native Windows applications) and will create and
distinguish between such paths. Gate the relevant bit of that test
accordingly.
Reported-by: RyotaK <security@ryotak.me>
Helped-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Adam Dinwoodie <adam@dinwoodie.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-04-29 20:11:44 +00:00
|
|
|
test_set_prereq WINDOWS
|
tests(mingw): avoid very slow `mingw_test_cmp`
When Git's test suite uses `test_cmp`, it is not actually trying to
compare binary files as the name `cmp` would suggest to users familiar
with Unix' tools, but the tests instead verify that actual output
matches the expected text.
On Unix, `cmp` works well enough for Git's purposes because only Line
Feed characters are used as line endings. However, on Windows, while
most tools accept Line Feeds as line endings, many tools produce
Carriage Return + Line Feed line endings, including some of the tools
used by the test suite (which are therefore provided via Git for Windows
SDK). Therefore, `cmp` would frequently fail merely due to different
line endings.
To accommodate for that, the `mingw_test_cmp` function was introduced
into Git's test suite to perform a line-by-line comparison that ignores
line endings. This function is a Bash function that is only used on
Windows, everywhere else `cmp` is used.
This is a double whammy because `cmp` is fast, and `mingw_test_cmp` is
slow, even more so on Windows because it is a Bash script function, and
Bash scripts are known to run particularly slowly on Windows due to
Bash's need for the POSIX emulation layer provided by the MSYS2 runtime.
The commit message of 32ed3314c104 (t5351: avoid using `test_cmp` for
binary data, 2022-07-29) provides an illuminating account of the
consequences: On Windows, the platform on which Git could really use all
the help it can get to improve its performance, the time spent on one
entire test script was reduced from half an hour to less than half a
minute merely by avoiding a single call to `mingw_test_cmp` in but a
single test case.
Learning the lesson to avoid shell scripting wherever possible, the Git
for Windows project implemented a minimal replacement for
`mingw_test_cmp` in the form of a `test-tool` subcommand that parses the
input files line by line, ignoring line endings, and compares them.
Essentially the same thing as `mingw_test_cmp`, but implemented in
C instead of Bash. This solution served the Git for Windows project
well, over years.
However, when this solution was finally upstreamed, the conclusion was
reached that a change to use `git diff --no-index` instead of
`mingw_test_cmp` was more easily reviewed and hence should be used
instead.
The reason why this approach was not even considered in Git for Windows
is that in 2007, there was already a motion on the table to use Git's
own diff machinery to perform comparisons in Git's test suite, but it
was dismissed in https://lore.kernel.org/git/xmqqbkrpo9or.fsf@gitster.g/
as undesirable because tests might potentially succeed due to bugs in
the diff machinery when they should not succeed, and those bugs could
therefore hide regressions that the tests try to prevent.
By the time Git for Windows' `mingw-test-cmp` in C was finally
contributed to the Git mailing list, reviewers agreed that the diff
machinery had matured enough and should be used instead.
When the concern was raised that the diff machinery, due to its
complexity, would perform substantially worse than the test helper
originally implemented in the Git for Windows project, a test
demonstrated that these performance differences are well lost within the
100+ minutes it takes to run Git's test suite on Windows.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-12-06 15:07:47 +00:00
|
|
|
GIT_TEST_CMP="GIT_DIR=/dev/null git diff --no-index --ignore-cr-at-eol --"
|
2010-12-14 18:32:12 +00:00
|
|
|
;;
|
|
|
|
*CYGWIN*)
|
|
|
|
test_set_prereq POSIXPERM
|
|
|
|
test_set_prereq EXECKEEPSPID
|
2013-01-27 03:11:11 +00:00
|
|
|
test_set_prereq CYGWIN
|
2010-12-14 18:32:12 +00:00
|
|
|
test_set_prereq SED_STRIPS_CR
|
2013-07-18 21:44:57 +00:00
|
|
|
test_set_prereq GREP_STRIPS_CR
|
cygwin: disallow backslashes in file names
The backslash character is not a valid part of a file name on Windows.
If, in Windows, Git attempts to write a file that has a backslash
character in the filename, it will be incorrectly interpreted as a
directory separator.
This caused CVE-2019-1354 in MinGW, as this behaviour can be manipulated
to cause the checkout to write to files it ought not write to, such as
adding code to the .git/hooks directory. This was fixed by e1d911dd4c
(mingw: disallow backslash characters in tree objects' file names,
2019-09-12). However, the vulnerability also exists in Cygwin: while
Cygwin mostly provides a POSIX-like path system, it will still interpret
a backslash as a directory separator.
To avoid this vulnerability, CVE-2021-29468, extend the previous fix to
also apply to Cygwin.
Similarly, extend the test case added by the previous version of the
commit. The test suite doesn't have an easy way to say "run this test
if in MinGW or Cygwin", so add a new test prerequisite that covers both.
As well as checking behaviour in the presence of paths containing
backslashes, the existing test also checks behaviour in the presence of
paths that differ only by the presence of a trailing ".". MinGW follows
normal Windows application behaviour and treats them as the same path,
but Cygwin more closely emulates *nix systems (at the expense of
compatibility with native Windows applications) and will create and
distinguish between such paths. Gate the relevant bit of that test
accordingly.
Reported-by: RyotaK <security@ryotak.me>
Helped-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Adam Dinwoodie <adam@dinwoodie.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-04-29 20:11:44 +00:00
|
|
|
test_set_prereq WINDOWS
|
2009-03-13 21:55:27 +00:00
|
|
|
;;
|
|
|
|
*)
|
|
|
|
test_set_prereq POSIXPERM
|
2009-03-13 22:00:15 +00:00
|
|
|
test_set_prereq BSLASHPSPEC
|
2009-03-25 12:21:15 +00:00
|
|
|
test_set_prereq EXECKEEPSPID
|
2009-03-11 20:17:26 +00:00
|
|
|
;;
|
|
|
|
esac
|
2009-03-04 21:38:24 +00:00
|
|
|
|
2020-05-16 05:33:38 +00:00
|
|
|
# Detect arches where a few things don't work
|
|
|
|
uname_m=$(uname -m)
|
|
|
|
case $uname_m in
|
|
|
|
parisc* | hppa*)
|
|
|
|
test_set_prereq HPPA
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
2021-05-31 16:56:29 +00:00
|
|
|
test_set_prereq REFFILES
|
|
|
|
|
2012-04-27 09:25:25 +00:00
|
|
|
( COLUMNS=1 && test $COLUMNS = 1 ) && test_set_prereq COLUMNS_CAN_BE_1
|
2022-06-15 10:36:32 +00:00
|
|
|
test -z "$NO_CURL" && test_set_prereq LIBCURL
|
2009-04-03 19:33:59 +00:00
|
|
|
test -z "$NO_PERL" && test_set_prereq PERL
|
2017-05-25 19:45:31 +00:00
|
|
|
test -z "$NO_PTHREADS" && test_set_prereq PTHREADS
|
2009-11-18 01:42:31 +00:00
|
|
|
test -z "$NO_PYTHON" && test_set_prereq PYTHON
|
2021-01-24 01:58:33 +00:00
|
|
|
test -n "$USE_LIBPCRE2" && test_set_prereq PCRE
|
2017-11-23 14:16:57 +00:00
|
|
|
test -n "$USE_LIBPCRE2" && test_set_prereq LIBPCRE2
|
i18n: add infrastructure for translating Git with gettext
Change the skeleton implementation of i18n in Git to one that can show
localized strings to users for our C, Shell and Perl programs using
either GNU libintl or the Solaris gettext implementation.
This new internationalization support is enabled by default. If
gettext isn't available, or if Git is compiled with
NO_GETTEXT=YesPlease, Git falls back on its current behavior of
showing interface messages in English. When using the autoconf script
we'll auto-detect if the gettext libraries are installed and act
appropriately.
This change is somewhat large because as well as adding a C, Shell and
Perl i18n interface we're adding a lot of tests for them, and for
those tests to work we need a skeleton PO file to actually test
translations. A minimal Icelandic translation is included for this
purpose. Icelandic includes multi-byte characters which makes it easy
to test various edge cases, and it's a language I happen to
understand.
The rest of the commit message goes into detail about various
sub-parts of this commit.
= Installation
Gettext .mo files will be installed and looked for in the standard
$(prefix)/share/locale path. GIT_TEXTDOMAINDIR can also be set to
override that, but that's only intended to be used to test Git itself.
= Perl
Perl code that's to be localized should use the new Git::I18n
module. It imports a __ function into the caller's package by default.
Instead of using the high level Locale::TextDomain interface I've
opted to use the low-level (equivalent to the C interface)
Locale::Messages module, which Locale::TextDomain itself uses.
Locale::TextDomain does a lot of redundant work we don't need, and
some of it would potentially introduce bugs. It tries to set the
$TEXTDOMAIN based on package of the caller, and has its own
hardcoded paths where it'll search for messages.
I found it easier just to completely avoid it rather than try to
circumvent its behavior. In any case, this is an issue wholly
internal Git::I18N. Its guts can be changed later if that's deemed
necessary.
See <AANLkTilYD_NyIZMyj9dHtVk-ylVBfvyxpCC7982LWnVd@mail.gmail.com> for
a further elaboration on this topic.
= Shell
Shell code that's to be localized should use the git-sh-i18n
library. It's basically just a wrapper for the system's gettext.sh.
If gettext.sh isn't available we'll fall back on gettext(1) if it's
available. The latter is available without the former on Solaris,
which has its own non-GNU gettext implementation. We also need to
emulate eval_gettext() there.
If neither are present we'll use a dumb printf(1) fall-through
wrapper.
= About libcharset.h and langinfo.h
We use libcharset to query the character set of the current locale if
it's available. I.e. we'll use it instead of nl_langinfo if
HAVE_LIBCHARSET_H is set.
The GNU gettext manual recommends using langinfo.h's
nl_langinfo(CODESET) to acquire the current character set, but on
systems that have libcharset.h's locale_charset() using the latter is
either saner, or the only option on those systems.
GNU and Solaris have a nl_langinfo(CODESET), FreeBSD can use either,
but MinGW and some others need to use libcharset.h's locale_charset()
instead.
=Credits
This patch is based on work by Jeff Epler <jepler@unpythonic.net> who
did the initial Makefile / C work, and a lot of comments from the Git
mailing list, including Jonathan Nieder, Jakub Narebski, Johannes
Sixt, Erik Faye-Lund, Peter Krefting, Junio C Hamano, Thomas Rast and
others.
[jc: squashed a small Makefile fix from Ramsay]
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Ramsay Jones <ramsay@ramsay1.demon.co.uk>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-11-17 23:14:42 +00:00
|
|
|
test -z "$NO_GETTEXT" && test_set_prereq GETTEXT
|
2021-09-23 09:20:45 +00:00
|
|
|
test -n "$SANITIZE_LEAK" && test_set_prereq SANITIZE_LEAK
|
2022-05-12 22:32:15 +00:00
|
|
|
test -n "$GIT_VALGRIND_ENABLED" && test_set_prereq VALGRIND
|
2009-04-03 19:33:59 +00:00
|
|
|
|
2018-08-18 14:41:28 +00:00
|
|
|
if test -z "$GIT_TEST_CHECK_CACHE_TREE"
|
|
|
|
then
|
|
|
|
GIT_TEST_CHECK_CACHE_TREE=true
|
|
|
|
export GIT_TEST_CHECK_CACHE_TREE
|
|
|
|
fi
|
|
|
|
|
2013-04-11 02:07:04 +00:00
|
|
|
test_lazy_prereq PIPE '
|
|
|
|
# test whether the filesystem supports FIFOs
|
2017-09-17 22:56:00 +00:00
|
|
|
test_have_prereq !MINGW,!CYGWIN &&
|
|
|
|
rm -f testfifo && mkfifo testfifo
|
2013-04-11 02:07:04 +00:00
|
|
|
'
|
|
|
|
|
2012-07-26 22:50:45 +00:00
|
|
|
test_lazy_prereq SYMLINKS '
|
|
|
|
# test whether the filesystem supports symbolic links
|
|
|
|
ln -s x y && test -h y
|
|
|
|
'
|
2010-08-06 22:09:09 +00:00
|
|
|
|
2021-08-02 21:07:30 +00:00
|
|
|
test_lazy_prereq SYMLINKS_WINDOWS '
|
|
|
|
# test whether symbolic links are enabled on Windows
|
|
|
|
test_have_prereq MINGW &&
|
|
|
|
cmd //c "mklink y x" &> /dev/null && test -h y
|
|
|
|
'
|
|
|
|
|
2013-11-25 21:02:16 +00:00
|
|
|
test_lazy_prereq FILEMODE '
|
|
|
|
test "$(git config --bool core.filemode)" = true
|
|
|
|
'
|
|
|
|
|
2012-07-26 13:39:53 +00:00
|
|
|
test_lazy_prereq CASE_INSENSITIVE_FS '
|
|
|
|
echo good >CamelCase &&
|
|
|
|
echo bad >camelcase &&
|
|
|
|
test "$(cat CamelCase)" != good
|
|
|
|
'
|
|
|
|
|
t: factor out FUNNYNAMES as shared lazy prereq
A fair number of tests need to check that the filesystem supports file
names including "funny" characters, like newline, tab, and double-quote.
Jonathan Nieder suggested that this be extracted into a lazy prereq in
the top-level `test-lib.sh`. This patch effects that change.
The FUNNYNAMES prereq now uniformly requires support for newlines, tabs,
and double-quotes in filenames. This very slightly decreases the power
of some tests, which might have run previously on a system that supports
(e.g.) newlines and tabs but not double-quotes, but now will not. This
seems to me like an acceptable tradeoff for consistency.
One test (`t/t9902-completion.sh`) defined FUNNYNAMES to further require
the separators \034 through \037, the test for which was implemented
using the Bash-specific $'\034' syntax. I've elected to leave this one
as is, renaming it to FUNNIERNAMES.
After this patch, `git grep 'test_\(set\|lazy\)_prereq.*FUNNYNAMES'` has
only one result.
Signed-off-by: William Chargin <wchargin@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-08-06 18:35:08 +00:00
|
|
|
test_lazy_prereq FUNNYNAMES '
|
|
|
|
test_have_prereq !MINGW &&
|
|
|
|
touch -- \
|
|
|
|
"FUNNYNAMES tab embedded" \
|
|
|
|
"FUNNYNAMES \"quote embedded\"" \
|
|
|
|
"FUNNYNAMES newline
|
|
|
|
embedded" 2>/dev/null &&
|
|
|
|
rm -- \
|
|
|
|
"FUNNYNAMES tab embedded" \
|
|
|
|
"FUNNYNAMES \"quote embedded\"" \
|
|
|
|
"FUNNYNAMES newline
|
|
|
|
embedded" 2>/dev/null
|
|
|
|
'
|
|
|
|
|
2012-07-26 13:39:56 +00:00
|
|
|
test_lazy_prereq UTF8_NFD_TO_NFC '
|
|
|
|
# check whether FS converts nfd unicode to nfc
|
|
|
|
auml=$(printf "\303\244")
|
|
|
|
aumlcdiar=$(printf "\141\314\210")
|
|
|
|
>"$auml" &&
|
2018-04-30 06:35:19 +00:00
|
|
|
test -f "$aumlcdiar"
|
2012-07-26 13:39:56 +00:00
|
|
|
'
|
|
|
|
|
2012-11-15 00:33:40 +00:00
|
|
|
test_lazy_prereq AUTOIDENT '
|
|
|
|
sane_unset GIT_AUTHOR_NAME &&
|
|
|
|
sane_unset GIT_AUTHOR_EMAIL &&
|
|
|
|
git var GIT_AUTHOR_IDENT
|
|
|
|
'
|
|
|
|
|
2014-06-09 19:23:30 +00:00
|
|
|
test_lazy_prereq EXPENSIVE '
|
|
|
|
test -n "$GIT_TEST_LONG"
|
|
|
|
'
|
|
|
|
|
2018-01-30 21:21:23 +00:00
|
|
|
test_lazy_prereq EXPENSIVE_ON_WINDOWS '
|
|
|
|
test_have_prereq EXPENSIVE || test_have_prereq !MINGW,!CYGWIN
|
|
|
|
'
|
|
|
|
|
2014-06-09 20:54:25 +00:00
|
|
|
test_lazy_prereq USR_BIN_TIME '
|
|
|
|
test -x /usr/bin/time
|
|
|
|
'
|
|
|
|
|
2015-01-16 09:16:49 +00:00
|
|
|
test_lazy_prereq NOT_ROOT '
|
|
|
|
uid=$(id -u) &&
|
|
|
|
test "$uid" != 0
|
|
|
|
'
|
|
|
|
|
2016-09-09 17:36:28 +00:00
|
|
|
test_lazy_prereq JGIT '
|
2019-05-15 01:36:33 +00:00
|
|
|
jgit --version
|
2016-09-09 17:36:28 +00:00
|
|
|
'
|
|
|
|
|
2016-01-19 22:09:37 +00:00
|
|
|
# SANITY is about "can you correctly predict what the filesystem would
|
|
|
|
# do by only looking at the permission bits of the files and
|
|
|
|
# directories?" A typical example of !SANITY is running the test
|
|
|
|
# suite as root, where a test may expect "chmod -r file && cat file"
|
|
|
|
# to fail because file is supposed to be unreadable after a successful
|
|
|
|
# chmod. In an environment (i.e. combination of what filesystem is
|
|
|
|
# being used and who is running the tests) that lacks SANITY, you may
|
|
|
|
# be able to delete or create a file when the containing directory
|
|
|
|
# doesn't have write permissions, or access a file even if the
|
|
|
|
# containing directory doesn't have read or execute permissions.
|
|
|
|
|
2015-01-27 15:39:01 +00:00
|
|
|
test_lazy_prereq SANITY '
|
|
|
|
mkdir SANETESTD.1 SANETESTD.2 &&
|
|
|
|
|
|
|
|
chmod +w SANETESTD.1 SANETESTD.2 &&
|
|
|
|
>SANETESTD.1/x 2>SANETESTD.2/x &&
|
|
|
|
chmod -w SANETESTD.1 &&
|
2016-01-19 22:09:37 +00:00
|
|
|
chmod -r SANETESTD.1/x &&
|
2015-01-27 15:39:01 +00:00
|
|
|
chmod -rx SANETESTD.2 ||
|
tests: send "bug in the test script" errors to the script's stderr
Some of the functions in our test library check that they were invoked
properly with conditions like this:
test "$#" = 2 ||
error "bug in the test script: not 2 parameters to test-expect-success"
If this particular condition is triggered, then 'error' will abort the
whole test script with a bold red error message [1] right away.
However, under certain circumstances the test script will be aborted
completely silently, namely if:
- a similar condition in a test helper function like
'test_line_count' is triggered,
- which is invoked from the test script's "main" shell [2],
- and the test script is run manually (i.e. './t1234-foo.sh' as
opposed to 'make t1234-foo.sh' or 'make test') [3]
- and without the '--verbose' option,
because the error message is printed from within 'test_eval_', where
standard output is redirected either to /dev/null or to a log file.
The only indication that something is wrong is that not all tests in
the script are executed and at the end of the test script's output
there is no "# passed all N tests" message, which are subtle and can
easily go unnoticed, as I had to experience myself.
Send these "bug in the test script" error messages directly to the
test scripts standard error and thus to the terminal, so those bugs
will be much harder to overlook. Instead of updating all ~20 such
'error' calls with a redirection, let's add a BUG() function to
'test-lib.sh', wrapping an 'error' call with the proper redirection
and also including the common prefix of those error messages, and
convert all those call sites [4] to use this new BUG() function
instead.
[1] That particular error message from 'test_expect_success' is
printed in color only when running with or without '--verbose';
with '--tee' or '--verbose-log' the error is printed without
color, but it is printed to the terminal nonetheless.
[2] If such a condition is triggered in a subshell of a test, then
'error' won't be able to abort the whole test script, but only the
subshell, which in turn causes the test to fail in the usual way,
indicating loudly and clearly that something is wrong.
[3] Well, 'error' aborts the test script the same way when run
manually or by 'make' or 'prove', but both 'make' and 'prove' pay
attention to the test script's exit status, and even a silently
aborted test script would then trigger those tools' usual
noticable error messages.
[4] Strictly speaking, not all those 'error' calls need that
redirection to send their output to the terminal, see e.g.
'test_expect_success' in the opening example, but I think it's
better to be consistent.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-11-19 13:13:26 +00:00
|
|
|
BUG "cannot prepare SANETESTD"
|
2015-01-27 15:39:01 +00:00
|
|
|
|
2016-01-19 22:09:37 +00:00
|
|
|
! test -r SANETESTD.1/x &&
|
2015-01-27 15:39:01 +00:00
|
|
|
! rm SANETESTD.1/x && ! test -f SANETESTD.2/x
|
|
|
|
status=$?
|
|
|
|
|
|
|
|
chmod +rwx SANETESTD.1 SANETESTD.2 &&
|
|
|
|
rm -rf SANETESTD.1 SANETESTD.2 ||
|
tests: send "bug in the test script" errors to the script's stderr
Some of the functions in our test library check that they were invoked
properly with conditions like this:
test "$#" = 2 ||
error "bug in the test script: not 2 parameters to test-expect-success"
If this particular condition is triggered, then 'error' will abort the
whole test script with a bold red error message [1] right away.
However, under certain circumstances the test script will be aborted
completely silently, namely if:
- a similar condition in a test helper function like
'test_line_count' is triggered,
- which is invoked from the test script's "main" shell [2],
- and the test script is run manually (i.e. './t1234-foo.sh' as
opposed to 'make t1234-foo.sh' or 'make test') [3]
- and without the '--verbose' option,
because the error message is printed from within 'test_eval_', where
standard output is redirected either to /dev/null or to a log file.
The only indication that something is wrong is that not all tests in
the script are executed and at the end of the test script's output
there is no "# passed all N tests" message, which are subtle and can
easily go unnoticed, as I had to experience myself.
Send these "bug in the test script" error messages directly to the
test scripts standard error and thus to the terminal, so those bugs
will be much harder to overlook. Instead of updating all ~20 such
'error' calls with a redirection, let's add a BUG() function to
'test-lib.sh', wrapping an 'error' call with the proper redirection
and also including the common prefix of those error messages, and
convert all those call sites [4] to use this new BUG() function
instead.
[1] That particular error message from 'test_expect_success' is
printed in color only when running with or without '--verbose';
with '--tee' or '--verbose-log' the error is printed without
color, but it is printed to the terminal nonetheless.
[2] If such a condition is triggered in a subshell of a test, then
'error' won't be able to abort the whole test script, but only the
subshell, which in turn causes the test to fail in the usual way,
indicating loudly and clearly that something is wrong.
[3] Well, 'error' aborts the test script the same way when run
manually or by 'make' or 'prove', but both 'make' and 'prove' pay
attention to the test script's exit status, and even a silently
aborted test script would then trigger those tools' usual
noticable error messages.
[4] Strictly speaking, not all those 'error' calls need that
redirection to send their output to the terminal, see e.g.
'test_expect_success' in the opening example, but I think it's
better to be consistent.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-11-19 13:13:26 +00:00
|
|
|
BUG "cannot clean SANETESTD"
|
2015-01-27 15:39:01 +00:00
|
|
|
return $status
|
|
|
|
'
|
2013-03-11 01:31:47 +00:00
|
|
|
|
2016-07-21 16:02:54 +00:00
|
|
|
test FreeBSD != $uname_s || GIT_UNZIP=${GIT_UNZIP:-/usr/local/bin/unzip}
|
2013-03-11 01:31:47 +00:00
|
|
|
GIT_UNZIP=${GIT_UNZIP:-unzip}
|
|
|
|
test_lazy_prereq UNZIP '
|
|
|
|
"$GIT_UNZIP" -v
|
|
|
|
test $? -ne 127
|
|
|
|
'
|
2015-03-13 04:53:07 +00:00
|
|
|
|
|
|
|
run_with_limited_cmdline () {
|
|
|
|
(ulimit -s 128 && "$@")
|
|
|
|
}
|
|
|
|
|
2017-09-14 17:24:41 +00:00
|
|
|
test_lazy_prereq CMDLINE_LIMIT '
|
2020-05-16 05:33:38 +00:00
|
|
|
test_have_prereq !HPPA,!MINGW,!CYGWIN &&
|
2017-09-14 17:24:41 +00:00
|
|
|
run_with_limited_cmdline true
|
|
|
|
'
|
2016-07-11 23:54:18 +00:00
|
|
|
|
2017-09-07 14:02:20 +00:00
|
|
|
run_with_limited_stack () {
|
|
|
|
(ulimit -s 128 && "$@")
|
|
|
|
}
|
|
|
|
|
2017-09-14 17:24:41 +00:00
|
|
|
test_lazy_prereq ULIMIT_STACK_SIZE '
|
2020-05-16 05:33:38 +00:00
|
|
|
test_have_prereq !HPPA,!MINGW,!CYGWIN &&
|
2017-09-14 17:24:41 +00:00
|
|
|
run_with_limited_stack true
|
|
|
|
'
|
2017-09-07 14:02:20 +00:00
|
|
|
|
2020-04-23 21:41:06 +00:00
|
|
|
run_with_limited_open_files () {
|
|
|
|
(ulimit -n 32 && "$@")
|
|
|
|
}
|
|
|
|
|
|
|
|
test_lazy_prereq ULIMIT_FILE_DESCRIPTORS '
|
|
|
|
test_have_prereq !MINGW,!CYGWIN &&
|
|
|
|
run_with_limited_open_files true
|
|
|
|
'
|
|
|
|
|
2016-07-11 23:54:18 +00:00
|
|
|
build_option () {
|
|
|
|
git version --build-options |
|
|
|
|
sed -ne "s/^$1: //p"
|
|
|
|
}
|
|
|
|
|
2021-11-02 15:46:06 +00:00
|
|
|
test_lazy_prereq SIZE_T_IS_64BIT '
|
|
|
|
test 8 -eq "$(build_option sizeof-size_t)"
|
|
|
|
'
|
|
|
|
|
2016-07-11 23:54:18 +00:00
|
|
|
test_lazy_prereq LONG_IS_64BIT '
|
|
|
|
test 8 -le "$(build_option sizeof-long)"
|
|
|
|
'
|
t0006 & t5000: prepare for 64-bit timestamps
Git's source code refers to timestamps as unsigned longs. On 32-bit
platforms, as well as on Windows, unsigned long is not large enough to
capture dates that are "absurdly far in the future".
It is perfectly valid by the C standard, of course, for the `long` data
type to refer to 32-bit integers. That is why the `time_t` data type
exists: so that it can be 64-bit even if `long` is 32-bit. Git's source
code simply uses an incorrect data type for timestamps, is all.
The earlier quick fix 6b9c38e14cd (t0006: skip "far in the future" test
when unsigned long is not long enough, 2016-07-11) papered over this
issue simply by skipping the respective test cases on platforms where
they would fail due to the data type in use.
This quick fix, however, tests for *long* to be 64-bit or not. What we
need, though, is a test that says whether *whatever data type we use for
timestamps* is 64-bit or not.
The same quick fix was used to handle the similar problem where Git's
source code uses `unsigned long` to represent size, instead of `size_t`,
conflating the two issues.
So let's just add another prerequisite to test specifically whether
timestamps are represented by a 64-bit data type or not. Later, after we
switch to a larger data type, we can flip that prerequisite to test
`time_t` instead of `long`.
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-04-20 20:52:13 +00:00
|
|
|
|
2018-03-24 07:44:36 +00:00
|
|
|
test_lazy_prereq TIME_IS_64BIT 'test-tool date is64bit'
|
|
|
|
test_lazy_prereq TIME_T_IS_64BIT 'test-tool date time_t-is64bit'
|
2018-04-03 14:01:57 +00:00
|
|
|
|
|
|
|
test_lazy_prereq CURL '
|
|
|
|
curl --version
|
|
|
|
'
|
2018-05-13 02:24:11 +00:00
|
|
|
|
|
|
|
# SHA1 is a test if the hash algorithm in use is SHA-1. This is both for tests
|
|
|
|
# which will not work with other hash algorithms and tests that work but don't
|
|
|
|
# test anything meaningful (e.g. special values which cause short collisions).
|
|
|
|
test_lazy_prereq SHA1 '
|
2020-07-29 23:14:25 +00:00
|
|
|
case "$GIT_DEFAULT_HASH" in
|
|
|
|
sha1) true ;;
|
|
|
|
"") test $(git hash-object /dev/null) = e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 ;;
|
|
|
|
*) false ;;
|
|
|
|
esac
|
2018-05-13 02:24:11 +00:00
|
|
|
'
|
2018-10-31 20:02:02 +00:00
|
|
|
|
2020-09-11 17:49:18 +00:00
|
|
|
# Ensure that no test accidentally triggers a Git command
|
2020-11-24 04:16:42 +00:00
|
|
|
# that runs the actual maintenance scheduler, affecting a user's
|
|
|
|
# system permanently.
|
|
|
|
# Tests that verify the scheduler integration must set this locally
|
2020-09-11 17:49:18 +00:00
|
|
|
# to avoid errors.
|
2020-11-24 04:16:42 +00:00
|
|
|
GIT_TEST_MAINT_SCHEDULER="none:exit 1"
|
2022-03-25 18:03:02 +00:00
|
|
|
|
|
|
|
# Does this platform support `git fsmonitor--daemon`
|
|
|
|
#
|
|
|
|
test_lazy_prereq FSMONITOR_DAEMON '
|
|
|
|
git version --build-options >output &&
|
|
|
|
grep "feature: fsmonitor--daemon" output
|
|
|
|
'
|