development/git
development/git
1
0
Fork 0
mirror of https://github.com/git/git synced 2024-08-27 03:29:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'sg/stress-test'

Browse source 
Flaky tests can now be repeatedly run under load with the
"--stress" option.

* sg/stress-test:
  test-lib: add the '--stress' option to run a test repeatedly under load
  test-lib-functions: introduce the 'test_set_port' helper function
  test-lib: set $TRASH_DIRECTORY earlier
  test-lib: consolidate naming of test-results paths
  test-lib: parse command line options earlier
  test-lib: parse options in a for loop to keep $@ intact
  test-lib: extract Bash version check for '-x' tracing
  test-lib: translate SIGTERM and SIGHUP to an exit
This commit is contained in:
Junio C Hamano 2019-01-18 13:49:56 -08:00
parent 2c0a645d9e fb7d1e3ac8
commit 3fe47ff444
9 changed files with 308 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
t/README
View file

@ -186,6 +186,22 @@ appropriately before running "make".
this feature by setting the GIT_TEST_CHAIN_LINT environment
variable to "1" or "0", respectively.
--stress::
--stress=<N>::
Run the test script repeatedly in multiple parallel jobs until
one of them fails. Useful for reproducing rare failures in
flaky tests. The number of parallel jobs is, in order of
precedence: <N>, or the value of the GIT_TEST_STRESS_LOAD
environment variable, or twice the number of available
processors (as shown by the 'getconf' utility), or 8.
Implies `--verbose -x --immediate` to get the most information
about the failure. Note that the verbose output of each test
job is saved to 't/test-results/$TEST_NAME.stress-<nr>.out',
and only the output of the failed test job is shown on the
terminal. The names of the trash directories get a
'.stress-<nr>' suffix, and the trash directory of the failed
test job is renamed to end with a '.stress-failed' suffix.
You can also set the GIT_TEST_INSTALLED environment variable to
the bindir of an existing git installation to test that installation.
You still need to have built this git sandbox, from which various
@ -425,7 +441,8 @@ This test harness library does the following things:
- Creates an empty test directory with an empty .git/objects database
and chdir(2) into it. This directory is 't/trash
directory.$test_name_without_dotsh', with t/ subject to change by
the --root option documented above.
the --root option documented above, and a '.stress-<N>' suffix
appended by the --stress option.
- Defines standard test helper functions for your scripts to
use. These functions are designed to make all scripts behave

2
t/lib-git-daemon.sh
View file

@ -28,7 +28,7 @@ then
test_skip_or_die $GIT_TEST_GIT_DAEMON "file system does not support FIFOs"
fi
LIB_GIT_DAEMON_PORT=${LIB_GIT_DAEMON_PORT-${this_test#t}}
test_set_port LIB_GIT_DAEMON_PORT
GIT_DAEMON_PID=
GIT_DAEMON_DOCUMENT_ROOT_PATH="$PWD"/repo

9
t/lib-git-p4.sh
View file

@ -53,14 +53,7 @@ time_in_seconds () {
(cd / && "$PYTHON_PATH" -c 'import time; print(int(time.time()))')
}
# Try to pick a unique port: guess a large number, then hope
# no more than one of each test is running.
#
# This does not handle the case where somebody else is running the
# same tests and has chosen the same ports.
testid=${this_test#t}
git_p4_test_start=9800
P4DPORT=$((10669 + ($testid - $git_p4_test_start)))
test_set_port P4DPORT
P4PORT=localhost:$P4DPORT
P4CLIENT=client

2
t/lib-git-svn.sh
View file

@ -13,6 +13,7 @@ fi
GIT_DIR=$PWD/.git
GIT_SVN_DIR=$GIT_DIR/svn/refs/remotes/git-svn
SVN_TREE=$GIT_SVN_DIR/svn-tree
test_set_port SVNSERVE_PORT
svn >/dev/null 2>&1
if test $? -ne 1
@ -119,7 +120,6 @@ require_svnserve () {
}
start_svnserve () {
SVNSERVE_PORT=${SVNSERVE_PORT-${this_test#t}}
svnserve --listen-port $SVNSERVE_PORT \
--root "$rawsvnrepo" \
--listen-once \

2
t/lib-httpd.sh
View file

@ -82,7 +82,7 @@ case $(uname) in
esac
LIB_HTTPD_PATH=${LIB_HTTPD_PATH-"$DEFAULT_HTTPD_PATH"}
LIB_HTTPD_PORT=${LIB_HTTPD_PORT-${this_test#t}}
test_set_port LIB_HTTPD_PORT
TEST_PATH="$TEST_DIRECTORY"/lib-httpd
HTTPD_ROOT_PATH="$PWD"/httpd

1
t/t0410-partial-clone.sh
View file

@ -492,7 +492,6 @@ test_expect_success 'gc stops traversal when a missing but promised object is re
! grep "$TREE_HASH" out
'
LIB_HTTPD_PORT=12345 # default port, 410, cannot be used as non-root
. "$TEST_DIRECTORY"/lib-httpd.sh
start_httpd

2
t/t5512-ls-remote.sh
View file

@ -260,7 +260,7 @@ test_lazy_prereq GIT_DAEMON '
# This test spawns a daemon, so run it only if the user would be OK with
# testing with git-daemon.
test_expect_success PIPE,JGIT,GIT_DAEMON 'indicate no refs in standards-compliant empty remote' '
JGIT_DAEMON_PORT=${JGIT_DAEMON_PORT-${this_test#t}} &&
test_set_port JGIT_DAEMON_PORT &&
JGIT_DAEMON_PID= &&
git init --bare empty.git &&
>empty.git/git-daemon-export-ok &&

39
t/test-lib-functions.sh
View file

@ -1263,3 +1263,42 @@ test_oid () {
fi &&
eval "printf '%s' \"\${$var}\""
}
# Choose a port number based on the test script's number and store it in
# the given variable name, unless that variable already contains a number.
test_set_port () {
local var=$1 port
if test $# -ne 1 || test -z "$var"
then
BUG "test_set_port requires a variable name"
fi
eval port=\$$var
case "$port" in
"")
# No port is set in the given env var, use the test
# number as port number instead.
# Remove not only the leading 't', but all leading zeros
# as well, so the arithmetic below won't (mis)interpret
# a test number like '0123' as an octal value.
port=${this_test#${this_test%%[1-9]*}}
if test "${port:-0}" -lt 1024
then
# root-only port, use a larger one instead.
port=$(($port + 10000))
fi
;;
*[^0-9]*|0*)
error >&7 "invalid port number: $port"
;;
*)
# The user has specified the port.
;;
esac
# Make sure that parallel '--stress' test jobs get different
# ports.
port=$(($port + ${GIT_TEST_STRESS_JOB_NR:-0}))
eval $var=$port
}

365
t/test-lib.sh
View file

@ -71,19 +71,222 @@ then
exit 1
fi
# Parse options while taking care to leave $@ intact, so we will still
# have all the original command line options when executing the test
# script again for '--tee' and '--verbose-log' below.
store_arg_to=
prev_opt=
for opt
do
if test -n "$store_arg_to"
then
eval $store_arg_to=\$opt
store_arg_to=
prev_opt=
continue
fi
case "$opt" in
-d|--d|--de|--deb|--debu|--debug)
debug=t ;;
-i|--i|--im|--imm|--imme|--immed|--immedi|--immedia|--immediat|--immediate)
immediate=t ;;
-l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests)
GIT_TEST_LONG=t; export GIT_TEST_LONG ;;
-r)
store_arg_to=run_list
;;
--run=*)
run_list=${opt#--*=} ;;
-h|--h|--he|--hel|--help)
help=t ;;
-v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose)
verbose=t ;;
--verbose-only=*)
verbose_only=${opt#--*=}
;;
-q|--q|--qu|--qui|--quie|--quiet)
# Ignore --quiet under a TAP::Harness. Saying how many tests
# passed without the ok/not ok details is always an error.
test -z "$HARNESS_ACTIVE" && quiet=t ;;
--with-dashes)
with_dashes=t ;;
--no-color)
color= ;;
--va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind)
valgrind=memcheck
tee=t
;;
--valgrind=*)
valgrind=${opt#--*=}
tee=t
;;
--valgrind-only=*)
valgrind_only=${opt#--*=}
tee=t
;;
--tee)
tee=t ;;
--root=*)
root=${opt#--*=} ;;
--chain-lint)
GIT_TEST_CHAIN_LINT=1 ;;
--no-chain-lint)
GIT_TEST_CHAIN_LINT=0 ;;
-x)
trace=t ;;
-V|--verbose-log)
verbose_log=t
tee=t
;;
--stress)
stress=t ;;
--stress=*)
stress=${opt#--*=}
case "$stress" in
*[^0-9]*|0*|"")
echo "error: --stress=<N> requires the number of jobs to run" >&2
exit 1
;;
*) # Good.
;;
esac
;;
*)
echo "error: unknown test option '$opt'" >&2; exit 1 ;;
esac
prev_opt=$opt
done
if test -n "$store_arg_to"
then
echo "error: $prev_opt requires an argument" >&2
exit 1
fi
if test -n "$valgrind_only"
then
test -z "$valgrind" && valgrind=memcheck
test -z "$verbose" && verbose_only="$valgrind_only"
elif test -n "$valgrind"
then
test -z "$verbose_log" && verbose=t
fi
if test -n "$stress"
then
verbose=t
trace=t
immediate=t
fi
TEST_STRESS_JOB_SFX="${GIT_TEST_STRESS_JOB_NR:+.stress-$GIT_TEST_STRESS_JOB_NR}"
TEST_NAME="$(basename "$0" .sh)"
TEST_RESULTS_DIR="$TEST_OUTPUT_DIRECTORY/test-results"
TEST_RESULTS_BASE="$TEST_RESULTS_DIR/$TEST_NAME$TEST_STRESS_JOB_SFX"
TRASH_DIRECTORY="trash directory.$TEST_NAME$TEST_STRESS_JOB_SFX"
test -n "$root" && TRASH_DIRECTORY="$root/$TRASH_DIRECTORY"
case "$TRASH_DIRECTORY" in
/*) ;; # absolute path is good
*) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$TRASH_DIRECTORY" ;;
esac
# If --stress was passed, run this test repeatedly in several parallel loops.
if test "$GIT_TEST_STRESS_STARTED" = "done"
then
: # Don't stress test again.
elif test -n "$stress"
then
if test "$stress" != t
then
job_count=$stress
elif test -n "$GIT_TEST_STRESS_LOAD"
then
job_count="$GIT_TEST_STRESS_LOAD"
elif job_count=$(getconf _NPROCESSORS_ONLN 2>/dev/null) &&
test -n "$job_count"
then
job_count=$((2 * $job_count))
else
job_count=8
fi
mkdir -p "$TEST_RESULTS_DIR"
stressfail="$TEST_RESULTS_BASE.stress-failed"
rm -f "$stressfail"
stress_exit=0
trap '
kill $job_pids 2>/dev/null
wait
stress_exit=1
' TERM INT HUP
job_pids=
job_nr=0
while test $job_nr -lt "$job_count"
do
(
GIT_TEST_STRESS_STARTED=done
GIT_TEST_STRESS_JOB_NR=$job_nr
export GIT_TEST_STRESS_STARTED GIT_TEST_STRESS_JOB_NR
trap '
kill $test_pid 2>/dev/null
wait
exit 1
' TERM INT
cnt=0
while ! test -e "$stressfail"
do
$TEST_SHELL_PATH "$0" "$@" >"$TEST_RESULTS_BASE.stress-$job_nr.out" 2>&1 &
test_pid=$!
if wait $test_pid
then
printf "OK %2d.%d\n" $GIT_TEST_STRESS_JOB_NR $cnt
else
echo $GIT_TEST_STRESS_JOB_NR >>"$stressfail"
printf "FAIL %2d.%d\n" $GIT_TEST_STRESS_JOB_NR $cnt
fi
cnt=$(($cnt + 1))
done
) &
job_pids="$job_pids $!"
job_nr=$(($job_nr + 1))
done
wait
if test -f "$stressfail"
then
echo "Log(s) of failed test run(s):"
for failed_job_nr in $(sort -n "$stressfail")
do
echo "Contents of '$TEST_RESULTS_BASE.stress-$failed_job_nr.out':"
cat "$TEST_RESULTS_BASE.stress-$failed_job_nr.out"
done
rm -rf "$TRASH_DIRECTORY.stress-failed"
# Move the last one.
mv "$TRASH_DIRECTORY.stress-$failed_job_nr" "$TRASH_DIRECTORY.stress-failed"
fi
exit $stress_exit
fi
# if --tee was passed, write the output not only to the terminal, but
# additionally to the file test-results/$BASENAME.out, too.
case "$GIT_TEST_TEE_STARTED, $* " in
done,*)
# do not redirect again
;;
*' --tee '*|*' --va'*|*' -V '*|*' --verbose-log '*)
mkdir -p "$TEST_OUTPUT_DIRECTORY/test-results"
BASE="$TEST_OUTPUT_DIRECTORY/test-results/$(basename "$0" .sh)"
if test "$GIT_TEST_TEE_STARTED" = "done"
then
: # do not redirect again
elif test -n "$tee"
then
mkdir -p "$TEST_RESULTS_DIR"
# Make this filename available to the sub-process in case it is using
# --verbose-log.
GIT_TEST_TEE_OUTPUT_FILE=$BASE.out
GIT_TEST_TEE_OUTPUT_FILE=$TEST_RESULTS_BASE.out
export GIT_TEST_TEE_OUTPUT_FILE
# Truncate before calling "tee -a" to get rid of the results
@ -91,11 +294,38 @@ done,*)
>"$GIT_TEST_TEE_OUTPUT_FILE"
(GIT_TEST_TEE_STARTED=done ${TEST_SHELL_PATH} "$0" "$@" 2>&1;
echo $? >"$BASE.exit") | tee -a "$GIT_TEST_TEE_OUTPUT_FILE"
test "$(cat "$BASE.exit")" = 0
echo $? >"$TEST_RESULTS_BASE.exit") | tee -a "$GIT_TEST_TEE_OUTPUT_FILE"
test "$(cat "$TEST_RESULTS_BASE.exit")" = 0
exit
;;
esac
fi
if test -n "$trace" && test -n "$test_untraceable"
then
# '-x' tracing requested, but this test script can't be reliably
# traced, unless it is run with a Bash version supporting
# BASH_XTRACEFD (introduced in Bash v4.1).
#
# Perform this version check _after_ the test script was
# potentially re-executed with $TEST_SHELL_PATH for '--tee' or
# '--verbose-log', so the right shell is checked and the
# warning is issued only once.
if test -n "$BASH_VERSION" && eval '
test ${BASH_VERSINFO[0]} -gt 4 || {
test ${BASH_VERSINFO[0]} -eq 4 &&
test ${BASH_VERSINFO[1]} -ge 1
}
'
then
: Executed by a Bash version supporting BASH_XTRACEFD. Good.
else
echo >&2 "warning: ignoring -x; '$0' is untraceable without BASH_XTRACEFD"
trace=
fi
fi
if test -n "$trace" && test -z "$verbose_log"
then
verbose=t
fi
# For repeatability, reset the environment to known value.
# TERM is sanitized below, after saving color control sequences.
@ -193,7 +423,7 @@ fi
# Add libc MALLOC and MALLOC_PERTURB test
# only if we are not executing the test with valgrind
if expr " $GIT_TEST_OPTS " : ".* --valgrind " >/dev/null ||
if test -n "$valgrind" ||
test -n "$TEST_NO_MALLOC_CHECK"
then
setup_malloc_check () {
@ -264,100 +494,6 @@ test "x$TERM" != "xdumb" && (
) &&
color=t
while test "$#" -ne 0
do
case "$1" in
-d|--d|--de|--deb|--debu|--debug)
debug=t; shift ;;
-i|--i|--im|--imm|--imme|--immed|--immedi|--immedia|--immediat|--immediate)
immediate=t; shift ;;
-l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests)
GIT_TEST_LONG=t; export GIT_TEST_LONG; shift ;;
-r)
shift; test "$#" -ne 0 || {
echo 'error: -r requires an argument' >&2;
exit 1;
}
run_list=$1; shift ;;
--run=*)
run_list=${1#--*=}; shift ;;
-h|--h|--he|--hel|--help)
help=t; shift ;;
-v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose)
verbose=t; shift ;;
--verbose-only=*)
verbose_only=${1#--*=}
shift ;;
-q|--q|--qu|--qui|--quie|--quiet)
# Ignore --quiet under a TAP::Harness. Saying how many tests
# passed without the ok/not ok details is always an error.
test -z "$HARNESS_ACTIVE" && quiet=t; shift ;;
--with-dashes)
with_dashes=t; shift ;;
--no-color)
color=; shift ;;
--va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind)
valgrind=memcheck
shift ;;
--valgrind=*)
valgrind=${1#--*=}
shift ;;
--valgrind-only=*)
valgrind_only=${1#--*=}
shift ;;
--tee)
shift ;; # was handled already
--root=*)
root=${1#--*=}
shift ;;
--chain-lint)
GIT_TEST_CHAIN_LINT=1
shift ;;
--no-chain-lint)
GIT_TEST_CHAIN_LINT=0
shift ;;
-x)
# Some test scripts can't be reliably traced with '-x',
# unless the test is run with a Bash version supporting
# BASH_XTRACEFD (introduced in Bash v4.1). Check whether
# this test is marked as such, and ignore '-x' if it
# isn't executed with a suitable Bash version.
if test -z "$test_untraceable" || {
test -n "$BASH_VERSION" && eval '
test ${BASH_VERSINFO[0]} -gt 4 || {
test ${BASH_VERSINFO[0]} -eq 4 &&
test ${BASH_VERSINFO[1]} -ge 1
}
'
}
then
trace=t
else
echo >&2 "warning: ignoring -x; '$0' is untraceable without BASH_XTRACEFD"
fi
shift ;;
-V|--verbose-log)
verbose_log=t
shift ;;
*)
echo "error: unknown test option '$1'" >&2; exit 1 ;;
esac
done
if test -n "$valgrind_only"
then
test -z "$valgrind" && valgrind=memcheck
test -z "$verbose" && verbose_only="$valgrind_only"
elif test -n "$valgrind"
then
test -z "$verbose_log" && verbose=t
fi
if test -n "$trace" && test -z "$verbose_log"
then
verbose=t
fi
if test -n "$color"
then
# Save the color control sequences now rather than run tput
@ -476,7 +612,7 @@ die () {
GIT_EXIT_OK=
trap 'die' EXIT
trap 'exit $?' INT
trap 'exit $?' INT TERM HUP
# The user-facing functions are loaded from a separate file so that
# test_perf subshells can have them too
@ -818,12 +954,9 @@ test_done () {
if test -z "$HARNESS_ACTIVE"
then
test_results_dir="$TEST_OUTPUT_DIRECTORY/test-results"
mkdir -p "$test_results_dir"
base=${0##*/}
test_results_path="$test_results_dir/${base%.sh}.counts"
mkdir -p "$TEST_RESULTS_DIR"
cat >"$test_results_path" <<-EOF
cat >"$TEST_RESULTS_BASE.counts" <<-EOF
total $test_count
success $test_success
fixed $test_fixed
@ -1029,12 +1162,6 @@ then
fi
# Test repository
TRASH_DIRECTORY="trash directory.$(basename "$0" .sh)"
test -n "$root" && TRASH_DIRECTORY="$root/$TRASH_DIRECTORY"
case "$TRASH_DIRECTORY" in
/*) ;; # absolute path is good
*) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$TRASH_DIRECTORY" ;;
esac
rm -fr "$TRASH_DIRECTORY" || {
GIT_EXIT_OK=t
echo >&5 "FATAL: Cannot prepare test area"