mirror of
https://github.com/git/git
synced 2024-07-17 11:07:55 +00:00
93a04bb105
This patch automates the process of determining which tests failed previously and re-running them. While developing patch series, it is a good practice to run the test suite from time to time, just to make sure that obvious bugs are caught early. With complex patch series, it is common to run `make -j15 -k test`, i.e. run the tests in parallel and *not* stop at the first failing test but continue. This has the advantage of identifying possibly multiple problems in one big test run. It is particularly important to reduce the turn-around time thusly on Windows, where the test suite spends 45 minutes on the computer on which this patch was developed. It is the most convenient way to determine which tests failed after running the entire test suite, in parallel, to look for left-over "trash directory.t*" subdirectories in the t/ subdirectory. However, those directories might live outside t/ when overridden using the --root=<directory> option, to which the Makefile has no access. The next best method is to grep explicitly for failed tests in the test-results/ directory, which the Makefile *can* access. Please note that the often-recommended `prove` tool requires Perl, and that opens a whole new can of worms on Windows. As no native Windows Perl comes with Subversion bindings, we have to use a Perl in Git for Windows that uses the POSIX emulation layer named MSYS2 (which is a portable version of Cygwin). When using this emulation layer under stress, e.g. when running massively-parallel tests, unexplicable crashes occur quite frequently, and instead of having a solution to the original problem, the developer now has an additional, quite huge problem. For that reason, this developer rejected `prove` as a solution and went with this patch instead. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Reviewed-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
104 lines
2.7 KiB
Makefile
104 lines
2.7 KiB
Makefile
# Run tests
|
|
#
|
|
# Copyright (c) 2005 Junio C Hamano
|
|
#
|
|
|
|
-include ../config.mak.autogen
|
|
-include ../config.mak
|
|
|
|
#GIT_TEST_OPTS = --verbose --debug
|
|
SHELL_PATH ?= $(SHELL)
|
|
PERL_PATH ?= /usr/bin/perl
|
|
TAR ?= $(TAR)
|
|
RM ?= rm -f
|
|
PROVE ?= prove
|
|
DEFAULT_TEST_TARGET ?= test
|
|
TEST_LINT ?= test-lint
|
|
|
|
ifdef TEST_OUTPUT_DIRECTORY
|
|
TEST_RESULTS_DIRECTORY = $(TEST_OUTPUT_DIRECTORY)/test-results
|
|
else
|
|
TEST_RESULTS_DIRECTORY = test-results
|
|
endif
|
|
|
|
# Shell quote;
|
|
SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH))
|
|
PERL_PATH_SQ = $(subst ','\'',$(PERL_PATH))
|
|
TEST_RESULTS_DIRECTORY_SQ = $(subst ','\'',$(TEST_RESULTS_DIRECTORY))
|
|
|
|
T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh))
|
|
TGITWEB = $(sort $(wildcard t95[0-9][0-9]-*.sh))
|
|
THELPERS = $(sort $(filter-out $(T),$(wildcard *.sh)))
|
|
|
|
all: $(DEFAULT_TEST_TARGET)
|
|
|
|
test: pre-clean $(TEST_LINT)
|
|
$(MAKE) aggregate-results-and-cleanup
|
|
|
|
failed:
|
|
@failed=$$(cd '$(TEST_RESULTS_DIRECTORY_SQ)' && \
|
|
grep -l '^failed [1-9]' *.counts | \
|
|
sed -n 's/\.counts$$/.sh/p') && \
|
|
test -z "$$failed" || $(MAKE) $$failed
|
|
|
|
prove: pre-clean $(TEST_LINT)
|
|
@echo "*** prove ***"; $(PROVE) --exec '$(SHELL_PATH_SQ)' $(GIT_PROVE_OPTS) $(T) :: $(GIT_TEST_OPTS)
|
|
$(MAKE) clean-except-prove-cache
|
|
|
|
$(T):
|
|
@echo "*** $@ ***"; '$(SHELL_PATH_SQ)' $@ $(GIT_TEST_OPTS)
|
|
|
|
pre-clean:
|
|
$(RM) -r '$(TEST_RESULTS_DIRECTORY_SQ)'
|
|
|
|
clean-except-prove-cache:
|
|
$(RM) -r 'trash directory'.* '$(TEST_RESULTS_DIRECTORY_SQ)'
|
|
$(RM) -r valgrind/bin
|
|
|
|
clean: clean-except-prove-cache
|
|
$(RM) .prove
|
|
|
|
test-lint: test-lint-duplicates test-lint-executable test-lint-shell-syntax \
|
|
test-lint-filenames
|
|
|
|
test-lint-duplicates:
|
|
@dups=`echo $(T) | tr ' ' '\n' | sed 's/-.*//' | sort | uniq -d` && \
|
|
test -z "$$dups" || { \
|
|
echo >&2 "duplicate test numbers:" $$dups; exit 1; }
|
|
|
|
test-lint-executable:
|
|
@bad=`for i in $(T); do test -x "$$i" || echo $$i; done` && \
|
|
test -z "$$bad" || { \
|
|
echo >&2 "non-executable tests:" $$bad; exit 1; }
|
|
|
|
test-lint-shell-syntax:
|
|
@'$(PERL_PATH_SQ)' check-non-portable-shell.pl $(T) $(THELPERS)
|
|
|
|
test-lint-filenames:
|
|
@# We do *not* pass a glob to ls-files but use grep instead, to catch
|
|
@# non-ASCII characters (which are quoted within double-quotes)
|
|
@bad="$$(git -c core.quotepath=true ls-files 2>/dev/null | \
|
|
grep '["*:<>?\\|]')"; \
|
|
test -z "$$bad" || { \
|
|
echo >&2 "non-portable file name(s): $$bad"; exit 1; }
|
|
|
|
aggregate-results-and-cleanup: $(T)
|
|
$(MAKE) aggregate-results
|
|
$(MAKE) clean
|
|
|
|
aggregate-results:
|
|
for f in '$(TEST_RESULTS_DIRECTORY_SQ)'/t*-*.counts; do \
|
|
echo "$$f"; \
|
|
done | '$(SHELL_PATH_SQ)' ./aggregate-results.sh
|
|
|
|
gitweb-test:
|
|
$(MAKE) $(TGITWEB)
|
|
|
|
valgrind:
|
|
$(MAKE) GIT_TEST_OPTS="$(GIT_TEST_OPTS) --valgrind"
|
|
|
|
perf:
|
|
$(MAKE) -C perf/ all
|
|
|
|
.PHONY: pre-clean $(T) aggregate-results clean valgrind perf
|