From ee38ed80d8ae00a5b7e6c80b8022804df52e9265 Mon Sep 17 00:00:00 2001 From: Pekka Paalanen Date: Wed, 3 Mar 2021 14:21:01 +0200 Subject: [PATCH] tests: add build option to force skips as failures This will be useful in CI, where we do not want to see any skips. If something starts to skip, that's a mistake somewhere, and want to catch it. Signed-off-by: Pekka Paalanen --- meson_options.txt | 6 ++++++ tests/meson.build | 1 + tests/weston-test-runner.c | 25 ++++++++++++++++++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/meson_options.txt b/meson_options.txt index 239bd2da..ea225a90 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -210,6 +210,12 @@ option( value: true, description: 'Tests: output JUnit XML results' ) +option( + 'test-skip-is-failure', + type: 'boolean', + value: false, + description: 'Tests: consider skip to be a failure' +) option( 'test-gl-renderer', type: 'boolean', diff --git a/tests/meson.build b/tests/meson.build index 90d776f5..212bba0b 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -276,6 +276,7 @@ test_config_h.set_quoted('WESTON_TEST_REFERENCE_PATH', meson.current_source_dir( test_config_h.set_quoted('WESTON_MODULE_MAP', env_modmap) test_config_h.set_quoted('WESTON_DATA_DIR', join_paths(meson.current_source_dir(), '..', 'data')) test_config_h.set_quoted('TESTSUITE_PLUGIN_PATH', exe_plugin_test.full_path()) +test_config_h.set10('WESTON_TEST_SKIP_IS_FAILURE', get_option('test-skip-is-failure')) configure_file(output: 'test-config.h', configuration: test_config_h) foreach t : tests diff --git a/tests/weston-test-runner.c b/tests/weston-test-runner.c index d8becc31..0ab7bc1b 100644 --- a/tests/weston-test-runner.c +++ b/tests/weston-test-runner.c @@ -35,6 +35,7 @@ #include #include +#include "test-config.h" #include "weston-test-runner.h" #include "weston-testsuite-data.h" #include "shared/string-helpers.h" @@ -243,7 +244,11 @@ result_to_str(enum test_result_code ret) [RESULT_FAIL] = "fail", [RESULT_HARD_ERROR] = "hard error", [RESULT_OK] = "ok", +#if WESTON_TEST_SKIP_IS_FAILURE + [RESULT_SKIP] = "skip error", +#else [RESULT_SKIP] = "skip", +#endif }; assert(ret >= 0 && ret < ARRAY_LENGTH(names)); @@ -284,6 +289,9 @@ run_case(struct wet_testsuite_data *suite_data, case RESULT_SKIP: suite_data->skipped++; skip = " # SKIP"; +#if WESTON_TEST_SKIP_IS_FAILURE + fail = "not "; +#endif break; } @@ -327,10 +335,16 @@ skip_case(struct wet_testsuite_data *suite_data, const void *test_data, int iteration) { + const char *skip_error = ""; int iteration_nr = iteration + 1; +#if WESTON_TEST_SKIP_IS_FAILURE + skip_error = "not "; +#endif + suite_data->counter++; - printf("ok %d %s %s/%d # SKIP fixture\n", suite_data->counter, + printf("%sok %d %s %s/%d # SKIP fixture\n", + skip_error, suite_data->counter, suite_data->fixture_name, t->name, iteration_nr); } @@ -477,6 +491,11 @@ weston_test_harness_destroy(struct weston_test_harness *harness) static enum test_result_code counts_to_result(const struct wet_testsuite_data *data) { +#if WESTON_TEST_SKIP_IS_FAILURE + if (data->skipped > 0) + return RESULT_FAIL; +#endif + /* RESULT_SKIP is reserved for fixture setup itself skipping everything */ if (data->total == data->passed + data->skipped) return RESULT_OK; @@ -644,7 +663,11 @@ main(int argc, char *argv[]) if (ret == RESULT_SKIP) { tap_skip_fixture(&harness->data); +#if WESTON_TEST_SKIP_IS_FAILURE + ret = RESULT_FAIL; +#else continue; +#endif } if (ret != RESULT_OK && result != RESULT_HARD_ERROR)