mirror of
git://source.winehq.org/git/wine.git
synced 2024-10-31 14:50:48 +00:00
tests: Allow marking unreliable tests as flaky.
This commit is contained in:
parent
e37769b108
commit
f9b836b1c1
1 changed files with 77 additions and 12 deletions
|
@ -44,6 +44,9 @@ extern int winetest_time;
|
||||||
/* running in interactive mode? */
|
/* running in interactive mode? */
|
||||||
extern int winetest_interactive;
|
extern int winetest_interactive;
|
||||||
|
|
||||||
|
/* always count flaky tests as successful (BOOL) */
|
||||||
|
extern int winetest_allow_flaky;
|
||||||
|
|
||||||
/* report successful tests (BOOL) */
|
/* report successful tests (BOOL) */
|
||||||
extern int winetest_report_success;
|
extern int winetest_report_success;
|
||||||
|
|
||||||
|
@ -59,6 +62,9 @@ extern void winetest_ignore_exceptions( BOOL ignore );
|
||||||
extern void winetest_start_todo( int is_todo );
|
extern void winetest_start_todo( int is_todo );
|
||||||
extern int winetest_loop_todo(void);
|
extern int winetest_loop_todo(void);
|
||||||
extern void winetest_end_todo(void);
|
extern void winetest_end_todo(void);
|
||||||
|
extern void winetest_start_flaky( int is_flaky );
|
||||||
|
extern int winetest_loop_flaky(void);
|
||||||
|
extern void winetest_end_flaky(void);
|
||||||
extern int winetest_get_mainargs( char*** pargv );
|
extern int winetest_get_mainargs( char*** pargv );
|
||||||
extern LONG winetest_get_failures(void);
|
extern LONG winetest_get_failures(void);
|
||||||
extern void winetest_add_failures( LONG new_failures );
|
extern void winetest_add_failures( LONG new_failures );
|
||||||
|
@ -111,6 +117,13 @@ extern void winetest_pop_context(void);
|
||||||
#define trace trace_(__FILE__, __LINE__)
|
#define trace trace_(__FILE__, __LINE__)
|
||||||
#define wait_child_process wait_child_process_(__FILE__, __LINE__)
|
#define wait_child_process wait_child_process_(__FILE__, __LINE__)
|
||||||
|
|
||||||
|
#define flaky_if(is_flaky) for (winetest_start_flaky(is_flaky); \
|
||||||
|
winetest_loop_flaky(); \
|
||||||
|
winetest_end_flaky())
|
||||||
|
#define flaky flaky_if(TRUE)
|
||||||
|
#define flaky_wine flaky_if(!strcmp(winetest_platform, "wine"))
|
||||||
|
#define flaky_wine_if(is_flaky) flaky_if((is_flaky) && !strcmp(winetest_platform, "wine"))
|
||||||
|
|
||||||
#define todo_if(is_todo) for (winetest_start_todo(is_todo); \
|
#define todo_if(is_todo) for (winetest_start_todo(is_todo); \
|
||||||
winetest_loop_todo(); \
|
winetest_loop_todo(); \
|
||||||
winetest_end_todo())
|
winetest_end_todo())
|
||||||
|
@ -194,6 +207,9 @@ int winetest_interactive = 0;
|
||||||
/* current platform */
|
/* current platform */
|
||||||
const char *winetest_platform = "windows";
|
const char *winetest_platform = "windows";
|
||||||
|
|
||||||
|
/* always count flaky tests as successful (BOOL) */
|
||||||
|
int winetest_allow_flaky;
|
||||||
|
|
||||||
/* report successful tests (BOOL) */
|
/* report successful tests (BOOL) */
|
||||||
int winetest_report_success = 0;
|
int winetest_report_success = 0;
|
||||||
|
|
||||||
|
@ -211,6 +227,7 @@ static const struct test *current_test; /* test currently being run */
|
||||||
|
|
||||||
static LONG successes; /* number of successful tests */
|
static LONG successes; /* number of successful tests */
|
||||||
static LONG failures; /* number of failures */
|
static LONG failures; /* number of failures */
|
||||||
|
static LONG flaky_failures; /* number of failures inside flaky block */
|
||||||
static LONG skipped; /* number of skipped test chunks */
|
static LONG skipped; /* number of skipped test chunks */
|
||||||
static LONG todo_successes; /* number of successful tests inside todo block */
|
static LONG todo_successes; /* number of successful tests inside todo block */
|
||||||
static LONG todo_failures; /* number of failures inside todo block */
|
static LONG todo_failures; /* number of failures inside todo block */
|
||||||
|
@ -226,6 +243,8 @@ struct tls_data
|
||||||
{
|
{
|
||||||
const char* current_file; /* file of current check */
|
const char* current_file; /* file of current check */
|
||||||
int current_line; /* line of current check */
|
int current_line; /* line of current check */
|
||||||
|
unsigned int flaky_level; /* current flaky nesting level */
|
||||||
|
int flaky_do_loop;
|
||||||
unsigned int todo_level; /* current todo nesting level */
|
unsigned int todo_level; /* current todo nesting level */
|
||||||
int todo_do_loop;
|
int todo_do_loop;
|
||||||
char *str_pos; /* position in debug buffer */
|
char *str_pos; /* position in debug buffer */
|
||||||
|
@ -283,10 +302,12 @@ const char *winetest_elapsed(void)
|
||||||
|
|
||||||
static const char color_reset[] = "\e[0m";
|
static const char color_reset[] = "\e[0m";
|
||||||
static const char color_dark_red[] = "\e[31m";
|
static const char color_dark_red[] = "\e[31m";
|
||||||
|
static const char color_dark_purple[] = "\e[35m";
|
||||||
static const char color_green[] = "\e[32m";
|
static const char color_green[] = "\e[32m";
|
||||||
static const char color_yellow[] = "\e[33m";
|
static const char color_yellow[] = "\e[33m";
|
||||||
static const char color_blue[] = "\e[34m";
|
static const char color_blue[] = "\e[34m";
|
||||||
static const char color_bright_red[] = "\e[1;91m";
|
static const char color_bright_red[] = "\e[1;91m";
|
||||||
|
static const char color_bright_purple[] = "\e[1;95m";
|
||||||
|
|
||||||
static void winetest_printf( const char *msg, ... ) __WINE_PRINTF_ATTR(1,2);
|
static void winetest_printf( const char *msg, ... ) __WINE_PRINTF_ATTR(1,2);
|
||||||
static void winetest_printf( const char *msg, ... )
|
static void winetest_printf( const char *msg, ... )
|
||||||
|
@ -359,11 +380,21 @@ int winetest_vok( int condition, const char *msg, va_list args )
|
||||||
{
|
{
|
||||||
if (condition)
|
if (condition)
|
||||||
{
|
{
|
||||||
if (winetest_color) printf( color_dark_red );
|
if (data->flaky_level)
|
||||||
winetest_print_context( "Test succeeded inside todo block: " );
|
{
|
||||||
vprintf(msg, args);
|
if (winetest_color) printf( color_dark_purple );
|
||||||
|
winetest_print_context( "Test succeeded inside flaky todo block: " );
|
||||||
|
vprintf(msg, args);
|
||||||
|
InterlockedIncrement(&flaky_failures);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (winetest_color) printf( color_dark_red );
|
||||||
|
winetest_print_context( "Test succeeded inside todo block: " );
|
||||||
|
vprintf(msg, args);
|
||||||
|
InterlockedIncrement(&todo_failures);
|
||||||
|
}
|
||||||
if (winetest_color) printf( color_reset );
|
if (winetest_color) printf( color_reset );
|
||||||
InterlockedIncrement(&todo_failures);
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -389,11 +420,21 @@ int winetest_vok( int condition, const char *msg, va_list args )
|
||||||
{
|
{
|
||||||
if (!condition)
|
if (!condition)
|
||||||
{
|
{
|
||||||
if (winetest_color) printf( color_bright_red );
|
if (data->flaky_level)
|
||||||
winetest_print_context( "Test failed: " );
|
{
|
||||||
vprintf(msg, args);
|
if (winetest_color) printf( color_bright_purple );
|
||||||
|
winetest_print_context( "Test marked flaky: " );
|
||||||
|
vprintf(msg, args);
|
||||||
|
InterlockedIncrement(&flaky_failures);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (winetest_color) printf( color_bright_red );
|
||||||
|
winetest_print_context( "Test failed: " );
|
||||||
|
vprintf(msg, args);
|
||||||
|
InterlockedIncrement(&failures);
|
||||||
|
}
|
||||||
if (winetest_color) printf( color_reset );
|
if (winetest_color) printf( color_reset );
|
||||||
InterlockedIncrement(&failures);
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -470,6 +511,27 @@ void winetest_win_skip( const char *msg, ... )
|
||||||
va_end(valist);
|
va_end(valist);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void winetest_start_flaky( int is_flaky )
|
||||||
|
{
|
||||||
|
struct tls_data *data = get_tls_data();
|
||||||
|
data->flaky_level = (data->flaky_level << 1) | (is_flaky != 0);
|
||||||
|
data->flaky_do_loop = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
int winetest_loop_flaky(void)
|
||||||
|
{
|
||||||
|
struct tls_data *data = get_tls_data();
|
||||||
|
int do_flaky = data->flaky_do_loop;
|
||||||
|
data->flaky_do_loop = 0;
|
||||||
|
return do_flaky;
|
||||||
|
}
|
||||||
|
|
||||||
|
void winetest_end_flaky(void)
|
||||||
|
{
|
||||||
|
struct tls_data *data = get_tls_data();
|
||||||
|
data->flaky_level >>= 1;
|
||||||
|
}
|
||||||
|
|
||||||
void winetest_start_todo( int is_todo )
|
void winetest_start_todo( int is_todo )
|
||||||
{
|
{
|
||||||
struct tls_data *data = get_tls_data();
|
struct tls_data *data = get_tls_data();
|
||||||
|
@ -615,14 +677,16 @@ static int run_test( const char *name )
|
||||||
printf( "%04x:%s:%s Silenced %d todos, %d skips and %d traces.\n",
|
printf( "%04x:%s:%s Silenced %d todos, %d skips and %d traces.\n",
|
||||||
(UINT)GetCurrentProcessId(), test->name, winetest_elapsed(),
|
(UINT)GetCurrentProcessId(), test->name, winetest_elapsed(),
|
||||||
(UINT)muted_todo_successes, (UINT)muted_skipped, (UINT)muted_traces);
|
(UINT)muted_todo_successes, (UINT)muted_skipped, (UINT)muted_traces);
|
||||||
printf( "%04x:%s:%s %d tests executed (%d marked as todo, %d %s), %d skipped.\n",
|
printf( "%04x:%s:%s %d tests executed (%d marked as todo, %d as flaky, %d %s), %d skipped.\n",
|
||||||
(UINT)GetCurrentProcessId(), test->name, winetest_elapsed(),
|
(UINT)GetCurrentProcessId(), test->name, winetest_elapsed(),
|
||||||
(UINT)(successes + failures + todo_successes + todo_failures),
|
(UINT)(successes + failures + flaky_failures + todo_successes + todo_failures),
|
||||||
(UINT)todo_successes, (UINT)(failures + todo_failures),
|
(UINT)todo_successes, (UINT)flaky_failures, (UINT)(failures + todo_failures),
|
||||||
(failures + todo_failures != 1) ? "failures" : "failure",
|
(failures + todo_failures != 1) ? "failures" : "failure",
|
||||||
(UINT)skipped );
|
(UINT)skipped );
|
||||||
}
|
}
|
||||||
status = (failures + todo_failures < 255) ? failures + todo_failures : 255;
|
status = failures + todo_failures;
|
||||||
|
if (!winetest_allow_flaky) status += flaky_failures;
|
||||||
|
if (status > 255) status = 255;
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -694,6 +758,7 @@ int main( int argc, char **argv )
|
||||||
}
|
}
|
||||||
if (GetEnvironmentVariableA( "WINETEST_DEBUG", p, sizeof(p) )) winetest_debug = atoi(p);
|
if (GetEnvironmentVariableA( "WINETEST_DEBUG", p, sizeof(p) )) winetest_debug = atoi(p);
|
||||||
if (GetEnvironmentVariableA( "WINETEST_INTERACTIVE", p, sizeof(p) )) winetest_interactive = atoi(p);
|
if (GetEnvironmentVariableA( "WINETEST_INTERACTIVE", p, sizeof(p) )) winetest_interactive = atoi(p);
|
||||||
|
if (GetEnvironmentVariableA( "WINETEST_ALLOW_FLAKY", p, sizeof(p) )) winetest_allow_flaky = atoi(p);
|
||||||
if (GetEnvironmentVariableA( "WINETEST_REPORT_SUCCESS", p, sizeof(p) )) winetest_report_success = atoi(p);
|
if (GetEnvironmentVariableA( "WINETEST_REPORT_SUCCESS", p, sizeof(p) )) winetest_report_success = atoi(p);
|
||||||
if (GetEnvironmentVariableA( "WINETEST_TIME", p, sizeof(p) )) winetest_time = atoi(p);
|
if (GetEnvironmentVariableA( "WINETEST_TIME", p, sizeof(p) )) winetest_time = atoi(p);
|
||||||
winetest_last_time = winetest_start_time = GetTickCount();
|
winetest_last_time = winetest_start_time = GetTickCount();
|
||||||
|
|
Loading…
Reference in a new issue