diff --git a/contrib/atf/.cirrus.yml b/contrib/atf/.cirrus.yml new file mode 100644 index 000000000000..fd9b6e4a47df --- /dev/null +++ b/contrib/atf/.cirrus.yml @@ -0,0 +1,26 @@ +env: + CIRRUS_CLONE_DEPTH: 1 + ARCH: amd64 + +task: + matrix: + - name: 13.0-CURRENT + freebsd_instance: + image_family: freebsd-13-0-snap + - name: 12.2-STABLE + freebsd_instance: + image_family: freebsd-12-2-snap + - name: 12.1-RELEASE + freebsd_instance: + image_family: freebsd-12-1 + install_script: + - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf + - ASSUME_ALWAYS_YES=yes pkg bootstrap -f + - pkg install -y autoconf automake libtool kyua + script: + - env JUNIT_OUTPUT=$(pwd)/test-results.xml ./admin/travis-build.sh + always: + junit_artifacts: + path: "test-results.xml" + type: text/xml + format: junit diff --git a/contrib/atf/.gitignore b/contrib/atf/.gitignore new file mode 100644 index 000000000000..396785ce2052 --- /dev/null +++ b/contrib/atf/.gitignore @@ -0,0 +1,25 @@ +*.la +*.lo +*.o +*.pc +*_helper +*_helpers +*_test +.deps +.dirstamp +.libs + +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +config.h +config.h.in +config.h.in~ +config.log +config.status +configure +installcheck.log +libtool +stamp-h1 +testsuite.log diff --git a/contrib/atf/.travis.yml b/contrib/atf/.travis.yml new file mode 100644 index 000000000000..1949aae54468 --- /dev/null +++ b/contrib/atf/.travis.yml @@ -0,0 +1,25 @@ +language: cpp + +compiler: + - gcc + - clang + +before_install: + - ./admin/travis-install-deps.sh + +env: + - ARCH=amd64 AS_ROOT=no + - ARCH=amd64 AS_ROOT=yes + - ARCH=i386 AS_ROOT=no + +matrix: + exclude: + - compiler: clang + env: ARCH=i386 AS_ROOT=no + +script: + - ./admin/travis-build.sh + +notifications: + email: + - atf-log@googlegroups.com diff --git a/contrib/atf/NEWS b/contrib/atf/NEWS index f1764e0d9dda..671ee81ff6ff 100644 --- a/contrib/atf/NEWS +++ b/contrib/atf/NEWS @@ -1,6 +1,24 @@ Major changes between releases Automated Testing Framework =========================================================================== +Changes in version 0.22 +*********************** + +STILL UNDER DEVELOPMENT; NOT RELEASED YET. +DON'T FORGET TO BUMP THE -version-info PRE-RELEASE IF NECESSARY! + +* Issue #23: Fix double-free triggered by atf_map_insert in low memory + scenarios, caused by an overlook in the atf_list code. + +* Issue #29: Fixed various typos and formatting errors in manual pages. + +* Issue #31: Added require.progs metadata properties to the tests that + need a compiler to run. + +* Added the atf_check_not_equal function to atf-sh to check for + unequal values. + + Changes in version 0.21 *********************** diff --git a/contrib/atf/README.md b/contrib/atf/README.md new file mode 100644 index 000000000000..d245552f35c9 --- /dev/null +++ b/contrib/atf/README.md @@ -0,0 +1,47 @@ +# Welcome to the ATF project! + +ATF, or Automated Testing Framework, is a **collection of libraries** to +write test programs in **C, C++ and POSIX shell**. + +The ATF libraries offer a simple API. The API is orthogonal through the +various bindings, allowing developers to quickly learn how to write test +programs in different languages. + +ATF-based test programs offer a **consistent end-user command-line +interface** to allow both humans and automation to run the tests. + +ATF-based test programs **rely on an execution engine** to be run and +this execution engine is *not* shipped with ATF. +**[Kyua](https://github.com/jmmv/kyua/) is the engine of choice.** + +## Download + +Formal releases for source files are available for download from GitHub: + +* [atf 0.20](../../releases/tag/atf-0.20), released on February 7th, 2014. + +## Installation + +You are encouraged to install binary packages for your operating system +wherever available: + +* Fedora 20 and above: install the `atf` package with `yum install atf`. + +* FreeBSD 10.0 and above: install the `atf` package with `pkg install atf`. + +* NetBSD with pkgsrc: install the `pkgsrc/devel/atf` package. + +* OpenBSD: install the `devel/atf` package with `pkg_add atf`. + +Should you want to build and install ATF from the source tree provided +here, follow the instructions in the [INSTALL file](INSTALL). + +## Support + +Please use the +[atf-discuss mailing list](https://groups.google.com/forum/#!forum/atf-discuss) +for any support inquiries related to `atf-c`, `atf-c++` or `atf-sh`. + +If you have any questions on Kyua proper, please use the +[kyua-discuss mailing list](https://groups.google.com/forum/#!forum/kyua-discuss) +instead. diff --git a/contrib/atf/atf-c++/atf-c++.3 b/contrib/atf/atf-c++/atf-c++.3 index 601efaf6fd5b..fead776755af 100644 --- a/contrib/atf/atf-c++/atf-c++.3 +++ b/contrib/atf/atf-c++/atf-c++.3 @@ -403,8 +403,8 @@ in the collection. takes the name of an exception and a statement and raises a failure if the statement does not throw the specified exception. .Fn ATF_REQUIRE_THROW_RE -takes the name of an exception, a regular expresion and a statement and raises a -failure if the statement does not throw the specified exception and if the +takes the name of an exception, a regular expression and a statement, and raises +a failure if the statement does not throw the specified exception and if the message of the exception does not match the regular expression. .Pp .Fn ATF_CHECK_ERRNO diff --git a/contrib/atf/atf-c++/detail/test_helpers.hpp b/contrib/atf/atf-c++/detail/test_helpers.hpp index f166ee218a13..c1171801a3a7 100644 --- a/contrib/atf/atf-c++/detail/test_helpers.hpp +++ b/contrib/atf/atf-c++/detail/test_helpers.hpp @@ -36,6 +36,7 @@ #include +#include #include #define HEADER_TC(name, hdrname) \ @@ -44,6 +45,8 @@ { \ set_md_var("descr", "Tests that the " hdrname " file can be " \ "included on its own, without any prerequisites"); \ + const std::string cxx = atf::env::get("ATF_BUILD_CXX", ATF_BUILD_CXX); \ + set_md_var("require.progs", cxx); \ } \ ATF_TEST_CASE_BODY(name) \ { \ @@ -55,6 +58,8 @@ ATF_TEST_CASE_HEAD(name) \ { \ set_md_var("descr", descr); \ + const std::string cxx = atf::env::get("ATF_BUILD_CXX", ATF_BUILD_CXX); \ + set_md_var("require.progs", cxx); \ } \ ATF_TEST_CASE_BODY(name) \ { \ diff --git a/contrib/atf/atf-c++/tests.hpp b/contrib/atf/atf-c++/tests.hpp index ce2fb1d165c8..a03cc852dcf8 100644 --- a/contrib/atf/atf-c++/tests.hpp +++ b/contrib/atf/atf-c++/tests.hpp @@ -73,7 +73,7 @@ class tc { tc(const tc&); tc& operator=(const tc&); - std::auto_ptr< tc_impl > pimpl; + std::unique_ptr< tc_impl > pimpl; protected: virtual void head(void); diff --git a/contrib/atf/atf-c++/utils.cpp b/contrib/atf/atf-c++/utils.cpp index a6ab08f0d770..995d78c6542e 100644 --- a/contrib/atf/atf-c++/utils.cpp +++ b/contrib/atf/atf-c++/utils.cpp @@ -70,6 +70,13 @@ atf::utils::fork(void) return atf_utils_fork(); } +void +atf::utils::reset_resultsfile(void) +{ + + atf_utils_reset_resultsfile(); +} + bool atf::utils::grep_file(const std::string& regex, const std::string& path) { diff --git a/contrib/atf/atf-c++/utils.hpp b/contrib/atf/atf-c++/utils.hpp index 8f5c5e337455..34d77a126df7 100644 --- a/contrib/atf/atf-c++/utils.hpp +++ b/contrib/atf/atf-c++/utils.hpp @@ -41,6 +41,7 @@ void copy_file(const std::string&, const std::string&); void create_file(const std::string&, const std::string&); bool file_exists(const std::string&); pid_t fork(void); +void reset_resultsfile(void); bool grep_file(const std::string&, const std::string&); bool grep_string(const std::string&, const std::string&); void redirect(const int, const std::string&); diff --git a/contrib/atf/atf-c++/utils_test.cpp b/contrib/atf/atf-c++/utils_test.cpp index 34e0709f580a..93e16652bac1 100644 --- a/contrib/atf/atf-c++/utils_test.cpp +++ b/contrib/atf/atf-c++/utils_test.cpp @@ -335,6 +335,7 @@ fork_and_wait(const int exitstatus, const char* expout, const char* experr) std::cerr << "Some error\n"; exit(123); } + atf::utils::reset_resultsfile(); atf::utils::wait(pid, exitstatus, expout, experr); exit(EXIT_SUCCESS); } diff --git a/contrib/atf/atf-c/.gitignore b/contrib/atf/atf-c/.gitignore new file mode 100644 index 000000000000..e7f0fb647c32 --- /dev/null +++ b/contrib/atf/atf-c/.gitignore @@ -0,0 +1 @@ +defs.h diff --git a/contrib/atf/atf-c/check.c b/contrib/atf/atf-c/check.c index 38afdf3743a6..1aec01bcca6b 100644 --- a/contrib/atf/atf-c/check.c +++ b/contrib/atf/atf-c/check.c @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -106,7 +107,7 @@ static int const_execvp(const char *file, const char *const *argv) { -#define UNCONST(a) ((void *)(unsigned long)(const void *)(a)) +#define UNCONST(a) ((void *)(uintptr_t)(const void *)(a)) return execvp(file, UNCONST(argv)); #undef UNCONST } diff --git a/contrib/atf/atf-c/detail/fs_test.c b/contrib/atf/atf-c/detail/fs_test.c index 3dbc4d3ba7ef..7812be0334b8 100644 --- a/contrib/atf/atf-c/detail/fs_test.c +++ b/contrib/atf/atf-c/detail/fs_test.c @@ -779,7 +779,7 @@ ATF_TC_BODY(rmdir_enotempty, tc) atf_fs_path_fini(&p); } -ATF_TC(rmdir_eperm); +ATF_TC_WITH_CLEANUP(rmdir_eperm); ATF_TC_HEAD(rmdir_eperm, tc) { atf_tc_set_md_var(tc, "descr", "Tests the atf_fs_rmdir function"); @@ -808,6 +808,13 @@ ATF_TC_BODY(rmdir_eperm, tc) atf_fs_path_fini(&p); } +ATF_TC_CLEANUP(rmdir_eperm, tc) +{ + if (chmod("test-dir", 0755) == -1) { + fprintf(stderr, "Failed to unprotect test-dir; test directory " + "cleanup will fail\n"); + } +} ATF_TC(mkdtemp_ok); ATF_TC_HEAD(mkdtemp_ok, tc) diff --git a/contrib/atf/atf-c/detail/list.c b/contrib/atf/atf-c/detail/list.c index d14216eb409f..7ac9f1fc948b 100644 --- a/contrib/atf/atf-c/detail/list.c +++ b/contrib/atf/atf-c/detail/list.c @@ -74,7 +74,7 @@ new_entry(void *object, bool managed) le->m_prev = le->m_next = NULL; le->m_object = object; le->m_managed = managed; - } else + } else if (managed) free(object); return le; diff --git a/contrib/atf/atf-c/detail/process.c b/contrib/atf/atf-c/detail/process.c index 8e08b6c57466..a6189bf78e20 100644 --- a/contrib/atf/atf-c/detail/process.c +++ b/contrib/atf/atf-c/detail/process.c @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -552,7 +553,7 @@ static int const_execvp(const char *file, const char *const *argv) { -#define UNCONST(a) ((void *)(unsigned long)(const void *)(a)) +#define UNCONST(a) ((void *)(uintptr_t)(const void *)(a)) return execvp(file, UNCONST(argv)); #undef UNCONST } diff --git a/contrib/atf/atf-c/detail/test_helpers.h b/contrib/atf/atf-c/detail/test_helpers.h index a601c293ffe4..90841f803c59 100644 --- a/contrib/atf/atf-c/detail/test_helpers.h +++ b/contrib/atf/atf-c/detail/test_helpers.h @@ -33,6 +33,7 @@ #include +#include #include #include @@ -46,8 +47,11 @@ struct atf_fs_path; ATF_TC(name); \ ATF_TC_HEAD(name, tc) \ { \ + const char *cc; \ atf_tc_set_md_var(tc, "descr", "Tests that the " hdrname " file can " \ "be included on its own, without any prerequisites"); \ + cc = atf_env_get_with_default("ATF_BUILD_CC", ATF_BUILD_CC); \ + atf_tc_set_md_var(tc, "require.progs", cc); \ } \ ATF_TC_BODY(name, tc) \ { \ @@ -58,7 +62,10 @@ struct atf_fs_path; ATF_TC(name); \ ATF_TC_HEAD(name, tc) \ { \ + const char *cc; \ atf_tc_set_md_var(tc, "descr", descr); \ + cc = atf_env_get_with_default("ATF_BUILD_CC", ATF_BUILD_CC); \ + atf_tc_set_md_var(tc, "require.progs", cc); \ } \ ATF_TC_BODY(name, tc) \ { \ diff --git a/contrib/atf/atf-c/tc.c b/contrib/atf/atf-c/tc.c index 92c3e12c99b1..69b31123f3a3 100644 --- a/contrib/atf/atf-c/tc.c +++ b/contrib/atf/atf-c/tc.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -62,6 +63,7 @@ enum expect_type { struct context { const atf_tc_t *tc; const char *resfile; + int resfilefd; size_t fail_count; enum expect_type expect; @@ -73,12 +75,14 @@ struct context { }; static void context_init(struct context *, const atf_tc_t *, const char *); +static void context_set_resfile(struct context *, const char *); +static void context_close_resfile(struct context *); static void check_fatal_error(atf_error_t); static void report_fatal_error(const char *, ...) ATF_DEFS_ATTRIBUTE_NORETURN; static atf_error_t write_resfile(const int, const char *, const int, const atf_dynstr_t *); -static void create_resfile(const char *, const char *, const int, +static void create_resfile(struct context *, const char *, const int, atf_dynstr_t *); static void error_in_expect(struct context *, const char *, ...) ATF_DEFS_ATTRIBUTE_NORETURN; @@ -102,11 +106,16 @@ static void errno_test(struct context *, const char *, const size_t, static atf_error_t check_prog_in_dir(const char *, void *); static atf_error_t check_prog(struct context *, const char *); +/* No prototype in header for this one, it's a little sketchy (internal). */ +void atf_tc_set_resultsfile(const char *); + static void context_init(struct context *ctx, const atf_tc_t *tc, const char *resfile) { + ctx->tc = tc; - ctx->resfile = resfile; + ctx->resfilefd = -1; + context_set_resfile(ctx, resfile); ctx->fail_count = 0; ctx->expect = EXPECT_PASS; check_fatal_error(atf_dynstr_init(&ctx->expect_reason)); @@ -116,6 +125,41 @@ context_init(struct context *ctx, const atf_tc_t *tc, const char *resfile) ctx->expect_signo = 0; } +static void +context_set_resfile(struct context *ctx, const char *resfile) +{ + atf_error_t err; + + context_close_resfile(ctx); + ctx->resfile = resfile; + if (strcmp(resfile, "/dev/stdout") == 0) + ctx->resfilefd = STDOUT_FILENO; + else if (strcmp(resfile, "/dev/stderr") == 0) + ctx->resfilefd = STDERR_FILENO; + else + ctx->resfilefd = open(resfile, O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (ctx->resfilefd == -1) { + err = atf_libc_error(errno, + "Cannot create results file '%s'", resfile); + check_fatal_error(err); + } + + ctx->resfile = resfile; +} + +static void +context_close_resfile(struct context *ctx) +{ + + if (ctx->resfilefd == -1) + return; + if (ctx->resfilefd != STDOUT_FILENO && ctx->resfilefd != STDERR_FILENO) + close(ctx->resfilefd); + ctx->resfilefd = -1; + ctx->resfile = NULL; +} + static void check_fatal_error(atf_error_t err) { @@ -162,7 +206,7 @@ write_resfile(const int fd, const char *result, const int arg, INV(arg == -1 || reason != NULL); -#define UNCONST(a) ((void *)(unsigned long)(const void *)(a)) +#define UNCONST(a) ((void *)(uintptr_t)(const void *)(a)) iov[count].iov_base = UNCONST(result); iov[count++].iov_len = strlen(result); @@ -202,26 +246,23 @@ write_resfile(const int fd, const char *result, const int arg, * not return any error code. */ static void -create_resfile(const char *resfile, const char *result, const int arg, +create_resfile(struct context *ctx, const char *result, const int arg, atf_dynstr_t *reason) { atf_error_t err; - if (strcmp("/dev/stdout", resfile) == 0) { - err = write_resfile(STDOUT_FILENO, result, arg, reason); - } else if (strcmp("/dev/stderr", resfile) == 0) { - err = write_resfile(STDERR_FILENO, result, arg, reason); - } else { - const int fd = open(resfile, O_WRONLY | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); - if (fd == -1) { - err = atf_libc_error(errno, "Cannot create results file '%s'", - resfile); - } else { - err = write_resfile(fd, result, arg, reason); - close(fd); - } - } + /* + * We'll attempt to truncate the results file, but only if it's not pointed + * at stdout/stderr. We could just blindly ftruncate() here, but it may + * be that stdout/stderr have been redirected to a file that we want to + * validate expectations on, for example. Kyua will want the truncation, + * but it will also redirect the results directly to some file and we'll + * have no issue here. + */ + if (ctx->resfilefd != STDOUT_FILENO && ctx->resfilefd != STDERR_FILENO && + ftruncate(ctx->resfilefd, 0) != -1) + lseek(ctx->resfilefd, 0, SEEK_SET); + err = write_resfile(ctx->resfilefd, result, arg, reason); if (reason != NULL) atf_dynstr_fini(reason); @@ -280,7 +321,8 @@ expected_failure(struct context *ctx, atf_dynstr_t *reason) { check_fatal_error(atf_dynstr_prepend_fmt(reason, "%s: ", atf_dynstr_cstring(&ctx->expect_reason))); - create_resfile(ctx->resfile, "expected_failure", -1, reason); + create_resfile(ctx, "expected_failure", -1, reason); + context_close_resfile(ctx); exit(EXIT_SUCCESS); } @@ -290,7 +332,8 @@ fail_requirement(struct context *ctx, atf_dynstr_t *reason) if (ctx->expect == EXPECT_FAIL) { expected_failure(ctx, reason); } else if (ctx->expect == EXPECT_PASS) { - create_resfile(ctx->resfile, "failed", -1, reason); + create_resfile(ctx, "failed", -1, reason); + context_close_resfile(ctx); exit(EXIT_FAILURE); } else { error_in_expect(ctx, "Test case raised a failure but was not " @@ -325,7 +368,8 @@ pass(struct context *ctx) error_in_expect(ctx, "Test case was expecting a failure but got " "a pass instead"); } else if (ctx->expect == EXPECT_PASS) { - create_resfile(ctx->resfile, "passed", -1, NULL); + create_resfile(ctx, "passed", -1, NULL); + context_close_resfile(ctx); exit(EXIT_SUCCESS); } else { error_in_expect(ctx, "Test case asked to explicitly pass but was " @@ -338,7 +382,8 @@ static void skip(struct context *ctx, atf_dynstr_t *reason) { if (ctx->expect == EXPECT_PASS) { - create_resfile(ctx->resfile, "skipped", -1, reason); + create_resfile(ctx, "skipped", -1, reason); + context_close_resfile(ctx); exit(EXIT_SUCCESS); } else { error_in_expect(ctx, "Can only skip a test case when running in " @@ -942,7 +987,7 @@ _atf_tc_expect_exit(struct context *ctx, const int exitcode, const char *reason, check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2)); va_end(ap2); - create_resfile(ctx->resfile, "expected_exit", exitcode, &formatted); + create_resfile(ctx, "expected_exit", exitcode, &formatted); } static void @@ -959,7 +1004,7 @@ _atf_tc_expect_signal(struct context *ctx, const int signo, const char *reason, check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2)); va_end(ap2); - create_resfile(ctx->resfile, "expected_signal", signo, &formatted); + create_resfile(ctx, "expected_signal", signo, &formatted); } static void @@ -975,7 +1020,7 @@ _atf_tc_expect_death(struct context *ctx, const char *reason, va_list ap) check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2)); va_end(ap2); - create_resfile(ctx->resfile, "expected_death", -1, &formatted); + create_resfile(ctx, "expected_death", -1, &formatted); } static void @@ -991,7 +1036,14 @@ _atf_tc_expect_timeout(struct context *ctx, const char *reason, va_list ap) check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2)); va_end(ap2); - create_resfile(ctx->resfile, "expected_timeout", -1, &formatted); + create_resfile(ctx, "expected_timeout", -1, &formatted); +} + +static void +_atf_tc_set_resultsfile(struct context *ctx, const char *file) +{ + + context_set_resfile(ctx, file); } /* --------------------------------------------------------------------- @@ -1215,3 +1267,13 @@ atf_tc_expect_timeout(const char *reason, ...) _atf_tc_expect_timeout(&Current, reason, ap); va_end(ap); } + +/* Internal! */ +void +atf_tc_set_resultsfile(const char *file) +{ + + PRE(Current.tc != NULL); + + _atf_tc_set_resultsfile(&Current, file); +} diff --git a/contrib/atf/atf-c/utils.c b/contrib/atf/atf-c/utils.c index 1e2aac1ed3b6..d8355bc68936 100644 --- a/contrib/atf/atf-c/utils.c +++ b/contrib/atf/atf-c/utils.c @@ -41,6 +41,9 @@ #include "atf-c/detail/dynstr.h" +/* No prototype in header for this one, it's a little sketchy (internal). */ +void atf_tc_set_resultsfile(const char *); + /** Allocate a filename to be used by atf_utils_{fork,wait}. * * In case of a failure, marks the calling test as failed when in_parent is @@ -271,6 +274,13 @@ atf_utils_fork(void) return pid; } +void +atf_utils_reset_resultsfile(void) +{ + + atf_tc_set_resultsfile("/dev/null"); +} + /** Frees an dynamically-allocated "argv" array. * * \param argv A dynamically-allocated array of dynamically-allocated diff --git a/contrib/atf/atf-c/utils.h b/contrib/atf/atf-c/utils.h index e4162b215fe5..422186a31e76 100644 --- a/contrib/atf/atf-c/utils.h +++ b/contrib/atf/atf-c/utils.h @@ -46,5 +46,6 @@ bool atf_utils_grep_string(const char *, const char *, ...) char *atf_utils_readline(int); void atf_utils_redirect(const int, const char *); void atf_utils_wait(const pid_t, const int, const char *, const char *); +void atf_utils_reset_resultsfile(void); #endif /* !defined(ATF_C_UTILS_H) */ diff --git a/contrib/atf/atf-c/utils_test.c b/contrib/atf/atf-c/utils_test.c index fb81cd3a1d9e..9d8f69683b9a 100644 --- a/contrib/atf/atf-c/utils_test.c +++ b/contrib/atf/atf-c/utils_test.c @@ -395,6 +395,7 @@ fork_and_wait(const int exitstatus, const char* expout, const char* experr) fprintf(stderr, "Some error\n"); exit(123); } + atf_utils_reset_resultsfile(); atf_utils_wait(pid, exitstatus, expout, experr); exit(EXIT_SUCCESS); } diff --git a/contrib/atf/atf-sh/.gitignore b/contrib/atf/atf-sh/.gitignore new file mode 100644 index 000000000000..a29438f1a9b8 --- /dev/null +++ b/contrib/atf/atf-sh/.gitignore @@ -0,0 +1,2 @@ +atf-check +atf-sh diff --git a/contrib/atf/atf-sh/atf-check.1 b/contrib/atf/atf-sh/atf-check.1 index a423e3ac3b1c..b03058e8442c 100644 --- a/contrib/atf/atf-sh/atf-check.1 +++ b/contrib/atf/atf-sh/atf-check.1 @@ -22,7 +22,7 @@ .\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN .\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Dd March 6, 2017 +.Dd June 21, 2020 .Dt ATF-CHECK 1 .Os .Sh NAME @@ -120,6 +120,14 @@ as a shell command line, executing it with the system shell defined by .Va ATF_SHELL . You should avoid using this flag if at all possible to prevent shell quoting issues. +.It Fl r Ar timeout[:interval] +Repeats failed checks until the +.Ar timeout +(in seconds) expires. +If unspecified, the default +.Ar interval +(in milliseconds) is 50 ms. +This can be used to wait for an expected update to the contents of a file. .El .Sh ENVIRONMENT .Bl -tag -width ATFXSHELLXX -compact @@ -157,6 +165,11 @@ atf_check -s signal:sigsegv my_program # Combined checks atf_check -o match:foo -o not-match:bar echo foo baz + +# Wait 5 seconds for a line to show up in a file +( sleep 2 ; echo "testing 123" > $test_path ) & +atf-check -o ignore -e ignore -s exit:0 -r 5 \e + grep "testing 123" $test_path .Ed .Sh SEE ALSO .Xr atf-sh 1 diff --git a/contrib/atf/atf-sh/atf-check.cpp b/contrib/atf/atf-sh/atf-check.cpp index 414b64ea91f0..38ab527aab54 100644 --- a/contrib/atf/atf-sh/atf-check.cpp +++ b/contrib/atf/atf-sh/atf-check.cpp @@ -29,6 +29,7 @@ extern "C" { #include #include +#include #include } @@ -53,6 +54,10 @@ extern "C" { #include "atf-c++/detail/sanity.hpp" #include "atf-c++/detail/text.hpp" +static const useconds_t seconds_in_useconds = (1000 * 1000); +static const useconds_t mseconds_in_useconds = 1000; +static const useconds_t useconds_in_nseconds = 1000; + // ------------------------------------------------------------------------ // Auxiliary functions. // ------------------------------------------------------------------------ @@ -162,6 +167,33 @@ class temp_file : public std::ostream { } // anonymous namespace +static useconds_t +get_monotonic_useconds(void) +{ + struct timespec ts; + useconds_t res; + int rc; + + rc = clock_gettime(CLOCK_MONOTONIC, &ts); + if (rc != 0) + throw std::runtime_error("clock_gettime: " + + std::string(strerror(errno))); + + res = ts.tv_sec * seconds_in_useconds; + res += ts.tv_nsec / useconds_in_nseconds; + return res; +} + +static bool +timo_expired(useconds_t timeout) +{ + + if (get_monotonic_useconds() >= timeout) + return true; + return false; +} + + static int parse_exit_code(const std::string& str) { @@ -216,7 +248,7 @@ parse_signal(const std::string& str) if (signo == INT_MIN) { try { return atf::text::to_type< int >(str); - } catch (std::runtime_error) { + } catch (const std::runtime_error&) { throw atf::application::usage_error("Invalid signal name or number " "in -s option"); } @@ -306,6 +338,62 @@ parse_output_check_arg(const std::string& arg) return output_check(type, negated, arg.substr(delimiter + 1)); } +static void +parse_repeat_check_arg(const std::string& arg, useconds_t *m_timo, + useconds_t *m_interval) +{ + const std::string::size_type delimiter = arg.find(':'); + const bool has_interval = (delimiter != std::string::npos); + const std::string timo_str = arg.substr(0, delimiter); + + long l; + char *end; + + // There is no reason this couldn't be a non-integer number of seconds, + // this was just easy to do for now. + errno = 0; + l = strtol(timo_str.c_str(), &end, 10); + if (errno == ERANGE) + throw atf::application::usage_error("Bogus timeout in seconds"); + else if (errno != 0) + throw atf::application::usage_error("Timeout must be a number"); + + if (*end != 0) + throw atf::application::usage_error("Timeout must be a number"); + + *m_timo = get_monotonic_useconds() + (l * seconds_in_useconds); + // 50 milliseconds is chosen arbitrarily. There is a tradeoff between + // longer and shorter poll times. A shorter poll time makes for faster + // tests. A longer poll time makes for lower CPU overhead for the polled + // operation. 50ms is chosen with these tradeoffs in mind: on + // microcontrollers, the hope is that we can still avoid meaningful CPU use + // with a small test every 50ms. And on typical fast x86 hardware, our + // tests can be much more precise with time wasted than they typically are + // without this feature. + *m_interval = 50 * mseconds_in_useconds; + + if (!has_interval) + return; + + const std::string intv_str = arg.substr(delimiter + 1, std::string::npos); + + // Same -- this could be non-integer milliseconds. + errno = 0; + l = strtol(intv_str.c_str(), &end, 10); + if (errno == ERANGE) + throw atf::application::usage_error( + "Bogus repeat interval in milliseconds"); + else if (errno != 0) + throw atf::application::usage_error( + "Repeat interval must be a number"); + + if (*end != 0) + throw atf::application::usage_error( + "Repeat interval must be a number"); + + *m_interval = l * mseconds_in_useconds; +} + static std::string flatten_argv(char* const* argv) @@ -694,8 +782,12 @@ run_output_checks(const std::vector< output_check >& checks, namespace { class atf_check : public atf::application::app { + bool m_rflag; bool m_xflag; + useconds_t m_timo; + useconds_t m_interval; + std::vector< status_check > m_status_checks; std::vector< output_check > m_stdout_checks; std::vector< output_check > m_stderr_checks; @@ -722,6 +814,7 @@ const char* atf_check::m_description = atf_check::atf_check(void) : app(m_description, "atf-check(1)"), + m_rflag(false), m_xflag(false) { } @@ -765,6 +858,8 @@ atf_check::specific_options(void) opts.insert(option('e', "action:arg", "Handle stderr. Action must be " "one of: empty ignore file: inline: match:regexp " "save:")); + opts.insert(option('r', "timeout[:interval]", "Repeat failed check until " + "the timeout expires.")); opts.insert(option('x', "", "Execute command as a shell command")); return opts; @@ -786,6 +881,11 @@ atf_check::process_option(int ch, const char* arg) m_stderr_checks.push_back(parse_output_check_arg(arg)); break; + case 'r': + m_rflag = true; + parse_repeat_check_arg(arg, &m_timo, &m_interval); + break; + case 'x': m_xflag = true; break; @@ -803,9 +903,6 @@ atf_check::main(void) int status = EXIT_FAILURE; - std::auto_ptr< atf::check::check_result > r = - m_xflag ? execute_with_shell(m_argv) : execute(m_argv); - if (m_status_checks.empty()) m_status_checks.push_back(status_check(sc_exit, false, EXIT_SUCCESS)); else if (m_status_checks.size() > 1) { @@ -818,12 +915,23 @@ atf_check::main(void) if (m_stderr_checks.empty()) m_stderr_checks.push_back(output_check(oc_empty, false, "")); - if ((run_status_checks(m_status_checks, *r) == false) || - (run_output_checks(*r, "stderr") == false) || - (run_output_checks(*r, "stdout") == false)) - status = EXIT_FAILURE; - else - status = EXIT_SUCCESS; + do { + std::auto_ptr< atf::check::check_result > r = + m_xflag ? execute_with_shell(m_argv) : execute(m_argv); + + if ((run_status_checks(m_status_checks, *r) == false) || + (run_output_checks(*r, "stderr") == false) || + (run_output_checks(*r, "stdout") == false)) + status = EXIT_FAILURE; + else + status = EXIT_SUCCESS; + + if (m_rflag && status == EXIT_FAILURE) { + if (timo_expired(m_timo)) + break; + usleep(m_interval); + } + } while (m_rflag && status == EXIT_FAILURE); return status; } diff --git a/contrib/atf/atf-sh/atf-sh.3 b/contrib/atf/atf-sh/atf-sh.3 index 974c2aee1009..5d1119b2b5dc 100644 --- a/contrib/atf/atf-sh/atf-sh.3 +++ b/contrib/atf/atf-sh/atf-sh.3 @@ -22,13 +22,14 @@ .\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN .\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Dd March 6, 2017 +.Dd June 08, 2017 .Dt ATF-SH 3 .Os .Sh NAME .Nm atf_add_test_case , .Nm atf_check , .Nm atf_check_equal , +.Nm atf_check_not_equal , .Nm atf_config_get , .Nm atf_config_has , .Nm atf_expect_death , @@ -55,6 +56,9 @@ .Nm atf_check_equal .Qq expected_expression .Qq actual_expression +.Nm atf_check_not_equal +.Qq expected_expression +.Qq actual_expression .Nm atf_config_get .Qq var_name .Nm atf_config_has @@ -310,6 +314,11 @@ This function takes two expressions, evaluates them and, if their results differ, aborts the test case with an appropriate failure message. The common style is to put the expected value in the first parameter and the actual value in the second parameter. +.It Nm atf_check_not_equal Qo expected_expression Qc Qo actual_expression Qc +This function takes two expressions, evaluates them and, if their +results are equal, aborts the test case with an appropriate failure message. +The common style is to put the expected value in the first parameter and the +actual value in the second parameter. .El .Sh EXAMPLES The following shows a complete test program with a single test case that diff --git a/contrib/atf/atf-sh/atf_check_test.sh b/contrib/atf/atf-sh/atf_check_test.sh index 9e3cfb955f68..6fe2bb775274 100644 --- a/contrib/atf/atf-sh/atf_check_test.sh +++ b/contrib/atf/atf-sh/atf_check_test.sh @@ -164,6 +164,30 @@ equal_body() grep '^failed: \${x} != \${y} (a != b)$' resfile } +atf_test_case not_equal +not_equal_head() +{ + atf_set "descr" "Verifies that atf_check_not_equal works" +} +not_equal_body() +{ + h="$(atf_get_srcdir)/misc_helpers -s $(atf_get_srcdir)" + + atf_check -s eq:0 -o ignore -e ignore -x "${h} atf_check_not_equal_ok" + + atf_check -s eq:1 -o ignore -e ignore -x \ + "${h} -r resfile atf_check_not_equal_fail" + atf_check -s eq:0 -o ignore -e empty grep '^failed: a == b (a == b)$' \ + resfile + + atf_check -s eq:0 -o ignore -e ignore -x "${h} atf_check_not_equal_eval_ok" + + atf_check -s eq:1 -o ignore -e ignore -x \ + "${h} -r resfile atf_check_not_equal_eval_fail" + atf_check -s eq:0 -o ignore -e empty \ + grep '^failed: \${x} == \${y} (a == b)$' resfile +} + atf_test_case flush_stdout_on_death flush_stdout_on_death_body() { diff --git a/contrib/atf/atf-sh/libatf-sh.subr b/contrib/atf/atf-sh/libatf-sh.subr index a078975400af..a2478b6a9be1 100644 --- a/contrib/atf/atf-sh/libatf-sh.subr +++ b/contrib/atf/atf-sh/libatf-sh.subr @@ -100,6 +100,23 @@ atf_check_equal() atf_fail "${1} != ${2} (${_val1} != ${_val2})" } +# +# atf_check_not_equal expected_expression actual_expression +# +# Checks that expected_expression's value does not match actual_expression's +# and, if it does, raises an error. Ideally expected_expression and +# actual_expression should be provided quoted (not expanded) so that +# the error message is helpful; otherwise it will only show the values, +# not the expressions themselves. +# +atf_check_not_equal() +{ + eval _val1=\"${1}\" + eval _val2=\"${2}\" + test "${_val1}" != "${_val2}" || \ + atf_fail "${1} == ${2} (${_val1} == ${_val2})" +} + # # atf_config_get varname [defvalue] # @@ -536,7 +553,18 @@ _atf_list_tcs() # _atf_normalize() { - echo ${1} | tr .- __ + # Check if the string contains any of the forbidden characters using + # POSIX parameter expansion (the ${var//} string substitution is + # unfortunately not supported in POSIX sh) and only use tr(1) then. + # tr(1) is generally not a builtin, so doing the substring check first + # avoids unnecessary fork()+execve() calls. As this function is called + # many times in each test script startup, those overheads add up + # (especially when running on emulated platforms such as QEMU). + if [ "${1#*[.-]}" != "$1" ]; then + echo "$1" | tr .- __ + else + echo "$1" + fi } # @@ -734,7 +762,7 @@ main() ;; esac done - shift `expr ${OPTIND} - 1` + shift $((OPTIND - 1)) case ${Source_Dir} in /*) diff --git a/contrib/atf/atf-sh/misc_helpers.sh b/contrib/atf/atf-sh/misc_helpers.sh index ca0f4650d99b..a2b2c0b53d73 100644 --- a/contrib/atf/atf-sh/misc_helpers.sh +++ b/contrib/atf/atf-sh/misc_helpers.sh @@ -139,6 +139,50 @@ atf_check_equal_eval_fail_body() atf_check_equal '${x}' '${y}' } +atf_test_case atf_check_not_equal_ok +atf_check_not_equal_ok_head() +{ + atf_set "descr" "Helper test case for the t_atf_check test program" +} +atf_check_not_equal_ok_body() +{ + atf_check_not_equal a b +} + +atf_test_case atf_check_not_equal_fail +atf_check_not_equal_fail_head() +{ + atf_set "descr" "Helper test case for the t_atf_check test program" +} +atf_check_not_equal_fail_body() +{ + atf_check_not_equal a a +} + +atf_test_case atf_check_not_equal_eval_ok +atf_check_not_equal_eval_ok_head() +{ + atf_set "descr" "Helper test case for the t_atf_check test program" +} +atf_check_not_equal_eval_ok_body() +{ + x=a + y=b + atf_check_not_equal '${x}' '${y}' +} + +atf_test_case atf_check_not_equal_eval_fail +atf_check_not_equal_eval_fail_head() +{ + atf_set "descr" "Helper test case for the t_atf_check test program" +} +atf_check_not_equal_eval_fail_body() +{ + x=a + y=a + atf_check_not_equal '${x}' '${y}' +} + atf_test_case atf_check_flush_stdout atf_check_flush_stdout_head() { @@ -285,6 +329,10 @@ atf_init_test_cases() atf_add_test_case atf_check_equal_fail atf_add_test_case atf_check_equal_eval_ok atf_add_test_case atf_check_equal_eval_fail + atf_add_test_case atf_check_not_equal_ok + atf_add_test_case atf_check_not_equal_fail + atf_add_test_case atf_check_not_equal_eval_ok + atf_add_test_case atf_check_not_equal_eval_fail atf_add_test_case atf_check_flush_stdout # Add helper tests for t_config. diff --git a/contrib/atf/doc/.gitignore b/contrib/atf/doc/.gitignore new file mode 100644 index 000000000000..7c3185645015 --- /dev/null +++ b/contrib/atf/doc/.gitignore @@ -0,0 +1 @@ +atf.7 diff --git a/contrib/atf/doc/atf-test-case.4 b/contrib/atf/doc/atf-test-case.4 index 34f5e1bce1e9..46690bdcb0ef 100644 --- a/contrib/atf/doc/atf-test-case.4 +++ b/contrib/atf/doc/atf-test-case.4 @@ -149,11 +149,7 @@ APIs to implement the test cases. The standard input of the test cases is unconditionally connected to .Sq /dev/zero . .Ss Meta-data -The following list describes all meta-data properties interpreted -internally by ATF. -You are free to define new properties in your test cases and use them as -you wish, but non-standard properties must be prefixed by -.Sq X- . +The following metadata properties can be exposed via the test case's head: .Bl -tag -width requireXmachineXX .It descr Type: textual. @@ -275,6 +271,17 @@ test program. Can optionally be set to zero, in which case the test case has no run-time limit. This is discouraged. +.It X- Ns Sq NAME +Type: textual. +Optional. +.Pp +A user-defined property named +.Sq NAME . +These properties are free form, have no special meaning within ATF, and can +be specified at will by the test case. +The runtime engine should propagate these properties from the test case to +the end user so that the end user can rely on custom properties for test case +tagging and classification. .El .Ss Environment Every time a test case is executed, several environment variables are diff --git a/lib/atf/libatf-c++/tests/Makefile b/lib/atf/libatf-c++/tests/Makefile index 851be7af97c4..d045b0526ed6 100644 --- a/lib/atf/libatf-c++/tests/Makefile +++ b/lib/atf/libatf-c++/tests/Makefile @@ -10,6 +10,8 @@ ATF= ${SRCTOP}/contrib/atf .PATH: ${ATF}/atf-c++ .PATH: ${ATF}/atf-c++/detail +CFLAGS+= -DATF_BUILD_CXX='"c++"' +CFLAGS+= -DATF_BUILD_CXXFLAGS='"-Wall"' CFLAGS+= -DATF_C_TESTS_BASE='"${TESTSBASE}/lib/atf/libatf-c"' CFLAGS+= -DATF_INCLUDEDIR='"${INCLUDEDIR}"' CFLAGS+= -I${ATF}