Import atf 0.22 snapshot ca73d08c3fc1ecffc1f1c97458c31ab82c12bb01

This includes improvements to the atf-sh helper functions that
significantly reduce the number of spawned processes for each test
and therefore speeds up running the testsuite noticeably.

contrib/atf/.cirrus.yml

@@ -0,0 +1,26 @@
+env:
+  CIRRUS_CLONE_DEPTH: 1
+  ARCH: amd64
+
+task:
+  matrix:
+    - name: 13.0-CURRENT
+      freebsd_instance:
+        image_family: freebsd-13-0-snap
+    - name: 12.2-STABLE
+      freebsd_instance:
+        image_family: freebsd-12-2-snap
+    - name: 12.1-RELEASE
+      freebsd_instance:
+        image_family: freebsd-12-1
+  install_script:
+    - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf
+    - ASSUME_ALWAYS_YES=yes pkg bootstrap -f
+    - pkg install -y autoconf automake libtool kyua
+  script:
+    - env JUNIT_OUTPUT=$(pwd)/test-results.xml ./admin/travis-build.sh
+  always:
+    junit_artifacts:
+      path: "test-results.xml"
+      type: text/xml
+      format: junit

contrib/atf/.gitignore

@@ -0,0 +1,25 @@
+*.la
+*.lo
+*.o
+*.pc
+*_helper
+*_helpers
+*_test
+.deps
+.dirstamp
+.libs
+
+Makefile
+Makefile.in
+aclocal.m4
+autom4te.cache
+config.h
+config.h.in
+config.h.in~
+config.log
+config.status
+configure
+installcheck.log
+libtool
+stamp-h1
+testsuite.log

contrib/atf/.travis.yml

@@ -0,0 +1,25 @@
+language: cpp
+
+compiler:
+    - gcc
+    - clang
+
+before_install:
+    - ./admin/travis-install-deps.sh
+
+env:
+    - ARCH=amd64 AS_ROOT=no
+    - ARCH=amd64 AS_ROOT=yes
+    - ARCH=i386 AS_ROOT=no
+
+matrix:
+    exclude:
+        - compiler: clang
+          env: ARCH=i386 AS_ROOT=no
+
+script:
+    - ./admin/travis-build.sh
+
+notifications:
+    email:
+        - atf-log@googlegroups.com

contrib/atf/NEWS

@@ -1,6 +1,24 @@
 Major changes between releases                  Automated Testing Framework
 ===========================================================================
 
+Changes in version 0.22
+***********************
+
+STILL UNDER DEVELOPMENT; NOT RELEASED YET.
+DON'T FORGET TO BUMP THE -version-info PRE-RELEASE IF NECESSARY!
+
+* Issue #23: Fix double-free triggered by atf_map_insert in low memory
+  scenarios, caused by an overlook in the atf_list code.
+
+* Issue #29: Fixed various typos and formatting errors in manual pages.
+
+* Issue #31: Added require.progs metadata properties to the tests that
+  need a compiler to run.
+
+* Added the atf_check_not_equal function to atf-sh to check for
+  unequal values.
+
+
 Changes in version 0.21
 ***********************
 

contrib/atf/README.md

@@ -0,0 +1,47 @@
+# Welcome to the ATF project!
+
+ATF, or Automated Testing Framework, is a **collection of libraries** to
+write test programs in **C, C++ and POSIX shell**.
+
+The ATF libraries offer a simple API.  The API is orthogonal through the
+various bindings, allowing developers to quickly learn how to write test
+programs in different languages.
+
+ATF-based test programs offer a **consistent end-user command-line
+interface** to allow both humans and automation to run the tests.
+
+ATF-based test programs **rely on an execution engine** to be run and
+this execution engine is *not* shipped with ATF.
+**[Kyua](https://github.com/jmmv/kyua/) is the engine of choice.**
+
+## Download
+
+Formal releases for source files are available for download from GitHub:
+
+* [atf 0.20](../../releases/tag/atf-0.20), released on February 7th, 2014.
+
+## Installation
+
+You are encouraged to install binary packages for your operating system
+wherever available:
+
+* Fedora 20 and above: install the `atf` package with `yum install atf`.
+
+* FreeBSD 10.0 and above: install the `atf` package with `pkg install atf`.
+
+* NetBSD with pkgsrc: install the `pkgsrc/devel/atf` package.
+
+* OpenBSD: install the `devel/atf` package with `pkg_add atf`.
+
+Should you want to build and install ATF from the source tree provided
+here, follow the instructions in the [INSTALL file](INSTALL).
+
+## Support
+
+Please use the
+[atf-discuss mailing list](https://groups.google.com/forum/#!forum/atf-discuss)
+for any support inquiries related to `atf-c`, `atf-c++` or `atf-sh`.
+
+If you have any questions on Kyua proper, please use the
+[kyua-discuss mailing list](https://groups.google.com/forum/#!forum/kyua-discuss)
+instead.

contrib/atf/atf-c++/atf-c++.3

@@ -403,8 +403,8 @@ in the collection.
 takes the name of an exception and a statement and raises a failure if
 the statement does not throw the specified exception.
 .Fn ATF_REQUIRE_THROW_RE
-takes the name of an exception, a regular expresion and a statement and raises a
-failure if the statement does not throw the specified exception and if the
+takes the name of an exception, a regular expression and a statement, and raises
+a failure if the statement does not throw the specified exception and if the
 message of the exception does not match the regular expression.
 .Pp
 .Fn ATF_CHECK_ERRNO

contrib/atf/atf-c++/detail/test_helpers.hpp

@@ -36,6 +36,7 @@
 
 #include <atf-c++.hpp>
 
+#include <atf-c++/detail/env.hpp>
 #include <atf-c++/detail/process.hpp>
 
 #define HEADER_TC(name, hdrname) \
@@ -44,6 +45,8 @@
     { \
         set_md_var("descr", "Tests that the " hdrname " file can be " \
             "included on its own, without any prerequisites"); \
+        const std::string cxx = atf::env::get("ATF_BUILD_CXX", ATF_BUILD_CXX); \
+        set_md_var("require.progs", cxx); \
     } \
     ATF_TEST_CASE_BODY(name) \
     { \
@@ -55,6 +58,8 @@
     ATF_TEST_CASE_HEAD(name) \
     { \
         set_md_var("descr", descr); \
+        const std::string cxx = atf::env::get("ATF_BUILD_CXX", ATF_BUILD_CXX); \
+        set_md_var("require.progs", cxx); \
     } \
     ATF_TEST_CASE_BODY(name) \
     { \

contrib/atf/atf-c++/tests.hpp

@@ -73,7 +73,7 @@ class tc {
     tc(const tc&);
     tc& operator=(const tc&);
 
-    std::auto_ptr< tc_impl > pimpl;
+    std::unique_ptr< tc_impl > pimpl;
 
 protected:
     virtual void head(void);

contrib/atf/atf-c++/utils.cpp

@@ -70,6 +70,13 @@ atf::utils::fork(void)
     return atf_utils_fork();
 }
 
+void
+atf::utils::reset_resultsfile(void)
+{
+
+    atf_utils_reset_resultsfile();
+}
+
 bool
 atf::utils::grep_file(const std::string& regex, const std::string& path)
 {

contrib/atf/atf-c++/utils.hpp

@@ -41,6 +41,7 @@ void copy_file(const std::string&, const std::string&);
 void create_file(const std::string&, const std::string&);
 bool file_exists(const std::string&);
 pid_t fork(void);
+void reset_resultsfile(void);
 bool grep_file(const std::string&, const std::string&);
 bool grep_string(const std::string&, const std::string&);
 void redirect(const int, const std::string&);

contrib/atf/atf-c++/utils_test.cpp

@@ -335,6 +335,7 @@ fork_and_wait(const int exitstatus, const char* expout, const char* experr)
         std::cerr << "Some error\n";
         exit(123);
     }
+    atf::utils::reset_resultsfile();
     atf::utils::wait(pid, exitstatus, expout, experr);
     exit(EXIT_SUCCESS);
 }

contrib/atf/atf-c/.gitignore

@@ -0,0 +1 @@
+defs.h

contrib/atf/atf-c/check.c

@@ -29,6 +29,7 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -106,7 +107,7 @@ static
 int
 const_execvp(const char *file, const char *const *argv)
 {
-#define UNCONST(a) ((void *)(unsigned long)(const void *)(a))
+#define UNCONST(a) ((void *)(uintptr_t)(const void *)(a))
     return execvp(file, UNCONST(argv));
 #undef UNCONST
 }

contrib/atf/atf-c/detail/fs_test.c

@@ -779,7 +779,7 @@ ATF_TC_BODY(rmdir_enotempty, tc)
     atf_fs_path_fini(&p);
 }
 
-ATF_TC(rmdir_eperm);
+ATF_TC_WITH_CLEANUP(rmdir_eperm);
 ATF_TC_HEAD(rmdir_eperm, tc)
 {
     atf_tc_set_md_var(tc, "descr", "Tests the atf_fs_rmdir function");
@@ -808,6 +808,13 @@ ATF_TC_BODY(rmdir_eperm, tc)
 
     atf_fs_path_fini(&p);
 }
+ATF_TC_CLEANUP(rmdir_eperm, tc)
+{
+    if (chmod("test-dir", 0755) == -1) {
+        fprintf(stderr, "Failed to unprotect test-dir; test directory "
+                "cleanup will fail\n");
+    }
+}
 
 ATF_TC(mkdtemp_ok);
 ATF_TC_HEAD(mkdtemp_ok, tc)

contrib/atf/atf-c/detail/list.c

@@ -74,7 +74,7 @@ new_entry(void *object, bool managed)
         le->m_prev = le->m_next = NULL;
         le->m_object = object;
         le->m_managed = managed;
-    } else
+    } else if (managed)
         free(object);
 
     return le;

contrib/atf/atf-c/detail/process.c

@@ -30,6 +30,7 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -552,7 +553,7 @@ static
 int
 const_execvp(const char *file, const char *const *argv)
 {
-#define UNCONST(a) ((void *)(unsigned long)(const void *)(a))
+#define UNCONST(a) ((void *)(uintptr_t)(const void *)(a))
     return execvp(file, UNCONST(argv));
 #undef UNCONST
 }

contrib/atf/atf-c/detail/test_helpers.h

@@ -33,6 +33,7 @@
 
 #include <atf-c.h>
 
+#include <atf-c/detail/env.h>
 #include <atf-c/error_fwd.h>
 #include <atf-c/tc.h>
 
@@ -46,8 +47,11 @@ struct atf_fs_path;
     ATF_TC(name); \
     ATF_TC_HEAD(name, tc) \
     { \
+        const char *cc; \
         atf_tc_set_md_var(tc, "descr", "Tests that the " hdrname " file can " \
             "be included on its own, without any prerequisites"); \
+        cc = atf_env_get_with_default("ATF_BUILD_CC", ATF_BUILD_CC); \
+        atf_tc_set_md_var(tc, "require.progs", cc); \
     } \
     ATF_TC_BODY(name, tc) \
     { \
@@ -58,7 +62,10 @@ struct atf_fs_path;
     ATF_TC(name); \
     ATF_TC_HEAD(name, tc) \
     { \
+        const char *cc; \
         atf_tc_set_md_var(tc, "descr", descr); \
+        cc = atf_env_get_with_default("ATF_BUILD_CC", ATF_BUILD_CC); \
+        atf_tc_set_md_var(tc, "require.progs", cc); \
     } \
     ATF_TC_BODY(name, tc) \
     { \

contrib/atf/atf-c/tc.c

@@ -33,6 +33,7 @@
 #include <fcntl.h>
 #include <stdarg.h>
 #include <stdbool.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -62,6 +63,7 @@ enum expect_type {
 struct context {
     const atf_tc_t *tc;
     const char *resfile;
+    int resfilefd;
     size_t fail_count;
 
     enum expect_type expect;
@@ -73,12 +75,14 @@ struct context {
 };
 
 static void context_init(struct context *, const atf_tc_t *, const char *);
+static void context_set_resfile(struct context *, const char *);
+static void context_close_resfile(struct context *);
 static void check_fatal_error(atf_error_t);
 static void report_fatal_error(const char *, ...)
     ATF_DEFS_ATTRIBUTE_NORETURN;
 static atf_error_t write_resfile(const int, const char *, const int,
                                  const atf_dynstr_t *);
-static void create_resfile(const char *, const char *, const int,
+static void create_resfile(struct context *, const char *, const int,
                            atf_dynstr_t *);
 static void error_in_expect(struct context *, const char *, ...)
     ATF_DEFS_ATTRIBUTE_NORETURN;
@@ -102,11 +106,16 @@ static void errno_test(struct context *, const char *, const size_t,
 static atf_error_t check_prog_in_dir(const char *, void *);
 static atf_error_t check_prog(struct context *, const char *);
 
+/* No prototype in header for this one, it's a little sketchy (internal). */
+void atf_tc_set_resultsfile(const char *);
+
 static void
 context_init(struct context *ctx, const atf_tc_t *tc, const char *resfile)
 {
+
     ctx->tc = tc;
-    ctx->resfile = resfile;
+    ctx->resfilefd = -1;
+    context_set_resfile(ctx, resfile);
     ctx->fail_count = 0;
     ctx->expect = EXPECT_PASS;
     check_fatal_error(atf_dynstr_init(&ctx->expect_reason));
@@ -116,6 +125,41 @@ context_init(struct context *ctx, const atf_tc_t *tc, const char *resfile)
     ctx->expect_signo = 0;
 }
 
+static void
+context_set_resfile(struct context *ctx, const char *resfile)
+{
+    atf_error_t err;
+
+    context_close_resfile(ctx);
+    ctx->resfile = resfile;
+    if (strcmp(resfile, "/dev/stdout") == 0)
+        ctx->resfilefd = STDOUT_FILENO;
+    else if (strcmp(resfile, "/dev/stderr") == 0)
+        ctx->resfilefd = STDERR_FILENO;
+    else
+        ctx->resfilefd = open(resfile, O_WRONLY | O_CREAT | O_TRUNC,
+            S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (ctx->resfilefd == -1) {
+            err = atf_libc_error(errno,
+                "Cannot create results file '%s'", resfile);
+            check_fatal_error(err);
+    }
+
+    ctx->resfile = resfile;
+}
+
+static void
+context_close_resfile(struct context *ctx)
+{
+
+    if (ctx->resfilefd == -1)
+        return;
+    if (ctx->resfilefd != STDOUT_FILENO && ctx->resfilefd != STDERR_FILENO)
+        close(ctx->resfilefd);
+    ctx->resfilefd = -1;
+    ctx->resfile = NULL;
+}
+
 static void
 check_fatal_error(atf_error_t err)
 {
@@ -162,7 +206,7 @@ write_resfile(const int fd, const char *result, const int arg,
 
     INV(arg == -1 || reason != NULL);
 
-#define UNCONST(a) ((void *)(unsigned long)(const void *)(a))
+#define UNCONST(a) ((void *)(uintptr_t)(const void *)(a))
     iov[count].iov_base = UNCONST(result);
     iov[count++].iov_len = strlen(result);
 
@@ -202,26 +246,23 @@ write_resfile(const int fd, const char *result, const int arg,
  * not return any error code.
  */
 static void
-create_resfile(const char *resfile, const char *result, const int arg,
+create_resfile(struct context *ctx, const char *result, const int arg,
                atf_dynstr_t *reason)
 {
     atf_error_t err;
 
-    if (strcmp("/dev/stdout", resfile) == 0) {
-        err = write_resfile(STDOUT_FILENO, result, arg, reason);
-    } else if (strcmp("/dev/stderr", resfile) == 0) {
-        err = write_resfile(STDERR_FILENO, result, arg, reason);
-    } else {
-        const int fd = open(resfile, O_WRONLY | O_CREAT | O_TRUNC,
-            S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd == -1) {
-            err = atf_libc_error(errno, "Cannot create results file '%s'",
-                                 resfile);
-        } else {
-            err = write_resfile(fd, result, arg, reason);
-            close(fd);
-        }
-    }
+    /*
+     * We'll attempt to truncate the results file, but only if it's not pointed
+     * at stdout/stderr.  We could just blindly ftruncate() here, but it may
+     * be that stdout/stderr have been redirected to a file that we want to
+     * validate expectations on, for example.  Kyua will want the truncation,
+     * but it will also redirect the results directly to some file and we'll
+     * have no issue here.
+     */
+    if (ctx->resfilefd != STDOUT_FILENO && ctx->resfilefd != STDERR_FILENO &&
+        ftruncate(ctx->resfilefd, 0) != -1)
+        lseek(ctx->resfilefd, 0, SEEK_SET);
+    err = write_resfile(ctx->resfilefd, result, arg, reason);
 
     if (reason != NULL)
         atf_dynstr_fini(reason);
@@ -280,7 +321,8 @@ expected_failure(struct context *ctx, atf_dynstr_t *reason)
 {
     check_fatal_error(atf_dynstr_prepend_fmt(reason, "%s: ",
         atf_dynstr_cstring(&ctx->expect_reason)));
-    create_resfile(ctx->resfile, "expected_failure", -1, reason);
+    create_resfile(ctx, "expected_failure", -1, reason);
+    context_close_resfile(ctx);
     exit(EXIT_SUCCESS);
 }
 
@@ -290,7 +332,8 @@ fail_requirement(struct context *ctx, atf_dynstr_t *reason)
     if (ctx->expect == EXPECT_FAIL) {
         expected_failure(ctx, reason);
     } else if (ctx->expect == EXPECT_PASS) {
-        create_resfile(ctx->resfile, "failed", -1, reason);
+        create_resfile(ctx, "failed", -1, reason);
+        context_close_resfile(ctx);
         exit(EXIT_FAILURE);
     } else {
         error_in_expect(ctx, "Test case raised a failure but was not "
@@ -325,7 +368,8 @@ pass(struct context *ctx)
         error_in_expect(ctx, "Test case was expecting a failure but got "
             "a pass instead");
     } else if (ctx->expect == EXPECT_PASS) {
-        create_resfile(ctx->resfile, "passed", -1, NULL);
+        create_resfile(ctx, "passed", -1, NULL);
+        context_close_resfile(ctx);
         exit(EXIT_SUCCESS);
     } else {
         error_in_expect(ctx, "Test case asked to explicitly pass but was "
@@ -338,7 +382,8 @@ static void
 skip(struct context *ctx, atf_dynstr_t *reason)
 {
     if (ctx->expect == EXPECT_PASS) {
-        create_resfile(ctx->resfile, "skipped", -1, reason);
+        create_resfile(ctx, "skipped", -1, reason);
+        context_close_resfile(ctx);
         exit(EXIT_SUCCESS);
     } else {
         error_in_expect(ctx, "Can only skip a test case when running in "
@@ -942,7 +987,7 @@ _atf_tc_expect_exit(struct context *ctx, const int exitcode, const char *reason,
     check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2));
     va_end(ap2);
 
-    create_resfile(ctx->resfile, "expected_exit", exitcode, &formatted);
+    create_resfile(ctx, "expected_exit", exitcode, &formatted);
 }
 
 static void
@@ -959,7 +1004,7 @@ _atf_tc_expect_signal(struct context *ctx, const int signo, const char *reason,
     check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2));
     va_end(ap2);
 
-    create_resfile(ctx->resfile, "expected_signal", signo, &formatted);
+    create_resfile(ctx, "expected_signal", signo, &formatted);
 }
 
 static void
@@ -975,7 +1020,7 @@ _atf_tc_expect_death(struct context *ctx, const char *reason, va_list ap)
     check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2));
     va_end(ap2);
 
-    create_resfile(ctx->resfile, "expected_death", -1, &formatted);
+    create_resfile(ctx, "expected_death", -1, &formatted);
 }
 
 static void
@@ -991,7 +1036,14 @@ _atf_tc_expect_timeout(struct context *ctx, const char *reason, va_list ap)
     check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2));
     va_end(ap2);
 
-    create_resfile(ctx->resfile, "expected_timeout", -1, &formatted);
+    create_resfile(ctx, "expected_timeout", -1, &formatted);
+}
+
+static void
+_atf_tc_set_resultsfile(struct context *ctx, const char *file)
+{
+
+    context_set_resfile(ctx, file);
 }
 
 /* ---------------------------------------------------------------------
@@ -1215,3 +1267,13 @@ atf_tc_expect_timeout(const char *reason, ...)
     _atf_tc_expect_timeout(&Current, reason, ap);
     va_end(ap);
 }
+
+/* Internal! */
+void
+atf_tc_set_resultsfile(const char *file)
+{
+
+    PRE(Current.tc != NULL);
+
+    _atf_tc_set_resultsfile(&Current, file);
+}

contrib/atf/atf-c/utils.c

@@ -41,6 +41,9 @@
 
 #include "atf-c/detail/dynstr.h"
 
+/* No prototype in header for this one, it's a little sketchy (internal). */
+void atf_tc_set_resultsfile(const char *);
+
 /** Allocate a filename to be used by atf_utils_{fork,wait}.
  *
  * In case of a failure, marks the calling test as failed when in_parent is
@@ -271,6 +274,13 @@ atf_utils_fork(void)
     return pid;
 }
 
+void
+atf_utils_reset_resultsfile(void)
+{
+
+    atf_tc_set_resultsfile("/dev/null");
+}
+
 /** Frees an dynamically-allocated "argv" array.
  *
  * \param argv A dynamically-allocated array of dynamically-allocated

contrib/atf/atf-c/utils.h

@@ -46,5 +46,6 @@ bool atf_utils_grep_string(const char *, const char *, ...)
 char *atf_utils_readline(int);
 void atf_utils_redirect(const int, const char *);
 void atf_utils_wait(const pid_t, const int, const char *, const char *);
+void atf_utils_reset_resultsfile(void);
 
 #endif /* !defined(ATF_C_UTILS_H) */

contrib/atf/atf-c/utils_test.c

@@ -395,6 +395,7 @@ fork_and_wait(const int exitstatus, const char* expout, const char* experr)
         fprintf(stderr, "Some error\n");
         exit(123);
     }
+    atf_utils_reset_resultsfile();
     atf_utils_wait(pid, exitstatus, expout, experr);
     exit(EXIT_SUCCESS);
 }

contrib/atf/atf-sh/.gitignore

@@ -0,0 +1,2 @@
+atf-check
+atf-sh

contrib/atf/atf-sh/atf-check.1

@@ -22,7 +22,7 @@
 .\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
 .\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.Dd March 6, 2017
+.Dd June 21, 2020
 .Dt ATF-CHECK 1
 .Os
 .Sh NAME
@@ -120,6 +120,14 @@ as a shell command line, executing it with the system shell defined by
 .Va ATF_SHELL .
 You should avoid using this flag if at all possible to prevent shell quoting
 issues.
+.It Fl r Ar timeout[:interval]
+Repeats failed checks until the
+.Ar timeout
+(in seconds) expires.
+If unspecified, the default
+.Ar interval
+(in milliseconds) is 50 ms.
+This can be used to wait for an expected update to the contents of a file.
 .El
 .Sh ENVIRONMENT
 .Bl -tag -width ATFXSHELLXX -compact
@@ -157,6 +165,11 @@ atf_check -s signal:sigsegv my_program
 
 # Combined checks
 atf_check -o match:foo -o not-match:bar echo foo baz
+
+# Wait 5 seconds for a line to show up in a file
+( sleep 2 ; echo "testing 123" > $test_path ) &
+atf-check -o ignore -e ignore -s exit:0 -r 5 \e
+    grep "testing 123" $test_path
 .Ed
 .Sh SEE ALSO
 .Xr atf-sh 1

contrib/atf/atf-sh/atf-check.cpp

@@ -29,6 +29,7 @@ extern "C" {
 
 #include <limits.h>
 #include <signal.h>
+#include <stdint.h>
 #include <unistd.h>
 }
 
@@ -53,6 +54,10 @@ extern "C" {
 #include "atf-c++/detail/sanity.hpp"
 #include "atf-c++/detail/text.hpp"
 
+static const useconds_t seconds_in_useconds = (1000 * 1000);
+static const useconds_t mseconds_in_useconds = 1000;
+static const useconds_t useconds_in_nseconds = 1000;
+
 // ------------------------------------------------------------------------
 // Auxiliary functions.
 // ------------------------------------------------------------------------
@@ -162,6 +167,33 @@ class temp_file : public std::ostream {
 
 } // anonymous namespace
 
+static useconds_t
+get_monotonic_useconds(void)
+{
+    struct timespec ts;
+    useconds_t res;
+    int rc;
+
+    rc = clock_gettime(CLOCK_MONOTONIC, &ts);
+    if (rc != 0)
+        throw std::runtime_error("clock_gettime: " +
+            std::string(strerror(errno)));
+
+    res = ts.tv_sec * seconds_in_useconds;
+    res += ts.tv_nsec / useconds_in_nseconds;
+    return res;
+}
+
+static bool
+timo_expired(useconds_t timeout)
+{
+
+    if (get_monotonic_useconds() >= timeout)
+        return true;
+    return false;
+}
+
+
 static int
 parse_exit_code(const std::string& str)
 {
@@ -216,7 +248,7 @@ parse_signal(const std::string& str)
     if (signo == INT_MIN) {
         try {
             return atf::text::to_type< int >(str);
-        } catch (std::runtime_error) {
+        } catch (const std::runtime_error&) {
             throw atf::application::usage_error("Invalid signal name or number "
                 "in -s option");
         }
@@ -306,6 +338,62 @@ parse_output_check_arg(const std::string& arg)
     return output_check(type, negated, arg.substr(delimiter + 1));
 }
 
+static void
+parse_repeat_check_arg(const std::string& arg, useconds_t *m_timo,
+    useconds_t *m_interval)
+{
+    const std::string::size_type delimiter = arg.find(':');
+    const bool has_interval = (delimiter != std::string::npos);
+    const std::string timo_str = arg.substr(0, delimiter);
+
+    long l;
+    char *end;
+
+    // There is no reason this couldn't be a non-integer number of seconds,
+    // this was just easy to do for now.
+    errno = 0;
+    l = strtol(timo_str.c_str(), &end, 10);
+    if (errno == ERANGE)
+        throw atf::application::usage_error("Bogus timeout in seconds");
+    else if (errno != 0)
+        throw atf::application::usage_error("Timeout must be a number");
+
+    if (*end != 0)
+        throw atf::application::usage_error("Timeout must be a number");
+
+    *m_timo = get_monotonic_useconds() + (l * seconds_in_useconds);
+    // 50 milliseconds is chosen arbitrarily.  There is a tradeoff between
+    // longer and shorter poll times.  A shorter poll time makes for faster
+    // tests.  A longer poll time makes for lower CPU overhead for the polled
+    // operation.  50ms is chosen with these tradeoffs in mind: on
+    // microcontrollers, the hope is that we can still avoid meaningful CPU use
+    // with a small test every 50ms.  And on typical fast x86 hardware, our
+    // tests can be much more precise with time wasted than they typically are
+    // without this feature.
+    *m_interval = 50 * mseconds_in_useconds;
+
+    if (!has_interval)
+        return;
+
+    const std::string intv_str = arg.substr(delimiter + 1, std::string::npos);
+
+    // Same -- this could be non-integer milliseconds.
+    errno = 0;
+    l = strtol(intv_str.c_str(), &end, 10);
+    if (errno == ERANGE)
+        throw atf::application::usage_error(
+            "Bogus repeat interval in milliseconds");
+    else if (errno != 0)
+        throw atf::application::usage_error(
+            "Repeat interval must be a number");
+
+    if (*end != 0)
+        throw atf::application::usage_error(
+            "Repeat interval must be a number");
+
+    *m_interval = l * mseconds_in_useconds;
+}
+
 static
 std::string
 flatten_argv(char* const* argv)
@@ -694,8 +782,12 @@ run_output_checks(const std::vector< output_check >& checks,
 namespace {
 
 class atf_check : public atf::application::app {
+    bool m_rflag;
     bool m_xflag;
 
+    useconds_t m_timo;
+    useconds_t m_interval;
+
     std::vector< status_check > m_status_checks;
     std::vector< output_check > m_stdout_checks;
     std::vector< output_check > m_stderr_checks;
@@ -722,6 +814,7 @@ const char* atf_check::m_description =
 
 atf_check::atf_check(void) :
     app(m_description, "atf-check(1)"),
+    m_rflag(false),
     m_xflag(false)
 {
 }
@@ -765,6 +858,8 @@ atf_check::specific_options(void)
     opts.insert(option('e', "action:arg", "Handle stderr. Action must be "
                 "one of: empty ignore file:<path> inline:<val> match:regexp "
                 "save:<path>"));
+    opts.insert(option('r', "timeout[:interval]", "Repeat failed check until "
+                "the timeout expires."));
     opts.insert(option('x', "", "Execute command as a shell command"));
 
     return opts;
@@ -786,6 +881,11 @@ atf_check::process_option(int ch, const char* arg)
         m_stderr_checks.push_back(parse_output_check_arg(arg));
         break;
 
+    case 'r':
+        m_rflag = true;
+        parse_repeat_check_arg(arg, &m_timo, &m_interval);
+        break;
+
     case 'x':
         m_xflag = true;
         break;
@@ -803,9 +903,6 @@ atf_check::main(void)
 
     int status = EXIT_FAILURE;
 
-    std::auto_ptr< atf::check::check_result > r =
-        m_xflag ? execute_with_shell(m_argv) : execute(m_argv);
-
     if (m_status_checks.empty())
         m_status_checks.push_back(status_check(sc_exit, false, EXIT_SUCCESS));
     else if (m_status_checks.size() > 1) {
@@ -818,12 +915,23 @@ atf_check::main(void)
     if (m_stderr_checks.empty())
         m_stderr_checks.push_back(output_check(oc_empty, false, ""));
 
-    if ((run_status_checks(m_status_checks, *r) == false) ||
-        (run_output_checks(*r, "stderr") == false) ||
-        (run_output_checks(*r, "stdout") == false))
-        status = EXIT_FAILURE;
-    else
-        status = EXIT_SUCCESS;
+    do {
+        std::auto_ptr< atf::check::check_result > r =
+            m_xflag ? execute_with_shell(m_argv) : execute(m_argv);
+
+        if ((run_status_checks(m_status_checks, *r) == false) ||
+            (run_output_checks(*r, "stderr") == false) ||
+            (run_output_checks(*r, "stdout") == false))
+            status = EXIT_FAILURE;
+        else
+            status = EXIT_SUCCESS;
+
+        if (m_rflag && status == EXIT_FAILURE) {
+            if (timo_expired(m_timo))
+                break;
+            usleep(m_interval);
+        }
+    } while (m_rflag && status == EXIT_FAILURE);
 
     return status;
 }

contrib/atf/atf-sh/atf-sh.3

@@ -22,13 +22,14 @@
 .\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
 .\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.Dd March 6, 2017
+.Dd June 08, 2017
 .Dt ATF-SH 3
 .Os
 .Sh NAME
 .Nm atf_add_test_case ,
 .Nm atf_check ,
 .Nm atf_check_equal ,
+.Nm atf_check_not_equal ,
 .Nm atf_config_get ,
 .Nm atf_config_has ,
 .Nm atf_expect_death ,
@@ -55,6 +56,9 @@
 .Nm atf_check_equal
 .Qq expected_expression
 .Qq actual_expression
+.Nm atf_check_not_equal
+.Qq expected_expression
+.Qq actual_expression
 .Nm atf_config_get
 .Qq var_name
 .Nm atf_config_has
@@ -310,6 +314,11 @@ This function takes two expressions, evaluates them and, if their
 results differ, aborts the test case with an appropriate failure message.
 The common style is to put the expected value in the first parameter and the
 actual value in the second parameter.
+.It Nm atf_check_not_equal Qo expected_expression Qc Qo actual_expression Qc
+This function takes two expressions, evaluates them and, if their
+results are equal, aborts the test case with an appropriate failure message.
+The common style is to put the expected value in the first parameter and the
+actual value in the second parameter.
 .El
 .Sh EXAMPLES
 The following shows a complete test program with a single test case that

contrib/atf/atf-sh/atf_check_test.sh

@@ -164,6 +164,30 @@ equal_body()
         grep '^failed: \${x} != \${y} (a != b)$' resfile
 }
 
+atf_test_case not_equal
+not_equal_head()
+{
+    atf_set "descr" "Verifies that atf_check_not_equal works"
+}
+not_equal_body()
+{
+    h="$(atf_get_srcdir)/misc_helpers -s $(atf_get_srcdir)"
+
+    atf_check -s eq:0 -o ignore -e ignore -x "${h} atf_check_not_equal_ok"
+
+    atf_check -s eq:1 -o ignore -e ignore -x \
+        "${h} -r resfile atf_check_not_equal_fail"
+    atf_check -s eq:0 -o ignore -e empty grep '^failed: a == b (a == b)$' \
+        resfile
+
+    atf_check -s eq:0 -o ignore -e ignore -x "${h} atf_check_not_equal_eval_ok"
+
+    atf_check -s eq:1 -o ignore -e ignore -x \
+        "${h} -r resfile atf_check_not_equal_eval_fail"
+    atf_check -s eq:0 -o ignore -e empty \
+        grep '^failed: \${x} == \${y} (a == b)$' resfile
+}
+
 atf_test_case flush_stdout_on_death
 flush_stdout_on_death_body()
 {

contrib/atf/atf-sh/libatf-sh.subr

@@ -100,6 +100,23 @@ atf_check_equal()
         atf_fail "${1} != ${2} (${_val1} != ${_val2})"
 }
 
+#
+# atf_check_not_equal expected_expression actual_expression
+#
+#   Checks that expected_expression's value does not match actual_expression's
+#   and, if it does, raises an error.  Ideally expected_expression and
+#   actual_expression should be provided quoted (not expanded) so that
+#   the error message is helpful; otherwise it will only show the values,
+#   not the expressions themselves.
+#
+atf_check_not_equal()
+{
+    eval _val1=\"${1}\"
+    eval _val2=\"${2}\"
+    test "${_val1}" != "${_val2}" || \
+        atf_fail "${1} == ${2} (${_val1} == ${_val2})"
+}
+
 #
 # atf_config_get varname [defvalue]
 #
@@ -536,7 +553,18 @@ _atf_list_tcs()
 #
 _atf_normalize()
 {
-    echo ${1} | tr .- __
+    # Check if the string contains any of the forbidden characters using
+    # POSIX parameter expansion (the ${var//} string substitution is
+    # unfortunately not supported in POSIX sh) and only use tr(1) then.
+    # tr(1) is generally not a builtin, so doing the substring check first
+    # avoids unnecessary fork()+execve() calls. As this function is called
+    # many times in each test script startup, those overheads add up
+    # (especially when running on emulated platforms such as QEMU).
+    if [ "${1#*[.-]}" != "$1" ]; then
+        echo "$1" | tr .- __
+    else
+        echo "$1"
+    fi
 }
 
 #
@@ -734,7 +762,7 @@ main()
             ;;
         esac
     done
-    shift `expr ${OPTIND} - 1`
+    shift $((OPTIND - 1))
 
     case ${Source_Dir} in
         /*)

contrib/atf/atf-sh/misc_helpers.sh

@@ -139,6 +139,50 @@ atf_check_equal_eval_fail_body()
     atf_check_equal '${x}' '${y}'
 }
 
+atf_test_case atf_check_not_equal_ok
+atf_check_not_equal_ok_head()
+{
+    atf_set "descr" "Helper test case for the t_atf_check test program"
+}
+atf_check_not_equal_ok_body()
+{
+    atf_check_not_equal a b
+}
+
+atf_test_case atf_check_not_equal_fail
+atf_check_not_equal_fail_head()
+{
+    atf_set "descr" "Helper test case for the t_atf_check test program"
+}
+atf_check_not_equal_fail_body()
+{
+    atf_check_not_equal a a
+}
+
+atf_test_case atf_check_not_equal_eval_ok
+atf_check_not_equal_eval_ok_head()
+{
+    atf_set "descr" "Helper test case for the t_atf_check test program"
+}
+atf_check_not_equal_eval_ok_body()
+{
+    x=a
+    y=b
+    atf_check_not_equal '${x}' '${y}'
+}
+
+atf_test_case atf_check_not_equal_eval_fail
+atf_check_not_equal_eval_fail_head()
+{
+    atf_set "descr" "Helper test case for the t_atf_check test program"
+}
+atf_check_not_equal_eval_fail_body()
+{
+    x=a
+    y=a
+    atf_check_not_equal '${x}' '${y}'
+}
+
 atf_test_case atf_check_flush_stdout
 atf_check_flush_stdout_head()
 {
@@ -285,6 +329,10 @@ atf_init_test_cases()
     atf_add_test_case atf_check_equal_fail
     atf_add_test_case atf_check_equal_eval_ok
     atf_add_test_case atf_check_equal_eval_fail
+    atf_add_test_case atf_check_not_equal_ok
+    atf_add_test_case atf_check_not_equal_fail
+    atf_add_test_case atf_check_not_equal_eval_ok
+    atf_add_test_case atf_check_not_equal_eval_fail
     atf_add_test_case atf_check_flush_stdout
 
     # Add helper tests for t_config.

contrib/atf/doc/.gitignore

@@ -0,0 +1 @@
+atf.7

contrib/atf/doc/atf-test-case.4

@@ -149,11 +149,7 @@ APIs to implement the test cases.
 The standard input of the test cases is unconditionally connected to
 .Sq /dev/zero .
 .Ss Meta-data
-The following list describes all meta-data properties interpreted
-internally by ATF.
-You are free to define new properties in your test cases and use them as
-you wish, but non-standard properties must be prefixed by
-.Sq X- .
+The following metadata properties can be exposed via the test case's head:
 .Bl -tag -width requireXmachineXX
 .It descr
 Type: textual.
@@ -275,6 +271,17 @@ test program.
 Can optionally be set to zero, in which case the test case has no run-time
 limit.
 This is discouraged.
+.It X- Ns Sq NAME
+Type: textual.
+Optional.
+.Pp
+A user-defined property named
+.Sq NAME .
+These properties are free form, have no special meaning within ATF, and can
+be specified at will by the test case.
+The runtime engine should propagate these properties from the test case to
+the end user so that the end user can rely on custom properties for test case
+tagging and classification.
 .El
 .Ss Environment
 Every time a test case is executed, several environment variables are

lib/atf/libatf-c++/tests/Makefile

@@ -10,6 +10,8 @@ ATF=		${SRCTOP}/contrib/atf
 .PATH:		${ATF}/atf-c++
 .PATH:		${ATF}/atf-c++/detail
 
+CFLAGS+=	-DATF_BUILD_CXX='"c++"'
+CFLAGS+=	-DATF_BUILD_CXXFLAGS='"-Wall"'
 CFLAGS+=	-DATF_C_TESTS_BASE='"${TESTSBASE}/lib/atf/libatf-c"'
 CFLAGS+=	-DATF_INCLUDEDIR='"${INCLUDEDIR}"'
 CFLAGS+=	-I${ATF}