Merge pull request #3948 from sylvestre/gnu-root

Agregate the root GNU testsuite results
2024-09-06 07:54:05 +00:00 · 2022-09-17 20:51:02 +02:00 · 2022-09-17 20:51:02 +02:00 · f8c9de706f
parent efe8a2ce9c 20af659f09
commit f8c9de706f
5 changed files with 116 additions and 15 deletions
--- a/.github/workflows/GnuTests.yml
+++ b/.github/workflows/GnuTests.yml
@ -34,12 +34,13 @@ jobs:
        outputs repo_default_branch repo_GNU_ref repo_reference_branch
        #
        SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log"
+        ROOT_SUITE_LOG_FILE="${path_GNU_tests}/test-suite-root.log"
        TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support
        TEST_FILESET_PREFIX='test-fileset-IDs.sha1#'
        TEST_FILESET_SUFFIX='.txt'
        TEST_SUMMARY_FILE='gnu-result.json'
        TEST_FULL_SUMMARY_FILE='gnu-full-result.json'
-        outputs SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE
+        outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE
    - name: Checkout code (uutil)
      uses: actions/checkout@v3
      with:
@ -104,6 +105,12 @@ jobs:
        path_GNU='${{ steps.vars.outputs.path_GNU }}'
        path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
        bash "${path_UUTILS}/util/run-gnu-test.sh"
+    - name: Run GNU root tests
+      shell: bash
+      run: |
+        path_GNU='${{ steps.vars.outputs.path_GNU }}'
+        path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
+        bash "${path_UUTILS}/util/run-gnu-test.sh" run-root
    - name: Extract testing info into JSON
      shell: bash
      run : |
@ -113,18 +120,17 @@ jobs:
      id: summary
      shell: bash
      run: |
+        path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
        ## Extract/summarize testing info
        outputs() { step_id="summary"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo ::set-output name=${var}::${!var}; done; }
        #
        SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}'
+        ROOT_SUITE_LOG_FILE='${{ steps.vars.outputs.ROOT_SUITE_LOG_FILE }}'
+        ls -al ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE}
+
        if test -f "${SUITE_LOG_FILE}"
        then
-            TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
-            PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
-            SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
-            FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
-            XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
-            ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+            source ${path_UUTILS}/util/analyze-gnu-results.sh ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE}
            if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then
              echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early"
              exit 1
--- a/DEVELOPER_INSTRUCTIONS.md
+++ b/DEVELOPER_INSTRUCTIONS.md
@ -21,7 +21,7 @@ Running GNU tests
 At the end you should have uutils, gnu and gnulib checked out next to each other.

 - Run `cd uutils && ./util/build-gnu.sh && cd ..` to get everything ready (this may take a while)
- Finally, you can run tests with `bash uutils/util/run-gnu-test.sh <test>`. Instead of `<test>` insert the test you want to run, e.g. `tests/misc/wc-proc.sh`.
+- Finally, you can run tests with `bash uutils/util/run-gnu-test.sh <tests>`. Instead of `<tests>` insert the tests you want to run, e.g. `tests/misc/wc-proc.sh`.


 Code Coverage Report Generation
--- a/README.md
+++ b/README.md
@ -369,6 +369,8 @@ $ bash util/build-gnu.sh
 $ bash util/run-gnu-test.sh
 # To run a single test:
 $ bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example
+# To run several tests:
+$ bash util/run-gnu-test.sh tests/touch/not-owner.sh tests/rm/no-give-up.sh # for example
 # If this is a perl (.pl) test, to run in debug:
 $ DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl
 ```
--- a/util/analyze-gnu-results.sh
+++ b/util/analyze-gnu-results.sh
@ -0,0 +1,75 @@
+#!/bin/sh
+# spell-checker:ignore xpass XPASS testsuite
+set -e
+
+# As we do two builds (with and without root), we need to do some trivial maths
+# to present the merge results
+# this script will export the values in the term
+
+if test $# -ne 2; then
+    echo "syntax:"
+    echo "$0 testsuite.log root-testsuite.log"
+fi
+
+SUITE_LOG_FILE=$1
+ROOT_SUITE_LOG_FILE=$2
+
+if test ! -f "${SUITE_LOG_FILE}"; then
+    echo "${SUITE_LOG_FILE} has not been found"
+    exit 1
+fi
+if test ! -f "${ROOT_SUITE_LOG_FILE}"; then
+    echo "${ROOT_SUITE_LOG_FILE} has not been found"
+    exit 1
+fi
+
+function get_total {
+    # Total of tests executed
+    # They are the normal number of tests as they are skipped in the normal run
+    NON_ROOT=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    echo $NON_ROOT
+}
+
+function get_pass {
+    # This is the sum of the two test suites.
+    # In the normal run, they are SKIP
+    NON_ROOT=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    AS_ROOT=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    echo $((NON_ROOT + AS_ROOT))
+}
+
+function get_skip {
+    # As some of the tests executed as root as still SKIP (ex: selinux), we
+    # need to some maths:
+    # Number of tests skip as user - total test as root + skipped as root
+    TOTAL_AS_ROOT=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    NON_ROOT=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    AS_ROOT=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    echo $((NON_ROOT - TOTAL_AS_ROOT + AS_ROOT))
+}
+
+function get_fail {
+    # They used to be SKIP, now they fail (this is a good news)
+    NON_ROOT=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    AS_ROOT=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    echo $((NON_ROOT + AS_ROOT))
+}
+
+function get_xpass {
+    NON_ROOT=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    echo $NON_ROOT
+}
+
+function get_error {
+    # They used to be SKIP, now they error (this is a good news)
+    NON_ROOT=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    AS_ROOT=$(sed -n "s/.*# ERROR:: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
+    echo $((NON_ROOT + AS_ROOT))
+}
+
+export TOTAL=$(get_total)
+export PASS=$(get_pass)
+export SKIP=$(get_skip)
+export FAIL=$(get_fail)
+export XPASS=$(get_xpass)
+export ERROR=$(get_error)
--- a/util/run-gnu-test.sh
+++ b/util/run-gnu-test.sh
@ -31,17 +31,35 @@ cd "${path_GNU}" && echo "[ pwd:'${PWD}' ]"

 export RUST_BACKTRACE=1

-if test -n "$1"; then
-    # if set, run only the test passed
-    export RUN_TEST="TESTS=$1"
+if test $# -ge 1; then
+    # if set, run only the tests passed
+    SPECIFIC_TESTS=""
+    for t in "$@"; do
+        SPECIFIC_TESTS="$SPECIFIC_TESTS $t"
+    done
+    # trim it
+    SPECIFIC_TESTS=$(echo $SPECIFIC_TESTS| xargs)
+    echo "Running specific tests: $SPECIFIC_TESTS"
 fi

 # * timeout used to kill occasionally errant/"stuck" processes (note: 'release' testing takes ~1 hour; 'debug' testing takes ~2.5 hours)
 # * `gl_public_submodule_commit=` disables testing for use of a "public" gnulib commit (which will fail when using shallow gnulib checkouts)
 # * `srcdir=..` specifies the GNU source directory for tests (fixing failing/confused 'tests/factor/tNN.sh' tests and causing no harm to other tests)
 #shellcheck disable=SC2086
-timeout -sKILL 4h make -j "$(nproc)" check ${RUN_TEST} SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make

-if test -z "$1" && test -n "$CI"; then
-    sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || :
-fi
+if test "$1" != "run-root"; then
+# run the regular tests
+    if test $# -ge 1; then
+        timeout -sKILL 4h make -j "$(nproc)" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make
+    else
+        timeout -sKILL 4h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make
+    fi
+else
+# in case we would like to run tests requiring root
+    if test -z "$1" -o "$1" == "run-root"; then
+        if test -n "$CI"; then
+            echo "Running check-root to run only root tests"
+            sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || :
+        fi
+    fi
+fi