name: GnuTests # spell-checker:ignore (abbrev/names) CodeCov gnulib GnuTests Swatinem # spell-checker:ignore (jargon) submodules # spell-checker:ignore (libs/utils) autopoint chksum gperf lcov libexpect pyinotify shopt texinfo valgrind libattr libcap taiki-e # spell-checker:ignore (options) Ccodegen Coverflow Cpanic Zpanic # spell-checker:ignore (people) Dawid Dziurla * dawidd dtolnay # spell-checker:ignore (vars) FILESET SUBDIRS XPASS # * note: to run a single test => `REPO/util/run-gnu-test.sh PATH/TO/TEST/SCRIPT` on: pull_request: push: branches: - main permissions: contents: read # End the current execution if there is a new changeset in the PR. concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: gnu: permissions: actions: read # for dawidd6/action-download-artifact to query and download artifacts contents: read # for actions/checkout to fetch code pull-requests: read # for dawidd6/action-download-artifact to query commit hash name: Run GNU tests runs-on: ubuntu-22.04 steps: - name: Initialize workflow variables id: vars shell: bash run: | ## VARs setup outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } # * config path_GNU="gnu" path_GNU_tests="${path_GNU}/tests" path_UUTILS="uutils" path_reference="reference" outputs path_GNU path_GNU_tests path_reference path_UUTILS # repo_default_branch="${{ github.event.repository.default_branch }}" repo_GNU_ref="v9.4" repo_reference_branch="${{ github.event.repository.default_branch }}" outputs repo_default_branch repo_GNU_ref repo_reference_branch # SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log" ROOT_SUITE_LOG_FILE="${path_GNU_tests}/test-suite-root.log" TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support TEST_FILESET_PREFIX='test-fileset-IDs.sha1#' TEST_FILESET_SUFFIX='.txt' TEST_SUMMARY_FILE='gnu-result.json' TEST_FULL_SUMMARY_FILE='gnu-full-result.json' outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE - name: Checkout code (uutil) uses: actions/checkout@v4 with: path: '${{ steps.vars.outputs.path_UUTILS }}' - uses: dtolnay/rust-toolchain@master with: toolchain: stable components: rustfmt - uses: Swatinem/rust-cache@v2 with: workspaces: "./${{ steps.vars.outputs.path_UUTILS }} -> target" - name: Checkout code (GNU coreutils) uses: actions/checkout@v4 with: repository: 'coreutils/coreutils' path: '${{ steps.vars.outputs.path_GNU }}' ref: ${{ steps.vars.outputs.repo_GNU_ref }} submodules: recursive - name: Retrieve reference artifacts uses: dawidd6/action-download-artifact@v3 # ref: continue-on-error: true ## don't break the build for missing reference artifacts (may be expired or just not generated yet) with: workflow: GnuTests.yml branch: "${{ steps.vars.outputs.repo_reference_branch }}" # workflow_conclusion: success ## (default); * but, if commit with failed GnuTests is merged into the default branch, future commits will all show regression errors in GnuTests CI until o/w fixed workflow_conclusion: completed ## continually recalibrates to last commit of default branch with a successful GnuTests (ie, "self-heals" from GnuTest regressions, but needs more supervision for/of regressions) path: "${{ steps.vars.outputs.path_reference }}" - name: Install dependencies shell: bash run: | ## Install dependencies sudo apt-get update sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev - name: Add various locales shell: bash run: | ## Add various locales echo "Before:" locale -a ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' ## Some others need a French locale sudo locale-gen sudo locale-gen --keep-existing fr_FR sudo locale-gen --keep-existing fr_FR.UTF-8 sudo locale-gen --keep-existing sv_SE sudo locale-gen --keep-existing sv_SE.UTF-8 sudo locale-gen --keep-existing en_US sudo locale-gen --keep-existing ru_RU.KOI8-R sudo update-locale echo "After:" locale -a - name: Build binaries shell: bash run: | ## Build binaries cd '${{ steps.vars.outputs.path_UUTILS }}' bash util/build-gnu.sh --release-build - name: Run GNU tests shell: bash run: | ## Run GNU tests path_GNU='${{ steps.vars.outputs.path_GNU }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' bash "${path_UUTILS}/util/run-gnu-test.sh" - name: Run GNU root tests shell: bash run: | ## Run GNU root tests path_GNU='${{ steps.vars.outputs.path_GNU }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' bash "${path_UUTILS}/util/run-gnu-test.sh" run-root - name: Extract testing info into JSON shell: bash run : | ## Extract testing info into JSON path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} - name: Extract/summarize testing info id: summary shell: bash run: | ## Extract/summarize testing info outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } # path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' # SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}' ROOT_SUITE_LOG_FILE='${{ steps.vars.outputs.ROOT_SUITE_LOG_FILE }}' ls -al ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} if test -f "${SUITE_LOG_FILE}" then source ${path_UUTILS}/util/analyze-gnu-results.sh ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early" exit 1 fi output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP" echo "${output}" if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi jq -n \ --arg date "$(date --rfc-email)" \ --arg sha "$GITHUB_SHA" \ --arg total "$TOTAL" \ --arg pass "$PASS" \ --arg skip "$SKIP" \ --arg fail "$FAIL" \ --arg xpass "$XPASS" \ --arg error "$ERROR" \ '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) outputs HASH else echo "::error ::Failed to find summary of test results (missing '${SUITE_LOG_FILE}'); failing early" exit 1 fi # Compress logs before upload (fails otherwise) gzip ${{ steps.vars.outputs.TEST_LOGS_GLOB }} - name: Reserve SHA1/ID of 'test-summary' uses: actions/upload-artifact@v4 with: name: "${{ steps.summary.outputs.HASH }}" path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Reserve test results summary uses: actions/upload-artifact@v4 with: name: test-summary path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Reserve test logs uses: actions/upload-artifact@v4 with: name: test-logs path: "${{ steps.vars.outputs.TEST_LOGS_GLOB }}" - name: Upload full json results uses: actions/upload-artifact@v4 with: name: gnu-full-result.json path: ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} - name: Compare test failures VS reference shell: bash run: | ## Compare test failures VS reference have_new_failures="" REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log' REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' # https://github.com/uutils/coreutils/issues/4294 # https://github.com/uutils/coreutils/issues/4295 IGNORE_INTERMITTENT="${path_UUTILS}/.github/workflows/ignore-intermittent.txt" mkdir -p ${{ steps.vars.outputs.path_reference }} COMMENT_DIR="${{ steps.vars.outputs.path_reference }}/comment" mkdir -p ${COMMENT_DIR} echo ${{ github.event.number }} > ${COMMENT_DIR}/NR COMMENT_LOG="${COMMENT_DIR}/result.txt" # The comment log might be downloaded from a previous run # We only want the new changes, so remove it if it exists. rm -f ${COMMENT_LOG} touch ${COMMENT_LOG} compare_tests() { local new_log_file=$1 local ref_log_file=$2 local test_type=$3 # "standard" or "root" if test -f "${ref_log_file}"; then echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}" REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) CURRENT_RUN_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) CURRENT_RUN_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) echo "Detailled information:" echo "REF_ERROR = ${REF_ERROR}" echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}" echo "REF_FAILING = ${REF_FAILING}" echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}" # Compare failing and error tests for LINE in ${CURRENT_RUN_FAILING} do if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" then if ! grep ${LINE} ${IGNORE_INTERMITTENT} then MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" echo "::error ::$MSG" echo $MSG >> ${COMMENT_LOG} have_new_failures="true" else MSG="Skip an intermittent issue ${LINE} (fails in this run but passes in the 'main' branch)" echo "::warning ::$MSG" echo $MSG >> ${COMMENT_LOG} echo "" fi fi done for LINE in ${REF_FAILING} do if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_FAILING}" then if ! grep ${LINE} ${IGNORE_INTERMITTENT} then MSG="Congrats! The gnu test ${LINE} is no longer failing!" echo "::warning ::$MSG" echo $MSG >> ${COMMENT_LOG} else MSG="Skipping an intermittent issue ${LINE} (passes in this run but fails in the 'main' branch)" echo "::warning ::$MSG" echo $MSG >> ${COMMENT_LOG} echo "" fi fi done for LINE in ${CURRENT_RUN_ERROR} do if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" then MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" echo "::error ::$MSG" echo $MSG >> ${COMMENT_LOG} have_new_failures="true" fi done for LINE in ${REF_ERROR} do if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_ERROR}" then MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" echo "::warning ::$MSG" echo $MSG >> ${COMMENT_LOG} fi done else echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available." fi } # Compare standard tests compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard" # Compare root tests compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root" if test -n "${have_new_failures}" ; then exit -1 ; fi - name: Upload comparison log (for GnuComment workflow) if: success() || failure() # run regardless of prior step success/failure uses: actions/upload-artifact@v4 with: name: comment path: ${{ steps.vars.outputs.path_reference }}/comment/ - name: Compare test summary VS reference if: success() || failure() # run regardless of prior step success/failure shell: bash run: | ## Compare test summary VS reference REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' if test -f "${REF_SUMMARY_FILE}"; then echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" mv "${REF_SUMMARY_FILE}" main-gnu-result.json python uutils/util/compare_gnu_result.py else echo "::warning ::Skipping test summary comparison; no prior reference summary is available." fi gnu_coverage: name: Run GNU tests with coverage runs-on: ubuntu-22.04 steps: - name: Checkout code uutil uses: actions/checkout@v4 with: path: 'uutils' - name: Checkout GNU coreutils uses: actions/checkout@v4 with: repository: 'coreutils/coreutils' path: 'gnu' ref: 'v9.4' submodules: recursive - uses: dtolnay/rust-toolchain@master with: toolchain: nightly components: rustfmt - uses: taiki-e/install-action@grcov - uses: Swatinem/rust-cache@v2 with: workspaces: "./uutils -> target" - name: Install dependencies run: | ## Install dependencies sudo apt-get update sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev - name: Add various locales run: | ## Add various locales echo "Before:" locale -a ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' ## Some others need a French locale sudo locale-gen sudo locale-gen --keep-existing fr_FR sudo locale-gen --keep-existing fr_FR.UTF-8 sudo update-locale echo "After:" locale -a - name: Build binaries env: RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" RUSTDOCFLAGS: "-Cpanic=abort" run: | ## Build binaries cd uutils bash util/build-gnu.sh - name: Run GNU tests run: bash uutils/util/run-gnu-test.sh - name: Generate coverage data (via `grcov`) id: coverage run: | ## Generate coverage data cd uutils COVERAGE_REPORT_DIR="target/debug" COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" mkdir -p "${COVERAGE_REPORT_DIR}" sudo chown -R "$(whoami)" "${COVERAGE_REPORT_DIR}" # display coverage files grcov . --output-type files --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique # generate coverage report grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" echo "report=${COVERAGE_REPORT_FILE}" >> $GITHUB_OUTPUT - name: Upload coverage results (to Codecov.io) uses: codecov/codecov-action@v3 with: file: ${{ steps.coverage.outputs.report }} flags: gnutests name: gnutests working-directory: uutils