name: GnuTests # spell-checker:ignore (names) gnulib ; (jargon) submodules ; (people) Dawid Dziurla * dawidd ; (utils) autopoint chksum gperf pyinotify shopt texinfo ; (vars) FILESET XPASS on: [push, pull_request] jobs: gnu: name: Run GNU tests runs-on: ubuntu-latest steps: - name: Initialize workflow variables id: vars shell: bash run: | ## VARs setup outputs() { step_id="vars"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo ::set-output name=${var}::${!var}; done; } # * config path_GNU="gnu" path_GNU_tests="${path_GNU}/tests" path_UUTILS="uutils" path_reference="reference" outputs path_GNU path_GNU_tests path_reference path_UUTILS # repo_default_branch="${{ github.event.repository.default_branch }}" repo_GNU_ref="v9.0" repo_GNULIB_ref="8e99f24c0931a38880c6ee9b8287c7da80b0036b" repo_reference_branch="${{ github.event.repository.default_branch }}" outputs repo_default_branch repo_GNU_ref repo_GNULIB_ref repo_reference_branch # SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log" TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support TEST_FILESET_PREFIX='test-fileset-IDs.sha1#' TEST_FILESET_SUFFIX='.txt' TEST_SUMMARY_FILE='gnu-result.json' outputs SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE - name: Checkout code (uutil) uses: actions/checkout@v2 with: path: '${{ steps.vars.outputs.path_UUTILS }}' - name: Checkout code (GNU coreutils) uses: actions/checkout@v2 with: repository: 'coreutils/coreutils' path: '${{ steps.vars.outputs.path_GNU }}' ref: ${{ steps.vars.outputs.repo_GNU_ref }} submodules: recursive - name: Retrieve reference artifacts uses: dawidd6/action-download-artifact@v2 # ref: continue-on-error: true ## don't break the build for missing reference artifacts (may be expired or just not generated yet) with: workflow: GnuTests.yml branch: "${{ steps.vars.outputs.repo_reference_branch }}" # workflow_conclusion: success ## (default); * but, if commit with failed GnuTests is merged into the default branch, future commits will all show regression errors in GnuTests CI until o/w fixed workflow_conclusion: completed ## continually recalibrates to last commit of default branch with a successful GnuTests (ie, "self-heals" from GnuTest regressions, but needs more supervision for/of regressions) path: "${{ steps.vars.outputs.path_reference }}" - name: Install `rust` toolchain uses: actions-rs/toolchain@v1 with: toolchain: stable default: true profile: minimal # minimal component installation (ie, no documentation) components: rustfmt - name: Install dependencies shell: bash run: | ## Install dependencies sudo apt-get update sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq - name: Build binaries shell: bash run: | ## Build binaries cd '${{ steps.vars.outputs.path_UUTILS }}' bash util/build-gnu.sh - name: Run GNU tests shell: bash run: | path_GNU='${{ steps.vars.outputs.path_GNU }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' bash "${path_UUTILS}/util/run-gnu-test.sh" - name: Extract/summarize testing info id: summary shell: bash run: | ## Extract/summarize testing info outputs() { step_id="summary"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo ::set-output name=${var}::${!var}; done; } # SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}' if test -f "${SUITE_LOG_FILE}" then TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early" exit 1 fi output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR" echo "${output}" if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi jq -n \ --arg date "$(date --rfc-email)" \ --arg sha "$GITHUB_SHA" \ --arg total "$TOTAL" \ --arg pass "$PASS" \ --arg skip "$SKIP" \ --arg fail "$FAIL" \ --arg xpass "$XPASS" \ --arg error "$ERROR" \ '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) outputs HASH else echo "::error ::Failed to find summary of test results (missing '${SUITE_LOG_FILE}'); failing early" exit 1 fi - name: Reserve SHA1/ID of 'test-summary' uses: actions/upload-artifact@v2 with: name: "${{ steps.summary.outputs.HASH }}" path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Reserve test results summary uses: actions/upload-artifact@v2 with: name: test-summary path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Reserve test logs uses: actions/upload-artifact@v2 with: name: test-logs path: "${{ steps.vars.outputs.TEST_LOGS_GLOB }}" - name: Compare test failures VS reference shell: bash run: | REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' if test -f "${REF_LOG_FILE}"; then echo "Reference SHA1/ID (of '${REF_SUMMARY_FILE}'): $(sha1sum -- "${REF_SUMMARY_FILE}")" REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) for LINE in $REF_FAILING do if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then echo "::warning ::Congrats! The gnu test $LINE is now passing!" fi done for LINE in $NEW_FAILING do if ! grep -Fxq $LINE<<<"$REF_FAILING" then echo "::error ::GNU test failed: $LINE. $LINE is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" fi done else echo "::warning ::Skipping test failure comparison; no prior reference test logs are available." fi - name: Compare test summary VS reference shell: bash run: | REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' if test -f "${REF_SUMMARY_FILE}"; then echo "Reference SHA1/ID (of '${REF_SUMMARY_FILE}'): $(sha1sum -- "${REF_SUMMARY_FILE}")" mv "${REF_SUMMARY_FILE}" main-gnu-result.json python uutils/util/compare_gnu_result.py else echo "::warning ::Skipping test summary comparison; no prior reference summary is available." fi