mirror of
https://github.com/uutils/coreutils
synced 2024-10-07 00:19:14 +00:00
maint/CICD ~ implement 'GnuTests' workflow fixes/refactor
- consolidate configuration - DRY improvements - improve flexibility/robustness in the face of missing reference test info - add reference test info IDs and additional logging to help diagnose testing failures - includes parallel refactor of 'util/run-gnu-test.sh'
This commit is contained in:
parent
cc61ea807e
commit
578e5c8aba
169
.github/workflows/GnuTests.yml
vendored
169
.github/workflows/GnuTests.yml
vendored
|
@ -1,6 +1,6 @@
|
||||||
name: GnuTests
|
name: GnuTests
|
||||||
|
|
||||||
# spell-checker:ignore (names) gnulib ; (utils) autopoint gperf pyinotify texinfo ; (vars) XPASS
|
# spell-checker:ignore (names) gnulib ; (people) Dawid Dziurla * dawidd6 ; (utils) autopoint chksum gperf pyinotify shopt texinfo ; (vars) FILESET XPASS
|
||||||
|
|
||||||
on: [push, pull_request]
|
on: [push, pull_request]
|
||||||
|
|
||||||
|
@ -9,23 +9,55 @@ jobs:
|
||||||
name: Run GNU tests
|
name: Run GNU tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
|
- name: Initialize workflow variables
|
||||||
|
id: vars
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
## VARs setup
|
||||||
|
outputs() { step_id="vars"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo ::set-output name=${var}::${!var}; done; }
|
||||||
|
# * config
|
||||||
|
path_GNU="gnu"
|
||||||
|
path_GNULIB="gnulib"
|
||||||
|
path_GNU_tests="gnu/tests"
|
||||||
|
path_UUTILS="uutils"
|
||||||
|
path_reference="reference"
|
||||||
|
outputs path_GNU path_GNU_tests path_GNULIB path_reference path_UUTILS
|
||||||
|
#
|
||||||
|
repo_GNU_ref="v9.0"
|
||||||
|
repo_GNULIB_ref="8e99f24c0931a38880c6ee9b8287c7da80b0036b"
|
||||||
|
repo_reference_branch="${{ github.event.repository.default_branch }}"
|
||||||
|
outputs repo_GNU_ref repo_GNULIB_ref repo_reference_branch
|
||||||
|
#
|
||||||
|
SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log"
|
||||||
|
TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support
|
||||||
|
TEST_FILESET_PREFIX='test-fileset-IDs.sha1#'
|
||||||
|
TEST_FILESET_SUFFIX='.txt'
|
||||||
|
TEST_SUMMARY_FILE='gnu-result.json'
|
||||||
|
outputs SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE
|
||||||
- name: Checkout code uutil
|
- name: Checkout code uutil
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
path: 'uutils'
|
path: '${{ steps.vars.outputs.path_UUTILS }}'
|
||||||
- name: Checkout GNU coreutils
|
- name: Checkout GNU coreutils
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
repository: 'coreutils/coreutils'
|
repository: 'coreutils/coreutils'
|
||||||
path: 'gnu'
|
path: '${{ steps.vars.outputs.path_GNU }}'
|
||||||
ref: v9.0
|
ref: ${{ steps.vars.outputs.repo_GNU_ref }}
|
||||||
- name: Checkout GNU coreutils library (gnulib)
|
- name: Checkout GNU coreutils library (gnulib)
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
repository: 'coreutils/gnulib'
|
repository: 'coreutils/gnulib'
|
||||||
path: 'gnulib'
|
path: '${{ steps.vars.outputs.path_GNULIB }}'
|
||||||
ref: 8e99f24c0931a38880c6ee9b8287c7da80b0036b
|
ref: ${{ steps.vars.outputs.repo_GNULIB_ref }}
|
||||||
fetch-depth: 0 # gnu gets upset if gnulib is a shallow checkout
|
fetch-depth: 0 # full depth checkout (o/w gnu gets upset if gnulib is a shallow checkout)
|
||||||
|
- name: Retrieve reference artifacts
|
||||||
|
uses: dawidd6/action-download-artifact@v2
|
||||||
|
continue-on-error: true ## don't break the build for missing reference artifacts (may be expired or just not generated yet)
|
||||||
|
with:
|
||||||
|
workflow: GnuTests.yml
|
||||||
|
branch: "${{ steps.vars.outputs.repo_reference_branch }}"
|
||||||
|
path: "${{ steps.vars.outputs.path_reference }}"
|
||||||
- name: Install `rust` toolchain
|
- name: Install `rust` toolchain
|
||||||
uses: actions-rs/toolchain@v1
|
uses: actions-rs/toolchain@v1
|
||||||
with:
|
with:
|
||||||
|
@ -43,27 +75,33 @@ jobs:
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
## Build binaries
|
## Build binaries
|
||||||
cd uutils
|
cd '${{ steps.vars.outputs.path_UUTILS }}'
|
||||||
bash util/build-gnu.sh
|
bash util/build-gnu.sh
|
||||||
- name: Run GNU tests
|
- name: Run GNU tests
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
bash uutils/util/run-gnu-test.sh
|
path_GNU='${{ steps.vars.outputs.path_GNU }}'
|
||||||
- name: Extract testing info
|
path_GNULIB='${{ steps.vars.outputs.path_GNULIB }}'
|
||||||
|
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
|
||||||
|
bash "${path_UUTILS}/util/run-gnu-test.sh"
|
||||||
|
- name: Extract/summarize testing info
|
||||||
|
id: summary
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
## Extract testing info
|
## Extract/summarize testing info
|
||||||
LOG_FILE=gnu/tests/test-suite.log
|
outputs() { step_id="summary"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo ::set-output name=${var}::${!var}; done; }
|
||||||
if test -f "$LOG_FILE"
|
#
|
||||||
|
SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}'
|
||||||
|
if test -f "${SUITE_LOG_FILE}"
|
||||||
then
|
then
|
||||||
TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
|
TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
|
||||||
PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
|
PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
|
||||||
SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
|
SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
|
||||||
FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
|
FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
|
||||||
XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
|
XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
|
||||||
ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
|
ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1)
|
||||||
if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then
|
if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then
|
||||||
echo "Error in the execution, failing early"
|
echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR"
|
output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR"
|
||||||
|
@ -78,54 +116,61 @@ jobs:
|
||||||
--arg fail "$FAIL" \
|
--arg fail "$FAIL" \
|
||||||
--arg xpass "$XPASS" \
|
--arg xpass "$XPASS" \
|
||||||
--arg error "$ERROR" \
|
--arg error "$ERROR" \
|
||||||
'{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > gnu-result.json
|
'{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}'
|
||||||
|
HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1)
|
||||||
|
outputs HASH
|
||||||
else
|
else
|
||||||
echo "::error ::Failed to get summary of test results"
|
echo "::error ::Failed to find summary of test results (missing '${SUITE_LOG_FILE}'); failing early"
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
- uses: actions/upload-artifact@v2
|
- name: Reserve SHA1/ID of 'test-summary'
|
||||||
|
uses: actions/upload-artifact@v2
|
||||||
with:
|
with:
|
||||||
name: test-report
|
name: "${{ steps.summary.outputs.HASH }}"
|
||||||
path: gnu/tests/**/*.log
|
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
|
||||||
- uses: actions/upload-artifact@v2
|
- name: Reserve test results summary
|
||||||
|
uses: actions/upload-artifact@v2
|
||||||
with:
|
with:
|
||||||
name: gnu-result
|
name: test-summary
|
||||||
path: gnu-result.json
|
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
|
||||||
- name: Download the result
|
- name: Reserve test logs
|
||||||
uses: dawidd6/action-download-artifact@v2
|
uses: actions/upload-artifact@v2
|
||||||
with:
|
with:
|
||||||
workflow: GnuTests.yml
|
name: test-logs
|
||||||
name: gnu-result
|
path: "${{ steps.vars.outputs.TEST_LOGS_GLOB }}"
|
||||||
repo: uutils/coreutils
|
- name: Compare test failures VS reference
|
||||||
branch: main
|
|
||||||
path: dl
|
|
||||||
- name: Download the log
|
|
||||||
uses: dawidd6/action-download-artifact@v2
|
|
||||||
with:
|
|
||||||
workflow: GnuTests.yml
|
|
||||||
name: test-report
|
|
||||||
repo: uutils/coreutils
|
|
||||||
branch: main
|
|
||||||
path: dl
|
|
||||||
- name: Compare failing tests against main
|
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
OLD_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" dl/test-suite.log | sort)
|
REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log'
|
||||||
NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" gnu/tests/test-suite.log | sort)
|
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json'
|
||||||
for LINE in $OLD_FAILING
|
if test -f "${REF_LOG_FILE}"; then
|
||||||
do
|
echo "Reference SHA1/ID (of '${REF_SUMMARY_FILE}'): $(sha1sum -- "${REF_SUMMARY_FILE}")"
|
||||||
if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then
|
REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort)
|
||||||
echo "::warning ::Congrats! The gnu test $LINE is now passing!"
|
NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort)
|
||||||
fi
|
for LINE in $REF_FAILING
|
||||||
done
|
do
|
||||||
for LINE in $NEW_FAILING
|
if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then
|
||||||
do
|
echo "::warning ::Congrats! The gnu test $LINE is now passing!"
|
||||||
if ! grep -Fxq $LINE<<<"$OLD_FAILING"
|
fi
|
||||||
then
|
done
|
||||||
echo "::error ::GNU test failed: $LINE. $LINE is passing on 'main'. Maybe you have to rebase?"
|
for LINE in $NEW_FAILING
|
||||||
fi
|
do
|
||||||
done
|
if ! grep -Fxq $LINE<<<"$REF_FAILING"
|
||||||
- name: Compare against main results
|
then
|
||||||
|
echo "::error ::GNU test failed: $LINE. $LINE is passing on 'main'. Maybe you have to rebase?"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
else
|
||||||
|
echo "::warning ::Skipping test failure comparison; no prior reference test logs are available."
|
||||||
|
fi
|
||||||
|
- name: Compare test summary VS reference
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
mv dl/gnu-result.json main-gnu-result.json
|
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json'
|
||||||
python uutils/util/compare_gnu_result.py
|
if test -f "${REF_SUMMARY_FILE}"; then
|
||||||
|
echo "Reference SHA1/ID (of '${REF_SUMMARY_FILE}'): $(sha1sum -- "${REF_SUMMARY_FILE}")"
|
||||||
|
mv "${REF_SUMMARY_FILE}" main-gnu-result.json
|
||||||
|
python uutils/util/compare_gnu_result.py
|
||||||
|
else
|
||||||
|
echo "::warning ::Skipping test summary comparison; no prior reference summary is available."
|
||||||
|
fi
|
||||||
|
|
|
@ -1,10 +1,28 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
# `$0 [TEST]`
|
||||||
|
# run GNU test (or all tests if TEST is missing/null)
|
||||||
# spell-checker:ignore (env/vars) BUILDDIR GNULIB SUBDIRS
|
# spell-checker:ignore (env/vars) BUILDDIR GNULIB SUBDIRS
|
||||||
cd "$(dirname -- "$(readlink -fm -- "$0")")/../.."
|
|
||||||
|
ME_dir="$(dirname -- "$(readlink -fm -- "$0")")"
|
||||||
|
REPO_main_dir="$(dirname -- "${ME_dir}")"
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
BUILDDIR="${PWD}/uutils/target/release"
|
|
||||||
GNULIB_DIR="${PWD}/gnulib"
|
### * config (from environment with fallback defaults)
|
||||||
pushd gnu
|
|
||||||
|
path_UUTILS=${path_UUTILS:-${REPO_main_dir}}
|
||||||
|
path_GNU=${path_GNU:-${path_UUTILS}/../gnu}
|
||||||
|
path_GNULIB=${path_GNULIB:-${path_UUTILS}/../gnulib}
|
||||||
|
|
||||||
|
###
|
||||||
|
|
||||||
|
BUILD_DIR="$(realpath -- "${path_UUTILS}/target/release")"
|
||||||
|
GNULIB_DIR="$(realpath -- "${path_GNULIB}")"
|
||||||
|
|
||||||
|
export BUILD_DIR
|
||||||
|
export GNULIB_DIR
|
||||||
|
|
||||||
|
pushd "$(realpath -- "${path_GNU}")"
|
||||||
|
|
||||||
export RUST_BACKTRACE=1
|
export RUST_BACKTRACE=1
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue