system/coreutils
system/coreutils
1
0
Fork 0
mirror of https://github.com/uutils/coreutils synced 2024-07-08 19:56:10 +00:00
Code Issues Projects Releases Wiki Activity

Improve the GNU ci analysis: display the root test changes too (#5728)

Browse Source 
* try to compare the root results too

* Add debug info in the logs

* rename the variable for something more explicit

* Improve intermittent doc
This commit is contained in:
Sylvestre Ledru 2023-12-27 08:46:33 +01:00 committed by GitHub
parent e26f4c7f36
commit 413638c789
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 77 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
.github/workflows/GnuTests.yml vendored
View File

@ -204,6 +204,7 @@ jobs:
## Compare test failures VS reference
have_new_failures=""
REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log'
ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log'
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json'
REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}'
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
@ -223,68 +224,92 @@ jobs:
rm -f ${COMMENT_LOG}
touch ${COMMENT_LOG}
if test -f "${REF_LOG_FILE}"; then
echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")"
REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort)
NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort)
REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort)
NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort)
for LINE in ${REF_FAILING}
do
if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
compare_tests() {
local new_log_file=$1
local ref_log_file=$2
local test_type=$3 # "standard" or "root"
if test -f "${ref_log_file}"; then
echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}"
REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
CURRENT_RUN_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
CURRENT_RUN_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
echo "Detailled information:"
echo "REF_ERROR = ${REF_ERROR}"
echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}"
echo "REF_FAILING = ${REF_FAILING}"
echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}"
# Compare failing and error tests
for LINE in ${CURRENT_RUN_FAILING}
do
if ! grep -Fxq ${LINE}<<<"${REF_FAILING}"
then
MSG="Congrats! The gnu test ${LINE} is no longer failing!"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
else
MSG="Skipping an intermittent issue ${LINE}"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
echo ""
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
then
MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
echo "::error ::$MSG"
echo $MSG >> ${COMMENT_LOG}
have_new_failures="true"
else
MSG="Skip an intermittent issue ${LINE} (fails in this run but passes in the 'main' branch)"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
echo ""
fi
fi
fi
done
for LINE in ${NEW_FAILING}
do
if ! grep -Fxq ${LINE}<<<"${REF_FAILING}"
then
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
done
for LINE in ${REF_FAILING}
do
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_FAILING}"
then
MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?"
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
then
MSG="Congrats! The gnu test ${LINE} is no longer failing!"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
else
MSG="Skipping an intermittent issue ${LINE} (passes in this run but fails in the 'main' branch)"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
echo ""
fi
fi
done
for LINE in ${CURRENT_RUN_ERROR}
do
if ! grep -Fxq ${LINE}<<<"${REF_ERROR}"
then
MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
echo "::error ::$MSG"
echo $MSG >> ${COMMENT_LOG}
have_new_failures="true"
else
MSG="Skip an intermittent issue ${LINE}"
fi
done
for LINE in ${REF_ERROR}
do
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_ERROR}"
then
MSG="Congrats! The gnu test ${LINE} is no longer ERROR!"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
echo ""
fi
fi
done
for LINE in ${REF_ERROR}
do
if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}"; then
MSG="Congrats! The gnu test ${LINE} is no longer ERROR!"
echo "::warning ::$MSG"
echo $MSG >> ${COMMENT_LOG}
fi
done
for LINE in ${NEW_ERROR}
do
if ! grep -Fxq ${LINE}<<<"${REF_ERROR}"
then
MSG="GNU test error: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?"
echo "::error ::$MSG"
echo $MSG >> ${COMMENT_LOG}
have_new_failures="true"
fi
done
done
else
echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available."
fi
}
# Compare standard tests
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard"
# Compare root tests
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root"
else
echo "::warning ::Skipping test failure comparison; no prior reference test logs are available."
fi
if test -n "${have_new_failures}" ; then exit -1 ; fi
- name: Upload comparison log (for GnuComment workflow)
if: success() || failure() # run regardless of prior step success/failure