mirror of
https://github.com/uutils/coreutils
synced 2024-10-07 08:29:17 +00:00
Improve the GNU ci analysis: display the root test changes too (#5728)
* try to compare the root results too * Add debug info in the logs * rename the variable for something more explicit * Improve intermittent doc
This commit is contained in:
parent
e26f4c7f36
commit
413638c789
129
.github/workflows/GnuTests.yml
vendored
129
.github/workflows/GnuTests.yml
vendored
|
@ -204,6 +204,7 @@ jobs:
|
||||||
## Compare test failures VS reference
|
## Compare test failures VS reference
|
||||||
have_new_failures=""
|
have_new_failures=""
|
||||||
REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log'
|
REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log'
|
||||||
|
ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log'
|
||||||
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json'
|
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json'
|
||||||
REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}'
|
REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}'
|
||||||
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
|
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
|
||||||
|
@ -223,68 +224,92 @@ jobs:
|
||||||
rm -f ${COMMENT_LOG}
|
rm -f ${COMMENT_LOG}
|
||||||
touch ${COMMENT_LOG}
|
touch ${COMMENT_LOG}
|
||||||
|
|
||||||
if test -f "${REF_LOG_FILE}"; then
|
compare_tests() {
|
||||||
echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")"
|
local new_log_file=$1
|
||||||
REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort)
|
local ref_log_file=$2
|
||||||
NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort)
|
local test_type=$3 # "standard" or "root"
|
||||||
REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort)
|
|
||||||
NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort)
|
if test -f "${ref_log_file}"; then
|
||||||
for LINE in ${REF_FAILING}
|
echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}"
|
||||||
do
|
REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
|
||||||
if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then
|
CURRENT_RUN_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
|
||||||
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
|
REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
|
||||||
|
CURRENT_RUN_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
|
||||||
|
echo "Detailled information:"
|
||||||
|
echo "REF_ERROR = ${REF_ERROR}"
|
||||||
|
echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}"
|
||||||
|
echo "REF_FAILING = ${REF_FAILING}"
|
||||||
|
echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}"
|
||||||
|
|
||||||
|
# Compare failing and error tests
|
||||||
|
for LINE in ${CURRENT_RUN_FAILING}
|
||||||
|
do
|
||||||
|
if ! grep -Fxq ${LINE}<<<"${REF_FAILING}"
|
||||||
then
|
then
|
||||||
MSG="Congrats! The gnu test ${LINE} is no longer failing!"
|
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
|
||||||
echo "::warning ::$MSG"
|
then
|
||||||
echo $MSG >> ${COMMENT_LOG}
|
MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
|
||||||
else
|
echo "::error ::$MSG"
|
||||||
MSG="Skipping an intermittent issue ${LINE}"
|
echo $MSG >> ${COMMENT_LOG}
|
||||||
echo "::warning ::$MSG"
|
have_new_failures="true"
|
||||||
echo $MSG >> ${COMMENT_LOG}
|
else
|
||||||
echo ""
|
MSG="Skip an intermittent issue ${LINE} (fails in this run but passes in the 'main' branch)"
|
||||||
|
echo "::warning ::$MSG"
|
||||||
|
echo $MSG >> ${COMMENT_LOG}
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
done
|
||||||
done
|
|
||||||
for LINE in ${NEW_FAILING}
|
for LINE in ${REF_FAILING}
|
||||||
do
|
do
|
||||||
if ! grep -Fxq ${LINE}<<<"${REF_FAILING}"
|
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_FAILING}"
|
||||||
then
|
|
||||||
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
|
|
||||||
then
|
then
|
||||||
MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?"
|
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
|
||||||
|
then
|
||||||
|
MSG="Congrats! The gnu test ${LINE} is no longer failing!"
|
||||||
|
echo "::warning ::$MSG"
|
||||||
|
echo $MSG >> ${COMMENT_LOG}
|
||||||
|
else
|
||||||
|
MSG="Skipping an intermittent issue ${LINE} (passes in this run but fails in the 'main' branch)"
|
||||||
|
echo "::warning ::$MSG"
|
||||||
|
echo $MSG >> ${COMMENT_LOG}
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
for LINE in ${CURRENT_RUN_ERROR}
|
||||||
|
do
|
||||||
|
if ! grep -Fxq ${LINE}<<<"${REF_ERROR}"
|
||||||
|
then
|
||||||
|
MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
|
||||||
echo "::error ::$MSG"
|
echo "::error ::$MSG"
|
||||||
echo $MSG >> ${COMMENT_LOG}
|
echo $MSG >> ${COMMENT_LOG}
|
||||||
have_new_failures="true"
|
have_new_failures="true"
|
||||||
else
|
fi
|
||||||
MSG="Skip an intermittent issue ${LINE}"
|
done
|
||||||
|
|
||||||
|
for LINE in ${REF_ERROR}
|
||||||
|
do
|
||||||
|
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_ERROR}"
|
||||||
|
then
|
||||||
|
MSG="Congrats! The gnu test ${LINE} is no longer ERROR!"
|
||||||
echo "::warning ::$MSG"
|
echo "::warning ::$MSG"
|
||||||
echo $MSG >> ${COMMENT_LOG}
|
echo $MSG >> ${COMMENT_LOG}
|
||||||
echo ""
|
|
||||||
fi
|
fi
|
||||||
fi
|
done
|
||||||
done
|
else
|
||||||
for LINE in ${REF_ERROR}
|
echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available."
|
||||||
do
|
fi
|
||||||
if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}"; then
|
}
|
||||||
MSG="Congrats! The gnu test ${LINE} is no longer ERROR!"
|
|
||||||
echo "::warning ::$MSG"
|
# Compare standard tests
|
||||||
echo $MSG >> ${COMMENT_LOG}
|
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard"
|
||||||
fi
|
|
||||||
done
|
# Compare root tests
|
||||||
for LINE in ${NEW_ERROR}
|
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root"
|
||||||
do
|
|
||||||
if ! grep -Fxq ${LINE}<<<"${REF_ERROR}"
|
|
||||||
then
|
|
||||||
MSG="GNU test error: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?"
|
|
||||||
echo "::error ::$MSG"
|
|
||||||
echo $MSG >> ${COMMENT_LOG}
|
|
||||||
have_new_failures="true"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
else
|
|
||||||
echo "::warning ::Skipping test failure comparison; no prior reference test logs are available."
|
|
||||||
fi
|
|
||||||
if test -n "${have_new_failures}" ; then exit -1 ; fi
|
if test -n "${have_new_failures}" ; then exit -1 ; fi
|
||||||
- name: Upload comparison log (for GnuComment workflow)
|
- name: Upload comparison log (for GnuComment workflow)
|
||||||
if: success() || failure() # run regardless of prior step success/failure
|
if: success() || failure() # run regardless of prior step success/failure
|
||||||
|
|
Loading…
Reference in a new issue