mirror of
https://github.com/RGBCube/uutils-coreutils
synced 2025-07-28 11:37:44 +00:00
GNU/CI: use the aggregated-result.json files and move to python (#7471)
* GNU/CI: use the aggregated-result.json files and move to python Co-authored-by: Daniel Hofstetter <daniel.hofstetter@42dh.com> * simplify code Co-authored-by: Daniel Hofstetter <daniel.hofstetter@42dh.com> --------- Co-authored-by: Daniel Hofstetter <daniel.hofstetter@42dh.com>
This commit is contained in:
Sylvestre Ledru
GitHub
parent
c3098cce35
commit
38aee73fe5
2 changed files with 226 additions and 117 deletions
139
.github/workflows/GnuTests.yml
vendored
139
.github/workflows/GnuTests.yml
vendored
|
|
@ -349,137 +349,42 @@ jobs:
|
|||
- name: Compare test failures VS reference
|
||||
shell: bash
|
||||
run: |
|
||||
## Compare test failures VS reference
|
||||
have_new_failures=""
|
||||
REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log'
|
||||
ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log'
|
||||
SELINUX_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/selinux-test-suite.log'
|
||||
SELINUX_ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/selinux-test-suite-root.log'
|
||||
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json'
|
||||
|
||||
|
||||
## Compare test failures VS reference using JSON files
|
||||
REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/aggregated-result/aggregated-result.json'
|
||||
CURRENT_SUMMARY_FILE='${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }}'
|
||||
REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}'
|
||||
path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}'
|
||||
# https://github.com/uutils/coreutils/issues/4294
|
||||
# https://github.com/uutils/coreutils/issues/4295
|
||||
|
||||
# Path to ignore file for intermittent issues
|
||||
IGNORE_INTERMITTENT="${path_UUTILS}/.github/workflows/ignore-intermittent.txt"
|
||||
|
||||
mkdir -p ${{ steps.vars.outputs.path_reference }}
|
||||
|
||||
# Set up comment directory
|
||||
COMMENT_DIR="${{ steps.vars.outputs.path_reference }}/comment"
|
||||
mkdir -p ${COMMENT_DIR}
|
||||
echo ${{ github.event.number }} > ${COMMENT_DIR}/NR
|
||||
COMMENT_LOG="${COMMENT_DIR}/result.txt"
|
||||
|
||||
# The comment log might be downloaded from a previous run
|
||||
# We only want the new changes, so remove it if it exists.
|
||||
rm -f ${COMMENT_LOG}
|
||||
touch ${COMMENT_LOG}
|
||||
COMPARISON_RESULT=0
|
||||
if test -f "${CURRENT_SUMMARY_FILE}"; then
|
||||
if test -f "${REF_SUMMARY_FILE}"; then
|
||||
echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")"
|
||||
echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")"
|
||||
|
||||
compare_tests() {
|
||||
local new_log_file=$1
|
||||
local ref_log_file=$2
|
||||
local test_type=$3 # "standard" or "root"
|
||||
|
||||
if test -f "${ref_log_file}"; then
|
||||
echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}"
|
||||
REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
|
||||
CURRENT_RUN_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
|
||||
REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
|
||||
CURRENT_RUN_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
|
||||
REF_SKIP=$(sed -n "s/^SKIP: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort)
|
||||
CURRENT_RUN_SKIP=$(sed -n "s/^SKIP: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort)
|
||||
|
||||
echo "Detailed information:"
|
||||
echo "REF_ERROR = ${REF_ERROR}"
|
||||
echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}"
|
||||
echo "REF_FAILING = ${REF_FAILING}"
|
||||
echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}"
|
||||
echo "REF_SKIP_PASS = ${REF_SKIP}"
|
||||
echo "CURRENT_RUN_SKIP = ${CURRENT_RUN_SKIP}"
|
||||
|
||||
# Compare failing and error tests
|
||||
for LINE in ${CURRENT_RUN_FAILING}
|
||||
do
|
||||
if ! grep -Fxq ${LINE}<<<"${REF_FAILING}"
|
||||
then
|
||||
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
|
||||
then
|
||||
MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
|
||||
echo "::error ::$MSG"
|
||||
echo $MSG >> ${COMMENT_LOG}
|
||||
have_new_failures="true"
|
||||
else
|
||||
MSG="Skip an intermittent issue ${LINE} (fails in this run but passes in the 'main' branch)"
|
||||
echo "::notice ::$MSG"
|
||||
echo $MSG >> ${COMMENT_LOG}
|
||||
echo ""
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
for LINE in ${REF_FAILING}
|
||||
do
|
||||
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_FAILING}"
|
||||
then
|
||||
if ! grep ${LINE} ${IGNORE_INTERMITTENT}
|
||||
then
|
||||
MSG="Congrats! The gnu test ${LINE} is no longer failing!"
|
||||
echo "::notice ::$MSG"
|
||||
echo $MSG >> ${COMMENT_LOG}
|
||||
else
|
||||
MSG="Skipping an intermittent issue ${LINE} (passes in this run but fails in the 'main' branch)"
|
||||
echo "::notice ::$MSG"
|
||||
echo $MSG >> ${COMMENT_LOG}
|
||||
echo ""
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
for LINE in ${CURRENT_RUN_ERROR}
|
||||
do
|
||||
if ! grep -Fxq ${LINE}<<<"${REF_ERROR}"
|
||||
then
|
||||
MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?"
|
||||
echo "::error ::$MSG"
|
||||
echo $MSG >> ${COMMENT_LOG}
|
||||
have_new_failures="true"
|
||||
fi
|
||||
done
|
||||
|
||||
for LINE in ${REF_ERROR}
|
||||
do
|
||||
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_ERROR}"
|
||||
then
|
||||
MSG="Congrats! The gnu test ${LINE} is no longer ERROR! (might be PASS or FAIL)"
|
||||
echo "::warning ::$MSG"
|
||||
echo $MSG >> ${COMMENT_LOG}
|
||||
fi
|
||||
done
|
||||
|
||||
for LINE in ${REF_SKIP}
|
||||
do
|
||||
if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_SKIP}"
|
||||
then
|
||||
MSG="Congrats! The gnu test ${LINE} is no longer SKIP! (might be PASS, ERROR or FAIL)"
|
||||
echo "::warning ::$MSG"
|
||||
echo $MSG >> ${COMMENT_LOG}
|
||||
fi
|
||||
done
|
||||
python3 ${path_UUTILS}/util/compare_test_results.py \
|
||||
--ignore-file "${IGNORE_INTERMITTENT}" \
|
||||
--output "${COMMENT_LOG}" \
|
||||
"${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}"
|
||||
|
||||
COMPARISON_RESULT=$?
|
||||
else
|
||||
echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available."
|
||||
echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'."
|
||||
fi
|
||||
}
|
||||
else
|
||||
echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Compare standard tests
|
||||
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard"
|
||||
|
||||
# Compare root tests
|
||||
compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root"
|
||||
|
||||
# Set environment variable to indicate whether all failures are intermittent
|
||||
if [ -n "${have_new_failures}" ]; then
|
||||
if [ ${COMPARISON_RESULT} -eq 1 ]; then
|
||||
echo "ONLY_INTERMITTENT=false" >> $GITHUB_ENV
|
||||
echo "::error ::Found new non-intermittent test failures"
|
||||
exit 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue