mirror of
https://github.com/RGBCube/uutils-coreutils
synced 2025-07-28 11:37:44 +00:00
Merge pull request #2549 from jaggededgedjustice/compare-gnu-test-results
Add step to GnuTest workflow to compare results against master
This commit is contained in:
commit
13b6d003bb
2 changed files with 71 additions and 0 deletions
39
.github/workflows/GnuTests.yml
vendored
39
.github/workflows/GnuTests.yml
vendored
|
@ -90,3 +90,42 @@ jobs:
|
||||||
with:
|
with:
|
||||||
name: gnu-result
|
name: gnu-result
|
||||||
path: gnu-result.json
|
path: gnu-result.json
|
||||||
|
- name: Download the result
|
||||||
|
uses: dawidd6/action-download-artifact@v2
|
||||||
|
with:
|
||||||
|
workflow: GnuTests.yml
|
||||||
|
name: gnu-result
|
||||||
|
repo: uutils/coreutils
|
||||||
|
branch: master
|
||||||
|
path: dl
|
||||||
|
- name: Download the log
|
||||||
|
uses: dawidd6/action-download-artifact@v2
|
||||||
|
with:
|
||||||
|
workflow: GnuTests.yml
|
||||||
|
name: test-report
|
||||||
|
repo: uutils/coreutils
|
||||||
|
branch: master
|
||||||
|
path: dl
|
||||||
|
- name: Compare failing tests against master
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
OLD_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" dl/test-suite.log | sort)
|
||||||
|
NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" gnu/tests/test-suite.log | sort)
|
||||||
|
for LINE in $OLD_FAILING
|
||||||
|
do
|
||||||
|
if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then
|
||||||
|
echo "::warning ::Congrats! The gnu test $LINE is now passing!"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
for LINE in $NEW_FAILING
|
||||||
|
do
|
||||||
|
if ! grep -Fxq $LINE<<<"$OLD_FAILING"
|
||||||
|
then
|
||||||
|
echo "::error ::GNU test failed: $LINE. $LINE is passing on 'master'. Maybe you have to rebase?"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
- name: Compare against master results
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
mv dl/gnu-result.json master-gnu-result.json
|
||||||
|
python uutils/util/compare_gnu_result.py
|
||||||
|
|
32
util/compare_gnu_result.py
Normal file
32
util/compare_gnu_result.py
Normal file
|
@ -0,0 +1,32 @@
|
||||||
|
#! /usr/bin/python
|
||||||
|
|
||||||
|
"""
|
||||||
|
Compare the current results to the last results gathered from the master branch to highlight
|
||||||
|
if a PR is making the results better/worse
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from os import environ
|
||||||
|
|
||||||
|
NEW = json.load(open("gnu-result.json"))
|
||||||
|
OLD = json.load(open("master-gnu-result.json"))
|
||||||
|
|
||||||
|
# Extract the specific results from the dicts
|
||||||
|
last = OLD[list(OLD.keys())[0]]
|
||||||
|
current = NEW[list(NEW.keys())[0]]
|
||||||
|
|
||||||
|
|
||||||
|
pass_d = int(current["pass"]) - int(last["pass"])
|
||||||
|
fail_d = int(current["fail"]) - int(last["fail"])
|
||||||
|
error_d = int(current["error"]) - int(last["error"])
|
||||||
|
skip_d = int(current["skip"]) - int(last["skip"])
|
||||||
|
|
||||||
|
# Get an annotation to highlight changes
|
||||||
|
print(
|
||||||
|
f"::warning ::Changes from master: PASS {pass_d:+d} / FAIL {fail_d:+d} / ERROR {error_d:+d} / SKIP {skip_d:+d} "
|
||||||
|
)
|
||||||
|
|
||||||
|
# If results are worse fail the job to draw attention
|
||||||
|
if pass_d < 0:
|
||||||
|
sys.exit(1)
|
Loading…
Add table
Add a link
Reference in a new issue