From 81a5f0a4dc7f9a4de7598339d39558171e69940f Mon Sep 17 00:00:00 2001
From: James Robson <jaggededgedjustice@gmail.com>
Date: Thu, 5 Aug 2021 15:44:03 +0100
Subject: [PATCH 1/2] Add step to GnuTest workflow to compare results against
 master

---
 .github/workflows/GnuTests.yml | 13 +++++++++++++
 util/compare_gnu_result.py     | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 util/compare_gnu_result.py

diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml
index 8bf6c091b..fe225d2fa 100644
--- a/.github/workflows/GnuTests.yml
+++ b/.github/workflows/GnuTests.yml
@@ -90,3 +90,16 @@ jobs:
       with:
         name: gnu-result
         path: gnu-result.json
+    - name: Download the result
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        workflow: GnuTests.yml
+        name: gnu-result
+        repo: uutils/coreutils
+        branch: master
+        path: dl
+    - name: Compare against master results
+      shell: bash
+      run: |
+        mv dl/gnu-result.json master-gnu-result.json
+        python uutils/util/compare_gnu_result.py
diff --git a/util/compare_gnu_result.py b/util/compare_gnu_result.py
new file mode 100644
index 000000000..52aa96abe
--- /dev/null
+++ b/util/compare_gnu_result.py
@@ -0,0 +1,32 @@
+#! /usr/bin/python
+
+"""
+Compare the current results to the last results gathered from the master branch to highlight
+if a PR is making the results better/worse
+"""
+
+import json
+import sys
+from os import environ
+
+NEW = json.load(open("gnu-result.json"))
+OLD = json.load(open("master-gnu-result.json"))
+
+# Extract the specific results from the dicts
+last = OLD[list(OLD.keys())[0]]
+current = NEW[list(NEW.keys())[0]]
+
+
+pass_d = int(current["pass"]) - int(last["pass"])
+fail_d = int(current["fail"]) - int(last["fail"])
+error_d = int(current["error"]) - int(last["error"])
+skip_d = int(current["skip"]) - int(last["skip"])
+
+# Get an annotation to highlight changes
+print(
+    f"::warning ::Changes from master: PASS {pass_d:+d} / FAIL {fail_d:+d} / ERROR {error_d:+d} / SKIP {skip_d:+d} "
+)
+
+# If results are worse fail the job to draw attention
+if pass_d < 0:
+    sys.exit(1)

From 882b5ad1f162da14d591c56b1006d7ef26ca5375 Mon Sep 17 00:00:00 2001
From: James Robson <jaggededgedjustice@gmail.com>
Date: Sun, 8 Aug 2021 15:48:38 +0100
Subject: [PATCH 2/2] Display changes in the failing GNU tests

Co-authored-by: Michael Debertol <michael.debertol@gmail.com>
---
 .github/workflows/GnuTests.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml
index fe225d2fa..dad53f20c 100644
--- a/.github/workflows/GnuTests.yml
+++ b/.github/workflows/GnuTests.yml
@@ -98,6 +98,32 @@ jobs:
         repo: uutils/coreutils
         branch: master
         path: dl
+    - name: Download the log
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        workflow: GnuTests.yml
+        name: test-report
+        repo: uutils/coreutils
+        branch: master
+        path: dl
+    - name: Compare failing tests against master
+      shell: bash
+      run: |
+            OLD_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" dl/test-suite.log | sort)
+            NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" gnu/tests/test-suite.log | sort)
+            for LINE in $OLD_FAILING
+            do
+              if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then
+                echo "::warning ::Congrats! The gnu test $LINE is now passing!"
+              fi
+            done
+            for LINE in $NEW_FAILING
+            do
+              if ! grep -Fxq $LINE<<<"$OLD_FAILING"
+              then
+                echo "::error ::GNU test failed: $LINE. $LINE is passing on 'master'. Maybe you have to rebase?"
+              fi
+            done
     - name: Compare against master results
       shell: bash
       run: |