From 55bedbb68e2735adf94e56e0c203cb3693998e47 Mon Sep 17 00:00:00 2001
From: Sylvestre Ledru <sylvestre@debian.org>
Date: Fri, 7 Mar 2025 10:03:59 +0100
Subject: [PATCH] CI: improve the intermittent ignore

---
 .github/workflows/GnuTests.yml |  7 ++++++-
 util/compare_gnu_result.py     | 17 ++++++++++++++---
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml
index 68aa12744..17d2edf2f 100644
--- a/.github/workflows/GnuTests.yml
+++ b/.github/workflows/GnuTests.yml
@@ -340,9 +340,14 @@ jobs:
         # Compare root tests
         compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root"
 
+        # Set environment variable to indicate whether all failures are intermittent
         if [ -n "${have_new_failures}" ]; then
-          echo "::error ::Found new test failures"
+          echo "ONLY_INTERMITTENT=false" >> $GITHUB_ENV
+          echo "::error ::Found new non-intermittent test failures"
           exit 1
+        else
+          echo "ONLY_INTERMITTENT=true" >> $GITHUB_ENV
+          echo "::notice ::No new test failures detected"
         fi
     - name: Upload comparison log (for GnuComment workflow)
       if: success() || failure() # run regardless of prior step success/failure
diff --git a/util/compare_gnu_result.py b/util/compare_gnu_result.py
index 0ea55210d..b18d47065 100755
--- a/util/compare_gnu_result.py
+++ b/util/compare_gnu_result.py
@@ -2,7 +2,8 @@
 
 """
 Compare the current results to the last results gathered from the main branch to highlight
-if a PR is making the results better/worse
+if a PR is making the results better/worse.
+Don't exit with error code if all failing tests are in the ignore-intermittent.txt list.
 """
 
 import json
@@ -10,6 +11,7 @@ import sys
 from os import environ
 
 REPO_DEFAULT_BRANCH = environ.get("REPO_DEFAULT_BRANCH", "main")
+ONLY_INTERMITTENT = environ.get("ONLY_INTERMITTENT", "false")
 
 NEW = json.load(open("gnu-result.json"))
 OLD = json.load(open("main-gnu-result.json"))
@@ -29,9 +31,18 @@ print(
     f"::warning ::Changes from '{REPO_DEFAULT_BRANCH}': PASS {pass_d:+d} / FAIL {fail_d:+d} / ERROR {error_d:+d} / SKIP {skip_d:+d} "
 )
 
-# If results are worse fail the job to draw attention
+# If results are worse, check if we should fail the job
 if pass_d < 0:
     print(
         f"::error ::PASS count is reduced from '{REPO_DEFAULT_BRANCH}': PASS {pass_d:+d} "
     )
-    sys.exit(1)
+
+    # Check if all failing tests are intermittent based on the environment variable
+    only_intermittent = ONLY_INTERMITTENT.lower() == 'true'
+
+    if only_intermittent:
+        print("::notice ::All failing tests are in the ignored intermittent list")
+        print("::notice ::Not failing the build")
+    else:
+        print("::error ::Found non-ignored failing tests")
+        sys.exit(1)