Merge pull request #3351 from sylvestre/python-script

GNU: add a script to list the failing tests & document how to fix them

README.md

@@ -373,6 +373,25 @@ $ bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example
 
 Note that it relies on individual utilities (not the multicall binary).
 
+### Improving the GNU compatibility
+
+The Python script `./util/remaining-gnu-error.py` shows the list of failing tests in the CI.
+
+To improve the GNU compatibility, the following process is recommended:
+
+1. Identify a test (the smaller, the better) on a program that you understand or is easy to understand. You can use the `./util/remaining-gnu-error.py` script to help with this decision.
+1. Build both the GNU and Rust coreutils using: `bash util/build-gnu.sh`
+1. Run the test with `bash util/run-gnu-test.sh <your test>`
+1. Start to modify `<your test>` to understand what is wrong. Examples:
+    1. Add `set -v` to have the bash verbose mode
+    1. Add `echo $?` where needed
+    1. Bump the content of the output (ex: `cat err`)
+    1. ...
+1. Or, if the test is simple, extract the relevant information to create a new test case running both GNU & Rust implementation
+1. Start to modify the Rust implementation to match the expected behavior
+1. Add a test to make sure that we don't regress (our test suite is super quick)
+
+
 ## Contributing
 
 To contribute to uutils, please see [CONTRIBUTING](CONTRIBUTING.md).

util/remaining-gnu-error.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# This script lists the GNU failing tests by size
+# Just like with util/run-gnu-test.sh, we expect the gnu sources
+# to be in ../
+import urllib.request
+
+import urllib
+import os
+import glob
+import json
+
+base = "../gnu/tests/"
+urllib.request.urlretrieve(
+    "https://raw.githubusercontent.com/uutils/coreutils-tracking/main/gnu-full-result.json",
+    "result.json",
+)
+
+tests = glob.glob(base + "/*/*.sh")
+tests_pl = glob.glob(base + "/*/*.pl")
+tests_xpl = glob.glob(base + "/*/*.xpl")
+tests = tests + tests_pl + tests_xpl
+
+# sort by size
+list_of_files = sorted(tests, key=lambda x: os.stat(x).st_size)
+
+with open("result.json", "r") as json_file:
+    data = json.load(json_file)
+
+for d in data:
+    for e in data[d]:
+        # Not all the tests are .sh files, rename them if not.
+        script = e.replace(".log", ".sh")
+        a = f"{base}{d}{script}"
+        if not os.path.exists(a):
+            a = a.replace(".sh", ".pl")
+            if not os.path.exists(a):
+                a = a.replace(".pl", ".xpl")
+
+        # the tests pass, we don't care anymore
+        if data[d][e] == "PASS":
+            list_of_files.remove(a)
+
+# Remove the factor tests and reverse the list (bigger first)
+tests = list(filter(lambda k: "factor" not in k, list_of_files))
+
+for f in reversed(tests):
+    print("%s: %s" % (f, os.stat(f).st_size))
+print("")
+print("%s tests remaining" % len(tests))