Merge pull request #5774 from lcheylus/bsd-scripts

Fix util shell scripts to be compatible with BSD

.github/workflows/freebsd.yml

@@ -44,8 +44,8 @@ jobs:
         usesh: true
         sync: rsync
         copyback: false
-        # We need jq to run show-utils.sh and bash to use inline shell string replacement
+        # We need jq and GNU coreutils to run show-utils.sh and bash to use inline shell string replacement
-        prepare: pkg install -y curl sudo jq bash
+        prepare: pkg install -y curl sudo jq coreutils bash
         run: |
           ## Prepare, build, and test
           # implementation modelled after ref: <https://github.com/rust-lang/rustup/pull/2783>

DEVELOPMENT.md

@@ -1,4 +1,4 @@
-<!-- spell-checker:ignore (flags) Ccodegen Coverflow Cpanic Zinstrument Zpanic reimplementing toybox RUNTEST CARGOFLAGS nextest prereq autopoint gettext texinfo automake findutils shellenv libexec gnubin toolchains -->
+<!-- spell-checker:ignore (flags) Ccodegen Coverflow Cpanic Zinstrument Zpanic reimplementing toybox RUNTEST CARGOFLAGS nextest prereq autopoint gettext texinfo automake findutils shellenv libexec gnubin toolchains gsed -->
 
 # Setting up your local development environment
 
@@ -241,6 +241,12 @@ DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl
 
 Note that GNU test suite relies on individual utilities (not the multicall binary).
 
+On FreeBSD, you need to install packages for GNU coreutils and sed (used in shell scripts instead of system commands):
+
+```shell
+pkg install coreutils gsed
+```
+
 ## Code coverage report
 
 Code coverage report can be generated using [grcov](https://github.com/mozilla/grcov).

util/analyze-gnu-results.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # spell-checker:ignore xpass XPASS testsuite
 set -e
 

util/android-commands.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill nextest logfile
 
 # There are three shells: the host's, adb, and termux. Only adb lets us run

util/build-code_coverage.sh

@@ -1,15 +1,25 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 # spell-checker:ignore (abbrevs/acronyms) HTML gcno llvm
 # spell-checker:ignore (jargon) toolchain
 # spell-checker:ignore (rust) Ccodegen Cinline Coverflow Cpanic RUSTC RUSTDOCFLAGS RUSTFLAGS RUSTUP Zpanic
-# spell-checker:ignore (shell) OSID esac
+# spell-checker:ignore (shell) OSID OSTYPE esac
-# spell-checker:ignore (utils) genhtml grcov lcov readlink sccache shellcheck uutils
+# spell-checker:ignore (utils) genhtml grcov lcov greadlink readlink sccache shellcheck uutils
 
 FEATURES_OPTION="--features feat_os_unix"
 
+# Use GNU coreutils for readlink on *BSD
+case "$OSTYPE" in
+    *bsd*)
+        READLINK="greadlink"
+        ;;
+    *)
+        READLINK="readlink"
+        ;;
+esac
+
 ME="${0}"
-ME_dir="$(dirname -- "$(readlink -fm -- "${ME}")")"
+ME_dir="$(dirname -- "$("${READLINK}" -fm -- "${ME}")")"
 REPO_main_dir="$(dirname -- "${ME_dir}")"
 
 cd "${REPO_main_dir}" &&

util/build-gnu.sh

@@ -1,13 +1,29 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # `build-gnu.bash` ~ builds GNU coreutils (from supplied sources)
 #
 
-# spell-checker:ignore (paths) abmon deref discrim eacces getlimits getopt ginstall inacc infloop inotify reflink ; (misc) INT_OFLOW OFLOW baddecode submodules ; (vars/env) SRCDIR vdir rcexp xpart dired
+# spell-checker:ignore (paths) abmon deref discrim eacces getlimits getopt ginstall inacc infloop inotify reflink ; (misc) INT_OFLOW OFLOW baddecode submodules ; (vars/env) SRCDIR vdir rcexp xpart dired OSTYPE ; (utils) gnproc greadlink gsed
 
 set -e
 
+# Use GNU version for make, nproc, readlink and sed on *BSD
+case "$OSTYPE" in
+    *bsd*)
+        MAKE="gmake"
+        NPROC="gnproc"
+        READLINK="greadlink"
+        SED="gsed"
+        ;;
+    *)
+        MAKE="make"
+        NPROC="nproc"
+        READLINK="readlink"
+        SED="sed"
+        ;;
+esac
+
 ME="${0}"
-ME_dir="$(dirname -- "$(readlink -fm -- "${ME}")")"
+ME_dir="$(dirname -- "$("${READLINK}" -fm -- "${ME}")")"
 REPO_main_dir="$(dirname -- "${ME_dir}")"
 
 # Default profile is 'debug'
@@ -26,7 +42,7 @@ echo "UU_MAKE_PROFILE='${UU_MAKE_PROFILE}'"
 ### * config (from environment with fallback defaults); note: GNU is expected to be a sibling repo directory
 
 path_UUTILS=${path_UUTILS:-${REPO_main_dir}}
-path_GNU="$(readlink -fm -- "${path_GNU:-${path_UUTILS}/../gnu}")"
+path_GNU="$("${READLINK}" -fm -- "${path_GNU:-${path_UUTILS}/../gnu}")"
 
 ###
 
@@ -78,7 +94,7 @@ if [ "$(uname)" == "Linux" ]; then
     export SELINUX_ENABLED=1
 fi
 
-make PROFILE="${UU_MAKE_PROFILE}"
+"${MAKE}" PROFILE="${UU_MAKE_PROFILE}"
 
 cp "${UU_BUILD_DIR}/install" "${UU_BUILD_DIR}/ginstall" # The GNU tests rename this script before running, to avoid confusion with the make target
 # Create *sum binaries
@@ -115,7 +131,7 @@ else
     sed -i 's| tr | /usr/bin/tr |' tests/init.sh
     # Use a better diff
     sed -i 's|diff -c|diff -u|g' tests/Coreutils.pm
-    make -j "$(nproc)"
+    "${MAKE}" -j "$("${NPROC}")"
     touch gnu-built
 fi
 
@@ -135,7 +151,7 @@ t_max=36
 #         done
 #     )
 #     for i in ${seq}; do
-#         make "tests/factor/t${i}.sh"
+#         "${MAKE}" "tests/factor/t${i}.sh"
 #     done
 #     cat
 #     sed -i -e 's|^seq |/usr/bin/seq |' -e 's|sha1sum |/usr/bin/sha1sum |' tests/factor/t*.sh
@@ -194,8 +210,8 @@ sed -i 's|timeout |'"${SYSTEM_TIMEOUT}"' |' tests/tail/follow-stdin.sh
 sed -i 's|\(^\s*\)seq \$|\1'"${SYSTEM_TIMEOUT}"' 0.1 seq \$|' tests/seq/seq-precision.sh tests/seq/seq-long-double.sh
 
 # Remove dup of /usr/bin/ and /usr/local/bin/ when executed several times
-grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g'
+grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs -r sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g'
-grep -rlE '/usr/local/bin/\s?/usr/local/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/local/bin/\s?/usr/local/bin/|/usr/local/bin/|g'
+grep -rlE '/usr/local/bin/\s?/usr/local/bin' init.cfg tests/* | xargs -r sed -Ei 's|/usr/local/bin/\s?/usr/local/bin/|/usr/local/bin/|g'
 
 #### Adjust tests to make them work with Rust/coreutils
 # in some cases, what we are doing in rust/coreutils is good (or better)
@@ -275,7 +291,8 @@ sed -i -e "s/ginstall: creating directory/install: creating directory/g" tests/i
 
 # GNU doesn't support padding < -LONG_MAX
 # disable this test case
-sed -i -Ez "s/\n([^\n#]*pad-3\.2[^\n]*)\n([^\n]*)\n([^\n]*)/\n# uutils\/numfmt supports padding = LONG_MIN\n#\1\n#\2\n#\3/" tests/misc/numfmt.pl
+# Use GNU sed because option -z is not available on BSD sed
+"${SED}" -i -Ez "s/\n([^\n#]*pad-3\.2[^\n]*)\n([^\n]*)\n([^\n]*)/\n# uutils\/numfmt supports padding = LONG_MIN\n#\1\n#\2\n#\3/" tests/misc/numfmt.pl
 
 # Update the GNU error message to match the one generated by clap
 sed -i -e "s/\$prog: multiple field specifications/error: The argument '--field <FIELDS>' was provided more than once, but cannot be used multiple times\n\nUsage: numfmt [OPTION]... [NUMBER]...\n\n\nFor more information try '--help'/g" tests/misc/numfmt.pl
@@ -300,7 +317,8 @@ awk 'BEGIN {count=0} /compare exp out2/ && count < 6 {sub(/compare exp out2/, "g
 sed -i -e "s|44 45|48 49|" tests/ls/stat-failed.sh
 
 # small difference in the error message
-sed -i -e "/ls: invalid argument 'XX' for 'time style'/,/Try 'ls --help' for more information\./c\
+# Use GNU sed for /c command
+"${SED}" -i -e "/ls: invalid argument 'XX' for 'time style'/,/Try 'ls --help' for more information\./c\
 ls: invalid --time-style argument 'XX'\nPossible values are: [\"full-iso\", \"long-iso\", \"iso\", \"locale\", \"+FORMAT (e.g., +%H:%M) for a 'date'-style format\"]\n\nFor more information try --help" tests/ls/time-style-diag.sh
 
 # disable two kind of tests:
@@ -309,7 +327,8 @@ ls: invalid --time-style argument 'XX'\nPossible values are: [\"full-iso\", \"lo
 sed -i -e "s/env \$prog \$BEFORE \$opt > out2/env \$prog \$BEFORE \$opt > out2 #/" -e "s/env \$prog \$BEFORE \$opt AFTER > out3/env \$prog \$BEFORE \$opt AFTER > out3 #/" -e "s/compare exp out2/compare exp out2 #/" -e "s/compare exp out3/compare exp out3 #/" tests/help/help-version-getopt.sh
 
 # Add debug info + we have less syscall then GNU's. Adjust our check.
-sed -i -e '/test \$n_stat1 = \$n_stat2 \\/c\
+# Use GNU sed for /c command
+"${SED}" -i -e '/test \$n_stat1 = \$n_stat2 \\/c\
 echo "n_stat1 = \$n_stat1"\n\
 echo "n_stat2 = \$n_stat2"\n\
 test \$n_stat1 -ge \$n_stat2 \\' tests/ls/stat-free-color.sh

util/run-gnu-test.sh

@@ -1,13 +1,27 @@
-#!/bin/sh
+#!/usr/bin/env bash
 # `run-gnu-test.bash [TEST]`
 # run GNU test (or all tests if TEST is missing/null)
 
-# spell-checker:ignore (env/vars) GNULIB SRCDIR SUBDIRS ; (utils) shellcheck
+# spell-checker:ignore (env/vars) GNULIB SRCDIR SUBDIRS OSTYPE ; (utils) shellcheck gnproc greadlink
 
 # ref: [How the GNU coreutils are tested](https://www.pixelbeat.org/docs/coreutils-testing.html) @@ <https://archive.is/p2ITW>
 # * note: to run a single test => `make check TESTS=PATH/TO/TEST/SCRIPT SUBDIRS=. VERBOSE=yes`
 
-ME_dir="$(dirname -- "$(readlink -fm -- "$0")")"
+# Use GNU version for make, nproc, readlink on *BSD
+case "$OSTYPE" in
+    *bsd*)
+        MAKE="gmake"
+        NPROC="gnproc"
+        READLINK="greadlink"
+        ;;
+    *)
+        MAKE="make"
+        NPROC="nproc"
+        READLINK="readlink"
+        ;;
+esac
+
+ME_dir="$(dirname -- "$("${READLINK}" -fm -- "$0")")"
 REPO_main_dir="$(dirname -- "${ME_dir}")"
 
 echo "ME_dir='${ME_dir}'"
@@ -18,7 +32,7 @@ set -e
 ### * config (from environment with fallback defaults); note: GNU and GNULIB are expected to be sibling repo directories
 
 path_UUTILS=${path_UUTILS:-${REPO_main_dir}}
-path_GNU="$(readlink -fm -- "${path_GNU:-${path_UUTILS}/../gnu}")"
+path_GNU="$("${READLINK}" -fm -- "${path_GNU:-${path_UUTILS}/../gnu}")"
 
 echo "path_UUTILS='${path_UUTILS}'"
 echo "path_GNU='${path_GNU}'"
@@ -47,7 +61,7 @@ if test "$1" != "run-root"; then
             fi
         done
         # trim it
-        SPECIFIC_TESTS=$(echo $SPECIFIC_TESTS | xargs)
+        SPECIFIC_TESTS=$(echo "$SPECIFIC_TESTS" | xargs)
         echo "Running specific tests: $SPECIFIC_TESTS"
     fi
 fi
@@ -60,16 +74,16 @@ fi
 if test "$1" != "run-root"; then
     # run the regular tests
     if test $# -ge 1; then
-        timeout -sKILL 4h make -j "$(nproc)" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make
+        timeout -sKILL 4h "${MAKE}" -j "$("${NPROC}")" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make
     else
-        timeout -sKILL 4h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make
+        timeout -sKILL 4h "${MAKE}" -j "$("${NPROC}")" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make
     fi
 else
     # in case we would like to run tests requiring root
     if test -z "$1" -o "$1" == "run-root"; then
         if test -n "$CI"; then
             echo "Running check-root to run only root tests"
-            sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || :
+            sudo "${MAKE}" -j "$("${NPROC}")" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || :
         fi
     fi
 fi

util/show-code_coverage.sh

@@ -1,9 +1,19 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-# spell-checker:ignore (vars) OSID binfmt
+# spell-checker:ignore (vars) OSID OSTYPE binfmt greadlink
+
+# Use GNU coreutils for readlink on *BSD
+case "$OSTYPE" in
+    *bsd*)
+        READLINK="greadlink"
+        ;;
+    *)
+        READLINK="readlink"
+        ;;
+esac
 
 ME="${0}"
-ME_dir="$(dirname -- "$(readlink -fm -- "${ME}")")"
+ME_dir="$(dirname -- "$("${READLINK}" -fm -- "${ME}")")"
 REPO_main_dir="$(dirname -- "${ME_dir}")"
 
 export COVERAGE_REPORT_DIR="${REPO_main_dir}/target/debug/coverage-nix"

util/show-utils.sh

@@ -1,13 +1,23 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-# spell-checker:ignore (utils) cksum coreutils dircolors hashsum mkdir mktemp printenv printf readlink realpath  rmdir shuf tsort unexpand
+# spell-checker:ignore (shell) OSTYPE
+# spell-checker:ignore (utils) cksum coreutils dircolors hashsum mkdir mktemp printenv printf readlink realpath grealpath rmdir shuf tsort unexpand
 # spell-checker:ignore (jq) deps startswith
 
+# Use GNU version for realpath on *BSD
+case "$OSTYPE" in
+    *bsd*)
+        REALPATH="grealpath"
+        ;;
+    *)
+        REALPATH="realpath"
+        ;;
+esac
+
 ME="${0}"
 ME_dir="$(dirname -- "${ME}")"
 ME_parent_dir="$(dirname -- "${ME_dir}")"
-# NOTE: On FreeBSD, `-mP` arguments are not available.
+ME_parent_dir_abs="$("${REALPATH}" -mP -- "${ME_parent_dir}" || "${REALPATH}" -- "${ME_parent_dir}")"
-ME_parent_dir_abs="$(realpath -mP -- "${ME_parent_dir}" || realpath -- "${ME_parent_dir}")"
 
 # refs: <https://forge.rust-lang.org/release/platform-support.html> , <https://docs.rs/platforms/0.2.1/platforms/platform/tier1/index.html>