From 6d3345cfec4496afd75b8e8f9ed46c6ba4375c16 Mon Sep 17 00:00:00 2001 From: Laurent Cheylus Date: Mon, 1 Jan 2024 15:11:58 +0100 Subject: [PATCH 1/7] util: use env to call bash in shell scripts - For shell scripts using bash, use #!/usr/bin/env bash instead of #!/bin/bash. On some OS, bash is not the default shell and is not installed as /bin/bash Signed-off-by: Laurent Cheylus --- util/analyze-gnu-results.sh | 2 +- util/android-commands.sh | 2 +- util/build-gnu.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 util/analyze-gnu-results.sh diff --git a/util/analyze-gnu-results.sh b/util/analyze-gnu-results.sh old mode 100644 new mode 100755 index 045e3b93d..76ade340f --- a/util/analyze-gnu-results.sh +++ b/util/analyze-gnu-results.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # spell-checker:ignore xpass XPASS testsuite set -e diff --git a/util/android-commands.sh b/util/android-commands.sh index 4b504de56..652e3d992 100755 --- a/util/android-commands.sh +++ b/util/android-commands.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill nextest logfile # There are three shells: the host's, adb, and termux. Only adb lets us run diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 2d79bd710..c8bd908b0 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # `build-gnu.bash` ~ builds GNU coreutils (from supplied sources) # From 7aca1f932ab72ddea6ae2b0ea0288f6ebfcdb578 Mon Sep 17 00:00:00 2001 From: Laurent Cheylus Date: Mon, 1 Jan 2024 20:31:47 +0100 Subject: [PATCH 2/7] build-gnu.sh: modify script to be compatible with BSD OS - Define variables for GNU version of make, nproc, readlink and sed and use them on BSD. - In specific cases (option -z not available on BSD and with command /c), use GNU sed instead of BSD sed. - For xargs, --no-run-if-empty option is a GNU extension. Replace it by -r to be compatible with FreeBSD and OpenBSD xargs command. Signed-off-by: Laurent Cheylus --- util/build-gnu.sh | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index c8bd908b0..5f851a9e2 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -2,12 +2,28 @@ # `build-gnu.bash` ~ builds GNU coreutils (from supplied sources) # -# spell-checker:ignore (paths) abmon deref discrim eacces getlimits getopt ginstall inacc infloop inotify reflink ; (misc) INT_OFLOW OFLOW baddecode submodules ; (vars/env) SRCDIR vdir rcexp xpart dired +# spell-checker:ignore (paths) abmon deref discrim eacces getlimits getopt ginstall inacc infloop inotify reflink ; (misc) INT_OFLOW OFLOW baddecode submodules ; (vars/env) SRCDIR vdir rcexp xpart dired OSTYPE ; (utils) gnproc greadlink gsed set -e +# Use GNU version for make, nproc, readlink and sed on *BSD +case "$OSTYPE" in + *bsd*) + MAKE="gmake" + NPROC="gnproc" + READLINK="greadlink" + SED="gsed" + ;; + *) + MAKE="make" + NPROC="nproc" + READLINK="readlink" + SED="sed" + ;; +esac + ME="${0}" -ME_dir="$(dirname -- "$(readlink -fm -- "${ME}")")" +ME_dir="$(dirname -- "$("${READLINK}" -fm -- "${ME}")")" REPO_main_dir="$(dirname -- "${ME_dir}")" # Default profile is 'debug' @@ -26,7 +42,7 @@ echo "UU_MAKE_PROFILE='${UU_MAKE_PROFILE}'" ### * config (from environment with fallback defaults); note: GNU is expected to be a sibling repo directory path_UUTILS=${path_UUTILS:-${REPO_main_dir}} -path_GNU="$(readlink -fm -- "${path_GNU:-${path_UUTILS}/../gnu}")" +path_GNU="$("${READLINK}" -fm -- "${path_GNU:-${path_UUTILS}/../gnu}")" ### @@ -78,7 +94,7 @@ if [ "$(uname)" == "Linux" ]; then export SELINUX_ENABLED=1 fi -make PROFILE="${UU_MAKE_PROFILE}" +"${MAKE}" PROFILE="${UU_MAKE_PROFILE}" cp "${UU_BUILD_DIR}/install" "${UU_BUILD_DIR}/ginstall" # The GNU tests rename this script before running, to avoid confusion with the make target # Create *sum binaries @@ -115,7 +131,7 @@ else sed -i 's| tr | /usr/bin/tr |' tests/init.sh # Use a better diff sed -i 's|diff -c|diff -u|g' tests/Coreutils.pm - make -j "$(nproc)" + "${MAKE}" -j "$("${NPROC}")" touch gnu-built fi @@ -135,7 +151,7 @@ t_max=36 # done # ) # for i in ${seq}; do -# make "tests/factor/t${i}.sh" +# "${MAKE}" "tests/factor/t${i}.sh" # done # cat # sed -i -e 's|^seq |/usr/bin/seq |' -e 's|sha1sum |/usr/bin/sha1sum |' tests/factor/t*.sh @@ -194,8 +210,8 @@ sed -i 's|timeout |'"${SYSTEM_TIMEOUT}"' |' tests/tail/follow-stdin.sh sed -i 's|\(^\s*\)seq \$|\1'"${SYSTEM_TIMEOUT}"' 0.1 seq \$|' tests/seq/seq-precision.sh tests/seq/seq-long-double.sh # Remove dup of /usr/bin/ and /usr/local/bin/ when executed several times -grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g' -grep -rlE '/usr/local/bin/\s?/usr/local/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/local/bin/\s?/usr/local/bin/|/usr/local/bin/|g' +grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs -r sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g' +grep -rlE '/usr/local/bin/\s?/usr/local/bin' init.cfg tests/* | xargs -r sed -Ei 's|/usr/local/bin/\s?/usr/local/bin/|/usr/local/bin/|g' #### Adjust tests to make them work with Rust/coreutils # in some cases, what we are doing in rust/coreutils is good (or better) @@ -275,7 +291,8 @@ sed -i -e "s/ginstall: creating directory/install: creating directory/g" tests/i # GNU doesn't support padding < -LONG_MAX # disable this test case -sed -i -Ez "s/\n([^\n#]*pad-3\.2[^\n]*)\n([^\n]*)\n([^\n]*)/\n# uutils\/numfmt supports padding = LONG_MIN\n#\1\n#\2\n#\3/" tests/misc/numfmt.pl +# Use GNU sed because option -z is not available on BSD sed +"${SED}" -i -Ez "s/\n([^\n#]*pad-3\.2[^\n]*)\n([^\n]*)\n([^\n]*)/\n# uutils\/numfmt supports padding = LONG_MIN\n#\1\n#\2\n#\3/" tests/misc/numfmt.pl # Update the GNU error message to match the one generated by clap sed -i -e "s/\$prog: multiple field specifications/error: The argument '--field ' was provided more than once, but cannot be used multiple times\n\nUsage: numfmt [OPTION]... [NUMBER]...\n\n\nFor more information try '--help'/g" tests/misc/numfmt.pl @@ -300,7 +317,8 @@ awk 'BEGIN {count=0} /compare exp out2/ && count < 6 {sub(/compare exp out2/, "g sed -i -e "s|44 45|48 49|" tests/ls/stat-failed.sh # small difference in the error message -sed -i -e "/ls: invalid argument 'XX' for 'time style'/,/Try 'ls --help' for more information\./c\ +# Use GNU sed for /c command +"${SED}" -i -e "/ls: invalid argument 'XX' for 'time style'/,/Try 'ls --help' for more information\./c\ ls: invalid --time-style argument 'XX'\nPossible values are: [\"full-iso\", \"long-iso\", \"iso\", \"locale\", \"+FORMAT (e.g., +%H:%M) for a 'date'-style format\"]\n\nFor more information try --help" tests/ls/time-style-diag.sh # disable two kind of tests: @@ -309,7 +327,8 @@ ls: invalid --time-style argument 'XX'\nPossible values are: [\"full-iso\", \"lo sed -i -e "s/env \$prog \$BEFORE \$opt > out2/env \$prog \$BEFORE \$opt > out2 #/" -e "s/env \$prog \$BEFORE \$opt AFTER > out3/env \$prog \$BEFORE \$opt AFTER > out3 #/" -e "s/compare exp out2/compare exp out2 #/" -e "s/compare exp out3/compare exp out3 #/" tests/help/help-version-getopt.sh # Add debug info + we have less syscall then GNU's. Adjust our check. -sed -i -e '/test \$n_stat1 = \$n_stat2 \\/c\ +# Use GNU sed for /c command +"${SED}" -i -e '/test \$n_stat1 = \$n_stat2 \\/c\ echo "n_stat1 = \$n_stat1"\n\ echo "n_stat2 = \$n_stat2"\n\ test \$n_stat1 -ge \$n_stat2 \\' tests/ls/stat-free-color.sh From d450d5a46342b3c50bccb1b61ff655f509db4538 Mon Sep 17 00:00:00 2001 From: Laurent Cheylus Date: Mon, 1 Jan 2024 22:21:10 +0100 Subject: [PATCH 3/7] show-utils.sh: modify script to be compatible with BSD OS - Use /usr/bin/env bash instead of /bin/sh to define OSTYPE - Use GNU realpath on BSD OS (FreeBSD and OpenBSD) Signed-off-by: Laurent Cheylus --- util/show-utils.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/util/show-utils.sh b/util/show-utils.sh index dda01abe2..f53a92cc8 100755 --- a/util/show-utils.sh +++ b/util/show-utils.sh @@ -1,13 +1,23 @@ -#!/bin/sh +#!/usr/bin/env bash -# spell-checker:ignore (utils) cksum coreutils dircolors hashsum mkdir mktemp printenv printf readlink realpath rmdir shuf tsort unexpand +# spell-checker:ignore (shell) OSTYPE +# spell-checker:ignore (utils) cksum coreutils dircolors hashsum mkdir mktemp printenv printf readlink realpath grealpath rmdir shuf tsort unexpand # spell-checker:ignore (jq) deps startswith +# Use GNU version for realpath on *BSD +case "$OSTYPE" in + *bsd*) + REALPATH="grealpath" + ;; + *) + REALPATH="realpath" + ;; +esac + ME="${0}" ME_dir="$(dirname -- "${ME}")" ME_parent_dir="$(dirname -- "${ME_dir}")" -# NOTE: On FreeBSD, `-mP` arguments are not available. -ME_parent_dir_abs="$(realpath -mP -- "${ME_parent_dir}" || realpath -- "${ME_parent_dir}")" +ME_parent_dir_abs="$("${REALPATH}" -mP -- "${ME_parent_dir}" || "${REALPATH}" -- "${ME_parent_dir}")" # refs: , From aef204461c93ae3de8a33d278d0e508643d9e1da Mon Sep 17 00:00:00 2001 From: Laurent Cheylus Date: Mon, 1 Jan 2024 23:44:10 +0100 Subject: [PATCH 4/7] util: modify scripts for code coverage to be compatible with BSD OS - Use /usr/bin/env bash instead of /bin/sh to use OSTYPE - Use readlink from GNU coreutils on BSD OS in util/build-code_coverage.sh and util/show-code_coverage.sh scripts Signed-off-by: Laurent Cheylus --- util/build-code_coverage.sh | 18 ++++++++++++++---- util/show-code_coverage.sh | 16 +++++++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/util/build-code_coverage.sh b/util/build-code_coverage.sh index 4082bc13d..bbe4abaab 100755 --- a/util/build-code_coverage.sh +++ b/util/build-code_coverage.sh @@ -1,15 +1,25 @@ -#!/bin/sh +#!/usr/bin/env bash # spell-checker:ignore (abbrevs/acronyms) HTML gcno llvm # spell-checker:ignore (jargon) toolchain # spell-checker:ignore (rust) Ccodegen Cinline Coverflow Cpanic RUSTC RUSTDOCFLAGS RUSTFLAGS RUSTUP Zpanic -# spell-checker:ignore (shell) OSID esac -# spell-checker:ignore (utils) genhtml grcov lcov readlink sccache shellcheck uutils +# spell-checker:ignore (shell) OSID OSTYPE esac +# spell-checker:ignore (utils) genhtml grcov lcov greadlink readlink sccache shellcheck uutils FEATURES_OPTION="--features feat_os_unix" +# Use GNU coreutils for readlink on *BSD +case "$OSTYPE" in + *bsd*) + READLINK="greadlink" + ;; + *) + READLINK="readlink" + ;; +esac + ME="${0}" -ME_dir="$(dirname -- "$(readlink -fm -- "${ME}")")" +ME_dir="$(dirname -- "$("${READLINK}" -fm -- "${ME}")")" REPO_main_dir="$(dirname -- "${ME_dir}")" cd "${REPO_main_dir}" && diff --git a/util/show-code_coverage.sh b/util/show-code_coverage.sh index 3f51462c9..8c6f5e20a 100755 --- a/util/show-code_coverage.sh +++ b/util/show-code_coverage.sh @@ -1,9 +1,19 @@ -#!/bin/sh +#!/usr/bin/env bash -# spell-checker:ignore (vars) OSID binfmt +# spell-checker:ignore (vars) OSID OSTYPE binfmt greadlink + +# Use GNU coreutils for readlink on *BSD +case "$OSTYPE" in + *bsd*) + READLINK="greadlink" + ;; + *) + READLINK="readlink" + ;; +esac ME="${0}" -ME_dir="$(dirname -- "$(readlink -fm -- "${ME}")")" +ME_dir="$(dirname -- "$("${READLINK}" -fm -- "${ME}")")" REPO_main_dir="$(dirname -- "${ME_dir}")" export COVERAGE_REPORT_DIR="${REPO_main_dir}/target/debug/coverage-nix" From 13665da85ed1cbe06a76ae2673d8eac3f5891af8 Mon Sep 17 00:00:00 2001 From: Laurent Cheylus Date: Tue, 2 Jan 2024 19:11:34 +0100 Subject: [PATCH 5/7] run-gnu-test.sh: modify script to be compatible with BSD OS - Use /usr/bin/env bash instead of /bin/sh to use OSTYPE - Define variables for GNU version of make, nproc, readlink and use them on BSD. Signed-off-by: Laurent Cheylus --- util/run-gnu-test.sh | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index 1abb476b7..4148c3f96 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -1,13 +1,27 @@ -#!/bin/sh +#!/usr/bin/env bash # `run-gnu-test.bash [TEST]` # run GNU test (or all tests if TEST is missing/null) -# spell-checker:ignore (env/vars) GNULIB SRCDIR SUBDIRS ; (utils) shellcheck +# spell-checker:ignore (env/vars) GNULIB SRCDIR SUBDIRS OSTYPE ; (utils) shellcheck gnproc greadlink # ref: [How the GNU coreutils are tested](https://www.pixelbeat.org/docs/coreutils-testing.html) @@ # * note: to run a single test => `make check TESTS=PATH/TO/TEST/SCRIPT SUBDIRS=. VERBOSE=yes` -ME_dir="$(dirname -- "$(readlink -fm -- "$0")")" +# Use GNU version for make, nproc, readlink on *BSD +case "$OSTYPE" in + *bsd*) + MAKE="gmake" + NPROC="gnproc" + READLINK="greadlink" + ;; + *) + MAKE="make" + NPROC="nproc" + READLINK="readlink" + ;; +esac + +ME_dir="$(dirname -- "$("${READLINK}" -fm -- "$0")")" REPO_main_dir="$(dirname -- "${ME_dir}")" echo "ME_dir='${ME_dir}'" @@ -18,7 +32,7 @@ set -e ### * config (from environment with fallback defaults); note: GNU and GNULIB are expected to be sibling repo directories path_UUTILS=${path_UUTILS:-${REPO_main_dir}} -path_GNU="$(readlink -fm -- "${path_GNU:-${path_UUTILS}/../gnu}")" +path_GNU="$("${READLINK}" -fm -- "${path_GNU:-${path_UUTILS}/../gnu}")" echo "path_UUTILS='${path_UUTILS}'" echo "path_GNU='${path_GNU}'" @@ -47,7 +61,7 @@ if test "$1" != "run-root"; then fi done # trim it - SPECIFIC_TESTS=$(echo $SPECIFIC_TESTS | xargs) + SPECIFIC_TESTS=$(echo "$SPECIFIC_TESTS" | xargs) echo "Running specific tests: $SPECIFIC_TESTS" fi fi @@ -60,16 +74,16 @@ fi if test "$1" != "run-root"; then # run the regular tests if test $# -ge 1; then - timeout -sKILL 4h make -j "$(nproc)" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make + timeout -sKILL 4h "${MAKE}" -j "$("${NPROC}")" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make else - timeout -sKILL 4h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make + timeout -sKILL 4h "${MAKE}" -j "$("${NPROC}")" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make fi else # in case we would like to run tests requiring root if test -z "$1" -o "$1" == "run-root"; then if test -n "$CI"; then echo "Running check-root to run only root tests" - sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : + sudo "${MAKE}" -j "$("${NPROC}")" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : fi fi fi From 0bc70e3ba1535206ac1194d7a683f34c7ffd23bd Mon Sep 17 00:00:00 2001 From: Laurent Cheylus Date: Tue, 2 Jan 2024 19:28:28 +0100 Subject: [PATCH 6/7] CI: install GNU coreutils package for FreeBSD workflow - util/show-utils.sh script is used by FreeBSD workflow => use realpath command from GNU coreutils instead of FreeBSD realpath. - install GNU coreutils (FreeBSD package coreutils) in style job Signed-off-by: Laurent Cheylus --- .github/workflows/freebsd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index c4d441a55..25655f091 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -44,8 +44,8 @@ jobs: usesh: true sync: rsync copyback: false - # We need jq to run show-utils.sh and bash to use inline shell string replacement - prepare: pkg install -y curl sudo jq bash + # We need jq and GNU coreutils to run show-utils.sh and bash to use inline shell string replacement + prepare: pkg install -y curl sudo jq coreutils bash run: | ## Prepare, build, and test # implementation modelled after ref: From be587980780cbed0a7ece95a61ed8f52d7a159e4 Mon Sep 17 00:00:00 2001 From: Laurent Cheylus Date: Tue, 2 Jan 2024 20:31:41 +0100 Subject: [PATCH 7/7] DEVELOPMENT.md: on FreeBSD, install GNU coreutils package to build/run tests Signed-off-by: Laurent Cheylus --- DEVELOPMENT.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 29cfe2b2a..6f1de3b54 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,4 +1,4 @@ - + # Setting up your local development environment @@ -241,6 +241,12 @@ DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl Note that GNU test suite relies on individual utilities (not the multicall binary). +On FreeBSD, you need to install packages for GNU coreutils and sed (used in shell scripts instead of system commands): + +```shell +pkg install coreutils gsed +``` + ## Code coverage report Code coverage report can be generated using [grcov](https://github.com/mozilla/grcov).