Merge pull request #4740 from Joining7943/ci-use-nextest-as-test-runner

`ci`: Run tests with nextest

.config/nextest.toml

@@ -0,0 +1,6 @@
+[profile.ci]
+retries = 2
+status-level = "all"
+final-status-level = "skip"
+failure-output = "immediate-final"
+fail-fast = false

.github/workflows/CICD.yml

@@ -1,8 +1,8 @@
 name: CICD
 
-# spell-checker:ignore (abbrev/names) CICD CodeCOV MacOS MinGW MSVC musl
+# spell-checker:ignore (abbrev/names) CICD CodeCOV MacOS MinGW MSVC musl taiki
-# spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic
+# spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic CARGOFLAGS
-# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain
+# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers
 # spell-checker:ignore (people) Peltoche rivy
 # spell-checker:ignore (shell/tools) choco clippy dmake dpkg esac fakeroot fdesc fdescfs gmake grcov halium lcov libssl mkdir popd printf pushd rsync rustc rustfmt rustup shopt utmpdump xargs
 # spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos gnueabihf issuecomment maint multisize nullglob onexitbegin onexitend pell runtest Swatinem tempfile testsuite toybox uutils
@@ -357,6 +357,7 @@ jobs:
         rustup toolchain install --no-self-update ${{ env.RUST_MIN_SRV }} --profile minimal
         rustup default ${{ env.RUST_MIN_SRV }}
     - uses: Swatinem/rust-cache@v2
+    - uses: taiki-e/install-action@nextest
     - name: Run sccache-cache
       uses: mozilla-actions/sccache-action@v0.0.3
     - name: Initialize workflow variables
@@ -404,10 +405,11 @@ jobs:
         RUSTUP_TOOLCHAIN=stable cargo fetch --locked --quiet
         RUSTUP_TOOLCHAIN=stable cargo tree --all --locked --no-dev-dependencies --no-indent ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} | grep -vE "$PWD" | sort --unique
     - name: Test
-      run: cargo test -v ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} -p uucore -p coreutils
+      run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} -p uucore -p coreutils
       env:
         RUSTFLAGS: "-Awarnings"
         RUST_BACKTRACE: "1"
+        CARGO_TERM_COLOR: always
 
   deps:
     name: Dependencies
@@ -451,6 +453,7 @@ jobs:
         ## Install `rust` toolchain
         rustup toolchain install stable --no-self-update --profile minimal
         rustup default stable
+    - uses: taiki-e/install-action@nextest
     - uses: Swatinem/rust-cache@v2
     - name: Run sccache-cache
       uses: mozilla-actions/sccache-action@v0.0.3
@@ -458,12 +461,12 @@ jobs:
       shell: bash
       run: |
         make build
-    - name: "`make test`"
+    - name: "`make nextest`"
       shell: bash
-      run: |
+      run: make nextest CARGOFLAGS="--profile ci --hide-progress-bar"
-        make test
       env:
         RUST_BACKTRACE: "1"
+        CARGO_TERM_COLOR: "always"
     - name: "`make install`"
       shell: bash
       run: |
@@ -497,13 +500,15 @@ jobs:
         ## Install `rust` toolchain
         rustup toolchain install stable --no-self-update --profile minimal
         rustup default stable
+    - uses: taiki-e/install-action@nextest
     - uses: Swatinem/rust-cache@v2
     - name: Run sccache-cache
       uses: mozilla-actions/sccache-action@v0.0.3
     - name: Test
-      run: cargo test ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }}
+      run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }}
       env:
         RUST_BACKTRACE: "1"
+        CARGO_TERM_COLOR: "always"
 
   build_rust_nightly:
     name: Build/nightly
@@ -527,13 +532,15 @@ jobs:
         ## Install `rust` toolchain
         rustup toolchain install nightly --no-self-update --profile minimal
         rustup default nightly
+    - uses: taiki-e/install-action@nextest
     - uses: Swatinem/rust-cache@v2
     - name: Run sccache-cache
       uses: mozilla-actions/sccache-action@v0.0.3
     - name: Test
-      run: cargo test ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }}
+      run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }}
       env:
         RUST_BACKTRACE: "1"
+        CARGO_TERM_COLOR: "always"
 
   compute_size:
     name: Binary sizes
@@ -1060,6 +1067,7 @@ jobs:
         ## rust toolchain ~ install
         rustup toolchain install ${{ matrix.job.toolchain }} --no-self-update --profile minimal
         rustup default ${{ matrix.job.toolchain }}
+    - uses: taiki-e/install-action@nextest
     - uses: Swatinem/rust-cache@v2
     - name: Run sccache-cache
       uses: mozilla-actions/sccache-action@v0.0.3
@@ -1122,7 +1130,7 @@ jobs:
         CARGO_UTILITY_LIST_OPTIONS="$(for u in ${UTILITY_LIST}; do echo -n "-puu_${u} "; done;)"
         outputs CARGO_UTILITY_LIST_OPTIONS
     - name: Test uucore
-      run: cargo test --no-fail-fast -p uucore
+      run: cargo nextest run --profile ci --hide-progress-bar -p uucore
       env:
         CARGO_INCREMENTAL: "0"
         RUSTC_WRAPPER: ""
@@ -1131,7 +1139,7 @@ jobs:
         RUST_BACKTRACE: "1"
         # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }}
     - name: Test
-      run: cargo test ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --no-fail-fast
+      run: cargo nextest run --profile ci --hide-progress-bar ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }}
       env:
         CARGO_INCREMENTAL: "0"
         RUSTC_WRAPPER: ""
@@ -1140,7 +1148,7 @@ jobs:
         RUST_BACKTRACE: "1"
         # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }}
     - name: Test individual utilities
-      run: cargo test --no-fail-fast ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }}
+      run: cargo nextest run --profile ci --hide-progress-bar ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }}
       env:
         CARGO_INCREMENTAL: "0"
         RUSTC_WRAPPER: ""

.github/workflows/android.yml

@@ -1,5 +1,7 @@
 name: Android
 
+# spell-checker:ignore TERMUX reactivecircus Swatinem  noaudio pkill swiftshader
+
 on: [push, pull_request]
 
 permissions:
@@ -30,15 +32,15 @@ jobs:
     - uses: Swatinem/rust-cache@v2
     - name: Run sccache-cache
       uses: mozilla-actions/sccache-action@v0.0.3
-    - name: AVD cache
+    - name: Restore AVD cache
-      uses: actions/cache@v3
+      uses: actions/cache/restore@v3
       id: avd-cache
       with:
         path: |
           ~/.android/avd/*
           ~/.android/avd/*/snapshots/*
           ~/.android/adb*
-        key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}
+        key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest
     - name: Create and cache emulator image
       if: steps.avd-cache.outputs.cache-hit != 'true'
       uses: reactivecircus/android-emulator-runner@v2
@@ -51,11 +53,21 @@ jobs:
         force-avd-creation: true
         emulator-options: -no-snapshot-load -noaudio -no-boot-anim -camera-back none
         script: |
+          set -e
           wget https://github.com/termux/termux-app/releases/download/${{ env.TERMUX }}/termux-app_${{ env.TERMUX }}+github-debug_${{ matrix.arch }}.apk
           util/android-commands.sh snapshot termux-app_${{ env.TERMUX }}+github-debug_${{ matrix.arch }}.apk
           adb -s emulator-5554 emu avd snapshot save ${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}
           echo "Emulator image created."
           pkill -9 qemu-system-x86_64
+    - name: Save AVD cache
+      if: steps.avd-cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v3
+      with:
+        path: |
+          ~/.android/avd/*
+          ~/.android/avd/*/snapshots/*
+          ~/.android/adb*
+        key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest
     - name: Build and Test
       uses: reactivecircus/android-emulator-runner@v2
       with:

.github/workflows/freebsd.yml

@@ -1,6 +1,6 @@
 name: FreeBSD
 
-# spell-checker:ignore sshfs usesh vmactions
+# spell-checker:ignore sshfs usesh vmactions taiki Swatinem esac fdescfs fdesc
 
 env:
   # * style job configuration
@@ -156,6 +156,9 @@ jobs:
           curl https://sh.rustup.rs -sSf --output rustup.sh
           sh rustup.sh -y --profile=minimal
           . $HOME/.cargo/env
+          # Install nextest
+          mkdir -p ~/.cargo/bin
+          curl -LsSf https://get.nexte.st/latest/freebsd | tar zxf - -C ~/.cargo/bin
           ## Info
           # environment
           echo "## environment"
@@ -168,6 +171,7 @@ jobs:
           # tooling info
           echo "## tooling info"
           cargo -V
+          cargo nextest --version
           rustc -V
           #
           # To ensure that files are cleaned up, we don't want to exit on error
@@ -175,9 +179,11 @@ jobs:
           cd "${WORKSPACE}"
           unset FAULT
           cargo build || FAULT=1
+          export PATH=~/.cargo/bin:${PATH}
           export RUST_BACKTRACE=1
-          if (test -z "\$FAULT"); then cargo test --features '${{ matrix.job.features }}' || FAULT=1 ; fi
+          export CARGO_TERM_COLOR=always
-          if (test -z "\$FAULT"); then cargo test --all-features -p uucore || FAULT=1 ; fi
+          if (test -z "\$FAULT"); then cargo nextest run --hide-progress-bar --profile ci --features '${{ matrix.job.features }}' || FAULT=1 ; fi
+          if (test -z "\$FAULT"); then cargo nextest run --hide-progress-bar --profile ci --all-features -p uucore || FAULT=1 ; fi
           # Clean to avoid to rsync back the files
           cargo clean
           if (test -n "\$FAULT"); then exit 1 ; fi

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-<!-- spell-checker:ignore reimplementing toybox RUNTEST -->
+<!-- spell-checker:ignore reimplementing toybox RUNTEST CARGOFLAGS nextest -->
 
 # Contributing to coreutils
 
@@ -137,6 +137,14 @@ If you also want to test the core utilities:
 cargo test  -p uucore -p coreutils
 ```
 
+Running the complete test suite might take a while. We use [nextest](https://nexte.st/index.html) in
+the CI and you might want to try it out locally. It can speed up the execution time of the whole
+test run significantly if the cpu has multiple cores.
+
+```shell
+cargo nextest run --features unix --no-fail-fast
+```
+
 To debug:
 
 ```shell
@@ -171,6 +179,15 @@ To include tests for unimplemented behavior:
 make UTILS='UTILITY_1 UTILITY_2' SPEC=y test
 ```
 
+To run tests with `nextest` just use the nextest target. Note you'll need to
+[install](https://nexte.st/book/installation.html) `nextest` first. The `nextest` target accepts the
+same arguments like the default `test` target, so it's possible to pass arguments to `nextest run`
+via `CARGOFLAGS`:
+
+```shell
+make CARGOFLAGS='--no-fail-fast' UTILS='UTILITY_1 UTILITY_2' nextest
+```
+
 ### Run Busybox Tests
 
 This testing functionality is only available on *nix operating systems and
@@ -326,7 +343,6 @@ if changes are not reflected in the report then run `cargo clean` and run the ab
 If you are using stable version of Rust that doesn't enable code coverage instrumentation by default
 then add `-Z-Zinstrument-coverage` flag to `RUSTFLAGS` env variable specified above.
 
-
 ## Other implementations
 
 The Coreutils have different implementations, with different levels of completions:
@@ -342,7 +358,6 @@ The Coreutils have different implementations, with different levels of completio
 However, when reimplementing the tools/options in Rust, don't read their source codes
 when they are using reciprocal licenses (ex: GNU GPL, GNU LGPL, etc).
 
-
 ## Licensing
 
 uutils is distributed under the terms of the MIT License; see the `LICENSE` file

GNUmakefile

@@ -1,4 +1,4 @@
-# spell-checker:ignore (misc) testsuite runtest findstring (targets) busytest toybox distclean pkgs ; (vars/env) BINDIR BUILDDIR CARGOFLAGS DESTDIR DOCSDIR INSTALLDIR INSTALLEES MULTICALL DATAROOTDIR TESTDIR
+# spell-checker:ignore (misc) testsuite runtest findstring (targets) busytest toybox distclean pkgs nextest ; (vars/env) BINDIR BUILDDIR CARGOFLAGS DESTDIR DOCSDIR INSTALLDIR INSTALLEES MULTICALL DATAROOTDIR TESTDIR
 
 # Config options
 PROFILE         ?= debug
@@ -289,6 +289,9 @@ $(foreach test,$(filter-out $(SKIP_UTILS),$(PROGS)),$(eval $(call TEST_BUSYBOX,$
 test:
 	${CARGO} test ${CARGOFLAGS} --features "$(TESTS) $(TEST_SPEC_FEATURE)" --no-default-features $(TEST_NO_FAIL_FAST)
 
+nextest:
+	${CARGO} nextest run ${CARGOFLAGS} --features "$(TESTS) $(TEST_SPEC_FEATURE)" --no-default-features $(TEST_NO_FAIL_FAST)
+
 test_toybox:
 	-(cd $(TOYBOX_SRC)/ && make tests)
 

util/android-commands.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill
+# spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill nextest
 
 # There are three shells: the host's, adb, and termux. Only adb lets us run
 # commands directly on the emulated device, only termux provides a GNU
@@ -14,7 +14,6 @@
 # success, some other number for errors (an empty file is basically the same as
 # 0). Note that the return codes are text, not raw bytes.
 
-
 this_repo="$(dirname $(dirname -- "$(readlink -- "${0}")"))"
 
 help() {
@@ -62,7 +61,10 @@ run_termux_command() {
     probe="$2"   # unique file that indicates the command is complete
     launch_termux
     adb shell input text "$command" && hit_enter
-    while ! adb shell "ls $probe" 2>/dev/null; do echo "waiting for $probe"; sleep 30; done
+    while ! adb shell "ls $probe" 2>/dev/null; do
+        echo "waiting for $probe"
+        sleep 30
+    done
     return_code=$(adb shell "cat $probe")
     adb shell "rm $probe"
     echo "return code: $return_code"
@@ -73,9 +75,50 @@ snapshot () {
     apk="$1"
     echo "running snapshot"
     adb install -g "$apk"
+
+    echo "Prepare and install system packages"
     probe='/sdcard/pkg.probe'
-    command="'yes | pkg install rust binutils openssl -y; touch $probe'"
+    log='/sdcard/pkg.log'
+    command="'{ mkdir -vp ~/.cargo/bin; yes | pkg install rust binutils openssl -y; echo \$? > $probe; } &> $log'"
     run_termux_command "$command" "$probe"
+    return_code=$?
+
+    adb pull "$log" .
+    cat $(basename "$log")
+
+    if [[ $return_code -ne 0 ]]; then return $return_code; fi
+
+    echo "Installing cargo-nextest"
+    probe='/sdcard/nextest.probe'
+    log='/sdcard/nextest.log'
+    # We need to install nextest via cargo currently, since there is no pre-built binary for android x86
+    command="'cargo install cargo-nextest &> $log; touch $probe'"
+    run_termux_command "$command" "$probe"
+
+    adb pull "$log" .
+    cat $(basename "$log")
+
+    echo "Info about cargo and rust"
+    probe='/sdcard/info.probe'
+    log='/sdcard/info.log'
+    command="'{ \
+        set -x; \
+        echo \$HOME; \
+        PATH=\$HOME/.cargo/bin:\$PATH; \
+        export PATH; \
+        echo \$PATH; \
+        pwd; \
+        command -v rustc && rustc --version; \
+        ls -la ~/.cargo/bin; \
+        cargo --list; \
+        cargo nextest --version; \
+        set +x; \
+        } &> $log; touch $probe'"
+    run_termux_command "$command" "$probe"
+
+    adb pull "$log" .
+    cat $(basename "$log")
+
     echo "snapshot complete"
     adb shell input text "exit" && hit_enter && hit_enter
 }
@@ -120,8 +163,15 @@ build () {
 
 tests() {
     probe='/sdcard/tests.probe'
-    export RUST_BACKTRACE=1
+    command="'\
-    command="'cd ~/coreutils && timeout --preserve-status --verbose -k 1m 60m cargo test --features feat_os_unix_android --no-fail-fast >/sdcard/tests.log 2>&1; echo \$? >$probe'"
+        export PATH=\$HOME/.cargo/bin:\$PATH; \
+        export RUST_BACKTRACE=1; \
+        export CARGO_TERM_COLOR=always; \
+        cd ~/coreutils || { echo 1 > $probe; exit; }; \
+        timeout --preserve-status --verbose -k 1m 60m \
+            cargo nextest run --profile ci --hide-progress-bar --features feat_os_unix_android \
+            &>/sdcard/tests.log; \
+        echo \$? >$probe'"
     run_termux_command "$command" "$probe"
     return_code=$?
     adb pull /sdcard/tests.log .
@@ -134,16 +184,34 @@ exit_code=0
 
 if [ $# -eq 1 ]; then
     case "$1" in
-        sync)     sync "$this_repo"; exit_code=$?;;
+    sync)
-        build)    build; exit_code=$?;;
+        sync "$this_repo"
-        tests)    tests; exit_code=$?;;
+        exit_code=$?
+        ;;
+    build)
+        build
+        exit_code=$?
+        ;;
+    tests)
+        tests
+        exit_code=$?
+        ;;
     *) help ;;
     esac
 elif [ $# -eq 2 ]; then
     case "$1" in
-        snapshot) snapshot "$2"; exit_code=$?;;
+    snapshot)
-        sync)     sync "$2"; exit_code=$?;;
+        snapshot "$2"
-        *)        help; exit 1;;
+        exit_code=$?
+        ;;
+    sync)
+        sync "$2"
+        exit_code=$?
+        ;;
+    *)
+        help
+        exit 1
+        ;;
     esac
 else
     help