From 78ce521c01f902955a50d6cf8b88b42b05df37a4 Mon Sep 17 00:00:00 2001 From: Joining7943 <111500881+Joining7943@users.noreply.github.com> Date: Fri, 14 Apr 2023 19:19:57 +0200 Subject: [PATCH 1/4] ci: Use nextest as test runner instead of cargo test --- .config/nextest.toml | 6 ++++ .github/workflows/CICD.yml | 32 ++++++++++++-------- .github/workflows/android.yml | 2 ++ .github/workflows/freebsd.yml | 12 ++++++-- GNUmakefile | 5 +++- util/android-commands.sh | 56 ++++++++++++++++++++++++++++++++--- 6 files changed, 93 insertions(+), 20 deletions(-) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 000000000..3ba8bb393 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,6 @@ +[profile.ci] +retries = 2 +status-level = "all" +final-status-level = "skip" +failure-output = "immediate-final" +fail-fast = false diff --git a/.github/workflows/CICD.yml b/.github/workflows/CICD.yml index 2a67c7d97..32fd30536 100644 --- a/.github/workflows/CICD.yml +++ b/.github/workflows/CICD.yml @@ -1,8 +1,8 @@ name: CICD -# spell-checker:ignore (abbrev/names) CICD CodeCOV MacOS MinGW MSVC musl -# spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic -# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain +# spell-checker:ignore (abbrev/names) CICD CodeCOV MacOS MinGW MSVC musl taiki +# spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic CARGOFLAGS +# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers # spell-checker:ignore (people) Peltoche rivy # spell-checker:ignore (shell/tools) choco clippy dmake dpkg esac fakeroot fdesc fdescfs gmake grcov halium lcov libssl mkdir popd printf pushd rsync rustc rustfmt rustup shopt utmpdump xargs # spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos gnueabihf issuecomment maint multisize nullglob onexitbegin onexitend pell runtest Swatinem tempfile testsuite toybox uutils @@ -357,6 +357,7 @@ jobs: rustup toolchain install --no-self-update ${{ env.RUST_MIN_SRV }} --profile minimal rustup default ${{ env.RUST_MIN_SRV }} - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 - name: Initialize workflow variables @@ -404,10 +405,11 @@ jobs: RUSTUP_TOOLCHAIN=stable cargo fetch --locked --quiet RUSTUP_TOOLCHAIN=stable cargo tree --all --locked --no-dev-dependencies --no-indent ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} | grep -vE "$PWD" | sort --unique - name: Test - run: cargo test -v ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} -p uucore -p coreutils + run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} -p uucore -p coreutils env: RUSTFLAGS: "-Awarnings" RUST_BACKTRACE: "1" + CARGO_TERM_COLOR: always deps: name: Dependencies @@ -451,6 +453,7 @@ jobs: ## Install `rust` toolchain rustup toolchain install stable --no-self-update --profile minimal rustup default stable + - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 @@ -458,12 +461,12 @@ jobs: shell: bash run: | make build - - name: "`make test`" + - name: "`make nextest`" shell: bash - run: | - make test + run: make nextest CARGOFLAGS="--profile ci --hide-progress-bar" env: RUST_BACKTRACE: "1" + CARGO_TERM_COLOR: "always" - name: "`make install`" shell: bash run: | @@ -497,13 +500,15 @@ jobs: ## Install `rust` toolchain rustup toolchain install stable --no-self-update --profile minimal rustup default stable + - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 - name: Test - run: cargo test ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} env: RUST_BACKTRACE: "1" + CARGO_TERM_COLOR: "always" build_rust_nightly: name: Build/nightly @@ -527,13 +532,15 @@ jobs: ## Install `rust` toolchain rustup toolchain install nightly --no-self-update --profile minimal rustup default nightly + - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 - name: Test - run: cargo test ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} env: RUST_BACKTRACE: "1" + CARGO_TERM_COLOR: "always" compute_size: name: Binary sizes @@ -1060,6 +1067,7 @@ jobs: ## rust toolchain ~ install rustup toolchain install ${{ matrix.job.toolchain }} --no-self-update --profile minimal rustup default ${{ matrix.job.toolchain }} + - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 @@ -1122,7 +1130,7 @@ jobs: CARGO_UTILITY_LIST_OPTIONS="$(for u in ${UTILITY_LIST}; do echo -n "-puu_${u} "; done;)" outputs CARGO_UTILITY_LIST_OPTIONS - name: Test uucore - run: cargo test --no-fail-fast -p uucore + run: cargo nextest run --profile ci --hide-progress-bar -p uucore env: CARGO_INCREMENTAL: "0" RUSTC_WRAPPER: "" @@ -1131,7 +1139,7 @@ jobs: RUST_BACKTRACE: "1" # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }} - name: Test - run: cargo test ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --no-fail-fast + run: cargo nextest run --profile ci --hide-progress-bar ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} env: CARGO_INCREMENTAL: "0" RUSTC_WRAPPER: "" @@ -1140,7 +1148,7 @@ jobs: RUST_BACKTRACE: "1" # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }} - name: Test individual utilities - run: cargo test --no-fail-fast ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} + run: cargo nextest run --profile ci --hide-progress-bar ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} env: CARGO_INCREMENTAL: "0" RUSTC_WRAPPER: "" diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 059fe6d81..06c39f961 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -1,5 +1,7 @@ name: Android +# spell-checker:ignore TERMUX reactivecircus Swatinem noaudio pkill swiftshader + on: [push, pull_request] permissions: diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index 2ff167434..9507b3a56 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -1,6 +1,6 @@ name: FreeBSD -# spell-checker:ignore sshfs usesh vmactions +# spell-checker:ignore sshfs usesh vmactions taiki Swatinem esac fdescfs fdesc env: # * style job configuration @@ -156,6 +156,9 @@ jobs: curl https://sh.rustup.rs -sSf --output rustup.sh sh rustup.sh -y --profile=minimal . $HOME/.cargo/env + # Install nextest + mkdir -p ~/.cargo/bin + curl -LsSf https://get.nexte.st/latest/freebsd | tar zxf - -C ~/.cargo/bin ## Info # environment echo "## environment" @@ -168,6 +171,7 @@ jobs: # tooling info echo "## tooling info" cargo -V + cargo nextest --version rustc -V # # To ensure that files are cleaned up, we don't want to exit on error @@ -175,9 +179,11 @@ jobs: cd "${WORKSPACE}" unset FAULT cargo build || FAULT=1 + export PATH=~/.cargo/bin:${PATH} export RUST_BACKTRACE=1 - if (test -z "\$FAULT"); then cargo test --features '${{ matrix.job.features }}' || FAULT=1 ; fi - if (test -z "\$FAULT"); then cargo test --all-features -p uucore || FAULT=1 ; fi + export CARGO_TERM_COLOR=always + if (test -z "\$FAULT"); then cargo nextest run --hide-progress-bar --profile ci --features '${{ matrix.job.features }}' || FAULT=1 ; fi + if (test -z "\$FAULT"); then cargo nextest run --hide-progress-bar --profile ci --all-features -p uucore || FAULT=1 ; fi # Clean to avoid to rsync back the files cargo clean if (test -n "\$FAULT"); then exit 1 ; fi diff --git a/GNUmakefile b/GNUmakefile index 81b90d32f..26d6de5ba 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,4 +1,4 @@ -# spell-checker:ignore (misc) testsuite runtest findstring (targets) busytest toybox distclean pkgs ; (vars/env) BINDIR BUILDDIR CARGOFLAGS DESTDIR DOCSDIR INSTALLDIR INSTALLEES MULTICALL DATAROOTDIR TESTDIR +# spell-checker:ignore (misc) testsuite runtest findstring (targets) busytest toybox distclean pkgs nextest ; (vars/env) BINDIR BUILDDIR CARGOFLAGS DESTDIR DOCSDIR INSTALLDIR INSTALLEES MULTICALL DATAROOTDIR TESTDIR # Config options PROFILE ?= debug @@ -289,6 +289,9 @@ $(foreach test,$(filter-out $(SKIP_UTILS),$(PROGS)),$(eval $(call TEST_BUSYBOX,$ test: ${CARGO} test ${CARGOFLAGS} --features "$(TESTS) $(TEST_SPEC_FEATURE)" --no-default-features $(TEST_NO_FAIL_FAST) +nextest: + ${CARGO} nextest run ${CARGOFLAGS} --features "$(TESTS) $(TEST_SPEC_FEATURE)" --no-default-features $(TEST_NO_FAIL_FAST) + test_toybox: -(cd $(TOYBOX_SRC)/ && make tests) diff --git a/util/android-commands.sh b/util/android-commands.sh index 42e27ad88..02999f5da 100755 --- a/util/android-commands.sh +++ b/util/android-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -# spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill +# spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill nextest # There are three shells: the host's, adb, and termux. Only adb lets us run # commands directly on the emulated device, only termux provides a GNU @@ -73,9 +73,50 @@ snapshot () { apk="$1" echo "running snapshot" adb install -g "$apk" + + echo "Prepare and install system packages" probe='/sdcard/pkg.probe' - command="'yes | pkg install rust binutils openssl -y; touch $probe'" + log='/sdcard/pkg.log' + command="'{ mkdir -vp ~/.cargo/bin; yes | pkg install rust binutils openssl -y; echo \$? > $probe; } &> $log'" run_termux_command "$command" "$probe" + return_code=$? + + adb pull "$log" . + cat $(basename "$log") + + if [[ $return_code -ne 0 ]]; then return $return_code; fi + + echo "Installing cargo-nextest" + probe='/sdcard/nextest.probe' + log='/sdcard/nextest.log' + # We need to install nextest via cargo currently, since there is no pre-built binary for android x86 + command="'cargo install cargo-nextest &> $log; touch $probe'" + run_termux_command "$command" "$probe" + + adb pull "$log" . + cat $(basename "$log") + + echo "Info about cargo and rust" + probe='/sdcard/info.probe' + log='/sdcard/info.log' + command="'{ \ + set -x; \ + echo \$HOME; \ + PATH=\$HOME/.cargo/bin:\$PATH; \ + export PATH; \ + echo \$PATH; \ + pwd; \ + command -v rustc && rustc --version; \ + ls -la ~/.cargo/bin; \ + cargo --list; \ + cargo nextest --version; \ + set +x; \ + } &> $log; touch $probe'" + run_termux_command "$command" "$probe" + + adb pull "$log" . + cat $(basename "$log") + echo "snapshot complete" adb shell input text "exit" && hit_enter && hit_enter } @@ -120,8 +161,15 @@ build () { tests () { probe='/sdcard/tests.probe' - export RUST_BACKTRACE=1 - command="'cd ~/coreutils && timeout --preserve-status --verbose -k 1m 60m cargo test --features feat_os_unix_android --no-fail-fast >/sdcard/tests.log 2>&1; echo \$? >$probe'" + command="'\ + export PATH=\$HOME/.cargo/bin:\$PATH; \ + export RUST_BACKTRACE=1; \ + export CARGO_TERM_COLOR=always; \ + cd ~/coreutils || { echo 1 > $probe; exit; }; \ + timeout --preserve-status --verbose -k 1m 60m \ + cargo nextest run --profile ci --hide-progress-bar --features feat_os_unix_android \ + &>/sdcard/tests.log; \ + echo \$? >$probe'" run_termux_command "$command" "$probe" return_code=$? adb pull /sdcard/tests.log . From 037aaf0a3679914994533cdd5fddb82468ac4531 Mon Sep 17 00:00:00 2001 From: Joining7943 <111500881+Joining7943@users.noreply.github.com> Date: Fri, 14 Apr 2023 19:23:43 +0200 Subject: [PATCH 2/4] android-commands.sh: Reformat file with shftm --- util/android-commands.sh | 56 +++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/util/android-commands.sh b/util/android-commands.sh index 02999f5da..5d6039413 100755 --- a/util/android-commands.sh +++ b/util/android-commands.sh @@ -14,12 +14,11 @@ # success, some other number for errors (an empty file is basically the same as # 0). Note that the return codes are text, not raw bytes. - this_repo="$(dirname $(dirname -- "$(readlink -- "${0}")"))" -help () { +help() { echo \ -"Usage: $0 COMMAND [ARG] + "Usage: $0 COMMAND [ARG] where COMMAND is one of: snapshot APK install APK and dependencies on an emulator to prep a snapshot @@ -43,7 +42,7 @@ hit_enter() { launch_termux() { echo "launching termux" - if ! adb shell 'am start -n com.termux/.HomeActivity' ; then + if ! adb shell 'am start -n com.termux/.HomeActivity'; then echo "failed to launch termux" exit 1 fi @@ -58,18 +57,21 @@ launch_termux() { } run_termux_command() { - command="$1" # text of the escaped command, including creating the probe! - probe="$2" # unique file that indicates the command is complete + command="$1" # text of the escaped command, including creating the probe! + probe="$2" # unique file that indicates the command is complete launch_termux adb shell input text "$command" && hit_enter - while ! adb shell "ls $probe" 2>/dev/null; do echo "waiting for $probe"; sleep 30; done + while ! adb shell "ls $probe" 2>/dev/null; do + echo "waiting for $probe" + sleep 30 + done return_code=$(adb shell "cat $probe") adb shell "rm $probe" echo "return code: $return_code" return $return_code } -snapshot () { +snapshot() { apk="$1" echo "running snapshot" adb install -g "$apk" @@ -121,7 +123,7 @@ snapshot () { adb shell input text "exit" && hit_enter && hit_enter } -sync () { +sync() { repo="$1" echo "running sync $1" # android doesn't allow symlinks on shared dirs, and adb can't selectively push files @@ -148,7 +150,7 @@ sync () { run_termux_command "$command" "$probe" } -build () { +build() { probe='/sdcard/build.probe' command="'cd ~/coreutils && cargo build --features feat_os_unix_android 2>/sdcard/build.log; echo \$? >$probe'" echo "running build" @@ -159,7 +161,7 @@ build () { return $return_code } -tests () { +tests() { probe='/sdcard/tests.probe' command="'\ export PATH=\$HOME/.cargo/bin:\$PATH; \ @@ -182,16 +184,34 @@ exit_code=0 if [ $# -eq 1 ]; then case "$1" in - sync) sync "$this_repo"; exit_code=$?;; - build) build; exit_code=$?;; - tests) tests; exit_code=$?;; - *) help;; + sync) + sync "$this_repo" + exit_code=$? + ;; + build) + build + exit_code=$? + ;; + tests) + tests + exit_code=$? + ;; + *) help ;; esac elif [ $# -eq 2 ]; then case "$1" in - snapshot) snapshot "$2"; exit_code=$?;; - sync) sync "$2"; exit_code=$?;; - *) help; exit 1;; + snapshot) + snapshot "$2" + exit_code=$? + ;; + sync) + sync "$2" + exit_code=$? + ;; + *) + help + exit 1 + ;; esac else help From 18d5e9a3bb287ded426b28e4597e499693540906 Mon Sep 17 00:00:00 2001 From: Joining7943 <111500881+Joining7943@users.noreply.github.com> Date: Sat, 15 Apr 2023 10:30:43 +0200 Subject: [PATCH 3/4] Update CONTRIBUTING.md: Add nextest usage examples --- CONTRIBUTING.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8199b6d87..8f66b2bba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ - + # Contributing to coreutils @@ -137,6 +137,14 @@ If you also want to test the core utilities: cargo test -p uucore -p coreutils ``` +Running the complete test suite might take a while. We use [nextest](https://nexte.st/index.html) in +the CI and you might want to try it out locally. It can speed up the execution time of the whole +test run significantly if the cpu has multiple cores. + +```shell +cargo nextest run --features unix --no-fail-fast +``` + To debug: ```shell @@ -171,6 +179,15 @@ To include tests for unimplemented behavior: make UTILS='UTILITY_1 UTILITY_2' SPEC=y test ``` +To run tests with `nextest` just use the nextest target. Note you'll need to +[install](https://nexte.st/book/installation.html) `nextest` first. The `nextest` target accepts the +same arguments like the default `test` target, so it's possible to pass arguments to `nextest run` +via `CARGOFLAGS`: + +```shell +make CARGOFLAGS='--no-fail-fast' UTILS='UTILITY_1 UTILITY_2' nextest +``` + ### Run Busybox Tests This testing functionality is only available on *nix operating systems and @@ -326,7 +343,6 @@ if changes are not reflected in the report then run `cargo clean` and run the ab If you are using stable version of Rust that doesn't enable code coverage instrumentation by default then add `-Z-Zinstrument-coverage` flag to `RUSTFLAGS` env variable specified above. - ## Other implementations The Coreutils have different implementations, with different levels of completions: @@ -339,10 +355,9 @@ The Coreutils have different implementations, with different levels of completio * [SerenityOS](https://github.com/SerenityOS/serenity/tree/master/Userland/Utilities) * [Initial Unix](https://github.com/dspinellis/unix-history-repo) -However, when reimplementing the tools/options in Rust, don't read their source codes +However, when reimplementing the tools/options in Rust, don't read their source codes when they are using reciprocal licenses (ex: GNU GPL, GNU LGPL, etc). - ## Licensing uutils is distributed under the terms of the MIT License; see the `LICENSE` file From 3495b82499a0cdfb782f300016428709a8bbb803 Mon Sep 17 00:00:00 2001 From: Joining7943 <111500881+Joining7943@users.noreply.github.com> Date: Sat, 15 Apr 2023 13:22:51 +0200 Subject: [PATCH 4/4] ci/android: Use different caching strategy --- .github/workflows/android.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 06c39f961..97e1e60f1 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -32,15 +32,15 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 - - name: AVD cache - uses: actions/cache@v3 + - name: Restore AVD cache + uses: actions/cache/restore@v3 id: avd-cache with: path: | ~/.android/avd/* ~/.android/avd/*/snapshots/* ~/.android/adb* - key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }} + key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest - name: Create and cache emulator image if: steps.avd-cache.outputs.cache-hit != 'true' uses: reactivecircus/android-emulator-runner@v2 @@ -53,11 +53,21 @@ jobs: force-avd-creation: true emulator-options: -no-snapshot-load -noaudio -no-boot-anim -camera-back none script: | + set -e wget https://github.com/termux/termux-app/releases/download/${{ env.TERMUX }}/termux-app_${{ env.TERMUX }}+github-debug_${{ matrix.arch }}.apk util/android-commands.sh snapshot termux-app_${{ env.TERMUX }}+github-debug_${{ matrix.arch }}.apk adb -s emulator-5554 emu avd snapshot save ${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }} echo "Emulator image created." pkill -9 qemu-system-x86_64 + - name: Save AVD cache + if: steps.avd-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + ~/.android/avd/* + ~/.android/avd/*/snapshots/* + ~/.android/adb* + key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest - name: Build and Test uses: reactivecircus/android-emulator-runner@v2 with: