From acef46b6295bea61439fbd77b84a5f6a0d7a7b22 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 16 Sep 2022 10:26:29 +0200 Subject: [PATCH 1/3] add the capability to run several tests at once --- DEVELOPER_INSTRUCTIONS.md | 2 +- README.md | 2 ++ util/run-gnu-test.sh | 18 ++++++++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/DEVELOPER_INSTRUCTIONS.md b/DEVELOPER_INSTRUCTIONS.md index c007fba7e..d6171bef8 100644 --- a/DEVELOPER_INSTRUCTIONS.md +++ b/DEVELOPER_INSTRUCTIONS.md @@ -21,7 +21,7 @@ Running GNU tests At the end you should have uutils, gnu and gnulib checked out next to each other. - Run `cd uutils && ./util/build-gnu.sh && cd ..` to get everything ready (this may take a while) -- Finally, you can run tests with `bash uutils/util/run-gnu-test.sh `. Instead of `` insert the test you want to run, e.g. `tests/misc/wc-proc.sh`. +- Finally, you can run tests with `bash uutils/util/run-gnu-test.sh `. Instead of `` insert the tests you want to run, e.g. `tests/misc/wc-proc.sh`. Code Coverage Report Generation diff --git a/README.md b/README.md index 0bf8ca039..f04c58cea 100644 --- a/README.md +++ b/README.md @@ -369,6 +369,8 @@ $ bash util/build-gnu.sh $ bash util/run-gnu-test.sh # To run a single test: $ bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example +# To run several tests: +$ bash util/run-gnu-test.sh tests/touch/not-owner.sh tests/rm/no-give-up.sh # for example # If this is a perl (.pl) test, to run in debug: $ DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl ``` diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index 9b784699c..fdd928221 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -31,16 +31,26 @@ cd "${path_GNU}" && echo "[ pwd:'${PWD}' ]" export RUST_BACKTRACE=1 -if test -n "$1"; then - # if set, run only the test passed - export RUN_TEST="TESTS=$1" +if test $# -ge 1; then + # if set, run only the tests passed + SPECIFIC_TESTS="" + for t in "$@"; do + SPECIFIC_TESTS="$SPECIFIC_TESTS $t" + done + # trim it + SPECIFIC_TESTS=$(echo $SPECIFIC_TESTS| xargs) + echo "Running specific tests: $SPECIFIC_TESTS" fi # * timeout used to kill occasionally errant/"stuck" processes (note: 'release' testing takes ~1 hour; 'debug' testing takes ~2.5 hours) # * `gl_public_submodule_commit=` disables testing for use of a "public" gnulib commit (which will fail when using shallow gnulib checkouts) # * `srcdir=..` specifies the GNU source directory for tests (fixing failing/confused 'tests/factor/tNN.sh' tests and causing no harm to other tests) #shellcheck disable=SC2086 -timeout -sKILL 4h make -j "$(nproc)" check ${RUN_TEST} SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make +if test $# -ge 1; then + timeout -sKILL 4h make -j "$(nproc)" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make +else + timeout -sKILL 4h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make +fi if test -z "$1" && test -n "$CI"; then sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : From cfa7ba2ce2076db144fe8bc8cccdb79760d51bfe Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 16 Sep 2022 09:14:13 +0200 Subject: [PATCH 2/3] gnu: merge the root tests results into the main one --- .github/workflows/GnuTests.yml | 12 +++--- util/analyze-gnu-results.sh | 75 ++++++++++++++++++++++++++++++++++ util/run-gnu-test.sh | 1 + 3 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 util/analyze-gnu-results.sh diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 3afaa1723..3b35f1871 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -34,6 +34,7 @@ jobs: outputs repo_default_branch repo_GNU_ref repo_reference_branch # SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log" + ROOT_SUITE_LOG_FILE="${path_GNU_tests}/test-suite-root.log" TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support TEST_FILESET_PREFIX='test-fileset-IDs.sha1#' TEST_FILESET_SUFFIX='.txt' @@ -108,18 +109,17 @@ jobs: id: summary shell: bash run: | + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' ## Extract/summarize testing info outputs() { step_id="summary"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo ::set-output name=${var}::${!var}; done; } # SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}' + ROOT_SUITE_LOG_FILE='${{ steps.vars.outputs.ROOT_SUITE_LOG_FILE }}' + ls -al ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} + if test -f "${SUITE_LOG_FILE}" then - TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + source ${path_UUTILS}/util/analyze-gnu-results.sh ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early" exit 1 diff --git a/util/analyze-gnu-results.sh b/util/analyze-gnu-results.sh new file mode 100644 index 000000000..2bc08a9a4 --- /dev/null +++ b/util/analyze-gnu-results.sh @@ -0,0 +1,75 @@ +#!/bin/sh +# spell-checker:ignore xpass XPASS testsuite +set -e + +# As we do two builds (with and without root), we need to do some trivial maths +# to present the merge results +# this script will export the values in the term + +if test $# -ne 2; then + echo "syntax:" + echo "$0 testsuite.log root-testsuite.log" +fi + +SUITE_LOG_FILE=$1 +ROOT_SUITE_LOG_FILE=$2 + +if test ! -f "${SUITE_LOG_FILE}"; then + echo "${SUITE_LOG_FILE} has not been found" + exit 1 +fi +if test ! -f "${ROOT_SUITE_LOG_FILE}"; then + echo "${ROOT_SUITE_LOG_FILE} has not been found" + exit 1 +fi + +function get_total { + # Total of tests executed + # They are the normal number of tests as they are skipped in the normal run + NON_ROOT=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $NON_ROOT +} + +function get_pass { + # This is the sum of the two test suites. + # In the normal run, they are SKIP + NON_ROOT=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + AS_ROOT=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $((NON_ROOT + AS_ROOT)) +} + +function get_skip { + # As some of the tests executed as root as still SKIP (ex: selinux), we + # need to some maths: + # Number of tests skip as user - total test as root + skipped as root + TOTAL_AS_ROOT=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + NON_ROOT=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + AS_ROOT=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $((NON_ROOT - TOTAL_AS_ROOT + AS_ROOT)) +} + +function get_fail { + # They used to be SKIP, now they fail (this is a good news) + NON_ROOT=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + AS_ROOT=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $((NON_ROOT + AS_ROOT)) +} + +function get_xpass { + NON_ROOT=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $NON_ROOT +} + +function get_error { + # They used to be SKIP, now they error (this is a good news) + NON_ROOT=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + AS_ROOT=$(sed -n "s/.*# ERROR:: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $((NON_ROOT + AS_ROOT)) +} + +export TOTAL=$(get_total) +export PASS=$(get_pass) +export SKIP=$(get_skip) +export FAIL=$(get_fail) +export XPASS=$(get_xpass) +export ERROR=$(get_error) diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index fdd928221..2fbeba501 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -53,5 +53,6 @@ else fi if test -z "$1" && test -n "$CI"; then + echo "Running check-root to run only root tests" sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : fi From 20af659f09f122d1122157aaa29a6fac0fe936a6 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sat, 17 Sep 2022 11:08:12 +0200 Subject: [PATCH 3/3] Run the GNU root tests in a separate task --- .github/workflows/GnuTests.yml | 8 +++++++- util/run-gnu-test.sh | 25 ++++++++++++++++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 3b35f1871..d932538e5 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -40,7 +40,7 @@ jobs: TEST_FILESET_SUFFIX='.txt' TEST_SUMMARY_FILE='gnu-result.json' TEST_FULL_SUMMARY_FILE='gnu-full-result.json' - outputs SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE + outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE - name: Checkout code (uutil) uses: actions/checkout@v3 with: @@ -100,6 +100,12 @@ jobs: path_GNU='${{ steps.vars.outputs.path_GNU }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' bash "${path_UUTILS}/util/run-gnu-test.sh" + - name: Run GNU root tests + shell: bash + run: | + path_GNU='${{ steps.vars.outputs.path_GNU }}' + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' + bash "${path_UUTILS}/util/run-gnu-test.sh" run-root - name: Extract testing info into JSON shell: bash run : | diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index 2fbeba501..f5c47e645 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -46,13 +46,20 @@ fi # * `gl_public_submodule_commit=` disables testing for use of a "public" gnulib commit (which will fail when using shallow gnulib checkouts) # * `srcdir=..` specifies the GNU source directory for tests (fixing failing/confused 'tests/factor/tNN.sh' tests and causing no harm to other tests) #shellcheck disable=SC2086 -if test $# -ge 1; then - timeout -sKILL 4h make -j "$(nproc)" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make -else - timeout -sKILL 4h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make -fi -if test -z "$1" && test -n "$CI"; then - echo "Running check-root to run only root tests" - sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : -fi +if test "$1" != "run-root"; then +# run the regular tests + if test $# -ge 1; then + timeout -sKILL 4h make -j "$(nproc)" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make + else + timeout -sKILL 4h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make + fi +else +# in case we would like to run tests requiring root + if test -z "$1" -o "$1" == "run-root"; then + if test -n "$CI"; then + echo "Running check-root to run only root tests" + sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : + fi + fi +fi \ No newline at end of file