1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-05-31 15:48:12 +00:00

Base+Utilities: Add run-tests program to run system tests with LibTest

This test program heavily pulls from the JavaScriptTestRunner/test-js,
but with a twist. Instead of loading JavaScript files into the current
process, constructing a JS environment for them, and executing test
suites/tests directly, run-tests posix_spawns each test file.

Test file stdout is written to a temp file, and only dumped to console
if the test fails or the verbose option is passed to the program. Unlike
test-js, times are always printed for every test executed for better
visibility in CI.
This commit is contained in:
Andrew Kaster 2021-06-27 14:22:25 -06:00 committed by Ali Mohammad Pur
parent 601c9e89de
commit cc0a376c95
5 changed files with 344 additions and 48 deletions

View file

@ -3,54 +3,8 @@
echo
echo "==== Running Tests on SerenityOS ===="
run(index) {
shift
test_dir=$(dirname $1)
if test -z $test_dir {
test_dir="."
}
test_cmd=($*)
echo "==== Running test $index out of $count_of_all_tests -- $test_cmd ===="
echo "::group::$test_cmd"
cd $test_dir
if $test_cmd {
echo "::endgroup::"
echo "==== $test_cmd passed! ===="
} else {
echo "::endgroup::"
failed_tests=${concat_lists $failed_tests ($test_cmd)}
echo "==== Added $test_cmd to list of failed tests. Failed tests so far: $failed_tests ===="
echo "::error file=$test_cmd:: $test_cmd returned non-zero exit code, check logs!"
}
cd -
}
# Files in /usr/Tests/* that we don't want to execute match these patterns:
# Kernel/Legacy: Kernel tests that are either old patched exploits, or hang the system/shell
# .inc: Shell test helper file that's not a test
# UserEmulator: Tests designed to run inside the Userspace Emulator
# stack-smash: Intentionally crashes by smashing the stack
# .frm,.txt: Test inputs that are not tests
# test-web: Requires the window server in order to work
# test-js: We start this one manually with the show progress flag set to false
exclude_patterns='Kernel/Legacy|.inc|UserEmulator|stack-smash|.txt|.frm|test-web|test-js'
system_tests=((test-js --show-progress=false) (test-crypto -c -t test))
all_tests=${concat_lists $system_tests $(find /usr/Tests -type f | grep -E -v $exclude_patterns | shuf) }
count_of_all_tests=${length $all_tests}
failed_tests=()
for index i cmd in $all_tests {
run $(expr $i + 1) $cmd
}
fail_count=${length $failed_tests}
echo "==== Out of $count_of_all_tests tests, $(expr $count_of_all_tests - $fail_count) passed and $fail_count failed. ===="
if test $fail_count -gt 0 {
echo "==== Failing tests: $failed_tests ===="
}
run-tests
fail_count=$?
echo "Failed: $fail_count" > ./test-results.log