Base/CI: Run tests from /usr/Tests in GitHub Actions

Uncomment the tests that were disabled due to frequent freezes when
running without KVM. This also adds a new github actions group for
every single test, which makes it easier to browse test boundaries
during test runs.

Move catting the serial output log back to its own step, so that it
has higher visibility. The previous solution was also shown to not
actually cat the log in the case of a failed boot and timeout :^(.

.github/workflows/cmake.yml

@@ -141,24 +141,20 @@ jobs:
       if: ${{ matrix.debug-macros == 'NORMAL_DEBUG'}}
       working-directory: ${{ github.workspace }}/Build
       env:
+        SERENITY_QEMU_CPU: "max,vmx=off"
         SERENITY_KERNEL_CMDLINE: "boot_mode=self-test"
         SERENITY_RUN: "ci"
       run: |
         echo "::group::ninja run # Qemu output"
-        run_worked=y
+        ninja run
-        ninja run || run_worked=n
         echo "::endgroup::"
-        echo "::group::cat debug.log # Serenity output"
+      timeout-minutes: 20
-        if [ "y" = "${run_worked}" ] ; then
+
-          cat debug.log
+    - name: Print Target Logs
-        else
+      # Extremely useful if Serenity hangs trying to run one of the tests
-          echo "skipped (qemu had non-zero exit-code)"
+      if: ${{ !cancelled() && matrix.debug-macros == 'NORMAL_DEBUG'}}
-        fi
+      working-directory: ${{ github.workspace }}/Build
-        echo "::endgroup::"
+      run: '[ ! -e debug.log ] || cat debug.log'
-        [ "y" = ${run_worked} ]
-      timeout-minutes: 10
-      # FIXME: When stable, remove continue on error. (See issue #5541)
-      continue-on-error: true
 
   build_and_test_lagom:
     runs-on: ${{ matrix.os }}

Base/home/anon/tests/run-tests-and-shutdown.sh

@@ -6,26 +6,49 @@ echo "==== Running Tests on SerenityOS ===="
 run(index) {
     shift
     test_cmd=($*)
-    echo "Running test $index out of $count_of_all_tests -- $test_cmd"
+    echo "==== Running test $index out of $count_of_all_tests -- $test_cmd ===="
+    echo "::group::$test_cmd"
     if $test_cmd {
-        echo "::debug file=$test_cmd:: $test_cmd passed!"
+        echo "::endgroup::"
+        echo "==== $test_cmd passed! ===="
     } else {
+        echo "::endgroup::"
+        failed_tests=${concat_lists $failed_tests ($test_cmd)}
+        echo "==== Added $test_cmd to list of failed tests. Failed tests so far: $failed_tests ===="
         echo "::error file=$test_cmd:: $test_cmd returned non-zero exit code, check logs!"
     }
 }
 
-# TODO: test-web requires the window server
+# Files in /usr/Tests/* that we don't want to execute match these patterns:
-system_tests=((test-js --show-progress=false) test-pthread /usr/Tests/LibM/test-math (test-crypto -t bigint))
+#     Kernel/Legacy: Kernel tests that are either old patched exploits, or hang the system/shell
-# FIXME: Running too much at once is likely to run into #5541. Remove commented out find below when stable
+#     .inc: Shell test helper file that's not a test
-all_tests=${concat_lists $system_tests} #$(find /usr/Tests -type f | grep -v Kernel | grep -v .inc | shuf))
+#     UserEmulator: Tests designed to run inside the Userspace Emulator
+#     stack-smash: Intentionally crashes by smashing the stack
+#     TestJSON: AK/TestJSON makes assumptions about $PWD to load its input files
+#     .frm: Test inputs that are not tests
+#     test-web: Requires the window server in order to work
+#     test-js: We start this one manually with the show progress flag set to false
+exclude_patterns='Kernel/Legacy|.inc|UserEmulator|stack-smash|TestJSON|.frm|test-web|test-js'
+
+system_tests=((test-js --show-progress=false) (test-crypto -c -t test))
+all_tests=${concat_lists $system_tests $(find /usr/Tests -type f | grep -E -v $exclude_patterns | shuf) }
 count_of_all_tests=${length $all_tests}
+failed_tests=()
 
 for index i cmd in $all_tests {
     run $(expr $i + 1) $cmd
 }
 
-echo "==== Done running tests ===="
+fail_count=${length $failed_tests}
+
+echo "==== Out of $count_of_all_tests tests, $(expr $count_of_all_tests - $fail_count) passed and $fail_count failed. ===="
+
+if test $fail_count -gt 0 {
+    echo "==== Failing tests: $failed_tests ===="
+}
 
 if test $DO_SHUTDOWN_AFTER_TESTS {
     shutdown -n
 }
+
+exit $fail_count