From 6bf91d00eff87b8904239db5126165485741d731 Mon Sep 17 00:00:00 2001
From: Brian Gianforcaro <bgianf@serenityos.org>
Date: Wed, 5 Jan 2022 02:17:45 -0800
Subject: [PATCH] run-tests: Make reproducing test failures behave closer to
 selftest mode

In CI / or local testing when you run `serenity.sh tests`, the system
will boot itself in self test mode, and the test runner will be launched
by SystemServer. Previously we were setting up settings for that
environment in the `SystemServer.ini`. This makes reproducing CI
failures a bit confusing, as the system will behavior differently if you
run in self-test mode, vs running `run-tests-and-shutdown.sh` or
`run-tests` manually in a session.

This change moves the settings to `run-tests`, so no matter how you try
to run the test runner, it will always behave the same.
---
 Base/etc/SystemServer.ini        |  2 +-
 Userland/Utilities/run-tests.cpp | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/Base/etc/SystemServer.ini b/Base/etc/SystemServer.ini
index 3a2a05a6eb..54bb9958b3 100644
--- a/Base/etc/SystemServer.ini
+++ b/Base/etc/SystemServer.ini
@@ -155,7 +155,7 @@ User=anon
 [TestRunner@ttyS0]
 Executable=/home/anon/tests/run-tests-and-shutdown.sh
 StdIO=/dev/ttyS0
-Environment=DO_SHUTDOWN_AFTER_TESTS=1 TERM=xterm PATH=/bin:/usr/bin:/usr/local/bin TESTS_ONLY=1 UBSAN_OPTIONS=halt_on_error=1
+Environment=DO_SHUTDOWN_AFTER_TESTS=1 TERM=xterm PATH=/bin:/usr/bin:/usr/local/bin
 User=anon
 WorkingDirectory=/home/anon
 SystemModes=self-test
diff --git a/Userland/Utilities/run-tests.cpp b/Userland/Utilities/run-tests.cpp
index 8c6645a547..a263743073 100644
--- a/Userland/Utilities/run-tests.cpp
+++ b/Userland/Utilities/run-tests.cpp
@@ -12,6 +12,7 @@
 #include <LibTest/TestRunner.h>
 #include <signal.h>
 #include <spawn.h>
+#include <stdlib.h>
 #include <sys/wait.h>
 #include <unistd.h>
 
@@ -255,6 +256,7 @@ FileResult TestRunner::run_test_file(const String& test_path)
 
 int main(int argc, char** argv)
 {
+
     auto program_name = LexicalPath::basename(argv[0]);
 
 #ifdef SIGINFO
@@ -274,6 +276,7 @@ int main(int argc, char** argv)
 #endif
     bool print_json = false;
     bool print_all_output = false;
+    bool run_benchmarks = false;
     const char* specified_test_root = nullptr;
     String test_glob;
     String exclude_pattern;
@@ -297,6 +300,7 @@ int main(int argc, char** argv)
     });
     args_parser.add_option(print_json, "Show results as JSON", "json", 'j');
     args_parser.add_option(print_all_output, "Show all test output", "verbose", 'v');
+    args_parser.add_option(run_benchmarks, "Run benchmarks as well", "benchmarks", 'b');
     args_parser.add_option(test_glob, "Only run tests matching the given glob", "filter", 'f', "glob");
     args_parser.add_option(exclude_pattern, "Regular expression to use to exclude paths from being considered tests", "exclude-pattern", 'e', "pattern");
     args_parser.add_option(config_file, "Configuration file to use", "config-file", 'c', "filename");
@@ -309,6 +313,12 @@ int main(int argc, char** argv)
         AK::set_debug_enabled(false);
     }
 
+    // Make UBSAN deadly for all tests we run by default.
+    setenv("UBSAN_OPTIONS", "halt_on_error=1", true);
+
+    if (!run_benchmarks)
+        setenv("TESTS_ONLY", "1", true);
+
     String test_root;
 
     if (specified_test_root) {