LibTest: Add ability to turn test failure reporting on/off

This will be very useful as we add the randomized test cases and their
two loops ("generate+test many times" and "shrink once failure is
found"), because without this failure reporting we'd get many FAIL error
messages while still searching for the minimal one.

So, inside randomized test cases we want to only turn the error
reporting on for one last time after all the generating and shrinking.

Userland/Libraries/LibTest/Macros.h

@@ -26,6 +26,10 @@ TestResult current_test_result();
 
 Randomized::RandomnessSource& randomness_source();
 void set_randomness_source(Randomized::RandomnessSource);
+
+bool is_reporting_enabled();
+void enable_reporting();
+void disable_reporting();
 }
 
 #define EXPECT_EQ(a, b)                                                                                       \
@@ -33,7 +37,9 @@ void set_randomness_source(Randomized::RandomnessSource);
         auto lhs = (a);                                                                                       \
         auto rhs = (b);                                                                                       \
         if (lhs != rhs) {                                                                                     \
-            ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \
+            if (::Test::is_reporting_enabled())                                                               \
+                ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", \
+                    __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs });        \
             ::Test::set_current_test_result(::Test::TestResult::Failed);                                      \
         }                                                                                                     \
     } while (false)
@@ -45,6 +51,7 @@ void set_randomness_source(Randomized::RandomnessSource);
         bool ltruth = static_cast<bool>(lhs);                                                                                 \
         bool rtruth = static_cast<bool>(rhs);                                                                                 \
         if (ltruth != rtruth) {                                                                                               \
+            if (::Test::is_reporting_enabled())                                                                               \
                 ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \
                     __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth);        \
             ::Test::set_current_test_result(::Test::TestResult::Failed);                                                      \
@@ -58,7 +65,9 @@ void set_randomness_source(Randomized::RandomnessSource);
         auto lhs = (a);                                                                                       \
         auto rhs = (b);                                                                                       \
         if (lhs != rhs) {                                                                                     \
-            ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \
+            if (::Test::is_reporting_enabled())                                                               \
+                ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", \
+                    __FILE__, __LINE__, #a, #b, lhs, rhs);                                                    \
             ::Test::set_current_test_result(::Test::TestResult::Failed);                                      \
         }                                                                                                     \
     } while (false)
@@ -68,7 +77,9 @@ void set_randomness_source(Randomized::RandomnessSource);
         auto lhs = (a);                                                                                       \
         auto rhs = (b);                                                                                       \
         if (lhs == rhs) {                                                                                     \
-            ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \
+            if (::Test::is_reporting_enabled())                                                               \
+                ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", \
+                    __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs });        \
             ::Test::set_current_test_result(::Test::TestResult::Failed);                                      \
         }                                                                                                     \
     } while (false)
@@ -76,6 +87,7 @@ void set_randomness_source(Randomized::RandomnessSource);
 #define EXPECT(x)                                                                                        \
     do {                                                                                                 \
         if (!(x)) {                                                                                      \
+            if (::Test::is_reporting_enabled())                                                          \
                 ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \
             ::Test::set_current_test_result(::Test::TestResult::Failed);                                 \
         }                                                                                                \
@@ -87,6 +99,7 @@ void set_randomness_source(Randomized::RandomnessSource);
         auto expect_close_rhs = b;                                                                              \
         auto expect_close_diff = static_cast<double>(expect_close_lhs) - static_cast<double>(expect_close_rhs); \
         if (AK::fabs(expect_close_diff) > (err)) {                                                              \
+            if (::Test::is_reporting_enabled())                                                                 \
                 ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})"                         \
                              " failed with lhs={}, rhs={}, (lhs-rhs)={}",                                       \
                     __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff);         \
@@ -98,6 +111,7 @@ void set_randomness_source(Randomized::RandomnessSource);
 
 #define FAIL(message)                                                                      \
     do {                                                                                   \
+        if (::Test::is_reporting_enabled())                                                \
             ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \
         ::Test::set_current_test_result(::Test::TestResult::Failed);                       \
     } while (false)

Userland/Libraries/LibTest/TestSuite.cpp

@@ -76,6 +76,24 @@ void set_suite_setup_function(Function<void()> setup)
     TestSuite::the().set_suite_setup(move(setup));
 }
 
+// Declared in Macros.h
+bool is_reporting_enabled()
+{
+    return TestSuite::the().is_reporting_enabled();
+}
+
+// Declared in Macros.h
+void enable_reporting()
+{
+    TestSuite::the().enable_reporting();
+}
+
+// Declared in Macros.h
+void disable_reporting()
+{
+    TestSuite::the().disable_reporting();
+}
+
 static DeprecatedString test_result_to_string(TestResult result)
 {
     switch (result) {
@@ -164,6 +182,7 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
 
         warnln("Running {} '{}'.", test_type, t->name());
         m_current_test_result = TestResult::NotRun;
+        enable_reporting();
 
         u64 total_time = 0;
         u64 sum_of_squared_times = 0;

Userland/Libraries/LibTest/TestSuite.h

@@ -53,6 +53,14 @@ public:
     void set_randomness_source(Randomized::RandomnessSource source) { m_randomness_source = move(source); }
     Randomized::RandomnessSource& randomness_source() { return m_randomness_source; }
 
+    // Dictates whether FAIL(), EXPECT() and similar macros in LibTest/Macros.h
+    // print messages or not. This is important for randomized tests because
+    // they run the test function many times in a row, and we only want to
+    // report the _minimal_ (shrunk) failure to the user, not all of them.
+    bool is_reporting_enabled() { return m_reporting_enabled; }
+    void enable_reporting() { m_reporting_enabled = true; }
+    void disable_reporting() { m_reporting_enabled = false; }
+
 private:
     static TestSuite* s_global;
     Vector<NonnullRefPtr<TestCase>> m_cases;
@@ -63,6 +71,7 @@ private:
     Function<void()> m_setup;
     TestResult m_current_test_result = TestResult::NotRun;
     Randomized::RandomnessSource m_randomness_source = Randomized::RandomnessSource::live();
+    bool m_reporting_enabled = true;
 };
 
 }