diff --git a/Userland/Libraries/LibTest/Macros.h b/Userland/Libraries/LibTest/Macros.h index 0a43ec8977..472e73752f 100644 --- a/Userland/Libraries/LibTest/Macros.h +++ b/Userland/Libraries/LibTest/Macros.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace AK { template @@ -18,8 +19,9 @@ void warnln(CheckedFormatString&& fmtstr, Parameters const&...); } namespace Test { -// Declare a helper so that we can call it from VERIFY in included headers -void current_test_case_did_fail(); +// Declare helpers so that we can call them from VERIFY in included headers +// the setter for TestResult is already declared in TestResult.h +TestResult current_test_result(); } #define EXPECT_EQ(a, b) \ @@ -28,7 +30,7 @@ void current_test_case_did_fail(); auto rhs = (b); \ if (lhs != rhs) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -41,7 +43,7 @@ void current_test_case_did_fail(); if (ltruth != rtruth) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \ __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -53,7 +55,7 @@ void current_test_case_did_fail(); auto rhs = (b); \ if (lhs != rhs) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -63,7 +65,7 @@ void current_test_case_did_fail(); auto rhs = (b); \ if (lhs == rhs) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -71,7 +73,7 @@ void current_test_case_did_fail(); do { \ if (!(x)) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -84,7 +86,7 @@ void current_test_case_did_fail(); ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \ " failed with lhs={}, rhs={}, (lhs-rhs)={}", \ __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -93,32 +95,32 @@ void current_test_case_did_fail(); #define FAIL(message) \ do { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) // To use, specify the lambda to execute in a sub process and verify it exits: // EXPECT_CRASH("This should fail", []{ // return Test::Crash::Failure::DidNotCrash; // }); -#define EXPECT_CRASH(test_message, test_func) \ - do { \ - Test::Crash crash(test_message, test_func); \ - if (!crash.run()) \ - ::Test::current_test_case_did_fail(); \ +#define EXPECT_CRASH(test_message, test_func) \ + do { \ + Test::Crash crash(test_message, test_func); \ + if (!crash.run()) \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) -#define EXPECT_CRASH_WITH_SIGNAL(test_message, signal, test_func) \ - do { \ - Test::Crash crash(test_message, test_func, (signal)); \ - if (!crash.run()) \ - ::Test::current_test_case_did_fail(); \ +#define EXPECT_CRASH_WITH_SIGNAL(test_message, signal, test_func) \ + do { \ + Test::Crash crash(test_message, test_func, (signal)); \ + if (!crash.run()) \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) -#define EXPECT_NO_CRASH(test_message, test_func) \ - do { \ - Test::Crash crash(test_message, test_func, 0); \ - if (!crash.run()) \ - ::Test::current_test_case_did_fail(); \ +#define EXPECT_NO_CRASH(test_message, test_func) \ + do { \ + Test::Crash crash(test_message, test_func, 0); \ + if (!crash.run()) \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) #define TRY_OR_FAIL(expression) \ diff --git a/Userland/Libraries/LibTest/TestResult.h b/Userland/Libraries/LibTest/TestResult.h new file mode 100644 index 0000000000..2405bd6871 --- /dev/null +++ b/Userland/Libraries/LibTest/TestResult.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023, Martin Janiczek + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +namespace Test { + +// TestResult signals to the TestSuite how the TestCase execution went. +enum class TestResult { + NotRun, + + // Test fn ran to completion without setting any of the below flags + Passed, + + // Didn't get through EXPECT(...). + Failed, +}; + +// Used eg. to signal we've ran out of prerecorded random bits. +// Defined in TestSuite.cpp +void set_current_test_result(TestResult); + +} // namespace Test diff --git a/Userland/Libraries/LibTest/TestSuite.cpp b/Userland/Libraries/LibTest/TestSuite.cpp index f8c7f9a174..94acd72370 100644 --- a/Userland/Libraries/LibTest/TestSuite.cpp +++ b/Userland/Libraries/LibTest/TestSuite.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -40,9 +41,15 @@ private: }; // Declared in Macros.h -void current_test_case_did_fail() +TestResult current_test_result() { - TestSuite::the().current_test_case_did_fail(); + return TestSuite::the().current_test_result(); +} + +// Declared in Macros.h +void set_current_test_result(TestResult result) +{ + TestSuite::the().set_current_test_result(result); } // Declared in TestCase.h @@ -57,6 +64,20 @@ void set_suite_setup_function(Function setup) TestSuite::the().set_suite_setup(move(setup)); } +static DeprecatedString test_result_to_string(TestResult result) +{ + switch (result) { + case TestResult::NotRun: + return "Not run"; + case TestResult::Passed: + return "Completed"; + case TestResult::Failed: + return "Failed"; + default: + return "Unknown TestResult"; + } +} + int TestSuite::main(DeprecatedString const& suite_name, Span arguments) { m_suite_name = suite_name; @@ -116,8 +137,11 @@ Vector> TestSuite::find_cases(DeprecatedString const& se int TestSuite::run(Vector> const& tests) { size_t test_count = 0; + size_t test_passed_count = 0; size_t test_failed_count = 0; size_t benchmark_count = 0; + size_t benchmark_passed_count = 0; + size_t benchmark_failed_count = 0; TestElapsedTimer global_timer; for (auto const& t : tests) { @@ -125,7 +149,7 @@ int TestSuite::run(Vector> const& tests) auto const repetitions = t->is_benchmark() ? m_benchmark_repetitions : 1; warnln("Running {} '{}'.", test_type, t->name()); - m_current_test_case_passed = true; + m_current_test_result = TestResult::NotRun; u64 total_time = 0; u64 sum_of_squared_times = 0; @@ -140,6 +164,10 @@ int TestSuite::run(Vector> const& tests) sum_of_squared_times += iteration_time * iteration_time; min_time = min(min_time, iteration_time); max_time = max(max_time, iteration_time); + + // Non-randomized tests don't touch the test result when passing. + if (m_current_test_result == TestResult::NotRun) + m_current_test_result = TestResult::Passed; } if (repetitions != 1) { @@ -148,22 +176,40 @@ int TestSuite::run(Vector> const& tests) double standard_deviation = sqrt((sum_of_squared_times + repetitions * average_squared - 2 * total_time * average) / (repetitions - 1)); dbgln("{} {} '{}' on average in {:.1f}±{:.1f}ms (min={}ms, max={}ms, total={}ms)", - m_current_test_case_passed ? "Completed" : "Failed", test_type, t->name(), + test_result_to_string(m_current_test_result), test_type, t->name(), average, standard_deviation, min_time, max_time, total_time); } else { - dbgln("{} {} '{}' in {}ms", m_current_test_case_passed ? "Completed" : "Failed", test_type, t->name(), total_time); + dbgln("{} {} '{}' in {}ms", test_result_to_string(m_current_test_result), test_type, t->name(), total_time); } if (t->is_benchmark()) { m_benchtime += total_time; benchmark_count++; + + switch (m_current_test_result) { + case TestResult::Passed: + benchmark_passed_count++; + break; + case TestResult::Failed: + benchmark_failed_count++; + break; + default: + break; + } } else { m_testtime += total_time; test_count++; - } - if (!m_current_test_case_passed) { - test_failed_count++; + switch (m_current_test_result) { + case TestResult::Passed: + test_passed_count++; + break; + case TestResult::Failed: + test_failed_count++; + break; + default: + break; + } } } @@ -175,10 +221,29 @@ int TestSuite::run(Vector> const& tests) m_benchtime, global_timer.elapsed_milliseconds() - (m_testtime + m_benchtime)); - if (test_count != 0) - dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_count - test_failed_count, test_failed_count); + if (test_count != 0) { + if (test_passed_count == test_count) { + dbgln("All {} tests passed.", test_count); + } else if (test_passed_count + test_failed_count == test_count) { + dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_passed_count, test_failed_count); + } else { + dbgln("Out of {} tests, {} passed, {} failed and {} didn't finish for other reasons.", test_count, test_passed_count, test_failed_count, test_count - test_passed_count - test_failed_count); + } + } - return (int)test_failed_count; + if (benchmark_count != 0) { + if (benchmark_passed_count == benchmark_count) { + dbgln("All {} benchmarks passed.", benchmark_count); + } else if (benchmark_passed_count + benchmark_failed_count == benchmark_count) { + dbgln("Out of {} benchmarks, {} passed and {} failed.", benchmark_count, benchmark_passed_count, benchmark_failed_count); + } else { + dbgln("Out of {} benchmarks, {} passed, {} failed and {} didn't finish for other reasons.", benchmark_count, benchmark_passed_count, benchmark_failed_count, benchmark_count - benchmark_passed_count - benchmark_failed_count); + } + } + + // We have multiple TestResults, all except for Passed being "bad". + // Let's get a count of them: + return (int)(test_count - test_passed_count + benchmark_count - benchmark_passed_count); } -} +} // namespace Test diff --git a/Userland/Libraries/LibTest/TestSuite.h b/Userland/Libraries/LibTest/TestSuite.h index fd8d2afc6f..298cae5d10 100644 --- a/Userland/Libraries/LibTest/TestSuite.h +++ b/Userland/Libraries/LibTest/TestSuite.h @@ -13,6 +13,7 @@ #include #include #include +#include namespace Test { @@ -40,7 +41,8 @@ public: m_cases.append(test_case); } - void current_test_case_did_fail() { m_current_test_case_passed = false; } + TestResult current_test_result() const { return m_current_test_result; } + void set_current_test_result(TestResult result) { m_current_test_result = result; } void set_suite_setup(Function setup) { m_setup = move(setup); } @@ -51,8 +53,8 @@ private: u64 m_benchtime = 0; DeprecatedString m_suite_name; u64 m_benchmark_repetitions = 1; - bool m_current_test_case_passed = true; Function m_setup; + TestResult m_current_test_result = TestResult::NotRun; }; }