From a60e3b17b1503da62f9764c61b9427315144b262 Mon Sep 17 00:00:00 2001 From: Martin Janiczek Date: Tue, 24 Oct 2023 00:19:04 +0200 Subject: [PATCH] LibTest: Expand test result bool to a TestResult This will be used in the randomized tests a lot more than it is in the unit tests / benchmarks; randomized tests will run the test function multiple times, check the result and optionally start shrinking the failing input. Generators will also be able to fail, resulting in some of the new TestResult variants. --- Userland/Libraries/LibTest/Macros.h | 50 ++++++------- Userland/Libraries/LibTest/TestResult.h | 26 +++++++ Userland/Libraries/LibTest/TestSuite.cpp | 89 ++++++++++++++++++++---- Userland/Libraries/LibTest/TestSuite.h | 6 +- 4 files changed, 133 insertions(+), 38 deletions(-) create mode 100644 Userland/Libraries/LibTest/TestResult.h diff --git a/Userland/Libraries/LibTest/Macros.h b/Userland/Libraries/LibTest/Macros.h index 0a43ec8977..472e73752f 100644 --- a/Userland/Libraries/LibTest/Macros.h +++ b/Userland/Libraries/LibTest/Macros.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace AK { template @@ -18,8 +19,9 @@ void warnln(CheckedFormatString&& fmtstr, Parameters const&...); } namespace Test { -// Declare a helper so that we can call it from VERIFY in included headers -void current_test_case_did_fail(); +// Declare helpers so that we can call them from VERIFY in included headers +// the setter for TestResult is already declared in TestResult.h +TestResult current_test_result(); } #define EXPECT_EQ(a, b) \ @@ -28,7 +30,7 @@ void current_test_case_did_fail(); auto rhs = (b); \ if (lhs != rhs) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -41,7 +43,7 @@ void current_test_case_did_fail(); if (ltruth != rtruth) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \ __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -53,7 +55,7 @@ void current_test_case_did_fail(); auto rhs = (b); \ if (lhs != rhs) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -63,7 +65,7 @@ void current_test_case_did_fail(); auto rhs = (b); \ if (lhs == rhs) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -71,7 +73,7 @@ void current_test_case_did_fail(); do { \ if (!(x)) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -84,7 +86,7 @@ void current_test_case_did_fail(); ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \ " failed with lhs={}, rhs={}, (lhs-rhs)={}", \ __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) @@ -93,32 +95,32 @@ void current_test_case_did_fail(); #define FAIL(message) \ do { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \ - ::Test::current_test_case_did_fail(); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) // To use, specify the lambda to execute in a sub process and verify it exits: // EXPECT_CRASH("This should fail", []{ // return Test::Crash::Failure::DidNotCrash; // }); -#define EXPECT_CRASH(test_message, test_func) \ - do { \ - Test::Crash crash(test_message, test_func); \ - if (!crash.run()) \ - ::Test::current_test_case_did_fail(); \ +#define EXPECT_CRASH(test_message, test_func) \ + do { \ + Test::Crash crash(test_message, test_func); \ + if (!crash.run()) \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) -#define EXPECT_CRASH_WITH_SIGNAL(test_message, signal, test_func) \ - do { \ - Test::Crash crash(test_message, test_func, (signal)); \ - if (!crash.run()) \ - ::Test::current_test_case_did_fail(); \ +#define EXPECT_CRASH_WITH_SIGNAL(test_message, signal, test_func) \ + do { \ + Test::Crash crash(test_message, test_func, (signal)); \ + if (!crash.run()) \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) -#define EXPECT_NO_CRASH(test_message, test_func) \ - do { \ - Test::Crash crash(test_message, test_func, 0); \ - if (!crash.run()) \ - ::Test::current_test_case_did_fail(); \ +#define EXPECT_NO_CRASH(test_message, test_func) \ + do { \ + Test::Crash crash(test_message, test_func, 0); \ + if (!crash.run()) \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) #define TRY_OR_FAIL(expression) \ diff --git a/Userland/Libraries/LibTest/TestResult.h b/Userland/Libraries/LibTest/TestResult.h new file mode 100644 index 0000000000..2405bd6871 --- /dev/null +++ b/Userland/Libraries/LibTest/TestResult.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023, Martin Janiczek + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +namespace Test { + +// TestResult signals to the TestSuite how the TestCase execution went. +enum class TestResult { + NotRun, + + // Test fn ran to completion without setting any of the below flags + Passed, + + // Didn't get through EXPECT(...). + Failed, +}; + +// Used eg. to signal we've ran out of prerecorded random bits. +// Defined in TestSuite.cpp +void set_current_test_result(TestResult); + +} // namespace Test diff --git a/Userland/Libraries/LibTest/TestSuite.cpp b/Userland/Libraries/LibTest/TestSuite.cpp index f8c7f9a174..94acd72370 100644 --- a/Userland/Libraries/LibTest/TestSuite.cpp +++ b/Userland/Libraries/LibTest/TestSuite.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -40,9 +41,15 @@ private: }; // Declared in Macros.h -void current_test_case_did_fail() +TestResult current_test_result() { - TestSuite::the().current_test_case_did_fail(); + return TestSuite::the().current_test_result(); +} + +// Declared in Macros.h +void set_current_test_result(TestResult result) +{ + TestSuite::the().set_current_test_result(result); } // Declared in TestCase.h @@ -57,6 +64,20 @@ void set_suite_setup_function(Function setup) TestSuite::the().set_suite_setup(move(setup)); } +static DeprecatedString test_result_to_string(TestResult result) +{ + switch (result) { + case TestResult::NotRun: + return "Not run"; + case TestResult::Passed: + return "Completed"; + case TestResult::Failed: + return "Failed"; + default: + return "Unknown TestResult"; + } +} + int TestSuite::main(DeprecatedString const& suite_name, Span arguments) { m_suite_name = suite_name; @@ -116,8 +137,11 @@ Vector> TestSuite::find_cases(DeprecatedString const& se int TestSuite::run(Vector> const& tests) { size_t test_count = 0; + size_t test_passed_count = 0; size_t test_failed_count = 0; size_t benchmark_count = 0; + size_t benchmark_passed_count = 0; + size_t benchmark_failed_count = 0; TestElapsedTimer global_timer; for (auto const& t : tests) { @@ -125,7 +149,7 @@ int TestSuite::run(Vector> const& tests) auto const repetitions = t->is_benchmark() ? m_benchmark_repetitions : 1; warnln("Running {} '{}'.", test_type, t->name()); - m_current_test_case_passed = true; + m_current_test_result = TestResult::NotRun; u64 total_time = 0; u64 sum_of_squared_times = 0; @@ -140,6 +164,10 @@ int TestSuite::run(Vector> const& tests) sum_of_squared_times += iteration_time * iteration_time; min_time = min(min_time, iteration_time); max_time = max(max_time, iteration_time); + + // Non-randomized tests don't touch the test result when passing. + if (m_current_test_result == TestResult::NotRun) + m_current_test_result = TestResult::Passed; } if (repetitions != 1) { @@ -148,22 +176,40 @@ int TestSuite::run(Vector> const& tests) double standard_deviation = sqrt((sum_of_squared_times + repetitions * average_squared - 2 * total_time * average) / (repetitions - 1)); dbgln("{} {} '{}' on average in {:.1f}±{:.1f}ms (min={}ms, max={}ms, total={}ms)", - m_current_test_case_passed ? "Completed" : "Failed", test_type, t->name(), + test_result_to_string(m_current_test_result), test_type, t->name(), average, standard_deviation, min_time, max_time, total_time); } else { - dbgln("{} {} '{}' in {}ms", m_current_test_case_passed ? "Completed" : "Failed", test_type, t->name(), total_time); + dbgln("{} {} '{}' in {}ms", test_result_to_string(m_current_test_result), test_type, t->name(), total_time); } if (t->is_benchmark()) { m_benchtime += total_time; benchmark_count++; + + switch (m_current_test_result) { + case TestResult::Passed: + benchmark_passed_count++; + break; + case TestResult::Failed: + benchmark_failed_count++; + break; + default: + break; + } } else { m_testtime += total_time; test_count++; - } - if (!m_current_test_case_passed) { - test_failed_count++; + switch (m_current_test_result) { + case TestResult::Passed: + test_passed_count++; + break; + case TestResult::Failed: + test_failed_count++; + break; + default: + break; + } } } @@ -175,10 +221,29 @@ int TestSuite::run(Vector> const& tests) m_benchtime, global_timer.elapsed_milliseconds() - (m_testtime + m_benchtime)); - if (test_count != 0) - dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_count - test_failed_count, test_failed_count); + if (test_count != 0) { + if (test_passed_count == test_count) { + dbgln("All {} tests passed.", test_count); + } else if (test_passed_count + test_failed_count == test_count) { + dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_passed_count, test_failed_count); + } else { + dbgln("Out of {} tests, {} passed, {} failed and {} didn't finish for other reasons.", test_count, test_passed_count, test_failed_count, test_count - test_passed_count - test_failed_count); + } + } - return (int)test_failed_count; + if (benchmark_count != 0) { + if (benchmark_passed_count == benchmark_count) { + dbgln("All {} benchmarks passed.", benchmark_count); + } else if (benchmark_passed_count + benchmark_failed_count == benchmark_count) { + dbgln("Out of {} benchmarks, {} passed and {} failed.", benchmark_count, benchmark_passed_count, benchmark_failed_count); + } else { + dbgln("Out of {} benchmarks, {} passed, {} failed and {} didn't finish for other reasons.", benchmark_count, benchmark_passed_count, benchmark_failed_count, benchmark_count - benchmark_passed_count - benchmark_failed_count); + } + } + + // We have multiple TestResults, all except for Passed being "bad". + // Let's get a count of them: + return (int)(test_count - test_passed_count + benchmark_count - benchmark_passed_count); } -} +} // namespace Test diff --git a/Userland/Libraries/LibTest/TestSuite.h b/Userland/Libraries/LibTest/TestSuite.h index fd8d2afc6f..298cae5d10 100644 --- a/Userland/Libraries/LibTest/TestSuite.h +++ b/Userland/Libraries/LibTest/TestSuite.h @@ -13,6 +13,7 @@ #include #include #include +#include namespace Test { @@ -40,7 +41,8 @@ public: m_cases.append(test_case); } - void current_test_case_did_fail() { m_current_test_case_passed = false; } + TestResult current_test_result() const { return m_current_test_result; } + void set_current_test_result(TestResult result) { m_current_test_result = result; } void set_suite_setup(Function setup) { m_setup = move(setup); } @@ -51,8 +53,8 @@ private: u64 m_benchtime = 0; DeprecatedString m_suite_name; u64 m_benchmark_repetitions = 1; - bool m_current_test_case_passed = true; Function m_setup; + TestResult m_current_test_result = TestResult::NotRun; }; }