RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-27 03:47:35 +00:00
Code Issues Activity Actions

LibTest: Expand test result bool to a TestResult

Browse source 
This will be used in the randomized tests a lot more than it is in the
unit tests / benchmarks; randomized tests will run the test function
multiple times, check the result and optionally start shrinking the
failing input. Generators will also be able to fail, resulting in some
of the new TestResult variants.
This commit is contained in:
Martin Janiczek 2023-10-24 00:19:04 +02:00 • committed by Andrew Kaster
parent 5a7f43ad38
commit a60e3b17b1
4 changed files with 133 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

50
Userland/Libraries/LibTest/Macros.h
View file

@ -11,6 +11,7 @@
#include <AK/CheckedFormatString.h> #include <AK/CheckedFormatString.h>
#include <AK/Math.h> #include <AK/Math.h>
#include <LibTest/CrashTest.h> #include <LibTest/CrashTest.h>
#include <LibTest/TestResult.h>
namespace AK { namespace AK {
template<typename... Parameters> template<typename... Parameters>
@ -18,8 +19,9 @@ void warnln(CheckedFormatString<Parameters...>&& fmtstr, Parameters const&...);
} }
namespace Test { namespace Test {
// Declare a helper so that we can call it from VERIFY in included headers // Declare helpers so that we can call them from VERIFY in included headers
void current_test_case_did_fail(); // the setter for TestResult is already declared in TestResult.h
TestResult current_test_result();
} }
#define EXPECT_EQ(a, b) \ #define EXPECT_EQ(a, b) \
@ -28,7 +30,7 @@ void current_test_case_did_fail();
auto rhs = (b); \ auto rhs = (b); \
if (lhs != rhs) { \ if (lhs != rhs) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} \ } \
} while (false) } while (false)
@ -41,7 +43,7 @@ void current_test_case_did_fail();
if (ltruth != rtruth) { \ if (ltruth != rtruth) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \
__FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \ __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} \ } \
} while (false) } while (false)
@ -53,7 +55,7 @@ void current_test_case_did_fail();
auto rhs = (b); \ auto rhs = (b); \
if (lhs != rhs) { \ if (lhs != rhs) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} \ } \
} while (false) } while (false)
@ -63,7 +65,7 @@ void current_test_case_did_fail();
auto rhs = (b); \ auto rhs = (b); \
if (lhs == rhs) { \ if (lhs == rhs) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} \ } \
} while (false) } while (false)
@ -71,7 +73,7 @@ void current_test_case_did_fail();
do { \ do { \
if (!(x)) { \ if (!(x)) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} \ } \
} while (false) } while (false)
@ -84,7 +86,7 @@ void current_test_case_did_fail();
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \
" failed with lhs={}, rhs={}, (lhs-rhs)={}", \ " failed with lhs={}, rhs={}, (lhs-rhs)={}", \
__FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \ __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} \ } \
} while (false) } while (false)
@ -93,32 +95,32 @@ void current_test_case_did_fail();
#define FAIL(message) \ #define FAIL(message) \
do { \ do { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} while (false) } while (false)
// To use, specify the lambda to execute in a sub process and verify it exits: // To use, specify the lambda to execute in a sub process and verify it exits:
// EXPECT_CRASH("This should fail", []{ // EXPECT_CRASH("This should fail", []{
// return Test::Crash::Failure::DidNotCrash; // return Test::Crash::Failure::DidNotCrash;
// }); // });
#define EXPECT_CRASH(test_message, test_func) \ #define EXPECT_CRASH(test_message, test_func) \
do { \ do { \
Test::Crash crash(test_message, test_func); \ Test::Crash crash(test_message, test_func); \
if (!crash.run()) \ if (!crash.run()) \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} while (false) } while (false)
#define EXPECT_CRASH_WITH_SIGNAL(test_message, signal, test_func) \ #define EXPECT_CRASH_WITH_SIGNAL(test_message, signal, test_func) \
do { \ do { \
Test::Crash crash(test_message, test_func, (signal)); \ Test::Crash crash(test_message, test_func, (signal)); \
if (!crash.run()) \ if (!crash.run()) \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} while (false) } while (false)
#define EXPECT_NO_CRASH(test_message, test_func) \ #define EXPECT_NO_CRASH(test_message, test_func) \
do { \ do { \
Test::Crash crash(test_message, test_func, 0); \ Test::Crash crash(test_message, test_func, 0); \
if (!crash.run()) \ if (!crash.run()) \
::Test::current_test_case_did_fail(); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} while (false) } while (false)
#define TRY_OR_FAIL(expression) \ #define TRY_OR_FAIL(expression) \

26
Userland/Libraries/LibTest/TestResult.h Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright (c) 2023, Martin Janiczek <martin@janiczek.cz>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
namespace Test {
// TestResult signals to the TestSuite how the TestCase execution went.
enum class TestResult {
NotRun,
// Test fn ran to completion without setting any of the below flags
Passed,
// Didn't get through EXPECT(...).
Failed,
};
// Used eg. to signal we've ran out of prerecorded random bits.
// Defined in TestSuite.cpp
void set_current_test_result(TestResult);
} // namespace Test

89
Userland/Libraries/LibTest/TestSuite.cpp
View file

@ -9,6 +9,7 @@
#include <AK/Function.h> #include <AK/Function.h>
#include <LibCore/ArgsParser.h> #include <LibCore/ArgsParser.h>
#include <LibTest/TestResult.h>
#include <LibTest/TestSuite.h> #include <LibTest/TestSuite.h>
#include <math.h> #include <math.h>
#include <stdlib.h> #include <stdlib.h>
@ -40,9 +41,15 @@ private:
}; };
// Declared in Macros.h // Declared in Macros.h
void current_test_case_did_fail() TestResult current_test_result()
{ {
TestSuite::the().current_test_case_did_fail(); return TestSuite::the().current_test_result();
}
// Declared in Macros.h
void set_current_test_result(TestResult result)
{
TestSuite::the().set_current_test_result(result);
} }
// Declared in TestCase.h // Declared in TestCase.h
@ -57,6 +64,20 @@ void set_suite_setup_function(Function<void()> setup)
TestSuite::the().set_suite_setup(move(setup)); TestSuite::the().set_suite_setup(move(setup));
} }
static DeprecatedString test_result_to_string(TestResult result)
{
switch (result) {
case TestResult::NotRun:
return "Not run";
case TestResult::Passed:
return "Completed";
case TestResult::Failed:
return "Failed";
default:
return "Unknown TestResult";
}
}
int TestSuite::main(DeprecatedString const& suite_name, Span<StringView> arguments) int TestSuite::main(DeprecatedString const& suite_name, Span<StringView> arguments)
{ {
m_suite_name = suite_name; m_suite_name = suite_name;
@ -116,8 +137,11 @@ Vector<NonnullRefPtr<TestCase>> TestSuite::find_cases(DeprecatedString const& se
int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests) int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
{ {
size_t test_count = 0; size_t test_count = 0;
size_t test_passed_count = 0;
size_t test_failed_count = 0; size_t test_failed_count = 0;
size_t benchmark_count = 0; size_t benchmark_count = 0;
size_t benchmark_passed_count = 0;
size_t benchmark_failed_count = 0;
TestElapsedTimer global_timer; TestElapsedTimer global_timer;
for (auto const& t : tests) { for (auto const& t : tests) {
@ -125,7 +149,7 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
auto const repetitions = t->is_benchmark() ? m_benchmark_repetitions : 1; auto const repetitions = t->is_benchmark() ? m_benchmark_repetitions : 1;
warnln("Running {} '{}'.", test_type, t->name()); warnln("Running {} '{}'.", test_type, t->name());
m_current_test_case_passed = true; m_current_test_result = TestResult::NotRun;
u64 total_time = 0; u64 total_time = 0;
u64 sum_of_squared_times = 0; u64 sum_of_squared_times = 0;
@ -140,6 +164,10 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
sum_of_squared_times += iteration_time * iteration_time; sum_of_squared_times += iteration_time * iteration_time;
min_time = min(min_time, iteration_time); min_time = min(min_time, iteration_time);
max_time = max(max_time, iteration_time); max_time = max(max_time, iteration_time);
// Non-randomized tests don't touch the test result when passing.
if (m_current_test_result == TestResult::NotRun)
m_current_test_result = TestResult::Passed;
} }
if (repetitions != 1) { if (repetitions != 1) {
@ -148,22 +176,40 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
double standard_deviation = sqrt((sum_of_squared_times + repetitions * average_squared - 2 * total_time * average) / (repetitions - 1)); double standard_deviation = sqrt((sum_of_squared_times + repetitions * average_squared - 2 * total_time * average) / (repetitions - 1));
dbgln("{} {} '{}' on average in {:.1f}±{:.1f}ms (min={}ms, max={}ms, total={}ms)", dbgln("{} {} '{}' on average in {:.1f}±{:.1f}ms (min={}ms, max={}ms, total={}ms)",
m_current_test_case_passed ? "Completed" : "Failed", test_type, t->name(), test_result_to_string(m_current_test_result), test_type, t->name(),
average, standard_deviation, min_time, max_time, total_time); average, standard_deviation, min_time, max_time, total_time);
} else { } else {
dbgln("{} {} '{}' in {}ms", m_current_test_case_passed ? "Completed" : "Failed", test_type, t->name(), total_time); dbgln("{} {} '{}' in {}ms", test_result_to_string(m_current_test_result), test_type, t->name(), total_time);
} }
if (t->is_benchmark()) { if (t->is_benchmark()) {
m_benchtime += total_time; m_benchtime += total_time;
benchmark_count++; benchmark_count++;
switch (m_current_test_result) {
case TestResult::Passed:
benchmark_passed_count++;
break;
case TestResult::Failed:
benchmark_failed_count++;
break;
default:
break;
}
} else { } else {
m_testtime += total_time; m_testtime += total_time;
test_count++; test_count++;
}
if (!m_current_test_case_passed) { switch (m_current_test_result) {
test_failed_count++; case TestResult::Passed:
test_passed_count++;
break;
case TestResult::Failed:
test_failed_count++;
break;
default:
break;
}
} }
} }
@ -175,10 +221,29 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
m_benchtime, m_benchtime,
global_timer.elapsed_milliseconds() - (m_testtime + m_benchtime)); global_timer.elapsed_milliseconds() - (m_testtime + m_benchtime));
if (test_count != 0) if (test_count != 0) {
dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_count - test_failed_count, test_failed_count); if (test_passed_count == test_count) {
dbgln("All {} tests passed.", test_count);
} else if (test_passed_count + test_failed_count == test_count) {
dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_passed_count, test_failed_count);
} else {
dbgln("Out of {} tests, {} passed, {} failed and {} didn't finish for other reasons.", test_count, test_passed_count, test_failed_count, test_count - test_passed_count - test_failed_count);
}
}
return (int)test_failed_count; if (benchmark_count != 0) {
if (benchmark_passed_count == benchmark_count) {
dbgln("All {} benchmarks passed.", benchmark_count);
} else if (benchmark_passed_count + benchmark_failed_count == benchmark_count) {
dbgln("Out of {} benchmarks, {} passed and {} failed.", benchmark_count, benchmark_passed_count, benchmark_failed_count);
} else {
dbgln("Out of {} benchmarks, {} passed, {} failed and {} didn't finish for other reasons.", benchmark_count, benchmark_passed_count, benchmark_failed_count, benchmark_count - benchmark_passed_count - benchmark_failed_count);
}
}
// We have multiple TestResults, all except for Passed being "bad".
// Let's get a count of them:
return (int)(test_count - test_passed_count + benchmark_count - benchmark_passed_count);
} }
} } // namespace Test

6
Userland/Libraries/LibTest/TestSuite.h
View file

@ -13,6 +13,7 @@
#include <AK/Function.h> #include <AK/Function.h>
#include <AK/Vector.h> #include <AK/Vector.h>
#include <LibTest/TestCase.h> #include <LibTest/TestCase.h>
#include <LibTest/TestResult.h>
namespace Test { namespace Test {
@ -40,7 +41,8 @@ public:
m_cases.append(test_case); m_cases.append(test_case);
} }
void current_test_case_did_fail() { m_current_test_case_passed = false; } TestResult current_test_result() const { return m_current_test_result; }
void set_current_test_result(TestResult result) { m_current_test_result = result; }
void set_suite_setup(Function<void()> setup) { m_setup = move(setup); } void set_suite_setup(Function<void()> setup) { m_setup = move(setup); }
@ -51,8 +53,8 @@ private:
u64 m_benchtime = 0; u64 m_benchtime = 0;
DeprecatedString m_suite_name; DeprecatedString m_suite_name;
u64 m_benchmark_repetitions = 1; u64 m_benchmark_repetitions = 1;
bool m_current_test_case_passed = true;
Function<void()> m_setup; Function<void()> m_setup;
TestResult m_current_test_result = TestResult::NotRun;
}; };
} }