diff --git a/Userland/Libraries/LibTest/TestCase.h b/Userland/Libraries/LibTest/TestCase.h index a98cc6ee09..fa55210919 100644 --- a/Userland/Libraries/LibTest/TestCase.h +++ b/Userland/Libraries/LibTest/TestCase.h @@ -8,16 +8,36 @@ #pragma once #include // intentionally first -- we redefine VERIFY and friends in here +#include +#include #include #include #include #include +#ifndef MAX_GENERATED_VALUES_PER_TEST +# define MAX_GENERATED_VALUES_PER_TEST 100 +#endif + +#ifndef MAX_GEN_ATTEMPTS_PER_VALUE +# define MAX_GEN_ATTEMPTS_PER_VALUE 15 +#endif + namespace Test { using TestFunction = Function; +inline void run_with_randomness_source(Randomized::RandomnessSource source, TestFunction const& test_function) +{ + set_randomness_source(move(source)); + set_current_test_result(TestResult::NotRun); + test_function(); + if (current_test_result() == TestResult::NotRun) { + set_current_test_result(TestResult::Passed); + } +} + class TestCase : public RefCounted { public: TestCase(DeprecatedString const& name, TestFunction&& fn, bool is_benchmark) @@ -31,6 +51,60 @@ public: DeprecatedString const& name() const { return m_name; } TestFunction const& func() const { return m_function; } + static NonnullRefPtr randomized(DeprecatedString const& name, TestFunction&& test_function) + { + using namespace Randomized; + TestFunction test_case_function = [test_function = move(test_function)]() { + for (u32 i = 0; i < MAX_GENERATED_VALUES_PER_TEST; ++i) { + bool generated_successfully = false; + u8 gen_attempt; + for (gen_attempt = 0; gen_attempt < MAX_GEN_ATTEMPTS_PER_VALUE && !generated_successfully; ++gen_attempt) { + // We're going to run the test function many times, so let's turn off the reporting until we finish. + disable_reporting(); + + set_current_test_result(TestResult::NotRun); + run_with_randomness_source(RandomnessSource::live(), test_function); + switch (current_test_result()) { + case TestResult::NotRun: + VERIFY_NOT_REACHED(); + break; + case TestResult::Passed: { + generated_successfully = true; + break; + } + case TestResult::Failed: { + generated_successfully = true; + RandomRun first_failure = randomness_source().run(); + RandomRun best_failure = shrink(first_failure, test_function); + + // Run one last time with reporting on, so that the user can see the minimal failure + enable_reporting(); + run_with_randomness_source(RandomnessSource::recorded(best_failure), test_function); + return; + } + case TestResult::Rejected: + break; + case TestResult::Overrun: + break; + default: + VERIFY_NOT_REACHED(); + break; + } + } + enable_reporting(); + if (!generated_successfully) { + // The loop above got to the full MAX_GEN_ATTEMPTS_PER_VALUE and gave up. + // Run one last time with reporting on, so that the user gets the REJECTED message. + RandomRun last_failure = randomness_source().run(); + run_with_randomness_source(RandomnessSource::recorded(last_failure), test_function); + return; + } + } + // MAX_GENERATED_VALUES_PER_TEST values generated, all passed the test. + }; + return make_ref_counted(name, move(test_case_function), false); + } + private: DeprecatedString m_name; TestFunction m_function; @@ -53,6 +127,8 @@ void set_suite_setup_function(Function setup); static struct __setup_type __setup_type; \ static void __setup() +// Unit test + #define __TESTCASE_FUNC(x) __test_##x #define __TESTCASE_TYPE(x) __TestCase_##x @@ -68,6 +144,8 @@ void set_suite_setup_function(Function setup); static struct __TESTCASE_TYPE(x) __TESTCASE_TYPE(x); \ static void __TESTCASE_FUNC(x)() +// Benchmark + #define __BENCHMARK_FUNC(x) __benchmark_##x #define __BENCHMARK_TYPE(x) __BenchmarkCase_##x @@ -83,6 +161,23 @@ void set_suite_setup_function(Function setup); static struct __BENCHMARK_TYPE(x) __BENCHMARK_TYPE(x); \ static void __BENCHMARK_FUNC(x)() +// Randomized test + +#define __RANDOMIZED_TEST_FUNC(x) __randomized_test_##x +#define __RANDOMIZED_TEST_TYPE(x) __RandomizedTestCase_##x + +#define RANDOMIZED_TEST_CASE(x) \ + static void __RANDOMIZED_TEST_FUNC(x)(); \ + struct __RANDOMIZED_TEST_TYPE(x) { \ + __RANDOMIZED_TEST_TYPE(x) \ + () \ + { \ + add_test_case_to_suite(::Test::TestCase::randomized(#x, __RANDOMIZED_TEST_FUNC(x))); \ + } \ + }; \ + static struct __RANDOMIZED_TEST_TYPE(x) __RANDOMIZED_TEST_TYPE(x); \ + static void __RANDOMIZED_TEST_FUNC(x)() + // This allows us to print the generated locals in the test after a failure is fully shrunk. #define GEN(identifier, value) \ auto identifier = (value); \