From 7e5a3650fe2dbef042ad6420aa8e2d960d7cebb7 Mon Sep 17 00:00:00 2001 From: Martin Janiczek Date: Tue, 24 Oct 2023 00:35:15 +0200 Subject: [PATCH] LibTest: Add the RandomnessSource abstraction This will be a foundational part of bootstrapping generators: this is the way they'll get prerecorded values from / record random values into RandomRuns. (Generators don't get in contact with RandomRuns themselves, they just interact with the RandomnessSource.) --- Userland/Libraries/LibTest/Macros.h | 4 ++ .../LibTest/Randomized/RandomnessSource.h | 61 +++++++++++++++++++ Userland/Libraries/LibTest/TestResult.h | 4 ++ Userland/Libraries/LibTest/TestSuite.cpp | 14 +++++ Userland/Libraries/LibTest/TestSuite.h | 8 +++ 5 files changed, 91 insertions(+) create mode 100644 Userland/Libraries/LibTest/Randomized/RandomnessSource.h diff --git a/Userland/Libraries/LibTest/Macros.h b/Userland/Libraries/LibTest/Macros.h index 472e73752f..ed01cedfda 100644 --- a/Userland/Libraries/LibTest/Macros.h +++ b/Userland/Libraries/LibTest/Macros.h @@ -11,6 +11,7 @@ #include #include #include +#include #include namespace AK { @@ -22,6 +23,9 @@ namespace Test { // Declare helpers so that we can call them from VERIFY in included headers // the setter for TestResult is already declared in TestResult.h TestResult current_test_result(); + +Randomized::RandomnessSource& randomness_source(); +void set_randomness_source(Randomized::RandomnessSource); } #define EXPECT_EQ(a, b) \ diff --git a/Userland/Libraries/LibTest/Randomized/RandomnessSource.h b/Userland/Libraries/LibTest/Randomized/RandomnessSource.h new file mode 100644 index 0000000000..af3f34fee0 --- /dev/null +++ b/Userland/Libraries/LibTest/Randomized/RandomnessSource.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023, Martin Janiczek + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +namespace Test { +namespace Randomized { + +// RandomnessSource provides random bits to Generators. +// +// If it's live, a PRNG will be used and the random values will be recorded into +// its RandomRun. +// +// If it's recorded, its RandomRun will be used to "mock" the PRNG. This allows +// us to replay the generation of a particular value, and to test out +// "alternative histories": "what if the PRNG generated 0 instead of 13 here?" +class RandomnessSource { +public: + static RandomnessSource live() { return RandomnessSource(RandomRun(), true); } + static RandomnessSource recorded(RandomRun const& run) { return RandomnessSource(run, false); } + RandomRun& run() { return m_run; } + u32 draw_value(u32 max, Function random_generator) + { + // Live: use the random generator and remember the value. + if (m_is_live) { + u32 value = random_generator(); + m_run.append(value); + return value; + } + + // Not live! let's get another prerecorded value. + auto next = m_run.next(); + if (next.has_value()) { + return min(next.value(), max); + } + + // Signal a failure. The value returned doesn't matter at this point but + // we need to return something. + set_current_test_result(TestResult::Overrun); + return 0; + } + +private: + explicit RandomnessSource(RandomRun const& run, bool is_live) + : m_run(run) + , m_is_live(is_live) + { + } + RandomRun m_run; + bool m_is_live; +}; + +} // namespace Randomized +} // namespace Test diff --git a/Userland/Libraries/LibTest/TestResult.h b/Userland/Libraries/LibTest/TestResult.h index 2405bd6871..d63b763800 100644 --- a/Userland/Libraries/LibTest/TestResult.h +++ b/Userland/Libraries/LibTest/TestResult.h @@ -17,6 +17,10 @@ enum class TestResult { // Didn't get through EXPECT(...). Failed, + + // Ran out of RandomRun data (in a randomized test, when shrinking). + // This is fine, we'll just try some other shrink. + Overrun, }; // Used eg. to signal we've ran out of prerecorded random bits. diff --git a/Userland/Libraries/LibTest/TestSuite.cpp b/Userland/Libraries/LibTest/TestSuite.cpp index 94acd72370..f02321d01a 100644 --- a/Userland/Libraries/LibTest/TestSuite.cpp +++ b/Userland/Libraries/LibTest/TestSuite.cpp @@ -52,6 +52,18 @@ void set_current_test_result(TestResult result) TestSuite::the().set_current_test_result(result); } +// Declared in Macros.h +void set_randomness_source(Randomized::RandomnessSource source) +{ + TestSuite::the().set_randomness_source(move(source)); +} + +// Declared in Macros.h +Randomized::RandomnessSource& randomness_source() +{ + return TestSuite::the().randomness_source(); +} + // Declared in TestCase.h void add_test_case_to_suite(NonnullRefPtr const& test_case) { @@ -73,6 +85,8 @@ static DeprecatedString test_result_to_string(TestResult result) return "Completed"; case TestResult::Failed: return "Failed"; + case TestResult::Overrun: + return "Ran out of randomness"; default: return "Unknown TestResult"; } diff --git a/Userland/Libraries/LibTest/TestSuite.h b/Userland/Libraries/LibTest/TestSuite.h index 298cae5d10..9ee1a6466f 100644 --- a/Userland/Libraries/LibTest/TestSuite.h +++ b/Userland/Libraries/LibTest/TestSuite.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,12 @@ public: void set_current_test_result(TestResult result) { m_current_test_result = result; } void set_suite_setup(Function setup) { m_setup = move(setup); } + // The RandomnessSource is where generators record / replay random data + // from. Initially a live "truly random" RandomnessSource is used, and when + // a failure is found, a set of hardcoded RandomnessSources is used during + // shrinking. + void set_randomness_source(Randomized::RandomnessSource source) { m_randomness_source = move(source); } + Randomized::RandomnessSource& randomness_source() { return m_randomness_source; } private: static TestSuite* s_global; @@ -55,6 +62,7 @@ private: u64 m_benchmark_repetitions = 1; Function m_setup; TestResult m_current_test_result = TestResult::NotRun; + Randomized::RandomnessSource m_randomness_source = Randomized::RandomnessSource::live(); }; }