diff --git a/AK/TestSuite.h b/AK/TestSuite.h index 9276896286..b18ea0176e 100644 --- a/AK/TestSuite.h +++ b/AK/TestSuite.h @@ -34,6 +34,10 @@ namespace AK { template void warnln(CheckedFormatString&& fmtstr, const Parameters&...); +// Declare a helper so that we can call it from VERIFY in included headers +// before defining TestSuite +inline void current_test_case_did_fail(); + } using AK::warnln; @@ -41,8 +45,10 @@ using AK::warnln; #undef VERIFY #define VERIFY(x) \ do { \ - if (!(x)) \ + if (!(x)) { \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: VERIFY({}) failed", __FILE__, __LINE__, #x); \ + current_test_case_did_fail(); \ + } \ } while (false) #undef VERIFY_NOT_REACHED @@ -127,23 +133,28 @@ public: s_global = nullptr; } - void run(const NonnullRefPtrVector&); - void main(const String& suite_name, int argc, char** argv); + int run(const NonnullRefPtrVector&); + int main(const String& suite_name, int argc, char** argv); NonnullRefPtrVector find_cases(const String& search, bool find_tests, bool find_benchmarks); void add_case(const NonnullRefPtr& test_case) { m_cases.append(test_case); } + void current_test_case_did_fail() { m_current_test_case_passed = false; } + private: static TestSuite* s_global; NonnullRefPtrVector m_cases; u64 m_testtime = 0; u64 m_benchtime = 0; String m_suite_name; + bool m_current_test_case_passed = true; }; -void TestSuite::main(const String& suite_name, int argc, char** argv) +inline void current_test_case_did_fail() { TestSuite::the().current_test_case_did_fail(); } + +int TestSuite::main(const String& suite_name, int argc, char** argv) { m_suite_name = suite_name; @@ -167,11 +178,12 @@ void TestSuite::main(const String& suite_name, int argc, char** argv) for (const auto& test : matching_tests) { outln(" {}", test.name()); } - } else { - outln("Running {} cases out of {}.", matching_tests.size(), m_cases.size()); - - run(matching_tests); + return 0; } + + outln("Running {} cases out of {}.", matching_tests.size(), m_cases.size()); + + return run(matching_tests); } NonnullRefPtrVector TestSuite::find_cases(const String& search, bool find_tests, bool find_benchmarks) @@ -194,9 +206,10 @@ NonnullRefPtrVector TestSuite::find_cases(const String& search, bool f return matches; } -void TestSuite::run(const NonnullRefPtrVector& tests) +int TestSuite::run(const NonnullRefPtrVector& tests) { size_t test_count = 0; + size_t test_failed_count = 0; size_t benchmark_count = 0; TestElapsedTimer global_timer; @@ -204,12 +217,13 @@ void TestSuite::run(const NonnullRefPtrVector& tests) const auto test_type = t.is_benchmark() ? "benchmark" : "test"; warnln("Running {} '{}'.", test_type, t.name()); + m_current_test_case_passed = true; TestElapsedTimer timer; t.func()(); const auto time = timer.elapsed_milliseconds(); - dbgln("Completed {} '{}' in {}ms", test_type, t.name(), time); + dbgln("{} {} '{}' in {}ms", m_current_test_case_passed ? "Completed" : "Failed", test_type, t.name(), time); if (t.is_benchmark()) { m_benchtime += time; @@ -218,6 +232,10 @@ void TestSuite::run(const NonnullRefPtrVector& tests) m_testtime += time; test_count++; } + + if (!m_current_test_case_passed) { + test_failed_count++; + } } dbgln("Finished {} tests and {} benchmarks in {}ms ({}ms tests, {}ms benchmarks, {}ms other).", @@ -227,10 +245,14 @@ void TestSuite::run(const NonnullRefPtrVector& tests) m_testtime, m_benchtime, global_timer.elapsed_milliseconds() - (m_testtime + m_benchtime)); + dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_count - test_failed_count, test_failed_count); + + return (int)test_failed_count; } } +using AK::current_test_case_did_fail; using AK::TestCase; using AK::TestSuite; @@ -268,16 +290,19 @@ using AK::TestSuite; int main(int argc, char** argv) \ { \ static_assert(compiletime_lenof(#x) != 0, "Set SuiteName"); \ - TestSuite::the().main(#x, argc, argv); \ + int ret = TestSuite::the().main(#x, argc, argv); \ TestSuite::release(); \ + return ret; \ } #define EXPECT_EQ(a, b) \ do { \ auto lhs = (a); \ auto rhs = (b); \ - if (lhs != rhs) \ + if (lhs != rhs) { \ warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ + current_test_case_did_fail(); \ + } \ } while (false) // If you're stuck and `EXPECT_EQ` seems to refuse to print anything useful, @@ -286,12 +311,16 @@ using AK::TestSuite; do { \ auto lhs = (a); \ auto rhs = (b); \ - if (lhs != rhs) \ + if (lhs != rhs) { \ warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \ + current_test_case_did_fail(); \ + } \ } while (false) #define EXPECT(x) \ do { \ - if (!(x)) \ + if (!(x)) { \ warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \ + current_test_case_did_fail(); \ + } \ } while (false) diff --git a/Meta/Lagom/CMakeLists.txt b/Meta/Lagom/CMakeLists.txt index a91ff834fe..cd1f2e892e 100644 --- a/Meta/Lagom/CMakeLists.txt +++ b/Meta/Lagom/CMakeLists.txt @@ -177,13 +177,6 @@ if (BUILD_LAGOM) # FIXME: Only TestJSON needs this property WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../AK/Tests ) - - set_tests_properties( - ${name}_lagom - PROPERTIES - FAIL_REGULAR_EXPRESSION - "FAIL" - ) endforeach() foreach(source ${LIBREGEX_TESTS}) @@ -195,13 +188,6 @@ if (BUILD_LAGOM) COMMAND ${name}_lagom WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) - - set_tests_properties( - ${name}_lagom - PROPERTIES - FAIL_REGULAR_EXPRESSION - "FAIL" - ) endforeach() endif() endif()