From 82fa65135aeca74b42fd67a5103cb7a12348ece9 Mon Sep 17 00:00:00 2001 From: Matthew Olsson Date: Sat, 4 Jul 2020 12:57:12 -0700 Subject: [PATCH] test-js: Allow skipping tests with "test.skip(name, callback)" Skipped tests count as a "pass" rather than a "fail" (i.e. a test suite with a skipped test will pass), however it does display a message when the test is printing. This is intended for tests which _should_ work, but currently do not. This should be preferred over "// FIXME" notes if possible. --- Libraries/LibJS/Tests/test-common.js | 27 +++++++++--- Userland/test-js.cpp | 65 +++++++++++++++++++++------- 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/Libraries/LibJS/Tests/test-common.js b/Libraries/LibJS/Tests/test-common.js index 5b120a9807..190cb50d19 100644 --- a/Libraries/LibJS/Tests/test-common.js +++ b/Libraries/LibJS/Tests/test-common.js @@ -415,7 +415,7 @@ class Expector { expect = value => new Expector(value); // describe is able to lump test results inside of it by using this context -// variable. Top level tests are assumed to be in the default context +// variable. Top level tests have the default suite message const defaultSuiteMessage = "__$$TOP_LEVEL$$__"; let suiteMessage = defaultSuiteMessage; @@ -425,19 +425,18 @@ describe = (message, callback) => { suiteMessage = defaultSuiteMessage; } -const getTestFunction = successMessage => (message, callback) => { +test = (message, callback) => { if (!__TestResults__[suiteMessage]) __TestResults__[suiteMessage] = {}; const suite = __TestResults__[suiteMessage]; - - if (!suite[message]) - suite[message] = {}; + if (suite[message]) + throw new Error("Duplicate test name: " + message); try { callback(); suite[message] = { - result: successMessage, + result: "pass", }; } catch (e) { suite[message] = { @@ -446,6 +445,20 @@ const getTestFunction = successMessage => (message, callback) => { } } -test = getTestFunction("pass"); +test.skip = (message, callback) => { + if (typeof callback !== "function") + throw new Error("test.skip has invalid second argument (must be a function)"); + + if (!__TestResults__[suiteMessage]) + __TestResults__[suiteMessage] = {}; + + const suite = __TestResults__[suiteMessage]; + if (suite[message]) + throw new Error("Duplicate test name: " + message); + + suite[message] = { + result: "skip", + } +} })(); diff --git a/Userland/test-js.cpp b/Userland/test-js.cpp index c9f6c86e2f..c8f13f4862 100644 --- a/Userland/test-js.cpp +++ b/Userland/test-js.cpp @@ -192,6 +192,7 @@ Vector tests_to_run = { enum class TestResult { Pass, Fail, + Skip, }; struct JSTest { @@ -201,7 +202,9 @@ struct JSTest { struct JSSuite { String name; - bool has_failed_tests { false }; + // A failed test takes precedence over a skipped test, which both have + // precedence over a passed test + TestResult most_severe_test_result { TestResult::Pass }; Vector tests {}; }; @@ -213,13 +216,16 @@ struct ParserError { struct JSFileResult { String name; Optional error {}; - bool has_failed_tests { false }; + // A failed test takes precedence over a skipped test, which both have + // precedence over a passed test + TestResult most_severe_test_result { TestResult::Pass }; Vector suites {}; }; struct JSTestRunnerCounts { int tests_failed { 0 }; int tests_passed { 0 }; + int tests_skipped { 0 }; int suites_failed { 0 }; int suites_passed { 0 }; int files_total { 0 }; @@ -361,19 +367,26 @@ JSFileResult TestRunner::run_file_test(const String& test_path) if (result_string == "pass") { test.result = TestResult::Pass; m_counts.tests_passed++; - } else { + } else if (result_string == "fail") { test.result = TestResult::Fail; m_counts.tests_failed++; - suite.has_failed_tests = true; + suite.most_severe_test_result = TestResult::Fail; + } else { + test.result = TestResult::Skip; + if (suite.most_severe_test_result == TestResult::Pass) + suite.most_severe_test_result = TestResult::Skip; + m_counts.tests_skipped++; } suite.tests.append(test); }); - if (suite.has_failed_tests) { + if (suite.most_severe_test_result == TestResult::Fail) { m_counts.suites_failed++; - file_result.has_failed_tests = true; + file_result.most_severe_test_result = TestResult::Fail; } else { + if (suite.most_severe_test_result == TestResult::Skip && file_result.most_severe_test_result == TestResult::Pass) + file_result.most_severe_test_result = TestResult::Skip; m_counts.suites_passed++; } @@ -390,6 +403,7 @@ enum Modifier { BG_GREEN, FG_RED, FG_GREEN, + FG_ORANGE, FG_GRAY, FG_BLACK, FG_BOLD, @@ -409,6 +423,8 @@ void print_modifiers(Vector modifiers) return "\033[38;2;255;0;102m"; case FG_GREEN: return "\033[38;2;102;255;0m"; + case FG_ORANGE: + return "\033[38;2;255;102;0m"; case FG_GRAY: return "\033[38;2;135;139;148m"; case FG_BLACK: @@ -426,7 +442,8 @@ void print_modifiers(Vector modifiers) void TestRunner::print_file_result(const JSFileResult& file_result) { - if (file_result.has_failed_tests || file_result.error.has_value()) { + + if (file_result.most_severe_test_result == TestResult::Fail || file_result.error.has_value()) { print_modifiers({ BG_RED, FG_BLACK, FG_BOLD }); printf(" FAIL "); print_modifiers({ CLEAR }); @@ -453,18 +470,26 @@ void TestRunner::print_file_result(const JSFileResult& file_result) return; } - if (file_result.has_failed_tests) { + if (file_result.most_severe_test_result != TestResult::Pass) { for (auto& suite : file_result.suites) { - if (!suite.has_failed_tests) + if (suite.most_severe_test_result == TestResult::Pass) continue; + bool failed = suite.most_severe_test_result == TestResult::Fail; + print_modifiers({ FG_GRAY, FG_BOLD }); - printf(" ❌ Suite: "); + + if (failed) { + printf(" ❌ Suite: "); + } else { + printf(" ⚠️️ Suite: "); + } + + print_modifiers({ CLEAR, FG_GRAY }); + if (suite.name == TOP_LEVEL_TEST_NAME) { - print_modifiers({ CLEAR, FG_GRAY }); printf("\n"); } else { - print_modifiers({ CLEAR, FG_RED }); printf("%s\n", suite.name.characters()); } print_modifiers({ CLEAR }); @@ -474,9 +499,14 @@ void TestRunner::print_file_result(const JSFileResult& file_result) continue; print_modifiers({ FG_GRAY, FG_BOLD }); - printf(" Test: "); - print_modifiers({ CLEAR, FG_RED }); - printf("%s\n", test.name.characters()); + printf(" Test: "); + if (test.result == TestResult::Fail) { + print_modifiers({ CLEAR, FG_RED }); + printf("%s (failed)\n", test.name.characters()); + } else { + print_modifiers({ CLEAR, FG_ORANGE }); + printf("%s (skipped)\n", test.name.characters()); + } print_modifiers({ CLEAR }); } } @@ -504,6 +534,11 @@ void TestRunner::print_test_results() const printf("%d failed, ", m_counts.tests_failed); print_modifiers({ CLEAR }); } + if (m_counts.tests_skipped) { + print_modifiers({ FG_ORANGE }); + printf("%d skipped, ", m_counts.tests_skipped); + print_modifiers({ CLEAR }); + } if (m_counts.tests_passed) { print_modifiers({ FG_GREEN }); printf("%d passed, ", m_counts.tests_passed);