From 7948897688f84f4aa85adf3c1f9f96b07f27ddec Mon Sep 17 00:00:00 2001 From: davidot Date: Sun, 11 Sep 2022 09:14:32 +0200 Subject: [PATCH] LibJS: Import the libjs-test262-runner from linusg/libjs-test262 For now this is a lagom only application as it is not compatible with serenity in its current state. The only change is that it is released under a different license with permission from all the authors. --- Meta/Lagom/CMakeLists.txt | 4 + Tests/LibJS/test262-runner.cpp | 789 +++++++++++++++++++++++++++++++++ 2 files changed, 793 insertions(+) create mode 100644 Tests/LibJS/test262-runner.cpp diff --git a/Meta/Lagom/CMakeLists.txt b/Meta/Lagom/CMakeLists.txt index 7c7fad48fb..b8812bf6c6 100644 --- a/Meta/Lagom/CMakeLists.txt +++ b/Meta/Lagom/CMakeLists.txt @@ -621,6 +621,10 @@ if (BUILD_LAGOM) set_target_properties(shell_lagom PROPERTIES OUTPUT_NAME shell) target_link_libraries(shell_lagom LibCore LibShell LibMain) + add_executable(test262-runner_lagom ../../Tests/LibJS/test262-runner.cpp) + set_target_properties(test262-runner_lagom PROPERTIES OUTPUT_NAME test262-runner) + target_link_libraries(test262-runner_lagom LibJS LibCore) + add_executable(wasm_lagom ../../Userland/Utilities/wasm.cpp) set_target_properties(wasm_lagom PROPERTIES OUTPUT_NAME wasm) target_link_libraries(wasm_lagom LibCore LibWasm LibLine LibMain) diff --git a/Tests/LibJS/test262-runner.cpp b/Tests/LibJS/test262-runner.cpp new file mode 100644 index 0000000000..0b4e3e595a --- /dev/null +++ b/Tests/LibJS/test262-runner.cpp @@ -0,0 +1,789 @@ +/* + * Copyright (c) 2021, Linus Groh + * Copyright (c) 2021-2022, David Tuin + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static String s_current_test = ""; +static bool s_use_bytecode = false; +static bool s_parse_only = false; +static String s_harness_file_directory; + +enum class NegativePhase { + ParseOrEarly, + Resolution, + Runtime, + Harness +}; + +struct TestError { + NegativePhase phase { NegativePhase::ParseOrEarly }; + String type; + String details; + String harness_file; +}; + +using ScriptOrModuleProgram = Variant, JS::NonnullGCPtr>; + +template +static Result parse_program(JS::Realm& realm, StringView source, StringView filepath) +{ + auto script_or_error = ScriptType::parse(source, realm, filepath); + if (script_or_error.is_error()) { + return TestError { + NegativePhase::ParseOrEarly, + "SyntaxError", + script_or_error.error()[0].to_string(), + "" + }; + } + return ScriptOrModuleProgram { script_or_error.release_value() }; +} + +static Result parse_program(JS::Realm& realm, StringView source, StringView filepath, JS::Program::Type program_type) +{ + if (program_type == JS::Program::Type::Script) + return parse_program(realm, source, filepath); + return parse_program(realm, source, filepath); +} + +template +static Result run_program(InterpreterT& interpreter, ScriptOrModuleProgram& program) +{ + auto result = JS::ThrowCompletionOr { JS::js_undefined() }; + if constexpr (IsSame) { + result = program.visit( + [&](auto& visitor) { + return interpreter.run(*visitor); + }); + } else { + auto program_node = program.visit( + [](auto& visitor) -> NonnullRefPtr { + return visitor->parse_node(); + }); + + auto unit_result = JS::Bytecode::Generator::generate(program_node); + if (unit_result.is_error()) { + result = JS::throw_completion(JS::InternalError::create(interpreter.realm(), String::formatted("TODO({})", unit_result.error().to_string()))); + } else { + auto unit = unit_result.release_value(); + auto& passes = JS::Bytecode::Interpreter::optimization_pipeline(); + passes.perform(*unit); + result = interpreter.run(*unit); + } + } + + if (result.is_error()) { + auto error_value = *result.throw_completion().value(); + TestError error; + error.phase = NegativePhase::Runtime; + if (error_value.is_object()) { + auto& object = error_value.as_object(); + + auto name = object.get_without_side_effects("name"); + if (!name.is_empty() && !name.is_accessor()) { + error.type = name.to_string_without_side_effects(); + } else { + auto constructor = object.get_without_side_effects("constructor"); + if (constructor.is_object()) { + name = constructor.as_object().get_without_side_effects("name"); + if (!name.is_undefined()) + error.type = name.to_string_without_side_effects(); + } + } + + auto message = object.get_without_side_effects("message"); + if (!message.is_empty() && !message.is_accessor()) + error.details = message.to_string_without_side_effects(); + } + if (error.type.is_empty()) + error.type = error_value.to_string_without_side_effects(); + return error; + } + return {}; +} + +static HashMap s_cached_harness_files; + +static Result read_harness_file(StringView harness_file) +{ + auto cache = s_cached_harness_files.find(harness_file); + if (cache == s_cached_harness_files.end()) { + auto file = Core::File::construct(String::formatted("{}{}", s_harness_file_directory, harness_file)); + if (!file->open(Core::OpenMode::ReadOnly)) { + return TestError { + NegativePhase::Harness, + "filesystem", + String::formatted("Could not open file: {}", harness_file), + harness_file + }; + } + + auto contents = file->read_all(); + StringView contents_view = contents; + s_cached_harness_files.set(harness_file, contents_view.to_string()); + cache = s_cached_harness_files.find(harness_file); + VERIFY(cache != s_cached_harness_files.end()); + } + return cache->value.view(); +} + +static Result, TestError> parse_harness_files(JS::Realm& realm, StringView harness_file) +{ + auto source_or_error = read_harness_file(harness_file); + if (source_or_error.is_error()) + return source_or_error.release_error(); + auto program_or_error = parse_program(realm, source_or_error.value(), harness_file); + if (program_or_error.is_error()) { + return TestError { + NegativePhase::Harness, + program_or_error.error().type, + program_or_error.error().details, + harness_file + }; + } + return program_or_error.release_value().get>(); +} + +static Result run_test(StringView source, StringView filepath, JS::Program::Type program_type, Vector const& harness_files) +{ + if (s_parse_only) { + // Creating the vm and interpreter is heavy so we just parse directly here. + auto parser = JS::Parser(JS::Lexer(source, filepath), program_type); + auto program_or_error = parser.parse_program(); + if (parser.has_errors()) { + return TestError { + NegativePhase::ParseOrEarly, + "SyntaxError", + parser.errors()[0].to_string(), + "" + }; + } + return {}; + } + + auto vm = JS::VM::create(); + vm->enable_default_host_import_module_dynamically_hook(); + auto ast_interpreter = JS::Interpreter::create(*vm); + auto& realm = ast_interpreter->realm(); + + auto program_or_error = parse_program(realm, source, filepath, program_type); + if (program_or_error.is_error()) + return program_or_error.release_error(); + + OwnPtr bytecode_interpreter = nullptr; + if (s_use_bytecode) + bytecode_interpreter = make(realm); + + auto run_with_interpreter = [&](ScriptOrModuleProgram& program) { + if (s_use_bytecode) + return run_program(*bytecode_interpreter, program); + return run_program(*ast_interpreter, program); + }; + + for (auto& harness_file : harness_files) { + auto harness_program_or_error = parse_harness_files(realm, harness_file); + if (harness_program_or_error.is_error()) + return harness_program_or_error.release_error(); + ScriptOrModuleProgram harness_program { harness_program_or_error.release_value() }; + auto result = run_with_interpreter(harness_program); + if (result.is_error()) { + return TestError { + NegativePhase::Harness, + result.error().type, + result.error().details, + harness_file + }; + } + } + + return run_with_interpreter(program_or_error.value()); +} + +enum class StrictMode { + Both, + NoStrict, + OnlyStrict +}; + +static constexpr auto sta_harness_file = "sta.js"sv; +static constexpr auto assert_harness_file = "assert.js"sv; +static constexpr auto async_include = "doneprintHandle.js"sv; + +struct TestMetadata { + Vector harness_files { sta_harness_file, assert_harness_file }; + + StrictMode strict_mode { StrictMode::Both }; + JS::Program::Type program_type { JS::Program::Type::Script }; + bool is_async { false }; + + bool is_negative { false }; + NegativePhase phase { NegativePhase::ParseOrEarly }; + StringView type; +}; + +static Result extract_metadata(StringView source) +{ + auto lines = source.lines(); + + TestMetadata metadata; + + bool parsing_negative = false; + String failed_message; + + auto parse_list = [&](StringView line) { + auto start = line.find('['); + if (!start.has_value()) + return Vector {}; + + Vector items; + + auto end = line.find_last(']'); + if (!end.has_value() || end.value() <= start.value()) { + failed_message = String::formatted("Can't parse list in '{}'", line); + return items; + } + + auto list = line.substring_view(start.value() + 1, end.value() - start.value() - 1); + for (auto const& item : list.split_view(","sv)) + items.append(item.trim_whitespace(TrimMode::Both)); + return items; + }; + + auto second_word = [&](StringView line) { + auto separator = line.find(' '); + if (!separator.has_value() || separator.value() >= (line.length() - 1u)) { + failed_message = String::formatted("Can't parse value after space in '{}'", line); + return ""sv; + } + return line.substring_view(separator.value() + 1); + }; + + Vector include_list; + bool parsing_includes_list = false; + bool has_phase = false; + + for (auto raw_line : lines) { + if (!failed_message.is_empty()) + break; + + if (raw_line.starts_with("---*/"sv)) { + if (parsing_includes_list) { + for (auto& file : include_list) + metadata.harness_files.append(file); + } + return metadata; + } + + auto line = raw_line.trim_whitespace(); + + if (parsing_includes_list) { + if (line.starts_with('-')) { + include_list.append(second_word(line)); + continue; + } else { + if (include_list.is_empty()) { + failed_message = "Supposed to parse a list but found no entries"; + break; + } + + for (auto& file : include_list) + metadata.harness_files.append(file); + include_list.clear(); + + parsing_includes_list = false; + } + } + + if (parsing_negative) { + if (line.starts_with("phase:"sv)) { + auto phase = second_word(line); + has_phase = true; + if (phase == "early"sv || phase == "parse"sv) { + metadata.phase = NegativePhase::ParseOrEarly; + } else if (phase == "resolution"sv) { + metadata.phase = NegativePhase::Resolution; + } else if (phase == "runtime"sv) { + metadata.phase = NegativePhase::Runtime; + } else { + has_phase = false; + failed_message = String::formatted("Unknown negative phase: {}", phase); + break; + } + } else if (line.starts_with("type:"sv)) { + metadata.type = second_word(line); + } else { + if (!has_phase) { + failed_message = "Failed to find phase in negative attributes"; + break; + } + if (metadata.type.is_null()) { + failed_message = "Failed to find type in negative attributes"; + break; + } + + parsing_negative = false; + } + } + + if (line.starts_with("flags:"sv)) { + auto flags = parse_list(line); + + if (flags.is_empty()) { + failed_message = String::formatted("Failed to find flags in '{}'", line); + break; + } + + for (auto flag : flags) { + if (flag == "raw"sv) { + metadata.strict_mode = StrictMode::NoStrict; + metadata.harness_files.clear(); + } else if (flag == "noStrict"sv) { + metadata.strict_mode = StrictMode::NoStrict; + } else if (flag == "onlyStrict"sv) { + metadata.strict_mode = StrictMode::OnlyStrict; + } else if (flag == "module"sv) { + VERIFY(metadata.strict_mode == StrictMode::Both); + metadata.program_type = JS::Program::Type::Module; + metadata.strict_mode = StrictMode::NoStrict; + } else if (flag == "async"sv) { + metadata.harness_files.append(async_include); + metadata.is_async = true; + } + } + } else if (line.starts_with("includes:"sv)) { + auto files = parse_list(line); + if (files.is_empty()) { + parsing_includes_list = true; + } else { + for (auto& file : files) + metadata.harness_files.append(file); + } + } else if (line.starts_with("negative:"sv)) { + metadata.is_negative = true; + parsing_negative = true; + } + } + + if (failed_message.is_empty()) + failed_message = String::formatted("Never reached end of comment '---*/'"); + + return failed_message; +} + +static bool verify_test(Result& result, TestMetadata const& metadata, JsonObject& output) +{ + if (result.is_error()) { + if (result.error().phase == NegativePhase::Harness) { + output.set("harness_error", true); + output.set("harness_file", result.error().harness_file); + output.set("result", "harness_error"); + } else if (result.error().phase == NegativePhase::Runtime) { + auto& error_type = result.error().type; + auto& error_details = result.error().details; + if ((error_type == "InternalError"sv && error_details.starts_with("TODO("sv)) + || (error_type == "Test262Error"sv && error_details.ends_with(" but got a InternalError"sv))) { + output.set("todo_error", true); + output.set("result", "todo_error"); + } + } + } + + if (metadata.is_async && output.has("output"sv)) { + auto& output_messages = output.get("output"sv); + VERIFY(output_messages.is_string()); + if (output_messages.as_string().contains("AsyncTestFailure:InternalError: TODO("sv)) { + output.set("todo_error", true); + output.set("result", "todo_error"); + } + } + + auto phase_to_string = [](NegativePhase phase) { + switch (phase) { + case NegativePhase::ParseOrEarly: + return "parse"; + case NegativePhase::Resolution: + return "resolution"; + case NegativePhase::Runtime: + return "runtime"; + case NegativePhase::Harness: + return "harness"; + } + VERIFY_NOT_REACHED(); + }; + + auto error_to_json = [&phase_to_string](TestError const& error) { + JsonObject error_object; + error_object.set("phase", phase_to_string(error.phase)); + error_object.set("type", error.type); + error_object.set("details", error.details); + return error_object; + }; + + JsonValue expected_error; + JsonValue got_error; + + ScopeGuard set_error = [&] { + JsonObject error_object; + error_object.set("expected", expected_error); + error_object.set("got", got_error); + + output.set("error", error_object); + }; + + if (!metadata.is_negative) { + if (!result.is_error()) + return true; + + got_error = JsonValue { error_to_json(result.error()) }; + return false; + } + + JsonObject expected_error_object; + expected_error_object.set("phase", phase_to_string(metadata.phase)); + expected_error_object.set("type", metadata.type.to_string()); + + expected_error = expected_error_object; + + if (!result.is_error()) { + if (s_parse_only && metadata.phase != NegativePhase::ParseOrEarly) { + // Expected non-parse error but did not get it but we never got to that phase. + return true; + } + + return false; + } + + auto const& error = result.error(); + + got_error = JsonValue { error_to_json(error) }; + + if (metadata.program_type == JS::Program::Type::Module && metadata.type == "SyntaxError"sv) { + // NOTE: Since the "phase" of negative results is both not defined and hard to + // track throughout the entire Module life span we will just accept any + // SyntaxError as the correct one. + // See for example: + // - test/language/module-code/instn-star-err-not-found.js + // - test/language/module-code/instn-resolve-err-syntax-1.js + // - test/language/import/json-invalid.js + // The first fails in runtime because there is no 'x' to export + // However this is during the linking phase of the upper module. + // Whereas the second fails with a SyntaxError because the linked module + // has one. + // The third test is the same as the second, upper module is fine but + // import a module with SyntaxError, however here the phase is runtime. + // In conclusion all the test which would cause the inital module to not + // be evaluated !should! have '$DONOTEVALUATE();' at the top causing a + // Reference error, meaning we just ignore the phase in the SyntaxError case. + return error.type == metadata.type; + } + return error.phase == metadata.phase && error.type == metadata.type; +} + +static FILE* saved_stdout_fd; + +static void timer_handler(int signum) +{ + JsonObject timeout_result; + timeout_result.set("test", s_current_test); + timeout_result.set("timeout", true); + timeout_result.set("result", "timeout"); + outln(saved_stdout_fd, "RESULT {}{}", timeout_result.to_string(), '\0'); + // Make sure this message gets out and just die like the default action would be. + fflush(saved_stdout_fd); + + signal(signum, SIG_DFL); + raise(signum); +} + +void __assert_fail(char const* assertion, char const* file, unsigned int line, char const* function) +{ + JsonObject assert_fail_result; + assert_fail_result.set("test", s_current_test); + assert_fail_result.set("assert_fail", true); + assert_fail_result.set("result", "assert_fail"); + assert_fail_result.set("output", String::formatted("{}:{}: {}: Assertion `{}' failed.", file, line, function, assertion)); + outln(saved_stdout_fd, "RESULT {}{}", assert_fail_result.to_string(), '\0'); + // (Attempt to) Ensure that messages are written before quitting. + fflush(saved_stdout_fd); + fflush(stderr); + abort(); +} + +int main(int argc, char** argv) +{ + int timeout = 10; + bool enable_debug_printing = false; + + Core::ArgsParser args_parser; + args_parser.set_general_help("LibJS test262 runner for streaming tests"); + args_parser.add_option(s_harness_file_directory, "Directory containing the harness files", "harness-location", 'l', "harness-files"); + args_parser.add_option(s_use_bytecode, "Use the bytecode interpreter", "use-bytecode", 'b'); + args_parser.add_option(s_parse_only, "Only parse the files", "parse-only", 'p'); + args_parser.add_option(timeout, "Seconds before test should timeout", "timeout", 't', "seconds"); + args_parser.add_option(enable_debug_printing, "Enable debug printing", "debug", 'd'); + args_parser.parse(argc, argv); + + if (s_harness_file_directory.is_empty()) { + dbgln("You must specify the harness file directory via --harness-location"); + return 2; + } + + if (!s_harness_file_directory.ends_with('/')) { + s_harness_file_directory = String::formatted("{}/", s_harness_file_directory); + } + + if (timeout <= 0) { + dbgln("timeout must be atleast 1"); + return 2; + } + + AK::set_debug_enabled(enable_debug_printing); + + // The piping stuff is based on https://stackoverflow.com/a/956269. + constexpr auto BUFFER_SIZE = 1 * KiB; + char buffer[BUFFER_SIZE] = {}; + + auto saved_stdout = dup(STDOUT_FILENO); + if (saved_stdout < 0) { + perror("dup"); + return 1; + } + + saved_stdout_fd = fdopen(saved_stdout, "w"); + if (!saved_stdout_fd) { + perror("fdopen"); + return 1; + } + + int stdout_pipe[2]; + if (pipe(stdout_pipe) < 0) { + perror("pipe"); + return 1; + } + + auto flags = fcntl(stdout_pipe[0], F_GETFL); + flags |= O_NONBLOCK; + fcntl(stdout_pipe[0], F_SETFL, flags); + + auto flags2 = fcntl(stdout_pipe[1], F_GETFL); + flags2 |= O_NONBLOCK; + fcntl(stdout_pipe[1], F_SETFL, flags2); + + if (dup2(stdout_pipe[1], STDOUT_FILENO) < 0) { + perror("dup2"); + return 1; + } + + if (close(stdout_pipe[1]) < 0) { + perror("close"); + return 1; + } + + auto collect_output = [&] { + fflush(stdout); + auto nread = read(stdout_pipe[0], buffer, BUFFER_SIZE); + String value; + + if (nread > 0) { + value = String { buffer, static_cast(nread) }; + while (nread > 0) { + nread = read(stdout_pipe[0], buffer, BUFFER_SIZE); + } + } + + return value; + }; + + if (signal(SIGVTALRM, timer_handler) == SIG_ERR) { + perror("signal"); + return 1; + } + + timer_t timer_id; + struct sigevent timer_settings; + timer_settings.sigev_notify = SIGEV_SIGNAL; + timer_settings.sigev_signo = SIGVTALRM; + timer_settings.sigev_value.sival_ptr = &timer_id; + + if (timer_create(CLOCK_PROCESS_CPUTIME_ID, &timer_settings, &timer_id) < 0) { + perror("timer_create"); + return 1; + } + ScopeGuard destroy_timer = [timer_id] { timer_delete(timer_id); }; + + struct itimerspec timeout_timer; + timeout_timer.it_value.tv_sec = timeout; + timeout_timer.it_value.tv_nsec = 0; + timeout_timer.it_interval.tv_sec = 0; + timeout_timer.it_interval.tv_nsec = 0; + + struct itimerspec disarm; + disarm.it_value.tv_sec = 0; + disarm.it_value.tv_nsec = 0; + disarm.it_interval.tv_sec = 0; + disarm.it_interval.tv_nsec = 0; + +#define ARM_TIMER() \ + if (timer_settime(timer_id, 0, &timeout_timer, nullptr) < 0) { \ + perror("timer_settime"); \ + return 1; \ + } + +#define DISARM_TIMER() \ + if (timer_settime(timer_id, 0, &disarm, nullptr) < 0) { \ + perror("timer_settime"); \ + return 1; \ + } + + auto stdin = Core::File::standard_input(); + size_t count = 0; + + while (!stdin->eof()) { + auto path = stdin->read_line(); + if (path.is_empty()) { + continue; + } + + s_current_test = path; + + auto file = Core::File::construct(path); + if (!file->open(Core::OpenMode::ReadOnly)) { + dbgln("Could not open file: {}", path); + return 3; + } + + count++; + + String source_with_strict; + static String use_strict = "'use strict';\n"; + static size_t strict_length = use_strict.length(); + + { + auto contents = file->read_all(); + StringBuilder builder { contents.size() + strict_length }; + builder.append(use_strict); + builder.append(contents); + source_with_strict = builder.to_string(); + } + + StringView with_strict = source_with_strict.view(); + StringView original_contents = source_with_strict.substring_view(strict_length); + + JsonObject result_object; + result_object.set("test", path); + + ScopeGuard output_guard = [&] { + outln(saved_stdout_fd, "RESULT {}{}", result_object.to_string(), '\0'); + fflush(saved_stdout_fd); + }; + + auto metadata_or_error = extract_metadata(original_contents); + if (metadata_or_error.is_error()) { + result_object.set("result", "metadata_error"); + result_object.set("metadata_error", true); + result_object.set("metadata_output", metadata_or_error.error()); + continue; + } + + auto& metadata = metadata_or_error.value(); + + bool passed = true; + + if (metadata.strict_mode != StrictMode::OnlyStrict) { + result_object.set("strict_mode", false); + + ARM_TIMER(); + auto result = run_test(original_contents, path, metadata.program_type, metadata.harness_files); + DISARM_TIMER(); + + String first_output = collect_output(); + if (!first_output.is_null()) + result_object.set("output", first_output); + + passed = verify_test(result, metadata, result_object); + if (metadata.is_async && !s_parse_only) { + if (!first_output.contains("Test262:AsyncTestComplete"sv) || first_output.contains("Test262:AsyncTestFailure"sv)) { + result_object.set("async_fail", true); + if (first_output.is_null()) + result_object.set("output", JsonValue { AK::JsonValue::Type::Null }); + + passed = false; + } + } + } + + if (passed && metadata.strict_mode != StrictMode::NoStrict) { + result_object.set("strict_mode", true); + + ARM_TIMER(); + auto result = run_test(with_strict, path, metadata.program_type, metadata.harness_files); + DISARM_TIMER(); + + String first_output = collect_output(); + if (!first_output.is_null()) + result_object.set("strict_output", first_output); + + passed = verify_test(result, metadata, result_object); + if (metadata.is_async && !s_parse_only) { + if (!first_output.contains("Test262:AsyncTestComplete"sv) || first_output.contains("Test262:AsyncTestFailure"sv)) { + result_object.set("async_fail", true); + if (first_output.is_null()) + result_object.set("output", JsonValue { AK::JsonValue::Type::Null }); + + passed = false; + } + } + } + + if (passed) + result_object.remove("strict_mode"sv); + + if (!result_object.has("result"sv)) + result_object.set("result"sv, passed ? "passed"sv : "failed"sv); + } + + s_current_test = ""; + outln(saved_stdout_fd, "DONE {}", count); + + // After this point we have already written our output so pretend everything is fine if we get an error. + if (dup2(saved_stdout, STDOUT_FILENO) < 0) { + perror("dup2"); + return 0; + } + + if (fclose(saved_stdout_fd) < 0) { + perror("fclose"); + return 0; + } + + if (close(stdout_pipe[0]) < 0) { + perror("close"); + return 0; + } + + return 0; +}