LibJS+LibWeb: Bring script fetching closer to the spec

This patch updates various parts of the script fetching implementation
to match the current specification.

Notably, the implementation of changes to the import assertions /
attributes proposal are not part of this patch(series).

Userland/Libraries/LibJS/CyclicModule.cpp

@@ -33,6 +33,127 @@ void CyclicModule::visit_edges(Cell::Visitor& visitor)
         visitor.visit(loaded_module.module);
 }
 
+// 16.2.1.5.1 LoadRequestedModules ( [ hostDefined ] ), https://tc39.es/ecma262/#sec-LoadRequestedModules
+PromiseCapability& CyclicModule::load_requested_modules(JS::Realm& realm, Optional<GraphLoadingState::HostDefined> host_defined)
+{
+    // 1. If hostDefined is not present, let hostDefined be EMPTY.
+    // NOTE: The empty state is handled by hostDefined being an optional without value.
+
+    // 2. Let pc be ! NewPromiseCapability(%Promise%).
+    auto promise_capability = MUST(new_promise_capability(realm.vm(), realm.intrinsics().promise_constructor()));
+
+    // 3. Let state be the GraphLoadingState Record { [[IsLoading]]: true, [[PendingModulesCount]]: 1, [[Visited]]: « », [[PromiseCapability]]: pc, [[HostDefined]]: hostDefined }.
+    auto state = GraphLoadingState { .promise_capability = promise_capability, .is_loading = true, .pending_module_count = 1, .visited = {}, .host_defined = move(host_defined) };
+
+    // 4. Perform InnerModuleLoading(state, module).
+    inner_module_loading(state);
+
+    // NOTE: This is likely a spec bug, see https://matrixlogs.bakkot.com/WHATWG/2023-02-13#L1
+    // FIXME: 5. Return pc.[[Promise]].
+    return promise_capability;
+}
+
+// 16.2.1.5.1.1 InnerModuleLoading ( state, module ), https://tc39.es/ecma262/#sec-InnerModuleLoading
+void CyclicModule::inner_module_loading(JS::GraphLoadingState& state)
+{
+    // 1. Assert: state.[[IsLoading]] is true.
+    VERIFY(state.is_loading);
+
+    // 2. If module is a Cyclic Module Record, module.[[Status]] is NEW, and state.[[Visited]] does not contain module, then
+    if (m_status == ModuleStatus::New && !state.visited.contains(this)) {
+        // a. Append module to state.[[Visited]].
+        state.visited.set(this);
+
+        // b. Let requestedModulesCount be the number of elements in module.[[RequestedModules]].
+        auto requested_modules_count = m_requested_modules.size();
+
+        // c. Set state.[[PendingModulesCount]] to state.[[PendingModulesCount]] + requestedModulesCount.
+        state.pending_module_count += requested_modules_count;
+
+        // d. For each String required of module.[[RequestedModules]], do
+        for (auto const& required : m_requested_modules) {
+            bool found_record_in_loaded_modules = false;
+
+            // i. If module.[[LoadedModules]] contains a Record whose [[Specifier]] is required, then
+            for (auto const& record : m_loaded_modules) {
+                if (record.specifier == required.module_specifier) {
+                    // 1. Let record be that Record.
+
+                    // 2. Perform InnerModuleLoading(state, record.[[Module]]).
+                    static_cast<CyclicModule&>(*record.module).inner_module_loading(state);
+
+                    found_record_in_loaded_modules = true;
+                    break;
+                }
+            }
+
+            // ii. Else,
+            if (!found_record_in_loaded_modules) {
+                // 1. Perform HostLoadImportedModule(module, required, state.[[HostDefined]], state).
+                vm().host_load_imported_module(realm(), NonnullGCPtr<CyclicModule>(*this), required, state.host_defined, state);
+
+                // 2. NOTE: HostLoadImportedModule will call FinishLoadingImportedModule, which re-enters the graph loading process through ContinueModuleLoading.
+            }
+
+            // iii. If state.[[IsLoading]] is false, return UNUSED.
+            if (!state.is_loading)
+                return;
+        }
+    }
+
+    // 3. Assert: state.[[PendingModulesCount]] ≥ 1.
+    VERIFY(state.pending_module_count >= 1);
+
+    // 4. Set state.[[PendingModulesCount]] to state.[[PendingModulesCount]] - 1.
+    --state.pending_module_count;
+
+    // 5. If state.[[PendingModulesCount]] = 0, then
+    if (state.pending_module_count == 0) {
+        // a. Set state.[[IsLoading]] to false.
+        state.is_loading = false;
+
+        // b. For each Cyclic Module Record loaded of state.[[Visited]], do
+        for (auto const& loaded : state.visited) {
+            // i. If loaded.[[Status]] is NEW, set loaded.[[Status]] to UNLINKED.
+            if (loaded->m_status == ModuleStatus::New)
+                loaded->m_status = ModuleStatus::Linked;
+        }
+
+        // c. Perform ! Call(state.[[PromiseCapability]].[[Resolve]], undefined, « undefined »).
+        MUST(call(vm(), *state.promise_capability->resolve(), js_undefined(), js_undefined()));
+    }
+
+    // 6. Return unused.
+}
+
+// 16.2.1.5.1.2 ContinueModuleLoading ( state, moduleCompletion ), https://tc39.es/ecma262/#sec-ContinueModuleLoading
+void continue_module_loading(Realm& realm, GraphLoadingState& state, ThrowCompletionOr<Module*> const& module_completion)
+{
+    // 1. If state.[[IsLoading]] is false, return UNUSED.
+    if (state.is_loading)
+        return;
+
+    // 2. If moduleCompletion is a normal completion, then
+    if (!module_completion.is_error()) {
+        auto* module = const_cast<Module*>(module_completion.value());
+
+        // a. Perform InnerModuleLoading(state, moduleCompletion.[[Value]]).
+        static_cast<CyclicModule*>(module)->inner_module_loading(state);
+    }
+    // 3. Else,
+    else {
+        // a. Set state.[[IsLoading]] to false.
+        state.is_loading = false;
+
+        auto value = module_completion.throw_completion().value();
+
+        // b. Perform ! Call(state.[[PromiseCapability]].[[Reject]], undefined, « moduleCompletion.[[Value]] »).
+        MUST(call(realm.vm(), *state.promise_capability->reject(), js_undefined(), *value));
+    }
+
+    // 4. Return UNUSED.
+}
+
 // 16.2.1.5.1 Link ( ), https://tc39.es/ecma262/#sec-moduledeclarationlinking
 ThrowCompletionOr<void> CyclicModule::link(VM& vm)
 {

Userland/Libraries/LibJS/CyclicModule.h

@@ -49,6 +49,9 @@ public:
     virtual ThrowCompletionOr<void> link(VM& vm) override final;
     virtual ThrowCompletionOr<Promise*> evaluate(VM& vm) override final;
 
+    virtual PromiseCapability& load_requested_modules(Realm&, Optional<GraphLoadingState::HostDefined>);
+    virtual void inner_module_loading(GraphLoadingState& state);
+
     Vector<ModuleRequest> const& requested_modules() const { return m_requested_modules; }
     Vector<ModuleWithSpecifier> const& loaded_modules() const { return m_loaded_modules; }
 
@@ -82,4 +85,6 @@ protected:
     Optional<u32> m_pending_async_dependencies;         // [[PendingAsyncDependencies]]
 };
 
+void continue_module_loading(Realm&, GraphLoadingState& state, ThrowCompletionOr<Module*> const&);
+
 }

Userland/Libraries/LibJS/Module.cpp

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2021, Andreas Kling <kling@serenityos.org>
  * Copyright (c) 2022, David Tuin <davidot@serenityos.org>
+ * Copyright (c) 2023, networkException <networkexception@serenityos.org>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -8,6 +9,7 @@
 #include <LibJS/CyclicModule.h>
 #include <LibJS/Module.h>
 #include <LibJS/Runtime/ModuleNamespaceObject.h>
+#include <LibJS/Runtime/ModuleRequest.h>
 #include <LibJS/Runtime/Promise.h>
 #include <LibJS/Runtime/VM.h>
 
@@ -62,6 +64,51 @@ ThrowCompletionOr<u32> Module::inner_module_evaluation(VM& vm, Vector<Module*>&,
     return index;
 }
 
+// 16.2.1.9 FinishLoadingImportedModule ( referrer, specifier, payload, result ), https://tc39.es/ecma262/#sec-FinishLoadingImportedModule
+// FIXME: We currently implement an outdated version of https://tc39.es/proposal-import-attributes, as such it is not possible to
+//        use the exact steps from https://tc39.es/proposal-import-attributes/#sec-HostLoadImportedModule here.
+// FIXME: Support Realm for referrer.
+void finish_loading_imported_module(Realm& realm, Variant<JS::NonnullGCPtr<JS::Script>, JS::NonnullGCPtr<JS::CyclicModule>> referrer, ModuleRequest const& module_request, GraphLoadingState& payload, ThrowCompletionOr<Module*> const& result)
+{
+    // 1. If result is a normal completion, then
+    if (!result.is_error()) {
+        auto loaded_modules = referrer.visit(
+            [](JS::NonnullGCPtr<JS::Script> script) -> Vector<ModuleWithSpecifier> { return script->loaded_modules(); },
+            [](JS::NonnullGCPtr<JS::CyclicModule> module) -> Vector<ModuleWithSpecifier> { return module->loaded_modules(); });
+
+        bool found_record = false;
+
+        // a.a. If referrer.[[LoadedModules]] contains a Record whose [[Specifier]] is specifier, then
+        for (auto const& record : loaded_modules) {
+            if (record.specifier == module_request.module_specifier) {
+                // i. Assert: That Record's [[Module]] is result.[[Value]].
+                VERIFY(record.module == result.value());
+
+                found_record = true;
+            }
+        }
+
+        // b. Else,
+        if (!found_record) {
+            auto* module = const_cast<Module*>(result.value());
+
+            // i. Append the Record { [[Specifier]]: specifier, [[Module]]: result.[[Value]] } to referrer.[[LoadedModules]].
+            loaded_modules.append(ModuleWithSpecifier {
+                .specifier = module_request.module_specifier,
+                .module = NonnullGCPtr<Module>(*module) });
+        }
+    }
+
+    // FIXME: 2. If payload is a GraphLoadingState Record, then
+    // a. Perform ContinueModuleLoading(payload, result)
+    continue_module_loading(realm, payload, result);
+
+    // FIXME: Else,
+    // FIXME: a. Perform ContinueDynamicImport(payload, result).
+
+    // 4. Return unused.
+}
+
 // 16.2.1.10 GetModuleNamespace ( module ), https://tc39.es/ecma262/#sec-getmodulenamespace
 ThrowCompletionOr<Object*> Module::get_module_namespace(VM& vm)
 {

Userland/Libraries/LibJS/Module.h

@@ -109,4 +109,9 @@ private:
     DeprecatedString m_filename;
 };
 
+class CyclicModule;
+struct GraphLoadingState;
+
+void finish_loading_imported_module(Realm&, Variant<NonnullGCPtr<Script>, NonnullGCPtr<CyclicModule>>, ModuleRequest const&, GraphLoadingState&, ThrowCompletionOr<Module*> const&);
+
 }

Userland/Libraries/LibJS/Runtime/VM.h

@@ -2,6 +2,7 @@
  * Copyright (c) 2020-2023, Andreas Kling <kling@serenityos.org>
  * Copyright (c) 2020-2023, Linus Groh <linusg@serenityos.org>
  * Copyright (c) 2021-2022, David Tuin <davidot@serenityos.org>
+ * Copyright (c) 2023, networkException <networkexception@serenityos.org>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -14,6 +15,7 @@
 #include <AK/RefCounted.h>
 #include <AK/StackInfo.h>
 #include <AK/Variant.h>
+#include <LibJS/CyclicModule.h>
 #include <LibJS/Heap/Heap.h>
 #include <LibJS/Heap/MarkedVector.h>
 #include <LibJS/Runtime/CommonPropertyNames.h>
@@ -222,6 +224,13 @@ public:
 
     ScriptOrModule get_active_script_or_module() const;
 
+    // NOTE: The host defined implementation described in the web spec https://html.spec.whatwg.org/multipage/webappapis.html#hostloadimportedmodule
+    //       currently references proposal-import-attributes.
+    //       Our implementation of this proposal is outdated however, as such we try to adapt the proposal and living standard
+    //       to match our implementation for now.
+    // 16.2.1.8 HostLoadImportedModule ( referrer, moduleRequest, hostDefined, payload ), https://tc39.es/proposal-import-attributes/#sec-HostLoadImportedModule
+    Function<void(Realm&, Variant<NonnullGCPtr<Script>, NonnullGCPtr<CyclicModule>>, ModuleRequest const&, Optional<GraphLoadingState::HostDefined>, GraphLoadingState&)> host_load_imported_module;
+
     Function<ThrowCompletionOr<NonnullGCPtr<Module>>(ScriptOrModule, ModuleRequest const&)> host_resolve_imported_module;
     Function<ThrowCompletionOr<void>(ScriptOrModule, ModuleRequest, PromiseCapability const&)> host_import_module_dynamically;
     Function<void(ScriptOrModule, ModuleRequest const&, PromiseCapability const&, Promise*)> host_finish_dynamic_import;