LibJS: Add a mode to parse JS as a module

In a module strict mode should be enabled at the start of parsing and we
allow import and export statements.

Tests/LibJS/test-js.cpp

@@ -85,7 +85,9 @@ TESTJS_RUN_FILE_FUNCTION(const String& test_file, JS::Interpreter&)
     else
         return Test::JS::RunFileHookResult::SkipFile;
 
-    auto parse_result = Test::JS::parse_file(test_file);
+    auto program_type = path.basename().ends_with(".module.js") ? JS::Program::Type::Module : JS::Program::Type::Script;
+
+    auto parse_result = Test::JS::parse_file(test_file, program_type);
     bool test_passed = true;
     String message;
     String expectation_string;

Userland/Libraries/LibJS/AST.h

@@ -168,8 +168,14 @@ private:
 
 class Program final : public ScopeNode {
 public:
-    explicit Program(SourceRange source_range)
+    enum class Type {
+        Script,
+        Module
+    };
+
+    explicit Program(SourceRange source_range, Type program_type)
         : ScopeNode(source_range)
+        , m_type(program_type)
     {
     }
 
@@ -178,10 +184,13 @@ public:
     bool is_strict_mode() const { return m_is_strict_mode; }
     void set_strict_mode() { m_is_strict_mode = true; }
 
+    Type type() const { return m_type; }
+
 private:
     virtual bool is_program() const override { return true; }
 
     bool m_is_strict_mode { false };
+    Type m_type { Type::Script };
 };
 
 class BlockStatement final : public ScopeNode {

Userland/Libraries/LibJS/Lexer.cpp

@@ -330,11 +330,11 @@ bool Lexer::is_identifier_middle() const
 bool Lexer::is_line_comment_start(bool line_has_token_yet) const
 {
     return match('/', '/')
-        || match('<', '!', '-', '-')
+        || (m_allow_html_comments && match('<', '!', '-', '-'))
         // "-->" is considered a line comment start if the current line is only whitespace and/or
         // other block comment(s); or in other words: the current line does not have a token or
         // ongoing line comment yet
-        || (match('-', '-', '>') && !line_has_token_yet)
+        || (m_allow_html_comments && !line_has_token_yet && match('-', '-', '>'))
         // https://tc39.es/proposal-hashbang/out.html#sec-updated-syntax
         || (match('#', '!') && m_position == 1);
 }

Userland/Libraries/LibJS/Lexer.h

@@ -23,6 +23,8 @@ public:
     const StringView& source() const { return m_source; };
     const StringView& filename() const { return m_filename; };
 
+    void disallow_html_comments() { m_allow_html_comments = false; };
+
 private:
     void consume();
     bool consume_exponent();
@@ -63,6 +65,8 @@ private:
     };
     Vector<TemplateState> m_template_states;
 
+    bool m_allow_html_comments { true };
+
     static HashMap<String, TokenType> s_keywords;
     static HashMap<String, TokenType> s_three_char_tokens;
     static HashMap<String, TokenType> s_two_char_tokens;

Userland/Libraries/LibJS/Parser.cpp

@@ -208,10 +208,13 @@ private:
 
 constexpr OperatorPrecedenceTable g_operator_precedence;
 
-Parser::ParserState::ParserState(Lexer l)
+Parser::ParserState::ParserState(Lexer l, Program::Type program_type)
     : lexer(move(l))
-    , current_token(lexer.next())
+    , current_token(TokenType::Invalid, {}, {}, {}, {}, 0, 0, 0)
 {
+    if (program_type == Program::Type::Module)
+        lexer.disallow_html_comments();
+    current_token = lexer.next();
 }
 
 Parser::Scope::Scope(Parser::Scope::Type type, RefPtr<Parser::Scope> parent_scope)
@@ -232,8 +235,9 @@ RefPtr<Parser::Scope> Parser::Scope::get_current_function_scope()
     return result;
 }
 
-Parser::Parser(Lexer lexer)
+Parser::Parser(Lexer lexer, Program::Type program_type)
-    : m_state(move(lexer))
+    : m_state(move(lexer), program_type)
+    , m_program_type(program_type)
 {
 }
 
@@ -282,8 +286,8 @@ NonnullRefPtr<Program> Parser::parse_program(bool starts_in_strict_mode)
 {
     auto rule_start = push_start();
     ScopePusher scope(*this, ScopePusher::Var | ScopePusher::Let, Scope::Function);
-    auto program = adopt_ref(*new Program({ m_filename, rule_start.position(), position() }));
+    auto program = adopt_ref(*new Program({ m_filename, rule_start.position(), position() }, m_program_type));
-    if (starts_in_strict_mode) {
+    if (starts_in_strict_mode || m_program_type == Program::Type::Module) {
         program->set_strict_mode();
         m_state.strict_mode = true;
     }

Userland/Libraries/LibJS/Parser.h

@@ -35,7 +35,7 @@ struct FunctionNodeParseOptions {
 
 class Parser {
 public:
-    explicit Parser(Lexer lexer);
+    explicit Parser(Lexer lexer, Program::Type program_type = Program::Type::Script);
 
     NonnullRefPtr<Program> parse_program(bool starts_in_strict_mode = false);
 
@@ -246,7 +246,7 @@ private:
         bool in_continue_context { false };
         bool string_legacy_octal_escape_sequence_in_scope { false };
 
-        explicit ParserState(Lexer);
+        ParserState(Lexer, Program::Type);
     };
 
     class PositionKeyTraits {
@@ -267,5 +267,6 @@ private:
     FlyString m_filename;
     Vector<ParserState> m_saved_state;
     HashMap<Position, TokenMemoization, PositionKeyTraits> m_token_memoizations;
+    Program::Type m_program_type;
 };
 }

Userland/Libraries/LibTest/JavaScriptTestRunner.h

@@ -194,7 +194,7 @@ inline void TestRunnerGlobalObject::initialize_global_object()
     }
 }
 
-inline AK::Result<NonnullRefPtr<JS::Program>, ParserError> parse_file(const String& file_path)
+inline AK::Result<NonnullRefPtr<JS::Program>, ParserError> parse_file(const String& file_path, JS::Program::Type program_type = JS::Program::Type::Script)
 {
     auto file = Core::File::construct(file_path);
     auto result = file->open(Core::OpenMode::ReadOnly);
@@ -207,7 +207,7 @@ inline AK::Result<NonnullRefPtr<JS::Program>, ParserError> parse_file(const Stri
     String test_file_string(reinterpret_cast<const char*>(contents.data()), contents.size());
     file->close();
 
-    auto parser = JS::Parser(JS::Lexer(test_file_string));
+    auto parser = JS::Parser(JS::Lexer(test_file_string), program_type);
     auto program = parser.parse_program();
 
     if (parser.has_errors()) {

Userland/Utilities/js.cpp

@@ -91,6 +91,7 @@ static bool s_dump_ast = false;
 static bool s_dump_bytecode = false;
 static bool s_run_bytecode = false;
 static bool s_opt_bytecode = false;
+static bool s_as_module = false;
 static bool s_print_last_result = false;
 static RefPtr<Line::Editor> s_editor;
 static String s_history_path = String::formatted("{}/.js-history", Core::StandardPaths::home_directory());
@@ -633,7 +634,8 @@ static bool write_to_file(String const& path)
 
 static bool parse_and_run(JS::Interpreter& interpreter, StringView const& source)
 {
-    auto parser = JS::Parser(JS::Lexer(source));
+    auto program_type = s_as_module ? JS::Program::Type::Module : JS::Program::Type::Script;
+    auto parser = JS::Parser(JS::Lexer(source), program_type);
     auto program = parser.parse_program();
 
     if (s_dump_ast)
@@ -922,6 +924,7 @@ int main(int argc, char** argv)
     args_parser.add_option(s_dump_bytecode, "Dump the bytecode", "dump-bytecode", 'd');
     args_parser.add_option(s_run_bytecode, "Run the bytecode", "run-bytecode", 'b');
     args_parser.add_option(s_opt_bytecode, "Optimize the bytecode", "optimize-bytecode", 'p');
+    args_parser.add_option(s_as_module, "Treat as module", "as-module", 'm');
     args_parser.add_option(s_print_last_result, "Print last result", "print-last-result", 'l');
     args_parser.add_option(gc_on_every_allocation, "GC on every allocation", "gc-on-every-allocation", 'g');
     args_parser.add_option(disable_syntax_highlight, "Disable live syntax highlighting", "no-syntax-highlight", 's');