From 3194177dce7bcac1c53bc987447ea3685baa87c8 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Pur Date: Mon, 14 Jun 2021 15:46:41 +0430 Subject: [PATCH] LibJS: Correctly parse yield-from expressions This commit implements parsing for `yield *expr`, and the multiple ways something can or can't be parsed like that. Also makes yield-from a TODO in the bytecode generator. Behold, the glory of javascript syntax: ```js // 'yield' = expression in generators. function* foo() { yield *bar; // <- Syntax error here, expression can't start with * } // 'yield' = identifier anywhere else. function foo() { yield *bar; // Perfectly fine, this is just `yield * bar` } ``` --- Userland/Libraries/LibJS/AST.h | 5 +- .../Libraries/LibJS/Bytecode/ASTCodegen.cpp | 3 + Userland/Libraries/LibJS/Parser.cpp | 92 +++++++++++-------- Userland/Libraries/LibJS/Parser.h | 7 +- .../LibJS/Tests/syntax/generators.js | 2 +- 5 files changed, 67 insertions(+), 42 deletions(-) diff --git a/Userland/Libraries/LibJS/AST.h b/Userland/Libraries/LibJS/AST.h index d36b639251..27c891620a 100644 --- a/Userland/Libraries/LibJS/AST.h +++ b/Userland/Libraries/LibJS/AST.h @@ -348,13 +348,15 @@ public: class YieldExpression final : public Expression { public: - explicit YieldExpression(SourceRange source_range, RefPtr argument) + explicit YieldExpression(SourceRange source_range, RefPtr argument, bool is_yield_from) : Expression(move(source_range)) , m_argument(move(argument)) + , m_is_yield_from(is_yield_from) { } Expression const* argument() const { return m_argument; } + bool is_yield_from() const { return m_is_yield_from; } virtual Value execute(Interpreter&, GlobalObject&) const override; virtual void dump(int indent) const override; @@ -362,6 +364,7 @@ public: private: RefPtr m_argument; + bool m_is_yield_from { false }; }; class ReturnStatement final : public Statement { diff --git a/Userland/Libraries/LibJS/Bytecode/ASTCodegen.cpp b/Userland/Libraries/LibJS/Bytecode/ASTCodegen.cpp index 4dbbd94e76..fa2050c3c4 100644 --- a/Userland/Libraries/LibJS/Bytecode/ASTCodegen.cpp +++ b/Userland/Libraries/LibJS/Bytecode/ASTCodegen.cpp @@ -721,6 +721,9 @@ void YieldExpression::generate_bytecode(Bytecode::Generator& generator) const { VERIFY(generator.is_in_generator_function()); + if (m_is_yield_from) + TODO(); + if (m_argument) m_argument->generate_bytecode(generator); diff --git a/Userland/Libraries/LibJS/Parser.cpp b/Userland/Libraries/LibJS/Parser.cpp index 114e68b1ef..90461ff8d6 100644 --- a/Userland/Libraries/LibJS/Parser.cpp +++ b/Userland/Libraries/LibJS/Parser.cpp @@ -496,7 +496,9 @@ NonnullRefPtr Parser::parse_class_expression(bool expect_class_ if (match(TokenType::Extends)) { consume(); - super_class = parse_primary_expression(); + auto [expression, should_continue_parsing] = parse_primary_expression(); + super_class = move(expression); + (void)should_continue_parsing; } consume(TokenType::CurlyOpen); @@ -612,11 +614,11 @@ NonnullRefPtr Parser::parse_class_expression(bool expect_class_ return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, move(class_name), move(constructor), move(super_class), move(methods)); } -NonnullRefPtr Parser::parse_primary_expression() +Parser::PrimaryExpressionParseResult Parser::parse_primary_expression() { auto rule_start = push_start(); if (match_unary_prefixed_expression()) - return parse_unary_prefixed_expression(); + return { parse_unary_prefixed_expression() }; switch (m_parser_state.m_current_token.type()) { case TokenType::ParenOpen: { @@ -625,7 +627,7 @@ NonnullRefPtr Parser::parse_primary_expression() if ((match(TokenType::ParenClose) || match(TokenType::Identifier) || match(TokenType::TripleDot)) && !try_parse_arrow_function_expression_failed_at_position(paren_position)) { auto arrow_function_result = try_parse_arrow_function_expression(true); if (!arrow_function_result.is_null()) - return arrow_function_result.release_nonnull(); + return { arrow_function_result.release_nonnull() }; set_try_parse_arrow_function_expression_failed_at_position(paren_position, true); } @@ -634,24 +636,24 @@ NonnullRefPtr Parser::parse_primary_expression() if (is(*expression)) { static_cast(*expression).set_cannot_auto_rename(); } - return expression; + return { move(expression) }; } case TokenType::This: consume(); - return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }); + return { create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }) }; case TokenType::Class: - return parse_class_expression(false); + return { parse_class_expression(false) }; case TokenType::Super: consume(); if (!m_parser_state.m_allow_super_property_lookup) syntax_error("'super' keyword unexpected here"); - return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }); + return { create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }) }; case TokenType::Identifier: { read_as_identifier:; if (!try_parse_arrow_function_expression_failed_at_position(position())) { auto arrow_function_result = try_parse_arrow_function_expression(false); if (!arrow_function_result.is_null()) - return arrow_function_result.release_nonnull(); + return { arrow_function_result.release_nonnull() }; set_try_parse_arrow_function_expression_failed_at_position(position(), true); } @@ -671,47 +673,47 @@ NonnullRefPtr Parser::parse_primary_expression() ++i; } } - return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, string, argument_index); + return { create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, string, argument_index) }; } case TokenType::NumericLiteral: - return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, consume_and_validate_numeric_literal().double_value()); + return { create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, consume_and_validate_numeric_literal().double_value()) }; case TokenType::BigIntLiteral: - return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, consume().value()); + return { create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, consume().value()) }; case TokenType::BoolLiteral: - return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, consume().bool_value()); + return { create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, consume().bool_value()) }; case TokenType::StringLiteral: - return parse_string_literal(consume()); + return { parse_string_literal(consume()) }; case TokenType::NullLiteral: consume(); - return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }); + return { create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }) }; case TokenType::CurlyOpen: - return parse_object_expression(); + return { parse_object_expression() }; case TokenType::Function: - return parse_function_node(); + return { parse_function_node() }; case TokenType::BracketOpen: - return parse_array_expression(); + return { parse_array_expression() }; case TokenType::RegexLiteral: - return parse_regexp_literal(); + return { parse_regexp_literal() }; case TokenType::TemplateLiteralStart: - return parse_template_literal(false); + return { parse_template_literal(false) }; case TokenType::New: { auto new_start = position(); auto new_target_result = try_parse_new_target_expression(); if (!new_target_result.is_null()) { if (!m_parser_state.m_in_function_context) syntax_error("'new.target' not allowed outside of a function", new_start); - return new_target_result.release_nonnull(); + return { new_target_result.release_nonnull() }; } - return parse_new_expression(); + return { parse_new_expression() }; } case TokenType::Yield: if (!m_parser_state.m_in_generator_function_context) goto read_as_identifier; - return parse_yield_expression(); + return { parse_yield_expression(), false }; default: expected("primary expression"); consume(); - return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }); + return { create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }) }; } } @@ -1024,23 +1026,25 @@ NonnullRefPtr Parser::parse_template_literal(bool is_tagged) NonnullRefPtr Parser::parse_expression(int min_precedence, Associativity associativity, const Vector& forbidden) { auto rule_start = push_start(); - auto expression = parse_primary_expression(); + auto [expression, should_continue_parsing] = parse_primary_expression(); while (match(TokenType::TemplateLiteralStart)) { auto template_literal = parse_template_literal(true); expression = create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, move(expression), move(template_literal)); } - while (match_secondary_expression(forbidden)) { - int new_precedence = g_operator_precedence.get(m_parser_state.m_current_token.type()); - if (new_precedence < min_precedence) - break; - if (new_precedence == min_precedence && associativity == Associativity::Left) - break; + if (should_continue_parsing) { + while (match_secondary_expression(forbidden)) { + int new_precedence = g_operator_precedence.get(m_parser_state.m_current_token.type()); + if (new_precedence < min_precedence) + break; + if (new_precedence == min_precedence && associativity == Associativity::Left) + break; - Associativity new_associativity = operator_associativity(m_parser_state.m_current_token.type()); - expression = parse_secondary_expression(move(expression), new_precedence, new_associativity); - while (match(TokenType::TemplateLiteralStart)) { - auto template_literal = parse_template_literal(true); - expression = create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, move(expression), move(template_literal)); + Associativity new_associativity = operator_associativity(m_parser_state.m_current_token.type()); + expression = parse_secondary_expression(move(expression), new_precedence, new_associativity); + while (match(TokenType::TemplateLiteralStart)) { + auto template_literal = parse_template_literal(true); + expression = create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, move(expression), move(template_literal)); + } } } if (match(TokenType::Comma) && min_precedence <= 1) { @@ -1303,9 +1307,19 @@ NonnullRefPtr Parser::parse_yield_expression() auto rule_start = push_start(); consume(TokenType::Yield); RefPtr argument; - if (match_expression()) - argument = parse_expression(0); - return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, move(argument)); + bool yield_from = false; + + if (!m_parser_state.m_current_token.trivia_contains_line_terminator()) { + if (match(TokenType::Asterisk)) { + consume(); + yield_from = true; + } + + if (yield_from || match_expression()) + argument = parse_expression(0); + } + + return create_ast_node({ m_parser_state.m_current_token.filename(), rule_start.position(), position() }, move(argument), yield_from); } NonnullRefPtr Parser::parse_return_statement() diff --git a/Userland/Libraries/LibJS/Parser.h b/Userland/Libraries/LibJS/Parser.h index b1b364fcdc..6b63d8c985 100644 --- a/Userland/Libraries/LibJS/Parser.h +++ b/Userland/Libraries/LibJS/Parser.h @@ -44,6 +44,11 @@ public: Vector parse_formal_parameters(int& function_length, u8 parse_options = 0); RefPtr parse_binding_pattern(); + struct PrimaryExpressionParseResult { + NonnullRefPtr result; + bool should_continue_parsing_as_expression { true }; + }; + NonnullRefPtr parse_declaration(); NonnullRefPtr parse_statement(); NonnullRefPtr parse_block_statement(); @@ -66,7 +71,7 @@ public: NonnullRefPtr parse_debugger_statement(); NonnullRefPtr parse_conditional_expression(NonnullRefPtr test); NonnullRefPtr parse_expression(int min_precedence, Associativity associate = Associativity::Right, const Vector& forbidden = {}); - NonnullRefPtr parse_primary_expression(); + PrimaryExpressionParseResult parse_primary_expression(); NonnullRefPtr parse_unary_prefixed_expression(); NonnullRefPtr parse_regexp_literal(); NonnullRefPtr parse_object_expression(); diff --git a/Userland/Libraries/LibJS/Tests/syntax/generators.js b/Userland/Libraries/LibJS/Tests/syntax/generators.js index dbe0dcec4b..76c91b6dcb 100644 --- a/Userland/Libraries/LibJS/Tests/syntax/generators.js +++ b/Userland/Libraries/LibJS/Tests/syntax/generators.js @@ -12,7 +12,7 @@ describe("parsing freestanding generators", () => { expect(`function foo() { yield; }`).toEval(); expect(`function foo() { yield 3; }`).not.toEval(); }); - test.skip("yield-from expression", () => { + test("yield-from expression", () => { expect(`function* foo() { yield *bar; }`).toEval(); expect(`function* foo() { yield *(yield); }`).toEval(); expect(`function* foo() { yield