RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-05-28 11:55:12 +00:00
Code Issues Activity Actions

JSSpecCompiler: Issue meaningful errors in TextParser

Browse source
This commit is contained in:
Dan Klishch 2024-01-17 01:36:39 -05:00 • committed by Andrew Kaster
parent dee4978d67
commit 6b30c4d2f0
5 changed files with 272 additions and 147 deletions
Download patch file Download diff file Expand all files Collapse all files

60
Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.cpp
View file

@ -86,24 +86,29 @@ Optional<AlgorithmStep> AlgorithmStep::create(SpecificationParsingContext& ctx,
result.m_substeps = step_list.has_value() ? step_list->tree() : error_tree; result.m_substeps = step_list.has_value() ? step_list->tree() : error_tree;
} }
auto parse_result = result.parse(); if (!result.parse())
if (parse_result.is_error()) {
ctx.diag().error(ctx.location_from_xml_offset(parse_result.error()->offset()),
"{}", parse_result.error()->to_string());
return {}; return {};
}
result.m_expression = parse_result.release_value();
return result; return result;
} }
ParseErrorOr<Tree> AlgorithmStep::parse() bool AlgorithmStep::parse()
{ {
TextParser parser(m_tokens, m_node); TextParser parser(m_ctx, m_tokens, m_node);
TextParseErrorOr<Tree> parse_result = TextParseError {};
if (m_substeps) if (m_substeps)
return parser.parse_step_with_substeps(RefPtr(m_substeps).release_nonnull()); parse_result = parser.parse_step_with_substeps(RefPtr(m_substeps).release_nonnull());
else else
return parser.parse_step_without_substeps(); parse_result = parser.parse_step_without_substeps();
if (parse_result.is_error()) {
auto [location, message] = parser.get_diagnostic();
m_ctx.diag().error(location, "{}", message);
return false;
} else {
m_expression = parse_result.release_value();
return true;
}
} }
Optional<AlgorithmStepList> AlgorithmStepList::create(SpecificationParsingContext& ctx, XML::Node const* element) Optional<AlgorithmStepList> AlgorithmStepList::create(SpecificationParsingContext& ctx, XML::Node const* element)
@ -250,12 +255,26 @@ void SpecificationClause::collect_into(TranslationUnitRef translation_unit)
subclause->collect_into(translation_unit); subclause->collect_into(translation_unit);
} }
ParseErrorOr<void> SpecificationClause::parse_header(XML::Node const* element) Optional<FailedTextParseDiagnostic> SpecificationClause::parse_header(XML::Node const* element)
{ {
auto& ctx = *m_ctx_pointer;
VERIFY(element->as_element().name == tag_h1); VERIFY(element->as_element().name == tag_h1);
auto tokens = TRY(tokenize_tree(*m_ctx_pointer, element));
TextParser parser(tokens.tokens, element); auto tokenization_result = tokenize_tree(ctx, element, false);
m_header = TRY(parser.parse_clause_header()); if (tokenization_result.is_error()) {
return FailedTextParseDiagnostic {
ctx.location_from_xml_offset(tokenization_result.error()->offset()),
tokenization_result.error()->to_string()
};
}
auto const& tokens = tokenization_result.release_value().tokens;
TextParser parser(ctx, tokens, element);
auto parse_result = parser.parse_clause_header();
if (parse_result.is_error())
return parser.get_diagnostic();
m_header = parse_result.value();
return {}; return {};
} }
@ -264,7 +283,7 @@ void SpecificationClause::parse(XML::Node const* element)
auto& ctx = context(); auto& ctx = context();
u32 child_index = 0; u32 child_index = 0;
Optional<NonnullRefPtr<ParseError>> header_parse_error; Optional<FailedTextParseDiagnostic> header_parse_error;
for (auto const& child : element->as_element().children) { for (auto const& child : element->as_element().children) {
child->content.visit( child->content.visit(
@ -275,10 +294,8 @@ void SpecificationClause::parse(XML::Node const* element)
"<h1> must be the first child of <emu-clause>"); "<h1> must be the first child of <emu-clause>");
return; return;
} }
header_parse_error = parse_header(child);
if (auto error = parse_header(child); error.is_error()) if (!header_parse_error.has_value())
header_parse_error = error.release_error();
else
ctx.current_logical_scope().section = MUST(String::from_utf8(m_header.section_number)); ctx.current_logical_scope().section = MUST(String::from_utf8(m_header.section_number));
} else { } else {
if (element.name == tag_h1) { if (element.name == tag_h1) {
@ -294,10 +311,7 @@ void SpecificationClause::parse(XML::Node const* element)
if (header_parse_error.has_value()) { if (header_parse_error.has_value()) {
ctx.diag().warn(ctx.location_from_xml_offset(child->offset), ctx.diag().warn(ctx.location_from_xml_offset(child->offset),
"node content will be ignored since section header was not parsed successfully"); "node content will be ignored since section header was not parsed successfully");
// TODO: Integrate backtracing parser errors better ctx.diag().note(header_parse_error->location, "{}", header_parse_error->message);
ctx.diag().note(ctx.location_from_xml_offset(header_parse_error.value()->offset()),
"{}", header_parse_error.value()->to_string());
header_parse_error.clear();
} }
} }
++child_index; ++child_index;

4
Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.h
View file

@ -70,7 +70,7 @@ private:
{ {
} }
ParseErrorOr<Tree> parse(); bool parse();
SpecificationParsingContext& m_ctx; SpecificationParsingContext& m_ctx;
Vector<Token> m_tokens; Vector<Token> m_tokens;
@ -113,7 +113,7 @@ private:
{ {
} }
ParseErrorOr<void> parse_header(XML::Node const* element); Optional<FailedTextParseDiagnostic> parse_header(XML::Node const* element);
void parse(XML::Node const* element); void parse(XML::Node const* element);
SpecificationParsingContext* m_ctx_pointer; SpecificationParsingContext* m_ctx_pointer;

204
Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.cpp
View file

@ -6,10 +6,21 @@
#include <AK/ScopeGuard.h> #include <AK/ScopeGuard.h>
#include "Parser/SpecParser.h"
#include "Parser/TextParser.h" #include "Parser/TextParser.h"
namespace JSSpecCompiler { namespace JSSpecCompiler {
void TextParser::save_error(Variant<TokenType, StringView, CustomMessage>&& expected)
{
if (m_max_parsed_tokens > m_next_token_index)
return;
if (m_max_parsed_tokens < m_next_token_index)
m_suitable_continuations.clear();
m_max_parsed_tokens = m_next_token_index;
m_suitable_continuations.append(move(expected));
}
void TextParser::retreat() void TextParser::retreat()
{ {
--m_next_token_index; --m_next_token_index;
@ -24,48 +35,69 @@ auto TextParser::rollback_point()
}; };
} }
ParseErrorOr<Token const*> TextParser::peek_token() Optional<Token> TextParser::peek_token()
{ {
if (m_next_token_index == m_tokens.size()) if (m_next_token_index == m_tokens.size())
return ParseError::create("Expected token but found EOF"sv, m_node); return {};
return &m_tokens[m_next_token_index]; return m_tokens[m_next_token_index];
} }
ParseErrorOr<Token const*> TextParser::consume_token() Optional<Token> TextParser::consume_token()
{ {
auto result = peek_token(); auto result = peek_token();
if (!result.is_error()) if (result.has_value())
++m_next_token_index; ++m_next_token_index;
return result; return result;
} }
ParseErrorOr<Token const*> TextParser::consume_token_with_one_of_types(std::initializer_list<TokenType> types) TextParseErrorOr<Token> TextParser::consume_token_with_one_of_types(std::initializer_list<TokenType> types)
{ {
auto token = TRY(consume_token()); auto token = peek_token();
for (TokenType type : types) if (token.has_value()) {
if (token->type == type) for (TokenType type : types) {
return token; if (token->type == type) {
retreat(); (void)consume_token();
return *token;
} else {
save_error(type);
}
}
} else {
for (TokenType type : types)
save_error(type);
}
return ParseError::create(String::formatted("Unexpected token type {}", token->name()), token->node); return TextParseError {};
} }
ParseErrorOr<Token const*> TextParser::consume_token_with_type(TokenType type) TextParseErrorOr<Token> TextParser::consume_token_with_type(TokenType type)
{ {
return consume_token_with_one_of_types({ type }); return consume_token_with_one_of_types({ type });
} }
ParseErrorOr<void> TextParser::consume_word(StringView word) TextParseErrorOr<void> TextParser::consume_token(TokenType type, StringView data)
{ {
auto token = TRY(consume_token_with_type(TokenType::Word)); auto token = consume_token();
if (!token->data.equals_ignoring_ascii_case(word)) { if (!token.has_value() || token->type != type || !token->data.equals_ignoring_ascii_case(data)) {
retreat(); retreat();
return ParseError::create("Unexpected word"sv, token->node); save_error(data);
return TextParseError {};
} }
return {}; return {};
} }
ParseErrorOr<void> TextParser::consume_words(std::initializer_list<StringView> words) TextParseErrorOr<void> TextParser::consume_word(StringView word)
{
auto token = consume_token();
if (!token.has_value() || token->type != TokenType::Word || !token->data.equals_ignoring_ascii_case(word)) {
retreat();
save_error(word);
return TextParseError {};
}
return {};
}
TextParseErrorOr<void> TextParser::consume_words(std::initializer_list<StringView> words)
{ {
for (auto word : words) for (auto word : words)
TRY(consume_word(word)); TRY(consume_word(word));
@ -77,14 +109,17 @@ bool TextParser::is_eof() const
return m_next_token_index == m_tokens.size(); return m_next_token_index == m_tokens.size();
} }
ParseErrorOr<void> TextParser::expect_eof() const TextParseErrorOr<void> TextParser::expect_eof()
{ {
if (!is_eof()) if (!is_eof()) {
return ParseError::create("Expected EOF"sv, m_node); save_error(CustomMessage { "EOF"sv });
return TextParseError {};
}
return {}; return {};
} }
ParseErrorOr<Tree> TextParser::parse_record_direct_list_initialization() // (the)? <record_name> { (<name>: <value>,)* }
TextParseErrorOr<Tree> TextParser::parse_record_direct_list_initialization()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -96,36 +131,37 @@ ParseErrorOr<Tree> TextParser::parse_record_direct_list_initialization()
while (true) { while (true) {
auto name = TRY(consume_token_with_one_of_types({ TokenType::Identifier, TokenType::BraceClose })); auto name = TRY(consume_token_with_one_of_types({ TokenType::Identifier, TokenType::BraceClose }));
if (name->is_bracket()) { if (name.is_bracket()) {
break; break;
} else { } else {
TRY(consume_token_with_type(TokenType::Colon)); TRY(consume_token_with_type(TokenType::Colon));
auto value = TRY(parse_expression()); auto value = TRY(parse_expression());
(void)consume_token_with_type(TokenType::Comma); (void)consume_token_with_type(TokenType::Comma);
arguments.append({ make_ref_counted<UnresolvedReference>(name->data), value }); arguments.append({ make_ref_counted<UnresolvedReference>(name.data), value });
} }
} }
rollback.disarm(); rollback.disarm();
return make_ref_counted<RecordDirectListInitialization>( return make_ref_counted<RecordDirectListInitialization>(
make_ref_counted<UnresolvedReference>(identifier->data), move(arguments)); make_ref_counted<UnresolvedReference>(identifier.data), move(arguments));
} }
// <expr> // <expr>
ParseErrorOr<Tree> TextParser::parse_expression() TextParseErrorOr<Tree> TextParser::parse_expression()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
// (the)? <record_name> { (<name>: <value>,)* }
if (auto record_init = parse_record_direct_list_initialization(); !record_init.is_error()) { if (auto record_init = parse_record_direct_list_initialization(); !record_init.is_error()) {
rollback.disarm(); rollback.disarm();
return record_init.release_value(); return record_init.release_value();
} }
#define THROW_PARSE_ERROR_IF(expr) \ #define THROW_PARSE_ERROR_IF(expr) \
do { \ do { \
if (expr) \ if (expr) { \
return ParseError::create("Expected expression"sv, m_node); \ save_error(CustomMessage { "valid expression continuation (not valid because " #expr ")"##sv }); \
return TextParseError {}; \
} \
} while (false) } while (false)
#define THROW_PARSE_ERROR THROW_PARSE_ERROR_IF(true) #define THROW_PARSE_ERROR THROW_PARSE_ERROR_IF(true)
@ -183,9 +219,9 @@ ParseErrorOr<Tree> TextParser::parse_expression()
while (true) { while (true) {
auto token_or_error = peek_token(); auto token_or_error = peek_token();
if (token_or_error.is_error()) if (!token_or_error.has_value())
break; break;
auto token = *token_or_error.release_value(); auto token = token_or_error.release_value();
enum { enum {
NoneType, NoneType,
@ -274,7 +310,7 @@ ParseErrorOr<Tree> TextParser::parse_expression()
merge_pre_merged(); merge_pre_merged();
} }
MUST(consume_token()); VERIFY(consume_token().has_value());
} }
THROW_PARSE_ERROR_IF(stack.is_empty()); THROW_PARSE_ERROR_IF(stack.is_empty());
@ -288,7 +324,7 @@ ParseErrorOr<Tree> TextParser::parse_expression()
} }
// <condition> :== <expr> | (<expr> is <expr> (or <expr>)?) // <condition> :== <expr> | (<expr> is <expr> (or <expr>)?)
ParseErrorOr<Tree> TextParser::parse_condition() TextParseErrorOr<Tree> TextParser::parse_condition()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
auto expression = TRY(parse_expression()); auto expression = TRY(parse_expression());
@ -307,7 +343,7 @@ ParseErrorOr<Tree> TextParser::parse_condition()
} }
// return <expr> // return <expr>
ParseErrorOr<Tree> TextParser::parse_return_statement() TextParseErrorOr<Tree> TextParser::parse_return_statement()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -319,15 +355,11 @@ ParseErrorOr<Tree> TextParser::parse_return_statement()
} }
// assert: <condition> // assert: <condition>
ParseErrorOr<Tree> TextParser::parse_assert() TextParseErrorOr<Tree> TextParser::parse_assert()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
auto identifier = TRY(consume_token_with_type(TokenType::Identifier))->data; TRY(consume_token(TokenType::Identifier, "assert"sv));
if (!identifier.equals_ignoring_ascii_case("assert"sv)) {
return ParseError::create("Expected identifier \"Assert\""sv, m_node);
}
TRY(consume_token_with_type(TokenType::Colon)); TRY(consume_token_with_type(TokenType::Colon));
auto condition = TRY(parse_condition()); auto condition = TRY(parse_condition());
@ -336,7 +368,7 @@ ParseErrorOr<Tree> TextParser::parse_assert()
} }
// (let <expr> be <expr>) | (set <expr> to <expr>) // (let <expr> be <expr>) | (set <expr> to <expr>)
ParseErrorOr<Tree> TextParser::parse_assignment() TextParseErrorOr<Tree> TextParser::parse_assignment()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -353,7 +385,7 @@ ParseErrorOr<Tree> TextParser::parse_assignment()
} }
// <simple_step> // <simple_step>
ParseErrorOr<Tree> TextParser::parse_simple_step_or_inline_if_branch() TextParseErrorOr<Tree> TextParser::parse_simple_step_or_inline_if_branch()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -382,11 +414,11 @@ ParseErrorOr<Tree> TextParser::parse_simple_step_or_inline_if_branch()
return result.release_value(); return result.release_value();
} }
return ParseError::create("Unable to parse simple step or inline if branch"sv, m_node); return TextParseError {};
} }
// <if_condition> :== (If <condition>) | (Else) | (Else if <condition>), // <if_condition> :== (If <condition>) | (Else) | (Else if <condition>),
ParseErrorOr<TextParser::IfConditionParseResult> TextParser::parse_if_beginning() TextParseErrorOr<TextParser::IfConditionParseResult> TextParser::parse_if_beginning()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -406,7 +438,7 @@ ParseErrorOr<TextParser::IfConditionParseResult> TextParser::parse_if_beginning(
} }
// <inline_if> :== <if_condition> <simple_step>.$ // <inline_if> :== <if_condition> <simple_step>.$
ParseErrorOr<Tree> TextParser::parse_inline_if_else() TextParseErrorOr<Tree> TextParser::parse_inline_if_else()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -420,7 +452,7 @@ ParseErrorOr<Tree> TextParser::parse_inline_if_else()
} }
// <if> :== <if_condition> then$ <substeps> // <if> :== <if_condition> then$ <substeps>
ParseErrorOr<Tree> TextParser::parse_if(Tree then_branch) TextParseErrorOr<Tree> TextParser::parse_if(Tree then_branch)
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -436,7 +468,7 @@ ParseErrorOr<Tree> TextParser::parse_if(Tree then_branch)
} }
// <else> :== Else,$ <substeps> // <else> :== Else,$ <substeps>
ParseErrorOr<Tree> TextParser::parse_else(Tree else_branch) TextParseErrorOr<Tree> TextParser::parse_else(Tree else_branch)
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -449,7 +481,7 @@ ParseErrorOr<Tree> TextParser::parse_else(Tree else_branch)
} }
// <simple_step> | <inline_if> // <simple_step> | <inline_if>
ParseErrorOr<Tree> TextParser::parse_step_without_substeps() TextParseErrorOr<Tree> TextParser::parse_step_without_substeps()
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -465,11 +497,11 @@ ParseErrorOr<Tree> TextParser::parse_step_without_substeps()
return result.release_value(); return result.release_value();
} }
return ParseError::create("Unable to parse step without substeps"sv, m_node); return TextParseError {};
} }
// <if> | <else> // <if> | <else>
ParseErrorOr<Tree> TextParser::parse_step_with_substeps(Tree substeps) TextParseErrorOr<Tree> TextParser::parse_step_with_substeps(Tree substeps)
{ {
auto rollback = rollback_point(); auto rollback = rollback_point();
@ -485,32 +517,32 @@ ParseErrorOr<Tree> TextParser::parse_step_with_substeps(Tree substeps)
return result.release_value(); return result.release_value();
} }
return ParseError::create("Unable to parse step with substeps"sv, m_node); return TextParseError {};
} }
ParseErrorOr<ClauseHeader> TextParser::parse_clause_header() TextParseErrorOr<ClauseHeader> TextParser::parse_clause_header()
{ {
ClauseHeader result; ClauseHeader result;
auto section_number_token = TRY(consume_token_with_type(TokenType::SectionNumber)); auto section_number_token = TRY(consume_token_with_type(TokenType::SectionNumber));
result.section_number = section_number_token->data; result.section_number = section_number_token.data;
ClauseHeader::FunctionDefinition function_definition; ClauseHeader::FunctionDefinition function_definition;
function_definition.name = TRY(consume_token())->data; function_definition.name = TRY(consume_token_with_type(TokenType::Word)).data;
TRY(consume_token_with_type(TokenType::ParenOpen)); TRY(consume_token_with_type(TokenType::ParenOpen));
while (true) { while (true) {
if (function_definition.arguments.is_empty()) { if (function_definition.arguments.is_empty()) {
auto argument = TRY(consume_token_with_one_of_types({ TokenType::ParenClose, TokenType::Identifier })); auto argument = TRY(consume_token_with_one_of_types({ TokenType::ParenClose, TokenType::Identifier }));
if (argument->type == TokenType::ParenClose) if (argument.type == TokenType::ParenClose)
break; break;
function_definition.arguments.append({ argument->data }); function_definition.arguments.append({ argument.data });
} else { } else {
function_definition.arguments.append({ TRY(consume_token_with_type(TokenType::Identifier))->data }); function_definition.arguments.append({ TRY(consume_token_with_type(TokenType::Identifier)).data });
} }
auto next_token = TRY(consume_token_with_one_of_types({ TokenType::ParenClose, TokenType::Comma })); auto next_token = TRY(consume_token_with_one_of_types({ TokenType::ParenClose, TokenType::Comma }));
if (next_token->type == TokenType::ParenClose) if (next_token.type == TokenType::ParenClose)
break; break;
} }
TRY(expect_eof()); TRY(expect_eof());
@ -520,4 +552,56 @@ ParseErrorOr<ClauseHeader> TextParser::parse_clause_header()
return result; return result;
} }
FailedTextParseDiagnostic TextParser::get_diagnostic() const
{
StringBuilder message;
message.append("unexpected "sv);
if (m_max_parsed_tokens == m_tokens.size()) {
message.append("EOF"sv);
} else {
auto token = m_tokens[m_max_parsed_tokens];
if (token.type == TokenType::Word)
message.appendff("'{}'", token.data);
else if (token.type == TokenType::Identifier)
message.appendff("identifier '{}'", token.data);
else
message.append(token.name_for_diagnostic());
}
message.appendff(", expected ");
size_t size = m_suitable_continuations.size();
VERIFY(size > 0);
for (size_t i = 0; i < size; ++i) {
m_suitable_continuations[i].visit(
[&](TokenType type) { message.append(token_info[to_underlying(type)].name_for_diagnostic); },
[&](StringView word) { message.appendff("'{}'", word); },
[&](CustomMessage continuation) { message.append(continuation.message); });
if (i + 1 != size) {
if (size == 2)
message.append(" or "sv);
else if (i + 2 == size)
message.append(", or "sv);
else
message.append(", "sv);
}
}
Location location = Location::global_scope();
if (m_max_parsed_tokens < m_tokens.size()) {
location = m_tokens[m_max_parsed_tokens].location;
} else {
// FIXME: Would be nice to point to the closing tag not the opening one. This is also the
// only place where we use m_location.
location = m_ctx.location_from_xml_offset(m_node->offset);
}
return { location, MUST(message.to_string()) };
}
} }

72
Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.h
View file

@ -23,17 +23,30 @@ struct ClauseHeader {
Variant<AK::Empty, FunctionDefinition> header; Variant<AK::Empty, FunctionDefinition> header;
}; };
struct TextParseError { };
struct FailedTextParseDiagnostic {
Location location;
String message;
};
template<typename T>
using TextParseErrorOr = ErrorOr<T, TextParseError>;
class TextParser { class TextParser {
public: public:
TextParser(Vector<Token>& tokens_, XML::Node const* node_) TextParser(SpecificationParsingContext& ctx, Vector<Token> const& tokens, XML::Node const* node)
: m_tokens(tokens_) : m_ctx(ctx)
, m_node(node_) , m_tokens(tokens)
, m_node(node)
{ {
} }
ParseErrorOr<ClauseHeader> parse_clause_header(); TextParseErrorOr<ClauseHeader> parse_clause_header();
ParseErrorOr<Tree> parse_step_without_substeps(); TextParseErrorOr<Tree> parse_step_without_substeps();
ParseErrorOr<Tree> parse_step_with_substeps(Tree substeps); TextParseErrorOr<Tree> parse_step_with_substeps(Tree substeps);
FailedTextParseDiagnostic get_diagnostic() const;
private: private:
struct IfConditionParseResult { struct IfConditionParseResult {
@ -41,32 +54,43 @@ private:
NullableTree condition; NullableTree condition;
}; };
struct CustomMessage {
StringView message;
};
void save_error(Variant<TokenType, StringView, CustomMessage>&& expected);
void retreat(); void retreat();
[[nodiscard]] auto rollback_point(); [[nodiscard]] auto rollback_point();
ParseErrorOr<Token const*> peek_token(); Optional<Token> peek_token();
ParseErrorOr<Token const*> consume_token(); Optional<Token> consume_token();
ParseErrorOr<Token const*> consume_token_with_one_of_types(std::initializer_list<TokenType> types); TextParseErrorOr<Token> consume_token_with_one_of_types(std::initializer_list<TokenType> types);
ParseErrorOr<Token const*> consume_token_with_type(TokenType type); TextParseErrorOr<Token> consume_token_with_type(TokenType type);
ParseErrorOr<void> consume_word(StringView word); TextParseErrorOr<void> consume_token(TokenType type, StringView data);
ParseErrorOr<void> consume_words(std::initializer_list<StringView> words); TextParseErrorOr<void> consume_word(StringView word);
TextParseErrorOr<void> consume_words(std::initializer_list<StringView> words);
bool is_eof() const; bool is_eof() const;
ParseErrorOr<void> expect_eof() const; TextParseErrorOr<void> expect_eof();
ParseErrorOr<Tree> parse_record_direct_list_initialization(); TextParseErrorOr<Tree> parse_record_direct_list_initialization();
ParseErrorOr<Tree> parse_expression(); TextParseErrorOr<Tree> parse_expression();
ParseErrorOr<Tree> parse_condition(); TextParseErrorOr<Tree> parse_condition();
ParseErrorOr<Tree> parse_return_statement(); TextParseErrorOr<Tree> parse_return_statement();
ParseErrorOr<Tree> parse_assert(); TextParseErrorOr<Tree> parse_assert();
ParseErrorOr<Tree> parse_assignment(); TextParseErrorOr<Tree> parse_assignment();
ParseErrorOr<Tree> parse_simple_step_or_inline_if_branch(); TextParseErrorOr<Tree> parse_simple_step_or_inline_if_branch();
ParseErrorOr<IfConditionParseResult> parse_if_beginning(); TextParseErrorOr<IfConditionParseResult> parse_if_beginning();
ParseErrorOr<Tree> parse_inline_if_else(); TextParseErrorOr<Tree> parse_inline_if_else();
ParseErrorOr<Tree> parse_if(Tree then_branch); TextParseErrorOr<Tree> parse_if(Tree then_branch);
ParseErrorOr<Tree> parse_else(Tree else_branch); TextParseErrorOr<Tree> parse_else(Tree else_branch);
SpecificationParsingContext& m_ctx;
Vector<Token> const& m_tokens; Vector<Token> const& m_tokens;
size_t m_next_token_index = 0; size_t m_next_token_index = 0;
XML::Node const* m_node; XML::Node const* m_node;
size_t m_max_parsed_tokens = 0;
Vector<Variant<TokenType, StringView, CustomMessage>, 8> m_suitable_continuations;
}; };
} }

79
Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/Token.h
View file

@ -20,38 +20,38 @@ constexpr i32 closing_bracket_precedence = 18;
// NOTE: Operator precedence is generally the same as in // NOTE: Operator precedence is generally the same as in
// https://en.cppreference.com/w/cpp/language/operator_precedence (common sense applies). // https://en.cppreference.com/w/cpp/language/operator_precedence (common sense applies).
#define ENUMERATE_TOKENS(F) \ #define ENUMERATE_TOKENS(F) \
F(Invalid, -1, Invalid, Invalid, Invalid) \ F(Invalid, -1, Invalid, Invalid, Invalid, "") \
F(SectionNumber, -1, Invalid, Invalid, Invalid) \ F(SectionNumber, -1, Invalid, Invalid, Invalid, "section number") \
F(Identifier, -1, Invalid, Invalid, Invalid) \ F(Identifier, -1, Invalid, Invalid, Invalid, "identifier") \
F(Number, -1, Invalid, Invalid, Invalid) \ F(Number, -1, Invalid, Invalid, Invalid, "number") \
F(String, -1, Invalid, Invalid, Invalid) \ F(String, -1, Invalid, Invalid, Invalid, "string literal") \
F(Undefined, -1, Invalid, Invalid, Invalid) \ F(Undefined, -1, Invalid, Invalid, Invalid, "constant") \
F(Word, -1, Invalid, Invalid, Invalid) \ F(Word, -1, Invalid, Invalid, Invalid, "word") \
F(ParenOpen, -1, Invalid, Invalid, ParenClose) \ F(ParenOpen, -1, Invalid, Invalid, ParenClose, "'('") \
F(ParenClose, 18, Invalid, Invalid, ParenOpen) \ F(ParenClose, 18, Invalid, Invalid, ParenOpen, "')'") \
F(BraceOpen, -1, Invalid, Invalid, BraceClose) \ F(BraceOpen, -1, Invalid, Invalid, BraceClose, "'{'") \
F(BraceClose, 18, Invalid, Invalid, BraceOpen) \ F(BraceClose, 18, Invalid, Invalid, BraceOpen, "'}'") \
F(Comma, 17, Invalid, Comma, Invalid) \ F(Comma, 17, Invalid, Comma, Invalid, "','") \
F(MemberAccess, 2, Invalid, MemberAccess, Invalid) \ F(MemberAccess, 2, Invalid, MemberAccess, Invalid, "member access operator '.'") \
F(Dot, -1, Invalid, Invalid, Invalid) \ F(Dot, -1, Invalid, Invalid, Invalid, "punctuation mark '.'") \
F(Colon, -1, Invalid, Invalid, Invalid) \ F(Colon, -1, Invalid, Invalid, Invalid, "':'") \
F(Less, 9, Invalid, CompareLess, Invalid) \ F(Less, 9, Invalid, CompareLess, Invalid, "less than") \
F(Greater, 9, Invalid, CompareGreater, Invalid) \ F(Greater, 9, Invalid, CompareGreater, Invalid, "greater than") \
F(NotEquals, 10, Invalid, CompareNotEqual, Invalid) \ F(NotEquals, 10, Invalid, CompareNotEqual, Invalid, "not equals") \
F(Equals, 10, Invalid, CompareEqual, Invalid) \ F(Equals, 10, Invalid, CompareEqual, Invalid, "equals") \
F(Plus, 6, Invalid, Plus, Invalid) \ F(Plus, 6, Invalid, Plus, Invalid, "plus") \
F(AmbiguousMinus, -2, Invalid, Invalid, Invalid) \ F(AmbiguousMinus, -2, Invalid, Invalid, Invalid, "minus") \
F(UnaryMinus, 3, Minus, Invalid, Invalid) \ F(UnaryMinus, 3, Minus, Invalid, Invalid, "unary minus") \
F(BinaryMinus, 6, Invalid, Minus, Invalid) \ F(BinaryMinus, 6, Invalid, Minus, Invalid, "binary minus") \
F(Multiplication, 5, Invalid, Multiplication, Invalid) \ F(Multiplication, 5, Invalid, Multiplication, Invalid, "multiplication") \
F(Division, 5, Invalid, Division, Invalid) \ F(Division, 5, Invalid, Division, Invalid, "division") \
F(FunctionCall, 2, Invalid, FunctionCall, Invalid) \ F(FunctionCall, 2, Invalid, FunctionCall, Invalid, "function call token") \
F(ExclamationMark, 3, AssertCompletion, Invalid, Invalid) \ F(ExclamationMark, 3, AssertCompletion, Invalid, Invalid, "exclamation mark") \
F(Is, -1, Invalid, Invalid, Invalid) F(Is, -1, Invalid, Invalid, Invalid, "operator is")
enum class TokenType { enum class TokenType {
#define ID(name, precedence, unary_name, binary_name, matching_bracket) name, #define ID(name, precedence, unary_name, binary_name, matching_bracket, name_for_diagnostic) name,
ENUMERATE_TOKENS(ID) ENUMERATE_TOKENS(ID)
#undef ID #undef ID
}; };
@ -62,14 +62,16 @@ constexpr struct TokenInfo {
UnaryOperator as_unary_operator; UnaryOperator as_unary_operator;
BinaryOperator as_binary_operator; BinaryOperator as_binary_operator;
TokenType matching_bracket; TokenType matching_bracket;
StringView name_for_diagnostic;
} token_info[] = { } token_info[] = {
#define TOKEN_INFO(name, precedence, unary_name, binary_name, matching_bracket) \ #define TOKEN_INFO(name, precedence, unary_name, binary_name, matching_bracket, name_for_diagnostic) \
{ \ { \
#name##sv, \ #name##sv, \
precedence, \ precedence, \
UnaryOperator::unary_name, \ UnaryOperator::unary_name, \
BinaryOperator::binary_name, \ BinaryOperator::binary_name, \
TokenType::matching_bracket, \ TokenType::matching_bracket, \
name_for_diagnostic##sv \
}, },
ENUMERATE_TOKENS(TOKEN_INFO) ENUMERATE_TOKENS(TOKEN_INFO)
#undef TOKEN_INFO #undef TOKEN_INFO
@ -78,7 +80,8 @@ constexpr struct TokenInfo {
struct Token { struct Token {
TokenInfo const& info() const { return token_info[to_underlying(type)]; } TokenInfo const& info() const { return token_info[to_underlying(type)]; }
StringView name() const { return token_info[to_underlying(type)].name; } StringView name() const { return info().name; }
StringView name_for_diagnostic() const { return info().name_for_diagnostic; }
i32 precedence() const { return info().precedence; } i32 precedence() const { return info().precedence; }
bool is_operator() const { return precedence() > 0 && precedence() < closing_bracket_precedence; } bool is_operator() const { return precedence() > 0 && precedence() < closing_bracket_precedence; }
bool is_ambiguous_operator() const { return precedence() == ambiguous_operator_precedence; } bool is_ambiguous_operator() const { return precedence() == ambiguous_operator_precedence; }