From 6b30c4d2f0a98a904e2cdddddd41637546a3216c Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 17 Jan 2024 01:36:39 -0500 Subject: [PATCH] JSSpecCompiler: Issue meaningful errors in TextParser --- .../JSSpecCompiler/Parser/SpecParser.cpp | 60 ++++-- .../JSSpecCompiler/Parser/SpecParser.h | 4 +- .../JSSpecCompiler/Parser/TextParser.cpp | 204 ++++++++++++------ .../JSSpecCompiler/Parser/TextParser.h | 72 ++++--- .../JSSpecCompiler/Parser/Token.h | 79 +++---- 5 files changed, 272 insertions(+), 147 deletions(-) diff --git a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.cpp b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.cpp index 663d9ecb59..c2061d9033 100644 --- a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.cpp +++ b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.cpp @@ -86,24 +86,29 @@ Optional AlgorithmStep::create(SpecificationParsingContext& ctx, result.m_substeps = step_list.has_value() ? step_list->tree() : error_tree; } - auto parse_result = result.parse(); - if (parse_result.is_error()) { - ctx.diag().error(ctx.location_from_xml_offset(parse_result.error()->offset()), - "{}", parse_result.error()->to_string()); + if (!result.parse()) return {}; - } - result.m_expression = parse_result.release_value(); return result; } -ParseErrorOr AlgorithmStep::parse() +bool AlgorithmStep::parse() { - TextParser parser(m_tokens, m_node); + TextParser parser(m_ctx, m_tokens, m_node); + TextParseErrorOr parse_result = TextParseError {}; if (m_substeps) - return parser.parse_step_with_substeps(RefPtr(m_substeps).release_nonnull()); + parse_result = parser.parse_step_with_substeps(RefPtr(m_substeps).release_nonnull()); else - return parser.parse_step_without_substeps(); + parse_result = parser.parse_step_without_substeps(); + + if (parse_result.is_error()) { + auto [location, message] = parser.get_diagnostic(); + m_ctx.diag().error(location, "{}", message); + return false; + } else { + m_expression = parse_result.release_value(); + return true; + } } Optional AlgorithmStepList::create(SpecificationParsingContext& ctx, XML::Node const* element) @@ -250,12 +255,26 @@ void SpecificationClause::collect_into(TranslationUnitRef translation_unit) subclause->collect_into(translation_unit); } -ParseErrorOr SpecificationClause::parse_header(XML::Node const* element) +Optional SpecificationClause::parse_header(XML::Node const* element) { + auto& ctx = *m_ctx_pointer; VERIFY(element->as_element().name == tag_h1); - auto tokens = TRY(tokenize_tree(*m_ctx_pointer, element)); - TextParser parser(tokens.tokens, element); - m_header = TRY(parser.parse_clause_header()); + + auto tokenization_result = tokenize_tree(ctx, element, false); + if (tokenization_result.is_error()) { + return FailedTextParseDiagnostic { + ctx.location_from_xml_offset(tokenization_result.error()->offset()), + tokenization_result.error()->to_string() + }; + } + auto const& tokens = tokenization_result.release_value().tokens; + + TextParser parser(ctx, tokens, element); + auto parse_result = parser.parse_clause_header(); + if (parse_result.is_error()) + return parser.get_diagnostic(); + + m_header = parse_result.value(); return {}; } @@ -264,7 +283,7 @@ void SpecificationClause::parse(XML::Node const* element) auto& ctx = context(); u32 child_index = 0; - Optional> header_parse_error; + Optional header_parse_error; for (auto const& child : element->as_element().children) { child->content.visit( @@ -275,10 +294,8 @@ void SpecificationClause::parse(XML::Node const* element) "

must be the first child of "); return; } - - if (auto error = parse_header(child); error.is_error()) - header_parse_error = error.release_error(); - else + header_parse_error = parse_header(child); + if (!header_parse_error.has_value()) ctx.current_logical_scope().section = MUST(String::from_utf8(m_header.section_number)); } else { if (element.name == tag_h1) { @@ -294,10 +311,7 @@ void SpecificationClause::parse(XML::Node const* element) if (header_parse_error.has_value()) { ctx.diag().warn(ctx.location_from_xml_offset(child->offset), "node content will be ignored since section header was not parsed successfully"); - // TODO: Integrate backtracing parser errors better - ctx.diag().note(ctx.location_from_xml_offset(header_parse_error.value()->offset()), - "{}", header_parse_error.value()->to_string()); - header_parse_error.clear(); + ctx.diag().note(header_parse_error->location, "{}", header_parse_error->message); } } ++child_index; diff --git a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.h b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.h index d613cc25ea..c5802630ca 100644 --- a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.h +++ b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/SpecParser.h @@ -70,7 +70,7 @@ private: { } - ParseErrorOr parse(); + bool parse(); SpecificationParsingContext& m_ctx; Vector m_tokens; @@ -113,7 +113,7 @@ private: { } - ParseErrorOr parse_header(XML::Node const* element); + Optional parse_header(XML::Node const* element); void parse(XML::Node const* element); SpecificationParsingContext* m_ctx_pointer; diff --git a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.cpp b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.cpp index 21ae23e8f7..c70ad98af1 100644 --- a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.cpp +++ b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.cpp @@ -6,10 +6,21 @@ #include +#include "Parser/SpecParser.h" #include "Parser/TextParser.h" namespace JSSpecCompiler { +void TextParser::save_error(Variant&& expected) +{ + if (m_max_parsed_tokens > m_next_token_index) + return; + if (m_max_parsed_tokens < m_next_token_index) + m_suitable_continuations.clear(); + m_max_parsed_tokens = m_next_token_index; + m_suitable_continuations.append(move(expected)); +} + void TextParser::retreat() { --m_next_token_index; @@ -24,48 +35,69 @@ auto TextParser::rollback_point() }; } -ParseErrorOr TextParser::peek_token() +Optional TextParser::peek_token() { if (m_next_token_index == m_tokens.size()) - return ParseError::create("Expected token but found EOF"sv, m_node); - return &m_tokens[m_next_token_index]; + return {}; + return m_tokens[m_next_token_index]; } -ParseErrorOr TextParser::consume_token() +Optional TextParser::consume_token() { auto result = peek_token(); - if (!result.is_error()) + if (result.has_value()) ++m_next_token_index; return result; } -ParseErrorOr TextParser::consume_token_with_one_of_types(std::initializer_list types) +TextParseErrorOr TextParser::consume_token_with_one_of_types(std::initializer_list types) { - auto token = TRY(consume_token()); - for (TokenType type : types) - if (token->type == type) - return token; - retreat(); + auto token = peek_token(); + if (token.has_value()) { + for (TokenType type : types) { + if (token->type == type) { + (void)consume_token(); + return *token; + } else { + save_error(type); + } + } + } else { + for (TokenType type : types) + save_error(type); + } - return ParseError::create(String::formatted("Unexpected token type {}", token->name()), token->node); + return TextParseError {}; } -ParseErrorOr TextParser::consume_token_with_type(TokenType type) +TextParseErrorOr TextParser::consume_token_with_type(TokenType type) { return consume_token_with_one_of_types({ type }); } -ParseErrorOr TextParser::consume_word(StringView word) +TextParseErrorOr TextParser::consume_token(TokenType type, StringView data) { - auto token = TRY(consume_token_with_type(TokenType::Word)); - if (!token->data.equals_ignoring_ascii_case(word)) { + auto token = consume_token(); + if (!token.has_value() || token->type != type || !token->data.equals_ignoring_ascii_case(data)) { retreat(); - return ParseError::create("Unexpected word"sv, token->node); + save_error(data); + return TextParseError {}; } return {}; } -ParseErrorOr TextParser::consume_words(std::initializer_list words) +TextParseErrorOr TextParser::consume_word(StringView word) +{ + auto token = consume_token(); + if (!token.has_value() || token->type != TokenType::Word || !token->data.equals_ignoring_ascii_case(word)) { + retreat(); + save_error(word); + return TextParseError {}; + } + return {}; +} + +TextParseErrorOr TextParser::consume_words(std::initializer_list words) { for (auto word : words) TRY(consume_word(word)); @@ -77,14 +109,17 @@ bool TextParser::is_eof() const return m_next_token_index == m_tokens.size(); } -ParseErrorOr TextParser::expect_eof() const +TextParseErrorOr TextParser::expect_eof() { - if (!is_eof()) - return ParseError::create("Expected EOF"sv, m_node); + if (!is_eof()) { + save_error(CustomMessage { "EOF"sv }); + return TextParseError {}; + } return {}; } -ParseErrorOr TextParser::parse_record_direct_list_initialization() +// (the)? { (: ,)* } +TextParseErrorOr TextParser::parse_record_direct_list_initialization() { auto rollback = rollback_point(); @@ -96,36 +131,37 @@ ParseErrorOr TextParser::parse_record_direct_list_initialization() while (true) { auto name = TRY(consume_token_with_one_of_types({ TokenType::Identifier, TokenType::BraceClose })); - if (name->is_bracket()) { + if (name.is_bracket()) { break; } else { TRY(consume_token_with_type(TokenType::Colon)); auto value = TRY(parse_expression()); (void)consume_token_with_type(TokenType::Comma); - arguments.append({ make_ref_counted(name->data), value }); + arguments.append({ make_ref_counted(name.data), value }); } } rollback.disarm(); return make_ref_counted( - make_ref_counted(identifier->data), move(arguments)); + make_ref_counted(identifier.data), move(arguments)); } // -ParseErrorOr TextParser::parse_expression() +TextParseErrorOr TextParser::parse_expression() { auto rollback = rollback_point(); - // (the)? { (: ,)* } if (auto record_init = parse_record_direct_list_initialization(); !record_init.is_error()) { rollback.disarm(); return record_init.release_value(); } -#define THROW_PARSE_ERROR_IF(expr) \ - do { \ - if (expr) \ - return ParseError::create("Expected expression"sv, m_node); \ +#define THROW_PARSE_ERROR_IF(expr) \ + do { \ + if (expr) { \ + save_error(CustomMessage { "valid expression continuation (not valid because " #expr ")"##sv }); \ + return TextParseError {}; \ + } \ } while (false) #define THROW_PARSE_ERROR THROW_PARSE_ERROR_IF(true) @@ -183,9 +219,9 @@ ParseErrorOr TextParser::parse_expression() while (true) { auto token_or_error = peek_token(); - if (token_or_error.is_error()) + if (!token_or_error.has_value()) break; - auto token = *token_or_error.release_value(); + auto token = token_or_error.release_value(); enum { NoneType, @@ -274,7 +310,7 @@ ParseErrorOr TextParser::parse_expression() merge_pre_merged(); } - MUST(consume_token()); + VERIFY(consume_token().has_value()); } THROW_PARSE_ERROR_IF(stack.is_empty()); @@ -288,7 +324,7 @@ ParseErrorOr TextParser::parse_expression() } // :== | ( is (or )?) -ParseErrorOr TextParser::parse_condition() +TextParseErrorOr TextParser::parse_condition() { auto rollback = rollback_point(); auto expression = TRY(parse_expression()); @@ -307,7 +343,7 @@ ParseErrorOr TextParser::parse_condition() } // return -ParseErrorOr TextParser::parse_return_statement() +TextParseErrorOr TextParser::parse_return_statement() { auto rollback = rollback_point(); @@ -319,15 +355,11 @@ ParseErrorOr TextParser::parse_return_statement() } // assert: -ParseErrorOr TextParser::parse_assert() +TextParseErrorOr TextParser::parse_assert() { auto rollback = rollback_point(); - auto identifier = TRY(consume_token_with_type(TokenType::Identifier))->data; - if (!identifier.equals_ignoring_ascii_case("assert"sv)) { - return ParseError::create("Expected identifier \"Assert\""sv, m_node); - } - + TRY(consume_token(TokenType::Identifier, "assert"sv)); TRY(consume_token_with_type(TokenType::Colon)); auto condition = TRY(parse_condition()); @@ -336,7 +368,7 @@ ParseErrorOr TextParser::parse_assert() } // (let be ) | (set to ) -ParseErrorOr TextParser::parse_assignment() +TextParseErrorOr TextParser::parse_assignment() { auto rollback = rollback_point(); @@ -353,7 +385,7 @@ ParseErrorOr TextParser::parse_assignment() } // -ParseErrorOr TextParser::parse_simple_step_or_inline_if_branch() +TextParseErrorOr TextParser::parse_simple_step_or_inline_if_branch() { auto rollback = rollback_point(); @@ -382,11 +414,11 @@ ParseErrorOr TextParser::parse_simple_step_or_inline_if_branch() return result.release_value(); } - return ParseError::create("Unable to parse simple step or inline if branch"sv, m_node); + return TextParseError {}; } // :== (If ) | (Else) | (Else if ), -ParseErrorOr TextParser::parse_if_beginning() +TextParseErrorOr TextParser::parse_if_beginning() { auto rollback = rollback_point(); @@ -406,7 +438,7 @@ ParseErrorOr TextParser::parse_if_beginning( } // :== .$ -ParseErrorOr TextParser::parse_inline_if_else() +TextParseErrorOr TextParser::parse_inline_if_else() { auto rollback = rollback_point(); @@ -420,7 +452,7 @@ ParseErrorOr TextParser::parse_inline_if_else() } // :== then$ -ParseErrorOr TextParser::parse_if(Tree then_branch) +TextParseErrorOr TextParser::parse_if(Tree then_branch) { auto rollback = rollback_point(); @@ -436,7 +468,7 @@ ParseErrorOr TextParser::parse_if(Tree then_branch) } // :== Else,$ -ParseErrorOr TextParser::parse_else(Tree else_branch) +TextParseErrorOr TextParser::parse_else(Tree else_branch) { auto rollback = rollback_point(); @@ -449,7 +481,7 @@ ParseErrorOr TextParser::parse_else(Tree else_branch) } // | -ParseErrorOr TextParser::parse_step_without_substeps() +TextParseErrorOr TextParser::parse_step_without_substeps() { auto rollback = rollback_point(); @@ -465,11 +497,11 @@ ParseErrorOr TextParser::parse_step_without_substeps() return result.release_value(); } - return ParseError::create("Unable to parse step without substeps"sv, m_node); + return TextParseError {}; } // | -ParseErrorOr TextParser::parse_step_with_substeps(Tree substeps) +TextParseErrorOr TextParser::parse_step_with_substeps(Tree substeps) { auto rollback = rollback_point(); @@ -485,32 +517,32 @@ ParseErrorOr TextParser::parse_step_with_substeps(Tree substeps) return result.release_value(); } - return ParseError::create("Unable to parse step with substeps"sv, m_node); + return TextParseError {}; } -ParseErrorOr TextParser::parse_clause_header() +TextParseErrorOr TextParser::parse_clause_header() { ClauseHeader result; auto section_number_token = TRY(consume_token_with_type(TokenType::SectionNumber)); - result.section_number = section_number_token->data; + result.section_number = section_number_token.data; ClauseHeader::FunctionDefinition function_definition; - function_definition.name = TRY(consume_token())->data; + function_definition.name = TRY(consume_token_with_type(TokenType::Word)).data; TRY(consume_token_with_type(TokenType::ParenOpen)); while (true) { if (function_definition.arguments.is_empty()) { auto argument = TRY(consume_token_with_one_of_types({ TokenType::ParenClose, TokenType::Identifier })); - if (argument->type == TokenType::ParenClose) + if (argument.type == TokenType::ParenClose) break; - function_definition.arguments.append({ argument->data }); + function_definition.arguments.append({ argument.data }); } else { - function_definition.arguments.append({ TRY(consume_token_with_type(TokenType::Identifier))->data }); + function_definition.arguments.append({ TRY(consume_token_with_type(TokenType::Identifier)).data }); } auto next_token = TRY(consume_token_with_one_of_types({ TokenType::ParenClose, TokenType::Comma })); - if (next_token->type == TokenType::ParenClose) + if (next_token.type == TokenType::ParenClose) break; } TRY(expect_eof()); @@ -520,4 +552,56 @@ ParseErrorOr TextParser::parse_clause_header() return result; } +FailedTextParseDiagnostic TextParser::get_diagnostic() const +{ + StringBuilder message; + + message.append("unexpected "sv); + + if (m_max_parsed_tokens == m_tokens.size()) { + message.append("EOF"sv); + } else { + auto token = m_tokens[m_max_parsed_tokens]; + if (token.type == TokenType::Word) + message.appendff("'{}'", token.data); + else if (token.type == TokenType::Identifier) + message.appendff("identifier '{}'", token.data); + else + message.append(token.name_for_diagnostic()); + } + + message.appendff(", expected "); + + size_t size = m_suitable_continuations.size(); + VERIFY(size > 0); + + for (size_t i = 0; i < size; ++i) { + m_suitable_continuations[i].visit( + [&](TokenType type) { message.append(token_info[to_underlying(type)].name_for_diagnostic); }, + [&](StringView word) { message.appendff("'{}'", word); }, + [&](CustomMessage continuation) { message.append(continuation.message); }); + + if (i + 1 != size) { + if (size == 2) + message.append(" or "sv); + else if (i + 2 == size) + message.append(", or "sv); + else + message.append(", "sv); + } + } + + Location location = Location::global_scope(); + + if (m_max_parsed_tokens < m_tokens.size()) { + location = m_tokens[m_max_parsed_tokens].location; + } else { + // FIXME: Would be nice to point to the closing tag not the opening one. This is also the + // only place where we use m_location. + location = m_ctx.location_from_xml_offset(m_node->offset); + } + + return { location, MUST(message.to_string()) }; +} + } diff --git a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.h b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.h index 627f9f353d..0e7c7756dd 100644 --- a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.h +++ b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/TextParser.h @@ -23,17 +23,30 @@ struct ClauseHeader { Variant header; }; +struct TextParseError { }; + +struct FailedTextParseDiagnostic { + Location location; + String message; +}; + +template +using TextParseErrorOr = ErrorOr; + class TextParser { public: - TextParser(Vector& tokens_, XML::Node const* node_) - : m_tokens(tokens_) - , m_node(node_) + TextParser(SpecificationParsingContext& ctx, Vector const& tokens, XML::Node const* node) + : m_ctx(ctx) + , m_tokens(tokens) + , m_node(node) { } - ParseErrorOr parse_clause_header(); - ParseErrorOr parse_step_without_substeps(); - ParseErrorOr parse_step_with_substeps(Tree substeps); + TextParseErrorOr parse_clause_header(); + TextParseErrorOr parse_step_without_substeps(); + TextParseErrorOr parse_step_with_substeps(Tree substeps); + + FailedTextParseDiagnostic get_diagnostic() const; private: struct IfConditionParseResult { @@ -41,32 +54,43 @@ private: NullableTree condition; }; + struct CustomMessage { + StringView message; + }; + + void save_error(Variant&& expected); + void retreat(); [[nodiscard]] auto rollback_point(); - ParseErrorOr peek_token(); - ParseErrorOr consume_token(); - ParseErrorOr consume_token_with_one_of_types(std::initializer_list types); - ParseErrorOr consume_token_with_type(TokenType type); - ParseErrorOr consume_word(StringView word); - ParseErrorOr consume_words(std::initializer_list words); + Optional peek_token(); + Optional consume_token(); + TextParseErrorOr consume_token_with_one_of_types(std::initializer_list types); + TextParseErrorOr consume_token_with_type(TokenType type); + TextParseErrorOr consume_token(TokenType type, StringView data); + TextParseErrorOr consume_word(StringView word); + TextParseErrorOr consume_words(std::initializer_list words); bool is_eof() const; - ParseErrorOr expect_eof() const; + TextParseErrorOr expect_eof(); - ParseErrorOr parse_record_direct_list_initialization(); - ParseErrorOr parse_expression(); - ParseErrorOr parse_condition(); - ParseErrorOr parse_return_statement(); - ParseErrorOr parse_assert(); - ParseErrorOr parse_assignment(); - ParseErrorOr parse_simple_step_or_inline_if_branch(); - ParseErrorOr parse_if_beginning(); - ParseErrorOr parse_inline_if_else(); - ParseErrorOr parse_if(Tree then_branch); - ParseErrorOr parse_else(Tree else_branch); + TextParseErrorOr parse_record_direct_list_initialization(); + TextParseErrorOr parse_expression(); + TextParseErrorOr parse_condition(); + TextParseErrorOr parse_return_statement(); + TextParseErrorOr parse_assert(); + TextParseErrorOr parse_assignment(); + TextParseErrorOr parse_simple_step_or_inline_if_branch(); + TextParseErrorOr parse_if_beginning(); + TextParseErrorOr parse_inline_if_else(); + TextParseErrorOr parse_if(Tree then_branch); + TextParseErrorOr parse_else(Tree else_branch); + SpecificationParsingContext& m_ctx; Vector const& m_tokens; size_t m_next_token_index = 0; XML::Node const* m_node; + + size_t m_max_parsed_tokens = 0; + Vector, 8> m_suitable_continuations; }; } diff --git a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/Token.h b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/Token.h index e968079b4c..2f11167039 100644 --- a/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/Token.h +++ b/Meta/Lagom/Tools/CodeGenerators/JSSpecCompiler/Parser/Token.h @@ -20,38 +20,38 @@ constexpr i32 closing_bracket_precedence = 18; // NOTE: Operator precedence is generally the same as in // https://en.cppreference.com/w/cpp/language/operator_precedence (common sense applies). -#define ENUMERATE_TOKENS(F) \ - F(Invalid, -1, Invalid, Invalid, Invalid) \ - F(SectionNumber, -1, Invalid, Invalid, Invalid) \ - F(Identifier, -1, Invalid, Invalid, Invalid) \ - F(Number, -1, Invalid, Invalid, Invalid) \ - F(String, -1, Invalid, Invalid, Invalid) \ - F(Undefined, -1, Invalid, Invalid, Invalid) \ - F(Word, -1, Invalid, Invalid, Invalid) \ - F(ParenOpen, -1, Invalid, Invalid, ParenClose) \ - F(ParenClose, 18, Invalid, Invalid, ParenOpen) \ - F(BraceOpen, -1, Invalid, Invalid, BraceClose) \ - F(BraceClose, 18, Invalid, Invalid, BraceOpen) \ - F(Comma, 17, Invalid, Comma, Invalid) \ - F(MemberAccess, 2, Invalid, MemberAccess, Invalid) \ - F(Dot, -1, Invalid, Invalid, Invalid) \ - F(Colon, -1, Invalid, Invalid, Invalid) \ - F(Less, 9, Invalid, CompareLess, Invalid) \ - F(Greater, 9, Invalid, CompareGreater, Invalid) \ - F(NotEquals, 10, Invalid, CompareNotEqual, Invalid) \ - F(Equals, 10, Invalid, CompareEqual, Invalid) \ - F(Plus, 6, Invalid, Plus, Invalid) \ - F(AmbiguousMinus, -2, Invalid, Invalid, Invalid) \ - F(UnaryMinus, 3, Minus, Invalid, Invalid) \ - F(BinaryMinus, 6, Invalid, Minus, Invalid) \ - F(Multiplication, 5, Invalid, Multiplication, Invalid) \ - F(Division, 5, Invalid, Division, Invalid) \ - F(FunctionCall, 2, Invalid, FunctionCall, Invalid) \ - F(ExclamationMark, 3, AssertCompletion, Invalid, Invalid) \ - F(Is, -1, Invalid, Invalid, Invalid) +#define ENUMERATE_TOKENS(F) \ + F(Invalid, -1, Invalid, Invalid, Invalid, "") \ + F(SectionNumber, -1, Invalid, Invalid, Invalid, "section number") \ + F(Identifier, -1, Invalid, Invalid, Invalid, "identifier") \ + F(Number, -1, Invalid, Invalid, Invalid, "number") \ + F(String, -1, Invalid, Invalid, Invalid, "string literal") \ + F(Undefined, -1, Invalid, Invalid, Invalid, "constant") \ + F(Word, -1, Invalid, Invalid, Invalid, "word") \ + F(ParenOpen, -1, Invalid, Invalid, ParenClose, "'('") \ + F(ParenClose, 18, Invalid, Invalid, ParenOpen, "')'") \ + F(BraceOpen, -1, Invalid, Invalid, BraceClose, "'{'") \ + F(BraceClose, 18, Invalid, Invalid, BraceOpen, "'}'") \ + F(Comma, 17, Invalid, Comma, Invalid, "','") \ + F(MemberAccess, 2, Invalid, MemberAccess, Invalid, "member access operator '.'") \ + F(Dot, -1, Invalid, Invalid, Invalid, "punctuation mark '.'") \ + F(Colon, -1, Invalid, Invalid, Invalid, "':'") \ + F(Less, 9, Invalid, CompareLess, Invalid, "less than") \ + F(Greater, 9, Invalid, CompareGreater, Invalid, "greater than") \ + F(NotEquals, 10, Invalid, CompareNotEqual, Invalid, "not equals") \ + F(Equals, 10, Invalid, CompareEqual, Invalid, "equals") \ + F(Plus, 6, Invalid, Plus, Invalid, "plus") \ + F(AmbiguousMinus, -2, Invalid, Invalid, Invalid, "minus") \ + F(UnaryMinus, 3, Minus, Invalid, Invalid, "unary minus") \ + F(BinaryMinus, 6, Invalid, Minus, Invalid, "binary minus") \ + F(Multiplication, 5, Invalid, Multiplication, Invalid, "multiplication") \ + F(Division, 5, Invalid, Division, Invalid, "division") \ + F(FunctionCall, 2, Invalid, FunctionCall, Invalid, "function call token") \ + F(ExclamationMark, 3, AssertCompletion, Invalid, Invalid, "exclamation mark") \ + F(Is, -1, Invalid, Invalid, Invalid, "operator is") enum class TokenType { -#define ID(name, precedence, unary_name, binary_name, matching_bracket) name, +#define ID(name, precedence, unary_name, binary_name, matching_bracket, name_for_diagnostic) name, ENUMERATE_TOKENS(ID) #undef ID }; @@ -62,14 +62,16 @@ constexpr struct TokenInfo { UnaryOperator as_unary_operator; BinaryOperator as_binary_operator; TokenType matching_bracket; + StringView name_for_diagnostic; } token_info[] = { -#define TOKEN_INFO(name, precedence, unary_name, binary_name, matching_bracket) \ - { \ - #name##sv, \ - precedence, \ - UnaryOperator::unary_name, \ - BinaryOperator::binary_name, \ - TokenType::matching_bracket, \ +#define TOKEN_INFO(name, precedence, unary_name, binary_name, matching_bracket, name_for_diagnostic) \ + { \ + #name##sv, \ + precedence, \ + UnaryOperator::unary_name, \ + BinaryOperator::binary_name, \ + TokenType::matching_bracket, \ + name_for_diagnostic##sv \ }, ENUMERATE_TOKENS(TOKEN_INFO) #undef TOKEN_INFO @@ -78,7 +80,8 @@ constexpr struct TokenInfo { struct Token { TokenInfo const& info() const { return token_info[to_underlying(type)]; } - StringView name() const { return token_info[to_underlying(type)].name; } + StringView name() const { return info().name; } + StringView name_for_diagnostic() const { return info().name_for_diagnostic; } i32 precedence() const { return info().precedence; } bool is_operator() const { return precedence() > 0 && precedence() < closing_bracket_precedence; } bool is_ambiguous_operator() const { return precedence() == ambiguous_operator_precedence; }