diff --git a/Libraries/LibJS/Lexer.cpp b/Libraries/LibJS/Lexer.cpp index d61e1a184e..070f80aaaf 100644 --- a/Libraries/LibJS/Lexer.cpp +++ b/Libraries/LibJS/Lexer.cpp @@ -42,7 +42,7 @@ HashMap Lexer::s_single_char_tokens; Lexer::Lexer(StringView source) : m_source(source) - , m_current_token(TokenType::Eof, StringView(nullptr), StringView(nullptr), 0, 0) + , m_current_token(TokenType::Eof, {}, StringView(nullptr), StringView(nullptr), 0, 0) { if (s_keywords.is_empty()) { s_keywords.set("await", TokenType::Await); @@ -392,6 +392,10 @@ Token Lexer::next() size_t value_start_line_number = m_line_number; size_t value_start_column_number = m_line_column; auto token_type = TokenType::Invalid; + // This is being used to communicate info about invalid tokens to the parser, which then + // can turn that into more specific error messages - instead of us having to make up a + // bunch of Invalid* tokens (bad numeric literals, unterminated comments etc.) + String token_message; if (m_current_token.type() == TokenType::RegexLiteral && !is_eof() && isalpha(m_current_char)) { token_type = TokenType::RegexFlags; @@ -602,6 +606,7 @@ Token Lexer::next() m_current_token = Token( token_type, + token_message, m_source.substring_view(trivia_start - 1, value_start - trivia_start), m_source.substring_view(value_start - 1, m_position - value_start), value_start_line_number, diff --git a/Libraries/LibJS/Token.h b/Libraries/LibJS/Token.h index b9d4bb0d60..a266cd32ce 100644 --- a/Libraries/LibJS/Token.h +++ b/Libraries/LibJS/Token.h @@ -181,8 +181,9 @@ enum class TokenCategory { class Token { public: - Token(TokenType type, StringView trivia, StringView value, size_t line_number, size_t line_column) + Token(TokenType type, String message, StringView trivia, StringView value, size_t line_number, size_t line_column) : m_type(type) + , m_message(message) , m_trivia(trivia) , m_value(value) , m_line_number(line_number) @@ -196,6 +197,7 @@ public: const char* name() const; static const char* name(TokenType); + const String& message() const { return m_message; } const StringView& trivia() const { return m_trivia; } const StringView& value() const { return m_value; } size_t line_number() const { return m_line_number; } @@ -217,6 +219,7 @@ public: private: TokenType m_type; + String m_message; StringView m_trivia; StringView m_value; size_t m_line_number;