RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-10-31 17:52:45 +00:00
Code Issues Activity Actions

LibJS: Add message string to Token

Browse source 
This allows us to communicate details about invalid tokens to the parser
without having to invent a bunch of specific invalid tokens like
TokenType::InvalidNumericLiteral.
This commit is contained in:
Linus Groh 2020-10-26 20:08:01 +00:00 • committed by Andreas Kling
parent 2ac734b7a8
commit 03c1d43f6e
2 changed files with 10 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Libraries/LibJS/Lexer.cpp
View file

@ -42,7 +42,7 @@ HashMap<char, TokenType> Lexer::s_single_char_tokens;
Lexer::Lexer(StringView source) Lexer::Lexer(StringView source)
: m_source(source) : m_source(source)
, m_current_token(TokenType::Eof, StringView(nullptr), StringView(nullptr), 0, 0) , m_current_token(TokenType::Eof, {}, StringView(nullptr), StringView(nullptr), 0, 0)
{ {
if (s_keywords.is_empty()) { if (s_keywords.is_empty()) {
s_keywords.set("await", TokenType::Await); s_keywords.set("await", TokenType::Await);
@ -392,6 +392,10 @@ Token Lexer::next()
size_t value_start_line_number = m_line_number; size_t value_start_line_number = m_line_number;
size_t value_start_column_number = m_line_column; size_t value_start_column_number = m_line_column;
auto token_type = TokenType::Invalid; auto token_type = TokenType::Invalid;
// This is being used to communicate info about invalid tokens to the parser, which then
// can turn that into more specific error messages - instead of us having to make up a
// bunch of Invalid* tokens (bad numeric literals, unterminated comments etc.)
String token_message;
if (m_current_token.type() == TokenType::RegexLiteral && !is_eof() && isalpha(m_current_char)) { if (m_current_token.type() == TokenType::RegexLiteral && !is_eof() && isalpha(m_current_char)) {
token_type = TokenType::RegexFlags; token_type = TokenType::RegexFlags;
@ -602,6 +606,7 @@ Token Lexer::next()
m_current_token = Token( m_current_token = Token(
token_type, token_type,
token_message,
m_source.substring_view(trivia_start - 1, value_start - trivia_start), m_source.substring_view(trivia_start - 1, value_start - trivia_start),
m_source.substring_view(value_start - 1, m_position - value_start), m_source.substring_view(value_start - 1, m_position - value_start),
value_start_line_number, value_start_line_number,

5
Libraries/LibJS/Token.h
View file

@ -181,8 +181,9 @@ enum class TokenCategory {
class Token { class Token {
public: public:
Token(TokenType type, StringView trivia, StringView value, size_t line_number, size_t line_column) Token(TokenType type, String message, StringView trivia, StringView value, size_t line_number, size_t line_column)
: m_type(type) : m_type(type)
, m_message(message)
, m_trivia(trivia) , m_trivia(trivia)
, m_value(value) , m_value(value)
, m_line_number(line_number) , m_line_number(line_number)
@ -196,6 +197,7 @@ public:
const char* name() const; const char* name() const;
static const char* name(TokenType); static const char* name(TokenType);
const String& message() const { return m_message; }
const StringView& trivia() const { return m_trivia; } const StringView& trivia() const { return m_trivia; }
const StringView& value() const { return m_value; } const StringView& value() const { return m_value; }
size_t line_number() const { return m_line_number; } size_t line_number() const { return m_line_number; }
@ -217,6 +219,7 @@ public:
private: private:
TokenType m_type; TokenType m_type;
String m_message;
StringView m_trivia; StringView m_trivia;
StringView m_value; StringView m_value;
size_t m_line_number; size_t m_line_number;