RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-25 22:07:35 +00:00
Code Issues Activity Actions

LibRegex: Convert regex::Lexer to inherit from GenericLexer

Browse source 
This will allow regex::Lexer users to invoke GenericLexer consumption
methods, such as GenericLexer::consume_escaped_codepoint().

This also allows for de-duplicating common methods between the lexers.
This commit is contained in:
Timothy Flynn 2021-08-18 14:10:08 -04:00 • committed by Andreas Kling
parent dd44a5e948
commit 5ff9596678
3 changed files with 28 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

68
Userland/Libraries/LibRegex/RegexLexer.cpp
View file

@ -31,86 +31,58 @@ char const* Token::name() const
return name(m_type);
}
Lexer::Lexer(StringView const source)
: m_source(source)
Lexer::Lexer()
: GenericLexer(StringView { nullptr })
{
}
ALWAYS_INLINE int Lexer::peek(size_t offset) const
Lexer::Lexer(StringView const source)
: GenericLexer(source)
{
if ((m_position + offset) >= m_source.length())
return EOF;
return (unsigned char)m_source[m_position + offset];
}
void Lexer::back(size_t offset)
{
if (offset == m_position + 1)
offset = m_position; // 'position == 0' occurs twice.
if (offset == m_index + 1)
offset = m_index; // 'position == 0' occurs twice.
VERIFY(offset <= m_position);
VERIFY(offset <= m_index);
if (!offset)
return;
m_position -= offset;
m_previous_position = (m_position > 0) ? m_position - 1 : 0;
m_current_char = m_source[m_position];
m_index -= offset;
m_previous_position = (m_index > 0) ? m_index - 1 : 0;
}
ALWAYS_INLINE void Lexer::consume()
char Lexer::consume()
{
m_previous_position = m_position;
if (m_position >= m_source.length()) {
m_position = m_source.length() + 1;
m_current_char = EOF;
return;
}
m_current_char = m_source[m_position++];
m_previous_position = m_index;
return GenericLexer::consume();
}
void Lexer::reset()
{
m_position = 0;
m_index = 0;
m_current_token = { TokenType::Eof, 0, StringView(nullptr) };
m_current_char = 0;
m_previous_position = 0;
}
bool Lexer::try_skip(char c)
{
if (peek() != c)
return false;
consume();
return true;
}
char Lexer::skip()
{
auto c = peek();
consume();
VERIFY(c != EOF);
return c;
}
Token Lexer::next()
{
size_t token_start_position;
auto begin_token = [&] {
token_start_position = m_position;
token_start_position = m_index;
};
auto commit_token = [&](auto type) -> Token& {
VERIFY(token_start_position + m_previous_position - token_start_position + 1 <= m_source.length());
auto substring = m_source.substring_view(token_start_position, m_previous_position - token_start_position + 1);
VERIFY(token_start_position + m_previous_position - token_start_position + 1 <= m_input.length());
auto substring = m_input.substring_view(token_start_position, m_previous_position - token_start_position + 1);
m_current_token = Token(type, token_start_position, substring);
return m_current_token;
};
auto emit_token = [&](auto type) -> Token& {
m_current_token = Token(type, m_position, m_source.substring_view(m_position, 1));
m_current_token = Token(type, m_index, m_input.substring_view(m_index, 1));
consume();
return m_current_token;
};
@ -137,7 +109,7 @@ Token Lexer::next()
}
};
while (m_position <= m_source.length()) {
while (m_index <= m_input.length()) {
auto ch = peek();
if (ch == '(')
return emit_token(TokenType::LeftParen);
@ -203,13 +175,13 @@ Token Lexer::next()
}
}
if (ch == EOF)
if (ch == '\0')
break;
return emit_token(TokenType::Char);
}
return Token(TokenType::Eof, m_position, nullptr);
return Token(TokenType::Eof, m_index, nullptr);
}
}