RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-05-20 16:45:07 +00:00
Code Issues Activity Actions

HackStudio: Make CppTokens have (line,column) positions

Browse source 
These are infinitely more useful than raw indices into the input text.
This commit is contained in:
Andreas Kling 2019-10-25 21:07:45 +02:00
parent 0d53d74d5f
commit 0604fcf9fd
2 changed files with 27 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

23
DevTools/HackStudio/CppLexer.cpp
View file

@ -17,7 +17,15 @@ char CppLexer::peek(int offset) const
char CppLexer::consume()
{
ASSERT(m_index < m_input.length());
return m_input[m_index++];
char ch = m_input[m_index++];
m_previous_position = m_position;
if (ch == '\n') {
m_position.line++;
m_position.column = 0;
} else {
m_position.column++;
}
return ch;
}
static bool is_valid_first_character_of_identifier(char ch)
@ -41,22 +49,27 @@ Vector<CppToken> CppLexer::lex()
{
Vector<CppToken> tokens;
int token_start_index = 0;
CppPosition token_start_position;
auto emit_token = [&](auto type) {
CppToken token;
token.m_type = type;
token.m_view = StringView(m_input.characters_without_null_termination() + m_index, 1);
token.m_start = m_position;
token.m_end = m_position;
tokens.append(token);
m_index++;
consume();
};
int token_start_index = 0;
auto begin_token = [&] {
token_start_index = m_index;
token_start_position = m_position;
};
auto commit_token = [&](auto type) {
CppToken token;
token.m_type = type;
token.m_view = StringView(m_input.characters_without_null_termination() + token_start_index, m_index - token_start_index);
token.m_start = token_start_position;
token.m_end = m_previous_position;
tokens.append(token);
};