RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-05-22 16:15:08 +00:00
Code Issues Activity Actions

HackStudio: Make CppTokens have (line,column) positions

Browse source 
These are infinitely more useful than raw indices into the input text.
This commit is contained in:
Andreas Kling 2019-10-25 21:07:45 +02:00
parent 0d53d74d5f
commit 0604fcf9fd
2 changed files with 27 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

23
DevTools/HackStudio/CppLexer.cpp
View file

@ -17,7 +17,15 @@ char CppLexer::peek(int offset) const
char CppLexer::consume() char CppLexer::consume()
{ {
ASSERT(m_index < m_input.length()); ASSERT(m_index < m_input.length());
return m_input[m_index++]; char ch = m_input[m_index++];
m_previous_position = m_position;
if (ch == '\n') {
m_position.line++;
m_position.column = 0;
} else {
m_position.column++;
}
return ch;
} }
static bool is_valid_first_character_of_identifier(char ch) static bool is_valid_first_character_of_identifier(char ch)
@ -41,22 +49,27 @@ Vector<CppToken> CppLexer::lex()
{ {
Vector<CppToken> tokens; Vector<CppToken> tokens;
int token_start_index = 0;
CppPosition token_start_position;
auto emit_token = [&](auto type) { auto emit_token = [&](auto type) {
CppToken token; CppToken token;
token.m_type = type; token.m_type = type;
token.m_view = StringView(m_input.characters_without_null_termination() + m_index, 1); token.m_start = m_position;
token.m_end = m_position;
tokens.append(token); tokens.append(token);
m_index++; consume();
}; };
int token_start_index = 0;
auto begin_token = [&] { auto begin_token = [&] {
token_start_index = m_index; token_start_index = m_index;
token_start_position = m_position;
}; };
auto commit_token = [&](auto type) { auto commit_token = [&](auto type) {
CppToken token; CppToken token;
token.m_type = type; token.m_type = type;
token.m_view = StringView(m_input.characters_without_null_termination() + token_start_index, m_index - token_start_index); token.m_start = token_start_position;
token.m_end = m_previous_position;
tokens.append(token); tokens.append(token);
}; };

10
DevTools/HackStudio/CppLexer.h
View file

@ -23,6 +23,11 @@
__TOKEN(Keyword) \ __TOKEN(Keyword) \
__TOKEN(Identifier) __TOKEN(Identifier)
struct CppPosition {
int line { -1 };
int column { -1 };
};
struct CppToken { struct CppToken {
enum class Type { enum class Type {
#define __TOKEN(x) x, #define __TOKEN(x) x,
@ -43,7 +48,8 @@ struct CppToken {
} }
Type m_type { Type::Invalid }; Type m_type { Type::Invalid };
StringView m_view; CppPosition m_start;
CppPosition m_end;
}; };
class CppLexer { class CppLexer {
@ -58,4 +64,6 @@ private:
StringView m_input; StringView m_input;
int m_index { 0 }; int m_index { 0 };
CppPosition m_previous_position { 0, 0 };
CppPosition m_position { 0, 0 };
}; };