RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-24 22:17:42 +00:00
Code Issues Activity Actions

LibWeb: Record position information in CSS Tokens

Browse source 
This is a requirement to be able to use the Tokens for syntax
highlighting.
This commit is contained in:
Sam Atkins 2021-10-21 21:25:14 +01:00 • committed by Andreas Kling
parent 9a2eecaca4
commit ecf5368535
3 changed files with 30 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

18
Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp
View file

@ -241,7 +241,10 @@ Vector<Token> Tokenizer::parse()
{
Vector<Token> tokens;
for (;;) {
auto token_start = m_position;
auto token = consume_a_token();
token.m_start_position = token_start;
token.m_end_position = m_position;
tokens.append(token);
if (token.is(Token::Type::EndOfFile)) {
@ -256,8 +259,18 @@ u32 Tokenizer::next_code_point()
return TOKENIZER_EOF;
m_prev_utf8_iterator = m_utf8_iterator;
++m_utf8_iterator;
dbgln_if(CSS_TOKENIZER_DEBUG, "(Tokenizer) Next code_point: {:d}", *m_prev_utf8_iterator);
return *m_prev_utf8_iterator;
auto code_point = *m_prev_utf8_iterator;
m_prev_position = m_position;
if (is_newline(code_point)) {
m_position.line++;
m_position.column = 0;
} else {
m_position.column++;
}
dbgln_if(CSS_TOKENIZER_DEBUG, "(Tokenizer) Next code_point: {:d}", code_point);
return code_point;
}
u32 Tokenizer::peek_code_point(size_t offset) const
@ -579,6 +592,7 @@ void Tokenizer::consume_as_much_whitespace_as_possible()
void Tokenizer::reconsume_current_input_code_point()
{
m_utf8_iterator = m_prev_utf8_iterator;
m_position = m_prev_position;
}
// https://www.w3.org/TR/css-syntax-3/#consume-numeric-token