1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-25 15:57:45 +00:00

LibWeb: Move Token and Tokenizer into Parser namespace

This commit is contained in:
Sam Atkins 2022-04-12 15:10:08 +01:00 committed by Andreas Kling
parent 7272997b1b
commit bf786d66b1
8 changed files with 52 additions and 51 deletions

View file

@ -12,7 +12,7 @@ namespace Web::CSS {
bool SyntaxHighlighter::is_identifier(u64 token) const
{
return static_cast<CSS::Token::Type>(token) == CSS::Token::Type::Ident;
return static_cast<CSS::Parser::Token::Type>(token) == CSS::Parser::Token::Type::Ident;
}
bool SyntaxHighlighter::is_navigatable(u64) const
@ -27,7 +27,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
Vector<GUI::TextDocumentSpan> spans;
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Token::Type type) {
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Parser::Token::Type type) {
if (start_line > end_line || (start_line == end_line && start_column >= end_column)) {
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(CSS::SyntaxHighlighter) discarding ({}-{}) to ({}-{}) because it has zero or negative length", start_line, start_column, end_line, end_column);
return;
@ -43,85 +43,85 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
false);
};
CSS::Tokenizer tokenizer { text, "utf-8" };
CSS::Parser::Tokenizer tokenizer { text, "utf-8" };
auto tokens = tokenizer.parse();
for (auto const& token : tokens) {
if (token.is(Token::Type::EndOfFile))
if (token.is(Parser::Token::Type::EndOfFile))
break;
switch (token.type()) {
case Token::Type::Ident:
case Parser::Token::Type::Ident:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_identifier(), {} }, token.type());
break;
case Token::Type::String:
case Parser::Token::Type::String:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_string(), {} }, token.type());
break;
case Token::Type::Whitespace:
case Parser::Token::Type::Whitespace:
// CSS doesn't produce comment tokens, they're just included as part of whitespace.
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
break;
case Token::Type::AtKeyword:
case Parser::Token::Type::AtKeyword:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_keyword(), {} }, token.type());
break;
case Token::Type::Function:
case Parser::Token::Type::Function:
// Function tokens include the opening '(', so we split that into two tokens for highlighting purposes.
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column - 1, { palette.syntax_keyword(), {} }, token.type());
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
break;
case Token::Type::Url:
case Parser::Token::Type::Url:
// An Url token is a `url()` function with its parameter string unquoted.
// url
highlight(token.start_position().line, token.start_position().column, token.start_position().line, token.start_position().column + 3, { palette.syntax_keyword(), {} }, token.type());
// (
highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
// <string>
highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Token::Type::String);
highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Parser::Token::Type::String);
// )
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::CloseParen);
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::CloseParen);
break;
case Token::Type::Number:
case Token::Type::Dimension:
case Token::Type::Percentage:
case Parser::Token::Type::Number:
case Parser::Token::Type::Dimension:
case Parser::Token::Type::Percentage:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
break;
case Token::Type::Delim:
case Token::Type::Colon:
case Token::Type::Comma:
case Token::Type::Semicolon:
case Token::Type::OpenCurly:
case Token::Type::OpenParen:
case Token::Type::OpenSquare:
case Token::Type::CloseCurly:
case Token::Type::CloseParen:
case Token::Type::CloseSquare:
case Parser::Token::Type::Delim:
case Parser::Token::Type::Colon:
case Parser::Token::Type::Comma:
case Parser::Token::Type::Semicolon:
case Parser::Token::Type::OpenCurly:
case Parser::Token::Type::OpenParen:
case Parser::Token::Type::OpenSquare:
case Parser::Token::Type::CloseCurly:
case Parser::Token::Type::CloseParen:
case Parser::Token::Type::CloseSquare:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, token.type());
break;
case Token::Type::CDO:
case Token::Type::CDC:
case Parser::Token::Type::CDO:
case Parser::Token::Type::CDC:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
break;
case Token::Type::Hash:
case Parser::Token::Type::Hash:
// FIXME: Hash tokens can be ID selectors or colors, we don't know which without parsing properly.
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
break;
case Token::Type::Invalid:
case Token::Type::BadUrl:
case Token::Type::BadString:
case Parser::Token::Type::Invalid:
case Parser::Token::Type::BadUrl:
case Parser::Token::Type::BadString:
// FIXME: Error highlighting color in palette?
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { Color(Color::NamedColor::Red), {}, false, true }, token.type());
break;
case Token::Type::EndOfFile:
case Parser::Token::Type::EndOfFile:
default:
break;
}
@ -144,10 +144,10 @@ Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token
{
static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenCurly), static_cast<u64>(CSS::Token::Type::CloseCurly) });
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenParen), static_cast<u64>(CSS::Token::Type::CloseParen) });
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenSquare), static_cast<u64>(CSS::Token::Type::CloseSquare) });
pairs.append({ static_cast<u64>(CSS::Token::Type::CDO), static_cast<u64>(CSS::Token::Type::CDC) });
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenCurly), static_cast<u64>(CSS::Parser::Token::Type::CloseCurly) });
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenParen), static_cast<u64>(CSS::Parser::Token::Type::CloseParen) });
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenSquare), static_cast<u64>(CSS::Parser::Token::Type::CloseSquare) });
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::CDO), static_cast<u64>(CSS::Parser::Token::Type::CDC) });
}
return pairs;
}