mirror of
https://github.com/RGBCube/serenity
synced 2025-07-26 03:17:34 +00:00
LibWeb: Move Token and Tokenizer into Parser namespace
This commit is contained in:
Sam Atkins
Andreas Kling
parent
7272997b1b
commit
bf786d66b1
8 changed files with 52 additions and 51 deletions
|
|
@ -12,7 +12,7 @@ namespace Web::CSS {
|
|||
|
||||
bool SyntaxHighlighter::is_identifier(u64 token) const
|
||||
{
|
||||
return static_cast<CSS::Token::Type>(token) == CSS::Token::Type::Ident;
|
||||
return static_cast<CSS::Parser::Token::Type>(token) == CSS::Parser::Token::Type::Ident;
|
||||
}
|
||||
|
||||
bool SyntaxHighlighter::is_navigatable(u64) const
|
||||
|
|
@ -27,7 +27,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
|
|||
|
||||
Vector<GUI::TextDocumentSpan> spans;
|
||||
|
||||
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Token::Type type) {
|
||||
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Parser::Token::Type type) {
|
||||
if (start_line > end_line || (start_line == end_line && start_column >= end_column)) {
|
||||
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(CSS::SyntaxHighlighter) discarding ({}-{}) to ({}-{}) because it has zero or negative length", start_line, start_column, end_line, end_column);
|
||||
return;
|
||||
|
|
@ -43,85 +43,85 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
|
|||
false);
|
||||
};
|
||||
|
||||
CSS::Tokenizer tokenizer { text, "utf-8" };
|
||||
CSS::Parser::Tokenizer tokenizer { text, "utf-8" };
|
||||
auto tokens = tokenizer.parse();
|
||||
for (auto const& token : tokens) {
|
||||
if (token.is(Token::Type::EndOfFile))
|
||||
if (token.is(Parser::Token::Type::EndOfFile))
|
||||
break;
|
||||
|
||||
switch (token.type()) {
|
||||
case Token::Type::Ident:
|
||||
case Parser::Token::Type::Ident:
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_identifier(), {} }, token.type());
|
||||
break;
|
||||
|
||||
case Token::Type::String:
|
||||
case Parser::Token::Type::String:
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_string(), {} }, token.type());
|
||||
break;
|
||||
|
||||
case Token::Type::Whitespace:
|
||||
case Parser::Token::Type::Whitespace:
|
||||
// CSS doesn't produce comment tokens, they're just included as part of whitespace.
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
|
||||
break;
|
||||
|
||||
case Token::Type::AtKeyword:
|
||||
case Parser::Token::Type::AtKeyword:
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_keyword(), {} }, token.type());
|
||||
break;
|
||||
|
||||
case Token::Type::Function:
|
||||
case Parser::Token::Type::Function:
|
||||
// Function tokens include the opening '(', so we split that into two tokens for highlighting purposes.
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column - 1, { palette.syntax_keyword(), {} }, token.type());
|
||||
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
|
||||
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
|
||||
break;
|
||||
|
||||
case Token::Type::Url:
|
||||
case Parser::Token::Type::Url:
|
||||
// An Url token is a `url()` function with its parameter string unquoted.
|
||||
// url
|
||||
highlight(token.start_position().line, token.start_position().column, token.start_position().line, token.start_position().column + 3, { palette.syntax_keyword(), {} }, token.type());
|
||||
// (
|
||||
highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
|
||||
highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
|
||||
// <string>
|
||||
highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Token::Type::String);
|
||||
highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Parser::Token::Type::String);
|
||||
// )
|
||||
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::CloseParen);
|
||||
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::CloseParen);
|
||||
break;
|
||||
|
||||
case Token::Type::Number:
|
||||
case Token::Type::Dimension:
|
||||
case Token::Type::Percentage:
|
||||
case Parser::Token::Type::Number:
|
||||
case Parser::Token::Type::Dimension:
|
||||
case Parser::Token::Type::Percentage:
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
|
||||
break;
|
||||
|
||||
case Token::Type::Delim:
|
||||
case Token::Type::Colon:
|
||||
case Token::Type::Comma:
|
||||
case Token::Type::Semicolon:
|
||||
case Token::Type::OpenCurly:
|
||||
case Token::Type::OpenParen:
|
||||
case Token::Type::OpenSquare:
|
||||
case Token::Type::CloseCurly:
|
||||
case Token::Type::CloseParen:
|
||||
case Token::Type::CloseSquare:
|
||||
case Parser::Token::Type::Delim:
|
||||
case Parser::Token::Type::Colon:
|
||||
case Parser::Token::Type::Comma:
|
||||
case Parser::Token::Type::Semicolon:
|
||||
case Parser::Token::Type::OpenCurly:
|
||||
case Parser::Token::Type::OpenParen:
|
||||
case Parser::Token::Type::OpenSquare:
|
||||
case Parser::Token::Type::CloseCurly:
|
||||
case Parser::Token::Type::CloseParen:
|
||||
case Parser::Token::Type::CloseSquare:
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, token.type());
|
||||
break;
|
||||
|
||||
case Token::Type::CDO:
|
||||
case Token::Type::CDC:
|
||||
case Parser::Token::Type::CDO:
|
||||
case Parser::Token::Type::CDC:
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
|
||||
break;
|
||||
|
||||
case Token::Type::Hash:
|
||||
case Parser::Token::Type::Hash:
|
||||
// FIXME: Hash tokens can be ID selectors or colors, we don't know which without parsing properly.
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
|
||||
break;
|
||||
|
||||
case Token::Type::Invalid:
|
||||
case Token::Type::BadUrl:
|
||||
case Token::Type::BadString:
|
||||
case Parser::Token::Type::Invalid:
|
||||
case Parser::Token::Type::BadUrl:
|
||||
case Parser::Token::Type::BadString:
|
||||
// FIXME: Error highlighting color in palette?
|
||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { Color(Color::NamedColor::Red), {}, false, true }, token.type());
|
||||
break;
|
||||
|
||||
case Token::Type::EndOfFile:
|
||||
case Parser::Token::Type::EndOfFile:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -144,10 +144,10 @@ Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token
|
|||
{
|
||||
static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
|
||||
if (pairs.is_empty()) {
|
||||
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenCurly), static_cast<u64>(CSS::Token::Type::CloseCurly) });
|
||||
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenParen), static_cast<u64>(CSS::Token::Type::CloseParen) });
|
||||
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenSquare), static_cast<u64>(CSS::Token::Type::CloseSquare) });
|
||||
pairs.append({ static_cast<u64>(CSS::Token::Type::CDO), static_cast<u64>(CSS::Token::Type::CDC) });
|
||||
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenCurly), static_cast<u64>(CSS::Parser::Token::Type::CloseCurly) });
|
||||
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenParen), static_cast<u64>(CSS::Parser::Token::Type::CloseParen) });
|
||||
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenSquare), static_cast<u64>(CSS::Parser::Token::Type::CloseSquare) });
|
||||
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::CDO), static_cast<u64>(CSS::Parser::Token::Type::CDC) });
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue