mirror of
https://github.com/RGBCube/serenity
synced 2025-07-24 13:57:35 +00:00
LibWeb: Move Token and Tokenizer into Parser namespace
This commit is contained in:
parent
7272997b1b
commit
bf786d66b1
8 changed files with 52 additions and 51 deletions
|
@ -7,14 +7,13 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <AK/Types.h>
|
#include <AK/Types.h>
|
||||||
|
#include <LibWeb/Forward.h>
|
||||||
#include <math.h>
|
#include <math.h>
|
||||||
|
|
||||||
namespace Web::CSS {
|
namespace Web::CSS {
|
||||||
|
|
||||||
class Tokenizer;
|
|
||||||
|
|
||||||
class Number {
|
class Number {
|
||||||
friend class Tokenizer;
|
friend class Parser::Tokenizer;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
enum class Type {
|
enum class Type {
|
||||||
|
|
|
@ -9,7 +9,7 @@
|
||||||
#include <LibWeb/CSS/Parser/Token.h>
|
#include <LibWeb/CSS/Parser/Token.h>
|
||||||
#include <LibWeb/CSS/Serialize.h>
|
#include <LibWeb/CSS/Serialize.h>
|
||||||
|
|
||||||
namespace Web::CSS {
|
namespace Web::CSS::Parser {
|
||||||
|
|
||||||
String Token::to_string() const
|
String Token::to_string() const
|
||||||
{
|
{
|
||||||
|
|
|
@ -12,7 +12,7 @@
|
||||||
#include <AK/Utf8View.h>
|
#include <AK/Utf8View.h>
|
||||||
#include <LibWeb/CSS/Number.h>
|
#include <LibWeb/CSS/Number.h>
|
||||||
|
|
||||||
namespace Web::CSS {
|
namespace Web::CSS::Parser {
|
||||||
|
|
||||||
class Token {
|
class Token {
|
||||||
friend class Tokenizer;
|
friend class Tokenizer;
|
||||||
|
|
|
@ -13,6 +13,8 @@
|
||||||
#include <LibWeb/CSS/Parser/Tokenizer.h>
|
#include <LibWeb/CSS/Parser/Tokenizer.h>
|
||||||
#include <math.h>
|
#include <math.h>
|
||||||
|
|
||||||
|
namespace Web::CSS::Parser {
|
||||||
|
|
||||||
// U+FFFD REPLACEMENT CHARACTER (<28>)
|
// U+FFFD REPLACEMENT CHARACTER (<28>)
|
||||||
#define REPLACEMENT_CHARACTER 0xFFFD
|
#define REPLACEMENT_CHARACTER 0xFFFD
|
||||||
static constexpr u32 TOKENIZER_EOF = 0xFFFFFFFF;
|
static constexpr u32 TOKENIZER_EOF = 0xFFFFFFFF;
|
||||||
|
@ -190,8 +192,6 @@ static inline bool is_E(u32 code_point)
|
||||||
return code_point == 0x45;
|
return code_point == 0x45;
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace Web::CSS {
|
|
||||||
|
|
||||||
Tokenizer::Tokenizer(StringView input, String const& encoding)
|
Tokenizer::Tokenizer(StringView input, String const& encoding)
|
||||||
{
|
{
|
||||||
auto* decoder = TextCodec::decoder_for(encoding);
|
auto* decoder = TextCodec::decoder_for(encoding);
|
||||||
|
|
|
@ -14,7 +14,7 @@
|
||||||
#include <LibWeb/CSS/Parser/Token.h>
|
#include <LibWeb/CSS/Parser/Token.h>
|
||||||
#include <LibWeb/Forward.h>
|
#include <LibWeb/Forward.h>
|
||||||
|
|
||||||
namespace Web::CSS {
|
namespace Web::CSS::Parser {
|
||||||
|
|
||||||
class U32Twin {
|
class U32Twin {
|
||||||
public:
|
public:
|
||||||
|
|
|
@ -557,7 +557,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
auto const& custom_property_name_token = var_contents.first();
|
auto const& custom_property_name_token = var_contents.first();
|
||||||
if (!custom_property_name_token.is(Token::Type::Ident))
|
if (!custom_property_name_token.is(Parser::Token::Type::Ident))
|
||||||
return false;
|
return false;
|
||||||
auto custom_property_name = custom_property_name_token.token().ident();
|
auto custom_property_name = custom_property_name_token.token().ident();
|
||||||
if (!custom_property_name.starts_with("--"))
|
if (!custom_property_name.starts_with("--"))
|
||||||
|
@ -582,7 +582,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use the provided fallback value, if any.
|
// Use the provided fallback value, if any.
|
||||||
if (var_contents.size() > 2 && var_contents[1].is(Token::Type::Comma)) {
|
if (var_contents.size() > 2 && var_contents[1].is(Parser::Token::Type::Comma)) {
|
||||||
if (!expand_unresolved_values(element, property_name, dependencies, var_contents, dest, 2))
|
if (!expand_unresolved_values(element, property_name, dependencies, var_contents, dest, 2))
|
||||||
return false;
|
return false;
|
||||||
continue;
|
continue;
|
||||||
|
@ -594,7 +594,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
auto const& attr_name_token = attr_contents.first();
|
auto const& attr_name_token = attr_contents.first();
|
||||||
if (!attr_name_token.is(Token::Type::Ident))
|
if (!attr_name_token.is(Parser::Token::Type::Ident))
|
||||||
return false;
|
return false;
|
||||||
auto attr_name = attr_name_token.token().ident();
|
auto attr_name = attr_name_token.token().ident();
|
||||||
|
|
||||||
|
@ -602,13 +602,13 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
|
||||||
// 1. If the attr() function has a substitution value, replace the attr() function by the substitution value.
|
// 1. If the attr() function has a substitution value, replace the attr() function by the substitution value.
|
||||||
if (!attr_value.is_null()) {
|
if (!attr_value.is_null()) {
|
||||||
// FIXME: attr() should also accept an optional type argument, not just strings.
|
// FIXME: attr() should also accept an optional type argument, not just strings.
|
||||||
dest.empend(Token::of_string(attr_value));
|
dest.empend(Parser::Token::of_string(attr_value));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Otherwise, if the attr() function has a fallback value as its last argument, replace the attr() function by the fallback value.
|
// 2. Otherwise, if the attr() function has a fallback value as its last argument, replace the attr() function by the fallback value.
|
||||||
// If there are any var() or attr() references in the fallback, substitute them as well.
|
// If there are any var() or attr() references in the fallback, substitute them as well.
|
||||||
if (attr_contents.size() > 2 && attr_contents[1].is(Token::Type::Comma)) {
|
if (attr_contents.size() > 2 && attr_contents[1].is(Parser::Token::Type::Comma)) {
|
||||||
if (!expand_unresolved_values(element, property_name, dependencies, attr_contents, dest, 2))
|
if (!expand_unresolved_values(element, property_name, dependencies, attr_contents, dest, 2))
|
||||||
return false;
|
return false;
|
||||||
continue;
|
continue;
|
||||||
|
|
|
@ -12,7 +12,7 @@ namespace Web::CSS {
|
||||||
|
|
||||||
bool SyntaxHighlighter::is_identifier(u64 token) const
|
bool SyntaxHighlighter::is_identifier(u64 token) const
|
||||||
{
|
{
|
||||||
return static_cast<CSS::Token::Type>(token) == CSS::Token::Type::Ident;
|
return static_cast<CSS::Parser::Token::Type>(token) == CSS::Parser::Token::Type::Ident;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool SyntaxHighlighter::is_navigatable(u64) const
|
bool SyntaxHighlighter::is_navigatable(u64) const
|
||||||
|
@ -27,7 +27,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
|
||||||
|
|
||||||
Vector<GUI::TextDocumentSpan> spans;
|
Vector<GUI::TextDocumentSpan> spans;
|
||||||
|
|
||||||
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Token::Type type) {
|
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Parser::Token::Type type) {
|
||||||
if (start_line > end_line || (start_line == end_line && start_column >= end_column)) {
|
if (start_line > end_line || (start_line == end_line && start_column >= end_column)) {
|
||||||
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(CSS::SyntaxHighlighter) discarding ({}-{}) to ({}-{}) because it has zero or negative length", start_line, start_column, end_line, end_column);
|
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(CSS::SyntaxHighlighter) discarding ({}-{}) to ({}-{}) because it has zero or negative length", start_line, start_column, end_line, end_column);
|
||||||
return;
|
return;
|
||||||
|
@ -43,85 +43,85 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
|
||||||
false);
|
false);
|
||||||
};
|
};
|
||||||
|
|
||||||
CSS::Tokenizer tokenizer { text, "utf-8" };
|
CSS::Parser::Tokenizer tokenizer { text, "utf-8" };
|
||||||
auto tokens = tokenizer.parse();
|
auto tokens = tokenizer.parse();
|
||||||
for (auto const& token : tokens) {
|
for (auto const& token : tokens) {
|
||||||
if (token.is(Token::Type::EndOfFile))
|
if (token.is(Parser::Token::Type::EndOfFile))
|
||||||
break;
|
break;
|
||||||
|
|
||||||
switch (token.type()) {
|
switch (token.type()) {
|
||||||
case Token::Type::Ident:
|
case Parser::Token::Type::Ident:
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_identifier(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_identifier(), {} }, token.type());
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::String:
|
case Parser::Token::Type::String:
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_string(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_string(), {} }, token.type());
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::Whitespace:
|
case Parser::Token::Type::Whitespace:
|
||||||
// CSS doesn't produce comment tokens, they're just included as part of whitespace.
|
// CSS doesn't produce comment tokens, they're just included as part of whitespace.
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::AtKeyword:
|
case Parser::Token::Type::AtKeyword:
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_keyword(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_keyword(), {} }, token.type());
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::Function:
|
case Parser::Token::Type::Function:
|
||||||
// Function tokens include the opening '(', so we split that into two tokens for highlighting purposes.
|
// Function tokens include the opening '(', so we split that into two tokens for highlighting purposes.
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column - 1, { palette.syntax_keyword(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column - 1, { palette.syntax_keyword(), {} }, token.type());
|
||||||
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
|
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::Url:
|
case Parser::Token::Type::Url:
|
||||||
// An Url token is a `url()` function with its parameter string unquoted.
|
// An Url token is a `url()` function with its parameter string unquoted.
|
||||||
// url
|
// url
|
||||||
highlight(token.start_position().line, token.start_position().column, token.start_position().line, token.start_position().column + 3, { palette.syntax_keyword(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.start_position().line, token.start_position().column + 3, { palette.syntax_keyword(), {} }, token.type());
|
||||||
// (
|
// (
|
||||||
highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
|
highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
|
||||||
// <string>
|
// <string>
|
||||||
highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Token::Type::String);
|
highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Parser::Token::Type::String);
|
||||||
// )
|
// )
|
||||||
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::CloseParen);
|
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::CloseParen);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::Number:
|
case Parser::Token::Type::Number:
|
||||||
case Token::Type::Dimension:
|
case Parser::Token::Type::Dimension:
|
||||||
case Token::Type::Percentage:
|
case Parser::Token::Type::Percentage:
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::Delim:
|
case Parser::Token::Type::Delim:
|
||||||
case Token::Type::Colon:
|
case Parser::Token::Type::Colon:
|
||||||
case Token::Type::Comma:
|
case Parser::Token::Type::Comma:
|
||||||
case Token::Type::Semicolon:
|
case Parser::Token::Type::Semicolon:
|
||||||
case Token::Type::OpenCurly:
|
case Parser::Token::Type::OpenCurly:
|
||||||
case Token::Type::OpenParen:
|
case Parser::Token::Type::OpenParen:
|
||||||
case Token::Type::OpenSquare:
|
case Parser::Token::Type::OpenSquare:
|
||||||
case Token::Type::CloseCurly:
|
case Parser::Token::Type::CloseCurly:
|
||||||
case Token::Type::CloseParen:
|
case Parser::Token::Type::CloseParen:
|
||||||
case Token::Type::CloseSquare:
|
case Parser::Token::Type::CloseSquare:
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, token.type());
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::CDO:
|
case Parser::Token::Type::CDO:
|
||||||
case Token::Type::CDC:
|
case Parser::Token::Type::CDC:
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::Hash:
|
case Parser::Token::Type::Hash:
|
||||||
// FIXME: Hash tokens can be ID selectors or colors, we don't know which without parsing properly.
|
// FIXME: Hash tokens can be ID selectors or colors, we don't know which without parsing properly.
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::Invalid:
|
case Parser::Token::Type::Invalid:
|
||||||
case Token::Type::BadUrl:
|
case Parser::Token::Type::BadUrl:
|
||||||
case Token::Type::BadString:
|
case Parser::Token::Type::BadString:
|
||||||
// FIXME: Error highlighting color in palette?
|
// FIXME: Error highlighting color in palette?
|
||||||
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { Color(Color::NamedColor::Red), {}, false, true }, token.type());
|
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { Color(Color::NamedColor::Red), {}, false, true }, token.type());
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Token::Type::EndOfFile:
|
case Parser::Token::Type::EndOfFile:
|
||||||
default:
|
default:
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -144,10 +144,10 @@ Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token
|
||||||
{
|
{
|
||||||
static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
|
static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
|
||||||
if (pairs.is_empty()) {
|
if (pairs.is_empty()) {
|
||||||
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenCurly), static_cast<u64>(CSS::Token::Type::CloseCurly) });
|
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenCurly), static_cast<u64>(CSS::Parser::Token::Type::CloseCurly) });
|
||||||
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenParen), static_cast<u64>(CSS::Token::Type::CloseParen) });
|
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenParen), static_cast<u64>(CSS::Parser::Token::Type::CloseParen) });
|
||||||
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenSquare), static_cast<u64>(CSS::Token::Type::CloseSquare) });
|
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenSquare), static_cast<u64>(CSS::Parser::Token::Type::CloseSquare) });
|
||||||
pairs.append({ static_cast<u64>(CSS::Token::Type::CDO), static_cast<u64>(CSS::Token::Type::CDC) });
|
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::CDO), static_cast<u64>(CSS::Parser::Token::Type::CDC) });
|
||||||
}
|
}
|
||||||
return pairs;
|
return pairs;
|
||||||
}
|
}
|
||||||
|
|
|
@ -106,6 +106,8 @@ class DeclarationOrAtRule;
|
||||||
class Function;
|
class Function;
|
||||||
class Parser;
|
class Parser;
|
||||||
class StyleRule;
|
class StyleRule;
|
||||||
|
class Token;
|
||||||
|
class Tokenizer;
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace Web::DOM {
|
namespace Web::DOM {
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue