RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-24 13:57:35 +00:00
Code Issues Activity Actions

LibWeb: Move Token and Tokenizer into Parser namespace

Browse source
This commit is contained in:
Sam Atkins 2022-04-12 15:10:08 +01:00 • committed by Andreas Kling
parent 7272997b1b
commit bf786d66b1
8 changed files with 52 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Userland/Libraries/LibWeb/CSS/Number.h
View file

@ -7,14 +7,13 @@
#pragma once #pragma once
#include <AK/Types.h> #include <AK/Types.h>
#include <LibWeb/Forward.h>
#include <math.h> #include <math.h>
namespace Web::CSS { namespace Web::CSS {
class Tokenizer;
class Number { class Number {
friend class Tokenizer; friend class Parser::Tokenizer;
public: public:
enum class Type { enum class Type {

2
Userland/Libraries/LibWeb/CSS/Parser/Token.cpp
View file

@ -9,7 +9,7 @@
#include <LibWeb/CSS/Parser/Token.h> #include <LibWeb/CSS/Parser/Token.h>
#include <LibWeb/CSS/Serialize.h> #include <LibWeb/CSS/Serialize.h>
namespace Web::CSS { namespace Web::CSS::Parser {
String Token::to_string() const String Token::to_string() const
{ {

2
Userland/Libraries/LibWeb/CSS/Parser/Token.h
View file

@ -12,7 +12,7 @@
#include <AK/Utf8View.h> #include <AK/Utf8View.h>
#include <LibWeb/CSS/Number.h> #include <LibWeb/CSS/Number.h>
namespace Web::CSS { namespace Web::CSS::Parser {
class Token { class Token {
friend class Tokenizer; friend class Tokenizer;

4
Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp
View file

@ -13,6 +13,8 @@
#include <LibWeb/CSS/Parser/Tokenizer.h> #include <LibWeb/CSS/Parser/Tokenizer.h>
#include <math.h> #include <math.h>
namespace Web::CSS::Parser {
// U+FFFD REPLACEMENT CHARACTER (<28>) // U+FFFD REPLACEMENT CHARACTER (<28>)
#define REPLACEMENT_CHARACTER 0xFFFD #define REPLACEMENT_CHARACTER 0xFFFD
static constexpr u32 TOKENIZER_EOF = 0xFFFFFFFF; static constexpr u32 TOKENIZER_EOF = 0xFFFFFFFF;
@ -190,8 +192,6 @@ static inline bool is_E(u32 code_point)
return code_point == 0x45; return code_point == 0x45;
} }
namespace Web::CSS {
Tokenizer::Tokenizer(StringView input, String const& encoding) Tokenizer::Tokenizer(StringView input, String const& encoding)
{ {
auto* decoder = TextCodec::decoder_for(encoding); auto* decoder = TextCodec::decoder_for(encoding);

2
Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.h
View file

@ -14,7 +14,7 @@
#include <LibWeb/CSS/Parser/Token.h> #include <LibWeb/CSS/Parser/Token.h>
#include <LibWeb/Forward.h> #include <LibWeb/Forward.h>
namespace Web::CSS { namespace Web::CSS::Parser {
class U32Twin { class U32Twin {
public: public:

10
Userland/Libraries/LibWeb/CSS/StyleComputer.cpp
View file

@ -557,7 +557,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
return false; return false;
auto const& custom_property_name_token = var_contents.first(); auto const& custom_property_name_token = var_contents.first();
if (!custom_property_name_token.is(Token::Type::Ident)) if (!custom_property_name_token.is(Parser::Token::Type::Ident))
return false; return false;
auto custom_property_name = custom_property_name_token.token().ident(); auto custom_property_name = custom_property_name_token.token().ident();
if (!custom_property_name.starts_with("--")) if (!custom_property_name.starts_with("--"))
@ -582,7 +582,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
} }
// Use the provided fallback value, if any. // Use the provided fallback value, if any.
if (var_contents.size() > 2 && var_contents[1].is(Token::Type::Comma)) { if (var_contents.size() > 2 && var_contents[1].is(Parser::Token::Type::Comma)) {
if (!expand_unresolved_values(element, property_name, dependencies, var_contents, dest, 2)) if (!expand_unresolved_values(element, property_name, dependencies, var_contents, dest, 2))
return false; return false;
continue; continue;
@ -594,7 +594,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
return false; return false;
auto const& attr_name_token = attr_contents.first(); auto const& attr_name_token = attr_contents.first();
if (!attr_name_token.is(Token::Type::Ident)) if (!attr_name_token.is(Parser::Token::Type::Ident))
return false; return false;
auto attr_name = attr_name_token.token().ident(); auto attr_name = attr_name_token.token().ident();
@ -602,13 +602,13 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
// 1. If the attr() function has a substitution value, replace the attr() function by the substitution value. // 1. If the attr() function has a substitution value, replace the attr() function by the substitution value.
if (!attr_value.is_null()) { if (!attr_value.is_null()) {
// FIXME: attr() should also accept an optional type argument, not just strings. // FIXME: attr() should also accept an optional type argument, not just strings.
dest.empend(Token::of_string(attr_value)); dest.empend(Parser::Token::of_string(attr_value));
continue; continue;
} }
// 2. Otherwise, if the attr() function has a fallback value as its last argument, replace the attr() function by the fallback value. // 2. Otherwise, if the attr() function has a fallback value as its last argument, replace the attr() function by the fallback value.
// If there are any var() or attr() references in the fallback, substitute them as well. // If there are any var() or attr() references in the fallback, substitute them as well.
if (attr_contents.size() > 2 && attr_contents[1].is(Token::Type::Comma)) { if (attr_contents.size() > 2 && attr_contents[1].is(Parser::Token::Type::Comma)) {
if (!expand_unresolved_values(element, property_name, dependencies, attr_contents, dest, 2)) if (!expand_unresolved_values(element, property_name, dependencies, attr_contents, dest, 2))
return false; return false;
continue; continue;

76
Userland/Libraries/LibWeb/CSS/SyntaxHighlighter/SyntaxHighlighter.cpp
View file

@ -12,7 +12,7 @@ namespace Web::CSS {
bool SyntaxHighlighter::is_identifier(u64 token) const bool SyntaxHighlighter::is_identifier(u64 token) const
{ {
return static_cast<CSS::Token::Type>(token) == CSS::Token::Type::Ident; return static_cast<CSS::Parser::Token::Type>(token) == CSS::Parser::Token::Type::Ident;
} }
bool SyntaxHighlighter::is_navigatable(u64) const bool SyntaxHighlighter::is_navigatable(u64) const
@ -27,7 +27,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
Vector<GUI::TextDocumentSpan> spans; Vector<GUI::TextDocumentSpan> spans;
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Token::Type type) { auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Parser::Token::Type type) {
if (start_line > end_line || (start_line == end_line && start_column >= end_column)) { if (start_line > end_line || (start_line == end_line && start_column >= end_column)) {
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(CSS::SyntaxHighlighter) discarding ({}-{}) to ({}-{}) because it has zero or negative length", start_line, start_column, end_line, end_column); dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(CSS::SyntaxHighlighter) discarding ({}-{}) to ({}-{}) because it has zero or negative length", start_line, start_column, end_line, end_column);
return; return;
@ -43,85 +43,85 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
false); false);
}; };
CSS::Tokenizer tokenizer { text, "utf-8" }; CSS::Parser::Tokenizer tokenizer { text, "utf-8" };
auto tokens = tokenizer.parse(); auto tokens = tokenizer.parse();
for (auto const& token : tokens) { for (auto const& token : tokens) {
if (token.is(Token::Type::EndOfFile)) if (token.is(Parser::Token::Type::EndOfFile))
break; break;
switch (token.type()) { switch (token.type()) {
case Token::Type::Ident: case Parser::Token::Type::Ident:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_identifier(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_identifier(), {} }, token.type());
break; break;
case Token::Type::String: case Parser::Token::Type::String:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_string(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_string(), {} }, token.type());
break; break;
case Token::Type::Whitespace: case Parser::Token::Type::Whitespace:
// CSS doesn't produce comment tokens, they're just included as part of whitespace. // CSS doesn't produce comment tokens, they're just included as part of whitespace.
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
break; break;
case Token::Type::AtKeyword: case Parser::Token::Type::AtKeyword:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_keyword(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_keyword(), {} }, token.type());
break; break;
case Token::Type::Function: case Parser::Token::Type::Function:
// Function tokens include the opening '(', so we split that into two tokens for highlighting purposes. // Function tokens include the opening '(', so we split that into two tokens for highlighting purposes.
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column - 1, { palette.syntax_keyword(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column - 1, { palette.syntax_keyword(), {} }, token.type());
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen); highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
break; break;
case Token::Type::Url: case Parser::Token::Type::Url:
// An Url token is a `url()` function with its parameter string unquoted. // An Url token is a `url()` function with its parameter string unquoted.
// url // url
highlight(token.start_position().line, token.start_position().column, token.start_position().line, token.start_position().column + 3, { palette.syntax_keyword(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.start_position().line, token.start_position().column + 3, { palette.syntax_keyword(), {} }, token.type());
// ( // (
highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen); highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
// <string> // <string>
highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Token::Type::String); highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Parser::Token::Type::String);
// ) // )
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::CloseParen); highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::CloseParen);
break; break;
case Token::Type::Number: case Parser::Token::Type::Number:
case Token::Type::Dimension: case Parser::Token::Type::Dimension:
case Token::Type::Percentage: case Parser::Token::Type::Percentage:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
break; break;
case Token::Type::Delim: case Parser::Token::Type::Delim:
case Token::Type::Colon: case Parser::Token::Type::Colon:
case Token::Type::Comma: case Parser::Token::Type::Comma:
case Token::Type::Semicolon: case Parser::Token::Type::Semicolon:
case Token::Type::OpenCurly: case Parser::Token::Type::OpenCurly:
case Token::Type::OpenParen: case Parser::Token::Type::OpenParen:
case Token::Type::OpenSquare: case Parser::Token::Type::OpenSquare:
case Token::Type::CloseCurly: case Parser::Token::Type::CloseCurly:
case Token::Type::CloseParen: case Parser::Token::Type::CloseParen:
case Token::Type::CloseSquare: case Parser::Token::Type::CloseSquare:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, token.type());
break; break;
case Token::Type::CDO: case Parser::Token::Type::CDO:
case Token::Type::CDC: case Parser::Token::Type::CDC:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
break; break;
case Token::Type::Hash: case Parser::Token::Type::Hash:
// FIXME: Hash tokens can be ID selectors or colors, we don't know which without parsing properly. // FIXME: Hash tokens can be ID selectors or colors, we don't know which without parsing properly.
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
break; break;
case Token::Type::Invalid: case Parser::Token::Type::Invalid:
case Token::Type::BadUrl: case Parser::Token::Type::BadUrl:
case Token::Type::BadString: case Parser::Token::Type::BadString:
// FIXME: Error highlighting color in palette? // FIXME: Error highlighting color in palette?
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { Color(Color::NamedColor::Red), {}, false, true }, token.type()); highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { Color(Color::NamedColor::Red), {}, false, true }, token.type());
break; break;
case Token::Type::EndOfFile: case Parser::Token::Type::EndOfFile:
default: default:
break; break;
} }
@ -144,10 +144,10 @@ Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token
{ {
static Vector<Syntax::Highlighter::MatchingTokenPair> pairs; static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
if (pairs.is_empty()) { if (pairs.is_empty()) {
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenCurly), static_cast<u64>(CSS::Token::Type::CloseCurly) }); pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenCurly), static_cast<u64>(CSS::Parser::Token::Type::CloseCurly) });
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenParen), static_cast<u64>(CSS::Token::Type::CloseParen) }); pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenParen), static_cast<u64>(CSS::Parser::Token::Type::CloseParen) });
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenSquare), static_cast<u64>(CSS::Token::Type::CloseSquare) }); pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenSquare), static_cast<u64>(CSS::Parser::Token::Type::CloseSquare) });
pairs.append({ static_cast<u64>(CSS::Token::Type::CDO), static_cast<u64>(CSS::Token::Type::CDC) }); pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::CDO), static_cast<u64>(CSS::Parser::Token::Type::CDC) });
} }
return pairs; return pairs;
} }

2
Userland/Libraries/LibWeb/Forward.h
View file

@ -106,6 +106,8 @@ class DeclarationOrAtRule;
class Function; class Function;
class Parser; class Parser;
class StyleRule; class StyleRule;
class Token;
class Tokenizer;
} }
namespace Web::DOM { namespace Web::DOM {