mirror of
https://github.com/RGBCube/serenity
synced 2025-07-02 22:22:07 +00:00
Everywhere: codepoint => code point
This commit is contained in:
parent
a8ae8b24de
commit
12a42edd13
18 changed files with 240 additions and 240 deletions
|
@ -20,179 +20,179 @@ static inline void log_parse_error(const SourceLocation& location = SourceLocati
|
|||
dbgln_if(CSS_TOKENIZER_TRACE, "Parse error (css tokenization) {} ", location);
|
||||
}
|
||||
|
||||
static inline bool is_surrogate(u32 codepoint)
|
||||
static inline bool is_surrogate(u32 code_point)
|
||||
{
|
||||
return (codepoint & 0xfffff800) == 0xd800;
|
||||
return (code_point & 0xfffff800) == 0xd800;
|
||||
}
|
||||
|
||||
static inline bool is_quotation_mark(u32 codepoint)
|
||||
static inline bool is_quotation_mark(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x22;
|
||||
return code_point == 0x22;
|
||||
}
|
||||
|
||||
static inline bool is_greater_than_maximum_allowed_codepoint(u32 codepoint)
|
||||
static inline bool is_greater_than_maximum_allowed_code_point(u32 code_point)
|
||||
{
|
||||
return codepoint > 0x10FFFF;
|
||||
return code_point > 0x10FFFF;
|
||||
}
|
||||
|
||||
static inline bool is_hex_digit(u32 codepoint)
|
||||
static inline bool is_hex_digit(u32 code_point)
|
||||
{
|
||||
return isxdigit(codepoint);
|
||||
return isxdigit(code_point);
|
||||
}
|
||||
|
||||
static inline bool is_low_line(u32 codepoint)
|
||||
static inline bool is_low_line(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x5F;
|
||||
return code_point == 0x5F;
|
||||
}
|
||||
|
||||
static inline bool is_non_ascii(u32 codepoint)
|
||||
static inline bool is_non_ascii(u32 code_point)
|
||||
{
|
||||
return codepoint >= 0x80;
|
||||
return code_point >= 0x80;
|
||||
}
|
||||
|
||||
static inline bool is_name_start_codepoint(u32 codepoint)
|
||||
static inline bool is_name_start_code_point(u32 code_point)
|
||||
{
|
||||
return isalpha(codepoint) || is_non_ascii(codepoint) || is_low_line(codepoint);
|
||||
return isalpha(code_point) || is_non_ascii(code_point) || is_low_line(code_point);
|
||||
}
|
||||
|
||||
static inline bool is_hyphen_minus(u32 codepoint)
|
||||
static inline bool is_hyphen_minus(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x2D;
|
||||
return code_point == 0x2D;
|
||||
}
|
||||
|
||||
static inline bool is_name_codepoint(u32 codepoint)
|
||||
static inline bool is_name_code_point(u32 code_point)
|
||||
{
|
||||
return is_name_start_codepoint(codepoint) || isdigit(codepoint) || is_hyphen_minus(codepoint);
|
||||
return is_name_start_code_point(code_point) || isdigit(code_point) || is_hyphen_minus(code_point);
|
||||
}
|
||||
|
||||
static inline bool is_non_printable(u32 codepoint)
|
||||
static inline bool is_non_printable(u32 code_point)
|
||||
{
|
||||
return codepoint <= 0x8 || codepoint == 0xB || (codepoint >= 0xE && codepoint <= 0x1F) || codepoint == 0x7F;
|
||||
return code_point <= 0x8 || code_point == 0xB || (code_point >= 0xE && code_point <= 0x1F) || code_point == 0x7F;
|
||||
}
|
||||
|
||||
static inline bool is_number_sign(u32 codepoint)
|
||||
static inline bool is_number_sign(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x23;
|
||||
return code_point == 0x23;
|
||||
}
|
||||
|
||||
static inline bool is_reverse_solidus(u32 codepoint)
|
||||
static inline bool is_reverse_solidus(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x5C;
|
||||
return code_point == 0x5C;
|
||||
}
|
||||
|
||||
static inline bool is_apostrophe(u32 codepoint)
|
||||
static inline bool is_apostrophe(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x27;
|
||||
return code_point == 0x27;
|
||||
}
|
||||
|
||||
static inline bool is_left_paren(u32 codepoint)
|
||||
static inline bool is_left_paren(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x28;
|
||||
return code_point == 0x28;
|
||||
}
|
||||
|
||||
static inline bool is_right_paren(u32 codepoint)
|
||||
static inline bool is_right_paren(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x29;
|
||||
return code_point == 0x29;
|
||||
}
|
||||
|
||||
static inline bool is_plus_sign(u32 codepoint)
|
||||
static inline bool is_plus_sign(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x2B;
|
||||
return code_point == 0x2B;
|
||||
}
|
||||
|
||||
static inline bool is_comma(u32 codepoint)
|
||||
static inline bool is_comma(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x2C;
|
||||
return code_point == 0x2C;
|
||||
}
|
||||
|
||||
static inline bool is_full_stop(u32 codepoint)
|
||||
static inline bool is_full_stop(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x2E;
|
||||
return code_point == 0x2E;
|
||||
}
|
||||
|
||||
static inline bool is_newline(u32 codepoint)
|
||||
static inline bool is_newline(u32 code_point)
|
||||
{
|
||||
return codepoint == 0xA;
|
||||
return code_point == 0xA;
|
||||
}
|
||||
|
||||
static inline bool is_asterisk(u32 codepoint)
|
||||
static inline bool is_asterisk(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x2A;
|
||||
return code_point == 0x2A;
|
||||
}
|
||||
|
||||
static inline bool is_solidus(u32 codepoint)
|
||||
static inline bool is_solidus(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x2F;
|
||||
return code_point == 0x2F;
|
||||
}
|
||||
|
||||
static inline bool is_colon(u32 codepoint)
|
||||
static inline bool is_colon(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x3A;
|
||||
return code_point == 0x3A;
|
||||
}
|
||||
|
||||
static inline bool is_semicolon(u32 codepoint)
|
||||
static inline bool is_semicolon(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x3B;
|
||||
return code_point == 0x3B;
|
||||
}
|
||||
|
||||
static inline bool is_less_than_sign(u32 codepoint)
|
||||
static inline bool is_less_than_sign(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x3C;
|
||||
return code_point == 0x3C;
|
||||
}
|
||||
|
||||
static inline bool is_greater_than_sign(u32 codepoint)
|
||||
static inline bool is_greater_than_sign(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x3E;
|
||||
return code_point == 0x3E;
|
||||
}
|
||||
|
||||
static inline bool is_at(u32 codepoint)
|
||||
static inline bool is_at(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x40;
|
||||
return code_point == 0x40;
|
||||
}
|
||||
|
||||
static inline bool is_open_square_bracket(u32 codepoint)
|
||||
static inline bool is_open_square_bracket(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x5B;
|
||||
return code_point == 0x5B;
|
||||
}
|
||||
|
||||
static inline bool is_closed_square_bracket(u32 codepoint)
|
||||
static inline bool is_closed_square_bracket(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x5D;
|
||||
return code_point == 0x5D;
|
||||
}
|
||||
|
||||
static inline bool is_open_curly_bracket(u32 codepoint)
|
||||
static inline bool is_open_curly_bracket(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x7B;
|
||||
return code_point == 0x7B;
|
||||
}
|
||||
|
||||
static inline bool is_closed_curly_bracket(u32 codepoint)
|
||||
static inline bool is_closed_curly_bracket(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x7D;
|
||||
return code_point == 0x7D;
|
||||
}
|
||||
|
||||
static inline bool is_whitespace(u32 codepoint)
|
||||
static inline bool is_whitespace(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x9 || codepoint == 0xA || codepoint == 0x20;
|
||||
return code_point == 0x9 || code_point == 0xA || code_point == 0x20;
|
||||
}
|
||||
|
||||
static inline bool is_percent(u32 codepoint)
|
||||
static inline bool is_percent(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x25;
|
||||
return code_point == 0x25;
|
||||
}
|
||||
|
||||
static inline bool is_exclamation_mark(u32 codepoint)
|
||||
static inline bool is_exclamation_mark(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x21;
|
||||
return code_point == 0x21;
|
||||
}
|
||||
|
||||
static inline bool is_e(u32 codepoint)
|
||||
static inline bool is_e(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x65;
|
||||
return code_point == 0x65;
|
||||
}
|
||||
|
||||
static inline bool is_E(u32 codepoint)
|
||||
static inline bool is_E(u32 code_point)
|
||||
{
|
||||
return codepoint == 0x45;
|
||||
return code_point == 0x45;
|
||||
}
|
||||
|
||||
namespace Web::CSS {
|
||||
|
@ -222,17 +222,17 @@ Vector<Token> Tokenizer::parse()
|
|||
}
|
||||
}
|
||||
|
||||
Optional<u32> Tokenizer::next_codepoint()
|
||||
Optional<u32> Tokenizer::next_code_point()
|
||||
{
|
||||
if (m_utf8_iterator == m_utf8_view.end())
|
||||
return {};
|
||||
m_prev_utf8_iterator = m_utf8_iterator;
|
||||
++m_utf8_iterator;
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "(Tokenizer) Next codepoint: {:c}", (char)*m_prev_utf8_iterator);
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "(Tokenizer) Next code_point: {:c}", (char)*m_prev_utf8_iterator);
|
||||
return *m_prev_utf8_iterator;
|
||||
}
|
||||
|
||||
Optional<u32> Tokenizer::peek_codepoint(size_t offset) const
|
||||
Optional<u32> Tokenizer::peek_code_point(size_t offset) const
|
||||
{
|
||||
auto it = m_utf8_iterator;
|
||||
for (size_t i = 0; i < offset && it != m_utf8_view.end(); ++i)
|
||||
|
@ -292,32 +292,32 @@ Token Tokenizer::create_value_token(Token::TokenType type, u32 value)
|
|||
return token;
|
||||
}
|
||||
|
||||
u32 Tokenizer::consume_escaped_codepoint()
|
||||
u32 Tokenizer::consume_escaped_code_point()
|
||||
{
|
||||
auto codepoint = next_codepoint();
|
||||
auto code_point = next_code_point();
|
||||
|
||||
if (!codepoint.has_value()) {
|
||||
if (!code_point.has_value()) {
|
||||
log_parse_error();
|
||||
return REPLACEMENT_CHARACTER;
|
||||
}
|
||||
|
||||
auto input = codepoint.value();
|
||||
auto input = code_point.value();
|
||||
|
||||
if (is_hex_digit(input)) {
|
||||
StringBuilder builder;
|
||||
builder.append_code_point(input);
|
||||
|
||||
size_t counter = 0;
|
||||
while (is_hex_digit(peek_codepoint().value()) && counter++ < 5) {
|
||||
builder.append_code_point(next_codepoint().value());
|
||||
while (is_hex_digit(peek_code_point().value()) && counter++ < 5) {
|
||||
builder.append_code_point(next_code_point().value());
|
||||
}
|
||||
|
||||
if (is_whitespace(peek_codepoint().value())) {
|
||||
(void)next_codepoint();
|
||||
if (is_whitespace(peek_code_point().value())) {
|
||||
(void)next_code_point();
|
||||
}
|
||||
|
||||
auto unhexed = strtoul(builder.to_string().characters(), nullptr, 16);
|
||||
if (unhexed == 0 || is_surrogate(unhexed) || is_greater_than_maximum_allowed_codepoint(unhexed)) {
|
||||
if (unhexed == 0 || is_surrogate(unhexed) || is_greater_than_maximum_allowed_code_point(unhexed)) {
|
||||
return REPLACEMENT_CHARACTER;
|
||||
}
|
||||
|
||||
|
@ -336,8 +336,8 @@ Token Tokenizer::consume_an_ident_like_token()
|
|||
{
|
||||
auto string = consume_a_name();
|
||||
|
||||
if (string.equals_ignoring_case("url") && is_left_paren(peek_codepoint().value())) {
|
||||
(void)next_codepoint();
|
||||
if (string.equals_ignoring_case("url") && is_left_paren(peek_code_point().value())) {
|
||||
(void)next_code_point();
|
||||
|
||||
for (;;) {
|
||||
auto maybe_whitespace = peek_twin().value();
|
||||
|
@ -345,7 +345,7 @@ Token Tokenizer::consume_an_ident_like_token()
|
|||
break;
|
||||
}
|
||||
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
}
|
||||
|
||||
auto next_two = peek_twin().value();
|
||||
|
@ -357,8 +357,8 @@ Token Tokenizer::consume_an_ident_like_token()
|
|||
return consume_a_url_token();
|
||||
}
|
||||
|
||||
if (is_left_paren(peek_codepoint().value())) {
|
||||
(void)next_codepoint();
|
||||
if (is_left_paren(peek_code_point().value())) {
|
||||
(void)next_code_point();
|
||||
|
||||
return create_value_token(Token::TokenType::Function, string);
|
||||
}
|
||||
|
@ -371,32 +371,32 @@ CSSNumber Tokenizer::consume_a_number()
|
|||
StringBuilder repr;
|
||||
Token::NumberType type = Token::NumberType::Integer;
|
||||
|
||||
auto next_input = peek_codepoint().value();
|
||||
auto next_input = peek_code_point().value();
|
||||
if (is_plus_sign(next_input) || is_hyphen_minus(next_input)) {
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
auto digits = peek_codepoint().value();
|
||||
auto digits = peek_code_point().value();
|
||||
if (!isdigit(digits))
|
||||
break;
|
||||
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
}
|
||||
|
||||
auto maybe_number = peek_twin().value();
|
||||
if (is_full_stop(maybe_number.first) && isdigit(maybe_number.second)) {
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
|
||||
type = Token::NumberType::Number;
|
||||
|
||||
for (;;) {
|
||||
auto digits = peek_codepoint();
|
||||
auto digits = peek_code_point();
|
||||
if (digits.has_value() && !isdigit(digits.value()))
|
||||
break;
|
||||
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -404,23 +404,23 @@ CSSNumber Tokenizer::consume_a_number()
|
|||
if (is_E(maybe_exp.first) || is_e(maybe_exp.first)) {
|
||||
if (is_plus_sign(maybe_exp.second) || is_hyphen_minus(maybe_exp.second)) {
|
||||
if (isdigit(maybe_exp.third)) {
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
}
|
||||
} else if (isdigit(maybe_exp.second)) {
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
}
|
||||
|
||||
type = Token::NumberType::Number;
|
||||
|
||||
for (;;) {
|
||||
auto digits = peek_codepoint().value();
|
||||
auto digits = peek_code_point().value();
|
||||
if (!isdigit(digits))
|
||||
break;
|
||||
|
||||
repr.append_code_point(next_codepoint().value());
|
||||
repr.append_code_point(next_code_point().value());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -432,66 +432,66 @@ String Tokenizer::consume_a_name()
|
|||
StringBuilder result;
|
||||
|
||||
for (;;) {
|
||||
auto input = next_codepoint().value();
|
||||
auto input = next_code_point().value();
|
||||
|
||||
if (is_name_codepoint(input)) {
|
||||
if (is_name_code_point(input)) {
|
||||
result.append_code_point(input);
|
||||
continue;
|
||||
}
|
||||
|
||||
auto next = peek_codepoint();
|
||||
auto next = peek_code_point();
|
||||
if (next.has_value() && is_valid_escape_sequence({ input, next.value() })) {
|
||||
result.append_code_point(consume_escaped_codepoint());
|
||||
result.append_code_point(consume_escaped_code_point());
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
reconsume_current_input_codepoint();
|
||||
reconsume_current_input_code_point();
|
||||
return result.to_string();
|
||||
}
|
||||
Token Tokenizer::consume_a_url_token()
|
||||
{
|
||||
auto token = create_new_token(Token::TokenType::Url);
|
||||
for (;;) {
|
||||
if (!is_whitespace(peek_codepoint().value())) {
|
||||
if (!is_whitespace(peek_code_point().value())) {
|
||||
break;
|
||||
}
|
||||
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
|
||||
auto codepoint = peek_codepoint();
|
||||
if (!codepoint.has_value()) {
|
||||
auto code_point = peek_code_point();
|
||||
if (!code_point.has_value()) {
|
||||
log_parse_error();
|
||||
return token;
|
||||
}
|
||||
|
||||
auto input = codepoint.value();
|
||||
auto input = code_point.value();
|
||||
|
||||
if (is_right_paren(input)) {
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
return token;
|
||||
}
|
||||
|
||||
if (is_whitespace(input)) {
|
||||
for (;;) {
|
||||
if (!is_whitespace(peek_codepoint().value())) {
|
||||
if (!is_whitespace(peek_code_point().value())) {
|
||||
break;
|
||||
}
|
||||
|
||||
codepoint = next_codepoint();
|
||||
code_point = next_code_point();
|
||||
}
|
||||
|
||||
if (!codepoint.has_value()) {
|
||||
if (!code_point.has_value()) {
|
||||
log_parse_error();
|
||||
return token;
|
||||
}
|
||||
|
||||
input = codepoint.value();
|
||||
input = code_point.value();
|
||||
|
||||
if (is_right_paren(input)) {
|
||||
return token;
|
||||
|
@ -503,30 +503,30 @@ Token Tokenizer::consume_a_url_token()
|
|||
|
||||
if (is_quotation_mark(input) || is_apostrophe(input) || is_left_paren(input) || is_non_printable(input)) {
|
||||
log_parse_error();
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
consume_the_remnants_of_a_bad_url();
|
||||
return create_new_token(Token::TokenType::BadUrl);
|
||||
}
|
||||
|
||||
if (is_reverse_solidus(input)) {
|
||||
if (is_valid_escape_sequence()) {
|
||||
token.m_value.append_code_point(consume_escaped_codepoint());
|
||||
token.m_value.append_code_point(consume_escaped_code_point());
|
||||
} else {
|
||||
log_parse_error();
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
consume_the_remnants_of_a_bad_url();
|
||||
return create_new_token(Token::TokenType::BadUrl);
|
||||
}
|
||||
}
|
||||
|
||||
token.m_value.append_code_point(next_codepoint().value());
|
||||
token.m_value.append_code_point(next_code_point().value());
|
||||
}
|
||||
}
|
||||
|
||||
void Tokenizer::consume_the_remnants_of_a_bad_url()
|
||||
{
|
||||
for (;;) {
|
||||
auto next = peek_codepoint();
|
||||
auto next = peek_code_point();
|
||||
|
||||
if (!next.has_value()) {
|
||||
return;
|
||||
|
@ -535,19 +535,19 @@ void Tokenizer::consume_the_remnants_of_a_bad_url()
|
|||
auto input = next.value();
|
||||
|
||||
if (is_right_paren(input)) {
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
return;
|
||||
}
|
||||
|
||||
if (is_valid_escape_sequence()) {
|
||||
[[maybe_unused]] auto cp = consume_escaped_codepoint();
|
||||
[[maybe_unused]] auto cp = consume_escaped_code_point();
|
||||
}
|
||||
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
}
|
||||
}
|
||||
|
||||
void Tokenizer::reconsume_current_input_codepoint()
|
||||
void Tokenizer::reconsume_current_input_code_point()
|
||||
{
|
||||
m_utf8_iterator = m_prev_utf8_iterator;
|
||||
}
|
||||
|
@ -566,8 +566,8 @@ Token Tokenizer::consume_a_numeric_token()
|
|||
return token;
|
||||
}
|
||||
|
||||
if (is_percent(peek_codepoint().value())) {
|
||||
(void)next_codepoint();
|
||||
if (is_percent(peek_code_point().value())) {
|
||||
(void)next_code_point();
|
||||
|
||||
auto token = create_new_token(Token::TokenType::Percentage);
|
||||
token.m_value.append(number.value);
|
||||
|
@ -632,12 +632,12 @@ bool Tokenizer::would_start_an_identifier()
|
|||
bool Tokenizer::would_start_an_identifier(U32Triplet values)
|
||||
{
|
||||
if (is_hyphen_minus(values.first)) {
|
||||
if (is_name_start_codepoint(values.second) || is_hyphen_minus(values.second) || is_valid_escape_sequence(values.to_twin_23()))
|
||||
if (is_name_start_code_point(values.second) || is_hyphen_minus(values.second) || is_valid_escape_sequence(values.to_twin_23()))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (is_name_start_codepoint(values.first)) {
|
||||
if (is_name_start_code_point(values.first)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -650,39 +650,39 @@ bool Tokenizer::would_start_an_identifier(U32Triplet values)
|
|||
return false;
|
||||
}
|
||||
|
||||
Token Tokenizer::consume_string_token(u32 ending_codepoint)
|
||||
Token Tokenizer::consume_string_token(u32 ending_code_point)
|
||||
{
|
||||
auto token = create_new_token(Token::TokenType::String);
|
||||
|
||||
for (;;) {
|
||||
auto codepoint = next_codepoint();
|
||||
auto code_point = next_code_point();
|
||||
|
||||
if (!codepoint.has_value()) {
|
||||
if (!code_point.has_value()) {
|
||||
log_parse_error();
|
||||
return token;
|
||||
}
|
||||
|
||||
auto input = codepoint.value();
|
||||
auto input = code_point.value();
|
||||
|
||||
if (input == ending_codepoint)
|
||||
if (input == ending_code_point)
|
||||
return token;
|
||||
|
||||
if (is_newline(input)) {
|
||||
reconsume_current_input_codepoint();
|
||||
reconsume_current_input_code_point();
|
||||
return create_new_token(Token::TokenType::BadString);
|
||||
}
|
||||
|
||||
if (is_reverse_solidus(input)) {
|
||||
auto next_input = peek_codepoint();
|
||||
auto next_input = peek_code_point();
|
||||
if (!next_input.has_value())
|
||||
continue;
|
||||
|
||||
if (is_newline(next_input.value())) {
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
continue;
|
||||
}
|
||||
|
||||
auto escaped = consume_escaped_codepoint();
|
||||
auto escaped = consume_escaped_code_point();
|
||||
token.m_value.append_code_point(escaped);
|
||||
}
|
||||
|
||||
|
@ -703,8 +703,8 @@ start:
|
|||
if (!(is_solidus(twin.first) && is_asterisk(twin.second)))
|
||||
return;
|
||||
|
||||
(void)next_codepoint();
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
(void)next_code_point();
|
||||
|
||||
for (;;) {
|
||||
auto peek_inner = peek_twin();
|
||||
|
@ -716,12 +716,12 @@ start:
|
|||
auto twin_inner = peek_inner.value();
|
||||
|
||||
if (is_asterisk(twin_inner.first) && is_solidus(twin_inner.second)) {
|
||||
(void)next_codepoint();
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
(void)next_code_point();
|
||||
goto start;
|
||||
}
|
||||
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -729,19 +729,19 @@ Token Tokenizer::consume_a_token()
|
|||
{
|
||||
consume_comments();
|
||||
|
||||
auto codepoint = next_codepoint();
|
||||
auto code_point = next_code_point();
|
||||
|
||||
if (!codepoint.has_value()) {
|
||||
if (!code_point.has_value()) {
|
||||
return create_new_token(Token::TokenType::EndOfFile);
|
||||
}
|
||||
|
||||
auto input = codepoint.value();
|
||||
auto input = code_point.value();
|
||||
|
||||
if (is_whitespace(input)) {
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "is whitespace");
|
||||
|
||||
while (is_whitespace(peek_codepoint().value()))
|
||||
(void)next_codepoint();
|
||||
while (is_whitespace(peek_code_point().value()))
|
||||
(void)next_code_point();
|
||||
|
||||
return create_new_token(Token::TokenType::Whitespace);
|
||||
}
|
||||
|
@ -754,10 +754,10 @@ Token Tokenizer::consume_a_token()
|
|||
if (is_number_sign(input)) {
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "is number sign");
|
||||
|
||||
auto next_input = peek_codepoint().value();
|
||||
auto next_input = peek_code_point().value();
|
||||
auto maybe_escape = peek_twin().value();
|
||||
|
||||
if (is_name_codepoint(next_input) || is_valid_escape_sequence(maybe_escape)) {
|
||||
if (is_name_code_point(next_input) || is_valid_escape_sequence(maybe_escape)) {
|
||||
auto token = create_new_token(Token::TokenType::Hash);
|
||||
|
||||
if (would_start_an_identifier())
|
||||
|
@ -790,7 +790,7 @@ Token Tokenizer::consume_a_token()
|
|||
if (is_plus_sign(input)) {
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "is plus sign");
|
||||
if (starts_with_a_number()) {
|
||||
reconsume_current_input_codepoint();
|
||||
reconsume_current_input_code_point();
|
||||
return consume_a_numeric_token();
|
||||
}
|
||||
|
||||
|
@ -805,20 +805,20 @@ Token Tokenizer::consume_a_token()
|
|||
if (is_hyphen_minus(input)) {
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "is hyphen minus");
|
||||
if (starts_with_a_number()) {
|
||||
reconsume_current_input_codepoint();
|
||||
reconsume_current_input_code_point();
|
||||
return consume_a_numeric_token();
|
||||
}
|
||||
|
||||
auto next_twin = peek_twin().value();
|
||||
if (is_hyphen_minus(next_twin.first) && is_greater_than_sign(next_twin.second)) {
|
||||
(void)next_codepoint();
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
(void)next_code_point();
|
||||
|
||||
return create_new_token(Token::TokenType::CDC);
|
||||
}
|
||||
|
||||
if (would_start_an_identifier()) {
|
||||
reconsume_current_input_codepoint();
|
||||
reconsume_current_input_code_point();
|
||||
return consume_an_ident_like_token();
|
||||
}
|
||||
|
||||
|
@ -828,7 +828,7 @@ Token Tokenizer::consume_a_token()
|
|||
if (is_full_stop(input)) {
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "is full stop");
|
||||
if (starts_with_a_number()) {
|
||||
reconsume_current_input_codepoint();
|
||||
reconsume_current_input_code_point();
|
||||
return consume_a_numeric_token();
|
||||
}
|
||||
|
||||
|
@ -850,9 +850,9 @@ Token Tokenizer::consume_a_token()
|
|||
auto maybe_cdo = peek_triplet().value();
|
||||
|
||||
if (is_exclamation_mark(maybe_cdo.first) && is_hyphen_minus(maybe_cdo.second) && is_hyphen_minus(maybe_cdo.third)) {
|
||||
(void)next_codepoint();
|
||||
(void)next_codepoint();
|
||||
(void)next_codepoint();
|
||||
(void)next_code_point();
|
||||
(void)next_code_point();
|
||||
(void)next_code_point();
|
||||
|
||||
return create_new_token(Token::TokenType::CDO);
|
||||
}
|
||||
|
@ -879,7 +879,7 @@ Token Tokenizer::consume_a_token()
|
|||
if (is_reverse_solidus(input)) {
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "is reverse solidus");
|
||||
if (is_valid_escape_sequence()) {
|
||||
reconsume_current_input_codepoint();
|
||||
reconsume_current_input_code_point();
|
||||
return consume_an_ident_like_token();
|
||||
}
|
||||
|
||||
|
@ -904,13 +904,13 @@ Token Tokenizer::consume_a_token()
|
|||
|
||||
if (isdigit(input)) {
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "is digit");
|
||||
reconsume_current_input_codepoint();
|
||||
reconsume_current_input_code_point();
|
||||
return consume_a_numeric_token();
|
||||
}
|
||||
|
||||
if (is_name_start_codepoint(input)) {
|
||||
if (is_name_start_code_point(input)) {
|
||||
dbgln_if(CSS_TOKENIZER_TRACE, "is name start");
|
||||
reconsume_current_input_codepoint();
|
||||
reconsume_current_input_code_point();
|
||||
return consume_an_ident_like_token();
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue