From 981badb45f124f010c73adc8a52a904bda014908 Mon Sep 17 00:00:00 2001 From: Sam Atkins Date: Fri, 24 Dec 2021 16:08:35 +0000 Subject: [PATCH] LibWeb: Add CSS::Tokenizer::start_of_input_stream_[twin|triplet]() These correspond to "If the input stream starts with..." in the spec, which up until now we were not handling correctly, which led to some fun bugs. As noted, reconsuming the input code point in order to read its value is hacky, but works. Keeping track of the current code point in Tokenizer would be nicer, when I'm feeling brave enough to mess with it! --- .../Libraries/LibWeb/CSS/Parser/Tokenizer.cpp | 24 +++++++++++++++++++ .../Libraries/LibWeb/CSS/Parser/Tokenizer.h | 3 +++ 2 files changed, 27 insertions(+) diff --git a/Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp b/Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp index 545ee4cfbd..b0736ae489 100644 --- a/Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp +++ b/Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp @@ -309,6 +309,30 @@ U32Triplet Tokenizer::peek_triplet() const return values; } +U32Twin Tokenizer::start_of_input_stream_twin() +{ + U32Twin twin; + // FIXME: Reconsuming just to read the current code point again is weird. + reconsume_current_input_code_point(); + twin.first = next_code_point(); + twin.second = peek_code_point(); + + return twin; +} + +U32Triplet Tokenizer::start_of_input_stream_triplet() +{ + U32Triplet triplet; + // FIXME: Reconsuming just to read the current code point again is weird. + reconsume_current_input_code_point(); + triplet.first = next_code_point(); + auto next_two = peek_twin(); + triplet.second = next_two.first; + triplet.third = next_two.second; + + return triplet; +} + Token Tokenizer::create_new_token(Token::Type type) { Token token = {}; diff --git a/Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.h b/Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.h index 1e60df0a7c..2d54d1ea8e 100644 --- a/Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.h +++ b/Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.h @@ -79,6 +79,9 @@ private: [[nodiscard]] U32Twin peek_twin() const; [[nodiscard]] U32Triplet peek_triplet() const; + [[nodiscard]] U32Twin start_of_input_stream_twin(); + [[nodiscard]] U32Triplet start_of_input_stream_triplet(); + [[nodiscard]] static Token create_new_token(Token::Type); [[nodiscard]] static Token create_value_token(Token::Type, String value); [[nodiscard]] static Token create_value_token(Token::Type, u32 value);