1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-25 09:07:44 +00:00

LibRegex: Implement section B.1.4. of the ECMA262 spec

This allows the parser to deal with crazy patterns like the one
in #5517.
This commit is contained in:
AnotherTest 2021-02-26 22:49:34 +03:30 committed by Andreas Kling
parent ce5fe2a6e8
commit f05e518cbc
7 changed files with 382 additions and 91 deletions

View file

@ -1,5 +1,6 @@
/*
* Copyright (c) 2020, Emanuel Sprung <emanuel.sprung@gmail.com>
* Copyright (c) 2020-2021, the SerenityOS developers.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -51,6 +52,11 @@ ALWAYS_INLINE bool Parser::match(TokenType type) const
return m_parser_state.current_token.type() == type;
}
ALWAYS_INLINE bool Parser::match(char ch) const
{
return m_parser_state.current_token.type() == TokenType::Char && m_parser_state.current_token.value().length() == 1 && m_parser_state.current_token.value()[0] == ch;
}
ALWAYS_INLINE Token Parser::consume()
{
auto old_token = m_parser_state.current_token;
@ -108,6 +114,16 @@ ALWAYS_INLINE bool Parser::try_skip(StringView str)
return true;
}
ALWAYS_INLINE bool Parser::lookahead_any(StringView str)
{
for (auto ch : str) {
if (match(ch))
return true;
}
return false;
}
ALWAYS_INLINE char Parser::skip()
{
char ch;
@ -122,6 +138,12 @@ ALWAYS_INLINE char Parser::skip()
return ch;
}
ALWAYS_INLINE void Parser::back(size_t count)
{
m_parser_state.lexer.back(count);
m_parser_state.current_token = m_parser_state.lexer.next();
}
ALWAYS_INLINE void Parser::reset()
{
m_parser_state.bytecode.clear();
@ -702,10 +724,25 @@ bool ECMA262Parser::parse_term(ByteCode& stack, size_t& match_length_minimum, bo
ByteCode atom_stack;
size_t minimum_atom_length = 0;
if (!parse_atom(atom_stack, minimum_atom_length, unicode, named))
return false;
auto parse_with_quantifier = [&] {
bool did_parse_one = false;
if (m_should_use_browser_extended_grammar)
did_parse_one = parse_extended_atom(atom_stack, minimum_atom_length, named);
if (!parse_quantifier(atom_stack, minimum_atom_length, unicode, named))
if (!did_parse_one)
did_parse_one = parse_atom(atom_stack, minimum_atom_length, unicode, named);
if (!did_parse_one)
return false;
VERIFY(did_parse_one);
if (!parse_quantifier(atom_stack, minimum_atom_length, unicode, named))
return false;
return true;
};
if (!parse_with_quantifier())
return false;
stack.append(move(atom_stack));
@ -749,35 +786,36 @@ bool ECMA262Parser::parse_assertion(ByteCode& stack, [[maybe_unused]] size_t& ma
ByteCode assertion_stack;
size_t length_dummy = 0;
auto parse_inner_disjunction = [&] {
auto disjunction_ok = parse_disjunction(assertion_stack, length_dummy, unicode, named);
if (!disjunction_ok)
return false;
consume(TokenType::RightParen, Error::MismatchingParen);
return true;
};
if (try_skip("=")) {
if (!parse_inner_disjunction())
bool should_parse_forward_assertion = m_should_use_browser_extended_grammar ? unicode : true;
if (should_parse_forward_assertion && try_skip("=")) {
if (!parse_inner_disjunction(assertion_stack, length_dummy, unicode, named))
return false;
stack.insert_bytecode_lookaround(move(assertion_stack), ByteCode::LookAroundType::LookAhead);
return true;
}
if (try_skip("!")) {
if (!parse_inner_disjunction())
if (should_parse_forward_assertion && try_skip("!")) {
if (!parse_inner_disjunction(assertion_stack, length_dummy, unicode, named))
return false;
stack.insert_bytecode_lookaround(move(assertion_stack), ByteCode::LookAroundType::NegatedLookAhead);
return true;
}
if (m_should_use_browser_extended_grammar) {
if (!unicode) {
if (parse_quantifiable_assertion(assertion_stack, match_length_minimum, named)) {
stack.append(move(assertion_stack));
return true;
}
}
}
if (try_skip("<=")) {
if (!parse_inner_disjunction())
if (!parse_inner_disjunction(assertion_stack, length_dummy, unicode, named))
return false;
// FIXME: Somehow ensure that this assertion regexp has a fixed length.
stack.insert_bytecode_lookaround(move(assertion_stack), ByteCode::LookAroundType::LookBehind, length_dummy);
return true;
}
if (try_skip("<!")) {
if (!parse_inner_disjunction())
if (!parse_inner_disjunction(assertion_stack, length_dummy, unicode, named))
return false;
stack.insert_bytecode_lookaround(move(assertion_stack), ByteCode::LookAroundType::NegatedLookBehind, length_dummy);
return true;
@ -792,7 +830,40 @@ bool ECMA262Parser::parse_assertion(ByteCode& stack, [[maybe_unused]] size_t& ma
return false;
}
Optional<unsigned> ECMA262Parser::read_digits(ECMA262Parser::ReadDigitsInitialZeroState initial_zero, ECMA262Parser::ReadDigitFollowPolicy follow_policy, bool hex, int max_count)
bool ECMA262Parser::parse_inner_disjunction(ByteCode& bytecode_stack, size_t& length, bool unicode, bool named)
{
auto disjunction_ok = parse_disjunction(bytecode_stack, length, unicode, named);
if (!disjunction_ok)
return false;
consume(TokenType::RightParen, Error::MismatchingParen);
return true;
}
bool ECMA262Parser::parse_quantifiable_assertion(ByteCode& stack, size_t&, bool named)
{
VERIFY(m_should_use_browser_extended_grammar);
ByteCode assertion_stack;
size_t match_length_minimum = 0;
if (try_skip("=")) {
if (!parse_inner_disjunction(assertion_stack, match_length_minimum, false, named))
return false;
stack.insert_bytecode_lookaround(move(assertion_stack), ByteCode::LookAroundType::LookAhead);
return true;
}
if (try_skip("!")) {
if (!parse_inner_disjunction(assertion_stack, match_length_minimum, false, named))
return false;
stack.insert_bytecode_lookaround(move(assertion_stack), ByteCode::LookAroundType::NegatedLookAhead);
return true;
}
return false;
}
StringView ECMA262Parser::read_digits_as_string(ReadDigitsInitialZeroState initial_zero, ReadDigitFollowPolicy follow_policy, bool hex, int max_count)
{
if (!match(TokenType::Char))
return {};
@ -832,10 +903,16 @@ Optional<unsigned> ECMA262Parser::read_digits(ECMA262Parser::ReadDigitsInitialZe
++count;
}
StringView str { start_token.value().characters_without_null_termination(), offset };
return StringView { start_token.value().characters_without_null_termination(), offset };
}
Optional<unsigned> ECMA262Parser::read_digits(ECMA262Parser::ReadDigitsInitialZeroState initial_zero, ECMA262Parser::ReadDigitFollowPolicy follow_policy, bool hex, int max_count)
{
auto str = read_digits_as_string(initial_zero, follow_policy, hex, max_count);
if (str.is_empty())
return {};
if (hex)
return AK::StringUtils::convert_to_uint_from_hex(str);
return str.to_uint();
}
@ -948,12 +1025,12 @@ bool ECMA262Parser::parse_atom(ByteCode& stack, size_t& match_length_minimum, bo
if (match(TokenType::LeftBracket)) {
// Character class.
return parse_character_class(stack, match_length_minimum, unicode, named);
return parse_character_class(stack, match_length_minimum, unicode && !m_should_use_browser_extended_grammar, named);
}
if (match(TokenType::LeftParen)) {
// Non-capturing group, or a capture group.
return parse_capture_group(stack, match_length_minimum, unicode, named);
return parse_capture_group(stack, match_length_minimum, unicode && !m_should_use_browser_extended_grammar, named);
}
if (match(TokenType::Period)) {
@ -963,13 +1040,24 @@ bool ECMA262Parser::parse_atom(ByteCode& stack, size_t& match_length_minimum, bo
return true;
}
if (match(TokenType::Circumflex) || match(TokenType::Dollar) || match(TokenType::RightBracket)
|| match(TokenType::RightCurly) || match(TokenType::RightParen) || match(TokenType::Pipe)
|| match(TokenType::Plus) || match(TokenType::Asterisk) || match(TokenType::Questionmark)) {
if (match(TokenType::Circumflex) || match(TokenType::Dollar) || match(TokenType::RightParen)
|| match(TokenType::Pipe) || match(TokenType::Plus) || match(TokenType::Asterisk)
|| match(TokenType::Questionmark)) {
return false;
}
if (match(TokenType::RightBracket) || match(TokenType::RightCurly) || match(TokenType::LeftCurly)) {
if (m_should_use_browser_extended_grammar) {
auto token = consume();
match_length_minimum += 1;
stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)token.value()[0] } });
return true;
} else {
return false;
}
}
if (match_ordinary_characters()) {
auto token = consume().value();
match_length_minimum += 1;
@ -981,17 +1069,67 @@ bool ECMA262Parser::parse_atom(ByteCode& stack, size_t& match_length_minimum, bo
return false;
}
bool ECMA262Parser::parse_extended_atom(ByteCode&, size_t&, bool)
{
// Note: This includes only rules *not* present in parse_atom()
VERIFY(m_should_use_browser_extended_grammar);
if (parse_invalid_braced_quantifier())
return true; // FAIL FAIL FAIL
return false;
}
bool ECMA262Parser::parse_invalid_braced_quantifier()
{
if (!match(TokenType::LeftCurly))
return false;
consume();
size_t chars_consumed = 1;
auto low_bound = read_digits_as_string();
StringView high_bound;
if (low_bound.is_empty()) {
back(chars_consumed + 1);
return false;
}
chars_consumed += low_bound.length();
if (match(TokenType::Comma)) {
consume();
++chars_consumed;
high_bound = read_digits_as_string();
chars_consumed += high_bound.length();
}
if (!match(TokenType::RightCurly)) {
back(chars_consumed + 1);
return false;
}
consume();
set_error(Error::InvalidPattern);
return true;
}
bool ECMA262Parser::parse_atom_escape(ByteCode& stack, size_t& match_length_minimum, bool unicode, bool named)
{
if (auto escape = read_digits(ReadDigitsInitialZeroState::Disallow, ReadDigitFollowPolicy::DisallowNonDigit); escape.has_value()) {
auto maybe_length = m_parser_state.capture_group_minimum_lengths.get(escape.value());
if (!maybe_length.has_value()) {
set_error(Error::InvalidNumber);
return false;
if (auto escape_str = read_digits_as_string(ReadDigitsInitialZeroState::Disallow, ReadDigitFollowPolicy::DisallowNonDigit); !escape_str.is_empty()) {
if (auto escape = escape_str.to_uint(); escape.has_value()) {
auto maybe_length = m_parser_state.capture_group_minimum_lengths.get(escape.value());
if (maybe_length.has_value()) {
match_length_minimum += maybe_length.value();
stack.insert_bytecode_compare_values({ { CharacterCompareType::Reference, (ByteCodeValueType)escape.value() } });
return true;
}
if (!m_should_use_browser_extended_grammar) {
set_error(Error::InvalidNumber);
return false;
}
}
match_length_minimum += maybe_length.value();
stack.insert_bytecode_compare_values({ { CharacterCompareType::Reference, (ByteCodeValueType)escape.value() } });
return true;
// If not, put the characters back.
back(escape_str.length());
}
// CharacterEscape > ControlEscape
@ -1030,11 +1168,18 @@ bool ECMA262Parser::parse_atom_escape(ByteCode& stack, size_t& match_length_mini
for (auto c = 'A'; c <= 'z'; ++c) {
if (try_skip({ &c, 1 })) {
match_length_minimum += 1;
stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)(c & 0x3f) } });
stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)(c % 32) } });
return true;
}
}
if (m_should_use_browser_extended_grammar) {
back(2);
stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)'\\' } });
match_length_minimum += 1;
return true;
}
if (unicode) {
set_error(Error::InvalidPattern);
return false;
@ -1046,6 +1191,17 @@ bool ECMA262Parser::parse_atom_escape(ByteCode& stack, size_t& match_length_mini
return true;
}
// LegacyOctalEscapeSequence
if (m_should_use_browser_extended_grammar) {
if (!unicode) {
if (auto escape = parse_legacy_octal_escape(); escape.has_value()) {
stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)escape.value() } });
match_length_minimum += 1;
return true;
}
}
}
// '\0'
if (read_digits(ReadDigitsInitialZeroState::Require, ReadDigitFollowPolicy::DisallowDigit).has_value()) {
match_length_minimum += 1;
@ -1092,7 +1248,8 @@ bool ECMA262Parser::parse_atom_escape(ByteCode& stack, size_t& match_length_mini
}
// IdentityEscape
for (auto ch : StringView { "^$\\.*+?()[]{}|" }) {
auto source_characters = m_should_use_browser_extended_grammar ? "^$\\.*+?()[|"sv : "^$\\.*+?()[]{}|"sv;
for (auto ch : source_characters) {
if (try_skip({ &ch, 1 })) {
match_length_minimum += 1;
stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)ch } });
@ -1162,6 +1319,59 @@ bool ECMA262Parser::parse_atom_escape(ByteCode& stack, size_t& match_length_mini
stack.insert_bytecode_compare_values(move(compares));
return true;
}
Optional<u8> ECMA262Parser::parse_legacy_octal_escape()
{
constexpr auto all_octal_digits = "01234567";
auto read_octal_digit = [&](auto start, auto end, bool should_ensure_no_following_octal_digit) -> Optional<u8> {
for (char c = '0' + start; c <= '0' + end; ++c) {
if (try_skip({ &c, 1 })) {
if (!should_ensure_no_following_octal_digit || !lookahead_any(all_octal_digits))
return c - '0';
back(2);
return {};
}
}
return {};
};
// OctalDigit(1)
if (auto digit = read_octal_digit(0, 7, true); digit.has_value()) {
return digit.value();
}
// OctalDigit(2)
if (auto left_digit = read_octal_digit(0, 3, false); left_digit.has_value()) {
if (auto right_digit = read_octal_digit(0, 7, true); right_digit.has_value()) {
return left_digit.value() * 8 + right_digit.value();
}
back(2);
}
// OctalDigit(2)
if (auto left_digit = read_octal_digit(4, 7, false); left_digit.has_value()) {
if (auto right_digit = read_octal_digit(0, 7, false); right_digit.has_value()) {
return left_digit.value() * 8 + right_digit.value();
}
back(2);
}
// OctalDigit(3)
if (auto left_digit = read_octal_digit(0, 3, false); left_digit.has_value()) {
size_t chars_consumed = 1;
if (auto mid_digit = read_octal_digit(0, 7, false); mid_digit.has_value()) {
++chars_consumed;
if (auto right_digit = read_octal_digit(0, 7, false); right_digit.has_value()) {
return left_digit.value() * 64 + mid_digit.value() * 8 + right_digit.value();
}
}
back(chars_consumed);
}
return {};
}
Optional<CharClass> ECMA262Parser::parse_character_class_escape(bool& negate, bool expect_backslash)
{
@ -1260,7 +1470,18 @@ bool ECMA262Parser::parse_nonempty_class_ranges(Vector<CompareTypeAndValuePair>&
if (try_skip("c")) {
for (auto c = 'A'; c <= 'z'; ++c) {
if (try_skip({ &c, 1 }))
return { { .code_point = (u32)(c & 0x3f), .is_character_class = false } };
return { { .code_point = (u32)(c % 32), .is_character_class = false } };
}
if (m_should_use_browser_extended_grammar) {
for (auto c = '0'; c <= '9'; ++c) {
if (try_skip({ &c, 1 }))
return { { .code_point = (u32)(c % 32), .is_character_class = false } };
}
if (try_skip("_"))
return { { .code_point = (u32)('_' % 32), .is_character_class = false } };
back(2);
return { { .code_point = '\\', .is_character_class = false } };
}
}
@ -1361,8 +1582,28 @@ bool ECMA262Parser::parse_nonempty_class_ranges(Vector<CompareTypeAndValuePair>&
return false;
if (first_atom.value().is_character_class || second_atom.value().is_character_class) {
set_error(Error::InvalidRange);
return false;
if (m_should_use_browser_extended_grammar) {
if (unicode) {
set_error(Error::InvalidRange);
return false;
}
// CharacterRangeOrUnion > !Unicode > CharClass
if (first_atom->is_character_class)
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::CharClass, (ByteCodeValueType)first_atom->character_class });
else
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::Char, (ByteCodeValueType)first_atom->code_point });
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::Char, (ByteCodeValueType)'-' });
if (second_atom->is_character_class)
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::CharClass, (ByteCodeValueType)second_atom->character_class });
else
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::Char, (ByteCodeValueType)second_atom->code_point });
continue;
} else {
set_error(Error::InvalidRange);
return false;
}
}
if (first_atom.value().code_point > second_atom.value().code_point) {