diff --git a/Libraries/LibRegex/CMakeLists.txt b/Libraries/LibRegex/CMakeLists.txt index 033ad018c2..5ce08544ac 100644 --- a/Libraries/LibRegex/CMakeLists.txt +++ b/Libraries/LibRegex/CMakeLists.txt @@ -1,21 +1,15 @@ -set(SOURCES - RegexByteCode.cpp - RegexLexer.cpp - RegexMatcher.cpp - RegexOptimizer.cpp - RegexParser.cpp -) - -if(SERENITYOS) - list(APPEND SOURCES C/Regex.cpp) +if (NOT ENABLE_RUST) + message(FATAL_ERROR "LibRegex requires ENABLE_RUST; the legacy C++ regex engine has been removed") endif() +set(SOURCES + ECMAScriptRegex.cpp + RustRegex.cpp +) + ladybird_lib(LibRegex regex EXPLICIT_SYMBOL_EXPORT) target_link_libraries(LibRegex PRIVATE LibUnicode) -if (ENABLE_RUST) - target_sources(LibRegex PRIVATE RustRegex.cpp) - import_rust_crate(MANIFEST_PATH Rust/Cargo.toml CRATE_NAME libregex_rust) - target_link_libraries(LibRegex PRIVATE libregex_rust) - target_compile_definitions(LibRegex PRIVATE ENABLE_RUST) -endif() +import_rust_crate(MANIFEST_PATH Rust/Cargo.toml CRATE_NAME libregex_rust) +target_link_libraries(LibRegex PRIVATE libregex_rust) +target_compile_definitions(LibRegex PRIVATE ENABLE_RUST) diff --git a/Libraries/LibRegex/Forward.h b/Libraries/LibRegex/Forward.h deleted file mode 100644 index 650cc34fe2..0000000000 --- a/Libraries/LibRegex/Forward.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include - -namespace regex { - -struct CompareTypeAndValuePair; - -enum class Error : u8; -class Lexer; -class PosixExtendedParser; -class ECMA262Parser; - -class ByteCode; - -class RegexStringView; - -} - -using regex::ECMA262Parser; -using regex::Lexer; -using regex::PosixExtendedParser; -using regex::RegexStringView; diff --git a/Libraries/LibRegex/Regex.h b/Libraries/LibRegex/Regex.h deleted file mode 100644 index 91dcf50f94..0000000000 --- a/Libraries/LibRegex/Regex.h +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include diff --git a/Libraries/LibRegex/RegexByteCode.cpp b/Libraries/LibRegex/RegexByteCode.cpp deleted file mode 100644 index 7e4f600323..0000000000 --- a/Libraries/LibRegex/RegexByteCode.cpp +++ /dev/null @@ -1,524 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include "RegexByteCode.h" - -#include -#include -#include - -namespace regex { - -StringView execution_result_name(ExecutionResult result) -{ - switch (result) { -#define __ENUMERATE_EXECUTION_RESULT(x) \ - case ExecutionResult::x: \ - return #x##sv; - ENUMERATE_EXECUTION_RESULTS -#undef __ENUMERATE_EXECUTION_RESULT - default: - VERIFY_NOT_REACHED(); - return ""sv; - } -} - -StringView opcode_id_name(OpCodeId opcode) -{ - switch (opcode) { -#define __ENUMERATE_OPCODE(x) \ - case OpCodeId::x: \ - return #x##sv; - - ENUMERATE_OPCODES - -#undef __ENUMERATE_OPCODE - default: - VERIFY_NOT_REACHED(); - return ""sv; - } -} - -StringView fork_if_condition_name(ForkIfCondition condition) -{ - switch (condition) { -#define __ENUMERATE_FORK_IF_CONDITION(x) \ - case ForkIfCondition::x: \ - return #x##sv; - ENUMERATE_FORK_IF_CONDITIONS -#undef __ENUMERATE_FORK_IF_CONDITION - default: - return ""sv; - } -} - -StringView boundary_check_type_name(BoundaryCheckType ty) -{ - switch (ty) { -#define __ENUMERATE_BOUNDARY_CHECK_TYPE(x) \ - case BoundaryCheckType::x: \ - return #x##sv; - ENUMERATE_BOUNDARY_CHECK_TYPES -#undef __ENUMERATE_BOUNDARY_CHECK_TYPE - default: - VERIFY_NOT_REACHED(); - return ""sv; - } -} - -StringView character_compare_type_name(CharacterCompareType ch_compare_type) -{ - switch (ch_compare_type) { -#define __ENUMERATE_CHARACTER_COMPARE_TYPE(x) \ - case CharacterCompareType::x: \ - return #x##sv; - ENUMERATE_CHARACTER_COMPARE_TYPES -#undef __ENUMERATE_CHARACTER_COMPARE_TYPE - default: - VERIFY_NOT_REACHED(); - return ""sv; - } -} - -StringView character_class_name(CharClass ch_class) -{ - switch (ch_class) { -#define __ENUMERATE_CHARACTER_CLASS(x) \ - case CharClass::x: \ - return #x##sv; - ENUMERATE_CHARACTER_CLASSES -#undef __ENUMERATE_CHARACTER_CLASS - default: - VERIFY_NOT_REACHED(); - return ""sv; - } -} - -static bool is_word_character(u32 code_point, bool case_insensitive, bool unicode_mode) -{ - if (is_ascii_alphanumeric(code_point) || code_point == '_') - return true; - - if (case_insensitive && unicode_mode) { - auto canonical = Unicode::canonicalize(code_point, unicode_mode); - if (is_ascii_alphanumeric(canonical) || canonical == '_') - return true; - } - - return false; -} - -size_t ByteCode::s_next_checkpoint_serial_id { 0 }; -u32 s_next_string_table_serial { 1 }; -static u32 s_next_string_set_table_serial { 1 }; - -StringSetTable::StringSetTable() - : m_serial(s_next_string_set_table_serial++) -{ -} - -StringSetTable::~StringSetTable() -{ - if (m_serial == s_next_string_set_table_serial - 1 && m_u8_tries.is_empty()) - --s_next_string_set_table_serial; -} - -StringSetTable::StringSetTable(StringSetTable const& other) - : m_serial(s_next_string_set_table_serial++) -{ - for (auto const& entry : other.m_u8_tries) - m_u8_tries.set(entry.key, MUST(const_cast(entry.value).deep_copy())); - for (auto const& entry : other.m_u16_tries) - m_u16_tries.set(entry.key, MUST(const_cast(entry.value).deep_copy())); -} - -StringSetTable& StringSetTable::operator=(StringSetTable const& other) -{ - if (this != &other) { - m_u8_tries.clear(); - m_u16_tries.clear(); - for (auto const& entry : other.m_u8_tries) - m_u8_tries.set(entry.key, MUST(const_cast(entry.value).deep_copy())); - for (auto const& entry : other.m_u16_tries) - m_u16_tries.set(entry.key, MUST(const_cast(entry.value).deep_copy())); - } - return *this; -} - -bool matches_character_class(CharClass character_class, u32 ch, bool insensitive, bool unicode_mode) -{ - constexpr auto is_space_or_line_terminator = [](u32 code_point) { - if ((code_point == 0x0a) || (code_point == 0x0d) || (code_point == 0x2028) || (code_point == 0x2029)) - return true; - if ((code_point == 0x09) || (code_point == 0x0b) || (code_point == 0x0c) || (code_point == 0xfeff)) - return true; - return Unicode::code_point_has_space_separator_general_category(code_point); - }; - - switch (character_class) { - case CharClass::Alnum: - return is_ascii_alphanumeric(ch); - case CharClass::Alpha: - return is_ascii_alpha(ch); - case CharClass::Blank: - return is_ascii_blank(ch); - case CharClass::Cntrl: - return is_ascii_control(ch); - case CharClass::Digit: - return is_ascii_digit(ch); - case CharClass::Graph: - return is_ascii_graphical(ch); - case CharClass::Lower: - return is_ascii_lower_alpha(ch) || (insensitive && is_ascii_upper_alpha(ch)); - case CharClass::Print: - return is_ascii_printable(ch); - case CharClass::Punct: - return is_ascii_punctuation(ch); - case CharClass::Space: - return is_space_or_line_terminator(ch); - case CharClass::Upper: - return is_ascii_upper_alpha(ch) || (insensitive && is_ascii_lower_alpha(ch)); - case CharClass::Word: - return is_word_character(ch, insensitive, unicode_mode); - case CharClass::Xdigit: - return is_ascii_hex_digit(ch); - } - - VERIFY_NOT_REACHED(); -} - -ByteString opcode_arguments_string(OpCodeId id, ByteCodeValueType const* data, size_t ip, MatchState const& state, ByteCodeBase const& bytecode) -{ - // argument(N) = data[ip + 1 + N] - auto arg = [&](size_t n) -> ByteCodeValueType { return data[ip + 1 + n]; }; - auto sz = opcode_size(id, data, ip); - - switch (id) { - case OpCodeId::SaveModifiers: - return ByteString::formatted("new_modifiers={:#x}", arg(0)); - case OpCodeId::RestoreModifiers: - case OpCodeId::Exit: - case OpCodeId::FailForks: - case OpCodeId::PopSaved: - case OpCodeId::Save: - case OpCodeId::Restore: - case OpCodeId::CheckBegin: - case OpCodeId::CheckEnd: - return ByteString::empty(); - case OpCodeId::GoBack: - return ByteString::formatted("count={}", arg(0)); - case OpCodeId::SetStepBack: - return ByteString::formatted("step={}", static_cast(arg(0))); - case OpCodeId::IncStepBack: - return ByteString::formatted("inc step back"); - case OpCodeId::CheckStepBack: - return ByteString::formatted("check step back"); - case OpCodeId::CheckSavedPosition: - return ByteString::formatted("check saved back"); - case OpCodeId::Jump: - return ByteString::formatted("offset={} [&{}]", static_cast(arg(0)), ip + sz + static_cast(arg(0))); - case OpCodeId::ForkJump: - return ByteString::formatted("offset={} [&{}], sp: {}", static_cast(arg(0)), ip + sz + static_cast(arg(0)), state.string_position); - case OpCodeId::ForkReplaceJump: - return ByteString::formatted("offset={} [&{}], sp: {}", static_cast(arg(0)), ip + sz + static_cast(arg(0)), state.string_position); - case OpCodeId::ForkStay: - return ByteString::formatted("offset={} [&{}], sp: {}", static_cast(arg(0)), ip + sz + static_cast(arg(0)), state.string_position); - case OpCodeId::ForkReplaceStay: - return ByteString::formatted("offset={} [&{}], sp: {}", static_cast(arg(0)), ip + sz + static_cast(arg(0)), state.string_position); - case OpCodeId::CheckBoundary: - return ByteString::formatted("kind={} ({})", static_cast(arg(0)), boundary_check_type_name(static_cast(arg(0)))); - case OpCodeId::ClearCaptureGroup: - case OpCodeId::SaveLeftCaptureGroup: - case OpCodeId::SaveRightCaptureGroup: - case OpCodeId::Checkpoint: - return ByteString::formatted("id={}", arg(0)); - case OpCodeId::FailIfEmpty: - return ByteString::formatted("checkpoint={}", arg(0)); - case OpCodeId::SaveRightNamedCaptureGroup: - return ByteString::formatted("name_id={}, id={}", arg(0), arg(1)); - case OpCodeId::RSeekTo: { - auto ch = arg(0); - if (ch <= 0x7f) - return ByteString::formatted("before '{}'", ch); - return ByteString::formatted("before u+{:04x}", arg(0)); - } - case OpCodeId::Compare: - return ByteString::formatted("argc={}, args={} ", arg(0), arg(1)); - case OpCodeId::CompareSimple: { - StringBuilder builder; - auto type = static_cast(arg(1)); - builder.append(character_compare_type_name(type)); - switch (type) { - case CharacterCompareType::Char: { - auto ch = arg(2); - if (is_ascii_printable(ch)) - builder.append(ByteString::formatted(" '{:c}'", static_cast(ch))); - else - builder.append(ByteString::formatted(" 0x{:x}", ch)); - break; - } - case CharacterCompareType::String: { - auto string_index = arg(2); - auto string = bytecode.get_u16_string(string_index); - builder.appendff(" \"{}\"", string); - break; - } - case CharacterCompareType::CharClass: { - auto character_class = static_cast(arg(2)); - builder.appendff(" {}", character_class_name(character_class)); - break; - } - case CharacterCompareType::Reference: { - auto ref = arg(2); - builder.appendff(" number={}", ref); - break; - } - case CharacterCompareType::NamedReference: { - auto ref = arg(2); - builder.appendff(" named_number={}", ref); - break; - } - case CharacterCompareType::GeneralCategory: - case CharacterCompareType::Property: - case CharacterCompareType::Script: - case CharacterCompareType::ScriptExtension: - case CharacterCompareType::StringSet: { - builder.appendff(" value={}", arg(2)); - break; - } - case CharacterCompareType::LookupTable: { - auto count_sensitive = arg(2); - auto count_insensitive = arg(3); - for (size_t j = 0; j < count_sensitive; ++j) { - auto range = static_cast(arg(4 + j)); - builder.appendff(" {:x}-{:x}", range.from, range.to); - } - if (count_insensitive > 0) { - builder.append(" [insensitive ranges:"sv); - for (size_t j = 0; j < count_insensitive; ++j) { - auto range = static_cast(arg(4 + count_sensitive + j)); - builder.appendff(" {:x}-{:x}", range.from, range.to); - } - builder.append(" ]"sv); - } - break; - } - case CharacterCompareType::CharRange: { - auto value = arg(2); - auto range = static_cast(value); - builder.appendff(" {:x}-{:x}", range.from, range.to); - break; - } - default: - break; - } - return builder.to_byte_string(); - } - case OpCodeId::Repeat: { - auto repeat_id = arg(2); - auto reps = repeat_id < state.repetition_marks.size() ? state.repetition_marks.at(repeat_id) : 0; - return ByteString::formatted("offset={} [&{}] count={} id={} rep={}, sp: {}", - static_cast(arg(0)), - ip - arg(0), - arg(1) + 1, - repeat_id, - reps + 1, - state.string_position); - } - case OpCodeId::ResetRepeat: { - auto repeat_id = arg(0); - auto reps = repeat_id < state.repetition_marks.size() ? state.repetition_marks.at(repeat_id) : 0; - return ByteString::formatted("id={} rep={}", repeat_id, reps + 1); - } - case OpCodeId::JumpNonEmpty: - return ByteString::formatted("{} offset={} [&{}], cp={}", - opcode_id_name(static_cast(arg(2))), - static_cast(arg(0)), ip + sz + static_cast(arg(0)), - arg(1)); - case OpCodeId::ForkIf: - return ByteString::formatted("{} {} offset={} [&{}]", - opcode_id_name(static_cast(arg(1))), - fork_if_condition_name(static_cast(arg(2))), - static_cast(arg(0)), ip + sz + static_cast(arg(0))); - } - VERIFY_NOT_REACHED(); -} - -Vector compare_variable_arguments_to_byte_string(ByteCodeValueType const* data, size_t ip, MatchState const& state, ByteCodeBase const& bytecode, Optional input) -{ - Vector result; - - size_t offset = ip + 3; - RegexStringView const& view = input.has_value() ? input.value().view : StringView {}; - - auto argument_count = data[ip + 1]; // arguments_count for Compare - - for (size_t i = 0; i < argument_count; ++i) { - auto compare_type = static_cast(data[offset++]); - result.empend(ByteString::formatted("type={} [{}]", static_cast(compare_type), character_compare_type_name(compare_type))); - - auto string_start_offset = state.string_position_before_match; - - if (compare_type == CharacterCompareType::Char) { - auto ch = data[offset++]; - auto is_ascii = is_ascii_printable(ch); - if (is_ascii) - result.empend(ByteString::formatted(" value='{:c}'", static_cast(ch))); - else - result.empend(ByteString::formatted(" value={:x}", ch)); - - if (!view.is_null() && view.length() > string_start_offset) { - if (is_ascii) { - result.empend(ByteString::formatted( - " compare against: '{}'", - view.substring_view(string_start_offset, string_start_offset > view.length() ? 0 : 1).to_byte_string())); - } else { - auto str = view.substring_view(string_start_offset, string_start_offset > view.length() ? 0 : 1).to_byte_string(); - u8 buf[8] { 0 }; - __builtin_memcpy(buf, str.characters(), min(str.length(), sizeof(buf))); - result.empend(ByteString::formatted(" compare against: {:x},{:x},{:x},{:x},{:x},{:x},{:x},{:x}", - buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7])); - } - } - } else if (compare_type == CharacterCompareType::Reference) { - auto ref = data[offset++]; - result.empend(ByteString::formatted(" number={}", ref)); - if (input.has_value()) { - if (state.capture_group_matches_size() > input->match_index) { - auto match = state.capture_group_matches(input->match_index); - if (match.size() > ref) { - auto& group = match[ref]; - result.empend(ByteString::formatted(" left={}", group.left_column)); - result.empend(ByteString::formatted(" right={}", group.left_column + group.view.length_in_code_units())); - result.empend(ByteString::formatted(" contents='{}'", group.view)); - } else { - result.empend(ByteString::formatted(" (invalid ref, max={})", match.size() - 1)); - } - } else { - result.empend(ByteString::formatted(" (invalid index {}, max={})", input->match_index, state.capture_group_matches_size() - 1)); - } - } - } else if (compare_type == CharacterCompareType::NamedReference) { - auto ref = data[offset++]; - result.empend(ByteString::formatted(" named_number={}", ref)); - if (input.has_value()) { - if (state.capture_group_matches_size() > input->match_index) { - auto match = state.capture_group_matches(input->match_index); - if (match.size() > ref) { - auto& group = match[ref]; - result.empend(ByteString::formatted(" left={}", group.left_column)); - result.empend(ByteString::formatted(" right={}", group.left_column + group.view.length_in_code_units())); - result.empend(ByteString::formatted(" contents='{}'", group.view)); - } else { - result.empend(ByteString::formatted(" (invalid ref {}, max={})", ref, match.size() - 1)); - } - } else { - result.empend(ByteString::formatted(" (invalid index {}, max={})", input->match_index, state.capture_group_matches_size() - 1)); - } - } - } else if (compare_type == CharacterCompareType::String) { - auto str_id = data[offset++]; - auto string = bytecode.get_u16_string(str_id); - result.empend(ByteString::formatted(" value=\"{}\"", string)); - if (!view.is_null() && view.length() > state.string_position) - result.empend(ByteString::formatted( - " compare against: \"{}\"", - input.value().view.substring_view(string_start_offset, string_start_offset + string.length_in_code_units() > view.length() ? 0 : string.length_in_code_units()).to_byte_string())); - } else if (compare_type == CharacterCompareType::CharClass) { - auto character_class = static_cast(data[offset++]); - result.empend(ByteString::formatted(" ch_class={} [{}]", static_cast(character_class), character_class_name(character_class))); - if (!view.is_null() && view.length() > state.string_position) - result.empend(ByteString::formatted( - " compare against: '{}'", - input.value().view.substring_view(string_start_offset, state.string_position > view.length() ? 0 : 1).to_byte_string())); - } else if (compare_type == CharacterCompareType::CharRange) { - auto value = static_cast(data[offset++]); - result.empend(ByteString::formatted(" ch_range={:x}-{:x}", value.from, value.to)); - if (!view.is_null() && view.length() > state.string_position) - result.empend(ByteString::formatted( - " compare against: '{}'", - input.value().view.substring_view(string_start_offset, state.string_position > view.length() ? 0 : 1).to_byte_string())); - } else if (compare_type == CharacterCompareType::LookupTable) { - auto count_sensitive = data[offset++]; - auto count_insensitive = data[offset++]; - for (size_t j = 0; j < count_sensitive; ++j) { - auto range = static_cast(data[offset++]); - result.append(ByteString::formatted(" {:x}-{:x}", range.from, range.to)); - } - if (count_insensitive > 0) { - result.append(" [insensitive ranges:"); - for (size_t j = 0; j < count_insensitive; ++j) { - auto range = static_cast(data[offset++]); - result.append(ByteString::formatted(" {:x}-{:x}", range.from, range.to)); - } - result.append(" ]"); - } - - if (!view.is_null() && view.length() > state.string_position) - result.empend(ByteString::formatted( - " compare against: '{}'", - input.value().view.substring_view(string_start_offset, state.string_position > view.length() ? 0 : 1).to_byte_string())); - } else if (compare_type == CharacterCompareType::GeneralCategory - || compare_type == CharacterCompareType::Property - || compare_type == CharacterCompareType::Script - || compare_type == CharacterCompareType::ScriptExtension - || compare_type == CharacterCompareType::StringSet) { - auto value = data[offset++]; - result.empend(ByteString::formatted(" value={}", value)); - } - } - return result; -} - -Vector flat_compares_at(ByteCodeValueType const* data, size_t ip, bool is_simple) -{ - Vector result; - - size_t offset = ip + (is_simple ? 2 : 3); - auto argument_count = is_simple ? 1 : data[ip + OpArgs::Compare::arguments_count]; - - for (size_t i = 0; i < argument_count; ++i) { - auto compare_type = (CharacterCompareType)data[offset++]; - - if (compare_type == CharacterCompareType::Char) { - auto ch = data[offset++]; - result.append({ compare_type, ch }); - } else if (compare_type == CharacterCompareType::Reference) { - auto ref = data[offset++]; - result.append({ compare_type, ref }); - } else if (compare_type == CharacterCompareType::NamedReference) { - auto ref = data[offset++]; - result.append({ compare_type, ref }); - } else if (compare_type == CharacterCompareType::String) { - auto string_index = data[offset++]; - result.append({ compare_type, string_index }); - } else if (compare_type == CharacterCompareType::CharClass) { - auto character_class = data[offset++]; - result.append({ compare_type, character_class }); - } else if (compare_type == CharacterCompareType::CharRange) { - auto value = data[offset++]; - result.append({ compare_type, value }); - } else if (compare_type == CharacterCompareType::LookupTable) { - auto count_sensitive = data[offset++]; - auto count_insensitive = data[offset++]; - for (size_t j = 0; j < count_sensitive; ++j) - result.append({ CharacterCompareType::CharRange, data[offset++] }); - offset += count_insensitive; // Skip insensitive ranges - } else if (compare_type == CharacterCompareType::GeneralCategory - || compare_type == CharacterCompareType::Property - || compare_type == CharacterCompareType::Script - || compare_type == CharacterCompareType::ScriptExtension - || compare_type == CharacterCompareType::StringSet) { - auto value = data[offset++]; - result.append({ compare_type, value }); - } else { - result.append({ compare_type, 0 }); - } - } - return result; -} - -} diff --git a/Libraries/LibRegex/RegexByteCode.h b/Libraries/LibRegex/RegexByteCode.h deleted file mode 100644 index 3e36bcbee2..0000000000 --- a/Libraries/LibRegex/RegexByteCode.h +++ /dev/null @@ -1,1027 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include "RegexBytecodeStreamOptimizer.h" -#include "RegexMatch.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace regex { - -using ByteCodeValueType = u64; - -#define ENUMERATE_OPCODES \ - __ENUMERATE_OPCODE(Compare) \ - __ENUMERATE_OPCODE(Jump) \ - __ENUMERATE_OPCODE(JumpNonEmpty) \ - __ENUMERATE_OPCODE(ForkJump) \ - __ENUMERATE_OPCODE(ForkStay) \ - __ENUMERATE_OPCODE(ForkReplaceJump) \ - __ENUMERATE_OPCODE(ForkReplaceStay) \ - __ENUMERATE_OPCODE(ForkIf) \ - __ENUMERATE_OPCODE(FailForks) \ - __ENUMERATE_OPCODE(PopSaved) \ - __ENUMERATE_OPCODE(SaveLeftCaptureGroup) \ - __ENUMERATE_OPCODE(SaveRightCaptureGroup) \ - __ENUMERATE_OPCODE(SaveRightNamedCaptureGroup) \ - __ENUMERATE_OPCODE(RSeekTo) \ - __ENUMERATE_OPCODE(CheckBegin) \ - __ENUMERATE_OPCODE(CheckEnd) \ - __ENUMERATE_OPCODE(CheckBoundary) \ - __ENUMERATE_OPCODE(Save) \ - __ENUMERATE_OPCODE(Restore) \ - __ENUMERATE_OPCODE(GoBack) \ - __ENUMERATE_OPCODE(SetStepBack) \ - __ENUMERATE_OPCODE(IncStepBack) \ - __ENUMERATE_OPCODE(CheckStepBack) \ - __ENUMERATE_OPCODE(CheckSavedPosition) \ - __ENUMERATE_OPCODE(ClearCaptureGroup) \ - __ENUMERATE_OPCODE(FailIfEmpty) \ - __ENUMERATE_OPCODE(Repeat) \ - __ENUMERATE_OPCODE(ResetRepeat) \ - __ENUMERATE_OPCODE(Checkpoint) \ - __ENUMERATE_OPCODE(CompareSimple) \ - __ENUMERATE_OPCODE(SaveModifiers) \ - __ENUMERATE_OPCODE(RestoreModifiers) \ - __ENUMERATE_OPCODE(Exit) - -// clang-format off -enum class OpCodeId : ByteCodeValueType { -#define __ENUMERATE_OPCODE(x) x, - ENUMERATE_OPCODES -#undef __ENUMERATE_OPCODE - - First = Compare, - Last = Exit, -}; -// clang-format on - -#define ENUMERATE_CHARACTER_COMPARE_TYPES \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(Undefined) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(Inverse) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(TemporaryInverse) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(AnyChar) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(Char) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(String) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(CharClass) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(CharRange) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(Reference) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(NamedReference) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(Property) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(GeneralCategory) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(Script) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(ScriptExtension) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(RangeExpressionDummy) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(LookupTable) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(And) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(Or) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(EndAndOr) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(Subtract) \ - __ENUMERATE_CHARACTER_COMPARE_TYPE(StringSet) - -enum class CharacterCompareType : ByteCodeValueType { -#define __ENUMERATE_CHARACTER_COMPARE_TYPE(x) x, - ENUMERATE_CHARACTER_COMPARE_TYPES -#undef __ENUMERATE_CHARACTER_COMPARE_TYPE -}; - -#define ENUMERATE_CHARACTER_CLASSES \ - __ENUMERATE_CHARACTER_CLASS(Alnum) \ - __ENUMERATE_CHARACTER_CLASS(Cntrl) \ - __ENUMERATE_CHARACTER_CLASS(Lower) \ - __ENUMERATE_CHARACTER_CLASS(Space) \ - __ENUMERATE_CHARACTER_CLASS(Alpha) \ - __ENUMERATE_CHARACTER_CLASS(Digit) \ - __ENUMERATE_CHARACTER_CLASS(Print) \ - __ENUMERATE_CHARACTER_CLASS(Upper) \ - __ENUMERATE_CHARACTER_CLASS(Blank) \ - __ENUMERATE_CHARACTER_CLASS(Graph) \ - __ENUMERATE_CHARACTER_CLASS(Punct) \ - __ENUMERATE_CHARACTER_CLASS(Word) \ - __ENUMERATE_CHARACTER_CLASS(Xdigit) - -enum class CharClass : ByteCodeValueType { -#define __ENUMERATE_CHARACTER_CLASS(x) x, - ENUMERATE_CHARACTER_CLASSES -#undef __ENUMERATE_CHARACTER_CLASS -}; - -#define ENUMERATE_BOUNDARY_CHECK_TYPES \ - __ENUMERATE_BOUNDARY_CHECK_TYPE(Word) \ - __ENUMERATE_BOUNDARY_CHECK_TYPE(NonWord) - -enum class BoundaryCheckType : ByteCodeValueType { -#define __ENUMERATE_BOUNDARY_CHECK_TYPE(x) x, - ENUMERATE_BOUNDARY_CHECK_TYPES -#undef __ENUMERATE_BOUNDARY_CHECK_TYPE -}; - -#define ENUMERATE_FORK_IF_CONDITIONS \ - __ENUMERATE_FORK_IF_CONDITION(AtStartOfLine) \ - __ENUMERATE_FORK_IF_CONDITION(Invalid) /* Must be last */ - -enum class ForkIfCondition : ByteCodeValueType { -#define __ENUMERATE_FORK_IF_CONDITION(x) x, - ENUMERATE_FORK_IF_CONDITIONS -#undef __ENUMERATE_FORK_IF_CONDITION -}; - -struct CharRange { - u32 from; - u32 to; - - CharRange(u64 value) - : from(value >> 32) - , to(value & 0xffffffff) - { - } - - CharRange(u32 from, u32 to) - : from(from) - , to(to) - { - } - - operator ByteCodeValueType() const { return ((u64)from << 32) | to; } -}; - -struct CompareTypeAndValuePair { - CharacterCompareType type; - ByteCodeValueType value; -}; - -REGEX_API extern u32 s_next_string_table_serial; - -template -struct StringTable { - StringTable() - : m_serial(s_next_string_table_serial++) - { - } - ~StringTable() - { - if (m_serial != 0) { - if (m_serial == s_next_string_table_serial - 1 && m_table.is_empty()) - --s_next_string_table_serial; // We didn't use this serial, put it back. - } - } - StringTable(StringTable const& other) - { - // Pull a new serial for this copy - m_serial = s_next_string_table_serial++; - m_table = other.m_table; - m_inverse_table = other.m_inverse_table; - } - StringTable(StringTable&& other) - { - m_serial = other.m_serial; - m_table = move(other.m_table); - m_inverse_table = move(other.m_inverse_table); - // Clear other's data to avoid double-deletion of serial - other.m_serial = 0; - } - StringTable& operator=(StringTable const& other) - { - if (this != &other) { - m_serial = s_next_string_table_serial++; - m_table = other.m_table; - m_inverse_table = other.m_inverse_table; - } - return *this; - } - StringTable& operator=(StringTable&& other) - { - if (this != &other) { - m_serial = other.m_serial; - m_table = move(other.m_table); - m_inverse_table = move(other.m_inverse_table); - // Clear other's data to avoid double-deletion of serial - other.m_serial = 0; - } - return *this; - } - - ByteCodeValueType set(StringType string) - { - u32 local_index = m_table.size() + 0x4242; - ByteCodeValueType global_index; - if (auto maybe_local_index = m_table.get(string); maybe_local_index.has_value()) { - local_index = maybe_local_index.value(); - global_index = static_cast(m_serial) << 32 | static_cast(local_index); - } else { - global_index = static_cast(m_serial) << 32 | static_cast(local_index); - m_table.set(string, global_index); - m_inverse_table.set(global_index, string); - } - - return global_index; - } - - StringType get(ByteCodeValueType index) const - { - return m_inverse_table.get(index).value(); - } - - u32 m_serial { 0 }; - HashMap m_table; - HashMap m_inverse_table; -}; - -using StringSetTrie = Trie; - -struct REGEX_API StringSetTable { - StringSetTable(); - ~StringSetTable(); - StringSetTable(StringSetTable const& other); - StringSetTable(StringSetTable&&) = default; - StringSetTable& operator=(StringSetTable const& other); - StringSetTable& operator=(StringSetTable&&) = default; - - ByteCodeValueType set(Vector const& strings) - { - u32 local_index = m_u8_tries.size(); - ByteCodeValueType global_index = static_cast(m_serial) << 32 | static_cast(local_index); - - StringSetTrie u8_trie { 0, false }; - StringSetTrie u16_trie { 0, false }; - - for (auto const& str : strings) { - Vector code_points; - Utf8View utf8_view { str.bytes_as_string_view() }; - for (auto code_point : utf8_view) - code_points.append(code_point); - - (void)u8_trie.insert(code_points.begin(), code_points.end(), true, [](auto&, auto) { return false; }); - - auto utf16_string = Utf16String::from_utf32({ code_points.data(), code_points.size() }); - Vector u16_code_units; - auto utf16_view = utf16_string.utf16_view(); - for (size_t i = 0; i < utf16_view.length_in_code_units(); i++) { - auto code_unit = utf16_view.code_unit_at(i); - u16_code_units.append(code_unit); - } - (void)u16_trie.insert(u16_code_units.begin(), u16_code_units.end(), true, [](auto&, auto) { return false; }); - } - - m_u8_tries.set(global_index, move(u8_trie)); - m_u16_tries.set(global_index, move(u16_trie)); - return global_index; - } - - StringSetTrie const& get_u8_trie(ByteCodeValueType index) const - { - return m_u8_tries.get(index).value(); - } - - StringSetTrie const& get_u16_trie(ByteCodeValueType index) const - { - return m_u16_tries.get(index).value(); - } - - u32 m_serial { 0 }; - HashMap m_u8_tries; - HashMap m_u16_tries; -}; - -struct ByteCodeBase { - FlyString get_string(size_t index) const { return m_string_table.get(index); } - auto const& string_table() const { return m_string_table; } - - auto get_u16_string(size_t index) const { return m_u16_string_table.get(index); } - auto const& u16_string_table() const { return m_u16_string_table; } - - auto const& string_set_table() const { return m_string_set_table; } - auto& string_set_table() { return m_string_set_table; } - - Optional get_group_name_index(size_t group_index) const - { - return m_group_name_mappings.get(group_index); - } - -protected: - StringTable m_string_table; - StringTable m_u16_string_table; - StringSetTable m_string_set_table; - HashMap m_group_name_mappings; -}; - -class REGEX_API ByteCode : public ByteCodeBase - , public DisjointChunks { - using Base = DisjointChunks; - friend class FlatByteCode; - -public: - using Base::append; - - ByteCode() = default; - - ByteCode(ByteCode const&) = default; - ByteCode(ByteCode&&) = default; - - ByteCode(Base&&) = delete; - ByteCode(Base const&) = delete; - - ~ByteCode() = default; - - ByteCode& operator=(ByteCode const&) = default; - ByteCode& operator=(ByteCode&&) = default; - - ByteCode& operator=(Base&& value) = delete; - ByteCode& operator=(Base const& value) = delete; - - void extend(ByteCode&& other) - { - merge_string_tables_from({ &other, 1 }); - Base::extend(move(other)); - } - - void extend(ByteCode const& other) - { - merge_string_tables_from({ &other, 1 }); - Base::extend(other); - } - - template> T> - void extend(T other) - { - Base::append(move(other)); - } - - template - void empend(Args&&... args) - { - if (is_empty()) - Base::append({}); - Base::last_chunk().empend(forward(args)...); - } - template - void append(T&& value) - { - if (is_empty()) - Base::append({}); - Base::last_chunk().append(forward(value)); - } - template - void prepend(T&& value) - { - if (is_empty()) - return append(forward(value)); - Base::first_chunk().prepend(forward(value)); - } - - void append(Span value) - { - if (is_empty()) - Base::append({}); - auto& last = Base::last_chunk(); - last.ensure_capacity(value.size()); - for (auto v : value) - last.unchecked_append(v); - } - - void ensure_capacity(size_t capacity) - { - if (is_empty()) - Base::append({}); - Base::last_chunk().ensure_capacity(capacity); - } - - void last_chunk() const = delete; - void first_chunk() const = delete; - - void merge_string_tables_from(Span others) - { - for (auto const& other : others) { - for (auto const& entry : other.m_string_table.m_table) { - auto const result = m_string_table.m_inverse_table.set(entry.value, entry.key); - if (result != HashSetResult::InsertedNewEntry) { - if (m_string_table.m_inverse_table.get(entry.value) == entry.key) // Already in inverse table. - continue; - dbgln("StringTable: Detected ID clash in string tables! ID {} seems to be reused", entry.value); - dbgln("Old: {}, New: {}", m_string_table.m_inverse_table.get(entry.value), entry.key); - VERIFY_NOT_REACHED(); - } - m_string_table.m_table.set(entry.key, entry.value); - } - m_string_table.m_inverse_table.update(other.m_string_table.m_inverse_table); - - for (auto const& entry : other.m_u16_string_table.m_table) { - auto const result = m_u16_string_table.m_inverse_table.set(entry.value, entry.key); - if (result != HashSetResult::InsertedNewEntry) { - if (m_u16_string_table.m_inverse_table.get(entry.value) == entry.key) // Already in inverse table. - continue; - dbgln("StringTable: Detected ID clash in string tables! ID {} seems to be reused", entry.value); - dbgln("Old: {}, New: {}", m_u16_string_table.m_inverse_table.get(entry.value), entry.key); - VERIFY_NOT_REACHED(); - } - m_u16_string_table.m_table.set(entry.key, entry.value); - } - m_u16_string_table.m_inverse_table.update(other.m_u16_string_table.m_inverse_table); - - for (auto const& entry : other.m_string_set_table.m_u8_tries) { - m_string_set_table.m_u8_tries.set(entry.key, MUST(const_cast(entry.value).deep_copy())); - } - for (auto const& entry : other.m_string_set_table.m_u16_tries) { - m_string_set_table.m_u16_tries.set(entry.key, MUST(const_cast(entry.value).deep_copy())); - } - - for (auto const& mapping : other.m_group_name_mappings) { - m_group_name_mappings.set(mapping.key, mapping.value); - } - } - } - - void insert_bytecode_compare_values(Vector&& pairs) - { - Optimizer::append_character_class(*this, move(pairs)); - } - - void insert_bytecode_check_boundary(BoundaryCheckType type) - { - ByteCode bytecode; - bytecode.empend((ByteCodeValueType)OpCodeId::CheckBoundary); - bytecode.empend((ByteCodeValueType)type); - - extend(move(bytecode)); - } - - void insert_bytecode_clear_capture_group(size_t index) - { - empend(static_cast(OpCodeId::ClearCaptureGroup)); - empend(index); - } - - void insert_bytecode_compare_string(Utf16FlyString string) - { - empend(static_cast(OpCodeId::Compare)); - empend(static_cast(1)); // number of arguments - empend(static_cast(2)); // size of arguments - empend(static_cast(CharacterCompareType::String)); - auto index = m_u16_string_table.set(move(string)); - empend(index); - } - - void insert_bytecode_group_capture_left(size_t capture_groups_count) - { - empend(static_cast(OpCodeId::SaveLeftCaptureGroup)); - empend(capture_groups_count); - } - - void insert_bytecode_group_capture_right(size_t capture_groups_count) - { - empend(static_cast(OpCodeId::SaveRightCaptureGroup)); - empend(capture_groups_count); - } - - void insert_bytecode_group_capture_right(size_t capture_groups_count, FlyString name) - { - empend(static_cast(OpCodeId::SaveRightNamedCaptureGroup)); - auto name_string_index = m_string_table.set(move(name)); - empend(name_string_index); - empend(capture_groups_count); - - m_group_name_mappings.set(capture_groups_count - 1, name_string_index); - } - - void insert_bytecode_save_modifiers(FlagsUnderlyingType new_modifiers) - { - empend(static_cast(OpCodeId::SaveModifiers)); - empend(static_cast(new_modifiers)); - } - - void insert_bytecode_restore_modifiers() - { - empend(static_cast(OpCodeId::RestoreModifiers)); - } - - enum class LookAroundType { - LookAhead, - LookBehind, - NegatedLookAhead, - NegatedLookBehind, - }; - void insert_bytecode_lookaround(ByteCode&& lookaround_body, LookAroundType type, size_t match_length = 0, bool greedy_lookaround = true) - { - // FIXME: The save stack will grow infinitely with repeated failures - // as we do not discard that on failure (we don't necessarily know how many to pop with the current architecture). - switch (type) { - case LookAroundType::LookAhead: { - // SAVE - // FORKJUMP _BODY - // POPSAVED - // LABEL _BODY - // REGEXP BODY - // RESTORE - empend((ByteCodeValueType)OpCodeId::Save); - empend((ByteCodeValueType)OpCodeId::ForkJump); - empend((ByteCodeValueType)1); - empend((ByteCodeValueType)OpCodeId::PopSaved); - extend(move(lookaround_body)); - empend((ByteCodeValueType)OpCodeId::Restore); - return; - } - case LookAroundType::NegatedLookAhead: { - // JUMP _A - // LABEL _L - // REGEXP BODY - // FAIL - // LABEL _A - // SAVE - // FORKJUMP _L - // RESTORE - auto body_length = lookaround_body.size(); - empend((ByteCodeValueType)OpCodeId::Jump); - empend((ByteCodeValueType)body_length + 1); // JUMP to label _A - extend(move(lookaround_body)); - empend((ByteCodeValueType)OpCodeId::FailForks); - empend((ByteCodeValueType)OpCodeId::Save); - empend((ByteCodeValueType)OpCodeId::ForkJump); - empend((ByteCodeValueType) - (body_length + 4)); // JUMP to label _L - empend((ByteCodeValueType)OpCodeId::Restore); - return; - } - case LookAroundType::LookBehind: { - // SAVE - // SET_STEPBACK match_length(BODY)-1 - // LABEL _START - // INC_STEPBACK - // FORK_JUMP _BODY - // CHECK_STEPBACK - // JUMP _START - // LABEL _BODY - // REGEX BODY - // CHECK_SAVED_POSITION - // RESTORE - auto body_length = lookaround_body.size(); - empend((ByteCodeValueType)OpCodeId::Save); - empend((ByteCodeValueType)OpCodeId::SetStepBack); - empend((ByteCodeValueType)match_length - 1); - empend((ByteCodeValueType)OpCodeId::IncStepBack); - empend((ByteCodeValueType)OpCodeId::ForkJump); - empend((ByteCodeValueType)1 + 2); // JUMP to label _BODY - empend((ByteCodeValueType)OpCodeId::CheckStepBack); - empend((ByteCodeValueType)OpCodeId::Jump); - empend((ByteCodeValueType)-6); // JUMP to label _START - extend(move(lookaround_body)); - if (greedy_lookaround) { - empend((ByteCodeValueType)OpCodeId::ForkJump); - empend((ByteCodeValueType)(0 - 2 - body_length - 6)); - } - empend((ByteCodeValueType)OpCodeId::CheckSavedPosition); - empend((ByteCodeValueType)OpCodeId::Restore); - return; - } - case LookAroundType::NegatedLookBehind: { - // JUMP _A - // LABEL _L - // GOBACK match_length(BODY) - // REGEXP BODY - // FAIL - // LABEL _A - // SAVE - // FORKJUMP _L - // RESTORE - auto body_length = lookaround_body.size(); - empend((ByteCodeValueType)OpCodeId::Jump); - empend((ByteCodeValueType)body_length + 3); // JUMP to label _A - empend((ByteCodeValueType)OpCodeId::GoBack); - empend((ByteCodeValueType)match_length); - extend(move(lookaround_body)); - empend((ByteCodeValueType)OpCodeId::FailForks); - empend((ByteCodeValueType)OpCodeId::Save); - empend((ByteCodeValueType)OpCodeId::ForkJump); - empend((ByteCodeValueType) - (body_length + 6)); // JUMP to label _L - empend((ByteCodeValueType)OpCodeId::Restore); - return; - } - } - - VERIFY_NOT_REACHED(); - } - - void insert_bytecode_alternation(ByteCode&& left, ByteCode&& right) - { - - // FORKJUMP _ALT - // REGEXP ALT2 - // JUMP _END - // LABEL _ALT - // REGEXP ALT1 - // LABEL _END - - // Optimisation: Eliminate extra work by unifying common pre-and-postfix exprs. - Optimizer::append_alternation(*this, move(left), move(right)); - } - - template - static void transform_bytecode_repetition_min_max(ByteCode& bytecode_to_repeat, T minimum, Optional maximum, size_t min_repetition_mark_id, size_t max_repetition_mark_id, bool greedy = true) - { - if (!maximum.has_value()) { - if (minimum == 0) - return transform_bytecode_repetition_any(bytecode_to_repeat, greedy); - if (minimum == 1) - return transform_bytecode_repetition_min_one(bytecode_to_repeat, greedy); - } - - if (minimum == 0 && maximum.has_value() && maximum.value() == 1) { - return transform_bytecode_repetition_zero_or_one(bytecode_to_repeat, greedy); - } - - ByteCode new_bytecode; - new_bytecode.insert_bytecode_repetition_n(bytecode_to_repeat, minimum, min_repetition_mark_id); - - if (maximum.has_value()) { - // (REPEAT REGEXP MIN) - // LABEL _MAX_LOOP | - // FORK END | - // CHECKPOINT (if min==0) | - // REGEXP | - // FAILIFEMPTY (if min==0) | - // REPEAT _MAX_LOOP MAX-MIN | if max > min - // FORK END | - // CHECKPOINT (if min==0) | - // REGEXP | - // FAILIFEMPTY (if min==0) | - // LABEL END | - // RESET _MAX_LOOP | - auto jump_kind = static_cast(greedy ? OpCodeId::ForkStay : OpCodeId::ForkJump); - if (maximum.value() > minimum) { - new_bytecode.empend(jump_kind); - new_bytecode.empend((ByteCodeValueType)0); // Placeholder for the jump target. - auto pre_loop_fork_jump_index = new_bytecode.size(); - - auto checkpoint1 = minimum == 0 ? s_next_checkpoint_serial_id++ : 0; - if (minimum == 0) { - new_bytecode.empend(static_cast(OpCodeId::Checkpoint)); - new_bytecode.empend(static_cast(checkpoint1)); - } - - new_bytecode.extend(bytecode_to_repeat); - - if (minimum == 0) { - new_bytecode.empend(static_cast(OpCodeId::FailIfEmpty)); - new_bytecode.empend(checkpoint1); - } - - auto repetitions = maximum.value() - minimum; - auto fork_jump_address = new_bytecode.size(); - if (repetitions > 1) { - auto repeated_bytecode_size = bytecode_to_repeat.size(); - if (minimum == 0) - repeated_bytecode_size += 4; // Checkpoint + FailIfEmpty - - new_bytecode.empend((ByteCodeValueType)OpCodeId::Repeat); - new_bytecode.empend(repeated_bytecode_size + 2); - new_bytecode.empend(static_cast(repetitions - 1)); - new_bytecode.empend(max_repetition_mark_id); - new_bytecode.empend(jump_kind); - new_bytecode.empend((ByteCodeValueType)0); // Placeholder for the jump target. - auto post_loop_fork_jump_index = new_bytecode.size(); - - auto checkpoint2 = minimum == 0 ? s_next_checkpoint_serial_id++ : 0; - if (minimum == 0) { - new_bytecode.empend(static_cast(OpCodeId::Checkpoint)); - new_bytecode.empend(static_cast(checkpoint2)); - } - - new_bytecode.extend(bytecode_to_repeat); - - if (minimum == 0) { - new_bytecode.empend(static_cast(OpCodeId::FailIfEmpty)); - new_bytecode.empend(checkpoint2); - } - - fork_jump_address = new_bytecode.size(); - - new_bytecode[post_loop_fork_jump_index - 1] = (ByteCodeValueType)(fork_jump_address - post_loop_fork_jump_index); - - new_bytecode.empend((ByteCodeValueType)OpCodeId::ResetRepeat); - new_bytecode.empend((ByteCodeValueType)max_repetition_mark_id); - } - new_bytecode[pre_loop_fork_jump_index - 1] = (ByteCodeValueType)(fork_jump_address - pre_loop_fork_jump_index); - } - } else { - // no maximum value set, repeat finding if possible: - // (REPEAT REGEXP MIN) - // LABEL _START - // CHECKPOINT _C - // REGEXP - // JUMP_NONEMPTY _C _START FORK - - // Note: This is only safe because REPEAT will leave one iteration outside (see repetition_n) - auto checkpoint = s_next_checkpoint_serial_id++; - new_bytecode.insert(new_bytecode.size() - bytecode_to_repeat.size(), (ByteCodeValueType)OpCodeId::Checkpoint); - new_bytecode.insert(new_bytecode.size() - bytecode_to_repeat.size(), (ByteCodeValueType)checkpoint); - - auto jump_kind = static_cast(greedy ? OpCodeId::ForkJump : OpCodeId::ForkStay); - new_bytecode.empend((ByteCodeValueType)OpCodeId::JumpNonEmpty); - new_bytecode.empend(-bytecode_to_repeat.size() - 4 - 2); // Jump to the last iteration - new_bytecode.empend(checkpoint); // if _C is not empty. - new_bytecode.empend(jump_kind); - } - - bytecode_to_repeat = move(new_bytecode); - } - - template - void insert_bytecode_repetition_n(ByteCode& bytecode_to_repeat, T n, size_t repetition_mark_id) - { - // LABEL _LOOP - // REGEXP - // REPEAT _LOOP N-1 - // REGEXP - if (n == 0) - return; - - // Note: this bytecode layout allows callers to repeat the last REGEXP instruction without the - // REPEAT instruction forcing another loop. - extend(bytecode_to_repeat); - - if (n > 1) { - empend(static_cast(OpCodeId::Repeat)); - empend(bytecode_to_repeat.size()); - empend(static_cast(n - 1)); - empend(repetition_mark_id); - - extend(bytecode_to_repeat); - } - } - - static void transform_bytecode_repetition_min_one(ByteCode& bytecode_to_repeat, bool greedy) - { - // LABEL _START = -bytecode_to_repeat.size() - // CHECKPOINT _C - // REGEXP - // JUMP_NONEMPTY _C _START FORKSTAY (FORKJUMP -> Greedy) - - auto checkpoint = s_next_checkpoint_serial_id++; - bytecode_to_repeat.prepend((ByteCodeValueType)checkpoint); - bytecode_to_repeat.prepend((ByteCodeValueType)OpCodeId::Checkpoint); - - bytecode_to_repeat.empend((ByteCodeValueType)OpCodeId::JumpNonEmpty); - bytecode_to_repeat.empend(-bytecode_to_repeat.size() - 3); // Jump to the _START label... - bytecode_to_repeat.empend(checkpoint); // ...if _C is not empty - - if (greedy) - bytecode_to_repeat.empend(static_cast(OpCodeId::ForkJump)); - else - bytecode_to_repeat.empend(static_cast(OpCodeId::ForkStay)); - } - - static void transform_bytecode_repetition_any(ByteCode& bytecode_to_repeat, bool greedy) - { - // LABEL _START - // FORKJUMP _END (FORKSTAY -> Greedy) - // CHECKPOINT _C - // REGEXP - // FAILIFEMPTY _C - // JUMP_NONEMPTY _C _START JUMP - // LABEL _END - - // LABEL _START = m_bytes.size(); - ByteCode bytecode; - - if (greedy) - bytecode.empend(static_cast(OpCodeId::ForkStay)); - else - bytecode.empend(static_cast(OpCodeId::ForkJump)); - - bytecode.empend(bytecode_to_repeat.size() + 2 + 4 + 2); // Jump to the _END label - - auto checkpoint = s_next_checkpoint_serial_id++; - bytecode.empend(static_cast(OpCodeId::Checkpoint)); - bytecode.empend(static_cast(checkpoint)); - - bytecode.extend(bytecode_to_repeat); - - bytecode.empend(static_cast(OpCodeId::FailIfEmpty)); - bytecode.empend(checkpoint); - - bytecode.empend(static_cast(OpCodeId::JumpNonEmpty)); - bytecode.empend(-bytecode.size() - 3); // Jump(...) to the _START label... - bytecode.empend(checkpoint); // ...only if _C passes. - bytecode.empend((ByteCodeValueType)OpCodeId::Jump); - // LABEL _END = bytecode.size() - - bytecode_to_repeat = move(bytecode); - } - - static void transform_bytecode_repetition_zero_or_one(ByteCode& bytecode_to_repeat, bool greedy) - { - // FORKJUMP _END (FORKSTAY -> Greedy) - // CHECKPOINT _C - // REGEXP - // FAILIFEMPTY _C - // LABEL _END - ByteCode bytecode; - - if (greedy) - bytecode.empend(static_cast(OpCodeId::ForkStay)); - else - bytecode.empend(static_cast(OpCodeId::ForkJump)); - - bytecode.empend(bytecode_to_repeat.size() + 4); // Jump to the _END label - - auto checkpoint = s_next_checkpoint_serial_id++; - bytecode.empend(static_cast(OpCodeId::Checkpoint)); - bytecode.empend(static_cast(checkpoint)); - - bytecode.extend(move(bytecode_to_repeat)); - - bytecode.empend(static_cast(OpCodeId::FailIfEmpty)); - bytecode.empend(checkpoint); - // LABEL _END = bytecode.size() - - bytecode_to_repeat = move(bytecode); - } - - static void reset_checkpoint_serial_id() { s_next_checkpoint_serial_id = 0; } - -private: - static size_t s_next_checkpoint_serial_id; -}; - -class REGEX_API FlatByteCode : public ByteCodeBase { -public: - static FlatByteCode from(ByteCode&& bytecode) - { - FlatByteCode flat_bytecode; - if (!bytecode.is_empty()) - flat_bytecode.m_data = move(static_cast&>(bytecode).first_chunk()); - flat_bytecode.m_string_table = move(bytecode.m_string_table); - flat_bytecode.m_u16_string_table = move(bytecode.m_u16_string_table); - flat_bytecode.m_string_set_table = move(bytecode.m_string_set_table); - flat_bytecode.m_group_name_mappings = move(bytecode.m_group_name_mappings); - return flat_bytecode; - } - - Span flat_data() const { return m_data.span(); } - auto& at(size_t index) { return m_data.data()[index]; } - auto const& at(size_t index) const { return m_data.data()[index]; } - auto& operator[](size_t index) { return m_data.data()[index]; } - auto const& operator[](size_t index) const { return m_data.data()[index]; } - auto size() const { return m_data.size(); } - - auto begin() const { return m_data.begin(); } - auto end() const { return m_data.end(); } - -private: - Vector m_data; -}; - -#define ENUMERATE_EXECUTION_RESULTS \ - __ENUMERATE_EXECUTION_RESULT(Continue) \ - __ENUMERATE_EXECUTION_RESULT(Fork_PrioHigh) \ - __ENUMERATE_EXECUTION_RESULT(Fork_PrioLow) \ - __ENUMERATE_EXECUTION_RESULT(Failed) \ - __ENUMERATE_EXECUTION_RESULT(Failed_ExecuteLowPrioForks) \ - __ENUMERATE_EXECUTION_RESULT(Failed_ExecuteLowPrioForksButNoFurtherPossibleMatches) \ - __ENUMERATE_EXECUTION_RESULT(Succeeded) - -enum class ExecutionResult : u8 { -#define __ENUMERATE_EXECUTION_RESULT(x) x, - ENUMERATE_EXECUTION_RESULTS -#undef __ENUMERATE_EXECUTION_RESULT -}; - -StringView execution_result_name(ExecutionResult result); -REGEX_API StringView opcode_id_name(OpCodeId opcode_id); -StringView boundary_check_type_name(BoundaryCheckType); -StringView character_compare_type_name(CharacterCompareType result); -StringView character_class_name(CharClass ch_class); -StringView fork_if_condition_name(ForkIfCondition condition); - -namespace OpArgs { - -struct Jump { - static constexpr size_t offset = 1; -}; -struct GoBack { - static constexpr size_t count = 1; -}; -struct SetStepBack { - static constexpr size_t step = 1; -}; -struct CheckBoundary { - static constexpr size_t type = 1; -}; -struct ClearCaptureGroup { - static constexpr size_t id = 1; -}; -struct FailIfEmpty { - static constexpr size_t checkpoint = 1; -}; -struct SaveLeftCaptureGroup { - static constexpr size_t id = 1; -}; -struct SaveRightCaptureGroup { - static constexpr size_t id = 1; -}; -struct SaveModifiers { - static constexpr size_t new_modifiers = 1; -}; -struct ResetRepeat { - static constexpr size_t id = 1; -}; -struct Checkpoint { - static constexpr size_t id = 1; -}; -struct RSeekTo { - static constexpr size_t ch = 1; -}; -struct SaveRightNamedCaptureGroup { - static constexpr size_t name_index = 1; - static constexpr size_t id = 2; -}; -struct Repeat { - static constexpr size_t offset = 1; - static constexpr size_t count = 2; - static constexpr size_t id = 3; -}; -struct JumpNonEmpty { - static constexpr size_t offset = 1; - static constexpr size_t checkpoint = 2; - static constexpr size_t form = 3; -}; -struct ForkIf { - static constexpr size_t offset = 1; - static constexpr size_t form = 2; - static constexpr size_t condition = 3; -}; -struct Compare { - static constexpr size_t arguments_count = 1; - static constexpr size_t arguments_size = 2; - static constexpr size_t data_start = 3; -}; -struct CompareSimple { - static constexpr size_t arguments_size = 1; - static constexpr size_t data_start = 2; -}; - -} - -inline size_t opcode_size(OpCodeId id, ByteCodeValueType const* data, size_t ip) -{ - switch (id) { - case OpCodeId::Exit: - case OpCodeId::FailForks: - case OpCodeId::PopSaved: - case OpCodeId::Save: - case OpCodeId::Restore: - case OpCodeId::IncStepBack: - case OpCodeId::CheckStepBack: - case OpCodeId::CheckSavedPosition: - case OpCodeId::CheckBegin: - case OpCodeId::CheckEnd: - case OpCodeId::RestoreModifiers: - return 1; - case OpCodeId::Jump: - case OpCodeId::ForkJump: - case OpCodeId::ForkStay: - case OpCodeId::ForkReplaceJump: - case OpCodeId::ForkReplaceStay: - case OpCodeId::GoBack: - case OpCodeId::SetStepBack: - case OpCodeId::CheckBoundary: - case OpCodeId::ClearCaptureGroup: - case OpCodeId::FailIfEmpty: - case OpCodeId::SaveLeftCaptureGroup: - case OpCodeId::SaveRightCaptureGroup: - case OpCodeId::SaveModifiers: - case OpCodeId::ResetRepeat: - case OpCodeId::Checkpoint: - case OpCodeId::RSeekTo: - return 2; - case OpCodeId::SaveRightNamedCaptureGroup: - return 3; - case OpCodeId::Repeat: - case OpCodeId::JumpNonEmpty: - case OpCodeId::ForkIf: - return 4; - case OpCodeId::Compare: - return data[ip + OpArgs::Compare::arguments_size] + 3; - case OpCodeId::CompareSimple: - return 2 + data[ip + OpArgs::CompareSimple::arguments_size]; - } - VERIFY_NOT_REACHED(); -} - -Vector flat_compares_at(ByteCodeValueType const* data, size_t ip, bool is_simple); -bool matches_character_class(CharClass, u32 ch, bool insensitive, bool unicode_mode); -REGEX_API ByteString opcode_arguments_string(OpCodeId id, ByteCodeValueType const* data, size_t ip, MatchState const& state, ByteCodeBase const& bytecode); -REGEX_API Vector compare_variable_arguments_to_byte_string(ByteCodeValueType const* data, size_t ip, MatchState const& state, ByteCodeBase const& bytecode, Optional input = {}); - -} diff --git a/Libraries/LibRegex/RegexBytecodeStreamOptimizer.h b/Libraries/LibRegex/RegexBytecodeStreamOptimizer.h deleted file mode 100644 index cd3176dc65..0000000000 --- a/Libraries/LibRegex/RegexBytecodeStreamOptimizer.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2021, Ali Mohammad Pur - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include "Forward.h" -#include - -namespace regex { - -class Optimizer { -public: - static void append_alternation(ByteCode& target, ByteCode&& left, ByteCode&& right); - static void append_alternation(ByteCode& target, Span alternatives); - static void append_character_class(ByteCode& target, Vector&& pairs); -}; - -} diff --git a/Libraries/LibRegex/RegexDebug.h b/Libraries/LibRegex/RegexDebug.h deleted file mode 100644 index f1d0068aa8..0000000000 --- a/Libraries/LibRegex/RegexDebug.h +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include - -namespace regex { - -template -class RegexDebug { -public: - RegexDebug(FILE* file = stdout) - : m_file(file) - { - } - - virtual ~RegexDebug() = default; - - template - void print_raw_bytecode(Regex& regex) const - { - auto& bytecode = regex.parser_result.bytecode.template get(); - size_t index { 0 }; - for (auto& value : bytecode) { - outln(m_file, "OpCode i={:3} [{:#02X}]", index, value); - ++index; - } - } - - template - void print_bytecode(Regex const& regex) const - { - print_bytecode(regex.parser_result.bytecode.template get()); - } - - void print_bytecode(ByteCode const& bytecode) const - { - auto state = MatchState::only_for_enumeration(); - ByteCodeValueType const* data; - auto data_size = bytecode.size(); - Optional> flat_storage; - - if constexpr (IsSame) { - data = bytecode.flat_data().data(); - } else { - flat_storage.emplace(); - flat_storage->ensure_capacity(data_size); - for (size_t i = 0; i < data_size; ++i) - flat_storage->unchecked_append(bytecode[i]); - data = flat_storage->data(); - } - - for (;;) { - auto id = (data_size <= state.instruction_position) - ? OpCodeId::Exit - : static_cast(data[state.instruction_position]); - auto sz = opcode_size(id, data, state.instruction_position); - print_opcode("PrintBytecode", id, data, state, bytecode); - out(m_file, "{}", m_debug_stripline); - - if (id == OpCodeId::Exit) - break; - - state.instruction_position += sz; - } - - out(m_file, "String Table:\n"); - for (auto const& entry : bytecode.string_table().m_table) - outln(m_file, "+ {} -> {:x}", entry.key, entry.value); - out(m_file, "Reverse String Table:\n"); - for (auto const& entry : bytecode.string_table().m_inverse_table) - outln(m_file, "+ {:x} -> {}", entry.key, entry.value); - - out(m_file, "(u16) String Table:\n"); - for (auto const& entry : bytecode.u16_string_table().m_table) - outln(m_file, "+ {} -> {:x}", entry.key, entry.value); - out(m_file, "Reverse (u16) String Table:\n"); - for (auto const& entry : bytecode.u16_string_table().m_inverse_table) - outln(m_file, "+ {:x} -> {}", entry.key, entry.value); - - fflush(m_file); - } - - void print_opcode(ByteString const& system, OpCodeId id, ByteCodeValueType const* data, MatchState& state, ByteCodeBase const& bytecode, size_t recursion = 0, bool newline = true) const - { - auto opcode_str = ByteString::formatted("[{:#02X}] {}", (int)id, opcode_id_name(id)); - out(m_file, "{:15} | {:5} | {:9} | {:35} | {:30} | {:20}", - system.characters(), - state.instruction_position, - recursion, - opcode_str.characters(), - opcode_arguments_string(id, data, state.instruction_position, state, bytecode).characters(), - ByteString::formatted("ip: {:3}, sp: {:3}", state.instruction_position, state.string_position)); - if (newline) - outln(); - if (newline && id == OpCodeId::Compare) { - for (auto& line : compare_variable_arguments_to_byte_string(data, state.instruction_position, state, bytecode)) - outln(m_file, "{:15} | {:5} | {:9} | {:35} | {:30} | {:20}", "", "", "", "", line, ""); - } - } - - void print_result(OpCodeId id, ByteCodeValueType const* data, size_t data_size, ByteCodeBase const& bytecode, MatchInput const& input, MatchState& state, size_t current_opcode_size, ExecutionResult result) const - { - StringBuilder builder; - builder.append(execution_result_name(result)); - builder.appendff(", fc: {}, ss: {}", input.fail_counter, input.saved_positions.size()); - if (result == ExecutionResult::Succeeded) { - builder.appendff(", ip: {}/{}, sp: {}/{}", state.instruction_position, data_size - 1, state.string_position, input.view.length() - 1); - } else if (result == ExecutionResult::Fork_PrioHigh) { - builder.appendff(", next ip: {}", state.fork_at_position + current_opcode_size); - } else if (result != ExecutionResult::Failed) { - builder.appendff(", next ip: {}", state.instruction_position + current_opcode_size); - } - - outln(m_file, " | {:20}", builder.to_byte_string()); - - if (id == OpCodeId::CheckSavedPosition) { - auto last_saved = input.saved_positions.is_empty() - ? "saved: "_string - : MUST(String::formatted("saved: {}", input.saved_positions.last())); - outln(m_file, "{:15} | {:5} | {:9} | {:35} | {:30} | {:20}", "", "", "", "", last_saved, ""); - } - if (id == OpCodeId::CheckStepBack || id == OpCodeId::IncStepBack) { - auto last_step_back = state.step_backs.is_empty() - ? "step: "_string - : MUST(String::formatted("step: {}", state.step_backs.last())); - outln(m_file, "{:15} | {:5} | {:9} | {:35} | {:30} | {:20}", "", "", "", "", last_step_back, ""); - } - - if (id == OpCodeId::Compare) { - for (auto& line : compare_variable_arguments_to_byte_string(data, state.instruction_position, state, bytecode, input)) { - outln(m_file, "{:15} | {:5} | {:9} | {:35} | {:30} | {:20}", "", "", "", "", line, ""); - } - } - - out(m_file, "{}", m_debug_stripline); - } - - void print_header() - { - StringBuilder builder; - builder.appendff("{:15} | {:5} | {:9} | {:35} | {:30} | {:20} | {:20}\n", "System", "Index", "Recursion", "OpCode", "Arguments", "State", "Result"); - auto length = builder.length(); - for (size_t i = 0; i < length; ++i) { - builder.append('='); - } - auto str = builder.to_byte_string(); - VERIFY(!str.is_empty()); - - outln(m_file, "{}", str); - fflush(m_file); - - builder.clear(); - for (size_t i = 0; i < length; ++i) { - builder.append('-'); - } - builder.append('\n'); - m_debug_stripline = builder.to_byte_string(); - } - -private: - ByteString m_debug_stripline; - FILE* m_file; -}; - -} - -using regex::RegexDebug; diff --git a/Libraries/LibRegex/RegexDefs.h b/Libraries/LibRegex/RegexDefs.h deleted file mode 100644 index 759b5359c6..0000000000 --- a/Libraries/LibRegex/RegexDefs.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * Copyright (c) 2020-2022, Ali Mohammad Pur - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -enum __Regex_Error { - __Regex_NoError, - __Regex_InvalidPattern, // Invalid regular expression. - __Regex_InvalidCollationElement, // Invalid collating element referenced. - __Regex_InvalidCharacterClass, // Invalid character class type referenced. - __Regex_InvalidTrailingEscape, // Trailing \ in pattern. - __Regex_InvalidNumber, // Number in \digit invalid or in error. - __Regex_MismatchingBracket, // [ ] imbalance. - __Regex_MismatchingParen, // ( ) imbalance. - __Regex_MismatchingBrace, // { } imbalance. - __Regex_InvalidBraceContent, // Content of {} invalid: not a number, number too large, more than two numbers, first larger than second. - __Regex_InvalidBracketContent, // Content of [] invalid. - __Regex_InvalidRange, // Invalid endpoint in range expression. - __Regex_InvalidRepetitionMarker, // ?, * or + not preceded by valid regular expression. - __Regex_ReachedMaxRecursion, // MaximumRecursion has been reached. - __Regex_EmptySubExpression, // Sub expression has empty content. - __Regex_InvalidCaptureGroup, // Content of capture group is invalid. - __Regex_InvalidNameForCaptureGroup, // Name of capture group is invalid. - __Regex_InvalidNameForProperty, // Name of property is invalid. - __Regex_DuplicateNamedCapture, // Duplicate named capture group - __Regex_InvalidCharacterClassEscape, // Invalid escaped entity in character class. - __Regex_NegatedCharacterClassStrings, // Negated character class cannot contain strings. - __Regex_InvalidModifierGroup, // Invalid modifier group. - __Regex_RepeatedModifierFlag, // Repeated flag in modifier group. -}; - -enum __RegexAllFlags { - __Regex_Global = 1, // All matches (don't return after first match) - __Regex_Insensitive = __Regex_Global << 1, // Case insensitive match (ignores case of [a-zA-Z]) - __Regex_Ungreedy = __Regex_Global << 2, // The match becomes lazy by default. Now a ? following a quantifier makes it greedy - __Regex_Unicode = __Regex_Global << 3, // Enable all unicode features and interpret all unicode escape sequences as such - __Regex_Extended = __Regex_Global << 4, // Ignore whitespaces. Spaces and text after a # in the pattern are ignored - __Regex_Extra = __Regex_Global << 5, // Disallow meaningless escapes. A \ followed by a letter with no special meaning is faulted - __Regex_MatchNotBeginOfLine = __Regex_Global << 6, // Pattern is not forced to ^ -> search in whole string! - __Regex_MatchNotEndOfLine = __Regex_Global << 7, // Don't Force the dollar sign, $, to always match end of the string, instead of end of the line. This option is ignored if the Multiline-flag is set - __Regex_SkipSubExprResults = __Regex_Global << 8, // Do not return sub expressions in the result - __Regex_SingleLine = __Regex_Global << 10, // Dot matches newline characters - __Regex_Sticky = __Regex_Global << 11, // Force the pattern to only match consecutive matches from where the previous match ended. - __Regex_Multiline = __Regex_Global << 12, // Handle newline characters. Match each line, one by one. - __Regex_SingleMatch = __Regex_Global << 13, // Stop after acquiring a single match. - __Regex_UnicodeSets = __Regex_Global << 14, // ECMA262 Parser specific: Allow set operations in char classes. - __Regex_Internal_Stateful = __Regex_Global << 15, // Internal flag; enables stateful matches. - __Regex_Internal_BrowserExtended = __Regex_Global << 16, // Internal flag; enable browser-specific ECMA262 extensions. - __Regex_Internal_ConsiderNewline = __Regex_Global << 17, // Internal flag; allow matchers to consider newlines as line separators. - __Regex_Internal_ECMA262DotSemantics = __Regex_Global << 18, // Internal flag; use ECMA262 semantics for dot ('.') - disallow CR/LF/LS/PS instead of just CR. - __Regex_Last = __Regex_Internal_ECMA262DotSemantics, -}; diff --git a/Libraries/LibRegex/RegexError.h b/Libraries/LibRegex/RegexError.h deleted file mode 100644 index 4539aff6a3..0000000000 --- a/Libraries/LibRegex/RegexError.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include "RegexDefs.h" -#include -#include - -namespace regex { - -enum class Error : u8 { - NoError = __Regex_NoError, - InvalidPattern = __Regex_InvalidPattern, // Invalid regular expression. - InvalidCollationElement = __Regex_InvalidCollationElement, // Invalid collating element referenced. - InvalidCharacterClass = __Regex_InvalidCharacterClass, // Invalid character class type referenced. - InvalidTrailingEscape = __Regex_InvalidTrailingEscape, // Trailing \ in pattern. - InvalidNumber = __Regex_InvalidNumber, // Number in \digit invalid or in error. - MismatchingBracket = __Regex_MismatchingBracket, // [ ] imbalance. - MismatchingParen = __Regex_MismatchingParen, // ( ) imbalance. - MismatchingBrace = __Regex_MismatchingBrace, // { } imbalance. - InvalidBraceContent = __Regex_InvalidBraceContent, // Content of {} invalid: not a number, number too large, more than two numbers, first larger than second. - InvalidBracketContent = __Regex_InvalidBracketContent, // Content of [] invalid. - InvalidRange = __Regex_InvalidRange, // Invalid endpoint in range expression. - InvalidRepetitionMarker = __Regex_InvalidRepetitionMarker, // ?, * or + not preceded by valid regular expression. - ReachedMaxRecursion = __Regex_ReachedMaxRecursion, // MaximumRecursion has been reached. - EmptySubExpression = __Regex_EmptySubExpression, // Sub expression has empty content. - InvalidCaptureGroup = __Regex_InvalidCaptureGroup, // Content of capture group is invalid. - InvalidNameForCaptureGroup = __Regex_InvalidNameForCaptureGroup, // Name of capture group is invalid. - InvalidNameForProperty = __Regex_InvalidNameForProperty, // Name of property is invalid. - DuplicateNamedCapture = __Regex_DuplicateNamedCapture, // Name of property is invalid. - InvalidCharacterClassEscape = __Regex_InvalidCharacterClassEscape, // Invalid escaped entity in character class. - NegatedCharacterClassStrings = __Regex_NegatedCharacterClassStrings, // Negated character class may contain strings. - InvalidModifierGroup = __Regex_InvalidModifierGroup, // Invalid modifier group. - RepeatedModifierFlag = __Regex_RepeatedModifierFlag, // Repeated flag in modifier group. -}; - -inline StringView get_error_string(Error error) -{ - switch (error) { - case Error::NoError: - return "No error"sv; - case Error::InvalidPattern: - return "Invalid regular expression."sv; - case Error::InvalidCollationElement: - return "Invalid collating element referenced."sv; - case Error::InvalidCharacterClass: - return "Invalid character class type referenced."sv; - case Error::InvalidTrailingEscape: - return "Trailing \\ in pattern."sv; - case Error::InvalidNumber: - return "Number in \\digit invalid or in error."sv; - case Error::MismatchingBracket: - return "[ ] imbalance."sv; - case Error::MismatchingParen: - return "( ) imbalance."sv; - case Error::MismatchingBrace: - return "{ } imbalance."sv; - case Error::InvalidBraceContent: - return "Content of {} invalid: not a number, number too large, more than two numbers, first larger than second."sv; - case Error::InvalidBracketContent: - return "Content of [] invalid."sv; - case Error::InvalidRange: - return "Invalid endpoint in range expression."sv; - case Error::InvalidRepetitionMarker: - return "?, * or + not preceded by valid regular expression."sv; - case Error::ReachedMaxRecursion: - return "Maximum recursion has been reached."sv; - case Error::EmptySubExpression: - return "Sub expression has empty content."sv; - case Error::InvalidCaptureGroup: - return "Content of capture group is invalid."sv; - case Error::InvalidNameForCaptureGroup: - return "Name of capture group is invalid."sv; - case Error::InvalidNameForProperty: - return "Name of property is invalid."sv; - case Error::DuplicateNamedCapture: - return "Duplicate capture group name"sv; - case Error::InvalidCharacterClassEscape: - return "Invalid escaped entity in character class."sv; - case Error::NegatedCharacterClassStrings: - return "Negated character class cannot contain strings."sv; - case Error::InvalidModifierGroup: - return "Invalid modifier group."sv; - case Error::RepeatedModifierFlag: - return "Repeated flag in modifier group."sv; - } - return "Undefined error."sv; -} - -} - -using regex::get_error_string; diff --git a/Libraries/LibRegex/RegexLexer.cpp b/Libraries/LibRegex/RegexLexer.cpp deleted file mode 100644 index b9d9c3dd3a..0000000000 --- a/Libraries/LibRegex/RegexLexer.cpp +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include "RegexLexer.h" -#include -#include -#include - -namespace regex { - -char const* Token::name(TokenType const type) -{ - switch (type) { -#define __ENUMERATE_REGEX_TOKEN(x) \ - case TokenType::x: \ - return #x; - ENUMERATE_REGEX_TOKENS -#undef __ENUMERATE_REGEX_TOKEN - default: - VERIFY_NOT_REACHED(); - return ""; - } -} - -char const* Token::name() const -{ - return name(m_type); -} - -Lexer::Lexer() - : GenericLexer(StringView {}) -{ -} - -Lexer::Lexer(StringView const source) - : GenericLexer(source) -{ -} - -void Lexer::back(size_t offset) -{ - if (offset == m_index + 1) - offset = m_index; // 'position == 0' occurs twice. - - VERIFY(offset <= m_index); - if (!offset) - return; - m_index -= offset; - m_previous_position = (m_index > 0) ? m_index - 1 : 0; -} - -char Lexer::consume() -{ - m_previous_position = m_index; - return GenericLexer::consume(); -} - -void Lexer::reset() -{ - m_index = 0; - m_current_token = { TokenType::Eof, 0, {} }; - m_previous_position = 0; -} - -Token Lexer::next() -{ - size_t token_start_position; - - auto begin_token = [&] { - token_start_position = m_index; - }; - - auto commit_token = [&](auto type) -> Token& { - VERIFY(token_start_position + m_previous_position - token_start_position + 1 <= m_input.length()); - auto substring = m_input.substring_view(token_start_position, m_previous_position - token_start_position + 1); - m_current_token = Token(type, token_start_position, substring); - return m_current_token; - }; - - auto emit_token = [&](auto type) -> Token& { - m_current_token = Token(type, m_index, m_input.substring_view(m_index, 1)); - consume(); - return m_current_token; - }; - - auto match_escape_sequence = [&]() -> size_t { - switch (peek(1)) { - case '^': - case '.': - case '[': - case ']': - case '$': - case '(': - case ')': - case '|': - case '*': - case '+': - case '?': - case '{': - case '\\': - return 2; - default: - dbgln_if(REGEX_DEBUG, "[LEXER] Found invalid escape sequence: \\{:c} (the parser will have to deal with this!)", peek(1)); - return 0; - } - }; - - while (m_index < m_input.length()) { - auto ch = peek(); - if (ch == '(') - return emit_token(TokenType::LeftParen); - - if (ch == ')') - return emit_token(TokenType::RightParen); - - if (ch == '{') - return emit_token(TokenType::LeftCurly); - - if (ch == '}') - return emit_token(TokenType::RightCurly); - - if (ch == '[') - return emit_token(TokenType::LeftBracket); - - if (ch == ']') - return emit_token(TokenType::RightBracket); - - if (ch == '.') - return emit_token(TokenType::Period); - - if (ch == '*') - return emit_token(TokenType::Asterisk); - - if (ch == '+') - return emit_token(TokenType::Plus); - - if (ch == '$') - return emit_token(TokenType::Dollar); - - if (ch == '^') - return emit_token(TokenType::Circumflex); - - if (ch == '|') - return emit_token(TokenType::Pipe); - - if (ch == '?') - return emit_token(TokenType::Questionmark); - - if (ch == ',') - return emit_token(TokenType::Comma); - - if (ch == '/') - return emit_token(TokenType::Slash); - - if (ch == '=') - return emit_token(TokenType::EqualSign); - - if (ch == ':') - return emit_token(TokenType::Colon); - - if (ch == '-') - return emit_token(TokenType::HyphenMinus); - - if (ch == '\\') { - size_t escape = match_escape_sequence(); - if (escape > 0) { - begin_token(); - for (size_t i = 0; i < escape; ++i) - consume(); - return commit_token(TokenType::EscapeSequence); - } - } - - return emit_token(TokenType::Char); - } - - return Token(TokenType::Eof, m_index, {}); -} - -} diff --git a/Libraries/LibRegex/RegexLexer.h b/Libraries/LibRegex/RegexLexer.h deleted file mode 100644 index bf35d46c8a..0000000000 --- a/Libraries/LibRegex/RegexLexer.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include - -namespace regex { - -#define ENUMERATE_REGEX_TOKENS \ - __ENUMERATE_REGEX_TOKEN(Eof) \ - __ENUMERATE_REGEX_TOKEN(Char) \ - __ENUMERATE_REGEX_TOKEN(Circumflex) \ - __ENUMERATE_REGEX_TOKEN(Period) \ - __ENUMERATE_REGEX_TOKEN(LeftParen) \ - __ENUMERATE_REGEX_TOKEN(RightParen) \ - __ENUMERATE_REGEX_TOKEN(LeftCurly) \ - __ENUMERATE_REGEX_TOKEN(RightCurly) \ - __ENUMERATE_REGEX_TOKEN(LeftBracket) \ - __ENUMERATE_REGEX_TOKEN(RightBracket) \ - __ENUMERATE_REGEX_TOKEN(Asterisk) \ - __ENUMERATE_REGEX_TOKEN(EscapeSequence) \ - __ENUMERATE_REGEX_TOKEN(Dollar) \ - __ENUMERATE_REGEX_TOKEN(Pipe) \ - __ENUMERATE_REGEX_TOKEN(Plus) \ - __ENUMERATE_REGEX_TOKEN(Comma) \ - __ENUMERATE_REGEX_TOKEN(Slash) \ - __ENUMERATE_REGEX_TOKEN(EqualSign) \ - __ENUMERATE_REGEX_TOKEN(HyphenMinus) \ - __ENUMERATE_REGEX_TOKEN(Colon) \ - __ENUMERATE_REGEX_TOKEN(Questionmark) - -enum class TokenType { -#define __ENUMERATE_REGEX_TOKEN(x) x, - ENUMERATE_REGEX_TOKENS -#undef __ENUMERATE_REGEX_TOKEN -}; - -class Token { -public: - Token() = default; - Token(TokenType const type, size_t const start_position, StringView const value) - : m_type(type) - , m_position(start_position) - , m_value(value) - { - } - - TokenType type() const { return m_type; } - StringView value() const { return m_value; } - size_t position() const { return m_position; } - - char const* name() const; - static char const* name(TokenType); - -private: - TokenType m_type { TokenType::Eof }; - size_t m_position { 0 }; - StringView m_value {}; -}; - -class REGEX_API Lexer : public GenericLexer { -public: - Lexer(); - explicit Lexer(StringView source); - Token next(); - void reset(); - void back(size_t offset); - char consume(); - void set_source(StringView const source) { m_input = source; } - auto const& source() const { return m_input; } - -private: - size_t m_previous_position { 0 }; - Token m_current_token { TokenType::Eof, 0, {} }; -}; - -} - -using regex::Lexer; diff --git a/Libraries/LibRegex/RegexMatch.h b/Libraries/LibRegex/RegexMatch.h deleted file mode 100644 index 2fabeeec95..0000000000 --- a/Libraries/LibRegex/RegexMatch.h +++ /dev/null @@ -1,576 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include "Forward.h" -#include "RegexOptions.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace regex { - -class RegexStringView { -public: - RegexStringView() = default; - - RegexStringView(String const& string) - : m_view(string.bytes_as_string_view()) - { - } - - RegexStringView(StringView const view) - : m_view(view) - { - } - - RegexStringView(Utf16View view) - : m_view(view) - { - } - - RegexStringView(String&&) = delete; - - Utf16View const& u16_view() const - { - return m_view.get(); - } - - bool is_u16_view() const - { - return m_view.has(); - } - - bool unicode() const { return m_unicode; } - void set_unicode(bool unicode) { m_unicode = unicode; } - - bool is_empty() const - { - return m_view.visit([](auto& view) { return view.is_empty(); }); - } - - bool is_null() const - { - return m_view.visit([](auto& view) { return view.is_null(); }); - } - - size_t length() const - { - if (unicode()) { - return m_view.visit( - [](Utf16View const& view) { return view.length_in_code_points(); }, - [](auto const& view) { return view.length(); }); - } - - return length_in_code_units(); - } - - size_t length_in_code_units() const - { - return m_view.visit( - [](Utf16View const& view) { return view.length_in_code_units(); }, - [](auto const& view) { return view.length(); }); - } - - size_t length_of_code_point(u32 code_point) const - { - return m_view.visit( - [&](Utf16View const&) { - if (code_point < 0x10000) - return 1; - return 2; - }, - [&](auto const&) { - if (code_point <= 0x7f) - return 1; - if (code_point <= 0x07ff) - return 2; - if (code_point <= 0xffff) - return 3; - return 4; - }); - } - - RegexStringView typed_null_view() - { - auto view = m_view.visit( - [&](T const&) { - return RegexStringView { T {} }; - }); - view.set_unicode(unicode()); - return view; - } - - RegexStringView construct_as_same(Span data, Optional& optional_string_storage, Utf16String& optional_utf16_storage) const - { - auto view = m_view.visit( - [&optional_string_storage, data](T const&) { - StringBuilder builder; - for (auto ch : data) - builder.append(ch); // Note: The type conversion is intentional. - optional_string_storage = builder.to_byte_string(); - return RegexStringView { T { *optional_string_storage } }; - }, - [&optional_utf16_storage, data](Utf16View) { - optional_utf16_storage = Utf16String::from_utf32({ data.data(), data.size() }); - return RegexStringView { optional_utf16_storage.utf16_view() }; - }); - - view.set_unicode(unicode()); - return view; - } - - Vector lines() const - { - return m_view.visit( - [](StringView view) { - auto views = view.lines(StringView::ConsiderCarriageReturn::No); - Vector new_views; - for (auto& view : views) - new_views.empend(view); - return new_views; - }, - [](Utf16View view) { - if (view.is_empty()) - return Vector { view }; - - Vector views; - while (!view.is_empty()) { - auto position = view.find_code_unit_offset(u'\n'); - if (!position.has_value()) - break; - auto offset = position.value() / sizeof(u16); - views.empend(view.substring_view(0, offset)); - view = view.substring_view(offset + 1, view.length_in_code_units() - offset - 1); - } - if (!view.is_empty()) - views.empend(view); - return views; - }); - } - - RegexStringView substring_view(size_t offset, size_t length) const - { - if (unicode()) { - auto view = m_view.visit( - [&](auto view) { return RegexStringView { view.substring_view(offset, length) }; }, - [&](Utf16View const& view) { return RegexStringView { view.unicode_substring_view(offset, length) }; }); - - view.set_unicode(unicode()); - return view; - } - - auto view = m_view.visit([&](auto view) { return RegexStringView { view.substring_view(offset, length) }; }); - view.set_unicode(unicode()); - return view; - } - - ByteString to_byte_string() const - { - return m_view.visit( - [](StringView view) { return view.to_byte_string(); }, - [](Utf16View view) { return view.to_byte_string().release_value_but_fixme_should_propagate_errors(); }); - } - - ErrorOr to_string() const - { - return m_view.visit( - [](StringView view) { return String::from_utf8(view); }, - [](Utf16View view) { return view.to_utf8(); }); - } - - u32 code_point_at(size_t code_unit_index) const - { - return m_view.visit( - [&](StringView view) -> u32 { - auto ch = view[code_unit_index]; - if constexpr (IsSigned) { - if (ch < 0) - return 256u + ch; - return ch; - } - }, - [&](Utf16View const& view) -> u32 { return view.code_point_at(code_unit_index); }); - } - - // Returns the code point at the code unit offset if the Unicode flag is set. Otherwise, returns the code unit. - u32 unicode_aware_code_point_at(size_t code_unit_index) const - { - if (unicode()) - return code_point_at(code_unit_index); - - return m_view.visit( - [&](StringView view) -> u32 { - auto ch = view[code_unit_index]; - if constexpr (IsSigned) { - if (ch < 0) - return 256u + ch; - return ch; - } - }, - [&](Utf16View const& view) -> u32 { return view.code_unit_at(code_unit_index); }); - } - - size_t code_unit_offset_of(size_t code_point_index) const - { - return m_view.visit( - [&](StringView view) -> u32 { - Utf8View utf8_view { view }; - return utf8_view.byte_offset_of(code_point_index); - }, - [&](Utf16View const& view) -> u32 { - return view.code_unit_offset_of(code_point_index); - }); - } - - bool operator==(char const* cstring) const - { - return m_view.visit( - [&](Utf16View) { return to_byte_string() == cstring; }, - [&](StringView view) { return view == cstring; }); - } - - bool operator==(StringView string) const - { - return m_view.visit( - [&](Utf16View) { return to_byte_string() == string; }, - [&](StringView view) { return view == string; }); - } - - bool operator==(Utf16View const& other) const - { - return m_view.visit( - [&](Utf16View const& view) { return view == other; }, - [&](StringView view) { return view == RegexStringView { other }.to_byte_string(); }); - } - - bool equals(RegexStringView other) const - { - return other.m_view.visit([this](auto const& view) { return operator==(view); }); - } - - bool equals_ignoring_case(RegexStringView other, bool unicode_mode) const - { - return m_view.visit( - [&](StringView view) { - return other.m_view.visit( - [&](StringView other_view) { - if (!unicode_mode) - return view.equals_ignoring_ascii_case(other_view); - - Utf8View view_utf8(view); - Utf8View other_utf8(other_view); - return Unicode::ranges_equal_ignoring_case(view_utf8, other_utf8, unicode_mode); - }, - [&](Utf16View other_view) { - Utf8View view_utf8(view); - return Unicode::ranges_equal_ignoring_case(view_utf8, other_view, unicode_mode); - }, - [](auto&) -> bool { TODO(); }); - }, - [&](Utf16View view) { - return other.m_view.visit( - [&](StringView other_view) { - Utf8View other_utf8(other_view); - return Unicode::ranges_equal_ignoring_case(view, other_utf8, unicode_mode); - }, - [&](Utf16View other_view) { - if (!unicode_mode) - return view.equals_ignoring_ascii_case(other_view); - - return Unicode::ranges_equal_ignoring_case(view, other_view, unicode_mode); - }, - [](auto&) -> bool { TODO(); }); - }, - [](auto&) -> bool { TODO(); }); - } - - bool starts_with(StringView str) const - { - return m_view.visit( - [&](Utf16View) -> bool { - TODO(); - }, - [&](StringView view) { return view.starts_with(str); }); - } - - struct FoundIndex { - size_t code_unit_index; - size_t code_point_index; - }; - Optional find_index_of_previous(u32 code_point, size_t end_code_point_index, size_t end_code_unit_index) const - { - return m_view.visit( - [&](Utf16View const& view) -> Optional { - auto result = view.find_last_code_point_offset(code_point, end_code_unit_index); - if (!result.has_value()) - return {}; - return FoundIndex { result.value(), view.code_point_offset_of(result.value()) }; - }, - [&](StringView const& view) -> Optional { - if (unicode()) { - Utf8View utf8_view { view }; - auto it = utf8_view.begin(); - size_t current_code_point_index = 0; - Optional found_index; - - for (; it != utf8_view.end(); ++it, ++current_code_point_index) { - if (current_code_point_index >= end_code_point_index) - break; - if (*it == code_point) { - auto byte_index = utf8_view.byte_offset_of(it); - found_index = { byte_index, current_code_point_index }; - } - } - - return found_index; - } - - auto byte_index = view.substring_view(0, min(end_code_unit_index, view.length())).find_last(code_point); - if (!byte_index.has_value()) - return {}; - return FoundIndex { byte_index.value(), byte_index.value() }; - }); - } - - FoundIndex find_end_of_line(size_t start_code_point_index, size_t start_code_unit_index) const - { - constexpr auto is_newline = [](u32 ch) { return ch == '\n' || ch == '\r' || ch == 0x2028 || ch == 0x2029; }; - - return m_view.visit( - [&](Utf16View const& view) -> FoundIndex { - size_t code_unit_index = start_code_unit_index; - size_t code_point_index = start_code_point_index; - while (code_unit_index < view.length_in_code_units()) { - auto code_unit = view.code_unit_at(code_unit_index); - u32 ch = code_unit; - size_t code_units_for_this = 1; - if (AK::UnicodeUtils::is_utf16_high_surrogate(code_unit) && code_unit_index + 1 < view.length_in_code_units()) { - auto next_code_unit = view.code_unit_at(code_unit_index + 1); - if (AK::UnicodeUtils::is_utf16_low_surrogate(next_code_unit)) { - ch = AK::UnicodeUtils::decode_utf16_surrogate_pair(code_unit, next_code_unit); - code_units_for_this = 2; - } - } - - if (is_newline(ch)) - return FoundIndex { code_unit_index, code_point_index }; - code_unit_index += code_units_for_this; - ++code_point_index; - } - return FoundIndex { view.length_in_code_units(), code_point_index }; - }, - [&](StringView const& view) -> FoundIndex { - if (unicode()) { - Utf8View utf8_view { view }; - auto it = utf8_view.begin(); - size_t current_code_point_index = 0; - - // Skip to start position - while (it != utf8_view.end() && current_code_point_index < start_code_point_index) { - ++it; - ++current_code_point_index; - } - - for (; it != utf8_view.end(); ++it, ++current_code_point_index) { - if (is_newline(*it)) { - return FoundIndex { utf8_view.byte_offset_of(it), current_code_point_index }; - } - } - - return FoundIndex { view.length(), utf8_view.length() }; - } - - for (size_t i = start_code_unit_index; i < view.length(); ++i) { - if (is_newline(static_cast(view[i]))) - return FoundIndex { i, i }; - } - return FoundIndex { view.length(), view.length() }; - }); - } - -private: - NO_UNIQUE_ADDRESS Variant m_view { StringView {} }; - NO_UNIQUE_ADDRESS bool m_unicode { false }; -}; - -class Match final { -public: - Match() = default; - ~Match() = default; - - Match(RegexStringView view_, size_t const line_, size_t const column_, size_t const global_offset_) - : view(view_) - , line(line_) - , column(column_) - , global_offset(global_offset_) - , left_column(column_) - { - } - - Match(RegexStringView const view_, size_t capture_group_name_, size_t const line_, size_t const column_, size_t const global_offset_) - : view(view_) - , capture_group_name(capture_group_name_) - , line(line_) - , column(column_) - , global_offset(global_offset_) - , left_column(column_) - { - } - - void reset() - { - view = view.typed_null_view(); - capture_group_name = -1; - line = 0; - column = 0; - global_offset = 0; - left_column = 0; - } - - RegexStringView view {}; - - // This is a string table index. -1 if none. Not using Optional to keep the struct trivially copyable. - ssize_t capture_group_name { -1 }; - - size_t line { 0 }; - size_t column { 0 }; - size_t global_offset { 0 }; - - // ugly, as not usable by user, but needed to prevent to create extra vectors that are - // able to store the column when the left paren has been found - size_t left_column { 0 }; -}; - -struct MatchInput { - RegexStringView view {}; - AllOptions regex_options {}; - size_t start_offset { 0 }; // For Stateful matches, saved and restored from Regex::start_offset. - - size_t match_index { 0 }; - size_t line { 0 }; - size_t column { 0 }; - - size_t global_offset { 0 }; // For multiline matching, knowing the offset from start could be important - - mutable size_t fail_counter { 0 }; - mutable Vector saved_positions; - mutable Vector saved_code_unit_positions; - mutable Vector saved_forks_since_last_save; - mutable Optional fork_to_replace; - - bool in_the_middle_of_a_line { false }; - StringView pattern {}; -}; - -struct MatchState { - size_t capture_group_count; - size_t string_position_before_match { 0 }; - size_t string_position { 0 }; - size_t string_position_in_code_units { 0 }; - size_t instruction_position { 0 }; - size_t fork_at_position { 0 }; - size_t forks_since_last_save { 0 }; - size_t string_position_before_rseek { NumericLimits::max() }; - size_t string_position_in_code_units_before_rseek { NumericLimits::max() }; - Optional initiating_fork; - COWVector matches; - COWVector flat_capture_group_matches; // Vector> indexed by match index, then by capture group id; flattened for performance - COWVector repetition_marks; - Vector checkpoints; - Vector step_backs; - Vector modifier_stack; - AllOptions current_options; - - explicit MatchState(size_t capture_group_count, AllOptions options = {}) - : capture_group_count(capture_group_count) - , current_options(options) - { - } - - MatchState(MatchState const&) = default; - MatchState(MatchState&&) = default; - - MatchState& operator=(MatchState const&) = default; - MatchState& operator=(MatchState&&) = default; - - static MatchState only_for_enumeration() { return MatchState { 0 }; } - - size_t capture_group_matches_size() const - { - return flat_capture_group_matches.size() / capture_group_count; - } - - Span capture_group_matches(size_t match_index) const - { - return flat_capture_group_matches.span().slice(match_index * capture_group_count, capture_group_count); - } - - Span mutable_capture_group_matches(size_t match_index) - { - return flat_capture_group_matches.mutable_span().slice(match_index * capture_group_count, capture_group_count); - } - - // For size_t in {0..300}, ips in {0..750} and repetitions in {0..50}, there are zero collisions. - u64 u64_hash() const - { - u64 hash = 0xcbf29ce484222325; - auto combine = [&hash](auto value) { - hash ^= static_cast(value); - hash *= 0x9e3779b97f4a7c15; - }; - auto combine_vector = [&combine](auto const& vector, auto tag) { - combine(tag); - combine(vector.size()); - for (auto& value : vector) - combine(value); - }; - - combine(string_position_before_match); - combine(string_position); - combine(string_position_in_code_units); - combine(instruction_position); - combine(fork_at_position); - combine(initiating_fork.value_or(0) + initiating_fork.has_value()); - combine_vector(repetition_marks, 0xbeefbeefbeefbeef); - combine_vector(checkpoints, 0xfacefacefaceface); - combine_vector(step_backs, 0xfedefedefedefede); - - return hash; - } -}; - -} - -using regex::RegexStringView; - -template<> -struct AK::Formatter : Formatter { - ErrorOr format(FormatBuilder& builder, regex::RegexStringView value) - { - auto string = value.to_byte_string(); - return Formatter::format(builder, string); - } -}; - -template<> -struct AK::Traits : public AK::DefaultTraits { - constexpr static bool is_trivial() { return true; } -}; diff --git a/Libraries/LibRegex/RegexMatcher.cpp b/Libraries/LibRegex/RegexMatcher.cpp deleted file mode 100644 index 672f58488d..0000000000 --- a/Libraries/LibRegex/RegexMatcher.cpp +++ /dev/null @@ -1,1941 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#if REGEX_DEBUG -# include -#endif - -// U+2028 LINE SEPARATOR -constexpr static u32 const LineSeparator { 0x2028 }; -// U+2029 PARAGRAPH SEPARATOR -constexpr static u32 const ParagraphSeparator { 0x2029 }; - -namespace regex { - -static void advance_string_position(MatchState& state, RegexStringView view, Optional code_point = {}) -{ - ++state.string_position; - - if (view.unicode()) { - if (!code_point.has_value() && (state.string_position_in_code_units < view.length_in_code_units())) - code_point = view.code_point_at(state.string_position_in_code_units); - if (code_point.has_value()) - state.string_position_in_code_units += view.length_of_code_point(*code_point); - } else { - ++state.string_position_in_code_units; - } -} - -static void advance_string_position(MatchState& state, RegexStringView, RegexStringView advance_by) -{ - state.string_position += advance_by.length(); - state.string_position_in_code_units += advance_by.length_in_code_units(); -} - -static void reverse_string_position(MatchState& state, RegexStringView view, size_t amount) -{ - VERIFY(state.string_position >= amount); - state.string_position -= amount; - - if (view.unicode()) - state.string_position_in_code_units = view.code_unit_offset_of(state.string_position); - else - state.string_position_in_code_units -= amount; -} - -static void save_string_position(MatchInput const& input, MatchState const& state) -{ - input.saved_positions.append(state.string_position); - input.saved_forks_since_last_save.append(state.forks_since_last_save); - input.saved_code_unit_positions.append(state.string_position_in_code_units); -} - -static bool restore_string_position(MatchInput const& input, MatchState& state) -{ - if (input.saved_positions.is_empty()) - return false; - - state.string_position = input.saved_positions.take_last(); - state.string_position_in_code_units = input.saved_code_unit_positions.take_last(); - state.forks_since_last_save = input.saved_forks_since_last_save.take_last(); - return true; -} - -static bool is_word_character(u32 code_point, bool case_insensitive, bool unicode_mode) -{ - if (is_ascii_alphanumeric(code_point) || code_point == '_') - return true; - - if (case_insensitive && unicode_mode) { - auto canonical = Unicode::canonicalize(code_point, unicode_mode); - if (is_ascii_alphanumeric(canonical) || canonical == '_') - return true; - } - - return false; -} - -static ALWAYS_INLINE void compare_char(MatchInput const& input, MatchState& state, u32 ch1, bool inverse, bool& inverse_matched) -{ - if (state.string_position == input.view.length()) - return; - - // FIXME: Figure out how to do this if unicode() without performing a substring split first. - auto input_view = input.view.unicode() - ? input.view.substring_view(state.string_position, 1).code_point_at(0) - : input.view.unicode_aware_code_point_at(state.string_position_in_code_units); - - bool equal; - if (state.current_options & AllFlags::Insensitive) { - equal = Unicode::canonicalize(input_view, input.view.unicode()) == Unicode::canonicalize(ch1, input.view.unicode()); - } else { - equal = input_view == ch1; - } - - if (equal) { - if (inverse) - inverse_matched = true; - else - advance_string_position(state, input.view, ch1); - } -} - -static ALWAYS_INLINE bool compare_string(MatchInput const& input, MatchState& state, RegexStringView str, bool& had_zero_length_match) -{ - if (state.string_position + str.length() > input.view.length()) { - if (str.is_empty()) { - had_zero_length_match = true; - return true; - } - return false; - } - - if (str.length() == 0) { - had_zero_length_match = true; - return true; - } - - if (str.length() == 1) { - auto inverse_matched = false; - compare_char(input, state, str.code_point_at(0), false, inverse_matched); - return !inverse_matched; - } - - auto subject = input.view.substring_view(state.string_position, str.length()); - bool equals; - if (state.current_options & AllFlags::Insensitive) - equals = subject.equals_ignoring_case(str, input.view.unicode()); - else - equals = subject.equals(str); - - if (equals) - advance_string_position(state, input.view, str); - - return equals; -} - -static ALWAYS_INLINE void compare_character_class(MatchInput const& input, MatchState& state, CharClass character_class, u32 ch, bool inverse, bool& inverse_matched) -{ - if (matches_character_class(character_class, ch, state.current_options & AllFlags::Insensitive, input.view.unicode())) { - if (inverse) - inverse_matched = true; - else - advance_string_position(state, input.view, ch); - } -} - -static ALWAYS_INLINE void compare_character_range(MatchInput const& input, MatchState& state, u32 from, u32 to, u32 ch, bool inverse, bool& inverse_matched) -{ - bool matched = false; - if (state.current_options & AllFlags::Insensitive) { - matched = Unicode::code_point_matches_range_ignoring_case(ch, from, to, input.view.unicode()); - } else { - matched = (ch >= from && ch <= to); - } - - if (matched) { - if (inverse) - inverse_matched = true; - else - advance_string_position(state, input.view, ch); - } -} - -static ALWAYS_INLINE void compare_property(MatchInput const& input, MatchState& state, Unicode::Property property, bool inverse, bool is_double_negation, bool& inverse_matched) -{ - if (state.string_position == input.view.length()) - return; - - u32 code_point = input.view.code_point_at(state.string_position_in_code_units); - bool case_insensitive = (state.current_options & AllFlags::Insensitive) && input.view.unicode(); - bool is_unicode_sets_mode = state.current_options.has_flag_set(AllFlags::UnicodeSets); - - // In /u mode, case folding happens after complementing: \P{x} matches caseFold(allChars - charsWithX). - // This means a code point matches the inverted property if ANY of its case variants lacks the property. - // In /v mode, case folding happens before complementing: \P{x} matches caseFold(allChars) - caseFold(charsWithX), - // so we can just use normal case-insensitive matching and invert the result. - if ((inverse || is_double_negation) && case_insensitive && !is_unicode_sets_mode) { - bool any_variant_lacks_property = false; - Unicode::for_each_case_folded_code_point(code_point, [&](u32 variant) { - if (!Unicode::code_point_has_property(variant, property)) { - any_variant_lacks_property = true; - return IterationDecision::Break; - } - return IterationDecision::Continue; - }); - - if (is_double_negation) { - if (any_variant_lacks_property) - return; - advance_string_position(state, input.view, code_point); - } else if (!any_variant_lacks_property) { - inverse_matched = true; - return; - } - } else { - auto case_sensitivity = case_insensitive && (is_unicode_sets_mode || !inverse) ? CaseSensitivity::CaseInsensitive : CaseSensitivity::CaseSensitive; - if (Unicode::code_point_has_property(code_point, property, case_sensitivity)) { - if (inverse) - inverse_matched = true; - else - advance_string_position(state, input.view, code_point); - } - } -} - -static ALWAYS_INLINE void compare_general_category(MatchInput const& input, MatchState& state, Unicode::GeneralCategory general_category, bool inverse, bool is_double_negation, bool& inverse_matched) -{ - if (state.string_position == input.view.length()) - return; - - u32 code_point = input.view.code_point_at(state.string_position_in_code_units); - bool case_insensitive = (state.current_options & AllFlags::Insensitive) && input.view.unicode(); - bool is_unicode_sets_mode = state.current_options.has_flag_set(AllFlags::UnicodeSets); - - // See comment in compare_property for /u vs /v mode case folding semantics. - if ((inverse || is_double_negation) && case_insensitive && !is_unicode_sets_mode) { - bool any_variant_lacks_category = false; - Unicode::for_each_case_folded_code_point(code_point, [&](u32 variant) { - if (!Unicode::code_point_has_general_category(variant, general_category)) { - any_variant_lacks_category = true; - return IterationDecision::Break; - } - return IterationDecision::Continue; - }); - - if (is_double_negation) { - if (any_variant_lacks_category) - return; - advance_string_position(state, input.view, code_point); - } else if (!any_variant_lacks_category) { - inverse_matched = true; - return; - } - } else { - auto case_sensitivity = case_insensitive && (is_unicode_sets_mode || !inverse) ? CaseSensitivity::CaseInsensitive : CaseSensitivity::CaseSensitive; - if (Unicode::code_point_has_general_category(code_point, general_category, case_sensitivity)) { - if (inverse) - inverse_matched = true; - else - advance_string_position(state, input.view, code_point); - } - } -} - -static ALWAYS_INLINE void compare_script(MatchInput const& input, MatchState& state, Unicode::Script script, bool inverse, bool& inverse_matched) -{ - if (state.string_position == input.view.length()) - return; - - u32 code_point = input.view.code_point_at(state.string_position_in_code_units); - bool equal = Unicode::code_point_has_script(code_point, script); - - if (equal) { - if (inverse) - inverse_matched = true; - else - advance_string_position(state, input.view, code_point); - } -} - -static ALWAYS_INLINE void compare_script_extension(MatchInput const& input, MatchState& state, Unicode::Script script, bool inverse, bool& inverse_matched) -{ - if (state.string_position == input.view.length()) - return; - - u32 code_point = input.view.code_point_at(state.string_position_in_code_units); - bool equal = Unicode::code_point_has_script_extension(code_point, script); - - if (equal) { - if (inverse) - inverse_matched = true; - else - advance_string_position(state, input.view, code_point); - } -} - -#if REGEX_DEBUG -static RegexDebug s_regex_dbg(stderr); -#endif - -template -ALWAYS_INLINE ExecutionResult compare_execute(ByteCode const& bc, MatchInput const& input, MatchState& state) -{ - auto const argument_count = IsSimple ? 1 : bc.at(state.instruction_position + 1); - auto has_single_argument = argument_count == 1; - - bool inverse { false }; - bool temporary_inverse { false }; - bool reset_temp_inverse { false }; - struct DisjunctionState { - bool active { false }; - bool is_conjunction { false }; - bool is_subtraction { false }; - bool is_and_operation { false }; - bool fail { false }; - bool inverse_matched { false }; - size_t subtraction_operand_index { 0 }; - size_t initial_position; - size_t initial_code_unit_position; - Optional last_accepted_position {}; - Optional last_accepted_code_unit_position {}; - }; - - Vector disjunction_states; - disjunction_states.unchecked_empend(); - - auto current_disjunction_state = [&]() -> DisjunctionState& { return disjunction_states.last(); }; - - auto current_inversion_state = [&]() -> bool { - if constexpr (IsSimple) - return false; - else - return temporary_inverse ^ inverse; - }; - - size_t string_position = state.string_position; - bool inverse_matched { false }; - bool had_zero_length_match { false }; - - state.string_position_before_match = state.string_position; - - bool has_string_set = false; - bool string_set_matched = false; - size_t best_match_position = state.string_position; - size_t best_match_position_in_code_units = state.string_position_in_code_units; - - size_t offset { state.instruction_position + (IsSimple ? 2 : 3) }; - CharacterCompareType last_compare_type = CharacterCompareType::Undefined; - - auto const* bytecode_data = bc.flat_data().data(); - - for (size_t i = 0; i < argument_count; ++i) { - if (state.string_position > string_position) - break; - - if (has_string_set) { - state.string_position = string_position; - state.string_position_in_code_units = current_disjunction_state().initial_code_unit_position; - } - - auto compare_type = (CharacterCompareType)bytecode_data[offset++]; - - if constexpr (!IsSimple) { - if (reset_temp_inverse) { - reset_temp_inverse = false; - if (compare_type != CharacterCompareType::Property || last_compare_type != CharacterCompareType::StringSet) { - temporary_inverse = false; - } - } else { - reset_temp_inverse = true; - } - - last_compare_type = compare_type; - } - - switch (compare_type) { - case CharacterCompareType::Inverse: - inverse = !inverse; - continue; - case CharacterCompareType::TemporaryInverse: - // If "TemporaryInverse" is given, negate the current inversion state only for the next opcode. - // it follows that this cannot be the last compare element. - VERIFY(!IsSimple); - VERIFY(i != argument_count - 1); - - temporary_inverse = true; - reset_temp_inverse = false; - continue; - case CharacterCompareType::Char: { - u32 ch = bytecode_data[offset++]; - - // We want to compare a string that is longer or equal in length to the available string - if (input.view.length() <= state.string_position) - return ExecutionResult::Failed_ExecuteLowPrioForks; - - compare_char(input, state, ch, current_inversion_state(), inverse_matched); - break; - } - case CharacterCompareType::AnyChar: { - // We want to compare a string that is definitely longer than the available string - if (input.view.length() <= state.string_position) - return ExecutionResult::Failed_ExecuteLowPrioForks; - - auto input_view = input.view.substring_view(state.string_position, 1).code_point_at(0); - auto is_equivalent_to_newline = input_view == '\n' - || (state.current_options.has_flag_set(AllFlags::Internal_ECMA262DotSemantics) - ? (input_view == '\r' || input_view == LineSeparator || input_view == ParagraphSeparator) - : false); - - if (!is_equivalent_to_newline || (state.current_options.has_flag_set(AllFlags::SingleLine) && state.current_options.has_flag_set(AllFlags::Internal_ConsiderNewline))) { - if (current_inversion_state()) - inverse_matched = true; - else - advance_string_position(state, input.view, input_view); - } - break; - } - case CharacterCompareType::String: { - VERIFY(!current_inversion_state()); - - auto string_index = bytecode_data[offset++]; - auto string = bc.get_u16_string(string_index); - - // We want to compare a string that is definitely longer than the available string - if (input.view.unicode()) { - if (input.view.length() < state.string_position + string.length_in_code_points()) - return ExecutionResult::Failed_ExecuteLowPrioForks; - } else { - if (input.view.length() < state.string_position_in_code_units + string.length_in_code_units()) - return ExecutionResult::Failed_ExecuteLowPrioForks; - } - - auto view = RegexStringView(string); - view.set_unicode(input.view.unicode()); - if (compare_string(input, state, view, had_zero_length_match)) { - if (current_inversion_state()) - inverse_matched = true; - } - break; - } - case CharacterCompareType::CharClass: { - if (input.view.length_in_code_units() <= state.string_position_in_code_units) - return ExecutionResult::Failed_ExecuteLowPrioForks; - - auto character_class = (CharClass)bytecode_data[offset++]; - auto ch = input.view.unicode_aware_code_point_at(state.string_position_in_code_units); - - compare_character_class(input, state, character_class, ch, current_inversion_state(), inverse_matched); - break; - } - case CharacterCompareType::LookupTable: { - if (input.view.length() <= state.string_position) - return ExecutionResult::Failed_ExecuteLowPrioForks; - - auto count_sensitive = bytecode_data[offset++]; - auto count_insensitive = bytecode_data[offset++]; - auto sensitive_range_data = bc.flat_data().slice(offset, count_sensitive); - offset += count_sensitive; - auto insensitive_range_data = bc.flat_data().slice(offset, count_insensitive); - offset += count_insensitive; - - bool const insensitive = state.current_options & AllFlags::Insensitive; - auto ch = input.view.unicode_aware_code_point_at(state.string_position_in_code_units); - - if (insensitive) - ch = to_ascii_lowercase(ch); - - auto const ranges = insensitive && !insensitive_range_data.is_empty() ? insensitive_range_data : sensitive_range_data; - auto const* matching_range = binary_search(ranges, ch, nullptr, [](auto needle, CharRange range) { - if (needle >= range.from && needle <= range.to) - return 0; - if (needle > range.to) - return 1; - return -1; - }); - - if (matching_range) { - if (current_inversion_state()) - inverse_matched = true; - else - advance_string_position(state, input.view, ch); - } - break; - } - case CharacterCompareType::CharRange: { - if (input.view.length() <= state.string_position) - return ExecutionResult::Failed_ExecuteLowPrioForks; - - auto value = (CharRange)bytecode_data[offset++]; - - auto from = value.from; - auto to = value.to; - auto ch = input.view.unicode_aware_code_point_at(state.string_position_in_code_units); - - compare_character_range(input, state, from, to, ch, current_inversion_state(), inverse_matched); - break; - } - case CharacterCompareType::Reference: { - auto reference_number = ((size_t)bytecode_data[offset++]) - 1; - if (input.match_index >= state.capture_group_matches_size()) { - had_zero_length_match = true; - if (current_inversion_state()) - inverse_matched = true; - break; - } - - auto groups = state.capture_group_matches(input.match_index); - - if (groups.size() <= reference_number) { - had_zero_length_match = true; - if (current_inversion_state()) - inverse_matched = true; - break; - } - - auto str = groups.at(reference_number).view; - - // We want to compare a string that is definitely longer than the available string - if (input.view.length() < state.string_position + str.length()) - return ExecutionResult::Failed_ExecuteLowPrioForks; - - if (compare_string(input, state, str, had_zero_length_match)) { - if (current_inversion_state()) - inverse_matched = true; - } - break; - } - case CharacterCompareType::NamedReference: { - auto reference_number = ((size_t)bytecode_data[offset++]) - 1; - - if (input.match_index >= state.capture_group_matches_size()) { - had_zero_length_match = true; - if (current_inversion_state()) - inverse_matched = true; - break; - } - - auto groups = state.capture_group_matches(input.match_index); - - if (groups.size() <= reference_number) { - had_zero_length_match = true; - if (current_inversion_state()) - inverse_matched = true; - break; - } - - RegexStringView str {}; - - auto reference_name_index = bc.get_group_name_index(reference_number); - - if (reference_name_index.has_value()) { - auto target_name_string = bc.get_string(reference_name_index.value()); - - for (size_t i = 0; i < groups.size(); ++i) { - if (groups[i].view.is_null()) - continue; - - auto group_name_index = bc.get_group_name_index(i); - - if (group_name_index.has_value()) { - auto group_name_string = bc.get_string(group_name_index.value()); - - if (group_name_string == target_name_string) { - str = groups[i].view; - break; - } - } - } - } - - if (input.view.length() < state.string_position + str.length()) { - return ExecutionResult::Failed_ExecuteLowPrioForks; - } - - if (compare_string(input, state, str, had_zero_length_match)) { - if (current_inversion_state()) - inverse_matched = true; - } - break; - } - case CharacterCompareType::Property: { - auto property = static_cast(bytecode_data[offset++]); - compare_property(input, state, property, current_inversion_state(), temporary_inverse && inverse, inverse_matched); - break; - } - case CharacterCompareType::GeneralCategory: { - auto general_category = static_cast(bytecode_data[offset++]); - compare_general_category(input, state, general_category, current_inversion_state(), temporary_inverse && inverse, inverse_matched); - break; - } - case CharacterCompareType::Script: { - auto script = static_cast(bytecode_data[offset++]); - compare_script(input, state, script, current_inversion_state(), inverse_matched); - break; - } - case CharacterCompareType::ScriptExtension: { - auto script = static_cast(bytecode_data[offset++]); - compare_script_extension(input, state, script, current_inversion_state(), inverse_matched); - break; - } - case CharacterCompareType::StringSet: { - has_string_set = true; - auto string_set_index = bytecode_data[offset++]; - - bool matched = false; - size_t longest_match_length = 0; - - auto find_longest_match = [&](auto const& view, auto const& trie) { - auto const* current = ≜ - size_t current_code_unit_offset = state.string_position_in_code_units; - - if (current->has_metadata() && current->metadata_value()) { - matched = true; - longest_match_length = 0; - } - - while (true) { - u32 value; - - if constexpr (IsSame) { - if (current_code_unit_offset >= view.length_in_code_units()) - break; - value = view.code_unit_at(current_code_unit_offset); - } else { - if (current_code_unit_offset >= input.view.length_in_code_units()) - break; - value = input.view.code_point_at(current_code_unit_offset); - } - - if (state.current_options & AllFlags::Insensitive) { - bool found_child = false; - for (auto const& [key, child] : current->children()) { - if (Unicode::canonicalize(key, input.view.unicode()) == Unicode::canonicalize(value, input.view.unicode())) { - current = static_cast(child.ptr()); - current_code_unit_offset++; - found_child = true; - break; - } - } - if (!found_child) - break; - } else { - auto it = current->children().find(value); - if (it == current->children().end()) - break; - - current = static_cast(it->value.ptr()); - current_code_unit_offset++; - } - - auto is_terminal = current->has_metadata() && current->metadata_value(); - if (is_terminal) { - size_t match_length_in_code_points; - if constexpr (IsSame) { - size_t code_points = 0; - for (size_t i = state.string_position_in_code_units; i < current_code_unit_offset;) { - auto code_point = view.code_point_at(i); - i += code_point >= 0x10000 ? 2 : 1; - code_points++; - } - match_length_in_code_points = code_points; - } else { - size_t code_points = 0; - for (size_t i = state.string_position_in_code_units; i < current_code_unit_offset;) { - auto code_point = input.view.code_point_at(i); - if (code_point <= 0x7F) - i += 1; - else if (code_point <= 0x7FF) - i += 2; - else if (code_point <= 0xFFFF) - i += 3; - else - i += 4; - code_points++; - } - match_length_in_code_points = code_points; - } - - if (match_length_in_code_points > longest_match_length) { - matched = true; - longest_match_length = match_length_in_code_points; - } - } - } - }; - - if (input.view.u16_view().is_null()) { - auto const& trie = bc.string_set_table().get_u8_trie(string_set_index); - StringView view; - find_longest_match(view, trie); - } else { - auto const& view = input.view.u16_view(); - auto const& trie = bc.string_set_table().get_u16_trie(string_set_index); - find_longest_match(view, trie); - } - - if (matched) { - if (longest_match_length == 0) - had_zero_length_match = true; - if (current_inversion_state()) { - inverse_matched = true; - } else { - state.string_position += longest_match_length; - if (input.view.unicode()) { - state.string_position_in_code_units = input.view.code_unit_offset_of(state.string_position); - } else { - state.string_position_in_code_units = state.string_position; - } - } - } - break; - } - case CharacterCompareType::And: - VERIFY(!IsSimple); - if constexpr (!IsSimple) { - disjunction_states.append({ - .active = true, - .is_conjunction = current_inversion_state(), - .is_and_operation = true, - .fail = current_inversion_state(), - .inverse_matched = current_inversion_state(), - .initial_position = state.string_position, - .initial_code_unit_position = state.string_position_in_code_units, - }); - } - continue; - case CharacterCompareType::Subtract: - VERIFY(!IsSimple); - if constexpr (!IsSimple) { - disjunction_states.append({ - .active = true, - .is_conjunction = true, - .is_subtraction = true, - .fail = true, - .inverse_matched = false, - .initial_position = state.string_position, - .initial_code_unit_position = state.string_position_in_code_units, - }); - } - continue; - case CharacterCompareType::Or: - VERIFY(!IsSimple); - if constexpr (!IsSimple) { - disjunction_states.append({ - .active = true, - .is_conjunction = !current_inversion_state(), - .fail = !current_inversion_state(), - .inverse_matched = !current_inversion_state(), - .initial_position = state.string_position, - .initial_code_unit_position = state.string_position_in_code_units, - }); - } - continue; - case CharacterCompareType::EndAndOr: { - VERIFY(!IsSimple); - if constexpr (!IsSimple) { - auto disjunction_state = disjunction_states.take_last(); - if (!disjunction_state.fail) { - state.string_position = disjunction_state.last_accepted_position.value_or(disjunction_state.initial_position); - state.string_position_in_code_units = disjunction_state.last_accepted_code_unit_position.value_or(disjunction_state.initial_code_unit_position); - } else if (has_string_set) { - string_set_matched = false; - best_match_position = disjunction_state.initial_position; - best_match_position_in_code_units = disjunction_state.initial_code_unit_position; - } - inverse_matched = disjunction_state.inverse_matched || disjunction_state.fail; - } - break; - } - default: - warnln("Undefined comparison: {}", (int)compare_type); - VERIFY_NOT_REACHED(); - break; - } - - if constexpr (!IsSimple) { - auto& new_disjunction_state = current_disjunction_state(); - if (current_inversion_state() && (!inverse || new_disjunction_state.active) && !inverse_matched) { - advance_string_position(state, input.view); - inverse_matched = true; - } - } - - if (has_string_set && state.string_position > best_match_position) { - best_match_position = state.string_position; - best_match_position_in_code_units = state.string_position_in_code_units; - string_set_matched = true; - } - - if constexpr (!IsSimple) { - auto& new_disjunction_state = current_disjunction_state(); - if (!has_single_argument && new_disjunction_state.active) { - auto failed = (!had_zero_length_match && string_position == state.string_position) || state.string_position > input.view.length(); - - if (!failed && new_disjunction_state.is_and_operation - && new_disjunction_state.last_accepted_position.has_value() - && new_disjunction_state.last_accepted_position.value() != state.string_position) { - - failed = true; - } - - if (!failed) { - new_disjunction_state.last_accepted_position = state.string_position; - new_disjunction_state.last_accepted_code_unit_position = state.string_position_in_code_units; - new_disjunction_state.inverse_matched |= inverse_matched; - } - - if (new_disjunction_state.is_subtraction) { - if (new_disjunction_state.subtraction_operand_index == 0) { - new_disjunction_state.fail = failed && new_disjunction_state.fail; - } else if (!failed && (!has_string_set || state.string_position >= best_match_position)) { - new_disjunction_state.fail = true; - } - new_disjunction_state.subtraction_operand_index++; - } else if (new_disjunction_state.is_conjunction) { - new_disjunction_state.fail = failed && new_disjunction_state.fail; - } else { - new_disjunction_state.fail = failed || new_disjunction_state.fail; - } - - state.string_position = new_disjunction_state.initial_position; - state.string_position_in_code_units = new_disjunction_state.initial_code_unit_position; - inverse_matched = false; - } - } - } - - if constexpr (!IsSimple) { - if (!has_single_argument) { - auto& new_disjunction_state = current_disjunction_state(); - if (new_disjunction_state.active && !new_disjunction_state.fail) { - state.string_position = new_disjunction_state.last_accepted_position.value_or(new_disjunction_state.initial_position); - state.string_position_in_code_units = new_disjunction_state.last_accepted_code_unit_position.value_or(new_disjunction_state.initial_code_unit_position); - } - } - } - - if (has_string_set && string_set_matched) { - if (has_single_argument || best_match_position > string_position) { - state.string_position = best_match_position; - state.string_position_in_code_units = best_match_position_in_code_units; - } - } - - if (current_inversion_state() && !inverse_matched && state.string_position == string_position) - advance_string_position(state, input.view); - - if ((!had_zero_length_match && string_position == state.string_position) || state.string_position > input.view.length()) - return ExecutionResult::Failed_ExecuteLowPrioForks; - - return ExecutionResult::Continue; -} - -struct InstructionResult { - ExecutionResult result; - size_t size; -}; - -// Switch-based interpreter: executes a single bytecode instruction. -// Returns the ExecutionResult and the instruction size. -static ALWAYS_INLINE InstructionResult execute_instruction( - OpCodeId id, - ByteCodeValueType const* data, - size_t data_size, - FlatByteCode const& bytecode, - MatchInput const& input, - MatchState& state) -{ - auto ip = state.instruction_position; - auto current_ip_size = opcode_size(id, data, ip); - - switch (id) { - case OpCodeId::Exit: { - if (state.string_position > input.view.length() || state.instruction_position >= data_size) - return { ExecutionResult::Succeeded, current_ip_size }; - return { ExecutionResult::Failed, current_ip_size }; - } - case OpCodeId::Save: { - save_string_position(input, state); - state.forks_since_last_save = 0; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::Restore: { - if (!restore_string_position(input, state)) - return { ExecutionResult::Failed, current_ip_size }; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::GoBack: { - auto count = data[ip + OpArgs::GoBack::count]; - if (count > state.string_position) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - reverse_string_position(state, input.view, count); - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::SetStepBack: { - state.step_backs.append(static_cast(data[ip + OpArgs::SetStepBack::step])); - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::IncStepBack: { - if (state.step_backs.is_empty()) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - size_t last_step_back = static_cast(++state.step_backs.last()); - if (last_step_back > state.string_position) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - reverse_string_position(state, input.view, last_step_back); - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::CheckStepBack: { - if (state.step_backs.is_empty()) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - if (input.saved_positions.is_empty()) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - if (static_cast(state.step_backs.last()) > input.saved_positions.last()) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - state.string_position = input.saved_positions.last(); - state.string_position_in_code_units = input.saved_code_unit_positions.last(); - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::CheckSavedPosition: { - if (input.saved_positions.is_empty()) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - if (state.string_position != input.saved_positions.last()) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - state.step_backs.take_last(); - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::FailForks: { - input.fail_counter += state.forks_since_last_save; - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - } - case OpCodeId::PopSaved: { - if (input.saved_positions.is_empty() || input.saved_code_unit_positions.is_empty()) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - input.saved_positions.take_last(); - input.saved_code_unit_positions.take_last(); - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - } - case OpCodeId::Jump: { - state.instruction_position += static_cast(data[ip + OpArgs::Jump::offset]); - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::ForkJump: { - auto offset = static_cast(data[ip + OpArgs::Jump::offset]); - state.fork_at_position = ip + current_ip_size + offset; - state.forks_since_last_save++; - return { ExecutionResult::Fork_PrioHigh, current_ip_size }; - } - case OpCodeId::ForkReplaceJump: { - auto offset = static_cast(data[ip + OpArgs::Jump::offset]); - state.fork_at_position = ip + current_ip_size + offset; - input.fork_to_replace = ip; - state.forks_since_last_save++; - return { ExecutionResult::Fork_PrioHigh, current_ip_size }; - } - case OpCodeId::ForkStay: { - auto offset = static_cast(data[ip + OpArgs::Jump::offset]); - state.fork_at_position = ip + current_ip_size + offset; - state.forks_since_last_save++; - return { ExecutionResult::Fork_PrioLow, current_ip_size }; - } - case OpCodeId::ForkReplaceStay: { - auto offset = static_cast(data[ip + OpArgs::Jump::offset]); - state.fork_at_position = ip + current_ip_size + offset; - input.fork_to_replace = ip; - return { ExecutionResult::Fork_PrioLow, current_ip_size }; - } - case OpCodeId::ForkIf: { - auto offset = static_cast(data[ip + OpArgs::ForkIf::offset]); - auto form = static_cast(data[ip + OpArgs::ForkIf::form]); - auto condition = static_cast(data[ip + OpArgs::ForkIf::condition]); - - auto next_step = [&](bool do_fork) -> ExecutionResult { - switch (form) { - case OpCodeId::ForkJump: - if (do_fork) { - state.fork_at_position = ip + current_ip_size + offset; - state.forks_since_last_save++; - return ExecutionResult::Fork_PrioHigh; - } - return ExecutionResult::Continue; - case OpCodeId::ForkReplaceJump: - if (do_fork) { - state.fork_at_position = ip + current_ip_size + offset; - input.fork_to_replace = ip; - state.forks_since_last_save++; - return ExecutionResult::Fork_PrioHigh; - } - return ExecutionResult::Continue; - case OpCodeId::ForkStay: - if (do_fork) { - state.fork_at_position = ip + current_ip_size + offset; - state.forks_since_last_save++; - return ExecutionResult::Fork_PrioLow; - } - state.instruction_position += offset; - return ExecutionResult::Continue; - case OpCodeId::ForkReplaceStay: - if (do_fork) { - state.fork_at_position = ip + current_ip_size + offset; - input.fork_to_replace = ip; - return ExecutionResult::Fork_PrioLow; - } - state.instruction_position += offset; - return ExecutionResult::Continue; - default: - VERIFY_NOT_REACHED(); - } - }; - - switch (condition) { - case ForkIfCondition::AtStartOfLine: - return { next_step(!input.in_the_middle_of_a_line), current_ip_size }; - case ForkIfCondition::Invalid: - default: - VERIFY_NOT_REACHED(); - } - } - case OpCodeId::CheckBegin: { - auto is_at_line_boundary = [&] { - if (state.string_position == 0) - return true; - if (state.current_options.has_flag_set(AllFlags::Multiline) && state.current_options.has_flag_set(AllFlags::Internal_ConsiderNewline)) { - auto input_view = input.view.substring_view(state.string_position - 1, 1).code_point_at(0); - return input_view == '\r' || input_view == '\n' || input_view == LineSeparator || input_view == ParagraphSeparator; - } - return false; - }(); - if (is_at_line_boundary && (state.current_options & AllFlags::MatchNotBeginOfLine)) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - if ((is_at_line_boundary && !(state.current_options & AllFlags::MatchNotBeginOfLine)) - || (!is_at_line_boundary && (state.current_options & AllFlags::MatchNotBeginOfLine)) - || (is_at_line_boundary && (state.current_options & AllFlags::Global))) - return { ExecutionResult::Continue, current_ip_size }; - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - } - case OpCodeId::CheckEnd: { - auto is_at_line_boundary = [&] { - if (state.string_position == input.view.length()) - return true; - if (state.current_options.has_flag_set(AllFlags::Multiline) && state.current_options.has_flag_set(AllFlags::Internal_ConsiderNewline)) { - auto input_view = input.view.substring_view(state.string_position, 1).code_point_at(0); - return input_view == '\r' || input_view == '\n' || input_view == LineSeparator || input_view == ParagraphSeparator; - } - return false; - }(); - if (is_at_line_boundary && (state.current_options & AllFlags::MatchNotEndOfLine)) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - if ((is_at_line_boundary && !(state.current_options & AllFlags::MatchNotEndOfLine)) - || (!is_at_line_boundary && (state.current_options & AllFlags::MatchNotEndOfLine || state.current_options & AllFlags::MatchNotBeginOfLine))) - return { ExecutionResult::Continue, current_ip_size }; - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - } - case OpCodeId::CheckBoundary: { - auto boundary_type = static_cast(data[ip + OpArgs::CheckBoundary::type]); - auto isword = [&](auto ch) { - return is_word_character(ch, state.current_options & AllFlags::Insensitive, input.view.unicode()); - }; - auto is_word_boundary = [&] { - if (state.string_position == input.view.length()) - return (state.string_position > 0 && isword(input.view.code_point_at(state.string_position_in_code_units - 1))); - if (state.string_position == 0) - return (isword(input.view.code_point_at(0))); - return !!(isword(input.view.code_point_at(state.string_position_in_code_units)) ^ isword(input.view.code_point_at(state.string_position_in_code_units - 1))); - }; - switch (boundary_type) { - case BoundaryCheckType::Word: - if (is_word_boundary()) - return { ExecutionResult::Continue, current_ip_size }; - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - case BoundaryCheckType::NonWord: - if (!is_word_boundary()) - return { ExecutionResult::Continue, current_ip_size }; - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - } - VERIFY_NOT_REACHED(); - } - case OpCodeId::ClearCaptureGroup: { - auto group_id = data[ip + OpArgs::ClearCaptureGroup::id]; - if (input.match_index < state.capture_group_matches_size()) { - auto group = state.mutable_capture_group_matches(input.match_index); - group[group_id - 1].reset(); - } - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::FailIfEmpty: { - auto checkpoint_id = data[ip + OpArgs::FailIfEmpty::checkpoint]; - u64 current_position = state.string_position + 1; - auto checkpoint_position = state.checkpoints.get(checkpoint_id).value_or(current_position); - if (checkpoint_position == current_position) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::SaveLeftCaptureGroup: { - auto group_id = data[ip + OpArgs::SaveLeftCaptureGroup::id]; - if (input.match_index >= state.capture_group_matches_size()) { - state.flat_capture_group_matches.ensure_capacity((input.match_index + 1) * state.capture_group_count); - for (size_t i = state.capture_group_matches_size(); i <= input.match_index; ++i) - for (size_t j = 0; j < state.capture_group_count; ++j) - state.flat_capture_group_matches.append({}); - } - state.mutable_capture_group_matches(input.match_index).at(group_id - 1).left_column = state.string_position; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::SaveRightCaptureGroup: { - auto group_id = data[ip + OpArgs::SaveRightCaptureGroup::id]; - auto& match = state.capture_group_matches(input.match_index).at(group_id - 1); - auto start_position = match.left_column; - if (state.string_position < start_position) { - dbgln("Right capture group {} is before left capture group {}!", state.string_position, start_position); - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - } - auto length = state.string_position - start_position; - if (start_position < match.column && state.step_backs.is_empty()) - return { ExecutionResult::Continue, current_ip_size }; - VERIFY(start_position + length <= input.view.length_in_code_units()); - auto captured_text = input.view.substring_view(start_position, length); - auto& existing_capture = state.mutable_capture_group_matches(input.match_index).at(group_id - 1); - if (length == 0 && !existing_capture.view.is_null() && existing_capture.view.length() > 0) { - auto existing_end_position = existing_capture.global_offset - input.global_offset + existing_capture.view.length(); - if (existing_end_position == state.string_position) - return { ExecutionResult::Continue, current_ip_size }; - } - state.mutable_capture_group_matches(input.match_index).at(group_id - 1) = { captured_text, input.line, start_position, input.global_offset + start_position }; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::SaveRightNamedCaptureGroup: { - auto name_index = data[ip + OpArgs::SaveRightNamedCaptureGroup::name_index]; - auto group_id = data[ip + OpArgs::SaveRightNamedCaptureGroup::id]; - auto& match = state.capture_group_matches(input.match_index).at(group_id - 1); - auto start_position = match.left_column; - if (state.string_position < start_position) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - auto length = state.string_position - start_position; - if (start_position < match.column) - return { ExecutionResult::Continue, current_ip_size }; - VERIFY(start_position + length <= input.view.length_in_code_units()); - auto view = input.view.substring_view(start_position, length); - auto& existing_capture = state.mutable_capture_group_matches(input.match_index).at(group_id - 1); - if (length == 0 && !existing_capture.view.is_null() && existing_capture.view.length() > 0) { - auto existing_end_position = existing_capture.global_offset - input.global_offset + existing_capture.view.length(); - if (existing_end_position == state.string_position) - return { ExecutionResult::Continue, current_ip_size }; - } - state.mutable_capture_group_matches(input.match_index).at(group_id - 1) = { view, name_index, input.line, start_position, input.global_offset + start_position }; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::RSeekTo: { - auto ch = data[ip + OpArgs::RSeekTo::ch]; - - size_t search_from; - size_t search_from_in_code_units; - auto line_limited = false; - - if (state.string_position_before_rseek == NumericLimits::max()) { - state.string_position_before_rseek = state.string_position; - state.string_position_in_code_units_before_rseek = state.string_position_in_code_units; - - if (!input.regex_options.has_flag_set(AllFlags::SingleLine)) { - auto end_of_line = input.view.find_end_of_line(state.string_position, state.string_position_in_code_units); - search_from = end_of_line.code_point_index + 1; - search_from_in_code_units = end_of_line.code_unit_index + 1; - line_limited = true; - } else { - search_from = NumericLimits::max(); - search_from_in_code_units = NumericLimits::max(); - } - } else { - search_from = state.string_position; - search_from_in_code_units = state.string_position_in_code_units; - } - - auto next = input.view.find_index_of_previous(ch, search_from, search_from_in_code_units); - if (!next.has_value() || next->code_unit_index < state.string_position_in_code_units_before_rseek) { - if (line_limited) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - return { ExecutionResult::Failed_ExecuteLowPrioForksButNoFurtherPossibleMatches, current_ip_size }; - } - state.string_position = next->code_point_index; - state.string_position_in_code_units = next->code_unit_index; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::SaveModifiers: { - auto current_flags = to_underlying(state.current_options.value()); - state.modifier_stack.append(current_flags); - state.current_options = AllOptions { static_cast(data[ip + OpArgs::SaveModifiers::new_modifiers]) }; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::RestoreModifiers: { - if (state.modifier_stack.is_empty()) - return { ExecutionResult::Failed, current_ip_size }; - auto previous_modifiers = state.modifier_stack.take_last(); - state.current_options = AllOptions { static_cast(previous_modifiers) }; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::Repeat: { - auto repeat_offset = data[ip + OpArgs::Repeat::offset]; - auto repeat_count = data[ip + OpArgs::Repeat::count]; - auto repeat_id = data[ip + OpArgs::Repeat::id]; - VERIFY(repeat_count > 0); - if (repeat_id >= state.repetition_marks.size()) - state.repetition_marks.resize(repeat_id + 1); - auto& repetition_mark = state.repetition_marks.mutable_at(repeat_id); - if (repetition_mark == repeat_count - 1) { - repetition_mark = 0; - } else { - state.instruction_position -= repeat_offset + current_ip_size; - ++repetition_mark; - } - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::ResetRepeat: { - auto repeat_id = data[ip + OpArgs::ResetRepeat::id]; - if (repeat_id >= state.repetition_marks.size()) - state.repetition_marks.resize(repeat_id + 1); - state.repetition_marks.mutable_at(repeat_id) = 0; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::Checkpoint: { - auto checkpoint_id = data[ip + OpArgs::Checkpoint::id]; - if (checkpoint_id >= state.checkpoints.size()) - state.checkpoints.resize(checkpoint_id + 1); - state.checkpoints[checkpoint_id] = state.string_position + 1; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::JumpNonEmpty: { - auto offset = static_cast(data[ip + OpArgs::JumpNonEmpty::offset]); - auto checkpoint_id = data[ip + OpArgs::JumpNonEmpty::checkpoint]; - auto form = static_cast(data[ip + OpArgs::JumpNonEmpty::form]); - - u64 current_position = state.string_position; - auto checkpoint_position = state.checkpoints.get(checkpoint_id).value_or(0); - - if (checkpoint_position != 0 && checkpoint_position != current_position + 1) { - if (form == OpCodeId::Jump) { - state.instruction_position += offset; - return { ExecutionResult::Continue, current_ip_size }; - } - state.fork_at_position = ip + current_ip_size + offset; - if (form == OpCodeId::ForkJump) { - state.forks_since_last_save++; - return { ExecutionResult::Fork_PrioHigh, current_ip_size }; - } - if (form == OpCodeId::ForkStay) { - state.forks_since_last_save++; - return { ExecutionResult::Fork_PrioLow, current_ip_size }; - } - if (form == OpCodeId::ForkReplaceStay) { - input.fork_to_replace = ip; - return { ExecutionResult::Fork_PrioLow, current_ip_size }; - } - if (form == OpCodeId::ForkReplaceJump) { - input.fork_to_replace = ip; - return { ExecutionResult::Fork_PrioHigh, current_ip_size }; - } - } - if (form == OpCodeId::Jump && state.string_position < input.view.length()) - return { ExecutionResult::Failed_ExecuteLowPrioForks, current_ip_size }; - return { ExecutionResult::Continue, current_ip_size }; - } - case OpCodeId::Compare: - return { compare_execute(bytecode, input, state), current_ip_size }; - case OpCodeId::CompareSimple: - return { compare_execute(bytecode, input, state), current_ip_size }; - } - VERIFY_NOT_REACHED(); -} - -template -regex::Parser::Result Regex::parse_pattern(StringView pattern, typename ParserTraits::OptionsType regex_options) -{ - regex::Lexer lexer(pattern); - - Parser parser(lexer, regex_options); - return parser.parse(); -} - -template -struct CacheKey { - ByteString pattern; - typename ParserTraits::OptionsType options; - - bool operator==(CacheKey const& other) const - { - return pattern == other.pattern && options.value() == other.options.value(); - } -}; -template -static OrderedHashMap, regex::Parser::Result> s_parser_cache; - -template -static size_t s_cached_bytecode_size = 0; - -static constexpr auto MaxRegexCachedBytecodeSize = 1 * MiB; - -template -static void cache_parse_result(regex::Parser::Result const& result, CacheKey const& key) -{ - auto bytecode_size = result.bytecode.visit([](auto& bytecode) { return bytecode.size() * sizeof(ByteCodeValueType); }); - if (bytecode_size > MaxRegexCachedBytecodeSize) - return; - - while (bytecode_size + s_cached_bytecode_size > MaxRegexCachedBytecodeSize) - s_cached_bytecode_size -= s_parser_cache.take_first().bytecode.visit([](auto& bytecode) { return bytecode.size() * sizeof(ByteCodeValueType); }); - - s_parser_cache.set(key, result); - s_cached_bytecode_size += bytecode_size; -} - -template -Regex::Regex(ByteString pattern, typename ParserTraits::OptionsType regex_options) - : pattern_value(move(pattern)) - , parser_result(ByteCode {}) -{ - if (auto cache_entry = s_parser_cache.get({ pattern_value, regex_options }); cache_entry.has_value()) { - parser_result = cache_entry.value(); - } else { - regex::Lexer lexer(pattern_value); - - Parser parser(lexer, regex_options); - parser_result = parser.parse(); - parser_result.bytecode.template get().flatten(); - - run_optimization_passes(); - - if (parser_result.error == regex::Error::NoError) - cache_parse_result(parser_result, { pattern_value, regex_options }); - } - - if (parser_result.error == regex::Error::NoError) - matcher = make>(this, static_cast(parser_result.options.value())); -} - -template -Regex::Regex(regex::Parser::Result parse_result, ByteString pattern, typename ParserTraits::OptionsType regex_options) - : pattern_value(move(pattern)) - , parser_result(move(parse_result)) -{ - parser_result.bytecode.template get().flatten(); - run_optimization_passes(); - if (parser_result.error == regex::Error::NoError) - matcher = make>(this, regex_options | static_cast(parser_result.options.value())); -} - -template -Regex::Regex(Regex const& other) - : pattern_value(other.pattern_value) - , parser_result(other.parser_result) -{ - if (other.matcher) - matcher = make>(this, other.matcher->options()); -} - -template -Regex::Regex(Regex&& regex) - : pattern_value(move(regex.pattern_value)) - , parser_result(move(regex.parser_result)) - , matcher(move(regex.matcher)) - , start_offset(regex.start_offset) -{ - if (matcher) - matcher->reset_pattern({}, this); -} - -template -Regex& Regex::operator=(Regex&& regex) -{ - pattern_value = move(regex.pattern_value); - parser_result = move(regex.parser_result); - matcher = move(regex.matcher); - if (matcher) - matcher->reset_pattern({}, this); - start_offset = regex.start_offset; - return *this; -} - -template -typename ParserTraits::OptionsType Regex::options() const -{ - if (!matcher || parser_result.error != Error::NoError) - return {}; - - return matcher->options(); -} - -template -ByteString Regex::error_string(Optional message) const -{ - StringBuilder eb; - eb.append("Error during parsing of regular expression:\n"sv); - eb.appendff(" {}\n ", pattern_value); - for (size_t i = 0; i < parser_result.error_token.position(); ++i) - eb.append(' '); - - eb.appendff("^---- {}", message.value_or(get_error_string(parser_result.error))); - return eb.to_byte_string(); -} - -template -RegexResult Matcher::match(RegexStringView view, Optional::OptionsType> regex_options) const -{ - AllOptions options = m_regex_options | regex_options.value_or({}).value(); - - if constexpr (!IsSame) { - if (options.has_flag_set(AllFlags::Multiline)) - return match(view.lines(), regex_options); // FIXME: how do we know, which line ending a line has (1char or 2char)? This is needed to get the correct match offsets from start of string... - } - - Vector views; - views.append(view); - return match(views, regex_options); -} - -template -RegexResult Matcher::match(Vector const& views, Optional::OptionsType> regex_options) const -{ - // If the pattern *itself* isn't stateful, reset any changes to start_offset. - if (!((AllFlags)m_regex_options.value() & AllFlags::Internal_Stateful)) - m_pattern->start_offset = 0; - - size_t match_count { 0 }; - - MatchInput input; - size_t operations = 0; - - input.pattern = m_pattern->pattern_value; - - input.regex_options = m_regex_options | regex_options.value_or({}).value(); - input.start_offset = m_pattern->start_offset; - MatchState state(m_pattern->parser_result.capture_groups_count, input.regex_options); - size_t lines_to_skip = 0; - - bool unicode = input.regex_options.has_flag_set(AllFlags::Unicode) || input.regex_options.has_flag_set(AllFlags::UnicodeSets); - for (auto const& view : views) - const_cast(view).set_unicode(unicode); - - if constexpr (REGEX_DEBUG) { - if (input.regex_options.has_flag_set(AllFlags::Internal_Stateful)) { - if (views.size() > 1 && input.start_offset > views.first().length()) { - dbgln("Started with start={}, goff={}, skip={}", input.start_offset, input.global_offset, lines_to_skip); - for (auto const& view : views) { - if (input.start_offset < view.length() + 1) - break; - ++lines_to_skip; - input.start_offset -= view.length() + 1; - input.global_offset += view.length() + 1; - } - dbgln("Ended with start={}, goff={}, skip={}", input.start_offset, input.global_offset, lines_to_skip); - } - } - } - - auto append_match = [](auto& input, auto& state, auto& start_position) { - if (state.matches.size() == input.match_index) - state.matches.empend(); - - VERIFY(start_position + state.string_position - start_position <= input.view.length()); - state.matches.mutable_at(input.match_index) = { input.view.substring_view(start_position, state.string_position - start_position), input.line, start_position, input.global_offset + start_position }; - }; - -#if REGEX_DEBUG - s_regex_dbg.print_header(); -#endif - - bool continue_search = input.regex_options.has_flag_set(AllFlags::Global) || input.regex_options.has_flag_set(AllFlags::Multiline); - if (input.regex_options.has_flag_set(AllFlags::Sticky)) - continue_search = false; - - auto single_match_only = input.regex_options.has_flag_set(AllFlags::SingleMatch); - auto only_start_of_line = m_pattern->parser_result.optimization_data.only_start_of_line && !input.regex_options.has_flag_set(AllFlags::Multiline); - - auto compare_range = [insensitive = input.regex_options & AllFlags::Insensitive](auto needle, CharRange range) { - auto upper_case_needle = needle; - auto lower_case_needle = needle; - if (insensitive) { - upper_case_needle = to_ascii_uppercase(needle); - lower_case_needle = to_ascii_lowercase(needle); - } - - if (lower_case_needle >= range.from && lower_case_needle <= range.to) - return 0; - if (upper_case_needle >= range.from && upper_case_needle <= range.to) - return 0; - if (lower_case_needle > range.to || upper_case_needle > range.to) - return 1; - return -1; - }; - - for (auto const& view : views) { - input.in_the_middle_of_a_line = false; - if (lines_to_skip != 0) { - ++input.line; - --lines_to_skip; - continue; - } - input.view = view; - dbgln_if(REGEX_DEBUG, "[match] Starting match with view ({}): _{}_", view.length(), view); - - auto view_length = view.length(); - size_t view_index = m_pattern->start_offset; - state.string_position = view_index; - if (view.unicode()) { - if (view_index < view_length) - state.string_position_in_code_units = view.code_unit_offset_of(view_index); - else - state.string_position_in_code_units = view.length_in_code_units(); - } else { - state.string_position_in_code_units = view_index; - } - bool succeeded = false; - - if (view_index == view_length && m_pattern->parser_result.match_length_minimum == 0) { - // Run the code until it tries to consume something. - // This allows non-consuming code to run on empty strings, for instance - // e.g. "Exit" - size_t temp_operations = operations; - - input.column = match_count; - input.match_index = match_count; - - state.instruction_position = 0; - state.repetition_marks.clear(); - state.modifier_stack.clear(); - state.current_options = input.regex_options; - - auto result = execute(input, state, temp_operations); - // This success is acceptable only if it doesn't read anything from the input (input length is 0). - if (result == ExecuteResult::Matched && (state.string_position <= view_index)) { - operations = temp_operations; - if (!match_count) { - // Nothing was *actually* matched, so append an empty match. - append_match(input, state, view_index); - ++match_count; - - // This prevents a regex pattern like ".*" from matching the empty string - // multiple times, once in this block and once in the following for loop. - if (view_index == 0 && view_length == 0) - ++view_index; - } - } - } - - for (; view_index <= view_length; ++view_index, input.in_the_middle_of_a_line = true) { - if (view_index == view_length) { - if (input.regex_options.has_flag_set(AllFlags::Multiline)) - break; - } - - // FIXME: More performant would be to know the remaining minimum string - // length needed to match from the current position onwards within - // the vm. Add new OpCode for MinMatchLengthFromSp with the value of - // the remaining string length from the current path. The value though - // has to be filled in reverse. That implies a second run over bytecode - // after generation has finished. - auto const match_length_minimum = m_pattern->parser_result.match_length_minimum; - if (match_length_minimum && match_length_minimum > view_length - view_index) - break; - - auto const insensitive = input.regex_options.has_flag_set(AllFlags::Insensitive); - if (auto& starting_ranges = m_pattern->parser_result.optimization_data.starting_ranges; !starting_ranges.is_empty()) { - auto ranges = insensitive ? m_pattern->parser_result.optimization_data.starting_ranges_insensitive.span() : starting_ranges.span(); - auto code_unit_index = input.view.unicode() ? input.view.code_unit_offset_of(view_index) : view_index; - auto ch = input.view.unicode_aware_code_point_at(code_unit_index); - if (insensitive) - ch = to_ascii_lowercase(ch); - - if (!binary_search(ranges, ch, nullptr, compare_range)) - goto done_matching; - } - - input.column = match_count; - input.match_index = match_count; - - state.string_position = view_index; - if (input.view.unicode()) { - if (view_index < view_length) - state.string_position_in_code_units = input.view.code_unit_offset_of(view_index); - else - state.string_position_in_code_units = input.view.length_in_code_units(); - } else { - state.string_position_in_code_units = view_index; - } - state.instruction_position = 0; - state.repetition_marks.clear(); - state.modifier_stack.clear(); - state.current_options = input.regex_options; - state.string_position_before_rseek = NumericLimits::max(); - state.string_position_in_code_units_before_rseek = NumericLimits::max(); - - if (auto const result = execute(input, state, operations); result == ExecuteResult::Matched) { - succeeded = true; - - if (input.regex_options.has_flag_set(AllFlags::MatchNotEndOfLine) && state.string_position == input.view.length()) { - if (!continue_search) - break; - continue; - } - if (input.regex_options.has_flag_set(AllFlags::MatchNotBeginOfLine) && view_index == 0) { - if (!continue_search) - break; - continue; - } - - dbgln_if(REGEX_DEBUG, "state.string_position={}, view_index={}", state.string_position, view_index); - dbgln_if(REGEX_DEBUG, "[match] Found a match (length={}): '{}'", state.string_position - view_index, input.view.substring_view(view_index, state.string_position - view_index)); - - ++match_count; - - if (continue_search) { - append_match(input, state, view_index); - - bool has_zero_length = state.string_position == view_index; - view_index = state.string_position - (has_zero_length ? 0 : 1); - if (single_match_only) - break; - continue; - } - if (input.regex_options.has_flag_set(AllFlags::Internal_Stateful)) { - append_match(input, state, view_index); - break; - } - if (state.string_position < view_length) { - return { false, 0, {}, {}, {}, operations }; - } - - append_match(input, state, view_index); - break; - } else if (result == ExecuteResult::DidNotMatchAndNoFurtherPossibleMatchesInView) { - break; - } - - done_matching: - if (!continue_search || only_start_of_line) - break; - } - - ++input.line; - input.global_offset += view.length() + 1; // +1 includes the line break character - - if (input.regex_options.has_flag_set(AllFlags::Internal_Stateful)) - m_pattern->start_offset = state.string_position; - - if (succeeded && !continue_search) - break; - } - - auto flat_capture_group_matches = move(state.flat_capture_group_matches).release(); - if (flat_capture_group_matches.size() < state.capture_group_count * match_count) { - flat_capture_group_matches.ensure_capacity(match_count * state.capture_group_count); - for (size_t i = flat_capture_group_matches.size(); i < match_count * state.capture_group_count; ++i) - flat_capture_group_matches.unchecked_empend(); - } - - Vector> capture_group_matches; - for (size_t i = 0; i < match_count; ++i) { - auto span = flat_capture_group_matches.span().slice(state.capture_group_count * i, state.capture_group_count); - capture_group_matches.append(span); - } - - RegexResult result { - match_count != 0, - match_count, - move(state.matches).release(), - move(flat_capture_group_matches), - move(capture_group_matches), - operations, - m_pattern->parser_result.capture_groups_count, - m_pattern->parser_result.named_capture_groups_count, - }; - - if (match_count > 0) - VERIFY(result.capture_group_matches.size() >= match_count); - else - result.capture_group_matches.clear_with_capacity(); - - return result; -} - -template -class BumpAllocatedLinkedList { -public: - BumpAllocatedLinkedList() = default; - - ALWAYS_INLINE void append(T value) - { - auto node_ptr = m_allocator.allocate(move(value)); - VERIFY(node_ptr); - - if (!m_first) { - m_first = node_ptr; - m_last = node_ptr; - return; - } - - node_ptr->previous = m_last; - m_last->next = node_ptr; - m_last = node_ptr; - } - - ALWAYS_INLINE T take_last() - { - VERIFY(m_last); - T value = move(m_last->value); - if (m_last == m_first) { - m_last = nullptr; - m_first = nullptr; - } else { - m_last = m_last->previous; - m_last->next = nullptr; - } - return value; - } - - ALWAYS_INLINE T& last() - { - return m_last->value; - } - - ALWAYS_INLINE bool is_empty() const - { - return m_first == nullptr; - } - - auto reverse_begin() { return ReverseIterator(m_last); } - auto reverse_end() { return ReverseIterator(); } - -private: - struct Node { - T value; - Node* next { nullptr }; - Node* previous { nullptr }; - }; - - struct ReverseIterator { - ReverseIterator() = default; - explicit ReverseIterator(Node* node) - : m_node(node) - { - } - - T* operator->() { return &m_node->value; } - T& operator*() { return m_node->value; } - bool operator==(ReverseIterator const& it) const { return m_node == it.m_node; } - ReverseIterator& operator++() - { - if (m_node) - m_node = m_node->previous; - return *this; - } - - private: - Node* m_node; - }; - - UniformBumpAllocator m_allocator; - Node* m_first { nullptr }; - Node* m_last { nullptr }; -}; - -template -Matcher::ExecuteResult Matcher::execute(MatchInput const& input, MatchState& state, size_t& operations) const -{ - if (m_pattern->parser_result.optimization_data.pure_substring_search.has_value() && input.view.is_u16_view()) { - // Yay, we can do a simple substring search! - auto is_insensitive = input.regex_options.has_flag_set(AllFlags::Insensitive); - auto is_unicode = input.view.unicode() || input.regex_options.has_flag_set(AllFlags::Unicode) || input.regex_options.has_flag_set(AllFlags::UnicodeSets); - // Utf16View::equals_ignoring_case can't handle unicode case folding, so we can only use it for ASCII case insensitivity. - if (!(is_insensitive && is_unicode)) { - auto input_view = input.view.u16_view(); - Span needle = m_pattern->parser_result.optimization_data.pure_substring_search->span(); - Utf16View needle_view { bit_cast(needle.data()), needle.size() }; - - if (is_unicode) { - if (needle_view.length_in_code_points() + state.string_position > input_view.length_in_code_points()) - return ExecuteResult::DidNotMatch; - } else { - if (needle_view.length_in_code_units() + state.string_position_in_code_units > input_view.length_in_code_units()) - return ExecuteResult::DidNotMatch; - } - - Utf16View haystack; - if (is_unicode) - haystack = input_view.unicode_substring_view(state.string_position, needle_view.length_in_code_points()); - else - haystack = input_view.substring_view(state.string_position_in_code_units, needle_view.length_in_code_units()); - - if (is_insensitive) { - if (!Unicode::ranges_equal_ignoring_case(haystack, needle_view, input.view.unicode())) - return ExecuteResult::DidNotMatch; - } else { - if (haystack != needle_view) - return ExecuteResult::DidNotMatch; - } - - if (input.view.unicode()) - state.string_position += haystack.length_in_code_points(); - else - state.string_position += haystack.length_in_code_units(); - state.string_position_in_code_units += haystack.length_in_code_units(); - return ExecuteResult::Matched; - } - } - - BumpAllocatedLinkedList states_to_try_next; - HashTable> seen_state_hashes; - - auto& bytecode = m_pattern->parser_result.bytecode.template get(); - auto const* data = bytecode.flat_data().data(); - auto const data_size = bytecode.size(); - - for (;;) { - auto const ip = state.instruction_position; - OpCodeId id = (data_size <= ip) - ? OpCodeId::Exit - : static_cast(data[ip]); - ++operations; - - ExecutionResult result; - size_t current_opcode_size; - if (input.fail_counter > 0) { - --input.fail_counter; - result = ExecutionResult::Failed_ExecuteLowPrioForks; - current_opcode_size = opcode_size(id, data, ip); - } else { - auto insn_result = execute_instruction(id, data, data_size, bytecode, input, state); - result = insn_result.result; - current_opcode_size = insn_result.size; - } - - state.instruction_position += current_opcode_size; - - switch (result) { - case ExecutionResult::Fork_PrioLow: { - bool found = false; - if (input.fork_to_replace.has_value()) { - for (auto it = states_to_try_next.reverse_begin(); it != states_to_try_next.reverse_end(); ++it) { - if (it->initiating_fork == input.fork_to_replace.value()) { - (*it) = state; - it->instruction_position = state.fork_at_position; - it->initiating_fork = *input.fork_to_replace; - found = true; - break; - } - } - input.fork_to_replace.clear(); - } - if (!found) { - states_to_try_next.append(state); - states_to_try_next.last().initiating_fork = state.instruction_position - current_opcode_size; - states_to_try_next.last().instruction_position = state.fork_at_position; - } - state.string_position_before_rseek = NumericLimits::max(); - state.string_position_in_code_units_before_rseek = NumericLimits::max(); - continue; - } - case ExecutionResult::Fork_PrioHigh: { - bool found = false; - if (input.fork_to_replace.has_value()) { - for (auto it = states_to_try_next.reverse_begin(); it != states_to_try_next.reverse_end(); ++it) { - if (it->initiating_fork == input.fork_to_replace.value()) { - (*it) = state; - it->initiating_fork = *input.fork_to_replace; - found = true; - break; - } - } - input.fork_to_replace.clear(); - } - if (!found) { - states_to_try_next.append(state); - states_to_try_next.last().initiating_fork = state.instruction_position - current_opcode_size; - states_to_try_next.last().string_position_before_rseek = NumericLimits::max(); - states_to_try_next.last().string_position_in_code_units_before_rseek = NumericLimits::max(); - } - state.instruction_position = state.fork_at_position; - continue; - } - case ExecutionResult::Continue: - continue; - case ExecutionResult::Succeeded: - return ExecuteResult::Matched; - case ExecutionResult::Failed: { - bool found = false; - while (!states_to_try_next.is_empty()) { - state = states_to_try_next.take_last(); - if (auto hash = state.u64_hash(); seen_state_hashes.set(hash) != HashSetResult::InsertedNewEntry) { - dbgln_if(REGEX_DEBUG, "Already seen state, skipping: {}", hash); - continue; - } - found = true; - break; - } - if (found) - continue; - return ExecuteResult::DidNotMatch; - } - case ExecutionResult::Failed_ExecuteLowPrioForks: { - bool found = false; - while (!states_to_try_next.is_empty()) { - state = states_to_try_next.take_last(); - if (auto hash = state.u64_hash(); seen_state_hashes.set(hash) != HashSetResult::InsertedNewEntry) { - dbgln_if(REGEX_DEBUG, "Already seen state, skipping: {}", hash); - continue; - } - found = true; - break; - } - if (!found) - return ExecuteResult::DidNotMatch; - continue; - } - case ExecutionResult::Failed_ExecuteLowPrioForksButNoFurtherPossibleMatches: { - bool found = false; - while (!states_to_try_next.is_empty()) { - state = states_to_try_next.take_last(); - if (auto hash = state.u64_hash(); seen_state_hashes.set(hash) != HashSetResult::InsertedNewEntry) { - dbgln_if(REGEX_DEBUG, "Already seen state, skipping: {}", hash); - continue; - } - found = true; - break; - } - if (!found) - return ExecuteResult::DidNotMatchAndNoFurtherPossibleMatchesInView; - continue; - } - } - } - - VERIFY_NOT_REACHED(); -} - -template class Matcher; -template class Regex; - -template class Matcher; -template class Regex; - -template class Matcher; -template class Regex; - -} - -template -struct AK::Traits> : public AK::DefaultTraits> { - static unsigned hash(regex::CacheKey const& key) - { - return pair_int_hash(key.pattern.hash(), to_underlying(key.options.value())); - } -}; diff --git a/Libraries/LibRegex/RegexMatcher.h b/Libraries/LibRegex/RegexMatcher.h deleted file mode 100644 index 54cbf57389..0000000000 --- a/Libraries/LibRegex/RegexMatcher.h +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include "RegexByteCode.h" -#include "RegexMatch.h" -#include "RegexOptions.h" -#include "RegexParser.h" - -#include -#include -#include -#include - -#include - -namespace regex { - -namespace Detail { - -struct Block { - size_t start; - size_t end; - StringView comment { "N/A"sv }; -}; - -} - -static constexpr size_t const c_max_recursion = 5000; - -struct REGEX_API RegexResult final { - bool success { false }; - size_t count { 0 }; - Vector matches; - Vector flat_capture_group_matches; - Vector> capture_group_matches; - size_t n_operations { 0 }; - size_t n_capture_groups { 0 }; - size_t n_named_capture_groups { 0 }; -}; - -template -class REGEX_API Regex; - -template -class REGEX_API Matcher final { - -public: - Matcher(Regex const* pattern, Optional::OptionsType> regex_options = {}) - : m_pattern(pattern) - , m_regex_options(regex_options.value_or({})) - { - } - ~Matcher() = default; - - RegexResult match(RegexStringView, Optional::OptionsType> = {}) const; - RegexResult match(Vector const&, Optional::OptionsType> = {}) const; - - typename ParserTraits::OptionsType options() const - { - return m_regex_options; - } - - void reset_pattern(Badge>, Regex const* pattern) - { - m_pattern = pattern; - } - -private: - enum class ExecuteResult { - DidNotMatch, - Matched, - DidNotMatchAndNoFurtherPossibleMatchesInView, - }; - ExecuteResult execute(MatchInput const& input, MatchState& state, size_t& operations) const; - - Regex const* m_pattern; - typename ParserTraits::OptionsType const m_regex_options; -}; - -template -class REGEX_API Regex final { -public: - ByteString pattern_value; - regex::Parser::Result parser_result; - OwnPtr> matcher { nullptr }; - mutable size_t start_offset { 0 }; - - static regex::Parser::Result parse_pattern(StringView pattern, typename ParserTraits::OptionsType regex_options = {}); - - explicit Regex(ByteString pattern, typename ParserTraits::OptionsType regex_options = {}); - Regex(regex::Parser::Result parse_result, ByteString pattern, typename ParserTraits::OptionsType regex_options = {}); - Regex(Regex const&); - ~Regex() = default; - Regex(Regex&&); - Regex& operator=(Regex&&); - - typename ParserTraits::OptionsType options() const; - ByteString error_string(Optional message = {}) const; - - RegexResult match(RegexStringView view, Optional::OptionsType> regex_options = {}) const - { - if (!matcher || parser_result.error != Error::NoError) - return {}; - return matcher->match(view, regex_options); - } - - RegexResult match(Vector const& views, Optional::OptionsType> regex_options = {}) const - { - if (!matcher || parser_result.error != Error::NoError) - return {}; - return matcher->match(views, regex_options); - } - - ByteString replace(RegexStringView view, StringView replacement_pattern, Optional::OptionsType> regex_options = {}) const - { - if (!matcher || parser_result.error != Error::NoError) - return {}; - - StringBuilder builder; - size_t start_offset = 0; - RegexResult result = matcher->match(view, regex_options); - if (!result.success) - return view.to_byte_string(); - - for (size_t i = 0; i < result.matches.size(); ++i) { - auto& match = result.matches[i]; - builder.append(view.substring_view(start_offset, match.global_offset - start_offset).to_byte_string()); - start_offset = match.global_offset + match.view.length(); - GenericLexer lexer(replacement_pattern); - while (!lexer.is_eof()) { - if (lexer.consume_specific('\\')) { - if (lexer.consume_specific('\\')) { - builder.append('\\'); - continue; - } - auto number = lexer.consume_while(isdigit); - if (auto index = number.to_number(); index.has_value() && result.n_capture_groups >= index.value()) { - builder.append(result.capture_group_matches[i][index.value() - 1].view.to_byte_string()); - } else { - builder.appendff("\\{}", number); - } - } else { - builder.append(lexer.consume_while([](auto ch) { return ch != '\\'; })); - } - } - } - - builder.append(view.substring_view(start_offset, view.length() - start_offset).to_byte_string()); - - return builder.to_byte_string(); - } - - // FIXME: replace(Vector const , ...) - - RegexResult search(RegexStringView view, Optional::OptionsType> regex_options = {}) const - { - if (!matcher || parser_result.error != Error::NoError) - return {}; - - AllOptions options = (AllOptions)regex_options.value_or({}); - if ((options & AllFlags::MatchNotBeginOfLine) && (options & AllFlags::MatchNotEndOfLine)) { - options.reset_flag(AllFlags::MatchNotEndOfLine); - options.reset_flag(AllFlags::MatchNotBeginOfLine); - } - options.reset_flag(AllFlags::Internal_Stateful); - options |= AllFlags::Global; - - return matcher->match(view, options); - } - - RegexResult search(Vector const& views, Optional::OptionsType> regex_options = {}) const - { - if (!matcher || parser_result.error != Error::NoError) - return {}; - - AllOptions options = (AllOptions)regex_options.value_or({}); - if ((options & AllFlags::MatchNotBeginOfLine) && (options & AllFlags::MatchNotEndOfLine)) { - options.reset_flag(AllFlags::MatchNotEndOfLine); - options.reset_flag(AllFlags::MatchNotBeginOfLine); - } - options.reset_flag(AllFlags::Internal_Stateful); - options |= AllFlags::Global; - - return matcher->match(views, options); - } - - bool match(RegexStringView view, RegexResult& m, Optional::OptionsType> regex_options = {}) const - { - m = match(view, regex_options); - return m.success; - } - - bool match(Vector const& views, RegexResult& m, Optional::OptionsType> regex_options = {}) const - { - m = match(views, regex_options); - return m.success; - } - - bool search(RegexStringView view, RegexResult& m, Optional::OptionsType> regex_options = {}) const - { - m = search(view, regex_options); - return m.success; - } - - bool search(Vector const& views, RegexResult& m, Optional::OptionsType> regex_options = {}) const - { - m = search(views, regex_options); - return m.success; - } - - bool has_match(RegexStringView view, Optional::OptionsType> regex_options = {}) const - { - if (!matcher || parser_result.error != Error::NoError) - return false; - RegexResult result = matcher->match(view, AllOptions { regex_options.value_or({}) } | AllFlags::SkipSubExprResults); - return result.success; - } - - bool has_match(Vector const& views, Optional::OptionsType> regex_options = {}) const - { - if (!matcher || parser_result.error != Error::NoError) - return false; - RegexResult result = matcher->match(views, AllOptions { regex_options.value_or({}) } | AllFlags::SkipSubExprResults); - return result.success; - } - - using BasicBlockList = Vector; - static BasicBlockList split_basic_blocks(ByteCode const&); - -private: - void run_optimization_passes(); - void rewrite_with_useless_jumps_removed(); - void attempt_rewrite_loops_as_atomic_groups(BasicBlockList const&); - bool attempt_rewrite_entire_match_as_substring_search(BasicBlockList const&); - void attempt_rewrite_adjacent_compares_as_string_compare(BasicBlockList const&); - void attempt_rewrite_dot_star_sequences_as_seek(BasicBlockList const&); - void rewrite_simple_compares(BasicBlockList const&); - void fill_optimization_data(BasicBlockList const&); -}; - -// free standing functions for match, search and has_match -template -RegexResult match(RegexStringView view, Regex& pattern, Optional::OptionsType> regex_options = {}) -{ - return pattern.match(view, regex_options); -} - -template -RegexResult match(Vector const& view, Regex& pattern, Optional::OptionsType> regex_options = {}) -{ - return pattern.match(view, regex_options); -} - -template -bool match(RegexStringView view, Regex& pattern, RegexResult&, Optional::OptionsType> regex_options = {}) -{ - return pattern.match(view, regex_options); -} - -template -bool match(Vector const& view, Regex& pattern, RegexResult&, Optional::OptionsType> regex_options = {}) -{ - return pattern.match(view, regex_options); -} - -template -RegexResult search(RegexStringView view, Regex& pattern, Optional::OptionsType> regex_options = {}) -{ - return pattern.search(view, regex_options); -} - -template -RegexResult search(Vector const& views, Regex& pattern, Optional::OptionsType> regex_options = {}) -{ - return pattern.search(views, regex_options); -} - -template -bool search(RegexStringView view, Regex& pattern, RegexResult&, Optional::OptionsType> regex_options = {}) -{ - return pattern.search(view, regex_options); -} - -template -bool search(Vector const& views, Regex& pattern, RegexResult&, Optional::OptionsType> regex_options = {}) -{ - return pattern.search(views, regex_options); -} - -template -bool has_match(RegexStringView view, Regex& pattern, Optional::OptionsType> regex_options = {}) -{ - return pattern.has_match(view, regex_options); -} - -template -bool has_match(Vector const& views, Regex& pattern, Optional::OptionsType> regex_options = {}) -{ - return pattern.has_match(views, regex_options); -} - -} - -using regex::has_match; -using regex::match; -using regex::Regex; -using regex::RegexResult; diff --git a/Libraries/LibRegex/RegexOptimizer.cpp b/Libraries/LibRegex/RegexOptimizer.cpp deleted file mode 100644 index b971758240..0000000000 --- a/Libraries/LibRegex/RegexOptimizer.cpp +++ /dev/null @@ -1,2788 +0,0 @@ -/* - * Copyright (c) 2021, Ali Mohammad Pur - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if REGEX_DEBUG -# include -# include -#endif - -namespace regex { - -using Detail::Block; - -struct BytecodeRewriter { - struct Instruction { - size_t old_ip; - size_t size; - OpCodeId id; - bool skip; - }; - Vector instructions; - HashMap new_ip_mapping; - StringView target_pattern; - - BytecodeRewriter(ByteCode const& bytecode, StringView pattern = {}) - : target_pattern(pattern) - { - auto flat = bytecode.flat_data(); - - for (size_t old_ip = 0; old_ip < flat.size();) { - auto id = static_cast(flat[old_ip]); - auto sz = opcode_size(id, flat.data(), old_ip); - - instructions.append({ old_ip, sz, id, false }); - old_ip += sz; - } - } - - void mark_range_for_skip(size_t start, size_t end) - { - for (auto& instr : instructions) { - if (instr.old_ip >= start && instr.old_ip < end) - instr.skip = true; - } - } - - void build_ip_mapping(ByteCode const& bytecode, Span replacements) - { - new_ip_mapping.ensure_capacity(instructions.size() + 1); - size_t current_new_ip = 0; - auto replacements_it = replacements.begin(); - - for (auto& instr : instructions) { - new_ip_mapping.set(instr.old_ip, current_new_ip); - auto& replacement = *replacements_it; - ++replacements_it; - - if (!instr.skip) - current_new_ip += instr.size; - else - current_new_ip += replacement.size(); - } - - new_ip_mapping.set(bytecode.size(), current_new_ip); - } - - template - requires(requires(Range r) { r.start_ip; r.end_ip; }) - void build_ip_mapping(ByteCode const& bytecode, Span replacement_ranges, Span replacements) - { - new_ip_mapping.ensure_capacity(instructions.size() + 1); - size_t current_new_ip = 0; - auto instruction_it = instructions.begin(); - - for (auto i = 0uz; i < replacements.size(); ++i) { - auto& range = replacement_ranges[i]; - auto& replacement = replacements[i]; - - while (instruction_it != instructions.end()) { - auto& instr = *instruction_it; - if (instr.old_ip >= range.start_ip) { - ASSERT(instr.old_ip < range.end_ip); - new_ip_mapping.set(instr.old_ip, current_new_ip); - break; - } - new_ip_mapping.set(instr.old_ip, current_new_ip); - current_new_ip += instr.size; - ++instruction_it; - } - current_new_ip += replacement.size(); - - // Skip instructions in the replacement range - while (instruction_it != instructions.end()) { - auto& instr = *instruction_it; - if (instr.old_ip >= range.end_ip) - break; - ++instruction_it; - } - } - // Map any remaining instructions - for (; instruction_it != instructions.end(); ++instruction_it) { - auto& instr = *instruction_it; - new_ip_mapping.set(instr.old_ip, current_new_ip); - current_new_ip += instr.size; - } - new_ip_mapping.set(bytecode.size(), current_new_ip); - } - - template - requires(requires(Range r) { r.start_ip; r.end_ip; }) - ByteCode rebuild(ByteCode const& bytecode, Span replacement_ranges, Span replacements) - { - // Assumes that replacement_ranges and replacements are the same size - // As well as in order - VERIFY(replacement_ranges.size() == replacements.size()); - auto flat = bytecode.flat_data(); - ByteCode result; - result.merge_string_tables_from({ &bytecode, 1 }); - - size_t total_new_size = bytecode.size(); - // FIXME: Get a zip(it...) helper - for (auto i = 0uz; i < replacement_ranges.size(); ++i) { - mark_range_for_skip(replacement_ranges[i].start_ip, replacement_ranges[i].end_ip); - total_new_size -= (replacement_ranges[i].end_ip - replacement_ranges[i].start_ip); - total_new_size += replacements[i].size(); - } - build_ip_mapping(bytecode, replacement_ranges, replacements); - result.ensure_capacity(total_new_size); - - // FIXME: Use a zip(...) helper - auto instructions_it = instructions.begin(); - for (auto i = 0uz; i < replacement_ranges.size(); ++i) { - auto& range = replacement_ranges[i]; - auto& replacement = replacements[i]; - - // Append and adjust all instructions before the replacement range - for (; instructions_it != instructions.end(); ++instructions_it) { - auto& instr = *instructions_it; - if (instr.old_ip >= range.start_ip) { - ASSERT(instr.old_ip < range.end_ip); - ++instructions_it; - break; - } - VERIFY(instr.skip == false); - auto slice = Vector { flat.slice(instr.old_ip, instr.size) }; - adjust_jump_in_slice(bytecode, slice, instr); - result.append(move(slice)); - } - // Finally insert the replacement - result.extend(replacement); - - // Skip instructions in the replacement range - while (instructions_it != instructions.end()) { - auto& instr = *instructions_it; - if (instr.old_ip >= range.end_ip) - break; - ++instructions_it; - } - } - // Append any remaining instructions - for (; instructions_it != instructions.end(); ++instructions_it) { - auto& instr = *instructions_it; - auto slice = Vector { flat.slice(instr.old_ip, instr.size) }; - adjust_jump_in_slice(bytecode, slice, instr); - result.append(move(slice)); - VERIFY(instr.skip == false); - } - - result.flatten(); - return result; - } - - ByteCode rebuild(ByteCode const& bytecode, Function insert_replacement = nullptr) - { - auto flat = bytecode.flat_data(); - ByteCode result; - result.merge_string_tables_from({ &bytecode, 1 }); - - Vector replacements; - replacements.resize_with_default_value(instructions.size(), ByteCode {}); - - size_t total_new_size = 0; - for (auto const& [i, instr] : enumerate(instructions)) { - if (!instr.skip) { - total_new_size += instr.size; - } else if (insert_replacement) { - ByteCode temp; - insert_replacement(instr, temp); - total_new_size += temp.size(); - replacements[i] = move(temp); - } - } - build_ip_mapping(bytecode, replacements); - result.ensure_capacity(total_new_size); - - auto replacements_it = replacements.begin(); - for (auto& instr : instructions) { - auto& replacement = *replacements_it; - ++replacements_it; - - if (instr.skip) { - result.extend(move(replacement)); - continue; - } - - auto slice = Vector { flat.slice(instr.old_ip, instr.size) }; - adjust_jump_in_slice(bytecode, slice, instr); - result.append(move(slice)); - } - - result.flatten(); - return result; - } - -private: - void adjust_jump_in_slice(ByteCode const& bytecode, Vector& slice, Instruction const& instr) - { - auto adjust = [&](size_t idx, bool is_repeat) { - auto old_offset = slice[idx]; - auto target_old = is_repeat - ? instr.old_ip - old_offset - : instr.old_ip + instr.size + old_offset; - - if (!new_ip_mapping.contains(target_old)) { - dbgln("In pattern /{}/", target_pattern); - dbgln("Target {} not found in new_ip mapping (in {})", target_old, instr.old_ip); - RegexDebug dbg(stderr); - dbg.print_bytecode(bytecode); - VERIFY_NOT_REACHED(); - } - - size_t target_new = *new_ip_mapping.get(target_old); - size_t source_new = *new_ip_mapping.get(instr.old_ip); - auto new_offset = is_repeat - ? source_new - target_new - : target_new - source_new - instr.size; - - slice[idx] = static_cast(new_offset); - }; - - switch (instr.id) { - case OpCodeId::Jump: - case OpCodeId::ForkJump: - case OpCodeId::ForkStay: - case OpCodeId::ForkReplaceJump: - case OpCodeId::ForkReplaceStay: - case OpCodeId::JumpNonEmpty: - case OpCodeId::ForkIf: - adjust(1, false); - break; - case OpCodeId::Repeat: - adjust(1, true); - break; - default: - break; - } - } -}; - -template -static typename Regex::BasicBlockList split_basic_blocks_for_atomic_groups(ByteCode const& bytecode) -{ - typename Regex::BasicBlockList block_boundaries; - size_t end_of_last_block = 0; - - auto bytecode_size = bytecode.size(); - - auto flat = bytecode.flat_data(); - auto const* flat_data = flat.data(); - - auto check_jump = [&](size_t ip, size_t sz, ssize_t offset) { - ssize_t jump_offset = static_cast(sz) + offset; - if (jump_offset >= 0) { - block_boundaries.append({ end_of_last_block, ip, "Jump ahead"sv }); - end_of_last_block = ip + sz; - } else { - if (static_cast(ip) + jump_offset > static_cast(end_of_last_block)) { - block_boundaries.append({ end_of_last_block, static_cast(static_cast(ip) + jump_offset), "Jump back 1"sv }); - block_boundaries.append({ static_cast(static_cast(ip) + jump_offset), ip, "Jump back 2"sv }); - end_of_last_block = ip + sz; - } else { - block_boundaries.append({ end_of_last_block, ip, "Jump"sv }); - end_of_last_block = ip + sz; - } - } - }; - for (size_t ip = 0; ip < bytecode_size;) { - auto id = static_cast(flat_data[ip]); - auto sz = opcode_size(id, flat_data, ip); - - switch (id) { - case OpCodeId::Jump: - case OpCodeId::JumpNonEmpty: - case OpCodeId::ForkJump: - case OpCodeId::ForkStay: - case OpCodeId::ForkIf: - check_jump(ip, sz, static_cast(flat_data[ip + OpArgs::Jump::offset])); - break; - case OpCodeId::FailForks: - block_boundaries.append({ end_of_last_block, ip, "FailForks"sv }); - end_of_last_block = ip + sz; - break; - case OpCodeId::Repeat: { - auto repeat_offset = flat_data[ip + OpArgs::Repeat::offset]; - auto repeat_start = ip - repeat_offset; - if (repeat_start > end_of_last_block) - block_boundaries.append({ end_of_last_block, repeat_start, "Repeat"sv }); - block_boundaries.append({ repeat_start, ip, "Repeat after"sv }); - end_of_last_block = ip + sz; - break; - } - default: - break; - } - - ip += sz; - } - - if (end_of_last_block < bytecode_size) - block_boundaries.append({ end_of_last_block, bytecode_size, "End"sv }); - - quick_sort(block_boundaries, [](auto& a, auto& b) { return a.start < b.start; }); - - return block_boundaries; -} - -template -void Regex::run_optimization_passes() -{ - ScopeGuard switch_to_flat = [&] { - parser_result.bytecode = FlatByteCode::from(move(parser_result.bytecode.template get())); - }; - rewrite_with_useless_jumps_removed(); - - auto blocks = split_basic_blocks(parser_result.bytecode.get()); - if (attempt_rewrite_entire_match_as_substring_search(blocks)) { - return; - } - - // Rewrite fork loops as atomic groups - // e.g. a*b -> (ATOMIC a*)b - blocks = split_basic_blocks_for_atomic_groups(parser_result.bytecode.get()); - attempt_rewrite_loops_as_atomic_groups(blocks); - - // Join adjacent compares that only match single characters into a single compare that matches a string. - blocks = split_basic_blocks(parser_result.bytecode.get()); - attempt_rewrite_adjacent_compares_as_string_compare(blocks); - - // Rewrite /.*x/ as a seek to x - blocks = split_basic_blocks(parser_result.bytecode.get()); - attempt_rewrite_dot_star_sequences_as_seek(blocks); - - // Simplify compares where possible - blocks = split_basic_blocks(parser_result.bytecode.get()); - rewrite_simple_compares(blocks); - - fill_optimization_data(split_basic_blocks(parser_result.bytecode.template get())); -} - -struct StaticallyInterpretedCompares { - RedBlackTree ranges; - RedBlackTree negated_ranges; - HashTable char_classes; - HashTable negated_char_classes; - - bool has_any_unicode_property = false; - HashTable unicode_general_categories; - HashTable unicode_properties; - HashTable unicode_scripts; - HashTable unicode_script_extensions; - HashTable negated_unicode_general_categories; - HashTable negated_unicode_properties; - HashTable negated_unicode_scripts; - HashTable negated_unicode_script_extensions; -}; - -static bool interpret_compares(Vector const& lhs, StaticallyInterpretedCompares& compares, ByteCodeBase const* bytecode = nullptr, bool as_follow = false) -{ - bool inverse { false }; - bool temporary_inverse { false }; - bool reset_temporary_inverse { false }; - - auto current_lhs_inversion_state = [&]() -> bool { return temporary_inverse ^ inverse; }; - - auto& lhs_ranges = compares.ranges; - auto& lhs_negated_ranges = compares.negated_ranges; - auto& lhs_char_classes = compares.char_classes; - auto& lhs_negated_char_classes = compares.negated_char_classes; - auto& has_any_unicode_property = compares.has_any_unicode_property; - auto& lhs_unicode_general_categories = compares.unicode_general_categories; - auto& lhs_unicode_properties = compares.unicode_properties; - auto& lhs_unicode_scripts = compares.unicode_scripts; - auto& lhs_unicode_script_extensions = compares.unicode_script_extensions; - auto& lhs_negated_unicode_general_categories = compares.negated_unicode_general_categories; - auto& lhs_negated_unicode_properties = compares.negated_unicode_properties; - auto& lhs_negated_unicode_scripts = compares.negated_unicode_scripts; - auto& lhs_negated_unicode_script_extensions = compares.negated_unicode_script_extensions; - - for (auto const& pair : lhs) { - if (reset_temporary_inverse) { - reset_temporary_inverse = false; - temporary_inverse = false; - } else { - reset_temporary_inverse = true; - } - - switch (pair.type) { - case CharacterCompareType::Inverse: - inverse = !inverse; - break; - case CharacterCompareType::TemporaryInverse: - temporary_inverse = true; - reset_temporary_inverse = false; - break; - case CharacterCompareType::AnyChar: - // Special case: if not inverted, AnyChar is always in the range. - if (!current_lhs_inversion_state()) - return false; - break; - case CharacterCompareType::Char: - if (!current_lhs_inversion_state()) - lhs_ranges.insert(pair.value, pair.value); - else - lhs_negated_ranges.insert(pair.value, pair.value); - break; - case CharacterCompareType::String: { - if (!as_follow) - return false; - auto string = bytecode->get_u16_string(pair.value); - u32 ch = string.code_point_at(0); - - if (!current_lhs_inversion_state()) - lhs_ranges.insert(ch, ch); - else - lhs_negated_ranges.insert(ch, ch); - break; - } - case CharacterCompareType::StringSet: - return false; - case CharacterCompareType::CharClass: - if (!current_lhs_inversion_state()) - lhs_char_classes.set(static_cast(pair.value)); - else - lhs_negated_char_classes.set(static_cast(pair.value)); - break; - case CharacterCompareType::CharRange: { - auto range = CharRange(pair.value); - if (!current_lhs_inversion_state()) - lhs_ranges.insert(range.from, range.to); - else - lhs_negated_ranges.insert(range.from, range.to); - break; - } - case CharacterCompareType::LookupTable: - // We've transformed this into a series of ranges in flat_compares(), so bail out if we see it. - return false; - case CharacterCompareType::Reference: - case CharacterCompareType::NamedReference: - // We've handled this before coming here. - break; - case CharacterCompareType::Property: - has_any_unicode_property = true; - if (!current_lhs_inversion_state()) - lhs_unicode_properties.set(static_cast(pair.value)); - else - lhs_negated_unicode_properties.set(static_cast(pair.value)); - break; - case CharacterCompareType::GeneralCategory: - has_any_unicode_property = true; - if (!current_lhs_inversion_state()) - lhs_unicode_general_categories.set(static_cast(pair.value)); - else - lhs_negated_unicode_general_categories.set(static_cast(pair.value)); - break; - case CharacterCompareType::Script: - has_any_unicode_property = true; - if (!current_lhs_inversion_state()) - lhs_unicode_scripts.set(static_cast(pair.value)); - else - lhs_negated_unicode_scripts.set(static_cast(pair.value)); - break; - case CharacterCompareType::ScriptExtension: - has_any_unicode_property = true; - if (!current_lhs_inversion_state()) - lhs_unicode_script_extensions.set(static_cast(pair.value)); - else - lhs_negated_unicode_script_extensions.set(static_cast(pair.value)); - break; - case CharacterCompareType::Or: - case CharacterCompareType::EndAndOr: - // These are the default behaviour for [...], so we don't need to do anything (unless we add support for 'And' below). - break; - case CharacterCompareType::And: - case CharacterCompareType::Subtract: - // FIXME: These are too difficult to handle, so bail out. - return false; - case CharacterCompareType::Undefined: - case CharacterCompareType::RangeExpressionDummy: - // These do not occur in valid bytecode. - VERIFY_NOT_REACHED(); - } - } - - return true; -} - -template -void Regex::fill_optimization_data(BasicBlockList const& blocks) -{ - if (blocks.is_empty()) - return; - - if constexpr (REGEX_DEBUG) { - dbgln("Pulling out optimization data from bytecode:"); - RegexDebug dbg; - dbg.print_bytecode(*this); - for (auto const& block : blocks) - dbgln("block from {} to {} (comment: {})", block.start, block.end, block.comment); - } - - ScopeGuard print = [&] { - if constexpr (REGEX_DEBUG) { - dbgln("Optimization data:"); - if (parser_result.optimization_data.starting_ranges.is_empty()) - dbgln("; - no starting ranges"); - for (auto const& range : parser_result.optimization_data.starting_ranges) - dbgln(" - starting range: {}-{}", range.from, range.to); - dbgln("; - only start of line: {}", parser_result.optimization_data.only_start_of_line); - } - }; - - auto& bytecode = parser_result.bytecode.get(); - auto flat = bytecode.flat_data(); - auto const* flat_data = flat.data(); - - auto block = blocks.first(); - for (size_t ip = block.start; ip < block.end;) { - auto id = static_cast(flat_data[ip]); - auto sz = opcode_size(id, flat_data, ip); - switch (id) { - case OpCodeId::Compare: { - if (flat_data[ip + OpArgs::Compare::arguments_count] == 0) - return; // This matches 'nothing', so there are no starting ranges that can satisfy it. - auto fc = flat_compares_at(flat_data, ip, false); - StaticallyInterpretedCompares compares; - if (!interpret_compares(fc, compares)) - return; // No idea, the bytecode is too complex. - - if (compares.has_any_unicode_property) - return; // Faster to just run the bytecode. - - // FIXME: We should be able to handle these cases (jump ahead while...) - if (!compares.char_classes.is_empty() || !compares.negated_char_classes.is_empty() || !compares.negated_ranges.is_empty()) - return; - - for (auto it = compares.ranges.begin(); it != compares.ranges.end(); ++it) { - parser_result.optimization_data.starting_ranges.append({ it.key(), *it }); - for (auto range : Unicode::expand_range_case_insensitive(it.key(), *it)) - parser_result.optimization_data.starting_ranges_insensitive.append({ range.from, range.to }); - } - quick_sort(parser_result.optimization_data.starting_ranges_insensitive, [](CharRange a, CharRange b) { return a.from < b.from; }); - return; - } - case OpCodeId::CompareSimple: { - auto fc = flat_compares_at(flat_data, ip, true); - StaticallyInterpretedCompares compares; - if (!interpret_compares(fc, compares)) - return; // No idea, the bytecode is too complex. - - if (compares.has_any_unicode_property) - return; // Faster to just run the bytecode. - - // FIXME: We should be able to handle these cases (jump ahead while...) - if (!compares.char_classes.is_empty() || !compares.negated_char_classes.is_empty() || !compares.negated_ranges.is_empty()) - return; - - for (auto it = compares.ranges.begin(); it != compares.ranges.end(); ++it) { - parser_result.optimization_data.starting_ranges.append({ it.key(), *it }); - for (auto range : Unicode::expand_range_case_insensitive(it.key(), *it)) - parser_result.optimization_data.starting_ranges_insensitive.append({ range.from, range.to }); - } - quick_sort(parser_result.optimization_data.starting_ranges_insensitive, [](CharRange a, CharRange b) { return a.from < b.from; }); - return; - } - case OpCodeId::CheckBegin: - parser_result.optimization_data.only_start_of_line = true; - return; - case OpCodeId::Checkpoint: - case OpCodeId::Save: - case OpCodeId::ClearCaptureGroup: - case OpCodeId::SaveLeftCaptureGroup: - // These do not 'match' anything, so look through them. - ip += sz; - continue; - default: - return; - } - } -} - -template -typename Regex::BasicBlockList Regex::split_basic_blocks(ByteCode const& bytecode) -{ - BasicBlockList block_boundaries; - HashTable block_starts; - - auto bytecode_size = bytecode.size(); - - block_starts.set(0); - - auto flat = bytecode.flat_data(); - auto const* flat_data = flat.data(); - - for (size_t ip = 0; ip < bytecode_size;) { - auto id = static_cast(flat_data[ip]); - auto sz = opcode_size(id, flat_data, ip); - - switch (id) { - case OpCodeId::Jump: - case OpCodeId::JumpNonEmpty: - case OpCodeId::ForkJump: - case OpCodeId::ForkStay: - case OpCodeId::ForkIf: { - auto offset = static_cast(flat_data[ip + OpArgs::Jump::offset]); - ssize_t target = static_cast(ip) + static_cast(sz) + offset; - block_starts.set(target); - block_starts.set(ip + sz); - break; - } - case OpCodeId::FailForks: - block_starts.set(ip + sz); - break; - case OpCodeId::Repeat: { - auto repeat_offset = flat_data[ip + OpArgs::Repeat::offset]; - auto repeat_start = ip - repeat_offset; - block_starts.set(repeat_start); - block_starts.set(ip + sz); - break; - } - default: - break; - } - - ip += sz; - } - - Vector sorted_starts; - for (auto start : block_starts) - sorted_starts.append(start); - quick_sort(sorted_starts); - - for (size_t i = 0; i < sorted_starts.size(); ++i) { - size_t start = sorted_starts[i]; - size_t end; - - if (i + 1 < sorted_starts.size()) { - size_t next_block_start = sorted_starts[i + 1]; - - size_t cur_ip = start; - size_t last_ip = start; - while (cur_ip < next_block_start) { - last_ip = cur_ip; - cur_ip += opcode_size(static_cast(flat_data[cur_ip]), flat_data, cur_ip); - } - end = last_ip; - } else { - size_t cur_ip = start; - size_t last_ip = start; - while (cur_ip < bytecode_size) { - last_ip = cur_ip; - auto next_ip = cur_ip + opcode_size(static_cast(flat_data[cur_ip]), flat_data, cur_ip); - if (next_ip >= bytecode_size) - break; - cur_ip = next_ip; - } - end = last_ip; - } - - block_boundaries.append({ start, end, "Block"sv }); - } - - return block_boundaries; -} - -static bool has_overlap(Vector const& lhs, Vector const& rhs, bool insensitive = false, bool unicode_mode = false) -{ - // We have to fully interpret the two sequences to determine if they overlap (that is, keep track of inversion state and what ranges they cover). - bool inverse { false }; - bool temporary_inverse { false }; - bool reset_temporary_inverse { false }; - - auto current_lhs_inversion_state = [&]() -> bool { return temporary_inverse ^ inverse; }; - - StaticallyInterpretedCompares compares; - auto& lhs_ranges = compares.ranges; - auto& lhs_negated_ranges = compares.negated_ranges; - auto& lhs_char_classes = compares.char_classes; - auto& lhs_negated_char_classes = compares.negated_char_classes; - auto& has_any_unicode_property = compares.has_any_unicode_property; - auto& lhs_unicode_general_categories = compares.unicode_general_categories; - auto& lhs_unicode_properties = compares.unicode_properties; - auto& lhs_unicode_scripts = compares.unicode_scripts; - auto& lhs_unicode_script_extensions = compares.unicode_script_extensions; - auto& lhs_negated_unicode_general_categories = compares.negated_unicode_general_categories; - auto& lhs_negated_unicode_properties = compares.negated_unicode_properties; - auto& lhs_negated_unicode_scripts = compares.negated_unicode_scripts; - auto& lhs_negated_unicode_script_extensions = compares.negated_unicode_script_extensions; - - auto any_unicode_property_matches = [&](u32 code_point) { - if (any_of(lhs_negated_unicode_general_categories, [code_point](auto category) { return Unicode::code_point_has_general_category(code_point, category); })) - return false; - if (any_of(lhs_negated_unicode_properties, [code_point](auto property) { return Unicode::code_point_has_property(code_point, property); })) - return false; - if (any_of(lhs_negated_unicode_scripts, [code_point](auto script) { return Unicode::code_point_has_script(code_point, script); })) - return false; - if (any_of(lhs_negated_unicode_script_extensions, [code_point](auto script) { return Unicode::code_point_has_script_extension(code_point, script); })) - return false; - - if (any_of(lhs_unicode_general_categories, [code_point](auto category) { return Unicode::code_point_has_general_category(code_point, category); })) - return true; - if (any_of(lhs_unicode_properties, [code_point](auto property) { return Unicode::code_point_has_property(code_point, property); })) - return true; - if (any_of(lhs_unicode_scripts, [code_point](auto script) { return Unicode::code_point_has_script(code_point, script); })) - return true; - if (any_of(lhs_unicode_script_extensions, [code_point](auto script) { return Unicode::code_point_has_script_extension(code_point, script); })) - return true; - return false; - }; - - auto range_contains = [&](T& value) -> bool { - u32 start; - u32 end; - - if constexpr (IsSame) { - start = value.from; - end = value.to; - } else { - start = value; - end = value; - } - - if (has_any_unicode_property) { - // We have some properties, and a range is present - // Instead of checking every single code point in the range, assume it's a match. - return start != end || any_unicode_property_matches(start); - } - - for (auto it = lhs_ranges.begin(); it != lhs_ranges.end(); ++it) { - auto lhs_start = it.key(); - auto lhs_end = *it; - if (lhs_start <= end && start <= lhs_end) - return true; - } - - if (insensitive) { - auto expanded_ranges = Unicode::expand_range_case_insensitive(start, end); - for (auto const& expanded : expanded_ranges) { - for (auto it = lhs_ranges.begin(); it != lhs_ranges.end(); ++it) { - auto lhs_start = it.key(); - auto lhs_end = *it; - if (lhs_start <= expanded.to && expanded.from <= lhs_end) - return true; - } - } - } - - return false; - }; - - auto char_class_contains = [&](CharClass const& value) -> bool { - if (lhs_char_classes.contains(value)) - return true; - - if (lhs_negated_char_classes.contains(value)) - return false; - - for (auto const& lhs_class : lhs_char_classes) { - for (u32 ch = 0; ch < 128; ++ch) { - if (matches_character_class(value, ch, insensitive, unicode_mode) && matches_character_class(lhs_class, ch, insensitive, unicode_mode)) - return true; - } - } - - if (lhs_ranges.is_empty()) - return false; - - for (auto it = lhs_ranges.begin(); it != lhs_ranges.end(); ++it) { - auto start = it.key(); - auto end = *it; - for (u32 ch = start; ch <= end; ++ch) { - if (matches_character_class(value, ch, insensitive, unicode_mode)) - return true; - } - } - - return false; - }; - - if (!interpret_compares(lhs, compares)) - return true; // We can't interpret this, so we can't optimize it. - - if constexpr (REGEX_DEBUG) { - dbgln("lhs ranges:"); - for (auto it = lhs_ranges.begin(); it != lhs_ranges.end(); ++it) - dbgln(" {}..{}", it.key(), *it); - dbgln("lhs negated ranges:"); - for (auto it = lhs_negated_ranges.begin(); it != lhs_negated_ranges.end(); ++it) - dbgln(" {}..{}", it.key(), *it); - } - - temporary_inverse = false; - reset_temporary_inverse = false; - inverse = false; - struct DisjunctionState { - bool in_or = false; // We're in an OR block, so we should wait for the EndAndOr to decide if we would match. - bool matched_in_or = false; - bool inverse_matched_in_or = false; - }; - - Vector disjunction_stack; - disjunction_stack.empend(); - - auto in_or = [&] -> bool& { return disjunction_stack.last().in_or; }; - auto matched_in_or = [&] -> bool& { return disjunction_stack.last().matched_in_or; }; - auto inverse_matched_in_or = [&] -> bool& { return disjunction_stack.last().inverse_matched_in_or; }; - - for (auto const& pair : rhs) { - if (reset_temporary_inverse) { - reset_temporary_inverse = false; - temporary_inverse = false; - } else { - reset_temporary_inverse = true; - } - - if constexpr (REGEX_DEBUG) { - dbgln("check {} ({}) [inverted? {}] against {{", character_compare_type_name(pair.type), pair.value, current_lhs_inversion_state()); - for (auto it = lhs_ranges.begin(); it != lhs_ranges.end(); ++it) - dbgln(" {}..{}", it.key(), *it); - for (auto it = lhs_negated_ranges.begin(); it != lhs_negated_ranges.end(); ++it) - dbgln(" ^[{}..{}]", it.key(), *it); - for (auto& char_class : lhs_char_classes) - dbgln(" {}", character_class_name(char_class)); - for (auto& char_class : lhs_negated_char_classes) - dbgln(" ^{}", character_class_name(char_class)); - dbgln("}}, in or: {}, matched in or: {}, inverse matched in or: {}", in_or(), matched_in_or(), inverse_matched_in_or()); - } - - switch (pair.type) { - case CharacterCompareType::Inverse: - inverse = !inverse; - break; - case CharacterCompareType::TemporaryInverse: - temporary_inverse = true; - reset_temporary_inverse = false; - break; - case CharacterCompareType::AnyChar: - // Special case: if not inverted, AnyChar is always in the range. - if (!in_or() && !current_lhs_inversion_state()) - return true; - if (in_or()) { - matched_in_or() = true; - inverse_matched_in_or() = false; - } - break; - case CharacterCompareType::Char: { - auto matched = range_contains(pair.value); - if (!matched) { - for (auto const& char_class : lhs_char_classes) { - if (matches_character_class(char_class, pair.value, insensitive, unicode_mode)) { - matched = true; - break; - } - } - } - if (!in_or() && (current_lhs_inversion_state() ^ matched)) - return true; - if (in_or()) { - matched_in_or() |= matched; - inverse_matched_in_or() |= !matched; - } - break; - } - case CharacterCompareType::String: - // FIXME: We just need to look at the last character of this string, but we only have the first character here. - // Just bail out to avoid false positives. - return true; - case CharacterCompareType::StringSet: - return true; - case CharacterCompareType::CharClass: { - auto contains = char_class_contains(static_cast(pair.value)); - if (!in_or() && (current_lhs_inversion_state() ^ contains)) - return true; - if (in_or()) { - matched_in_or() |= contains; - inverse_matched_in_or() |= !contains; - } - break; - } - case CharacterCompareType::CharRange: { - auto range = CharRange(pair.value); - auto contains = range_contains(range); - if (!contains) { - for (auto const& char_class : lhs_char_classes) { - for (u32 ch = range.from; ch <= range.to; ++ch) { - if (matches_character_class(char_class, ch, insensitive, unicode_mode)) { - contains = true; - break; - } - } - if (contains) - break; - } - } - if (!in_or() && (contains ^ current_lhs_inversion_state())) - return true; - - if (in_or()) { - matched_in_or() |= contains; - inverse_matched_in_or() |= !contains; - } - - break; - } - case CharacterCompareType::LookupTable: - // We've transformed this into a series of ranges in flat_compares(), so bail out if we see it. - return true; - case CharacterCompareType::Reference: - case CharacterCompareType::NamedReference: - // We've handled this before coming here. - break; - case CharacterCompareType::Property: - // The only reasonable scenario where we can check these properties without spending too much time is if: - // - the ranges are empty - // - the char classes are empty - // - the unicode properties are empty or contain only this property - if (!lhs_ranges.is_empty() || !lhs_negated_ranges.is_empty() || !lhs_char_classes.is_empty() || !lhs_negated_char_classes.is_empty()) - return true; - if (has_any_unicode_property && !lhs_unicode_properties.is_empty() && !lhs_negated_unicode_properties.is_empty()) { - auto contains = lhs_unicode_properties.contains(static_cast(pair.value)); - if (!in_or() && (current_lhs_inversion_state() ^ contains)) - return true; - - auto inverse_contains = lhs_negated_unicode_properties.contains(static_cast(pair.value)); - if (!in_or() && !(current_lhs_inversion_state() ^ inverse_contains)) - return true; - - if (in_or()) { - matched_in_or() |= contains; - inverse_matched_in_or() |= inverse_contains; - } - } - break; - case CharacterCompareType::GeneralCategory: - if (!lhs_ranges.is_empty() || !lhs_negated_ranges.is_empty() || !lhs_char_classes.is_empty() || !lhs_negated_char_classes.is_empty()) - return true; - if (has_any_unicode_property && !lhs_unicode_general_categories.is_empty() && !lhs_negated_unicode_general_categories.is_empty()) { - auto contains = lhs_unicode_general_categories.contains(static_cast(pair.value)); - if (!in_or() && (current_lhs_inversion_state() ^ contains)) - return true; - auto inverse_contains = lhs_negated_unicode_general_categories.contains(static_cast(pair.value)); - if (!in_or() && !(current_lhs_inversion_state() ^ inverse_contains)) - return true; - if (in_or()) { - matched_in_or() |= contains; - inverse_matched_in_or() |= inverse_contains; - } - } - break; - case CharacterCompareType::Script: - if (!lhs_ranges.is_empty() || !lhs_negated_ranges.is_empty() || !lhs_char_classes.is_empty() || !lhs_negated_char_classes.is_empty()) - return true; - if (has_any_unicode_property && !lhs_unicode_scripts.is_empty() && !lhs_negated_unicode_scripts.is_empty()) { - auto contains = lhs_unicode_scripts.contains(static_cast(pair.value)); - if (!in_or() && (current_lhs_inversion_state() ^ contains)) - return true; - auto inverse_contains = lhs_negated_unicode_scripts.contains(static_cast(pair.value)); - if (!in_or() && !(current_lhs_inversion_state() ^ inverse_contains)) - return true; - if (in_or()) { - matched_in_or() |= contains; - inverse_matched_in_or() |= inverse_contains; - } - } - break; - case CharacterCompareType::ScriptExtension: - if (!lhs_ranges.is_empty() || !lhs_negated_ranges.is_empty() || !lhs_char_classes.is_empty() || !lhs_negated_char_classes.is_empty()) - return true; - if (has_any_unicode_property && !lhs_unicode_script_extensions.is_empty() && !lhs_negated_unicode_script_extensions.is_empty()) { - auto contains = lhs_unicode_script_extensions.contains(static_cast(pair.value)); - if (!in_or() && (current_lhs_inversion_state() ^ contains)) - return true; - auto inverse_contains = lhs_negated_unicode_script_extensions.contains(static_cast(pair.value)); - if (!in_or() && !(current_lhs_inversion_state() ^ inverse_contains)) - return true; - if (in_or()) { - matched_in_or() |= contains; - inverse_matched_in_or() |= inverse_contains; - } - } - break; - case CharacterCompareType::Or: - disjunction_stack.empend(true); - break; - case CharacterCompareType::EndAndOr: { - // FIXME: Handle And when we support it below. - VERIFY(in_or()); - auto state = disjunction_stack.take_last(); - if (current_lhs_inversion_state()) { - if (!state.inverse_matched_in_or) - return true; - } else { - if (state.matched_in_or) - return true; - } - - break; - } - case CharacterCompareType::And: - case CharacterCompareType::Subtract: - // FIXME: These are too difficult to handle, so bail out. - return true; - case CharacterCompareType::Undefined: - case CharacterCompareType::RangeExpressionDummy: - // These do not occur in valid bytecode. - VERIFY_NOT_REACHED(); - } - } - - // We got to the end, just double-check that the inverse flag was not left on (which would match everything). - return current_lhs_inversion_state(); -} - -static bool has_overlap(StaticallyInterpretedCompares const& lhs, StaticallyInterpretedCompares const& rhs, bool insensitive = false) -{ - if (lhs.has_any_unicode_property || rhs.has_any_unicode_property || !lhs.negated_ranges.is_empty() || !rhs.negated_ranges.is_empty() || !lhs.negated_char_classes.is_empty() || !rhs.negated_char_classes.is_empty()) - return true; - - for (auto it_lhs = lhs.ranges.begin(); it_lhs != lhs.ranges.end(); ++it_lhs) { - auto lhs_start = it_lhs.key(); - auto lhs_end = *it_lhs; - - for (auto it_rhs = rhs.ranges.begin(); it_rhs != rhs.ranges.end(); ++it_rhs) { - auto rhs_start = it_rhs.key(); - auto rhs_end = *it_rhs; - - // Check if ranges overlap - if (lhs_start <= rhs_end && rhs_start <= lhs_end) - return true; - - if (insensitive) { - auto expanded_ranges = Unicode::expand_range_case_insensitive(rhs_start, rhs_end); - for (auto const& expanded : expanded_ranges) { - if (lhs_start <= expanded.to && expanded.from <= lhs_end) - return true; - } - } - } - } - - for (auto& lhs_class : lhs.char_classes) { - for (auto& rhs_class : rhs.char_classes) { - if (lhs_class == rhs_class) - return true; - } - } - - return false; -} -enum class AtomicRewritePreconditionResult { - SatisfiedWithProperHeader, - SatisfiedWithEmptyHeader, - NotSatisfied, -}; -static AtomicRewritePreconditionResult block_satisfies_atomic_rewrite_precondition(ByteCode const& bytecode, Block repeated_block, Block following_block, auto const& all_blocks, bool insensitive = false, bool unicode_mode = false) -{ - Vector> repeated_values; - auto flat = bytecode.flat_data(); - auto const* flat_data = flat.data(); - auto has_seen_actionable_opcode = false; - for (size_t ip = repeated_block.start; ip < repeated_block.end;) { - auto id = static_cast(flat_data[ip]); - auto sz = opcode_size(id, flat_data, ip); - switch (id) { - case OpCodeId::Compare: { - has_seen_actionable_opcode = true; - auto compares = flat_compares_at(flat_data, ip, false); - if (repeated_values.is_empty() && any_of(compares, [](auto& compare) { return compare.type == CharacterCompareType::AnyChar; })) - return AtomicRewritePreconditionResult::NotSatisfied; - repeated_values.append(move(compares)); - break; - } - case OpCodeId::CheckBegin: - case OpCodeId::CheckEnd: - has_seen_actionable_opcode = true; - if (repeated_values.is_empty()) - return AtomicRewritePreconditionResult::SatisfiedWithProperHeader; - break; - case OpCodeId::CheckBoundary: - return AtomicRewritePreconditionResult::NotSatisfied; - case OpCodeId::Restore: - case OpCodeId::GoBack: - return AtomicRewritePreconditionResult::NotSatisfied; - case OpCodeId::ForkJump: - case OpCodeId::ForkReplaceJump: - case OpCodeId::ForkIf: - case OpCodeId::JumpNonEmpty: - if (!has_seen_actionable_opcode) - return AtomicRewritePreconditionResult::NotSatisfied; - break; - case OpCodeId::Jump: { - auto offset = static_cast(flat_data[ip + OpArgs::Jump::offset]); - auto jump_target = ip + offset + sz; - auto next_block_it = find_if(all_blocks.begin(), all_blocks.end(), [jump_target](auto& block) { return block.start == jump_target; }); - if (next_block_it == all_blocks.end()) - return AtomicRewritePreconditionResult::NotSatisfied; - repeated_block = *next_block_it; - ip = repeated_block.start; - continue; - } - default: - break; - } - - ip += sz; - } - dbgln_if(REGEX_DEBUG, "Found {} entries in reference", repeated_values.size()); - - auto accept_empty_follow = false; - while (following_block.start == following_block.end && !accept_empty_follow) { - dbgln_if(REGEX_DEBUG, "Following empty block {}", following_block.start); - size_t ip = following_block.start; - auto id = static_cast(flat_data[ip]); - auto sz = opcode_size(id, flat_data, ip); - switch (id) { - case OpCodeId::Jump: { - auto offset = static_cast(flat_data[ip + OpArgs::Jump::offset]); - auto jump_target = ip + offset + sz; - if (jump_target < ip) { - dbgln_if(REGEX_DEBUG, "Jump to {} is backwards, I'm scared of loops", jump_target); - return AtomicRewritePreconditionResult::NotSatisfied; - } - dbgln_if(REGEX_DEBUG, "Following jump to {}", jump_target); - auto next_block_it = find_if(all_blocks.begin(), all_blocks.end(), [jump_target](auto& block) { return block.start == jump_target; }); - if (next_block_it == all_blocks.end()) - return AtomicRewritePreconditionResult::NotSatisfied; - following_block = *next_block_it; - continue; - } - case OpCodeId::ForkJump: - case OpCodeId::ForkIf: - case OpCodeId::ForkReplaceJump: - case OpCodeId::JumpNonEmpty: - return AtomicRewritePreconditionResult::NotSatisfied; - default: - dbgln_if(REGEX_DEBUG, "Empty follow had instruction [{:#02X}] {}", (int)id, opcode_id_name(id)); - accept_empty_follow = true; - break; - } - } - - bool following_block_has_at_least_one_compare = false; - auto final_instruction = following_block.start; - for (size_t ip = following_block.start; ip < following_block.end;) { - final_instruction = ip; - auto id = static_cast(flat_data[ip]); - auto sz = opcode_size(id, flat_data, ip); - switch (id) { - case OpCodeId::Compare: { - following_block_has_at_least_one_compare = true; - auto compares = flat_compares_at(flat_data, ip, false); - if (compares.is_empty()) - break; - - if (any_of(compares, [&](auto& compare) { - return compare.type == CharacterCompareType::AnyChar || compare.type == CharacterCompareType::Reference || compare.type == CharacterCompareType::NamedReference; - })) - return AtomicRewritePreconditionResult::NotSatisfied; - - if (any_of(repeated_values, [&](auto& repeated_value) { return has_overlap(compares, repeated_value, insensitive, unicode_mode); })) - return AtomicRewritePreconditionResult::NotSatisfied; - - return AtomicRewritePreconditionResult::SatisfiedWithProperHeader; - } - case OpCodeId::CheckBegin: - case OpCodeId::CheckEnd: - return AtomicRewritePreconditionResult::SatisfiedWithProperHeader; - case OpCodeId::CheckBoundary: - return AtomicRewritePreconditionResult::NotSatisfied; - case OpCodeId::ForkJump: - case OpCodeId::ForkIf: - case OpCodeId::ForkReplaceJump: - case OpCodeId::JumpNonEmpty: - if (!following_block_has_at_least_one_compare) - return AtomicRewritePreconditionResult::NotSatisfied; - break; - default: - break; - } - - ip += sz; - } - - // If the following block falls through, we can't rewrite it. - auto final_id = static_cast(flat_data[final_instruction]); - switch (final_id) { - case OpCodeId::Jump: - case OpCodeId::JumpNonEmpty: - case OpCodeId::ForkJump: - case OpCodeId::ForkReplaceJump: - case OpCodeId::ForkIf: - break; - default: - return AtomicRewritePreconditionResult::NotSatisfied; - } - - if (following_block_has_at_least_one_compare) - return AtomicRewritePreconditionResult::SatisfiedWithProperHeader; - return AtomicRewritePreconditionResult::SatisfiedWithEmptyHeader; -} - -template -bool Regex::attempt_rewrite_entire_match_as_substring_search(BasicBlockList const& basic_blocks) -{ - // If there's no jumps, we can probably rewrite this as a substring search (Compare { string = str }). - if (basic_blocks.size() > 1) - return false; - - if (basic_blocks.is_empty()) { - parser_result.optimization_data.pure_substring_search.emplace(); - return true; // Empty regex, sure. - } - - auto& bytecode = parser_result.bytecode.get(); - auto flat = bytecode.flat_data(); - auto const* flat_data = flat.data(); - auto flat_size = flat.size(); - - // We have a single basic block, let's see if it's a series of character or string compares. - Vector u16_units; - for (size_t ip = 0; ip < flat_size;) { - auto id = static_cast(flat_data[ip]); - auto sz = opcode_size(id, flat_data, ip); - switch (id) { - case OpCodeId::Compare: { - if (flat_data[ip + OpArgs::Compare::arguments_count] == 0) - return false; // This matches 'nothing', so we can't do a substring search. - for (auto& flat_compare : flat_compares_at(flat_data, ip, false)) { - if (flat_compare.type != CharacterCompareType::Char) - return false; - (void)AK::UnicodeUtils::code_point_to_utf16(flat_compare.value, [&](auto code_unit) { u16_units.append(code_unit); }); - } - break; - } - default: - return false; - } - ip += sz; - } - - parser_result.optimization_data.pure_substring_search.emplace(move(u16_units)); - return true; -} - -template -void Regex::rewrite_with_useless_jumps_removed() -{ - auto& bytecode = parser_result.bytecode.get(); - - if constexpr (REGEX_DEBUG) { - RegexDebug dbg; - dbg.print_bytecode(*this); - } - - BytecodeRewriter rewriter(bytecode, pattern_value); - - auto flat = bytecode.flat_data(); - auto const* flat_data = flat.data(); - - for (auto& instr : rewriter.instructions) { - bool is_useless = false; - auto id = instr.id; - auto ip = instr.old_ip; - if (id == OpCodeId::Jump || id == OpCodeId::JumpNonEmpty - || id == OpCodeId::ForkJump || id == OpCodeId::ForkReplaceJump - || id == OpCodeId::ForkStay || id == OpCodeId::ForkReplaceStay - || id == OpCodeId::ForkIf) { - is_useless = static_cast(flat_data[ip + OpArgs::Jump::offset]) == 0; - } - - instr.skip = is_useless; - } - - parser_result.bytecode = rewriter.rebuild(bytecode); -} - -template -void Regex::attempt_rewrite_loops_as_atomic_groups(BasicBlockList const& basic_blocks) -{ - auto& bytecode = parser_result.bytecode.get(); - bool const insensitive = parser_result.options.has_flag_set(AllFlags::Insensitive); - bool const unicode_mode = parser_result.options.has_flag_set(AllFlags::Unicode); - if constexpr (REGEX_DEBUG) { - RegexDebug dbg; - dbg.print_bytecode(*this); - for (auto const& block : basic_blocks) - dbgln("block from {} to {} (comment: {})", block.start, block.end, block.comment); - } - - // A pattern such as: - // bb0 | RE0 - // | ForkX bb0 - // ------------------------- - // bb1 | RE1 - // can be rewritten as: - // ------------------------- - // bb0 | RE0 - // | ForkReplaceX bb0 - // ------------------------- - // bb1 | RE1 - // provided that first(RE1) not-in end(RE0), which is to say - // that RE1 cannot start with whatever RE0 has matched (ever). - // - // Alternatively, a second form of this pattern can also occur: - // bb0 | * - // | ForkX bb2 - // ------------------------ - // bb1 | RE0 - // | Jump bb0 - // ------------------------ - // bb2 | RE1 - // which can be transformed (with the same preconditions) to: - // bb0 | * - // | ForkReplaceX bb2 - // ------------------------ - // bb1 | RE0 - // | Jump bb0 - // ------------------------ - // bb2 | RE1 - - enum class AlternateForm { - DirectLoopWithoutHeader, // loop without proper header, a block forking to itself. i.e. the first form. - DirectLoopWithoutHeaderAndEmptyFollow, // loop without proper header, a block forking to itself. i.e. the first form but with RE1 being empty. - DirectLoopWithHeader, // loop with proper header, i.e. the second form. - }; - struct CandidateBlock { - Block forking_block; - Optional new_target_block; - AlternateForm form; - }; - Vector candidate_blocks; - auto flat = bytecode.flat_data(); - auto const* flat_data = flat.data(); - auto flat_size = flat.size(); - - auto is_an_eligible_jump = [&flat_data, flat_size](size_t ip, size_t block_start, AlternateForm alternate_form) { - if (ip >= flat_size) - return false; - auto id = static_cast(flat_data[ip]); - auto sz = opcode_size(id, flat_data, ip); - auto offset = static_cast(flat_data[ip + OpArgs::Jump::offset]); - switch (id) { - case OpCodeId::JumpNonEmpty: { - auto form = static_cast(flat_data[ip + OpArgs::JumpNonEmpty::form]); - if (form != OpCodeId::Jump && alternate_form == AlternateForm::DirectLoopWithHeader) - return false; - if (form != OpCodeId::ForkJump && form != OpCodeId::ForkStay && alternate_form == AlternateForm::DirectLoopWithoutHeader) - return false; - return offset + ip + sz == block_start; - } - case OpCodeId::ForkJump: - if (alternate_form == AlternateForm::DirectLoopWithHeader) - return false; - return offset + ip + sz == block_start; - case OpCodeId::ForkStay: - if (alternate_form == AlternateForm::DirectLoopWithHeader) - return false; - return offset + ip + sz == block_start; - case OpCodeId::Jump: - if (alternate_form == AlternateForm::DirectLoopWithoutHeader) - return false; - if (alternate_form == AlternateForm::DirectLoopWithHeader) - return offset + ip + sz == block_start; - VERIFY_NOT_REACHED(); - default: - return false; - } - }; - for (size_t i = 0; i < basic_blocks.size(); ++i) { - auto forking_block = basic_blocks[i]; - Optional fork_fallback_block; - if (i + 1 < basic_blocks.size()) - fork_fallback_block = basic_blocks[i + 1]; - // Check if the last instruction in this block is a jump to the block itself: - { - auto ip = forking_block.end; - if (is_an_eligible_jump(ip, forking_block.start, AlternateForm::DirectLoopWithoutHeader)) { - // We've found RE0 (and RE1 is just the following block, if any), let's see if the precondition applies. - // if RE1 is empty, there's no first(RE1), so this is an automatic pass. - if (!fork_fallback_block.has_value() - || (fork_fallback_block->end == fork_fallback_block->start && block_satisfies_atomic_rewrite_precondition(bytecode, forking_block, *fork_fallback_block, basic_blocks, insensitive, unicode_mode) != AtomicRewritePreconditionResult::NotSatisfied)) { - candidate_blocks.append({ forking_block, fork_fallback_block, AlternateForm::DirectLoopWithoutHeader }); - break; - } - - auto precondition = block_satisfies_atomic_rewrite_precondition(bytecode, forking_block, *fork_fallback_block, basic_blocks, insensitive, unicode_mode); - if (precondition == AtomicRewritePreconditionResult::SatisfiedWithProperHeader) { - candidate_blocks.append({ forking_block, fork_fallback_block, AlternateForm::DirectLoopWithoutHeader }); - break; - } - if (precondition == AtomicRewritePreconditionResult::SatisfiedWithEmptyHeader) { - candidate_blocks.append({ forking_block, fork_fallback_block, AlternateForm::DirectLoopWithoutHeaderAndEmptyFollow }); - break; - } - } - } - // Check if the last instruction in the last block is a direct jump to this block - if (fork_fallback_block.has_value()) { - auto fb_ip = fork_fallback_block->end; - if (is_an_eligible_jump(fb_ip, forking_block.start, AlternateForm::DirectLoopWithHeader)) { - // We've found bb1 and bb0, let's just make sure that bb0 forks to bb2. - auto fork_id = forking_block.end < flat_size ? static_cast(flat_data[forking_block.end]) : OpCodeId::Exit; - if (fork_id == OpCodeId::ForkJump || fork_id == OpCodeId::ForkStay) { - Optional block_following_fork_fallback; - if (i + 2 < basic_blocks.size()) - block_following_fork_fallback = basic_blocks[i + 2]; - if (!block_following_fork_fallback.has_value() - || block_satisfies_atomic_rewrite_precondition(bytecode, *fork_fallback_block, *block_following_fork_fallback, basic_blocks, insensitive, unicode_mode) != AtomicRewritePreconditionResult::NotSatisfied) { - candidate_blocks.append({ forking_block, {}, AlternateForm::DirectLoopWithHeader }); - break; - } - } - } - // We've found a slightly degenerate case, where the next block jumps back to the _jump_ instruction in the forking block. - // This is a direct loop without a proper header that is posing as a loop with a header. - if (is_an_eligible_jump(fb_ip, forking_block.end, AlternateForm::DirectLoopWithHeader)) { - // We've found bb1 and bb0, let's just make sure that bb0 forks to bb2. - auto fork_id2 = forking_block.end < flat_size ? static_cast(flat_data[forking_block.end]) : OpCodeId::Exit; - if (fork_id2 == OpCodeId::ForkJump || fork_id2 == OpCodeId::ForkStay) { - Optional block_following_fork_fallback; - if (i + 2 < basic_blocks.size()) - block_following_fork_fallback = basic_blocks[i + 2]; - if (!block_following_fork_fallback.has_value() - || block_satisfies_atomic_rewrite_precondition(bytecode, *fork_fallback_block, *block_following_fork_fallback, basic_blocks, insensitive, unicode_mode) != AtomicRewritePreconditionResult::NotSatisfied) { - candidate_blocks.append({ forking_block, {}, AlternateForm::DirectLoopWithoutHeader }); - break; - } - } - } - } - } - - dbgln_if(REGEX_DEBUG, "Found {} candidate blocks", candidate_blocks.size()); - if constexpr (REGEX_DEBUG) { - for (auto const& candidate : candidate_blocks) { - dbgln("Candidate block from {} to {} (comment: {})", candidate.forking_block.start, candidate.forking_block.end, candidate.forking_block.comment); - if (candidate.new_target_block.has_value()) - dbgln(" with target block from {} to {} (comment: {})", candidate.new_target_block->start, candidate.new_target_block->end, candidate.new_target_block->comment); - switch (candidate.form) { - case AlternateForm::DirectLoopWithoutHeader: - dbgln(" form: DirectLoopWithoutHeader"); - break; - case AlternateForm::DirectLoopWithoutHeaderAndEmptyFollow: - dbgln(" form: DirectLoopWithoutHeaderAndEmptyFollow"); - break; - case AlternateForm::DirectLoopWithHeader: - dbgln(" form: DirectLoopWithHeader"); - break; - default: - dbgln(" form: Unknown"); - break; - } - } - } - if (candidate_blocks.is_empty()) { - dbgln_if(REGEX_DEBUG, "Failed to find anything for {}", pattern_value); - return; - } - - RedBlackTree needed_patches; - - // Reverse the blocks, so we can patch the bytecode without messing with the latter patches. - quick_sort(candidate_blocks, [](auto& a, auto& b) { return b.forking_block.start > a.forking_block.start; }); - for (auto& candidate : candidate_blocks) { - // Note that both forms share a ForkReplace patch in forking_block. - // Patch the ForkX in forking_block to be a ForkReplaceX instead. - auto& opcode_id = bytecode[candidate.forking_block.end]; - if (opcode_id == (ByteCodeValueType)OpCodeId::ForkStay) { - opcode_id = (ByteCodeValueType)OpCodeId::ForkReplaceStay; - } else if (opcode_id == (ByteCodeValueType)OpCodeId::ForkJump) { - opcode_id = (ByteCodeValueType)OpCodeId::ForkReplaceJump; - } else if (opcode_id == (ByteCodeValueType)OpCodeId::JumpNonEmpty) { - auto& jump_opcode_id = bytecode[candidate.forking_block.end + 3]; - if (jump_opcode_id == (ByteCodeValueType)OpCodeId::ForkStay) - jump_opcode_id = (ByteCodeValueType)OpCodeId::ForkReplaceStay; - else if (jump_opcode_id == (ByteCodeValueType)OpCodeId::ForkJump) - jump_opcode_id = (ByteCodeValueType)OpCodeId::ForkReplaceJump; - else - VERIFY_NOT_REACHED(); - } else { - VERIFY_NOT_REACHED(); - } - } - - if (!needed_patches.is_empty()) { - auto bytecode_size = bytecode.size(); - struct Patch { - ssize_t value; - size_t offset; - bool should_negate { false }; - }; - for (size_t ip = 0; ip < bytecode_size;) { - auto id = static_cast(flat_data[ip]); - auto sz = opcode_size(id, flat_data, ip); - Stack patch_points; - - switch (id) { - case OpCodeId::Jump: - patch_points.push({ static_cast(flat_data[ip + OpArgs::Jump::offset]), ip + 1 }); - break; - case OpCodeId::JumpNonEmpty: - patch_points.push({ static_cast(flat_data[ip + OpArgs::JumpNonEmpty::offset]), ip + 1 }); - patch_points.push({ static_cast(flat_data[ip + OpArgs::JumpNonEmpty::checkpoint]), ip + 2 }); - break; - case OpCodeId::ForkJump: - patch_points.push({ static_cast(flat_data[ip + OpArgs::Jump::offset]), ip + 1 }); - break; - case OpCodeId::ForkStay: - patch_points.push({ static_cast(flat_data[ip + OpArgs::Jump::offset]), ip + 1 }); - break; - case OpCodeId::ForkIf: - patch_points.push({ static_cast(flat_data[ip + OpArgs::ForkIf::offset]), ip + 1 }); - break; - case OpCodeId::Repeat: - patch_points.push({ -static_cast(flat_data[ip + OpArgs::Repeat::offset]), ip + 1, true }); - break; - default: - break; - } - - while (!patch_points.is_empty()) { - auto& patch_point = patch_points.top(); - auto target_offset = patch_point.value + static_cast(ip + sz); - - constexpr auto do_patch = [](auto& patch_it, auto& patch_point, auto& target_offset, auto& bytecode, auto current_ip) { - if (patch_it.key() == current_ip) - return; - - if (patch_point.value < 0 && target_offset <= static_cast(patch_it.key()) && current_ip > patch_it.key()) - bytecode[patch_point.offset] += (patch_point.should_negate ? 1 : -1) * (*patch_it); - else if (patch_point.value > 0 && target_offset >= static_cast(patch_it.key()) && current_ip < patch_it.key()) - bytecode[patch_point.offset] += (patch_point.should_negate ? -1 : 1) * (*patch_it); - }; - - if (auto patch_it = needed_patches.find_largest_not_above_iterator(target_offset); !patch_it.is_end()) - do_patch(patch_it, patch_point, target_offset, bytecode, ip); - else if (auto patch_it = needed_patches.find_largest_not_above_iterator(ip); !patch_it.is_end()) - do_patch(patch_it, patch_point, target_offset, bytecode, ip); - - patch_points.pop(); - } - - ip += sz; - } - } - - if constexpr (REGEX_DEBUG) { - warnln("Transformed to:"); - RegexDebug dbg; - dbg.print_bytecode(*this); - } -} - -template -void Regex::attempt_rewrite_adjacent_compares_as_string_compare(BasicBlockList const& basic_blocks) -{ - auto& bytecode = parser_result.bytecode.get(); - - if (basic_blocks.is_empty()) - return; - - // Find sequences of single-character compares - struct StringSequence { - size_t start_ip; - size_t end_ip; - Vector characters; - }; - Vector sequences; - - auto flat = bytecode.flat_data(); - auto const* flat_d = flat.data(); - auto flat_sz = flat.size(); - - for (auto const& block : basic_blocks) { - Vector current_chars; - size_t sequence_start = 0; - bool in_sequence = false; - size_t ip = block.start; - - for (; ip <= block.end && ip < flat_sz;) { - auto current_ip = ip; - auto id = static_cast(flat_d[ip]); - auto sz = opcode_size(id, flat_d, ip); - - bool is_single_char = false; - u32 character = 0; - - if (id == OpCodeId::Compare) { - auto fc = flat_compares_at(flat_d, ip, false); - - if (fc.size() == 1 - && fc[0].type == CharacterCompareType::Char) { - is_single_char = true; - character = fc[0].value; - } - } - - if (is_single_char) { - if (!in_sequence) { - sequence_start = current_ip; - current_chars.clear(); - in_sequence = true; - } - current_chars.append(character); - } else { - if (in_sequence && current_chars.size() >= 2) { - sequences.append({ sequence_start, current_ip, move(current_chars) }); - current_chars.clear(); - } - in_sequence = false; - } - - ip += sz; - } - - if (in_sequence && current_chars.size() >= 2) { - sequences.append({ sequence_start, ip, move(current_chars) }); - } - } - - if (sequences.is_empty()) - return; - - BytecodeRewriter rewriter(bytecode, pattern_value); - Vector replacements; - replacements.ensure_capacity(sequences.size()); - for (auto const& seq : sequences) { - StringBuilder string_builder(StringBuilder::Mode::UTF16); - for (auto ch : seq.characters) - string_builder.append_code_point(ch); - ByteCode replacement; - replacement.insert_bytecode_compare_string(string_builder.to_utf16_string()); - replacements.append(move(replacement)); - } - - parser_result.bytecode = rewriter.rebuild(bytecode, sequences.span(), replacements.span()); -} - -template -void Regex::attempt_rewrite_dot_star_sequences_as_seek(BasicBlockList const& basic_blocks) -{ - auto& bytecode = parser_result.bytecode.get(); - - if (basic_blocks.is_empty()) { - dbgln_if(REGEX_DEBUG, "No basic blocks, skipping /.*/ rewrite"); - return; - } - - // If a /.*/ sequence is followed by a compare C (with some non-matching ops {O} in between), we can rewrite: - // bbN: {O0} (optional non-matching ops before the pattern) - // ForkStay bbM - // Checkpoint p - // Compare AnyChar - // FailIfEmpty (optional, noop for .*) - // JumpNonEmpty (back to ForkStay) p - // bbM: {O1} (optional non-matching ops) - // Compare C - // as - // bbN: {O0} - // bbR: RSeekTo C - // ForkStay bbR - // bbM: {O1} - // Compare C - // - // Note: bbM is determined by the ForkStay's target, not necessarily the next sequential block - // Note: The pattern may span across multiple basic blocks - - struct DotStarCandidate { - size_t fork_ip; - size_t checkpoint_ip; - size_t compare_ip; - size_t jump_ip; - size_t following_block_start; - u64 checkpoint_id; - u32 seek_code_point; - }; - OrderedHashMap candidates; - - auto flat2 = bytecode.flat_data(); - auto const* flat_d2 = flat2.data(); - auto flat2_size = flat2.size(); - - for (size_t i = 0; i < basic_blocks.size(); ++i) { - auto const& block = basic_blocks[i]; - - size_t ip = block.start; - - if (ip > block.end || ip >= flat2_size) - continue; - - // Skip non-matching ops at the start of the block - while (ip <= block.end && ip < flat2_size) { - auto op_id = static_cast(flat_d2[ip]); - - switch (op_id) { - case OpCodeId::Checkpoint: - case OpCodeId::Save: - case OpCodeId::SaveLeftCaptureGroup: - case OpCodeId::SaveRightCaptureGroup: - case OpCodeId::SaveRightNamedCaptureGroup: - case OpCodeId::ClearCaptureGroup: - ip += opcode_size(op_id, flat_d2, ip); - continue; - - default: - goto found_potential_fork; - } - } - - { - if (ip >= bytecode.size()) - continue; - - if (static_cast(flat_d2[ip]) != OpCodeId::ForkStay) - continue; - } - - found_potential_fork: - // (1) ForkStay bbM - dbgln_if(REGEX_DEBUG, "Examining block {} from {} to {}", i, block.start, block.end); - if (static_cast(flat_d2[ip]) != OpCodeId::ForkStay) { - dbgln_if(REGEX_DEBUG, " did not find ForkStay at {}", ip); - continue; - } - - auto fork_ip = ip; - auto fork_sz = opcode_size(OpCodeId::ForkStay, flat_d2, fork_ip); - auto fork_offset = static_cast(flat_d2[fork_ip + OpArgs::Jump::offset]); - - // Find the actual following block by the fork target - auto fork_target = fork_ip + fork_sz + fork_offset; - - size_t following_block_idx = 0; - bool found_following_block = false; - for (size_t j = 0; j < basic_blocks.size(); ++j) { - if (basic_blocks[j].start == fork_target) { - if (basic_blocks[j].start <= basic_blocks[j].end) { - following_block_idx = j; - found_following_block = true; - break; - } - continue; - } - } - - if (!found_following_block) { - dbgln_if(REGEX_DEBUG, " did not find non-empty following block for fork target {}", fork_target); - continue; - } - - auto const& following_block = basic_blocks[following_block_idx]; - dbgln_if(REGEX_DEBUG, " Fork target {} is in block {} (from {} to {})", fork_target, following_block_idx, following_block.start, following_block.end); - - ip += fork_sz; - - // (2) Checkpoint p - if (static_cast(flat_d2[ip]) != OpCodeId::Checkpoint) { - dbgln_if(REGEX_DEBUG, " did not find Checkpoint at {} (found opcode {})", ip, (int)static_cast(flat_d2[ip])); - continue; - } - - auto checkpoint_ip = ip; - auto checkpoint_id = flat_d2[checkpoint_ip + OpArgs::Checkpoint::id]; - - ip += opcode_size(OpCodeId::Checkpoint, flat_d2, ip); - - // (3) Compare AnyChar - if (static_cast(flat_d2[ip]) != OpCodeId::Compare) { - dbgln_if(REGEX_DEBUG, " did not find Compare at {} (found opcode {})", ip, (int)static_cast(flat_d2[ip])); - continue; - } - - { - auto compare_ip = ip; - auto fc = flat_compares_at(flat_d2, ip, false); - - if (fc.size() != 1 || fc[0].type != CharacterCompareType::AnyChar) { - dbgln_if(REGEX_DEBUG, " Compare at {} is not AnyChar", ip); - continue; - } - - ip += opcode_size(OpCodeId::Compare, flat_d2, ip); - - // (3.5) Skip FailIfEmpty if present - if (static_cast(flat_d2[ip]) == OpCodeId::FailIfEmpty) - ip += opcode_size(OpCodeId::FailIfEmpty, flat_d2, ip); - - // (4) JumpNonEmpty back to ForkStay - if (static_cast(flat_d2[ip]) != OpCodeId::JumpNonEmpty) { - dbgln_if(REGEX_DEBUG, " did not find JumpNonEmpty at {} (found opcode {})", ip, (int)static_cast(flat_d2[ip])); - continue; - } - - auto jump_ip = ip; - auto jump_sz = opcode_size(OpCodeId::JumpNonEmpty, flat_d2, jump_ip); - auto jump_offset = static_cast(flat_d2[jump_ip + OpArgs::JumpNonEmpty::offset]); - auto jump_checkpoint = flat_d2[jump_ip + OpArgs::JumpNonEmpty::checkpoint]; - - if (jump_ip + jump_sz + jump_offset != fork_ip) { - dbgln_if(REGEX_DEBUG, " JumpNonEmpty at {} does not jump back to ForkStay at {} (instead jumps to {})", - ip, fork_ip, jump_ip + jump_sz + jump_offset); - continue; - } - if (jump_checkpoint != checkpoint_id) { - dbgln_if(REGEX_DEBUG, " JumpNonEmpty at {} does not reference Checkpoint id {} (instead references {})", - ip, checkpoint_id, jump_checkpoint); - continue; - } - - dbgln_if(REGEX_DEBUG, " Found .* pattern from IP {} to {}", fork_ip, jump_ip + jump_sz); - - // The following block must contain a Compare C, with only non-matching ops in between - size_t fip = following_block.start; - while (fip <= following_block.end && fip < flat2_size) { - auto fop_id = static_cast(flat_d2[fip]); - auto fop_sz = opcode_size(fop_id, flat_d2, fip); - - switch (fop_id) { - case OpCodeId::Checkpoint: - case OpCodeId::Save: - case OpCodeId::SaveLeftCaptureGroup: - case OpCodeId::SaveRightCaptureGroup: - case OpCodeId::SaveRightNamedCaptureGroup: - case OpCodeId::ClearCaptureGroup: - fip += fop_sz; - continue; - - case OpCodeId::Compare: { - auto following_compares = flat_compares_at(flat_d2, fip, false); - - StaticallyInterpretedCompares compares; - if (!interpret_compares(following_compares, compares, &bytecode, true)) { - dbgln_if(REGEX_DEBUG, " could not statically interpret compares at {} in following block", fip); - goto next_block; - } - - // Must be able to pull a single code point from this compare (i.e. a single 1-long range, no negations, no char classes, and no unicode properties) - if (compares.ranges.size() != 1 || !compares.negated_ranges.is_empty() - || !compares.char_classes.is_empty() || !compares.negated_char_classes.is_empty() - || compares.has_any_unicode_property - || !compares.unicode_general_categories.is_empty() - || !compares.unicode_properties.is_empty() - || !compares.unicode_scripts.is_empty() - || !compares.unicode_script_extensions.is_empty() - || !compares.negated_unicode_general_categories.is_empty() - || !compares.negated_unicode_properties.is_empty() - || !compares.negated_unicode_scripts.is_empty() - || !compares.negated_unicode_script_extensions.is_empty()) { - dbgln_if(REGEX_DEBUG, " compares at {} in following block are too complex to rewrite as SeekTo", fip); - goto next_block; - } - - auto it = compares.ranges.begin(); - if (it.key() != *it) { - // Not a single code point - dbgln_if(REGEX_DEBUG, " compares at {} in following block are a range, not a single code point ({}..{})", fip, it.key(), *it); - goto next_block; - } - - auto seeked_code_point = it.key(); - - candidates.set(fork_ip, { fork_ip, checkpoint_ip, compare_ip, jump_ip, following_block.start, checkpoint_id, seeked_code_point }); - - dbgln_if(REGEX_DEBUG, " Found sequence from {} to {} followed by Compare '{}' at {}, can rewrite as SeekTo", - fork_ip, jump_ip + 4, (char)seeked_code_point, fip); - goto next_block; - } - - default: - dbgln_if(REGEX_DEBUG, " Hit non-matching, non-skippable opcode {} at {} in following block", (int)fop_id, fip); - goto next_block; - } - } - } - - next_block: - continue; - } - - dbgln_if(REGEX_DEBUG, "Found {} dot-star sequences to rewrite as SeekTo", candidates.size()); - - if (candidates.is_empty()) - return; - - BytecodeRewriter rewriter(bytecode, pattern_value); - - struct Range { - size_t start_ip; - size_t end_ip; - }; - - Vector ranges_to_skip; - Vector replacements; - ranges_to_skip.ensure_capacity(candidates.size()); - replacements.ensure_capacity(candidates.size()); - - for (auto& [_, candidate] : candidates) { - ranges_to_skip.empend(candidate.fork_ip, candidate.jump_ip + 4); // JumpNonEmpty = 4 - ByteCode replacement; - replacement.empend(static_cast(OpCodeId::RSeekTo)); - replacement.empend(candidate.seek_code_point); - replacement.empend(static_cast(OpCodeId::ForkStay)); - replacement.empend(static_cast(-4)); // Offset back to RSeekTo - replacements.append(move(replacement)); - } - - parser_result.bytecode = rewriter.rebuild(bytecode, ranges_to_skip.span(), replacements.span()); - - if constexpr (REGEX_DEBUG) { - dbgln("After dot-star rewrite as SeekTo:"); - RegexDebug dbg; - dbg.print_bytecode(*this); - } -} - -template -void Regex::rewrite_simple_compares(BasicBlockList const& basic_blocks) -{ - // If a Compare opcode only has a single compare and that's a match opcode - // we can rewrite it as a CompareSimple to avoid the overhead of handling multiple compares: - // Compare argc=1 args=S - // Char 'a' - // --> - // CompareSimple args=S - // Char 'a' - - auto& bytecode = parser_result.bytecode.get(); - - if (basic_blocks.is_empty()) - return; - - struct SimpleCompareCandidate { - size_t compare_ip; - Vector compare_data; - }; - Vector candidates; - - auto flat3 = bytecode.flat_data(); - auto const* flat_d3 = flat3.data(); - auto flat3_size = flat3.size(); - - for (auto const& block : basic_blocks) { - for (size_t ip = block.start; ip <= block.end && ip < flat3_size;) { - auto current_ip = ip; - auto id = static_cast(flat_d3[ip]); - auto sz = opcode_size(id, flat_d3, ip); - - if (id == OpCodeId::Compare) { - auto fc = flat_compares_at(flat_d3, ip, false); - - if (fc.size() == 1 && !first_is_one_of(fc[0].type, CharacterCompareType::And, CharacterCompareType::Or, CharacterCompareType::Inverse, CharacterCompareType::TemporaryInverse, CharacterCompareType::Subtract, CharacterCompareType::Undefined)) { - auto slice = bytecode.spans().slice(current_ip + 2, sz - 2); - Vector data; - data.ensure_capacity(slice.size()); - for (auto value : slice) - data.append(value); - candidates.append({ current_ip, move(data) }); // +2 to skip opcode id and argc - } - } - - ip += sz; - } - } - - if (candidates.is_empty()) - return; - - dbgln_if(REGEX_DEBUG, "Found {} simple compare candidates to rewrite", candidates.size()); - - BytecodeRewriter rewriter(bytecode, pattern_value); - - for (auto& candidate : candidates) { - auto& instr = *rewriter.instructions.find_if([&](auto& i) { return i.old_ip == candidate.compare_ip; }); - instr.skip = true; - } - - size_t candidate_index = 0; - auto insert_replacement = [&](auto const& instr, ByteCode& result) { - while (candidate_index < candidates.size()) { - auto& candidate = candidates[candidate_index]; - - if (instr.old_ip == candidate.compare_ip) { - result.empend(static_cast(OpCodeId::CompareSimple)); - result.extend(move(candidate.compare_data)); - candidate_index++; - return; - } - - if (instr.old_ip < candidate.compare_ip) - return; - - candidate_index++; - } - }; - - parser_result.bytecode = rewriter.rebuild(bytecode, move(insert_replacement)); - - if constexpr (REGEX_DEBUG) { - dbgln("After simple compare rewrite:"); - RegexDebug dbg; - dbg.print_bytecode(*this); - } -} - -void Optimizer::append_alternation(ByteCode& target, ByteCode&& left, ByteCode&& right) -{ - Array alternatives; - alternatives[0] = move(left); - alternatives[1] = move(right); - - append_alternation(target, alternatives); -} - -template -using OrderedHashMapForTrie = OrderedHashMap; - -void Optimizer::append_alternation(ByteCode& target, Span alternatives) -{ - // Assume we have N alternatives A0..AN, each with M basic blocks bb0..bbM, each with I instructions 0..I (denoted Ai.bbj[k]) - // We can create the alternation is two ways: - // - Lay them out sequentially, such that A0 is tried, then A1, then A2, etc. - // - Generate a prefix tree for A*.bb*[*], and walk the tree at runtime. - // For the first case, assuming we have two A0.bb0[0..2] and A1.bb0[0..2]: - // out.bb0: - // ForkStay out.bb1 - // A0.bb0[*] - // Jump out.bb2 - // out.bb1: - // A1.bb0[*] - // out.bb2: - // - // For the second case, assuming the following alternatives: - // A0.bb0: - // Compare 'a' - // Compare 'b' - // Compare 'd' - // A1.bb0: - // Compare 'a' - // Compare 'c' - // Compare 'd' - // We can first generate a prefix tree (trie here), with each node denoted by [insn, insn*]: - // (root) - // |- [A0.bb0[0], A1.bb0[0]] - // | |- [A0.bb0[1]] - // | | |- [A0.bb0[2]] - // | |- [A1.bb0[1]] - // | | |- [A1.bb0[2]] - // i.e. the first instruction of A0 and A1 are the same, so we can merge them into one node; - // everything following that is different (A1.bb0[2] is not considered equivalent to A0.bb0[2] as they are jumped-to by different instructions, - // in this case their previous instruction) - // Then, each trie node N { insn, children } can be represented as: - // out for N: - // N.insn[*] - // ForkJump out for N.children[0] - // ForkJump out for N.children[1] - // ... - // or if there's a single child, we can directly jump to it: - // out for N: // if N.children.size() == 1 - // N.insn[*] - // Jump out for N.children[0] - // For our example, this would yield: - // out for root: - // Jump out for [A0.bb0[0], A1.bb0[0]] - // out for [A0.bb0[0], A1.bb0[0]]: - // Compare 'a' - // ForkJump out for A0.bb0[1] - // ForkJump out for A1.bb0[1] - // out for A0.bb0[1]: - // Compare 'b' - // Jump out for A0.bb0[2] - // out for A1.bb0[1]: - // Compare 'c' - // Jump out for A1.bb0[2] - // out for A0.bb0[2]: - // Compare 'd' - // out for A1.bb0[2]: - // Compare 'd' - if (alternatives.size() == 0) - return; - - if (alternatives.size() == 1) - return target.extend(move(alternatives[0])); - - target.merge_string_tables_from(alternatives); - if (all_of(alternatives, [](auto& x) { return x.is_empty(); })) - return; - - for (auto& entry : alternatives) - entry.flatten(); - -#if REGEX_DEBUG - ScopeLogger log; - warnln("Alternations:"); - RegexDebug dbg; - for (auto& entry : alternatives) { - warnln("----------"); - dbg.print_bytecode(entry); - } - ScopeGuard print_at_end { - [&] { - warnln("======================"); - RegexDebug dbg; - dbg.print_bytecode(target); - } - }; -#endif - - // First, find incoming jump edges. - // We need them for two reasons: - // - We need to distinguish between insn-A-jumped-to-by-insn-B and insn-A-jumped-to-by-insn-C (as otherwise we'd break trie invariants) - // - We need to know which jumps to patch when we're done - - struct JumpEdge { - Span jump_insn; - }; - Vector>> incoming_jump_edges_for_each_alternative; - incoming_jump_edges_for_each_alternative.resize(alternatives.size()); - - auto has_any_backwards_jump = false; - - for (size_t i = 0; i < alternatives.size(); ++i) { - auto& alternative = alternatives[i]; - // Add a jump to the "end" of the block; this is implicit in the bytecode, but we need it to be explicit in the trie. - // Jump{offset=0} - alternative.append(static_cast(OpCodeId::Jump)); - alternative.append(0); - - auto& incoming_jump_edges = incoming_jump_edges_for_each_alternative[i]; - - auto alternative_bytes = alternative.spans<1>().singular_span(); - auto const* alt_data = alternative_bytes.data(); - for (size_t ip = 0; ip < alternative.size();) { - auto id = static_cast(alt_data[ip]); - auto sz = opcode_size(id, alt_data, ip); - auto op_bytes = alternative_bytes.slice(ip, sz); - - switch (id) { - case OpCodeId::Jump: - case OpCodeId::JumpNonEmpty: - case OpCodeId::ForkJump: - case OpCodeId::ForkStay: - case OpCodeId::ForkReplaceJump: - case OpCodeId::ForkReplaceStay: - case OpCodeId::ForkIf: { - auto offset = static_cast(alt_data[ip + OpArgs::Jump::offset]); - incoming_jump_edges.ensure(offset + sz + ip).append({ op_bytes }); - has_any_backwards_jump |= offset < 0; - break; - } - case OpCodeId::Repeat: { - auto repeat_offset = alt_data[ip + OpArgs::Repeat::offset]; - incoming_jump_edges.ensure(ip - repeat_offset).append({ op_bytes }); - has_any_backwards_jump = true; - break; - } - default: - break; - } - ip += sz; - } - } - - struct QualifiedIP { - size_t alternative_index; - size_t instruction_position; - - bool operator==(QualifiedIP const& other) const = default; - }; - struct NodeMetadataEntry { - QualifiedIP ip; - NonnullOwnPtr first_compare_from_here; - }; - using Tree = Trie, Vector, Traits>, void, OrderedHashMapForTrie>; - Tree trie { {} }; // Root node is empty, key{ instruction_bytes, dependent_instruction_bytes... } -> IP - - size_t common_hits = 0; - size_t total_nodes = 0; - size_t total_bytecode_entries_in_tree = 0; - for (size_t i = 0; i < alternatives.size(); ++i) { - auto& alternative = alternatives[i]; - auto& incoming_jump_edges = incoming_jump_edges_for_each_alternative[i]; - - auto* active_node = ≜ - auto alternative_span = alternative.spans<1>().singular_span(); - auto const* alt_d = alternative_span.data(); - for (size_t ip = 0; ip < alternative_span.size();) { - total_nodes += 1; - auto id = static_cast(alt_d[ip]); - auto sz = opcode_size(id, alt_d, ip); - auto op_bytes = alternative_span.slice(ip, sz); - Vector> node_key_bytes; - node_key_bytes.append(op_bytes); - - if (auto edges = incoming_jump_edges.get(ip); edges.has_value()) { - for (auto& edge : *edges) - node_key_bytes.append(edge.jump_insn); - } - - active_node = static_cast(MUST(active_node->ensure_child(DisjointSpans { move(node_key_bytes) }, [] -> Vector { return {}; }))); - - auto next_compare = [alt_d, ip](StaticallyInterpretedCompares& compares) { - size_t temp_ip = ip; - auto temp_id = static_cast(alt_d[temp_ip]); - while (temp_id == OpCodeId::Checkpoint || temp_id == OpCodeId::SaveLeftCaptureGroup - || temp_id == OpCodeId::SaveRightCaptureGroup || temp_id == OpCodeId::SaveRightNamedCaptureGroup - || temp_id == OpCodeId::Save) { - temp_ip += opcode_size(temp_id, alt_d, temp_ip); - temp_id = static_cast(alt_d[temp_ip]); - } - - // We found something functional, if it's a compare, we need to care. - if (temp_id != OpCodeId::Compare) - return; - - auto fc = flat_compares_at(alt_d, temp_ip, false); - interpret_compares(fc, compares); - }; - - auto node_metadata = NodeMetadataEntry { { i, ip }, make() }; - auto& metadata = active_node->metadata_value(); - if (metadata.is_empty()) - total_bytecode_entries_in_tree += sz; - else - common_hits++; - metadata.append(move(node_metadata)); - next_compare(*active_node->metadata_value().last().first_compare_from_here); - - ip += sz; - } - } - - if constexpr (REGEX_DEBUG) { - Function print_tree = [&](decltype(trie)& node, size_t indent = 0) mutable { - ByteString name = "(no ip)"; - ByteString insn; - if (node.has_metadata()) { - name = ByteString::formatted( - "{}@{} ({} node{})", - node.metadata_value().first().ip.instruction_position, - node.metadata_value().first().ip.alternative_index, - node.metadata_value().size(), - node.metadata_value().size() == 1 ? "" : "s"); - - auto state = MatchState::only_for_enumeration(); - state.instruction_position = node.metadata_value().first().ip.instruction_position; - auto& alt = alternatives[node.metadata_value().first().ip.alternative_index]; - auto alt_flat = alt.flat_data(); - auto const* alt_data = alt_flat.data(); - auto alt_id = static_cast(alt_data[state.instruction_position]); - insn = ByteString::formatted("[{:#02X}] {} {}", (int)alt_id, opcode_id_name(alt_id), opcode_arguments_string(alt_id, alt_data, state.instruction_position, state, alt)); - } - dbgln("{:->{}}| {} -- {}", "", indent * 2, name, insn); - for (auto& child : node.children()) - print_tree(static_cast(*child.value), indent + 1); - }; - - print_tree(trie, 0); - } - - // This is really only worth it if we don't blow up the size by the 2-extra-instruction-per-node scheme, similarly, if no nodes are shared, we're better off not using a tree. - auto tree_cost = (total_nodes - common_hits) * 2; - auto chain_cost = total_bytecode_entries_in_tree + alternatives.size() * 2; - dbgln_if(REGEX_DEBUG, "Total nodes: {}, common hits: {} (tree cost = {}, chain cost = {})", total_nodes, common_hits, tree_cost, chain_cost); - - // Make sure we're not breaking the order requirements (a should be tried before b in a|b) - Queue nodes_to_visit; - nodes_to_visit.enqueue(&trie); - while (!nodes_to_visit.is_empty()) { - auto& node = *nodes_to_visit.dequeue(); - auto& children = node.children(); - for (auto& entry : children) - nodes_to_visit.enqueue(entry.value.ptr()); - // If the children are not sorted right, we've got a problem. - if (children.size() <= 1) - continue; - - size_t max_index = 0; - NodeMetadataEntry const* child_with_max_index = nullptr; - for (auto& entry : children) { - auto& child = *entry.value; - if (child.has_metadata()) { - for (auto& child_entry : child.metadata_value()) { - if (max_index > child_entry.ip.alternative_index) { - // We have a problem, an alternative later in the list is being tried before an earlier one. - // we can't use this trie...unless the first compare in this child is not the same as the one in the entry with max-index - // then there's no overlap and the order doesn't matter anyhow. - if (!has_overlap(*child_with_max_index->first_compare_from_here, *child_entry.first_compare_from_here)) { - // We can use this trie after all. - continue; - } - tree_cost = NumericLimits::max(); - goto exit_useless_loop; - } - max_index = child_entry.ip.alternative_index; - child_with_max_index = &child_entry; - } - } - } - continue; - exit_useless_loop: - break; - } - - if (common_hits == 0 || tree_cost > chain_cost) { - dbgln_if(REGEX_DEBUG, "Choosing sequential alternation layout over trie-based layout"); - // It's better to lay these out as a normal sequence of instructions. - // We can avoid trying alternatives that we know cannot match in certain cases: - // - If the alternative starts with an assertion, we can lift the assertion to the fork op itself (currently only ^). - Vector fork_conditions; - fork_conditions.resize_with_default_value(alternatives.size(), ForkIfCondition::Invalid); - - for (size_t i = 0; i < alternatives.size(); ++i) { - auto& alternative = alternatives[i]; - if (!alternative.is_empty() && static_cast(alternative.flat_data()[0]) == OpCodeId::CheckBegin) - fork_conditions[i] = ForkIfCondition::AtStartOfLine; - } - - Vector jump_op_positions; - Vector jump_sizes; - jump_op_positions.resize(alternatives.size() - 1); - jump_sizes.resize(alternatives.size() - 1); - - for (size_t i = 1; i < alternatives.size(); ++i) { - jump_op_positions[i - 1] = target.size(); - if (fork_conditions[i - 1] != ForkIfCondition::Invalid) { - jump_sizes[i - 1] = 4; - target.empend(static_cast(OpCodeId::ForkIf)); - target.empend(0u); // To be filled later. - target.empend(static_cast(OpCodeId::ForkJump)); - target.empend(static_cast(fork_conditions[i - 1])); - } else { - jump_sizes[i - 1] = 2; - target.empend(static_cast(OpCodeId::ForkJump)); - target.empend(0u); // To be filled later. - } - } - - bool seen_one_empty = false; - - Vector jump_to_end_patch_positions; - jump_to_end_patch_positions.resize_with_default_value(alternatives.size(), NumericLimits::max()); - - for (size_t i = alternatives.size(); i > 0; --i) { - auto& chunk = alternatives[i - 1]; - if (chunk.is_empty()) { - if (seen_one_empty) - continue; - seen_one_empty = true; - } - - if (i < alternatives.size()) { - auto this_block_start = target.size(); - auto position = jump_op_positions[i - 1]; - auto jump_size = jump_sizes[i - 1]; - target[position + 1] = static_cast(this_block_start - position - jump_size); - } - - target.extend(move(chunk)); - target.empend(static_cast(OpCodeId::Jump)); - target.empend(0u); // Jump to the _END label - jump_to_end_patch_positions[i - 1] = target.size() - 1; - } - - auto end_position = target.size(); - for (size_t i = 0; i < alternatives.size(); ++i) { - if (auto& position = jump_to_end_patch_positions[i]; position != NumericLimits::max()) - target[position] = static_cast(end_position - (position + 1)); - } - } else { - dbgln_if(REGEX_DEBUG, "Choosing trie-based alternation layout"); - target.ensure_capacity(total_bytecode_entries_in_tree + common_hits * 6); - - auto node_is = [](Tree const* node, QualifiedIP ip) { - return node->metadata_value().span().first_matching([&](auto& entry) { return entry.ip == ip; }).has_value(); - }; - - struct Patch { - QualifiedIP source_ip; - size_t target_ip; - size_t size_delta { 0 }; - bool done { false }; - }; - Vector patch_locations; - patch_locations.ensure_capacity(total_nodes); - - HashMap>> instruction_positions; - if (has_any_backwards_jump) - MUST(instruction_positions.try_ensure_capacity(alternatives.size())); - - auto ip_mapping_for_alternative = [&](size_t i) -> RedBlackTree& { - return *instruction_positions.ensure(i, [] { - return make>(); - }); - }; - - auto add_patch_point = [&](Tree const* node, size_t target_ip) { - if (!node->has_metadata()) - return; - - patch_locations.append({ node->metadata_value().first().ip, target_ip }); - }; - - Vector nodes_to_visit; - nodes_to_visit.append(&trie); - - // each node: - // node.re - // forkjump child1 - // forkjump child2 - // ... - while (!nodes_to_visit.is_empty()) { - auto const* node = nodes_to_visit.take_last(); - for (auto& patch : patch_locations) { - if (!patch.done && node_is(node, patch.source_ip)) { - auto value = static_cast(target.size() - patch.target_ip - 1 - patch.size_delta); - if (value == 0) - target[patch.target_ip - 1] = static_cast(OpCodeId::Jump); - target[patch.target_ip] = value; - patch.done = true; - } - } - - if (!node->value().individual_spans().is_empty()) { - auto insn_bytes = node->value().individual_spans().first(); - - target.ensure_capacity(target.size() + insn_bytes.size()); - auto insn_ip = target.size(); - target.append(insn_bytes); - - if (has_any_backwards_jump) { - for (auto const& entry : node->metadata_value()) - ip_mapping_for_alternative(entry.ip.alternative_index).insert(entry.ip.instruction_position, insn_ip); - } - - auto insn_id = static_cast(insn_bytes[0]); - auto insn_sz = opcode_size(insn_id, insn_bytes.data(), 0); - - ssize_t jump_offset; - auto is_jump = true; - auto patch_location = insn_ip + 1; - bool should_negate = false; - size_t size_delta = insn_sz - 2; - - switch (insn_id) { - case OpCodeId::Jump: - case OpCodeId::JumpNonEmpty: - case OpCodeId::ForkJump: - case OpCodeId::ForkStay: - case OpCodeId::ForkReplaceJump: - case OpCodeId::ForkReplaceStay: - case OpCodeId::ForkIf: - jump_offset = static_cast(insn_bytes[OpArgs::Jump::offset]); - break; - case OpCodeId::Repeat: - jump_offset = static_cast(0) - static_cast(insn_bytes[OpArgs::Repeat::offset]) - static_cast(insn_sz); - should_negate = true; - break; - default: - is_jump = false; - break; - } - - if (is_jump) { - VERIFY(node->has_metadata()); - if (node->metadata_value().size() > 1) - target[patch_location] = static_cast(0); // Fall through instead. - - auto only_one = node->metadata_value().size() == 1; - auto patch_size = insn_sz - 1; - for (auto const& entry : node->metadata_value()) { - auto& [alternative_index, instruction_position] = entry.ip; - if (!only_one) { - target.append(static_cast(OpCodeId::ForkJump)); - patch_location = target.size(); - should_negate = false; - patch_size = 1; - target.append(static_cast(0)); - } - - auto intended_jump_ip = instruction_position + jump_offset + static_cast(insn_sz); - if (jump_offset < 0) { - VERIFY(has_any_backwards_jump); - // We should've already seen this instruction, so we can just patch it in. - auto& ip_mapping = ip_mapping_for_alternative(alternative_index); - auto target_ip = ip_mapping.find(intended_jump_ip); - if (!target_ip) { - RegexDebug dbg; - size_t x = 0; - for (auto& entry : alternatives) { - warnln("----------- {} ----------", x++); - dbg.print_bytecode(entry); - } - - dbgln("Regex Tree / Unknown backwards jump: {}@{} -> {}", - instruction_position, - alternative_index, - intended_jump_ip); - VERIFY_NOT_REACHED(); - } - ssize_t target_value = *target_ip - patch_location - patch_size; - if (should_negate) - target_value = -target_value - static_cast(insn_sz); - target[patch_location] = static_cast(target_value); - } else { - patch_locations.append({ QualifiedIP { alternative_index, static_cast(intended_jump_ip) }, patch_location, size_delta }); - } - } - } - } - - for (auto const& child : node->children()) { - auto* child_node = static_cast(child.value.ptr()); - target.append(static_cast(OpCodeId::ForkJump)); - add_patch_point(child_node, target.size()); - target.append(static_cast(0)); - nodes_to_visit.append(child_node); - } - } - - for (auto& patch : patch_locations) { - if (patch.done) - continue; - - auto& alternative = alternatives[patch.source_ip.alternative_index]; - if (patch.source_ip.instruction_position >= alternative.size()) { - // This just wants to jump to the end of the alternative, which is fine. - // Patch it to jump to the end of the target instead. - target[patch.target_ip] = static_cast(target.size() - patch.target_ip - 1); - continue; - } - - dbgln("Regex Tree / Unpatched jump: {}@{} -> {}@{}", - patch.source_ip.instruction_position, - patch.source_ip.alternative_index, - patch.target_ip, - target[patch.target_ip]); - VERIFY_NOT_REACHED(); - } - } -} - -enum class LookupTableInsertionOutcome { - Successful, - ReplaceWithAnyChar, - TemporaryInversionNeeded, - PermanentInversionNeeded, - FlushOnInsertion, - FinishFlushOnInsertion, - CannotPlaceInTable, -}; -static LookupTableInsertionOutcome insert_into_lookup_table(RedBlackTree& table, CompareTypeAndValuePair pair) -{ - switch (pair.type) { - case CharacterCompareType::Inverse: - return LookupTableInsertionOutcome::PermanentInversionNeeded; - case CharacterCompareType::TemporaryInverse: - return LookupTableInsertionOutcome::TemporaryInversionNeeded; - case CharacterCompareType::AnyChar: - return LookupTableInsertionOutcome::ReplaceWithAnyChar; - case CharacterCompareType::CharClass: - return LookupTableInsertionOutcome::CannotPlaceInTable; - case CharacterCompareType::Char: - table.insert(pair.value, { (u32)pair.value, (u32)pair.value }); - break; - case CharacterCompareType::CharRange: { - CharRange range { pair.value }; - table.insert(range.from, range); - break; - } - case CharacterCompareType::EndAndOr: - return LookupTableInsertionOutcome::FinishFlushOnInsertion; - case CharacterCompareType::And: - case CharacterCompareType::Subtract: - return LookupTableInsertionOutcome::FlushOnInsertion; - case CharacterCompareType::Reference: - case CharacterCompareType::NamedReference: - case CharacterCompareType::Property: - case CharacterCompareType::GeneralCategory: - case CharacterCompareType::Script: - case CharacterCompareType::ScriptExtension: - case CharacterCompareType::StringSet: - case CharacterCompareType::Or: - return LookupTableInsertionOutcome::CannotPlaceInTable; - case CharacterCompareType::Undefined: - case CharacterCompareType::RangeExpressionDummy: - case CharacterCompareType::String: - case CharacterCompareType::LookupTable: - VERIFY_NOT_REACHED(); - } - - return LookupTableInsertionOutcome::Successful; -} - -void Optimizer::append_character_class(ByteCode& target, Vector&& pairs) -{ - ByteCode arguments; - size_t argument_count = 0; - - if (pairs.size() <= 1) { - for (auto& pair : pairs) { - arguments.append(to_underlying(pair.type)); - if (pair.type != CharacterCompareType::AnyChar - && pair.type != CharacterCompareType::TemporaryInverse - && pair.type != CharacterCompareType::Inverse - && pair.type != CharacterCompareType::And - && pair.type != CharacterCompareType::Or - && pair.type != CharacterCompareType::Subtract - && pair.type != CharacterCompareType::EndAndOr) - arguments.append(pair.value); - ++argument_count; - } - } else { - RedBlackTree table; - RedBlackTree inverted_table; - auto* current_table = &table; - auto* current_inverted_table = &inverted_table; - bool invert_for_next_iteration = false; - bool is_currently_inverted = false; - - auto flush_tables = [&] { - auto merge_overlapping_ranges = [](auto& source) { - Optional active_range; - Vector result; - for (auto& range : source) { - if (!active_range.has_value()) { - active_range = CharRange(range); - continue; - } - CharRange char_range(range); - if (char_range.from <= active_range->to + 1 && char_range.to + 1 >= active_range->from) { - active_range = CharRange { min(char_range.from, active_range->from), max(char_range.to, active_range->to) }; - } else { - result.append(active_range.release_value()); - active_range = char_range; - } - } - if (active_range.has_value()) - result.append(active_range.release_value()); - return result; - }; - - auto append_table = [&](auto& table) { - ++argument_count; - arguments.append(to_underlying(CharacterCompareType::LookupTable)); - auto sensitive_size_index = arguments.size(); - auto insensitive_size_index = sensitive_size_index + 1; - arguments.append(0); - arguments.append(0); - - auto range_data = merge_overlapping_ranges(table); - arguments.extend(range_data); - arguments[sensitive_size_index] = range_data.size(); - - Vector insensitive_data; - for (CharRange range : range_data) { - for (auto expanded : Unicode::expand_range_case_insensitive(range.from, range.to)) - insensitive_data.append(CharRange { expanded.from, expanded.to }); - } - quick_sort(insensitive_data, [](CharRange a, CharRange b) { return a.from < b.from; }); - - auto merged_data = merge_overlapping_ranges(insensitive_data); - arguments.extend(merged_data); - arguments[insensitive_size_index] = merged_data.size(); - }; - - auto contains_regular_table = !table.is_empty(); - auto contains_inverted_table = !inverted_table.is_empty(); - if (contains_regular_table) - append_table(table); - - if (contains_inverted_table) { - ++argument_count; - arguments.append(to_underlying(CharacterCompareType::TemporaryInverse)); - append_table(inverted_table); - } - - table.clear(); - inverted_table.clear(); - }; - - auto flush_on_every_insertion = false; - for (auto& value : pairs) { - auto should_invert_after_this_iteration = invert_for_next_iteration; - invert_for_next_iteration = false; - - auto insertion_result = insert_into_lookup_table(*current_table, value); - switch (insertion_result) { - case LookupTableInsertionOutcome::Successful: - if (flush_on_every_insertion) - flush_tables(); - break; - case LookupTableInsertionOutcome::ReplaceWithAnyChar: { - table.clear(); - inverted_table.clear(); - arguments.append(to_underlying(CharacterCompareType::AnyChar)); - ++argument_count; - break; - } - case LookupTableInsertionOutcome::TemporaryInversionNeeded: - swap(current_table, current_inverted_table); - invert_for_next_iteration = true; - is_currently_inverted = !is_currently_inverted; - break; - case LookupTableInsertionOutcome::PermanentInversionNeeded: - flush_tables(); - arguments.append(to_underlying(CharacterCompareType::Inverse)); - ++argument_count; - break; - case LookupTableInsertionOutcome::FlushOnInsertion: - case LookupTableInsertionOutcome::FinishFlushOnInsertion: - flush_tables(); - flush_on_every_insertion = insertion_result == LookupTableInsertionOutcome::FlushOnInsertion; - [[fallthrough]]; - case LookupTableInsertionOutcome::CannotPlaceInTable: - if (is_currently_inverted) { - arguments.append(to_underlying(CharacterCompareType::TemporaryInverse)); - ++argument_count; - } - arguments.append(to_underlying(value.type)); - - if (value.type != CharacterCompareType::AnyChar - && value.type != CharacterCompareType::TemporaryInverse - && value.type != CharacterCompareType::Inverse - && value.type != CharacterCompareType::And - && value.type != CharacterCompareType::Or - && value.type != CharacterCompareType::Subtract - && value.type != CharacterCompareType::EndAndOr) - arguments.append(value.value); - ++argument_count; - break; - } - - if (should_invert_after_this_iteration) { - swap(current_table, current_inverted_table); - is_currently_inverted = !is_currently_inverted; - } - } - - flush_tables(); - } - - target.empend(static_cast(OpCodeId::Compare)); - target.empend(argument_count); // number of arguments - target.empend(arguments.size()); // size of arguments - target.extend(move(arguments)); -} - -template void Regex::run_optimization_passes(); -template void Regex::run_optimization_passes(); -template void Regex::run_optimization_passes(); - -} diff --git a/Libraries/LibRegex/RegexOptions.h b/Libraries/LibRegex/RegexOptions.h deleted file mode 100644 index ae7e71839b..0000000000 --- a/Libraries/LibRegex/RegexOptions.h +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include "RegexDefs.h" -#include -#include - -namespace regex { - -using FlagsUnderlyingType = u32; - -enum class AllFlags { - Default = 0, - Global = __Regex_Global, // All matches (don't return after first match) - Insensitive = __Regex_Insensitive, // Case insensitive match (ignores case of [a-zA-Z]) - Ungreedy = __Regex_Ungreedy, // The match becomes lazy by default. Now a ? following a quantifier makes it greedy - Unicode = __Regex_Unicode, // Enable all unicode features and interpret all unicode escape sequences as such - Extended = __Regex_Extended, // Ignore whitespaces. Spaces and text after a # in the pattern are ignored - Extra = __Regex_Extra, // Disallow meaningless escapes. A \ followed by a letter with no special meaning is faulted - MatchNotBeginOfLine = __Regex_MatchNotBeginOfLine, // Pattern is not forced to ^ -> search in whole string! - MatchNotEndOfLine = __Regex_MatchNotEndOfLine, // Don't Force the dollar sign, $, to always match end of the string, instead of end of the line. This option is ignored if the Multiline-flag is set - SkipSubExprResults = __Regex_SkipSubExprResults, // Do not return sub expressions in the result - SingleLine = __Regex_SingleLine, // Dot matches newline characters - Sticky = __Regex_Sticky, // Force the pattern to only match consecutive matches from where the previous match ended. - Multiline = __Regex_Multiline, // Handle newline characters. Match each line, one by one. - SingleMatch = __Regex_SingleMatch, // Stop after acquiring a single match. - UnicodeSets = __Regex_UnicodeSets, // Only for ECMA262, Allow set operations in character classes. - Internal_Stateful = __Regex_Internal_Stateful, // Make global matches match one result at a time, and further match() calls on the same instance continue where the previous one left off. - Internal_BrowserExtended = __Regex_Internal_BrowserExtended, // Only for ECMA262, Enable the behaviors defined in section B.1.4. of the ECMA262 spec. - Internal_ConsiderNewline = __Regex_Internal_ConsiderNewline, // Only for ECMA262, Allow multiline matches to consider newlines as line boundaries. - Internal_ECMA262DotSemantics = __Regex_Internal_ECMA262DotSemantics, // Use ECMA262 dot semantics: disallow matching CR/LF/LS/PS instead of just CR. - Last = Internal_BrowserExtended, -}; - -enum class PosixFlags : FlagsUnderlyingType { - Default = 0, - Global = (FlagsUnderlyingType)AllFlags::Global, - Insensitive = (FlagsUnderlyingType)AllFlags::Insensitive, - Ungreedy = (FlagsUnderlyingType)AllFlags::Ungreedy, - Unicode = (FlagsUnderlyingType)AllFlags::Unicode, - Extended = (FlagsUnderlyingType)AllFlags::Extended, - Extra = (FlagsUnderlyingType)AllFlags::Extra, - MatchNotBeginOfLine = (FlagsUnderlyingType)AllFlags::MatchNotBeginOfLine, - MatchNotEndOfLine = (FlagsUnderlyingType)AllFlags::MatchNotEndOfLine, - SkipSubExprResults = (FlagsUnderlyingType)AllFlags::SkipSubExprResults, - Multiline = (FlagsUnderlyingType)AllFlags::Multiline, - SingleMatch = (FlagsUnderlyingType)AllFlags::SingleMatch, -}; - -enum class ECMAScriptFlags : FlagsUnderlyingType { - Default = (FlagsUnderlyingType)AllFlags::Internal_ECMA262DotSemantics, - Global = (FlagsUnderlyingType)AllFlags::Global | (FlagsUnderlyingType)AllFlags::Internal_Stateful, // Note: ECMAScript "Global" creates a stateful regex. - Insensitive = (FlagsUnderlyingType)AllFlags::Insensitive, - Ungreedy = (FlagsUnderlyingType)AllFlags::Ungreedy, - Unicode = (FlagsUnderlyingType)AllFlags::Unicode, - Extended = (FlagsUnderlyingType)AllFlags::Extended, - Extra = (FlagsUnderlyingType)AllFlags::Extra, - SingleLine = (FlagsUnderlyingType)AllFlags::SingleLine, - Sticky = (FlagsUnderlyingType)AllFlags::Sticky, - Multiline = (FlagsUnderlyingType)AllFlags::Multiline, - UnicodeSets = (FlagsUnderlyingType)AllFlags::UnicodeSets, - BrowserExtended = (FlagsUnderlyingType)AllFlags::Internal_BrowserExtended, -}; - -template -class RegexOptions { -public: - using FlagsType = T; - - RegexOptions() = default; - - constexpr RegexOptions(T flags) - : m_flags(static_cast(to_underlying(flags) | to_underlying(T::Default))) - { - } - - template - constexpr RegexOptions(RegexOptions other) - : RegexOptions(static_cast(to_underlying(other.value()))) - { - } - - operator bool() const { return !!*this; } - bool operator!() const { return (FlagsUnderlyingType)m_flags == 0; } - - constexpr RegexOptions operator|(T flag) const { return RegexOptions { (T)((FlagsUnderlyingType)m_flags | (FlagsUnderlyingType)flag) }; } - constexpr RegexOptions operator&(T flag) const { return RegexOptions { (T)((FlagsUnderlyingType)m_flags & (FlagsUnderlyingType)flag) }; } - - constexpr RegexOptions& operator|=(T flag) - { - m_flags = (T)((FlagsUnderlyingType)m_flags | (FlagsUnderlyingType)flag); - return *this; - } - - constexpr RegexOptions& operator&=(T flag) - { - m_flags = (T)((FlagsUnderlyingType)m_flags & (FlagsUnderlyingType)flag); - return *this; - } - - void reset_flags() { m_flags = (T)0; } - void reset_flag(T flag) { m_flags = (T)((FlagsUnderlyingType)m_flags & ~(FlagsUnderlyingType)flag); } - void set_flag(T flag) { *this |= flag; } - bool has_flag_set(T flag) const { return (FlagsUnderlyingType)flag == ((FlagsUnderlyingType)m_flags & (FlagsUnderlyingType)flag); } - constexpr T value() const { return m_flags; } - -private: - T m_flags { T::Default }; -}; - -template -constexpr RegexOptions operator|(T lhs, T rhs) -{ - return RegexOptions { lhs } |= rhs; -} - -template -constexpr RegexOptions operator&(T lhs, T rhs) -{ - return RegexOptions { lhs } &= rhs; -} - -template -constexpr T operator~(T flag) -{ - return (T) ~((FlagsUnderlyingType)flag); -} - -using AllOptions = RegexOptions; -using ECMAScriptOptions = RegexOptions; -using PosixOptions = RegexOptions; - -} - -using regex::ECMAScriptFlags; -using regex::ECMAScriptOptions; -using regex::PosixFlags; -using regex::PosixOptions; diff --git a/Libraries/LibRegex/RegexParser.cpp b/Libraries/LibRegex/RegexParser.cpp deleted file mode 100644 index 7189f7b727..0000000000 --- a/Libraries/LibRegex/RegexParser.cpp +++ /dev/null @@ -1,3147 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * Copyright (c) 2020-2021, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include "RegexParser.h" -#include "RegexDebug.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace regex { - -static constexpr size_t s_maximum_repetition_count = 1024 * 1024; -static constexpr u64 s_ecma262_maximum_repetition_count = (1ull << 31) - 1; -static constexpr auto s_alphabetic_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"sv; -static constexpr auto s_decimal_characters = "0123456789"sv; - -static constexpr StringView identity_escape_characters(bool unicode, bool browser_extended) -{ - if (unicode) - return "^$\\.*+?()[]{}|/"sv; - if (browser_extended) - return "^$\\.*+?()[|"sv; - return "^$\\.*+?()[]{}|"sv; -} - -static bool has_multiple_code_points(String const& str) -{ - Utf8View utf8_view { str.bytes_as_string_view() }; - auto it = utf8_view.begin(); - if (it == utf8_view.end()) - return false; - return ++it != utf8_view.end(); -} - -ALWAYS_INLINE bool Parser::set_error(Error error) -{ - if (m_parser_state.error == Error::NoError) { - m_parser_state.error = error; - m_parser_state.error_token = m_parser_state.current_token; - } - return false; // always return false, that eases the API usage (return set_error(...)) :^) -} - -ALWAYS_INLINE bool Parser::done() const -{ - return match(TokenType::Eof); -} - -ALWAYS_INLINE bool Parser::match(TokenType type) const -{ - return m_parser_state.current_token.type() == type; -} - -ALWAYS_INLINE bool Parser::match(char ch) const -{ - return m_parser_state.current_token.type() == TokenType::Char && m_parser_state.current_token.value().length() == 1 && m_parser_state.current_token.value()[0] == ch; -} - -ALWAYS_INLINE Token Parser::consume() -{ - auto old_token = m_parser_state.current_token; - m_parser_state.current_token = m_parser_state.lexer.next(); - return old_token; -} - -ALWAYS_INLINE Token Parser::consume(TokenType type, Error error) -{ - if (m_parser_state.current_token.type() != type) { - set_error(error); - dbgln_if(REGEX_DEBUG, "[PARSER] Error: Unexpected token {}. Expected: {}", m_parser_state.current_token.name(), Token::name(type)); - } - return consume(); -} - -ALWAYS_INLINE bool Parser::consume(ByteString const& str) -{ - size_t potentially_go_back { 1 }; - for (auto ch : str) { - if (match(TokenType::Char)) { - if (m_parser_state.current_token.value()[0] != ch) { - m_parser_state.lexer.back(potentially_go_back); - m_parser_state.current_token = m_parser_state.lexer.next(); - return false; - } - } else { - m_parser_state.lexer.back(potentially_go_back); - m_parser_state.current_token = m_parser_state.lexer.next(); - return false; - } - consume(TokenType::Char, Error::NoError); - ++potentially_go_back; - } - return true; -} - -ALWAYS_INLINE Optional Parser::consume_escaped_code_point(bool unicode) -{ - if (match(TokenType::LeftCurly) && !unicode) { - // In non-Unicode mode, this should be parsed as a repetition symbol (repeating the 'u'). - return static_cast('u'); - } - - m_parser_state.lexer.retreat(2 + !done()); // Go back to just before '\u' (+1 char, because we will have consumed an extra character) - auto position_before_escape = m_parser_state.lexer.tell(); - - if (auto code_point_or_error = m_parser_state.lexer.consume_escaped_code_point(unicode); !code_point_or_error.is_error()) { - m_parser_state.current_token = m_parser_state.lexer.next(); - return code_point_or_error.value(); - } - - if (!unicode) { - // '\u' is allowed in non-unicode mode, just matches 'u'. - m_parser_state.lexer.retreat(m_parser_state.lexer.tell() - (position_before_escape + 2)); - m_parser_state.current_token = m_parser_state.lexer.next(); - return static_cast('u'); - } - - set_error(Error::InvalidPattern); - return {}; -} - -ALWAYS_INLINE bool Parser::try_skip(StringView str) -{ - if (!str.starts_with(m_parser_state.current_token.value())) - return false; - str = str.substring_view(m_parser_state.current_token.value().length(), str.length() - m_parser_state.current_token.value().length()); - - size_t potentially_go_back { 0 }; - for (auto ch : str) { - if (!m_parser_state.lexer.consume_specific(ch)) { - m_parser_state.lexer.back(potentially_go_back); - return false; - } - ++potentially_go_back; - } - - m_parser_state.current_token = m_parser_state.lexer.next(); - return true; -} - -ALWAYS_INLINE bool Parser::lookahead_any(StringView str) -{ - return AK::any_of(str, [this](auto ch) { return match(ch); }); -} - -ALWAYS_INLINE unsigned char Parser::skip() -{ - unsigned char ch; - if (m_parser_state.current_token.value().length() == 1) { - ch = m_parser_state.current_token.value()[0]; - } else { - m_parser_state.lexer.back(m_parser_state.current_token.value().length()); - ch = m_parser_state.lexer.consume(); - } - - m_parser_state.current_token = m_parser_state.lexer.next(); - return ch; -} - -ALWAYS_INLINE void Parser::back(size_t count) -{ - m_parser_state.lexer.back(count); - m_parser_state.current_token = m_parser_state.lexer.next(); -} - -ALWAYS_INLINE void Parser::reset() -{ - m_parser_state.bytecode.clear(); - m_parser_state.lexer.reset(); - m_parser_state.current_token = m_parser_state.lexer.next(); - m_parser_state.error = Error::NoError; - m_parser_state.error_token = { TokenType::Eof, 0, {} }; - m_parser_state.capture_group_minimum_lengths.clear(); - m_parser_state.optional_capture_groups.clear(); - m_parser_state.capture_groups_count = 0; - m_parser_state.named_capture_groups_count = 0; - m_parser_state.named_capture_groups.clear(); - m_parser_state.unresolved_named_references.clear(); -} - -Parser::Result Parser::parse(Optional regex_options) -{ - ByteCode::reset_checkpoint_serial_id(); - - reset(); - if (regex_options.has_value()) - m_parser_state.regex_options = regex_options.value(); - if (parse_internal(m_parser_state.bytecode, m_parser_state.match_length_minimum)) { - consume(TokenType::Eof, Error::InvalidPattern); - if (!resolve_forward_named_references()) - set_error(Error::InvalidNameForCaptureGroup); - } else { - set_error(Error::InvalidPattern); - } - - auto capture_groups = m_parser_state.named_capture_groups.keys(); - - dbgln_if(REGEX_DEBUG, "[PARSER] Produced bytecode with {} entries (opcodes + arguments)", m_parser_state.bytecode.size()); - return { - move(m_parser_state.bytecode), - move(m_parser_state.capture_groups_count), - move(m_parser_state.named_capture_groups_count), - move(m_parser_state.match_length_minimum), - move(m_parser_state.error), - move(m_parser_state.error_token), - move(capture_groups), - m_parser_state.regex_options, - }; -} - -ALWAYS_INLINE bool Parser::match_ordinary_characters() -{ - // NOTE: This method must not be called during bracket and repetition parsing! - // FIXME: Add assertion for that? - auto type = m_parser_state.current_token.type(); - return ((type == TokenType::Char && m_parser_state.current_token.value() != "\\"sv) // NOTE: Backslash will only be matched as 'char' if it does not form a valid escape. - || type == TokenType::Comma - || type == TokenType::Slash - || type == TokenType::EqualSign - || type == TokenType::HyphenMinus - || type == TokenType::Colon); -} - -// ============================= -// Abstract Posix Parser -// ============================= - -ALWAYS_INLINE bool AbstractPosixParser::parse_bracket_expression(Vector& values, size_t& match_length_minimum) -{ - for (; !done();) { - if (match(TokenType::HyphenMinus)) { - consume(); - - if (values.is_empty() || (values.size() == 1 && values.last().type == CharacterCompareType::Inverse)) { - // first in the bracket expression - values.append({ CharacterCompareType::Char, (ByteCodeValueType)'-' }); - } else if (match(TokenType::RightBracket)) { - // Last in the bracket expression - values.append({ CharacterCompareType::Char, (ByteCodeValueType)'-' }); - } else if (values.last().type == CharacterCompareType::Char) { - values.append({ CharacterCompareType::RangeExpressionDummy, 0 }); - - if (done()) - return set_error(Error::MismatchingBracket); - - if (match(TokenType::HyphenMinus)) { - consume(); - // Valid range, add ordinary character - values.append({ CharacterCompareType::Char, (ByteCodeValueType)'-' }); - } - } else { - return set_error(Error::InvalidRange); - } - - } else if (match(TokenType::Circumflex)) { - auto t = consume(); - - if (values.is_empty()) - values.append({ CharacterCompareType::Inverse, 0 }); - else - values.append({ CharacterCompareType::Char, (ByteCodeValueType)*t.value().characters_without_null_termination() }); - - } else if (match(TokenType::LeftBracket)) { - consume(); - - if (match(TokenType::Period)) { - consume(); - - // FIXME: Parse collating element, this is needed when we have locale support - // This could have impact on length parameter, I guess. - set_error(Error::InvalidCollationElement); - - consume(TokenType::Period, Error::InvalidCollationElement); - consume(TokenType::RightBracket, Error::MismatchingBracket); - - } else if (match(TokenType::EqualSign)) { - consume(); - // FIXME: Parse collating element, this is needed when we have locale support - // This could have impact on length parameter, I guess. - set_error(Error::InvalidCollationElement); - - consume(TokenType::EqualSign, Error::InvalidCollationElement); - consume(TokenType::RightBracket, Error::MismatchingBracket); - - } else if (match(TokenType::Colon)) { - consume(); - - CharClass ch_class; - // parse character class - if (match(TokenType::Char)) { - if (consume("alnum")) - ch_class = CharClass::Alnum; - else if (consume("alpha")) - ch_class = CharClass::Alpha; - else if (consume("blank")) - ch_class = CharClass::Blank; - else if (consume("cntrl")) - ch_class = CharClass::Cntrl; - else if (consume("digit")) - ch_class = CharClass::Digit; - else if (consume("graph")) - ch_class = CharClass::Graph; - else if (consume("lower")) - ch_class = CharClass::Lower; - else if (consume("print")) - ch_class = CharClass::Print; - else if (consume("punct")) - ch_class = CharClass::Punct; - else if (consume("space")) - ch_class = CharClass::Space; - else if (consume("upper")) - ch_class = CharClass::Upper; - else if (consume("xdigit")) - ch_class = CharClass::Xdigit; - else - return set_error(Error::InvalidCharacterClass); - - values.append({ CharacterCompareType::CharClass, (ByteCodeValueType)ch_class }); - - } else - return set_error(Error::InvalidCharacterClass); - - // FIXME: we do not support locale specific character classes until locales are implemented - - consume(TokenType::Colon, Error::InvalidCharacterClass); - consume(TokenType::RightBracket, Error::MismatchingBracket); - } else { - return set_error(Error::MismatchingBracket); - } - - } else if (match(TokenType::RightBracket)) { - - if (values.is_empty() || (values.size() == 1 && values.last().type == CharacterCompareType::Inverse)) { - // handle bracket as ordinary character - values.append({ CharacterCompareType::Char, (ByteCodeValueType)*consume().value().characters_without_null_termination() }); - } else { - // closing bracket expression - break; - } - } else { - values.append({ CharacterCompareType::Char, (ByteCodeValueType)skip() }); - } - - // check if range expression has to be completed... - if (values.size() >= 3 && values.at(values.size() - 2).type == CharacterCompareType::RangeExpressionDummy) { - if (values.last().type != CharacterCompareType::Char) - return set_error(Error::InvalidRange); - - auto value2 = values.take_last(); - values.take_last(); // RangeExpressionDummy - auto value1 = values.take_last(); - - values.append({ CharacterCompareType::CharRange, static_cast(CharRange { (u32)value1.value, (u32)value2.value }) }); - } - } - - if (!values.is_empty()) { - match_length_minimum = 1; - if (values.first().type == CharacterCompareType::Inverse) - match_length_minimum = 0; - } - - return true; -} - -// ============================= -// PosixBasic Parser -// ============================= - -bool PosixBasicParser::parse_internal(ByteCode& stack, size_t& match_length_minimum) -{ - return parse_root(stack, match_length_minimum); -} - -bool PosixBasicParser::parse_root(ByteCode& bytecode, size_t& match_length_minimum) -{ - // basic_reg_exp : L_ANCHOR? RE_expression R_ANCHOR? - if (match(TokenType::Circumflex)) { - consume(); - bytecode.empend((ByteCodeValueType)OpCodeId::CheckBegin); - } - - if (!parse_re_expression(bytecode, match_length_minimum)) - return false; - - if (match(TokenType::Dollar)) { - consume(); - bytecode.empend((ByteCodeValueType)OpCodeId::CheckEnd); - } - - return !has_error(); -} - -bool PosixBasicParser::parse_re_expression(ByteCode& bytecode, size_t& match_length_minimum) -{ - // RE_expression : RE_expression? simple_RE - while (!done()) { - if (!parse_simple_re(bytecode, match_length_minimum)) - break; - } - - return !has_error(); -} - -bool PosixBasicParser::parse_simple_re(ByteCode& bytecode, size_t& match_length_minimum) -{ - // simple_RE : nondupl_RE RE_dupl_symbol? - ByteCode simple_re_bytecode; - size_t re_match_length_minimum = 0; - if (!parse_nonduplicating_re(simple_re_bytecode, re_match_length_minimum)) - return false; - - // RE_dupl_symbol : '*' | Back_open_brace DUP_COUNT (',' DUP_COUNT?)? Back_close_brace - if (match(TokenType::Asterisk)) { - consume(); - ByteCode::transform_bytecode_repetition_any(simple_re_bytecode, true); - } else if (try_skip("\\{"sv)) { - auto read_number = [&]() -> Optional { - if (!match(TokenType::Char)) - return {}; - size_t value = 0; - while (match(TokenType::Char)) { - auto c = m_parser_state.current_token.value().substring_view(0, 1); - auto c_value = c.to_number(); - if (!c_value.has_value()) - break; - value *= 10; - value += *c_value; - consume(); - } - return value; - }; - - size_t min_limit; - Optional max_limit; - - if (auto limit = read_number(); !limit.has_value()) - return set_error(Error::InvalidRepetitionMarker); - else - min_limit = *limit; - - if (match(TokenType::Comma)) { - consume(); - max_limit = read_number(); - } - - if (!try_skip("\\}"sv)) - return set_error(Error::MismatchingBrace); - - if (max_limit.value_or(min_limit) < min_limit) - return set_error(Error::InvalidBraceContent); - - if (min_limit > s_maximum_repetition_count || (max_limit.has_value() && *max_limit > s_maximum_repetition_count)) - return set_error(Error::InvalidBraceContent); - - auto min_repetition_mark_id = m_parser_state.repetition_mark_count++; - auto max_repetition_mark_id = m_parser_state.repetition_mark_count++; - ByteCode::transform_bytecode_repetition_min_max(simple_re_bytecode, min_limit, max_limit, min_repetition_mark_id, max_repetition_mark_id, true); - match_length_minimum += re_match_length_minimum * min_limit; - } else { - match_length_minimum += re_match_length_minimum; - } - - bytecode.extend(move(simple_re_bytecode)); - return true; -} - -bool PosixBasicParser::parse_nonduplicating_re(ByteCode& bytecode, size_t& match_length_minimum) -{ - // nondupl_RE : one_char_or_coll_elem_RE | Back_open_paren RE_expression Back_close_paren | BACKREF - if (try_skip("\\("sv)) { - TemporaryChange change { m_current_capture_group_depth, m_current_capture_group_depth + 1 }; - // Max number of addressable capture groups is 10, let's just be lenient - // and accept 20; anything past that is probably a silly pattern anyway. - if (m_current_capture_group_depth > 20) - return set_error(Error::InvalidPattern); - ByteCode capture_bytecode; - size_t capture_length_minimum = 0; - auto capture_group_index = ++m_parser_state.capture_groups_count; - - if (!parse_re_expression(capture_bytecode, capture_length_minimum)) - return false; - - if (!try_skip("\\)"sv)) - return set_error(Error::MismatchingParen); - - match_length_minimum += capture_length_minimum; - if (capture_group_index <= number_of_addressable_capture_groups) { - m_capture_group_minimum_lengths[capture_group_index - 1] = capture_length_minimum; - m_capture_group_seen[capture_group_index - 1] = true; - bytecode.insert_bytecode_group_capture_left(capture_group_index); - } - - bytecode.extend(capture_bytecode); - - if (capture_group_index <= number_of_addressable_capture_groups) - bytecode.insert_bytecode_group_capture_right(capture_group_index); - return true; - } - - for (size_t i = 1; i < 10; ++i) { - char backref_name[2] { '\\', '0' }; - backref_name[1] += i; - if (try_skip({ backref_name, 2 })) { - if (!m_capture_group_seen[i - 1]) - return set_error(Error::InvalidNumber); - bytecode.insert_bytecode_compare_values({ { CharacterCompareType::Reference, (ByteCodeValueType)i } }); - return true; - } - } - - return parse_one_char_or_collation_element(bytecode, match_length_minimum); -} - -bool PosixBasicParser::parse_one_char_or_collation_element(ByteCode& bytecode, size_t& match_length_minimum) -{ - // one_char_or_coll_elem_RE : ORD_CHAR | QUOTED_CHAR | '.' | bracket_expression - if (match(TokenType::Period)) { - consume(); - bytecode.insert_bytecode_compare_values({ { CharacterCompareType::AnyChar, 0 } }); - match_length_minimum += 1; - return true; - } - - // Dollars are special if at the end of a pattern. - if (match(TokenType::Dollar)) { - consume(); - - // If we are at the end of a pattern, emit an end check instruction. - if (match(TokenType::Eof)) { - bytecode.empend((ByteCodeValueType)OpCodeId::CheckEnd); - return true; - } - - // We are not at the end of the string, so we should roll back and continue as normal. - back(2); - } - - if (match(TokenType::Char)) { - auto ch = consume().value()[0]; - if (ch == '\\') { - if (m_parser_state.regex_options.has_flag_set(AllFlags::Extra)) - return set_error(Error::InvalidPattern); - - // This was \, the spec does not define any behaviour for this but glibc regex ignores it - and so do we. - return true; - } - - bytecode.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)ch } }); - match_length_minimum += 1; - return true; - } - - // None of these are special in BRE. - if (match(TokenType::Questionmark) || match(TokenType::RightParen) || match(TokenType::HyphenMinus) - || match(TokenType::Circumflex) || match(TokenType::RightCurly) || match(TokenType::Comma) || match(TokenType::Colon) - || match(TokenType::Dollar) || match(TokenType::EqualSign) || match(TokenType::LeftCurly) || match(TokenType::LeftParen) - || match(TokenType::Pipe) || match(TokenType::Slash) || match(TokenType::RightBracket) || match(TokenType::RightParen)) { - - auto ch = consume().value()[0]; - bytecode.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)ch } }); - match_length_minimum += 1; - return true; - } - - if (match(TokenType::EscapeSequence)) { - if (m_parser_state.current_token.value().is_one_of("\\)"sv, "\\}"sv, "\\("sv, "\\{"sv)) - return false; - auto ch = consume().value()[1]; - bytecode.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)ch } }); - match_length_minimum += 1; - return true; - } - - Vector values; - size_t bracket_minimum_length = 0; - - if (match(TokenType::LeftBracket)) { - consume(); - if (!AbstractPosixParser::parse_bracket_expression(values, bracket_minimum_length)) - return false; - - consume(TokenType::RightBracket, Error::MismatchingBracket); - - if (!has_error()) - bytecode.insert_bytecode_compare_values(move(values)); - match_length_minimum += bracket_minimum_length; - return !has_error(); - } - - return set_error(Error::InvalidPattern); -} - -// ============================= -// PosixExtended Parser -// ============================= - -bool PosixExtendedParser::parse_internal(ByteCode& stack, size_t& match_length_minimum) -{ - return parse_root(stack, match_length_minimum); -} - -ALWAYS_INLINE bool PosixExtendedParser::match_repetition_symbol() -{ - auto type = m_parser_state.current_token.type(); - return (type == TokenType::Asterisk - || type == TokenType::Plus - || type == TokenType::Questionmark - || type == TokenType::LeftCurly); -} - -ALWAYS_INLINE bool PosixExtendedParser::parse_repetition_symbol(ByteCode& bytecode_to_repeat, size_t& match_length_minimum) -{ - if (match(TokenType::LeftCurly)) { - consume(); - - StringBuilder number_builder; - - while (match(TokenType::Char)) { - number_builder.append(consume().value()); - } - - auto maybe_minimum = number_builder.to_byte_string().to_number(); - if (!maybe_minimum.has_value()) - return set_error(Error::InvalidBraceContent); - - auto minimum = maybe_minimum.value(); - match_length_minimum *= minimum; - - if (minimum > s_maximum_repetition_count) - return set_error(Error::InvalidBraceContent); - - if (match(TokenType::Comma)) { - consume(); - } else { - auto repetition_mark_id = m_parser_state.repetition_mark_count++; - - ByteCode bytecode; - bytecode.insert_bytecode_repetition_n(bytecode_to_repeat, minimum, repetition_mark_id); - bytecode_to_repeat = move(bytecode); - - consume(TokenType::RightCurly, Error::MismatchingBrace); - return !has_error(); - } - - Optional maybe_maximum {}; - number_builder.clear(); - while (match(TokenType::Char)) { - number_builder.append(consume().value()); - } - if (!number_builder.is_empty()) { - auto value = number_builder.to_byte_string().to_number(); - if (!value.has_value() || minimum > value.value() || *value > s_maximum_repetition_count) - return set_error(Error::InvalidBraceContent); - - maybe_maximum = value.value(); - } - - auto min_repetition_mark_id = m_parser_state.repetition_mark_count++; - auto max_repetition_mark_id = m_parser_state.repetition_mark_count++; - ByteCode::transform_bytecode_repetition_min_max(bytecode_to_repeat, minimum, maybe_maximum, min_repetition_mark_id, max_repetition_mark_id); - - consume(TokenType::RightCurly, Error::MismatchingBrace); - return !has_error(); - } - if (match(TokenType::Plus)) { - consume(); - - bool nongreedy = match(TokenType::Questionmark); - if (nongreedy) - consume(); - - // Note: don't touch match_length_minimum, it's already correct - ByteCode::transform_bytecode_repetition_min_one(bytecode_to_repeat, !nongreedy); - return !has_error(); - } - if (match(TokenType::Asterisk)) { - consume(); - match_length_minimum = 0; - - bool nongreedy = match(TokenType::Questionmark); - if (nongreedy) - consume(); - - ByteCode::transform_bytecode_repetition_any(bytecode_to_repeat, !nongreedy); - - return !has_error(); - } - if (match(TokenType::Questionmark)) { - consume(); - match_length_minimum = 0; - - bool nongreedy = match(TokenType::Questionmark); - if (nongreedy) - consume(); - - ByteCode::transform_bytecode_repetition_zero_or_one(bytecode_to_repeat, !nongreedy); - return !has_error(); - } - - return false; -} - -ALWAYS_INLINE bool PosixExtendedParser::parse_bracket_expression(ByteCode& stack, size_t& match_length_minimum) -{ - Vector values; - if (!AbstractPosixParser::parse_bracket_expression(values, match_length_minimum)) - return false; - - if (!has_error()) - stack.insert_bytecode_compare_values(move(values)); - - return !has_error(); -} - -ALWAYS_INLINE bool PosixExtendedParser::parse_sub_expression(ByteCode& stack, size_t& match_length_minimum) -{ - ByteCode bytecode; - size_t length = 0; - bool should_parse_repetition_symbol { false }; - - for (;;) { - if (match_ordinary_characters()) { - Token start_token = m_parser_state.current_token; - Token last_token = m_parser_state.current_token; - for (;;) { - if (!match_ordinary_characters()) - break; - ++length; - last_token = consume(); - } - - if (length > 1) { - // last character is inserted into 'bytecode' for duplication symbol handling - auto new_length = length - (match_repetition_symbol() ? 1 : 0); - auto substring = m_parser_state.lexer.source().substring_view_starting_from_substring(start_token.value()).substring_view(0, new_length); - stack.insert_bytecode_compare_string(Utf16FlyString::from_utf8(substring)); - } - - if ((match_repetition_symbol() && length > 1) || length == 1) // Create own compare opcode for last character before duplication symbol - bytecode.insert_bytecode_compare_values({ { CharacterCompareType::Char, (ByteCodeValueType)last_token.value().characters_without_null_termination()[0] } }); - - should_parse_repetition_symbol = true; - break; - } - - if (m_parser_state.current_token.value() == "\\"sv) { - if (m_parser_state.regex_options.has_flag_set(AllFlags::Extra)) - return set_error(Error::InvalidPattern); - - consume(); - continue; - } - - if (match_repetition_symbol()) - return set_error(Error::InvalidRepetitionMarker); - - if (match(TokenType::Period)) { - length = 1; - consume(); - bytecode.insert_bytecode_compare_values({ { CharacterCompareType::AnyChar, 0 } }); - should_parse_repetition_symbol = true; - break; - } - - if (match(TokenType::EscapeSequence)) { - length = 1; - Token t = consume(); - dbgln_if(REGEX_DEBUG, "[PARSER] EscapeSequence with substring {}", t.value()); - - bytecode.insert_bytecode_compare_values({ { CharacterCompareType::Char, (u32)t.value().characters_without_null_termination()[1] } }); - should_parse_repetition_symbol = true; - break; - } - - if (match(TokenType::LeftBracket)) { - consume(); - - ByteCode sub_ops; - if (!parse_bracket_expression(sub_ops, length) || !sub_ops.size()) - return set_error(Error::InvalidBracketContent); - - bytecode.extend(move(sub_ops)); - - consume(TokenType::RightBracket, Error::MismatchingBracket); - should_parse_repetition_symbol = true; - break; - } - - if (match(TokenType::RightBracket)) { - return set_error(Error::MismatchingBracket); - } - - if (match(TokenType::RightCurly)) { - return set_error(Error::MismatchingBrace); - } - - if (match(TokenType::Circumflex)) { - consume(); - bytecode.empend((ByteCodeValueType)OpCodeId::CheckBegin); - break; - } - - if (match(TokenType::Dollar)) { - consume(); - bytecode.empend((ByteCodeValueType)OpCodeId::CheckEnd); - break; - } - - if (match(TokenType::RightParen)) - return false; - - if (match(TokenType::LeftParen)) { - enum GroupMode { - Normal, - Lookahead, - NegativeLookahead, - Lookbehind, - NegativeLookbehind, - } group_mode { Normal }; - consume(); - Optional capture_group_name; - bool prevent_capture_group = false; - if (match(TokenType::Questionmark)) { - consume(); - - if (match(TokenType::Colon)) { - consume(); - prevent_capture_group = true; - } else if (consume("<")) { // named capturing group - - Token start_token = m_parser_state.current_token; - Token last_token = m_parser_state.current_token; - size_t capture_group_name_length = 0; - for (;;) { - if (!match_ordinary_characters()) - return set_error(Error::InvalidNameForCaptureGroup); - if (match(TokenType::Char) && m_parser_state.current_token.value()[0] == '>') { - consume(); - break; - } - ++capture_group_name_length; - last_token = consume(); - } - capture_group_name = MUST(FlyString::from_utf8(m_parser_state.lexer.input().substring_view_starting_from_substring(start_token.value()).substring_view(0, capture_group_name_length))); - ++m_parser_state.named_capture_groups_count; - - } else if (match(TokenType::EqualSign)) { // positive lookahead - consume(); - group_mode = Lookahead; - } else if (consume("!")) { // negative lookahead - group_mode = NegativeLookahead; - } else if (consume("<")) { - if (match(TokenType::EqualSign)) { // positive lookbehind - consume(); - group_mode = Lookbehind; - } - if (consume("!")) // negative lookbehind - group_mode = NegativeLookbehind; - } else { - return set_error(Error::InvalidRepetitionMarker); - } - } - - auto current_capture_group = m_parser_state.capture_groups_count; - if (!(m_parser_state.regex_options & AllFlags::SkipSubExprResults || prevent_capture_group)) { - bytecode.insert_bytecode_group_capture_left(current_capture_group + 1); - m_parser_state.capture_groups_count++; - } - - ByteCode capture_group_bytecode; - - if (!parse_root(capture_group_bytecode, length)) - return set_error(Error::InvalidPattern); - - switch (group_mode) { - case Normal: - bytecode.extend(move(capture_group_bytecode)); - break; - case Lookahead: - bytecode.insert_bytecode_lookaround(move(capture_group_bytecode), ByteCode::LookAroundType::LookAhead, length); - break; - case NegativeLookahead: - bytecode.insert_bytecode_lookaround(move(capture_group_bytecode), ByteCode::LookAroundType::NegatedLookAhead, length); - break; - case Lookbehind: - bytecode.insert_bytecode_lookaround(move(capture_group_bytecode), ByteCode::LookAroundType::LookBehind, length); - break; - case NegativeLookbehind: - bytecode.insert_bytecode_lookaround(move(capture_group_bytecode), ByteCode::LookAroundType::NegatedLookBehind, length); - break; - } - - consume(TokenType::RightParen, Error::MismatchingParen); - - if (!(m_parser_state.regex_options & AllFlags::SkipSubExprResults || prevent_capture_group)) { - if (capture_group_name.has_value()) - bytecode.insert_bytecode_group_capture_right(current_capture_group + 1, capture_group_name.value()); - else - bytecode.insert_bytecode_group_capture_right(current_capture_group + 1); - } - should_parse_repetition_symbol = true; - break; - } - - return false; - } - - if (match_repetition_symbol()) { - if (should_parse_repetition_symbol) - parse_repetition_symbol(bytecode, length); - else - return set_error(Error::InvalidRepetitionMarker); - } - - stack.extend(move(bytecode)); - match_length_minimum += length; - - return true; -} - -bool PosixExtendedParser::parse_root(ByteCode& stack, size_t& match_length_minimum) -{ - ByteCode bytecode_left; - size_t match_length_minimum_left { 0 }; - - if (match_repetition_symbol()) - return set_error(Error::InvalidRepetitionMarker); - - for (;;) { - if (!parse_sub_expression(bytecode_left, match_length_minimum_left)) - break; - - if (match(TokenType::Pipe)) { - consume(); - - ByteCode bytecode_right; - size_t match_length_minimum_right { 0 }; - - if (!parse_root(bytecode_right, match_length_minimum_right) || bytecode_right.is_empty()) - return set_error(Error::InvalidPattern); - - ByteCode new_bytecode; - new_bytecode.insert_bytecode_alternation(move(bytecode_left), move(bytecode_right)); - bytecode_left = move(new_bytecode); - match_length_minimum_left = min(match_length_minimum_right, match_length_minimum_left); - } - } - - if (bytecode_left.is_empty()) - set_error(Error::EmptySubExpression); - - stack.extend(move(bytecode_left)); - match_length_minimum = match_length_minimum_left; - return !has_error(); -} - -// ============================= -// ECMA262 Parser -// ============================= - -bool ECMA262Parser::parse_internal(ByteCode& stack, size_t& match_length_minimum) -{ - auto unicode = m_parser_state.regex_options.has_flag_set(AllFlags::Unicode); - auto unicode_sets = m_parser_state.regex_options.has_flag_set(AllFlags::UnicodeSets); - if (unicode || unicode_sets) { - return parse_pattern(stack, match_length_minimum, { .unicode = true, .named = true, .unicode_sets = unicode_sets }); - } - - ByteCode new_stack; - size_t new_match_length = 0; - auto res = parse_pattern(new_stack, new_match_length, { .unicode = false, .named = false, .unicode_sets = false }); - if (m_parser_state.named_capture_groups_count > 0) { - reset(); - return parse_pattern(stack, match_length_minimum, { .unicode = false, .named = true, .unicode_sets = false }); - } - - if (!res) - return false; - - stack.extend(new_stack); - match_length_minimum = new_match_length; - return res; -} - -bool ECMA262Parser::parse_pattern(ByteCode& stack, size_t& match_length_minimum, ParseFlags flags) -{ - return parse_disjunction(stack, match_length_minimum, flags); -} - -bool ECMA262Parser::has_duplicate_in_current_alternative(FlyString const& name) -{ - auto it = m_parser_state.named_capture_groups.find(name); - if (it == m_parser_state.named_capture_groups.end()) - return false; - - return it->value.contains([&](auto& group) { return group.alternative_id == m_current_alternative_id; }); -} - -bool ECMA262Parser::parse_disjunction(ByteCode& stack, size_t& match_length_minimum, ParseFlags flags) -{ - size_t total_match_length_minimum = NumericLimits::max(); - Vector alternatives; - size_t initial_capture_groups_count = m_parser_state.capture_groups_count; - - TemporaryChange alternative_id_change { m_current_alternative_id, 1 }; - - while (true) { - ByteCode alternative_stack; - size_t alternative_minimum_length = 0; - auto alt_ok = parse_alternative(alternative_stack, alternative_minimum_length, flags); - if (!alt_ok) - return false; - - alternatives.append(move(alternative_stack)); - total_match_length_minimum = min(alternative_minimum_length, total_match_length_minimum); - - if (!match(TokenType::Pipe)) - break; - consume(); - - m_current_alternative_id += 1; - } - - if (alternatives.size() > 1) { - m_parser_state.greedy_lookaround = false; - mark_capture_groups_as_optional_from(initial_capture_groups_count); - } - Optimizer::append_alternation(stack, alternatives.span()); - match_length_minimum = total_match_length_minimum; - - return true; -} - -bool ECMA262Parser::parse_alternative(ByteCode& stack, size_t& match_length_minimum, ParseFlags flags) -{ - for (;;) { - if (match(TokenType::Eof)) - return true; - - if (parse_term(stack, match_length_minimum, flags)) - continue; - - return !has_error(); - } -} - -bool ECMA262Parser::parse_term(ByteCode& stack, size_t& match_length_minimum, ParseFlags flags) -{ - if (parse_assertion(stack, match_length_minimum, flags)) - return true; - - ByteCode atom_stack; - size_t minimum_atom_length = 0; - size_t initial_capture_groups_count = m_parser_state.capture_groups_count; - - auto parse_with_quantifier = [&] { - bool did_parse_one = false; - if (m_should_use_browser_extended_grammar) - did_parse_one = parse_extended_atom(atom_stack, minimum_atom_length, flags); - - if (!did_parse_one) - did_parse_one = parse_atom(atom_stack, minimum_atom_length, flags); - - if (!did_parse_one) - return false; - - VERIFY(did_parse_one); - return parse_quantifier(atom_stack, minimum_atom_length, flags); - }; - - if (!parse_with_quantifier()) - return false; - - if (minimum_atom_length == 0) - mark_capture_groups_as_optional_from(initial_capture_groups_count); - - stack.extend(move(atom_stack)); - match_length_minimum += minimum_atom_length; - return true; -} - -bool ECMA262Parser::parse_assertion(ByteCode& stack, [[maybe_unused]] size_t& match_length_minimum, ParseFlags flags) -{ - if (match(TokenType::Circumflex)) { - consume(); - stack.empend((ByteCodeValueType)OpCodeId::CheckBegin); - return true; - } - - if (match(TokenType::Dollar)) { - consume(); - stack.empend((ByteCodeValueType)OpCodeId::CheckEnd); - return true; - } - - if (try_skip("\\b"sv)) { - stack.insert_bytecode_check_boundary(BoundaryCheckType::Word); - return true; - } - - if (try_skip("\\B"sv)) { - stack.insert_bytecode_check_boundary(BoundaryCheckType::NonWord); - return true; - } - - if (match(TokenType::LeftParen)) { - if (!try_skip("(?"sv)) - return false; - - if (done()) { - set_error(Error::InvalidCaptureGroup); - return false; - } - - ByteCode assertion_stack; - size_t length_dummy = 0; - - bool should_parse_forward_assertion = !m_should_use_browser_extended_grammar || flags.unicode; - if (should_parse_forward_assertion && try_skip("="sv)) { - if (!parse_inner_disjunction(assertion_stack, length_dummy, flags)) - return false; - stack.insert_bytecode_lookaround(move(assertion_stack), ByteCode::LookAroundType::LookAhead); - return true; - } - if (should_parse_forward_assertion && try_skip("!"sv)) { - enter_capture_group_scope(); - size_t initial_capture_groups_count = m_parser_state.capture_groups_count; - ScopeGuard quit_scope { - [this] { - exit_capture_group_scope(); - } - }; - if (!parse_inner_disjunction(assertion_stack, length_dummy, flags)) - return false; - stack.insert_bytecode_lookaround(move(assertion_stack), ByteCode::LookAroundType::NegatedLookAhead); - clear_all_capture_groups_in_scope(stack); - mark_capture_groups_as_optional_from(initial_capture_groups_count); - - return true; - } - if (m_should_use_browser_extended_grammar) { - if (!flags.unicode) { - if (parse_quantifiable_assertion(assertion_stack, match_length_minimum, flags)) { - size_t assertion_match_length_minimum = 0; - if (!parse_quantifier(assertion_stack, assertion_match_length_minimum, flags)) - return false; - - stack.extend(move(assertion_stack)); - return true; - } - } - } - if (try_skip("<="sv)) { - if (!parse_inner_disjunction(assertion_stack, length_dummy, flags)) - return false; - // FIXME: Somehow ensure that this assertion regexp has a fixed length. - stack.insert_bytecode_lookaround(move(assertion_stack), ByteCode::LookAroundType::LookBehind, length_dummy, m_parser_state.greedy_lookaround); - return true; - } - if (try_skip(" 0 && count >= max_count) - break; - - if (hex && !AK::parse_hexadecimal_number(c).has_value()) - break; - if (!hex && !c.to_number().has_value()) - break; - - offset += consume().value().length(); - ++count; - } - - if (count < min_count) { - if (offset > 0) - back(offset + (done() ? 0 : 1)); - return {}; - } - - return StringView { start_token.value().characters_without_null_termination(), offset }; -} - -Optional ECMA262Parser::read_digits(ECMA262Parser::ReadDigitsInitialZeroState initial_zero, bool hex, int max_count, int min_count) -{ - auto str = read_digits_as_string(initial_zero, hex, max_count, min_count); - if (str.is_empty()) - return {}; - if (hex) - return AK::parse_hexadecimal_number(str); - return str.to_number(); -} - -bool ECMA262Parser::parse_quantifier(ByteCode& stack, size_t& match_length_minimum, ParseFlags flags) -{ - enum class Repetition { - OneOrMore, - ZeroOrMore, - Optional, - Explicit, - None, - } repetition_mark { Repetition::None }; - - bool ungreedy = false; - Optional repeat_min, repeat_max; - - if (match(TokenType::Asterisk)) { - consume(); - repetition_mark = Repetition::ZeroOrMore; - } else if (match(TokenType::Plus)) { - consume(); - repetition_mark = Repetition::OneOrMore; - } else if (match(TokenType::Questionmark)) { - consume(); - repetition_mark = Repetition::Optional; - } else if (match(TokenType::LeftCurly)) { - repetition_mark = Repetition::Explicit; - if (!parse_interval_quantifier(repeat_min, repeat_max)) { - if (flags.unicode) { - // Invalid interval quantifiers are disallowed in Unicode mod - they must be escaped with '\{'. - set_error(Error::InvalidPattern); - } - return !has_error(); - } - } else { - return true; - } - - if (match(TokenType::Questionmark)) { - consume(); - ungreedy = true; - } - - switch (repetition_mark) { - case Repetition::OneOrMore: - ByteCode::transform_bytecode_repetition_min_one(stack, !ungreedy); - break; - case Repetition::ZeroOrMore: - ByteCode::transform_bytecode_repetition_any(stack, !ungreedy); - match_length_minimum = 0; - break; - case Repetition::Optional: - ByteCode::transform_bytecode_repetition_zero_or_one(stack, !ungreedy); - match_length_minimum = 0; - break; - case Repetition::Explicit: { - auto min_repetition_mark_id = m_parser_state.repetition_mark_count++; - auto max_repetition_mark_id = m_parser_state.repetition_mark_count++; - ByteCode::transform_bytecode_repetition_min_max(stack, repeat_min.value(), repeat_max, min_repetition_mark_id, max_repetition_mark_id, !ungreedy); - match_length_minimum *= repeat_min.value(); - break; - } - case Repetition::None: - VERIFY_NOT_REACHED(); - } - - return true; -} - -bool ECMA262Parser::parse_interval_quantifier(Optional& repeat_min, Optional& repeat_max) -{ - VERIFY(match(TokenType::LeftCurly)); - consume(); - auto chars_consumed = 1; - - auto low_bound_string = read_digits_as_string(); - chars_consumed += low_bound_string.length(); - - if (low_bound_string.is_empty()) { - if (!m_should_use_browser_extended_grammar && done()) - return set_error(Error::MismatchingBrace); - - back(chars_consumed + !done()); - return false; - } - - auto low_bound = low_bound_string.to_number(); - - if (!low_bound.has_value() || low_bound.value() > s_ecma262_maximum_repetition_count) { - repeat_min = s_ecma262_maximum_repetition_count; - } else { - repeat_min = low_bound.value(); - } - - if (match(TokenType::Comma)) { - consume(); - ++chars_consumed; - auto high_bound_string = read_digits_as_string(); - auto high_bound = high_bound_string.to_number(); - if (!high_bound_string.is_empty()) { - chars_consumed += high_bound_string.length(); - - if (!high_bound.has_value() || high_bound.value() > s_ecma262_maximum_repetition_count) { - repeat_max = s_ecma262_maximum_repetition_count; - } else { - repeat_max = high_bound.value(); - } - } - } else { - repeat_max = repeat_min; - } - - if (!match(TokenType::RightCurly)) { - if (!m_should_use_browser_extended_grammar && done()) - return set_error(Error::MismatchingBrace); - - back(chars_consumed + !done()); - return false; - } - - consume(); - ++chars_consumed; - - if (repeat_max.has_value()) { - if (repeat_min.value() > repeat_max.value()) - set_error(Error::InvalidBraceContent); - } - - return true; -} - -bool ECMA262Parser::parse_atom(ByteCode& stack, size_t& match_length_minimum, ParseFlags flags) -{ - if (match(TokenType::EscapeSequence)) { - // Also part of AtomEscape. - auto token = consume(); - match_length_minimum += 1; - stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (u8)token.value()[1] } }); - return true; - } - if (try_skip("\\"sv)) { - // AtomEscape. - return parse_atom_escape(stack, match_length_minimum, flags); - } - - if (match(TokenType::LeftBracket)) { - // Character class. - return parse_character_class(stack, match_length_minimum, flags); - } - - if (match(TokenType::LeftParen)) { - // Non-capturing group, or a capture group. - return parse_capture_group(stack, match_length_minimum, flags); - } - - if (match(TokenType::Period)) { - consume(); - match_length_minimum += 1; - stack.insert_bytecode_compare_values({ { CharacterCompareType::AnyChar, 0 } }); - return true; - } - - if (match(TokenType::Circumflex) || match(TokenType::Dollar) || match(TokenType::RightParen) - || match(TokenType::Pipe) || match(TokenType::Plus) || match(TokenType::Asterisk) - || match(TokenType::Questionmark)) { - - return false; - } - - if (match(TokenType::RightBracket) || match(TokenType::RightCurly) || match(TokenType::LeftCurly)) { - if (flags.unicode) - return set_error(Error::InvalidPattern); - - if (m_should_use_browser_extended_grammar) { - auto token = consume(); - match_length_minimum += 1; - stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (u8)token.value()[0] } }); - return true; - } - return false; - } - - if (match_ordinary_characters()) { - auto token = consume().value(); - match_length_minimum += 1; - stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (u8)token[0] } }); - return true; - } - - set_error(Error::InvalidPattern); - return false; -} - -bool ECMA262Parser::parse_extended_atom(ByteCode&, size_t&, ParseFlags) -{ - // Note: This includes only rules *not* present in parse_atom() - VERIFY(m_should_use_browser_extended_grammar); - - return parse_invalid_braced_quantifier(); // FAIL FAIL FAIL -} - -bool ECMA262Parser::parse_invalid_braced_quantifier() -{ - if (!match(TokenType::LeftCurly)) - return false; - consume(); - size_t chars_consumed = 1; - auto low_bound = read_digits_as_string(); - StringView high_bound; - - if (low_bound.is_empty()) { - back(chars_consumed + !done()); - return false; - } - chars_consumed += low_bound.length(); - if (match(TokenType::Comma)) { - consume(); - ++chars_consumed; - - high_bound = read_digits_as_string(); - chars_consumed += high_bound.length(); - } - - if (!match(TokenType::RightCurly)) { - back(chars_consumed + !done()); - return false; - } - - consume(); - set_error(Error::InvalidPattern); - return true; -} - -bool ECMA262Parser::parse_character_escape(Vector& compares, size_t& match_length_minimum, ParseFlags flags) -{ - // CharacterEscape > ControlEscape - if (try_skip("f"sv)) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)'\f' }); - return true; - } - - if (try_skip("n"sv)) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)'\n' }); - return true; - } - - if (try_skip("r"sv)) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)'\r' }); - return true; - } - - if (try_skip("t"sv)) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)'\t' }); - return true; - } - - if (try_skip("v"sv)) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)'\v' }); - return true; - } - - // CharacterEscape > ControlLetter - if (try_skip("c"sv)) { - for (auto c : s_alphabetic_characters) { - if (try_skip({ &c, 1 })) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)(c % 32) }); - return true; - } - } - - if (flags.unicode) { - set_error(Error::InvalidPattern); - return false; - } - - if (m_should_use_browser_extended_grammar) { - back(1 + (done() ? 0 : 1)); - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)'\\' }); - match_length_minimum += 1; - return true; - } - - // Allow '\c' in non-unicode mode, just matches 'c'. - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)'c' }); - return true; - } - - // '\0' - if (try_skip("0"sv)) { - if (!lookahead_any(s_decimal_characters)) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)0 }); - return true; - } - - back(2); - } - - // LegacyOctalEscapeSequence - if (m_should_use_browser_extended_grammar) { - if (!flags.unicode) { - if (auto escape = parse_legacy_octal_escape(); escape.has_value()) { - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)escape.value() }); - match_length_minimum += 1; - return true; - } - } - } - - // HexEscape - if (try_skip("x"sv)) { - if (auto hex_escape = read_digits(ReadDigitsInitialZeroState::Allow, true, 2, 2); hex_escape.has_value()) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)hex_escape.value() }); - return true; - } - if (!flags.unicode) { - // '\x' is allowed in non-unicode mode, just matches 'x'. - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)'x' }); - return true; - } - - set_error(Error::InvalidPattern); - return false; - } - - if (try_skip("u"sv)) { - if (auto code_point = consume_escaped_code_point(flags.unicode); code_point.has_value()) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)code_point.value() }); - return true; - } - - return false; - } - - // IdentityEscape - for (auto ch : identity_escape_characters(flags.unicode, m_should_use_browser_extended_grammar)) { - if (try_skip({ &ch, 1 })) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)ch }); - return true; - } - } - - if (flags.unicode) { - if (try_skip("/"sv)) { - match_length_minimum += 1; - compares.append({ CharacterCompareType::Char, (ByteCodeValueType)'/' }); - return true; - } - } - - return false; -} - -bool ECMA262Parser::parse_atom_escape(ByteCode& stack, size_t& match_length_minimum, ParseFlags flags) -{ - if (auto escape_str = read_digits_as_string(ReadDigitsInitialZeroState::Disallow); !escape_str.is_empty()) { - if (auto escape = escape_str.to_number(); escape.has_value()) { - // See if this is a "back"-reference (we've already parsed the group it refers to) - auto maybe_length = m_parser_state.capture_group_minimum_lengths.get(escape.value()); - if (maybe_length.has_value()) { - if (!m_parser_state.optional_capture_groups.contains(escape.value())) - match_length_minimum += maybe_length.value(); - stack.insert_bytecode_compare_values({ { CharacterCompareType::Reference, (ByteCodeValueType)escape.value() } }); - return true; - } - // It's not a pattern seen before, so we have to see if it's a valid reference to a future group. - if (escape.value() <= ensure_total_number_of_capturing_parenthesis()) { - // This refers to a future group, and it will _always_ be matching an empty string - // So just match nothing and move on. - return true; - } - if (!m_should_use_browser_extended_grammar) { - set_error(Error::InvalidNumber); - return false; - } - } - - // If not, put the characters back. - back(escape_str.length() + (done() ? 0 : 1)); - } - - Vector escape_compares; - if (parse_character_escape(escape_compares, match_length_minimum, flags)) { - stack.insert_bytecode_compare_values(move(escape_compares)); - return true; - } - - if (flags.named && try_skip("k"sv)) { - auto name = read_capture_group_specifier(true); - if (name.is_empty()) { - set_error(Error::InvalidNameForCaptureGroup); - return false; - } - - auto it = m_parser_state.named_capture_groups.find(name); - if (it != m_parser_state.named_capture_groups.end()) { - - // Use the first occurrence of the named group for the backreference - // This follows ECMAScript behavior where \k refers to the first - // group with that name in left-to-right order, regardless of alternative - auto group_index = it->value.first().group_index; - auto maybe_length = m_parser_state.capture_group_minimum_lengths.get(group_index); - if (maybe_length.has_value()) { - // Backward reference - if (!m_parser_state.optional_capture_groups.contains(group_index)) - match_length_minimum += maybe_length.value(); - stack.insert_bytecode_compare_values({ { CharacterCompareType::NamedReference, static_cast(group_index) } }); - } else { - // Self-reference or forward reference - auto placeholder_index = 0; - auto bytecode_offset = stack.size(); - stack.insert_bytecode_compare_values({ { CharacterCompareType::NamedReference, static_cast(placeholder_index) } }); - - m_parser_state.unresolved_named_references.append({ name, bytecode_offset + 1 }); - } - } else { - // Forward reference - auto placeholder_index = 0; - auto bytecode_offset = stack.size(); - stack.insert_bytecode_compare_values({ { CharacterCompareType::NamedReference, static_cast(placeholder_index) } }); - - m_parser_state.unresolved_named_references.append({ name, bytecode_offset + 1 }); - } - return true; - } - - if (flags.unicode) { - PropertyEscape property {}; - bool negated = false; - - if (parse_unicode_property_escape(property, negated)) { - Vector compares; - if (negated) - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Inverse, 0 }); - property.visit( - [&](Unicode::Property property) { - if (Unicode::is_ecma262_string_property(property) && !negated) { - auto strings = Unicode::get_property_strings(property); - if (!strings.is_empty()) { - auto string_set_index = m_parser_state.bytecode.string_set_table().set(move(strings)); - compares.empend(CompareTypeAndValuePair { CharacterCompareType::StringSet, string_set_index }); - } - } else { - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Property, (ByteCodeValueType)property.value() }); - } - }, - [&](Unicode::GeneralCategory general_category) { - compares.empend(CompareTypeAndValuePair { CharacterCompareType::GeneralCategory, (ByteCodeValueType)general_category.value() }); - }, - [&](Script script) { - if (script.is_extension) - compares.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)script.script.value() }); - else - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)script.script.value() }); - }, - [](Empty&) { VERIFY_NOT_REACHED(); }); - stack.insert_bytecode_compare_values(move(compares)); - match_length_minimum += 1; - return true; - } - } - - if (done()) - return set_error(Error::InvalidTrailingEscape); - - bool negate = false; - auto ch = parse_character_class_escape(negate); - if (!ch.has_value()) { - if (!flags.unicode) { - // Allow all SourceCharacter's as escapes here. - auto token = consume(); - match_length_minimum += 1; - stack.insert_bytecode_compare_values({ { CharacterCompareType::Char, (u8)token.value()[0] } }); - return true; - } - - set_error(Error::InvalidCharacterClass); - return false; - } - - Vector compares; - if (negate) - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Inverse, 0 }); - compares.empend(CompareTypeAndValuePair { CharacterCompareType::CharClass, (ByteCodeValueType)ch.value() }); - match_length_minimum += 1; - stack.insert_bytecode_compare_values(move(compares)); - return true; -} -Optional ECMA262Parser::parse_legacy_octal_escape() -{ - constexpr auto all_octal_digits = "01234567"sv; - auto read_octal_digit = [&](auto start, auto end, bool should_ensure_no_following_octal_digit) -> Optional { - for (char c = '0' + start; c <= '0' + end; ++c) { - if (try_skip({ &c, 1 })) { - if (!should_ensure_no_following_octal_digit || !lookahead_any(all_octal_digits)) - return c - '0'; - back(2); - return {}; - } - } - return {}; - }; - - // OctalDigit(1) - if (auto digit = read_octal_digit(0, 7, true); digit.has_value()) { - return digit.value(); - } - - // OctalDigit(2) - if (auto left_digit = read_octal_digit(0, 3, false); left_digit.has_value()) { - if (auto right_digit = read_octal_digit(0, 7, true); right_digit.has_value()) { - return left_digit.value() * 8 + right_digit.value(); - } - - back(2); - } - - // OctalDigit(2) - if (auto left_digit = read_octal_digit(4, 7, false); left_digit.has_value()) { - if (auto right_digit = read_octal_digit(0, 7, false); right_digit.has_value()) { - return left_digit.value() * 8 + right_digit.value(); - } - - back(2); - } - - // OctalDigit(3) - if (auto left_digit = read_octal_digit(0, 3, false); left_digit.has_value()) { - size_t chars_consumed = 1; - if (auto mid_digit = read_octal_digit(0, 7, false); mid_digit.has_value()) { - ++chars_consumed; - if (auto right_digit = read_octal_digit(0, 7, false); right_digit.has_value()) { - return left_digit.value() * 64 + mid_digit.value() * 8 + right_digit.value(); - } - } - - back(chars_consumed); - } - - return {}; -} - -Optional ECMA262Parser::parse_character_class_escape(bool& negate, bool expect_backslash) -{ - if (expect_backslash && !try_skip("\\"sv)) - return {}; - - // CharacterClassEscape - CharClass ch_class; - if (try_skip("d"sv)) { - ch_class = CharClass::Digit; - } else if (try_skip("D"sv)) { - ch_class = CharClass::Digit; - negate = true; - } else if (try_skip("s"sv)) { - ch_class = CharClass::Space; - } else if (try_skip("S"sv)) { - ch_class = CharClass::Space; - negate = true; - } else if (try_skip("w"sv)) { - ch_class = CharClass::Word; - } else if (try_skip("W"sv)) { - ch_class = CharClass::Word; - negate = true; - } else { - return {}; - } - - return ch_class; -} - -bool ECMA262Parser::parse_character_class(ByteCode& stack, size_t& match_length_minimum, ParseFlags flags) -{ - consume(TokenType::LeftBracket, Error::InvalidPattern); - - Vector compares; - - auto uses_explicit_or_semantics = false; - bool is_negated = false; - if (match(TokenType::Circumflex)) { - // Negated charclass - consume(); - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Inverse, 0 }); - uses_explicit_or_semantics = true; - is_negated = true; - } - - auto previous_negated_state = m_parser_state.in_negated_character_class; - m_parser_state.in_negated_character_class = is_negated; - ArmedScopeGuard restore_negated_state { [&] { - m_parser_state.in_negated_character_class = previous_negated_state; - } }; - - // ClassContents :: [empty] - if (match(TokenType::RightBracket)) { - consume(); - // Should only have at most an 'Inverse' - VERIFY(compares.size() <= 1); - stack.insert_bytecode_compare_values(move(compares)); - return true; - } - - // ClassContents :: [~UnicodeSetsMode] NonemptyClassRanges[?UnicodeMode] - if (!flags.unicode_sets && !parse_nonempty_class_ranges(compares, flags)) { - restore_negated_state.disarm(); - return false; - } - - // ClassContents :: [+UnicodeSetsMode] ClassSetExpression - if (flags.unicode_sets && !parse_class_set_expression(compares)) { - restore_negated_state.disarm(); - return false; - } - - restore_negated_state.disarm(); - - if (uses_explicit_or_semantics && compares.size() > 2) { - compares.insert(1, CompareTypeAndValuePair { CharacterCompareType::Or, 0 }); - compares.empend(CompareTypeAndValuePair { CharacterCompareType::EndAndOr, 0 }); - } - - bool has_empty_string_set = any_of(compares, [this](auto const& compare) { - if (compare.type != CharacterCompareType::StringSet) - return false; - - auto const& trie = m_parser_state.bytecode.string_set_table().get_u8_trie(compare.value); - return trie.has_metadata() && trie.metadata_value(); - }); - - if (!has_empty_string_set) - match_length_minimum += 1; - stack.insert_bytecode_compare_values(move(compares)); - return true; -} - -struct CharClassRangeElement { - union { - CharClass character_class; - u32 code_point { 0 }; - Unicode::Property property; - Unicode::GeneralCategory general_category; - Unicode::Script script; - }; - - bool is_negated { false }; - bool is_character_class { false }; - bool is_property { false }; - bool is_general_category { false }; - bool is_script { false }; - bool is_script_extension { false }; -}; - -bool ECMA262Parser::parse_nonempty_class_ranges(Vector& ranges, ParseFlags flags) -{ - auto read_class_atom_no_dash = [&]() -> Optional { - if (match(TokenType::EscapeSequence)) { - auto token = consume().value(); - return { CharClassRangeElement { .code_point = (u32)token[1], .is_character_class = false } }; - } - - if (try_skip("\\"sv)) { - if (done()) { - set_error(Error::InvalidTrailingEscape); - return {}; - } - - if (try_skip("f"sv)) - return { CharClassRangeElement { .code_point = '\f', .is_character_class = false } }; - if (try_skip("n"sv)) - return { CharClassRangeElement { .code_point = '\n', .is_character_class = false } }; - if (try_skip("r"sv)) - return { CharClassRangeElement { .code_point = '\r', .is_character_class = false } }; - if (try_skip("t"sv)) - return { CharClassRangeElement { .code_point = '\t', .is_character_class = false } }; - if (try_skip("v"sv)) - return { CharClassRangeElement { .code_point = '\v', .is_character_class = false } }; - if (try_skip("b"sv)) - return { CharClassRangeElement { .code_point = '\b', .is_character_class = false } }; - if (try_skip("/"sv)) - return { CharClassRangeElement { .code_point = '/', .is_character_class = false } }; - - // CharacterEscape > ControlLetter - if (try_skip("c"sv)) { - for (auto c : s_alphabetic_characters) { - if (try_skip({ &c, 1 })) { - return { CharClassRangeElement { .code_point = (u32)(c % 32), .is_character_class = false } }; - } - } - - if (flags.unicode) { - set_error(Error::InvalidPattern); - return {}; - } - - if (m_should_use_browser_extended_grammar) { - for (auto c = '0'; c <= '9'; ++c) { - if (try_skip({ &c, 1 })) - return { CharClassRangeElement { .code_point = (u32)(c % 32), .is_character_class = false } }; - } - if (try_skip("_"sv)) - return { CharClassRangeElement { .code_point = (u32)('_' % 32), .is_character_class = false } }; - - back(1 + !done()); - return { CharClassRangeElement { .code_point = '\\', .is_character_class = false } }; - } - } - - // '\0' - if (try_skip("0"sv)) { - if (!lookahead_any(s_decimal_characters)) - return { CharClassRangeElement { .code_point = 0, .is_character_class = false } }; - back(); - } - - // LegacyOctalEscapeSequence - if (m_should_use_browser_extended_grammar && !flags.unicode) { - if (auto escape = parse_legacy_octal_escape(); escape.has_value()) - return { CharClassRangeElement { .code_point = escape.value(), .is_character_class = false } }; - } - - // HexEscape - if (try_skip("x"sv)) { - if (auto hex_escape = read_digits(ReadDigitsInitialZeroState::Allow, true, 2, 2); hex_escape.has_value()) { - return { CharClassRangeElement { .code_point = hex_escape.value(), .is_character_class = false } }; - } else if (!flags.unicode) { - // '\x' is allowed in non-unicode mode, just matches 'x'. - return { CharClassRangeElement { .code_point = 'x', .is_character_class = false } }; - } else { - set_error(Error::InvalidPattern); - return {}; - } - } - - if (try_skip("u"sv)) { - if (auto code_point = consume_escaped_code_point(flags.unicode); code_point.has_value()) { - // FIXME: While code point ranges are supported, code point matches as "Char" are not! - return { CharClassRangeElement { .code_point = code_point.value(), .is_character_class = false } }; - } - return {}; - } - - // IdentityEscape - for (auto ch : identity_escape_characters(flags.unicode, m_should_use_browser_extended_grammar)) { - if (try_skip({ &ch, 1 })) - return { CharClassRangeElement { .code_point = (u32)ch, .is_character_class = false } }; - } - - if (flags.unicode) { - if (try_skip("-"sv)) - return { CharClassRangeElement { .code_point = '-', .is_character_class = false } }; - - PropertyEscape property {}; - bool negated = false; - if (parse_unicode_property_escape(property, negated)) { - return property.visit( - [&](Unicode::Property property) { - return CharClassRangeElement { .property = property, .is_negated = negated, .is_character_class = true, .is_property = true }; - }, - [&](Unicode::GeneralCategory general_category) { - return CharClassRangeElement { .general_category = general_category, .is_negated = negated, .is_character_class = true, .is_general_category = true }; - }, - [&](Script script) { - if (script.is_extension) - return CharClassRangeElement { .script = script.script, .is_negated = negated, .is_character_class = true, .is_script_extension = true }; - - return CharClassRangeElement { .script = script.script, .is_negated = negated, .is_character_class = true, .is_script = true }; - }, - [](Empty&) -> CharClassRangeElement { VERIFY_NOT_REACHED(); }); - } - } - - if (try_skip("d"sv)) - return { CharClassRangeElement { .character_class = CharClass::Digit, .is_character_class = true } }; - if (try_skip("s"sv)) - return { CharClassRangeElement { .character_class = CharClass::Space, .is_character_class = true } }; - if (try_skip("w"sv)) - return { CharClassRangeElement { .character_class = CharClass::Word, .is_character_class = true } }; - if (try_skip("D"sv)) - return { CharClassRangeElement { .character_class = CharClass::Digit, .is_negated = true, .is_character_class = true } }; - if (try_skip("S"sv)) - return { CharClassRangeElement { .character_class = CharClass::Space, .is_negated = true, .is_character_class = true } }; - if (try_skip("W"sv)) - return { CharClassRangeElement { .character_class = CharClass::Word, .is_negated = true, .is_character_class = true } }; - - if (!flags.unicode) { - // Any unrecognised escape is allowed in non-unicode mode. - return { CharClassRangeElement { .code_point = (u32)skip(), .is_character_class = false } }; - } - - set_error(Error::InvalidPattern); - return {}; - } - - if (match(TokenType::Eof)) { - set_error(Error::MismatchingBracket); - return {}; - } - - if (match(TokenType::RightBracket) || match(TokenType::HyphenMinus)) - return {}; - - // Allow any (other) SourceCharacter. - return { CharClassRangeElement { .code_point = (u32)skip(), .is_character_class = false } }; - }; - auto read_class_atom = [&]() -> Optional { - if (match(TokenType::HyphenMinus)) { - consume(); - return { CharClassRangeElement { .code_point = '-', .is_character_class = false } }; - } - - return read_class_atom_no_dash(); - }; - - auto empend_atom = [&](auto& atom) { - if (atom.is_character_class) { - if (atom.is_negated) - ranges.empend(CompareTypeAndValuePair { CharacterCompareType::TemporaryInverse, 0 }); - - if (atom.is_property) - ranges.empend(CompareTypeAndValuePair { CharacterCompareType::Property, (ByteCodeValueType)(atom.property.value()) }); - else if (atom.is_general_category) - ranges.empend(CompareTypeAndValuePair { CharacterCompareType::GeneralCategory, (ByteCodeValueType)(atom.general_category.value()) }); - else if (atom.is_script) - ranges.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)(atom.script.value()) }); - else if (atom.is_script_extension) - ranges.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)(atom.script.value()) }); - else - ranges.empend(CompareTypeAndValuePair { CharacterCompareType::CharClass, (ByteCodeValueType)atom.character_class }); - } else { - VERIFY(!atom.is_negated); - ranges.empend(CompareTypeAndValuePair { CharacterCompareType::Char, atom.code_point }); - } - }; - - while (!match(TokenType::RightBracket)) { - if (match(TokenType::Eof)) { - set_error(Error::MismatchingBracket); - return false; - } - - auto first_atom = read_class_atom(); - if (!first_atom.has_value()) - return false; - - if (match(TokenType::HyphenMinus)) { - consume(); - if (match(TokenType::RightBracket)) { - // Allow '-' as the last element in a charclass, even after an atom. - m_parser_state.lexer.back(2); // -] - m_parser_state.current_token = m_parser_state.lexer.next(); - goto read_as_single_atom; - } - auto second_atom = read_class_atom(); - if (!second_atom.has_value()) - return false; - - if (first_atom.value().is_character_class || second_atom.value().is_character_class) { - if (m_should_use_browser_extended_grammar) { - if (flags.unicode) { - set_error(Error::InvalidRange); - return false; - } - - // CharacterRangeOrUnion > !Unicode > CharClass - empend_atom(*first_atom); - ranges.empend(CompareTypeAndValuePair { CharacterCompareType::Char, (ByteCodeValueType)'-' }); - empend_atom(*second_atom); - continue; - } - - set_error(Error::InvalidRange); - return false; - } - - if (first_atom.value().code_point > second_atom.value().code_point) { - set_error(Error::InvalidRange); - return false; - } - - VERIFY(!first_atom.value().is_negated); - VERIFY(!second_atom.value().is_negated); - - ranges.empend(CompareTypeAndValuePair { CharacterCompareType::CharRange, CharRange { first_atom.value().code_point, second_atom.value().code_point } }); - continue; - } - - read_as_single_atom:; - - auto atom = first_atom.value(); - empend_atom(atom); - } - - consume(TokenType::RightBracket, Error::MismatchingBracket); - - return true; -} - -bool ECMA262Parser::parse_class_set_expression(Vector& compares) -{ - auto start_token = m_parser_state.current_token; - auto start_lexer_index = m_parser_state.lexer.tell(); - - // ClassSetExpression :: ClassUnion | ClassIntersection | ClassSubtraction - if (parse_class_subtraction(compares)) { - consume(TokenType::RightBracket, Error::MismatchingBracket); - return true; - } - if (has_error()) - return false; - - m_parser_state.current_token = start_token; - m_parser_state.lexer.back(m_parser_state.lexer.tell() - start_lexer_index); - - if (parse_class_intersection(compares)) { - consume(TokenType::RightBracket, Error::MismatchingBracket); - return true; - } - if (has_error()) - return false; - - m_parser_state.current_token = start_token; - m_parser_state.lexer.back(m_parser_state.lexer.tell() - start_lexer_index); - - if (parse_class_union(compares)) { - consume(TokenType::RightBracket, Error::MismatchingBracket); - return true; - } - - return false; -} - -bool ECMA262Parser::parse_class_union(Vector& compares) -{ - auto restore_position = save_parser_state(); - - auto first = true; - - // ClassUnion :: ClassSetRange ClassUnion[opt] | ClassSetOperand ClassUnion[opt] - for (;;) { - if (!parse_class_set_range(compares)) { - if (has_error() || match(TokenType::RightBracket)) - break; - - if (!parse_class_set_operand(compares)) { - if (first || has_error()) - return false; - break; - } - } - first = false; - } - - if (!first) { - compares.prepend({ CharacterCompareType::Or, 0 }); - compares.append({ CharacterCompareType::EndAndOr, 0 }); - } - - restore_position.disarm(); - return !has_error(); -} - -bool ECMA262Parser::parse_class_intersection(Vector& compares) -{ - // ClassIntersection :: ClassSetOperand "&&" [lookahead != "&"] ClassSetOperand - // | ClassIntersection "&&" [lookahead != "&"] ClassSetOperand - Vector lhs; - Vector rhs; - - auto restore_position = save_parser_state(); - - if (!parse_class_set_operand(lhs)) - return false; - - if (!try_skip("&&"sv)) - return false; - - compares.append({ CharacterCompareType::And, 0 }); - compares.extend(move(lhs)); - - do { - rhs.clear_with_capacity(); - if (!parse_class_set_operand(rhs)) - return false; - - compares.extend(rhs); - - if (try_skip("&&&"sv)) - return false; - } while (!has_error() && try_skip("&&"sv)); - - compares.append({ CharacterCompareType::EndAndOr, 0 }); - - restore_position.disarm(); - return true; -} - -bool ECMA262Parser::parse_class_subtraction(Vector& compares) -{ - // ClassSubtraction :: ClassSetOperand "--" ClassSetOperand | ClassSubtraction "--" ClassSetOperand - Vector lhs; - Vector rhs; - - auto restore_position = save_parser_state(); - - if (!parse_class_set_operand(lhs)) - return false; - - if (!try_skip("--"sv)) - return false; - - compares.append({ CharacterCompareType::Subtract, 0 }); - compares.extend(move(lhs)); - - do { - rhs.clear_with_capacity(); - if (!parse_class_set_operand(rhs)) - return false; - - compares.extend(rhs); - } while (!has_error() && try_skip("--"sv)); - - compares.append({ CharacterCompareType::EndAndOr, 0 }); - - restore_position.disarm(); - return true; -} - -bool ECMA262Parser::parse_class_set_range(Vector& compares) -{ - // ClassSetRange :: ClassSetCharacter "-" ClassSetCharacter - auto restore_position = save_parser_state(); - - auto lhs = parse_class_set_character(); - if (!lhs.has_value()) - return false; - - if (!match(TokenType::HyphenMinus)) - return false; - consume(); - - auto rhs = parse_class_set_character(); - if (!rhs.has_value()) - return false; - - compares.append({ - CharacterCompareType::CharRange, - CharRange { lhs.value(), rhs.value() }, - }); - restore_position.disarm(); - return true; -} - -Optional ECMA262Parser::parse_class_set_character() -{ - // ClassSetCharacter :: [lookahead βˆ‰ ClassSetReservedDoublePunctuator] SourceCharacter but not ClassSetSyntaxCharacter - // | "\" CharacterEscape[+UnicodeMode] - // | "\" ClassSetReservedPunctuator - // | "\" b - // ClassSetReservedDoublePunctuator :: one of "&&" "!!" "##" "$$" "%%" "**" "++" ",," ".." "::" ";;" "<<" "==" ">>" "??" "@@" "^^" "``" "~~" - // ClassSetSyntaxCharacter :: one of "(" ")" "{" "}" "[" "]" "/" "-" "\" "|" - // ClassSetReservedPunctuator :: one of "&" "-" "!" "#" "%" "," ":" ";" "<" "=" ">" "@" "`" "~" - - constexpr auto class_set_reserved_double_punctuator = Array { - "&&"sv, "!!"sv, "##"sv, "$$"sv, "%%"sv, "**"sv, "++"sv, ",,"sv, ".."sv, "::"sv, ";;"sv, "<<"sv, "=="sv, ">>"sv, "??"sv, "@@"sv, "^^"sv, "``"sv, "~~"sv - }; - - constexpr auto class_set_reserved_punctuator = Array { - "&"sv, "-"sv, "!"sv, "#"sv, "%"sv, ","sv, ":"sv, ";"sv, "<"sv, "="sv, ">"sv, "@"sv, "`"sv, "~"sv - }; - - auto restore = save_parser_state(); - - if (done()) { - set_error(Error::InvalidPattern); - return {}; - } - - if (match(TokenType::EscapeSequence)) { - auto escape_value = m_parser_state.current_token.value(); - consume(); - - if (escape_value[0] == '\\' && escape_value.length() == 2) { - restore.disarm(); - return escape_value[1]; - } - } - - if (try_skip("\\"sv)) { - if (done()) { - set_error(Error::InvalidTrailingEscape); - return {}; - } - - // "\" ClassSetReservedPunctuator - for (auto const& reserved : class_set_reserved_punctuator) { - if (try_skip(reserved)) { - // "\" ClassSetReservedPunctuator (ClassSetReservedPunctuator) - back(); - - restore.disarm(); - return reserved[0]; - } - } - // "\" b - if (try_skip("b"sv)) { - restore.disarm(); - return '\b'; - } - - // "\" CharacterEscape[+UnicodeMode] - Vector compares; - size_t minimum_length = 0; - if (parse_character_escape(compares, minimum_length, { .unicode = true })) { - VERIFY(compares.size() == 1); - auto& compare = compares.first(); - VERIFY(compare.type == CharacterCompareType::Char); - restore.disarm(); - return compare.value; - } - - return {}; - } - - // [lookahead βˆ‰ ClassSetReservedDoublePunctuator] SourceCharacter but not ClassSetSyntaxCharacter - auto lookahead_matches = any_of(class_set_reserved_double_punctuator, [this](auto& reserved) { - return try_skip(reserved); - }); - - if (lookahead_matches) - return {}; - - for (auto character : { "("sv, ")"sv, "{"sv, "}"sv, "["sv, "]"sv, "/"sv, "-"sv, "\\"sv, "|"sv }) { - if (try_skip(character)) - return {}; - } - - restore.disarm(); - return skip(); -} - -bool ECMA262Parser::parse_class_set_operand(Vector& compares) -{ - auto start_token = m_parser_state.current_token; - auto start_lexer_index = m_parser_state.lexer.tell(); - - // ClassStringDisjunction :: "\q{" ClassStringDisjunctionContents "}" - // ClassStringDisjunctionContents :: ClassString | ClassString "|" ClassStringDisjunctionContents - // ClassString :: [empty] | NonEmptyClassString - // NonEmptyClassString :: ClassCharacter NonEmptyClassString[opt] - if (try_skip("\\q"sv)) { - if (!match(TokenType::LeftCurly)) { - back(2); - return false; - } - consume(); - - Vector strings; - StringBuilder current_string; - - while (!match(TokenType::RightCurly)) { - if (done()) { - set_error(Error::MismatchingBrace); - return false; - } - - if (match(TokenType::Pipe)) { - consume(); - strings.append(MUST(current_string.to_string())); - current_string.clear(); - continue; - } - - auto character = parse_class_set_character(); - if (!character.has_value()) { - if (has_error()) - return false; - set_error(Error::InvalidCharacterClass); - return false; - } - - current_string.append_code_point(character.value()); - } - - strings.append(MUST(current_string.to_string())); - consume(TokenType::RightCurly, Error::MismatchingBrace); - - if (m_parser_state.in_negated_character_class && any_of(strings, has_multiple_code_points)) { - set_error(Error::NegatedCharacterClassStrings); - return false; - } - - auto string_set_index = m_parser_state.bytecode.string_set_table().set(strings); - compares.empend(CompareTypeAndValuePair { CharacterCompareType::StringSet, string_set_index }); - - return true; - } - - // ClassSetOperand :: ClassSetCharacter | ClassStringDisjunction | NestedClass - if (auto character = parse_class_set_character(); character.has_value()) { - compares.append({ CharacterCompareType::Char, character.value() }); - return true; - } - - // NestedClass :: "[" [lookahead != "^"] ClassContents[+UnicodeMode +UnicodeSetsMode] "]" - // | "[" "^" ClassContents[+UnicodeMode +UnicodeSetsMode] "]" - // | "\" CharacterClassEscape[+UnicodeMode] - if (parse_nested_class(compares)) - return true; - - if (has_error()) - return false; - - auto negated = false; - if (auto ch = parse_character_class_escape(negated, true); ch.has_value()) { - if (negated) - compares.append({ CharacterCompareType::TemporaryInverse, 1 }); - compares.append({ CharacterCompareType::CharClass, (ByteCodeValueType)ch.value() }); - return true; - } - - PropertyEscape property {}; - if (parse_unicode_property_escape(property, negated)) { - if (negated) - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Inverse, 0 }); - property.visit( - [&](Unicode::Property property) { - if (Unicode::is_ecma262_string_property(property) && !negated) { - auto strings = Unicode::get_property_strings(property); - if (!strings.is_empty()) { - auto string_set_index = m_parser_state.bytecode.string_set_table().set(move(strings)); - compares.empend(CompareTypeAndValuePair { CharacterCompareType::StringSet, string_set_index }); - } - } else { - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Property, (ByteCodeValueType)property.value() }); - } - }, - [&](Unicode::GeneralCategory general_category) { - compares.empend(CompareTypeAndValuePair { CharacterCompareType::GeneralCategory, (ByteCodeValueType)general_category.value() }); - }, - [&](Script script) { - if (script.is_extension) - compares.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)script.script.value() }); - else - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)script.script.value() }); - }, - [](Empty&) { VERIFY_NOT_REACHED(); }); - return true; - } - - if (has_error()) - return false; - - m_parser_state.current_token = start_token; - m_parser_state.lexer.back(m_parser_state.lexer.tell() - start_lexer_index); - return false; -} - -bool ECMA262Parser::parse_nested_class(Vector& compares) -{ - auto start_token = m_parser_state.current_token; - auto start_lexer_index = m_parser_state.lexer.tell(); - - // NestedClass :: "[" [lookahead β‰  ^ ] ClassContents [+UnicodeMode, +UnicodeSetsMode] "]" - // | "[" "^" ClassContents[+UnicodeMode, +UnicodeSetsMode] "]" - // | "\" CharacterClassEscape[+UnicodeMode] - - if (match(TokenType::LeftBracket)) { - consume(); - - auto initial_compares_size = compares.size(); - compares.append(CompareTypeAndValuePair { CharacterCompareType::Or, 0 }); - - if (match(TokenType::Circumflex)) { - // Negated charclass - consume(); - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Inverse, 0 }); - } - - // ClassContents :: [empty] - if (match(TokenType::RightBracket)) { - consume(); - auto added_compares = compares.size() - initial_compares_size; - // Should only have at most an 'Inverse' (after an 'Or') - if (m_parser_state.regex_options.has_flag_set(regex::AllFlags::UnicodeSets)) { - // In unicode sets mode, we can have an additional 'And'/'Or' before the 'Inverse'. - VERIFY(added_compares <= 3); - } else { - VERIFY(added_compares <= 2); - } - compares.append(CompareTypeAndValuePair { CharacterCompareType::EndAndOr, 0 }); - return true; - } - - // ClassContents :: [+UnicodeSetsMode] ClassSetExpression - if (!parse_class_set_expression(compares)) - return false; - - compares.append(CompareTypeAndValuePair { CharacterCompareType::EndAndOr, 0 }); - return true; - } - - if (try_skip("\\"sv)) { - auto negated = false; - if (auto char_class = parse_character_class_escape(negated); char_class.has_value()) { - if (negated) - compares.append({ CharacterCompareType::TemporaryInverse, 1 }); - compares.append({ CharacterCompareType::CharClass, (ByteCodeValueType)char_class.value() }); - return true; - } - - PropertyEscape property {}; - if (parse_unicode_property_escape(property, negated)) { - if (negated) - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Inverse, 0 }); - property.visit( - [&](Unicode::Property property) { - if (Unicode::is_ecma262_string_property(property)) { - if (negated) { - set_error(Error::InvalidNameForProperty); - return; - } - - auto strings = Unicode::get_property_strings(property); - - if (m_parser_state.in_negated_character_class && any_of(strings, has_multiple_code_points)) { - set_error(Error::NegatedCharacterClassStrings); - return; - } - - if (!strings.is_empty()) { - auto string_set_index = m_parser_state.bytecode.string_set_table().set(move(strings)); - compares.empend(CompareTypeAndValuePair { CharacterCompareType::StringSet, string_set_index }); - } - } else { - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Property, (ByteCodeValueType)property.value() }); - } - }, - [&](Unicode::GeneralCategory general_category) { - compares.empend(CompareTypeAndValuePair { CharacterCompareType::GeneralCategory, (ByteCodeValueType)general_category.value() }); - }, - [&](Script script) { - if (script.is_extension) - compares.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)script.script.value() }); - else - compares.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)script.script.value() }); - }, - [](Empty&) { VERIFY_NOT_REACHED(); }); - return true; - } - - if (has_error()) - return false; - } - - m_parser_state.current_token = start_token; - m_parser_state.lexer.back(m_parser_state.lexer.tell() - start_lexer_index); - return false; -} - -bool ECMA262Parser::parse_unicode_property_escape(PropertyEscape& property, bool& negated) -{ - negated = false; - - if (try_skip("p"sv)) - negated = false; - else if (try_skip("P"sv)) - negated = true; - else - return false; - - auto parsed_property = read_unicode_property_escape(); - if (!parsed_property.has_value()) { - set_error(Error::InvalidNameForProperty); - return false; - } - - property = move(*parsed_property); - - return property.visit( - [this, negated](Unicode::Property property) { - if (Unicode::is_ecma262_string_property(property)) { - if (!m_parser_state.regex_options.has_flag_set(AllFlags::UnicodeSets) || negated) { - set_error(Error::InvalidNameForProperty); - return false; - } - } else if (!Unicode::is_ecma262_property(property)) { - set_error(Error::InvalidNameForProperty); - return false; - } - return true; - }, - [](Unicode::GeneralCategory) { return true; }, - [](Script) { return true; }, - [](Empty&) -> bool { VERIFY_NOT_REACHED(); }); -} - -FlyString ECMA262Parser::read_capture_group_specifier(bool take_starting_angle_bracket) -{ - static constexpr u32 const REPLACEMENT_CHARACTER = 0xFFFD; - constexpr u32 const ZERO_WIDTH_NON_JOINER { 0x200C }; - constexpr u32 const ZERO_WIDTH_JOINER { 0x200D }; - - if (take_starting_angle_bracket && !consume("<")) - return {}; - - StringBuilder builder; - - auto consume_code_point = [&] { - Utf8View utf_8_view { m_parser_state.lexer.source().substring_view(m_parser_state.lexer.tell() - 1) }; - if (utf_8_view.is_empty()) - return REPLACEMENT_CHARACTER; - u32 code_point = *utf_8_view.begin(); - auto characters = utf_8_view.byte_offset_of(1); - - while (characters-- > 0) - consume(); - - return code_point; - }; - - { - // The first character is limited to: https://tc39.es/ecma262/#prod-RegExpIdentifierStart - // RegExpIdentifierStart[UnicodeMode] :: - // IdentifierStartChar - // \ RegExpUnicodeEscapeSequence[+UnicodeMode] - // [~UnicodeMode] UnicodeLeadSurrogate UnicodeTrailSurrogate - - auto code_point = consume_code_point(); - - if (code_point == '\\' && match('u')) { - consume(); - - if (auto maybe_code_point = consume_escaped_code_point(true); maybe_code_point.has_value()) { - code_point = *maybe_code_point; - } else { - set_error(Error::InvalidNameForCaptureGroup); - return {}; - } - } - - if (is_ascii(code_point)) { - // The only valid ID_Start unicode characters in ascii are the letters. - if (!is_ascii_alpha(code_point) && code_point != '$' && code_point != '_') { - set_error(Error::InvalidNameForCaptureGroup); - return {}; - } - } else if (!Unicode::code_point_has_identifier_start_property(code_point)) { - set_error(Error::InvalidNameForCaptureGroup); - return {}; - } - builder.append_code_point(code_point); - } - - bool hit_end = false; - - // Any following characters are limited to: - // RegExpIdentifierPart[UnicodeMode] :: - // IdentifierPartChar - // \ RegExpUnicodeEscapeSequence[+UnicodeMode] - // [~UnicodeMode] UnicodeLeadSurrogate UnicodeTrailSurrogate - - while (match(TokenType::Char) || match(TokenType::Dollar) || match(TokenType::LeftCurly) || match(TokenType::RightCurly)) { - auto code_point = consume_code_point(); - - if (code_point == '>') { - hit_end = true; - break; - } - - if (code_point == '\\') { - if (!try_skip("u"sv)) { - set_error(Error::InvalidNameForCaptureGroup); - return {}; - } - if (auto maybe_code_point = consume_escaped_code_point(true); maybe_code_point.has_value()) { - code_point = *maybe_code_point; - } else { - set_error(Error::InvalidNameForCaptureGroup); - return {}; - } - } - - if (is_ascii(code_point)) { - // The only valid ID_Continue unicode characters in ascii are the letters and numbers. - if (!is_ascii_alphanumeric(code_point) && code_point != '$' && code_point != '_') { - set_error(Error::InvalidNameForCaptureGroup); - return {}; - } - } else if (code_point != ZERO_WIDTH_JOINER && code_point != ZERO_WIDTH_NON_JOINER) { - if (!Unicode::code_point_has_identifier_continue_property(code_point)) { - set_error(Error::InvalidNameForCaptureGroup); - return {}; - } - } - builder.append_code_point(code_point); - } - - auto name = MUST(builder.to_fly_string()); - if (!hit_end || name.is_empty()) - set_error(Error::InvalidNameForCaptureGroup); - - return name; -} - -Optional ECMA262Parser::read_unicode_property_escape() -{ - consume(TokenType::LeftCurly, Error::InvalidPattern); - - auto read_until = [&](Ts&&... terminators) { - auto start_token = m_parser_state.current_token; - size_t offset = 0; - - while (match(TokenType::Char)) { - if (m_parser_state.current_token.value().is_one_of(forward(terminators)...)) - break; - offset += consume().value().length(); - } - - return StringView { start_token.value().characters_without_null_termination(), offset }; - }; - - StringView property_type; - StringView property_name = read_until("="sv, "}"sv); - - if (try_skip("="sv)) { - if (property_name.is_empty()) - return {}; - property_type = property_name; - property_name = read_until("}"sv); - } - - consume(TokenType::RightCurly, Error::InvalidPattern); - - if (property_type.is_empty()) { - if (auto property = Unicode::property_from_string(property_name); property.has_value()) - return { *property }; - if (auto general_category = Unicode::general_category_from_string(property_name); general_category.has_value()) - return { *general_category }; - } else if ((property_type == "General_Category"sv) || (property_type == "gc"sv)) { - if (auto general_category = Unicode::general_category_from_string(property_name); general_category.has_value()) - return { *general_category }; - } else if ((property_type == "Script"sv) || (property_type == "sc"sv)) { - if (auto script = Unicode::script_from_string(property_name); script.has_value()) - return Script { *script, false }; - } else if ((property_type == "Script_Extensions"sv) || (property_type == "scx"sv)) { - if (auto script = Unicode::script_from_string(property_name); script.has_value()) - return Script { *script, true }; - } - - return {}; -} - -bool ECMA262Parser::parse_capture_group(ByteCode& stack, size_t& match_length_minimum, ParseFlags flags) -{ - consume(TokenType::LeftParen, Error::InvalidPattern); - - auto register_capture_group_in_current_scope = [&](auto identifier) { - m_capture_groups_in_scope.last().empend(identifier); - }; - - if (match(TokenType::Questionmark)) { - // Non-capturing group or group with specifier. - consume(); - - if (match(TokenType::Colon)) { - consume(); - ByteCode noncapture_group_bytecode; - size_t length = 0; - - enter_capture_group_scope(); - if (!parse_disjunction(noncapture_group_bytecode, length, flags)) - return set_error(Error::InvalidPattern); - clear_all_capture_groups_in_scope(stack); - exit_capture_group_scope(); - - consume(TokenType::RightParen, Error::MismatchingParen); - - stack.extend(move(noncapture_group_bytecode)); - match_length_minimum += length; - return true; - } - - if (match(TokenType::Char) || match(TokenType::HyphenMinus)) { - if (!match(TokenType::HyphenMinus)) { - auto value = m_parser_state.current_token.value(); - if (value.length() != 1 || !"ims"sv.contains(value[0])) - goto not_a_modifier; - } - - auto saved_position = m_parser_state.lexer.tell(); - auto reset_to_saved_position = [&] { - m_parser_state.lexer.retreat(m_parser_state.lexer.tell() - saved_position); - m_parser_state.current_token = m_parser_state.lexer.next(); - }; - - enum class ModifierParseResult : u8 { - Success, - InvalidGroup, - RepeatedFlag, - }; - - auto parse_modifier_flags = [&](bool& has_i, bool& has_m, bool& has_s) -> ModifierParseResult { - has_i = false; - has_m = false; - has_s = false; - - while (match(TokenType::Char)) { - auto value = m_parser_state.current_token.value(); - bool* flag_ptr; - switch (value[0]) { - case 'i': - flag_ptr = &has_i; - break; - case 'm': - flag_ptr = &has_m; - break; - case 's': - flag_ptr = &has_s; - break; - default: - return ModifierParseResult::InvalidGroup; - } - - if (*flag_ptr) - return ModifierParseResult::RepeatedFlag; - *flag_ptr = true; - consume(); - } - - return ModifierParseResult::Success; - }; - - bool add_i = false; - bool add_m = false; - bool add_s = false; - auto add_result = parse_modifier_flags(add_i, add_m, add_s); - - if (add_result == ModifierParseResult::InvalidGroup) - return set_error(Error::InvalidModifierGroup); - - if (add_result == ModifierParseResult::RepeatedFlag) - return set_error(Error::RepeatedModifierFlag); - - bool remove_i = false; - bool remove_m = false; - bool remove_s = false; - if (match(TokenType::HyphenMinus)) { - consume(); - auto remove_result = parse_modifier_flags(remove_i, remove_m, remove_s); - - if (remove_result == ModifierParseResult::InvalidGroup) - return set_error(Error::InvalidModifierGroup); - - if (remove_result == ModifierParseResult::RepeatedFlag) - return set_error(Error::RepeatedModifierFlag); - - if ((add_i && remove_i) || (add_m && remove_m) || (add_s && remove_s)) - return set_error(Error::RepeatedModifierFlag); - - if (!add_i && !add_m && !add_s && !remove_i && !remove_m && !remove_s) - return set_error(Error::InvalidModifierGroup); - } - - if (!match(TokenType::Colon)) { - reset_to_saved_position(); - goto not_a_modifier; - } - - consume(); - - auto current_options = to_underlying(m_parser_state.regex_options.value()); - FlagsUnderlyingType updated_options = current_options; - - auto update_modifier_flag = [&](bool add, bool remove, AllFlags flag) { - auto flag_value = static_cast(flag); - if (add) - updated_options |= flag_value; - if (remove) - updated_options &= ~flag_value; - }; - - update_modifier_flag(add_i, remove_i, AllFlags::Insensitive); - update_modifier_flag(add_m, remove_m, AllFlags::Multiline); - update_modifier_flag(add_s, remove_s, AllFlags::SingleLine); - - ByteCode modifier_group_bytecode; - size_t length = 0; - - stack.insert_bytecode_save_modifiers(updated_options); - - auto saved_parser_options = m_parser_state.regex_options; - m_parser_state.regex_options = AllOptions { static_cast(updated_options) }; - - enter_capture_group_scope(); - if (!parse_disjunction(modifier_group_bytecode, length, flags)) { - m_parser_state.regex_options = saved_parser_options; - return set_error(Error::InvalidPattern); - } - clear_all_capture_groups_in_scope(stack); - exit_capture_group_scope(); - - m_parser_state.regex_options = saved_parser_options; - - consume(TokenType::RightParen, Error::MismatchingParen); - - stack.extend(move(modifier_group_bytecode)); - stack.insert_bytecode_restore_modifiers(); - match_length_minimum += length; - return true; - } - - not_a_modifier: - if (consume("<")) { - ++m_parser_state.named_capture_groups_count; - auto group_index = ++m_parser_state.capture_groups_count; // Named capture groups count as normal capture groups too. - auto name = read_capture_group_specifier(); - - if (name.is_empty()) { - set_error(Error::InvalidNameForCaptureGroup); - return false; - } - - if (has_duplicate_in_current_alternative(name)) { - set_error(Error::DuplicateNamedCapture); - return false; - } - - auto& group_vector = m_parser_state.named_capture_groups.ensure(name); - group_vector.append({ group_index, m_current_alternative_id }); - - ByteCode capture_group_bytecode; - size_t length = 0; - enter_capture_group_scope(); - if (!parse_disjunction(capture_group_bytecode, length, flags)) - return set_error(Error::InvalidPattern); - clear_all_capture_groups_in_scope(stack); - exit_capture_group_scope(); - - register_capture_group_in_current_scope(group_index); - - consume(TokenType::RightParen, Error::MismatchingParen); - - stack.insert_bytecode_group_capture_left(group_index); - stack.extend(move(capture_group_bytecode)); - stack.insert_bytecode_group_capture_right(group_index, name); - - match_length_minimum += length; - - m_parser_state.capture_group_minimum_lengths.set(group_index, length); - return true; - } - - set_error(Error::InvalidCaptureGroup); - return false; - } - - auto group_index = ++m_parser_state.capture_groups_count; - enter_capture_group_scope(); - - ByteCode capture_group_bytecode; - size_t length = 0; - - if (!parse_disjunction(capture_group_bytecode, length, flags)) - return set_error(Error::InvalidPattern); - - clear_all_capture_groups_in_scope(stack); - exit_capture_group_scope(); - - register_capture_group_in_current_scope(group_index); - - stack.insert_bytecode_group_capture_left(group_index); - stack.extend(move(capture_group_bytecode)); - - m_parser_state.capture_group_minimum_lengths.set(group_index, length); - - consume(TokenType::RightParen, Error::MismatchingParen); - - stack.insert_bytecode_group_capture_right(group_index); - - match_length_minimum += length; - - return true; -} - -size_t ECMA262Parser::ensure_total_number_of_capturing_parenthesis() -{ - if (m_total_number_of_capturing_parenthesis.has_value()) - return m_total_number_of_capturing_parenthesis.value(); - - GenericLexer lexer { m_parser_state.lexer.source() }; - size_t count = 0; - while (!lexer.is_eof()) { - switch (lexer.peek()) { - case '\\': - lexer.consume(min(lexer.tell_remaining(), 2)); - continue; - case '[': - while (!lexer.is_eof()) { - if (lexer.consume_specific('\\')) { - if (lexer.is_eof()) - break; - lexer.consume(); - continue; - } - if (lexer.consume_specific(']')) { - break; - } - - if (lexer.is_eof()) - break; - lexer.consume(); - } - break; - case '(': - lexer.consume(); - if (lexer.consume_specific('?')) { - // non-capturing group '(?:', lookaround '(?<='/'(?value.first().group_index; - - m_parser_state.bytecode.at(unresolved_ref.bytecode_offset) = (ByteCodeValueType)group_index; - } - - return true; -} - -} diff --git a/Libraries/LibRegex/RegexParser.h b/Libraries/LibRegex/RegexParser.h deleted file mode 100644 index 3ae3d33a5c..0000000000 --- a/Libraries/LibRegex/RegexParser.h +++ /dev/null @@ -1,360 +0,0 @@ -/* - * Copyright (c) 2020, Emanuel Sprung - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include "RegexByteCode.h" -#include "RegexError.h" -#include "RegexLexer.h" -#include "RegexOptions.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace regex { - -class PosixExtendedParser; -class PosixBasicParser; -class ECMA262Parser; - -template -struct GenericParserTraits { - using OptionsType = T; -}; - -template -struct ParserTraits : public GenericParserTraits { -}; - -template<> -struct ParserTraits : public GenericParserTraits { -}; - -template<> -struct ParserTraits : public GenericParserTraits { -}; - -template<> -struct ParserTraits : public GenericParserTraits { -}; - -struct NamedCaptureGroup { - size_t group_index; - size_t alternative_id; -}; - -class REGEX_API Parser { -public: - struct Result { - Variant bytecode; - size_t capture_groups_count { 0 }; - size_t named_capture_groups_count { 0 }; - size_t match_length_minimum { 0 }; - Error error { Error::NoError }; - Token error_token {}; - Vector capture_groups {}; - AllOptions options {}; - - struct { - Optional> pure_substring_search; - // If populated, the pattern only accepts strings that start with a character in these ranges. - Vector starting_ranges; - Vector starting_ranges_insensitive; - bool only_start_of_line = false; - } optimization_data {}; - }; - - explicit Parser(Lexer& lexer) - : m_parser_state(lexer) - { - } - - Parser(Lexer& lexer, AllOptions regex_options) - : m_parser_state(lexer, regex_options) - { - } - - virtual ~Parser() = default; - - Result parse(Optional regex_options = {}); - bool has_error() const { return m_parser_state.error != Error::NoError; } - Error error() const { return m_parser_state.error; } - AllOptions options() const { return m_parser_state.regex_options; } - -protected: - virtual bool parse_internal(ByteCode&, size_t& match_length_minimum) = 0; - bool resolve_forward_named_references(); - - ALWAYS_INLINE bool match(TokenType type) const; - ALWAYS_INLINE bool match(char ch) const; - ALWAYS_INLINE bool match_ordinary_characters(); - ALWAYS_INLINE Token consume(); - ALWAYS_INLINE Token consume(TokenType type, Error error); - ALWAYS_INLINE bool consume(ByteString const&); - ALWAYS_INLINE Optional consume_escaped_code_point(bool unicode); - ALWAYS_INLINE bool try_skip(StringView); - ALWAYS_INLINE bool lookahead_any(StringView); - ALWAYS_INLINE unsigned char skip(); - ALWAYS_INLINE void back(size_t = 1); - ALWAYS_INLINE void reset(); - ALWAYS_INLINE bool done() const; - ALWAYS_INLINE bool set_error(Error error); - - size_t tell() const { return m_parser_state.current_token.position(); } - - struct ParserState { - Lexer& lexer; - Token current_token; - Error error = Error::NoError; - Token error_token { TokenType::Eof, 0, {} }; - ByteCode bytecode; - size_t capture_groups_count { 0 }; - size_t named_capture_groups_count { 0 }; - size_t match_length_minimum { 0 }; - bool greedy_lookaround { true }; - size_t repetition_mark_count { 0 }; - bool in_negated_character_class { false }; - AllOptions regex_options; - HashMap capture_group_minimum_lengths; - HashTable optional_capture_groups; - OrderedHashMap> named_capture_groups; - - struct UnresolvedNamedReference { - FlyString name; - size_t bytecode_offset; - }; - Vector unresolved_named_references; - - explicit ParserState(Lexer& lexer) - : lexer(lexer) - , current_token(lexer.next()) - { - } - explicit ParserState(Lexer& lexer, AllOptions regex_options) - : lexer(lexer) - , current_token(lexer.next()) - , regex_options(regex_options) - { - } - }; - - ParserState m_parser_state; -}; - -class REGEX_API AbstractPosixParser : public Parser { -protected: - explicit AbstractPosixParser(Lexer& lexer) - : Parser(lexer) - { - } - - AbstractPosixParser(Lexer& lexer, Optional::OptionsType> regex_options) - : Parser(lexer, regex_options.value_or({})) - { - } - - ALWAYS_INLINE bool parse_bracket_expression(Vector&, size_t&); -}; - -class REGEX_API PosixBasicParser final : public AbstractPosixParser { -public: - explicit PosixBasicParser(Lexer& lexer) - : AbstractPosixParser(lexer) - { - } - - PosixBasicParser(Lexer& lexer, Optional::OptionsType> regex_options) - : AbstractPosixParser(lexer, regex_options.value_or({})) - { - } - - ~PosixBasicParser() = default; - -private: - bool parse_internal(ByteCode&, size_t&) override; - - bool parse_root(ByteCode&, size_t&); - bool parse_re_expression(ByteCode&, size_t&); - bool parse_simple_re(ByteCode&, size_t&); - bool parse_nonduplicating_re(ByteCode&, size_t&); - bool parse_one_char_or_collation_element(ByteCode&, size_t&); - - constexpr static size_t number_of_addressable_capture_groups = 9; - size_t m_capture_group_minimum_lengths[number_of_addressable_capture_groups] { 0 }; - bool m_capture_group_seen[number_of_addressable_capture_groups] { false }; - size_t m_current_capture_group_depth { 0 }; -}; - -class REGEX_API PosixExtendedParser final : public AbstractPosixParser { - constexpr static auto default_options = static_cast(AllFlags::SingleLine) | static_cast(AllFlags::Internal_ConsiderNewline); - -public: - explicit PosixExtendedParser(Lexer& lexer) - : AbstractPosixParser(lexer, default_options) - { - } - - PosixExtendedParser(Lexer& lexer, Optional::OptionsType> regex_options) - : AbstractPosixParser(lexer, regex_options.value_or({}) | default_options.value()) - { - } - - ~PosixExtendedParser() = default; - -private: - ALWAYS_INLINE bool match_repetition_symbol(); - - bool parse_internal(ByteCode&, size_t&) override; - - bool parse_root(ByteCode&, size_t&); - ALWAYS_INLINE bool parse_sub_expression(ByteCode&, size_t&); - ALWAYS_INLINE bool parse_bracket_expression(ByteCode&, size_t&); - ALWAYS_INLINE bool parse_repetition_symbol(ByteCode&, size_t&); -}; - -class REGEX_API ECMA262Parser final : public Parser { - constexpr static ECMAScriptOptions default_options = static_cast(AllFlags::Internal_ConsiderNewline); - -public: - explicit ECMA262Parser(Lexer& lexer) - : Parser(lexer, default_options) - { - m_capture_groups_in_scope.empend(); - } - - ECMA262Parser(Lexer& lexer, Optional::OptionsType> regex_options) - : Parser(lexer, regex_options.value_or({}) | default_options.value()) - { - m_should_use_browser_extended_grammar = regex_options.has_value() && regex_options->has_flag_set(ECMAScriptFlags::BrowserExtended); - m_capture_groups_in_scope.empend(); - } - - ~ECMA262Parser() = default; - -private: - bool parse_internal(ByteCode&, size_t&) override; - - struct ParseFlags { - bool unicode { false }; - bool named { false }; - bool unicode_sets { false }; - }; - - enum class ReadDigitsInitialZeroState { - Allow, - Disallow, - }; - StringView read_digits_as_string(ReadDigitsInitialZeroState initial_zero = ReadDigitsInitialZeroState::Allow, bool hex = false, int max_count = -1, int min_count = -1); - Optional read_digits(ReadDigitsInitialZeroState initial_zero = ReadDigitsInitialZeroState::Allow, bool hex = false, int max_count = -1, int min_count = -1); - FlyString read_capture_group_specifier(bool take_starting_angle_bracket = false); - - struct Script { - Unicode::Script script {}; - bool is_extension { false }; - }; - using PropertyEscape = Variant; - Optional read_unicode_property_escape(); - - bool parse_pattern(ByteCode&, size_t&, ParseFlags); - bool parse_disjunction(ByteCode&, size_t&, ParseFlags); - bool parse_alternative(ByteCode&, size_t&, ParseFlags); - bool parse_term(ByteCode&, size_t&, ParseFlags); - bool parse_assertion(ByteCode&, size_t&, ParseFlags); - bool parse_atom(ByteCode&, size_t&, ParseFlags); - bool parse_quantifier(ByteCode&, size_t&, ParseFlags); - bool parse_interval_quantifier(Optional& repeat_min, Optional& repeat_max); - bool parse_atom_escape(ByteCode&, size_t&, ParseFlags); - bool parse_character_class(ByteCode&, size_t&, ParseFlags); - bool parse_capture_group(ByteCode&, size_t&, ParseFlags); - Optional parse_character_class_escape(bool& out_inverse, bool expect_backslash = false); - bool parse_nonempty_class_ranges(Vector&, ParseFlags); - bool parse_unicode_property_escape(PropertyEscape& property, bool& negated); - - bool parse_character_escape(Vector&, size_t&, ParseFlags); - - bool parse_class_set_expression(Vector&); - bool parse_class_union(Vector&); - bool parse_class_intersection(Vector&); - bool parse_class_subtraction(Vector&); - bool parse_class_set_range(Vector&); - bool parse_class_set_operand(Vector&); - bool parse_nested_class(Vector&); - Optional parse_class_set_character(); - - // Used only by B.1.4, Regular Expression Patterns (Extended for use in browsers) - bool parse_quantifiable_assertion(ByteCode&, size_t&, ParseFlags); - bool parse_extended_atom(ByteCode&, size_t&, ParseFlags); - bool parse_inner_disjunction(ByteCode& bytecode_stack, size_t& length, ParseFlags); - bool parse_invalid_braced_quantifier(); // Note: This function either parses and *fails*, or doesn't parse anything and returns false. - Optional parse_legacy_octal_escape(); - - bool has_duplicate_in_current_alternative(FlyString const& name); - - size_t ensure_total_number_of_capturing_parenthesis(); - - auto save_parser_state() - { - auto saved_token = m_parser_state.current_token; - auto saved_lexer_index = m_parser_state.lexer.tell(); - - return ArmedScopeGuard { [this, saved_token, saved_lexer_index] { - m_parser_state.current_token = saved_token; - m_parser_state.lexer.back(m_parser_state.lexer.tell() - saved_lexer_index); - } }; - } - - void enter_capture_group_scope() { m_capture_groups_in_scope.empend(); } - - void exit_capture_group_scope() - { - auto last = m_capture_groups_in_scope.take_last(); - m_capture_groups_in_scope.last().extend(move(last)); - } - - void clear_all_capture_groups_in_scope(ByteCode& stack) - { - for (auto& index : m_capture_groups_in_scope.last()) - stack.insert_bytecode_clear_capture_group(index); - } - - void mark_capture_groups_as_optional_from(size_t first_group) - { - for (size_t i = first_group + 1; i <= m_parser_state.capture_groups_count; ++i) - m_parser_state.optional_capture_groups.set(i); - } - - // ECMA-262's flavour of regex is a bit weird in that it allows backrefs to reference "future" captures, and such backrefs - // always match the empty string. So we have to know how many capturing parenthesis there are, but we don't want to always - // parse it twice, so we'll just do so when it's actually needed. - // Most patterns should have no need to ever populate this field. - Optional m_total_number_of_capturing_parenthesis; - - // We need to keep track of the current alternative's named capture groups, so we can check for duplicates. - size_t m_current_alternative_id { 0 }; - - // Keep the Annex B. behavior behind a flag, the users can enable it by passing the `ECMAScriptFlags::BrowserExtended` flag. - bool m_should_use_browser_extended_grammar { false }; - - // ECMA-262 basically requires that we clear the inner captures of a capture group before trying to match it, - // by requiring that (...)+ only contain the matches for the last iteration. - // To do that, we have to keep track of which capture groups are "in scope", so we can clear them as needed. - Vector> m_capture_groups_in_scope; -}; - -using PosixExtended = PosixExtendedParser; -using PosixBasic = PosixBasicParser; -using ECMA262 = ECMA262Parser; - -} - -using regex::ECMA262; -using regex::PosixBasic; -using regex::PosixExtended; diff --git a/Libraries/LibWeb/DOM/Node.cpp b/Libraries/LibWeb/DOM/Node.cpp index d4d80827a4..d7804f818f 100644 --- a/Libraries/LibWeb/DOM/Node.cpp +++ b/Libraries/LibWeb/DOM/Node.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include diff --git a/Libraries/LibWeb/HTML/StructuredSerialize.cpp b/Libraries/LibWeb/HTML/StructuredSerialize.cpp index 81ccbd8e10..66ca14033f 100644 --- a/Libraries/LibWeb/HTML/StructuredSerialize.cpp +++ b/Libraries/LibWeb/HTML/StructuredSerialize.cpp @@ -346,7 +346,7 @@ public: // { [[Type]]: "RegExp", [[RegExpMatcher]]: value.[[RegExpMatcher]], [[OriginalSource]]: value.[[OriginalSource]], // [[OriginalFlags]]: value.[[OriginalFlags]] }. else if (auto const* reg_exp_object = as_if(*object)) { - // NOTE: A Regex object is perfectly happy to be reconstructed with just the source+flags. + // NOTE: ECMAScriptRegex is perfectly happy to be reconstructed with just the source+flags. // In the future, we could optimize the work being done on the deserialize step by serializing // more of the internal state (the [[RegExpMatcher]] internal slot). serialized.encode(ValueTag::RegExpObject); diff --git a/Meta/Lagom/Fuzzers/FuzzRegexECMA262.cpp b/Meta/Lagom/Fuzzers/FuzzRegexECMA262.cpp index 39d79fd6dc..88b08c1e46 100644 --- a/Meta/Lagom/Fuzzers/FuzzRegexECMA262.cpp +++ b/Meta/Lagom/Fuzzers/FuzzRegexECMA262.cpp @@ -5,7 +5,7 @@ */ #include -#include +#include #include #include @@ -13,6 +13,6 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size) { AK::set_debug_enabled(false); auto pattern = StringView(static_cast(data), size); - [[maybe_unused]] auto re = Regex(pattern); + [[maybe_unused]] auto re = regex::ECMAScriptRegex::compile(pattern, {}); return 0; } diff --git a/Meta/Lagom/Fuzzers/FuzzRegexPosixBasic.cpp b/Meta/Lagom/Fuzzers/FuzzRegexPosixBasic.cpp deleted file mode 100644 index 79dbbcb5df..0000000000 --- a/Meta/Lagom/Fuzzers/FuzzRegexPosixBasic.cpp +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2021, Luke Wilde - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include - -extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size) -{ - AK::set_debug_enabled(false); - auto pattern = StringView(static_cast(data), size); - [[maybe_unused]] auto re = Regex(pattern); - return 0; -} diff --git a/Meta/Lagom/Fuzzers/FuzzRegexPosixExtended.cpp b/Meta/Lagom/Fuzzers/FuzzRegexPosixExtended.cpp deleted file mode 100644 index bded37547f..0000000000 --- a/Meta/Lagom/Fuzzers/FuzzRegexPosixExtended.cpp +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2020, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include - -extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size) -{ - AK::set_debug_enabled(false); - auto pattern = StringView(static_cast(data), size); - [[maybe_unused]] auto re = Regex(pattern); - return 0; -} diff --git a/Meta/Lagom/Fuzzers/fuzzers.cmake b/Meta/Lagom/Fuzzers/fuzzers.cmake index ebbf10f640..30f3abe76e 100644 --- a/Meta/Lagom/Fuzzers/fuzzers.cmake +++ b/Meta/Lagom/Fuzzers/fuzzers.cmake @@ -17,8 +17,6 @@ set(FUZZER_TARGETS PEM PNGLoader RegexECMA262 - RegexPosixBasic - RegexPosixExtended RSAKeyParsing SHA1 SHA256 @@ -59,8 +57,6 @@ set(FUZZER_DEPENDENCIES_PEM LibCrypto) set(FUZZER_DEPENDENCIES_PNGLoader LibGfx) set(FUZZER_DEPENDENCIES_Poly1305 LibCrypto) set(FUZZER_DEPENDENCIES_RegexECMA262 LibRegex) -set(FUZZER_DEPENDENCIES_RegexPosixBasic LibRegex) -set(FUZZER_DEPENDENCIES_RegexPosixExtended LibRegex) set(FUZZER_DEPENDENCIES_RSAKeyParsing LibCrypto) set(FUZZER_DEPENDENCIES_SHA1 LibCrypto) set(FUZZER_DEPENDENCIES_SHA256 LibCrypto) diff --git a/Tests/LibRegex/TestRegex.cpp b/Tests/LibRegex/TestRegex.cpp index 06c482e711..1fef979a76 100644 --- a/Tests/LibRegex/TestRegex.cpp +++ b/Tests/LibRegex/TestRegex.cpp @@ -1,1912 +1,80 @@ /* - * Copyright (c) 2020, Emanuel Sprung + * Copyright (c) 2026-present, the Ladybird developers. * * SPDX-License-Identifier: BSD-2-Clause */ -#include // import first, to prevent warning of VERIFY* redefinition +#include -#include -#include -#include -#include -#include -#include -#include +#include -static ECMAScriptOptions match_test_api_options(ECMAScriptOptions const options) +TEST_CASE(compile_rejects_invalid_pattern) { - return options; + auto regex = regex::ECMAScriptRegex::compile("("sv, {}); + EXPECT(regex.is_error()); } -static PosixOptions match_test_api_options(PosixOptions const options) +TEST_CASE(exec_tracks_named_capture_slots) { - return options; -} - -template -static constexpr ECMAScriptFlags combine_flags(Flags&&... flags) -requires((IsSame && ...)) -{ - return static_cast((static_cast(flags) | ...)); -} - -TEST_CASE(regex_options_ecmascript) -{ - ECMAScriptOptions eo; - eo |= ECMAScriptFlags::Global; - - EXPECT(eo.has_flag_set(ECMAScriptFlags::Global)); - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Insensitive)); - - eo = match_test_api_options(ECMAScriptFlags::Global | ECMAScriptFlags::Insensitive | ECMAScriptFlags::Sticky); - EXPECT(eo.has_flag_set(ECMAScriptFlags::Global)); - EXPECT(eo.has_flag_set(ECMAScriptFlags::Insensitive)); - EXPECT(eo.has_flag_set(ECMAScriptFlags::Sticky)); - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Unicode)); - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Multiline)); - EXPECT(!eo.has_flag_set(ECMAScriptFlags::SingleLine)); - - eo &= ECMAScriptFlags::Insensitive; - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Global)); - EXPECT(eo.has_flag_set(ECMAScriptFlags::Insensitive)); - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Multiline)); - - eo &= ECMAScriptFlags::Sticky; - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Global)); - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Insensitive)); - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Multiline)); - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Sticky)); - - eo = ~ECMAScriptFlags::Insensitive; - EXPECT(eo.has_flag_set(ECMAScriptFlags::Global)); - EXPECT(!eo.has_flag_set(ECMAScriptFlags::Insensitive)); - EXPECT(eo.has_flag_set(ECMAScriptFlags::Multiline)); - EXPECT(eo.has_flag_set(ECMAScriptFlags::Sticky)); -} - -TEST_CASE(regex_options_posix) -{ - PosixOptions eo; - eo |= PosixFlags::Global; - - EXPECT(eo.has_flag_set(PosixFlags::Global)); - EXPECT(!eo.has_flag_set(PosixFlags::Insensitive)); - - eo = match_test_api_options(PosixFlags::Global | PosixFlags::Insensitive | PosixFlags::MatchNotBeginOfLine); - EXPECT(eo.has_flag_set(PosixFlags::Global)); - EXPECT(eo.has_flag_set(PosixFlags::Insensitive)); - EXPECT(eo.has_flag_set(PosixFlags::MatchNotBeginOfLine)); - EXPECT(!eo.has_flag_set(PosixFlags::Unicode)); - EXPECT(!eo.has_flag_set(PosixFlags::Multiline)); - - eo &= PosixFlags::Insensitive; - EXPECT(!eo.has_flag_set(PosixFlags::Global)); - EXPECT(eo.has_flag_set(PosixFlags::Insensitive)); - EXPECT(!eo.has_flag_set(PosixFlags::Multiline)); - - eo &= PosixFlags::MatchNotBeginOfLine; - EXPECT(!eo.has_flag_set(PosixFlags::Global)); - EXPECT(!eo.has_flag_set(PosixFlags::Insensitive)); - EXPECT(!eo.has_flag_set(PosixFlags::Multiline)); - - eo = ~PosixFlags::Insensitive; - EXPECT(eo.has_flag_set(PosixFlags::Global)); - EXPECT(!eo.has_flag_set(PosixFlags::Insensitive)); - EXPECT(eo.has_flag_set(PosixFlags::Multiline)); -} - -TEST_CASE(regex_lexer) -{ - Lexer l("/[.*+?^${}()|[\\]\\\\]/g"sv); - EXPECT(l.next().type() == regex::TokenType::Slash); - EXPECT(l.next().type() == regex::TokenType::LeftBracket); - EXPECT(l.next().type() == regex::TokenType::Period); - EXPECT(l.next().type() == regex::TokenType::Asterisk); - EXPECT(l.next().type() == regex::TokenType::Plus); - EXPECT(l.next().type() == regex::TokenType::Questionmark); - EXPECT(l.next().type() == regex::TokenType::Circumflex); - EXPECT(l.next().type() == regex::TokenType::Dollar); - EXPECT(l.next().type() == regex::TokenType::LeftCurly); - EXPECT(l.next().type() == regex::TokenType::RightCurly); - EXPECT(l.next().type() == regex::TokenType::LeftParen); - EXPECT(l.next().type() == regex::TokenType::RightParen); - EXPECT(l.next().type() == regex::TokenType::Pipe); - EXPECT(l.next().type() == regex::TokenType::LeftBracket); - EXPECT(l.next().type() == regex::TokenType::EscapeSequence); - EXPECT(l.next().type() == regex::TokenType::EscapeSequence); - EXPECT(l.next().type() == regex::TokenType::RightBracket); - EXPECT(l.next().type() == regex::TokenType::Slash); - EXPECT(l.next().type() == regex::TokenType::Char); -} - -TEST_CASE(parser_error_parens) -{ - ByteString pattern = "test()test"; - Lexer l(pattern); - PosixExtendedParser p(l); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::EmptySubExpression); -} - -TEST_CASE(parser_error_special_characters_used_at_wrong_place) -{ - ByteString pattern; - Vector chars = { '*', '+', '?', '{' }; - StringBuilder b; - - Lexer l; - PosixExtended p(l); - - for (auto& ch : chars) { - // First in ere - b.clear(); - b.append(ch); - pattern = b.to_byte_string(); - l.set_source(pattern); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::InvalidRepetitionMarker); - - // After vertical line - b.clear(); - b.append("a|"sv); - b.append(ch); - pattern = b.to_byte_string(); - l.set_source(pattern); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::InvalidRepetitionMarker); - - // After circumflex - b.clear(); - b.append('^'); - b.append(ch); - pattern = b.to_byte_string(); - l.set_source(pattern); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::InvalidRepetitionMarker); + auto regex = MUST(regex::ECMAScriptRegex::compile("(?foo)(bar)"sv, {})); - // After dollar - b.clear(); - b.append('$'); - b.append(ch); - pattern = b.to_byte_string(); - l.set_source(pattern); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::InvalidRepetitionMarker); + EXPECT_EQ(regex.capture_count(), 2u); + EXPECT_EQ(regex.total_groups(), 3u); + EXPECT_EQ(regex.named_groups().size(), 1u); + EXPECT_EQ(regex.named_groups()[0].name, "word"sv); + EXPECT_EQ(regex.named_groups()[0].index, 1u); - // After left parens - b.clear(); - b.append('('); - b.append(ch); - b.append(')'); - pattern = b.to_byte_string(); - l.set_source(pattern); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::InvalidRepetitionMarker); - } + EXPECT_EQ(regex.exec(u"foobar"sv, 0), regex::MatchResult::Match); + EXPECT_EQ(regex.capture_slot(0), 0); + EXPECT_EQ(regex.capture_slot(1), 6); + EXPECT_EQ(regex.capture_slot(2), 0); + EXPECT_EQ(regex.capture_slot(3), 3); + EXPECT_EQ(regex.capture_slot(4), 3); + EXPECT_EQ(regex.capture_slot(5), 6); } -TEST_CASE(parser_error_vertical_line_used_at_wrong_place) +TEST_CASE(exec_reports_unmatched_optional_groups) { - Lexer l; - PosixExtended p(l); + auto regex = MUST(regex::ECMAScriptRegex::compile("(foo)?bar"sv, {})); - // First in ere - l.set_source("|asdf"sv); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::EmptySubExpression); - - // Last in ere - l.set_source("asdf|"sv); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::EmptySubExpression); - - // After left parens - l.set_source("(|asdf)"sv); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::EmptySubExpression); - - // Proceed right parens - l.set_source("(asdf)|"sv); - p.parse(); - EXPECT(p.has_error()); - EXPECT(p.error() == regex::Error::EmptySubExpression); -} - -TEST_CASE(catch_all_first) -{ - Regex re("^.*$"); - RegexResult m; - re.match("Hello World"sv, m); - EXPECT(m.count == 1); - EXPECT(re.match("Hello World"sv, m)); + EXPECT_EQ(regex.exec(u"bar"sv, 0), regex::MatchResult::Match); + EXPECT_EQ(regex.capture_slot(0), 0); + EXPECT_EQ(regex.capture_slot(1), 3); + EXPECT_EQ(regex.capture_slot(2), -1); + EXPECT_EQ(regex.capture_slot(3), -1); } -TEST_CASE(catch_all) +TEST_CASE(test_honors_ignore_case) { - Regex re("^.*$", PosixFlags::Global); - - EXPECT(re.has_match("Hello World"sv)); - EXPECT(re.match("Hello World"sv).success); - EXPECT(re.match("Hello World"sv).count == 1); + auto regex = MUST(regex::ECMAScriptRegex::compile("casesensitive"sv, { .ignore_case = true })); - EXPECT(has_match("Hello World"sv, re)); - auto res = match("Hello World"sv, re); - EXPECT(res.success); - EXPECT(res.count == 1); - EXPECT(res.matches.size() == 1); - EXPECT(res.matches.first().view == "Hello World"); -} - -TEST_CASE(catch_all_again) -{ - Regex re("^.*$", PosixFlags::Extra); - EXPECT_EQ(has_match("Hello World"sv, re), true); + EXPECT_EQ(regex.test(u"CaseSensitive"sv, 0), regex::MatchResult::Match); + EXPECT_EQ(regex.test(u"something else"sv, 0), regex::MatchResult::NoMatch); } -TEST_CASE(catch_all_newline) +TEST_CASE(find_all_returns_non_overlapping_matches) { - Regex re("^.*$", PosixFlags::Multiline); - RegexResult result; - String aaa = "Hello World\nTest\n1234\n"_string; - auto lambda = [&]() { - result = match(aaa, re); - EXPECT_EQ(result.success, true); - }; - lambda(); - EXPECT_EQ(result.count, 3u); - EXPECT_EQ(result.matches.at(0).view, "Hello World"); - EXPECT_EQ(result.matches.at(1).view, "Test"); - EXPECT_EQ(result.matches.at(2).view, "1234"); -} - -TEST_CASE(catch_all_newline_view) -{ - Regex re("^.*$", PosixFlags::Multiline); - RegexResult result; - - String aaa = "Hello World\nTest\n1234\n"_string; - result = match(aaa, re); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.count, 3u); - ByteString str = "Hello World"; - EXPECT_EQ(result.matches.at(0).view, str.view()); - EXPECT_EQ(result.matches.at(1).view, "Test"); - EXPECT_EQ(result.matches.at(2).view, "1234"); -} - -TEST_CASE(catch_all_newline_2) -{ - Regex re("^.*$"); - RegexResult result; - result = match("Hello World\nTest\n1234\n"sv, re, PosixFlags::Multiline); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.count, 3u); - EXPECT_EQ(result.matches.at(0).view, "Hello World"); - EXPECT_EQ(result.matches.at(1).view, "Test"); - EXPECT_EQ(result.matches.at(2).view, "1234"); - - result = match("Hello World\nTest\n1234\n"sv, re); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.count, 1u); - EXPECT_EQ(result.matches.at(0).view, "Hello World\nTest\n1234\n"); -} - -TEST_CASE(match_all_character_class) -{ - Regex re("[[:alpha:]]"); - String str = "[Window]\nOpacity=255\nAudibleBeep=0\n"_string; - RegexResult result = match(str, re, PosixFlags::Global); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.count, 24u); - EXPECT_EQ(result.matches.at(0).view, "W"); - EXPECT_EQ(result.matches.at(1).view, "i"); - EXPECT_EQ(result.matches.at(2).view, "n"); -} - -TEST_CASE(match_character_class_with_assertion) -{ - Regex re("[[:alpha:]]+$"); - String str = "abcdef"_string; - RegexResult result = match(str, re); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.count, 1u); -} - -TEST_CASE(example_for_git_commit) -{ - Regex re("^.*$"); - auto result = re.match("Well, hello friends!\nHello World!"sv); - - EXPECT(result.success); - EXPECT(result.count == 1); - EXPECT(result.matches.at(0).view.starts_with("Well"sv)); - EXPECT(result.matches.at(0).view.length() == 33); - - EXPECT(re.has_match("Well,...."sv)); - - result = re.match("Well, hello friends!\nHello World!"sv, PosixFlags::Multiline); - - EXPECT(result.success); - EXPECT(result.count == 2); - EXPECT(result.matches.at(0).view == "Well, hello friends!"); - EXPECT(result.matches.at(1).view == "Hello World!"); -} - -TEST_CASE(email_address) -{ - Regex re("^[A-Z0-9a-z._%+-]{1,64}@([A-Za-z0-9-]{1,63}\\.){1,125}[A-Za-z]{2,63}$"); - EXPECT(re.has_match("hello.world@domain.tld"sv)); - EXPECT(re.has_match("this.is.a.very_long_email_address@world.wide.web"sv)); -} - -TEST_CASE(ini_file_entries) -{ - Regex re("[[:alpha:]]*=([[:digit:]]*)|\\[(.*)\\]"); - RegexResult result; - - if constexpr (REGEX_DEBUG) { - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - } - - ByteString haystack = "[Window]\nOpacity=255\nAudibleBeep=0\n"; - EXPECT_EQ(re.search(haystack.view(), result, PosixFlags::Multiline), true); - EXPECT_EQ(result.count, 3u); - - if constexpr (REGEX_DEBUG) { - for (auto& v : result.matches) - fprintf(stderr, "%s\n", v.view.to_byte_string().characters()); - } - - EXPECT_EQ(result.matches.at(0).view, "[Window]"); - EXPECT_EQ(result.capture_group_matches.at(0).at(1).view, "Window"); - EXPECT_EQ(result.matches.at(1).view, "Opacity=255"); - EXPECT_EQ(result.matches.at(1).line, 1u); - EXPECT_EQ(result.matches.at(1).column, 0u); - EXPECT_EQ(result.capture_group_matches.at(1).at(0).view, "255"); - EXPECT_EQ(result.capture_group_matches.at(1).at(0).line, 1u); - EXPECT_EQ(result.capture_group_matches.at(1).at(0).column, 8u); - EXPECT_EQ(result.matches.at(2).view, "AudibleBeep=0"); - EXPECT_EQ(result.capture_group_matches.at(2).at(0).view, "0"); - EXPECT_EQ(result.capture_group_matches.at(2).at(0).line, 2u); - EXPECT_EQ(result.capture_group_matches.at(2).at(0).column, 12u); -} - -TEST_CASE(ini_file_entries2) -{ - Regex re("[[:alpha:]]*=([[:digit:]]*)"); - RegexResult result; - - ByteString haystack = "ViewMode=Icon"; - - EXPECT_EQ(re.match(haystack.view(), result), false); - EXPECT_EQ(result.count, 0u); - - EXPECT_EQ(re.search(haystack.view(), result), true); - EXPECT_EQ(result.count, 1u); -} - -TEST_CASE(named_capture_group) -{ - Regex re("[[:alpha:]]*=(?[[:digit:]]*)"); - RegexResult result; - - if constexpr (REGEX_DEBUG) { - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - } - - String haystack = "[Window]\nOpacity=255\nAudibleBeep=0\n"_string; - EXPECT_EQ(re.search(haystack, result, PosixFlags::Multiline), true); - EXPECT_EQ(result.count, 2u); - EXPECT_EQ(result.matches.at(0).view, "Opacity=255"); - EXPECT_EQ(result.capture_group_matches.at(0).at(0).view, "255"); - EXPECT_EQ(re.parser_result.bytecode.visit([&](auto& bytecode) { return bytecode.get_string(result.capture_group_matches.at(0).at(0).capture_group_name); }), "Test"); - EXPECT_EQ(result.matches.at(1).view, "AudibleBeep=0"); - EXPECT_EQ(result.capture_group_matches.at(1).at(0).view, "0"); - EXPECT_EQ(re.parser_result.bytecode.visit([&](auto& bytecode) { return bytecode.get_string(result.capture_group_matches.at(1).at(0).capture_group_name); }), "Test"); -} - -TEST_CASE(ecma262_named_capture_group_with_dollar_sign) -{ - Regex re("[a-zA-Z]*=(?<$Test$>[0-9]*)"); - RegexResult result; - - if constexpr (REGEX_DEBUG) { - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - } - - String haystack = "[Window]\nOpacity=255\nAudibleBeep=0\n"_string; - EXPECT_EQ(re.search(haystack, result, ECMAScriptFlags::Multiline), true); - EXPECT_EQ(result.count, 2u); - EXPECT_EQ(result.matches.at(0).view, "Opacity=255"); - EXPECT_EQ(result.capture_group_matches.at(0).at(0).view, "255"); - EXPECT_EQ(re.parser_result.bytecode.visit([&](auto& bytecode) { return bytecode.get_string(result.capture_group_matches.at(0).at(0).capture_group_name); }), "$Test$"); - EXPECT_EQ(result.matches.at(1).view, "AudibleBeep=0"); - EXPECT_EQ(result.capture_group_matches.at(1).at(0).view, "0"); - EXPECT_EQ(re.parser_result.bytecode.visit([&](auto& bytecode) { return bytecode.get_string(result.capture_group_matches.at(1).at(0).capture_group_name); }), "$Test$"); -} - -TEST_CASE(a_star) -{ - Regex re("a*"); - RegexResult result; - - if constexpr (REGEX_DEBUG) { - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - } - - ByteString haystack = "[Window]\nOpacity=255\nAudibleBeep=0\n"; - EXPECT_EQ(re.search(haystack.view(), result, PosixFlags::Multiline), true); - EXPECT_EQ(result.count, 32u); - if (result.count == 32u) { - EXPECT_EQ(result.matches.at(0).view.length(), 0u); - EXPECT_EQ(result.matches.at(10).view.length(), 1u); - EXPECT_EQ(result.matches.at(10).view, "a"); - EXPECT_EQ(result.matches.at(31).view.length(), 0u); - } -} - -TEST_CASE(simple_period_end_benchmark) -{ - Regex re("hello.$"); - RegexResult m; - EXPECT_EQ(re.search("Hello1"sv, m), false); - EXPECT_EQ(re.search("hello1hello1"sv, m), true); - EXPECT_EQ(re.search("hello2hell"sv, m), false); - EXPECT_EQ(re.search("hello?"sv, m), true); -} - -TEST_CASE(posix_extended_nested_capture_group) -{ - Regex re("(h(e(?llo)))"); // group 0 -> "hello", group 1 -> "ello", group 2/"llo" -> "llo" - auto result = re.match("hello"sv); - EXPECT(result.success); - EXPECT_EQ(result.capture_group_matches.size(), 1u); - EXPECT_EQ(result.capture_group_matches[0].size(), 3u); - EXPECT_EQ(result.capture_group_matches[0][0].view, "hello"sv); - EXPECT_EQ(result.capture_group_matches[0][1].view, "ello"sv); - EXPECT_EQ(result.capture_group_matches[0][2].view, "llo"sv); -} - -auto parse_test_case_long_disjunction_chain = ByteString::repeated("a|"sv, 100000); - -TEST_CASE(ECMA262_parse) -{ - struct _test { - StringView pattern; - regex::Error expected_error { regex::Error::NoError }; - regex::ECMAScriptFlags flags {}; - }; - - _test const tests[] { - { "^hello.$"sv }, - { "^(hello.)$"sv }, - { "^h{0,1}ello.$"sv }, - { "^hello\\W$"sv }, - { "^hell\\w.$"sv }, - { "^hell\\x6f1$"sv }, // ^hello1$ - { "^hel(?:l\\w).$"sv }, - { "^hel(?l\\w).$"sv }, - { "^[-a-zA-Z\\w\\s]+$"sv }, - { "\\bhello\\B"sv }, - { "^[\\w+/_-]+[=]{0,2}$"sv }, // #4189 - { "^(?:[^<]*(<[\\w\\W]+>)[^>]*$|#([\\w\\-]*)$)"sv }, // #4189 - { "\\/"sv }, // #4189 - { ",/=-:"sv }, // #4243 - { "\\x"sv }, // Even invalid escapes are allowed if ~unicode. - { "\\x1"sv }, // Even invalid escapes are allowed if ~unicode. - { "\\x1"sv, regex::Error::InvalidPattern, regex::ECMAScriptFlags::Unicode }, - { "\\x11"sv }, - { "\\x11"sv, regex::Error::NoError, regex::ECMAScriptFlags::Unicode }, - { "\\"sv, regex::Error::InvalidTrailingEscape }, - { "(?"sv, regex::Error::InvalidCaptureGroup }, - { "\\u1234"sv, regex::Error::NoError, regex::ECMAScriptFlags::Unicode }, - { "[\\u1234]"sv, regex::Error::NoError, regex::ECMAScriptFlags::Unicode }, - { "\\u1"sv, regex::Error::InvalidPattern, regex::ECMAScriptFlags::Unicode }, - { "[\\u1]"sv, regex::Error::InvalidPattern, regex::ECMAScriptFlags::Unicode }, - { ",(?"sv, regex::Error::InvalidCaptureGroup }, // #4583 - { "{1}"sv, regex::Error::InvalidPattern }, - { "{1,2}"sv, regex::Error::InvalidPattern }, - { "\\uxxxx"sv, regex::Error::NoError }, - { "\\uxxxx"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\ud83d"sv, regex::Error::NoError, ECMAScriptFlags::Unicode }, - { "\\ud83d\\uxxxx"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\u{0}"sv }, - { "\\u{0}"sv, regex::Error::NoError, ECMAScriptFlags::Unicode }, - { "\\u{10ffff}"sv, regex::Error::NoError, ECMAScriptFlags::Unicode }, - { "\\u{10ffff"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\u{10ffffx"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\u{110000}"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\p"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\p{"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\p{}"sv, regex::Error::InvalidNameForProperty, ECMAScriptFlags::Unicode }, - { "\\p{AsCiI}"sv, regex::Error::InvalidNameForProperty, ECMAScriptFlags::Unicode }, - { "\\p{hello friends}"sv, regex::Error::InvalidNameForProperty, ECMAScriptFlags::Unicode }, - { "\\p{Prepended_Concatenation_Mark}"sv, regex::Error::InvalidNameForProperty, ECMAScriptFlags::Unicode }, - { "\\p{ASCII}"sv, regex::Error::NoError, ECMAScriptFlags::Unicode }, - { "\\\\p{1}"sv, regex::Error::NoError, ECMAScriptFlags::Unicode }, - { "\\\\p{AsCiI}"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\\\p{ASCII}"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\c"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "\\c"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "[\\c]"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "[\\c]"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\c`"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "\\c`"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "[\\c`]"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "[\\c`]"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\A"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "\\A"sv, regex::Error::InvalidCharacterClass, ECMAScriptFlags::Unicode }, - { "[\\A]"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "[\\A]"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\0"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "\\0"sv, regex::Error::NoError, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::BrowserExtended) }, - { "\\00"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "\\00"sv, regex::Error::InvalidCharacterClass, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::BrowserExtended) }, - { "[\\0]"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "[\\0]"sv, regex::Error::NoError, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::BrowserExtended) }, - { "[\\00]"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "[\\00]"sv, regex::Error::InvalidPattern, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::BrowserExtended) }, - { "\\^\\$\\\\\\.\\*\\+\\?\\(\\)\\[\\]\\{\\}\\|\\/"sv, regex::Error::NoError, ECMAScriptFlags::Unicode }, - { "[\\^\\$\\\\\\.\\*\\+\\?\\(\\)\\[\\]\\{\\}\\|\\/]"sv, regex::Error::NoError, ECMAScriptFlags::Unicode }, - { "]"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "]"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\]"sv, regex::Error::NoError, ECMAScriptFlags::Unicode }, - { "}"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, - { "}"sv, regex::Error::InvalidPattern, ECMAScriptFlags::Unicode }, - { "\\}"sv, regex::Error::NoError, ECMAScriptFlags::Unicode }, - { "a{9007199254740991}"sv }, // 2^53 - 1 - { "a{9007199254740991,}"sv }, - { "a{9007199254740991,9007199254740991}"sv }, - { "a{9007199254740992}"sv }, - { "a{9007199254740992,}"sv }, - { "a{9007199254740991,9007199254740992}"sv }, - { "a{9007199254740992,9007199254740991}"sv }, - { "a{9007199254740992,9007199254740992}"sv }, - { "a{1,99999999999999999999999999999999999999999999999999}"sv }, - { "a{99999999999999999999999999999999999999999999999999,1}"sv, regex::Error::InvalidBraceContent }, - { "a{99999999999999999999999999999999999999999999999999}"sv }, - { "a{2147483647}"sv }, // 2^31 - 1 - { "a{2147483648}"sv }, // 2^31 - { "a{2147483648,2147483647}"sv }, - { "a{2147483647,2147483646}"sv, regex::Error::InvalidBraceContent }, - { "(?a)(?b)"sv, regex::Error::DuplicateNamedCapture }, - { "(?a)(?b)(?c)"sv, regex::Error::DuplicateNamedCapture }, - { "(?(?a))"sv, regex::Error::DuplicateNamedCapture }, - { "(?:(?a)|(?a)(?b))(?:(?c)|(?d))"sv }, // Duplicate named capturing groups in separate alternatives should parse correctly - { "(?<1a>a)"sv, regex::Error::InvalidNameForCaptureGroup }, - { "(?<\\a>a)"sv, regex::Error::InvalidNameForCaptureGroup }, - { "(?<\ta>a)"sv, regex::Error::InvalidNameForCaptureGroup }, - { "(?<$$_$$>a)"sv }, - { "(?<ΓΏ>a)"sv }, - { "(?<𝓑𝓻𝓸𝔀𝓷>a)"sv }, - { "((?=lg)?[vl]k\\-?\\d{3}) bui| 3\\.[-\\w; ]{10}lg?-([06cv9]{3,4})"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, // #12373, quantifiable assertions. - { parse_test_case_long_disjunction_chain.view() }, // A whole lot of disjunctions, should not overflow the stack. - { "(\"|')(?:(?!\\2)[^\\\\\\r\\n]|\\\\.)*\\2"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, // LegacyOctalEscapeSequence should not consume too many chars (and should not crash) - // #18324, Capture group counter skipped past EOF. - { "\\1[\\"sv, regex::Error::InvalidNumber }, - { "(?ii:a)"sv, regex::Error::RepeatedModifierFlag }, - { "(?i-i:a)"sv, regex::Error::RepeatedModifierFlag }, - { "(?-ii:a)"sv, regex::Error::RepeatedModifierFlag }, - { "(?-:a)"sv, regex::Error::InvalidModifierGroup }, - { "(?-ig:a)"sv, regex::Error::InvalidModifierGroup }, - { "(?-x:a)"sv, regex::Error::InvalidModifierGroup }, - { "(?i)"sv, regex::Error::InvalidCaptureGroup }, - { "(?-i)"sv, regex::Error::InvalidCaptureGroup }, - }; - - for (auto& test : tests) { - Regex re(test.pattern, test.flags); - EXPECT_EQ(re.parser_result.error, test.expected_error); - if constexpr (REGEX_DEBUG) { - dbgln("\n"); - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - dbgln("\n"); - } - } -} - -TEST_CASE(ECMA262_match) -{ - constexpr auto global_multiline = ECMAScriptFlags::Global | ECMAScriptFlags::Multiline; - - struct _test { - StringView pattern; - StringView subject; - bool matches { true }; - ECMAScriptFlags options {}; - }; - constexpr _test tests[] { - { "^hello.$"sv, "hello1"sv }, - { "^(hello.)$"sv, "hello1"sv }, - { "^h{0,1}ello.$"sv, "ello1"sv }, - { "^hello\\W$"sv, "hello!"sv }, - { "^hell\\w.$"sv, "hellx!"sv }, - { "^hell\\x6f1$"sv, "hello1"sv }, - { "^hel(?l.)1$"sv, "hello1"sv }, - { "^hel(?l.)1*\\k.$"sv, "hello1lo1"sv }, - { "^[-a-z1-3\\s]+$"sv, "hell2 o1"sv }, - { "^[\\0-\\x1f]$"sv, "\n"sv }, - { .pattern = "\\bhello\\B"sv, .subject = "hello1"sv, .options = ECMAScriptFlags::Global }, - { "\\b.*\\b"sv, "hello1"sv }, - { "[^\\D\\S]{2}"sv, "1 "sv, false }, - { "bar(?=f.)foo"sv, "barfoo"sv }, - { "bar(?=foo)bar"sv, "barbar"sv, false }, - { "bar(?!foo)bar"sv, "barbar"sv, true }, - { "bar(?!bar)bar"sv, "barbar"sv, false }, - { "bar.*(?<=foo)"sv, "barbar"sv, false }, - { "bar.*(?|\\/|\\/=|:|::|;|<|<<|<<=|<=|=|==|===|>|>=|>>|>>=|>>>|>>>=|[?@[^]|\\^=|\\^\\^|\\^\\^=|{|\\||\\|=|\\|\\||\\|\\|=|~|break|case|continue|delete|do|else|finally|instanceof|return|throw|try|typeof)\\s*(\\/(?=[^*/])(?:[^/[\\\\]|\\\\[\\S\\s]|\\[(?:[^\\\\\\]]|\\\\[\\S\\s])*(?:]|$))+\\/)"sv, - "return /xx/"sv, - true, - ECMAScriptFlags::BrowserExtended, - }, - // #5518 - { "a{2,}"sv, "aaaa"sv }, - { "\\0"sv, "\0"sv, true, ECMAScriptFlags::BrowserExtended }, - { "\\0"sv, "\0"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::BrowserExtended) }, - { "\\01"sv, "\1"sv, true, ECMAScriptFlags::BrowserExtended }, - { "[\\0]"sv, "\0"sv, true, ECMAScriptFlags::BrowserExtended }, - { "[\\0]"sv, "\0"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::BrowserExtended) }, - { "[\\01]"sv, "\1"sv, true, ECMAScriptFlags::BrowserExtended }, - // #9686, Should allow null bytes in pattern - { "(\0|a)"sv, "a"sv, true }, - // #6042, Groups inside lookarounds may be referenced outside, but their contents appear empty if the pattern in the lookaround fails. - { "(.*?)a(?!(a+)b\\2c)\\2(.*)"sv, "baaabaac"sv, true }, - // #11940, Global (not the 'g' flag) regexps should attempt to match the zero-length end of the string too. - { "a|$"sv, "x"sv, true, (ECMAScriptFlags)regex::AllFlags::Global }, - // #12126, ECMA262 regexp should match literal newlines without the 's' flag. - { "foo\nbar"sv, "foo\nbar"sv, true }, - // #12126, ECMA262 regexp should match newline with [^]. - { "foo[^]bar"sv, "foo\nbar"sv, true }, - // Insensitive lookup table: characters in a range do not necessarily lie in the same range after being converted to lowercase. - { "^[_A-Z]+$"sv, "_aA"sv, true, ECMAScriptFlags::Insensitive }, - { "^[a-sy-z]$"sv, "b"sv, true, ECMAScriptFlags::Insensitive }, - { "^[a-sy-z]$"sv, "y"sv, true, ECMAScriptFlags::Insensitive }, - { "^[a-sy-z]$"sv, "u"sv, false, ECMAScriptFlags::Insensitive }, - // Dot should not match any of CR/LF/LS/PS in ECMA262 mode without DotAll. - { "."sv, "\n\r\u2028\u2029"sv, false }, - // $ should accept all LineTerminators in ECMA262 mode with Multiline. - { "a$"sv, "a\r\n"sv, true, global_multiline.value() }, - { "^a"sv, "\ra"sv, true, global_multiline.value() }, - { "^(.*?):[ \\t]*([^\\r\\n]*)$"sv, "content-length: 488\r\ncontent-type: application/json; charset=utf-8\r\n"sv, true, global_multiline.value() }, - // ladybird#968, ?+ should not loop forever. */ - { "^\\?((&?category=[0-9]+)?(&?shippable=1)?(&?ad_type=demand)?(&?page=[0-9]+)?(&?locations=(r|d)_[0-9]+)?)+$"sv, "?category=54&shippable=1&baby_age=p,0,1,3"sv, false }, - // optimizer bug, blindly accepting inverted char classes [^x] as atomic rewrite opportunities. - { "([^\\s]+):\\s*([^;]+);"sv, "font-family: 'Inter';"sv, true }, - // Optimizer bug, ignoring references that weren't bound in the current or past block, ladybird#2281 - { "(a)(?=a*\\1)"sv, "aaaa"sv, true, global_multiline.value() }, - // Optimizer bug, wrong Repeat basic block splits. - { "[ a](b{2})"sv, "abb"sv, true }, - // See above. - { "^ {0,3}(([\\`\\~])\\2{2,})\\s*([\\*_]*)\\s*([^\\*_\\s]*).*$"sv, ""sv, false }, - // See above, also ladybird#2931. - { - "^(\\d{4}|[+-]\\d{6})(?:-?(\\d{2})(?:-?(\\d{2}))?)?(?:[ T]?(\\d{2}):?(\\d{2})(?::?(\\d{2})(?:[,.](\\d{1,}))?)?(?:(Z)|([+-])(\\d{2})(?::?(\\d{2}))?)?)?$"sv, - ""sv, - false, - }, - // Optimizer bug, ignoring an enabled trailing 'invert' when comparing blocks, ladybird#3421. - { "[^]*[^]"sv, "i"sv, true }, - { "xx|...|...."sv, "cd"sv, false }, - // Tests nested lookahead with alternation - verifies proper save/restore stack cleanup - { "a(?=.(?=c)|b)b"sv, "ab"sv, true }, - { "(?=)(?=\\d)"sv, "smart"sv, false }, - // Backrefs are cleared after lookaheads, the indices should be checked before lookup. - { "(?!(b))\\1"sv, "a"sv, false }, - // String table merge bug: inverse map should be merged regardless of available direct mappings. - { "((?a)|(?b))"sv, "aa"sv, false }, - // Insensitive charclasses should accept upper/lowercase in pattern (lookup table should still be ordered if insensitive lookup is used), ladybird#5399. - { "[aBc]"sv, "b"sv, true, ECMAScriptFlags::Insensitive }, - // Optimizer bug: nested 'or' compare ops caused a crash, ladybird#6647. - { "([[[]]])*0"sv, ""sv, false, ECMAScriptFlags::UnicodeSets }, - { "(([[[]]]{2,})\\s)*"sv, ""sv, true, (ECMAScriptFlags::UnicodeSets | ECMAScriptFlags::Global).value() }, - // Optimizer bug: duplicated rseekto ops output for the same fork. - { "(.*a)?(x)"sv, "x"sv, true }, - // Optimizer bug: invalid forkif jump target calculation in tree-layout alternatives - { "ab|a(?:^|x)"sv, "ab"sv, true }, - // Optimizer bug: process rseekto candidates in the correct order. - { "(.*)/client-(.*)\\.js$"sv, "/client-abc.js"sv, true }, - // Optimizer bug: overlapping character classes and ranges not detected. - { "^a*\\w"sv, "aa"sv, true }, - { "^a*[a-z]"sv, "aa"sv, true }, - { "^\\w*\\d"sv, "1"sv, true }, - { "^\\w*[\\u212A]"sv, "K"sv, true, combine_flags(ECMAScriptFlags::Insensitive, ECMAScriptFlags::Unicode) }, - // Optimizer bug: case-insensitive matching was not considered during atomic rewrite. - { "^a*A\\d"sv, "aaaa5"sv, true, ECMAScriptFlags::Insensitive }, - // Quantified lookahead assertions should not affect match_length_minimum. - { "[a-e](?!Z){2}"sv, "aZZZZ bZZZ cZZ dZ e"sv, true, combine_flags(ECMAScriptFlags::Global, ECMAScriptFlags::BrowserExtended) }, - { "[a-e](?!Z){2,}"sv, "aZZZZ bZZZ cZZ dZ e"sv, true, combine_flags(ECMAScriptFlags::Global, ECMAScriptFlags::BrowserExtended) }, - { "[a-e](?!Z){2,3}"sv, "aZZZZ bZZZ cZZ dZ e"sv, true, combine_flags(ECMAScriptFlags::Global, ECMAScriptFlags::BrowserExtended) }, - }; - - for (auto& test : tests) { - Regex re(test.pattern, test.options); - if constexpr (REGEX_DEBUG) { - dbgln("\n"); - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - dbgln("\n"); - } - EXPECT_EQ(re.parser_result.error, regex::Error::NoError); - EXPECT_EQ(re.match(test.subject).success, test.matches); - } -} - -TEST_CASE(lookbehind) -{ - struct _test { - StringView pattern; - StringView subject; - bool matches { true }; - ECMAScriptFlags options {}; - }; - constexpr _test tests[] { - { "(?<=(ab|abc))d"sv, "abcd"sv, true, (ECMAScriptFlags)regex::AllFlags::Global }, - { "(?<=a.*)b"sv, "a b"sv, true, (ECMAScriptFlags)regex::AllFlags::Global }, - { "(?<=[a|b|c]*)[^a|b|c]{3}"sv, "abcdef"sv, true, (ECMAScriptFlags)regex::AllFlags::Global }, - { "(?<=\\b)\\b"sv, "ab"sv, true, (ECMAScriptFlags)regex::AllFlags::Global }, - }; - - for (auto& test : tests) { - Regex re(test.pattern, test.options); - if constexpr (REGEX_DEBUG) { - dbgln("\n"); - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - dbgln("\n"); - } - EXPECT_EQ(re.parser_result.error, regex::Error::NoError); - EXPECT_EQ(re.match(test.subject).success, test.matches); - } - - struct _captureTest { - StringView pattern; - StringView subject; - size_t capture_index; - StringView expected_match; - ECMAScriptFlags options {}; - }; - - constexpr _captureTest capture_tests[] { - { "(?<=(a|cc))b"sv, "ccb"sv, 0, "cc"sv, ECMAScriptFlags::Global }, - { "((?<=\\b)[d-f]{3})"sv, "abc def"sv, 0, "def"sv, (ECMAScriptFlags)regex::AllFlags::Global }, - { "(?<=(b+))c"sv, "abbbbbbc"sv, 0, "bbbbbb"sv, ECMAScriptFlags::Global }, - { "(?<=((?:b\\d{2})+))c"sv, "ab12b23b34c"sv, 0, "b12b23b34"sv, ECMAScriptFlags::Global }, - }; - - for (auto& test : capture_tests) { - Regex re(test.pattern, test.options); - auto result = re.match(test.subject); - EXPECT_EQ(result.capture_group_matches.first()[test.capture_index].view.to_byte_string(), test.expected_match); - } -} - -TEST_CASE(ECMA262_unicode_parser_error) -{ - struct _test { - StringView pattern; - regex::Error error; - }; - - constexpr _test tests[] { - { "([^\\:]+?)"sv, regex::Error::InvalidPattern }, - }; - - for (auto test : tests) { - Regex re(test.pattern, (ECMAScriptFlags)regex::AllFlags::Unicode); - EXPECT_EQ(re.parser_result.error, test.error); - } -} - -TEST_CASE(ECMA262_unicode_match) -{ - constexpr auto space_and_line_terminator_code_points = Array { 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x0020, 0x00A0, 0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000, 0xFEFF }; - - StringBuilder builder; - for (u32 code_point : space_and_line_terminator_code_points) - builder.append_code_point(code_point); - auto space_and_line_terminators = builder.to_byte_string(); - - struct _test { - StringView pattern; - StringView subject; - bool matches { true }; - ECMAScriptFlags options {}; - }; - _test tests[] { - { "\xf0\x9d\x8c\x86"sv, "abcdef"sv, false, ECMAScriptFlags::Unicode }, - { "[\xf0\x9d\x8c\x86]"sv, "abcdef"sv, false, ECMAScriptFlags::Unicode }, - { "\\ud83d"sv, "πŸ˜€"sv, true }, - { "\\ud83d"sv, "πŸ˜€"sv, false, ECMAScriptFlags::Unicode }, - { "\\ude00"sv, "πŸ˜€"sv, true }, - { "\\ude00"sv, "πŸ˜€"sv, false, ECMAScriptFlags::Unicode }, - { "\\ud83d\\ude00"sv, "πŸ˜€"sv, true }, - { "\\ud83d\\ude00"sv, "πŸ˜€"sv, true, ECMAScriptFlags::Unicode }, - { "\\u{1f600}"sv, "πŸ˜€"sv, true, ECMAScriptFlags::Unicode }, - { "\\ud83d\\ud83d"sv, "\xed\xa0\xbd\xed\xa0\xbd"sv, true }, - { "\\ud83d\\ud83d"sv, "\xed\xa0\xbd\xed\xa0\xbd"sv, true, ECMAScriptFlags::Unicode }, - { "(?<=.{3})f"sv, "abcdef"sv, true, ECMAScriptFlags::Unicode }, - { "(?<=.{3})f"sv, "abcπŸ˜€ef"sv, true, ECMAScriptFlags::Unicode }, - { "(?<𝓑𝓻𝓸𝔀𝓷>brown)"sv, "brown"sv, true, ECMAScriptFlags::Unicode }, - { "(?<\\u{1d4d1}\\u{1d4fb}\\u{1d4f8}\\u{1d500}\\u{1d4f7}>brown)"sv, "brown"sv, true, ECMAScriptFlags::Unicode }, - { "(?<\\ud835\\udcd1\\ud835\\udcfb\\ud835\\udcf8\\ud835\\udd00\\ud835\\udcf7>brown)"sv, "brown"sv, true, ECMAScriptFlags::Unicode }, - { "^\\s+$"sv, space_and_line_terminators }, - { "^\\s+$"sv, space_and_line_terminators, true, ECMAScriptFlags::Unicode }, - { "[\\u0390]"sv, "\u1fd3"sv, false, ECMAScriptFlags::Unicode }, - { "[\\u1fd3]"sv, "\u0390"sv, false, ECMAScriptFlags::Unicode }, - { "[\\u0390]"sv, "\u1fd3"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::Insensitive) }, - { "[\\u1fd3]"sv, "\u0390"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::Insensitive) }, - { "[\\u03b0]"sv, "\u1fe3"sv, false, ECMAScriptFlags::Unicode }, - { "[\\u1fe3]"sv, "\u03b0"sv, false, ECMAScriptFlags::Unicode }, - { "[\\u03b0]"sv, "\u1fe3"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::Insensitive) }, - { "[\\u1fe3]"sv, "\u03b0"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::Insensitive) }, - { "[\\ufb05]"sv, "\ufb06"sv, false, ECMAScriptFlags::Unicode }, - { "[\\ufb06]"sv, "\ufb05"sv, false, ECMAScriptFlags::Unicode }, - { "[\\ufb05]"sv, "\ufb06"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::Insensitive) }, - { "[\\ufb06]"sv, "\ufb05"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::Insensitive) }, - - // https://github.com/LadybirdBrowser/ladybird/issues/5549 - { "[\\ud800-\\udbff][\\udc00-\\udfff]"sv, "πŸ˜€"sv, true }, - { "[\\ud800-\\udbff][\\udc00-\\udfff]"sv, "πŸ˜€"sv, false, ECMAScriptFlags::Unicode }, - { "[\\ud800-\\udbff][\\udc00-\\udfff]"sv, "a"sv, false }, - { "[\\ud800-\\udbff][\\udc00-\\udfff]"sv, "a"sv, false, ECMAScriptFlags::Unicode }, - { - "\\ud83c[\\udffb-\\udfff](?=\\ud83c[\\udffb-\\udfff])|(?:[^\\ud800-\\udfff][\\u0300-\\u036f\\ufe20-\\ufe2f\\u20d0-\\u20ff]?|[\\u0300-\\u036f\\ufe20-\\ufe2f\\u20d0-\\u20ff]|(?:\\ud83c[\\udde6-\\uddff]){2}|[\\ud800-\\udbff][\\udc00-\\udfff]|[\\ud800-\\udfff])[\\ufe0e\\ufe0f]?(?:[\\u0300-\\u036f\\ufe20-\\ufe2f\\u20d0-\\u20ff]|\\ud83c[\\udffb-\\udfff])?(?:\\u200d(?:[^\\ud800-\\udfff]|(?:\\ud83c[\\udde6-\\uddff]){2}|[\\ud800-\\udbff][\\udc00-\\udfff])[\\ufe0e\\ufe0f]?(?:[\\u0300-\\u036f\\ufe20-\\ufe2f\\u20d0-\\u20ff]|\\ud83c[\\udffb-\\udfff])?)*"sv, - "πŸ˜€"sv, - true, - }, - { "(?\\w*)\\s*(?\\p{Emoji}+)\\s*(?\\w*)"sv, "Hey πŸŽ‰ there! I love πŸ• pizza"sv, true, ECMAScriptFlags::Unicode }, - // Optimizer bug: case-insensitive matching was not considered during atomic rewrite. - { "^\\u{017f}*s"sv, "\u017fs"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::Insensitive) }, - { "^\\u{212A}*k"sv, "\u212Ak"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::Insensitive) }, - { "^\\u{03C3}*\\u{03A3}"sv, "\u03C3\u03A3"sv, true, combine_flags(ECMAScriptFlags::Unicode, ECMAScriptFlags::Insensitive) }, - }; - - for (auto& test : tests) { - Regex re(test.pattern, (ECMAScriptFlags)regex::AllFlags::Global | test.options); - - auto subject = Utf16String::from_utf8(test.subject); - Utf16View view { subject }; - - if constexpr (REGEX_DEBUG) { - dbgln("\n"); - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - dbgln("\n"); - } - - EXPECT_EQ(re.parser_result.error, regex::Error::NoError); - EXPECT_EQ(re.match(view).success, test.matches); - } -} - -TEST_CASE(ECMA262_unicode_sets_parser_error) -{ - struct _test { - StringView pattern; - regex::Error error; - }; - - constexpr _test tests[] { - { "[[]"sv, regex::Error::InvalidPattern }, - { "[[x[]]]"sv, regex::Error::NoError }, // #23691, should not crash on empty charclass within AndOr. - { "[[^\\u0430-\\u044f][\\p{RGI_Emoji}]]"sv, regex::Error::NoError }, - { "[^[[\\p{RGI_Emoji}]--[A-Z]]]"sv, regex::Error::NegatedCharacterClassStrings }, - { "[^[^\\p{RGI_Emoji}]]"sv, regex::Error::NegatedCharacterClassStrings }, - { "[\\[]"sv, regex::Error::NoError }, - { "[\\[\\]]"sv, regex::Error::NoError }, - { "[\\S[\\[]]"sv, regex::Error::NoError }, - { "[\\S&&[\\[]]"sv, regex::Error::NoError }, - { "[\\S--[\\[]]"sv, regex::Error::NoError }, - }; - - for (auto test : tests) { - Regex re(test.pattern, (ECMAScriptFlags)regex::AllFlags::UnicodeSets); - EXPECT_EQ(re.parser_result.error, test.error); - } -} - -TEST_CASE(ECMA262_unicode_sets_match) -{ - struct _test { - StringView pattern; - StringView subject; - bool matches { true }; - ECMAScriptFlags options {}; - }; - - constexpr _test tests[] { - { "[\\w--x]"sv, "x"sv, false }, - { "[\\w&&x]"sv, "y"sv, false }, - { "[\\w--x]"sv, "y"sv, true }, - { "[\\w&&x]"sv, "x"sv, true }, - { "[[0-9\\w]--x--6]"sv, "6"sv, false }, - { "[[0-9\\w]--x--6]"sv, "x"sv, false }, - { "[[0-9\\w]--x--6]"sv, "y"sv, true }, - { "[[0-9\\w]--x--6]"sv, "9"sv, true }, - { "[\\w&&\\d]"sv, "a"sv, false }, - { "[\\w&&\\d]"sv, "4"sv, true }, - { "([^\\:]+?)"sv, "a"sv, true }, - { "[[a][]]"sv, "a"sv, true }, // ladybird#6647 - }; - - for (auto& test : tests) { - Regex re(test.pattern, (ECMAScriptFlags)regex::AllFlags::UnicodeSets | test.options); - if constexpr (REGEX_DEBUG) { - dbgln("\n"); - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - dbgln("\n"); - } - - EXPECT_EQ(re.parser_result.error, regex::Error::NoError); - auto result = re.match(test.subject).success; - EXPECT_EQ(result, test.matches); - } -} - -TEST_CASE(ECMA262_property_match) -{ - struct _test { - StringView pattern; - StringView subject; - bool matches { true }; - ECMAScriptFlags options {}; - }; - - constexpr _test tests[] { - { "\\p{ASCII}"sv, "a"sv, false }, - { "\\p{ASCII}"sv, "p{ASCII}"sv, true }, - { "\\p{ASCII}"sv, "a"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{ASCII}"sv, "πŸ˜€"sv, false, ECMAScriptFlags::Unicode }, - { "\\P{ASCII}"sv, "a"sv, false, ECMAScriptFlags::Unicode }, - { "\\P{ASCII}"sv, "πŸ˜€"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{ASCII_Hex_Digit}"sv, "1"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{ASCII_Hex_Digit}"sv, "a"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{ASCII_Hex_Digit}"sv, "x"sv, false, ECMAScriptFlags::Unicode }, - { "\\P{ASCII_Hex_Digit}"sv, "1"sv, false, ECMAScriptFlags::Unicode }, - { "\\P{ASCII_Hex_Digit}"sv, "a"sv, false, ECMAScriptFlags::Unicode }, - { "\\P{ASCII_Hex_Digit}"sv, "x"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{Any}"sv, "\xcd\xb8"sv, true, ECMAScriptFlags::Unicode }, // U+0378, which is an unassigned code point. - { "\\P{Any}"sv, "\xcd\xb8"sv, false, ECMAScriptFlags::Unicode }, // U+0378, which is an unassigned code point. - { "\\p{Assigned}"sv, "\xcd\xb8"sv, false, ECMAScriptFlags::Unicode }, // U+0378, which is an unassigned code point. - { "\\P{Assigned}"sv, "\xcd\xb8"sv, true, ECMAScriptFlags::Unicode }, // U+0378, which is an unassigned code point. - { "\\p{Lu}"sv, "a"sv, false, ECMAScriptFlags::Unicode }, - { "\\p{Lu}"sv, "A"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{Lu}"sv, "9"sv, false, ECMAScriptFlags::Unicode }, - { "\\p{Cased_Letter}"sv, "a"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{Cased_Letter}"sv, "A"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{Cased_Letter}"sv, "9"sv, false, ECMAScriptFlags::Unicode }, - { "\\P{Cased_Letter}"sv, "a"sv, false, ECMAScriptFlags::Unicode }, - { "\\P{Cased_Letter}"sv, "A"sv, false, ECMAScriptFlags::Unicode }, - { "\\P{Cased_Letter}"sv, "9"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{General_Category=Cased_Letter}"sv, "a"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{General_Category=Cased_Letter}"sv, "A"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{General_Category=Cased_Letter}"sv, "9"sv, false, ECMAScriptFlags::Unicode }, - { "\\p{gc=Cased_Letter}"sv, "a"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{gc=Cased_Letter}"sv, "A"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{gc=Cased_Letter}"sv, "9"sv, false, ECMAScriptFlags::Unicode }, - { "\\p{Script=Latin}"sv, "a"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{Script=Latin}"sv, "A"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{Script=Latin}"sv, "9"sv, false, ECMAScriptFlags::Unicode }, - { "\\p{sc=Latin}"sv, "a"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{sc=Latin}"sv, "A"sv, true, ECMAScriptFlags::Unicode }, - { "\\p{sc=Latin}"sv, "9"sv, false, ECMAScriptFlags::Unicode }, - { "\\p{Script_Extensions=Deva}"sv, "a"sv, false, ECMAScriptFlags::Unicode }, - { "\\p{Script_Extensions=Beng}"sv, "\xe1\xb3\x95"sv, true, ECMAScriptFlags::Unicode }, // U+01CD5 - { "\\p{Script_Extensions=Deva}"sv, "\xe1\xb3\x95"sv, true, ECMAScriptFlags::Unicode }, // U+01CD5 - { "\\p{scx=Deva}"sv, "a"sv, false, ECMAScriptFlags::Unicode }, - { "\\p{scx=Beng}"sv, "\xe1\xb3\x95"sv, true, ECMAScriptFlags::Unicode }, // U+01CD5 - { "\\p{scx=Deva}"sv, "\xe1\xb3\x95"sv, true, ECMAScriptFlags::Unicode }, // U+01CD5 - }; - - for (auto& test : tests) { - Regex re(test.pattern, (ECMAScriptFlags)regex::AllFlags::Global | regex::ECMAScriptFlags::BrowserExtended | test.options); - - auto subject = Utf16String::from_utf8(test.subject); - Utf16View view { subject }; - - if constexpr (REGEX_DEBUG) { - dbgln("\n"); - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - dbgln("\n"); - } - - EXPECT_EQ(re.parser_result.error, regex::Error::NoError); - EXPECT_EQ(re.match(view).success, test.matches); - } -} - -TEST_CASE(replace) -{ - struct _test { - StringView pattern; - StringView replacement; - StringView subject; - StringView expected; - ECMAScriptFlags options {}; - }; - - constexpr _test tests[] { - { "foo(.+)"sv, "aaa"sv, "test"sv, "test"sv }, - { "foo(.+)"sv, "test\\1"sv, "foobar"sv, "testbar"sv }, - { "foo(.+)"sv, "\\2\\1"sv, "foobar"sv, "\\2bar"sv }, - { "foo(.+)"sv, "\\\\\\1"sv, "foobar"sv, "\\bar"sv }, - { "foo(.)"sv, "a\\1"sv, "fooxfooy"sv, "axay"sv, ECMAScriptFlags::Multiline }, - }; - - for (auto& test : tests) { - Regex re(test.pattern, test.options); - if constexpr (REGEX_DEBUG) { - dbgln("\n"); - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - dbgln("\n"); - } - EXPECT_EQ(re.parser_result.error, regex::Error::NoError); - EXPECT_EQ(re.replace(test.subject, test.replacement), test.expected); - } -} - -TEST_CASE(case_insensitive_match) -{ - Regex re("cd", PosixFlags::Insensitive | PosixFlags::Global); - auto result = re.match("AEKFCD"sv); - - EXPECT_EQ(result.success, true); - if (result.success) { - EXPECT_EQ(result.matches.at(0).column, 4ul); - } -} - -TEST_CASE(extremely_long_fork_chain) -{ - Regex re("(?:aa)*"); - auto input = MUST(String::repeated('a', 1000)); - auto result = re.match(input); - EXPECT_EQ(result.success, true); -} - -TEST_CASE(nullable_quantifiers) -{ - Regex re("(a?b?\x3f)*"); // Pattern (a?b??)* isn't written plain to avoid "??)", which is a trigraph. - auto result = re.match("ab"sv); - EXPECT_EQ(result.matches.at(0).view, "ab"sv); -} - -TEST_CASE(theoretically_infinite_loop) -{ - Array patterns { - "(a*)*"sv, // Infinitely matching empty substrings, the outer loop should short-circuit. - "(a*?)*"sv, // Infinitely matching empty substrings, the outer loop should short-circuit. - "(a*)*?"sv, // Should match exactly nothing. - "(?:)*?"sv, // Should not generate an infinite fork loop. - "(a?)+$"sv, // Infinitely matching empty strings, but with '+' instead of '*'. - }; - for (auto& pattern : patterns) { - Regex re(pattern); - auto result = re.match(""sv); - EXPECT_EQ(result.success, true); - } -} - -static auto g_lots_of_a_s = String::repeated('a', 10'000'000).release_value(); - -BENCHMARK_CASE(fork_performance) -{ - { - Regex re("(?:aa)*"); - auto result = re.match(g_lots_of_a_s); - EXPECT_EQ(result.success, true); - } - { - Regex re("(a+)+b"); - auto result = re.match(g_lots_of_a_s.bytes_as_string_view().substring_view(0, 100)); - EXPECT_EQ(result.success, false); - } - { - Regex re("^(a|a?)+$"); - auto input = MUST(String::formatted("{}b", g_lots_of_a_s.bytes_as_string_view().substring_view(0, 100))); - auto result = re.match(input); - EXPECT_EQ(result.success, false); - } -} - -BENCHMARK_CASE(anchor_performance) -{ - Regex re("^b"); - for (auto i = 0; i < 100'000; i++) { - auto result = re.match(g_lots_of_a_s); - EXPECT_EQ(result.success, false); - } -} - -TEST_CASE(optimizer_atomic_groups) -{ - Array tests { - // Fork -> ForkReplace - Tuple { "a*b"sv, "aaaaa"sv, false }, - Tuple { "a+b"sv, "aaaaa"sv, false }, - Tuple { "\\\\(\\d+)"sv, "\\\\"sv, false }, // Rewrite bug turning a+ to a*, see #10952. - Tuple { "[a-z.]+\\."sv, "..."sv, true }, // Rewrite bug, incorrect interpretation of Compare. - Tuple { "[.-]+\\."sv, ".-."sv, true }, - // Alternative fuse - Tuple { "(abcfoo|abcbar|abcbaz).*x"sv, "abcbarx"sv, true }, - Tuple { "(a|a)"sv, "a"sv, true }, - Tuple { "(a|)"sv, ""sv, true }, // Ensure that empty alternatives are not outright removed - Tuple { "a{2,3}|a{5,8}"sv, "abc"sv, false }, // Optimizer should not mess up the instruction stream by ignoring inter-insn dependencies, see #11247. - Tuple { "^(a{2,3}|a{5,8})$"sv, "aaaa"sv, false }, // Optimizer should not mess up the instruction stream by ignoring inter-insn dependencies, see #11247. - // Optimizer should not chop off *half* of an instruction when fusing instructions. - Tuple { "cubic-bezier\\(\\s*(-?\\d+\\.?\\d*|-?\\.\\d+)\\s*,\\s*(-?\\d+\\.?\\d*|-?\\.\\d+)\\s*,\\s*(-?\\d+\\.?\\d*|-?\\.\\d+)\\s*,\\s*(-?\\d+\\.?\\d*|-?\\.\\d+)\\s*\\)"sv, "cubic-bezier(.05, 0, 0, 1)"sv, true }, - // ForkReplace shouldn't be applied where it would change the semantics - Tuple { "(1+)\\1"sv, "11"sv, true }, - Tuple { "(1+)1"sv, "11"sv, true }, - Tuple { "(1+)0"sv, "10"sv, true }, - // Rewrite should not skip over first required iteration of +. - Tuple { "a+"sv, ""sv, false }, - // 'y' and [^x] have an overlap ('y'), the loop should not be rewritten here. - Tuple { "[^x]+y"sv, "ay"sv, true }, - // .+ should not be rewritten here, as it's followed by something that would be matched by `.`. - Tuple { ".+(a|b|c)"sv, "xxa"sv, true }, - // (b+)(b+) produces an intermediate block with no matching ops, the optimiser should ignore that block when looking for following matches and correctly detect the overlap between (b+) and (b+). - // note that the second loop may be rewritten to a ForkReplace, but the first loop should not be rewritten. - Tuple { "(b+)(b+)"sv, "bbb"sv, true }, - // Don't treat [\S] as [\s]; see ladybird#2296. - Tuple { "([^\\s]+?)\\(([\\s\\S]*)\\)"sv, "a(b)"sv, true }, - // Follow direct jumps in the optimizer instead of assuming they're a noop. - Tuple { "(|[^]*)\\)"sv, "p)"sv, true }, - }; - - for (auto& test : tests) { - Regex re(test.get<0>()); - auto result = re.match(test.get<1>()); - EXPECT_EQ(result.success, test.get<2>()); - } -} - -TEST_CASE(optimizer_char_class_lut) -{ - Regex re(R"([\f\n\r\t\v\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+$)"); - - if constexpr (REGEX_DEBUG) { - dbgln("\n"); - RegexDebug regex_dbg(stderr); - regex_dbg.print_raw_bytecode(re); - regex_dbg.print_header(); - regex_dbg.print_bytecode(re); - dbgln("\n"); - } - - // This will go through _all_ alternatives in the character class, and then fail. - for (size_t i = 0; i < 1'000'000; ++i) - EXPECT_EQ(re.match("1635488940000"sv).success, false); -} - -TEST_CASE(optimizer_alternation) -{ - Array tests { - // Pattern, Subject, Expected length [0 == fail] - Tuple { "a|"sv, "a"sv, 1u }, - Tuple { "a|a|a|a|a|a|a|a|a|b"sv, "a"sv, 1u }, - Tuple { "ab|ac|ad|bc"sv, "bc"sv, 2u }, - // Should not crash on backwards jumps introduced by '.*'. - Tuple { "\\bDroid\\b.*Build|XT912|XT928|XT926|XT915|XT919|XT925|XT1021|\\bMoto E\\b|XT1068|XT1092|XT1052"sv, "XT1068"sv, 6u }, - // Backwards jumps to IP 0 are normal jumps too. - Tuple { "^(\\d+|x)"sv, "42"sv, 2u }, - // `Repeat' does not add its insn size to the jump target. - Tuple { "[0-9]{2}|[0-9]"sv, "92"sv, 2u }, - // Don't ForkJump to the next instruction, rerunning it would produce the same result. see ladybird#2398. - Tuple { "(xxxxxxxxxxxxxxxxxxxxxxx|xxxxxxxxxxxxxxxxxxxxxxx)?b"sv, "xxxxxxxxxxxxxxxxxxxxxxx"sv, 0u }, - // Don't take the jump in JumpNonEmpty with nonexistent checkpoints (also don't crash). - Tuple { "(?!\\d*|[g-ta-r]+|[h-l]|\\S|\\S|\\S){,9}|\\S{7,8}|\\d|(?)|[c-mj-tb-o]*|\\s"sv, "rjvogg7pm|li4nmct mjb2|pk7s8e0"sv, 0u }, - // Use the right offset when patching jumps through a fork-tree - Tuple { "(?!a)|(?!a)b"sv, "b"sv, 0u }, - // Optimizer should maintain the correct ordering between the alternatives - Tuple { "\\\\junk|(\\\\[a-zA-Z@]+)|\\\\[^X]"sv, "\\sqrt"sv, 5u }, - }; - - for (auto& test : tests) { - Regex re(test.get<0>()); - auto result = re.match(test.get<1>()); - if (test.get<2>() != 0) { - EXPECT(result.success); - EXPECT_EQ(result.matches.first().view.length(), test.get<2>()); - } else { - EXPECT(!result.success); - } - } -} - -TEST_CASE(optimizer_rseekto) -{ - Regex re("^(.*)\\/(?:\\/(.*))$"); // should backtrack from the second '/'. - - auto result = re.match("foo//bar"sv); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.at(0).view, "foo//bar"sv); - EXPECT_EQ(result.capture_group_matches.at(0).at(0).view, "foo"sv); - EXPECT_EQ(result.capture_group_matches.at(0).at(1).view, "bar"sv); -} - -TEST_CASE(start_anchor) -{ - // Ensure that a circumflex at the start only matches the start of the line. - { - Regex re("^abc"); - EXPECT_EQ(re.match("123abcdef"sv, PosixFlags::Global).success, false); - EXPECT_EQ(re.match("abc123"sv, PosixFlags::Global).success, true); - EXPECT_EQ(re.match("123^abcdef"sv, PosixFlags::Global).success, false); - EXPECT_EQ(re.match("^abc123"sv, PosixFlags::Global).success, false); - - // Multiple lines - EXPECT_EQ(re.match("123\nabc"sv, PosixFlags::Multiline).success, true); - } -} - -TEST_CASE(posix_basic_dollar_is_end_anchor) -{ - // Ensure that a dollar sign at the end only matches the end of the line. - { - Regex re("abc$"); - EXPECT_EQ(re.match("123abcdef"sv, PosixFlags::Global).success, false); - EXPECT_EQ(re.match("123abc"sv, PosixFlags::Global).success, true); - EXPECT_EQ(re.match("123abc$def"sv, PosixFlags::Global).success, false); - EXPECT_EQ(re.match("123abc$"sv, PosixFlags::Global).success, false); - } -} - -TEST_CASE(posix_basic_dollar_is_literal) -{ - // Ensure that a dollar sign in the middle is treated as a literal. - { - Regex re("abc$d"); - EXPECT_EQ(re.match("123abcdef"sv, PosixFlags::Global).success, false); - EXPECT_EQ(re.match("123abc"sv, PosixFlags::Global).success, false); - EXPECT_EQ(re.match("123abc$def"sv, PosixFlags::Global).success, true); - EXPECT_EQ(re.match("123abc$"sv, PosixFlags::Global).success, false); - } - - // Ensure that a dollar sign is always treated as a literal if escaped, even if at the end of the pattern. - { - Regex re("abc\\$"); - EXPECT_EQ(re.match("123abcdef"sv, PosixFlags::Global).success, false); - EXPECT_EQ(re.match("123abc"sv, PosixFlags::Global).success, false); - EXPECT_EQ(re.match("123abc$def"sv, PosixFlags::Global).success, true); - EXPECT_EQ(re.match("123abc$"sv, PosixFlags::Global).success, true); - } -} - -TEST_CASE(negative_lookahead) -{ - { - // Negative lookahead with more than 2 forks difference between lookahead init and finish. - auto options = ECMAScriptOptions { ECMAScriptFlags::Global }; - options.reset_flag((ECMAScriptFlags)regex::AllFlags::Internal_Stateful); - Regex re(":(?!\\^\\)|1)", options); - EXPECT_EQ(re.match(":^)"sv).success, false); - EXPECT_EQ(re.match(":1"sv).success, false); - EXPECT_EQ(re.match(":foobar"sv).success, true); - } - { - // Correctly count forks with nested groups and optimised loops - Regex re("^((?:[^\\n]|\\n(?! *\\n))+)(?:\\n *)+\\n"); - EXPECT_EQ(re.match("foo\n\n"sv).success, true); - EXPECT_EQ(re.match("foo\n"sv).success, false); - } -} - -TEST_CASE(single_match_flag) -{ - { - // Ensure that only a single match is produced and nothing past that. - Regex re("[\\u0008-\\uffff]"sv, ECMAScriptFlags::Global | (ECMAScriptFlags)regex::AllFlags::SingleMatch); - auto result = re.match("ABC"sv); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "A"sv); - } -} - -TEST_CASE(empty_string_wildcard_match) -{ - { - // Ensure that the wildcard ".*" matches the empty string exactly once - Regex re(".*"sv, ECMAScriptFlags::Global); - auto result = re.match(""sv); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), ""sv); - } -} - -TEST_CASE(inversion_state_in_char_class) -{ - { - // #13755, /[\S\s]/.exec("hello") should be [ "h" ], not null. - Regex re("[\\S\\s]", ECMAScriptFlags::Global | (ECMAScriptFlags)regex::AllFlags::SingleMatch); - - auto result = re.match("hello"sv); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "h"sv); - } - { - Regex re("^(?:([^\\s!\"#%-,\\./;->@\\[-\\^`\\{-~]+(?=([=~}\\s/.)|]))))"sv, ECMAScriptFlags::Global); - - auto result = re.match("slideNumbers}}"sv); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "slideNumbers"sv); - EXPECT_EQ(result.capture_group_matches.first()[0].view.to_byte_string(), "slideNumbers"sv); - EXPECT_EQ(result.capture_group_matches.first()[1].view.to_byte_string(), "}"sv); - } - { - // #21786, /[^\S\n]/.exec("\n") should be null, not [ "\n" ]. - // This was a general confusion between the inversion state and the negation state (temp inverse). - Regex re("[^\\S\\n]", ECMAScriptFlags::Global | (ECMAScriptFlags)regex::AllFlags::SingleMatch); - - auto result = re.match("\n"sv); - EXPECT_EQ(result.success, false); - } - { - // /[^\S]/ should match whitespace characters - Regex re("[^\\S]", ECMAScriptFlags::Global | (ECMAScriptFlags)regex::AllFlags::SingleMatch); - - auto result = re.match("\t"sv); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "\t"sv); - } -} - -TEST_CASE(mismatching_brackets) -{ - auto const test_cases = Array { - "["sv, - "[ -"sv, - }; - - for (auto const& test_case : test_cases) { - Regex re(test_case); - EXPECT_EQ(re.parser_result.error, regex::Error::MismatchingBracket); - } -} - -TEST_CASE(optimizer_repeat_offset) -{ - { - // Miscalculating the repeat offset in table reconstruction of alternatives would lead to crash here - // make sure that doesn't happen :) - Regex re("\\/?\\??#?([\\/?#]|[\\uD800-\\uDBFF]|%[c-f][0-9a-f](%[89ab][0-9a-f]){0,2}(%[89ab]?)?|%[0-9a-f]?)$"sv); - } -} - -TEST_CASE(quantified_alternation_capture_groups) -{ - { - // Ensure that (a|a?)+ captures the last meaningful match, not empty string - Regex re("^(a|a?)+$"); - auto result = re.match("a"sv); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "a"sv); - EXPECT_EQ(result.capture_group_matches.first()[0].view.to_byte_string(), "a"sv); - } - { - Regex re("^(a|a?)+$"); - auto result = re.match("aa"sv); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "aa"sv); - EXPECT_EQ(result.capture_group_matches.first()[0].view.to_byte_string(), "a"sv); - } -} - -TEST_CASE(zero_width_backreference) -{ - { - // Ensure that a zero-width backreference will match correctly. - Regex re("(a*)b\\1+", ECMAScriptFlags::Global); - auto result = re.match("baaac"sv); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "b"sv); - EXPECT_EQ(result.capture_group_matches.first()[0].view.to_byte_string(), ""sv); - } - { - Regex re("(x)?\\1y"sv); - auto result = re.match("y"sv); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.first().view, "y"sv); - EXPECT(result.capture_group_matches.first()[0].view.is_null()); - } - { - Regex re("(?!(y)y)(\\1)z"sv, ECMAScriptFlags::Global); - auto result = re.match("xyyz"sv); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.first().view, "z"sv); - EXPECT(result.capture_group_matches.first()[0].view.is_null()); - EXPECT_EQ(result.capture_group_matches.first()[1].view.to_byte_string(), ""sv); - } -} - -TEST_CASE(account_for_opcode_size_calculating_incoming_jump_edges) -{ - { - // The optimizer should not optimize the initial ForkStay for these alternatives as they are jumped to from different locations. - Regex re(".*a|.*b", ECMAScriptFlags::Global); - auto result = re.match("aa"sv); - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "aa"sv); - } -} - -TEST_CASE(backreference_to_undefined_capture_groups) -{ - { - // Test duplicate named groups in alternatives where backreference refers to participating group - Regex re("(?:(?a)|(?b))\\k"sv); - auto result = re.match("bb"sv); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "bb"sv); - EXPECT_EQ(result.capture_group_matches.first().size(), 2u); - EXPECT(result.capture_group_matches.first()[0].view.is_null()); - EXPECT_EQ(result.capture_group_matches.first()[1].view.to_byte_string(), "b"sv); - } - - { - // Test duplicate named groups with quantifier - Regex re("(?:(?:(?a)|(?b))\\k){2}"sv); - auto result = re.match("aabb"sv); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "aabb"sv); - EXPECT_EQ(result.capture_group_matches.first().size(), 2u); - EXPECT(result.capture_group_matches.first()[0].view.is_null()); - EXPECT_EQ(result.capture_group_matches.first()[1].view.to_byte_string(), "b"sv); - } - - { - // Test that first alternative works too - Regex re("(?:(?a)|(?b))\\k"sv); - auto result = re.match("aa"sv); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "aa"sv); - EXPECT_EQ(result.capture_group_matches.first().size(), 2u); - EXPECT_EQ(result.capture_group_matches.first()[0].view.to_byte_string(), "a"sv); - EXPECT(result.capture_group_matches.first()[1].view.is_null()); - } - - { - // Test numbered backreference to undefined group - Regex re("(.*?)a(?!(a+)b\\2c)\\2(.*)"sv); - auto result = re.match("baaabaac"sv); - - EXPECT_EQ(result.success, true); - EXPECT_EQ(result.matches.size(), 1u); - EXPECT_EQ(result.matches.first().view.to_byte_string(), "baaabaac"sv); - EXPECT_EQ(result.capture_group_matches.first().size(), 3u); - EXPECT_EQ(result.capture_group_matches.first()[0].view.to_byte_string(), "ba"sv); - EXPECT(result.capture_group_matches.first()[1].view.is_null()); - EXPECT_EQ(result.capture_group_matches.first()[2].view.to_byte_string(), "abaac"sv); - } - - { - Regex re("^(?:(?x)|(?y)|z)\\k$"sv); - - // Third alternative matches and backreference is undefined - auto result1 = re.match("z"sv); - EXPECT_EQ(result1.success, true); - EXPECT_EQ(result1.matches.size(), 1u); - EXPECT_EQ(result1.matches.first().view.to_byte_string(), "z"sv); - EXPECT_EQ(result1.capture_group_matches.first().size(), 2u); - EXPECT(result1.capture_group_matches.first()[0].view.is_null()); - EXPECT(result1.capture_group_matches.first()[1].view.is_null()); - } - - { - // Quantified version of the above pattern - Regex re("^(?:(?x)|(?y)|z){2}\\k$"sv); - - auto result1 = re.match("xz"sv); - EXPECT_EQ(result1.success, true); - EXPECT_EQ(result1.matches.size(), 1u); - EXPECT_EQ(result1.matches.first().view.to_byte_string(), "xz"sv); - EXPECT_EQ(result1.capture_group_matches.first().size(), 2u); - EXPECT(result1.capture_group_matches.first()[0].view.is_null()); - EXPECT(result1.capture_group_matches.first()[1].view.is_null()); - - auto result2 = re.match("yz"sv); - EXPECT_EQ(result2.success, true); - EXPECT_EQ(result2.matches.size(), 1u); - EXPECT_EQ(result2.matches.first().view.to_byte_string(), "yz"sv); - EXPECT_EQ(result2.capture_group_matches.first().size(), 2u); - EXPECT(result2.capture_group_matches.first()[0].view.is_null()); - EXPECT(result2.capture_group_matches.first()[1].view.is_null()); - } -} - -TEST_CASE(optional_groups_with_empty_matches) -{ - Regex re1("^(.*)(.*)?$"sv); - auto result1 = re1.match("a"sv); - EXPECT_EQ(result1.success, true); - EXPECT_EQ(result1.capture_group_matches.first()[0].view.to_byte_string(), "a"sv); - EXPECT(result1.capture_group_matches.first()[1].view.is_null()); - - Regex re2("()?"sv); - auto result3 = re2.match(""sv); - EXPECT_EQ(result3.success, true); - EXPECT(result3.capture_group_matches.first()[0].view.is_null()); - - Regex re3("(z)((a+)?(b+)?(c))*"sv); - auto result4 = re3.match("zaacbbbcac"sv); - EXPECT_EQ(result4.success, true); - EXPECT_EQ(result4.capture_group_matches.first()[0].view.to_byte_string(), "z"sv); - EXPECT_EQ(result4.capture_group_matches.first()[1].view.to_byte_string(), "ac"sv); - EXPECT_EQ(result4.capture_group_matches.first()[2].view.to_byte_string(), "a"sv); - EXPECT(result4.capture_group_matches.first()[3].view.is_null()); - EXPECT_EQ(result4.capture_group_matches.first()[4].view.to_byte_string(), "c"sv); - - Regex re4("(?:(?=(abc)))?a"sv); - auto result5 = re4.match("abc"sv, ECMAScriptFlags::Global); - EXPECT_EQ(result5.success, true); - EXPECT_EQ(result5.matches.first().view.to_byte_string(), "a"sv); - EXPECT(result5.capture_group_matches.first()[0].view.is_null()); - - Regex re5("^(?:(?=(abc))){0,1}a"sv); - auto result6 = re5.match("abc"sv, ECMAScriptFlags::Global); - EXPECT_EQ(result6.success, true); - EXPECT_EQ(result6.matches.first().view.to_byte_string(), "a"sv); - EXPECT(result6.capture_group_matches.first()[0].view.is_null()); -} - -TEST_CASE(ecma262_modifiers) -{ - struct Test { - StringView pattern; - StringView subject; - bool matches { true }; - ECMAScriptFlags flags {}; - }; - - constexpr Test tests[] { - { "a(?i:b)c"sv, "aBc"sv, true, {} }, - { "a(?i:b)c"sv, "aBC"sv, false, {} }, - { "a(?s:.)c"sv, "a\nc"sv, true, {} }, - { "(?ims:a.b)"sv, "A\nB"sv, true, {} }, - { "(?i:a(?-i:b)c)"sv, "AbC"sv, true, {} }, - { "(?i:a(?-i:b)c)"sv, "ABC"sv, false, {} }, - { "a(?-i:b)c"sv, "AbC"sv, true, ECMAScriptFlags::Insensitive }, - { "a(?-i:b)c"sv, "ABC"sv, false, ECMAScriptFlags::Insensitive }, - { "x.(?m:^a)"sv, "x\na"sv, true, ECMAScriptFlags::SingleLine }, - }; - - for (auto const& test : tests) { - Regex re(test.pattern, test.flags); - auto result = re.match(test.subject); - EXPECT_EQ(result.success, test.matches); - } -} - -#define EXPECT_PATTERNS_IN_DUMP(re, ...) \ - do { \ - auto dump = bytecode_dump(re); \ - if (!bytecode_matches_checks(dump, Array { __VA_ARGS__ })) { \ - warnln("Failed pattern expectation {} in dump lines:\n{}", Vector { __VA_ARGS__ }, dump); \ - EXPECT(false && #__VA_ARGS__); \ - } \ - } while (0); - -#define EXPECT_NO_PATTERN_IN_DUMP(re, pattern) \ - do { \ - auto dump = bytecode_dump(re); \ - if (bytecode_contains_pattern(dump, pattern)) { \ - warnln("Unexpected pattern '{}' found in dump:\n{}", pattern, dump); \ - EXPECT(false && #pattern); \ - } \ - } while (0); - -static Vector bytecode_dump(Regex const& re) -{ - Vector lines; - auto& bytecode = re.parser_result.bytecode.get(); - auto const* data = bytecode.flat_data().data(); - auto data_size = bytecode.size(); - auto state = regex::MatchState::only_for_enumeration(); - while (state.instruction_position < data_size) { - auto id = static_cast(data[state.instruction_position]); - auto sz = regex::opcode_size(id, data, state.instruction_position); - lines.append(ByteString::formatted("{} {}", regex::opcode_id_name(id), regex::opcode_arguments_string(id, data, state.instruction_position, state, bytecode))); - if (id == regex::OpCodeId::Exit) - break; - state.instruction_position += sz; - } - return lines; -} - -template -static bool bytecode_matches_checks(Span lines, Array checks) -{ - size_t line_idx = 0; - for (auto check : checks) { - bool found = false; - for (; line_idx < lines.size(); ++line_idx) { - if (lines[line_idx].contains(check)) { - found = true; - ++line_idx; - break; - } - } - if (!found) - return false; - } - return true; -} - -static bool bytecode_contains_pattern(Span lines, StringView pattern) -{ - for (auto const& line : lines) { - if (line.contains(pattern)) - return true; - } - return false; -} - -TEST_CASE(optimizer_dot_star_to_rseekto) -{ - Regex re(".*foo"); - - // 'f' = 102 - EXPECT_PATTERNS_IN_DUMP(re, "RSeekTo before '102'"sv, "ForkStay"sv); - - // Should still match correctly - EXPECT_EQ(re.match("xyzfoo"sv).success, true); - EXPECT_EQ(re.match("foo"sv).success, true); - EXPECT_EQ(re.match("xyzbar"sv).success, false); -} - -TEST_CASE(optimizer_simple_compare_string) -{ - Regex re(".?foo"); - - EXPECT_PATTERNS_IN_DUMP(re, "CompareSimple String \"foo\""sv); - - EXPECT_EQ(re.match("foo"sv).success, true); - EXPECT_EQ(re.match("xyzbar"sv).success, false); -} - -TEST_CASE(optimizer_dot_star_with_fail_if_empty) -{ - // FailIfEmpty within a .* loop should be ignored during RSeekTo detection. - Regex re(".*foo"); - - // 'f' = 102 - EXPECT_PATTERNS_IN_DUMP(re, "RSeekTo before '102'"sv); - - EXPECT_EQ(re.match("foo"sv).success, true); - EXPECT_EQ(re.match("xyzfoo"sv).success, true); - EXPECT_EQ(re.match("bar"sv).success, false); -} - -TEST_CASE(optimizer_dot_plus_no_rseekto) -{ - // .+ uses a `JumpNonEmpty ForkJump` loop structure without ForkStay at the start, - // so it is not eligible for the RSeekTo rewrite. - Regex re(".+foo"); - EXPECT_NO_PATTERN_IN_DUMP(re, "RSeekTo"sv); - - EXPECT_EQ(re.match("xfoo"sv).success, true); - EXPECT_EQ(re.match("xyzfoo"sv).success, true); - EXPECT_EQ(re.match("foo"sv).success, false); // .+ requires at least one character - EXPECT_EQ(re.match("bar"sv).success, false); -} - -TEST_CASE(optimizer_dot_star_in_capture_group) -{ - // .* inside a capture group should still produce RSeekTo - Regex re("(.*)x"); - - // 'x' = 120 - EXPECT_PATTERNS_IN_DUMP(re, "RSeekTo before '120'"sv); - - EXPECT_EQ(re.match("abcx"sv).success, true); - EXPECT_EQ(re.match("x"sv).success, true); - EXPECT_EQ(re.match("abc"sv).success, false); -} - -TEST_CASE(optimizer_no_rseekto_for_char_class) -{ - // .* followed by a char class cannot produce a RSeekTo (can't seek to a class) - { - Regex re(".*\\d"); - EXPECT_NO_PATTERN_IN_DUMP(re, "RSeekTo"sv); - - EXPECT_EQ(re.match("abc5"sv).success, true); - EXPECT_EQ(re.match("abc"sv).success, false); - } - - // .* followed by a range cannot produce RSeekTo - { - Regex re(".*[abc]"); - EXPECT_NO_PATTERN_IN_DUMP(re, "RSeekTo"sv); - - EXPECT_EQ(re.match("xyzc"sv).success, true); - EXPECT_EQ(re.match("xyz"sv).success, false); - } -} - -TEST_CASE(optimizer_atomic_rewrite_bytecode) -{ - // a+b: 'b' cannot be matched by 'a', so the a+ loop should be rewritten as atomic. - { - Regex re("a+b"); - EXPECT_PATTERNS_IN_DUMP(re, "ForkReplace"sv); - - EXPECT_EQ(re.match("ab"sv).success, true); - EXPECT_EQ(re.match("aab"sv).success, true); - EXPECT_EQ(re.match("b"sv).success, false); - EXPECT_EQ(re.match("aaa"sv).success, false); - } - - // [a-z]+[0-9]: char classes don't overlap, so it should be rewritten as atomic. - { - Regex re("[a-z]+[0-9]"); - EXPECT_PATTERNS_IN_DUMP(re, "ForkReplace"sv); - - EXPECT_EQ(re.match("abc5"sv).success, true); - EXPECT_EQ(re.match("5"sv).success, false); - EXPECT_EQ(re.match("abc"sv).success, false); - } -} - -TEST_CASE(optimizer_no_atomic_rewrite_with_overlap) -{ - // a+a: 'a' overlaps with 'a', so the loop cannot be rewritten as atomic - { - Regex re("a+a"); - EXPECT_NO_PATTERN_IN_DUMP(re, "ForkReplace"sv); - - EXPECT_EQ(re.match("aa"sv).success, true); - EXPECT_EQ(re.match("aaa"sv).success, true); - EXPECT_EQ(re.match("a"sv).success, false); - } - - // (a+)\1: backreference should prevent atomic rewrite. - { - Regex re("(a+)\\1"); - EXPECT_NO_PATTERN_IN_DUMP(re, "ForkReplace"sv); - - EXPECT_EQ(re.match("aa"sv).success, true); - EXPECT_EQ(re.match("aaaa"sv).success, true); - EXPECT_EQ(re.match("a"sv).success, false); - } -} - -TEST_CASE(optimizer_adjacent_char_to_string_compare) -{ - // Multiple adjacent single-character compares should be merged into a string compare - { - Regex re(".?hello"); - EXPECT_PATTERNS_IN_DUMP(re, "CompareSimple String \"hello\""sv); - - EXPECT_EQ(re.match("hello"sv).success, true); - EXPECT_EQ(re.match("xhello"sv).success, true); - EXPECT_EQ(re.match("world"sv).success, false); - } - - // Two characters should also be merged - { - Regex re(".?ab"); - EXPECT_PATTERNS_IN_DUMP(re, "CompareSimple String \"ab\""sv); + auto regex = MUST(regex::ECMAScriptRegex::compile("aba"sv, {})); - EXPECT_EQ(re.match("ab"sv).success, true); - EXPECT_EQ(re.match("xab"sv).success, true); - EXPECT_EQ(re.match("ba"sv).success, false); - } + EXPECT_EQ(regex.find_all(u"aba aba"sv, 0), 2); + EXPECT_EQ(regex.find_all_match(0).start, 0); + EXPECT_EQ(regex.find_all_match(0).end, 3); + EXPECT_EQ(regex.find_all_match(1).start, 4); + EXPECT_EQ(regex.find_all_match(1).end, 7); } -TEST_CASE(optimizer_simple_compare_char) +TEST_CASE(unicode_property_matching_works) { - // A single character compare should become 'CompareSimple Char' - { - Regex re(".*a"); - EXPECT_PATTERNS_IN_DUMP(re, "CompareSimple Char 'a'"sv); + auto regex = MUST(regex::ECMAScriptRegex::compile("\\p{ASCII}+"sv, { .unicode = true })); - EXPECT_EQ(re.match("a"sv).success, true); - EXPECT_EQ(re.match("ba"sv).success, true); - EXPECT_EQ(re.match("b"sv).success, false); - } + EXPECT_EQ(regex.test(u"ASCII"sv, 0), regex::MatchResult::Match); + EXPECT_EQ(regex.test(u"πŸ˜€"sv, 0), regex::MatchResult::NoMatch); } -TEST_CASE(optimizer_simple_compare_char_class) +TEST_CASE(end_anchored_suffix_patterns_preserve_behavior) { - // A single char class compare should become 'CompareSimple CharClass' - { - Regex re(".*\\d"); - EXPECT_PATTERNS_IN_DUMP(re, "CompareSimple CharClass"sv); + auto regex = MUST(regex::ECMAScriptRegex::compile("(.*)\\/client-(.*)\\.js$"sv, {})); - EXPECT_EQ(re.match("abc5"sv).success, true); - EXPECT_EQ(re.match("abc"sv).success, false); - } + EXPECT_EQ(regex.test(u"https://cdn.example.com/assets/client-main.js"sv, 0), regex::MatchResult::Match); + EXPECT_EQ(regex.test(u""sv, 0), regex::MatchResult::NoMatch); }