dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibRegex: Add support for regex modifiers

Browse source 
This commit implements the regexp-modifiers proposal. It allows us to
use modification of i,m,s flags within groups using
`(?flags:subpattern)` and `(?flags-flags:subpattern)` syntax.
This commit is contained in:
aplefull 2025-12-04 00:16:16 +01:00 committed by Ali Mohammad Pur
parent 3fb0e69c20
commit e4572aa9d7
9 changed files with 319 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
Libraries/LibJS/Tests/builtins/RegExp/RegExp.js
View file

@ -376,3 +376,61 @@ test("RegExp string literal", () => {
expect(() => new RegExp(pattern, "v")).toThrow(SyntaxError);
});
});
// https://github.com/tc39/test262/tree/main/test/built-ins/RegExp/regexp-modifiers
test("RegExp modifiers", () => {
const testModifiers = (pattern, flags, tests) => {
const re = new RegExp(pattern, flags);
tests.forEach(([input, expected]) => expect(re.test(input)).toBe(expected));
};
testModifiers("(^a$)|(?:^b$)|(?m:^c$)|(?:^d$)|(^e$)", "", [
["\na\n", false],
["\nb\n", false],
["\nc\n", true],
["\nd\n", false],
["\ne\n", false],
]);
testModifiers("(?m-:es$|(?-m:js$))", "", [
["es\ns", true],
["js", true],
["js\ns", false],
]);
testModifiers("(a)|(?:b)|(?-i:c)|(?:d)|(e)", "i", [
["A", true],
["B", true],
["C", false],
["D", true],
["E", true],
]);
testModifiers("(?m:es.$)", "", [
["esz\n", true],
["es\n\n", false],
]);
testModifiers("(?m-:es.$)", "s", [
["esz\n", true],
["es\n\n", true],
]);
testModifiers("(?-i:\\u{0061})b", "iu", [
["ab", true],
["aB", true],
["Ab", false],
]);
testModifiers("(?-i:\\p{Lu})", "iu", [
["A", true],
["a", false],
["Z", true],
["z", false],
]);
testModifiers("(?-m:^es)$", "m", [
["e\nes\n", false],
["es\n", true],
]);
});

50
Libraries/LibRegex/RegexByteCode.cpp
View file

@ -240,6 +240,26 @@ void FlatByteCode::ensure_opcodes_initialized()
s_opcodes_initialized = true;
}
template<typename ByteCode>
ALWAYS_INLINE ExecutionResult OpCode_SaveModifiers<ByteCode>::execute(MatchInput const&, MatchState& state) const
{
auto current_flags = to_underlying(state.current_options.value());
state.modifier_stack.append(current_flags);
state.current_options = AllOptions { static_cast<AllFlags>(new_modifiers()) };
return ExecutionResult::Continue;
}
template<typename ByteCode>
ALWAYS_INLINE ExecutionResult OpCode_RestoreModifiers<ByteCode>::execute(MatchInput const&, MatchState& state) const
{
if (state.modifier_stack.is_empty())
return ExecutionResult::Failed;
auto previous_modifiers = state.modifier_stack.take_last();
state.current_options = AllOptions { static_cast<AllFlags>(previous_modifiers) };
return ExecutionResult::Continue;
}
template<typename ByteCode>
ALWAYS_INLINE ExecutionResult OpCode_Exit<ByteCode>::execute(MatchInput const& input, MatchState& state) const
{
@ -439,7 +459,7 @@ ALWAYS_INLINE ExecutionResult OpCode_CheckBegin<ByteCode>::execute(MatchInput co
if (state.string_position == 0)
return true;
if (input.regex_options.has_flag_set(AllFlags::Multiline) && input.regex_options.has_flag_set(AllFlags::Internal_ConsiderNewline)) {
if (state.current_options.has_flag_set(AllFlags::Multiline) && state.current_options.has_flag_set(AllFlags::Internal_ConsiderNewline)) {
auto input_view = input.view.substring_view(state.string_position - 1, 1).code_point_at(0);
return input_view == '\r' || input_view == '\n' || input_view == LineSeparator || input_view == ParagraphSeparator;
}
@ -447,12 +467,12 @@ ALWAYS_INLINE ExecutionResult OpCode_CheckBegin<ByteCode>::execute(MatchInput co
return false;
}();
if (is_at_line_boundary && (input.regex_options & AllFlags::MatchNotBeginOfLine))
if (is_at_line_boundary && (state.current_options & AllFlags::MatchNotBeginOfLine))
return ExecutionResult::Failed_ExecuteLowPrioForks;
if ((is_at_line_boundary && !(input.regex_options & AllFlags::MatchNotBeginOfLine))
|| (!is_at_line_boundary && (input.regex_options & AllFlags::MatchNotBeginOfLine))
|| (is_at_line_boundary && (input.regex_options & AllFlags::Global)))
if ((is_at_line_boundary && !(state.current_options & AllFlags::MatchNotBeginOfLine))
|| (!is_at_line_boundary && (state.current_options & AllFlags::MatchNotBeginOfLine))
|| (is_at_line_boundary && (state.current_options & AllFlags::Global)))
return ExecutionResult::Continue;
return ExecutionResult::Failed_ExecuteLowPrioForks;
@ -495,18 +515,18 @@ ALWAYS_INLINE ExecutionResult OpCode_CheckEnd<ByteCode>::execute(MatchInput cons
if (state.string_position == input.view.length())
return true;
if (input.regex_options.has_flag_set(AllFlags::Multiline) && input.regex_options.has_flag_set(AllFlags::Internal_ConsiderNewline)) {
if (state.current_options.has_flag_set(AllFlags::Multiline) && state.current_options.has_flag_set(AllFlags::Internal_ConsiderNewline)) {
auto input_view = input.view.substring_view(state.string_position, 1).code_point_at(0);
return input_view == '\r' || input_view == '\n' || input_view == LineSeparator || input_view == ParagraphSeparator;
}
return false;
}();
if (is_at_line_boundary && (input.regex_options & AllFlags::MatchNotEndOfLine))
if (is_at_line_boundary && (state.current_options & AllFlags::MatchNotEndOfLine))
return ExecutionResult::Failed_ExecuteLowPrioForks;
if ((is_at_line_boundary && !(input.regex_options & AllFlags::MatchNotEndOfLine))
|| (!is_at_line_boundary && (input.regex_options & AllFlags::MatchNotEndOfLine || input.regex_options & AllFlags::MatchNotBeginOfLine)))
if ((is_at_line_boundary && !(state.current_options & AllFlags::MatchNotEndOfLine))
|| (!is_at_line_boundary && (state.current_options & AllFlags::MatchNotEndOfLine || state.current_options & AllFlags::MatchNotBeginOfLine)))
return ExecutionResult::Continue;
return ExecutionResult::Failed_ExecuteLowPrioForks;
@ -736,11 +756,11 @@ ALWAYS_INLINE ExecutionResult CompareInternals<ByteCode, IsSimple>::execute(Matc
auto input_view = input.view.substring_view(state.string_position, 1).code_point_at(0);
auto is_equivalent_to_newline = input_view == '\n'
|| (input.regex_options.has_flag_set(AllFlags::Internal_ECMA262DotSemantics)
|| (state.current_options.has_flag_set(AllFlags::Internal_ECMA262DotSemantics)
? (input_view == '\r' || input_view == LineSeparator || input_view == ParagraphSeparator)
: false);
if (!is_equivalent_to_newline || (input.regex_options.has_flag_set(AllFlags::SingleLine) && input.regex_options.has_flag_set(AllFlags::Internal_ConsiderNewline))) {
if (!is_equivalent_to_newline || (state.current_options.has_flag_set(AllFlags::SingleLine) && state.current_options.has_flag_set(AllFlags::Internal_ConsiderNewline))) {
if (current_inversion_state())
inverse_matched = true;
else
@ -792,7 +812,7 @@ ALWAYS_INLINE ExecutionResult CompareInternals<ByteCode, IsSimple>::execute(Matc
auto insensitive_range_data = bytecode().flat_data().slice(offset, count_insensitive);
offset += count_insensitive;
bool const insensitive = input.regex_options & AllFlags::Insensitive;
bool const insensitive = state.current_options & AllFlags::Insensitive;
auto ch = input.view.unicode_aware_code_point_at(state.string_position_in_code_units);
if (insensitive)
@ -960,7 +980,7 @@ ALWAYS_INLINE ExecutionResult CompareInternals<ByteCode, IsSimple>::execute(Matc
value = input.view.code_point_at(current_code_unit_offset);
}
if (input.regex_options & AllFlags::Insensitive) {
if (state.current_options & AllFlags::Insensitive) {
bool found_child = false;
for (auto const& [key, child] : current->children()) {
if (to_ascii_lowercase(key) == to_ascii_lowercase(value)) {
@ -1196,7 +1216,7 @@ ALWAYS_INLINE void CompareInternals<ByteCode, IsSimple>::compare_char(MatchInput
: input.view.unicode_aware_code_point_at(state.string_position_in_code_units);
bool equal;
if (input.regex_options & AllFlags::Insensitive) {
if (state.current_options & AllFlags::Insensitive) {
if (input.view.unicode()) {
auto lhs = String::from_code_point(input_view);
auto rhs = String::from_code_point(ch1);
@ -1241,7 +1261,7 @@ ALWAYS_INLINE bool CompareInternals<ByteCode, IsSimple>::compare_string(MatchInp
auto subject = input.view.substring_view(state.string_position, str.length());
bool equals;
if (input.regex_options & AllFlags::Insensitive)
if (state.current_options & AllFlags::Insensitive)
equals = subject.equals_ignoring_case(str);
else
equals = subject.equals(str);

42
Libraries/LibRegex/RegexByteCode.h
View file

@ -56,6 +56,8 @@ using ByteCodeValueType = u64;
__ENUMERATE_OPCODE(ResetRepeat) \
__ENUMERATE_OPCODE(Checkpoint) \
__ENUMERATE_OPCODE(CompareSimple) \
__ENUMERATE_OPCODE(SaveModifiers) \
__ENUMERATE_OPCODE(RestoreModifiers) \
__ENUMERATE_OPCODE(Exit)
// clang-format off
@ -498,6 +500,17 @@ public:
m_group_name_mappings.set(capture_groups_count - 1, name_string_index);
}
void insert_bytecode_save_modifiers(FlagsUnderlyingType new_modifiers)
{
empend(static_cast<ByteCodeValueType>(OpCodeId::SaveModifiers));
empend(static_cast<ByteCodeValueType>(new_modifiers));
}
void insert_bytecode_restore_modifiers()
{
empend(static_cast<ByteCodeValueType>(OpCodeId::RestoreModifiers));
}
enum class LookAroundType {
LookAhead,
LookBehind,
@ -946,6 +959,35 @@ protected:
MatchState const* m_state { nullptr };
};
template<typename ByteCode>
class OpCode_SaveModifiers final : public OpCode<ByteCode> {
public:
using OpCode<ByteCode>::argument;
using OpCode<ByteCode>::name;
using OpCode<ByteCode>::state;
using OpCode<ByteCode>::bytecode;
ExecutionResult execute(MatchInput const& input, MatchState& state) const override;
ALWAYS_INLINE OpCodeId opcode_id() const override { return OpCodeId::SaveModifiers; }
ALWAYS_INLINE size_t size() const override { return 2; }
ALWAYS_INLINE FlagsUnderlyingType new_modifiers() const { return argument(0); }
ByteString arguments_string() const override { return ByteString::formatted("new_modifiers={:#x}", new_modifiers()); }
};
template<typename ByteCode>
class OpCode_RestoreModifiers final : public OpCode<ByteCode> {
public:
using OpCode<ByteCode>::argument;
using OpCode<ByteCode>::name;
using OpCode<ByteCode>::state;
using OpCode<ByteCode>::bytecode;
ExecutionResult execute(MatchInput const& input, MatchState& state) const override;
ALWAYS_INLINE OpCodeId opcode_id() const override { return OpCodeId::RestoreModifiers; }
ALWAYS_INLINE size_t size() const override { return 1; }
ByteString arguments_string() const override { return ByteString::empty(); }
};
template<typename ByteCode>
class OpCode_Exit final : public OpCode<ByteCode> {
public:

2
Libraries/LibRegex/RegexDefs.h
View file

@ -29,6 +29,8 @@ enum __Regex_Error {
__Regex_DuplicateNamedCapture, // Duplicate named capture group
__Regex_InvalidCharacterClassEscape, // Invalid escaped entity in character class.
__Regex_NegatedCharacterClassStrings, // Negated character class cannot contain strings.
__Regex_InvalidModifierGroup, // Invalid modifier group.
__Regex_RepeatedModifierFlag, // Repeated flag in modifier group.
};
enum __RegexAllFlags {

8
Libraries/LibRegex/RegexError.h
View file

@ -34,6 +34,8 @@ enum class Error : u8 {
DuplicateNamedCapture = __Regex_DuplicateNamedCapture, // Name of property is invalid.
InvalidCharacterClassEscape = __Regex_InvalidCharacterClassEscape, // Invalid escaped entity in character class.
NegatedCharacterClassStrings = __Regex_NegatedCharacterClassStrings, // Negated character class may contain strings.
InvalidModifierGroup = __Regex_InvalidModifierGroup, // Invalid modifier group.
RepeatedModifierFlag = __Regex_RepeatedModifierFlag, // Repeated flag in modifier group.
};
inline StringView get_error_string(Error error)
@ -80,7 +82,11 @@ inline StringView get_error_string(Error error)
case Error::InvalidCharacterClassEscape:
return "Invalid escaped entity in character class."sv;
case Error::NegatedCharacterClassStrings:
return "Negated character class cannot contain strings"sv;
return "Negated character class cannot contain strings."sv;
case Error::InvalidModifierGroup:
return "Invalid modifier group."sv;
case Error::RepeatedModifierFlag:
return "Repeated flag in modifier group."sv;
}
return "Undefined error."sv;
}

5
Libraries/LibRegex/RegexMatch.h
View file

@ -424,9 +424,12 @@ struct MatchState {
COWVector<u64> repetition_marks;
Vector<u64, 64> checkpoints;
Vector<i64> step_backs;
Vector<FlagsUnderlyingType, 1> modifier_stack;
AllOptions current_options;
explicit MatchState(size_t capture_group_count)
explicit MatchState(size_t capture_group_count, AllOptions options = {})
: capture_group_count(capture_group_count)
, current_options(options)
{
}

6
Libraries/LibRegex/RegexMatcher.cpp
View file

@ -177,13 +177,13 @@ RegexResult Matcher<Parser>::match(Vector<RegexStringView> const& views, Optiona
size_t match_count { 0 };
MatchInput input;
MatchState state { m_pattern->parser_result.capture_groups_count };
size_t operations = 0;
input.pattern = m_pattern->pattern_value;
input.regex_options = m_regex_options | regex_options.value_or({}).value();
input.start_offset = m_pattern->start_offset;
MatchState state(m_pattern->parser_result.capture_groups_count, input.regex_options);
size_t lines_to_skip = 0;
bool unicode = input.regex_options.has_flag_set(AllFlags::Unicode) || input.regex_options.has_flag_set(AllFlags::UnicodeSets);
@ -276,6 +276,8 @@ RegexResult Matcher<Parser>::match(Vector<RegexStringView> const& views, Optiona
state.instruction_position = 0;
state.repetition_marks.clear();
state.modifier_stack.clear();
state.current_options = input.regex_options;
auto result = execute(input, state, temp_operations);
// This success is acceptable only if it doesn't read anything from the input (input length is 0).
@ -336,6 +338,8 @@ RegexResult Matcher<Parser>::match(Vector<RegexStringView> const& views, Optiona
}
state.instruction_position = 0;
state.repetition_marks.clear();
state.modifier_stack.clear();
state.current_options = input.regex_options;
if (auto const result = execute(input, state, operations); result == ExecuteResult::Matched) {
succeeded = true;

130
Libraries/LibRegex/RegexParser.cpp
View file

@ -2848,6 +2848,136 @@ bool ECMA262Parser::parse_capture_group(ByteCode& stack, size_t& match_length_mi
return true;
}
if (match(TokenType::Char) || match(TokenType::HyphenMinus)) {
if (!match(TokenType::HyphenMinus)) {
auto value = m_parser_state.current_token.value();
if (value.length() != 1 || !"ims"sv.contains(value[0]))
goto not_a_modifier;
}
auto saved_position = m_parser_state.lexer.tell();
auto reset_to_saved_position = [&] {
m_parser_state.lexer.retreat(m_parser_state.lexer.tell() - saved_position);
m_parser_state.current_token = m_parser_state.lexer.next();
};
enum class ModifierParseResult : u8 {
Success,
InvalidGroup,
RepeatedFlag,
};
auto parse_modifier_flags = [&](bool& has_i, bool& has_m, bool& has_s) -> ModifierParseResult {
has_i = false;
has_m = false;
has_s = false;
while (match(TokenType::Char)) {
auto value = m_parser_state.current_token.value();
bool* flag_ptr;
switch (value[0]) {
case 'i':
flag_ptr = &has_i;
break;
case 'm':
flag_ptr = &has_m;
break;
case 's':
flag_ptr = &has_s;
break;
default:
return ModifierParseResult::InvalidGroup;
}
if (*flag_ptr)
return ModifierParseResult::RepeatedFlag;
*flag_ptr = true;
consume();
}
return ModifierParseResult::Success;
};
bool add_i = false;
bool add_m = false;
bool add_s = false;
auto add_result = parse_modifier_flags(add_i, add_m, add_s);
if (add_result == ModifierParseResult::InvalidGroup)
return set_error(Error::InvalidModifierGroup);
if (add_result == ModifierParseResult::RepeatedFlag)
return set_error(Error::RepeatedModifierFlag);
bool remove_i = false;
bool remove_m = false;
bool remove_s = false;
if (match(TokenType::HyphenMinus)) {
consume();
auto remove_result = parse_modifier_flags(remove_i, remove_m, remove_s);
if (remove_result == ModifierParseResult::InvalidGroup)
return set_error(Error::InvalidModifierGroup);
if (remove_result == ModifierParseResult::RepeatedFlag)
return set_error(Error::RepeatedModifierFlag);
if ((add_i && remove_i) || (add_m && remove_m) || (add_s && remove_s))
return set_error(Error::RepeatedModifierFlag);
if (!add_i && !add_m && !add_s && !remove_i && !remove_m && !remove_s)
return set_error(Error::InvalidModifierGroup);
}
if (!match(TokenType::Colon)) {
reset_to_saved_position();
goto not_a_modifier;
}
consume();
auto current_options = to_underlying(m_parser_state.regex_options.value());
FlagsUnderlyingType updated_options = current_options;
auto update_modifier_flag = [&](bool add, bool remove, AllFlags flag) {
auto flag_value = static_cast<FlagsUnderlyingType>(flag);
if (add)
updated_options |= flag_value;
if (remove)
updated_options &= ~flag_value;
};
update_modifier_flag(add_i, remove_i, AllFlags::Insensitive);
update_modifier_flag(add_m, remove_m, AllFlags::Multiline);
update_modifier_flag(add_s, remove_s, AllFlags::SingleLine);
ByteCode modifier_group_bytecode;
size_t length = 0;
stack.insert_bytecode_save_modifiers(updated_options);
auto saved_parser_options = m_parser_state.regex_options;
m_parser_state.regex_options = AllOptions { static_cast<AllFlags>(updated_options) };
enter_capture_group_scope();
if (!parse_disjunction(modifier_group_bytecode, length, flags)) {
m_parser_state.regex_options = saved_parser_options;
return set_error(Error::InvalidPattern);
}
clear_all_capture_groups_in_scope(stack);
exit_capture_group_scope();
m_parser_state.regex_options = saved_parser_options;
consume(TokenType::RightParen, Error::MismatchingParen);
stack.extend(move(modifier_group_bytecode));
stack.insert_bytecode_restore_modifiers();
match_length_minimum += length;
return true;
}
not_a_modifier:
if (consume("<")) {
++m_parser_state.named_capture_groups_count;
auto group_index = ++m_parser_state.capture_groups_count; // Named capture groups count as normal capture groups too.

36
Tests/LibRegex/TestRegex.cpp
View file

@ -608,6 +608,14 @@ TEST_CASE(ECMA262_parse)
{ "(\"|')(?:(?!\\2)[^\\\\\\r\\n]|\\\\.)*\\2"sv, regex::Error::NoError, ECMAScriptFlags::BrowserExtended }, // LegacyOctalEscapeSequence should not consume too many chars (and should not crash)
// #18324, Capture group counter skipped past EOF.
{ "\\1[\\"sv, regex::Error::InvalidNumber },
{ "(?ii:a)"sv, regex::Error::RepeatedModifierFlag },
{ "(?i-i:a)"sv, regex::Error::RepeatedModifierFlag },
{ "(?-ii:a)"sv, regex::Error::RepeatedModifierFlag },
{ "(?-:a)"sv, regex::Error::InvalidModifierGroup },
{ "(?-ig:a)"sv, regex::Error::InvalidModifierGroup },
{ "(?-x:a)"sv, regex::Error::InvalidModifierGroup },
{ "(?i)"sv, regex::Error::InvalidCaptureGroup },
{ "(?-i)"sv, regex::Error::InvalidCaptureGroup },
};
for (auto& test : tests) {
@ -1589,3 +1597,31 @@ TEST_CASE(optional_groups_with_empty_matches)
EXPECT_EQ(result6.matches.first().view.to_byte_string(), "a"sv);
EXPECT(result6.capture_group_matches.first()[0].view.is_null());
}
TEST_CASE(ecma262_modifiers)
{
struct Test {
StringView pattern;
StringView subject;
bool matches { true };
ECMAScriptFlags flags {};
};
constexpr Test tests[] {
{ "a(?i:b)c"sv, "aBc"sv, true, {} },
{ "a(?i:b)c"sv, "aBC"sv, false, {} },
{ "a(?s:.)c"sv, "a\nc"sv, true, {} },
{ "(?ims:a.b)"sv, "A\nB"sv, true, {} },
{ "(?i:a(?-i:b)c)"sv, "AbC"sv, true, {} },
{ "(?i:a(?-i:b)c)"sv, "ABC"sv, false, {} },
{ "a(?-i:b)c"sv, "AbC"sv, true, ECMAScriptFlags::Insensitive },
{ "a(?-i:b)c"sv, "ABC"sv, false, ECMAScriptFlags::Insensitive },
{ "x.(?m:^a)"sv, "x\na"sv, true, ECMAScriptFlags::SingleLine },
};
for (auto const& test : tests) {
Regex<ECMA262> re(test.pattern, test.flags);
auto result = re.match(test.subject);
EXPECT_EQ(result.success, test.matches);
}
}