LibURL: Fix public suffix matching for canonical hosts

Public suffix matching could fail when callers passed host text that
was not already in the same form as the generated PSL table. In
particular, uppercase ASCII hosts like EXAMPLE.COM and UTF-8 IDN hosts
could miss matches even though URL hosts are canonically represented as
lowercase ASCII/IDNA.

Make the PublicSuffixData API difficult to misuse by routing all
invocations through URL::Host overloads so that the canonical hostname
is always what is matched against.

Meta/Generators/generate_public_suffix_data.py

@@ -16,6 +16,7 @@ def generate_header_file(output_path: Path) -> None:
 #include <AK/Forward.h>
 #include <AK/Trie.h>
 #include <AK/Variant.h>
+#include <LibURL/Forward.h>
 
 namespace URL {
 
@@ -36,8 +37,11 @@ public:
     }
 
     bool is_matching_public_suffix(StringView host);
+    bool is_matching_public_suffix(Host const& host);
     Optional<String> find_matching_public_suffix(StringView string);
+    Optional<String> find_matching_public_suffix(Host const& host);
     Optional<String> find_matching_registrable_domain(StringView string);
+    Optional<String> find_matching_registrable_domain(Host const& host);
 
 };
 
@@ -49,10 +53,27 @@ public:
         f.write(content)
 
 
+def canonicalize_public_suffix_rule(rule: str) -> str:
+    def canonicalize_label(label: str) -> str:
+        if label == "*":
+            return label
+
+        prefix = ""
+        if label.startswith("!"):
+            prefix = "!"
+            label = label[1:]
+
+        return prefix + label.encode("idna").decode("ascii").lower()
+
+    return ".".join(canonicalize_label(label) for label in rule.split("."))
+
+
 def generate_implementation_file(input_path: Path, output_path: Path) -> None:
     content = """#include <AK/String.h>
 #include <AK/BinarySearch.h>
 #include <AK/Vector.h>
+#include <LibURL/Host.h>
+#include <LibURL/Parser.h>
 #include <LibURL/PublicSuffixData.h>
 
 namespace URL {
@@ -67,6 +88,7 @@ static constexpr auto s_public_suffixes = Array {"""
             if line.startswith("//") or not line:
                 continue
 
+            line = canonicalize_public_suffix_rule(line)
             reversed_line = ".".join(line.split(".")[::-1])
             reversed_lines.append(reversed_line)
 
@@ -87,7 +109,15 @@ static bool is_reversed_public_suffix(StringView reversed_host)
     return binary_search(s_public_suffixes, reversed_host);
 }
 
-bool PublicSuffixData::is_matching_public_suffix(StringView host)
+static Optional<StringView> serialized_domain(Host const& host)
+{
+    if (!host.is_domain())
+        return OptionalNone {};
+
+    return host.get<String>().bytes_as_string_view();
+}
+
+static bool is_matching_public_suffix_impl(StringView host)
 {
     // Empty labels are kept so that inputs such as "com." do not match the bare "com" entry.
     auto labels = host.split_view('.', SplitBehavior::KeepEmpty);
@@ -99,7 +129,28 @@ bool PublicSuffixData::is_matching_public_suffix(StringView host)
     return is_reversed_public_suffix(reversed_host.string_view());
 }
 
-Optional<String> PublicSuffixData::find_matching_public_suffix(StringView string)
+bool PublicSuffixData::is_matching_public_suffix(StringView host)
+{
+    if (host.is_empty())
+        return false;
+
+    auto parsed_host = Parser::parse_host(host);
+    if (!parsed_host.has_value())
+        return false;
+
+    return is_matching_public_suffix(*parsed_host);
+}
+
+bool PublicSuffixData::is_matching_public_suffix(Host const& host)
+{
+    auto domain = serialized_domain(host);
+    if (!domain.has_value())
+        return false;
+
+    return is_matching_public_suffix_impl(*domain);
+}
+
+static Optional<String> find_matching_public_suffix_impl(StringView string)
 {
     auto input = string.split_view('.');
     input.reverse();
@@ -140,11 +191,32 @@ Optional<String> PublicSuffixData::find_matching_public_suffix(StringView string
     return MUST(return_string_builder.to_string());
 }
 
+Optional<String> PublicSuffixData::find_matching_public_suffix(StringView string)
+{
+    if (string.is_empty())
+        return {};
+
+    auto parsed_host = Parser::parse_host(string);
+    if (!parsed_host.has_value())
+        return {};
+
+    return find_matching_public_suffix(*parsed_host);
+}
+
+Optional<String> PublicSuffixData::find_matching_public_suffix(Host const& host)
+{
+    auto domain = serialized_domain(host);
+    if (!domain.has_value())
+        return {};
+
+    return find_matching_public_suffix_impl(*domain);
+}
+
 // https://github.com/publicsuffix/list/wiki/Format#algorithm
-Optional<String> PublicSuffixData::find_matching_registrable_domain(StringView host)
+static Optional<String> find_matching_registrable_domain_impl(StringView host)
 {
     // The registered or registrable domain is the public suffix plus one additional label.
-    auto public_suffix = find_matching_public_suffix(host);
+    auto public_suffix = find_matching_public_suffix_impl(host);
     if (!public_suffix.has_value() || !host.ends_with(*public_suffix))
         return {};
 
@@ -164,6 +236,27 @@ Optional<String> PublicSuffixData::find_matching_registrable_domain(StringView h
     return MUST(String::from_utf8(host.substring_view(start_index)));
 }
 
+Optional<String> PublicSuffixData::find_matching_registrable_domain(StringView string)
+{
+    if (string.is_empty())
+        return {};
+
+    auto parsed_host = Parser::parse_host(string);
+    if (!parsed_host.has_value())
+        return {};
+
+    return find_matching_registrable_domain(*parsed_host);
+}
+
+Optional<String> PublicSuffixData::find_matching_registrable_domain(Host const& host)
+{
+    auto domain = serialized_domain(host);
+    if (!domain.has_value())
+        return {};
+
+    return find_matching_registrable_domain_impl(*domain);
+}
+
 }
 
 """

Tests/LibURL/TestPublicSuffix.cpp

@@ -6,6 +6,7 @@
 
 #include <LibTest/TestCase.h>
 
+#include <LibURL/Parser.h>
 #include <LibURL/PublicSuffixData.h>
 
 TEST_CASE(is_public_suffix)
@@ -19,6 +20,9 @@ TEST_CASE(is_public_suffix)
     EXPECT(public_suffix_data->is_matching_public_suffix("gov.uk"sv));
     EXPECT(public_suffix_data->is_matching_public_suffix("com.au"sv));
     EXPECT(public_suffix_data->is_matching_public_suffix("co.jp"sv));
+    EXPECT(public_suffix_data->is_matching_public_suffix("COM"sv));
+    EXPECT(public_suffix_data->is_matching_public_suffix("公司.cn"sv));
+    EXPECT(public_suffix_data->is_matching_public_suffix("xn--55qx5d.cn"sv));
 
     EXPECT(!public_suffix_data->is_matching_public_suffix(""sv));
     EXPECT(!public_suffix_data->is_matching_public_suffix("."sv));
@@ -54,4 +58,7 @@ TEST_CASE(get_public_suffix)
     EXPECT_EQ(public_suffix_data->find_matching_public_suffix("co.uk"sv), "co.uk"sv);
     EXPECT_EQ(public_suffix_data->find_matching_public_suffix("bbc.co.uk"sv), "co.uk"sv);
     EXPECT_EQ(public_suffix_data->find_matching_public_suffix("www.bbc.co.uk"sv), "co.uk"sv);
+    EXPECT_EQ(public_suffix_data->find_matching_public_suffix("EXAMPLE.COM"sv), "com"sv);
+    EXPECT_EQ(public_suffix_data->find_matching_public_suffix("www.公司.cn"sv), "xn--55qx5d.cn"sv);
+    EXPECT_EQ(public_suffix_data->find_matching_public_suffix("www.xn--55qx5d.cn"sv), "xn--55qx5d.cn"sv);
 }