From cd3c72dfda4d8b11919ebb56cef9a4a7135d00c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Chr=C3=A1stek?= Date: Sat, 23 May 2026 11:00:51 +0200 Subject: [PATCH] LibWeb: Implement `chardetng`-based encoding detection for HTML parsing --- Cargo.lock | 21 ++++++ Libraries/LibWeb/CMakeLists.txt | 8 +- .../LibWeb/HTML/FormAssociatedElement.cpp | 3 +- Libraries/LibWeb/HTML/HTMLBaseElement.cpp | 5 +- Libraries/LibWeb/HTML/HTMLFormElement.cpp | 2 +- .../HTML/Parser/HTMLEncodingDetection.cpp | 47 ++++++++++-- .../HTML/Parser/HTMLEncodingDetection.h | 10 ++- Libraries/LibWeb/Rust/Cargo.toml | 1 + Libraries/LibWeb/Rust/build.rs | 24 +++++- .../LibWeb/Rust/src/encoding_detection.rs | 73 +++++++++++++++++++ Libraries/LibWeb/Rust/src/lib.rs | 1 + Meta/CMake/flatpak/cargo-sources.json | 16 ++++ .../has-concrete-structural-triggers.txt | 2 +- .../ar-ISO-8859-6.tentative.txt | 6 ++ .../ar-windows-1256.tentative.txt | 6 ++ .../ca-windows-1252.tentative.txt | 6 ++ .../el-ISO-8859-7.tentative.txt | 6 ++ .../el-windows-1253.tentative.txt | 6 ++ .../en-copyright-windows-1252.tentative.txt | 6 ++ .../en-euro-windows-1252.tentative.txt | 6 ++ .../en-windows-1252.tentative.txt | 6 ++ .../es-maria-windows-1252.txt | 6 ++ ...-nuestra-senora-windows-1252.tentative.txt | 6 ++ .../es-number-one-windows-1252.tentative.txt | 6 ++ .../es-ordinal-windows-1252.txt | 6 ++ .../fa-windows-1256.tentative.txt | 6 ++ .../fi-windows-1252.tentative.txt | 6 ++ .../he-ISO-8859-8.tentative.txt | 6 ++ .../he-windows-1255.tentative.txt | 6 ++ .../is-windows-1252.tentative.txt | 6 ++ .../it-number-one-windows-1252.tentative.txt | 6 ++ .../it-ordinal-windows-1252.txt | 6 ++ .../it-roman-ordinal-windows-1252.txt | 6 ++ .../it-windows-1252.tentative.txt | 6 ++ .../ja-EUC-JP.tentative.txt | 6 ++ .../ja-ISO-2022-JP.tentative.txt | 6 ++ .../ja-Shift_JIS.tentative.txt | 6 ++ .../ja-half-width.tentative.txt | 6 ++ .../ko-EUC-KR.tentative.txt | 6 ++ .../lt-windows-1257.tentative.txt | 6 ++ .../lv-windows-1257.tentative.txt | 6 ++ .../nbsp-even-windows-1252.tentative.txt | 6 ++ .../nbsp-odd-windows-1252.tentative.txt | 6 ++ .../pl-ISO-8859-2.tentative.txt | 6 ++ .../pl-windows-1250.tentative.txt | 6 ++ .../pt-windows-1252.tentative.txt | 6 ++ .../ru-IBM866.tentative.txt | 6 ++ .../ru-ISO-8859-5.tentative.txt | 6 ++ .../th-windows-874.tentative.txt | 6 ++ .../tr-windows-1254.tentative.txt | 6 ++ .../uk-KOI8-U.tentative.txt | 6 ++ .../uk-windows-1251.tentative.txt | 6 ++ .../wpt-import/encoding-detection/utf-8.txt | 6 ++ .../vi-windows-1258.tentative.txt | 6 ++ .../yi-windows-1255.tentative.txt | 6 ++ .../encoding-detection/zh-Big5.tentative.txt | 6 ++ .../encoding-detection/zh-GBK.tentative.txt | 6 ++ .../ar-ISO-8859-6.tentative.html | 14 ++++ .../ar-windows-1256.tentative.html | 14 ++++ .../ca-windows-1252.tentative.html | 14 ++++ .../el-ISO-8859-7.tentative.html | 15 ++++ .../el-windows-1253.tentative.html | 15 ++++ .../en-copyright-windows-1252.tentative.html | 14 ++++ .../en-euro-windows-1252.tentative.html | 14 ++++ .../en-windows-1252.tentative.html | 14 ++++ .../es-maria-windows-1252.html | 14 ++++ ...nuestra-senora-windows-1252.tentative.html | 14 ++++ .../es-number-one-windows-1252.tentative.html | 14 ++++ .../es-ordinal-windows-1252.html | 14 ++++ .../fa-windows-1256.tentative.html | 14 ++++ .../fi-windows-1252.tentative.html | 14 ++++ .../he-ISO-8859-8.tentative.html | 14 ++++ .../he-windows-1255.tentative.html | 14 ++++ .../is-windows-1252.tentative.html | 14 ++++ .../it-number-one-windows-1252.tentative.html | 14 ++++ .../it-ordinal-windows-1252.html | 14 ++++ .../it-roman-ordinal-windows-1252.html | 14 ++++ .../it-windows-1252.tentative.html | 14 ++++ .../ja-EUC-JP.tentative.html | 14 ++++ .../ja-ISO-2022-JP.tentative.html | 14 ++++ .../ja-Shift_JIS.tentative.html | 14 ++++ .../ja-half-width.tentative.html | 14 ++++ .../ko-EUC-KR.tentative.html | 14 ++++ .../lt-windows-1257.tentative.html | 14 ++++ .../lv-windows-1257.tentative.html | 14 ++++ .../nbsp-even-windows-1252.tentative.html | 14 ++++ .../nbsp-odd-windows-1252.tentative.html | 14 ++++ .../pl-ISO-8859-2.tentative.html | 14 ++++ .../pl-windows-1250.tentative.html | 14 ++++ .../pt-windows-1252.tentative.html | 14 ++++ .../ru-IBM866.tentative.html | 14 ++++ .../ru-ISO-8859-5.tentative.html | 14 ++++ .../th-windows-874.tentative.html | 14 ++++ .../tr-windows-1254.tentative.html | 14 ++++ .../uk-KOI8-U.tentative.html | 14 ++++ .../uk-windows-1251.tentative.html | 14 ++++ .../wpt-import/encoding-detection/utf-8.html | 14 ++++ .../vi-windows-1258.tentative.html | 14 ++++ .../yi-windows-1255.tentative.html | 14 ++++ .../encoding-detection/zh-Big5.tentative.html | 14 ++++ .../encoding-detection/zh-GBK.tentative.html | 14 ++++ 101 files changed, 1077 insertions(+), 18 deletions(-) create mode 100644 Libraries/LibWeb/Rust/src/encoding_detection.rs create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ar-ISO-8859-6.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ar-windows-1256.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ca-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/el-ISO-8859-7.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/el-windows-1253.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-copyright-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-euro-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-maria-windows-1252.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-nuestra-senora-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-number-one-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-ordinal-windows-1252.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/fa-windows-1256.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/fi-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/he-ISO-8859-8.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/he-windows-1255.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/is-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-number-one-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-ordinal-windows-1252.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-roman-ordinal-windows-1252.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-EUC-JP.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-ISO-2022-JP.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-Shift_JIS.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-half-width.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ko-EUC-KR.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/lt-windows-1257.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/lv-windows-1257.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/nbsp-even-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/nbsp-odd-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pl-ISO-8859-2.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pl-windows-1250.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pt-windows-1252.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ru-IBM866.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ru-ISO-8859-5.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/th-windows-874.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/tr-windows-1254.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/uk-KOI8-U.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/uk-windows-1251.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/utf-8.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/vi-windows-1258.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/yi-windows-1255.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/zh-Big5.tentative.txt create mode 100644 Tests/LibWeb/Text/expected/wpt-import/encoding-detection/zh-GBK.tentative.txt create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ar-ISO-8859-6.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ar-windows-1256.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ca-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/el-ISO-8859-7.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/el-windows-1253.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-copyright-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-euro-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-maria-windows-1252.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-nuestra-senora-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-number-one-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-ordinal-windows-1252.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/fa-windows-1256.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/fi-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/he-ISO-8859-8.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/he-windows-1255.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/is-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-number-one-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-ordinal-windows-1252.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-roman-ordinal-windows-1252.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-EUC-JP.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-ISO-2022-JP.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-Shift_JIS.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-half-width.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ko-EUC-KR.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/lt-windows-1257.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/lv-windows-1257.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/nbsp-even-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/nbsp-odd-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/pl-ISO-8859-2.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/pl-windows-1250.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/pt-windows-1252.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ru-IBM866.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/ru-ISO-8859-5.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/th-windows-874.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/tr-windows-1254.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/uk-KOI8-U.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/uk-windows-1251.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/utf-8.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/vi-windows-1258.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/yi-windows-1255.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/zh-Big5.tentative.html create mode 100644 Tests/LibWeb/Text/input/wpt-import/encoding-detection/zh-GBK.tentative.html diff --git a/Cargo.lock b/Cargo.lock index 14dca35439..a370ff4a24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chardetng" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + [[package]] name = "clap" version = "4.6.0" @@ -304,6 +315,15 @@ dependencies = [ "syn", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -597,6 +617,7 @@ name = "libweb_rust" version = "0.1.0" dependencies = [ "cbindgen", + "chardetng", "libweb_html_tokenizer", ] diff --git a/Libraries/LibWeb/CMakeLists.txt b/Libraries/LibWeb/CMakeLists.txt index 7ba64bd8cc..0fed5c9325 100644 --- a/Libraries/LibWeb/CMakeLists.txt +++ b/Libraries/LibWeb/CMakeLists.txt @@ -1276,10 +1276,12 @@ if(NOT BUILD_SHARED_LIBS) # POST_BUILD steps don't track inputs, so when libweb_rust.a changes # ninja leaves liblagom-web.a alone and the merged archive ends up with - # stale Rust objects. Force the C++ FFI bridge file to depend on the - # Rust archive: when it changes, RustTokenizer.cpp recompiles, LibWeb - # re-archives, and POST_BUILD re-merges the fresh Rust objects. + # stale Rust objects. Force both FFI bridge files to depend on the + # Rust archive: when it changes, they recompile, LibWeb re-archives, and + # POST_BUILD re-merges the fresh Rust objects. + get_target_property(_libweb_rust_lib libweb_rust IMPORTED_LOCATION) set_property(SOURCE CSS/Parser/RustTokenizer.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libweb_rust_lib}) + set_property(SOURCE HTML/Parser/HTMLEncodingDetection.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libweb_rust_lib}) endif() if (HAS_FONTCONFIG) diff --git a/Libraries/LibWeb/HTML/FormAssociatedElement.cpp b/Libraries/LibWeb/HTML/FormAssociatedElement.cpp index ce34c4245f..c18ef642a1 100644 --- a/Libraries/LibWeb/HTML/FormAssociatedElement.cpp +++ b/Libraries/LibWeb/HTML/FormAssociatedElement.cpp @@ -321,8 +321,7 @@ String FormAssociatedElement::form_action() const return html_element.document().url_string(); } - auto document_base_url = html_element.document().base_url(); - if (auto maybe_url = document_base_url.complete_url(form_action_attribute.value()); maybe_url.has_value()) + if (auto maybe_url = html_element.document().encoding_parse_url(form_action_attribute.value()); maybe_url.has_value()) return maybe_url->to_string(); return {}; } diff --git a/Libraries/LibWeb/HTML/HTMLBaseElement.cpp b/Libraries/LibWeb/HTML/HTMLBaseElement.cpp index 434154f2cd..bbce4ea78e 100644 --- a/Libraries/LibWeb/HTML/HTMLBaseElement.cpp +++ b/Libraries/LibWeb/HTML/HTMLBaseElement.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace Web::HTML { @@ -113,8 +114,8 @@ String HTMLBaseElement::href() const auto url = attribute(AttributeNames::href).value_or(String {}); // 3. Let urlRecord be the result of parsing url with document's fallback base URL, and document's character encoding. (Thus, the base element isn't affected by other base elements or itself.) - // FIXME: Pass in document's character encoding. - auto url_record = document.fallback_base_url().complete_url(url); + auto encoding = document.encoding_or_default(); + auto url_record = DOMURL::parse(url, document.fallback_base_url(), encoding.bytes_as_string_view()); // 4. If urlRecord is failure, return url. if (!url_record.has_value()) diff --git a/Libraries/LibWeb/HTML/HTMLFormElement.cpp b/Libraries/LibWeb/HTML/HTMLFormElement.cpp index 5b6b9d76ba..8bb6b41f4a 100644 --- a/Libraries/LibWeb/HTML/HTMLFormElement.cpp +++ b/Libraries/LibWeb/HTML/HTMLFormElement.cpp @@ -680,7 +680,7 @@ String HTMLFormElement::action() const return document().url_string(); } - if (auto maybe_url = document().base_url().complete_url(form_action_attribute.value()); maybe_url.has_value()) + if (auto maybe_url = document().encoding_parse_url(form_action_attribute.value()); maybe_url.has_value()) return maybe_url->to_string(); return {}; } diff --git a/Libraries/LibWeb/HTML/Parser/HTMLEncodingDetection.cpp b/Libraries/LibWeb/HTML/Parser/HTMLEncodingDetection.cpp index 3c19dae648..4688cb9d89 100644 --- a/Libraries/LibWeb/HTML/Parser/HTMLEncodingDetection.cpp +++ b/Libraries/LibWeb/HTML/Parser/HTMLEncodingDetection.cpp @@ -8,12 +8,13 @@ #include #include #include -#include #include +#include #include #include #include #include +#include #include namespace Web::HTML { @@ -459,6 +460,23 @@ Optional run_bom_sniff(ReadonlyBytes input) return {}; } +ByteString extract_tld_hint(URL::URL const& url) +{ + // Extract the rightmost DNS label from the URL's host as a TLD hint for chardetng. + // chardetng uses this to improve detection accuracy for country-code TLDs (.jp, .ru, .cn, …). + // Skip IP addresses — only domain names have meaningful TLDs. + auto const& maybe_host = url.host(); + if (!maybe_host.has_value() || !maybe_host->is_domain()) + return {}; + auto host_string = maybe_host->serialize(); + StringView host_view = host_string; + auto last_dot = host_view.find_last('.'); + StringView tld_label = last_dot.has_value() + ? host_view.substring_view(*last_dot + 1) + : host_view; + return tld_label.is_empty() ? ByteString {} : ByteString { tld_label }; +} + // https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding ByteString run_encoding_sniffing_algorithm(DOM::Document& document, ReadonlyBytes input, Optional maybe_mime_type) { @@ -499,12 +517,27 @@ ByteString run_encoding_sniffing_algorithm(DOM::Document& document, ReadonlyByte // 7. Otherwise, if the user agent has information on the likely encoding for this page, e.g. based on the encoding of the page when it was last visited, then return // that encoding, with the confidence tentative. - // 8. FIXME: The user agent may attempt to autodetect the character encoding from applying frequency analysis or other algorithms to the data stream. Such algorithms - // may use information about the resource other than the resource's contents, including the address of the resource. If autodetection succeeds in determining a - // character encoding, and that encoding is a supported encoding, then return that encoding, with the confidence tentative. [UNIVCHARDET] - if (!Utf8View(StringView(input)).validate()) { - // FIXME: As soon as Locale is supported, this should sometimes return a different encoding based on the locale. - return "windows-1252"; + // 8. The user agent may attempt to autodetect the character encoding from applying frequency analysis + // or other algorithms to the data stream. Such algorithms may use information about the resource + // other than the resource's contents, including the address of the resource. + // If autodetection succeeds in determining a character encoding, and that encoding is a supported encoding, + // then return that encoding, with the confidence tentative. [UNIVCHARDET] + + auto tld_hint = extract_tld_hint(document.url()); + u8 const* tld_data = tld_hint.is_empty() ? nullptr : reinterpret_cast(tld_hint.characters()); + size_t tld_size = tld_hint.length(); + + u8 const* encoding_name_ptr = nullptr; + size_t encoding_name_len = 0; + + if (Parser::rust_detect_encoding( + input.data(), input.size(), + tld_data, tld_size, + &encoding_name_ptr, &encoding_name_len)) { + auto detected = StringView { reinterpret_cast(encoding_name_ptr), encoding_name_len }; + auto standardized = TextCodec::get_standardized_encoding(detected); + if (standardized.has_value()) + return ByteString { standardized.value() }; } // 9. Otherwise, return an implementation-defined or user-specified default character encoding, with the confidence tentative. diff --git a/Libraries/LibWeb/HTML/Parser/HTMLEncodingDetection.h b/Libraries/LibWeb/HTML/Parser/HTMLEncodingDetection.h index 8b7910192f..53a099cf38 100644 --- a/Libraries/LibWeb/HTML/Parser/HTMLEncodingDetection.h +++ b/Libraries/LibWeb/HTML/Parser/HTMLEncodingDetection.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,13 @@ Optional extract_character_encoding_from_meta_element(ByteString con GC::Ptr prescan_get_attribute(DOM::Document&, ReadonlyBytes input, size_t& position); Optional run_prescan_byte_stream_algorithm(DOM::Document&, ReadonlyBytes input); Optional run_bom_sniff(ReadonlyBytes input); -ByteString run_encoding_sniffing_algorithm(DOM::Document&, ReadonlyBytes input, Optional maybe_mime_type = {}); + +// Extracts the rightmost DNS label from a URL's host as a TLD hint for chardetng. +// Returns an empty string if the host is absent or is an IP address. chardetng treats +// an absent/empty TLD as equivalent to ".com". +ByteString extract_tld_hint(URL::URL const&); + +ByteString run_encoding_sniffing_algorithm(DOM::Document&, ReadonlyBytes input, + Optional maybe_mime_type = {}); } diff --git a/Libraries/LibWeb/Rust/Cargo.toml b/Libraries/LibWeb/Rust/Cargo.toml index 4e9b5c8b22..50c59e00fa 100644 --- a/Libraries/LibWeb/Rust/Cargo.toml +++ b/Libraries/LibWeb/Rust/Cargo.toml @@ -10,6 +10,7 @@ crate-type = ["staticlib"] # python3 Meta/CMake/flatpak/generate-cargo-sources.py [dependencies] libweb_html_tokenizer = { path = "../HTML/Parser/Rust" } +chardetng = "1.0.0" [build-dependencies] cbindgen = "0.29" diff --git a/Libraries/LibWeb/Rust/build.rs b/Libraries/LibWeb/Rust/build.rs index cd3e410bbd..261ebbfb6b 100644 --- a/Libraries/LibWeb/Rust/build.rs +++ b/Libraries/LibWeb/Rust/build.rs @@ -24,7 +24,7 @@ fn main() -> Result<(), Box> { let base_config = cbindgen::Config::from_file(manifest_dir.join("cbindgen.toml"))?; // CSS tokenizer header — namespace Web::CSS::Parser::FFI, CSS types only. - let mut css_config = base_config; + let mut css_config = base_config.clone(); css_config.namespaces = Some(vec![ "Web".to_string(), "CSS".to_string(), @@ -51,5 +51,27 @@ fn main() -> Result<(), Box> { }, ); + // Encoding-detection header — namespace Web::HTML::Parser, rust_detect_encoding only. + let mut html_config = base_config; + html_config.namespaces = Some(vec!["Web".to_string(), "HTML".to_string(), "Parser".to_string()]); + html_config.export.include = vec!["rust_detect_encoding".to_string()]; + + cbindgen::generate_with_config(&manifest_dir, html_config).map_or_else( + |error| match error { + cbindgen::Error::ParseSyntaxError { .. } => {} + other => panic!("{other:?}"), + }, + |bindings| { + let html_header_dir = out_dir.join("HTML").join("Parser"); + std::fs::create_dir_all(&html_header_dir).unwrap(); + bindings.write_to_file(html_header_dir.join("RustFFI.h")); + if ffi_out_dir != out_dir { + let dest = ffi_out_dir.join("HTML").join("Parser"); + std::fs::create_dir_all(&dest).unwrap(); + bindings.write_to_file(dest.join("RustFFI.h")); + } + }, + ); + Ok(()) } diff --git a/Libraries/LibWeb/Rust/src/encoding_detection.rs b/Libraries/LibWeb/Rust/src/encoding_detection.rs new file mode 100644 index 0000000000..a2aea2d38b --- /dev/null +++ b/Libraries/LibWeb/Rust/src/encoding_detection.rs @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection}; + +/// Attempts to detect the character encoding of a byte stream using frequency analysis. +/// +/// This implements step 8 of the WHATWG encoding sniffing algorithm: +/// https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding +/// +/// # Safety +/// - `input` and `input_len` must describe a valid byte slice (or `input` may be null if +/// `input_len` is 0) +/// - `tld` if non-null, must describe a valid byte slice of `tld_len` bytes containing the +/// rightmost DNS label of the resource's host, with no dots, no uppercase, and only ASCII +/// characters — these constraints are required by chardetng and must be validated by the caller +/// - `out_encoding_name` and `out_encoding_name_len` must be non-null writable pointers +/// +/// Returns `true` if an encoding was detected (always, unless the input pointer is invalid). +/// When `true` is returned, `*out_encoding_name` is set to a pointer into a static ASCII +/// string naming the detected encoding (e.g. `"windows-1252"`, `"Shift_JIS"`), and +/// `*out_encoding_name_len` is set to its byte length. The pointer is valid for the lifetime +/// of the process. When `false` is returned (only on null-pointer error), the output pointers +/// are left unmodified. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_detect_encoding( + input: *const u8, + input_len: usize, + tld: *const u8, + tld_len: usize, + out_encoding_name: *mut *const u8, + out_encoding_name_len: *mut usize, +) -> bool { + unsafe { + crate::abort_on_panic(|| { + let Some(input_slice) = crate::bytes_from_raw(input, input_len) else { + return false; + }; + + let tld_slice = if tld.is_null() || tld_len == 0 { + None + } else { + Some(std::slice::from_raw_parts(tld, tld_len)) + }; + + // Web browsers must use `Iso2022JpDetection::Deny` and `Utf8Detection::Deny` to + // prevent charset confusion attacks. See the chardetng documentation for details. + // Japanese pages using ISO-2022-JP will have declared it in a tag, which + // is detected in step 5 (prescan) before this step is reached. + // We always call `guess()` even for pure-ASCII input because chardetng still + // tracks ESC sequences (ISO-2022-JP uses 7-bit escapes) and returns the + // correct locale-based fallback (windows-1252 for generic TLD) with + // `Utf8Detection::Deny` when no distinctive non-ASCII encoding evidence is found. + let mut detector = EncodingDetector::new(Iso2022JpDetection::Deny); + // Pass last=false because the caller may only be providing a sniff-bytes prefix + // of a longer stream. chardetng docs: "If you want to perform detection on just + // the prefix of a longer stream, do not pass last=true." + detector.feed(input_slice, false); + + // `Utf8Detection::Deny`: chardetng never returns UTF-8 here; valid-UTF-8 + // content (including pure ASCII) gets the TLD-based default (windows-1252 + // for generic), while content in a non-UTF-8 encoding gets that encoding. + let encoding = detector.guess(tld_slice, Utf8Detection::Deny); + let name = encoding.name().as_bytes(); + *out_encoding_name = name.as_ptr(); + *out_encoding_name_len = name.len(); + true + }) + } +} diff --git a/Libraries/LibWeb/Rust/src/lib.rs b/Libraries/LibWeb/Rust/src/lib.rs index 30a5e7fcdc..6f76b871be 100644 --- a/Libraries/LibWeb/Rust/src/lib.rs +++ b/Libraries/LibWeb/Rust/src/lib.rs @@ -8,6 +8,7 @@ mod rust_allocator; mod css_tokenizer; +mod encoding_detection; pub use libweb_html_tokenizer as html_tokenizer; diff --git a/Meta/CMake/flatpak/cargo-sources.json b/Meta/CMake/flatpak/cargo-sources.json index 489bb9d8f1..018b011cfa 100644 --- a/Meta/CMake/flatpak/cargo-sources.json +++ b/Meta/CMake/flatpak/cargo-sources.json @@ -110,6 +110,13 @@ "sha256": "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801", "dest": "cargo/vendor/cfg-if-1.0.4" }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/chardetng/chardetng-1.0.0.crate", + "sha256": "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53", + "dest": "cargo/vendor/chardetng-1.0.0" + }, { "type": "archive", "archive-type": "tar-gzip", @@ -229,6 +236,13 @@ "sha256": "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0", "dest": "cargo/vendor/displaydoc-0.2.5" }, + { + "type": "archive", + "archive-type": "tar-gzip", + "url": "https://static.crates.io/crates/encoding_rs/encoding_rs-0.8.35.crate", + "sha256": "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3", + "dest": "cargo/vendor/encoding_rs-0.8.35" + }, { "type": "archive", "archive-type": "tar-gzip", @@ -898,6 +912,7 @@ "echo '{\"files\": {}, \"package\": \"5abbd6eeda6885048d357edc66748eea6e0268e3dd11f326fff5bd248d779c26\"}' > cargo/vendor/calendrical_calculations-0.2.4/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799\"}' > cargo/vendor/cbindgen-0.29.2/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801\"}' > cargo/vendor/cfg-if-1.0.4/.cargo-checksum.json", + "echo '{\"files\": {}, \"package\": \"13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53\"}' > cargo/vendor/chardetng-1.0.0/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351\"}' > cargo/vendor/clap-4.6.0/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f\"}' > cargo/vendor/clap_builder-4.6.0/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9\"}' > cargo/vendor/clap_lex-1.1.0/.cargo-checksum.json", @@ -915,6 +930,7 @@ "echo '{\"files\": {}, \"package\": \"4d55612bebcf16ff7306c8a6f5bdb6d45662b8aa1ee058ecce8807ad87db719b\"}' > cargo/vendor/cranelift-module-0.116.1/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"b8dee82f3f1f2c4cba9177f1cc5e350fe98764379bcd29340caa7b01f85076c7\"}' > cargo/vendor/cranelift-native-0.116.1/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0\"}' > cargo/vendor/displaydoc-0.2.5/.cargo-checksum.json", + "echo '{\"files\": {}, \"package\": \"75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3\"}' > cargo/vendor/encoding_rs-0.8.35/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f\"}' > cargo/vendor/equivalent-1.0.2/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb\"}' > cargo/vendor/errno-0.3.14/.cargo-checksum.json", "echo '{\"files\": {}, \"package\": \"2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649\"}' > cargo/vendor/fallible-iterator-0.3.0/.cargo-checksum.json", diff --git a/Tests/LibWeb/Text/expected/css/style-invalidation/has-concrete-structural-triggers.txt b/Tests/LibWeb/Text/expected/css/style-invalidation/has-concrete-structural-triggers.txt index c74fc17d15..b9bac5babd 100644 --- a/Tests/LibWeb/Text/expected/css/style-invalidation/has-concrete-structural-triggers.txt +++ b/Tests/LibWeb/Text/expected/css/style-invalidation/has-concrete-structural-triggers.txt @@ -1,4 +1,4 @@ -PASS: inserted class argument invalidates descendant fanout | styleInvalidations=2, fullStyleInvalidations=0, elementStyleRecomputations=13, elementStyleNoopRecomputations=5, elementInheritedStyleRecomputations=0, elementInheritedStyleNoopRecomputations=0, hasAncestorWalkInvocations=1, hasInvalidationMetadataCandidates=1, hasMatchInvocations=2, hasResultCacheHits=0, hasResultCacheMisses=2 +PASS: inserted class argument invalidates descendant fanout | styleInvalidations=2, fullStyleInvalidations=0, elementStyleRecomputations=14, elementStyleNoopRecomputations=5, elementInheritedStyleRecomputations=0, elementInheritedStyleNoopRecomputations=0, hasAncestorWalkInvocations=1, hasInvalidationMetadataCandidates=1, hasMatchInvocations=2, hasResultCacheHits=0, hasResultCacheMisses=2 PASS: removed class argument invalidates descendant fanout | styleInvalidations=2, fullStyleInvalidations=0, elementStyleRecomputations=3, elementStyleNoopRecomputations=2, elementInheritedStyleRecomputations=0, elementInheritedStyleNoopRecomputations=0, hasAncestorWalkInvocations=1, hasInvalidationMetadataCandidates=0, hasMatchInvocations=2, hasResultCacheHits=0, hasResultCacheMisses=2 PASS: inserted id argument invalidates descendant fanout | styleInvalidations=3, fullStyleInvalidations=0, elementStyleRecomputations=4, elementStyleNoopRecomputations=2, elementInheritedStyleRecomputations=0, elementInheritedStyleNoopRecomputations=0, hasAncestorWalkInvocations=1, hasInvalidationMetadataCandidates=1, hasMatchInvocations=2, hasResultCacheHits=0, hasResultCacheMisses=2 PASS: removed id argument invalidates descendant fanout | styleInvalidations=2, fullStyleInvalidations=0, elementStyleRecomputations=3, elementStyleNoopRecomputations=2, elementInheritedStyleRecomputations=0, elementInheritedStyleNoopRecomputations=0, hasAncestorWalkInvocations=1, hasInvalidationMetadataCandidates=0, hasMatchInvocations=2, hasResultCacheHits=0, hasResultCacheMisses=2 diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ar-ISO-8859-6.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ar-ISO-8859-6.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ar-ISO-8859-6.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ar-windows-1256.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ar-windows-1256.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ar-windows-1256.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ca-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ca-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ca-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/el-ISO-8859-7.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/el-ISO-8859-7.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/el-ISO-8859-7.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/el-windows-1253.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/el-windows-1253.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/el-windows-1253.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-copyright-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-copyright-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-copyright-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-euro-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-euro-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-euro-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/en-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-maria-windows-1252.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-maria-windows-1252.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-maria-windows-1252.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-nuestra-senora-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-nuestra-senora-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-nuestra-senora-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-number-one-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-number-one-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-number-one-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-ordinal-windows-1252.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-ordinal-windows-1252.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/es-ordinal-windows-1252.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/fa-windows-1256.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/fa-windows-1256.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/fa-windows-1256.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/fi-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/fi-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/fi-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/he-ISO-8859-8.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/he-ISO-8859-8.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/he-ISO-8859-8.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/he-windows-1255.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/he-windows-1255.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/he-windows-1255.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/is-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/is-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/is-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-number-one-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-number-one-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-number-one-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-ordinal-windows-1252.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-ordinal-windows-1252.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-ordinal-windows-1252.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-roman-ordinal-windows-1252.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-roman-ordinal-windows-1252.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-roman-ordinal-windows-1252.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/it-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-EUC-JP.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-EUC-JP.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-EUC-JP.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-ISO-2022-JP.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-ISO-2022-JP.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-ISO-2022-JP.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-Shift_JIS.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-Shift_JIS.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-Shift_JIS.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-half-width.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-half-width.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ja-half-width.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ko-EUC-KR.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ko-EUC-KR.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ko-EUC-KR.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/lt-windows-1257.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/lt-windows-1257.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/lt-windows-1257.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/lv-windows-1257.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/lv-windows-1257.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/lv-windows-1257.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/nbsp-even-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/nbsp-even-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/nbsp-even-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/nbsp-odd-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/nbsp-odd-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/nbsp-odd-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pl-ISO-8859-2.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pl-ISO-8859-2.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pl-ISO-8859-2.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pl-windows-1250.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pl-windows-1250.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pl-windows-1250.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pt-windows-1252.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pt-windows-1252.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/pt-windows-1252.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ru-IBM866.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ru-IBM866.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ru-IBM866.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ru-ISO-8859-5.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ru-ISO-8859-5.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/ru-ISO-8859-5.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/th-windows-874.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/th-windows-874.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/th-windows-874.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/tr-windows-1254.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/tr-windows-1254.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/tr-windows-1254.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/uk-KOI8-U.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/uk-KOI8-U.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/uk-KOI8-U.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/uk-windows-1251.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/uk-windows-1251.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/uk-windows-1251.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/utf-8.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/utf-8.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/utf-8.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/vi-windows-1258.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/vi-windows-1258.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/vi-windows-1258.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/yi-windows-1255.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/yi-windows-1255.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/yi-windows-1255.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/zh-Big5.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/zh-Big5.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/zh-Big5.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/zh-GBK.tentative.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/zh-GBK.tentative.txt new file mode 100644 index 0000000000..f5f82e354c --- /dev/null +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding-detection/zh-GBK.tentative.txt @@ -0,0 +1,6 @@ +Harness status: OK + +Found 1 tests + +1 Pass +Pass Check detection result \ No newline at end of file diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ar-ISO-8859-6.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ar-ISO-8859-6.tentative.html new file mode 100644 index 0000000000..30f1013f0b --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ar-ISO-8859-6.tentative.html @@ -0,0 +1,14 @@ + +ar ISO-8859-6 + + +

.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ar-windows-1256.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ar-windows-1256.tentative.html new file mode 100644 index 0000000000..33cfd511ac --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ar-windows-1256.tentative.html @@ -0,0 +1,14 @@ + +ar windows-1256 + + +

.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ca-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ca-windows-1252.tentative.html new file mode 100644 index 0000000000..ab3cb1db1f --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ca-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +ca windows-1252 + + +

Es tracta duna prova de codificaci de carcters.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/el-ISO-8859-7.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/el-ISO-8859-7.tentative.html new file mode 100644 index 0000000000..4a59ac8c0e --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/el-ISO-8859-7.tentative.html @@ -0,0 +1,15 @@ + +el ISO-8859-7 + + +

: +

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/el-windows-1253.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/el-windows-1253.tentative.html new file mode 100644 index 0000000000..635f35be68 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/el-windows-1253.tentative.html @@ -0,0 +1,15 @@ + +el windows-1253 + + +

: +

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-copyright-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-copyright-windows-1252.tentative.html new file mode 100644 index 0000000000..1682970bd7 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-copyright-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +en windows-1252 copyright sign + + +

Copyright 2021

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-euro-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-euro-windows-1252.tentative.html new file mode 100644 index 0000000000..8eccd28867 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-euro-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +en windows-1252 euro sign + + +

It costs 9.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-windows-1252.tentative.html new file mode 100644 index 0000000000..e05f587561 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/en-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +en windows-1252 + + +

Ive made an encoding test.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-maria-windows-1252.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-maria-windows-1252.html new file mode 100644 index 0000000000..6c1534b860 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-maria-windows-1252.html @@ -0,0 +1,14 @@ + +es windows-1252 + + +

M

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-nuestra-senora-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-nuestra-senora-windows-1252.tentative.html new file mode 100644 index 0000000000..3ec9b5a21d --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-nuestra-senora-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +es windows-1252 + + +

N S

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-number-one-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-number-one-windows-1252.tentative.html new file mode 100644 index 0000000000..1d1b747fa2 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-number-one-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +es windows-1252 + + +

n.1

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-ordinal-windows-1252.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-ordinal-windows-1252.html new file mode 100644 index 0000000000..e8f5cfaf3a --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/es-ordinal-windows-1252.html @@ -0,0 +1,14 @@ + +es windows-1252 + + +

42.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/fa-windows-1256.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/fa-windows-1256.tentative.html new file mode 100644 index 0000000000..7d433c68dd --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/fa-windows-1256.tentative.html @@ -0,0 +1,14 @@ + +fa windows-1256 + + +

ی ی Ґی ǘ .

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/fi-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/fi-windows-1252.tentative.html new file mode 100644 index 0000000000..5b3e7a6f58 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/fi-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +fi windows-1252 + + +

Tm on merkkikoodaustesti.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/he-ISO-8859-8.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/he-ISO-8859-8.tentative.html new file mode 100644 index 0000000000..b537150a8a --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/he-ISO-8859-8.tentative.html @@ -0,0 +1,14 @@ + +he ISO-8859-8 + + +

.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/he-windows-1255.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/he-windows-1255.tentative.html new file mode 100644 index 0000000000..d771823f31 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/he-windows-1255.tentative.html @@ -0,0 +1,14 @@ + +he windows-1255 + + +

.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/is-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/is-windows-1252.tentative.html new file mode 100644 index 0000000000..55227dd2e6 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/is-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +is windows-1252 + + +

etta er kunarprf staf. Fyrir sum tunguml sem nota latneska stafi urfum vi meira inntak til a taka kvrunina.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-number-one-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-number-one-windows-1252.tentative.html new file mode 100644 index 0000000000..7e73a5c1b4 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-number-one-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +fi windows-1252 + + +

N1

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-ordinal-windows-1252.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-ordinal-windows-1252.html new file mode 100644 index 0000000000..7dd080a670 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-ordinal-windows-1252.html @@ -0,0 +1,14 @@ + +it windows-1252 + + +

42

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-roman-ordinal-windows-1252.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-roman-ordinal-windows-1252.html new file mode 100644 index 0000000000..61049da7ab --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-roman-ordinal-windows-1252.html @@ -0,0 +1,14 @@ + +it windows-1252 + + +

III

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-windows-1252.tentative.html new file mode 100644 index 0000000000..275c659436 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/it-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +it windows-1252 + + +

Questo un test di codifica dei caratteri.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-EUC-JP.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-EUC-JP.tentative.html new file mode 100644 index 0000000000..e8f9e32cf6 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-EUC-JP.tentative.html @@ -0,0 +1,14 @@ + +ja EUC-JP + + +

ʸ¸Ǥ

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-ISO-2022-JP.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-ISO-2022-JP.tentative.html new file mode 100644 index 0000000000..c562655c90 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-ISO-2022-JP.tentative.html @@ -0,0 +1,14 @@ + +ja ISO-2022-JP + + +

$B$3$l$OJ8;z + diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-Shift_JIS.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-Shift_JIS.tentative.html new file mode 100644 index 0000000000..e3169a6244 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-Shift_JIS.tentative.html @@ -0,0 +1,14 @@ + +ja Shift_JIS + + +

͕łB

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-half-width.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-half-width.tentative.html new file mode 100644 index 0000000000..0e4dd85f2e --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ja-half-width.tentative.html @@ -0,0 +1,14 @@ + +ja Shift_JIS half-width katakana + + +

ʰ޳ʰ޳ʰ޳ʰ޳ʰ޳

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ko-EUC-KR.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ko-EUC-KR.tentative.html new file mode 100644 index 0000000000..545ffe1d77 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ko-EUC-KR.tentative.html @@ -0,0 +1,14 @@ + +ko EUC-KR + + +

̰ ڵ ׽ƮԴϴ.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/lt-windows-1257.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/lt-windows-1257.tentative.html new file mode 100644 index 0000000000..0bac1db737 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/lt-windows-1257.tentative.html @@ -0,0 +1,14 @@ + +lt windows-1257 + + +

Tai simboli kodavimo testas. Kai kurioms kalboms, naudojanioms lotynikus ramenis, mums reikia daugiau informacijos, kad galtume priimti sprendim.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/lv-windows-1257.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/lv-windows-1257.tentative.html new file mode 100644 index 0000000000..3c34a4a6a4 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/lv-windows-1257.tentative.html @@ -0,0 +1,14 @@ + +lv windows-1257 + + +

is ir rakstzmju kodanas tests. Das valods, kurs tiek izmantotas latu valodas burti, lmuma pieemanai mums ir nepiecieams vairk ieguldjuma.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/nbsp-even-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/nbsp-even-windows-1252.tentative.html new file mode 100644 index 0000000000..745581a424 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/nbsp-even-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +nbsp even windows-1252 + + +

No-break spaces

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/nbsp-odd-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/nbsp-odd-windows-1252.tentative.html new file mode 100644 index 0000000000..504f9f1371 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/nbsp-odd-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +nbsp odd windows-1252 + + +

No-break spaces

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/pl-ISO-8859-2.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/pl-ISO-8859-2.tentative.html new file mode 100644 index 0000000000..34857226a6 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/pl-ISO-8859-2.tentative.html @@ -0,0 +1,14 @@ + +pl ISO-8859-2 + + +

To jest test kodowania znakw. W przypadku niektrych jzykw, ktre uywaj znakw aciskich, potrzebujemy wicej danych, aby podj decyzj.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/pl-windows-1250.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/pl-windows-1250.tentative.html new file mode 100644 index 0000000000..b32aeaeca3 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/pl-windows-1250.tentative.html @@ -0,0 +1,14 @@ + +pl windows-1250 + + +

To jest test kodowania znakw. W przypadku niektrych jzykw, ktre uywaj znakw aciskich, potrzebujemy wicej danych, aby podj decyzj.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/pt-windows-1252.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/pt-windows-1252.tentative.html new file mode 100644 index 0000000000..0b564ab05d --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/pt-windows-1252.tentative.html @@ -0,0 +1,14 @@ + +pt windows-1252 + + +

Este um teste de codificao de caracteres.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ru-IBM866.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ru-IBM866.tentative.html new file mode 100644 index 0000000000..17a957cf90 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ru-IBM866.tentative.html @@ -0,0 +1,14 @@ + +ru IBM866 + + +

஢ ᨬ.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ru-ISO-8859-5.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ru-ISO-8859-5.tentative.html new file mode 100644 index 0000000000..869e9d4385 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/ru-ISO-8859-5.tentative.html @@ -0,0 +1,14 @@ + +ru ISO-8859-5 + + +

.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/th-windows-874.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/th-windows-874.tentative.html new file mode 100644 index 0000000000..f355502457 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/th-windows-874.tentative.html @@ -0,0 +1,14 @@ + +th windows-874 + + +

͡÷ͺѡ

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/tr-windows-1254.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/tr-windows-1254.tentative.html new file mode 100644 index 0000000000..5aaf138b76 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/tr-windows-1254.tentative.html @@ -0,0 +1,14 @@ + +tr windows-1254 + + +

Bu bir karakter kodlama testidir. Latince karakterleri kullanan baz dillerde karar vermek iin daha fazla girdiye ihtiyacmz var.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/uk-KOI8-U.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/uk-KOI8-U.tentative.html new file mode 100644 index 0000000000..c743c435b6 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/uk-KOI8-U.tentative.html @@ -0,0 +1,14 @@ + +uk KOI8-U + + +

̦.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/uk-windows-1251.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/uk-windows-1251.tentative.html new file mode 100644 index 0000000000..0de1e0149d --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/uk-windows-1251.tentative.html @@ -0,0 +1,14 @@ + +uk windows-1251 + + +

.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/utf-8.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/utf-8.html new file mode 100644 index 0000000000..3415ab45e0 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/utf-8.html @@ -0,0 +1,14 @@ + +UTF-8 + + +

これは文字実験です。

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/vi-windows-1258.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/vi-windows-1258.tentative.html new file mode 100644 index 0000000000..cb38b5c7e4 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/vi-windows-1258.tentative.html @@ -0,0 +1,14 @@ + +vi windows-1258 + + +

y la mt th nghim ma hoa ky t.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/yi-windows-1255.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/yi-windows-1255.tentative.html new file mode 100644 index 0000000000..07a39924f2 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/yi-windows-1255.tentative.html @@ -0,0 +1,14 @@ + +yi windows-1255 + + +

.

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/zh-Big5.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/zh-Big5.tentative.html new file mode 100644 index 0000000000..f28ca5502c --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/zh-Big5.tentative.html @@ -0,0 +1,14 @@ + +zh Big5 + + +

oO@ӦrŽsXաC

+ diff --git a/Tests/LibWeb/Text/input/wpt-import/encoding-detection/zh-GBK.tentative.html b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/zh-GBK.tentative.html new file mode 100644 index 0000000000..c233a2566d --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/encoding-detection/zh-GBK.tentative.html @@ -0,0 +1,14 @@ + +zh GBK + + +

һַԡ

+