AK: Serialize URL hosts with 'concept-host-serializer'

In order to follow spec text to achieve this, we need to change the
underlying representation of a host in AK::URL to deserialized format.
Before this, we were parsing the host and then immediately serializing
it again.

Making that change resulted in a whole bunch of fallout.

After this change, callers can access the serialized data through
this concept-host-serializer. The functional end result of this
change is that IPv6 hosts are now correctly serialized to be
surrounded with '[' and ']'.

Userland/Libraries/LibWeb/DOM/Document.cpp

@@ -2316,8 +2316,7 @@ DeprecatedString Document::domain() const
         return DeprecatedString::empty();
 
     // 3. Return effectiveDomain, serialized.
-    // FIXME: Implement host serialization.
-    return effective_domain.release_value();
+    return URLParser::serialize_host(effective_domain.release_value()).release_value_but_fixme_should_propagate_errors().to_deprecated_string();
 }
 
 void Document::set_domain(DeprecatedString const& domain)

Userland/Libraries/LibWeb/Fetch/Fetching/Fetching.cpp

@@ -256,7 +256,7 @@ WebIDL::ExceptionOr<Optional<JS::NonnullGCPtr<PendingResponse>>> main_fetch(JS::
         // - request’s current URL’s scheme is "http"
         request->current_url().scheme() == "http"sv
         // - request’s current URL’s host is a domain
-        && URL::host_is_domain(request->current_url().host())
+        && URL::host_is_domain(request->current_url().serialized_host().release_value_but_fixme_should_propagate_errors())
         // FIXME: - Matching request’s current URL’s host per Known HSTS Host Domain Name Matching results in either a
         //          superdomain match with an asserted includeSubDomains directive or a congruent match (with or without an
         //          asserted includeSubDomains directive) [HSTS]; or DNS resolution for the request finds a matching HTTPS RR

Userland/Libraries/LibWeb/HTML/BrowsingContext.cpp

@@ -44,7 +44,7 @@ static bool url_matches_about_blank(AK::URL const& url)
         && url.serialize_path() == "blank"sv
         && url.username().is_empty()
         && url.password().is_empty()
-        && url.host().is_null();
+        && url.host().has<Empty>();
 }
 
 // FIXME: This is an outdated older version of "determining the origin" and should be removed.

Userland/Libraries/LibWeb/HTML/HTMLHyperlinkElementUtils.cpp

@@ -167,15 +167,15 @@ DeprecatedString HTMLHyperlinkElementUtils::host() const
     auto& url = m_url;
 
     // 3. If url or url's host is null, return the empty string.
-    if (!url.has_value() || url->host().is_null())
+    if (!url.has_value() || url->host().has<Empty>())
         return DeprecatedString::empty();
 
     // 4. If url's port is null, return url's host, serialized.
     if (!url->port().has_value())
-        return url->host();
+        return url->serialized_host().release_value_but_fixme_should_propagate_errors().to_deprecated_string();
 
     // 5. Return url's host, serialized, followed by ":" and url's port, serialized.
-    return DeprecatedString::formatted("{}:{}", url->host(), url->port().value());
+    return DeprecatedString::formatted("{}:{}", url->serialized_host().release_value_but_fixme_should_propagate_errors(), url->port().value());
 }
 
 // https://html.spec.whatwg.org/multipage/links.html#dom-hyperlink-host
@@ -205,11 +205,14 @@ DeprecatedString HTMLHyperlinkElementUtils::hostname() const
     // 1. Reinitialize url.
     //
     // 2. Let url be this element's url.
-    //
+    AK::URL url(href());
+
     // 3. If url or url's host is null, return the empty string.
-    //
+    if (url.host().has<Empty>())
+        return DeprecatedString::empty();
+
     // 4. Return url's host, serialized.
-    return AK::URL(href()).host();
+    return url.serialized_host().release_value_but_fixme_should_propagate_errors().to_deprecated_string();
 }
 
 void HTMLHyperlinkElementUtils::set_hostname(DeprecatedString hostname)

Userland/Libraries/LibWeb/HTML/Location.cpp

@@ -153,15 +153,15 @@ WebIDL::ExceptionOr<String> Location::host() const
     auto url = this->url();
 
     // 3. If url's host is null, return the empty string.
-    if (url.host().is_null())
+    if (url.host().has<Empty>())
         return String {};
 
     // 4. If url's port is null, return url's host, serialized.
     if (!url.port().has_value())
-        return TRY_OR_THROW_OOM(vm, String::from_deprecated_string(url.host()));
+        return TRY_OR_THROW_OOM(vm, url.serialized_host());
 
     // 5. Return url's host, serialized, followed by ":" and url's port, serialized.
-    return TRY_OR_THROW_OOM(vm, String::formatted("{}:{}", url.host(), *url.port()));
+    return TRY_OR_THROW_OOM(vm, String::formatted("{}:{}", TRY_OR_THROW_OOM(vm, url.serialized_host()), *url.port()));
 }
 
 WebIDL::ExceptionOr<void> Location::set_host(String const&)
@@ -183,11 +183,11 @@ WebIDL::ExceptionOr<String> Location::hostname() const
     auto url = this->url();
 
     // 2. If this's url's host is null, return the empty string.
-    if (url.host().is_null())
+    if (url.host().has<Empty>())
         return String {};
 
     // 3. Return this's url's host, serialized.
-    return TRY_OR_THROW_OOM(vm, String::from_deprecated_string(url.host()));
+    return TRY_OR_THROW_OOM(vm, url.serialized_host());
 }
 
 WebIDL::ExceptionOr<void> Location::set_hostname(String const&)

Userland/Libraries/LibWeb/HTML/NavigableContainer.cpp

@@ -217,7 +217,7 @@ static bool url_matches_about_blank(AK::URL const& url)
         && url.serialize_path() == "blank"sv
         && url.username().is_empty()
         && url.password().is_empty()
-        && url.host().is_null();
+        && url.host().has<Empty>();
 }
 
 // https://html.spec.whatwg.org/multipage/iframe-embed-object.html#shared-attribute-processing-steps-for-iframe-and-frame-elements

Userland/Libraries/LibWeb/HTML/Origin.h

@@ -8,13 +8,15 @@
 #pragma once
 
 #include <AK/DeprecatedString.h>
+#include <AK/URL.h>
+#include <AK/URLParser.h>
 
 namespace Web::HTML {
 
 class Origin {
 public:
     Origin() = default;
-    Origin(DeprecatedString const& scheme, DeprecatedString const& host, u16 port)
+    Origin(DeprecatedString const& scheme, AK::URL::Host const& host, u16 port)
         : m_scheme(scheme)
         , m_host(host)
         , m_port(port)
@@ -22,10 +24,10 @@ public:
     }
 
     // https://html.spec.whatwg.org/multipage/origin.html#concept-origin-opaque
-    bool is_opaque() const { return m_scheme.is_null() && m_host.is_null() && m_port == 0; }
+    bool is_opaque() const { return m_scheme.is_null() && m_host.has<Empty>() && m_port == 0; }
 
     DeprecatedString const& scheme() const { return m_scheme; }
-    DeprecatedString const& host() const { return m_host; }
+    AK::URL::Host const& host() const { return m_host; }
     u16 port() const { return m_port; }
 
     // https://html.spec.whatwg.org/multipage/origin.html#same-origin
@@ -81,7 +83,7 @@ public:
         result.append("://"sv);
 
         // 4. Append origin's host, serialized, to result.
-        result.append(host());
+        result.append(URLParser::serialize_host(host()).release_value_but_fixme_should_propagate_errors().to_deprecated_string());
 
         // 5. If origin's port is non-null, append a U+003A COLON character (:), and origin's port, serialized, to result.
         if (port() != 0) {
@@ -93,11 +95,11 @@ public:
     }
 
     // https://html.spec.whatwg.org/multipage/origin.html#concept-origin-effective-domain
-    Optional<DeprecatedString> effective_domain() const
+    Optional<AK::URL::Host> effective_domain() const
     {
         // 1. If origin is an opaque origin, then return null.
         if (is_opaque())
-            return Optional<DeprecatedString> {};
+            return {};
 
         // FIXME: 2. If origin's domain is non-null, then return origin's domain.
 
@@ -109,7 +111,7 @@ public:
 
 private:
     DeprecatedString m_scheme;
-    DeprecatedString m_host;
+    AK::URL::Host m_host;
     u16 m_port { 0 };
 };
 
@@ -120,7 +122,10 @@ template<>
 struct Traits<Web::HTML::Origin> : public GenericTraits<Web::HTML::Origin> {
     static unsigned hash(Web::HTML::Origin const& origin)
     {
-        return pair_int_hash(origin.scheme().hash(), pair_int_hash(int_hash(origin.port()), origin.host().hash()));
+        auto hash_without_host = pair_int_hash(origin.scheme().hash(), origin.port());
+        if (origin.host().has<Empty>())
+            return hash_without_host;
+        return pair_int_hash(hash_without_host, URLParser::serialize_host(origin.host()).release_value_but_fixme_should_propagate_errors().hash());
     }
 };
 } // namespace AK

Userland/Libraries/LibWeb/HTML/WorkerLocation.cpp

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
+#include <AK/URLParser.h>
 #include <LibWeb/HTML/WorkerGlobalScope.h>
 #include <LibWeb/HTML/WorkerLocation.h>
 
@@ -43,15 +44,15 @@ WebIDL::ExceptionOr<String> WorkerLocation::host() const
     auto const& url = m_global_scope->url();
 
     // 2. If url's host is null, return the empty string.
-    if (url.host().is_empty())
+    if (url.host().has<Empty>())
         return String {};
 
     // 3. If url's port is null, return url's host, serialized.
     if (!url.port().has_value())
-        return TRY_OR_THROW_OOM(vm, String::from_deprecated_string(url.host()));
+        return TRY_OR_THROW_OOM(vm, url.serialized_host());
 
     // 4. Return url's host, serialized, followed by ":" and url's port, serialized.
-    return TRY_OR_THROW_OOM(vm, String::formatted("{}:{}", url.host().view(), url.port().value()));
+    return TRY_OR_THROW_OOM(vm, String::formatted("{}:{}", TRY_OR_THROW_OOM(vm, url.serialized_host()), url.port().value()));
 }
 
 // https://html.spec.whatwg.org/multipage/workers.html#dom-workerlocation-hostname
@@ -64,11 +65,11 @@ WebIDL::ExceptionOr<String> WorkerLocation::hostname() const
     auto const& host = m_global_scope->url().host();
 
     // 2. If host is null, return the empty string.
-    if (host.is_empty())
+    if (host.has<Empty>())
         return String {};
 
     // 3. Return host, serialized.
-    return TRY_OR_THROW_OOM(vm, String::from_deprecated_string(host));
+    return TRY_OR_THROW_OOM(vm, URLParser::serialize_host(host));
 }
 
 // https://html.spec.whatwg.org/multipage/workers.html#dom-workerlocation-port

Userland/Libraries/LibWeb/SecureContexts/AbstractOperations.cpp

@@ -28,20 +28,29 @@ Trustworthiness is_origin_potentially_trustworthy(HTML::Origin const& origin)
         return Trustworthiness::PotentiallyTrustworthy;
 
     // 4. If origin’s host matches one of the CIDR notations 127.0.0.0/8 or ::1/128 [RFC4632], return "Potentially Trustworthy".
-    if (auto ipv4_address = IPv4Address::from_string(origin.host()); ipv4_address.has_value() && (ipv4_address->to_u32() & 0xff000000) != 0)
-        return Trustworthiness::PotentiallyTrustworthy;
-    if (auto ipv6_address = IPv6Address::from_string(origin.host()); ipv6_address.has_value() && ipv6_address == IPv6Address::loopback())
-        return Trustworthiness::PotentiallyTrustworthy;
+    // FIXME: This would be nicer if URL::IPv4Address and URL::IPv6Address were instances of AK::IPv4Address and AK::IPv6Address
+    if (origin.host().has<AK::URL::IPv4Address>()) {
+        if ((origin.host().get<AK::URL::IPv4Address>() & 0xff000000) != 0)
+            return Trustworthiness::PotentiallyTrustworthy;
+    } else if (origin.host().has<AK::URL::IPv6Address>()) {
+        auto ipv6_address = origin.host().get<AK::URL::IPv6Address>();
+        static constexpr AK::URL::IPv6Address loopback { 0, 0, 0, 0, 0, 0, 0, 1 };
+        if (ipv6_address == loopback)
+            return Trustworthiness::PotentiallyTrustworthy;
+    }
 
     // 5. If the user agent conforms to the name resolution rules in [let-localhost-be-localhost] and one of the following is true:
     // - origin’s host is "localhost" or "localhost."
     // - origin’s host ends with ".localhost" or ".localhost."
     // then return "Potentially Trustworthy".
     // Note: See § 5.2 localhost for details on the requirements here.
-    if (origin.host().is_one_of("localhost"sv, "localhost.")
-        || origin.host().ends_with(".localhost"sv)
-        || origin.host().ends_with(".localhost."sv)) {
-        return Trustworthiness::PotentiallyTrustworthy;
+    if (origin.host().has<String>()) {
+        auto const& host = origin.host().get<String>();
+        if (host.is_one_of("localhost"sv, "localhost.")
+            || host.ends_with_bytes(".localhost"sv)
+            || host.ends_with_bytes(".localhost."sv)) {
+            return Trustworthiness::PotentiallyTrustworthy;
+        }
     }
 
     // 6. If origin’s scheme is "file", return "Potentially Trustworthy".

Userland/Libraries/LibWeb/URL/URL.cpp

@@ -235,15 +235,15 @@ WebIDL::ExceptionOr<String> URL::host() const
     auto& url = m_url;
 
     // 2. If url’s host is null, then return the empty string.
-    if (url.host().is_null())
+    if (url.host().has<Empty>())
         return String {};
 
     // 3. If url’s port is null, return url’s host, serialized.
     if (!url.port().has_value())
-        return TRY_OR_THROW_OOM(vm, String::from_deprecated_string(url.host()));
+        return TRY_OR_THROW_OOM(vm, url.serialized_host());
 
     // 4. Return url’s host, serialized, followed by U+003A (:) and url’s port, serialized.
-    return TRY_OR_THROW_OOM(vm, String::formatted("{}:{}", url.host(), *url.port()));
+    return TRY_OR_THROW_OOM(vm, String::formatted("{}:{}", TRY_OR_THROW_OOM(vm, url.serialized_host()), *url.port()));
 }
 
 // https://url.spec.whatwg.org/#dom-url-hostref-for-dom-url-host%E2%91%A0
@@ -265,11 +265,11 @@ WebIDL::ExceptionOr<String> URL::hostname() const
     auto& vm = realm().vm();
 
     // 1. If this’s URL’s host is null, then return the empty string.
-    if (m_url.host().is_null())
+    if (m_url.host().has<Empty>())
         return String {};
 
     // 2. Return this’s URL’s host, serialized.
-    return TRY_OR_THROW_OOM(vm, String::from_deprecated_string(m_url.host()));
+    return TRY_OR_THROW_OOM(vm, m_url.serialized_host());
 }
 
 // https://url.spec.whatwg.org/#ref-for-dom-url-hostname①
@@ -473,7 +473,7 @@ HTML::Origin url_origin(AK::URL const& url)
     if (url.scheme() == "file"sv) {
         // Unfortunate as it is, this is left as an exercise to the reader. When in doubt, return a new opaque origin.
         // Note: We must return an origin with the `file://' protocol for `file://' iframes to work from `file://' pages.
-        return HTML::Origin(url.scheme(), DeprecatedString(), 0);
+        return HTML::Origin(url.scheme(), String {}, 0);
     }
 
     // -> Otherwise

Userland/Libraries/LibWeb/XHR/XMLHttpRequest.cpp

@@ -388,7 +388,7 @@ WebIDL::ExceptionOr<void> XMLHttpRequest::open(String const& method_string, Stri
     // NOTE: This is handled in the overload lacking the async argument.
 
     // 8. If parsedURL’s host is non-null, then:
-    if (!parsed_url.host().is_null()) {
+    if (!parsed_url.host().has<Empty>()) {
         // 1. If the username argument is not null, set the username given parsedURL and username.
         if (username.has_value())
             parsed_url.set_username(username.value().to_deprecated_string());