Ladybird: Add RequestServer process to optionally replace Qt Networking

LibTLS still can't access many parts of the web, so let's hide this
behind a flag (with all the plumbing that entails).

Hopefully this can encourage folks to improve LibTLS's algorithm support
:^).

Meta/Lagom/CMakeLists.txt

@@ -407,6 +407,7 @@ if (BUILD_LAGOM)
         Manual
         Markdown
         PDF
+        Protocol
         Regex
         SoftGPU
         SQL
@@ -430,21 +431,26 @@ if (BUILD_LAGOM)
         list(APPEND LIBWEBVIEW_SOURCES "../../Userland/Libraries/LibWebView/StylePropertiesModel.cpp")
         list(APPEND LIBWEBVIEW_SOURCES "../../Userland/Libraries/LibWebView/ViewImplementation.cpp")
         list(APPEND LIBWEBVIEW_SOURCES "../../Userland/Libraries/LibWebView/WebContentClient.cpp")
+        list(APPEND LIBWEBVIEW_SOURCES "../../Userland/Libraries/LibWebView/RequestServerAdapter.cpp")
 
         compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/WebContent/WebContentServer.ipc WebContent/WebContentServerEndpoint.h)
         compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/WebContent/WebContentClient.ipc WebContent/WebContentClientEndpoint.h)
         compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/WebContent/WebDriverClient.ipc WebContent/WebDriverClientEndpoint.h)
         compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/WebContent/WebDriverServer.ipc WebContent/WebDriverServerEndpoint.h)
+        compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/RequestServer/RequestClient.ipc RequestServer/RequestClientEndpoint.h)
+        compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/RequestServer/RequestServer.ipc RequestServer/RequestServerEndpoint.h)
 
         list(APPEND LIBWEBVIEW_GENERATED_SOURCES WebContent/WebContentClientEndpoint.h)
         list(APPEND LIBWEBVIEW_GENERATED_SOURCES WebContent/WebContentServerEndpoint.h)
         list(APPEND LIBWEBVIEW_GENERATED_SOURCES WebContent/WebDriverClientEndpoint.h)
         list(APPEND LIBWEBVIEW_GENERATED_SOURCES WebContent/WebDriverServerEndpoint.h)
+        list(APPEND LIBWEBVIEW_GENERATED_SOURCES RequestServer/RequestClientEndpoint.h)
+        list(APPEND LIBWEBVIEW_GENERATED_SOURCES RequestServer/RequestServerEndpoint.h)
 
         set(GENERATED_SOURCES ${LIBWEBVIEW_GENERATED_SOURCES})
         lagom_lib(LibWebView webview
             SOURCES ${LIBWEBVIEW_SOURCES} ${LIBWEBVIEW_GENERATED_SOURCES}
-            LIBS LibGfx LibGUI LibIPC LibWeb)
+            LIBS LibGfx LibGUI LibIPC LibWeb LibProtocol)
         unset(GENERATED_SOURCES)
     endif()
 

Meta/gn/secondary/Ladybird/BUILD.gn

@@ -40,7 +40,7 @@ link_qt("ladybird_qt_components") {
 config("ladybird_config") {
   include_dirs = [
     "//Userland/Applications",
-    "//Userland",
+    "//Userland/Services",
   ]
   defines = [ "AK_DONT_REPLACE_STD" ]
 }
@@ -52,6 +52,7 @@ executable("ladybird_executable") {
   ]
   data_deps = [
     ":headless-browser",
+    "RequestServer",
     "SQLServer",
     "WebContent",
     "WebDriver",
@@ -67,6 +68,7 @@ executable("ladybird_executable") {
     "//Userland/Libraries/LibIPC",
     "//Userland/Libraries/LibJS",
     "//Userland/Libraries/LibMain",
+    "//Userland/Libraries/LibProtocol",
     "//Userland/Libraries/LibSQL",
     "//Userland/Libraries/LibWeb",
     "//Userland/Libraries/LibWebView",
@@ -119,6 +121,7 @@ executable("headless-browser") {
     "//Userland/Libraries/LibIPC",
     "//Userland/Libraries/LibJS",
     "//Userland/Libraries/LibMain",
+    "//Userland/Libraries/LibProtocol",
     "//Userland/Libraries/LibTLS",
     "//Userland/Libraries/LibWeb",
     "//Userland/Libraries/LibWebSocket",
@@ -142,11 +145,13 @@ if (current_os == "mac") {
     public_deps = [
       ":headless-browser",
       ":ladybird_executable",
+      "RequestServer",
       "SQLServer",
       "WebContent",
       "WebDriver",
     ]
     sources = [
+      "$root_out_dir/bin/RequestServer",
       "$root_out_dir/bin/SQLServer",
       "$root_out_dir/bin/WebContent",
       "$root_out_dir/bin/WebDriver",
@@ -176,6 +181,7 @@ if (current_os == "mac") {
       "//Userland/Libraries/LibJS",
       "//Userland/Libraries/LibLine",
       "//Userland/Libraries/LibMarkdown",
+      "//Userland/Libraries/LibProtocol",
       "//Userland/Libraries/LibRegex",
       "//Userland/Libraries/LibSQL",
       "//Userland/Libraries/LibSoftGPU",
@@ -209,6 +215,7 @@ if (current_os == "mac") {
       "$root_out_dir/lib/liblagom-js.dylib",
       "$root_out_dir/lib/liblagom-line.dylib",
       "$root_out_dir/lib/liblagom-markdown.dylib",
+      "$root_out_dir/lib/liblagom-protocol.dylib",
       "$root_out_dir/lib/liblagom-regex.dylib",
       "$root_out_dir/lib/liblagom-softgpu.dylib",
       "$root_out_dir/lib/liblagom-sql.dylib",
@@ -239,7 +246,9 @@ if (current_os == "mac") {
   }
 
   bundle_data("ladybird_config_resources") {
+    public_deps = [ "//Userland/Libraries/LibTLS:ca_certificates_download" ]
     sources = [
+      "$root_build_dir/cacert.pem",
       "//Base/home/anon/.config/BrowserAutoplayAllowlist.txt",
       "//Base/home/anon/.config/BrowserContentFilters.txt",
     ]

Meta/gn/secondary/Ladybird/RequestServer/BUILD.gn

@@ -0,0 +1,42 @@
+import("//Ladybird/link_qt.gni")
+
+link_qt("RequestServer_qt") {
+  qt_components = [ "Core" ]
+}
+
+executable("RequestServer") {
+  configs += [
+    "//Ladybird:ladybird_config",
+    ":RequestServer_qt",
+  ]
+  include_dirs = [
+    "//Userland/Libraries",
+    "//Userland/Services",
+  ]
+  deps = [
+    "//AK",
+    "//Userland/Libraries/LibCore",
+    "//Userland/Libraries/LibCrypto",
+    "//Userland/Libraries/LibFileSystem",
+    "//Userland/Libraries/LibGemini",
+    "//Userland/Libraries/LibHTTP",
+    "//Userland/Libraries/LibIPC",
+    "//Userland/Libraries/LibMain",
+    "//Userland/Libraries/LibProtocol",
+    "//Userland/Libraries/LibTLS",
+  ]
+  sources = [
+    "../Utilities.cpp",
+    "//Userland/Services/RequestServer/ConnectionCache.cpp",
+    "//Userland/Services/RequestServer/ConnectionFromClient.cpp",
+    "//Userland/Services/RequestServer/GeminiProtocol.cpp",
+    "//Userland/Services/RequestServer/GeminiRequest.cpp",
+    "//Userland/Services/RequestServer/HttpProtocol.cpp",
+    "//Userland/Services/RequestServer/HttpRequest.cpp",
+    "//Userland/Services/RequestServer/HttpsProtocol.cpp",
+    "//Userland/Services/RequestServer/HttpsRequest.cpp",
+    "//Userland/Services/RequestServer/Protocol.cpp",
+    "//Userland/Services/RequestServer/Request.cpp",
+    "main.cpp",
+  ]
+}

Meta/gn/secondary/Ladybird/WebContent/BUILD.gn

@@ -12,7 +12,6 @@ moc_qt_objects("generate_moc") {
 link_qt("WebContent_qt") {
   qt_components = [
     "Core",
-    "Gui",
     "Network",
     "Multimedia",
   ]
@@ -36,8 +35,10 @@ executable("WebContent") {
     "//Userland/Libraries/LibIPC",
     "//Userland/Libraries/LibJS",
     "//Userland/Libraries/LibMain",
+    "//Userland/Libraries/LibProtocol",
     "//Userland/Libraries/LibWeb",
     "//Userland/Libraries/LibWebSocket",
+    "//Userland/Libraries/LibWebView",
     "//Userland/Libraries/LibWebView:WebContentClientEndpoint",
     "//Userland/Libraries/LibWebView:WebContentServerEndpoint",
     "//Userland/Libraries/LibWebView:WebDriverClientEndpoint",
@@ -48,6 +49,7 @@ executable("WebContent") {
     "../AudioThread.cpp",
     "../EventLoopImplementationQt.cpp",
     "../FontPluginLadybird.cpp",
+    "../HelperProcess.cpp",
     "../ImageCodecPluginLadybird.cpp",
     "../RequestManagerQt.cpp",
     "../Utilities.cpp",

Meta/gn/secondary/Userland/Libraries/LibProtocol/BUILD.gn

@@ -0,0 +1,46 @@
+import("//Meta/gn/build/compiled_action.gni")
+
+compiled_action("RequestClientEndpoint") {
+  tool = "//Meta/Lagom/Tools/CodeGenerators/IPCCompiler"
+  inputs = [ "//Userland/Services/RequestServer/RequestClient.ipc" ]
+  outputs = [ "$root_gen_dir/RequestServer/RequestClientEndpoint.h" ]
+  args = [
+    rebase_path(inputs[0], root_build_dir),
+    "-o",
+    rebase_path(outputs[0], root_build_dir),
+  ]
+}
+
+compiled_action("RequestServerEndpoint") {
+  tool = "//Meta/Lagom/Tools/CodeGenerators/IPCCompiler"
+  inputs = [ "//Userland/Services/RequestServer/RequestServer.ipc" ]
+  outputs = [ "$root_gen_dir/RequestServer/RequestServerEndpoint.h" ]
+  args = [
+    rebase_path(inputs[0], root_build_dir),
+    "-o",
+    rebase_path(outputs[0], root_build_dir),
+  ]
+}
+
+shared_library("LibProtocol") {
+  output_name = "protocol"
+  include_dirs = [
+    "//Userland/Libraries",
+    "//Userland/Services",
+  ]
+  deps = [
+    ":RequestClientEndpoint",
+    ":RequestServerEndpoint",
+    "//AK",
+    "//Userland/Libraries/LibCore",
+    "//Userland/Libraries/LibIPC",
+  ]
+  sources = [
+    "Request.cpp",
+    "RequestClient.cpp",
+
+    # TODO: Add WebSocket sources + IPC
+  ]
+  sources += get_target_outputs(":RequestClientEndpoint") +
+             get_target_outputs(":RequestServerEndpoint")
+}

Meta/gn/secondary/Userland/Libraries/LibWebView/BUILD.gn

@@ -48,7 +48,8 @@ shared_library("LibWebView") {
   output_name = "webview"
   include_dirs = [
     "//Userland/Libraries",
-    "//Userland",
+    "//Userland/Services",
+    "//Userland/",
   ]
   deps = [
     ":WebContentClientEndpoint",
@@ -60,11 +61,13 @@ shared_library("LibWebView") {
     "//Userland/Libraries/LibGUI",
     "//Userland/Libraries/LibGfx",
     "//Userland/Libraries/LibIPC",
+    "//Userland/Libraries/LibProtocol",
     "//Userland/Libraries/LibWeb",
   ]
   sources = [
     "AccessibilityTreeModel.cpp",
     "DOMTreeModel.cpp",
+    "RequestServerAdapter.cpp",
     "StylePropertiesModel.cpp",
     "ViewImplementation.cpp",
     "WebContentClient.cpp",