pro: Add ability to log request/response metadata for HTTP URLs

In order to debug WebServer and responses we can't handle yet, it's
beneficial to being able to see what we request and what we get back.
As a first measure, we just log URL, response code, reason phrase and
headers.

Userland/Utilities/CMakeLists.txt

@@ -109,7 +109,7 @@ target_link_libraries(paste PRIVATE LibGUI)
 target_link_libraries(pgrep PRIVATE LibRegex)
 target_link_libraries(pkill PRIVATE LibRegex)
 target_link_libraries(pls PRIVATE LibCrypt)
-target_link_libraries(pro PRIVATE LibProtocol)
+target_link_libraries(pro PRIVATE LibProtocol LibHTTP)
 target_link_libraries(run-tests PRIVATE LibRegex LibCoredump LibDebug)
 target_link_libraries(shot PRIVATE LibGfx LibGUI LibIPC)
 target_link_libraries(sql PRIVATE LibLine LibSQL LibIPC)

Userland/Utilities/pro.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2022, Thomas Keppler <serenity@tkeppler.de>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -13,6 +14,7 @@
 #include <LibCore/EventLoop.h>
 #include <LibCore/File.h>
 #include <LibCore/System.h>
+#include <LibHTTP/HttpResponse.h>
 #include <LibMain/Main.h>
 #include <LibProtocol/Request.h>
 #include <LibProtocol/RequestClient.h>
@@ -151,6 +153,7 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
     StringView url_str;
     bool save_at_provided_name = false;
     bool should_follow_url = false;
+    bool verbose_output = false;
     char const* data = nullptr;
     StringView proxy_spec;
     DeprecatedString method = "GET";
@@ -180,6 +183,7 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
             return true;
         } });
     args_parser.add_option(proxy_spec, "Specify a proxy server to use for this request (proto://ip:port)", "proxy", 'p', "proxy");
+    args_parser.add_option(verbose_output, "(HTTP only) Log request and response metadata", "verbose", 'v');
     args_parser.add_positional_argument(url_str, "URL to download from", "url");
     args_parser.parse(arguments);
 
@@ -196,6 +200,8 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
         return 1;
     }
 
+    bool const is_http_url = url.scheme().is_one_of("http"sv, "https"sv);
+
     Core::ProxyData proxy_data {};
     if (!proxy_spec.is_empty())
         proxy_data = TRY(Core::ProxyData::parse_url(proxy_spec));
@@ -222,6 +228,14 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
             exit(1);
         }
 
+        if (verbose_output && is_http_url) {
+            warnln("* Setting up request");
+            warnln("> Method={}, URL={}", method, url);
+            for (auto const& header : request_headers) {
+                warnln("> {}: {}", header.key, header.value);
+            }
+        }
+
         request->on_progress = [&](Optional<u32> maybe_total_size, u32 downloaded_size) {
             gettimeofday(&current_time, nullptr);
             timersub(&current_time, &previous_time, &time_diff);
@@ -252,6 +266,18 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
             received_actual_headers = true; // And not trailers!
             should_save_stream_data = true;
 
+            if (verbose_output && is_http_url) {
+                warnln("* Received headers");
+                auto const value = status_code.value_or(0);
+                auto const reason_phrase = (value != 0)
+                    ? HTTP::HttpResponse::reason_phrase_for_code(value)
+                    : "UNKNOWN"sv;
+                warnln("< Code={}, Reason={}", value, reason_phrase);
+                for (auto const& header : response_headers) {
+                    warnln("< {}: {}", header.key, header.value);
+                }
+            }
+
             if (!following_url && save_at_provided_name) {
                 DeprecatedString output_name;
                 if (auto content_disposition = response_headers.get("Content-Disposition"); content_disposition.has_value()) {