SystemServer: Limit service restarts

SystemServer will now track the number of restart attempts and the
run time of a service, and will also pay attention to its exit code.
If a service exits unsuccessfully and too quickly (in less than a
second), SystemServer will only attempt to restart it twice.

This means that if WindowServer crashes on startup, we will now see
just a few copies of the crash instead of the quickly scrolling log
flashing with colors :^)

Servers/SystemServer/Service.cpp

@@ -179,6 +179,7 @@ void Service::spawn()
 {
     dbg() << "Spawning " << name();
 
+    m_run_timer.start();
     m_pid = fork();
 
     if (m_pid < 0) {
@@ -256,15 +257,34 @@ void Service::spawn()
 void Service::did_exit(int exit_code)
 {
     ASSERT(m_pid > 0);
-    (void)exit_code;
 
-    dbg() << "Service " << name() << " has exited";
+    dbg() << "Service " << name() << " has exited with exit code " << exit_code;
 
     s_service_map.remove(m_pid);
     m_pid = -1;
 
-    if (m_keep_alive)
+    if (!m_keep_alive)
-        activate();
+        return;
+
+    int run_time_in_msec = m_run_timer.elapsed();
+    bool exited_successfully = exit_code == 0;
+
+    if (!exited_successfully && run_time_in_msec < 1000) {
+        switch (m_restart_attempts) {
+        case 0:
+            dbg() << "Trying again";
+            break;
+        case 1:
+            dbg() << "Third time's a charm?";
+            break;
+        default:
+            dbg() << "Giving up on " << name() << ". Good luck!";
+            return;
+        }
+        m_restart_attempts++;
+    }
+
+    activate();
 }
 
 Service::Service(const Core::ConfigFile& config, const StringView& name)
@@ -330,4 +350,6 @@ void Service::save_to(JsonObject& json)
         json.set("pid", m_pid);
     else
         json.set("pid", nullptr);
+
+    json.set("restart_attempts", m_restart_attempts);
 }

Servers/SystemServer/Service.h

@@ -28,6 +28,7 @@
 
 #include <AK/RefPtr.h>
 #include <AK/String.h>
+#include <LibCore/ElapsedTimer.h>
 #include <LibCore/Notifier.h>
 #include <LibCore/Object.h>
 
@@ -81,6 +82,12 @@ private:
     int m_socket_fd { -1 };
     RefPtr<Core::Notifier> m_socket_notifier;
 
+    // Timer since we last spawned the service.
+    Core::ElapsedTimer m_run_timer;
+    // How many times we have tried to restart this service, only counting those
+    // times where it has exited unsuccessfully and too quickly.
+    int m_restart_attempts { 0 };
+
     void resolve_user();
     void setup_socket();
     void setup_notifier();