LibSQL+SQLServer: Build SQLServer system service

This patch introduces the SQLServer system server. This service is
supposed to be the only process/application talking to database storage.
This makes things like locking and caching more reliable, easier to
implement, and more efficient.

In LibSQL we added a client component that does the ugly IPC nitty-
gritty for you. All that's needed is setting a number of event handler
lambdas and you can connect to databases and execute statements on them.

Applications that wish to use this SQLClient class obviously need to
link LibSQL and LibIPC.

Userland/Services/SQLServer/CMakeLists.txt

@@ -0,0 +1,20 @@
+serenity_component(
+    SQLServer
+    REQUIRED
+    TARGETS SQLServer
+)
+
+compile_ipc(SQLServer.ipc SQLServerEndpoint.h)
+compile_ipc(SQLClient.ipc SQLClientEndpoint.h)
+
+set(SOURCES
+    ClientConnection.cpp
+    DatabaseConnection.cpp
+    main.cpp
+    SQLClientEndpoint.h
+    SQLServerEndpoint.h
+    SQLStatement.cpp
+    )
+
+serenity_bin(SQLServer)
+target_link_libraries(SQLServer LibCore LibIPC LibSQL)

Userland/Services/SQLServer/ClientConnection.cpp

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/String.h>
+#include <AK/Vector.h>
+#include <LibSQL/SQLResult.h>
+#include <SQLServer/ClientConnection.h>
+#include <SQLServer/DatabaseConnection.h>
+#include <SQLServer/SQLStatement.h>
+
+namespace SQLServer {
+
+static HashMap<int, RefPtr<ClientConnection>> s_connections;
+
+RefPtr<ClientConnection> ClientConnection::client_connection_for(int client_id)
+{
+    if (s_connections.contains(client_id))
+        return *s_connections.get(client_id).value();
+    dbgln_if(SQLSERVER_DEBUG, "Invalid client_id {}", client_id);
+    return nullptr;
+}
+
+ClientConnection::ClientConnection(AK::NonnullRefPtr<Core::LocalSocket> socket, int client_id)
+    : IPC::ClientConnection<SQLClientEndpoint, SQLServerEndpoint>(*this, move(socket), client_id)
+{
+    s_connections.set(client_id, *this);
+}
+
+ClientConnection::~ClientConnection()
+{
+}
+
+void ClientConnection::die()
+{
+    s_connections.remove(client_id());
+}
+
+Messages::SQLServer::ConnectResponse ClientConnection::connect(String const& database_name)
+{
+    dbgln_if(SQLSERVER_DEBUG, "ClientConnection::connect(database_name: {})", database_name);
+    auto database_connection = DatabaseConnection::construct(database_name, client_id());
+    return { database_connection->connection_id() };
+}
+
+void ClientConnection::disconnect(int connection_id)
+{
+    dbgln_if(SQLSERVER_DEBUG, "ClientConnection::disconnect(connection_id: {})", connection_id);
+    auto database_connection = DatabaseConnection::connection_for(connection_id);
+    if (database_connection)
+        database_connection->disconnect();
+    else
+        dbgln("Database connection has disappeared");
+}
+
+Messages::SQLServer::SqlStatementResponse ClientConnection::sql_statement(int connection_id, String const& sql)
+{
+    dbgln_if(SQLSERVER_DEBUG, "ClientConnection::sql_statement(connection_id: {}, sql: '{}')", connection_id, sql);
+    auto database_connection = DatabaseConnection::connection_for(connection_id);
+    if (database_connection) {
+        auto statement_id = database_connection->sql_statement(sql);
+        dbgln_if(SQLSERVER_DEBUG, "ClientConnection::sql_statement -> statement_id = {}", statement_id);
+        return { statement_id };
+    } else {
+        dbgln("Database connection has disappeared");
+        return { -1 };
+    }
+}
+
+void ClientConnection::statement_execute(int statement_id)
+{
+    dbgln_if(SQLSERVER_DEBUG, "ClientConnection::statement_execute_query(statement_id: {})", statement_id);
+    auto statement = SQLStatement::statement_for(statement_id);
+    if (statement && statement->connection()->client_id() == client_id()) {
+        statement->execute();
+    } else {
+        dbgln_if(SQLSERVER_DEBUG, "Statement has disappeared");
+        async_execution_error(statement_id, (int)SQL::SQLErrorCode::StatementUnavailable, String::formatted("{}", statement_id));
+    }
+}
+
+}

Userland/Services/SQLServer/ClientConnection.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <LibIPC/ClientConnection.h>
+#include <SQLServer/SQLClientEndpoint.h>
+#include <SQLServer/SQLServerEndpoint.h>
+
+namespace SQLServer {
+
+class ClientConnection final
+    : public IPC::ClientConnection<SQLClientEndpoint, SQLServerEndpoint> {
+    C_OBJECT(ClientConnection);
+
+public:
+    explicit ClientConnection(NonnullRefPtr<Core::LocalSocket>, int client_id);
+    virtual ~ClientConnection() override;
+
+    virtual void die() override;
+
+    static RefPtr<ClientConnection> client_connection_for(int client_id);
+
+private:
+    virtual Messages::SQLServer::ConnectResponse connect(String const&) override;
+    virtual Messages::SQLServer::SqlStatementResponse sql_statement(int, String const&) override;
+    virtual void statement_execute(int) override;
+    virtual void disconnect(int) override;
+};
+
+}

Userland/Services/SQLServer/DatabaseConnection.cpp

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/LexicalPath.h>
+#include <SQLServer/ClientConnection.h>
+#include <SQLServer/DatabaseConnection.h>
+#include <SQLServer/SQLStatement.h>
+
+namespace SQLServer {
+
+static HashMap<int, NonnullRefPtr<DatabaseConnection>> s_connections;
+
+RefPtr<DatabaseConnection> DatabaseConnection::connection_for(int connection_id)
+{
+    if (s_connections.contains(connection_id))
+        return *s_connections.get(connection_id).value();
+    dbgln_if(SQLSERVER_DEBUG, "Invalid connection_id {}", connection_id);
+    return nullptr;
+}
+
+static int s_next_connection_id = 0;
+
+DatabaseConnection::DatabaseConnection(String database_name, int client_id)
+    : Object()
+    , m_database_name(move(database_name))
+    , m_connection_id(s_next_connection_id++)
+    , m_client_id(client_id)
+{
+    LexicalPath path(database_name);
+    if (path.title() != database_name) {
+        auto client_connection = ClientConnection::client_connection_for(m_client_id);
+        client_connection->async_connection_error(m_connection_id, (int)SQL::SQLErrorCode::InvalidDatabaseName, m_database_name);
+        return;
+    }
+
+    dbgln_if(SQLSERVER_DEBUG, "DatabaseConnection {} initiating connection with database '{}'", connection_id(), m_database_name);
+    s_connections.set(m_connection_id, *this);
+    deferred_invoke([&](Object&) {
+        m_database = SQL::Database::construct(String::formatted("/home/anon/sql/{}.db", m_database_name));
+        m_accept_statements = true;
+        auto client_connection = ClientConnection::client_connection_for(client_id);
+        if (client_connection)
+            client_connection->async_connected(m_connection_id);
+        else
+            warnln("Cannot notify client of database connection. Client disconnected");
+    });
+}
+
+void DatabaseConnection::disconnect()
+{
+    dbgln_if(SQLSERVER_DEBUG, "DatabaseConnection::disconnect(connection_id {}, database '{}'", connection_id(), m_database_name);
+    m_accept_statements = false;
+    deferred_invoke([&](Object&) {
+        m_database = nullptr;
+        s_connections.remove(m_connection_id);
+        auto client_connection = ClientConnection::client_connection_for(client_id());
+        if (client_connection)
+            client_connection->async_disconnected(m_connection_id);
+        else
+            warnln("Cannot notify client of database disconnection. Client disconnected");
+    });
+}
+
+int DatabaseConnection::sql_statement(String const& sql)
+{
+    dbgln_if(SQLSERVER_DEBUG, "DatabaseConnection::sql_statement(connection_id {}, database '{}', sql '{}'", connection_id(), m_database_name, sql);
+    auto client_connection = ClientConnection::client_connection_for(client_id());
+    if (!client_connection) {
+        warnln("Cannot notify client of database disconnection. Client disconnected");
+        return -1;
+    }
+    if (!m_accept_statements) {
+        client_connection->async_execution_error(-1, (int)SQL::SQLErrorCode::DatabaseUnavailable, m_database_name);
+        return -1;
+    }
+    auto statement = SQLStatement::construct(*this, sql);
+    return statement->statement_id();
+}
+
+}

Userland/Services/SQLServer/DatabaseConnection.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibCore/Object.h>
+#include <LibSQL/Database.h>
+#include <SQLServer/Forward.h>
+
+namespace SQLServer {
+
+class DatabaseConnection final : public Core::Object {
+    C_OBJECT(DatabaseConnection)
+    ~DatabaseConnection() override = default;
+
+    static RefPtr<DatabaseConnection> connection_for(int connection_id);
+    int connection_id() const { return m_connection_id; }
+    int client_id() const { return m_client_id; }
+    RefPtr<SQL::Database> database() { return m_database; }
+    void disconnect();
+    int sql_statement(String const& sql);
+
+private:
+    DatabaseConnection(String database_name, int client_id);
+
+    RefPtr<SQL::Database> m_database { nullptr };
+    String m_database_name;
+    int m_connection_id;
+    int m_client_id;
+    bool m_accept_statements { false };
+};
+
+}

Userland/Services/SQLServer/Forward.h

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+namespace SQLServer {
+class ClientConnection;
+class DatabaseConnection;
+class SQLStatement;
+}

Userland/Services/SQLServer/SQLClient.ipc

@@ -0,0 +1,10 @@
+endpoint SQLClient
+{
+    connected(int connection_id) =|
+    connection_error(int connection_id, int code, String message) =|
+    execution_success(int statement_id, bool has_results, int created, int updated, int deleted) =|
+    next_result(int statement_id, Vector<String> row) =|
+    results_exhausted(int statement_id, int total_rows) =|
+    execution_error(int statement_id, int code, String message) =|
+    disconnected(int connection_id) =|
+}

Userland/Services/SQLServer/SQLServer.ipc

@@ -0,0 +1,7 @@
+endpoint SQLServer [magic=5432]
+{
+    connect(String name) => (int connection_id)
+    sql_statement(int connection_id, String statement) => (int statement_id)
+    statement_execute(int statement_id) =|
+    disconnect(int connection_id) =|
+}

Userland/Services/SQLServer/SQLStatement.cpp

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibCore/Object.h>
+#include <LibSQL/AST/Parser.h>
+#include <SQLServer/ClientConnection.h>
+#include <SQLServer/DatabaseConnection.h>
+#include <SQLServer/SQLStatement.h>
+
+namespace SQLServer {
+
+static HashMap<int, NonnullRefPtr<SQLStatement>> s_statements;
+
+RefPtr<SQLStatement> SQLStatement::statement_for(int statement_id)
+{
+    if (s_statements.contains(statement_id))
+        return *s_statements.get(statement_id).value();
+    dbgln_if(SQLSERVER_DEBUG, "Invalid statement_id {}", statement_id);
+    return nullptr;
+}
+
+static int s_next_statement_id = 0;
+
+SQLStatement::SQLStatement(DatabaseConnection& connection, String sql)
+    : Core::Object(&connection)
+    , m_statement_id(s_next_statement_id++)
+    , m_sql(move(sql))
+{
+    dbgln_if(SQLSERVER_DEBUG, "SQLStatement({}, {})", connection.connection_id(), sql);
+    s_statements.set(m_statement_id, *this);
+}
+
+void SQLStatement::report_error(SQL::SQLError error)
+{
+    dbgln_if(SQLSERVER_DEBUG, "SQLStatement::report_error(statement_id {}, error {}", statement_id(), error.to_string());
+    auto client_connection = ClientConnection::client_connection_for(connection()->client_id());
+    m_statement = nullptr;
+    m_result = nullptr;
+    remove_from_parent();
+    s_statements.remove(statement_id());
+    if (!client_connection) {
+        warnln("Cannot return execution error. Client disconnected");
+        warnln("SQLStatement::report_error(statement_id {}, error {}", statement_id(), error.to_string());
+        m_result = nullptr;
+        return;
+    }
+    client_connection->async_execution_error(statement_id(), (int)error.code, error.to_string());
+    m_result = nullptr;
+}
+
+void SQLStatement::execute()
+{
+    dbgln_if(SQLSERVER_DEBUG, "SQLStatement::execute(statement_id {}", statement_id());
+    auto client_connection = ClientConnection::client_connection_for(connection()->client_id());
+    if (!client_connection) {
+        warnln("Cannot yield next result. Client disconnected");
+        return;
+    }
+
+    deferred_invoke([&](Object&) {
+        auto maybe_error = parse();
+        if (maybe_error.has_value()) {
+            report_error(maybe_error.value());
+            return;
+        }
+        m_result = m_statement->execute(*connection()->database());
+        if (m_result->error().code != SQL::SQLErrorCode::NoError) {
+            report_error(m_result->error());
+            return;
+        }
+        auto client_connection = ClientConnection::client_connection_for(connection()->client_id());
+        if (!client_connection) {
+            warnln("Cannot return statement execution results. Client disconnected");
+            return;
+        }
+        client_connection->async_execution_success(statement_id(), m_result->has_results(), m_result->updated(), m_result->inserted(), m_result->deleted());
+        if (m_result->has_results()) {
+            m_index = 0;
+            next();
+        }
+    });
+}
+
+Optional<SQL::SQLError> SQLStatement::parse()
+{
+    auto parser = SQL::AST::Parser(SQL::AST::Lexer(m_sql));
+    m_statement = parser.next_statement();
+    if (parser.has_errors()) {
+        return SQL::SQLError { SQL::SQLErrorCode::SyntaxError, parser.errors()[0].to_string() };
+    }
+    return {};
+}
+
+void SQLStatement::next()
+{
+    VERIFY(m_result->has_results());
+    auto client_connection = ClientConnection::client_connection_for(connection()->client_id());
+    if (!client_connection) {
+        warnln("Cannot yield next result. Client disconnected");
+        return;
+    }
+    if (m_index < m_result->results().size()) {
+        auto& tuple = m_result->results()[m_index++];
+        client_connection->async_next_result(statement_id(), tuple.to_string_vector());
+        deferred_invoke([&](Object&) {
+            next();
+        });
+    } else {
+        client_connection->async_results_exhausted(statement_id(), (int)m_index);
+    }
+}
+
+}

Userland/Services/SQLServer/SQLStatement.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/NonnullRefPtr.h>
+#include <AK/String.h>
+#include <LibCore/Object.h>
+#include <LibSQL/AST/AST.h>
+#include <LibSQL/SQLResult.h>
+#include <SQLServer/DatabaseConnection.h>
+#include <SQLServer/Forward.h>
+
+namespace SQLServer {
+
+class SQLStatement final : public Core::Object {
+    C_OBJECT(SQLStatement)
+    ~SQLStatement() override = default;
+
+    static RefPtr<SQLStatement> statement_for(int statement_id);
+    int statement_id() const { return m_statement_id; }
+    String const& sql() const { return m_sql; }
+    DatabaseConnection* connection() { return dynamic_cast<DatabaseConnection*>(parent()); }
+    void execute();
+
+private:
+    SQLStatement(DatabaseConnection&, String sql);
+    Optional<SQL::SQLError> parse();
+    void next();
+    void report_error(SQL::SQLError);
+
+    int m_statement_id;
+    String m_sql;
+    size_t m_index { 0 };
+    RefPtr<SQL::AST::Statement> m_statement { nullptr };
+    RefPtr<SQL::SQLResult> m_result { nullptr };
+};
+
+}

Userland/Services/SQLServer/main.cpp

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibCore/EventLoop.h>
+#include <LibCore/LocalServer.h>
+#include <SQLServer/ClientConnection.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv)
+{
+    if (pledge("stdio accept unix rpath wpath cpath", nullptr) < 0) {
+        perror("pledge");
+        return 1;
+    }
+
+    struct stat statbuf;
+    if (stat("/home/anon/sql", &statbuf) != 0) {
+        if (errno != ENOENT) {
+            perror("stat");
+            return 1;
+        }
+        if (mkdir("/home/anon/sql", 0700) != 0) {
+            perror("mkdir");
+            return 1;
+        }
+    }
+
+    if (unveil("/home/anon/sql", "rwc") < 0) {
+        perror("unveil");
+        return 1;
+    }
+    if (unveil(nullptr, nullptr) < 0) {
+        perror("unveil");
+        return 1;
+    }
+
+    Core::EventLoop event_loop;
+    auto server = Core::LocalServer::construct();
+    bool ok = server->take_over_from_system_server();
+    VERIFY(ok);
+
+    server->on_ready_to_accept = [&] {
+        auto client_socket = server->accept();
+        if (!client_socket) {
+            dbgln("SQLServer: accept failed.");
+            return;
+        }
+        static int s_next_client_id = 0;
+        int client_id = ++s_next_client_id;
+        IPC::new_client_connection<SQLServer::ClientConnection>(client_socket.release_nonnull(), client_id);
+    };
+
+    return event_loop.exec();
+}