LibCodeComprehension: Re-organize code comprehension related code

This moves all code comprehension-related code to a new library,
LibCodeComprehension.

This also moves some types related to code comprehension tasks (such as
autocomplete, find declaration) out of LibGUI and into
LibCodeComprehension.

Userland/Libraries/LibCodeComprehension/Cpp/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(SOURCES
+    CppComprehensionEngine.cpp
+)
+
+serenity_lib(LibCppComprehension cppcomprehension)
+target_link_libraries(LibCppComprehension LibCodeComprehension LibC)
+
+serenity_component(
+    CppComprehensionTests
+    TARGETS CppComprehensionTests
+)
+
+set(SOURCES
+    CppComprehensionEngine.cpp
+    Tests.cpp
+)
+
+serenity_bin(CppComprehensionTests)
+
+target_link_libraries(CppComprehensionTests LibCodeComprehension LibCpp LibRegex LibMain)

Userland/Libraries/LibCodeComprehension/Cpp/ConnectionFromClient.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "CppComprehensionEngine.h"
+#include <DevTools/HackStudio/LanguageServers/ConnectionFromClient.h>
+
+namespace LanguageServers::Cpp {
+
+class ConnectionFromClient final : public LanguageServers::ConnectionFromClient {
+    C_OBJECT(ConnectionFromClient);
+
+private:
+    ConnectionFromClient(NonnullOwnPtr<Core::Stream::LocalSocket> socket)
+        : LanguageServers::ConnectionFromClient(move(socket))
+    {
+        m_autocomplete_engine = make<CodeComprehension::Cpp::CppComprehensionEngine>(m_filedb);
+        m_autocomplete_engine->set_declarations_of_document_callback = [this](String const& filename, Vector<CodeComprehension::Declaration>&& declarations) {
+            async_declarations_in_document(filename, move(declarations));
+        };
+        m_autocomplete_engine->set_todo_entries_of_document_callback = [this](String const& filename, Vector<CodeComprehension::TodoEntry>&& todo_entries) {
+            async_todo_entries_in_document(filename, move(todo_entries));
+        };
+    }
+
+    virtual ~ConnectionFromClient() override = default;
+};
+}

Userland/Libraries/LibCodeComprehension/Cpp/CppComprehensionEngine.h

@@ -0,0 +1,204 @@
+/*
+ * Copyright (c) 2021-2022, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Function.h>
+#include <AK/String.h>
+#include <AK/Vector.h>
+#include <DevTools/HackStudio/AutoCompleteResponse.h>
+#include <DevTools/HackStudio/LanguageServers/FileDB.h>
+#include <LibCpp/AST.h>
+#include <LibCpp/Parser.h>
+#include <LibCpp/Preprocessor.h>
+#include <LibGUI/TextPosition.h>
+#include <Libraries/LibCodeComprehension/CodeComprehensionEngine.h>
+
+namespace CodeComprehension::Cpp {
+
+using namespace ::Cpp;
+
+class CppComprehensionEngine : public CodeComprehensionEngine {
+public:
+    CppComprehensionEngine(FileDB const& filedb);
+
+    virtual Vector<CodeComprehension::AutocompleteResultEntry> get_suggestions(String const& file, GUI::TextPosition const& autocomplete_position) override;
+    virtual void on_edit(String const& file) override;
+    virtual void file_opened([[maybe_unused]] String const& file) override;
+    virtual Optional<CodeComprehension::ProjectLocation> find_declaration_of(String const& filename, GUI::TextPosition const& identifier_position) override;
+    virtual Optional<FunctionParamsHint> get_function_params_hint(String const&, GUI::TextPosition const&) override;
+    virtual Vector<CodeComprehension::TokenInfo> get_tokens_info(String const& filename) override;
+
+private:
+    struct SymbolName {
+        StringView name;
+        Vector<StringView> scope;
+
+        static SymbolName create(StringView, Vector<StringView>&&);
+        static SymbolName create(StringView);
+        String scope_as_string() const;
+        String to_string() const;
+
+        bool operator==(SymbolName const&) const = default;
+    };
+
+    struct Symbol {
+        SymbolName name;
+        NonnullRefPtr<Cpp::Declaration> declaration;
+
+        // Local symbols are symbols that should not appear in a global symbol search.
+        // For example, a variable that is declared inside a function will have is_local = true.
+        bool is_local { false };
+
+        enum class IsLocal {
+            No,
+            Yes
+        };
+        static Symbol create(StringView name, Vector<StringView> const& scope, NonnullRefPtr<Cpp::Declaration>, IsLocal is_local);
+    };
+
+    friend Traits<SymbolName>;
+
+    struct DocumentData {
+        String const& filename() const { return m_filename; }
+        String const& text() const { return m_text; }
+        Preprocessor const& preprocessor() const
+        {
+            VERIFY(m_preprocessor);
+            return *m_preprocessor;
+        }
+        Preprocessor& preprocessor()
+        {
+            VERIFY(m_preprocessor);
+            return *m_preprocessor;
+        }
+        Parser const& parser() const
+        {
+            VERIFY(m_parser);
+            return *m_parser;
+        }
+        Parser& parser()
+        {
+            VERIFY(m_parser);
+            return *m_parser;
+        }
+
+        String m_filename;
+        String m_text;
+        OwnPtr<Preprocessor> m_preprocessor;
+        OwnPtr<Parser> m_parser;
+
+        HashMap<SymbolName, Symbol> m_symbols;
+        HashTable<String> m_available_headers;
+    };
+
+    Vector<CodeComprehension::AutocompleteResultEntry> autocomplete_property(DocumentData const&, MemberExpression const&, const String partial_text) const;
+    Vector<AutocompleteResultEntry> autocomplete_name(DocumentData const&, ASTNode const&, String const& partial_text) const;
+    String type_of(DocumentData const&, Expression const&) const;
+    String type_of_property(DocumentData const&, Identifier const&) const;
+    String type_of_variable(Identifier const&) const;
+    bool is_property(ASTNode const&) const;
+    RefPtr<Cpp::Declaration> find_declaration_of(DocumentData const&, ASTNode const&) const;
+    RefPtr<Cpp::Declaration> find_declaration_of(DocumentData const&, SymbolName const&) const;
+    RefPtr<Cpp::Declaration> find_declaration_of(DocumentData const&, const GUI::TextPosition& identifier_position);
+
+    enum class RecurseIntoScopes {
+        No,
+        Yes
+    };
+
+    Vector<Symbol> properties_of_type(DocumentData const& document, String const& type) const;
+    Vector<Symbol> get_child_symbols(ASTNode const&) const;
+    Vector<Symbol> get_child_symbols(ASTNode const&, Vector<StringView> const& scope, Symbol::IsLocal) const;
+
+    DocumentData const* get_document_data(String const& file) const;
+    DocumentData const* get_or_create_document_data(String const& file);
+    void set_document_data(String const& file, OwnPtr<DocumentData>&& data);
+
+    OwnPtr<DocumentData> create_document_data_for(String const& file);
+    String document_path_from_include_path(StringView include_path) const;
+    void update_declared_symbols(DocumentData&);
+    void update_todo_entries(DocumentData&);
+    CodeComprehension::DeclarationType type_of_declaration(Cpp::Declaration const&);
+    Vector<StringView> scope_of_node(ASTNode const&) const;
+    Vector<StringView> scope_of_reference_to_symbol(ASTNode const&) const;
+
+    Optional<CodeComprehension::ProjectLocation> find_preprocessor_definition(DocumentData const&, const GUI::TextPosition&);
+    Optional<Cpp::Preprocessor::Substitution> find_preprocessor_substitution(DocumentData const&, Cpp::Position const&);
+
+    OwnPtr<DocumentData> create_document_data(String text, String const& filename);
+    Optional<Vector<CodeComprehension::AutocompleteResultEntry>> try_autocomplete_property(DocumentData const&, ASTNode const&, Optional<Token> containing_token) const;
+    Optional<Vector<CodeComprehension::AutocompleteResultEntry>> try_autocomplete_name(DocumentData const&, ASTNode const&, Optional<Token> containing_token) const;
+    Optional<Vector<CodeComprehension::AutocompleteResultEntry>> try_autocomplete_include(DocumentData const&, Token include_path_token, Cpp::Position const& cursor_position) const;
+    static bool is_symbol_available(Symbol const&, Vector<StringView> const& current_scope, Vector<StringView> const& reference_scope);
+    Optional<FunctionParamsHint> get_function_params_hint(DocumentData const&, FunctionCall&, size_t argument_index);
+
+    template<typename Func>
+    void for_each_available_symbol(DocumentData const&, Func) const;
+
+    template<typename Func>
+    void for_each_included_document_recursive(DocumentData const&, Func) const;
+
+    CodeComprehension::TokenInfo::SemanticType get_token_semantic_type(DocumentData const&, Token const&);
+    CodeComprehension::TokenInfo::SemanticType get_semantic_type_for_identifier(DocumentData const&, Position);
+
+    HashMap<String, OwnPtr<DocumentData>> m_documents;
+
+    // A document's path will be in this set if we're currently processing it.
+    // A document is added to this set when we start processing it (e.g because it was #included) and removed when we're done.
+    // We use this to prevent circular #includes from looping indefinitely.
+    HashTable<String> m_unfinished_documents;
+};
+
+template<typename Func>
+void CppComprehensionEngine::for_each_available_symbol(DocumentData const& document, Func func) const
+{
+    for (auto& item : document.m_symbols) {
+        auto decision = func(item.value);
+        if (decision == IterationDecision::Break)
+            return;
+    }
+
+    for_each_included_document_recursive(document, [&](DocumentData const& document) {
+        for (auto& item : document.m_symbols) {
+            auto decision = func(item.value);
+            if (decision == IterationDecision::Break)
+                return IterationDecision::Break;
+        }
+        return IterationDecision::Continue;
+    });
+}
+
+template<typename Func>
+void CppComprehensionEngine::for_each_included_document_recursive(DocumentData const& document, Func func) const
+{
+    for (auto& included_path : document.m_available_headers) {
+        auto* included_document = get_document_data(included_path);
+        if (!included_document)
+            continue;
+        auto decision = func(*included_document);
+        if (decision == IterationDecision::Break)
+            continue;
+    }
+}
+}
+
+namespace AK {
+
+template<>
+struct Traits<CodeComprehension::Cpp::CppComprehensionEngine::SymbolName> : public GenericTraits<CodeComprehension::Cpp::CppComprehensionEngine::SymbolName> {
+    static unsigned hash(CodeComprehension::Cpp::CppComprehensionEngine::SymbolName const& key)
+    {
+        unsigned hash = 0;
+        hash = pair_int_hash(hash, string_hash(key.name.characters_without_null_termination(), key.name.length()));
+        for (auto& scope_part : key.scope) {
+            hash = pair_int_hash(hash, string_hash(scope_part.characters_without_null_termination(), scope_part.length()));
+        }
+        return hash;
+    }
+};
+
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests.cpp

@@ -0,0 +1,206 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "Tests.h"
+#include "../FileDB.h"
+#include "CppComprehensionEngine.h"
+#include <AK/LexicalPath.h>
+#include <LibCore/File.h>
+#include <LibMain/Main.h>
+
+static bool s_some_test_failed = false;
+
+#define I_TEST(name)                     \
+    {                                    \
+        printf("Testing " #name "... "); \
+        fflush(stdout);                  \
+    }
+
+#define PASS              \
+    do {                  \
+        printf("PASS\n"); \
+        fflush(stdout);   \
+        return;           \
+    } while (0)
+
+#define FAIL(reason)                   \
+    do {                               \
+        printf("FAIL: " #reason "\n"); \
+        s_some_test_failed = true;     \
+        return;                        \
+    } while (0)
+
+constexpr char TESTS_ROOT_DIR[] = "/home/anon/Tests/cpp-tests/comprehension";
+
+class FileDB : public CodeComprehension::FileDB {
+public:
+    FileDB() = default;
+
+    void add(String filename, String content)
+    {
+        m_map.set(filename, content);
+    }
+
+    virtual Optional<String> get_or_read_from_filesystem(StringView filename) const override
+    {
+        String target_filename = filename;
+        if (!project_root().is_null() && filename.starts_with(project_root())) {
+            target_filename = LexicalPath::relative_path(filename, project_root());
+        }
+        return m_map.get(target_filename);
+    }
+
+private:
+    HashMap<String, String> m_map;
+};
+
+static void test_complete_local_args();
+static void test_complete_local_vars();
+static void test_complete_type();
+static void test_find_variable_definition();
+static void test_complete_includes();
+static void test_parameters_hint();
+
+int run_tests()
+{
+    test_complete_local_args();
+    test_complete_local_vars();
+    test_complete_type();
+    test_find_variable_definition();
+    test_complete_includes();
+    test_parameters_hint();
+    return s_some_test_failed ? 1 : 0;
+}
+
+static void add_file(FileDB& filedb, String const& name)
+{
+    auto file = Core::File::open(LexicalPath::join(TESTS_ROOT_DIR, name).string(), Core::OpenMode::ReadOnly);
+    VERIFY(!file.is_error());
+    filedb.add(name, String::copy(file.value()->read_all()));
+}
+
+void test_complete_local_args()
+{
+    I_TEST(Complete Local Args)
+    FileDB filedb;
+    add_file(filedb, "complete_local_args.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine engine(filedb);
+    auto suggestions = engine.get_suggestions("complete_local_args.cpp", { 2, 6 });
+    if (suggestions.size() != 2)
+        FAIL(bad size);
+
+    if (suggestions[0].completion == "argc" && suggestions[1].completion == "argv")
+        PASS;
+
+    FAIL("wrong results");
+}
+
+void test_complete_local_vars()
+{
+    I_TEST(Complete Local Vars)
+    FileDB filedb;
+    add_file(filedb, "complete_local_vars.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine autocomplete(filedb);
+    auto suggestions = autocomplete.get_suggestions("complete_local_vars.cpp", { 3, 7 });
+    if (suggestions.size() != 1)
+        FAIL(bad size);
+
+    if (suggestions[0].completion == "myvar1")
+        PASS;
+
+    FAIL("wrong results");
+}
+
+void test_complete_type()
+{
+    I_TEST(Complete Type)
+    FileDB filedb;
+    add_file(filedb, "complete_type.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine autocomplete(filedb);
+    auto suggestions = autocomplete.get_suggestions("complete_type.cpp", { 5, 7 });
+    if (suggestions.size() != 1)
+        FAIL(bad size);
+
+    if (suggestions[0].completion == "MyStruct")
+        PASS;
+
+    FAIL("wrong results");
+}
+
+void test_find_variable_definition()
+{
+    I_TEST(Find Variable Declaration)
+    FileDB filedb;
+    add_file(filedb, "find_variable_declaration.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine engine(filedb);
+    auto position = engine.find_declaration_of("find_variable_declaration.cpp", { 2, 5 });
+    if (!position.has_value())
+        FAIL("declaration not found");
+
+    if (position.value().file == "find_variable_declaration.cpp" && position.value().line == 0 && position.value().column >= 19)
+        PASS;
+    FAIL("wrong declaration location");
+}
+
+void test_complete_includes()
+{
+    I_TEST(Complete include statements)
+    FileDB filedb;
+    filedb.set_project_root(TESTS_ROOT_DIR);
+    add_file(filedb, "complete_includes.cpp");
+    add_file(filedb, "sample_header.h");
+    CodeComprehension::Cpp::CppComprehensionEngine autocomplete(filedb);
+
+    auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 22 });
+    if (suggestions.size() != 1)
+        FAIL(project include - bad size);
+
+    if (suggestions[0].completion != "\"sample_header.h\"")
+        FAIL("project include - wrong results");
+
+    suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 18 });
+    if (suggestions.size() != 1)
+        FAIL(global include - bad size);
+
+    if (suggestions[0].completion != "<sys/cdefs.h>")
+        FAIL("global include - wrong results");
+
+    PASS;
+}
+
+void test_parameters_hint()
+{
+    I_TEST(Function Parameters hint)
+    FileDB filedb;
+    filedb.set_project_root(TESTS_ROOT_DIR);
+    add_file(filedb, "parameters_hint1.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine engine(filedb);
+
+    auto result = engine.get_function_params_hint("parameters_hint1.cpp", { 4, 9 });
+    if (!result.has_value())
+        FAIL("failed to get parameters hint (1)");
+    if (result->params != Vector<String> { "int x", "char y" } || result->current_index != 0)
+        FAIL("bad result (1)");
+
+    result = engine.get_function_params_hint("parameters_hint1.cpp", { 5, 15 });
+    if (!result.has_value())
+        FAIL("failed to get parameters hint (2)");
+    if (result->params != Vector<String> { "int x", "char y" } || result->current_index != 1)
+        FAIL("bad result (2)");
+
+    result = engine.get_function_params_hint("parameters_hint1.cpp", { 6, 8 });
+    if (!result.has_value())
+        FAIL("failed to get parameters hint (3)");
+    if (result->params != Vector<String> { "int x", "char y" } || result->current_index != 0)
+        FAIL("bad result (3)");
+
+    PASS;
+}
+
+ErrorOr<int> serenity_main(Main::Arguments)
+{
+    return run_tests();
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests.h

@@ -0,0 +1,9 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+int run_tests();

Userland/Libraries/LibCodeComprehension/Cpp/Tests/complete_includes.cpp

@@ -0,0 +1,5 @@
+#include "sample_heade
+#include <sys/cdef
+
+void foo() {}
+

Userland/Libraries/LibCodeComprehension/Cpp/Tests/complete_local_args.cpp

@@ -0,0 +1,4 @@
+int main(int argc, char** argv)
+{
+    ar
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/complete_local_vars.cpp

@@ -0,0 +1,5 @@
+int main(int argc, char** argv)
+{
+    int myvar1 = 3;
+    myv
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/complete_type.cpp

@@ -0,0 +1,7 @@
+struct MyStruct {
+    int x;
+};
+void foo()
+{
+    MyS
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/find_variable_declaration.cpp

@@ -0,0 +1,4 @@
+int main(int argc, char** argv)
+{
+    argv = nullptr;
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/parameters_hint1.cpp

@@ -0,0 +1,8 @@
+void foo(int x, char y);
+
+void bar()
+{
+    foo();
+    foo(123, 'b');
+    foo(
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/sample_header.h

@@ -0,0 +1,3 @@
+#pragma once
+
+int bar();