LibCodeComprehension: Re-organize code comprehension related code

This moves all code comprehension-related code to a new library,
LibCodeComprehension.

This also moves some types related to code comprehension tasks (such as
autocomplete, find declaration) out of LibGUI and into
LibCodeComprehension.

Userland/Libraries/CMakeLists.txt

@@ -3,6 +3,7 @@ add_subdirectory(LibAudio)
 add_subdirectory(LibC)
 add_subdirectory(LibCards)
 add_subdirectory(LibChess)
+add_subdirectory(LibCodeComprehension)
 add_subdirectory(LibCompress)
 add_subdirectory(LibConfig)
 add_subdirectory(LibCore)

Userland/Libraries/LibCodeComprehension/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(SOURCES
+    CodeComprehensionEngine.cpp
+    FileDB.cpp
+)
+
+serenity_lib(LibCodeComprehension codecomprehension)
+target_link_libraries(LibCodeComprehension LibC)
+
+add_subdirectory(Cpp)
+add_subdirectory(Shell)

Userland/Libraries/LibCodeComprehension/CodeComprehensionEngine.cpp

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ * Copyright (c) 2022, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "CodeComprehensionEngine.h"
+
+namespace CodeComprehension {
+
+CodeComprehensionEngine::CodeComprehensionEngine(FileDB const& filedb, bool should_store_all_declarations)
+    : m_filedb(filedb)
+    , m_store_all_declarations(should_store_all_declarations)
+{
+}
+
+void CodeComprehensionEngine::set_declarations_of_document(String const& filename, Vector<Declaration>&& declarations)
+{
+    // Callback may not be configured if we're running tests
+    if (!set_declarations_of_document_callback)
+        return;
+
+    // Optimization - Only notify callback if declarations have changed
+    if (auto previous_declarations = m_all_declarations.find(filename); previous_declarations != m_all_declarations.end()) {
+        if (previous_declarations->value == declarations)
+            return;
+    }
+    if (m_store_all_declarations)
+        m_all_declarations.set(filename, declarations);
+    set_declarations_of_document_callback(filename, move(declarations));
+}
+
+void CodeComprehensionEngine::set_todo_entries_of_document(String const& filename, Vector<TodoEntry>&& todo_entries)
+{
+    // Callback may not be configured if we're running tests
+    if (!set_todo_entries_of_document_callback)
+        return;
+    set_todo_entries_of_document_callback(filename, move(todo_entries));
+}
+
+}

Userland/Libraries/LibCodeComprehension/CodeComprehensionEngine.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ * Copyright (c) 2022, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "FileDB.h"
+#include "Types.h"
+#include <AK/Function.h>
+#include <AK/HashMap.h>
+#include <AK/Vector.h>
+#include <LibGUI/TextPosition.h>
+
+namespace CodeComprehension {
+
+class CodeComprehensionEngine {
+    AK_MAKE_NONCOPYABLE(CodeComprehensionEngine);
+    AK_MAKE_NONMOVABLE(CodeComprehensionEngine);
+
+public:
+    CodeComprehensionEngine(FileDB const& filedb, bool store_all_declarations = false);
+    virtual ~CodeComprehensionEngine() = default;
+
+    virtual Vector<AutocompleteResultEntry> get_suggestions(String const& file, GUI::TextPosition const& autocomplete_position) = 0;
+
+    // TODO: In the future we can pass the range that was edited and only re-parse what we have to.
+    virtual void on_edit([[maybe_unused]] String const& file) {};
+    virtual void file_opened([[maybe_unused]] String const& file) {};
+
+    virtual Optional<ProjectLocation> find_declaration_of(String const&, GUI::TextPosition const&) { return {}; }
+
+    struct FunctionParamsHint {
+        Vector<String> params;
+        size_t current_index { 0 };
+    };
+    virtual Optional<FunctionParamsHint> get_function_params_hint(String const&, GUI::TextPosition const&) { return {}; }
+
+    virtual Vector<TokenInfo> get_tokens_info(String const&) { return {}; }
+
+    Function<void(String const&, Vector<Declaration>&&)> set_declarations_of_document_callback;
+    Function<void(String const&, Vector<TodoEntry>&&)> set_todo_entries_of_document_callback;
+
+protected:
+    FileDB const& filedb() const { return m_filedb; }
+    void set_declarations_of_document(String const&, Vector<Declaration>&&);
+    void set_todo_entries_of_document(String const&, Vector<TodoEntry>&&);
+    HashMap<String, Vector<Declaration>> const& all_declarations() const { return m_all_declarations; }
+
+private:
+    HashMap<String, Vector<Declaration>> m_all_declarations;
+    FileDB const& m_filedb;
+    bool m_store_all_declarations { false };
+};
+}

Userland/Libraries/LibCodeComprehension/Cpp/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(SOURCES
+    CppComprehensionEngine.cpp
+)
+
+serenity_lib(LibCppComprehension cppcomprehension)
+target_link_libraries(LibCppComprehension LibCodeComprehension LibC)
+
+serenity_component(
+    CppComprehensionTests
+    TARGETS CppComprehensionTests
+)
+
+set(SOURCES
+    CppComprehensionEngine.cpp
+    Tests.cpp
+)
+
+serenity_bin(CppComprehensionTests)
+
+target_link_libraries(CppComprehensionTests LibCodeComprehension LibCpp LibRegex LibMain)

Userland/Libraries/LibCodeComprehension/Cpp/ConnectionFromClient.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "CppComprehensionEngine.h"
+#include <DevTools/HackStudio/LanguageServers/ConnectionFromClient.h>
+
+namespace LanguageServers::Cpp {
+
+class ConnectionFromClient final : public LanguageServers::ConnectionFromClient {
+    C_OBJECT(ConnectionFromClient);
+
+private:
+    ConnectionFromClient(NonnullOwnPtr<Core::Stream::LocalSocket> socket)
+        : LanguageServers::ConnectionFromClient(move(socket))
+    {
+        m_autocomplete_engine = make<CodeComprehension::Cpp::CppComprehensionEngine>(m_filedb);
+        m_autocomplete_engine->set_declarations_of_document_callback = [this](String const& filename, Vector<CodeComprehension::Declaration>&& declarations) {
+            async_declarations_in_document(filename, move(declarations));
+        };
+        m_autocomplete_engine->set_todo_entries_of_document_callback = [this](String const& filename, Vector<CodeComprehension::TodoEntry>&& todo_entries) {
+            async_todo_entries_in_document(filename, move(todo_entries));
+        };
+    }
+
+    virtual ~ConnectionFromClient() override = default;
+};
+}

Userland/Libraries/LibCodeComprehension/Cpp/CppComprehensionEngine.h

@@ -0,0 +1,204 @@
+/*
+ * Copyright (c) 2021-2022, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Function.h>
+#include <AK/String.h>
+#include <AK/Vector.h>
+#include <DevTools/HackStudio/AutoCompleteResponse.h>
+#include <DevTools/HackStudio/LanguageServers/FileDB.h>
+#include <LibCpp/AST.h>
+#include <LibCpp/Parser.h>
+#include <LibCpp/Preprocessor.h>
+#include <LibGUI/TextPosition.h>
+#include <Libraries/LibCodeComprehension/CodeComprehensionEngine.h>
+
+namespace CodeComprehension::Cpp {
+
+using namespace ::Cpp;
+
+class CppComprehensionEngine : public CodeComprehensionEngine {
+public:
+    CppComprehensionEngine(FileDB const& filedb);
+
+    virtual Vector<CodeComprehension::AutocompleteResultEntry> get_suggestions(String const& file, GUI::TextPosition const& autocomplete_position) override;
+    virtual void on_edit(String const& file) override;
+    virtual void file_opened([[maybe_unused]] String const& file) override;
+    virtual Optional<CodeComprehension::ProjectLocation> find_declaration_of(String const& filename, GUI::TextPosition const& identifier_position) override;
+    virtual Optional<FunctionParamsHint> get_function_params_hint(String const&, GUI::TextPosition const&) override;
+    virtual Vector<CodeComprehension::TokenInfo> get_tokens_info(String const& filename) override;
+
+private:
+    struct SymbolName {
+        StringView name;
+        Vector<StringView> scope;
+
+        static SymbolName create(StringView, Vector<StringView>&&);
+        static SymbolName create(StringView);
+        String scope_as_string() const;
+        String to_string() const;
+
+        bool operator==(SymbolName const&) const = default;
+    };
+
+    struct Symbol {
+        SymbolName name;
+        NonnullRefPtr<Cpp::Declaration> declaration;
+
+        // Local symbols are symbols that should not appear in a global symbol search.
+        // For example, a variable that is declared inside a function will have is_local = true.
+        bool is_local { false };
+
+        enum class IsLocal {
+            No,
+            Yes
+        };
+        static Symbol create(StringView name, Vector<StringView> const& scope, NonnullRefPtr<Cpp::Declaration>, IsLocal is_local);
+    };
+
+    friend Traits<SymbolName>;
+
+    struct DocumentData {
+        String const& filename() const { return m_filename; }
+        String const& text() const { return m_text; }
+        Preprocessor const& preprocessor() const
+        {
+            VERIFY(m_preprocessor);
+            return *m_preprocessor;
+        }
+        Preprocessor& preprocessor()
+        {
+            VERIFY(m_preprocessor);
+            return *m_preprocessor;
+        }
+        Parser const& parser() const
+        {
+            VERIFY(m_parser);
+            return *m_parser;
+        }
+        Parser& parser()
+        {
+            VERIFY(m_parser);
+            return *m_parser;
+        }
+
+        String m_filename;
+        String m_text;
+        OwnPtr<Preprocessor> m_preprocessor;
+        OwnPtr<Parser> m_parser;
+
+        HashMap<SymbolName, Symbol> m_symbols;
+        HashTable<String> m_available_headers;
+    };
+
+    Vector<CodeComprehension::AutocompleteResultEntry> autocomplete_property(DocumentData const&, MemberExpression const&, const String partial_text) const;
+    Vector<AutocompleteResultEntry> autocomplete_name(DocumentData const&, ASTNode const&, String const& partial_text) const;
+    String type_of(DocumentData const&, Expression const&) const;
+    String type_of_property(DocumentData const&, Identifier const&) const;
+    String type_of_variable(Identifier const&) const;
+    bool is_property(ASTNode const&) const;
+    RefPtr<Cpp::Declaration> find_declaration_of(DocumentData const&, ASTNode const&) const;
+    RefPtr<Cpp::Declaration> find_declaration_of(DocumentData const&, SymbolName const&) const;
+    RefPtr<Cpp::Declaration> find_declaration_of(DocumentData const&, const GUI::TextPosition& identifier_position);
+
+    enum class RecurseIntoScopes {
+        No,
+        Yes
+    };
+
+    Vector<Symbol> properties_of_type(DocumentData const& document, String const& type) const;
+    Vector<Symbol> get_child_symbols(ASTNode const&) const;
+    Vector<Symbol> get_child_symbols(ASTNode const&, Vector<StringView> const& scope, Symbol::IsLocal) const;
+
+    DocumentData const* get_document_data(String const& file) const;
+    DocumentData const* get_or_create_document_data(String const& file);
+    void set_document_data(String const& file, OwnPtr<DocumentData>&& data);
+
+    OwnPtr<DocumentData> create_document_data_for(String const& file);
+    String document_path_from_include_path(StringView include_path) const;
+    void update_declared_symbols(DocumentData&);
+    void update_todo_entries(DocumentData&);
+    CodeComprehension::DeclarationType type_of_declaration(Cpp::Declaration const&);
+    Vector<StringView> scope_of_node(ASTNode const&) const;
+    Vector<StringView> scope_of_reference_to_symbol(ASTNode const&) const;
+
+    Optional<CodeComprehension::ProjectLocation> find_preprocessor_definition(DocumentData const&, const GUI::TextPosition&);
+    Optional<Cpp::Preprocessor::Substitution> find_preprocessor_substitution(DocumentData const&, Cpp::Position const&);
+
+    OwnPtr<DocumentData> create_document_data(String text, String const& filename);
+    Optional<Vector<CodeComprehension::AutocompleteResultEntry>> try_autocomplete_property(DocumentData const&, ASTNode const&, Optional<Token> containing_token) const;
+    Optional<Vector<CodeComprehension::AutocompleteResultEntry>> try_autocomplete_name(DocumentData const&, ASTNode const&, Optional<Token> containing_token) const;
+    Optional<Vector<CodeComprehension::AutocompleteResultEntry>> try_autocomplete_include(DocumentData const&, Token include_path_token, Cpp::Position const& cursor_position) const;
+    static bool is_symbol_available(Symbol const&, Vector<StringView> const& current_scope, Vector<StringView> const& reference_scope);
+    Optional<FunctionParamsHint> get_function_params_hint(DocumentData const&, FunctionCall&, size_t argument_index);
+
+    template<typename Func>
+    void for_each_available_symbol(DocumentData const&, Func) const;
+
+    template<typename Func>
+    void for_each_included_document_recursive(DocumentData const&, Func) const;
+
+    CodeComprehension::TokenInfo::SemanticType get_token_semantic_type(DocumentData const&, Token const&);
+    CodeComprehension::TokenInfo::SemanticType get_semantic_type_for_identifier(DocumentData const&, Position);
+
+    HashMap<String, OwnPtr<DocumentData>> m_documents;
+
+    // A document's path will be in this set if we're currently processing it.
+    // A document is added to this set when we start processing it (e.g because it was #included) and removed when we're done.
+    // We use this to prevent circular #includes from looping indefinitely.
+    HashTable<String> m_unfinished_documents;
+};
+
+template<typename Func>
+void CppComprehensionEngine::for_each_available_symbol(DocumentData const& document, Func func) const
+{
+    for (auto& item : document.m_symbols) {
+        auto decision = func(item.value);
+        if (decision == IterationDecision::Break)
+            return;
+    }
+
+    for_each_included_document_recursive(document, [&](DocumentData const& document) {
+        for (auto& item : document.m_symbols) {
+            auto decision = func(item.value);
+            if (decision == IterationDecision::Break)
+                return IterationDecision::Break;
+        }
+        return IterationDecision::Continue;
+    });
+}
+
+template<typename Func>
+void CppComprehensionEngine::for_each_included_document_recursive(DocumentData const& document, Func func) const
+{
+    for (auto& included_path : document.m_available_headers) {
+        auto* included_document = get_document_data(included_path);
+        if (!included_document)
+            continue;
+        auto decision = func(*included_document);
+        if (decision == IterationDecision::Break)
+            continue;
+    }
+}
+}
+
+namespace AK {
+
+template<>
+struct Traits<CodeComprehension::Cpp::CppComprehensionEngine::SymbolName> : public GenericTraits<CodeComprehension::Cpp::CppComprehensionEngine::SymbolName> {
+    static unsigned hash(CodeComprehension::Cpp::CppComprehensionEngine::SymbolName const& key)
+    {
+        unsigned hash = 0;
+        hash = pair_int_hash(hash, string_hash(key.name.characters_without_null_termination(), key.name.length()));
+        for (auto& scope_part : key.scope) {
+            hash = pair_int_hash(hash, string_hash(scope_part.characters_without_null_termination(), scope_part.length()));
+        }
+        return hash;
+    }
+};
+
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests.cpp

@@ -0,0 +1,206 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "Tests.h"
+#include "../FileDB.h"
+#include "CppComprehensionEngine.h"
+#include <AK/LexicalPath.h>
+#include <LibCore/File.h>
+#include <LibMain/Main.h>
+
+static bool s_some_test_failed = false;
+
+#define I_TEST(name)                     \
+    {                                    \
+        printf("Testing " #name "... "); \
+        fflush(stdout);                  \
+    }
+
+#define PASS              \
+    do {                  \
+        printf("PASS\n"); \
+        fflush(stdout);   \
+        return;           \
+    } while (0)
+
+#define FAIL(reason)                   \
+    do {                               \
+        printf("FAIL: " #reason "\n"); \
+        s_some_test_failed = true;     \
+        return;                        \
+    } while (0)
+
+constexpr char TESTS_ROOT_DIR[] = "/home/anon/Tests/cpp-tests/comprehension";
+
+class FileDB : public CodeComprehension::FileDB {
+public:
+    FileDB() = default;
+
+    void add(String filename, String content)
+    {
+        m_map.set(filename, content);
+    }
+
+    virtual Optional<String> get_or_read_from_filesystem(StringView filename) const override
+    {
+        String target_filename = filename;
+        if (!project_root().is_null() && filename.starts_with(project_root())) {
+            target_filename = LexicalPath::relative_path(filename, project_root());
+        }
+        return m_map.get(target_filename);
+    }
+
+private:
+    HashMap<String, String> m_map;
+};
+
+static void test_complete_local_args();
+static void test_complete_local_vars();
+static void test_complete_type();
+static void test_find_variable_definition();
+static void test_complete_includes();
+static void test_parameters_hint();
+
+int run_tests()
+{
+    test_complete_local_args();
+    test_complete_local_vars();
+    test_complete_type();
+    test_find_variable_definition();
+    test_complete_includes();
+    test_parameters_hint();
+    return s_some_test_failed ? 1 : 0;
+}
+
+static void add_file(FileDB& filedb, String const& name)
+{
+    auto file = Core::File::open(LexicalPath::join(TESTS_ROOT_DIR, name).string(), Core::OpenMode::ReadOnly);
+    VERIFY(!file.is_error());
+    filedb.add(name, String::copy(file.value()->read_all()));
+}
+
+void test_complete_local_args()
+{
+    I_TEST(Complete Local Args)
+    FileDB filedb;
+    add_file(filedb, "complete_local_args.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine engine(filedb);
+    auto suggestions = engine.get_suggestions("complete_local_args.cpp", { 2, 6 });
+    if (suggestions.size() != 2)
+        FAIL(bad size);
+
+    if (suggestions[0].completion == "argc" && suggestions[1].completion == "argv")
+        PASS;
+
+    FAIL("wrong results");
+}
+
+void test_complete_local_vars()
+{
+    I_TEST(Complete Local Vars)
+    FileDB filedb;
+    add_file(filedb, "complete_local_vars.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine autocomplete(filedb);
+    auto suggestions = autocomplete.get_suggestions("complete_local_vars.cpp", { 3, 7 });
+    if (suggestions.size() != 1)
+        FAIL(bad size);
+
+    if (suggestions[0].completion == "myvar1")
+        PASS;
+
+    FAIL("wrong results");
+}
+
+void test_complete_type()
+{
+    I_TEST(Complete Type)
+    FileDB filedb;
+    add_file(filedb, "complete_type.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine autocomplete(filedb);
+    auto suggestions = autocomplete.get_suggestions("complete_type.cpp", { 5, 7 });
+    if (suggestions.size() != 1)
+        FAIL(bad size);
+
+    if (suggestions[0].completion == "MyStruct")
+        PASS;
+
+    FAIL("wrong results");
+}
+
+void test_find_variable_definition()
+{
+    I_TEST(Find Variable Declaration)
+    FileDB filedb;
+    add_file(filedb, "find_variable_declaration.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine engine(filedb);
+    auto position = engine.find_declaration_of("find_variable_declaration.cpp", { 2, 5 });
+    if (!position.has_value())
+        FAIL("declaration not found");
+
+    if (position.value().file == "find_variable_declaration.cpp" && position.value().line == 0 && position.value().column >= 19)
+        PASS;
+    FAIL("wrong declaration location");
+}
+
+void test_complete_includes()
+{
+    I_TEST(Complete include statements)
+    FileDB filedb;
+    filedb.set_project_root(TESTS_ROOT_DIR);
+    add_file(filedb, "complete_includes.cpp");
+    add_file(filedb, "sample_header.h");
+    CodeComprehension::Cpp::CppComprehensionEngine autocomplete(filedb);
+
+    auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 22 });
+    if (suggestions.size() != 1)
+        FAIL(project include - bad size);
+
+    if (suggestions[0].completion != "\"sample_header.h\"")
+        FAIL("project include - wrong results");
+
+    suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 18 });
+    if (suggestions.size() != 1)
+        FAIL(global include - bad size);
+
+    if (suggestions[0].completion != "<sys/cdefs.h>")
+        FAIL("global include - wrong results");
+
+    PASS;
+}
+
+void test_parameters_hint()
+{
+    I_TEST(Function Parameters hint)
+    FileDB filedb;
+    filedb.set_project_root(TESTS_ROOT_DIR);
+    add_file(filedb, "parameters_hint1.cpp");
+    CodeComprehension::Cpp::CppComprehensionEngine engine(filedb);
+
+    auto result = engine.get_function_params_hint("parameters_hint1.cpp", { 4, 9 });
+    if (!result.has_value())
+        FAIL("failed to get parameters hint (1)");
+    if (result->params != Vector<String> { "int x", "char y" } || result->current_index != 0)
+        FAIL("bad result (1)");
+
+    result = engine.get_function_params_hint("parameters_hint1.cpp", { 5, 15 });
+    if (!result.has_value())
+        FAIL("failed to get parameters hint (2)");
+    if (result->params != Vector<String> { "int x", "char y" } || result->current_index != 1)
+        FAIL("bad result (2)");
+
+    result = engine.get_function_params_hint("parameters_hint1.cpp", { 6, 8 });
+    if (!result.has_value())
+        FAIL("failed to get parameters hint (3)");
+    if (result->params != Vector<String> { "int x", "char y" } || result->current_index != 0)
+        FAIL("bad result (3)");
+
+    PASS;
+}
+
+ErrorOr<int> serenity_main(Main::Arguments)
+{
+    return run_tests();
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests.h

@@ -0,0 +1,9 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+int run_tests();

Userland/Libraries/LibCodeComprehension/Cpp/Tests/complete_includes.cpp

@@ -0,0 +1,5 @@
+#include "sample_heade
+#include <sys/cdef
+
+void foo() {}
+

Userland/Libraries/LibCodeComprehension/Cpp/Tests/complete_local_args.cpp

@@ -0,0 +1,4 @@
+int main(int argc, char** argv)
+{
+    ar
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/complete_local_vars.cpp

@@ -0,0 +1,5 @@
+int main(int argc, char** argv)
+{
+    int myvar1 = 3;
+    myv
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/complete_type.cpp

@@ -0,0 +1,7 @@
+struct MyStruct {
+    int x;
+};
+void foo()
+{
+    MyS
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/find_variable_declaration.cpp

@@ -0,0 +1,4 @@
+int main(int argc, char** argv)
+{
+    argv = nullptr;
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/parameters_hint1.cpp

@@ -0,0 +1,8 @@
+void foo(int x, char y);
+
+void bar()
+{
+    foo();
+    foo(123, 'b');
+    foo(
+}

Userland/Libraries/LibCodeComprehension/Cpp/Tests/sample_header.h

@@ -0,0 +1,3 @@
+#pragma once
+
+int bar();

Userland/Libraries/LibCodeComprehension/FileDB.cpp

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2022, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "FileDB.h"
+#include <AK/LexicalPath.h>
+
+namespace CodeComprehension {
+
+String FileDB::to_absolute_path(StringView filename) const
+{
+    if (LexicalPath { filename }.is_absolute()) {
+        return filename;
+    }
+    if (m_project_root.is_null())
+        return filename;
+    return LexicalPath { String::formatted("{}/{}", m_project_root, filename) }.string();
+}
+
+}

Userland/Libraries/LibCodeComprehension/FileDB.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2022, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/String.h>
+#include <AK/StringView.h>
+
+namespace CodeComprehension {
+
+class FileDB {
+    AK_MAKE_NONCOPYABLE(FileDB);
+    AK_MAKE_NONMOVABLE(FileDB);
+
+public:
+    virtual ~FileDB() = default;
+
+    virtual Optional<String> get_or_read_from_filesystem(StringView filename) const = 0;
+    void set_project_root(StringView project_root) { m_project_root = project_root; }
+    String const& project_root() const { return m_project_root; }
+    String to_absolute_path(StringView filename) const;
+
+protected:
+    FileDB() = default;
+
+private:
+    String m_project_root;
+};
+
+}

Userland/Libraries/LibCodeComprehension/Shell/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(SOURCES
+    ShellComprehensionEngine.cpp
+)
+
+serenity_lib(LibShellComprehension shellcomprehension)
+target_link_libraries(LibShellComprehension LibCodeComprehension LibC)

Userland/Libraries/LibCodeComprehension/Shell/ConnectionFromClient.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "ShellComprehensionEngine.h"
+#include <DevTools/HackStudio/LanguageServers/ConnectionFromClient.h>
+#include <LibCpp/Parser.h>
+
+namespace LanguageServers::Shell {
+
+class ConnectionFromClient final : public LanguageServers::ConnectionFromClient {
+    C_OBJECT(ConnectionFromClient);
+
+private:
+    ConnectionFromClient(NonnullOwnPtr<Core::Stream::LocalSocket> socket)
+        : LanguageServers::ConnectionFromClient(move(socket))
+    {
+        m_autocomplete_engine = make<ShellComprehensionEngine>(m_filedb);
+        m_autocomplete_engine->set_declarations_of_document_callback = [this](String const& filename, Vector<CodeComprehension::Declaration>&& declarations) {
+            async_declarations_in_document(filename, move(declarations));
+        };
+        m_autocomplete_engine->set_todo_entries_of_document_callback = [this](String const& filename, Vector<CodeComprehension::TodoEntry>&& todo_entries) {
+            async_todo_entries_in_document(filename, move(todo_entries));
+        };
+    }
+    virtual ~ConnectionFromClient() override = default;
+};
+}

Userland/Libraries/LibCodeComprehension/Shell/ShellComprehensionEngine.cpp

@@ -0,0 +1,236 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "ShellComprehensionEngine.h"
+#include <AK/Assertions.h>
+#include <AK/HashTable.h>
+#include <LibRegex/Regex.h>
+
+namespace CodeComprehension::Shell {
+
+RefPtr<::Shell::Shell> ShellComprehensionEngine::s_shell {};
+
+ShellComprehensionEngine::ShellComprehensionEngine(FileDB const& filedb)
+    : CodeComprehensionEngine(filedb, true)
+{
+}
+
+ShellComprehensionEngine::DocumentData const& ShellComprehensionEngine::get_or_create_document_data(String const& file)
+{
+    auto absolute_path = filedb().to_absolute_path(file);
+    if (!m_documents.contains(absolute_path)) {
+        set_document_data(absolute_path, create_document_data_for(absolute_path));
+    }
+    return get_document_data(absolute_path);
+}
+
+ShellComprehensionEngine::DocumentData const& ShellComprehensionEngine::get_document_data(String const& file) const
+{
+    auto absolute_path = filedb().to_absolute_path(file);
+    auto document_data = m_documents.get(absolute_path);
+    VERIFY(document_data.has_value());
+    return *document_data.value();
+}
+
+OwnPtr<ShellComprehensionEngine::DocumentData> ShellComprehensionEngine::create_document_data_for(String const& file)
+{
+    auto document = filedb().get_or_read_from_filesystem(file);
+    if (!document.has_value())
+        return {};
+
+    auto content = document.value();
+    auto document_data = make<DocumentData>(move(content), file);
+    for (auto& path : document_data->sourced_paths())
+        get_or_create_document_data(path);
+
+    update_declared_symbols(*document_data);
+    return document_data;
+}
+
+void ShellComprehensionEngine::set_document_data(String const& file, OwnPtr<DocumentData>&& data)
+{
+    m_documents.set(filedb().to_absolute_path(file), move(data));
+}
+
+ShellComprehensionEngine::DocumentData::DocumentData(String&& _text, String _filename)
+    : filename(move(_filename))
+    , text(move(_text))
+    , node(parse())
+{
+}
+
+Vector<String> const& ShellComprehensionEngine::DocumentData::sourced_paths() const
+{
+    if (all_sourced_paths.has_value())
+        return all_sourced_paths.value();
+
+    struct : public ::Shell::AST::NodeVisitor {
+        void visit(const ::Shell::AST::CastToCommand* node) override
+        {
+            auto& inner = node->inner();
+            if (inner->is_list()) {
+                if (auto* list = dynamic_cast<const ::Shell::AST::ListConcatenate*>(inner.ptr())) {
+                    auto& entries = list->list();
+                    if (entries.size() == 2 && entries.first()->is_bareword() && static_ptr_cast<::Shell::AST::BarewordLiteral>(entries.first())->text() == "source") {
+                        auto& filename = entries[1];
+                        if (filename->would_execute())
+                            return;
+                        auto name_list = const_cast<::Shell::AST::Node*>(filename.ptr())->run(nullptr)->resolve_as_list(nullptr);
+                        StringBuilder builder;
+                        builder.join(" ", name_list);
+                        sourced_files.set(builder.build());
+                    }
+                }
+            }
+            ::Shell::AST::NodeVisitor::visit(node);
+        }
+
+        HashTable<String> sourced_files;
+    } visitor;
+
+    node->visit(visitor);
+
+    Vector<String> sourced_paths;
+    for (auto& entry : visitor.sourced_files)
+        sourced_paths.append(move(entry));
+
+    all_sourced_paths = move(sourced_paths);
+    return all_sourced_paths.value();
+}
+
+NonnullRefPtr<::Shell::AST::Node> ShellComprehensionEngine::DocumentData::parse() const
+{
+    ::Shell::Parser parser { text };
+    if (auto node = parser.parse())
+        return node.release_nonnull();
+
+    return ::Shell::AST::make_ref_counted<::Shell::AST::SyntaxError>(::Shell::AST::Position {}, "Unable to parse file");
+}
+
+size_t ShellComprehensionEngine::resolve(ShellComprehensionEngine::DocumentData const& document, const GUI::TextPosition& position)
+{
+    size_t offset = 0;
+
+    if (position.line() > 0) {
+        auto first = true;
+        size_t line = 0;
+        for (auto& line_view : document.text.split_limit('\n', position.line() + 1, true)) {
+            if (line == position.line())
+                break;
+            if (first)
+                first = false;
+            else
+                ++offset; // For the newline.
+            offset += line_view.length();
+            ++line;
+        }
+    }
+
+    offset += position.column() + 1;
+    return offset;
+}
+
+Vector<CodeComprehension::AutocompleteResultEntry> ShellComprehensionEngine::get_suggestions(String const& file, const GUI::TextPosition& position)
+{
+    dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "ShellComprehensionEngine position {}:{}", position.line(), position.column());
+
+    auto const& document = get_or_create_document_data(file);
+    size_t offset_in_file = resolve(document, position);
+
+    ::Shell::AST::HitTestResult hit_test = document.node->hit_test_position(offset_in_file);
+    if (!hit_test.matching_node) {
+        dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "no node at position {}:{}", position.line(), position.column());
+        return {};
+    }
+
+    auto completions = const_cast<::Shell::AST::Node*>(document.node.ptr())->complete_for_editor(shell(), offset_in_file, hit_test);
+    Vector<CodeComprehension::AutocompleteResultEntry> entries;
+    for (auto& completion : completions)
+        entries.append({ completion.text_string, completion.input_offset });
+
+    return entries;
+}
+
+void ShellComprehensionEngine::on_edit(String const& file)
+{
+    set_document_data(file, create_document_data_for(file));
+}
+
+void ShellComprehensionEngine::file_opened([[maybe_unused]] String const& file)
+{
+    set_document_data(file, create_document_data_for(file));
+}
+
+Optional<CodeComprehension::ProjectLocation> ShellComprehensionEngine::find_declaration_of(String const& filename, const GUI::TextPosition& identifier_position)
+{
+    dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "find_declaration_of({}, {}:{})", filename, identifier_position.line(), identifier_position.column());
+    auto const& document = get_or_create_document_data(filename);
+    auto position = resolve(document, identifier_position);
+    auto result = document.node->hit_test_position(position);
+    if (!result.matching_node) {
+        dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "no node at position {}:{}", identifier_position.line(), identifier_position.column());
+        return {};
+    }
+
+    if (!result.matching_node->is_bareword()) {
+        dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "no bareword at position {}:{}", identifier_position.line(), identifier_position.column());
+        return {};
+    }
+
+    auto name = static_ptr_cast<::Shell::AST::BarewordLiteral>(result.matching_node)->text();
+    auto& declarations = all_declarations();
+    for (auto& entry : declarations) {
+        for (auto& declaration : entry.value) {
+            if (declaration.name == name)
+                return declaration.position;
+        }
+    }
+
+    return {};
+}
+
+void ShellComprehensionEngine::update_declared_symbols(DocumentData const& document)
+{
+    struct Visitor : public ::Shell::AST::NodeVisitor {
+        explicit Visitor(String const& filename)
+            : filename(filename)
+        {
+        }
+
+        void visit(const ::Shell::AST::VariableDeclarations* node) override
+        {
+            for (auto& entry : node->variables()) {
+                auto literal = entry.name->leftmost_trivial_literal();
+                if (!literal)
+                    continue;
+
+                String name;
+                if (literal->is_bareword())
+                    name = static_ptr_cast<::Shell::AST::BarewordLiteral>(literal)->text();
+
+                if (!name.is_empty()) {
+                    dbgln("Found variable {}", name);
+                    declarations.append({ move(name), { filename, entry.name->position().start_line.line_number, entry.name->position().start_line.line_column }, CodeComprehension::DeclarationType::Variable, {} });
+                }
+            }
+            ::Shell::AST::NodeVisitor::visit(node);
+        }
+
+        void visit(const ::Shell::AST::FunctionDeclaration* node) override
+        {
+            dbgln("Found function {}", node->name().name);
+            declarations.append({ node->name().name, { filename, node->position().start_line.line_number, node->position().start_line.line_column }, CodeComprehension::DeclarationType::Function, {} });
+        }
+
+        String const& filename;
+        Vector<CodeComprehension::Declaration> declarations;
+    } visitor { document.filename };
+
+    document.node->visit(visitor);
+
+    set_declarations_of_document(document.filename, move(visitor.declarations));
+}
+}

Userland/Libraries/LibCodeComprehension/Shell/ShellComprehensionEngine.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibCodeComprehension/CodeComprehensionEngine.h>
+#include <Shell/Shell.h>
+
+namespace CodeComprehension::Shell {
+
+class ShellComprehensionEngine : public CodeComprehensionEngine {
+public:
+    ShellComprehensionEngine(FileDB const& filedb);
+    virtual Vector<CodeComprehension::AutocompleteResultEntry> get_suggestions(String const& file, const GUI::TextPosition& position) override;
+    virtual void on_edit(String const& file) override;
+    virtual void file_opened([[maybe_unused]] String const& file) override;
+    virtual Optional<CodeComprehension::ProjectLocation> find_declaration_of(String const& filename, const GUI::TextPosition& identifier_position) override;
+
+private:
+    struct DocumentData {
+        DocumentData(String&& text, String filename);
+        String filename;
+        String text;
+        NonnullRefPtr<::Shell::AST::Node> node;
+
+        Vector<String> const& sourced_paths() const;
+
+    private:
+        NonnullRefPtr<::Shell::AST::Node> parse() const;
+
+        mutable Optional<Vector<String>> all_sourced_paths {};
+    };
+
+    DocumentData const& get_document_data(String const& file) const;
+    DocumentData const& get_or_create_document_data(String const& file);
+    void set_document_data(String const& file, OwnPtr<DocumentData>&& data);
+
+    OwnPtr<DocumentData> create_document_data_for(String const& file);
+    String document_path_from_include_path(StringView include_path) const;
+    void update_declared_symbols(DocumentData const&);
+
+    static size_t resolve(ShellComprehensionEngine::DocumentData const& document, const GUI::TextPosition& position);
+
+    ::Shell::Shell& shell()
+    {
+        if (s_shell)
+            return *s_shell;
+        s_shell = ::Shell::Shell::construct();
+        return *s_shell;
+    }
+
+    HashMap<String, OwnPtr<DocumentData>> m_documents;
+    static RefPtr<::Shell::Shell> s_shell;
+};
+}

Userland/Libraries/LibCodeComprehension/Shell/main.cpp

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "ConnectionFromClient.h"
+#include <LibCore/EventLoop.h>
+#include <LibCore/LocalServer.h>
+#include <LibCore/System.h>
+#include <LibIPC/SingleServer.h>
+#include <LibMain/Main.h>
+
+ErrorOr<int> serenity_main(Main::Arguments)
+{
+    Core::EventLoop event_loop;
+    TRY(Core::System::pledge("stdio unix rpath recvfd"));
+
+    auto client = TRY(IPC::take_over_accepted_client_from_system_server<LanguageServers::Shell::ConnectionFromClient>());
+
+    TRY(Core::System::pledge("stdio rpath recvfd"));
+    TRY(Core::System::unveil("/etc/passwd", "r"));
+
+    return event_loop.exec();
+}

Userland/Libraries/LibCodeComprehension/Types.h

@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2022, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/String.h>
+
+#pragma once
+
+namespace CodeComprehension {
+
+enum class Language {
+    Unspecified,
+    Cpp,
+};
+
+struct AutocompleteResultEntry {
+    String completion;
+    size_t partial_input_length { 0 };
+    // TODO: Actually assign the value of this field in more places (when applicable).
+    Language language { Language::Unspecified };
+    String display_text {};
+
+    enum class HideAutocompleteAfterApplying {
+        No,
+        Yes,
+    };
+    HideAutocompleteAfterApplying hide_autocomplete_after_applying { HideAutocompleteAfterApplying::Yes };
+};
+
+struct ProjectLocation {
+    String file;
+    size_t line { 0 };
+    size_t column { 0 };
+
+    bool operator==(ProjectLocation const& other) const
+    {
+        return file == other.file && line == other.line && column == other.column;
+    }
+};
+
+enum class DeclarationType {
+    Function,
+    Struct,
+    Class,
+    Variable,
+    PreprocessorDefinition,
+    Namespace,
+    Member,
+};
+
+struct Declaration {
+    String name;
+    ProjectLocation position;
+    DeclarationType type;
+    String scope;
+
+    bool operator==(Declaration const& other) const
+    {
+        return name == other.name && position == other.position && type == other.type && scope == other.scope;
+    }
+};
+
+#define FOR_EACH_SEMANTIC_TYPE        \
+    __SEMANTIC(Unknown)               \
+    __SEMANTIC(Regular)               \
+    __SEMANTIC(Keyword)               \
+    __SEMANTIC(Type)                  \
+    __SEMANTIC(Identifier)            \
+    __SEMANTIC(String)                \
+    __SEMANTIC(Number)                \
+    __SEMANTIC(IncludePath)           \
+    __SEMANTIC(PreprocessorStatement) \
+    __SEMANTIC(Comment)               \
+    __SEMANTIC(Whitespace)            \
+    __SEMANTIC(Function)              \
+    __SEMANTIC(Variable)              \
+    __SEMANTIC(CustomType)            \
+    __SEMANTIC(Namespace)             \
+    __SEMANTIC(Member)                \
+    __SEMANTIC(Parameter)             \
+    __SEMANTIC(PreprocessorMacro)
+
+struct TokenInfo {
+
+    enum class SemanticType : u32 {
+#define __SEMANTIC(x) x,
+        FOR_EACH_SEMANTIC_TYPE
+#undef __SEMANTIC
+
+    } type { SemanticType::Unknown };
+    size_t start_line { 0 };
+    size_t start_column { 0 };
+    size_t end_line { 0 };
+    size_t end_column { 0 };
+
+    static constexpr char const* type_to_string(SemanticType t)
+    {
+        switch (t) {
+#define __SEMANTIC(x)     \
+    case SemanticType::x: \
+        return #x;
+            FOR_EACH_SEMANTIC_TYPE
+#undef __SEMANTIC
+        }
+        VERIFY_NOT_REACHED();
+    };
+};
+
+struct TodoEntry {
+    String content;
+    String filename;
+    size_t line { 0 };
+    size_t column { 0 };
+};
+
+}

Userland/Libraries/LibCpp/Parser.cpp

@@ -1012,9 +1012,9 @@ void Parser::print_tokens() const
     }
 }
 
-Vector<Parser::TodoEntry> Parser::get_todo_entries() const
+Vector<CodeComprehension::TodoEntry> Parser::get_todo_entries() const
 {
-    Vector<TodoEntry> ret;
+    Vector<CodeComprehension::TodoEntry> ret;
     for (auto& token : m_tokens) {
         if (token.type() == Token::Type::Comment) {
             if (token.text().contains("TODO")) {

Userland/Libraries/LibCpp/Parser.h

@@ -10,6 +10,7 @@
 #include "AST.h"
 #include "Preprocessor.h"
 #include <AK/Noncopyable.h>
+#include <LibCodeComprehension/Types.h>
 #include <LibCpp/Lexer.h>
 
 namespace Cpp {
@@ -35,13 +36,7 @@ public:
     Vector<Token> const& tokens() const { return m_tokens; }
     Vector<String> const& errors() const { return m_errors; }
 
-    struct TodoEntry {
-        String content;
-        String filename;
-        size_t line { 0 };
-        size_t column { 0 };
-    };
-    Vector<TodoEntry> get_todo_entries() const;
+    Vector<CodeComprehension::TodoEntry> get_todo_entries() const;
 
     Vector<Token> tokens_in_range(Position start, Position end) const;
 

Userland/Libraries/LibCpp/SemanticSyntaxHighlighter.cpp

@@ -15,7 +15,7 @@ namespace Cpp {
 
 void SemanticSyntaxHighlighter::rehighlight(Palette const& palette)
 {
-    Vector<GUI::AutocompleteProvider::TokenInfo> new_tokens_info;
+    Vector<CodeComprehension::TokenInfo> new_tokens_info;
     auto text = m_client->get_text();
     {
         Threading::MutexLocker locker(m_lock);
@@ -39,7 +39,7 @@ void SemanticSyntaxHighlighter::rehighlight(Palette const& palette)
         //        An improvement over this could be only including the tokens that are in edited text ranges in the diff.
         auto diff_hunks = Diff::from_text(previous.view(), current.view());
         for (auto& token : current_tokens) {
-            new_tokens_info.append(GUI::AutocompleteProvider::TokenInfo { GUI::AutocompleteProvider::TokenInfo::SemanticType::Unknown,
+            new_tokens_info.append(CodeComprehension::TokenInfo { CodeComprehension::TokenInfo::SemanticType::Unknown,
                 token.start().line, token.start().column, token.end().line, token.end().column });
         }
         size_t previous_token_index = 0;
@@ -67,47 +67,47 @@ void SemanticSyntaxHighlighter::rehighlight(Palette const& palette)
     update_spans(new_tokens_info, palette);
 }
 
-static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, GUI::AutocompleteProvider::TokenInfo::SemanticType type)
+static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, CodeComprehension::TokenInfo::SemanticType type)
 {
     switch (type) {
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Unknown:
+    case CodeComprehension::TokenInfo::SemanticType::Unknown:
         return { palette.base_text(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Keyword:
+    case CodeComprehension::TokenInfo::SemanticType::Keyword:
         return { palette.syntax_keyword(), true };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Type:
+    case CodeComprehension::TokenInfo::SemanticType::Type:
         return { palette.syntax_type(), true };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Identifier:
+    case CodeComprehension::TokenInfo::SemanticType::Identifier:
         return { palette.syntax_identifier(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::String:
+    case CodeComprehension::TokenInfo::SemanticType::String:
         return { palette.syntax_string(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Number:
+    case CodeComprehension::TokenInfo::SemanticType::Number:
         return { palette.syntax_number(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::IncludePath:
+    case CodeComprehension::TokenInfo::SemanticType::IncludePath:
         return { palette.syntax_preprocessor_value(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::PreprocessorStatement:
+    case CodeComprehension::TokenInfo::SemanticType::PreprocessorStatement:
         return { palette.syntax_preprocessor_statement(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Comment:
+    case CodeComprehension::TokenInfo::SemanticType::Comment:
         return { palette.syntax_comment(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Function:
+    case CodeComprehension::TokenInfo::SemanticType::Function:
         return { palette.syntax_function(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Variable:
+    case CodeComprehension::TokenInfo::SemanticType::Variable:
         return { palette.syntax_variable(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::CustomType:
+    case CodeComprehension::TokenInfo::SemanticType::CustomType:
         return { palette.syntax_custom_type(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Namespace:
+    case CodeComprehension::TokenInfo::SemanticType::Namespace:
         return { palette.syntax_namespace(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Member:
+    case CodeComprehension::TokenInfo::SemanticType::Member:
         return { palette.syntax_member(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Parameter:
+    case CodeComprehension::TokenInfo::SemanticType::Parameter:
         return { palette.syntax_parameter(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::PreprocessorMacro:
+    case CodeComprehension::TokenInfo::SemanticType::PreprocessorMacro:
         return { palette.syntax_preprocessor_value(), false };
     default:
         VERIFY_NOT_REACHED();
         return { palette.base_text(), false };
     }
 }
-void SemanticSyntaxHighlighter::update_spans(Vector<GUI::AutocompleteProvider::TokenInfo> const& tokens_info, Gfx::Palette const& pallete)
+void SemanticSyntaxHighlighter::update_spans(Vector<CodeComprehension::TokenInfo> const& tokens_info, Gfx::Palette const& pallete)
 {
     Vector<GUI::TextDocumentSpan> spans;
     for (auto& token : tokens_info) {
@@ -118,7 +118,7 @@ void SemanticSyntaxHighlighter::update_spans(Vector<GUI::AutocompleteProvider::T
         auto style = style_for_token_type(pallete, token.type);
         span.attributes.color = style.color;
         span.attributes.bold = style.bold;
-        span.is_skippable = token.type == GUI::AutocompleteProvider::TokenInfo::SemanticType::Whitespace;
+        span.is_skippable = token.type == CodeComprehension::TokenInfo::SemanticType::Whitespace;
         span.data = static_cast<u64>(token.type);
         spans.append(span);
     }
@@ -130,7 +130,7 @@ void SemanticSyntaxHighlighter::update_spans(Vector<GUI::AutocompleteProvider::T
     m_client->do_update();
 }
 
-void SemanticSyntaxHighlighter::update_tokens_info(Vector<GUI::AutocompleteProvider::TokenInfo> tokens_info)
+void SemanticSyntaxHighlighter::update_tokens_info(Vector<CodeComprehension::TokenInfo> tokens_info)
 {
     {
         Threading::MutexLocker locker(m_lock);
@@ -145,22 +145,21 @@ void SemanticSyntaxHighlighter::update_tokens_info(Vector<GUI::AutocompleteProvi
 
 bool SemanticSyntaxHighlighter::is_identifier(u64 token_type) const
 {
-    using GUI::AutocompleteProvider;
-    auto type = static_cast<AutocompleteProvider::TokenInfo::SemanticType>(token_type);
+    auto type = static_cast<CodeComprehension::TokenInfo::SemanticType>(token_type);
 
-    return type == AutocompleteProvider::TokenInfo::SemanticType::Identifier
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Function
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Variable
-        || type == AutocompleteProvider::TokenInfo::SemanticType::CustomType
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Namespace
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Member
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Parameter
-        || type == AutocompleteProvider::TokenInfo::SemanticType::PreprocessorMacro;
+    return type == CodeComprehension::TokenInfo::SemanticType::Identifier
+        || type == CodeComprehension::TokenInfo::SemanticType::Function
+        || type == CodeComprehension::TokenInfo::SemanticType::Variable
+        || type == CodeComprehension::TokenInfo::SemanticType::CustomType
+        || type == CodeComprehension::TokenInfo::SemanticType::Namespace
+        || type == CodeComprehension::TokenInfo::SemanticType::Member
+        || type == CodeComprehension::TokenInfo::SemanticType::Parameter
+        || type == CodeComprehension::TokenInfo::SemanticType::PreprocessorMacro;
 }
 
 bool SemanticSyntaxHighlighter::is_navigatable(u64 token_type) const
 {
-    return static_cast<GUI::AutocompleteProvider::TokenInfo::SemanticType>(token_type) == GUI::AutocompleteProvider::TokenInfo::SemanticType::IncludePath;
+    return static_cast<CodeComprehension::TokenInfo::SemanticType>(token_type) == CodeComprehension::TokenInfo::SemanticType::IncludePath;
 }
 
 }

Userland/Libraries/LibCpp/SemanticSyntaxHighlighter.h

@@ -27,7 +27,7 @@ public:
     virtual Syntax::Language language() const override { return Syntax::Language::Cpp; }
     virtual void rehighlight(Palette const&) override;
 
-    void update_tokens_info(Vector<GUI::AutocompleteProvider::TokenInfo>);
+    void update_tokens_info(Vector<CodeComprehension::TokenInfo>);
 
     virtual bool is_cpp_semantic_highlighter() const override { return true; }
 
@@ -36,10 +36,10 @@ protected:
     virtual bool token_types_equal(u64 token1, u64 token2) const override { return m_simple_syntax_highlighter.token_types_equal(token1, token2); };
 
 private:
-    void update_spans(Vector<GUI::AutocompleteProvider::TokenInfo> const&, Gfx::Palette const&);
+    void update_spans(Vector<CodeComprehension::TokenInfo> const&, Gfx::Palette const&);
 
     Cpp::SyntaxHighlighter m_simple_syntax_highlighter;
-    Vector<GUI::AutocompleteProvider::TokenInfo> m_tokens_info;
+    Vector<CodeComprehension::TokenInfo> m_tokens_info;
     String m_saved_tokens_text;
     Vector<Token> m_saved_tokens;
     Threading::Mutex m_lock;

Userland/Libraries/LibGUI/AutocompleteProvider.cpp

@@ -19,7 +19,7 @@ namespace GUI {
 
 class AutocompleteSuggestionModel final : public GUI::Model {
 public:
-    explicit AutocompleteSuggestionModel(Vector<AutocompleteProvider::Entry>&& suggestions)
+    explicit AutocompleteSuggestionModel(Vector<CodeComprehension::AutocompleteResultEntry>&& suggestions)
         : m_suggestions(move(suggestions))
     {
     }
@@ -50,13 +50,13 @@ public:
                     return suggestion.completion;
             }
             if (index.column() == Column::Icon) {
-                if (suggestion.language == GUI::AutocompleteProvider::Language::Cpp) {
+                if (suggestion.language == CodeComprehension::Language::Cpp) {
                     if (!s_cpp_identifier_icon) {
                         s_cpp_identifier_icon = Gfx::Bitmap::try_load_from_file("/res/icons/16x16/completion/cpp-identifier.png").release_value_but_fixme_should_propagate_errors();
                     }
                     return *s_cpp_identifier_icon;
                 }
-                if (suggestion.language == GUI::AutocompleteProvider::Language::Unspecified) {
+                if (suggestion.language == CodeComprehension::Language::Unspecified) {
                     if (!s_unspecified_identifier_icon) {
                         s_unspecified_identifier_icon = Gfx::Bitmap::try_load_from_file("/res/icons/16x16/completion/unspecified-identifier.png").release_value_but_fixme_should_propagate_errors();
                     }
@@ -73,15 +73,15 @@ public:
             return suggestion.completion;
 
         if ((int)role == InternalRole::HideAutocompleteAfterApplying)
-            return suggestion.hide_autocomplete_after_applying == AutocompleteProvider::Entry::HideAutocompleteAfterApplying::Yes;
+            return suggestion.hide_autocomplete_after_applying == CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::Yes;
 
         return {};
     }
 
-    void set_suggestions(Vector<AutocompleteProvider::Entry>&& suggestions) { m_suggestions = move(suggestions); }
+    void set_suggestions(Vector<CodeComprehension::AutocompleteResultEntry>&& suggestions) { m_suggestions = move(suggestions); }
 
 private:
-    Vector<AutocompleteProvider::Entry> m_suggestions;
+    Vector<CodeComprehension::AutocompleteResultEntry> m_suggestions;
 };
 
 AutocompleteBox::AutocompleteBox(TextEditor& editor)
@@ -109,7 +109,7 @@ AutocompleteBox::AutocompleteBox(TextEditor& editor)
     m_no_suggestions_view = main_widget.add<GUI::Label>("No suggestions");
 }
 
-void AutocompleteBox::update_suggestions(Vector<AutocompleteProvider::Entry>&& suggestions)
+void AutocompleteBox::update_suggestions(Vector<CodeComprehension::AutocompleteResultEntry>&& suggestions)
 {
     // FIXME: There's a potential race here if, after the user selected an autocomplete suggestion,
     // the LanguageServer sends an update and this function is executed before AutocompleteBox::apply_suggestion()
@@ -182,9 +182,9 @@ void AutocompleteBox::previous_suggestion()
     }
 }
 
-AutocompleteProvider::Entry::HideAutocompleteAfterApplying AutocompleteBox::apply_suggestion()
+CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying AutocompleteBox::apply_suggestion()
 {
-    auto hide_when_done = AutocompleteProvider::Entry::HideAutocompleteAfterApplying::Yes;
+    auto hide_when_done = CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::Yes;
 
     if (m_editor.is_null())
         return hide_when_done;
@@ -202,7 +202,7 @@ AutocompleteProvider::Entry::HideAutocompleteAfterApplying AutocompleteBox::appl
     auto hide_after_applying = suggestion_index.data((GUI::ModelRole)AutocompleteSuggestionModel::InternalRole::HideAutocompleteAfterApplying).to_bool();
 
     if (!hide_after_applying)
-        hide_when_done = AutocompleteProvider::Entry::HideAutocompleteAfterApplying::No;
+        hide_when_done = CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::No;
 
     VERIFY(completion.length() >= partial_length);
     if (!m_editor->has_selection()) {
@@ -219,14 +219,4 @@ AutocompleteProvider::Entry::HideAutocompleteAfterApplying AutocompleteBox::appl
     return hide_when_done;
 }
 
-bool AutocompleteProvider::Declaration::operator==(AutocompleteProvider::Declaration const& other) const
-{
-    return name == other.name && position == other.position && type == other.type && scope == other.scope;
-}
-
-bool AutocompleteProvider::ProjectLocation::operator==(ProjectLocation const& other) const
-{
-    return file == other.file && line == other.line && column == other.column;
-}
-
 }

Userland/Libraries/LibGUI/AutocompleteProvider.h

@@ -6,6 +6,7 @@
 
 #pragma once
 
+#include <LibCodeComprehension/Types.h>
 #include <LibGUI/Forward.h>
 #include <LibGUI/Label.h>
 #include <LibGUI/TableView.h>
@@ -22,98 +23,7 @@ class AutocompleteProvider {
 public:
     virtual ~AutocompleteProvider() = default;
 
-    enum class Language {
-        Unspecified,
-        Cpp,
-    };
-
-    struct Entry {
-        String completion;
-        size_t partial_input_length { 0 };
-        Language language { Language::Unspecified };
-        String display_text {};
-
-        enum class HideAutocompleteAfterApplying {
-            No,
-            Yes,
-        };
-        HideAutocompleteAfterApplying hide_autocomplete_after_applying { HideAutocompleteAfterApplying::Yes };
-    };
-
-    struct ProjectLocation {
-        String file;
-        size_t line { 0 };
-        size_t column { 0 };
-
-        bool operator==(ProjectLocation const&) const;
-    };
-
-    enum class DeclarationType {
-        Function,
-        Struct,
-        Class,
-        Variable,
-        PreprocessorDefinition,
-        Namespace,
-        Member,
-    };
-
-    struct Declaration {
-        String name;
-        ProjectLocation position;
-        DeclarationType type;
-        String scope;
-
-        bool operator==(Declaration const&) const;
-    };
-
-    virtual void provide_completions(Function<void(Vector<Entry>)>) = 0;
-
-#define FOR_EACH_SEMANTIC_TYPE        \
-    __SEMANTIC(Unknown)               \
-    __SEMANTIC(Regular)               \
-    __SEMANTIC(Keyword)               \
-    __SEMANTIC(Type)                  \
-    __SEMANTIC(Identifier)            \
-    __SEMANTIC(String)                \
-    __SEMANTIC(Number)                \
-    __SEMANTIC(IncludePath)           \
-    __SEMANTIC(PreprocessorStatement) \
-    __SEMANTIC(Comment)               \
-    __SEMANTIC(Whitespace)            \
-    __SEMANTIC(Function)              \
-    __SEMANTIC(Variable)              \
-    __SEMANTIC(CustomType)            \
-    __SEMANTIC(Namespace)             \
-    __SEMANTIC(Member)                \
-    __SEMANTIC(Parameter)             \
-    __SEMANTIC(PreprocessorMacro)
-
-    struct TokenInfo {
-
-        enum class SemanticType : u32 {
-#define __SEMANTIC(x) x,
-            FOR_EACH_SEMANTIC_TYPE
-#undef __SEMANTIC
-
-        } type { SemanticType::Unknown };
-        size_t start_line { 0 };
-        size_t start_column { 0 };
-        size_t end_line { 0 };
-        size_t end_column { 0 };
-
-        static constexpr char const* type_to_string(SemanticType t)
-        {
-            switch (t) {
-#define __SEMANTIC(x)     \
-    case SemanticType::x: \
-        return #x;
-                FOR_EACH_SEMANTIC_TYPE
-#undef __SEMANTIC
-            }
-            VERIFY_NOT_REACHED();
-        };
-    };
+    virtual void provide_completions(Function<void(Vector<CodeComprehension::AutocompleteResultEntry>)>) = 0;
 
     void attach(TextEditor& editor)
     {
@@ -133,7 +43,7 @@ public:
     explicit AutocompleteBox(TextEditor&);
     ~AutocompleteBox() = default;
 
-    void update_suggestions(Vector<AutocompleteProvider::Entry>&& suggestions);
+    void update_suggestions(Vector<CodeComprehension::AutocompleteResultEntry>&& suggestions);
     bool is_visible() const;
     void show(Gfx::IntPoint suggestion_box_location);
     void close();
@@ -141,7 +51,7 @@ public:
     bool has_suggestions() { return m_suggestion_view->model()->row_count() > 0; }
     void next_suggestion();
     void previous_suggestion();
-    AutocompleteProvider::Entry::HideAutocompleteAfterApplying apply_suggestion();
+    CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying apply_suggestion();
 
 private:
     WeakPtr<TextEditor> m_editor;
@@ -149,5 +59,4 @@ private:
     RefPtr<GUI::TableView> m_suggestion_view;
     RefPtr<GUI::Label> m_no_suggestions_view;
 };
-
 }

Userland/Libraries/LibGUI/GML/AutocompleteProvider.cpp

@@ -11,7 +11,7 @@
 
 namespace GUI::GML {
 
-void AutocompleteProvider::provide_completions(Function<void(Vector<Entry>)> callback)
+void AutocompleteProvider::provide_completions(Function<void(Vector<CodeComprehension::AutocompleteResultEntry>)> callback)
 {
     auto cursor = m_editor->cursor();
     auto text = m_editor->text();
@@ -121,7 +121,7 @@ void AutocompleteProvider::provide_completions(Function<void(Vector<Entry>)> cal
         return fuzzy_str_builder.build();
     };
 
-    Vector<AutocompleteProvider::Entry> class_entries, identifier_entries;
+    Vector<CodeComprehension::AutocompleteResultEntry> class_entries, identifier_entries;
 
     auto register_layouts_matching_pattern = [&](String pattern, size_t partial_input_length) {
         Core::ObjectClassRegistration::for_each([&](const Core::ObjectClassRegistration& registration) {
@@ -146,15 +146,15 @@ void AutocompleteProvider::provide_completions(Function<void(Vector<Entry>)> cal
             if (auto instance = registration->construct()) {
                 for (auto& it : instance->properties()) {
                     if (!it.value->is_readonly() && it.key.matches(pattern))
-                        identifier_entries.empend(String::formatted("{}: ", it.key), partial_input_length, Language::Unspecified, it.key);
+                        identifier_entries.empend(String::formatted("{}: ", it.key), partial_input_length, CodeComprehension::Language::Unspecified, it.key);
                 }
             }
         }
 
         if (can_have_declared_layout(class_names.last()) && "layout"sv.matches(pattern))
-            identifier_entries.empend("layout: ", partial_input_length, Language::Unspecified, "layout", AutocompleteProvider::Entry::HideAutocompleteAfterApplying::No);
+            identifier_entries.empend("layout: ", partial_input_length, CodeComprehension::Language::Unspecified, "layout", CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::No);
         if (class_names.last() == "GUI::ScrollableContainerWidget" && "content_widget"sv.matches(pattern))
-            identifier_entries.empend("content_widget: ", partial_input_length, Language::Unspecified, "content_widget", AutocompleteProvider::Entry::HideAutocompleteAfterApplying::No);
+            identifier_entries.empend("content_widget: ", partial_input_length, CodeComprehension::Language::Unspecified, "content_widget", CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::No);
     };
 
     auto register_properties_and_widgets_matching_pattern = [&](String pattern, size_t partial_input_length) {
@@ -235,7 +235,7 @@ void AutocompleteProvider::provide_completions(Function<void(Vector<Entry>)> cal
     quick_sort(class_entries, [](auto& a, auto& b) { return a.completion < b.completion; });
     quick_sort(identifier_entries, [](auto& a, auto& b) { return a.completion < b.completion; });
 
-    Vector<GUI::AutocompleteProvider::Entry> entries;
+    Vector<CodeComprehension::AutocompleteResultEntry> entries;
     entries.extend(move(identifier_entries));
     entries.extend(move(class_entries));
 

Userland/Libraries/LibGUI/GML/AutocompleteProvider.h

@@ -22,7 +22,7 @@ private:
         return class_name.is_one_of("GUI::Widget", "GUI::Frame");
     }
 
-    virtual void provide_completions(Function<void(Vector<Entry>)> callback) override;
+    virtual void provide_completions(Function<void(Vector<CodeComprehension::AutocompleteResultEntry>)> callback) override;
 };
 
 }

Userland/Libraries/LibGUI/TextEditor.cpp

@@ -757,7 +757,7 @@ void TextEditor::keydown_event(KeyEvent& event)
 {
     if (m_autocomplete_box && m_autocomplete_box->is_visible() && (event.key() == KeyCode::Key_Return || event.key() == KeyCode::Key_Tab)) {
         TemporaryChange change { m_should_keep_autocomplete_box, true };
-        if (m_autocomplete_box->apply_suggestion() == AutocompleteProvider::Entry::HideAutocompleteAfterApplying::Yes)
+        if (m_autocomplete_box->apply_suggestion() == CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::Yes)
             hide_autocomplete();
         else
             try_update_autocomplete();