LanguageServers/Cpp: ParserAutoComplete engine inspects header files

... and performs preprocessing on the source code before parsing.

To support this, we are now able to keep track of multiple
files in the autocomplete engine. We re-parse a file whenever it is
edited.

Base/home/anon/Source/little/other.cpp

@@ -1,12 +1,13 @@
-#include <stdio.h>
 #include "other.h"
+#include <stdio.h>
 
 int func()
 {
     int x = 1;
     int y = 2;
+    StructInHeader mystruct;
     printf("x: %d\n", x);
     printf("y: %d\n", y);
-    printf("x+y: %d\n", x+y);
+    printf("x+y: %d\n", x + y);
     return x + y;
 }

Base/home/anon/Source/little/other.h

@@ -1 +1,12 @@
 int func();
+
+#define USE_VAR2
+
+struct StructInHeader {
+    int var1;
+#ifdef USE_VAR2
+    int var2;
+#else
+    int var3;
+#endif
+};

Userland/DevTools/HackStudio/LanguageServers/Cpp/ParserAutoComplete.cpp

@@ -25,13 +25,15 @@
  */
 
 #include "ParserAutoComplete.h"
-#include "AK/Assertions.h"
+#include <AK/Assertions.h>
-#include "LibCpp/AST.h"
-#include "LibCpp/Lexer.h"
 #include <AK/HashTable.h>
+#include <LibCpp/AST.h>
+#include <LibCpp/Lexer.h>
 #include <LibCpp/Parser.h>
+#include <LibCpp/Preprocessor.h>
+#include <LibRegex/Regex.h>
 
-// #define DEBUG_AUTOCOMPLETE
+// #define AUTOCOMPLETE_VERBOSE
 
 #ifdef AUTOCOMPLETE_VERBOSE
 #    define VERBOSE(fmt, ...) dbgln(fmt, ##__VA_ARGS__)
@@ -43,46 +45,86 @@
 
 namespace LanguageServers::Cpp {
 
-ParserAutoComplete::ParserAutoComplete(const String& code)
+ParserAutoComplete::ParserAutoComplete(const FileDB& filedb)
-    : m_code(code)
+    : AutoCompleteEngine(filedb)
-    , m_parser(code.view())
 {
+}
 
-    auto root = m_parser.parse();
+const ParserAutoComplete::DocumentData& ParserAutoComplete::get_or_create_document_data(const String& file)
+{
+    auto absolute_path = filedb().to_absolute_path(file);
+    if (!m_documents.contains(absolute_path)) {
+        set_document_data(absolute_path, create_document_data_for(absolute_path));
+    }
+    return get_document_data(absolute_path);
+}
+
+const ParserAutoComplete::DocumentData& ParserAutoComplete::get_document_data(const String& file) const
+{
+    auto absolute_path = filedb().to_absolute_path(file);
+    auto document_data = m_documents.get(absolute_path);
+    ASSERT(document_data.has_value());
+    return *document_data.value();
+}
+
+OwnPtr<ParserAutoComplete::DocumentData> ParserAutoComplete::create_document_data_for(const String& file)
+{
+    auto document = filedb().get(file);
+    ASSERT(document);
+    auto content = document->text();
+    auto document_data = make<DocumentData>(document->text());
+    auto root = document_data->parser.parse();
+    for (auto& path : document_data->preprocessor.included_paths()) {
+        get_or_create_document_data(document_path_from_include_path(path));
+    }
 #ifdef DEBUG_AUTOCOMPLETE
     root->dump(0);
 #endif
+    return move(document_data);
 }
 
-Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::get_suggestions(const GUI::TextPosition& autocomplete_position) const
+void ParserAutoComplete::set_document_data(const String& file, OwnPtr<DocumentData>&& data)
+{
+    m_documents.set(filedb().to_absolute_path(file), move(data));
+}
+
+ParserAutoComplete::DocumentData::DocumentData(String&& _text)
+    : text(move(_text))
+    , preprocessor(text.view())
+    , parser(preprocessor.process().view())
+{
+}
+
+Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::get_suggestions(const String& file, const GUI::TextPosition& autocomplete_position)
 {
     ASSERT(autocomplete_position.column() > 0);
     Cpp::Position position { autocomplete_position.line(), autocomplete_position.column() - 1 };
 
     VERBOSE("ParserAutoComplete position {}:{}", position.line, position.column);
 
-    auto node = m_parser.node_at(position);
+    const auto& document = get_or_create_document_data(file);
+    auto node = document.parser.node_at(position);
     if (!node) {
         VERBOSE("no node at position {}:{}", position.line, position.column);
         return {};
     }
 
     if (!node->is_identifier()) {
-        if (is_empty_property(*node, position)) {
+        if (is_empty_property(document, *node, position)) {
             ASSERT(node->is_member_expression());
-            return autocomplete_property((MemberExpression&)(*node), "");
+            return autocomplete_property(document, (MemberExpression&)(*node), "");
         }
         return {};
     }
 
     if (is_property(*node)) {
-        return autocomplete_property((MemberExpression&)(*node->parent()), m_parser.text_of_node(*node));
+        return autocomplete_property(document, (MemberExpression&)(*node->parent()), document.parser.text_of_node(*node));
     }
 
-    return autocomplete_identifier(*node);
+    return autocomplete_identifier(document, *node);
 }
 
-Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::autocomplete_identifier(const ASTNode& node) const
+Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::autocomplete_identifier(const DocumentData& document, const ASTNode& node) const
 {
     const Cpp::ASTNode* current = &node;
     NonnullRefPtrVector<Cpp::Declaration> available_declarations;
@@ -98,15 +140,12 @@ Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::autocomplete_identi
             available_names.append(name);
     };
     for (auto& decl : available_declarations) {
-        if (decl.start() > node.start())
-            continue;
-
         if (decl.is_variable_or_parameter_declaration()) {
             add_name(((Cpp::VariableOrParameterDeclaration&)decl).m_name);
         }
     }
 
-    auto partial_text = m_parser.text_of_node(node);
+    auto partial_text = document.parser.text_of_node(node);
     Vector<GUI::AutocompleteProvider::Entry> suggestions;
     for (auto& name : available_names) {
         if (name.starts_with(partial_text)) {
@@ -116,18 +155,16 @@ Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::autocomplete_identi
     return suggestions;
 }
 
-Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::autocomplete_property(const MemberExpression& parent, const StringView partial_text) const
+Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::autocomplete_property(const DocumentData& document, const MemberExpression& parent, const StringView partial_text) const
 {
-    auto type = type_of(*parent.m_object);
+    auto type = type_of(document, *parent.m_object);
     if (type.is_null()) {
         VERBOSE("Could not infer type of object");
         return {};
     }
 
-    VERBOSE("type of object: {}", type);
-
     Vector<GUI::AutocompleteProvider::Entry> suggestions;
-    for (auto& prop : properties_of_type(type)) {
+    for (auto& prop : properties_of_type(document, type)) {
         if (prop.name.starts_with(partial_text)) {
             suggestions.append({ prop.name, partial_text.length(), GUI::AutocompleteProvider::CompletionKind::Identifier });
         }
@@ -144,20 +181,20 @@ bool ParserAutoComplete::is_property(const ASTNode& node) const
     return parent.m_property.ptr() == &node;
 }
 
-bool ParserAutoComplete::is_empty_property(const ASTNode& node, const Position& autocomplete_position) const
+bool ParserAutoComplete::is_empty_property(const DocumentData& document, const ASTNode& node, const Position& autocomplete_position) const
 {
     if (!node.is_member_expression())
         return false;
-    auto previous_token = m_parser.token_at(autocomplete_position);
+    auto previous_token = document.parser.token_at(autocomplete_position);
     if (!previous_token.has_value())
         return false;
     return previous_token.value().type() == Token::Type::Dot;
 }
 
-String ParserAutoComplete::type_of_property(const Identifier& identifier) const
+String ParserAutoComplete::type_of_property(const DocumentData& document, const Identifier& identifier) const
 {
     auto& parent = (const MemberExpression&)(*identifier.parent());
-    auto properties = properties_of_type(type_of(*parent.m_object));
+    auto properties = properties_of_type(document, type_of(document, *parent.m_object));
     for (auto& prop : properties) {
         if (prop.name == identifier.m_name)
             return prop.type->m_name;
@@ -182,11 +219,11 @@ String ParserAutoComplete::type_of_variable(const Identifier& identifier) const
     return {};
 }
 
-String ParserAutoComplete::type_of(const Expression& expression) const
+String ParserAutoComplete::type_of(const DocumentData& document, const Expression& expression) const
 {
     if (expression.is_member_expression()) {
         auto& member_expression = (const MemberExpression&)expression;
-        return type_of_property(*member_expression.m_property);
+        return type_of_property(document, *member_expression.m_property);
     }
     if (!expression.is_identifier()) {
         ASSERT_NOT_REACHED(); // TODO
@@ -195,15 +232,16 @@ String ParserAutoComplete::type_of(const Expression& expression) const
     auto& identifier = (const Identifier&)expression;
 
     if (is_property(identifier))
-        return type_of_property(identifier);
+        return type_of_property(document, identifier);
 
     return type_of_variable(identifier);
 }
 
-Vector<ParserAutoComplete::PropertyInfo> ParserAutoComplete::properties_of_type(const String& type) const
+Vector<ParserAutoComplete::PropertyInfo> ParserAutoComplete::properties_of_type(const DocumentData& document, const String& type) const
 {
+    auto declarations = get_declarations_in_outer_scope_including_headers(document);
     Vector<PropertyInfo> properties;
-    for (auto& decl : m_parser.root_node()->declarations()) {
+    for (auto& decl : declarations) {
         if (!decl.is_struct_or_class())
             continue;
         auto& struct_or_class = (StructOrClassDeclaration&)decl;
@@ -215,4 +253,58 @@ Vector<ParserAutoComplete::PropertyInfo> ParserAutoComplete::properties_of_type(
     }
     return properties;
 }
+
+NonnullRefPtrVector<Declaration> ParserAutoComplete::get_declarations_in_outer_scope_including_headers(const DocumentData& document) const
+{
+    NonnullRefPtrVector<Declaration> declarations;
+    for (auto& include : document.preprocessor.included_paths()) {
+        auto included_document = get_document_data(document_path_from_include_path(include));
+        declarations.append(get_declarations_in_outer_scope_including_headers(included_document));
+    }
+    for (auto& decl : document.parser.root_node()->declarations()) {
+        declarations.append(decl);
+    }
+    return declarations;
+}
+
+String ParserAutoComplete::document_path_from_include_path(const StringView& include_path) const
+{
+
+    static Regex<PosixExtended> library_include("<(.+)>");
+    static Regex<PosixExtended> user_defined_include("\"(.+)\"");
+
+    auto document_path_for_library_include = [&](const StringView& include_path) -> String {
+        RegexResult result;
+        if (!library_include.search(include_path, result))
+            return {};
+
+        auto path = result.capture_group_matches.at(0).at(0).view.u8view();
+        return String::formatted("/usr/include/{}", path);
+    };
+
+    auto document_path_for_user_defined_include = [&](const StringView& include_path) -> String {
+        RegexResult result;
+        if (!user_defined_include.search(include_path, result))
+            return {};
+
+        return result.capture_group_matches.at(0).at(0).view.u8view();
+    };
+
+    auto result = document_path_for_library_include(include_path);
+    if (result.is_null())
+        result = document_path_for_user_defined_include(include_path);
+
+    return result;
+}
+
+void ParserAutoComplete::on_edit(const String& file)
+{
+    set_document_data(file, create_document_data_for(file));
+}
+
+void ParserAutoComplete::file_opened([[maybe_unused]] const String& file)
+{
+    set_document_data(file, create_document_data_for(file));
+}
+
 }

Userland/DevTools/HackStudio/LanguageServers/Cpp/ParserAutoComplete.h

@@ -26,41 +26,59 @@
 
 #pragma once
 
-#include "LibCpp/AST.h"
+#include "AutoCompleteEngine.h"
+#include "FileDB.h"
 #include <AK/String.h>
 #include <AK/Vector.h>
 #include <DevTools/HackStudio/AutoCompleteResponse.h>
+#include <LibCpp/AST.h>
 #include <LibCpp/Parser.h>
+#include <LibCpp/Preprocessor.h>
 #include <LibGUI/TextPosition.h>
 
 namespace LanguageServers::Cpp {
 
 using namespace ::Cpp;
 
-class ParserAutoComplete {
+class ParserAutoComplete : public AutoCompleteEngine {
 public:
-    ParserAutoComplete(const String& code);
+    ParserAutoComplete(const FileDB& filedb);
 
-    Vector<GUI::AutocompleteProvider::Entry> get_suggestions(const GUI::TextPosition& autocomplete_position) const;
+    virtual Vector<GUI::AutocompleteProvider::Entry> get_suggestions(const String& file, const GUI::TextPosition& autocomplete_position) override;
+    virtual void on_edit(const String& file) override;
+    virtual void file_opened([[maybe_unused]] const String& file) override;
 
 private:
-    Vector<GUI::AutocompleteProvider::Entry> autocomplete_property(const MemberExpression&, const StringView partial_text) const;
+    struct DocumentData {
-    Vector<GUI::AutocompleteProvider::Entry> autocomplete_identifier(const ASTNode&) const;
+        DocumentData(String&& text);
-    String type_of(const Expression&) const;
+        String text;
-    String type_of_property(const Identifier&) const;
+        Preprocessor preprocessor;
+        Parser parser;
+    };
+
+    Vector<GUI::AutocompleteProvider::Entry> autocomplete_property(const DocumentData&, const MemberExpression&, const StringView partial_text) const;
+    Vector<GUI::AutocompleteProvider::Entry> autocomplete_identifier(const DocumentData&, const ASTNode&) const;
+    String type_of(const DocumentData&, const Expression&) const;
+    String type_of_property(const DocumentData&, const Identifier&) const;
     String type_of_variable(const Identifier&) const;
     bool is_property(const ASTNode&) const;
-    bool is_empty_property(const ASTNode&, const Position& autocomplete_position) const;
+    bool is_empty_property(const DocumentData&, const ASTNode&, const Position& autocomplete_position) const;
 
     struct PropertyInfo {
         StringView name;
         RefPtr<Type> type;
     };
-    Vector<PropertyInfo> properties_of_type(const String& type) const;
+    Vector<PropertyInfo> properties_of_type(const DocumentData& document, const String& type) const;
+    NonnullRefPtrVector<Declaration> get_declarations_in_outer_scope_including_headers(const DocumentData& document) const;
 
-private:
+    const DocumentData& get_document_data(const String& file) const;
-    String m_code;
+    const DocumentData& get_or_create_document_data(const String& file);
-    Cpp::Parser m_parser;
+    void set_document_data(const String& file, OwnPtr<DocumentData>&& data);
+
+    OwnPtr<DocumentData> create_document_data_for(const String& file);
+    String document_path_from_include_path(const StringView& include_path) const;
+
+    HashMap<String, OwnPtr<DocumentData>> m_documents;
 };
 
 }