LibCpp: Do macro substitution in the preprocessor instead of the parser

After this change, the parser is completely separated from preprocessor
concepts.

Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp

@@ -163,7 +163,7 @@ Vector<GUI::AutocompleteProvider::Entry> CppComprehensionEngine::autocomplete_na
     }
 
     if (reference_scope.is_empty()) {
-        for (auto& preprocessor_name : document.parser().preprocessor_definitions().keys()) {
+        for (auto& preprocessor_name : document.preprocessor().definitions().keys()) {
             if (preprocessor_name.starts_with(partial_text)) {
                 suggestions.append({ preprocessor_name.to_string(), partial_text.length(), GUI::AutocompleteProvider::CompletionKind::PreprocessorDefinition });
             }
@@ -412,13 +412,13 @@ Optional<GUI::AutocompleteProvider::ProjectLocation> CppComprehensionEngine::fin
     Position cpp_position { text_position.line(), text_position.column() };
 
     // Search for a replaced preprocessor token that intersects with text_position
-    for (auto& replaced_token : document.parser().replaced_preprocessor_tokens()) {
+    for (auto& substitution : document.preprocessor().substitutions()) {
-        if (replaced_token.token.start() > cpp_position)
+        if (substitution.original_token.start() > cpp_position)
             continue;
-        if (replaced_token.token.end() < cpp_position)
+        if (substitution.original_token.end() < cpp_position)
             continue;
 
-        return GUI::AutocompleteProvider::ProjectLocation { replaced_token.preprocessor_value.filename, replaced_token.preprocessor_value.line, replaced_token.preprocessor_value.column };
+        return GUI::AutocompleteProvider::ProjectLocation { substitution.defined_value.filename, substitution.defined_value.line, substitution.defined_value.column };
     }
     return {};
 }
@@ -592,7 +592,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
             document_data->m_available_headers.set(header);
     }
 
-    document_data->m_parser = make<Parser>(move(tokens), filename, document_data->preprocessor().definitions());
+    document_data->m_parser = make<Parser>(move(tokens), filename);
 
     auto root = document_data->parser().parse();
 

Userland/Libraries/LibCpp/Parser.cpp

@@ -15,11 +15,10 @@
 
 namespace Cpp {
 
-Parser::Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions const& definitions)
+Parser::Parser(Vector<Token> tokens, String const& filename)
-    : m_preprocessor_definitions(move(definitions))
+    : m_filename(filename)
-    , m_filename(filename)
+    , m_tokens(move(tokens))
 {
-    initialize_program_tokens(tokens);
     if constexpr (CPP_DEBUG) {
         dbgln("Tokens:");
         for (size_t i = 0; i < m_tokens.size(); ++i) {
@@ -28,22 +27,6 @@ Parser::Parser(Vector<Token> const& tokens, const String& filename, Preprocessor
     }
 }
 
-void Parser::initialize_program_tokens(Vector<Token> const& tokens)
-{
-    for (auto& token : tokens) {
-        if (token.type() == Token::Type::Whitespace)
-            continue;
-        if (token.type() == Token::Type::Identifier) {
-            if (auto defined_value = m_preprocessor_definitions.find(text_of_token(token)); defined_value != m_preprocessor_definitions.end()) {
-                add_tokens_for_preprocessor(token, defined_value->value);
-                m_replaced_preprocessor_tokens.append({ token, defined_value->value });
-                continue;
-            }
-        }
-        m_tokens.append(move(token));
-    }
-}
-
 NonnullRefPtr<TranslationUnit> Parser::parse()
 {
     LOG_SCOPE();
@@ -1396,19 +1379,6 @@ bool Parser::match_ellipsis()
         return false;
     return peek().type() == Token::Type::Dot && peek(1).type() == Token::Type::Dot && peek(2).type() == Token::Type::Dot;
 }
-void Parser::add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue& definition)
-{
-    if (!definition.value.has_value())
-        return;
-    Lexer lexer(definition.value.value());
-    for (auto token : lexer.lex()) {
-        if (token.type() == Token::Type::Whitespace)
-            continue;
-        token.set_start(replaced_token.start());
-        token.set_end(replaced_token.end());
-        m_tokens.append(move(token));
-    }
-}
 
 NonnullRefPtr<NamespaceDeclaration> Parser::parse_namespace_declaration(ASTNode& parent, bool is_nested_namespace)
 {

Userland/Libraries/LibCpp/Parser.h

@@ -18,7 +18,7 @@ class Parser final {
     AK_MAKE_NONCOPYABLE(Parser);
 
 public:
-    explicit Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions const& = {});
+    explicit Parser(Vector<Token> tokens, String const& filename);
     ~Parser() = default;
 
     NonnullRefPtr<TranslationUnit> parse();
@@ -33,7 +33,6 @@ public:
     StringView text_of_token(const Cpp::Token& token) const;
     void print_tokens() const;
     const Vector<String>& errors() const { return m_errors; }
-    const Preprocessor::Definitions& preprocessor_definitions() const { return m_preprocessor_definitions; }
 
     struct TodoEntry {
         String content;
@@ -43,11 +42,6 @@ public:
     };
     Vector<TodoEntry> get_todo_entries() const;
 
-    struct TokenAndPreprocessorDefinition {
-        Token token;
-        Preprocessor::DefinedValue preprocessor_value;
-    };
-    const Vector<TokenAndPreprocessorDefinition>& replaced_preprocessor_tokens() const { return m_replaced_preprocessor_tokens; }
     Vector<Token> tokens_in_range(Position start, Position end) const;
 
 private:
@@ -185,8 +179,6 @@ private:
     void consume_attribute_specification();
     void consume_access_specifier();
     bool match_ellipsis();
-    void initialize_program_tokens(Vector<Token> const& tokens);
-    void add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue&);
     Vector<StringView> parse_type_qualifiers();
     Vector<StringView> parse_function_qualifiers();
 
@@ -196,7 +188,6 @@ private:
     };
     void parse_constructor_or_destructor_impl(FunctionDeclaration&, CtorOrDtor);
 
-    Preprocessor::Definitions m_preprocessor_definitions;
     String m_filename;
     Vector<Token> m_tokens;
     State m_state;
@@ -204,8 +195,6 @@ private:
     RefPtr<TranslationUnit> m_root_node;
     Vector<String> m_errors;
     NonnullRefPtrVector<ASTNode> m_nodes;
-
-    Vector<TokenAndPreprocessorDefinition> m_replaced_preprocessor_tokens;
 };
 
 }

Userland/Libraries/LibCpp/Preprocessor.cpp

@@ -39,7 +39,6 @@ Preprocessor::Preprocessor(const String& filename, const StringView& program)
 
 Vector<Token> Preprocessor::process_and_lex()
 {
-    Vector<Token> all_tokens;
     for (; m_line_index < m_lines.size(); ++m_line_index) {
         auto& line = m_lines[m_line_index];
 
@@ -53,14 +52,11 @@ Vector<Token> Preprocessor::process_and_lex()
         }
 
         if (include_in_processed_text) {
-            for (auto& token : process_line(line)) {
+            process_line(line);
-                if (token.type() != Token::Type::Whitespace)
-                    all_tokens.append(token);
-            }
         }
     }
 
-    return all_tokens;
+    return m_tokens;
 }
 
 static void consume_whitespace(GenericLexer& lexer)
@@ -231,14 +227,39 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
     }
 }
 
-Vector<Token> Preprocessor::process_line(const StringView& line)
+void Preprocessor::process_line(StringView const& line)
 {
     Lexer line_lexer { line, m_line_index };
     auto tokens = line_lexer.lex();
 
-    // TODO: Go over tokens of line, do token substitution
+    for (auto& token : tokens) {
+        if (token.type() == Token::Type::Whitespace)
+            continue;
+        if (token.type() == Token::Type::Identifier) {
+            if (auto defined_value = m_definitions.find(token.text()); defined_value != m_definitions.end()) {
+                do_substitution(token, defined_value->value);
+                continue;
+            }
+        }
+        m_tokens.append(token);
+    }
+}
 
-    return tokens;
+void Preprocessor::do_substitution(Token const& replaced_token, DefinedValue const& defined_value)
+{
+    m_substitutions.append({ replaced_token, defined_value });
+
+    if (defined_value.value.is_null())
+        return;
+
+    Lexer lexer(m_substitutions.last().defined_value.value);
+    for (auto& token : lexer.lex()) {
+        if (token.type() == Token::Type::Whitespace)
+            continue;
+        token.set_start(replaced_token.start());
+        token.set_end(replaced_token.end());
+        m_tokens.append(token);
+    }
 }
 
 };

Userland/Libraries/LibCpp/Preprocessor.h

@@ -25,14 +25,20 @@ public:
     Vector<StringView> included_paths() const { return m_included_paths; }
 
     struct DefinedValue {
-        Optional<StringView> value;
+        String value;
         FlyString filename;
         size_t line { 0 };
         size_t column { 0 };
     };
     using Definitions = HashMap<StringView, DefinedValue>;
 
-    const Definitions& definitions() const { return m_definitions; }
+    struct Substitution {
+        Token original_token;
+        DefinedValue defined_value;
+    };
+
+    Definitions const& definitions() const { return m_definitions; }
+    Vector<Substitution> const& substitutions() const { return m_substitutions; }
 
     void set_ignore_unsupported_keywords(bool ignore) { m_options.ignore_unsupported_keywords = ignore; }
     void set_keep_include_statements(bool keep) { m_options.keep_include_statements = keep; }
@@ -43,12 +49,17 @@ private:
     using PreprocessorKeyword = StringView;
     PreprocessorKeyword handle_preprocessor_line(StringView const&);
     void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer);
-    Vector<Token> process_line(StringView const& line);
+    void process_line(StringView const& line);
+    void do_substitution(Token const& replaced_token, DefinedValue const&);
 
     String m_filename;
     String m_program;
-    Definitions m_definitions;
     Vector<StringView> m_lines;
+
+    Vector<Token> m_tokens;
+    Definitions m_definitions;
+    Vector<Substitution> m_substitutions;
+
     size_t m_line_index { 0 };
     size_t m_current_depth { 0 };
     Vector<size_t> m_depths_of_taken_branches;
@@ -62,7 +73,6 @@ private:
     State m_state { State::Normal };
 
     Vector<StringView> m_included_paths;
-    String m_processed_text;
 
     struct Options {
         bool ignore_unsupported_keywords { false };