diff --git a/Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp b/Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp index 84c776faff..2f171a3012 100644 --- a/Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp +++ b/Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp @@ -163,7 +163,7 @@ Vector CppComprehensionEngine::autocomplete_na } if (reference_scope.is_empty()) { - for (auto& preprocessor_name : document.parser().preprocessor_definitions().keys()) { + for (auto& preprocessor_name : document.preprocessor().definitions().keys()) { if (preprocessor_name.starts_with(partial_text)) { suggestions.append({ preprocessor_name.to_string(), partial_text.length(), GUI::AutocompleteProvider::CompletionKind::PreprocessorDefinition }); } @@ -412,13 +412,13 @@ Optional CppComprehensionEngine::fin Position cpp_position { text_position.line(), text_position.column() }; // Search for a replaced preprocessor token that intersects with text_position - for (auto& replaced_token : document.parser().replaced_preprocessor_tokens()) { - if (replaced_token.token.start() > cpp_position) + for (auto& substitution : document.preprocessor().substitutions()) { + if (substitution.original_token.start() > cpp_position) continue; - if (replaced_token.token.end() < cpp_position) + if (substitution.original_token.end() < cpp_position) continue; - return GUI::AutocompleteProvider::ProjectLocation { replaced_token.preprocessor_value.filename, replaced_token.preprocessor_value.line, replaced_token.preprocessor_value.column }; + return GUI::AutocompleteProvider::ProjectLocation { substitution.defined_value.filename, substitution.defined_value.line, substitution.defined_value.column }; } return {}; } @@ -592,7 +592,7 @@ OwnPtr CppComprehensionEngine::create_docu document_data->m_available_headers.set(header); } - document_data->m_parser = make(move(tokens), filename, document_data->preprocessor().definitions()); + document_data->m_parser = make(move(tokens), filename); auto root = document_data->parser().parse(); diff --git a/Userland/Libraries/LibCpp/Parser.cpp b/Userland/Libraries/LibCpp/Parser.cpp index 4bcc5ecd67..1e33441044 100644 --- a/Userland/Libraries/LibCpp/Parser.cpp +++ b/Userland/Libraries/LibCpp/Parser.cpp @@ -15,11 +15,10 @@ namespace Cpp { -Parser::Parser(Vector const& tokens, const String& filename, Preprocessor::Definitions const& definitions) - : m_preprocessor_definitions(move(definitions)) - , m_filename(filename) +Parser::Parser(Vector tokens, String const& filename) + : m_filename(filename) + , m_tokens(move(tokens)) { - initialize_program_tokens(tokens); if constexpr (CPP_DEBUG) { dbgln("Tokens:"); for (size_t i = 0; i < m_tokens.size(); ++i) { @@ -28,22 +27,6 @@ Parser::Parser(Vector const& tokens, const String& filename, Preprocessor } } -void Parser::initialize_program_tokens(Vector const& tokens) -{ - for (auto& token : tokens) { - if (token.type() == Token::Type::Whitespace) - continue; - if (token.type() == Token::Type::Identifier) { - if (auto defined_value = m_preprocessor_definitions.find(text_of_token(token)); defined_value != m_preprocessor_definitions.end()) { - add_tokens_for_preprocessor(token, defined_value->value); - m_replaced_preprocessor_tokens.append({ token, defined_value->value }); - continue; - } - } - m_tokens.append(move(token)); - } -} - NonnullRefPtr Parser::parse() { LOG_SCOPE(); @@ -1396,19 +1379,6 @@ bool Parser::match_ellipsis() return false; return peek().type() == Token::Type::Dot && peek(1).type() == Token::Type::Dot && peek(2).type() == Token::Type::Dot; } -void Parser::add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue& definition) -{ - if (!definition.value.has_value()) - return; - Lexer lexer(definition.value.value()); - for (auto token : lexer.lex()) { - if (token.type() == Token::Type::Whitespace) - continue; - token.set_start(replaced_token.start()); - token.set_end(replaced_token.end()); - m_tokens.append(move(token)); - } -} NonnullRefPtr Parser::parse_namespace_declaration(ASTNode& parent, bool is_nested_namespace) { diff --git a/Userland/Libraries/LibCpp/Parser.h b/Userland/Libraries/LibCpp/Parser.h index 34e70bd7a6..4bdf381a0a 100644 --- a/Userland/Libraries/LibCpp/Parser.h +++ b/Userland/Libraries/LibCpp/Parser.h @@ -18,7 +18,7 @@ class Parser final { AK_MAKE_NONCOPYABLE(Parser); public: - explicit Parser(Vector const& tokens, const String& filename, Preprocessor::Definitions const& = {}); + explicit Parser(Vector tokens, String const& filename); ~Parser() = default; NonnullRefPtr parse(); @@ -33,7 +33,6 @@ public: StringView text_of_token(const Cpp::Token& token) const; void print_tokens() const; const Vector& errors() const { return m_errors; } - const Preprocessor::Definitions& preprocessor_definitions() const { return m_preprocessor_definitions; } struct TodoEntry { String content; @@ -43,11 +42,6 @@ public: }; Vector get_todo_entries() const; - struct TokenAndPreprocessorDefinition { - Token token; - Preprocessor::DefinedValue preprocessor_value; - }; - const Vector& replaced_preprocessor_tokens() const { return m_replaced_preprocessor_tokens; } Vector tokens_in_range(Position start, Position end) const; private: @@ -185,8 +179,6 @@ private: void consume_attribute_specification(); void consume_access_specifier(); bool match_ellipsis(); - void initialize_program_tokens(Vector const& tokens); - void add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue&); Vector parse_type_qualifiers(); Vector parse_function_qualifiers(); @@ -196,7 +188,6 @@ private: }; void parse_constructor_or_destructor_impl(FunctionDeclaration&, CtorOrDtor); - Preprocessor::Definitions m_preprocessor_definitions; String m_filename; Vector m_tokens; State m_state; @@ -204,8 +195,6 @@ private: RefPtr m_root_node; Vector m_errors; NonnullRefPtrVector m_nodes; - - Vector m_replaced_preprocessor_tokens; }; } diff --git a/Userland/Libraries/LibCpp/Preprocessor.cpp b/Userland/Libraries/LibCpp/Preprocessor.cpp index cb7be70f7a..ca2a28987b 100644 --- a/Userland/Libraries/LibCpp/Preprocessor.cpp +++ b/Userland/Libraries/LibCpp/Preprocessor.cpp @@ -39,7 +39,6 @@ Preprocessor::Preprocessor(const String& filename, const StringView& program) Vector Preprocessor::process_and_lex() { - Vector all_tokens; for (; m_line_index < m_lines.size(); ++m_line_index) { auto& line = m_lines[m_line_index]; @@ -53,14 +52,11 @@ Vector Preprocessor::process_and_lex() } if (include_in_processed_text) { - for (auto& token : process_line(line)) { - if (token.type() != Token::Type::Whitespace) - all_tokens.append(token); - } + process_line(line); } } - return all_tokens; + return m_tokens; } static void consume_whitespace(GenericLexer& lexer) @@ -231,14 +227,39 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi } } -Vector Preprocessor::process_line(const StringView& line) +void Preprocessor::process_line(StringView const& line) { Lexer line_lexer { line, m_line_index }; auto tokens = line_lexer.lex(); - // TODO: Go over tokens of line, do token substitution + for (auto& token : tokens) { + if (token.type() == Token::Type::Whitespace) + continue; + if (token.type() == Token::Type::Identifier) { + if (auto defined_value = m_definitions.find(token.text()); defined_value != m_definitions.end()) { + do_substitution(token, defined_value->value); + continue; + } + } + m_tokens.append(token); + } +} - return tokens; +void Preprocessor::do_substitution(Token const& replaced_token, DefinedValue const& defined_value) +{ + m_substitutions.append({ replaced_token, defined_value }); + + if (defined_value.value.is_null()) + return; + + Lexer lexer(m_substitutions.last().defined_value.value); + for (auto& token : lexer.lex()) { + if (token.type() == Token::Type::Whitespace) + continue; + token.set_start(replaced_token.start()); + token.set_end(replaced_token.end()); + m_tokens.append(token); + } } }; diff --git a/Userland/Libraries/LibCpp/Preprocessor.h b/Userland/Libraries/LibCpp/Preprocessor.h index 6459a7582c..91410d6590 100644 --- a/Userland/Libraries/LibCpp/Preprocessor.h +++ b/Userland/Libraries/LibCpp/Preprocessor.h @@ -25,14 +25,20 @@ public: Vector included_paths() const { return m_included_paths; } struct DefinedValue { - Optional value; + String value; FlyString filename; size_t line { 0 }; size_t column { 0 }; }; using Definitions = HashMap; - const Definitions& definitions() const { return m_definitions; } + struct Substitution { + Token original_token; + DefinedValue defined_value; + }; + + Definitions const& definitions() const { return m_definitions; } + Vector const& substitutions() const { return m_substitutions; } void set_ignore_unsupported_keywords(bool ignore) { m_options.ignore_unsupported_keywords = ignore; } void set_keep_include_statements(bool keep) { m_options.keep_include_statements = keep; } @@ -43,12 +49,17 @@ private: using PreprocessorKeyword = StringView; PreprocessorKeyword handle_preprocessor_line(StringView const&); void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer); - Vector process_line(StringView const& line); + void process_line(StringView const& line); + void do_substitution(Token const& replaced_token, DefinedValue const&); String m_filename; String m_program; - Definitions m_definitions; Vector m_lines; + + Vector m_tokens; + Definitions m_definitions; + Vector m_substitutions; + size_t m_line_index { 0 }; size_t m_current_depth { 0 }; Vector m_depths_of_taken_branches; @@ -62,7 +73,6 @@ private: State m_state { State::Normal }; Vector m_included_paths; - String m_processed_text; struct Options { bool ignore_unsupported_keywords { false };