LibCpp: Do lexing in the Preprocessor

We now call Preprocessor::process_and_lex() and pass the result to the
parser.

Doing the lexing in the preprocessor will allow us to maintain the
original position information of tokens after substituting definitions.

Tests/LibCpp/test-cpp-parser.cpp

@@ -43,7 +43,8 @@ TEST_CASE(test_regression)
         auto target_ast = read_all(ast_file_path);
 
         StringView source_view(source);
-        ::Cpp::Parser parser(source_view, file_path);
+        Cpp::Preprocessor preprocessor(file_path, source_view);
+        Cpp::Parser parser(preprocessor.process_and_lex(), file_path);
         auto root = parser.parse();
 
         EXPECT(parser.errors().is_empty());

Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp

@@ -569,7 +569,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
     document_data->m_preprocessor = make<Preprocessor>(document_data->m_filename, document_data->text());
     document_data->preprocessor().set_ignore_unsupported_keywords(true);
     document_data->preprocessor().set_keep_include_statements(true);
-    document_data->preprocessor().process();
+    auto tokens = document_data->preprocessor().process_and_lex();
 
     Preprocessor::Definitions preprocessor_definitions;
     for (auto item : document_data->preprocessor().definitions())
@@ -590,7 +590,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
             preprocessor_definitions.set(move(item.key), move(item.value));
     }
 
-    document_data->m_parser = make<Parser>(document_data->preprocessor().processed_text(), filename, move(preprocessor_definitions));
+    document_data->m_parser = make<Parser>(move(tokens), filename, move(preprocessor_definitions));
 
     auto root = document_data->parser().parse();
 

Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests.cpp

@@ -134,14 +134,14 @@ void test_complete_includes()
     add_file(filedb, "sample_header.h");
     CppComprehensionEngine autocomplete(filedb);
 
-    auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 23 });
+    auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 22 });
     if (suggestions.size() != 1)
         FAIL(project include - bad size);
 
     if (suggestions[0].completion != "sample_header.h")
         FAIL("project include - wrong results");
 
-    suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 19 });
+    suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 18 });
     if (suggestions.size() != 1)
         FAIL(global include - bad size);
 

Userland/Libraries/LibCpp/Parser.cpp

@@ -15,11 +15,11 @@
 
 namespace Cpp {
 
-Parser::Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& definitions)
+Parser::Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& definitions)
     : m_preprocessor_definitions(move(definitions))
     , m_filename(filename)
 {
-    initialize_program_tokens(program);
+    initialize_program_tokens(tokens);
     if constexpr (CPP_DEBUG) {
         dbgln("Tokens:");
         for (size_t i = 0; i < m_tokens.size(); ++i) {
@@ -28,10 +28,9 @@ Parser::Parser(const StringView& program, const String& filename, Preprocessor::
     }
 }
 
-void Parser::initialize_program_tokens(const StringView& program)
+void Parser::initialize_program_tokens(Vector<Token> const& tokens)
 {
-    Lexer lexer(program);
+    for (auto& token : tokens) {
-    for (auto& token : lexer.lex()) {
         if (token.type() == Token::Type::Whitespace)
             continue;
         if (token.type() == Token::Type::Identifier) {
@@ -1397,7 +1396,7 @@ bool Parser::match_ellipsis()
         return false;
     return peek().type() == Token::Type::Dot && peek(1).type() == Token::Type::Dot && peek(2).type() == Token::Type::Dot;
 }
-void Parser::add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue& definition)
+void Parser::add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue& definition)
 {
     if (!definition.value.has_value())
         return;

Userland/Libraries/LibCpp/Parser.h

@@ -18,7 +18,7 @@ class Parser final {
     AK_MAKE_NONCOPYABLE(Parser);
 
 public:
-    explicit Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& = {});
+    explicit Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& = {});
     ~Parser() = default;
 
     NonnullRefPtr<TranslationUnit> parse();
@@ -185,8 +185,8 @@ private:
     void consume_attribute_specification();
     void consume_access_specifier();
     bool match_ellipsis();
-    void initialize_program_tokens(const StringView& program);
+    void initialize_program_tokens(Vector<Token> const& tokens);
-    void add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue&);
+    void add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue&);
     Vector<StringView> parse_type_qualifiers();
     Vector<StringView> parse_function_qualifiers();
 

Userland/Libraries/LibCpp/Preprocessor.cpp

@@ -8,6 +8,7 @@
 #include <AK/Assertions.h>
 #include <AK/GenericLexer.h>
 #include <AK/StringBuilder.h>
+#include <LibCpp/Lexer.h>
 #include <ctype.h>
 
 namespace Cpp {
@@ -36,8 +37,9 @@ Preprocessor::Preprocessor(const String& filename, const StringView& program)
     }
 }
 
-const String& Preprocessor::process()
+Vector<Token> Preprocessor::process_and_lex()
 {
+    Vector<Token> all_tokens;
     for (; m_line_index < m_lines.size(); ++m_line_index) {
         auto& line = m_lines[m_line_index];
 
@@ -51,14 +53,14 @@ const String& Preprocessor::process()
         }
 
         if (include_in_processed_text) {
-            m_builder.append(line);
+            for (auto& token : process_line(line)) {
+                if (token.type() != Token::Type::Whitespace)
+                    all_tokens.append(token);
+            }
         }
-
-        m_builder.append("\n");
     }
 
-    m_processed_text = m_builder.to_string();
+    return all_tokens;
-    return m_processed_text;
 }
 
 static void consume_whitespace(GenericLexer& lexer)
@@ -224,10 +226,14 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
     }
 }
 
-const String& Preprocessor::processed_text()
+Vector<Token> Preprocessor::process_line(const StringView& line)
 {
-    VERIFY(!m_processed_text.is_null());
+    Lexer line_lexer { line, m_line_index };
-    return m_processed_text;
+    auto tokens = line_lexer.lex();
+
+    // TODO: Go over tokens of line, do token substitution
+
+    return tokens;
 }
 
 };

Userland/Libraries/LibCpp/Preprocessor.h

@@ -12,6 +12,7 @@
 #include <AK/String.h>
 #include <AK/StringView.h>
 #include <AK/Vector.h>
+#include <LibCpp/Token.h>
 
 namespace Cpp {
 
@@ -19,8 +20,7 @@ class Preprocessor {
 
 public:
     explicit Preprocessor(const String& filename, const StringView& program);
-    const String& process();
+    Vector<Token> process_and_lex();
-    const String& processed_text();
     Vector<StringView> included_paths() const { return m_included_paths; }
 
     struct DefinedValue {
@@ -38,13 +38,13 @@ public:
 
 private:
     using PreprocessorKeyword = StringView;
-    PreprocessorKeyword handle_preprocessor_line(const StringView&);
+    PreprocessorKeyword handle_preprocessor_line(StringView const&);
-    void handle_preprocessor_keyword(const StringView& keyword, GenericLexer& line_lexer);
+    void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer);
+    Vector<Token> process_line(StringView const& line);
 
+    String m_filename;
+    String m_program;
     Definitions m_definitions;
-    const String m_filename;
-    const StringView m_program;
-    StringBuilder m_builder;
     Vector<StringView> m_lines;
     size_t m_line_index { 0 };
     size_t m_current_depth { 0 };

Userland/Utilities/PreprocessorTest.cpp

@@ -16,5 +16,8 @@ int main(int, char**)
     }
     auto content = file->read_all();
     Cpp::Preprocessor cpp("other.h", StringView { content });
-    dbgln("{}", cpp.process());
+    auto tokens = cpp.process_and_lex();
+    for (auto& token : tokens) {
+        dbgln("{}", token.to_string());
+    }
 }

Userland/Utilities/cpp-parser.cpp

@@ -13,9 +13,7 @@ int main(int argc, char** argv)
     Core::ArgsParser args_parser;
     const char* path = nullptr;
     bool tokens_mode = false;
-    bool preprocess_mode = false;
     args_parser.add_option(tokens_mode, "Print Tokens", "tokens", 'T');
-    args_parser.add_option(preprocess_mode, "Print Preprocessed source", "preprocessed", 'P');
     args_parser.add_positional_argument(path, "Cpp File", "cpp-file", Core::ArgsParser::Required::No);
     args_parser.parse(argc, argv);
 
@@ -30,13 +28,9 @@ int main(int argc, char** argv)
     StringView content_view(content);
 
     ::Cpp::Preprocessor processor(path, content_view);
-    auto preprocessed_content = processor.process();
+    auto tokens = processor.process_and_lex();
-    if (preprocess_mode) {
-        outln(preprocessed_content);
-        return 0;
-    }
 
-    ::Cpp::Parser parser(preprocessed_content, path);
+    ::Cpp::Parser parser(tokens, path);
     if (tokens_mode) {
         parser.print_tokens();
         return 0;