LibCpp: Understand preprocessor macro definition and invocation

The preprocessor now understands when a function-like macro is defined,
and can also parse calls to such macros.

The actual evaluation of function-like macros will be done in a
separate commit.

Userland/Libraries/LibCpp/Preprocessor.cpp

@@ -140,18 +140,9 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
 
     if (keyword == "define") {
         if (m_state == State::Normal) {
-            auto key = line_lexer.consume_until(' ');
-            consume_whitespace(line_lexer);
-
-            DefinedValue value;
-            value.filename = m_filename;
-            value.line = m_line_index;
-
-            auto string_value = line_lexer.consume_all();
-            if (!string_value.is_empty())
-                value.value = string_value;
-
-            m_definitions.set(key, value);
+            auto definition = create_definition(line_lexer.consume_all());
+            if (definition.has_value())
+                m_definitions.set(definition->key, *definition);
         }
         return;
     }
@@ -230,14 +221,15 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
 void Preprocessor::process_line(StringView const& line)
 {
     Lexer line_lexer { line, m_line_index };
+    line_lexer.set_ignore_whitespace(true);
     auto tokens = line_lexer.lex();
 
-    for (auto& token : tokens) {
-        if (token.type() == Token::Type::Whitespace)
-            continue;
+    for (size_t i = 0; i < tokens.size(); ++i) {
+        auto& token = tokens[i];
         if (token.type() == Token::Type::Identifier) {
             if (auto defined_value = m_definitions.find(token.text()); defined_value != m_definitions.end()) {
-                do_substitution(token, defined_value->value);
+                auto last_substituted_token_index = do_substitution(tokens, i, defined_value->value);
+                i = last_substituted_token_index;
                 continue;
             }
         }
@@ -245,21 +237,128 @@ void Preprocessor::process_line(StringView const& line)
     }
 }
 
-void Preprocessor::do_substitution(Token const& replaced_token, DefinedValue const& defined_value)
+size_t Preprocessor::do_substitution(Vector<Token> const& tokens, size_t token_index, Definition const& defined_value)
 {
-    m_substitutions.append({ replaced_token, defined_value });
-
     if (defined_value.value.is_null())
-        return;
+        return token_index;
 
-    Lexer lexer(m_substitutions.last().defined_value.value);
+    Substitution sub;
+    sub.defined_value = defined_value;
+
+    auto macro_call = parse_macro_call(tokens, token_index);
+
+    if (!macro_call.has_value())
+        return token_index;
+
+    // TODO: Evaluate macro call
+    auto processed_value = defined_value.value;
+    Vector<Token> original_tokens;
+    for (size_t i = token_index; i <= macro_call->end_token_index; ++i) {
+        original_tokens.append(tokens[i]);
+    }
+    VERIFY(!original_tokens.is_empty());
+
+    m_substitutions.append({ original_tokens, defined_value, processed_value });
+
+    Lexer lexer(processed_value);
     for (auto& token : lexer.lex()) {
         if (token.type() == Token::Type::Whitespace)
             continue;
-        token.set_start(replaced_token.start());
-        token.set_end(replaced_token.end());
+        token.set_start(original_tokens.first().start());
+        token.set_end(original_tokens.first().end());
         m_tokens.append(token);
     }
+    return macro_call->end_token_index;
+}
+
+Optional<Preprocessor::MacroCall> Preprocessor::parse_macro_call(Vector<Token> const& tokens, size_t token_index)
+{
+    auto name = tokens[token_index];
+    ++token_index;
+
+    if (token_index >= tokens.size() || tokens[token_index].type() != Token::Type::LeftParen)
+        return MacroCall { name, {}, token_index - 1 };
+    ++token_index;
+
+    Vector<MacroCall::Argument> arguments;
+    MacroCall::Argument current_argument;
+
+    size_t paren_depth = 1;
+    for (; token_index < tokens.size(); ++token_index) {
+        auto& token = tokens[token_index];
+        if (token.type() == Token::Type::LeftParen)
+            ++paren_depth;
+        if (token.type() == Token::Type::RightParen)
+            --paren_depth;
+
+        if (paren_depth == 0) {
+            arguments.append(move(current_argument));
+            break;
+        }
+
+        if (paren_depth == 1 && token.type() == Token::Type::Comma) {
+            arguments.append(move(current_argument));
+            current_argument = {};
+        } else {
+            current_argument.tokens.append(token);
+        }
+    }
+
+    if (token_index >= tokens.size())
+        return {};
+
+    return MacroCall { name, move(arguments), token_index };
+}
+
+Optional<Preprocessor::Definition> Preprocessor::create_definition(StringView line)
+{
+    Lexer lexer { line };
+    lexer.set_ignore_whitespace(true);
+    auto tokens = lexer.lex();
+    if (tokens.is_empty())
+        return {};
+
+    if (tokens.first().type() != Token::Type::Identifier)
+        return {};
+
+    Definition definition;
+    definition.filename = m_filename;
+    definition.line = m_line_index;
+
+    definition.key = tokens.first().text();
+
+    if (tokens.size() == 1)
+        return definition;
+
+    size_t token_index = 1;
+    // Parse macro parameters (if any)
+    if (tokens[token_index].type() == Token::Type::LeftParen) {
+        ++token_index;
+        while (token_index < tokens.size() && tokens[token_index].type() != Token::Type::RightParen) {
+            auto param = tokens[token_index];
+            if (param.type() != Token::Type::Identifier)
+                return {};
+
+            if (token_index + 1 >= tokens.size())
+                return {};
+
+            ++token_index;
+
+            if (tokens[token_index].type() == Token::Type::Comma)
+                ++token_index;
+            else if (tokens[token_index].type() != Token::Type::RightParen)
+                return {};
+
+            definition.parameters.empend(param.text());
+        }
+        if (token_index >= tokens.size())
+            return {};
+        ++token_index;
+    }
+
+    definition.value = line.substring_view(tokens[token_index].start().column);
+
+    return definition;
 }
 
 };