mirror of
https://github.com/RGBCube/serenity
synced 2025-07-25 04:07:44 +00:00
LibCpp: Understand preprocessor macro definition and invocation
The preprocessor now understands when a function-like macro is defined, and can also parse calls to such macros. The actual evaluation of function-like macros will be done in a separate commit.
This commit is contained in:
parent
c7d3a7789c
commit
8505fcb8ae
4 changed files with 155 additions and 31 deletions
|
@ -165,7 +165,7 @@ Vector<GUI::AutocompleteProvider::Entry> CppComprehensionEngine::autocomplete_na
|
||||||
if (reference_scope.is_empty()) {
|
if (reference_scope.is_empty()) {
|
||||||
for (auto& preprocessor_name : document.preprocessor().definitions().keys()) {
|
for (auto& preprocessor_name : document.preprocessor().definitions().keys()) {
|
||||||
if (preprocessor_name.starts_with(partial_text)) {
|
if (preprocessor_name.starts_with(partial_text)) {
|
||||||
suggestions.append({ preprocessor_name.to_string(), partial_text.length(), GUI::AutocompleteProvider::CompletionKind::PreprocessorDefinition });
|
suggestions.append({ preprocessor_name, partial_text.length(), GUI::AutocompleteProvider::CompletionKind::PreprocessorDefinition });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -413,9 +413,9 @@ Optional<GUI::AutocompleteProvider::ProjectLocation> CppComprehensionEngine::fin
|
||||||
|
|
||||||
// Search for a replaced preprocessor token that intersects with text_position
|
// Search for a replaced preprocessor token that intersects with text_position
|
||||||
for (auto& substitution : document.preprocessor().substitutions()) {
|
for (auto& substitution : document.preprocessor().substitutions()) {
|
||||||
if (substitution.original_token.start() > cpp_position)
|
if (substitution.original_tokens.first().start() > cpp_position)
|
||||||
continue;
|
continue;
|
||||||
if (substitution.original_token.end() < cpp_position)
|
if (substitution.original_tokens.first().end() < cpp_position)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
return GUI::AutocompleteProvider::ProjectLocation { substitution.defined_value.filename, substitution.defined_value.line, substitution.defined_value.column };
|
return GUI::AutocompleteProvider::ProjectLocation { substitution.defined_value.filename, substitution.defined_value.line, substitution.defined_value.column };
|
||||||
|
|
|
@ -140,18 +140,9 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
|
||||||
|
|
||||||
if (keyword == "define") {
|
if (keyword == "define") {
|
||||||
if (m_state == State::Normal) {
|
if (m_state == State::Normal) {
|
||||||
auto key = line_lexer.consume_until(' ');
|
auto definition = create_definition(line_lexer.consume_all());
|
||||||
consume_whitespace(line_lexer);
|
if (definition.has_value())
|
||||||
|
m_definitions.set(definition->key, *definition);
|
||||||
DefinedValue value;
|
|
||||||
value.filename = m_filename;
|
|
||||||
value.line = m_line_index;
|
|
||||||
|
|
||||||
auto string_value = line_lexer.consume_all();
|
|
||||||
if (!string_value.is_empty())
|
|
||||||
value.value = string_value;
|
|
||||||
|
|
||||||
m_definitions.set(key, value);
|
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -230,14 +221,15 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
|
||||||
void Preprocessor::process_line(StringView const& line)
|
void Preprocessor::process_line(StringView const& line)
|
||||||
{
|
{
|
||||||
Lexer line_lexer { line, m_line_index };
|
Lexer line_lexer { line, m_line_index };
|
||||||
|
line_lexer.set_ignore_whitespace(true);
|
||||||
auto tokens = line_lexer.lex();
|
auto tokens = line_lexer.lex();
|
||||||
|
|
||||||
for (auto& token : tokens) {
|
for (size_t i = 0; i < tokens.size(); ++i) {
|
||||||
if (token.type() == Token::Type::Whitespace)
|
auto& token = tokens[i];
|
||||||
continue;
|
|
||||||
if (token.type() == Token::Type::Identifier) {
|
if (token.type() == Token::Type::Identifier) {
|
||||||
if (auto defined_value = m_definitions.find(token.text()); defined_value != m_definitions.end()) {
|
if (auto defined_value = m_definitions.find(token.text()); defined_value != m_definitions.end()) {
|
||||||
do_substitution(token, defined_value->value);
|
auto last_substituted_token_index = do_substitution(tokens, i, defined_value->value);
|
||||||
|
i = last_substituted_token_index;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -245,21 +237,128 @@ void Preprocessor::process_line(StringView const& line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Preprocessor::do_substitution(Token const& replaced_token, DefinedValue const& defined_value)
|
size_t Preprocessor::do_substitution(Vector<Token> const& tokens, size_t token_index, Definition const& defined_value)
|
||||||
{
|
{
|
||||||
m_substitutions.append({ replaced_token, defined_value });
|
|
||||||
|
|
||||||
if (defined_value.value.is_null())
|
if (defined_value.value.is_null())
|
||||||
return;
|
return token_index;
|
||||||
|
|
||||||
Lexer lexer(m_substitutions.last().defined_value.value);
|
Substitution sub;
|
||||||
|
sub.defined_value = defined_value;
|
||||||
|
|
||||||
|
auto macro_call = parse_macro_call(tokens, token_index);
|
||||||
|
|
||||||
|
if (!macro_call.has_value())
|
||||||
|
return token_index;
|
||||||
|
|
||||||
|
// TODO: Evaluate macro call
|
||||||
|
auto processed_value = defined_value.value;
|
||||||
|
Vector<Token> original_tokens;
|
||||||
|
for (size_t i = token_index; i <= macro_call->end_token_index; ++i) {
|
||||||
|
original_tokens.append(tokens[i]);
|
||||||
|
}
|
||||||
|
VERIFY(!original_tokens.is_empty());
|
||||||
|
|
||||||
|
m_substitutions.append({ original_tokens, defined_value, processed_value });
|
||||||
|
|
||||||
|
Lexer lexer(processed_value);
|
||||||
for (auto& token : lexer.lex()) {
|
for (auto& token : lexer.lex()) {
|
||||||
if (token.type() == Token::Type::Whitespace)
|
if (token.type() == Token::Type::Whitespace)
|
||||||
continue;
|
continue;
|
||||||
token.set_start(replaced_token.start());
|
token.set_start(original_tokens.first().start());
|
||||||
token.set_end(replaced_token.end());
|
token.set_end(original_tokens.first().end());
|
||||||
m_tokens.append(token);
|
m_tokens.append(token);
|
||||||
}
|
}
|
||||||
|
return macro_call->end_token_index;
|
||||||
|
}
|
||||||
|
|
||||||
|
Optional<Preprocessor::MacroCall> Preprocessor::parse_macro_call(Vector<Token> const& tokens, size_t token_index)
|
||||||
|
{
|
||||||
|
auto name = tokens[token_index];
|
||||||
|
++token_index;
|
||||||
|
|
||||||
|
if (token_index >= tokens.size() || tokens[token_index].type() != Token::Type::LeftParen)
|
||||||
|
return MacroCall { name, {}, token_index - 1 };
|
||||||
|
++token_index;
|
||||||
|
|
||||||
|
Vector<MacroCall::Argument> arguments;
|
||||||
|
MacroCall::Argument current_argument;
|
||||||
|
|
||||||
|
size_t paren_depth = 1;
|
||||||
|
for (; token_index < tokens.size(); ++token_index) {
|
||||||
|
auto& token = tokens[token_index];
|
||||||
|
if (token.type() == Token::Type::LeftParen)
|
||||||
|
++paren_depth;
|
||||||
|
if (token.type() == Token::Type::RightParen)
|
||||||
|
--paren_depth;
|
||||||
|
|
||||||
|
if (paren_depth == 0) {
|
||||||
|
arguments.append(move(current_argument));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (paren_depth == 1 && token.type() == Token::Type::Comma) {
|
||||||
|
arguments.append(move(current_argument));
|
||||||
|
current_argument = {};
|
||||||
|
} else {
|
||||||
|
current_argument.tokens.append(token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (token_index >= tokens.size())
|
||||||
|
return {};
|
||||||
|
|
||||||
|
return MacroCall { name, move(arguments), token_index };
|
||||||
|
}
|
||||||
|
|
||||||
|
Optional<Preprocessor::Definition> Preprocessor::create_definition(StringView line)
|
||||||
|
{
|
||||||
|
Lexer lexer { line };
|
||||||
|
lexer.set_ignore_whitespace(true);
|
||||||
|
auto tokens = lexer.lex();
|
||||||
|
if (tokens.is_empty())
|
||||||
|
return {};
|
||||||
|
|
||||||
|
if (tokens.first().type() != Token::Type::Identifier)
|
||||||
|
return {};
|
||||||
|
|
||||||
|
Definition definition;
|
||||||
|
definition.filename = m_filename;
|
||||||
|
definition.line = m_line_index;
|
||||||
|
|
||||||
|
definition.key = tokens.first().text();
|
||||||
|
|
||||||
|
if (tokens.size() == 1)
|
||||||
|
return definition;
|
||||||
|
|
||||||
|
size_t token_index = 1;
|
||||||
|
// Parse macro parameters (if any)
|
||||||
|
if (tokens[token_index].type() == Token::Type::LeftParen) {
|
||||||
|
++token_index;
|
||||||
|
while (token_index < tokens.size() && tokens[token_index].type() != Token::Type::RightParen) {
|
||||||
|
auto param = tokens[token_index];
|
||||||
|
if (param.type() != Token::Type::Identifier)
|
||||||
|
return {};
|
||||||
|
|
||||||
|
if (token_index + 1 >= tokens.size())
|
||||||
|
return {};
|
||||||
|
|
||||||
|
++token_index;
|
||||||
|
|
||||||
|
if (tokens[token_index].type() == Token::Type::Comma)
|
||||||
|
++token_index;
|
||||||
|
else if (tokens[token_index].type() != Token::Type::RightParen)
|
||||||
|
return {};
|
||||||
|
|
||||||
|
definition.parameters.empend(param.text());
|
||||||
|
}
|
||||||
|
if (token_index >= tokens.size())
|
||||||
|
return {};
|
||||||
|
++token_index;
|
||||||
|
}
|
||||||
|
|
||||||
|
definition.value = line.substring_view(tokens[token_index].start().column);
|
||||||
|
|
||||||
|
return definition;
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
|
@ -24,17 +24,20 @@ public:
|
||||||
Vector<Token> process_and_lex();
|
Vector<Token> process_and_lex();
|
||||||
Vector<StringView> included_paths() const { return m_included_paths; }
|
Vector<StringView> included_paths() const { return m_included_paths; }
|
||||||
|
|
||||||
struct DefinedValue {
|
struct Definition {
|
||||||
|
String key;
|
||||||
|
Vector<String> parameters;
|
||||||
String value;
|
String value;
|
||||||
FlyString filename;
|
FlyString filename;
|
||||||
size_t line { 0 };
|
size_t line { 0 };
|
||||||
size_t column { 0 };
|
size_t column { 0 };
|
||||||
};
|
};
|
||||||
using Definitions = HashMap<StringView, DefinedValue>;
|
using Definitions = HashMap<String, Definition>;
|
||||||
|
|
||||||
struct Substitution {
|
struct Substitution {
|
||||||
Token original_token;
|
Vector<Token> original_tokens;
|
||||||
DefinedValue defined_value;
|
Definition defined_value;
|
||||||
|
String processed_value;
|
||||||
};
|
};
|
||||||
|
|
||||||
Definitions const& definitions() const { return m_definitions; }
|
Definitions const& definitions() const { return m_definitions; }
|
||||||
|
@ -50,7 +53,18 @@ private:
|
||||||
PreprocessorKeyword handle_preprocessor_line(StringView const&);
|
PreprocessorKeyword handle_preprocessor_line(StringView const&);
|
||||||
void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer);
|
void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer);
|
||||||
void process_line(StringView const& line);
|
void process_line(StringView const& line);
|
||||||
void do_substitution(Token const& replaced_token, DefinedValue const&);
|
size_t do_substitution(Vector<Token> const& tokens, size_t token_index, Definition const&);
|
||||||
|
Optional<Definition> create_definition(StringView line);
|
||||||
|
|
||||||
|
struct MacroCall {
|
||||||
|
Token name;
|
||||||
|
struct Argument {
|
||||||
|
Vector<Token> tokens;
|
||||||
|
};
|
||||||
|
Vector<Argument> arguments;
|
||||||
|
size_t end_token_index { 0 };
|
||||||
|
};
|
||||||
|
Optional<MacroCall> parse_macro_call(Vector<Token> const& tokens, size_t token_index);
|
||||||
|
|
||||||
String m_filename;
|
String m_filename;
|
||||||
String m_program;
|
String m_program;
|
||||||
|
|
|
@ -17,6 +17,17 @@ int main(int, char**)
|
||||||
auto content = file->read_all();
|
auto content = file->read_all();
|
||||||
Cpp::Preprocessor cpp("other.h", StringView { content });
|
Cpp::Preprocessor cpp("other.h", StringView { content });
|
||||||
auto tokens = cpp.process_and_lex();
|
auto tokens = cpp.process_and_lex();
|
||||||
|
|
||||||
|
outln("Definitions:");
|
||||||
|
for (auto& definition : cpp.definitions()) {
|
||||||
|
if (definition.value.parameters.is_empty())
|
||||||
|
outln("{}: {}", definition.key, definition.value.value);
|
||||||
|
else
|
||||||
|
outln("{}({}): {}", definition.key, String::join(",", definition.value.parameters), definition.value.value);
|
||||||
|
}
|
||||||
|
|
||||||
|
outln("");
|
||||||
|
|
||||||
for (auto& token : tokens) {
|
for (auto& token : tokens) {
|
||||||
dbgln("{}", token.to_string());
|
dbgln("{}", token.to_string());
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue