mirror of
https://github.com/RGBCube/serenity
synced 2025-07-25 03:17:35 +00:00
LibCpp: Do lexing in the Preprocessor
We now call Preprocessor::process_and_lex() and pass the result to the parser. Doing the lexing in the preprocessor will allow us to maintain the original position information of tokens after substituting definitions.
This commit is contained in:
parent
bf7262681e
commit
4673a517f6
9 changed files with 42 additions and 39 deletions
|
@ -43,7 +43,8 @@ TEST_CASE(test_regression)
|
||||||
auto target_ast = read_all(ast_file_path);
|
auto target_ast = read_all(ast_file_path);
|
||||||
|
|
||||||
StringView source_view(source);
|
StringView source_view(source);
|
||||||
::Cpp::Parser parser(source_view, file_path);
|
Cpp::Preprocessor preprocessor(file_path, source_view);
|
||||||
|
Cpp::Parser parser(preprocessor.process_and_lex(), file_path);
|
||||||
auto root = parser.parse();
|
auto root = parser.parse();
|
||||||
|
|
||||||
EXPECT(parser.errors().is_empty());
|
EXPECT(parser.errors().is_empty());
|
||||||
|
|
|
@ -569,7 +569,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
|
||||||
document_data->m_preprocessor = make<Preprocessor>(document_data->m_filename, document_data->text());
|
document_data->m_preprocessor = make<Preprocessor>(document_data->m_filename, document_data->text());
|
||||||
document_data->preprocessor().set_ignore_unsupported_keywords(true);
|
document_data->preprocessor().set_ignore_unsupported_keywords(true);
|
||||||
document_data->preprocessor().set_keep_include_statements(true);
|
document_data->preprocessor().set_keep_include_statements(true);
|
||||||
document_data->preprocessor().process();
|
auto tokens = document_data->preprocessor().process_and_lex();
|
||||||
|
|
||||||
Preprocessor::Definitions preprocessor_definitions;
|
Preprocessor::Definitions preprocessor_definitions;
|
||||||
for (auto item : document_data->preprocessor().definitions())
|
for (auto item : document_data->preprocessor().definitions())
|
||||||
|
@ -590,7 +590,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
|
||||||
preprocessor_definitions.set(move(item.key), move(item.value));
|
preprocessor_definitions.set(move(item.key), move(item.value));
|
||||||
}
|
}
|
||||||
|
|
||||||
document_data->m_parser = make<Parser>(document_data->preprocessor().processed_text(), filename, move(preprocessor_definitions));
|
document_data->m_parser = make<Parser>(move(tokens), filename, move(preprocessor_definitions));
|
||||||
|
|
||||||
auto root = document_data->parser().parse();
|
auto root = document_data->parser().parse();
|
||||||
|
|
||||||
|
|
|
@ -134,14 +134,14 @@ void test_complete_includes()
|
||||||
add_file(filedb, "sample_header.h");
|
add_file(filedb, "sample_header.h");
|
||||||
CppComprehensionEngine autocomplete(filedb);
|
CppComprehensionEngine autocomplete(filedb);
|
||||||
|
|
||||||
auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 23 });
|
auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 22 });
|
||||||
if (suggestions.size() != 1)
|
if (suggestions.size() != 1)
|
||||||
FAIL(project include - bad size);
|
FAIL(project include - bad size);
|
||||||
|
|
||||||
if (suggestions[0].completion != "sample_header.h")
|
if (suggestions[0].completion != "sample_header.h")
|
||||||
FAIL("project include - wrong results");
|
FAIL("project include - wrong results");
|
||||||
|
|
||||||
suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 19 });
|
suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 18 });
|
||||||
if (suggestions.size() != 1)
|
if (suggestions.size() != 1)
|
||||||
FAIL(global include - bad size);
|
FAIL(global include - bad size);
|
||||||
|
|
||||||
|
|
|
@ -15,11 +15,11 @@
|
||||||
|
|
||||||
namespace Cpp {
|
namespace Cpp {
|
||||||
|
|
||||||
Parser::Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& definitions)
|
Parser::Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& definitions)
|
||||||
: m_preprocessor_definitions(move(definitions))
|
: m_preprocessor_definitions(move(definitions))
|
||||||
, m_filename(filename)
|
, m_filename(filename)
|
||||||
{
|
{
|
||||||
initialize_program_tokens(program);
|
initialize_program_tokens(tokens);
|
||||||
if constexpr (CPP_DEBUG) {
|
if constexpr (CPP_DEBUG) {
|
||||||
dbgln("Tokens:");
|
dbgln("Tokens:");
|
||||||
for (size_t i = 0; i < m_tokens.size(); ++i) {
|
for (size_t i = 0; i < m_tokens.size(); ++i) {
|
||||||
|
@ -28,10 +28,9 @@ Parser::Parser(const StringView& program, const String& filename, Preprocessor::
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Parser::initialize_program_tokens(const StringView& program)
|
void Parser::initialize_program_tokens(Vector<Token> const& tokens)
|
||||||
{
|
{
|
||||||
Lexer lexer(program);
|
for (auto& token : tokens) {
|
||||||
for (auto& token : lexer.lex()) {
|
|
||||||
if (token.type() == Token::Type::Whitespace)
|
if (token.type() == Token::Type::Whitespace)
|
||||||
continue;
|
continue;
|
||||||
if (token.type() == Token::Type::Identifier) {
|
if (token.type() == Token::Type::Identifier) {
|
||||||
|
@ -1397,7 +1396,7 @@ bool Parser::match_ellipsis()
|
||||||
return false;
|
return false;
|
||||||
return peek().type() == Token::Type::Dot && peek(1).type() == Token::Type::Dot && peek(2).type() == Token::Type::Dot;
|
return peek().type() == Token::Type::Dot && peek(1).type() == Token::Type::Dot && peek(2).type() == Token::Type::Dot;
|
||||||
}
|
}
|
||||||
void Parser::add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue& definition)
|
void Parser::add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue& definition)
|
||||||
{
|
{
|
||||||
if (!definition.value.has_value())
|
if (!definition.value.has_value())
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -18,7 +18,7 @@ class Parser final {
|
||||||
AK_MAKE_NONCOPYABLE(Parser);
|
AK_MAKE_NONCOPYABLE(Parser);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
explicit Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& = {});
|
explicit Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& = {});
|
||||||
~Parser() = default;
|
~Parser() = default;
|
||||||
|
|
||||||
NonnullRefPtr<TranslationUnit> parse();
|
NonnullRefPtr<TranslationUnit> parse();
|
||||||
|
@ -185,8 +185,8 @@ private:
|
||||||
void consume_attribute_specification();
|
void consume_attribute_specification();
|
||||||
void consume_access_specifier();
|
void consume_access_specifier();
|
||||||
bool match_ellipsis();
|
bool match_ellipsis();
|
||||||
void initialize_program_tokens(const StringView& program);
|
void initialize_program_tokens(Vector<Token> const& tokens);
|
||||||
void add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue&);
|
void add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue&);
|
||||||
Vector<StringView> parse_type_qualifiers();
|
Vector<StringView> parse_type_qualifiers();
|
||||||
Vector<StringView> parse_function_qualifiers();
|
Vector<StringView> parse_function_qualifiers();
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,7 @@
|
||||||
#include <AK/Assertions.h>
|
#include <AK/Assertions.h>
|
||||||
#include <AK/GenericLexer.h>
|
#include <AK/GenericLexer.h>
|
||||||
#include <AK/StringBuilder.h>
|
#include <AK/StringBuilder.h>
|
||||||
|
#include <LibCpp/Lexer.h>
|
||||||
#include <ctype.h>
|
#include <ctype.h>
|
||||||
|
|
||||||
namespace Cpp {
|
namespace Cpp {
|
||||||
|
@ -36,8 +37,9 @@ Preprocessor::Preprocessor(const String& filename, const StringView& program)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const String& Preprocessor::process()
|
Vector<Token> Preprocessor::process_and_lex()
|
||||||
{
|
{
|
||||||
|
Vector<Token> all_tokens;
|
||||||
for (; m_line_index < m_lines.size(); ++m_line_index) {
|
for (; m_line_index < m_lines.size(); ++m_line_index) {
|
||||||
auto& line = m_lines[m_line_index];
|
auto& line = m_lines[m_line_index];
|
||||||
|
|
||||||
|
@ -51,14 +53,14 @@ const String& Preprocessor::process()
|
||||||
}
|
}
|
||||||
|
|
||||||
if (include_in_processed_text) {
|
if (include_in_processed_text) {
|
||||||
m_builder.append(line);
|
for (auto& token : process_line(line)) {
|
||||||
|
if (token.type() != Token::Type::Whitespace)
|
||||||
|
all_tokens.append(token);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
m_builder.append("\n");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
m_processed_text = m_builder.to_string();
|
return all_tokens;
|
||||||
return m_processed_text;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void consume_whitespace(GenericLexer& lexer)
|
static void consume_whitespace(GenericLexer& lexer)
|
||||||
|
@ -224,10 +226,14 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const String& Preprocessor::processed_text()
|
Vector<Token> Preprocessor::process_line(const StringView& line)
|
||||||
{
|
{
|
||||||
VERIFY(!m_processed_text.is_null());
|
Lexer line_lexer { line, m_line_index };
|
||||||
return m_processed_text;
|
auto tokens = line_lexer.lex();
|
||||||
|
|
||||||
|
// TODO: Go over tokens of line, do token substitution
|
||||||
|
|
||||||
|
return tokens;
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
|
@ -12,6 +12,7 @@
|
||||||
#include <AK/String.h>
|
#include <AK/String.h>
|
||||||
#include <AK/StringView.h>
|
#include <AK/StringView.h>
|
||||||
#include <AK/Vector.h>
|
#include <AK/Vector.h>
|
||||||
|
#include <LibCpp/Token.h>
|
||||||
|
|
||||||
namespace Cpp {
|
namespace Cpp {
|
||||||
|
|
||||||
|
@ -19,8 +20,7 @@ class Preprocessor {
|
||||||
|
|
||||||
public:
|
public:
|
||||||
explicit Preprocessor(const String& filename, const StringView& program);
|
explicit Preprocessor(const String& filename, const StringView& program);
|
||||||
const String& process();
|
Vector<Token> process_and_lex();
|
||||||
const String& processed_text();
|
|
||||||
Vector<StringView> included_paths() const { return m_included_paths; }
|
Vector<StringView> included_paths() const { return m_included_paths; }
|
||||||
|
|
||||||
struct DefinedValue {
|
struct DefinedValue {
|
||||||
|
@ -38,13 +38,13 @@ public:
|
||||||
|
|
||||||
private:
|
private:
|
||||||
using PreprocessorKeyword = StringView;
|
using PreprocessorKeyword = StringView;
|
||||||
PreprocessorKeyword handle_preprocessor_line(const StringView&);
|
PreprocessorKeyword handle_preprocessor_line(StringView const&);
|
||||||
void handle_preprocessor_keyword(const StringView& keyword, GenericLexer& line_lexer);
|
void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer);
|
||||||
|
Vector<Token> process_line(StringView const& line);
|
||||||
|
|
||||||
|
String m_filename;
|
||||||
|
String m_program;
|
||||||
Definitions m_definitions;
|
Definitions m_definitions;
|
||||||
const String m_filename;
|
|
||||||
const StringView m_program;
|
|
||||||
StringBuilder m_builder;
|
|
||||||
Vector<StringView> m_lines;
|
Vector<StringView> m_lines;
|
||||||
size_t m_line_index { 0 };
|
size_t m_line_index { 0 };
|
||||||
size_t m_current_depth { 0 };
|
size_t m_current_depth { 0 };
|
||||||
|
|
|
@ -16,5 +16,8 @@ int main(int, char**)
|
||||||
}
|
}
|
||||||
auto content = file->read_all();
|
auto content = file->read_all();
|
||||||
Cpp::Preprocessor cpp("other.h", StringView { content });
|
Cpp::Preprocessor cpp("other.h", StringView { content });
|
||||||
dbgln("{}", cpp.process());
|
auto tokens = cpp.process_and_lex();
|
||||||
|
for (auto& token : tokens) {
|
||||||
|
dbgln("{}", token.to_string());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,9 +13,7 @@ int main(int argc, char** argv)
|
||||||
Core::ArgsParser args_parser;
|
Core::ArgsParser args_parser;
|
||||||
const char* path = nullptr;
|
const char* path = nullptr;
|
||||||
bool tokens_mode = false;
|
bool tokens_mode = false;
|
||||||
bool preprocess_mode = false;
|
|
||||||
args_parser.add_option(tokens_mode, "Print Tokens", "tokens", 'T');
|
args_parser.add_option(tokens_mode, "Print Tokens", "tokens", 'T');
|
||||||
args_parser.add_option(preprocess_mode, "Print Preprocessed source", "preprocessed", 'P');
|
|
||||||
args_parser.add_positional_argument(path, "Cpp File", "cpp-file", Core::ArgsParser::Required::No);
|
args_parser.add_positional_argument(path, "Cpp File", "cpp-file", Core::ArgsParser::Required::No);
|
||||||
args_parser.parse(argc, argv);
|
args_parser.parse(argc, argv);
|
||||||
|
|
||||||
|
@ -30,13 +28,9 @@ int main(int argc, char** argv)
|
||||||
StringView content_view(content);
|
StringView content_view(content);
|
||||||
|
|
||||||
::Cpp::Preprocessor processor(path, content_view);
|
::Cpp::Preprocessor processor(path, content_view);
|
||||||
auto preprocessed_content = processor.process();
|
auto tokens = processor.process_and_lex();
|
||||||
if (preprocess_mode) {
|
|
||||||
outln(preprocessed_content);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
::Cpp::Parser parser(preprocessed_content, path);
|
::Cpp::Parser parser(tokens, path);
|
||||||
if (tokens_mode) {
|
if (tokens_mode) {
|
||||||
parser.print_tokens();
|
parser.print_tokens();
|
||||||
return 0;
|
return 0;
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue