LibCpp: Start working on a C preprocessor

We currently handle basic #define statements as well as ifdef and else
branches.

Userland/Libraries/LibCpp/CMakeLists.txt

@@ -2,6 +2,7 @@ set(SOURCES
     AST.cpp
     Lexer.cpp
     Parser.cpp
+    Preprocessor.cpp
     SyntaxHighlighter.cpp
 )
 

Userland/Libraries/LibCpp/Preprocessor.cpp

@@ -0,0 +1,170 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "Preprocessor.h"
+#include <AK/Assertions.h>
+#include <AK/GenericLexer.h>
+#include <AK/StringBuilder.h>
+#include <ctype.h>
+
+namespace Cpp {
+Preprocessor::Preprocessor(const StringView& program)
+    : m_program(program)
+{
+    m_lines = m_program.split_view('\n', true);
+}
+
+const String& Preprocessor::process()
+{
+    for (; m_line_index < m_lines.size(); ++m_line_index) {
+        auto& line = m_lines[m_line_index];
+        if (line.starts_with("#")) {
+            handle_preprocessor_line(line);
+        } else if (m_state == State::Normal) {
+            m_builder.append(line);
+        }
+        m_builder.append("\n");
+    }
+
+    m_processed_text = m_builder.to_string();
+    return m_processed_text;
+}
+
+void Preprocessor::handle_preprocessor_line(const StringView& line)
+{
+    GenericLexer lexer(line);
+
+    auto consume_whitespace = [&] {
+        lexer.ignore_while([](char ch) { return isspace(ch); });
+        if (lexer.peek() == '/' && lexer.peek(1) == '/')
+            lexer.ignore_until([](char ch) { return ch == '\n'; });
+    };
+
+    consume_whitespace();
+    lexer.consume_specific('#');
+    consume_whitespace();
+    auto keyword = lexer.consume_until(' ');
+
+    if (keyword == "include") {
+        consume_whitespace();
+        m_included_paths.append(lexer.consume_all());
+        return;
+    }
+
+    if (keyword == "else") {
+        ASSERT(m_current_depth > 0);
+        if (m_depths_of_not_taken_branches.contains_slow(m_current_depth - 1)) {
+            m_depths_of_not_taken_branches.remove_all_matching([this](auto x) { return x == m_current_depth - 1; });
+            m_state = State::Normal;
+        }
+        if (m_depths_of_taken_branches.contains_slow(m_current_depth - 1)) {
+            m_depths_of_taken_branches.contains_slow(m_current_depth - 1);
+            m_state = State::SkipElseBranch;
+        }
+        return;
+    }
+
+    if (keyword == "endif") {
+        ASSERT(m_current_depth > 0);
+        --m_current_depth;
+        if (m_depths_of_not_taken_branches.contains_slow(m_current_depth)) {
+            m_depths_of_not_taken_branches.remove_all_matching([this](auto x) { return x == m_current_depth; });
+        }
+        if (m_depths_of_taken_branches.contains_slow(m_current_depth)) {
+            m_depths_of_taken_branches.contains_slow(m_current_depth);
+        }
+        m_state = State::Normal;
+        return;
+    }
+
+    if (keyword == "define") {
+        if (m_state == State::Normal) {
+            auto key = lexer.consume_until(' ');
+            consume_whitespace();
+            DefinedValue value;
+            auto string_value = lexer.consume_all();
+            if (!string_value.is_empty())
+                value.value = string_value;
+            m_definitions.set(key, value);
+        }
+        return;
+    }
+    if (keyword == "undef") {
+        if (m_state == State::Normal) {
+            auto key = lexer.consume_until(' ');
+            lexer.consume_all();
+            m_definitions.remove(key);
+        }
+        return;
+    }
+    if (keyword == "ifdef") {
+        ++m_current_depth;
+        if (m_state == State::Normal) {
+            auto key = lexer.consume_until(' ');
+            if (m_definitions.contains(key)) {
+                m_depths_of_taken_branches.append(m_current_depth - 1);
+                return;
+            } else {
+                m_depths_of_not_taken_branches.append(m_current_depth - 1);
+                m_state = State::SkipIfBranch;
+                return;
+            }
+        }
+        return;
+    }
+    if (keyword == "ifndef") {
+        ++m_current_depth;
+        if (m_state == State::Normal) {
+            auto key = lexer.consume_until(' ');
+            if (!m_definitions.contains(key)) {
+                m_depths_of_taken_branches.append(m_current_depth - 1);
+                return;
+            } else {
+                m_depths_of_not_taken_branches.append(m_current_depth - 1);
+                m_state = State::SkipIfBranch;
+                return;
+            }
+        }
+        return;
+    }
+    if (keyword == "if") {
+        ++m_current_depth;
+        if (m_state == State::Normal) {
+            // FIXME: Implement #if logic
+            // We currently always take #if branches.
+            m_depths_of_taken_branches.append(m_current_depth - 1);
+        }
+        return;
+    }
+    if (keyword == "pragma") {
+        lexer.consume_all();
+        return;
+    }
+    dbgln("Unsupported preprocessor keyword: {}", keyword);
+    ASSERT_NOT_REACHED();
+}
+
+};

Userland/Libraries/LibCpp/Preprocessor.h

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <AK/Optional.h>
+#include <AK/String.h>
+#include <AK/StringView.h>
+#include <AK/Vector.h>
+
+namespace Cpp {
+class Preprocessor {
+
+public:
+    explicit Preprocessor(const StringView&);
+    const String& process();
+    const String& processed_text();
+    Vector<StringView> included_paths() const { return m_included_paths; }
+
+private:
+    struct DefinedValue {
+        Optional<StringView> value;
+    };
+
+    void handle_preprocessor_line(const StringView&);
+
+    HashMap<StringView, DefinedValue> m_definitions;
+    const StringView m_program;
+    StringBuilder m_builder;
+    Vector<StringView> m_lines;
+    size_t m_line_index { 0 };
+    size_t m_current_depth { 0 };
+    Vector<size_t> m_depths_of_taken_branches;
+    Vector<size_t> m_depths_of_not_taken_branches;
+
+    enum class State {
+        Normal,
+        SkipIfBranch,
+        SkipElseBranch
+    };
+    State m_state { State::Normal };
+
+    Vector<StringView> m_included_paths;
+    String m_processed_text;
+};
+}

Userland/Utilities/CMakeLists.txt

@@ -49,3 +49,4 @@ target_link_libraries(tt LibPthread)
 target_link_libraries(grep LibRegex)
 target_link_libraries(gunzip LibCompress)
 target_link_libraries(CppParserTest LibCpp LibGUI)
+target_link_libraries(PreprocessorTest LibCpp LibGUI)

Userland/Utilities/PreprocessorTest.cpp

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2021, the SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "AK/Forward.h"
+#include <LibCore/File.h>
+#include <LibCpp/Preprocessor.h>
+
+int main(int, char**)
+{
+    auto file = Core::File::construct("/home/anon/Source/little/other.h");
+    if (!file->open(Core::IODevice::ReadOnly)) {
+        perror("open");
+        exit(1);
+    }
+    auto content = file->read_all();
+    Cpp::Preprocessor cpp(StringView { content });
+    dbgprintf("%s\n", cpp.process().characters());
+}