Utilities: Add cpp-lexer

Userland/Utilities/CMakeLists.txt

@@ -96,6 +96,7 @@ target_link_libraries(test-pthread LibThreading)
 target_link_libraries(tt LibPthread)
 target_link_libraries(unzip LibArchive LibCompress)
 target_link_libraries(zip LibArchive LibCompress LibCrypto)
+target_link_libraries(cpp-lexer LibCpp)
 target_link_libraries(cpp-parser LibCpp LibGUI)
 target_link_libraries(cpp-preprocessor LibCpp LibGUI)
 target_link_libraries(wasm LibWasm LibLine)

Userland/Utilities/cpp-lexer.cpp

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibCore/ArgsParser.h>
+#include <LibCore/File.h>
+#include <LibCpp/Lexer.h>
+
+int main(int argc, char** argv)
+{
+    Core::ArgsParser args_parser;
+    const char* path = nullptr;
+    args_parser.add_positional_argument(path, "Cpp File", "cpp-file", Core::ArgsParser::Required::Yes);
+    args_parser.parse(argc, argv);
+
+    auto file = Core::File::construct(path);
+    if (!file->open(Core::OpenMode::ReadOnly)) {
+        perror("open");
+        exit(1);
+    }
+    auto content = file->read_all();
+    StringView content_view(content);
+
+    Cpp::Lexer lexer(content);
+    auto tokens = lexer.lex();
+
+    for (auto& token : tokens) {
+        outln("{}", token.to_string());
+    }
+}