From d0b4f9cc0e7110ba540d09c0244ff69aa19ad27f Mon Sep 17 00:00:00 2001 From: Itamar Date: Fri, 12 Mar 2021 12:28:20 +0200 Subject: [PATCH] LibCpp: Move Cpp::Token to a separate file --- Userland/Libraries/LibCpp/CMakeLists.txt | 1 + Userland/Libraries/LibCpp/Lexer.cpp | 13 --- Userland/Libraries/LibCpp/Lexer.h | 110 +----------------- Userland/Libraries/LibCpp/Token.cpp | 44 +++++++ Userland/Libraries/LibCpp/Token.h | 142 +++++++++++++++++++++++ 5 files changed, 188 insertions(+), 122 deletions(-) create mode 100644 Userland/Libraries/LibCpp/Token.cpp create mode 100644 Userland/Libraries/LibCpp/Token.h diff --git a/Userland/Libraries/LibCpp/CMakeLists.txt b/Userland/Libraries/LibCpp/CMakeLists.txt index 25d2c62341..65ebc107f2 100644 --- a/Userland/Libraries/LibCpp/CMakeLists.txt +++ b/Userland/Libraries/LibCpp/CMakeLists.txt @@ -4,6 +4,7 @@ set(SOURCES Parser.cpp Preprocessor.cpp SyntaxHighlighter.cpp + Token.cpp ) serenity_lib(LibCpp cpp) diff --git a/Userland/Libraries/LibCpp/Lexer.cpp b/Userland/Libraries/LibCpp/Lexer.cpp index c08a74bb9e..0ceeb5d97b 100644 --- a/Userland/Libraries/LibCpp/Lexer.cpp +++ b/Userland/Libraries/LibCpp/Lexer.cpp @@ -791,17 +791,4 @@ Vector Lexer::lex() return tokens; } -bool Position::operator<(const Position& other) const -{ - return line < other.line || (line == other.line && column < other.column); -} -bool Position::operator>(const Position& other) const -{ - return !(*this < other) && !(*this == other); -} -bool Position::operator==(const Position& other) const -{ - return line == other.line && column == other.column; -} - } diff --git a/Userland/Libraries/LibCpp/Lexer.h b/Userland/Libraries/LibCpp/Lexer.h index 1fff2d146d..5a2c7f54c6 100644 --- a/Userland/Libraries/LibCpp/Lexer.h +++ b/Userland/Libraries/LibCpp/Lexer.h @@ -26,120 +26,12 @@ #pragma once +#include "LibCpp/Token.h" #include #include namespace Cpp { -#define FOR_EACH_TOKEN_TYPE \ - __TOKEN(Unknown) \ - __TOKEN(Whitespace) \ - __TOKEN(PreprocessorStatement) \ - __TOKEN(IncludeStatement) \ - __TOKEN(IncludePath) \ - __TOKEN(LeftParen) \ - __TOKEN(RightParen) \ - __TOKEN(LeftCurly) \ - __TOKEN(RightCurly) \ - __TOKEN(LeftBracket) \ - __TOKEN(RightBracket) \ - __TOKEN(Less) \ - __TOKEN(Greater) \ - __TOKEN(LessEquals) \ - __TOKEN(GreaterEquals) \ - __TOKEN(LessLess) \ - __TOKEN(GreaterGreater) \ - __TOKEN(LessLessEquals) \ - __TOKEN(GreaterGreaterEquals) \ - __TOKEN(LessGreater) \ - __TOKEN(Comma) \ - __TOKEN(Plus) \ - __TOKEN(PlusPlus) \ - __TOKEN(PlusEquals) \ - __TOKEN(Minus) \ - __TOKEN(MinusMinus) \ - __TOKEN(MinusEquals) \ - __TOKEN(Asterisk) \ - __TOKEN(AsteriskEquals) \ - __TOKEN(Slash) \ - __TOKEN(SlashEquals) \ - __TOKEN(Percent) \ - __TOKEN(PercentEquals) \ - __TOKEN(Caret) \ - __TOKEN(CaretEquals) \ - __TOKEN(ExclamationMark) \ - __TOKEN(ExclamationMarkEquals) \ - __TOKEN(Equals) \ - __TOKEN(EqualsEquals) \ - __TOKEN(And) \ - __TOKEN(AndAnd) \ - __TOKEN(AndEquals) \ - __TOKEN(Pipe) \ - __TOKEN(PipePipe) \ - __TOKEN(PipeEquals) \ - __TOKEN(Tilde) \ - __TOKEN(QuestionMark) \ - __TOKEN(Colon) \ - __TOKEN(ColonColon) \ - __TOKEN(ColonColonAsterisk) \ - __TOKEN(Semicolon) \ - __TOKEN(Dot) \ - __TOKEN(DotAsterisk) \ - __TOKEN(Arrow) \ - __TOKEN(ArrowAsterisk) \ - __TOKEN(DoubleQuotedString) \ - __TOKEN(SingleQuotedString) \ - __TOKEN(RawString) \ - __TOKEN(EscapeSequence) \ - __TOKEN(Comment) \ - __TOKEN(Integer) \ - __TOKEN(Float) \ - __TOKEN(Keyword) \ - __TOKEN(KnownType) \ - __TOKEN(Identifier) \ - __TOKEN(EOF_TOKEN) - -struct Position { - size_t line { 0 }; - size_t column { 0 }; - - bool operator<(const Position&) const; - bool operator>(const Position&) const; - bool operator==(const Position&) const; -}; - -struct Token { - enum class Type { -#define __TOKEN(x) x, - FOR_EACH_TOKEN_TYPE -#undef __TOKEN - }; - - static const char* type_to_string(Type t) - { - switch (t) { -#define __TOKEN(x) \ - case Type::x: \ - return #x; - FOR_EACH_TOKEN_TYPE -#undef __TOKEN - } - VERIFY_NOT_REACHED(); - } - - const char* to_string() const - { - return type_to_string(m_type); - } - Position start() const { return m_start; } - Position end() const { return m_end; } - Type type() const { return m_type; } - - Type m_type { Type::Unknown }; - Position m_start; - Position m_end; -}; - class Lexer { public: Lexer(const StringView&); diff --git a/Userland/Libraries/LibCpp/Token.cpp b/Userland/Libraries/LibCpp/Token.cpp new file mode 100644 index 0000000000..80b19e55e8 --- /dev/null +++ b/Userland/Libraries/LibCpp/Token.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020, the SerenityOS developers. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "Token.h" + +namespace Cpp { + +bool Position::operator<(const Position& other) const +{ + return line < other.line || (line == other.line && column < other.column); +} +bool Position::operator>(const Position& other) const +{ + return !(*this < other) && !(*this == other); +} +bool Position::operator==(const Position& other) const +{ + return line == other.line && column == other.column; +} + +} diff --git a/Userland/Libraries/LibCpp/Token.h b/Userland/Libraries/LibCpp/Token.h new file mode 100644 index 0000000000..e16d01153e --- /dev/null +++ b/Userland/Libraries/LibCpp/Token.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2020, the SerenityOS developers. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include + +namespace Cpp { + +#define FOR_EACH_TOKEN_TYPE \ + __TOKEN(Unknown) \ + __TOKEN(Whitespace) \ + __TOKEN(PreprocessorStatement) \ + __TOKEN(IncludeStatement) \ + __TOKEN(IncludePath) \ + __TOKEN(LeftParen) \ + __TOKEN(RightParen) \ + __TOKEN(LeftCurly) \ + __TOKEN(RightCurly) \ + __TOKEN(LeftBracket) \ + __TOKEN(RightBracket) \ + __TOKEN(Less) \ + __TOKEN(Greater) \ + __TOKEN(LessEquals) \ + __TOKEN(GreaterEquals) \ + __TOKEN(LessLess) \ + __TOKEN(GreaterGreater) \ + __TOKEN(LessLessEquals) \ + __TOKEN(GreaterGreaterEquals) \ + __TOKEN(LessGreater) \ + __TOKEN(Comma) \ + __TOKEN(Plus) \ + __TOKEN(PlusPlus) \ + __TOKEN(PlusEquals) \ + __TOKEN(Minus) \ + __TOKEN(MinusMinus) \ + __TOKEN(MinusEquals) \ + __TOKEN(Asterisk) \ + __TOKEN(AsteriskEquals) \ + __TOKEN(Slash) \ + __TOKEN(SlashEquals) \ + __TOKEN(Percent) \ + __TOKEN(PercentEquals) \ + __TOKEN(Caret) \ + __TOKEN(CaretEquals) \ + __TOKEN(ExclamationMark) \ + __TOKEN(ExclamationMarkEquals) \ + __TOKEN(Equals) \ + __TOKEN(EqualsEquals) \ + __TOKEN(And) \ + __TOKEN(AndAnd) \ + __TOKEN(AndEquals) \ + __TOKEN(Pipe) \ + __TOKEN(PipePipe) \ + __TOKEN(PipeEquals) \ + __TOKEN(Tilde) \ + __TOKEN(QuestionMark) \ + __TOKEN(Colon) \ + __TOKEN(ColonColon) \ + __TOKEN(ColonColonAsterisk) \ + __TOKEN(Semicolon) \ + __TOKEN(Dot) \ + __TOKEN(DotAsterisk) \ + __TOKEN(Arrow) \ + __TOKEN(ArrowAsterisk) \ + __TOKEN(DoubleQuotedString) \ + __TOKEN(SingleQuotedString) \ + __TOKEN(RawString) \ + __TOKEN(EscapeSequence) \ + __TOKEN(Comment) \ + __TOKEN(Integer) \ + __TOKEN(Float) \ + __TOKEN(Keyword) \ + __TOKEN(KnownType) \ + __TOKEN(Identifier) \ + __TOKEN(EOF_TOKEN) + +struct Position { + size_t line { 0 }; + size_t column { 0 }; + + bool operator<(const Position&) const; + bool operator>(const Position&) const; + bool operator==(const Position&) const; +}; + +struct Token { + enum class Type { +#define __TOKEN(x) x, + FOR_EACH_TOKEN_TYPE +#undef __TOKEN + }; + + static const char* type_to_string(Type t) + { + switch (t) { +#define __TOKEN(x) \ + case Type::x: \ + return #x; + FOR_EACH_TOKEN_TYPE +#undef __TOKEN + } + VERIFY_NOT_REACHED(); + } + + const char* to_string() const + { + return type_to_string(m_type); + } + Position start() const { return m_start; } + Position end() const { return m_end; } + Type type() const { return m_type; } + + Type m_type { Type::Unknown }; + Position m_start; + Position m_end; +}; + +}