RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-25 21:07:35 +00:00
Code Issues Activity Actions

LibSQL: Introduce a SQL library with a tokenizer

Browse source 
LibSQL aims to be a SQLite clone for SerenityOS. Step 1 is creating a
tokenizer to lex SQL tokens. This lexer is heavily influenced by the
LibJS lexer.
This commit is contained in:
Timothy Flynn 2021-04-18 17:35:40 -04:00 • committed by Andreas Kling
parent 55b7b8e930
commit 90517da9ca
6 changed files with 716 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

306
Userland/Libraries/LibSQL/Lexer.cpp Normal file
View file

@ -0,0 +1,306 @@
/*
* Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "Lexer.h"
#include <AK/Debug.h>
#include <ctype.h>
namespace SQL {
HashMap<String, TokenType> Lexer::s_keywords;
HashMap<char, TokenType> Lexer::s_one_char_tokens;
HashMap<String, TokenType> Lexer::s_two_char_tokens;
Lexer::Lexer(StringView source)
: m_source(source)
{
if (s_keywords.is_empty()) {
#define __ENUMERATE_SQL_TOKEN(value, type, category) \
if (TokenCategory::category == TokenCategory::Keyword) \
s_keywords.set(value, TokenType::type);
ENUMERATE_SQL_TOKENS
#undef __ENUMERATE_SQL_TOKEN
}
if (s_one_char_tokens.is_empty()) {
#define __ENUMERATE_SQL_TOKEN(value, type, category) \
if (TokenCategory::category != TokenCategory::Keyword && StringView(value).length() == 1) \
s_one_char_tokens.set(value[0], TokenType::type);
ENUMERATE_SQL_TOKENS
#undef __ENUMERATE_SQL_TOKEN
}
if (s_two_char_tokens.is_empty()) {
#define __ENUMERATE_SQL_TOKEN(value, type, category) \
if (TokenCategory::category != TokenCategory::Keyword && StringView(value).length() == 2) \
s_two_char_tokens.set(value, TokenType::type);
ENUMERATE_SQL_TOKENS
#undef __ENUMERATE_SQL_TOKEN
}
consume();
}
Token Lexer::next()
{
bool found_invalid_comment = consume_whitespace_and_comments();
size_t value_start = m_position;
size_t value_start_line_number = m_line_number;
size_t value_start_column_number = m_line_column;
auto token_type = TokenType::Invalid;
if (is_eof()) {
token_type = found_invalid_comment ? TokenType::Invalid : TokenType::Eof;
} else if (is_identifier_start()) {
do {
consume();
} while (is_identifier_middle());
if (auto it = s_keywords.find(m_source.substring_view(value_start - 1, m_position - value_start)); it != s_keywords.end()) {
token_type = it->value;
} else {
token_type = TokenType::Identifier;
}
} else if (is_numeric_literal_start()) {
token_type = TokenType::NumericLiteral;
if (!consume_numeric_literal())
token_type = TokenType::Invalid;
} else {
bool found_two_char_token = false;
if (m_position < m_source.length()) {
if (auto it = s_two_char_tokens.find(m_source.substring_view(m_position - 1, 2)); it != s_two_char_tokens.end()) {
found_two_char_token = true;
token_type = it->value;
consume();
consume();
}
}
bool found_one_char_token = false;
if (!found_two_char_token) {
if (auto it = s_one_char_tokens.find(m_current_char); it != s_one_char_tokens.end()) {
found_one_char_token = true;
token_type = it->value;
consume();
}
}
if (!found_two_char_token && !found_one_char_token) {
token_type = TokenType::Invalid;
consume();
}
}
Token token(token_type, m_source.substring_view(value_start - 1, m_position - value_start), value_start_line_number, value_start_column_number);
if constexpr (SQL_DEBUG) {
dbgln("------------------------------");
dbgln("Token: {}", token.name());
dbgln("Value: {}", token.value());
dbgln("Line: {}, Column: {}", token.line_number(), token.line_column());
dbgln("------------------------------");
}
return token;
}
void Lexer::consume()
{
auto did_reach_eof = [this] {
if (m_position != m_source.length())
return false;
m_current_char = EOF;
++m_line_column;
++m_position;
return true;
};
if (m_position > m_source.length())
return;
if (did_reach_eof())
return;
if (is_line_break()) {
++m_line_number;
m_line_column = 1;
} else {
++m_line_column;
}
m_current_char = m_source[m_position++];
}
bool Lexer::consume_whitespace_and_comments()
{
bool found_invalid_comment = false;
while (true) {
if (isspace(m_current_char)) {
do {
consume();
} while (isspace(m_current_char));
} else if (is_line_comment_start()) {
consume();
do {
consume();
} while (!is_eof() && !is_line_break());
} else if (is_block_comment_start()) {
consume();
do {
consume();
} while (!is_eof() && !is_block_comment_end());
if (is_eof())
found_invalid_comment = true;
consume(); // consume *
if (is_eof())
found_invalid_comment = true;
consume(); // consume /
} else {
break;
}
}
return found_invalid_comment;
}
bool Lexer::consume_numeric_literal()
{
// https://www.sqlite.org/draft/syntax/numeric-literal.html
bool is_valid_numeric_literal = true;
if (m_current_char == '0') {
consume();
if (m_current_char == '.') {
consume();
while (isdigit(m_current_char))
consume();
if (m_current_char == 'e' || m_current_char == 'E')
is_valid_numeric_literal = consume_exponent();
} else if (m_current_char == 'e' || m_current_char == 'E') {
is_valid_numeric_literal = consume_exponent();
} else if (m_current_char == 'x' || m_current_char == 'X') {
is_valid_numeric_literal = consume_hexadecimal_number();
} else if (isdigit(m_current_char)) {
do {
consume();
} while (isdigit(m_current_char));
}
} else {
do {
consume();
} while (isdigit(m_current_char));
if (m_current_char == '.') {
consume();
while (isdigit(m_current_char))
consume();
}
if (m_current_char == 'e' || m_current_char == 'E')
is_valid_numeric_literal = consume_exponent();
}
return is_valid_numeric_literal;
}
bool Lexer::consume_exponent()
{
consume();
if (m_current_char == '-' || m_current_char == '+')
consume();
if (!isdigit(m_current_char))
return false;
while (isdigit(m_current_char)) {
consume();
}
return true;
}
bool Lexer::consume_hexadecimal_number()
{
consume();
if (!isxdigit(m_current_char))
return false;
while (isxdigit(m_current_char))
consume();
return true;
}
bool Lexer::match(char a, char b) const
{
if (m_position >= m_source.length())
return false;
return m_current_char == a
&& m_source[m_position] == b;
}
bool Lexer::is_identifier_start() const
{
return isalpha(m_current_char) || m_current_char == '_';
}
bool Lexer::is_identifier_middle() const
{
return is_identifier_start() || isdigit(m_current_char);
}
bool Lexer::is_numeric_literal_start() const
{
return isdigit(m_current_char) || (m_current_char == '.' && m_position < m_source.length() && isdigit(m_source[m_position]));
}
bool Lexer::is_line_comment_start() const
{
return match('-', '-');
}
bool Lexer::is_block_comment_start() const
{
return match('/', '*');
}
bool Lexer::is_block_comment_end() const
{
return match('*', '/');
}
bool Lexer::is_line_break() const
{
return m_current_char == '\n';
}
bool Lexer::is_eof() const
{
return m_current_char == EOF;
}
}