mirror of
https://github.com/RGBCube/serenity
synced 2025-07-28 21:17:44 +00:00
LibSQL: Move Lexer and Parser machinery to AST directory
The SQL engine is expected to be a fairly sizeable piece of software. Therefore we're starting to restructure the codebase for growth.
This commit is contained in:
parent
e0f1c237d2
commit
4198f7e1af
24 changed files with 281 additions and 278 deletions
|
@ -36,7 +36,7 @@
|
|||
#include <LibGfx/Painter.h>
|
||||
#include <LibJS/SyntaxHighlighter.h>
|
||||
#include <LibMarkdown/Document.h>
|
||||
#include <LibSQL/SyntaxHighlighter.h>
|
||||
#include <LibSQL/AST/SyntaxHighlighter.h>
|
||||
#include <LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h>
|
||||
#include <LibWeb/OutOfProcessWebView.h>
|
||||
#include <Shell/SyntaxHighlighter.h>
|
||||
|
@ -582,7 +582,7 @@ void MainWidget::initialize_menubar(GUI::Menubar& menubar)
|
|||
syntax_menu.add_action(*m_shell_highlight);
|
||||
|
||||
m_sql_highlight = GUI::Action::create_checkable("S&QL File", [&](auto&) {
|
||||
m_editor->set_syntax_highlighter(make<SQL::SyntaxHighlighter>());
|
||||
m_editor->set_syntax_highlighter(make<SQL::AST::SyntaxHighlighter>());
|
||||
m_editor->update();
|
||||
});
|
||||
syntax_actions.add_action(*m_sql_highlight);
|
||||
|
|
|
@ -11,10 +11,10 @@
|
|||
#include <AK/RefCounted.h>
|
||||
#include <AK/RefPtr.h>
|
||||
#include <AK/String.h>
|
||||
#include <LibSQL/AST/Token.h>
|
||||
#include <LibSQL/Forward.h>
|
||||
#include <LibSQL/Token.h>
|
||||
|
||||
namespace SQL {
|
||||
namespace SQL::AST {
|
||||
|
||||
template<class T, class... Args>
|
||||
static inline NonnullRefPtr<T>
|
|
@ -8,7 +8,7 @@
|
|||
#include <AK/Debug.h>
|
||||
#include <ctype.h>
|
||||
|
||||
namespace SQL {
|
||||
namespace SQL::AST {
|
||||
|
||||
HashMap<String, TokenType> Lexer::s_keywords;
|
||||
HashMap<char, TokenType> Lexer::s_one_char_tokens;
|
|
@ -11,7 +11,7 @@
|
|||
#include <AK/String.h>
|
||||
#include <AK/StringView.h>
|
||||
|
||||
namespace SQL {
|
||||
namespace SQL::AST {
|
||||
|
||||
class Lexer {
|
||||
public:
|
|
@ -8,7 +8,7 @@
|
|||
#include <AK/ScopeGuard.h>
|
||||
#include <AK/TypeCasts.h>
|
||||
|
||||
namespace SQL {
|
||||
namespace SQL::AST {
|
||||
|
||||
Parser::Parser(Lexer lexer)
|
||||
: m_parser_state(move(lexer))
|
|
@ -8,11 +8,11 @@
|
|||
|
||||
#include <AK/String.h>
|
||||
#include <AK/StringView.h>
|
||||
#include <LibSQL/AST.h>
|
||||
#include <LibSQL/Lexer.h>
|
||||
#include <LibSQL/Token.h>
|
||||
#include <LibSQL/AST/AST.h>
|
||||
#include <LibSQL/AST/Lexer.h>
|
||||
#include <LibSQL/AST/Token.h>
|
||||
|
||||
namespace SQL {
|
||||
namespace SQL::AST {
|
||||
|
||||
namespace Limits {
|
||||
// https://www.sqlite.org/limits.html
|
|
@ -6,10 +6,10 @@
|
|||
|
||||
#include <AK/Debug.h>
|
||||
#include <LibGfx/Palette.h>
|
||||
#include <LibSQL/Lexer.h>
|
||||
#include <LibSQL/SyntaxHighlighter.h>
|
||||
#include <LibSQL/AST/Lexer.h>
|
||||
#include <LibSQL/AST/SyntaxHighlighter.h>
|
||||
|
||||
namespace SQL {
|
||||
namespace SQL::AST {
|
||||
|
||||
static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, TokenType type)
|
||||
{
|
||||
|
@ -35,19 +35,19 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Token
|
|||
|
||||
bool SyntaxHighlighter::is_identifier(u64 token) const
|
||||
{
|
||||
auto sql_token = static_cast<SQL::TokenType>(static_cast<size_t>(token));
|
||||
return sql_token == SQL::TokenType::Identifier;
|
||||
auto sql_token = static_cast<TokenType>(static_cast<size_t>(token));
|
||||
return sql_token == TokenType::Identifier;
|
||||
}
|
||||
|
||||
void SyntaxHighlighter::rehighlight(Palette const& palette)
|
||||
{
|
||||
auto text = m_client->get_text();
|
||||
|
||||
SQL::Lexer lexer(text);
|
||||
Lexer lexer(text);
|
||||
|
||||
Vector<GUI::TextDocumentSpan> spans;
|
||||
|
||||
auto append_token = [&](StringView str, SQL::Token const& token) {
|
||||
auto append_token = [&](StringView str, Token const& token) {
|
||||
if (str.is_empty())
|
||||
return;
|
||||
|
||||
|
@ -79,7 +79,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
|
|||
for (;;) {
|
||||
auto token = lexer.next();
|
||||
append_token(token.value(), token);
|
||||
if (token.type() == SQL::TokenType::Eof)
|
||||
if (token.type() == TokenType::Eof)
|
||||
break;
|
||||
}
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
#include <LibSyntax/Highlighter.h>
|
||||
|
||||
namespace SQL {
|
||||
namespace SQL::AST {
|
||||
|
||||
class SyntaxHighlighter final : public Syntax::Highlighter {
|
||||
public:
|
|
@ -9,7 +9,7 @@
|
|||
#include <AK/String.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
namespace SQL {
|
||||
namespace SQL::AST {
|
||||
|
||||
StringView Token::name(TokenType type)
|
||||
{
|
|
@ -9,7 +9,7 @@
|
|||
#include <AK/HashMap.h>
|
||||
#include <AK/StringView.h>
|
||||
|
||||
namespace SQL {
|
||||
namespace SQL::AST {
|
||||
|
||||
// https://sqlite.org/lang_keywords.html
|
||||
#define ENUMERATE_SQL_TOKENS \
|
|
@ -1,4 +1,8 @@
|
|||
set(SOURCES
|
||||
AST/Lexer.cpp
|
||||
AST/Parser.cpp
|
||||
AST/SyntaxHighlighter.cpp
|
||||
AST/Token.cpp
|
||||
BTree.cpp
|
||||
BTreeIterator.cpp
|
||||
Database.cpp
|
||||
|
@ -6,12 +10,8 @@ set(SOURCES
|
|||
Heap.cpp
|
||||
Index.cpp
|
||||
Key.cpp
|
||||
Lexer.cpp
|
||||
Meta.cpp
|
||||
Parser.cpp
|
||||
Row.cpp
|
||||
SyntaxHighlighter.cpp
|
||||
Token.cpp
|
||||
TreeNode.cpp
|
||||
Tuple.cpp
|
||||
Value.cpp
|
||||
|
|
|
@ -7,27 +7,45 @@
|
|||
#pragma once
|
||||
|
||||
namespace SQL {
|
||||
class BTree;
|
||||
class BTreeIterator;
|
||||
class ColumnDef;
|
||||
class Database;
|
||||
class HashBucket;
|
||||
class HashDirectoryNode;
|
||||
class HashIndex;
|
||||
class HashIndexIterator;
|
||||
class Heap;
|
||||
class Index;
|
||||
class IndexNode;
|
||||
class IndexDef;
|
||||
class Key;
|
||||
class KeyPartDef;
|
||||
class Row;
|
||||
class TableDef;
|
||||
class TreeNode;
|
||||
class Tuple;
|
||||
class TupleDescriptor;
|
||||
struct TupleElement;
|
||||
class Value;
|
||||
}
|
||||
|
||||
namespace SQL::AST {
|
||||
class AddColumn;
|
||||
class AlterTable;
|
||||
class ASTNode;
|
||||
class BetweenExpression;
|
||||
class BinaryOperatorExpression;
|
||||
class BlobLiteral;
|
||||
class BTree;
|
||||
class BTreeIterator;
|
||||
class CaseExpression;
|
||||
class CastExpression;
|
||||
class ChainedExpression;
|
||||
class CollateExpression;
|
||||
class ColumnDef;
|
||||
class ColumnDefinition;
|
||||
class ColumnNameExpression;
|
||||
class CommonTableExpression;
|
||||
class CommonTableExpressionList;
|
||||
class CreateTable;
|
||||
class Database;
|
||||
class TupleDescriptor;
|
||||
struct TupleElement;
|
||||
class Delete;
|
||||
class DropColumn;
|
||||
class DropTable;
|
||||
|
@ -36,23 +54,13 @@ class ErrorStatement;
|
|||
class ExistsExpression;
|
||||
class Expression;
|
||||
class GroupByClause;
|
||||
class HashBucket;
|
||||
class HashDirectoryNode;
|
||||
class HashIndex;
|
||||
class HashIndexIterator;
|
||||
class Heap;
|
||||
class InChainedExpression;
|
||||
class Index;
|
||||
class IndexNode;
|
||||
class IndexDef;
|
||||
class InSelectionExpression;
|
||||
class Insert;
|
||||
class InTableExpression;
|
||||
class InvertibleNestedDoubleExpression;
|
||||
class InvertibleNestedExpression;
|
||||
class IsExpression;
|
||||
class Key;
|
||||
class KeyPartDef;
|
||||
class Lexer;
|
||||
class LimitClause;
|
||||
class MatchExpression;
|
||||
|
@ -68,18 +76,13 @@ class RenameColumn;
|
|||
class RenameTable;
|
||||
class ResultColumn;
|
||||
class ReturningClause;
|
||||
class Row;
|
||||
class Select;
|
||||
class SignedNumber;
|
||||
class Statement;
|
||||
class StringLiteral;
|
||||
class TableDef;
|
||||
class TableOrSubquery;
|
||||
class Token;
|
||||
class TreeNode;
|
||||
class Tuple;
|
||||
class TypeName;
|
||||
class UnaryOperatorExpression;
|
||||
class Update;
|
||||
class Value;
|
||||
}
|
||||
|
|
|
@ -37,7 +37,7 @@ NonnullRefPtr<IndexDef> SchemaDef::index_def()
|
|||
{
|
||||
NonnullRefPtr<IndexDef> s_index_def = IndexDef::construct("$schema", true, 0);
|
||||
if (!s_index_def->size()) {
|
||||
s_index_def->append_column("schema_name", SQLType::Text, Order::Ascending);
|
||||
s_index_def->append_column("schema_name", SQLType::Text, AST::Order::Ascending);
|
||||
}
|
||||
return s_index_def;
|
||||
}
|
||||
|
@ -70,15 +70,15 @@ NonnullRefPtr<IndexDef> ColumnDef::index_def()
|
|||
{
|
||||
NonnullRefPtr<IndexDef> s_index_def = IndexDef::construct("$column", true, 0);
|
||||
if (!s_index_def->size()) {
|
||||
s_index_def->append_column("table_hash", SQLType::Integer, Order::Ascending);
|
||||
s_index_def->append_column("column_number", SQLType::Integer, Order::Ascending);
|
||||
s_index_def->append_column("column_name", SQLType::Text, Order::Ascending);
|
||||
s_index_def->append_column("column_type", SQLType::Integer, Order::Ascending);
|
||||
s_index_def->append_column("table_hash", SQLType::Integer, AST::Order::Ascending);
|
||||
s_index_def->append_column("column_number", SQLType::Integer, AST::Order::Ascending);
|
||||
s_index_def->append_column("column_name", SQLType::Text, AST::Order::Ascending);
|
||||
s_index_def->append_column("column_type", SQLType::Integer, AST::Order::Ascending);
|
||||
}
|
||||
return s_index_def;
|
||||
}
|
||||
|
||||
KeyPartDef::KeyPartDef(IndexDef* index, String name, SQLType sql_type, Order sort_order)
|
||||
KeyPartDef::KeyPartDef(IndexDef* index, String name, SQLType sql_type, AST::Order sort_order)
|
||||
: ColumnDef(index, index->size(), move(name), sql_type)
|
||||
, m_sort_order(sort_order)
|
||||
{
|
||||
|
@ -96,7 +96,7 @@ IndexDef::IndexDef(String name, bool unique, u32 pointer)
|
|||
{
|
||||
}
|
||||
|
||||
void IndexDef::append_column(String name, SQLType sql_type, Order sort_order)
|
||||
void IndexDef::append_column(String name, SQLType sql_type, AST::Order sort_order)
|
||||
{
|
||||
auto part = KeyPartDef::construct(this, move(name), sql_type, sort_order);
|
||||
m_key_definition.append(part);
|
||||
|
@ -131,9 +131,9 @@ NonnullRefPtr<IndexDef> IndexDef::index_def()
|
|||
{
|
||||
NonnullRefPtr<IndexDef> s_index_def = IndexDef::construct("$index", true, 0);
|
||||
if (!s_index_def->size()) {
|
||||
s_index_def->append_column("table_hash", SQLType::Integer, Order::Ascending);
|
||||
s_index_def->append_column("index_name", SQLType::Text, Order::Ascending);
|
||||
s_index_def->append_column("unique", SQLType::Integer, Order::Ascending);
|
||||
s_index_def->append_column("table_hash", SQLType::Integer, AST::Order::Ascending);
|
||||
s_index_def->append_column("index_name", SQLType::Text, AST::Order::Ascending);
|
||||
s_index_def->append_column("unique", SQLType::Integer, AST::Order::Ascending);
|
||||
}
|
||||
return s_index_def;
|
||||
}
|
||||
|
@ -149,7 +149,7 @@ TupleDescriptor TableDef::to_tuple_descriptor() const
|
|||
{
|
||||
TupleDescriptor ret;
|
||||
for (auto& part : m_columns) {
|
||||
ret.append({ part.name(), part.type(), Order::Ascending });
|
||||
ret.append({ part.name(), part.type(), AST::Order::Ascending });
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
@ -192,8 +192,8 @@ NonnullRefPtr<IndexDef> TableDef::index_def()
|
|||
{
|
||||
NonnullRefPtr<IndexDef> s_index_def = IndexDef::construct("$table", true, 0);
|
||||
if (!s_index_def->size()) {
|
||||
s_index_def->append_column("schema_hash", SQLType::Integer, Order::Ascending);
|
||||
s_index_def->append_column("table_name", SQLType::Text, Order::Ascending);
|
||||
s_index_def->append_column("schema_hash", SQLType::Integer, AST::Order::Ascending);
|
||||
s_index_def->append_column("table_name", SQLType::Text, AST::Order::Ascending);
|
||||
}
|
||||
return s_index_def;
|
||||
}
|
||||
|
|
|
@ -13,7 +13,7 @@
|
|||
#include <AK/String.h>
|
||||
#include <AK/Vector.h>
|
||||
#include <LibCore/Object.h>
|
||||
#include <LibSQL/AST.h>
|
||||
#include <LibSQL/AST/AST.h>
|
||||
#include <LibSQL/Forward.h>
|
||||
#include <LibSQL/Key.h>
|
||||
#include <LibSQL/Type.h>
|
||||
|
@ -90,11 +90,11 @@ class KeyPartDef : public ColumnDef {
|
|||
C_OBJECT(KeyPartDef);
|
||||
|
||||
public:
|
||||
KeyPartDef(IndexDef*, String, SQLType, Order = Order::Ascending);
|
||||
Order sort_order() const { return m_sort_order; }
|
||||
KeyPartDef(IndexDef*, String, SQLType, AST::Order = AST::Order::Ascending);
|
||||
AST::Order sort_order() const { return m_sort_order; }
|
||||
|
||||
private:
|
||||
Order m_sort_order { Order::Ascending };
|
||||
AST::Order m_sort_order { AST::Order::Ascending };
|
||||
};
|
||||
|
||||
class IndexDef : public Relation {
|
||||
|
@ -106,7 +106,7 @@ public:
|
|||
NonnullRefPtrVector<KeyPartDef> key_definition() const { return m_key_definition; }
|
||||
bool unique() const { return m_unique; }
|
||||
[[nodiscard]] size_t size() const { return m_key_definition.size(); }
|
||||
void append_column(String, SQLType, Order = Order::Ascending);
|
||||
void append_column(String, SQLType, AST::Order = AST::Order::Ascending);
|
||||
Key key() const override;
|
||||
[[nodiscard]] TupleDescriptor to_tuple_descriptor() const;
|
||||
static NonnullRefPtr<IndexDef> index_def();
|
||||
|
|
|
@ -203,7 +203,7 @@ int Tuple::compare(const Tuple& other) const
|
|||
for (auto ix = 0u; ix < num_values; ix++) {
|
||||
auto ret = m_data[ix].compare(other.m_data[ix]);
|
||||
if (ret != 0) {
|
||||
if ((ix < m_descriptor.size()) && m_descriptor[ix].order == Order::Descending)
|
||||
if ((ix < m_descriptor.size()) && m_descriptor[ix].order == AST::Order::Descending)
|
||||
ret = -ret;
|
||||
return ret;
|
||||
}
|
||||
|
@ -223,7 +223,7 @@ int Tuple::match(const Tuple& other) const
|
|||
return -1;
|
||||
auto ret = m_data[my_index.value()].compare(other_value);
|
||||
if (ret != 0)
|
||||
return (m_descriptor[my_index.value()].order == Order::Descending) ? -ret : ret;
|
||||
return (m_descriptor[my_index.value()].order == AST::Order::Descending) ? -ret : ret;
|
||||
other_index++;
|
||||
}
|
||||
return 0;
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
#pragma once
|
||||
|
||||
#include <AK/Vector.h>
|
||||
#include <LibSQL/AST.h>
|
||||
#include <LibSQL/AST/AST.h>
|
||||
#include <LibSQL/Type.h>
|
||||
|
||||
namespace SQL {
|
||||
|
@ -15,7 +15,7 @@ namespace SQL {
|
|||
struct TupleElement {
|
||||
String name { "" };
|
||||
SQLType type { SQLType::Text };
|
||||
Order order { Order::Ascending };
|
||||
AST::Order order { AST::Order::Ascending };
|
||||
|
||||
bool operator==(TupleElement const&) const = default;
|
||||
};
|
||||
|
|
|
@ -9,9 +9,9 @@
|
|||
#include <AK/StringBuilder.h>
|
||||
#include <LibCore/StandardPaths.h>
|
||||
#include <LibLine/Editor.h>
|
||||
#include <LibSQL/Lexer.h>
|
||||
#include <LibSQL/Parser.h>
|
||||
#include <LibSQL/Token.h>
|
||||
#include <LibSQL/AST/Lexer.h>
|
||||
#include <LibSQL/AST/Parser.h>
|
||||
#include <LibSQL/AST/Token.h>
|
||||
|
||||
namespace {
|
||||
|
||||
|
@ -48,7 +48,7 @@ String read_next_piece()
|
|||
}
|
||||
|
||||
auto& line = line_result.value();
|
||||
auto lexer = SQL::Lexer(line);
|
||||
auto lexer = SQL::AST::Lexer(line);
|
||||
|
||||
s_editor->add_to_history(line);
|
||||
piece.append(line);
|
||||
|
@ -57,18 +57,18 @@ String read_next_piece()
|
|||
bool is_command = false;
|
||||
bool last_token_ended_statement = false;
|
||||
|
||||
for (SQL::Token token = lexer.next(); token.type() != SQL::TokenType::Eof; token = lexer.next()) {
|
||||
for (SQL::AST::Token token = lexer.next(); token.type() != SQL::AST::TokenType::Eof; token = lexer.next()) {
|
||||
switch (token.type()) {
|
||||
case SQL::TokenType::ParenOpen:
|
||||
case SQL::AST::TokenType::ParenOpen:
|
||||
++s_repl_line_level;
|
||||
break;
|
||||
case SQL::TokenType::ParenClose:
|
||||
case SQL::AST::TokenType::ParenClose:
|
||||
--s_repl_line_level;
|
||||
break;
|
||||
case SQL::TokenType::SemiColon:
|
||||
case SQL::AST::TokenType::SemiColon:
|
||||
last_token_ended_statement = true;
|
||||
break;
|
||||
case SQL::TokenType::Period:
|
||||
case SQL::AST::TokenType::Period:
|
||||
if (is_first_token)
|
||||
is_command = true;
|
||||
break;
|
||||
|
@ -96,7 +96,7 @@ void handle_command(StringView command)
|
|||
|
||||
void handle_statement(StringView statement_string)
|
||||
{
|
||||
auto parser = SQL::Parser(SQL::Lexer(statement_string));
|
||||
auto parser = SQL::AST::Parser(SQL::AST::Lexer(statement_string));
|
||||
[[maybe_unused]] auto statement = parser.next_statement();
|
||||
|
||||
if (parser.has_errors()) {
|
||||
|
@ -132,38 +132,38 @@ int main()
|
|||
size_t open_indents = s_repl_line_level;
|
||||
|
||||
auto line = editor.line();
|
||||
SQL::Lexer lexer(line);
|
||||
SQL::AST::Lexer lexer(line);
|
||||
|
||||
bool indenters_starting_line = true;
|
||||
for (SQL::Token token = lexer.next(); token.type() != SQL::TokenType::Eof; token = lexer.next()) {
|
||||
for (SQL::AST::Token token = lexer.next(); token.type() != SQL::AST::TokenType::Eof; token = lexer.next()) {
|
||||
auto length = token.value().length();
|
||||
auto start = token.line_column() - 1;
|
||||
auto end = start + length;
|
||||
|
||||
if (indenters_starting_line) {
|
||||
if (token.type() != SQL::TokenType::ParenClose)
|
||||
if (token.type() != SQL::AST::TokenType::ParenClose)
|
||||
indenters_starting_line = false;
|
||||
else
|
||||
--open_indents;
|
||||
}
|
||||
|
||||
switch (token.category()) {
|
||||
case SQL::TokenCategory::Invalid:
|
||||
case SQL::AST::TokenCategory::Invalid:
|
||||
editor.stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Red), Line::Style::Underline });
|
||||
break;
|
||||
case SQL::TokenCategory::Number:
|
||||
case SQL::AST::TokenCategory::Number:
|
||||
editor.stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Magenta) });
|
||||
break;
|
||||
case SQL::TokenCategory::String:
|
||||
case SQL::AST::TokenCategory::String:
|
||||
editor.stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Green), Line::Style::Bold });
|
||||
break;
|
||||
case SQL::TokenCategory::Blob:
|
||||
case SQL::AST::TokenCategory::Blob:
|
||||
editor.stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Magenta), Line::Style::Bold });
|
||||
break;
|
||||
case SQL::TokenCategory::Keyword:
|
||||
case SQL::AST::TokenCategory::Keyword:
|
||||
editor.stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Blue), Line::Style::Bold });
|
||||
break;
|
||||
case SQL::TokenCategory::Identifier:
|
||||
case SQL::AST::TokenCategory::Identifier:
|
||||
editor.stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::White), Line::Style::Bold });
|
||||
default:
|
||||
break;
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue