diff --git a/Userland/Libraries/LibGLSL/AST.cpp b/Userland/Libraries/LibGLSL/AST.cpp new file mode 100644 index 0000000000..ba96ef7198 --- /dev/null +++ b/Userland/Libraries/LibGLSL/AST.cpp @@ -0,0 +1,517 @@ +/* + * Copyright (c) 2021, Itamar S. + * Copyright (c) 2023, Volodymyr V. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "AST.h" + +namespace GLSL { + +static ErrorOr print_indent(AK::Stream& output, int indent) +{ + for (int i = 0; i < indent * 2; ++i) + TRY(output.write_some(" "sv.bytes())); + return {}; +} + +ErrorOr ASTNode::dump(AK::Stream& output, size_t indent) const +{ + TRY(print_indent(output, indent)); + TRY(output.write_formatted("{}[{}:{}->{}:{}]\n", class_name(), start().line, start().column, end().line, end().column)); + return {}; +} + +ErrorOr TranslationUnit::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + for (auto const& child : m_declarations) { + TRY(child->dump(output, indent + 1)); + } + return {}; +} + +ErrorOr FunctionDeclaration::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + + TRY(m_return_type->dump(output, indent + 1)); + + if (!m_name.is_null()) { + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("{}\n", m_name->name())); + } + + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("(\n")); + + for (auto const& arg : m_parameters) { + TRY(arg->dump(output, indent + 1)); + } + + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted(")\n")); + + if (!m_definition.is_null()) { + TRY(m_definition->dump(output, indent + 1)); + } + + return {}; +} + +Vector> FunctionDeclaration::declarations() const +{ + Vector> declarations; + for (auto& arg : m_parameters) { + declarations.append(arg); + } + + if (m_definition) + declarations.extend(m_definition->declarations()); + + return declarations; +} + +ErrorOr Type::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(print_indent(output, indent + 1)); + + StringBuilder qualifiers_string; + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Const).has_value()) + qualifiers_string.append("const "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::In).has_value()) + qualifiers_string.append("in "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Out).has_value()) + qualifiers_string.append("out "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Inout).has_value()) + qualifiers_string.append("inout "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Centroid).has_value()) + qualifiers_string.append("centroid "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Patch).has_value()) + qualifiers_string.append("patch "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Sample).has_value()) + qualifiers_string.append("sample "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Uniform).has_value()) + qualifiers_string.append("uniform "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Buffer).has_value()) + qualifiers_string.append("buffer "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Shared).has_value()) + qualifiers_string.append("shared "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Coherent).has_value()) + qualifiers_string.append("coherent "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Volatile).has_value()) + qualifiers_string.append("volatile "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Restrict).has_value()) + qualifiers_string.append("restrict "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Readonly).has_value()) + qualifiers_string.append("readonly "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Writeonly).has_value()) + qualifiers_string.append("writeonly "sv); + if (m_storage_qualifiers.find_first_index(StorageTypeQualifier::Subroutine).has_value()) + qualifiers_string.append("subroutine "sv); + TRY(output.write_formatted("{}{}\n", qualifiers_string.string_view(), m_name->name())); + return {}; +} + +ErrorOr Parameter::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + if (!m_name.is_null()) { + TRY(print_indent(output, indent)); + TRY(output.write_formatted("{}\n", m_name->name())); + } + if (m_type) + TRY(m_type->dump(output, indent + 1)); + return {}; +} + +ErrorOr FunctionDefinition::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(print_indent(output, indent)); + TRY(output.write_formatted("{{\n")); + for (auto const& statement : m_statements) { + TRY(statement->dump(output, indent + 1)); + } + TRY(print_indent(output, indent)); + TRY(output.write_formatted("}}\n")); + return {}; +} + +Vector> FunctionDefinition::declarations() const +{ + Vector> declarations; + for (auto& statement : m_statements) { + declarations.extend(statement->declarations()); + } + return declarations; +} + +ErrorOr VariableDeclaration::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + if (m_type) + TRY(m_type->dump(output, indent + 1)); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("{}\n", m_name->name())); + if (m_initial_value) + TRY(m_initial_value->dump(output, indent + 1)); + return {}; +} + +ErrorOr NumericLiteral::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("{}\n", m_value)); + return {}; +} + +ErrorOr BinaryExpression::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + + char const* op_string = nullptr; + switch (m_op) { + case BinaryOp::Addition: + op_string = "+"; + break; + case BinaryOp::Subtraction: + op_string = "-"; + break; + case BinaryOp::Multiplication: + op_string = "*"; + break; + case BinaryOp::Division: + op_string = "/"; + break; + case BinaryOp::Modulo: + op_string = "%"; + break; + case BinaryOp::GreaterThan: + op_string = ">"; + break; + case BinaryOp::GreaterThanEquals: + op_string = ">="; + break; + case BinaryOp::LessThan: + op_string = "<"; + break; + case BinaryOp::LessThanEquals: + op_string = "<="; + break; + case BinaryOp::BitwiseAnd: + op_string = "&"; + break; + case BinaryOp::BitwiseOr: + op_string = "|"; + break; + case BinaryOp::BitwiseXor: + op_string = "^"; + break; + case BinaryOp::LeftShift: + op_string = "<<"; + break; + case BinaryOp::RightShift: + op_string = ">>"; + break; + case BinaryOp::EqualsEquals: + op_string = "=="; + break; + case BinaryOp::NotEqual: + op_string = "!="; + break; + case BinaryOp::LogicalOr: + op_string = "||"; + break; + case BinaryOp::LogicalXor: + op_string = "^^"; + break; + case BinaryOp::LogicalAnd: + op_string = "&&"; + break; + case BinaryOp::Assignment: + op_string = "="; + break; + case BinaryOp::AdditionAssignment: + op_string = "+="; + break; + case BinaryOp::SubtractionAssignment: + op_string = "-="; + break; + case BinaryOp::MultiplicationAssignment: + op_string = "*="; + break; + case BinaryOp::DivisionAssignment: + op_string = "/="; + break; + case BinaryOp::ModuloAssignment: + op_string = "%="; + break; + case BinaryOp::AndAssignment: + op_string = "&="; + break; + case BinaryOp::OrAssignment: + op_string = "|="; + break; + case BinaryOp::XorAssignment: + op_string = "^="; + break; + case BinaryOp::LeftShiftAssignment: + op_string = "<<="; + break; + case BinaryOp::RightShiftAssignment: + op_string = ">>="; + break; + } + + TRY(m_lhs->dump(output, indent + 1)); + + TRY(print_indent(output, indent + 1)); + VERIFY(op_string); + TRY(output.write_formatted("{}\n", op_string)); + + TRY(m_rhs->dump(output, indent + 1)); + + return {}; +} + +ErrorOr FunctionCall::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(m_callee->dump(output, indent + 1)); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("(\n")); + for (auto const& arg : m_arguments) { + TRY(arg->dump(output, indent + 1)); + } + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted(")\n")); + return {}; +} + +ErrorOr StringLiteral::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("{}\n", m_value)); + return {}; +} + +ErrorOr ReturnStatement::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + if (m_value) + TRY(m_value->dump(output, indent + 1)); + return {}; +} + +ErrorOr StructDeclaration::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("{}\n", m_name->name())); + for (auto& member : m_members) { + TRY(member->dump(output, indent + 1)); + } + return {}; +} +Vector> StructDeclaration::declarations() const +{ + Vector> declarations; + for (auto& member : m_members) + declarations.append(member); + return declarations; +} + +ErrorOr UnaryExpression::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + + char const* op_string = nullptr; + switch (m_op) { + case UnaryOp::BitwiseNot: + op_string = "~"; + break; + case UnaryOp::Not: + op_string = "!"; + break; + case UnaryOp::Plus: + op_string = "+"; + break; + case UnaryOp::Minus: + op_string = "-"; + break; + case UnaryOp::PlusPlus: + op_string = "++"; + break; + case UnaryOp::MinusMinus: + op_string = "--"; + break; + default: + op_string = ""; + } + + VERIFY(op_string); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("{} {}\n", m_is_postfix ? "postfix"sv : "prefix"sv, op_string)); + TRY(m_lhs->dump(output, indent + 1)); + return {}; +} + +ErrorOr BooleanLiteral::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("{}\n", m_value ? "true" : "false")); + return {}; +} + +ErrorOr MemberExpression::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(m_object->dump(output, indent + 1)); + TRY(m_property->dump(output, indent + 1)); + return {}; +} + +ErrorOr ArrayElementExpression::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(m_array->dump(output, indent + 1)); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("[\n")); + TRY(m_index->dump(output, indent + 1)); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("]\n")); + return {}; +} + +ErrorOr BlockStatement::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + for (auto& statement : m_statements) { + TRY(statement->dump(output, indent + 1)); + } + return {}; +} + +ErrorOr ForStatement::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + if (m_init) { + TRY(print_indent(output, indent)); + TRY(output.write_formatted("Initializer:\n")); + TRY(m_init->dump(output, indent + 1)); + } + if (m_test) { + TRY(print_indent(output, indent)); + TRY(output.write_formatted("Test expression:\n")); + TRY(m_test->dump(output, indent + 1)); + } + if (m_update) { + TRY(print_indent(output, indent)); + TRY(output.write_formatted("Update expression:\n")); + TRY(m_update->dump(output, indent + 1)); + } + if (m_body) { + TRY(print_indent(output, indent)); + TRY(output.write_formatted("Body:\n")); + TRY(m_body->dump(output, indent + 1)); + } + return {}; +} + +Vector> Statement::declarations() const +{ + if (is_declaration()) { + Vector> vec; + auto const& decl = static_cast(*this); + vec.empend(const_cast(decl)); + return vec; + } + return {}; +} + +Vector> ForStatement::declarations() const +{ + Vector> declarations; + if (m_init) + declarations.extend(m_init->declarations()); + if (m_body) + declarations.extend(m_body->declarations()); + return declarations; +} + +Vector> BlockStatement::declarations() const +{ + Vector> declarations; + for (auto& statement : m_statements) { + declarations.extend(statement->declarations()); + } + return declarations; +} + +ErrorOr IfStatement::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + if (m_predicate) { + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("Predicate:\n")); + TRY(m_predicate->dump(output, indent + 1)); + } + if (m_then) { + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("Then:\n")); + TRY(m_then->dump(output, indent + 1)); + } + if (m_else) { + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("Else:\n")); + TRY(m_else->dump(output, indent + 1)); + } + return {}; +} + +Vector> IfStatement::declarations() const +{ + Vector> declarations; + if (m_predicate) + declarations.extend(m_predicate->declarations()); + if (m_then) + declarations.extend(m_then->declarations()); + if (m_else) + declarations.extend(m_else->declarations()); + return declarations; +} + +ErrorOr Name::dump(AK::Stream& output, size_t indent) const +{ + TRY(ASTNode::dump(output, indent)); + TRY(print_indent(output, indent + 1)); + TRY(output.write_formatted("{}\n", name())); + return {}; +} + +ErrorOr SizedName::dump(AK::Stream& output, size_t indent) const +{ + TRY(Name::dump(output, indent)); + TRY(print_indent(output, indent + 1)); + + StringBuilder dimension_info; + for (auto const& dim : m_dimensions) { + dimension_info.append('['); + dimension_info.append(dim); + dimension_info.append(']'); + } + + if (dimension_info.is_empty()) { + dimension_info.append("[]"sv); + } + TRY(output.write_formatted("{}\n", dimension_info.string_view())); + return {}; +} + +} diff --git a/Userland/Libraries/LibGLSL/AST.h b/Userland/Libraries/LibGLSL/AST.h new file mode 100644 index 0000000000..ce63f44009 --- /dev/null +++ b/Userland/Libraries/LibGLSL/AST.h @@ -0,0 +1,715 @@ +/* + * Copyright (c) 2021, Itamar S. + * Copyright (c) 2023, Volodymyr V. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace GLSL { + +class ASTNode; +class TranslationUnit; +class Declaration; +class FunctionDefinition; +class Type; +class Parameter; +class Statement; +class Name; + +class ASTNode : public RefCounted { +public: + virtual ~ASTNode() = default; + virtual StringView class_name() const = 0; + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const; + + template + bool fast_is() const = delete; + + ASTNode const* parent() const { return m_parent; } + Position start() const + { + VERIFY(m_start.has_value()); + return m_start.value(); + } + Position end() const + { + VERIFY(m_end.has_value()); + return m_end.value(); + } + FlyString const& filename() const + { + return m_filename; + } + void set_end(Position const& end) { m_end = end; } + void set_parent(ASTNode const& parent) { m_parent = &parent; } + + virtual Vector> declarations() const { return {}; } + + virtual bool is_variable_or_parameter_declaration() const { return false; } + virtual bool is_function_call() const { return false; } + virtual bool is_type() const { return false; } + virtual bool is_declaration() const { return false; } + virtual bool is_name() const { return false; } + virtual bool is_member_expression() const { return true; } + virtual bool is_dummy_node() const { return false; } + +protected: + ASTNode(ASTNode const* parent, Optional start, Optional end, String const& filename) + : m_parent(parent) + , m_start(start) + , m_end(end) + , m_filename(filename) + { + } + +private: + ASTNode const* m_parent { nullptr }; + Optional m_start; + Optional m_end; + FlyString m_filename; +}; + +class TranslationUnit : public ASTNode { + +public: + virtual ~TranslationUnit() override = default; + virtual StringView class_name() const override { return "TranslationUnit"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + virtual Vector> declarations() const override { return m_declarations; } + + TranslationUnit(ASTNode const* parent, Optional start, Optional end, String const& filename) + : ASTNode(parent, start, end, filename) + { + } + + void set_declarations(Vector>&& declarations) { m_declarations = move(declarations); } + +private: + Vector> m_declarations; +}; + +class Statement : public ASTNode { +public: + virtual ~Statement() override = default; + virtual StringView class_name() const override { return "Statement"sv; } + + virtual Vector> declarations() const override; + +protected: + Statement(ASTNode const* parent, Optional start, Optional end, String const& filename) + : ASTNode(parent, start, end, filename) + { + } +}; + +class Declaration : public Statement { + +public: + virtual bool is_declaration() const override { return true; } + virtual bool is_variable_declaration() const { return false; } + virtual bool is_parameter() const { return false; } + virtual bool is_struct() const { return false; } + virtual bool is_function() const { return false; } + Name const* name() const { return m_name; } + void set_name(RefPtr name) { m_name = move(name); } + +protected: + Declaration(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Statement(parent, start, end, filename) + { + } + + RefPtr m_name; +}; + +class InvalidDeclaration : public Declaration { + +public: + virtual ~InvalidDeclaration() override = default; + virtual StringView class_name() const override { return "InvalidDeclaration"sv; } + InvalidDeclaration(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Declaration(parent, start, end, filename) + { + } +}; + +class FunctionDeclaration : public Declaration { +public: + virtual ~FunctionDeclaration() override = default; + virtual StringView class_name() const override { return "FunctionDeclaration"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + virtual bool is_function() const override { return true; } + RefPtr definition() { return m_definition; } + + FunctionDeclaration(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Declaration(parent, start, end, filename) + { + } + + virtual Vector> declarations() const override; + Type const* return_type() const { return m_return_type.ptr(); } + void set_return_type(RefPtr const& return_type) { m_return_type = return_type; } + Vector> const& parameters() const { return m_parameters; } + void set_parameters(Vector> const& parameters) { m_parameters = parameters; } + FunctionDefinition const* definition() const { return m_definition.ptr(); } + void set_definition(RefPtr&& definition) { m_definition = move(definition); } + +private: + RefPtr m_return_type; + Vector> m_parameters; + RefPtr m_definition; +}; + +class VariableOrParameterDeclaration : public Declaration { +public: + virtual ~VariableOrParameterDeclaration() override = default; + virtual bool is_variable_or_parameter_declaration() const override { return true; } + + void set_type(RefPtr&& type) { m_type = move(type); } + Type const* type() const { return m_type.ptr(); } + +protected: + VariableOrParameterDeclaration(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Declaration(parent, start, end, filename) + { + } + + RefPtr m_type; +}; + +class Parameter : public VariableOrParameterDeclaration { +public: + virtual ~Parameter() override = default; + virtual StringView class_name() const override { return "Parameter"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + virtual bool is_parameter() const override { return true; } + + Parameter(ASTNode const* parent, Optional start, Optional end, String const& filename, RefPtr name) + : VariableOrParameterDeclaration(parent, start, end, filename) + { + m_name = name; + } +}; + +enum class StorageTypeQualifier { + Const, + In, + Out, + Inout, + Centroid, + Patch, + Sample, + Uniform, + Buffer, + Shared, + Coherent, + Volatile, + Restrict, + Readonly, + Writeonly, + Subroutine, +}; + +class Type : public ASTNode { +public: + virtual ~Type() override = default; + virtual StringView class_name() const override { return "Type"sv; } + virtual bool is_type() const override { return true; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + Type(ASTNode const* parent, Optional start, Optional end, String const& filename) + : ASTNode(parent, start, end, filename) + { + } + + Name const* name() const { return m_name.ptr(); } + void set_name(RefPtr&& name) { m_name = move(name); } + + Vector const& storage_qualifiers() const { return m_storage_qualifiers; } + void set_storage_qualifiers(Vector&& storage_qualifiers) { m_storage_qualifiers = move(storage_qualifiers); } + +private: + RefPtr m_name; + Vector m_storage_qualifiers; +}; + +class FunctionDefinition : public ASTNode { +public: + virtual ~FunctionDefinition() override = default; + virtual StringView class_name() const override { return "FunctionDefinition"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + FunctionDefinition(ASTNode const* parent, Optional start, Optional end, String const& filename) + : ASTNode(parent, start, end, filename) + { + } + + virtual Vector> declarations() const override; + Vector> const& statements() { return m_statements; } + void add_statement(NonnullRefPtr&& statement) { m_statements.append(move(statement)); } + +private: + Vector> m_statements; +}; + +class InvalidStatement : public Statement { +public: + virtual ~InvalidStatement() override = default; + virtual StringView class_name() const override { return "InvalidStatement"sv; } + InvalidStatement(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Statement(parent, start, end, filename) + { + } +}; + +class Expression : public Statement { +public: + virtual ~Expression() override = default; + virtual StringView class_name() const override { return "Expression"sv; } + +protected: + Expression(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Statement(parent, start, end, filename) + { + } +}; + +class InvalidExpression : public Expression { +public: + virtual ~InvalidExpression() override = default; + virtual StringView class_name() const override { return "InvalidExpression"sv; } + InvalidExpression(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Expression(parent, start, end, filename) + { + } +}; + +class VariableDeclaration : public VariableOrParameterDeclaration { +public: + virtual ~VariableDeclaration() override = default; + virtual StringView class_name() const override { return "VariableDeclaration"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + VariableDeclaration(ASTNode const* parent, Optional start, Optional end, String const& filename) + : VariableOrParameterDeclaration(parent, start, end, filename) + { + } + + virtual bool is_variable_declaration() const override { return true; } + + Expression const* initial_value() const { return m_initial_value; } + void set_initial_value(RefPtr&& initial_value) { m_initial_value = move(initial_value); } + +private: + RefPtr m_initial_value; +}; + +class Name : public Expression { +public: + virtual ~Name() override = default; + virtual StringView class_name() const override { return "Name"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + virtual bool is_name() const override { return true; } + virtual bool is_sized() const { return false; } + + Name(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Expression(parent, start, end, filename) + { + } + + StringView name() const { return m_name; } + void set_name(StringView name) { m_name = name; } + +private: + StringView m_name; +}; + +class SizedName : public Name { +public: + virtual ~SizedName() override = default; + virtual StringView class_name() const override { return "SizedName"sv; } + virtual bool is_sized() const override { return true; } + ErrorOr dump(AK::Stream&, size_t indent) const override; + + SizedName(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Name(parent, start, end, filename) + { + } + + void append_dimension(StringView dim) { m_dimensions.append(dim); } + +private: + Vector m_dimensions; +}; + +class NumericLiteral : public Expression { +public: + virtual ~NumericLiteral() override = default; + virtual StringView class_name() const override { return "NumericLiteral"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + NumericLiteral(ASTNode const* parent, Optional start, Optional end, String const& filename, StringView value) + : Expression(parent, start, end, filename) + , m_value(value) + { + } + +private: + StringView m_value; +}; + +class BooleanLiteral : public Expression { +public: + virtual ~BooleanLiteral() override = default; + virtual StringView class_name() const override { return "BooleanLiteral"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + BooleanLiteral(ASTNode const* parent, Optional start, Optional end, String const& filename, bool value) + : Expression(parent, start, end, filename) + , m_value(value) + { + } + +private: + bool m_value; +}; + +enum class BinaryOp { + Addition, + Subtraction, + Multiplication, + Division, + Modulo, + GreaterThan, + GreaterThanEquals, + LessThan, + LessThanEquals, + BitwiseAnd, + BitwiseOr, + BitwiseXor, + LeftShift, + RightShift, + EqualsEquals, + NotEqual, + LogicalOr, + LogicalXor, + LogicalAnd, + Assignment, + AdditionAssignment, + SubtractionAssignment, + MultiplicationAssignment, + DivisionAssignment, + ModuloAssignment, + AndAssignment, + OrAssignment, + XorAssignment, + LeftShiftAssignment, + RightShiftAssignment, +}; + +class BinaryExpression : public Expression { +public: + BinaryExpression(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Expression(parent, start, end, filename) + { + } + + virtual ~BinaryExpression() override = default; + virtual StringView class_name() const override { return "BinaryExpression"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + BinaryOp op() const { return m_op; } + void set_op(BinaryOp op) { m_op = op; } + Expression const* lhs() const { return m_lhs.ptr(); } + void set_lhs(RefPtr&& e) { m_lhs = move(e); } + Expression const* rhs() const { return m_rhs.ptr(); } + void set_rhs(RefPtr&& e) { m_rhs = move(e); } + +private: + BinaryOp m_op; + RefPtr m_lhs; + RefPtr m_rhs; +}; + +class FunctionCall : public Expression { +public: + FunctionCall(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Expression(parent, start, end, filename) + { + } + + virtual ~FunctionCall() override = default; + virtual StringView class_name() const override { return "FunctionCall"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + virtual bool is_function_call() const override { return true; } + + Expression const* callee() const { return m_callee; } + void set_callee(RefPtr&& callee) { m_callee = move(callee); } + + void set_arguments(Vector>&& args) { m_arguments = move(args); } + Vector> const& arguments() const { return m_arguments; } + +private: + RefPtr m_callee; + Vector> m_arguments; +}; + +class StringLiteral final : public Expression { +public: + StringLiteral(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Expression(parent, start, end, filename) + { + } + + ~StringLiteral() override = default; + virtual StringView class_name() const override { return "StringLiteral"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + String const& value() const { return m_value; } + void set_value(String value) { m_value = move(value); } + +private: + String m_value; +}; + +class ReturnStatement : public Statement { +public: + virtual ~ReturnStatement() override = default; + virtual StringView class_name() const override { return "ReturnStatement"sv; } + + ReturnStatement(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Statement(parent, start, end, filename) + { + } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + Expression const* value() const { return m_value.ptr(); } + void set_value(RefPtr&& value) { m_value = move(value); } + +private: + RefPtr m_value; +}; + +class DiscardStatement : public Statement { +public: + virtual ~DiscardStatement() override = default; + virtual StringView class_name() const override { return "DiscardStatement"sv; } + + DiscardStatement(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Statement(parent, start, end, filename) + { + } +}; + +class StructDeclaration : public Declaration { +public: + virtual ~StructDeclaration() override = default; + virtual StringView class_name() const override { return "StructDeclaration"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + virtual bool is_struct() const override { return true; } + virtual Vector> declarations() const override; + + StructDeclaration(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Declaration(parent, start, end, filename) + { + } + + Vector> const& members() const { return m_members; } + void set_members(Vector>&& members) { m_members = move(members); } + +private: + Vector> m_members; +}; + +enum class UnaryOp { + BitwiseNot, + Not, + Plus, + Minus, + PlusPlus, + MinusMinus, +}; + +class UnaryExpression : public Expression { +public: + UnaryExpression(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Expression(parent, start, end, filename) + { + } + + virtual ~UnaryExpression() override = default; + virtual StringView class_name() const override { return "UnaryExpression"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + UnaryOp op() const { return m_op; } + void set_op(UnaryOp op) { m_op = op; } + Expression const* lhs() const { return m_lhs.ptr(); } + void set_lhs(RefPtr&& e) { m_lhs = move(e); } + bool is_postfix() const { return m_is_postfix; } + void set_is_postfix(bool is_postfix) { m_is_postfix = is_postfix; } + +private: + UnaryOp m_op; + RefPtr m_lhs; + bool m_is_postfix = false; +}; + +class MemberExpression : public Expression { +public: + MemberExpression(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Expression(parent, start, end, filename) + { + } + + virtual ~MemberExpression() override = default; + virtual StringView class_name() const override { return "MemberExpression"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + virtual bool is_member_expression() const override { return true; } + + Expression const* object() const { return m_object.ptr(); } + void set_object(RefPtr&& object) { m_object = move(object); } + Expression const* property() const { return m_property.ptr(); } + void set_property(RefPtr&& property) { m_property = move(property); } + +private: + RefPtr m_object; + RefPtr m_property; +}; + +class ArrayElementExpression : public Expression { +public: + ArrayElementExpression(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Expression(parent, start, end, filename) + { + } + + virtual ~ArrayElementExpression() override = default; + virtual StringView class_name() const override { return "ArrayElementExpression"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + Expression const* array() const { return m_array.ptr(); } + void set_array(RefPtr&& array) { m_array = move(array); } + Expression const* index() const { return m_index.ptr(); } + void set_index(RefPtr&& index) { m_index = move(index); } + +private: + RefPtr m_array; + RefPtr m_index; +}; + +class ForStatement : public Statement { +public: + ForStatement(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Statement(parent, start, end, filename) + { + } + + virtual ~ForStatement() override = default; + virtual StringView class_name() const override { return "ForStatement"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + virtual Vector> declarations() const override; + + void set_init(RefPtr&& init) { m_init = move(init); } + void set_test(RefPtr&& test) { m_test = move(test); } + void set_update(RefPtr&& update) { m_update = move(update); } + void set_body(RefPtr&& body) { m_body = move(body); } + Statement const* body() const { return m_body.ptr(); } + +private: + RefPtr m_init; + RefPtr m_test; + RefPtr m_update; + RefPtr m_body; +}; + +class BlockStatement final : public Statement { +public: + BlockStatement(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Statement(parent, start, end, filename) + { + } + + virtual ~BlockStatement() override = default; + virtual StringView class_name() const override { return "BlockStatement"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + + virtual Vector> declarations() const override; + + void add_statement(NonnullRefPtr&& statement) { m_statements.append(move(statement)); } + +private: + Vector> m_statements; +}; + +class IfStatement : public Statement { +public: + IfStatement(ASTNode const* parent, Optional start, Optional end, String const& filename) + : Statement(parent, start, end, filename) + { + } + + virtual ~IfStatement() override = default; + virtual StringView class_name() const override { return "IfStatement"sv; } + virtual ErrorOr dump(AK::Stream&, size_t indent = 0) const override; + virtual Vector> declarations() const override; + + void set_predicate(RefPtr&& predicate) { m_predicate = move(predicate); } + void set_then_statement(RefPtr&& then) { m_then = move(then); } + void set_else_statement(RefPtr&& _else) { m_else = move(_else); } + + Statement const* then_statement() const { return m_then.ptr(); } + Statement const* else_statement() const { return m_else.ptr(); } + +private: + RefPtr m_predicate; + RefPtr m_then; + RefPtr m_else; +}; + +class DummyAstNode : public ASTNode { +public: + DummyAstNode(ASTNode const* parent, Optional start, Optional end, String const& filename) + : ASTNode(parent, start, end, filename) + { + } + virtual bool is_dummy_node() const override { return true; } + virtual StringView class_name() const override { return "DummyAstNode"sv; } + virtual ErrorOr dump(AK::Stream&, size_t = 0) const override { return {}; } +}; + +template<> +inline bool ASTNode::fast_is() const { return is_member_expression(); } +template<> +inline bool ASTNode::fast_is() const { return is_variable_or_parameter_declaration(); } +template<> +inline bool ASTNode::fast_is() const { return is_function_call(); } +template<> +inline bool ASTNode::fast_is() const { return is_type(); } +template<> +inline bool ASTNode::fast_is() const { return is_declaration(); } +template<> +inline bool ASTNode::fast_is() const { return is_name(); } +template<> +inline bool ASTNode::fast_is() const { return is_dummy_node(); } + +template<> +inline bool ASTNode::fast_is() const { return is_declaration() && verify_cast(*this).is_variable_declaration(); } +template<> +inline bool ASTNode::fast_is() const { return is_declaration() && verify_cast(*this).is_function(); } + +template<> +inline bool ASTNode::fast_is() const { return is_name() && verify_cast(*this).is_sized(); } +} diff --git a/Userland/Libraries/LibGLSL/CMakeLists.txt b/Userland/Libraries/LibGLSL/CMakeLists.txt index 7c57d8d9dc..0e98f691c8 100644 --- a/Userland/Libraries/LibGLSL/CMakeLists.txt +++ b/Userland/Libraries/LibGLSL/CMakeLists.txt @@ -1,6 +1,11 @@ set(SOURCES + AST.cpp Compiler.cpp + Lexer.cpp Linker.cpp + Parser.cpp + Preprocessor.cpp + Token.cpp ) serenity_lib(LibGLSL glsl) diff --git a/Userland/Libraries/LibGLSL/Lexer.cpp b/Userland/Libraries/LibGLSL/Lexer.cpp new file mode 100644 index 0000000000..a75523476e --- /dev/null +++ b/Userland/Libraries/LibGLSL/Lexer.cpp @@ -0,0 +1,819 @@ +/* + * Copyright (c) 2018-2020, Andreas Kling + * Copyright (c) 2023, Volodymyr V. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "Lexer.h" +#include +#include +#include +#include +#include + +namespace GLSL { + +Lexer::Lexer(StringView input, size_t start_line) + : m_input(input) + , m_previous_position { start_line, 0 } + , m_position { start_line, 0 } +{ +} + +char Lexer::peek(size_t offset) const +{ + if ((m_index + offset) >= m_input.length()) + return 0; + return m_input[m_index + offset]; +} + +char Lexer::consume() +{ + VERIFY(m_index < m_input.length()); + char ch = m_input[m_index++]; + m_previous_position = m_position; + if (ch == '\n') { + m_position.line++; + m_position.column = 0; + } else { + m_position.column++; + } + return ch; +} + +constexpr bool is_valid_first_character_of_identifier(char ch) +{ + return is_ascii_alpha(ch) || ch == '_' || ch == '$'; +} + +constexpr bool is_valid_nonfirst_character_of_identifier(char ch) +{ + return is_valid_first_character_of_identifier(ch) || is_ascii_digit(ch); +} + +// NOTE: some of these keywords are not used at the moment, however they are reserved for future use and should not be used as identifiers +constexpr Array s_known_keywords = { + "asm"sv, + "attribute"sv, + "break"sv, + "case"sv, + "cast"sv, + "centroid"sv, + "class"sv, + "common" + "partition"sv, + "active"sv, + "const"sv, + "continue"sv, + "default"sv, + "discard"sv, + "do"sv, + "else"sv, + "enum"sv, + "extern"sv, + "external"sv, + "false"sv, + "filter"sv, + "fixed"sv, + "flat"sv, + "for"sv, + "goto"sv, + "half"sv, + "highp"sv, + "if"sv, + "in"sv, + "inline"sv, + "inout"sv, + "input"sv, + "interface"sv, + "invariant"sv, + "layout"sv, + "lowp"sv, + "mediump"sv, + "namespace"sv, + "noinline"sv, + "noperspective"sv, + "out"sv, + "output"sv, + "packed"sv, + "patch"sv, + "precision"sv, + "public"sv, + "return"sv, + "row_major"sv, + "sample"sv, + "sizeof"sv, + "smooth"sv, + "static"sv, + "struct"sv, + "subroutine"sv, + "superp"sv, + "switch"sv, + "template"sv, + "this"sv, + "true"sv, + "typedef"sv, + "uniform"sv, + "union"sv, + "using"sv, + "varying"sv, + "volatile"sv, + "while"sv, +}; + +constexpr Array s_known_types = { + "bool"sv, + "bvec2"sv, + "bvec3"sv, + "bvec4"sv, + "dmat2"sv, + "dmat2x2"sv, + "dmat2x3"sv, + "dmat2x4"sv, + "dmat3"sv, + "dmat3x2"sv, + "dmat3x3"sv, + "dmat3x4"sv, + "dmat4"sv, + "dmat4x2"sv, + "dmat4x3"sv, + "dmat4x4"sv, + "double"sv, + "dvec2"sv, + "dvec3"sv, + "dvec4"sv, + "float"sv, + "fvec2"sv, + "fvec3"sv, + "fvec4"sv, + "hvec2"sv, + "hvec3"sv, + "hvec4"sv, + "iimage1D"sv, + "iimage1DArray"sv, + "iimage2D"sv, + "iimage2DArray"sv, + "iimage3D"sv, + "iimageBuffer"sv, + "iimageCube"sv, + "image1D"sv, + "image1DArray"sv, + "image1DArrayShadow"sv, + "image1DShadow"sv, + "image2D"sv, + "image2DArray"sv, + "image2DArrayShadow"sv, + "image2DShadow"sv, + "image3D"sv, + "imageBuffer"sv, + "imageCube"sv, + "int"sv, + "isampler1D"sv, + "isampler1DArray"sv, + "isampler2D"sv, + "isampler2DArray"sv, + "isampler2DMS"sv, + "isampler2DMSArray"sv, + "isampler2DRect"sv, + "isampler3D"sv, + "isamplerBuffer"sv, + "isamplerCube"sv, + "isamplerCubeArray"sv, + "ivec2"sv, + "ivec3"sv, + "ivec4"sv, + "long"sv, + "mat2"sv, + "mat2x2"sv, + "mat2x3"sv, + "mat2x4"sv, + "mat3"sv, + "mat3x2"sv, + "mat3x3"sv, + "mat3x4"sv, + "mat4"sv, + "mat4x2"sv, + "mat4x3"sv, + "mat4x4"sv, + "sampler1D"sv, + "sampler1DArray"sv, + "sampler1DArrayShadow"sv, + "sampler1DShadow"sv, + "sampler2D"sv, + "sampler2DArray"sv, + "sampler2DArrayShadow"sv, + "sampler2DMS"sv, + "sampler2DMSArray"sv, + "sampler2DRect"sv, + "sampler2DRectShadow"sv, + "sampler2DShadow"sv, + "sampler3D"sv, + "sampler3DRect"sv, + "samplerBuffer"sv, + "samplerCube"sv, + "samplerCubeArray"sv, + "samplerCubeArrayShadow"sv, + "samplerCubeShadow"sv, + "short"sv, + "uimage1D"sv, + "uimage1DArray"sv, + "uimage2D"sv, + "uimage2DArray"sv, + "uimage3D"sv, + "uimageBuffer"sv, + "uimageCube"sv, + "uint"sv, + "unsigned"sv, + "usampler1D"sv, + "usampler1DArray"sv, + "usampler2D"sv, + "usampler2DArray"sv, + "usampler2DMS"sv, + "usampler2DMSArray"sv, + "usampler2DRect"sv, + "usampler3D"sv, + "usamplerBuffer"sv, + "usamplerCube"sv, + "usamplerCubeArray"sv, + "uvec2"sv, + "uvec3"sv, + "uvec4"sv, + "vec2"sv, + "vec3"sv, + "vec4"sv, + "void"sv, +}; + +static bool is_keyword(StringView string) +{ + return AK::find(s_known_keywords.begin(), s_known_keywords.end(), string) != s_known_keywords.end(); +} + +static bool is_known_type(StringView string) +{ + return AK::find(s_known_types.begin(), s_known_types.end(), string) != s_known_types.end(); +} + +void Lexer::lex_impl(Function callback) +{ + size_t token_start_index = 0; + Position token_start_position; + + auto emit_single_char_token = [&](auto type) { + callback(Token(type, m_position, m_position, m_input.substring_view(m_index, 1))); + consume(); + }; + + auto begin_token = [&] { + token_start_index = m_index; + token_start_position = m_position; + }; + auto commit_token = [&](auto type) { + if (m_options.ignore_whitespace && type == Token::Type::Whitespace) + return; + callback(Token(type, token_start_position, m_previous_position, m_input.substring_view(token_start_index, m_index - token_start_index))); + }; + + auto emit_token_equals = [&](auto type, auto equals_type) { + if (peek(1) == '=') { + begin_token(); + consume(); + consume(); + commit_token(equals_type); + return; + } + emit_single_char_token(type); + }; + + auto match_escape_sequence = [&]() -> size_t { + switch (peek(1)) { + case '\'': + case '"': + case '?': + case '\\': + case 'a': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + return 2; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': { + size_t octal_digits = 1; + for (size_t i = 0; i < 2; ++i) { + char next = peek(2 + i); + if (next < '0' || next > '7') + break; + ++octal_digits; + } + return 1 + octal_digits; + } + case 'x': { + size_t hex_digits = 0; + while (is_ascii_hex_digit(peek(2 + hex_digits))) + ++hex_digits; + return 2 + hex_digits; + } + case 'u': + case 'U': { + bool is_unicode = true; + size_t number_of_digits = peek(1) == 'u' ? 4 : 8; + for (size_t i = 0; i < number_of_digits; ++i) { + if (!is_ascii_hex_digit(peek(2 + i))) { + is_unicode = false; + break; + } + } + return is_unicode ? 2 + number_of_digits : 0; + } + default: + return 0; + } + }; + + auto match_string_prefix = [&](char quote) -> size_t { + if (peek() == quote) + return 1; + if (peek() == 'L' && peek(1) == quote) + return 2; + if (peek() == 'u') { + if (peek(1) == quote) + return 2; + if (peek(1) == '8' && peek(2) == quote) + return 3; + } + if (peek() == 'U' && peek(1) == quote) + return 2; + return 0; + }; + + while (m_index < m_input.length()) { + auto ch = peek(); + if (is_ascii_space(ch)) { + begin_token(); + while (is_ascii_space(peek())) + consume(); + commit_token(Token::Type::Whitespace); + continue; + } + if (ch == '(') { + emit_single_char_token(Token::Type::LeftParen); + continue; + } + if (ch == ')') { + emit_single_char_token(Token::Type::RightParen); + continue; + } + if (ch == '{') { + emit_single_char_token(Token::Type::LeftCurly); + continue; + } + if (ch == '}') { + emit_single_char_token(Token::Type::RightCurly); + continue; + } + if (ch == '[') { + emit_single_char_token(Token::Type::LeftBracket); + continue; + } + if (ch == ']') { + emit_single_char_token(Token::Type::RightBracket); + continue; + } + if (ch == '<') { + begin_token(); + consume(); + if (peek() == '<') { + consume(); + if (peek() == '=') { + consume(); + commit_token(Token::Type::LessLessEquals); + continue; + } + commit_token(Token::Type::LessLess); + continue; + } + if (peek() == '=') { + consume(); + commit_token(Token::Type::LessEquals); + continue; + } + commit_token(Token::Type::Less); + continue; + } + if (ch == '>') { + begin_token(); + consume(); + if (peek() == '>') { + consume(); + if (peek() == '=') { + consume(); + commit_token(Token::Type::GreaterGreaterEquals); + continue; + } + commit_token(Token::Type::GreaterGreater); + continue; + } + if (peek() == '=') { + consume(); + commit_token(Token::Type::GreaterEquals); + continue; + } + commit_token(Token::Type::Greater); + continue; + } + if (ch == ',') { + emit_single_char_token(Token::Type::Comma); + continue; + } + if (ch == '+') { + begin_token(); + consume(); + if (peek() == '+') { + consume(); + commit_token(Token::Type::PlusPlus); + continue; + } + if (peek() == '=') { + consume(); + commit_token(Token::Type::PlusEquals); + continue; + } + commit_token(Token::Type::Plus); + continue; + } + if (ch == '-') { + begin_token(); + consume(); + if (peek() == '-') { + consume(); + commit_token(Token::Type::MinusMinus); + continue; + } + if (peek() == '=') { + consume(); + commit_token(Token::Type::MinusEquals); + continue; + } + commit_token(Token::Type::Minus); + continue; + } + if (ch == '*') { + emit_token_equals(Token::Type::Asterisk, Token::Type::AsteriskEquals); + continue; + } + if (ch == '%') { + emit_token_equals(Token::Type::Percent, Token::Type::PercentEquals); + continue; + } + if (ch == '^') { + begin_token(); + consume(); + if (peek() == '^') { + consume(); + commit_token(Token::Type::CaretCaret); + continue; + } + if (peek() == '=') { + consume(); + commit_token(Token::Type::CaretEquals); + continue; + } + commit_token(Token::Type::Caret); + continue; + } + if (ch == '!') { + emit_token_equals(Token::Type::ExclamationMark, Token::Type::ExclamationMarkEquals); + continue; + } + if (ch == '=') { + emit_token_equals(Token::Type::Equals, Token::Type::EqualsEquals); + continue; + } + if (ch == '&') { + begin_token(); + consume(); + if (peek() == '&') { + consume(); + commit_token(Token::Type::AndAnd); + continue; + } + if (peek() == '=') { + consume(); + commit_token(Token::Type::AndEquals); + continue; + } + commit_token(Token::Type::And); + continue; + } + if (ch == '|') { + begin_token(); + consume(); + if (peek() == '|') { + consume(); + commit_token(Token::Type::PipePipe); + continue; + } + if (peek() == '=') { + consume(); + commit_token(Token::Type::PipeEquals); + continue; + } + commit_token(Token::Type::Pipe); + continue; + } + if (ch == '~') { + emit_single_char_token(Token::Type::Tilde); + continue; + } + if (ch == '?') { + emit_single_char_token(Token::Type::QuestionMark); + continue; + } + if (ch == ':') { + emit_single_char_token(Token::Type::Colon); + continue; + } + if (ch == ';') { + emit_single_char_token(Token::Type::Semicolon); + continue; + } + if (ch == '.') { + emit_single_char_token(Token::Type::Dot); + continue; + } + if (ch == '#') { + begin_token(); + consume(); + while (AK::is_ascii_space(peek())) + consume(); + + size_t directive_start = m_index; + if (is_valid_first_character_of_identifier(peek())) + while (peek() && is_valid_nonfirst_character_of_identifier(peek())) + consume(); + + auto directive = StringView(m_input.characters_without_null_termination() + directive_start, m_index - directive_start); + if (directive == "include"sv) { + commit_token(Token::Type::IncludeStatement); + + if (is_ascii_space(peek())) { + begin_token(); + do { + consume(); + } while (is_ascii_space(peek())); + commit_token(Token::Type::Whitespace); + } + + begin_token(); + if (peek() == '<' || peek() == '"') { + char closing = consume() == '<' ? '>' : '"'; + while (peek() && peek() != closing && peek() != '\n') + consume(); + + if (peek() && consume() == '\n') { + commit_token(Token::Type::IncludePath); + continue; + } + + commit_token(Token::Type::IncludePath); + begin_token(); + } + } else { + while (peek()) { + if (peek() == '\\' && peek(1) == '\n') { + consume(); + consume(); + } else if (peek() == '\n') { + break; + } else { + consume(); + } + } + + commit_token(Token::Type::PreprocessorStatement); + } + + continue; + } + if (ch == '/' && peek(1) == '/') { + while (peek() && peek() != '\n') + consume(); + continue; + } + if (ch == '/' && peek(1) == '*') { + consume(); + consume(); + bool comment_block_ends = false; + while (peek()) { + if (peek() == '*' && peek(1) == '/') { + comment_block_ends = true; + break; + } + + consume(); + } + + if (comment_block_ends) { + consume(); + consume(); + } + continue; + } + if (ch == '/') { + emit_token_equals(Token::Type::Slash, Token::Type::SlashEquals); + continue; + } + if (size_t prefix = match_string_prefix('"'); prefix > 0) { + begin_token(); + for (size_t i = 0; i < prefix; ++i) + consume(); + while (peek()) { + if (peek() == '\\') { + if (size_t escape = match_escape_sequence(); escape > 0) { + commit_token(Token::Type::DoubleQuotedString); + begin_token(); + for (size_t i = 0; i < escape; ++i) + consume(); + commit_token(Token::Type::EscapeSequence); + begin_token(); + continue; + } + } + + // If string is not terminated - stop before EOF + if (!peek(1)) + break; + + if (consume() == '"') + break; + } + commit_token(Token::Type::DoubleQuotedString); + continue; + } + if (size_t prefix = match_string_prefix('R'); prefix > 0 && peek(prefix) == '"') { + begin_token(); + for (size_t i = 0; i < prefix + 1; ++i) + consume(); + size_t prefix_start = m_index; + while (peek() && peek() != '(') + consume(); + StringView prefix_string = m_input.substring_view(prefix_start, m_index - prefix_start); + while (peek()) { + if (consume() == '"') { + VERIFY(m_index >= prefix_string.length() + 2); + VERIFY(m_input[m_index - 1] == '"'); + if (m_input[m_index - 1 - prefix_string.length() - 1] == ')') { + StringView suffix_string = m_input.substring_view(m_index - 1 - prefix_string.length(), prefix_string.length()); + if (prefix_string == suffix_string) + break; + } + } + } + commit_token(Token::Type::RawString); + continue; + } + if (size_t prefix = match_string_prefix('\''); prefix > 0) { + begin_token(); + for (size_t i = 0; i < prefix; ++i) + consume(); + while (peek()) { + if (peek() == '\\') { + if (size_t escape = match_escape_sequence(); escape > 0) { + commit_token(Token::Type::SingleQuotedString); + begin_token(); + for (size_t i = 0; i < escape; ++i) + consume(); + commit_token(Token::Type::EscapeSequence); + begin_token(); + continue; + } + } + + if (consume() == '\'') + break; + } + commit_token(Token::Type::SingleQuotedString); + continue; + } + if (is_ascii_digit(ch) || (ch == '.' && is_ascii_digit(peek(1)))) { + begin_token(); + consume(); + + auto type = ch == '.' ? Token::Type::Float : Token::Type::Integer; + bool is_hex = false; + bool is_binary = false; + + auto match_exponent = [&]() -> size_t { + char ch = peek(); + if (ch != 'e' && ch != 'E' && ch != 'p' && ch != 'P') + return 0; + + type = Token::Type::Float; + size_t length = 1; + ch = peek(length); + if (ch == '+' || ch == '-') { + ++length; + } + for (ch = peek(length); is_ascii_digit(ch); ch = peek(length)) { + ++length; + } + return length; + }; + + auto match_type_literal = [&]() -> size_t { + size_t length = 0; + for (;;) { + char ch = peek(length); + if ((ch == 'u' || ch == 'U') && type == Token::Type::Integer) { + ++length; + } else if ((ch == 'f' || ch == 'F') && !is_binary) { + type = Token::Type::Float; + ++length; + } else if (ch == 'l' || ch == 'L') { + ++length; + } else + return length; + } + }; + + if (peek() == 'b' || peek() == 'B') { + consume(); + is_binary = true; + for (char ch = peek(); ch == '0' || ch == '1' || (ch == '\'' && peek(1) != '\''); ch = peek()) { + consume(); + } + } else { + if (peek() == 'x' || peek() == 'X') { + consume(); + is_hex = true; + } + + for (char ch = peek(); (is_hex ? is_ascii_hex_digit(ch) : is_ascii_digit(ch)) || (ch == '\'' && peek(1) != '\'') || ch == '.'; ch = peek()) { + if (ch == '.') { + if (type == Token::Type::Integer) { + type = Token::Type::Float; + } else + break; + }; + consume(); + } + } + + if (!is_binary) { + size_t length = match_exponent(); + for (size_t i = 0; i < length; ++i) + consume(); + } + + size_t length = match_type_literal(); + for (size_t i = 0; i < length; ++i) + consume(); + + commit_token(type); + continue; + } + if (is_valid_first_character_of_identifier(ch)) { + begin_token(); + while (peek() && is_valid_nonfirst_character_of_identifier(peek())) + consume(); + auto token_view = StringView(m_input.characters_without_null_termination() + token_start_index, m_index - token_start_index); + if (is_keyword(token_view)) + commit_token(Token::Type::Keyword); + else if (is_known_type(token_view)) + commit_token(Token::Type::KnownType); + else + commit_token(Token::Type::Identifier); + continue; + } + + if (ch == '\\' && peek(1) == '\n') { + consume(); + consume(); + continue; + } + + dbgln("Unimplemented token character: {}", ch); + emit_single_char_token(Token::Type::Unknown); + } +} + +Vector Lexer::lex() +{ + Vector tokens; + lex_impl([&](auto token) { + tokens.append(move(token)); + }); + return tokens; +} + +} diff --git a/Userland/Libraries/LibGLSL/Lexer.h b/Userland/Libraries/LibGLSL/Lexer.h new file mode 100644 index 0000000000..9d9451ce56 --- /dev/null +++ b/Userland/Libraries/LibGLSL/Lexer.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2018-2020, Andreas Kling + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +namespace GLSL { + +class Lexer { +public: + explicit Lexer(StringView, size_t start_line = 0); + + Vector lex(); + template + void lex_iterable(Callback); + + void set_ignore_whitespace(bool value) { m_options.ignore_whitespace = value; } + +private: + char peek(size_t offset = 0) const; + char consume(); + void lex_impl(Function); + + StringView m_input; + size_t m_index { 0 }; + Position m_previous_position { 0, 0 }; + Position m_position { 0, 0 }; + + struct Options { + bool ignore_whitespace { false }; + } m_options; +}; + +template +void Lexer::lex_iterable(Callback callback) +{ + return lex_impl(move(callback)); +} + +} diff --git a/Userland/Libraries/LibGLSL/Parser.cpp b/Userland/Libraries/LibGLSL/Parser.cpp new file mode 100644 index 0000000000..aa3275883a --- /dev/null +++ b/Userland/Libraries/LibGLSL/Parser.cpp @@ -0,0 +1,1142 @@ +/* + * Copyright (c) 2021, Itamar S. + * Copyright (c) 2023, Volodymyr V. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "Parser.h" +#include "AST.h" +#include + +namespace GLSL { + +Parser::Parser(Vector tokens, String const& filename) + : m_filename(move(filename)) + , m_tokens(move(tokens)) +{ +} + +ErrorOr> Parser::parse() +{ + if (m_tokens.is_empty()) + return create_root_ast_node({}, {}); + auto unit = create_root_ast_node(m_tokens.first().start(), m_tokens.last().end()); + unit->set_declarations(TRY(parse_declarations_in_translation_unit(*unit))); + return unit; +} + +bool Parser::eof() const +{ + return m_state.token_index >= m_tokens.size(); +} + +void Parser::print_tokens() const +{ + for (auto& token : m_tokens) { + outln("{}", token.to_string()); + } +} + +ErrorOr> Parser::match_declaration_in_translation_unit() +{ + if (TRY(match_variable_declaration())) + return DeclarationType::Variable; + if (TRY(match_function_declaration())) + return DeclarationType::Function; + if (TRY(match_struct_declaration())) + return DeclarationType::Struct; + return Optional(); +} + +ErrorOr Parser::match_struct_declaration() +{ + save_state(); + ScopeGuard state_guard = [this] { load_state(); }; + + if (!match_keyword("struct"sv)) + return false; + TRY(consume(Token::Type::Keyword)); + + if (!match(Token::Type::Identifier)) + return false; + TRY(consume(Token::Type::Identifier)); + + return match(Token::Type::LeftCurly); +} + +ErrorOr Parser::match_function_declaration() +{ + save_state(); + ScopeGuard state_guard = [this] { load_state(); }; + + if (!TRY(match_type())) + return false; + + VERIFY(m_root_node); + (void)parse_type(get_dummy_node()); + + if (!TRY(match_name())) + return false; + + (void)parse_name(get_dummy_node()); + + if (!peek(Token::Type::LeftParen).has_value()) + return false; + TRY(consume()); + + while (TRY(consume()).type() != Token::Type::RightParen && !eof()) + ; + + if (peek(Token::Type::Semicolon).has_value() || peek(Token::Type::LeftCurly).has_value()) + return true; + + return false; +} + +ErrorOr Parser::match_variable_declaration() +{ + save_state(); + ScopeGuard state_guard = [this] { load_state(); }; + + if (!TRY(match_type())) { + return false; + } + + VERIFY(m_root_node); + (void)parse_type(get_dummy_node()); + + // Identifier + if (!TRY(match_name())) + return false; + + (void)TRY(parse_name(get_dummy_node())); + + while (!eof() && (peek().type() == Token::Type::LeftBracket)) { + TRY(consume(Token::Type::LeftBracket)); + + if (match(Token::Type::Integer)) { + TRY(consume(Token::Type::Integer)); + } + if (!match(Token::Type::RightBracket)) { + TRY(error("No closing right bracket"sv)); + return false; + } + TRY(consume(Token::Type::RightBracket)); + } + + if (match(Token::Type::Equals)) { + TRY(consume(Token::Type::Equals)); + if (!TRY(match_expression())) { + TRY(error("initial value of variable is not an expression"sv)); + return false; + } + return true; + } + + return match(Token::Type::Semicolon); +} + +ErrorOr Parser::match_block_statement() +{ + return peek().type() == Token::Type::LeftCurly; +} + +ErrorOr Parser::match_expression() +{ + return TRY(match_name()) + || match_unary_op() + || match(Token::Type::LeftParen) + || TRY(match_boolean_literal()) + || TRY(match_numeric_literal()) + || TRY(match_string_literal()); +} + +ErrorOr Parser::match_name() +{ + auto type = peek().type(); + return type == Token::Type::Identifier || type == Token::Type::KnownType; +} + +ErrorOr Parser::match_string_literal() +{ + return match(Token::Type::DoubleQuotedString) || match(Token::Type::SingleQuotedString); +} + +ErrorOr Parser::match_numeric_literal() +{ + return match(Token::Type::Float) || match(Token::Type::Integer); +} + +ErrorOr Parser::match_boolean_literal() +{ + auto token = peek(); + if (token.type() != Token::Type::Keyword) + return false; + auto text = token.text(); + return text == "true" || text == "false"; +} + +ErrorOr Parser::match_type() +{ + save_state(); + ScopeGuard state_guard = [this] { load_state(); }; + + if (match_storage_qualifier()) + TRY(consume_storage_qualifier()); + + if (!TRY(match_name())) + return false; + + return true; +} + +ErrorOr>> Parser::parse_declarations_in_translation_unit(ASTNode const& parent) +{ + Vector> declarations; + while (!eof()) { + auto declaration = TRY(parse_single_declaration_in_translation_unit(parent)); + if (declaration) { + declarations.append(declaration.release_nonnull()); + } else { + TRY(error("unexpected token"sv)); + TRY(consume()); + } + } + return declarations; +} + +ErrorOr> Parser::parse_single_declaration_in_translation_unit(ASTNode const& parent) +{ + while (!eof()) { + if (match_preprocessor()) { + TRY(consume_preprocessor()); + continue; + } + + auto declaration = TRY(match_declaration_in_translation_unit()); + if (declaration.has_value()) { + return parse_declaration(parent, declaration.value()); + } + return nullptr; + } + return nullptr; +} + +ErrorOr> Parser::parse_declaration(ASTNode const& parent, DeclarationType declaration_type) +{ + switch (declaration_type) { + case DeclarationType::Function: + return parse_function_declaration(parent); + case DeclarationType::Variable: + return parse_variable_declaration(parent); + case DeclarationType::Struct: + return parse_struct_declaration(parent); + default: + TRY(error("unexpected declaration type"sv)); + return create_ast_node(parent, position(), position()); + } +} + +ErrorOr> Parser::parse_struct_declaration(ASTNode const& parent) +{ + TRY(consume_keyword("struct"sv)); + + auto decl = create_ast_node(parent, position(), {}); + decl->set_name(TRY(parse_name(*decl))); + + TRY(consume(Token::Type::LeftCurly)); + while (!eof() && peek().type() != Token::Type::RightCurly) { + decl->set_members(TRY(parse_struct_members(*decl))); + } + TRY(consume(Token::Type::RightCurly)); + + TRY(consume(Token::Type::Semicolon)); + decl->set_end(position()); + + return decl; +} + +ErrorOr>> Parser::parse_struct_members(StructDeclaration& parent) +{ + Vector> members; + while (!eof() && peek().type() != Token::Type::RightCurly) { + auto member = TRY(parse_variable_declaration(parent)); + members.append(move(member)); + } + return members; +} + +ErrorOr> Parser::parse_function_declaration(ASTNode const& parent) +{ + auto func = create_ast_node(parent, position(), {}); + + func->set_return_type(TRY(parse_type(*func))); + func->set_name(TRY(parse_name(*func))); + + TRY(consume(Token::Type::LeftParen)); + func->set_parameters(TRY(parse_parameter_list(*func))); + TRY(consume(Token::Type::RightParen)); + + RefPtr body; + Position func_end {}; + if (peek(Token::Type::LeftCurly).has_value()) { + body = TRY(parse_function_definition(*func)); + func_end = body->end(); + } else { + func_end = position(); + TRY(consume(Token::Type::Semicolon)); + } + + func->set_definition(move(body)); + func->set_end(func_end); + return func; +} + +ErrorOr>> Parser::parse_parameter_list(ASTNode const& parent) +{ + Vector> parameters; + while (peek().type() != Token::Type::RightParen && !eof()) { + auto type = TRY(parse_type(parent)); + + RefPtr name; + if (TRY(match_name())) + name = TRY(parse_name(parent)); + + auto param = create_ast_node(parent, type->start(), !name.is_null() ? name->end() : type->end(), name); + const_cast(*type).set_parent(*param.ptr()); + + param->set_type(move(type)); + parameters.append(move(param)); + + if (peek(Token::Type::Comma).has_value()) + TRY(consume(Token::Type::Comma)); + } + return parameters; +} + +ErrorOr> Parser::parse_function_definition(ASTNode const& parent) +{ + auto func = create_ast_node(parent, position(), {}); + + TRY(consume(Token::Type::LeftCurly)); + while (!eof() && peek().type() != Token::Type::RightCurly) { + func->add_statement(TRY(parse_statement(func))); + } + func->set_end(position()); + TRY(consume(Token::Type::RightCurly)); + + return func; +} + +ErrorOr> Parser::parse_variable_declaration(ASTNode const& parent, bool expect_semicolon) +{ + auto var = create_ast_node(parent, position(), {}); + if (!TRY(match_variable_declaration())) { + TRY(error("unexpected token for variable type"sv)); + var->set_end(position()); + return var; + } + var->set_type(TRY(parse_type(var))); + auto name = TRY(parse_name(*var, true)); + RefPtr initial_value; + + if (match(Token::Type::Equals)) { + TRY(consume(Token::Type::Equals)); + initial_value = TRY(parse_expression(var)); + } + + if (expect_semicolon) + TRY(consume(Token::Type::Semicolon)); + + var->set_end(position()); + var->set_name(name); + var->set_initial_value(move(initial_value)); + + return var; +} + +ErrorOr> Parser::parse_statement(ASTNode const& parent) +{ + bool should_consume_semicolon = true; + RefPtr result; + + if (TRY(match_block_statement())) { + should_consume_semicolon = false; + result = TRY(parse_block_statement(parent)); + } else if (TRY(match_variable_declaration())) { + result = TRY(parse_variable_declaration(parent, false)); + } else if (TRY(match_expression())) { + result = TRY(parse_expression(parent)); + } else if (match_keyword("return"sv)) { + result = TRY(parse_return_statement(parent)); + } else if (match_keyword("discard"sv)) { + auto start = position(); + TRY(consume()); + result = create_ast_node(parent, start, position()); + } else if (match_keyword("for"sv)) { + should_consume_semicolon = false; + result = TRY(parse_for_statement(parent)); + } else if (match_keyword("if"sv)) { + should_consume_semicolon = false; + result = TRY(parse_if_statement(parent)); + } else { + TRY(error("unexpected statement type"sv)); + should_consume_semicolon = false; + TRY(consume()); + return create_ast_node(parent, position(), position()); + } + + if (should_consume_semicolon) + TRY(consume(Token::Type::Semicolon)); + return result.release_nonnull(); +} + +ErrorOr> Parser::parse_block_statement(ASTNode const& parent) +{ + auto block_statement = create_ast_node(parent, position(), {}); + + TRY(consume(Token::Type::LeftCurly)); + while (!eof() && peek().type() != Token::Type::RightCurly) { + block_statement->add_statement(TRY(parse_statement(*block_statement))); + } + TRY(consume(Token::Type::RightCurly)); + + block_statement->set_end(position()); + return block_statement; +} + +ErrorOr> Parser::parse_if_statement(ASTNode const& parent) +{ + auto if_statement = create_ast_node(parent, position(), {}); + TRY(consume_keyword("if"sv)); + TRY(consume(Token::Type::LeftParen)); + if_statement->set_predicate(TRY(parse_expression(*if_statement))); + TRY(consume(Token::Type::RightParen)); + if_statement->set_then_statement(TRY(parse_statement(*if_statement))); + if (match_keyword("else"sv)) { + TRY(consume(Token::Type::Keyword)); + if_statement->set_else_statement(TRY(parse_statement(*if_statement))); + if_statement->set_end(if_statement->else_statement()->end()); + } else { + if_statement->set_end(if_statement->then_statement()->end()); + } + return if_statement; +} + +ErrorOr> Parser::parse_for_statement(ASTNode const& parent) +{ + auto for_statement = create_ast_node(parent, position(), {}); + TRY(consume_keyword("for"sv)); + TRY(consume(Token::Type::LeftParen)); + if (peek().type() != Token::Type::Semicolon) + for_statement->set_init(TRY(parse_variable_declaration(*for_statement, false))); + TRY(consume(Token::Type::Semicolon)); + + if (peek().type() != Token::Type::Semicolon) + for_statement->set_test(TRY(parse_expression(*for_statement))); + TRY(consume(Token::Type::Semicolon)); + + if (peek().type() != Token::Type::RightParen) + for_statement->set_update(TRY(parse_expression(*for_statement))); + TRY(consume(Token::Type::RightParen)); + + for_statement->set_body(TRY(parse_statement(*for_statement))); + + for_statement->set_end(for_statement->body()->end()); + return for_statement; +} + +ErrorOr> Parser::parse_return_statement(ASTNode const& parent) +{ + auto return_statement = create_ast_node(parent, position(), {}); + TRY(consume_keyword("return"sv)); + if (!peek(Token::Type::Semicolon).has_value()) { + return_statement->set_value(TRY(parse_expression(*return_statement))); + } + return_statement->set_end(position()); + return return_statement; +} + +HashMap s_operator_precedence = { + { BinaryOp::Assignment, 1 }, + { BinaryOp::AdditionAssignment, 1 }, + { BinaryOp::SubtractionAssignment, 1 }, + { BinaryOp::MultiplicationAssignment, 1 }, + { BinaryOp::DivisionAssignment, 1 }, + { BinaryOp::ModuloAssignment, 1 }, + { BinaryOp::AndAssignment, 1 }, + { BinaryOp::XorAssignment, 1 }, + { BinaryOp::OrAssignment, 1 }, + { BinaryOp::LeftShiftAssignment, 1 }, + { BinaryOp::RightShiftAssignment, 1 }, + { BinaryOp::LogicalOr, 2 }, + { BinaryOp::LogicalXor, 3 }, + { BinaryOp::LogicalAnd, 4 }, + { BinaryOp::BitwiseOr, 5 }, + { BinaryOp::BitwiseXor, 6 }, + { BinaryOp::BitwiseAnd, 7 }, + { BinaryOp::EqualsEquals, 8 }, + { BinaryOp::NotEqual, 8 }, + { BinaryOp::LessThan, 9 }, + { BinaryOp::LessThanEquals, 9 }, + { BinaryOp::GreaterThan, 9 }, + { BinaryOp::GreaterThanEquals, 9 }, + { BinaryOp::LeftShift, 10 }, + { BinaryOp::RightShift, 10 }, + { BinaryOp::Addition, 11 }, + { BinaryOp::Subtraction, 11 }, + { BinaryOp::Multiplication, 12 }, + { BinaryOp::Division, 12 }, + { BinaryOp::Modulo, 12 }, +}; + +HashMap Parser::s_operator_associativity = { + { BinaryOp::Assignment, Associativity::RightToLeft }, + { BinaryOp::AdditionAssignment, Associativity::RightToLeft }, + { BinaryOp::SubtractionAssignment, Associativity::RightToLeft }, + { BinaryOp::MultiplicationAssignment, Associativity::RightToLeft }, + { BinaryOp::DivisionAssignment, Associativity::RightToLeft }, + { BinaryOp::ModuloAssignment, Associativity::RightToLeft }, + { BinaryOp::AndAssignment, Associativity::RightToLeft }, + { BinaryOp::XorAssignment, Associativity::RightToLeft }, + { BinaryOp::OrAssignment, Associativity::RightToLeft }, + { BinaryOp::LeftShiftAssignment, Associativity::RightToLeft }, + { BinaryOp::RightShiftAssignment, Associativity::RightToLeft }, + { BinaryOp::LogicalOr, Associativity::LeftToRight }, + { BinaryOp::LogicalXor, Associativity::LeftToRight }, + { BinaryOp::LogicalAnd, Associativity::LeftToRight }, + { BinaryOp::BitwiseOr, Associativity::LeftToRight }, + { BinaryOp::BitwiseXor, Associativity::LeftToRight }, + { BinaryOp::BitwiseAnd, Associativity::LeftToRight }, + { BinaryOp::EqualsEquals, Associativity::LeftToRight }, + { BinaryOp::NotEqual, Associativity::LeftToRight }, + { BinaryOp::LessThan, Associativity::LeftToRight }, + { BinaryOp::LessThanEquals, Associativity::LeftToRight }, + { BinaryOp::GreaterThan, Associativity::LeftToRight }, + { BinaryOp::GreaterThanEquals, Associativity::LeftToRight }, + { BinaryOp::LeftShift, Associativity::LeftToRight }, + { BinaryOp::RightShift, Associativity::LeftToRight }, + { BinaryOp::Addition, Associativity::LeftToRight }, + { BinaryOp::Subtraction, Associativity::LeftToRight }, + { BinaryOp::Multiplication, Associativity::LeftToRight }, + { BinaryOp::Division, Associativity::LeftToRight }, + { BinaryOp::Modulo, Associativity::LeftToRight }, +}; + +ErrorOr> Parser::parse_expression(ASTNode const& parent, int min_precedence, Associativity associativity) +{ + auto start_pos = position(); + + auto lhs = TRY(parse_unary_expression(get_dummy_node())); + + while (match_binary_op()) { + auto op = TRY(peek_binary_op()); + auto maybe_op_precedence = s_operator_precedence.get(op); + VERIFY(maybe_op_precedence.has_value()); + + auto op_precedence = maybe_op_precedence.value(); + if (op_precedence < min_precedence || (op_precedence == min_precedence && associativity == Associativity::LeftToRight)) + break; + TRY(consume()); + + auto maybe_op_associativity = s_operator_associativity.get(op); + VERIFY(maybe_op_associativity.has_value()); + auto op_associativity = maybe_op_associativity.value(); + + auto expr = create_ast_node(parent, start_pos, {}); + const_cast(*lhs).set_parent(expr); + expr->set_lhs(move(lhs)); + expr->set_op(op); + expr->set_rhs(TRY(parse_expression(expr, op_precedence, op_associativity))); + + expr->set_end(position()); + + lhs = move(expr); + } + + return lhs; +} + +// NOTE: this function should parse everything with precedence of prefix increment and above, e.g. ++/--/!/~, function call, member expressions and expressions in parentheses +ErrorOr> Parser::parse_unary_expression(const GLSL::ASTNode& parent) +{ + if (match(Token::Type::LeftParen)) { + TRY(consume(Token::Type::LeftParen)); + auto expr = TRY(parse_expression(parent)); + TRY(consume(Token::Type::RightParen)); + + return expr; + } + + if (TRY(match_boolean_literal())) + return parse_boolean_literal(parent); + + if (TRY(match_numeric_literal())) + return parse_numeric_literal(parent); + + if (TRY(match_string_literal())) + return parse_string_literal(parent); + + if (TRY(match_name())) { + NonnullRefPtr lhs = TRY(parse_name(parent)); + + while (true) { + if (match(Token::Type::LeftParen)) { + TRY(consume(Token::Type::LeftParen)); + auto expr = create_ast_node(parent, lhs->start(), {}); + auto args = TRY(parse_function_call_args(expr)); + + const_cast(*lhs).set_parent(expr); + expr->set_callee(move(lhs)); + expr->set_arguments(move(args)); + + TRY(consume(Token::Type::RightParen)); + expr->set_end(position()); + + lhs = move(expr); + } else if (match(Token::Type::Dot)) { + TRY(consume(Token::Type::Dot)); + + auto expr = create_ast_node(parent, lhs->start(), {}); + auto rhs = TRY(parse_name(expr)); + + const_cast(*lhs).set_parent(expr); + expr->set_object(move(lhs)); + expr->set_property(move(rhs)); + expr->set_end(position()); + + lhs = move(expr); + } else if (match(Token::Type::LeftBracket)) { + TRY(consume(Token::Type::LeftBracket)); + + auto expr = create_ast_node(parent, lhs->start(), {}); + auto index = TRY(parse_expression(expr)); + + TRY(consume(Token::Type::RightBracket)); + + const_cast(*lhs).set_parent(expr); + expr->set_array(move(lhs)); + expr->set_index(move(index)); + expr->set_end(position()); + + lhs = move(expr); + } else if (match(Token::Type::PlusPlus) || match(Token::Type::MinusMinus)) { + auto op = TRY(consume_unary_op()); + + auto expr = create_ast_node(parent, lhs->start(), position()); + + const_cast(*lhs).set_parent(expr); + expr->set_lhs(move(lhs)); + expr->set_op(op); + expr->set_is_postfix(true); + + lhs = move(expr); + } else { + break; + } + } + + return lhs; + } + + if (match_unary_op()) { + auto expr = create_ast_node(parent, position(), {}); + auto op = TRY(consume_unary_op()); + + auto lhs = TRY(parse_unary_expression(expr)); + expr->set_lhs(move(lhs)); + expr->set_op(op); + expr->set_end(position()); + + return expr; + } + + TRY(error(TRY(String::formatted("unable to parse unary expression starting with {}", TRY(peek().type_as_string()))))); + return create_ast_node(parent, position(), position()); +} + +ErrorOr>> Parser::parse_function_call_args(ASTNode const& parent) +{ + Vector> result; + while (!match(Token::Type::RightParen)) { + auto arg = TRY(parse_expression(parent)); + result.append(move(arg)); + + if (!match(Token::Type::RightParen)) + TRY(consume(Token::Type::Comma)); + } + return result; +} + +ErrorOr> Parser::parse_boolean_literal(ASTNode const& parent) +{ + auto token = TRY(consume(Token::Type::Keyword)); + auto text = token.text(); + bool value = (text == "true"); + return create_ast_node(parent, token.start(), token.end(), value); +} + +ErrorOr> Parser::parse_numeric_literal(GLSL::ASTNode const& parent) +{ + auto token = TRY(consume()); + auto text = token.text(); + return create_ast_node(parent, token.start(), token.end(), text); +} + +ErrorOr> Parser::parse_string_literal(ASTNode const& parent) +{ + Optional start_token_index; + Optional end_token_index; + while (!eof()) { + auto token = peek(); + if (token.type() != Token::Type::DoubleQuotedString && token.type() != Token::Type::SingleQuotedString && token.type() != Token::Type::EscapeSequence) { + VERIFY(start_token_index.has_value()); + end_token_index = m_state.token_index - 1; + break; + } + if (!start_token_index.has_value()) + start_token_index = m_state.token_index; + TRY(consume()); + } + + // String was not terminated + if (!end_token_index.has_value()) { + end_token_index = m_tokens.size() - 1; + } + + VERIFY(start_token_index.has_value()); + VERIFY(end_token_index.has_value()); + + Token start_token = m_tokens[start_token_index.value()]; + Token end_token = m_tokens[end_token_index.value()]; + + auto text = TRY(text_in_range(start_token.start(), end_token.end())); + auto string_literal = create_ast_node(parent, start_token.start(), end_token.end()); + string_literal->set_value(move(text)); + return string_literal; +} + +ErrorOr> Parser::parse_name(ASTNode const& parent, bool allow_sized_name) +{ + NonnullRefPtr name_node = create_ast_node(parent, position(), {}); + + if (peek().type() == Token::Type::Identifier || peek().type() == Token::Type::KnownType) { + auto token = TRY(consume()); + name_node->set_name(token.text()); + name_node->set_end(position()); + } else { + TRY(error("expected keyword or identifier while trying to parse name"sv)); + name_node->set_end(position()); + return name_node; + } + + if (peek().type() == Token::Type::LeftBracket && allow_sized_name) { + NonnullRefPtr sized_name = create_ast_node(parent, name_node->start(), {}); + sized_name->set_name(name_node->name()); + + while (peek().type() == Token::Type::LeftBracket) { + TRY(consume(Token::Type::LeftBracket)); + + StringView size = "0"sv; + if (peek().type() == Token::Type::Integer) + size = TRY(consume(Token::Type::Integer)).text(); + sized_name->append_dimension(size); + + TRY(consume(Token::Type::RightBracket)); + } + name_node->set_end(position()); + name_node = sized_name; + } + + name_node->set_end(previous_token_end()); + return name_node; +} + +ErrorOr> Parser::parse_type(ASTNode const& parent) +{ + auto type = create_ast_node(parent, position(), {}); + + Vector storage_qualifiers; + while (match_storage_qualifier()) { + storage_qualifiers.append(TRY(consume_storage_qualifier())); + } + type->set_storage_qualifiers(move(storage_qualifiers)); + + if (match_keyword("struct"sv)) { + TRY(consume(Token::Type::Keyword)); // Consume struct prefix + } + + if (!TRY(match_name())) { + type->set_end(position()); + TRY(error(TRY(String::formatted("expected name instead of: {}", peek().text())))); + return type; + } + type->set_name(TRY(parse_name(*type))); + + type->set_end(previous_token_end()); + + return type; +} + +bool Parser::match_unary_op() +{ + return match(Token::Type::Plus) + || match(Token::Type::Minus) + || match(Token::Type::PlusPlus) + || match(Token::Type::MinusMinus) + || match(Token::Type::ExclamationMark) + || match(Token::Type::Tilde); +} + +ErrorOr Parser::consume_unary_op() +{ + switch (TRY(consume()).type()) { + case Token::Type::Plus: + return UnaryOp::Plus; + case Token::Type::Minus: + return UnaryOp::Minus; + case Token::Type::PlusPlus: + return UnaryOp::PlusPlus; + case Token::Type::MinusMinus: + return UnaryOp::MinusMinus; + case Token::Type::ExclamationMark: + return UnaryOp::Not; + case Token::Type::Tilde: + return UnaryOp::BitwiseNot; + default: + VERIFY_NOT_REACHED(); + } +} + +bool Parser::match_binary_op() +{ + return match(Token::Type::Plus) + || match(Token::Type::Minus) + || match(Token::Type::Asterisk) + || match(Token::Type::Slash) + || match(Token::Type::Percent) + || match(Token::Type::And) + || match(Token::Type::Pipe) + || match(Token::Type::Caret) + || match(Token::Type::AndAnd) + || match(Token::Type::PipePipe) + || match(Token::Type::CaretCaret) + || match(Token::Type::LessLess) + || match(Token::Type::GreaterGreater) + || match(Token::Type::Less) + || match(Token::Type::LessEquals) + || match(Token::Type::Greater) + || match(Token::Type::GreaterEquals) + || match(Token::Type::EqualsEquals) + || match(Token::Type::ExclamationMarkEquals) + || match(Token::Type::Equals) + || match(Token::Type::PlusEquals) + || match(Token::Type::MinusEquals) + || match(Token::Type::AsteriskEquals) + || match(Token::Type::SlashEquals) + || match(Token::Type::PercentEquals) + || match(Token::Type::LessLessEquals) + || match(Token::Type::GreaterGreaterEquals) + || match(Token::Type::AndEquals) + || match(Token::Type::PipeEquals) + || match(Token::Type::CaretEquals); +} + +ErrorOr Parser::peek_binary_op() +{ + switch (peek().type()) { + case Token::Type::Plus: + return BinaryOp::Addition; + case Token::Type::Minus: + return BinaryOp::Subtraction; + case Token::Type::Asterisk: + return BinaryOp::Multiplication; + case Token::Type::Slash: + return BinaryOp::Division; + case Token::Type::Percent: + return BinaryOp::Modulo; + case Token::Type::And: + return BinaryOp::BitwiseAnd; + case Token::Type::Pipe: + return BinaryOp::BitwiseOr; + case Token::Type::Caret: + return BinaryOp::BitwiseXor; + case Token::Type::AndAnd: + return BinaryOp::LogicalAnd; + case Token::Type::PipePipe: + return BinaryOp::LogicalOr; + case Token::Type::CaretCaret: + return BinaryOp::LogicalXor; + case Token::Type::LessLess: + return BinaryOp::LeftShift; + case Token::Type::GreaterGreater: + return BinaryOp::RightShift; + case Token::Type::Less: + return BinaryOp::LessThan; + case Token::Type::LessEquals: + return BinaryOp::LessThanEquals; + case Token::Type::Greater: + return BinaryOp::GreaterThan; + case Token::Type::GreaterEquals: + return BinaryOp::GreaterThanEquals; + case Token::Type::EqualsEquals: + return BinaryOp::EqualsEquals; + case Token::Type::ExclamationMarkEquals: + return BinaryOp::NotEqual; + case Token::Type::Equals: + return BinaryOp::Assignment; + case Token::Type::PlusEquals: + return BinaryOp::AdditionAssignment; + case Token::Type::MinusEquals: + return BinaryOp::SubtractionAssignment; + case Token::Type::AsteriskEquals: + return BinaryOp::MultiplicationAssignment; + case Token::Type::SlashEquals: + return BinaryOp::DivisionAssignment; + case Token::Type::PercentEquals: + return BinaryOp::ModuloAssignment; + case Token::Type::LessLessEquals: + return BinaryOp::LeftShiftAssignment; + case Token::Type::GreaterGreaterEquals: + return BinaryOp::RightShiftAssignment; + case Token::Type::AndEquals: + return BinaryOp::AndAssignment; + case Token::Type::PipeEquals: + return BinaryOp::OrAssignment; + case Token::Type::CaretEquals: + return BinaryOp::XorAssignment; + default: + VERIFY_NOT_REACHED(); + } +} + +bool Parser::match_storage_qualifier() +{ + return match_keyword("const"sv) + || match_keyword("in"sv) + || match_keyword("out"sv) + || match_keyword("inout"sv) + || match_keyword("centroid"sv) + || match_keyword("patch"sv) + || match_keyword("sample"sv) + || match_keyword("uniform"sv) + || match_keyword("buffer"sv) + || match_keyword("shared"sv) + || match_keyword("coherent"sv) + || match_keyword("volatile"sv) + || match_keyword("restrict"sv) + || match_keyword("readonly"sv) + || match_keyword("writeonly"sv) + || match_keyword("subroutine"sv); +} + +ErrorOr Parser::consume_storage_qualifier() +{ + VERIFY(peek().type() == Token::Type::Keyword); + auto keyword = MUST(consume()).text(); + if (keyword == "buffer") + return StorageTypeQualifier::Buffer; + if (keyword == "centroid") + return StorageTypeQualifier::Centroid; + if (keyword == "coherent") + return StorageTypeQualifier::Coherent; + if (keyword == "const") + return StorageTypeQualifier::Const; + if (keyword == "in") + return StorageTypeQualifier::In; + if (keyword == "inout") + return StorageTypeQualifier::Inout; + if (keyword == "out") + return StorageTypeQualifier::Out; + if (keyword == "patch") + return StorageTypeQualifier::Patch; + if (keyword == "readonly") + return StorageTypeQualifier::Readonly; + if (keyword == "restrict") + return StorageTypeQualifier::Restrict; + if (keyword == "sample") + return StorageTypeQualifier::Sample; + if (keyword == "shared") + return StorageTypeQualifier::Shared; + if (keyword == "subroutine") + return StorageTypeQualifier::Subroutine; + if (keyword == "uniform") + return StorageTypeQualifier::Uniform; + if (keyword == "volatile") + return StorageTypeQualifier::Volatile; + if (keyword == "writeonly") + return StorageTypeQualifier::Writeonly; + VERIFY_NOT_REACHED(); +} + +Token Parser::peek(size_t offset) const +{ + if (m_state.token_index + offset >= m_tokens.size()) + return { Token::Type::EOF_TOKEN, position(), position(), {} }; + return m_tokens[m_state.token_index + offset]; +} + +Optional Parser::peek(Token::Type type) const +{ + auto token = peek(); + if (token.type() == type) + return token; + return {}; +} + +bool Parser::match(Token::Type type) +{ + return peek().type() == type; +} + +bool Parser::match_keyword(StringView keyword) +{ + auto token = peek(); + if (token.type() != Token::Type::Keyword) { + return false; + } + if (token.text() != keyword) { + return false; + } + return true; +} + +bool Parser::match_preprocessor() +{ + return match(Token::Type::PreprocessorStatement) || match(Token::Type::IncludeStatement); +} + +ErrorOr Parser::consume() +{ + if (eof()) { + TRY(error("GLSL Parser: out of tokens"sv)); + return Token { Token::Type::EOF_TOKEN, position(), position(), {} }; + } + return m_tokens[m_state.token_index++]; +} + +ErrorOr Parser::consume(Token::Type type) +{ + auto token = TRY(consume()); + if (token.type() != type) + TRY(error(TRY(String::formatted("expected {} at {}:{}, found: {}", Token::type_to_string(type), token.start().line, token.start().column, Token::type_to_string(token.type()))))); + return token; +} + +ErrorOr Parser::consume_keyword(StringView keyword) +{ + auto token = TRY(consume()); + if (token.type() != Token::Type::Keyword) { + TRY(error(TRY(String::formatted("unexpected token: {}, expected Keyword", TRY(token.to_string()))))); + return token; + } + if (token.text() != keyword) { + TRY(error(TRY(String::formatted("unexpected keyword: {}, expected {}", token.text(), keyword)))); + return token; + } + return token; +} + +ErrorOr Parser::consume_preprocessor() +{ + switch (peek().type()) { + case Token::Type::PreprocessorStatement: + TRY(consume()); + break; + case Token::Type::IncludeStatement: + TRY(consume()); + TRY(consume(Token::Type::IncludePath)); + break; + default: + TRY(error("unexpected token while parsing preprocessor statement"sv)); + TRY(consume()); + } + return {}; +} + +Position Parser::position() const +{ + if (m_tokens.is_empty()) + return {}; + + if (eof()) + return m_tokens.last().end(); + + return peek().start(); +} + +Position Parser::previous_token_end() const +{ + if (m_state.token_index < 1) + return {}; + return m_tokens[m_state.token_index - 1].end(); +} + +Optional Parser::index_of_token_at(Position pos) const +{ + for (size_t token_index = 0; token_index < m_tokens.size(); ++token_index) { + auto token = m_tokens[token_index]; + if (token.start() > pos || token.end() < pos) + continue; + return token_index; + } + return {}; +} + +Vector Parser::tokens_in_range(Position start, Position end) const +{ + auto start_token_index = index_of_token_at(start); + auto end_node_index = index_of_token_at(end); + VERIFY(start_token_index.has_value()); + VERIFY(end_node_index.has_value()); + + Vector tokens; + for (size_t i = start_token_index.value(); i <= end_node_index.value(); ++i) { + tokens.append(m_tokens[i]); + } + return tokens; +} + +ErrorOr Parser::text_in_range(Position start, Position end) const +{ + StringBuilder builder; + for (auto token : tokens_in_range(start, end)) { + builder.append(token.text()); + } + return builder.to_string(); +} + +ErrorOr Parser::error(StringView message) +{ + if (!m_saved_states.is_empty()) + return {}; + + if (message.is_null() || message.is_empty()) + message = ""sv; + String formatted_message; + if (m_state.token_index >= m_tokens.size()) { + formatted_message = TRY(String::formatted("GLSL Parsed error on EOF.{}", message)); + } else { + formatted_message = TRY(String::formatted("GLSL Parser error: {}. token: {} ({}:{})", + message, + m_state.token_index < m_tokens.size() ? m_tokens[m_state.token_index].text() : "EOF"sv, + m_tokens[m_state.token_index].start().line, + m_tokens[m_state.token_index].start().column)); + } + + m_errors.append(formatted_message); + return {}; +} + +void Parser::save_state() +{ + m_saved_states.append(m_state); +} + +void Parser::load_state() +{ + m_state = m_saved_states.take_last(); +} + +} diff --git a/Userland/Libraries/LibGLSL/Parser.h b/Userland/Libraries/LibGLSL/Parser.h new file mode 100644 index 0000000000..cdfee2a97d --- /dev/null +++ b/Userland/Libraries/LibGLSL/Parser.h @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2021, Itamar S. + * Copyright (c) 2023, Volodymyr V. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace GLSL { + +class Parser final { + AK_MAKE_NONCOPYABLE(Parser); + +public: + explicit Parser(Vector tokens, String const& filename); + ~Parser() = default; + + ErrorOr> parse(); + bool eof() const; + + RefPtr root_node() const { return m_root_node; } + void print_tokens() const; + Vector const& tokens() const { return m_tokens; } + Vector const& errors() const { return m_errors; } + +private: + enum class DeclarationType { + Function, + Variable, + Struct, + }; + ErrorOr> match_declaration_in_translation_unit(); + + ErrorOr match_struct_declaration(); + ErrorOr match_function_declaration(); + ErrorOr match_variable_declaration(); + + ErrorOr match_block_statement(); + + ErrorOr match_expression(); + ErrorOr match_name(); + ErrorOr match_string_literal(); + ErrorOr match_numeric_literal(); + ErrorOr match_boolean_literal(); + + ErrorOr match_type(); + + ErrorOr>> parse_declarations_in_translation_unit(ASTNode const& parent); + ErrorOr> parse_single_declaration_in_translation_unit(ASTNode const& parent); + ErrorOr> parse_declaration(ASTNode const& parent, DeclarationType); + ErrorOr> parse_struct_declaration(ASTNode const& parent); + ErrorOr>> parse_struct_members(StructDeclaration& parent); + ErrorOr> parse_function_declaration(ASTNode const& parent); + ErrorOr>> parse_parameter_list(ASTNode const& parent); + ErrorOr> parse_function_definition(ASTNode const& parent); + ErrorOr> parse_variable_declaration(ASTNode const& parent, bool expect_semicolon = true); + + ErrorOr> parse_statement(ASTNode const& parent); + ErrorOr> parse_block_statement(ASTNode const& parent); + ErrorOr> parse_if_statement(ASTNode const& parent); + ErrorOr> parse_for_statement(ASTNode const& parent); + ErrorOr> parse_return_statement(ASTNode const& parent); + + enum class Associativity { + LeftToRight, + RightToLeft + }; + static HashMap s_operator_associativity; + + ErrorOr> parse_expression(ASTNode const& parent, int min_precedence = 0, Associativity associativity = Associativity::LeftToRight); + ErrorOr> parse_unary_expression(ASTNode const& parent); + ErrorOr>> parse_function_call_args(ASTNode const& parent); + ErrorOr> parse_boolean_literal(ASTNode const& parent); + ErrorOr> parse_numeric_literal(ASTNode const& parent); + ErrorOr> parse_string_literal(ASTNode const& parent); + ErrorOr> parse_name(ASTNode const& parent, bool allow_sized_name = false); + + ErrorOr> parse_type(ASTNode const& parent); + + bool match_unary_op(); + ErrorOr consume_unary_op(); + bool match_binary_op(); + ErrorOr peek_binary_op(); + bool match_storage_qualifier(); + ErrorOr consume_storage_qualifier(); + + Token peek(size_t offset = 0) const; + Optional peek(Token::Type) const; + + bool match(Token::Type); + bool match_keyword(StringView); + bool match_preprocessor(); + + ErrorOr consume(); + ErrorOr consume(Token::Type); + ErrorOr consume_keyword(StringView); + ErrorOr consume_preprocessor(); + + Position position() const; + Position previous_token_end() const; + Optional index_of_token_at(Position pos) const; + Vector tokens_in_range(Position start, Position end) const; + ErrorOr text_in_range(Position start, Position end) const; + + ErrorOr error(StringView message = {}); + + template + NonnullRefPtr + create_ast_node(ASTNode const& parent, Position const& start, Optional end, Args&&... args) + { + auto node = adopt_ref(*new T(&parent, start, end, m_filename, forward(args)...)); + return node; + } + + NonnullRefPtr + create_root_ast_node(Position const& start, Position end) + { + auto node = adopt_ref(*new TranslationUnit(nullptr, start, end, m_filename)); + m_root_node = node; + return node; + } + + DummyAstNode& get_dummy_node() + { + static NonnullRefPtr dummy = adopt_ref(*new DummyAstNode(nullptr, {}, {}, {})); + return dummy; + } + + struct State { + size_t token_index { 0 }; + }; + + void save_state(); + void load_state(); + + State m_state; + Vector m_saved_states; + + String m_filename; + Vector m_tokens; + RefPtr m_root_node; + Vector m_errors; +}; + +} diff --git a/Userland/Libraries/LibGLSL/Preprocessor.cpp b/Userland/Libraries/LibGLSL/Preprocessor.cpp new file mode 100644 index 0000000000..b20ee7f91d --- /dev/null +++ b/Userland/Libraries/LibGLSL/Preprocessor.cpp @@ -0,0 +1,416 @@ +/* + * Copyright (c) 2021, Itamar S. + * Copyright (c) 2023, Volodymyr V. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "Preprocessor.h" +#include +#include +#include +#include +#include + +namespace GLSL { +Preprocessor::Preprocessor(String filename, String program) + : m_filename(move(filename)) + , m_program(move(program)) +{ +} + +ErrorOr> Preprocessor::process_and_lex() +{ + Lexer lexer { m_program }; + lexer.set_ignore_whitespace(true); + auto tokens = lexer.lex(); + + m_unprocessed_tokens = tokens; + + for (size_t token_index = 0; token_index < tokens.size(); ++token_index) { + auto& token = tokens[token_index]; + m_current_line = token.start().line; + if (token.type() == Token::Type::PreprocessorStatement) { + TRY(handle_preprocessor_statement(token.text())); + m_processed_tokens.append(tokens[token_index]); + continue; + } + + if (m_state != State::Normal) + continue; + + if (token.type() == Token::Type::IncludeStatement) { + if (token_index >= tokens.size() - 1 || tokens[token_index + 1].type() != Token::Type::IncludePath) + continue; + handle_include_statement(tokens[token_index + 1].text()); + if (m_options.keep_include_statements) { + m_processed_tokens.append(tokens[token_index]); + m_processed_tokens.append(tokens[token_index + 1]); + } + ++token_index; // Also skip IncludePath token + continue; + } + + if (token.type() == Token::Type::Identifier) { + if (auto defined_value = m_definitions.find(token.text()); defined_value != m_definitions.end()) { + auto last_substituted_token_index = TRY(do_substitution(tokens, token_index, defined_value->value)); + token_index = last_substituted_token_index; + continue; + } + } + + m_processed_tokens.append(token); + } + + return m_processed_tokens; +} + +static void consume_whitespace(GenericLexer& lexer) +{ + auto ignore_line = [&] { + for (;;) { + if (lexer.consume_specific("\\\n"sv)) { + lexer.ignore(2); + } else { + lexer.ignore_until('\n'); + lexer.ignore(); + break; + } + } + }; + for (;;) { + if (lexer.consume_specific("//"sv)) { + ignore_line(); + } else if (lexer.consume_specific("/*"sv)) { + lexer.ignore_until("*/"); + lexer.ignore(2); + } else if (lexer.next_is("\\\n"sv)) { + lexer.ignore(2); + } else if (lexer.is_eof() || !lexer.next_is(isspace)) { + break; + } else { + lexer.ignore(); + } + } +} + +ErrorOr Preprocessor::handle_preprocessor_statement(StringView line) +{ + GenericLexer lexer(line); + + consume_whitespace(lexer); + lexer.consume_specific('#'); + consume_whitespace(lexer); + auto keyword = lexer.consume_until(' '); + lexer.ignore(); + if (keyword.is_empty() || keyword.is_null() || keyword.is_whitespace()) + return {}; + + return TRY(handle_preprocessor_keyword(keyword, lexer)); +} + +void Preprocessor::handle_include_statement(StringView include_path) +{ + m_included_paths.append(include_path); + if (definitions_in_header_callback) { + for (auto& def : definitions_in_header_callback(include_path)) + m_definitions.set(def.key, def.value); + } +} + +ErrorOr Preprocessor::handle_preprocessor_keyword(StringView keyword, GenericLexer& line_lexer) +{ + if (keyword == "include") { + // Should have called 'handle_include_statement'. + VERIFY_NOT_REACHED(); + } + + if (keyword == "else") { + if (m_options.ignore_invalid_statements && m_current_depth == 0) + return {}; + VERIFY(m_current_depth > 0); + if (m_depths_of_not_taken_branches.contains_slow(m_current_depth - 1)) { + m_depths_of_not_taken_branches.remove_all_matching([this](auto x) { return x == m_current_depth - 1; }); + m_state = State::Normal; + } + if (m_depths_of_taken_branches.contains_slow(m_current_depth - 1)) { + m_state = State::SkipElseBranch; + } + return {}; + } + + if (keyword == "endif") { + if (m_options.ignore_invalid_statements && m_current_depth == 0) + return {}; + VERIFY(m_current_depth > 0); + --m_current_depth; + if (m_depths_of_not_taken_branches.contains_slow(m_current_depth)) { + m_depths_of_not_taken_branches.remove_all_matching([this](auto x) { return x == m_current_depth; }); + } + if (m_depths_of_taken_branches.contains_slow(m_current_depth)) { + m_depths_of_taken_branches.remove_all_matching([this](auto x) { return x == m_current_depth; }); + } + m_state = State::Normal; + return {}; + } + + if (keyword == "define") { + if (m_state == State::Normal) { + auto definition = TRY(create_definition(line_lexer.consume_all())); + if (definition.has_value()) + m_definitions.set(definition->key, *definition); + } + return {}; + } + if (keyword == "undef") { + if (m_state == State::Normal) { + auto key = line_lexer.consume_until(' '); + line_lexer.consume_all(); + m_definitions.remove(key); + } + return {}; + } + if (keyword == "ifdef") { + ++m_current_depth; + if (m_state == State::Normal) { + auto key = line_lexer.consume_until(' '); + line_lexer.ignore(); + if (m_definitions.contains(key)) { + m_depths_of_taken_branches.append(m_current_depth - 1); + return {}; + } else { + m_depths_of_not_taken_branches.append(m_current_depth - 1); + m_state = State::SkipIfBranch; + return {}; + } + } + return {}; + } + if (keyword == "ifndef") { + ++m_current_depth; + if (m_state == State::Normal) { + auto key = line_lexer.consume_until(' '); + line_lexer.ignore(); + if (!m_definitions.contains(key)) { + m_depths_of_taken_branches.append(m_current_depth - 1); + return {}; + } else { + m_depths_of_not_taken_branches.append(m_current_depth - 1); + m_state = State::SkipIfBranch; + return {}; + } + } + return {}; + } + if (keyword == "if") { + ++m_current_depth; + if (m_state == State::Normal) { + // FIXME: Implement #if logic + // We currently always take #if branches. + m_depths_of_taken_branches.append(m_current_depth - 1); + } + return {}; + } + + if (keyword == "elif") { + if (m_options.ignore_invalid_statements && m_current_depth == 0) + return {}; + VERIFY(m_current_depth > 0); + // FIXME: Evaluate the elif expression + // We currently always treat the expression in #elif as true. + if (m_depths_of_not_taken_branches.contains_slow(m_current_depth - 1) /* && should_take*/) { + m_depths_of_not_taken_branches.remove_all_matching([this](auto x) { return x == m_current_depth - 1; }); + m_state = State::Normal; + } + if (m_depths_of_taken_branches.contains_slow(m_current_depth - 1)) { + m_state = State::SkipElseBranch; + } + return {}; + } + if (keyword == "pragma") { + line_lexer.consume_all(); + return {}; + } + if (keyword == "error") { + line_lexer.consume_all(); + return {}; + } + + if (!m_options.ignore_unsupported_keywords) { + dbgln("Unsupported preprocessor keyword: {}", keyword); + VERIFY_NOT_REACHED(); + } + return {}; +} + +ErrorOr Preprocessor::do_substitution(Vector const& tokens, size_t token_index, Definition const& defined_value) +{ + if (defined_value.value.is_empty()) + return token_index; + + Substitution sub; + sub.defined_value = defined_value; + + auto macro_call = parse_macro_call(tokens, token_index); + + if (!macro_call.has_value()) + return token_index; + + Vector original_tokens; + for (size_t i = token_index; i <= macro_call->end_token_index; ++i) { + original_tokens.append(tokens[i]); + } + VERIFY(!original_tokens.is_empty()); + + auto processed_value = TRY(evaluate_macro_call(*macro_call, defined_value)); + m_substitutions.append({ original_tokens, defined_value, processed_value }); + + Lexer lexer(processed_value); + lexer.lex_iterable([&](auto token) { + if (token.type() == Token::Type::Whitespace) + return; + token.set_start(original_tokens.first().start()); + token.set_end(original_tokens.first().end()); + m_processed_tokens.append(token); + }); + return macro_call->end_token_index; +} + +Optional Preprocessor::parse_macro_call(Vector const& tokens, size_t token_index) +{ + auto name = tokens[token_index]; + ++token_index; + + if (token_index >= tokens.size() || tokens[token_index].type() != Token::Type::LeftParen) + return MacroCall { name, {}, token_index - 1 }; + ++token_index; + + Vector arguments; + Optional current_argument; + + size_t paren_depth = 1; + for (; token_index < tokens.size(); ++token_index) { + auto& token = tokens[token_index]; + if (token.type() == Token::Type::LeftParen) + ++paren_depth; + if (token.type() == Token::Type::RightParen) + --paren_depth; + + if (paren_depth == 0) { + if (current_argument.has_value()) + arguments.append(*current_argument); + break; + } + + if (paren_depth == 1 && token.type() == Token::Type::Comma) { + if (current_argument.has_value()) + arguments.append(*current_argument); + current_argument = {}; + } else { + if (!current_argument.has_value()) + current_argument = MacroCall::Argument {}; + current_argument->tokens.append(token); + } + } + + if (token_index >= tokens.size()) + return {}; + + return MacroCall { name, move(arguments), token_index }; +} + +ErrorOr> Preprocessor::create_definition(StringView line) +{ + Lexer lexer { line }; + lexer.set_ignore_whitespace(true); + auto tokens = lexer.lex(); + if (tokens.is_empty()) + return Optional {}; + + if (tokens.first().type() != Token::Type::Identifier) + return Optional {}; + + Definition definition; + definition.filename = m_filename; + definition.line = m_current_line; + + definition.key = tokens.first().text(); + + if (tokens.size() == 1) + return definition; + + size_t token_index = 1; + // Parse macro parameters (if any) + if (tokens[token_index].type() == Token::Type::LeftParen) { + ++token_index; + while (token_index < tokens.size() && tokens[token_index].type() != Token::Type::RightParen) { + auto param = tokens[token_index]; + if (param.type() != Token::Type::Identifier) + return Optional {}; + + if (token_index + 1 >= tokens.size()) + return Optional {}; + + ++token_index; + + if (tokens[token_index].type() == Token::Type::Comma) + ++token_index; + else if (tokens[token_index].type() != Token::Type::RightParen) + return Optional {}; + + definition.parameters.empend(param.text()); + } + if (token_index >= tokens.size()) + return Optional {}; + ++token_index; + } + + if (token_index < tokens.size()) + definition.value = TRY(remove_escaped_newlines(line.substring_view(tokens[token_index].start().column))).bytes_as_string_view(); + + return definition; +} + +ErrorOr Preprocessor::remove_escaped_newlines(StringView value) +{ + static constexpr auto escaped_newline = "\\\n"sv; + AK::StringBuilder processed_value; + GenericLexer lexer { value }; + while (!lexer.is_eof()) { + processed_value.append(lexer.consume_until(escaped_newline)); + lexer.ignore(escaped_newline.length()); + } + return processed_value.to_string(); +} + +ErrorOr Preprocessor::evaluate_macro_call(MacroCall const& macro_call, Definition const& definition) +{ + if (macro_call.arguments.size() != definition.parameters.size()) { + dbgln("mismatch in # of arguments for macro call: {}", macro_call.name.text()); + return String {}; + } + + Lexer lexer { definition.value }; + StringBuilder processed_value; + lexer.lex_iterable([&](auto token) { + if (token.type() != Token::Type::Identifier) { + processed_value.append(token.text()); + return; + } + + auto param_index = definition.parameters.find_first_index(token.text()); + if (!param_index.has_value()) { + processed_value.append(token.text()); + return; + } + + auto& argument = macro_call.arguments[*param_index]; + for (auto& arg_token : argument.tokens) { + processed_value.append(arg_token.text()); + } + }); + + return processed_value.to_string(); +} + +}; diff --git a/Userland/Libraries/LibGLSL/Preprocessor.h b/Userland/Libraries/LibGLSL/Preprocessor.h new file mode 100644 index 0000000000..8291c5f764 --- /dev/null +++ b/Userland/Libraries/LibGLSL/Preprocessor.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021, Itamar S. + * Copyright (c) 2023, Volodymyr V. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace GLSL { + +class Preprocessor { + +public: + explicit Preprocessor(String filename, String program); + ErrorOr> process_and_lex(); + Vector included_paths() const { return m_included_paths; } + + struct Definition { + StringView key; + Vector parameters; + StringView value; + FlyString filename; + size_t line { 0 }; + size_t column { 0 }; + }; + using Definitions = HashMap; + + struct Substitution { + Vector original_tokens; + Definition defined_value; + String processed_value; + }; + + Definitions const& definitions() const { return m_definitions; } + Vector const& substitutions() const { return m_substitutions; } + + void set_ignore_unsupported_keywords(bool ignore) { m_options.ignore_unsupported_keywords = ignore; } + void set_ignore_invalid_statements(bool ignore) { m_options.ignore_invalid_statements = ignore; } + void set_keep_include_statements(bool keep) { m_options.keep_include_statements = keep; } + + Function definitions_in_header_callback { nullptr }; + + Vector const& unprocessed_tokens() const { return m_unprocessed_tokens; } + +private: + ErrorOr handle_preprocessor_statement(StringView); + void handle_include_statement(StringView); + ErrorOr handle_preprocessor_keyword(StringView keyword, GenericLexer& line_lexer); + ErrorOr remove_escaped_newlines(StringView value); + + ErrorOr do_substitution(Vector const& tokens, size_t token_index, Definition const&); + ErrorOr> create_definition(StringView line); + + struct MacroCall { + Token name; + struct Argument { + Vector tokens; + }; + Vector arguments; + size_t end_token_index { 0 }; + }; + Optional parse_macro_call(Vector const& tokens, size_t token_index); + ErrorOr evaluate_macro_call(MacroCall const&, Definition const&); + + String m_filename; + String m_program; + + Vector m_unprocessed_tokens; + Vector m_processed_tokens; + Definitions m_definitions; + Vector m_substitutions; + + size_t m_current_line { 0 }; + size_t m_current_depth { 0 }; + Vector m_depths_of_taken_branches; + Vector m_depths_of_not_taken_branches; + + enum class State { + Normal, + SkipIfBranch, + SkipElseBranch + }; + State m_state { State::Normal }; + + Vector m_included_paths; + + struct Options { + bool ignore_unsupported_keywords { false }; + bool ignore_invalid_statements { false }; + bool keep_include_statements { false }; + } m_options; +}; +} diff --git a/Userland/Libraries/LibGLSL/Token.cpp b/Userland/Libraries/LibGLSL/Token.cpp new file mode 100644 index 0000000000..fe5d31ad69 --- /dev/null +++ b/Userland/Libraries/LibGLSL/Token.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023, the SerenityOS developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "Token.h" +#include + +namespace GLSL { + +bool Position::operator<(Position const& other) const +{ + return line < other.line || (line == other.line && column < other.column); +} + +bool Position::operator>(Position const& other) const +{ + return !(*this < other) && !(*this == other); +} + +bool Position::operator==(Position const& other) const +{ + return line == other.line && column == other.column; +} + +bool Position::operator<=(Position const& other) const +{ + return !(*this > other); +} + +ErrorOr Token::to_string() const +{ + return String::formatted("{} {}:{}-{}:{} ({})", type_to_string(m_type), start().line, start().column, end().line, end().column, text()); +} + +ErrorOr Token::type_as_string() const +{ + auto str = type_to_string(m_type); + auto view = StringView(str, strlen(str)); + return String::from_utf8(view); +} + +} diff --git a/Userland/Libraries/LibGLSL/Token.h b/Userland/Libraries/LibGLSL/Token.h new file mode 100644 index 0000000000..b0ca01d2ec --- /dev/null +++ b/Userland/Libraries/LibGLSL/Token.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2023, the SerenityOS developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include + +namespace GLSL { + +#define FOR_EACH_TOKEN_TYPE \ + __TOKEN(Unknown) \ + __TOKEN(Whitespace) \ + __TOKEN(PreprocessorStatement) \ + __TOKEN(IncludeStatement) \ + __TOKEN(IncludePath) \ + __TOKEN(LeftParen) \ + __TOKEN(RightParen) \ + __TOKEN(LeftCurly) \ + __TOKEN(RightCurly) \ + __TOKEN(LeftBracket) \ + __TOKEN(RightBracket) \ + __TOKEN(Less) \ + __TOKEN(Greater) \ + __TOKEN(LessEquals) \ + __TOKEN(GreaterEquals) \ + __TOKEN(LessLess) \ + __TOKEN(GreaterGreater) \ + __TOKEN(LessLessEquals) \ + __TOKEN(GreaterGreaterEquals) \ + __TOKEN(Comma) \ + __TOKEN(Plus) \ + __TOKEN(PlusPlus) \ + __TOKEN(PlusEquals) \ + __TOKEN(Minus) \ + __TOKEN(MinusMinus) \ + __TOKEN(MinusEquals) \ + __TOKEN(Asterisk) \ + __TOKEN(AsteriskEquals) \ + __TOKEN(Slash) \ + __TOKEN(SlashEquals) \ + __TOKEN(Percent) \ + __TOKEN(PercentEquals) \ + __TOKEN(Caret) \ + __TOKEN(CaretCaret) \ + __TOKEN(CaretEquals) \ + __TOKEN(ExclamationMark) \ + __TOKEN(ExclamationMarkEquals) \ + __TOKEN(Equals) \ + __TOKEN(EqualsEquals) \ + __TOKEN(And) \ + __TOKEN(AndAnd) \ + __TOKEN(AndEquals) \ + __TOKEN(Pipe) \ + __TOKEN(PipePipe) \ + __TOKEN(PipeEquals) \ + __TOKEN(Tilde) \ + __TOKEN(QuestionMark) \ + __TOKEN(Colon) \ + __TOKEN(Semicolon) \ + __TOKEN(Dot) \ + __TOKEN(DoubleQuotedString) \ + __TOKEN(SingleQuotedString) \ + __TOKEN(RawString) \ + __TOKEN(EscapeSequence) \ + __TOKEN(Integer) \ + __TOKEN(Float) \ + __TOKEN(Keyword) \ + __TOKEN(KnownType) \ + __TOKEN(Identifier) \ + __TOKEN(EOF_TOKEN) + +struct Position { + size_t line { 0 }; + size_t column { 0 }; + + bool operator<(Position const&) const; + bool operator<=(Position const&) const; + bool operator>(Position const&) const; + bool operator==(Position const&) const; +}; + +struct Token { + enum class Type { +#define __TOKEN(x) x, + FOR_EACH_TOKEN_TYPE +#undef __TOKEN + }; + + Token(Type type, Position const& start, Position const& end, StringView text) + : m_type(type) + , m_start(start) + , m_end(end) + , m_text(text) + { + } + + static char const* type_to_string(Type t) + { + switch (t) { +#define __TOKEN(x) \ + case Type::x: \ + return #x; + FOR_EACH_TOKEN_TYPE +#undef __TOKEN + } + VERIFY_NOT_REACHED(); + } + + ErrorOr to_string() const; + ErrorOr type_as_string() const; + + Position const& start() const { return m_start; } + Position const& end() const { return m_end; } + + void set_start(Position const& other) { m_start = other; } + void set_end(Position const& other) { m_end = other; } + Type type() const { return m_type; } + StringView text() const { return m_text; } + +private: + Type m_type { Type::Unknown }; + Position m_start; + Position m_end; + StringView m_text; +}; + +} + +#undef FOR_EACH_TOKEN_TYPE