Shell: Switch to a new parser and AST

This commit also completely reworks the execution, highlighting and
completion model to work with the new AST.
New additions:
- $(...) stdout captures
- fd>&fd redirections
- fd>&- redirections (close fd)
- read-write redirections (<> path)
- completely event-based execution
- the weird idea of allowing the user to redirect the shell's own fds
- variables in strings
- local variables
- minimal list support
- adding hyperlinks to all paths that exist

Shell/AST.h

@@ -0,0 +1,815 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "Forward.h"
+#include "Job.h"
+#include <AK/NonnullRefPtr.h>
+#include <AK/RefCounted.h>
+#include <AK/RefPtr.h>
+#include <AK/String.h>
+#include <AK/Types.h>
+#include <AK/Vector.h>
+#include <LibLine/Editor.h>
+
+using TheExecutionInputType = RefPtr<Shell>;
+
+namespace AST {
+
+struct HighlightMetadata {
+    bool is_first_in_list { true };
+};
+
+struct Position {
+    size_t start_offset { 0 };
+    size_t end_offset { 0 };
+    bool contains(size_t offset) const { return start_offset <= offset && offset <= end_offset; }
+};
+
+struct Rewiring {
+    int source_fd { -1 };
+    int dest_fd { -1 };
+    enum class Close {
+        None,
+        Source,
+        Destination,
+    } must_be_closed { Close::None };
+};
+
+struct Redirection : public RefCounted<Redirection> {
+    virtual Result<Rewiring, String> apply() const = 0;
+    virtual ~Redirection();
+    virtual bool is_path_redirection() const { return false; }
+    virtual bool is_fd_redirection() const { return false; }
+    virtual bool is_close_redirection() const { return false; }
+};
+
+struct CloseRedirection : public Redirection {
+    int fd { -1 };
+
+    virtual Result<Rewiring, String> apply() const override;
+    virtual ~CloseRedirection();
+    CloseRedirection(int fd)
+        : fd(fd)
+    {
+    }
+
+private:
+    virtual bool is_close_redirection() const override { return true; }
+};
+
+struct PathRedirection : public Redirection {
+    String path;
+    int fd { -1 };
+    enum {
+        Read,
+        Write,
+        WriteAppend,
+        ReadWrite,
+    } direction { Read };
+
+    virtual Result<Rewiring, String> apply() const override;
+    virtual ~PathRedirection();
+    PathRedirection(String path, int fd, decltype(direction) direction)
+        : path(move(path))
+        , fd(fd)
+        , direction(direction)
+    {
+    }
+
+private:
+    virtual bool is_path_redirection() const override { return true; }
+};
+
+struct FdRedirection : public Redirection
+    , public Rewiring {
+
+    virtual Result<Rewiring, String> apply() const override { return *this; }
+    virtual ~FdRedirection();
+    FdRedirection(int source, int dest, Rewiring::Close close)
+        : Rewiring({ source, dest, close })
+    {
+    }
+
+private:
+    virtual bool is_fd_redirection() const override { return true; }
+};
+
+struct Command {
+    Vector<String> argv;
+    Vector<NonnullRefPtr<Redirection>> redirections;
+    bool should_wait { true };
+    bool is_pipe_source { false };
+};
+
+struct HitTestResult {
+    RefPtr<Node> matching_node;
+    RefPtr<Node> closest_node_with_semantic_meaning; // This is used if matching_node is a bareword
+};
+
+class Value : public RefCounted<Value> {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) = 0;
+    virtual Vector<Command> resolve_as_commands(TheExecutionInputType);
+    virtual ~Value();
+    virtual bool is_command() const { return false; }
+    virtual bool is_glob() const { return false; }
+    virtual bool is_job() const { return false; }
+    virtual bool is_list() const { return false; }
+    virtual bool is_string() const { return false; }
+};
+
+class CommandValue final : public Value {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) override;
+    virtual Vector<Command> resolve_as_commands(TheExecutionInputType) override;
+    virtual ~CommandValue();
+    virtual bool is_command() const override { return true; }
+    CommandValue(Command command)
+        : m_command(move(command))
+    {
+    }
+
+    CommandValue(Vector<String> argv)
+        : m_command({ move(argv), {}, true, false })
+    {
+    }
+
+private:
+    Command m_command;
+};
+
+class CommandSequenceValue final : public Value {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) override;
+    virtual Vector<Command> resolve_as_commands(TheExecutionInputType) override;
+    virtual ~CommandSequenceValue();
+    virtual bool is_command() const override { return true; }
+    CommandSequenceValue(Vector<Command> commands)
+        : m_contained_values(move(commands))
+    {
+    }
+
+private:
+    Vector<Command> m_contained_values;
+};
+
+class JobValue final : public Value {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) override { ASSERT_NOT_REACHED(); }
+    virtual Vector<Command> resolve_as_commands(TheExecutionInputType) override { ASSERT_NOT_REACHED(); }
+    virtual ~JobValue();
+    virtual bool is_job() const override { return true; }
+    JobValue(RefPtr<Job> job)
+        : m_job(move(job))
+    {
+    }
+
+    const RefPtr<Job> job() const { return m_job; }
+
+private:
+    RefPtr<Job> m_job;
+};
+
+class ListValue final : public Value {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) override;
+    virtual ~ListValue();
+    virtual bool is_list() const override { return true; }
+    ListValue(Vector<String> values);
+    ListValue(Vector<RefPtr<Value>> values)
+        : m_contained_values(move(values))
+    {
+    }
+
+private:
+    Vector<RefPtr<Value>> m_contained_values;
+};
+
+class StringValue final : public Value {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) override;
+    virtual ~StringValue();
+    virtual bool is_string() const override { return m_split.is_null(); }
+    virtual bool is_list() const override { return !m_split.is_null(); }
+    StringValue(String string, String split_by = {})
+        : m_string(string)
+        , m_split(move(split_by))
+    {
+    }
+
+private:
+    String m_string;
+    String m_split;
+};
+
+class GlobValue final : public Value {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) override;
+    virtual ~GlobValue();
+    virtual bool is_glob() const override { return true; }
+    GlobValue(String glob)
+        : m_glob(glob)
+    {
+    }
+
+private:
+    String m_glob;
+};
+
+class SimpleVariableValue final : public Value {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) override;
+    virtual ~SimpleVariableValue();
+    // FIXME: Should override is_list and is_string,
+    //        as it might have different types of values.
+    SimpleVariableValue(String name)
+        : m_name(name)
+    {
+    }
+
+private:
+    String m_name;
+};
+
+class SpecialVariableValue final : public Value {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) override;
+    virtual ~SpecialVariableValue();
+    // FIXME: Should override is_list and is_string,
+    //        as it might have different types of values.
+    SpecialVariableValue(char name)
+        : m_name(name)
+    {
+    }
+
+private:
+    char m_name { -1 };
+};
+
+class TildeValue final : public Value {
+public:
+    virtual Vector<String> resolve_as_list(TheExecutionInputType) override;
+    virtual ~TildeValue();
+    virtual bool is_string() const override { return true; }
+    TildeValue(String name)
+        : m_username(name)
+    {
+    }
+
+private:
+    String m_username;
+};
+
+class Node : public RefCounted<Node> {
+public:
+    virtual void dump(int level) = const 0;
+    virtual RefPtr<Value> run(TheExecutionInputType) = 0;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) = 0;
+    virtual Vector<Line::CompletionSuggestion> complete_for_editor(Shell&, size_t, RefPtr<Node> matching_node);
+    Vector<Line::CompletionSuggestion> complete_for_editor(Shell& shell, size_t offset);
+    virtual HitTestResult hit_test_position(size_t offset)
+    {
+        if (m_position.contains(offset))
+            return { this, nullptr };
+        return { nullptr, nullptr };
+    }
+    virtual String class_name() const { return "Node"; }
+    Node(Position);
+    virtual ~Node();
+
+    virtual bool is_bareword() const { return false; }
+    virtual bool is_command() const { return false; }
+    virtual bool is_execute() const { return false; }
+    virtual bool is_glob() const { return false; }
+    virtual bool is_tilde() const { return false; }
+    virtual bool is_variable_decls() const { return false; }
+
+    virtual bool is_list() const { return false; }
+
+    const Position& position() const { return m_position; }
+
+protected:
+    Position m_position;
+};
+
+class PathRedirectionNode : public Node {
+public:
+    PathRedirectionNode(Position, int, RefPtr<Node>);
+    virtual ~PathRedirectionNode();
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual Vector<Line::CompletionSuggestion> complete_for_editor(Shell&, size_t, RefPtr<Node> matching_node) override;
+    virtual HitTestResult hit_test_position(size_t offset) override;
+    virtual bool is_command() const override { return true; }
+    virtual bool is_list() const override { return true; }
+
+protected:
+    int m_fd { -1 };
+    RefPtr<Node> m_path;
+};
+
+class And final : public Node {
+public:
+    And(Position, RefPtr<Node>, RefPtr<Node>);
+    virtual ~And();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "And"; }
+
+    RefPtr<Node> m_left;
+    RefPtr<Node> m_right;
+};
+
+class ListConcatenate final : public Node {
+public:
+    ListConcatenate(Position, RefPtr<Node>, RefPtr<Node>);
+    virtual ~ListConcatenate();
+
+private:
+    virtual void dump(int level) const override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "ListConcatenate"; }
+    virtual bool is_list() const override { return true; }
+
+    RefPtr<Node> m_element;
+    RefPtr<Node> m_list;
+};
+
+class Background final : public Node {
+public:
+    Background(Position, RefPtr<Node>);
+    virtual ~Background();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "Background"; }
+
+    RefPtr<Node> m_command;
+};
+
+class BarewordLiteral final : public Node {
+public:
+    BarewordLiteral(Position, String);
+    virtual ~BarewordLiteral();
+    const String& text() const { return m_text; }
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual String class_name() const override { return "BarewordLiteral"; }
+    virtual bool is_bareword() const override { return true; }
+
+    String m_text;
+};
+
+class CastToCommand final : public Node {
+public:
+    CastToCommand(Position, RefPtr<Node>);
+    virtual ~CastToCommand();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual Vector<Line::CompletionSuggestion> complete_for_editor(Shell&, size_t, RefPtr<Node> matching_node) override;
+    virtual String class_name() const override { return "CastToCommand"; }
+    virtual bool is_command() const override { return true; }
+    virtual bool is_list() const override { return true; }
+
+    RefPtr<Node> m_inner;
+};
+
+class CastToList final : public Node {
+public:
+    CastToList(Position, RefPtr<Node>);
+    virtual ~CastToList();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "CastToList"; }
+    virtual bool is_list() const override { return true; }
+
+    RefPtr<Node> m_inner;
+};
+
+class CloseFdRedirection final : public Node {
+public:
+    CloseFdRedirection(Position, int);
+    virtual ~CloseFdRedirection();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual String class_name() const override { return "CloseFdRedirection"; }
+    virtual bool is_command() const override { return true; }
+
+    int m_fd { -1 };
+};
+
+class CommandLiteral final : public Node {
+public:
+    CommandLiteral(Position, Command);
+    virtual ~CommandLiteral();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override { ASSERT_NOT_REACHED(); }
+    virtual String class_name() const override { return "CommandLiteral"; }
+    virtual bool is_command() const override { return true; }
+    virtual bool is_list() const override { return true; }
+
+    Command m_command;
+};
+
+class Comment : public Node {
+public:
+    Comment(Position, String);
+    virtual ~Comment();
+    const String& text() const { return m_text; }
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual String class_name() const override { return "Comment"; }
+
+    String m_text;
+};
+
+class DynamicEvaluate final : public Node {
+public:
+    DynamicEvaluate(Position, RefPtr<Node>);
+    virtual ~DynamicEvaluate();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "DynamicEvaluate"; }
+
+    virtual bool is_bareword() const override { return m_inner->is_bareword(); }
+    virtual bool is_command() const override { return is_list(); }
+    virtual bool is_execute() const override { return true; }
+    virtual bool is_glob() const override { return m_inner->is_glob(); }
+    virtual bool is_list() const override
+    {
+        return m_inner->is_list() || m_inner->is_command() || m_inner->is_glob(); // Anything that generates a list.
+    }
+
+    RefPtr<Node> m_inner;
+};
+
+class DoubleQuotedString final : public Node {
+public:
+    DoubleQuotedString(Position, RefPtr<Node>);
+    virtual ~DoubleQuotedString();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "DoubleQuotedString"; }
+
+    RefPtr<Node> m_inner;
+};
+
+class Fd2FdRedirection final : public Node {
+public:
+    Fd2FdRedirection(Position, int, int);
+    virtual ~Fd2FdRedirection();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual String class_name() const override { return "Fd2FdRedirection"; }
+    virtual bool is_command() const override { return true; }
+
+    int source_fd { -1 };
+    int dest_fd { -1 };
+};
+
+class Glob final : public Node {
+public:
+    Glob(Position, String);
+    virtual ~Glob();
+    const String& text() const { return m_text; }
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual String class_name() const override { return "Glob"; }
+    virtual bool is_glob() const override { return true; }
+    virtual bool is_list() const override { return true; }
+
+    String m_text;
+};
+
+class Execute final : public Node {
+public:
+    Execute(Position, RefPtr<Node>, bool capture_stdout = false);
+    virtual ~Execute();
+    void capture_stdout() { m_capture_stdout = true; }
+    RefPtr<Node> command() { return m_command; }
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual Vector<Line::CompletionSuggestion> complete_for_editor(Shell&, size_t, RefPtr<Node> matching_node) override;
+    virtual String class_name() const override { return "Execute"; }
+    virtual bool is_execute() const override { return true; }
+
+    RefPtr<Node> m_command;
+    bool m_capture_stdout { false };
+};
+
+class Join final : public Node {
+public:
+    Join(Position, RefPtr<Node>, RefPtr<Node>);
+    virtual ~Join();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "Join"; }
+    virtual bool is_command() const override { return true; }
+    virtual bool is_list() const override { return true; }
+
+    RefPtr<Node> m_left;
+    RefPtr<Node> m_right;
+};
+
+class Or final : public Node {
+public:
+    Or(Position, RefPtr<Node>, RefPtr<Node>);
+    virtual ~Or();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "Or"; }
+    virtual bool is_list() const override { return true; }
+
+    RefPtr<Node> m_left;
+    RefPtr<Node> m_right;
+};
+
+class Pipe final : public Node {
+public:
+    Pipe(Position, RefPtr<Node>, RefPtr<Node>);
+    virtual ~Pipe();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "Pipe"; }
+    virtual bool is_list() const override { return true; }
+
+    RefPtr<Node> m_left;
+    RefPtr<Node> m_right;
+};
+
+class ReadRedirection final : public PathRedirectionNode {
+public:
+    ReadRedirection(Position, int, RefPtr<Node>);
+    virtual ~ReadRedirection();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual String class_name() const override { return "ReadRedirection"; }
+};
+
+class ReadWriteRedirection final : public PathRedirectionNode {
+public:
+    ReadWriteRedirection(Position, int, RefPtr<Node>);
+    virtual ~ReadWriteRedirection();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual String class_name() const override { return "ReadWriteRedirection"; }
+};
+
+class Sequence final : public Node {
+public:
+    Sequence(Position, RefPtr<Node>, RefPtr<Node>);
+    virtual ~Sequence();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "Sequence"; }
+    virtual bool is_list() const override { return true; }
+
+    RefPtr<Node> m_left;
+    RefPtr<Node> m_right;
+};
+
+class SimpleVariable final : public Node {
+public:
+    SimpleVariable(Position, String);
+    virtual ~SimpleVariable();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual Vector<Line::CompletionSuggestion> complete_for_editor(Shell&, size_t, RefPtr<Node> matching_node) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "SimpleVariable"; }
+
+    String m_name;
+};
+
+class SpecialVariable final : public Node {
+public:
+    SpecialVariable(Position, char);
+    virtual ~SpecialVariable();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual Vector<Line::CompletionSuggestion> complete_for_editor(Shell&, size_t, RefPtr<Node> matching_node) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "SpecialVariable"; }
+
+    char m_name { -1 };
+};
+
+class StringConcatenate final : public Node {
+public:
+    StringConcatenate(Position, RefPtr<Node>, RefPtr<Node>);
+    virtual ~StringConcatenate();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "StringConcatenate"; }
+
+    RefPtr<Node> m_left;
+    RefPtr<Node> m_right;
+};
+
+class StringLiteral final : public Node {
+public:
+    StringLiteral(Position, String);
+    virtual ~StringLiteral();
+    const String& text() const { return m_text; }
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual String class_name() const override { return "StringLiteral"; }
+
+    String m_text;
+};
+
+class StringPartCompose final : public Node {
+public:
+    StringPartCompose(Position, RefPtr<Node>, RefPtr<Node>);
+    virtual ~StringPartCompose();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "StringPartCompose"; }
+
+    RefPtr<Node> m_left;
+    RefPtr<Node> m_right;
+};
+
+class SyntaxError final : public Node {
+public:
+    SyntaxError(Position);
+    virtual ~SyntaxError();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override { return { nullptr, nullptr }; }
+    virtual String class_name() const override { return "SyntaxError"; }
+};
+
+class Tilde final : public Node {
+public:
+    Tilde(Position, String);
+    virtual ~Tilde();
+    String text() const;
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual Vector<Line::CompletionSuggestion> complete_for_editor(Shell&, size_t, RefPtr<Node> matching_node) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "Tilde"; }
+    virtual bool is_tilde() const override { return true; }
+
+    String m_username;
+};
+
+class VariableDeclarations final : public Node {
+public:
+    struct Variable {
+        RefPtr<Node> name;
+        RefPtr<Node> value;
+    };
+    VariableDeclarations(Position, Vector<Variable> variables);
+    virtual ~VariableDeclarations();
+
+    const Vector<Variable>& variables() const { return m_variables; }
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    virtual HitTestResult hit_test_position(size_t) override;
+    virtual String class_name() const override { return "VariableDeclarations"; }
+    virtual bool is_variable_decls() const override { return true; }
+
+    Vector<Variable> m_variables;
+};
+
+class WriteAppendRedirection final : public PathRedirectionNode {
+public:
+    WriteAppendRedirection(Position, int, RefPtr<Node>);
+    virtual ~WriteAppendRedirection();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual String class_name() const override { return "WriteAppendRedirection"; }
+};
+
+class WriteRedirection final : public PathRedirectionNode {
+public:
+    WriteRedirection(Position, int, RefPtr<Node>);
+    virtual ~WriteRedirection();
+
+private:
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(TheExecutionInputType) override;
+    virtual String class_name() const override { return "WriteRedirection"; }
+};
+
+}

Shell/Builtin.cpp

@@ -0,0 +1,701 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "Shell.h"
+#include <AK/LexicalPath.h>
+#include <LibCore/ArgsParser.h>
+#include <LibCore/File.h>
+#include <signal.h>
+#include <sys/wait.h>
+
+extern RefPtr<Line::Editor> editor;
+
+int Shell::builtin_bg(int argc, const char** argv)
+{
+    int job_id = -1;
+
+    Core::ArgsParser parser;
+    parser.add_positional_argument(job_id, "Job ID to run in background", "job-id", Core::ArgsParser::Required::No);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    if (job_id == -1 && !jobs.is_empty())
+        job_id = find_last_job_id();
+
+    auto* job = const_cast<Job*>(find_job(job_id));
+
+    if (!job) {
+        if (job_id == -1) {
+            fprintf(stderr, "bg: no current job\n");
+        } else {
+            fprintf(stderr, "bg: job with id %d not found\n", job_id);
+        }
+        return 1;
+    }
+
+    job->set_running_in_background(true);
+
+    dbg() << "Resuming " << job->pid() << " (" << job->cmd() << ")";
+    fprintf(stderr, "Resuming job %llu - %s\n", job->job_id(), job->cmd().characters());
+
+    if (killpg(job->pgid(), SIGCONT) < 0) {
+        perror("killpg");
+        return 1;
+    }
+
+    return 0;
+}
+
+int Shell::builtin_cd(int argc, const char** argv)
+{
+    const char* arg_path = nullptr;
+
+    Core::ArgsParser parser;
+    parser.add_positional_argument(arg_path, "Path to change to", "path", Core::ArgsParser::Required::No);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    String new_path;
+
+    if (!arg_path) {
+        new_path = home;
+        if (cd_history.is_empty() || cd_history.last() != home)
+            cd_history.enqueue(home);
+    } else {
+        if (cd_history.is_empty() || cd_history.last() != arg_path)
+            cd_history.enqueue(arg_path);
+        if (strcmp(arg_path, "-") == 0) {
+            char* oldpwd = getenv("OLDPWD");
+            if (oldpwd == nullptr)
+                return 1;
+            new_path = oldpwd;
+        } else if (arg_path[0] == '/') {
+            new_path = argv[1];
+        } else {
+            StringBuilder builder;
+            builder.append(cwd);
+            builder.append('/');
+            builder.append(arg_path);
+            new_path = builder.to_string();
+        }
+    }
+
+    auto real_path = Core::File::real_path_for(new_path);
+    if (real_path.is_empty()) {
+        fprintf(stderr, "Invalid path '%s'\n", new_path.characters());
+        return 1;
+    }
+    const char* path = real_path.characters();
+
+    int rc = chdir(path);
+    if (rc < 0) {
+        if (errno == ENOTDIR) {
+            fprintf(stderr, "Not a directory: %s\n", path);
+        } else {
+            fprintf(stderr, "chdir(%s) failed: %s\n", path, strerror(errno));
+        }
+        return 1;
+    }
+    setenv("OLDPWD", cwd.characters(), 1);
+    cwd = real_path;
+    setenv("PWD", cwd.characters(), 1);
+    return 0;
+}
+
+int Shell::builtin_cdh(int argc, const char** argv)
+{
+    int index = -1;
+
+    Core::ArgsParser parser;
+    parser.add_positional_argument(index, "Index of the cd history entry (leave out for a list)", "index", Core::ArgsParser::Required::No);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    if (index == -1) {
+        if (cd_history.is_empty()) {
+            fprintf(stderr, "cdh: no history available\n");
+            return 0;
+        }
+
+        for (ssize_t i = cd_history.size() - 1; i >= 0; --i)
+            printf("%lu: %s\n", cd_history.size() - i, cd_history.at(i).characters());
+        return 0;
+    }
+
+    if (index < 1 || (size_t)index > cd_history.size()) {
+        fprintf(stderr, "cdh: history index out of bounds: %d not in (0, %zu)\n", index, cd_history.size());
+        return 1;
+    }
+
+    const char* path = cd_history.at(cd_history.size() - index).characters();
+    const char* cd_args[] = { "cd", path, nullptr };
+    return Shell::builtin_cd(2, cd_args);
+}
+
+int Shell::builtin_dirs(int argc, const char** argv)
+{
+    // The first directory in the stack is ALWAYS the current directory
+    directory_stack.at(0) = cwd.characters();
+
+    bool clear = false;
+    bool print = false;
+    bool number_when_printing = false;
+    char separator = ' ';
+
+    Vector<const char*> paths;
+
+    Core::ArgsParser parser;
+    parser.add_option(clear, "Clear the directory stack", "clear", 'c');
+    parser.add_option(print, "Print directory entries one per line", "print", 'p');
+    parser.add_option(number_when_printing, "Number the directories in the stack when printing", "number", 'v');
+    parser.add_positional_argument(paths, "Extra paths to put on the stack", "path", Core::ArgsParser::Required::No);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    // -v implies -p
+    print = print || number_when_printing;
+
+    if (print) {
+        if (!paths.is_empty()) {
+            fprintf(stderr, "dirs: 'print' and 'number' are not allowed when any path is specified");
+            return 1;
+        }
+        separator = '\n';
+    }
+
+    if (clear) {
+        for (size_t i = 1; i < directory_stack.size(); i++)
+            directory_stack.remove(i);
+    }
+
+    for (auto& path : paths)
+        directory_stack.append(path);
+
+    if (print || (!clear && paths.is_empty())) {
+        int index = 0;
+        for (auto& directory : directory_stack) {
+            if (number_when_printing)
+                printf("%d ", index++);
+            print_path(directory);
+            fputc(separator, stdout);
+        }
+    }
+
+    return 0;
+}
+
+int Shell::builtin_exit(int argc, const char** argv)
+{
+    int exit_code = 0;
+    Core::ArgsParser parser;
+    parser.add_positional_argument(exit_code, "Exit code", "code", Core::ArgsParser::Required::No);
+    if (!parser.parse(argc, const_cast<char**>(argv)))
+        return 1;
+
+    if (!jobs.is_empty()) {
+        if (!m_should_ignore_jobs_on_next_exit) {
+            fprintf(stderr, "Shell: You have %zu active job%s, run 'exit' again to really exit.\n", jobs.size(), jobs.size() > 1 ? "s" : "");
+            m_should_ignore_jobs_on_next_exit = true;
+            return 1;
+        }
+    }
+    stop_all_jobs();
+    save_history();
+    printf("Good-bye!\n");
+    exit(exit_code);
+    return 0;
+}
+
+int Shell::builtin_export(int argc, const char** argv)
+{
+    Vector<const char*> vars;
+
+    Core::ArgsParser parser;
+    parser.add_positional_argument(vars, "List of variable[=value]'s", "values", Core::ArgsParser::Required::No);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    if (vars.is_empty()) {
+        for (size_t i = 0; environ[i]; ++i)
+            puts(environ[i]);
+        return 0;
+    }
+
+    for (auto& value : vars) {
+        auto parts = String { value }.split_limit('=', 2);
+
+        if (parts.size() == 1) {
+            auto value = lookup_local_variable(parts[0]);
+            if (value) {
+                auto values = value->resolve_as_list(*this);
+                StringBuilder builder;
+                builder.join(" ", values);
+                parts.append(builder.to_string());
+            } else {
+                // Ignore the export.
+                continue;
+            }
+        }
+
+        int setenv_return = setenv(parts[0].characters(), parts[1].characters(), 1);
+
+        if (setenv_return != 0) {
+            perror("setenv");
+            return 1;
+        }
+
+        if (parts[0] == "PATH")
+            cache_path();
+    }
+
+    return 0;
+}
+
+int Shell::builtin_fg(int argc, const char** argv)
+{
+    int job_id = -1;
+
+    Core::ArgsParser parser;
+    parser.add_positional_argument(job_id, "Job ID to bring to foreground", "job-id", Core::ArgsParser::Required::No);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    if (job_id == -1 && !jobs.is_empty())
+        job_id = find_last_job_id();
+
+    auto* job = const_cast<Job*>(find_job(job_id));
+
+    if (!job) {
+        if (job_id == -1) {
+            fprintf(stderr, "fg: no current job\n");
+        } else {
+            fprintf(stderr, "fg: job with id %d not found\n", job_id);
+        }
+        return 1;
+    }
+
+    job->set_running_in_background(false);
+
+    dbg() << "Resuming " << job->pid() << " (" << job->cmd() << ")";
+    fprintf(stderr, "Resuming job %llu - %s\n", job->job_id(), job->cmd().characters());
+
+    if (killpg(job->pgid(), SIGCONT) < 0) {
+        perror("killpg");
+        return 1;
+    }
+
+    block_on_job(job);
+
+    return job->exit_code();
+}
+
+int Shell::builtin_disown(int argc, const char** argv)
+{
+    Vector<const char*> str_job_ids;
+
+    Core::ArgsParser parser;
+    parser.add_positional_argument(str_job_ids, "Id of the jobs to disown (omit for current job)", "job_ids", Core::ArgsParser::Required::No);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    Vector<size_t> job_ids;
+    for (auto& job_id : str_job_ids) {
+        auto id = StringView(job_id).to_uint();
+        if (id.has_value())
+            job_ids.append(id.value());
+        else
+            fprintf(stderr, "disown: Invalid job id %s\n", job_id);
+    }
+
+    if (job_ids.is_empty()) {
+        u64 id = 0;
+        for (auto& job : jobs)
+            id = max(id, job.value->job_id());
+        job_ids.append(id);
+    }
+
+    Vector<size_t> keys_of_jobs_to_disown;
+
+    for (auto id : job_ids) {
+        bool found = false;
+        for (auto& entry : jobs) {
+            if (entry.value->job_id() == id) {
+                keys_of_jobs_to_disown.append(entry.key);
+                found = true;
+                break;
+            }
+        }
+        if (!found) {
+            fprintf(stderr, "disown: job with id %zu not found\n", id);
+        }
+    }
+    if (keys_of_jobs_to_disown.is_empty()) {
+        if (str_job_ids.is_empty()) {
+            fprintf(stderr, "disown: no current job\n");
+        }
+        // An error message has already been printed about the nonexistence of each listed job.
+        return 1;
+    }
+    for (auto job_index : keys_of_jobs_to_disown) {
+        auto job = jobs.get(job_index).value();
+
+        job->deactivate();
+
+        if (!job->is_running_in_background())
+            fprintf(stderr, "disown warning: job %llu is currently not running, 'kill -%d %d' to make it continue\n", job->job_id(), SIGCONT, job->pid());
+
+        jobs.remove(job_index);
+    }
+
+    return 0;
+}
+
+int Shell::builtin_history(int, const char**)
+{
+    for (size_t i = 0; i < editor->history().size(); ++i) {
+        printf("%6zu  %s\n", i, editor->history()[i].characters());
+    }
+    return 0;
+}
+
+int Shell::builtin_jobs(int argc, const char** argv)
+{
+    bool list = false, show_pid = false;
+
+    Core::ArgsParser parser;
+    parser.add_option(list, "List all information about jobs", "list", 'l');
+    parser.add_option(show_pid, "Display the PID of the jobs", "pid", 'p');
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    enum {
+        Basic,
+        OnlyPID,
+        ListAll,
+    } mode { Basic };
+
+    if (show_pid)
+        mode = OnlyPID;
+
+    if (list)
+        mode = ListAll;
+
+    for (auto& job : jobs) {
+        auto pid = job.value->pid();
+        int wstatus;
+        auto rc = waitpid(pid, &wstatus, WNOHANG);
+        if (rc == -1) {
+            perror("waitpid");
+            return 1;
+        }
+        auto status = "running";
+
+        if (rc != 0) {
+            if (WIFEXITED(wstatus))
+                status = "exited";
+
+            if (WIFSTOPPED(wstatus))
+                status = "stopped";
+
+            if (WIFSIGNALED(wstatus))
+                status = "signaled";
+        }
+
+        char background_indicator = '-';
+
+        if (job.value->is_running_in_background())
+            background_indicator = '+';
+
+        switch (mode) {
+        case Basic:
+            printf("[%llu] %c %s %s\n", job.value->job_id(), background_indicator, status, job.value->cmd().characters());
+            break;
+        case OnlyPID:
+            printf("[%llu] %c %d %s %s\n", job.value->job_id(), background_indicator, pid, status, job.value->cmd().characters());
+            break;
+        case ListAll:
+            printf("[%llu] %c %d %d %s %s\n", job.value->job_id(), background_indicator, pid, job.value->pgid(), status, job.value->cmd().characters());
+            break;
+        }
+    }
+
+    return 0;
+}
+
+int Shell::builtin_popd(int argc, const char** argv)
+{
+    if (directory_stack.size() <= 1) {
+        fprintf(stderr, "Shell: popd: directory stack empty\n");
+        return 1;
+    }
+
+    bool should_not_switch = false;
+    String path = directory_stack.take_last();
+
+    Core::ArgsParser parser;
+    parser.add_option(should_not_switch, "Do not switch dirs", "no-switch", 'n');
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    bool should_switch = !should_not_switch;
+
+    // When no arguments are given, popd removes the top directory from the stack and performs a cd to the new top directory.
+    if (argc == 1) {
+        int rc = chdir(path.characters());
+        if (rc < 0) {
+            fprintf(stderr, "chdir(%s) failed: %s\n", path.characters(), strerror(errno));
+            return 1;
+        }
+
+        cwd = path;
+        return 0;
+    }
+
+    LexicalPath lexical_path(path.characters());
+    if (!lexical_path.is_valid()) {
+        fprintf(stderr, "LexicalPath failed to canonicalize '%s'\n", path.characters());
+        return 1;
+    }
+
+    const char* real_path = lexical_path.string().characters();
+
+    struct stat st;
+    int rc = stat(real_path, &st);
+    if (rc < 0) {
+        fprintf(stderr, "stat(%s) failed: %s\n", real_path, strerror(errno));
+        return 1;
+    }
+
+    if (!S_ISDIR(st.st_mode)) {
+        fprintf(stderr, "Not a directory: %s\n", real_path);
+        return 1;
+    }
+
+    if (should_switch) {
+        int rc = chdir(real_path);
+        if (rc < 0) {
+            fprintf(stderr, "chdir(%s) failed: %s\n", real_path, strerror(errno));
+            return 1;
+        }
+
+        cwd = lexical_path.string();
+    }
+
+    return 0;
+}
+
+int Shell::builtin_pushd(int argc, const char** argv)
+{
+    StringBuilder path_builder;
+    bool should_switch = true;
+
+    // From the BASH reference manual: https://www.gnu.org/software/bash/manual/html_node/Directory-Stack-Builtins.html
+    // With no arguments, pushd exchanges the top two directories and makes the new top the current directory.
+    if (argc == 1) {
+        if (directory_stack.size() < 2) {
+            fprintf(stderr, "pushd: no other directory\n");
+            return 1;
+        }
+
+        String dir1 = directory_stack.take_first();
+        String dir2 = directory_stack.take_first();
+        directory_stack.insert(0, dir2);
+        directory_stack.insert(1, dir1);
+
+        int rc = chdir(dir2.characters());
+        if (rc < 0) {
+            fprintf(stderr, "chdir(%s) failed: %s\n", dir2.characters(), strerror(errno));
+            return 1;
+        }
+
+        cwd = dir2;
+
+        return 0;
+    }
+
+    // Let's assume the user's typed in 'pushd <dir>'
+    if (argc == 2) {
+        directory_stack.append(cwd.characters());
+        if (argv[1][0] == '/') {
+            path_builder.append(argv[1]);
+        } else {
+            path_builder.appendf("%s/%s", cwd.characters(), argv[1]);
+        }
+    } else if (argc == 3) {
+        directory_stack.append(cwd.characters());
+        for (int i = 1; i < argc; i++) {
+            const char* arg = argv[i];
+
+            if (arg[0] != '-') {
+                if (arg[0] == '/') {
+                    path_builder.append(arg);
+                } else
+                    path_builder.appendf("%s/%s", cwd.characters(), arg);
+            }
+
+            if (!strcmp(arg, "-n"))
+                should_switch = false;
+        }
+    }
+
+    LexicalPath lexical_path(path_builder.to_string());
+    if (!lexical_path.is_valid()) {
+        fprintf(stderr, "LexicalPath failed to canonicalize '%s'\n", path_builder.to_string().characters());
+        return 1;
+    }
+
+    const char* real_path = lexical_path.string().characters();
+
+    struct stat st;
+    int rc = stat(real_path, &st);
+    if (rc < 0) {
+        fprintf(stderr, "stat(%s) failed: %s\n", real_path, strerror(errno));
+        return 1;
+    }
+
+    if (!S_ISDIR(st.st_mode)) {
+        fprintf(stderr, "Not a directory: %s\n", real_path);
+        return 1;
+    }
+
+    if (should_switch) {
+        int rc = chdir(real_path);
+        if (rc < 0) {
+            fprintf(stderr, "chdir(%s) failed: %s\n", real_path, strerror(errno));
+            return 1;
+        }
+
+        cwd = lexical_path.string();
+    }
+
+    return 0;
+}
+
+int Shell::builtin_pwd(int, const char**)
+{
+    print_path(cwd);
+    fputc('\n', stdout);
+    return 0;
+}
+
+int Shell::builtin_time(int argc, const char** argv)
+{
+    Vector<const char*> args;
+
+    Core::ArgsParser parser;
+    parser.add_positional_argument(args, "Command to execute with arguments", "command", Core::ArgsParser::Required::Yes);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    StringBuilder builder;
+    builder.join(' ', args);
+
+    Core::ElapsedTimer timer;
+    timer.start();
+    // TODO: Exit code
+    run_command(builder.string_view());
+    fprintf(stderr, "Time: %d ms\n", timer.elapsed());
+    return 0;
+}
+
+int Shell::builtin_umask(int argc, const char** argv)
+{
+    const char* mask_text = nullptr;
+
+    Core::ArgsParser parser;
+    parser.add_positional_argument(mask_text, "New mask (omit to get current mask)", "octal-mask", Core::ArgsParser::Required::No);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    if (!mask_text) {
+        mode_t old_mask = umask(0);
+        printf("%#o\n", old_mask);
+        umask(old_mask);
+        return 0;
+    }
+
+    unsigned mask;
+    int matches = sscanf(mask_text, "%o", &mask);
+    if (matches == 1) {
+        umask(mask);
+        return 0;
+    }
+
+    fprintf(stderr, "umask: Invalid mask '%s'\n", mask_text);
+    return 1;
+}
+
+int Shell::builtin_unset(int argc, const char** argv)
+{
+    Vector<const char*> vars;
+
+    Core::ArgsParser parser;
+    parser.add_positional_argument(vars, "List of variables", "variables", Core::ArgsParser::Required::Yes);
+
+    if (!parser.parse(argc, const_cast<char**>(argv), false))
+        return 1;
+
+    for (auto& value : vars) {
+        if (lookup_local_variable(value)) {
+            unset_local_variable(value);
+        } else {
+            unsetenv(value);
+        }
+    }
+
+    return 0;
+}
+
+bool Shell::run_builtin(int argc, const char** argv, int& retval)
+{
+    if (argc == 0)
+        return false;
+
+    StringView name { argv[0] };
+
+#define __ENUMERATE_SHELL_BUILTIN(builtin)      \
+    if (name == #builtin) {                     \
+        retval = builtin_##builtin(argc, argv); \
+        return true;                            \
+    }
+
+    ENUMERATE_SHELL_BUILTINS();
+
+#undef __ENUMERATE_SHELL_BUILTIN
+
+    return false;
+}

Shell/CMakeLists.txt

@@ -1,4 +1,6 @@
 set(SOURCES
+    AST.cpp
+    Builtin.cpp
     main.cpp
     Parser.cpp
     Shell.cpp

Shell/Forward.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+class Shell;
+namespace AST {
+
+class Node;
+class Value;
+
+}

Shell/Job.h

@@ -27,6 +27,7 @@
 #pragma once
 
 #include "Execution.h"
+#include <AK/Function.h>
 #include <AK/JsonObject.h>
 #include <AK/JsonValue.h>
 #include <AK/OwnPtr.h>
@@ -34,7 +35,12 @@
 #include <LibCore/ElapsedTimer.h>
 #include <LibCore/Object.h>
 
-class Job {
+#define JOB_TIME_INFO
+#ifndef __serenity__
+#    undef JOB_TIME_INFO
+#endif
+
+class Job : public RefCounted<Job> {
 public:
     explicit Job()
     {
@@ -42,10 +48,12 @@ public:
 
     ~Job()
     {
+#ifdef JOB_TIME_INFO
         if (m_active) {
             auto elapsed = m_command_timer.elapsed();
             dbg() << "Command \"" << m_cmd << "\" finished in " << elapsed << " ms";
         }
+#endif
     }
 
     Job(pid_t pid, unsigned pgid, String cmd, u64 job_id)
@@ -64,7 +72,10 @@ public:
     u64 job_id() const { return m_job_id; }
     bool exited() const { return m_exited; }
     int exit_code() const { return m_exit_code; }
+    bool should_be_disowned() const { return m_should_be_disowned; }
+    void disown() { m_should_be_disowned = true; }
     bool is_running_in_background() const { return m_running_in_background; }
+    Function<void(RefPtr<Job>)> on_exit;
 
     Core::ElapsedTimer& timer() { return m_command_timer; }
 
@@ -74,6 +85,8 @@ public:
             return;
         m_exit_code = exit_code;
         m_exited = true;
+        if (on_exit)
+            on_exit(*this);
     }
 
     void set_running_in_background(bool running_in_background)
@@ -93,4 +106,5 @@ private:
     int m_exit_code { -1 };
     Core::ElapsedTimer m_command_timer;
     mutable bool m_active { true };
+    bool m_should_be_disowned { false };
 };

Shell/Parser.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2020, the SerenityOS developers.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,111 +26,134 @@
 
 #pragma once
 
+#include "AST.h"
+#include <AK/Function.h>
+#include <AK/RefPtr.h>
 #include <AK/String.h>
+#include <AK/StringBuilder.h>
 #include <AK/Vector.h>
 
-struct Token {
-    enum Type {
-        Bare,
-        SingleQuoted,
-        DoubleQuoted,
-        UnterminatedSingleQuoted,
-        UnterminatedDoubleQuoted,
-        Comment,
-        Special,
-    };
-    String text;
-    size_t end;
-    size_t length;
-    Type type;
-};
-
-enum Attributes {
-    None = 0x0,
-    ShortCircuitOnFailure = 0x1,
-    InBackground = 0x2,
-};
-
-struct Redirection {
-    enum Type {
-        Pipe,
-        FileWrite,
-        FileWriteAppend,
-        FileRead,
-    };
-    Type type;
-    int fd { -1 };
-    int rewire_fd { -1 };
-    size_t redirection_op_start { 0 };
-    Token path {};
-};
-
-struct Rewiring {
-    int fd { -1 };
-    int rewire_fd { -1 };
-};
-
-struct Subcommand {
-    Vector<Token> args;
-    Vector<Redirection> redirections;
-    Vector<Rewiring> rewirings;
-};
-
-struct Command {
-    Vector<Subcommand> subcommands;
-    Attributes attributes;
-};
-
 class Parser {
 public:
-    explicit Parser(const String& input)
+    Parser(StringView input)
-        : m_input(input)
+        : m_input(move(input))
     {
     }
 
-    Vector<Command> parse();
+    RefPtr<AST::Node> parse();
 
 private:
-    enum class AllowEmptyToken {
+    RefPtr<AST::Node> parse_toplevel();
-        No,
+    RefPtr<AST::Node> parse_sequence();
-        Yes,
+    RefPtr<AST::Node> parse_variable_decls();
-    };
+    RefPtr<AST::Node> parse_pipe_sequence();
-    void commit_token(Token::Type, AllowEmptyToken = AllowEmptyToken::No);
+    RefPtr<AST::Node> parse_command();
-    void commit_subcommand();
+    RefPtr<AST::Node> parse_redirection();
-    void commit_command(Attributes = None);
+    RefPtr<AST::Node> parse_list_expression();
-    void do_pipe();
+    RefPtr<AST::Node> parse_expression();
-    void begin_redirect_read(int fd);
+    RefPtr<AST::Node> parse_string_composite();
-    void begin_redirect_write(int fd);
+    RefPtr<AST::Node> parse_string();
+    RefPtr<AST::Node> parse_doublequoted_string_inner();
+    RefPtr<AST::Node> parse_variable();
+    RefPtr<AST::Node> parse_evaluate();
+    RefPtr<AST::Node> parse_comment();
+    RefPtr<AST::Node> parse_bareword();
+    RefPtr<AST::Node> parse_glob();
 
-    enum State {
+    template<typename A, typename... Args>
-        Free,
+    RefPtr<A> create(Args... args);
-        InSingleQuotes,
+
-        InDoubleQuotes,
+    bool at_end() const { return m_input.length() <= m_offset; }
-        InWriteAppendOrRedirectionPath,
+    char peek();
-        InRedirectionPath,
+    char consume();
+    void putback();
+    bool expect(char);
+    bool expect(const StringView&);
+
+    StringView consume_while(Function<bool(char)>);
+
+    struct ScopedOffset {
+        ScopedOffset(Vector<size_t>& offsets, size_t offset)
+            : offsets(offsets)
+            , offset(offset)
+        {
+            offsets.append(offset);
+        }
+        ~ScopedOffset()
+        {
+            auto last = offsets.take_last();
+            ASSERT(last == offset);
+        }
+
+        Vector<size_t>& offsets;
+        size_t offset;
     };
 
-    State state() const { return m_state_stack.last(); }
+    OwnPtr<ScopedOffset> push_start();
 
-    void pop_state()
+    StringView m_input;
-    {
+    size_t m_offset { 0 };
-        m_state_stack.take_last();
+    Vector<size_t> m_rule_start_offsets;
-    }
-
-    void push_state(State state)
-    {
-        m_state_stack.append(state);
-    }
-
-    bool in_state(State) const;
-
-    Vector<State> m_state_stack { Free };
-    String m_input;
-
-    Vector<Command> m_commands;
-    Vector<Subcommand> m_subcommands;
-    Vector<Token> m_tokens;
-    Vector<Redirection> m_redirections;
-    Vector<char> m_token;
-    size_t m_position { 0 };
 };
+
+#if 0
+constexpr auto the_grammar = R"(
+toplevel :: sequence?
+
+sequence :: variable_decls? pipe_sequence ';' sequence
+          | variable_decls? pipe_sequence '&'
+          | variable_decls? pipe_sequence '&' '&' sequence
+          | variable_decls? pipe_sequence '|' '|' sequence
+          | variable_decls? pipe_sequence
+
+variable_decls :: identifier '=' expression (' '+ variable_decls)? ' '*
+
+pipe_sequence :: command '|' pipe_sequence
+               | command
+
+command :: redirection command
+         | list_expression command?
+
+redirection :: number? '>'{1,2} ' '* string_composite
+             | number? '<' ' '* string_composite
+             | number? '>' '&' number
+
+list_expression :: ' '* expression (' '+ list_expression)?
+
+expression :: evaluate
+            | string_composite
+            | comment
+            | '(' list_expression ')'
+
+evaluate :: '$' expression {eval / dynamic resolve}
+
+string_composite :: string string_composite?
+                  | variable string_composite?
+                  | bareword string_composite?
+                  | glob string_composite?
+
+string :: '"' dquoted_string_inner '"'
+        | "'" [^']* "'"
+
+dquoted_string_inner :: '\' . dquoted_string_inner?       {concat}
+                      | variable dquoted_string_inner?    {compose}
+                      | . dquoted_string_inner?
+                      | '\' 'x' digit digit dquoted_string_inner?
+                      | '\' [abefrn] dquoted_string_inner?
+
+variable :: '$' identifier
+          | '$' '$'
+          | '$' '?'
+          | ...
+
+comment :: '#' [^\n]*
+
+bareword :: [^"'*$&#|()[\]{} ?;<>] bareword?
+          | '\' [^"'*$&#|()[\]{} ?;<>] bareword?
+
+bareword_with_tilde_expansion :: '~' bareword?
+
+glob :: [*?] bareword?
+      | bareword [*?]
+)";
+#endif

Shell/Shell.h

@@ -39,37 +39,6 @@
 #include <LibLine/Editor.h>
 #include <termios.h>
 
-struct ExitCodeOrContinuationRequest {
-    enum ContinuationRequest {
-        Nothing,
-        Pipe,
-        DoubleQuotedString,
-        SingleQuotedString,
-    };
-
-    ExitCodeOrContinuationRequest(ContinuationRequest continuation)
-        : continuation(continuation)
-    {
-    }
-
-    ExitCodeOrContinuationRequest(int exit)
-        : exit_code(exit)
-    {
-    }
-
-    bool has_value() const { return exit_code.has_value(); }
-    int value() const
-    {
-        ASSERT(has_value());
-        return exit_code.value();
-    }
-
-    Optional<int> exit_code;
-    ContinuationRequest continuation { Nothing };
-};
-
-using ContinuationRequest = ExitCodeOrContinuationRequest::ContinuationRequest;
-
 #define ENUMERATE_SHELL_BUILTINS()     \
     __ENUMERATE_SHELL_BUILTIN(cd)      \
     __ENUMERATE_SHELL_BUILTIN(cdh)     \
@@ -94,13 +63,21 @@ class Shell : public Core::Object {
     C_OBJECT(Shell);
 
 public:
-    ExitCodeOrContinuationRequest run_command(const StringView&);
+    int run_command(const StringView&);
+    RefPtr<Job> run_command(AST::Command&);
     bool run_builtin(int argc, const char** argv, int& retval);
+    void block_on_job(RefPtr<Job>);
     String prompt() const;
 
     static String expand_tilde(const String&);
-    static Vector<String> expand_globs(const StringView& path, const StringView& base);
+    static Vector<String> expand_globs(const StringView& path, StringView base);
-    Vector<String> expand_parameters(const StringView&) const;
+    static Vector<String> expand_globs(Vector<StringView> path_segments, const StringView& base);
+    String resolve_path(String) const;
+
+    RefPtr<AST::Value> lookup_local_variable(const String&);
+    String local_variable_or(const String&, const String&);
+    void set_local_variable(const String&, RefPtr<AST::Value>);
+    void unset_local_variable(const String&);
 
     static String escape_token(const String& token);
     static String unescape_token(const String& token);
@@ -108,25 +85,22 @@ public:
     static bool is_glob(const StringView&);
     static Vector<StringView> split_path(const StringView&);
 
-    Vector<String> process_arguments(const Vector<Token>&);
-
-    static ContinuationRequest is_complete(const Vector<Command>&);
-
     void highlight(Line::Editor&) const;
     Vector<Line::CompletionSuggestion> complete(const Line::Editor&);
+    Vector<Line::CompletionSuggestion> complete_path(const String&, size_t offset);
+    Vector<Line::CompletionSuggestion> complete_program_name(const String&, size_t offset);
+    Vector<Line::CompletionSuggestion> complete_variable(const String&, size_t offset);
 
-    bool is_waiting_for(pid_t pid) const { return m_waiting_for_pid == pid; }
+    void take_back_stdin();
 
     u64 find_last_job_id() const;
+    const Job* find_job(u64 id);
 
     String get_history_path();
     void load_history();
     void save_history();
     void print_path(const String& path);
 
-    bool should_read_more() const { return m_should_continue != ContinuationRequest::Nothing; }
-    void finish_command() { m_should_break_current_command = true; }
-
     bool read_single_line();
 
     struct termios termios;
@@ -148,12 +122,11 @@ public:
     int last_return_code { 0 };
     Vector<String> directory_stack;
     CircularQueue<String, 8> cd_history; // FIXME: have a configurable cd history length
-    HashMap<u64, OwnPtr<Job>> jobs;
+    HashMap<u64, RefPtr<Job>> jobs;
     Vector<String, 256> cached_path;
 
     enum ShellEventType {
         ReadLine,
-        ChildExited,
     };
 
 private:
@@ -163,16 +136,9 @@ private:
     // ^Core::Object
     virtual void save_to(JsonObject&) override;
 
-    struct SpawnedProcess {
-        String name;
-        pid_t pid;
-    };
-
     void cache_path();
     void stop_all_jobs();
 
-    IterationDecision wait_for_pid(const SpawnedProcess&, bool is_first_command_in_chain, int& return_value);
-
     virtual void custom_event(Core::CustomEvent&) override;
 
 #define __ENUMERATE_SHELL_BUILTIN(builtin) \
@@ -190,11 +156,11 @@ private:
 #undef __ENUMERATE_SHELL_BUILTIN
     };
 
-    ExitCodeOrContinuationRequest::ContinuationRequest m_should_continue { ExitCodeOrContinuationRequest::Nothing };
     StringBuilder m_complete_line_builder;
-    bool m_should_break_current_command { false };
     bool m_should_ignore_jobs_on_next_exit { false };
-    pid_t m_waiting_for_pid { -1 };
+    pid_t m_pid { 0 };
+
+    HashMap<String, RefPtr<AST::Value>> m_local_variables;
 };
 
 static constexpr bool is_word_character(char c)

Shell/main.cpp

@@ -73,17 +73,22 @@ int main(int argc, char** argv)
 
     signal(SIGCHLD, [](int) {
         auto& jobs = s_shell->jobs;
+        Vector<u64> disowned_jobs;
         for (auto& job : jobs) {
-            if (s_shell->is_waiting_for(job.value->pid()))
-                continue;
             int wstatus = 0;
             auto child_pid = waitpid(job.value->pid(), &wstatus, WNOHANG);
             if (child_pid == job.value->pid()) {
-                if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) {
+                if (WIFEXITED(wstatus)) {
-                    Core::EventLoop::current().post_event(*s_shell, make<Core::CustomEvent>(Shell::ShellEventType::ChildExited, job.value));
+                    job.value->set_has_exit(WEXITSTATUS(wstatus));
+                } else if (WIFSIGNALED(wstatus) && !WIFSTOPPED(wstatus)) {
+                    job.value->set_has_exit(126);
                 }
             }
+            if (job.value->should_be_disowned())
+                disowned_jobs.append(job.key);
         }
+        for (auto key : disowned_jobs)
+            jobs.remove(key);
     });
 
     // Ignore SIGTSTP as the shell should not be suspended with ^Z.
@@ -133,10 +138,7 @@ int main(int argc, char** argv)
     }
 
     editor->on_interrupt_handled = [&] {
-        if (shell->should_read_more()) {
+        editor->finish();
-            shell->finish_command();
-            editor->finish();
-        }
     };
 
     shell->add_child(*editor);