Shell: Implement program-aware autocompletion

A program can either respond to `--complete -- some args to complete`
directly, or add a `_complete_<program name>` invokable (i.e. shell
function, or just a plain binary in PATH) that completes the given
command and lists the completions on stdout.
Should such a completion fail or yield no results, we'll fall back to
the previous completion algorithm.

Userland/Shell/AST.cpp

@@ -368,7 +368,7 @@ Vector<Line::CompletionSuggestion> Node::complete_for_editor(Shell& shell, size_
 
             // If the literal isn't an option, treat it as a path.
             if (!(text.starts_with("-") || text == "--" || text == "-"))
-                return set_results_trivia(shell.complete_path("", text, corrected_offset, Shell::ExecutableOnly::No, escape_mode));
+                return set_results_trivia(shell.complete_path("", text, corrected_offset, Shell::ExecutableOnly::No, hit_test_result.closest_command_node.ptr(), hit_test_result.matching_node, escape_mode));
 
             // If the literal is an option, we have to know the program name
             // should we have no way to get that, bail early.
@@ -386,7 +386,7 @@ Vector<Line::CompletionSuggestion> Node::complete_for_editor(Shell& shell, size_
             else
                 program_name = static_cast<StringLiteral*>(program_name_node.ptr())->text();
 
-            return set_results_trivia(shell.complete_option(program_name, text, corrected_offset));
+            return set_results_trivia(shell.complete_option(program_name, text, corrected_offset, hit_test_result.closest_command_node.ptr(), hit_test_result.matching_node));
         }
         return {};
     }
@@ -1589,6 +1589,9 @@ void Execute::for_each_entry(RefPtr<Shell> shell, Function<IterationDecision(Non
     if (shell && shell->has_any_error())
         return;
 
+    if (!shell)
+        return;
+
     auto commands = shell->expand_aliases(move(unexpanded_commands));
 
     if (m_capture_stdout) {
@@ -2464,7 +2467,7 @@ Vector<Line::CompletionSuggestion> PathRedirectionNode::complete_for_editor(Shel
     if (corrected_offset > node->text().length())
         return {};
 
-    return shell.complete_path("", node->text(), corrected_offset, Shell::ExecutableOnly::No);
+    return shell.complete_path("", node->text(), corrected_offset, Shell::ExecutableOnly::No, nullptr, nullptr);
 }
 
 PathRedirectionNode::~PathRedirectionNode()
@@ -3042,7 +3045,7 @@ Vector<Line::CompletionSuggestion> Juxtaposition::complete_for_editor(Shell& she
     if (corrected_offset > right_value.length())
         return {};
 
-    return shell.complete_path(left_value, right_value, corrected_offset, Shell::ExecutableOnly::No);
+    return shell.complete_path(left_value, right_value, corrected_offset, Shell::ExecutableOnly::No, hit_test_result.closest_command_node.ptr(), hit_test_result.matching_node);
 }
 
 HitTestResult Juxtaposition::hit_test_position(size_t offset) const
@@ -3408,6 +3411,14 @@ VariableDeclarations::~VariableDeclarations()
 Value::~Value()
 {
 }
+
+String Value::resolve_as_string(RefPtr<Shell> shell)
+{
+    if (shell)
+        shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "Conversion to string not allowed");
+    return {};
+}
+
 Vector<AST::Command> Value::resolve_as_commands(RefPtr<Shell> shell)
 {
     Command command;
@@ -3482,10 +3493,9 @@ Vector<Command> CommandSequenceValue::resolve_as_commands(RefPtr<Shell>)
     return m_contained_values;
 }
 
-Vector<String> CommandValue::resolve_as_list(RefPtr<Shell> shell)
+Vector<String> CommandValue::resolve_as_list(RefPtr<Shell>)
 {
-    shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "Unexpected cast of a command to a list");
+    return m_command.argv;
-    return {};
 }
 
 Vector<Command> CommandValue::resolve_as_commands(RefPtr<Shell>)
@@ -3500,6 +3510,14 @@ JobValue::~JobValue()
 StringValue::~StringValue()
 {
 }
+
+String StringValue::resolve_as_string(RefPtr<Shell> shell)
+{
+    if (m_split.is_null())
+        return m_string;
+    return Value::resolve_as_string(shell);
+}
+
 Vector<String> StringValue::resolve_as_list(RefPtr<Shell> shell)
 {
     if (is_list()) {
@@ -3539,6 +3557,19 @@ Vector<String> GlobValue::resolve_as_list(RefPtr<Shell> shell)
 SimpleVariableValue::~SimpleVariableValue()
 {
 }
+
+String SimpleVariableValue::resolve_as_string(RefPtr<Shell> shell)
+{
+    if (!shell)
+        return resolve_slices(shell, String {}, m_slices);
+
+    if (auto value = resolve_without_cast(shell); value != this)
+        return value->resolve_as_string(shell);
+
+    char* env_value = getenv(m_name.characters());
+    return resolve_slices(shell, env_value, m_slices);
+}
+
 Vector<String> SimpleVariableValue::resolve_as_list(RefPtr<Shell> shell)
 {
     if (!shell)
@@ -3574,6 +3605,21 @@ SpecialVariableValue::~SpecialVariableValue()
 {
 }
 
+String SpecialVariableValue::resolve_as_string(RefPtr<Shell> shell)
+{
+    if (!shell)
+        return {};
+
+    auto result = resolve_as_list(shell);
+    if (result.size() == 1)
+        return result[0];
+
+    if (result.is_empty())
+        return {};
+
+    return Value::resolve_as_string(shell);
+}
+
 Vector<String> SpecialVariableValue::resolve_as_list(RefPtr<Shell> shell)
 {
     if (!shell)
@@ -3605,6 +3651,12 @@ Vector<String> SpecialVariableValue::resolve_as_list(RefPtr<Shell> shell)
 TildeValue::~TildeValue()
 {
 }
+
+String TildeValue::resolve_as_string(RefPtr<Shell> shell)
+{
+    return resolve_as_list(shell).first();
+}
+
 Vector<String> TildeValue::resolve_as_list(RefPtr<Shell> shell)
 {
     StringBuilder builder;

Userland/Shell/AST.h

@@ -217,6 +217,7 @@ struct HitTestResult {
 class Value : public RefCounted<Value> {
 public:
     virtual Vector<String> resolve_as_list(RefPtr<Shell>) = 0;
+    virtual String resolve_as_string(RefPtr<Shell> shell);
     virtual Vector<Command> resolve_as_commands(RefPtr<Shell>);
     virtual NonnullRefPtr<Value> resolve_without_cast(RefPtr<Shell>) { return *this; }
     virtual NonnullRefPtr<Value> clone() const = 0;
@@ -279,6 +280,7 @@ private:
 class JobValue final : public Value {
 public:
     virtual Vector<String> resolve_as_list(RefPtr<Shell>) override { VERIFY_NOT_REACHED(); }
+    virtual String resolve_as_string(RefPtr<Shell>) override { return String::formatted("%{}", m_job->job_id()); }
     virtual Vector<Command> resolve_as_commands(RefPtr<Shell>) override { VERIFY_NOT_REACHED(); }
     virtual NonnullRefPtr<Value> clone() const override { return make_ref_counted<JobValue>(m_job)->set_slices(m_slices); }
     virtual ~JobValue();
@@ -322,6 +324,7 @@ private:
 class StringValue final : public Value {
 public:
     virtual Vector<String> resolve_as_list(RefPtr<Shell>) override;
+    virtual String resolve_as_string(RefPtr<Shell> shell) override;
     virtual NonnullRefPtr<Value> clone() const override { return make_ref_counted<StringValue>(m_string, m_split, m_keep_empty)->set_slices(m_slices); }
     virtual ~StringValue();
     virtual bool is_string() const override { return m_split.is_null(); }
@@ -360,6 +363,7 @@ private:
 class SimpleVariableValue final : public Value {
 public:
     virtual Vector<String> resolve_as_list(RefPtr<Shell>) override;
+    virtual String resolve_as_string(RefPtr<Shell>) override;
     virtual NonnullRefPtr<Value> resolve_without_cast(RefPtr<Shell>) override;
     virtual NonnullRefPtr<Value> clone() const override { return make_ref_counted<SimpleVariableValue>(m_name)->set_slices(m_slices); }
     virtual ~SimpleVariableValue();
@@ -375,6 +379,7 @@ private:
 class SpecialVariableValue final : public Value {
 public:
     virtual Vector<String> resolve_as_list(RefPtr<Shell>) override;
+    virtual String resolve_as_string(RefPtr<Shell>) override;
     virtual NonnullRefPtr<Value> clone() const override { return make_ref_counted<SpecialVariableValue>(m_name)->set_slices(m_slices); }
     virtual ~SpecialVariableValue();
     SpecialVariableValue(char name)
@@ -389,6 +394,7 @@ private:
 class TildeValue final : public Value {
 public:
     virtual Vector<String> resolve_as_list(RefPtr<Shell>) override;
+    virtual String resolve_as_string(RefPtr<Shell>) override;
     virtual NonnullRefPtr<Value> clone() const override { return make_ref_counted<TildeValue>(m_username)->set_slices(m_slices); }
     virtual ~TildeValue();
     virtual bool is_string() const override { return true; }

Userland/Shell/ImmediateFunctions.cpp

@@ -408,7 +408,7 @@ RefPtr<AST::Node> Shell::immediate_filter_glob(AST::ImmediateExpression& invokin
         if (value.size() == 1) {
             if (!value.first().matches(glob))
                 return IterationDecision::Continue;
-            result.append(AST::make_ref_counted<AST::StringLiteral>(arguments[1].position(), value.first()));
+            result.append(AST::make_ref_counted<AST::StringLiteral>(arguments[1].position(), value.first(), AST::StringLiteral::EnclosureType::None));
             return IterationDecision::Continue;
         }
 
@@ -416,7 +416,7 @@ RefPtr<AST::Node> Shell::immediate_filter_glob(AST::ImmediateExpression& invokin
             if (entry.matches(glob)) {
                 NonnullRefPtrVector<AST::Node> nodes;
                 for (auto& string : value)
-                    nodes.append(AST::make_ref_counted<AST::StringLiteral>(arguments[1].position(), string));
+                    nodes.append(AST::make_ref_counted<AST::StringLiteral>(arguments[1].position(), string, AST::StringLiteral::EnclosureType::None));
                 result.append(AST::make_ref_counted<AST::ListConcatenate>(arguments[1].position(), move(nodes)));
                 return IterationDecision::Continue;
             }
@@ -450,7 +450,7 @@ RefPtr<AST::Node> Shell::immediate_join(AST::ImmediateExpression& invoking_node,
     StringBuilder builder;
     builder.join(delimiter_str, value->resolve_as_list(*this));
 
-    return AST::make_ref_counted<AST::StringLiteral>(invoking_node.position(), builder.to_string());
+    return AST::make_ref_counted<AST::StringLiteral>(invoking_node.position(), builder.to_string(), AST::StringLiteral::EnclosureType::None);
 }
 
 RefPtr<AST::Node> Shell::run_immediate_function(StringView str, AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments)

Userland/Shell/Shell.cpp

@@ -11,6 +11,7 @@
 #include <AK/Debug.h>
 #include <AK/Function.h>
 #include <AK/GenericLexer.h>
+#include <AK/JsonParser.h>
 #include <AK/LexicalPath.h>
 #include <AK/QuickSort.h>
 #include <AK/ScopeGuard.h>
@@ -22,7 +23,9 @@
 #include <LibCore/Event.h>
 #include <LibCore/EventLoop.h>
 #include <LibCore/File.h>
+#include <LibCore/Stream.h>
 #include <LibCore/System.h>
+#include <LibCore/Timer.h>
 #include <LibLine/Editor.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -1410,7 +1413,7 @@ Vector<Line::CompletionSuggestion> Shell::complete()
     return ast->complete_for_editor(*this, line.length());
 }
 
-Vector<Line::CompletionSuggestion> Shell::complete_path(StringView base, StringView part, size_t offset, ExecutableOnly executable_only, EscapeMode escape_mode)
+Vector<Line::CompletionSuggestion> Shell::complete_path(StringView base, StringView part, size_t offset, ExecutableOnly executable_only, AST::Node const* command_node, AST::Node const* node, EscapeMode escape_mode)
 {
     auto token = offset ? part.substring_view(0, offset) : "";
     String path;
@@ -1419,6 +1422,12 @@ Vector<Line::CompletionSuggestion> Shell::complete_path(StringView base, StringV
     while (last_slash >= 0 && token[last_slash] != '/')
         --last_slash;
 
+    if (command_node) {
+        auto program_results = complete_via_program_itself(offset, command_node, node, escape_mode, {});
+        if (!program_results.is_error())
+            return program_results.release_value();
+    }
+
     StringBuilder path_builder;
     auto init_slash_part = token.substring_view(0, last_slash + 1);
     auto last_slash_part = token.substring_view(last_slash + 1, token.length() - last_slash - 1);
@@ -1502,7 +1511,7 @@ Vector<Line::CompletionSuggestion> Shell::complete_program_name(StringView name,
         });
 
     if (!match)
-        return complete_path("", name, offset, ExecutableOnly::Yes, escape_mode);
+        return complete_path("", name, offset, ExecutableOnly::Yes, nullptr, nullptr, escape_mode);
 
     String completion = *match;
     auto token_length = escape_token(name, escape_mode).length();
@@ -1603,8 +1612,14 @@ Vector<Line::CompletionSuggestion> Shell::complete_user(StringView name, size_t
     return suggestions;
 }
 
-Vector<Line::CompletionSuggestion> Shell::complete_option(StringView program_name, StringView option, size_t offset)
+Vector<Line::CompletionSuggestion> Shell::complete_option(StringView program_name, StringView option, size_t offset, AST::Node const* command_node, AST::Node const* node)
 {
+    if (command_node) {
+        auto program_results = complete_via_program_itself(offset, command_node, node, EscapeMode::Bareword, program_name);
+        if (!program_results.is_error())
+            return program_results.release_value();
+    }
+
     size_t start = 0;
     while (start < option.length() && option[start] == '-' && start < 2)
         ++start;
@@ -1614,44 +1629,237 @@ Vector<Line::CompletionSuggestion> Shell::complete_option(StringView program_nam
     if (m_editor)
         m_editor->transform_suggestion_offsets(invariant_offset, static_offset);
 
-    Vector<Line::CompletionSuggestion> suggestions;
-
     dbgln("Shell::complete_option({}, {})", program_name, option_pattern);
+    return {};
+}
 
-    // FIXME: Figure out how to do this stuff.
+ErrorOr<Vector<Line::CompletionSuggestion>> Shell::complete_via_program_itself(size_t, AST::Node const* command_node, AST::Node const* node, EscapeMode, StringView known_program_name)
-    if (has_builtin(program_name)) {
+{
-        // Complete builtins.
+    if (!command_node)
-        if (program_name == "setopt") {
+        return Error::from_string_literal("Cannot complete null command");
-            bool negate = false;
-            if (option_pattern.starts_with("no_")) {
-                negate = true;
-                option_pattern = option_pattern.substring_view(3, option_pattern.length() - 3);
-            }
-            auto maybe_negate = [&](StringView view) {
-                static StringBuilder builder;
-                builder.clear();
-                builder.append("--");
-                if (negate)
-                    builder.append("no_");
-                builder.append(view);
-                return builder.to_string();
-            };
-#define __ENUMERATE_SHELL_OPTION(name, d_, descr_) \
-    if (#name##sv.starts_with(option_pattern))     \
-        suggestions.append(maybe_negate(#name));
 
-            ENUMERATE_SHELL_OPTIONS();
+    if (command_node->would_execute())
-#undef __ENUMERATE_SHELL_OPTION
+        return Error::from_string_literal("Refusing to complete nodes that would execute");
 
-            for (auto& entry : suggestions) {
+    String program_name_storage;
-                entry.input_offset = offset;
+    if (known_program_name.is_null()) {
-                entry.invariant_offset = invariant_offset;
+        auto node = command_node->leftmost_trivial_literal();
-                entry.static_offset = static_offset;
+        if (!node)
-            }
+            return Error::from_string_literal("Cannot complete");
 
-            return suggestions;
+        program_name_storage = node->run(*this)->resolve_as_string(*this);
-        }
+        known_program_name = program_name_storage;
     }
+
+    auto program_name = known_program_name;
+
+    AST::Command completion_command;
+    completion_command.argv.append(program_name);
+    completion_command = expand_aliases({ completion_command }).last();
+
+    auto completion_utility_name = String::formatted("_complete_{}", completion_command.argv[0]);
+    if (binary_search(cached_path, completion_utility_name))
+        completion_command.argv[0] = completion_utility_name;
+
+    completion_command.argv.extend({ "--complete", "--" });
+
+    struct Visitor : public AST::NodeVisitor {
+        Visitor(Shell& shell, AST::Node const& node)
+            : shell(shell)
+            , completion_position(node.position())
+        {
+            lists.empend();
+        }
+
+        Shell& shell;
+        AST::Position completion_position;
+        Vector<Vector<String>> lists;
+        bool fail { false };
+
+        void push_list() { lists.empend(); }
+        Vector<String> pop_list() { return lists.take_last(); }
+        Vector<String>& list() { return lists.last(); }
+
+        bool should_include(AST::Node const* node) const { return node->position().end_offset <= completion_position.end_offset; }
+
+        virtual void visit(AST::BarewordLiteral const* node) override
+        {
+            if (should_include(node))
+                list().append(node->text());
+        }
+
+        virtual void visit(AST::BraceExpansion const* node) override
+        {
+            if (should_include(node))
+                list().extend(static_cast<AST::Node*>(const_cast<AST::BraceExpansion*>(node))->run(shell)->resolve_as_list(shell));
+        }
+
+        virtual void visit(AST::CommandLiteral const* node) override
+        {
+            if (should_include(node))
+                list().extend(node->command().argv);
+        }
+
+        virtual void visit(AST::DynamicEvaluate const* node) override
+        {
+            if (should_include(node))
+                fail = true;
+        }
+
+        virtual void visit(AST::DoubleQuotedString const* node) override
+        {
+            if (!should_include(node))
+                return;
+
+            push_list();
+            AST::NodeVisitor::visit(node);
+            auto list = pop_list();
+            StringBuilder builder;
+            builder.join("", list);
+            this->list().append(builder.build());
+        }
+
+        virtual void visit(AST::Glob const* node) override
+        {
+            if (should_include(node))
+                list().append(node->text());
+        }
+
+        virtual void visit(AST::Heredoc const* node) override
+        {
+            if (!should_include(node))
+                return;
+
+            push_list();
+            AST::NodeVisitor::visit(node);
+            auto list = pop_list();
+            StringBuilder builder;
+            builder.join("", list);
+            this->list().append(builder.build());
+        }
+
+        virtual void visit(AST::ImmediateExpression const* node) override
+        {
+            if (should_include(node))
+                fail = true;
+        }
+
+        virtual void visit(AST::Range const* node) override
+        {
+            if (!should_include(node))
+                return;
+
+            push_list();
+            node->start()->visit(*this);
+            list().append(pop_list().first());
+        }
+
+        virtual void visit(AST::SimpleVariable const* node) override
+        {
+            if (should_include(node))
+                list().extend(static_cast<AST::Node*>(const_cast<AST::SimpleVariable*>(node))->run(shell)->resolve_as_list(shell));
+        }
+
+        virtual void visit(AST::SpecialVariable const* node) override
+        {
+            if (should_include(node))
+                list().extend(static_cast<AST::Node*>(const_cast<AST::SpecialVariable*>(node))->run(shell)->resolve_as_list(shell));
+        }
+
+        virtual void visit(AST::Juxtaposition const* node) override
+        {
+            if (!should_include(node))
+                return;
+
+            push_list();
+            node->left()->visit(*this);
+            auto left = pop_list();
+
+            push_list();
+            node->right()->visit(*this);
+            auto right = pop_list();
+
+            StringBuilder builder;
+            for (auto& left_entry : left) {
+                for (auto& right_entry : right) {
+                    builder.append(left_entry);
+                    builder.append(right_entry);
+                    list().append(builder.build());
+                    builder.clear();
+                }
+            }
+        }
+
+        virtual void visit(AST::StringLiteral const* node) override
+        {
+            if (should_include(node))
+                list().append(node->text());
+        }
+
+        virtual void visit(AST::Tilde const* node) override
+        {
+            if (should_include(node))
+                list().extend(static_cast<AST::Node*>(const_cast<AST::Tilde*>(node))->run(shell)->resolve_as_list(shell));
+        }
+
+        virtual void visit(AST::PathRedirectionNode const*) override { }
+        virtual void visit(AST::CloseFdRedirection const*) override { }
+        virtual void visit(AST::Fd2FdRedirection const*) override { }
+        virtual void visit(AST::Execute const*) override { }
+        virtual void visit(AST::ReadRedirection const*) override { }
+        virtual void visit(AST::ReadWriteRedirection const*) override { }
+        virtual void visit(AST::WriteAppendRedirection const*) override { }
+        virtual void visit(AST::WriteRedirection const*) override { }
+    } visitor { *this, *node };
+
+    command_node->visit(visitor);
+    if (visitor.fail)
+        return Error::from_string_literal("Cannot complete");
+
+    completion_command.argv.extend(visitor.list());
+
+    completion_command.should_wait = true;
+    completion_command.redirections.append(AST::PathRedirection::create("/dev/null", STDERR_FILENO, AST::PathRedirection::Write));
+    completion_command.redirections.append(AST::PathRedirection::create("/dev/null", STDIN_FILENO, AST::PathRedirection::Read));
+
+    auto execute_node = make_ref_counted<AST::Execute>(
+        AST::Position {},
+        make_ref_counted<AST::CommandLiteral>(AST::Position {}, move(completion_command)),
+        true);
+
+    Vector<Line::CompletionSuggestion> suggestions;
+    auto timer = Core::Timer::create_single_shot(300, [&] {
+        Core::EventLoop::current().quit(1);
+    });
+    timer->start();
+
+    execute_node->for_each_entry(*this, [&](NonnullRefPtr<AST::Value> entry) -> IterationDecision {
+        auto result = entry->resolve_as_string(*this);
+        JsonParser parser(result);
+        auto parsed_result = parser.parse();
+        if (parsed_result.is_error())
+            return IterationDecision::Continue;
+        auto parsed = parsed_result.release_value();
+        if (parsed.is_object()) {
+            auto& object = parsed.as_object();
+            Line::CompletionSuggestion suggestion { object.get("completion").to_string() };
+            suggestion.static_offset = object.get("static_offset").to_u64(0);
+            suggestion.invariant_offset = object.get("invariant_offset").to_u64(0);
+            suggestions.append(move(suggestion));
+        } else {
+            suggestions.append(parsed.to_string());
+        }
+
+        return IterationDecision::Continue;
+    });
+
+    auto pgid = getpgrp();
+    tcsetpgrp(STDOUT_FILENO, pgid);
+    tcsetpgrp(STDIN_FILENO, pgid);
+
+    if (suggestions.is_empty())
+        return Error::from_string_literal("No results");
+
     return suggestions;
 }
 

Userland/Shell/Shell.h

@@ -186,13 +186,15 @@ public:
 
     void highlight(Line::Editor&) const;
     Vector<Line::CompletionSuggestion> complete();
-    Vector<Line::CompletionSuggestion> complete_path(StringView base, StringView, size_t offset, ExecutableOnly executable_only, EscapeMode = EscapeMode::Bareword);
     Vector<Line::CompletionSuggestion> complete_program_name(StringView, size_t offset, EscapeMode = EscapeMode::Bareword);
     Vector<Line::CompletionSuggestion> complete_variable(StringView, size_t offset);
     Vector<Line::CompletionSuggestion> complete_user(StringView, size_t offset);
-    Vector<Line::CompletionSuggestion> complete_option(StringView, StringView, size_t offset);
     Vector<Line::CompletionSuggestion> complete_immediate_function_name(StringView, size_t offset);
 
+    Vector<Line::CompletionSuggestion> complete_path(StringView base, StringView, size_t offset, ExecutableOnly executable_only, AST::Node const* command_node, AST::Node const*, EscapeMode = EscapeMode::Bareword);
+    Vector<Line::CompletionSuggestion> complete_option(StringView, StringView, size_t offset, AST::Node const* command_node, AST::Node const*);
+    ErrorOr<Vector<Line::CompletionSuggestion>> complete_via_program_itself(size_t offset, AST::Node const* command_node, AST::Node const*, EscapeMode escape_mode, StringView known_program_name);
+
     void restore_ios();
 
     u64 find_last_job_id() const;