Shell: Parse lists serially, and flatten them only when needed

This allows `((1 2 3) (4 5 6))` to remain nested until we explicitly
flatten it out.

Shell/AST.cpp

@@ -216,33 +216,59 @@ And::~And()
 void ListConcatenate::dump(int level) const
 {
     Node::dump(level);
-    m_element->dump(level + 1);
+    for (auto& element : m_list)
-    m_list->dump(level + 1);
+        element->dump(level + 1);
 }
 
 RefPtr<Value> ListConcatenate::run(RefPtr<Shell> shell)
 {
-    auto list = m_list->run(shell)->resolve_without_cast(shell);
+    RefPtr<Value> result = nullptr;
-    auto element = m_element->run(shell)->resolve_without_cast(shell);
 
-    if (list->is_command() || element->is_command()) {
+    for (auto& element : m_list) {
-        auto joined_commands = join_commands(element->resolve_as_commands(shell), list->resolve_as_commands(shell));
+        if (!result) {
+            result = create<ListValue>({ element->run(shell)->resolve_without_cast(shell) });
+            continue;
+        }
+        auto element_value = element->run(shell)->resolve_without_cast(shell);
 
-        if (joined_commands.size() == 1)
+        if (result->is_command() || element_value->is_command()) {
-            return create<CommandValue>(joined_commands[0]);
+            auto joined_commands = join_commands(result->resolve_as_commands(shell), element_value->resolve_as_commands(shell));
-        return create<CommandSequenceValue>(move(joined_commands));
+
+            if (joined_commands.size() == 1)
+                result = create<CommandValue>(joined_commands[0]);
+            else
+                result = create<CommandSequenceValue>(move(joined_commands));
+        } else {
+            Vector<RefPtr<Value>> values;
+
+            if (result->is_list_without_resolution()) {
+                values.append(static_cast<ListValue*>(result.ptr())->values());
+            } else {
+                for (auto& result : result->resolve_as_list(shell))
+                    values.append(create<StringValue>(result));
+            }
+
+            values.append(move(element_value));
+
+            result = create<ListValue>(move(values));
+        }
     }
+    if (!result)
+        return create<ListValue>({});
 
-    return create<ListValue>({ move(element), move(list) });
+    return result;
 }
 
 void ListConcatenate::highlight_in_editor(Line::Editor& editor, Shell& shell, HighlightMetadata metadata)
 {
     auto first = metadata.is_first_in_list;
     metadata.is_first_in_list = false;
-    m_list->highlight_in_editor(editor, shell, metadata);
+
     metadata.is_first_in_list = first;
-    m_element->highlight_in_editor(editor, shell, metadata);
+    for (auto& element : m_list) {
+        element->highlight_in_editor(editor, shell, metadata);
+        metadata.is_first_in_list = false;
+    }
 }
 
 HitTestResult ListConcatenate::hit_test_position(size_t offset)
@@ -250,29 +276,37 @@ HitTestResult ListConcatenate::hit_test_position(size_t offset)
     if (!position().contains(offset))
         return {};
 
-    auto result = m_element->hit_test_position(offset);
+    bool first = true;
-    if (result.matching_node)
+    for (auto& element : m_list) {
-        return result;
+        auto result = element->hit_test_position(offset);
-    result = m_list->hit_test_position(offset);
+        if (!result.closest_node_with_semantic_meaning && !first)
-    if (!result.closest_node_with_semantic_meaning)
+            result.closest_node_with_semantic_meaning = this;
-        result.closest_node_with_semantic_meaning = this;
+        if (result.matching_node)
-    return result;
+            return result;
+        first = false;
+    }
+
+    return {};
 }
 
 RefPtr<Node> ListConcatenate::leftmost_trivial_literal() const
 {
-    return m_element->leftmost_trivial_literal();
+    if (m_list.is_empty())
+        return nullptr;
+
+    return m_list.first()->leftmost_trivial_literal();
 }
 
-ListConcatenate::ListConcatenate(Position position, RefPtr<Node> element, RefPtr<Node> list)
+ListConcatenate::ListConcatenate(Position position, Vector<RefPtr<Node>> list)
     : Node(move(position))
-    , m_element(move(element))
     , m_list(move(list))
 {
-    if (m_element->is_syntax_error())
+    for (auto& element : m_list) {
-        set_is_syntax_error(m_element->syntax_error_node());
+        if (element->is_syntax_error()) {
-    else if (m_list->is_syntax_error())
+            set_is_syntax_error(element->syntax_error_node());
-        set_is_syntax_error(m_list->syntax_error_node());
+            break;
+        }
+    }
 }
 
 ListConcatenate::~ListConcatenate()
@@ -451,9 +485,9 @@ RefPtr<Value> CastToList::run(RefPtr<Shell> shell)
     if (!m_inner)
         return create<ListValue>({});
 
-    auto inner_value = m_inner->run(shell);
+    auto inner_value = m_inner->run(shell)->resolve_without_cast(shell);
 
-    if (inner_value->is_command())
+    if (inner_value->is_command() || inner_value->is_list())
         return inner_value;
 
     auto values = inner_value->resolve_as_list(shell);
@@ -1783,6 +1817,15 @@ Vector<String> ListValue::resolve_as_list(RefPtr<Shell> shell)
     return values;
 }
 
+RefPtr<Value> ListValue::resolve_without_cast(RefPtr<Shell> shell)
+{
+    Vector<RefPtr<Value>> values;
+    for (auto& value : m_contained_values)
+        values.append(value->resolve_without_cast(shell));
+
+    return create<ListValue>(move(values));
+}
+
 CommandValue::~CommandValue()
 {
 }

Shell/AST.h

@@ -164,6 +164,7 @@ public:
     virtual bool is_job() const { return false; }
     virtual bool is_list() const { return false; }
     virtual bool is_string() const { return false; }
+    virtual bool is_list_without_resolution() const { return false; }
 };
 
 class CommandValue final : public Value {
@@ -221,14 +222,18 @@ private:
 class ListValue final : public Value {
 public:
     virtual Vector<String> resolve_as_list(RefPtr<Shell>) override;
+    virtual RefPtr<Value> resolve_without_cast(RefPtr<Shell>) override;
     virtual ~ListValue();
     virtual bool is_list() const override { return true; }
+    virtual bool is_list_without_resolution() const override { return true; }
     ListValue(Vector<String> values);
     ListValue(Vector<RefPtr<Value>> values)
         : m_contained_values(move(values))
     {
     }
 
+    const Vector<RefPtr<Value>>& values() const { return m_contained_values; }
+
 private:
     Vector<RefPtr<Value>> m_contained_values;
 };
@@ -389,7 +394,7 @@ private:
 
 class ListConcatenate final : public Node {
 public:
-    ListConcatenate(Position, RefPtr<Node>, RefPtr<Node>);
+    ListConcatenate(Position, Vector<RefPtr<Node>>);
     virtual ~ListConcatenate();
 
 private:
@@ -401,8 +406,7 @@ private:
     virtual bool is_list() const override { return true; }
     virtual RefPtr<Node> leftmost_trivial_literal() const override;
 
-    RefPtr<Node> m_element;
+    Vector<RefPtr<Node>> m_list;
-    RefPtr<Node> m_list;
 };
 
 class Background final : public Node {

Shell/Parser.cpp

@@ -420,19 +420,19 @@ RefPtr<AST::Node> Parser::parse_list_expression()
     consume_while(is_whitespace);
 
     auto rule_start = push_start();
+    Vector<RefPtr<AST::Node>> nodes;
 
-    auto expr = parse_expression();
+    do {
-    if (!expr)
+        auto expr = parse_expression();
+        if (!expr)
+            break;
+        nodes.append(move(expr));
+    } while (!consume_while(is_whitespace).is_empty());
+
+    if (nodes.is_empty())
         return nullptr;
 
-    if (consume_while(is_whitespace).is_empty())
+    return create<AST::ListConcatenate>(move(nodes)); // Concatenate List
-        return expr;
-
-    auto list = parse_list_expression();
-    if (!list)
-        return create<AST::CastToList>(move(expr));
-
-    return create<AST::ListConcatenate>(move(expr), move(list)); // Join Element List
 }
 
 RefPtr<AST::Node> Parser::parse_expression()

Shell/Shell.cpp

@@ -361,19 +361,19 @@ int Shell::run_command(const StringView& cmd)
     if (!command)
         return 0;
 
-    if (command->is_syntax_error()) {
-        auto& error_node = command->syntax_error_node();
-        auto& position = error_node.position();
-        fprintf(stderr, "Shell: Syntax error in command: %s\n", error_node.error_text().characters());
-        fprintf(stderr, "Around '%.*s'\n", (int)min(position.end_offset - position.start_offset, (size_t)10), cmd.characters_without_null_termination() + position.start_offset);
-        return 1;
-    }
-
 #ifdef SH_DEBUG
     dbg() << "Command follows";
     command->dump(0);
 #endif
 
+    if (command->is_syntax_error()) {
+        auto& error_node = command->syntax_error_node();
+        auto& position = error_node.position();
+        fprintf(stderr, "Shell: Syntax error in command: %s\n", error_node.error_text().characters());
+        fprintf(stderr, "Around '%.*s' at %zu:%zu\n", (int)min(position.end_offset - position.start_offset, (size_t)10), cmd.characters_without_null_termination() + position.start_offset, position.start_offset, position.end_offset);
+        return 1;
+    }
+
     tcgetattr(0, &termios);
 
     auto result = command->run(*this);