LibVT: Store all-ASCII terminal lines as 8-bit characters

To conserve memory, we now use byte storage for terminal lines until we
encounter a non-ASCII codepoint. At that point, we transparently switch
to UTF-32 storage for that one line.

Libraries/LibVT/Line.cpp

@@ -36,27 +36,44 @@ Line::Line(u16 length)
 
 Line::~Line()
 {
-    delete[] m_codepoints;
+    if (m_utf32)
+        delete[] m_codepoints.as_u32;
+    else
+        delete[] m_codepoints.as_u8;
     delete[] m_attributes;
 }
 
+template<typename CodepointType>
+static CodepointType* create_new_codepoint_array(size_t new_length, const CodepointType* old_codepoints, size_t old_length)
+{
+    auto* new_codepoints = new CodepointType[new_length];
+    for (size_t i = 0; i < new_length; ++i)
+        new_codepoints[i] = ' ';
+    if (old_codepoints) {
+        for (size_t i = 0; i < min(old_length, new_length); ++i) {
+            new_codepoints[i] = old_codepoints[i];
+        }
+    }
+    delete[] old_codepoints;
+    return new_codepoints;
+}
+
 void Line::set_length(u16 new_length)
 {
     if (m_length == new_length)
         return;
-    auto* new_codepoints = new u32[new_length];
+
+    if (m_utf32)
+        m_codepoints.as_u32 = create_new_codepoint_array<u32>(new_length, m_codepoints.as_u32, m_length);
+    else
+        m_codepoints.as_u8 = create_new_codepoint_array<u8>(new_length, m_codepoints.as_u8, m_length);
+
     auto* new_attributes = new Attribute[new_length];
-    for (size_t i = 0; i < new_length; ++i)
+    if (m_attributes) {
-        new_codepoints[i] = ' ';
+        for (size_t i = 0; i < min(m_length, new_length); ++i)
-    if (m_codepoints && m_attributes) {
-        for (size_t i = 0; i < min(m_length, new_length); ++i) {
-            new_codepoints[i] = m_codepoints[i];
             new_attributes[i] = m_attributes[i];
-        }
     }
-    delete[] m_codepoints;
     delete[] m_attributes;
-    m_codepoints = new_codepoints;
     m_attributes = new_attributes;
     m_length = new_length;
 }
@@ -65,15 +82,15 @@ void Line::clear(Attribute attribute)
 {
     if (m_dirty) {
         for (u16 i = 0; i < m_length; ++i) {
-            m_codepoints[i] = ' ';
+            set_codepoint(i, ' ');
             m_attributes[i] = attribute;
         }
         return;
     }
     for (unsigned i = 0; i < m_length; ++i) {
-        if (m_codepoints[i] != ' ')
+        if (codepoint(i) != ' ')
             m_dirty = true;
-        m_codepoints[i] = ' ';
+        set_codepoint(i, ' ');
     }
     for (unsigned i = 0; i < m_length; ++i) {
         if (m_attributes[i] != attribute)
@@ -95,4 +112,16 @@ bool Line::has_only_one_background_color() const
     return true;
 }
 
+void Line::convert_to_utf32()
+{
+    ASSERT(!m_utf32);
+    auto* new_codepoints = new u32[m_length];
+    for (size_t i = 0; i < m_length; ++i) {
+        new_codepoints[i] = m_codepoints.as_u8[i];
+    }
+    delete m_codepoints.as_u8;
+    m_codepoints.as_u32 = new_codepoints;
+    m_utf32 = true;
+}
+
 }

Libraries/LibVT/Line.h

@@ -90,8 +90,23 @@ public:
 
     u16 length() const { return m_length; }
 
-    const u32* codepoints() const { return m_codepoints; }
+    u32 codepoint(size_t index) const
-    u32* codepoints() { return m_codepoints; }
+    {
+        if (m_utf32)
+            return m_codepoints.as_u32[index];
+        return m_codepoints.as_u8[index];
+    }
+
+    void set_codepoint(size_t index, u32 codepoint)
+    {
+        if (!m_utf32 && codepoint & 0xffffff80u)
+            convert_to_utf32();
+
+        if (m_utf32)
+            m_codepoints.as_u32[index] = codepoint;
+        else
+            m_codepoints.as_u8[index] = codepoint;
+    }
 
     bool is_dirty() const { return m_dirty; }
     void set_dirty(bool b) { m_dirty = b; }
@@ -99,10 +114,18 @@ public:
     const Attribute* attributes() const { return m_attributes; }
     Attribute* attributes() { return m_attributes; }
 
+    void convert_to_utf32();
+
+    bool is_utf32() const { return m_utf32; }
+
 private:
-    u32* m_codepoints { nullptr };
+    union {
+        u8* as_u8;
+        u32* as_u32;
+    } m_codepoints { nullptr };
     Attribute* m_attributes { nullptr };
     bool m_dirty { false };
+    bool m_utf32 { false };
     u16 m_length { 0 };
 };
 

Libraries/LibVT/Terminal.cpp

@@ -528,11 +528,11 @@ void Terminal::escape$P(const ParamVector& params)
 
     // Move n characters of line to the left
     for (int i = m_cursor_column; i < line.length() - num; i++)
-        line.codepoints()[i] = line.codepoints()[i + num];
+        line.set_codepoint(i, line.codepoint(i + num));
 
     // Fill remainder of line with blanks
     for (int i = line.length() - num; i < line.length(); i++)
-        line.codepoints()[i] = ' ';
+        line.set_codepoint(i, ' ');
 
     line.set_dirty(true);
 }
@@ -760,17 +760,17 @@ void Terminal::set_cursor(unsigned a_row, unsigned a_column)
     invalidate_cursor();
 }
 
-void Terminal::put_character_at(unsigned row, unsigned column, u32 ch)
+void Terminal::put_character_at(unsigned row, unsigned column, u32 codepoint)
 {
     ASSERT(row < rows());
     ASSERT(column < columns());
     auto& line = m_lines[row];
-    line.codepoints()[column] = ch;
+    line.set_codepoint(column, codepoint);
     line.attributes()[column] = m_current_attribute;
     line.attributes()[column].flags |= Attribute::Touched;
     line.set_dirty(true);
 
-    m_last_codepoint = ch;
+    m_last_codepoint = codepoint;
 }
 
 void Terminal::NEL()

Libraries/LibVT/TerminalWidget.cpp

@@ -339,7 +339,7 @@ void TerminalWidget::paint_event(GUI::PaintEvent& event)
             painter.clear_rect(row_rect, color_from_rgb(line.attributes()[0].background_color).with_alpha(m_opacity));
 
         for (size_t column = 0; column < line.length(); ++column) {
-            u32 codepoint = line.codepoints()[column];
+            u32 codepoint = line.codepoint(column);
             bool should_reverse_fill_for_cursor_or_selection = m_cursor_blink_state
                 && m_has_logical_focus
                 && visual_row == row_with_cursor
@@ -560,16 +560,16 @@ void TerminalWidget::doubleclick_event(GUI::MouseEvent& event)
 
         auto position = buffer_position_at(event.position());
         auto& line = m_terminal.line(position.row());
-        bool want_whitespace = line.codepoints()[position.column()] == ' ';
+        bool want_whitespace = line.codepoint(position.column()) == ' ';
 
         int start_column = 0;
         int end_column = 0;
 
-        for (int column = position.column(); column >= 0 && (line.codepoints()[column] == ' ') == want_whitespace; --column) {
+        for (int column = position.column(); column >= 0 && (line.codepoint(column) == ' ') == want_whitespace; --column) {
             start_column = column;
         }
 
-        for (int column = position.column(); column < m_terminal.columns() && (line.codepoints()[column] == ' ') == want_whitespace; ++column) {
+        for (int column = position.column(); column < m_terminal.columns() && (line.codepoint(column) == ' ') == want_whitespace; ++column) {
             end_column = column;
         }
 
@@ -739,7 +739,7 @@ String TerminalWidget::selected_text() const
                 builder.append('\n');
                 break;
             }
-            builder.append(line.codepoints()[column]);
+            builder.append(line.codepoint(column));
             if (column == line.length() - 1 || (m_rectangle_selection && column == last_column)) {
                 builder.append('\n');
             }