LibWeb+LibSyntax: Implement nested syntax highlighters

And use them to highlight javascript in HTML source.
This commit also changes how TextDocumentSpan::data is interpreted,
as it used to be an opaque pointer, but everyone stuffed an enum value
inside it, which made the values not unique to each highlighter;
that field is now a u64 serial id.
The syntax highlighters don't need to change their ways of stuffing
token types into that field, but a highlighter that calls another
nested highlighter needs to register the nested types for use with
token pairs.

Userland/Applications/Spreadsheet/CellSyntaxHighlighter.cpp

@@ -31,7 +31,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette)
             false,
             false,
         },
-        nullptr,
+        (u64)-1,
         false);
 
     if (m_cell && m_cell->exception()) {
@@ -47,7 +47,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette)
                     false,
                     false,
                 },
-                nullptr,
+                (u64)-1,
                 false });
     }
     m_client->do_update();

Userland/Libraries/LibCpp/SyntaxHighlighter.cpp

@@ -43,15 +43,15 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Cpp::
     }
 }
 
-bool SyntaxHighlighter::is_identifier(void* token) const
+bool SyntaxHighlighter::is_identifier(u64 token) const
 {
-    auto cpp_token = static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token));
+    auto cpp_token = static_cast<Cpp::Token::Type>(token);
     return cpp_token == Cpp::Token::Type::Identifier;
 }
 
-bool SyntaxHighlighter::is_navigatable(void* token) const
+bool SyntaxHighlighter::is_navigatable(u64 token) const
 {
-    auto cpp_token = static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token));
+    auto cpp_token = static_cast<Cpp::Token::Type>(token);
     return cpp_token == Cpp::Token::Type::IncludePath;
 }
 
@@ -72,7 +72,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
         span.attributes.color = style.color;
         span.attributes.bold = style.bold;
         span.is_skippable = token.type() == Cpp::Token::Type::Whitespace;
-        span.data = reinterpret_cast<void*>(token.type());
+        span.data = static_cast<u64>(token.type());
         spans.append(span);
     }
     m_client->do_set_spans(move(spans));
@@ -83,20 +83,20 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
     m_client->do_update();
 }
 
-Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
+Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
 {
     static Vector<SyntaxHighlighter::MatchingTokenPair> pairs;
     if (pairs.is_empty()) {
-        pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftCurly), reinterpret_cast<void*>(Cpp::Token::Type::RightCurly) });
+        pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftCurly), static_cast<u64>(Cpp::Token::Type::RightCurly) });
-        pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftParen), reinterpret_cast<void*>(Cpp::Token::Type::RightParen) });
+        pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftParen), static_cast<u64>(Cpp::Token::Type::RightParen) });
-        pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftBracket), reinterpret_cast<void*>(Cpp::Token::Type::RightBracket) });
+        pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftBracket), static_cast<u64>(Cpp::Token::Type::RightBracket) });
     }
     return pairs;
 }
 
-bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
-    return static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token1)) == static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token2));
+    return static_cast<Cpp::Token::Type>(token1) == static_cast<Cpp::Token::Type>(token2);
 }
 
 SyntaxHighlighter::~SyntaxHighlighter()

Userland/Libraries/LibCpp/SyntaxHighlighter.h

@@ -15,15 +15,15 @@ public:
     SyntaxHighlighter() { }
     virtual ~SyntaxHighlighter() override;
 
-    virtual bool is_identifier(void*) const override;
+    virtual bool is_identifier(u64) const override;
-    virtual bool is_navigatable(void*) const override;
+    virtual bool is_navigatable(u64) const override;
 
     virtual Syntax::Language language() const override { return Syntax::Language::Cpp; }
     virtual void rehighlight(Palette const&) override;
 
 protected:
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
-    virtual bool token_types_equal(void*, void*) const override;
+    virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibGUI/GMLSyntaxHighlighter.cpp

@@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, GMLTo
     }
 }
 
-bool GMLSyntaxHighlighter::is_identifier(void* token) const
+bool GMLSyntaxHighlighter::is_identifier(u64 token) const
 {
-    auto ini_token = static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token));
+    auto ini_token = static_cast<GUI::GMLToken::Type>(token);
     return ini_token == GUI::GMLToken::Type::Identifier;
 }
 
@@ -54,7 +54,7 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette)
         span.attributes.color = style.color;
         span.attributes.bold = style.bold;
         span.is_skippable = false;
-        span.data = reinterpret_cast<void*>(token.m_type);
+        span.data = static_cast<u64>(token.m_type);
         spans.append(span);
     }
     m_client->do_set_spans(move(spans));
@@ -65,18 +65,18 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette)
     m_client->do_update();
 }
 
-Vector<GMLSyntaxHighlighter::MatchingTokenPair> GMLSyntaxHighlighter::matching_token_pairs() const
+Vector<GMLSyntaxHighlighter::MatchingTokenPair> GMLSyntaxHighlighter::matching_token_pairs_impl() const
 {
     static Vector<MatchingTokenPair> pairs;
     if (pairs.is_empty()) {
-        pairs.append({ reinterpret_cast<void*>(GMLToken::Type::LeftCurly), reinterpret_cast<void*>(GMLToken::Type::RightCurly) });
+        pairs.append({ static_cast<u64>(GMLToken::Type::LeftCurly), static_cast<u64>(GMLToken::Type::RightCurly) });
     }
     return pairs;
 }
 
-bool GMLSyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool GMLSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
-    return static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token1)) == static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token2));
+    return static_cast<GUI::GMLToken::Type>(token1) == static_cast<GUI::GMLToken::Type>(token2);
 }
 
 GMLSyntaxHighlighter::~GMLSyntaxHighlighter()

Userland/Libraries/LibGUI/GMLSyntaxHighlighter.h

@@ -15,14 +15,14 @@ public:
     GMLSyntaxHighlighter() { }
     virtual ~GMLSyntaxHighlighter() override;
 
-    virtual bool is_identifier(void*) const override;
+    virtual bool is_identifier(u64) const override;
 
     virtual Syntax::Language language() const override { return Syntax::Language::GML; }
     virtual void rehighlight(const Palette&) override;
 
 protected:
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
-    virtual bool token_types_equal(void*, void*) const override;
+    virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibGUI/INISyntaxHighlighter.cpp

@@ -32,9 +32,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, IniTo
     }
 }
 
-bool IniSyntaxHighlighter::is_identifier(void* token) const
+bool IniSyntaxHighlighter::is_identifier(u64 token) const
 {
-    auto ini_token = static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token));
+    auto ini_token = static_cast<GUI::IniToken::Type>(token);
     return ini_token == GUI::IniToken::Type::Name;
 }
 
@@ -53,7 +53,7 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette)
         span.attributes.color = style.color;
         span.attributes.bold = style.bold;
         span.is_skippable = token.m_type == IniToken::Type::Whitespace;
-        span.data = reinterpret_cast<void*>(token.m_type);
+        span.data = static_cast<u64>(token.m_type);
         spans.append(span);
     }
     m_client->do_set_spans(move(spans));
@@ -64,18 +64,18 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette)
     m_client->do_update();
 }
 
-Vector<IniSyntaxHighlighter::MatchingTokenPair> IniSyntaxHighlighter::matching_token_pairs() const
+Vector<IniSyntaxHighlighter::MatchingTokenPair> IniSyntaxHighlighter::matching_token_pairs_impl() const
 {
     static Vector<MatchingTokenPair> pairs;
     if (pairs.is_empty()) {
-        pairs.append({ reinterpret_cast<void*>(IniToken::Type::LeftBracket), reinterpret_cast<void*>(IniToken::Type::RightBracket) });
+        pairs.append({ static_cast<u64>(IniToken::Type::LeftBracket), static_cast<u64>(IniToken::Type::RightBracket) });
     }
     return pairs;
 }
 
-bool IniSyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool IniSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
-    return static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token1)) == static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token2));
+    return static_cast<GUI::IniToken::Type>(token1) == static_cast<GUI::IniToken::Type>(token2);
 }
 
 IniSyntaxHighlighter::~IniSyntaxHighlighter()

Userland/Libraries/LibGUI/INISyntaxHighlighter.h

@@ -15,14 +15,14 @@ public:
     IniSyntaxHighlighter() { }
     virtual ~IniSyntaxHighlighter() override;
 
-    virtual bool is_identifier(void*) const override;
+    virtual bool is_identifier(u64) const override;
 
     virtual Syntax::Language language() const override { return Syntax::Language::INI; }
     virtual void rehighlight(const Palette&) override;
 
 protected:
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
-    virtual bool token_types_equal(void*, void*) const override;
+    virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibGUI/TextDocument.h

@@ -26,7 +26,7 @@ namespace GUI {
 struct TextDocumentSpan {
     TextRange range;
     Gfx::TextAttributes attributes;
-    void* data { nullptr };
+    u64 data { 0 };
     bool is_skippable { false };
 };
 

Userland/Libraries/LibJS/SyntaxHighlighter.cpp

@@ -38,13 +38,13 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, JS::T
     }
 }
 
-bool SyntaxHighlighter::is_identifier(void* token) const
+bool SyntaxHighlighter::is_identifier(u64 token) const
 {
-    auto js_token = static_cast<JS::TokenType>(reinterpret_cast<size_t>(token));
+    auto js_token = static_cast<JS::TokenType>(static_cast<size_t>(token));
     return js_token == JS::TokenType::Identifier;
 }
 
-bool SyntaxHighlighter::is_navigatable([[maybe_unused]] void* token) const
+bool SyntaxHighlighter::is_navigatable([[maybe_unused]] u64 token) const
 {
     return false;
 }
@@ -83,7 +83,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
         span.attributes.color = style.color;
         span.attributes.bold = style.bold;
         span.is_skippable = is_trivia;
-        span.data = reinterpret_cast<void*>(static_cast<size_t>(type));
+        span.data = static_cast<u64>(type);
         spans.append(span);
 
         dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{}{} @ '{}' {}:{} - {}:{}",
@@ -111,20 +111,20 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
     m_client->do_update();
 }
 
-Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
+Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
 {
     static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
     if (pairs.is_empty()) {
-        pairs.append({ reinterpret_cast<void*>(JS::TokenType::CurlyOpen), reinterpret_cast<void*>(JS::TokenType::CurlyClose) });
+        pairs.append({ static_cast<u64>(JS::TokenType::CurlyOpen), static_cast<u64>(JS::TokenType::CurlyClose) });
-        pairs.append({ reinterpret_cast<void*>(JS::TokenType::ParenOpen), reinterpret_cast<void*>(JS::TokenType::ParenClose) });
+        pairs.append({ static_cast<u64>(JS::TokenType::ParenOpen), static_cast<u64>(JS::TokenType::ParenClose) });
-        pairs.append({ reinterpret_cast<void*>(JS::TokenType::BracketOpen), reinterpret_cast<void*>(JS::TokenType::BracketClose) });
+        pairs.append({ static_cast<u64>(JS::TokenType::BracketOpen), static_cast<u64>(JS::TokenType::BracketClose) });
     }
     return pairs;
 }
 
-bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
-    return static_cast<JS::TokenType>(reinterpret_cast<size_t>(token1)) == static_cast<JS::TokenType>(reinterpret_cast<size_t>(token2));
+    return static_cast<JS::TokenType>(token1) == static_cast<JS::TokenType>(token2);
 }
 
 SyntaxHighlighter::~SyntaxHighlighter()

Userland/Libraries/LibJS/SyntaxHighlighter.h

@@ -15,15 +15,15 @@ public:
     SyntaxHighlighter() { }
     virtual ~SyntaxHighlighter() override;
 
-    virtual bool is_identifier(void*) const override;
+    virtual bool is_identifier(u64) const override;
-    virtual bool is_navigatable(void*) const override;
+    virtual bool is_navigatable(u64) const override;
 
     virtual Syntax::Language language() const override { return Syntax::Language::JavaScript; }
     virtual void rehighlight(const Palette&) override;
 
 protected:
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
-    virtual bool token_types_equal(void*, void*) const override;
+    virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibSQL/SyntaxHighlighter.cpp

@@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Token
     }
 }
 
-bool SyntaxHighlighter::is_identifier(void* token) const
+bool SyntaxHighlighter::is_identifier(u64 token) const
 {
-    auto sql_token = static_cast<SQL::TokenType>(reinterpret_cast<size_t>(token));
+    auto sql_token = static_cast<SQL::TokenType>(static_cast<size_t>(token));
     return sql_token == SQL::TokenType::Identifier;
 }
 
@@ -66,7 +66,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
         auto style = style_for_token_type(palette, token.type());
         span.attributes.color = style.color;
         span.attributes.bold = style.bold;
-        span.data = reinterpret_cast<void*>(static_cast<size_t>(token.type()));
+        span.data = static_cast<u64>(token.type());
         spans.append(span);
 
         dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{} @ '{}' {}:{} - {}:{}",
@@ -91,18 +91,18 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
     m_client->do_update();
 }
 
-Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
+Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
 {
     static Vector<SyntaxHighlighter::MatchingTokenPair> pairs;
     if (pairs.is_empty()) {
-        pairs.append({ reinterpret_cast<void*>(TokenType::ParenOpen), reinterpret_cast<void*>(TokenType::ParenClose) });
+        pairs.append({ static_cast<u64>(TokenType::ParenOpen), static_cast<u64>(TokenType::ParenClose) });
     }
     return pairs;
 }
 
-bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
-    return static_cast<TokenType>(reinterpret_cast<size_t>(token1)) == static_cast<TokenType>(reinterpret_cast<size_t>(token2));
+    return static_cast<TokenType>(token1) == static_cast<TokenType>(token2);
 }
 
 SyntaxHighlighter::~SyntaxHighlighter()

Userland/Libraries/LibSQL/SyntaxHighlighter.h

@@ -15,14 +15,14 @@ public:
     SyntaxHighlighter() { }
     virtual ~SyntaxHighlighter() override;
 
-    virtual bool is_identifier(void*) const override;
+    virtual bool is_identifier(u64) const override;
 
     virtual Syntax::Language language() const override { return Syntax::Language::SQL; }
     virtual void rehighlight(Palette const&) override;
 
 protected:
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
-    virtual bool token_types_equal(void*, void*) const override;
+    virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibSyntax/Highlighter.cpp

@@ -23,7 +23,7 @@ void Highlighter::highlight_matching_token_pair()
         Backward,
     };
 
-    auto find_span_of_type = [&](auto i, void* type, void* not_type, Direction direction) -> Optional<size_t> {
+    auto find_span_of_type = [&](auto i, u64 type, u64 not_type, Direction direction) -> Optional<size_t> {
         size_t nesting_level = 0;
         bool forward = direction == Direction::Forward;
 
@@ -130,4 +130,19 @@ void Highlighter::cursor_did_change()
     highlight_matching_token_pair();
 }
 
+Vector<Highlighter::MatchingTokenPair> Highlighter::matching_token_pairs() const
+{
+    auto own_pairs = matching_token_pairs_impl();
+    own_pairs.ensure_capacity(own_pairs.size() + m_nested_token_pairs.size());
+    for (auto& nested_pair : m_nested_token_pairs)
+        own_pairs.append(nested_pair);
+    return own_pairs;
+}
+
+void Highlighter::register_nested_token_pairs(Vector<MatchingTokenPair> pairs)
+{
+    for (auto& pair : pairs)
+        m_nested_token_pairs.set(pair);
+}
+
 }

Userland/Libraries/LibSyntax/Highlighter.h

@@ -41,26 +41,30 @@ public:
     virtual void rehighlight(const Palette&) = 0;
     virtual void highlight_matching_token_pair();
 
-    virtual bool is_identifier(void*) const { return false; };
+    virtual bool is_identifier(u64) const { return false; };
-    virtual bool is_navigatable(void*) const { return false; };
+    virtual bool is_navigatable(u64) const { return false; };
 
     void attach(HighlighterClient&);
     void detach();
     void cursor_did_change();
 
+    struct MatchingTokenPair {
+        u64 open;
+        u64 close;
+    };
+    Vector<MatchingTokenPair> matching_token_pairs() const;
+
 protected:
     Highlighter() { }
 
     // FIXME: This should be WeakPtr somehow
     HighlighterClient* m_client { nullptr };
 
-    struct MatchingTokenPair {
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const = 0;
-        void* open;
+    virtual bool token_types_equal(u64, u64) const = 0;
-        void* close;
+    void register_nested_token_pairs(Vector<MatchingTokenPair>);
-    };
+    void clear_nested_token_pairs() { m_nested_token_pairs.clear(); }
-
+    size_t first_free_token_kind_serial_value() const { return m_nested_token_pairs.size(); }
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const = 0;
-    virtual bool token_types_equal(void*, void*) const = 0;
 
     struct BuddySpan {
         int index { -1 };
@@ -69,6 +73,75 @@ protected:
 
     bool m_has_brace_buddies { false };
     BuddySpan m_brace_buddies[2];
+    HashTable<MatchingTokenPair> m_nested_token_pairs;
+};
+
+class ProxyHighlighterClient final : public Syntax::HighlighterClient {
+public:
+    ProxyHighlighterClient(Syntax::HighlighterClient& client, GUI::TextPosition start, u64 nested_kind_start_value, StringView source)
+        : m_document(client.get_document())
+        , m_text(source)
+        , m_start(start)
+        , m_nested_kind_start_value(nested_kind_start_value)
+    {
+    }
+
+    Vector<GUI::TextDocumentSpan> corrected_spans() const
+    {
+        Vector<GUI::TextDocumentSpan> spans { m_spans };
+        for (auto& entry : spans) {
+            entry.range.start() = {
+                entry.range.start().line() + m_start.line(),
+                entry.range.start().line() == 0 ? entry.range.start().column() + m_start.column() : entry.range.start().column(),
+            };
+            entry.range.end() = {
+                entry.range.end().line() + m_start.line(),
+                entry.range.end().line() == 0 ? entry.range.end().column() + m_start.column() : entry.range.end().column(),
+            };
+            if (entry.data != (u64)-1)
+                entry.data += m_nested_kind_start_value;
+        }
+
+        return spans;
+    }
+
+    Vector<Syntax::Highlighter::MatchingTokenPair> corrected_token_pairs(Vector<Syntax::Highlighter::MatchingTokenPair> pairs) const
+    {
+        for (auto& pair : pairs) {
+            pair.close += m_nested_kind_start_value;
+            pair.open += m_nested_kind_start_value;
+        }
+        return pairs;
+    }
+
+private:
+    virtual Vector<GUI::TextDocumentSpan>& spans() override { return m_spans; }
+    virtual const Vector<GUI::TextDocumentSpan>& spans() const override { return m_spans; }
+    virtual void set_span_at_index(size_t index, GUI::TextDocumentSpan span) override { m_spans.at(index) = move(span); }
+
+    virtual String highlighter_did_request_text() const override { return m_text; }
+    virtual void highlighter_did_request_update() override { }
+    virtual GUI::TextDocument& highlighter_did_request_document() override { return m_document; }
+    virtual GUI::TextPosition highlighter_did_request_cursor() const override { return {}; }
+    virtual void highlighter_did_set_spans(Vector<GUI::TextDocumentSpan> spans) override { m_spans = move(spans); }
+
+    Vector<GUI::TextDocumentSpan> m_spans;
+    GUI::TextDocument& m_document;
+    StringView m_text;
+    GUI::TextPosition m_start;
+    u64 m_nested_kind_start_value { 0 };
 };
 
 }
+
+template<>
+struct AK::Traits<Syntax::Highlighter::MatchingTokenPair> : public AK::GenericTraits<Syntax::Highlighter::MatchingTokenPair> {
+    static unsigned hash(Syntax::Highlighter::MatchingTokenPair const& pair)
+    {
+        return pair_int_hash(u64_hash(pair.open), u64_hash(pair.close));
+    }
+    static bool equals(Syntax::Highlighter::MatchingTokenPair const& a, Syntax::Highlighter::MatchingTokenPair const& b)
+    {
+        return a.open == b.open && a.close == b.close;
+    }
+};

Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp

@@ -6,6 +6,7 @@
  */
 
 #include <AK/Debug.h>
+#include <LibJS/SyntaxHighlighter.h>
 #include <LibWeb/HTML/Parser/HTMLTokenizer.h>
 #include <LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h>
 
@@ -18,16 +19,17 @@ enum class AugmentedTokenKind : u32 {
     CloseTag,
     Comment,
     Doctype,
+    __Count,
 };
 
-bool SyntaxHighlighter::is_identifier(void* token) const
+bool SyntaxHighlighter::is_identifier(u64 token) const
 {
     if (!token)
         return false;
     return false;
 }
 
-bool SyntaxHighlighter::is_navigatable(void*) const
+bool SyntaxHighlighter::is_navigatable(u64) const
 {
     return false;
 }
@@ -35,8 +37,8 @@ bool SyntaxHighlighter::is_navigatable(void*) const
 void SyntaxHighlighter::rehighlight(Palette const& palette)
 {
     dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(HTML::SyntaxHighlighter) starting rehighlight");
-    (void)palette;
     auto text = m_client->get_text();
+    clear_nested_token_pairs();
 
     Vector<GUI::TextDocumentSpan> spans;
     auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, AugmentedTokenKind kind) {
@@ -51,7 +53,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
                 { end_line, end_column },
             },
             move(attributes),
-            (void*)kind,
+            static_cast<u64>(kind),
             false);
     };
 
@@ -61,6 +63,9 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
         Javascript,
         CSS,
     } state { State::HTML };
+    StringBuilder substring_builder;
+    GUI::TextPosition substring_start_position;
+
     for (;;) {
         auto token = tokenizer.next_token();
         if (!token.has_value() || token.value().is_end_of_file())
@@ -71,19 +76,41 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
             if (token->tag_name() == "script"sv) {
                 tokenizer.switch_to(HTMLTokenizer::State::ScriptData);
                 state = State::Javascript;
+                substring_start_position = { token->end_position().line, token->end_position().column };
             } else if (token->tag_name() == "style"sv) {
                 tokenizer.switch_to(HTMLTokenizer::State::RAWTEXT);
                 state = State::CSS;
+                substring_start_position = { token->end_position().line, token->end_position().column };
             }
         } else if (token->is_end_tag()) {
             if (token->tag_name().is_one_of("script"sv, "style"sv)) {
                 if (state == State::Javascript) {
-                    // FIXME: Highlight javascript code here instead.
+                    Syntax::ProxyHighlighterClient proxy_client {
+                        *m_client,
+                        substring_start_position,
+                        static_cast<u64>(AugmentedTokenKind::__Count) + first_free_token_kind_serial_value(),
+                        substring_builder.string_view()
+                    };
+                    {
+                        JS::SyntaxHighlighter highlighter;
+                        highlighter.attach(proxy_client);
+                        highlighter.rehighlight(palette);
+                        highlighter.detach();
+                        register_nested_token_pairs(proxy_client.corrected_token_pairs(highlighter.matching_token_pairs()));
+                    }
+
+                    spans.append(proxy_client.corrected_spans());
+                    substring_builder.clear();
                 } else if (state == State::CSS) {
                     // FIXME: Highlight CSS code here instead.
+                    substring_builder.clear();
                 }
                 state = State::HTML;
             }
+        } else if (state != State::HTML) {
+            VERIFY(token->is_character());
+            substring_builder.append_code_point(token->code_point());
+            continue;
         }
 
         size_t token_start_offset = token->is_end_tag() ? 1 : 0;
@@ -135,7 +162,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
     if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) {
         dbgln("(HTML::SyntaxHighlighter) list of all spans:");
         for (auto& span : spans)
-            dbgln("{}, {}", span.range, span.attributes.color);
+            dbgln("{}, {} - {}", span.range, span.attributes.color, span.data);
         dbgln("(HTML::SyntaxHighlighter) end of list");
     }
 
@@ -145,16 +172,16 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
     m_client->do_update();
 }
 
-Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
+Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
 {
     static Vector<MatchingTokenPair> pairs;
     if (pairs.is_empty()) {
-        pairs.append({ (void*)AugmentedTokenKind::OpenTag, (void*)AugmentedTokenKind::CloseTag });
+        pairs.append({ static_cast<u64>(AugmentedTokenKind::OpenTag), static_cast<u64>(AugmentedTokenKind::CloseTag) });
     }
     return pairs;
 }
 
-bool SyntaxHighlighter::token_types_equal(void* token0, void* token1) const
+bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const
 {
     return token0 == token1;
 }

Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h

@@ -15,15 +15,15 @@ public:
     SyntaxHighlighter() = default;
     virtual ~SyntaxHighlighter() override = default;
 
-    virtual bool is_identifier(void*) const override;
+    virtual bool is_identifier(u64) const override;
-    virtual bool is_navigatable(void*) const override;
+    virtual bool is_navigatable(u64) const override;
 
     virtual Syntax::Language language() const override { return Syntax::Language::HTML; }
     virtual void rehighlight(Palette const&) override;
 
 protected:
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
-    virtual bool token_types_equal(void*, void*) const override;
+    virtual bool token_types_equal(u64, u64) const override;
 
     size_t m_line { 1 };
     size_t m_column { 0 };

Userland/Shell/SyntaxHighlighter.cpp

@@ -68,7 +68,7 @@ private:
         GUI::TextDocumentSpan span;
         set_offset_range_start(span.range, node->position().start_line);
         set_offset_range_end(span.range, node->position().end_line);
-        span.data = (void*)static_cast<size_t>(node->kind());
+        span.data = static_cast<u64>(node->kind());
         span.is_skippable = false;
         m_spans.append(move(span));
 
@@ -143,12 +143,12 @@ private:
         auto& start_span = span_for_node(node);
         start_span.attributes.color = m_palette.syntax_punctuation();
         start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
-        start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
+        start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
 
         auto& end_span = span_for_node(node);
         end_span.attributes.color = m_palette.syntax_punctuation();
         set_offset_range_start(end_span.range, node->position().end_line, 1);
-        end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
+        end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
     }
     virtual void visit(const AST::CloseFdRedirection* node) override
     {
@@ -289,12 +289,12 @@ private:
             auto& start_span = span_for_node(node);
             start_span.attributes.color = m_palette.syntax_punctuation();
             start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
-            start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
+            start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
 
             auto& end_span = span_for_node(node);
             end_span.attributes.color = m_palette.syntax_punctuation();
             set_offset_range_start(end_span.range, node->position().end_line, 1);
-            end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
+            end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
         }
     }
     virtual void visit(const AST::IfCond* node) override
@@ -328,7 +328,7 @@ private:
         auto& start_span = span_for_node(node);
         start_span.attributes.color = m_palette.syntax_punctuation();
         start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
-        start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
+        start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
 
         // Function name
         auto& name_span = span_for_node(node);
@@ -340,7 +340,7 @@ private:
         auto& end_span = span_for_node(node);
         end_span.attributes.color = m_palette.syntax_punctuation();
         set_offset_range_start(end_span.range, node->position().end_line, 1);
-        end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
+        end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
     }
 
     virtual void visit(const AST::Join* node) override
@@ -500,7 +500,7 @@ private:
             start_span.range.set_start({ decl.name->position().end_line.line_number, decl.name->position().end_line.line_column });
             start_span.range.set_end({ decl.value->position().start_line.line_number, decl.value->position().start_line.line_column + 1 });
             start_span.attributes.color = m_palette.syntax_punctuation();
-            start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
+            start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
         }
     }
     virtual void visit(const AST::WriteAppendRedirection* node) override
@@ -518,7 +518,7 @@ private:
     bool m_is_first_in_command { false };
 };
 
-bool SyntaxHighlighter::is_identifier(void* token) const
+bool SyntaxHighlighter::is_identifier(u64 token) const
 {
     if (!token)
         return false;
@@ -529,7 +529,7 @@ bool SyntaxHighlighter::is_identifier(void* token) const
         || kind == (size_t)AST::Node::Kind::Tilde;
 }
 
-bool SyntaxHighlighter::is_navigatable(void*) const
+bool SyntaxHighlighter::is_navigatable(u64) const
 {
     return false;
 }
@@ -551,7 +551,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
 
     if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) {
         for (auto& span : spans) {
-            dbgln("Kind {}, range {}.", reinterpret_cast<size_t>(span.data), span.range);
+            dbgln("Kind {}, range {}.", span.data, span.range);
         }
     }
 
@@ -561,19 +561,19 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
     m_client->do_update();
 }
 
-Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
+Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
 {
     static Vector<MatchingTokenPair> pairs;
     if (pairs.is_empty()) {
         pairs.append({
-            (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen),
+            static_cast<u64>(AugmentedTokenKind::OpenParen),
-            (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen),
+            static_cast<u64>(AugmentedTokenKind::CloseParen),
         });
     }
     return pairs;
 }
 
-bool SyntaxHighlighter::token_types_equal(void* token0, void* token1) const
+bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const
 {
     return token0 == token1;
 }

Userland/Shell/SyntaxHighlighter.h

@@ -15,15 +15,15 @@ public:
     SyntaxHighlighter() { }
     virtual ~SyntaxHighlighter() override;
 
-    virtual bool is_identifier(void*) const override;
+    virtual bool is_identifier(u64) const override;
-    virtual bool is_navigatable(void*) const override;
+    virtual bool is_navigatable(u64) const override;
 
     virtual Syntax::Language language() const override { return Syntax::Language::Shell; }
     virtual void rehighlight(const Palette&) override;
 
 protected:
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
-    virtual bool token_types_equal(void*, void*) const override;
+    virtual bool token_types_equal(u64, u64) const override;
 };
 
 }