LibCpp: Add lex_iterable() method to the Lexer

This allows us to collect the tokens iteratively instead of having to
lex the whole program and then get a tokens vector.

Userland/Libraries/LibCpp/Lexer.h

@@ -17,12 +17,15 @@ public:
     explicit Lexer(StringView const&, size_t start_line = 0);
 
     Vector<Token> lex();
+    template<typename Callback>
+    void lex_iterable(Callback);
 
     void set_ignore_whitespace(bool value) { m_options.ignore_whitespace = value; }
 
 private:
     char peek(size_t offset = 0) const;
     char consume();
+    void lex_impl(Function<void(Token)>);
 
     StringView m_input;
     size_t m_index { 0 };
@@ -34,4 +37,10 @@ private:
     } m_options;
 };
 
+template<typename Callback>
+void Lexer::lex_iterable(Callback callback)
+{
+    return lex_impl(move(callback));
+}
+
 }