1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-27 04:17:35 +00:00

LibCpp: Add lex_iterable() method to the Lexer

This allows us to collect the tokens iteratively instead of having to
lex the whole program and then get a tokens vector.
This commit is contained in:
Itamar 2021-08-21 16:36:40 +03:00 committed by Andreas Kling
parent 7a4a32b112
commit 606e05852f
2 changed files with 21 additions and 5 deletions

View file

@ -17,12 +17,15 @@ public:
explicit Lexer(StringView const&, size_t start_line = 0);
Vector<Token> lex();
template<typename Callback>
void lex_iterable(Callback);
void set_ignore_whitespace(bool value) { m_options.ignore_whitespace = value; }
private:
char peek(size_t offset = 0) const;
char consume();
void lex_impl(Function<void(Token)>);
StringView m_input;
size_t m_index { 0 };
@ -34,4 +37,10 @@ private:
} m_options;
};
template<typename Callback>
void Lexer::lex_iterable(Callback callback)
{
return lex_impl(move(callback));
}
}