mirror of
https://github.com/RGBCube/serenity
synced 2025-07-27 01:27:34 +00:00
LibJS: Allow Unicode escape sequences in identifiers
For example, "property.br\u{64}wn" should resolve to "property.brown". To support this behavior, this commit changes the Token class to hold both the evaluated identifier name and a view into the original source for the unevaluated name. There are some contexts in which identifiers are not allowed to contain Unicode escape sequences; for example, export statements of the form "export {} from foo.js" forbid escapes in the identifier "from". The test file is added to .prettierignore because prettier will replace all escaped Unicode sequences with their unescaped value.
This commit is contained in:
parent
c5b5c779ff
commit
1259dc3623
7 changed files with 163 additions and 54 deletions
|
@ -6,8 +6,10 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include <AK/FlyString.h>
|
||||
#include <AK/String.h>
|
||||
#include <AK/StringView.h>
|
||||
#include <AK/Variant.h>
|
||||
|
||||
namespace JS {
|
||||
|
||||
|
@ -172,10 +174,13 @@ enum class TokenCategory {
|
|||
|
||||
class Token {
|
||||
public:
|
||||
Token() = default;
|
||||
|
||||
Token(TokenType type, String message, StringView trivia, StringView value, StringView filename, size_t line_number, size_t line_column, size_t offset)
|
||||
: m_type(type)
|
||||
, m_message(message)
|
||||
, m_trivia(trivia)
|
||||
, m_original_value(value)
|
||||
, m_value(value)
|
||||
, m_filename(filename)
|
||||
, m_line_number(line_number)
|
||||
|
@ -184,6 +189,19 @@ public:
|
|||
{
|
||||
}
|
||||
|
||||
Token(TokenType type, String message, StringView trivia, StringView original_value, FlyString value, StringView filename, size_t line_number, size_t line_column, size_t offset)
|
||||
: m_type(type)
|
||||
, m_message(message)
|
||||
, m_trivia(trivia)
|
||||
, m_original_value(original_value)
|
||||
, m_value(move(value))
|
||||
, m_filename(filename)
|
||||
, m_line_number(line_number)
|
||||
, m_line_column(line_column)
|
||||
, m_offset(offset)
|
||||
{
|
||||
}
|
||||
|
||||
TokenType type() const { return m_type; }
|
||||
TokenCategory category() const;
|
||||
static TokenCategory category(TokenType);
|
||||
|
@ -192,7 +210,14 @@ public:
|
|||
|
||||
const String& message() const { return m_message; }
|
||||
const StringView& trivia() const { return m_trivia; }
|
||||
const StringView& value() const { return m_value; }
|
||||
const StringView& original_value() const { return m_original_value; }
|
||||
StringView value() const
|
||||
{
|
||||
return m_value.visit(
|
||||
[](StringView const& view) { return view; },
|
||||
[](FlyString const& identifier) { return identifier.view(); },
|
||||
[](Empty) -> StringView { VERIFY_NOT_REACHED(); });
|
||||
}
|
||||
const StringView& filename() const { return m_filename; }
|
||||
size_t line_number() const { return m_line_number; }
|
||||
size_t line_column() const { return m_line_column; }
|
||||
|
@ -213,14 +238,15 @@ public:
|
|||
bool trivia_contains_line_terminator() const;
|
||||
|
||||
private:
|
||||
TokenType m_type;
|
||||
TokenType m_type { TokenType::Invalid };
|
||||
String m_message;
|
||||
StringView m_trivia;
|
||||
StringView m_value;
|
||||
StringView m_original_value;
|
||||
Variant<Empty, StringView, FlyString> m_value { Empty {} };
|
||||
StringView m_filename;
|
||||
size_t m_line_number;
|
||||
size_t m_line_column;
|
||||
size_t m_offset;
|
||||
size_t m_line_number { 0 };
|
||||
size_t m_line_column { 0 };
|
||||
size_t m_offset { 0 };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue