RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-27 01:27:34 +00:00
Code Issues Activity Actions

LibJS: Allow Unicode escape sequences in identifiers

Browse source 
For example, "property.br\u{64}wn" should resolve to "property.brown".

To support this behavior, this commit changes the Token class to hold
both the evaluated identifier name and a view into the original source
for the unevaluated name. There are some contexts in which identifiers
are not allowed to contain Unicode escape sequences; for example, export
statements of the form "export {} from foo.js" forbid escapes in the
identifier "from".

The test file is added to .prettierignore because prettier will replace
all escaped Unicode sequences with their unescaped value.
This commit is contained in:
Timothy Flynn 2021-08-18 16:34:25 -04:00 • committed by Andreas Kling
parent c5b5c779ff
commit 1259dc3623
7 changed files with 163 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

38
Userland/Libraries/LibJS/Token.h
View file

@ -6,8 +6,10 @@
#pragma once
#include <AK/FlyString.h>
#include <AK/String.h>
#include <AK/StringView.h>
#include <AK/Variant.h>
namespace JS {
@ -172,10 +174,13 @@ enum class TokenCategory {
class Token {
public:
Token() = default;
Token(TokenType type, String message, StringView trivia, StringView value, StringView filename, size_t line_number, size_t line_column, size_t offset)
: m_type(type)
, m_message(message)
, m_trivia(trivia)
, m_original_value(value)
, m_value(value)
, m_filename(filename)
, m_line_number(line_number)
@ -184,6 +189,19 @@ public:
{
}
Token(TokenType type, String message, StringView trivia, StringView original_value, FlyString value, StringView filename, size_t line_number, size_t line_column, size_t offset)
: m_type(type)
, m_message(message)
, m_trivia(trivia)
, m_original_value(original_value)
, m_value(move(value))
, m_filename(filename)
, m_line_number(line_number)
, m_line_column(line_column)
, m_offset(offset)
{
}
TokenType type() const { return m_type; }
TokenCategory category() const;
static TokenCategory category(TokenType);
@ -192,7 +210,14 @@ public:
const String& message() const { return m_message; }
const StringView& trivia() const { return m_trivia; }
const StringView& value() const { return m_value; }
const StringView& original_value() const { return m_original_value; }
StringView value() const
{
return m_value.visit(
[](StringView const& view) { return view; },
[](FlyString const& identifier) { return identifier.view(); },
[](Empty) -> StringView { VERIFY_NOT_REACHED(); });
}
const StringView& filename() const { return m_filename; }
size_t line_number() const { return m_line_number; }
size_t line_column() const { return m_line_column; }
@ -213,14 +238,15 @@ public:
bool trivia_contains_line_terminator() const;
private:
TokenType m_type;
TokenType m_type { TokenType::Invalid };
String m_message;
StringView m_trivia;
StringView m_value;
StringView m_original_value;
Variant<Empty, StringView, FlyString> m_value { Empty {} };
StringView m_filename;
size_t m_line_number;
size_t m_line_column;
size_t m_offset;
size_t m_line_number { 0 };
size_t m_line_column { 0 };
size_t m_offset { 0 };
};
}