LibJS: Devirtualize and pack the bytecode stream :^)

This patch changes the LibJS bytecode to be a stream of instructions
packed one-after-the-other in contiguous memory, instead of a vector
of OwnPtr<Instruction>. This should be a lot more cache-friendly. :^)

Instructions are also devirtualized and instead have a type field
using a new Instruction::Type enum.

To iterate over a bytecode stream, one must now use
Bytecode::InstructionStreamIterator.

Userland/Libraries/LibJS/Bytecode/ASTCodegen.cpp

@@ -180,7 +180,7 @@ Optional<Bytecode::Register> CallExpression::generate_bytecode(Bytecode::Generat
     for (auto& arg : m_arguments)
         argument_registers.append(*arg.value->generate_bytecode(generator));
     auto dst_reg = generator.allocate_register();
-    generator.emit<Bytecode::Op::Call>(dst_reg, *callee_reg, this_reg, argument_registers);
+    generator.emit_with_extra_register_slots<Bytecode::Op::Call>(argument_registers.size(), dst_reg, *callee_reg, this_reg, argument_registers);
     return dst_reg;
 }
 

Userland/Libraries/LibJS/Bytecode/Block.cpp

@@ -6,7 +6,8 @@
 
 #include <AK/String.h>
 #include <LibJS/Bytecode/Block.h>
-#include <LibJS/Bytecode/Instruction.h>
+#include <LibJS/Bytecode/Op.h>
+#include <sys/mman.h>
 
 namespace JS::Bytecode {
 
@@ -17,22 +18,42 @@ NonnullOwnPtr<Block> Block::create()
 
 Block::Block()
 {
+    // FIXME: This is not the smartest solution ever. Find something cleverer!
+    // The main issue we're working around here is that we don't want pointers into the bytecode stream to become invalidated
+    // during code generation due to dynamic buffer resizing. Otherwise we could just use a Vector.
+    m_buffer_capacity = 64 * KiB;
+    m_buffer = (u8*)mmap(nullptr, m_buffer_capacity, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
+    VERIFY(m_buffer != MAP_FAILED);
 }
 
 Block::~Block()
 {
-}
+    Bytecode::InstructionStreamIterator it(instruction_stream());
-
+    while (!it.at_end()) {
-void Block::append(Badge<Bytecode::Generator>, NonnullOwnPtr<Instruction> instruction)
+        auto& to_destroy = (*it);
-{
+        ++it;
-    m_instructions.append(move(instruction));
+        Instruction::destroy(const_cast<Instruction&>(to_destroy));
+    }
 }
 
 void Block::dump() const
 {
-    for (size_t i = 0; i < m_instructions.size(); ++i) {
+    Bytecode::InstructionStreamIterator it(instruction_stream());
-        warnln("[{:3}] {}", i, m_instructions[i].to_string());
+    while (!it.at_end()) {
+        warnln("[{:4x}] {}", it.offset(), (*it).to_string());
+        ++it;
     }
 }
 
+void Block::grow(size_t additional_size)
+{
+    m_buffer_size += additional_size;
+    VERIFY(m_buffer_size <= m_buffer_capacity);
+}
+
+void InstructionStreamIterator::operator++()
+{
+    m_offset += dereference().length();
+}
+
 }

Userland/Libraries/LibJS/Bytecode/Block.h

@@ -12,24 +12,55 @@
 
 namespace JS::Bytecode {
 
+class InstructionStreamIterator {
+public:
+    explicit InstructionStreamIterator(ReadonlyBytes bytes)
+        : m_bytes(bytes)
+    {
+    }
+
+    size_t offset() const { return m_offset; }
+    bool at_end() const { return m_offset >= m_bytes.size(); }
+    void jump(size_t offset)
+    {
+        VERIFY(offset <= m_bytes.size());
+        m_offset = offset;
+    }
+
+    Instruction const& operator*() const { return dereference(); }
+    void operator++();
+
+private:
+    Instruction const& dereference() const { return *reinterpret_cast<Instruction const*>(m_bytes.data() + offset()); }
+
+    ReadonlyBytes m_bytes;
+    size_t m_offset { 0 };
+};
+
 class Block {
 public:
     static NonnullOwnPtr<Block> create();
     ~Block();
 
-    NonnullOwnPtrVector<Instruction> const& instructions() const { return m_instructions; }
     void dump() const;
+    ReadonlyBytes instruction_stream() const { return ReadonlyBytes { m_buffer, m_buffer_size }; }
 
     size_t register_count() const { return m_register_count; }
 
-    void append(Badge<Bytecode::Generator>, NonnullOwnPtr<Instruction>);
     void set_register_count(Badge<Bytecode::Generator>, size_t count) { m_register_count = count; }
 
+    void* next_slot() { return m_buffer + m_buffer_size; }
+    void grow(size_t additional_size);
+
 private:
     Block();
 
     size_t m_register_count { 0 };
-    NonnullOwnPtrVector<Instruction> m_instructions;
+    u8* m_buffer { nullptr };
+    size_t m_buffer_capacity { 0 };
+    size_t m_buffer_size { 0 };
+
+    u8 const* m_buffer_end { nullptr };
 };
 
 }

Userland/Libraries/LibJS/Bytecode/Generator.cpp

@@ -31,9 +31,14 @@ OwnPtr<Block> Generator::generate(ASTNode const& node)
     return move(generator.m_block);
 }
 
-void Generator::append(NonnullOwnPtr<Instruction> instruction)
+void Generator::grow(size_t additional_size)
 {
-    m_block->append({}, move(instruction));
+    m_block->grow(additional_size);
+}
+
+void* Generator::next_slot()
+{
+    return m_block->next_slot();
 }
 
 Register Generator::allocate_register()
@@ -44,7 +49,7 @@ Register Generator::allocate_register()
 
 Label Generator::make_label() const
 {
-    return Label { m_block->instructions().size() };
+    return Label { m_block->instruction_stream().size() };
 }
 
 Label Generator::nearest_continuable_scope() const

Userland/Libraries/LibJS/Bytecode/Generator.h

@@ -8,6 +8,7 @@
 
 #include <AK/OwnPtr.h>
 #include <LibJS/Bytecode/Label.h>
+#include <LibJS/Bytecode/Register.h>
 #include <LibJS/Forward.h>
 
 namespace JS::Bytecode {
@@ -21,10 +22,19 @@ public:
     template<typename OpType, typename... Args>
     OpType& emit(Args&&... args)
     {
-        auto instruction = make<OpType>(forward<Args>(args)...);
+        void* slot = next_slot();
-        auto* ptr = instruction.ptr();
+        grow(sizeof(OpType));
-        append(move(instruction));
+        new (slot) OpType(forward<Args>(args)...);
-        return *ptr;
+        return *static_cast<OpType*>(slot);
+    }
+
+    template<typename OpType, typename... Args>
+    OpType& emit_with_extra_register_slots(size_t extra_register_slots, Args&&... args)
+    {
+        void* slot = next_slot();
+        grow(sizeof(OpType) + extra_register_slots * sizeof(Register));
+        new (slot) OpType(forward<Args>(args)...);
+        return *static_cast<OpType*>(slot);
     }
 
     Label make_label() const;
@@ -38,7 +48,8 @@ private:
     Generator();
     ~Generator();
 
-    void append(NonnullOwnPtr<Instruction>);
+    void grow(size_t);
+    void* next_slot();
 
     OwnPtr<Block> m_block;
     u32 m_next_register { 1 };

Userland/Libraries/LibJS/Bytecode/Instruction.cpp

@@ -5,11 +5,40 @@
  */
 
 #include <LibJS/Bytecode/Instruction.h>
+#include <LibJS/Bytecode/Op.h>
 
 namespace JS::Bytecode {
 
-Instruction::~Instruction()
+void Instruction::destroy(Instruction& instruction)
 {
+#define __BYTECODE_OP(op)                        \
+    case Type::op:                               \
+        static_cast<Op::op&>(instruction).~op(); \
+        return;
+
+    switch (instruction.type()) {
+        ENUMERATE_BYTECODE_OPS(__BYTECODE_OP)
+    default:
+        VERIFY_NOT_REACHED();
+    }
+
+#undef __BYTECODE_OP
+}
+
+size_t Instruction::length() const
+{
+    if (type() == Type::Call)
+        return static_cast<Op::Call const&>(*this).length();
+
+#define __BYTECODE_OP(op) \
+    case Type::op:        \
+        return sizeof(Op::op);
+
+    switch (type()) {
+        ENUMERATE_BYTECODE_OPS(__BYTECODE_OP)
+    default:
+        VERIFY_NOT_REACHED();
+    }
 }
 
 }

Userland/Libraries/LibJS/Bytecode/Instruction.h

@@ -9,14 +9,51 @@
 #include <AK/Forward.h>
 #include <LibJS/Forward.h>
 
+#define ENUMERATE_BYTECODE_OPS(O) \
+    O(Load)                       \
+    O(Add)                        \
+    O(Sub)                        \
+    O(LessThan)                   \
+    O(AbstractInequals)           \
+    O(AbstractEquals)             \
+    O(NewString)                  \
+    O(NewObject)                  \
+    O(GetVariable)                \
+    O(SetVariable)                \
+    O(PutById)                    \
+    O(GetById)                    \
+    O(Jump)                       \
+    O(JumpIfFalse)                \
+    O(JumpIfTrue)                 \
+    O(Call)                       \
+    O(EnterScope)                 \
+    O(Return)
+
 namespace JS::Bytecode {
 
 class Instruction {
 public:
-    virtual ~Instruction();
+    enum class Type {
+#define __BYTECODE_OP(op) \
+    op,
+        ENUMERATE_BYTECODE_OPS(__BYTECODE_OP)
+#undef __BYTECODE_OP
+    };
 
-    virtual String to_string() const = 0;
+    Type type() const { return m_type; }
-    virtual void execute(Bytecode::Interpreter&) const = 0;
+    size_t length() const;
+    String to_string() const;
+    void execute(Bytecode::Interpreter&) const;
+    static void destroy(Instruction&);
+
+protected:
+    explicit Instruction(Type type)
+        : m_type(type)
+    {
+    }
+
+private:
+    Type m_type {};
 };
 
 }

Userland/Libraries/LibJS/Bytecode/Interpreter.cpp

@@ -52,12 +52,12 @@ Value Interpreter::run(Bytecode::Block const& block)
     m_register_windows.append(make<RegisterWindow>());
     registers().resize(block.register_count());
 
-    size_t pc = 0;
+    Bytecode::InstructionStreamIterator pc(block.instruction_stream());
-    while (pc < block.instructions().size()) {
+    while (!pc.at_end()) {
-        auto& instruction = block.instructions()[pc];
+        auto& instruction = *pc;
         instruction.execute(*this);
         if (m_pending_jump.has_value()) {
-            pc = m_pending_jump.release_value();
+            pc.jump(m_pending_jump.release_value());
             continue;
         }
         if (!m_return_value.is_empty())

Userland/Libraries/LibJS/Bytecode/Label.h

@@ -29,6 +29,6 @@ template<>
 struct AK::Formatter<JS::Bytecode::Label> : AK::Formatter<FormatString> {
     void format(FormatBuilder& builder, JS::Bytecode::Label const& value)
     {
-        return AK::Formatter<FormatString>::format(builder, "@{}", value.address());
+        return AK::Formatter<FormatString>::format(builder, "@{:x}", value.address());
     }
 };

Userland/Libraries/LibJS/Bytecode/Op.cpp

@@ -11,6 +11,40 @@
 #include <LibJS/Runtime/ScriptFunction.h>
 #include <LibJS/Runtime/Value.h>
 
+namespace JS::Bytecode {
+
+void Instruction::execute(Bytecode::Interpreter& interpreter) const
+{
+#define __BYTECODE_OP(op)       \
+    case Instruction::Type::op: \
+        return static_cast<Bytecode::Op::op const&>(*this).execute(interpreter);
+
+    switch (type()) {
+        ENUMERATE_BYTECODE_OPS(__BYTECODE_OP)
+    default:
+        VERIFY_NOT_REACHED();
+    }
+
+#undef __BYTECODE_OP
+}
+
+String Instruction::to_string() const
+{
+#define __BYTECODE_OP(op)       \
+    case Instruction::Type::op: \
+        return static_cast<Bytecode::Op::op const&>(*this).to_string();
+
+    switch (type()) {
+        ENUMERATE_BYTECODE_OPS(__BYTECODE_OP)
+    default:
+        VERIFY_NOT_REACHED();
+    }
+
+#undef __BYTECODE_OP
+}
+
+}
+
 namespace JS::Bytecode::Op {
 
 void Load::execute(Bytecode::Interpreter& interpreter) const
@@ -108,12 +142,12 @@ void Call::execute(Bytecode::Interpreter& interpreter) const
 
     Value return_value;
 
-    if (m_arguments.is_empty()) {
+    if (m_argument_count == 0) {
         return_value = interpreter.vm().call(function, this_value);
     } else {
         MarkedValueList argument_values { interpreter.vm().heap() };
-        for (auto& arg : m_arguments) {
+        for (size_t i = 0; i < m_argument_count; ++i) {
-            argument_values.append(interpreter.reg(arg));
+            argument_values.append(interpreter.reg(m_arguments[i]));
         }
         return_value = interpreter.vm().call(function, this_value, move(argument_values));
     }
@@ -227,11 +261,11 @@ String Call::to_string() const
 {
     StringBuilder builder;
     builder.appendff("Call dst:{}, callee:{}, this:{}", m_dst, m_callee, m_this_value);
-    if (!m_arguments.is_empty()) {
+    if (m_argument_count != 0) {
         builder.append(", arguments:[");
-        for (size_t i = 0; i < m_arguments.size(); ++i) {
+        for (size_t i = 0; i < m_argument_count; ++i) {
             builder.appendff("{}", m_arguments[i]);
-            if (i != m_arguments.size() - 1)
+            if (i != m_argument_count - 1)
                 builder.append(',');
         }
         builder.append(']');

Userland/Libraries/LibJS/Bytecode/Op.h

@@ -18,14 +18,14 @@ namespace JS::Bytecode::Op {
 class Load final : public Instruction {
 public:
     Load(Register dst, Value value)
-        : m_dst(dst)
+        : Instruction(Type::Load)
+        , m_dst(dst)
         , m_value(value)
     {
     }
 
-    virtual ~Load() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -35,15 +35,15 @@ private:
 class Add final : public Instruction {
 public:
     Add(Register dst, Register src1, Register src2)
-        : m_dst(dst)
+        : Instruction(Type::Add)
+        , m_dst(dst)
         , m_src1(src1)
         , m_src2(src2)
     {
     }
 
-    virtual ~Add() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -54,15 +54,15 @@ private:
 class Sub final : public Instruction {
 public:
     Sub(Register dst, Register src1, Register src2)
-        : m_dst(dst)
+        : Instruction(Type::Sub)
+        , m_dst(dst)
         , m_src1(src1)
         , m_src2(src2)
     {
     }
 
-    virtual ~Sub() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -73,15 +73,15 @@ private:
 class LessThan final : public Instruction {
 public:
     LessThan(Register dst, Register src1, Register src2)
-        : m_dst(dst)
+        : Instruction(Type::LessThan)
+        , m_dst(dst)
         , m_src1(src1)
         , m_src2(src2)
     {
     }
 
-    virtual ~LessThan() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -92,15 +92,15 @@ private:
 class AbstractInequals final : public Instruction {
 public:
     AbstractInequals(Register dst, Register src1, Register src2)
-        : m_dst(dst)
+        : Instruction(Type::AbstractEquals)
+        , m_dst(dst)
         , m_src1(src1)
         , m_src2(src2)
     {
     }
 
-    virtual ~AbstractInequals() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -111,15 +111,15 @@ private:
 class AbstractEquals final : public Instruction {
 public:
     AbstractEquals(Register dst, Register src1, Register src2)
-        : m_dst(dst)
+        : Instruction(Type::AbstractEquals)
+        , m_dst(dst)
         , m_src1(src1)
         , m_src2(src2)
     {
     }
 
-    virtual ~AbstractEquals() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -130,14 +130,14 @@ private:
 class NewString final : public Instruction {
 public:
     NewString(Register dst, String string)
-        : m_dst(dst)
+        : Instruction(Type::NewString)
+        , m_dst(dst)
         , m_string(move(string))
     {
     }
 
-    virtual ~NewString() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -147,13 +147,13 @@ private:
 class NewObject final : public Instruction {
 public:
     explicit NewObject(Register dst)
-        : m_dst(dst)
+        : Instruction(Type::NewObject)
+        , m_dst(dst)
     {
     }
 
-    virtual ~NewObject() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -162,14 +162,14 @@ private:
 class SetVariable final : public Instruction {
 public:
     SetVariable(FlyString identifier, Register src)
-        : m_identifier(move(identifier))
+        : Instruction(Type::SetVariable)
+        , m_identifier(move(identifier))
         , m_src(src)
     {
     }
 
-    virtual ~SetVariable() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     FlyString m_identifier;
@@ -179,14 +179,14 @@ private:
 class GetVariable final : public Instruction {
 public:
     GetVariable(Register dst, FlyString identifier)
-        : m_dst(dst)
+        : Instruction(Type::GetVariable)
+        , m_dst(dst)
         , m_identifier(move(identifier))
     {
     }
 
-    virtual ~GetVariable() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -196,15 +196,15 @@ private:
 class GetById final : public Instruction {
 public:
     GetById(Register dst, Register base, FlyString property)
-        : m_dst(dst)
+        : Instruction(Type::GetById)
+        , m_dst(dst)
         , m_base(base)
         , m_property(move(property))
     {
     }
 
-    virtual ~GetById() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_dst;
@@ -215,15 +215,15 @@ private:
 class PutById final : public Instruction {
 public:
     PutById(Register base, FlyString property, Register src)
-        : m_base(base)
+        : Instruction(Type::PutById)
+        , m_base(base)
         , m_property(move(property))
         , m_src(src)
     {
     }
 
-    virtual ~PutById() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_base;
@@ -234,15 +234,15 @@ private:
 class Jump final : public Instruction {
 public:
     explicit Jump(Optional<Label> target = {})
-        : m_target(move(target))
+        : Instruction(Type::Jump)
+        , m_target(move(target))
     {
     }
 
     void set_target(Optional<Label> target) { m_target = move(target); }
 
-    virtual ~Jump() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Optional<Label> m_target;
@@ -251,16 +251,16 @@ private:
 class JumpIfFalse final : public Instruction {
 public:
     explicit JumpIfFalse(Register result, Optional<Label> target = {})
-        : m_result(result)
+        : Instruction(Type::JumpIfFalse)
+        , m_result(result)
         , m_target(move(target))
     {
     }
 
     void set_target(Optional<Label> target) { m_target = move(target); }
 
-    virtual ~JumpIfFalse() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_result;
@@ -270,53 +270,59 @@ private:
 class JumpIfTrue final : public Instruction {
 public:
     explicit JumpIfTrue(Register result, Optional<Label> target = {})
-        : m_result(result)
+        : Instruction(Type::JumpIfTrue)
+        , m_result(result)
         , m_target(move(target))
     {
     }
 
     void set_target(Optional<Label> target) { m_target = move(target); }
 
-    virtual ~JumpIfTrue() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Register m_result;
     Optional<Label> m_target;
 };
 
+// NOTE: This instruction is variable-width depending on the number of arguments!
 class Call final : public Instruction {
 public:
-    Call(Register dst, Register callee, Register this_value, Vector<Register> arguments)
+    Call(Register dst, Register callee, Register this_value, Vector<Register> const& arguments)
-        : m_dst(dst)
+        : Instruction(Type::Call)
+        , m_dst(dst)
         , m_callee(callee)
         , m_this_value(this_value)
-        , m_arguments(move(arguments))
+        , m_argument_count(arguments.size())
     {
+        for (size_t i = 0; i < m_argument_count; ++i)
+            m_arguments[i] = arguments[i];
     }
 
-    virtual ~Call() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
+
+    size_t length() const { return sizeof(*this) + sizeof(Register) * m_argument_count; }
 
 private:
     Register m_dst;
     Register m_callee;
     Register m_this_value;
-    Vector<Register> m_arguments;
+    size_t m_argument_count { 0 };
+    Register m_arguments[];
 };
 
 class EnterScope final : public Instruction {
 public:
     explicit EnterScope(ScopeNode const& scope_node)
-        : m_scope_node(scope_node)
+        : Instruction(Type::EnterScope)
+        , m_scope_node(scope_node)
     {
     }
 
-    virtual ~EnterScope() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     ScopeNode const& m_scope_node;
@@ -325,13 +331,13 @@ private:
 class Return final : public Instruction {
 public:
     explicit Return(Optional<Register> argument)
-        : m_argument(move(argument))
+        : Instruction(Type::Return)
+        , m_argument(move(argument))
     {
     }
 
-    virtual ~Return() override { }
+    void execute(Bytecode::Interpreter&) const;
-    virtual void execute(Bytecode::Interpreter&) const override;
+    String to_string() const;
-    virtual String to_string() const override;
 
 private:
     Optional<Register> m_argument;