UserspaceEmulator+LibC: Use sys$emuctl() to pass malloc info to UE

Get rid of the awkward secret handshake sequence between malloc and UE
and simply use sys$emuctl() to notify UE of malloc, free and realloc.

Userland/DevTools/UserspaceEmulator/Emulator.cpp

@@ -519,7 +519,7 @@ u32 Emulator::virt_syscall(u32 function, u32 arg1, u32 arg2, u32 arg3)
     case SC_fork:
         return virt$fork();
     case SC_emuctl:
-        return virt$emuctl();
+        return virt$emuctl(arg1, arg2, arg3);
     case SC_sched_getparam:
         return virt$sched_getparam(arg1, arg2);
     case SC_sched_setparam:
@@ -1267,9 +1267,24 @@ int Emulator::virt$ioctl([[maybe_unused]] int fd, unsigned request, [[maybe_unus
     TODO();
 }
 
-int Emulator::virt$emuctl()
+int Emulator::virt$emuctl(FlatPtr arg1, FlatPtr arg2, FlatPtr arg3)
 {
-    return 0;
+    auto* tracer = malloc_tracer();
+    if (!tracer)
+        return 0;
+    switch (arg1) {
+    case 1:
+        tracer->target_did_malloc({}, arg3, arg2);
+        return 0;
+    case 2:
+        tracer->target_did_free({}, arg2);
+        return 0;
+    case 3:
+        tracer->target_did_realloc({}, arg3, arg2);
+        return 0;
+    default:
+        return -EINVAL;
+    }
 }
 
 int Emulator::virt$fork()

Userland/DevTools/UserspaceEmulator/Emulator.h

@@ -82,7 +82,7 @@ private:
     void register_signal_handlers();
     void setup_signal_trampoline();
 
-    int virt$emuctl();
+    int virt$emuctl(FlatPtr, FlatPtr, FlatPtr);
     int virt$fork();
     int virt$execve(FlatPtr);
     int virt$access(FlatPtr, size_t, int);

Userland/DevTools/UserspaceEmulator/MallocTracer.cpp

@@ -55,7 +55,7 @@ inline void MallocTracer::for_each_mallocation(Callback callback) const
     });
 }
 
-void MallocTracer::target_did_malloc(Badge<SoftCPU>, FlatPtr address, size_t size)
+void MallocTracer::target_did_malloc(Badge<Emulator>, FlatPtr address, size_t size)
 {
     if (m_emulator.is_in_loader_code())
         return;
@@ -114,7 +114,7 @@ ALWAYS_INLINE size_t MallocRegionMetadata::chunk_index_for_address(FlatPtr addre
     return chunk_offset / this->chunk_size;
 }
 
-void MallocTracer::target_did_free(Badge<SoftCPU>, FlatPtr address)
+void MallocTracer::target_did_free(Badge<Emulator>, FlatPtr address)
 {
     if (!address)
         return;
@@ -138,7 +138,7 @@ void MallocTracer::target_did_free(Badge<SoftCPU>, FlatPtr address)
     m_emulator.dump_backtrace();
 }
 
-void MallocTracer::target_did_realloc(Badge<SoftCPU>, FlatPtr address, size_t size)
+void MallocTracer::target_did_realloc(Badge<Emulator>, FlatPtr address, size_t size)
 {
     if (m_emulator.is_in_loader_code())
         return;

Userland/DevTools/UserspaceEmulator/MallocTracer.h

@@ -69,9 +69,9 @@ class MallocTracer {
 public:
     explicit MallocTracer(Emulator&);
 
-    void target_did_malloc(Badge<SoftCPU>, FlatPtr address, size_t);
+    void target_did_malloc(Badge<Emulator>, FlatPtr address, size_t);
-    void target_did_free(Badge<SoftCPU>, FlatPtr address);
+    void target_did_free(Badge<Emulator>, FlatPtr address);
-    void target_did_realloc(Badge<SoftCPU>, FlatPtr address, size_t);
+    void target_did_realloc(Badge<Emulator>, FlatPtr address, size_t);
 
     void audit_read(const Region&, FlatPtr address, size_t);
     void audit_write(const Region&, FlatPtr address, size_t);

Userland/DevTools/UserspaceEmulator/SoftCPU.cpp

@@ -112,22 +112,6 @@ void SoftCPU::dump() const
     fflush(stdout);
 }
 
-void SoftCPU::did_receive_secret_data()
-{
-    if (m_secret_data[0] == 1) {
-        if (auto* tracer = m_emulator.malloc_tracer())
-            tracer->target_did_malloc({}, m_secret_data[2], m_secret_data[1]);
-    } else if (m_secret_data[0] == 2) {
-        if (auto* tracer = m_emulator.malloc_tracer())
-            tracer->target_did_free({}, m_secret_data[1]);
-    } else if (m_secret_data[0] == 3) {
-        if (auto* tracer = m_emulator.malloc_tracer())
-            tracer->target_did_realloc({}, m_secret_data[2], m_secret_data[1]);
-    } else {
-        VERIFY_NOT_REACHED();
-    }
-}
-
 void SoftCPU::update_code_cache()
 {
     auto* region = m_emulator.mmu().find_region({ cs(), eip() });
@@ -2757,18 +2741,6 @@ void SoftCPU::PUSH_reg16(const X86::Instruction& insn)
 void SoftCPU::PUSH_reg32(const X86::Instruction& insn)
 {
     push32(gpr32(insn.reg32()));
-
-    if (m_secret_handshake_state == 2) {
-        m_secret_data[0] = gpr32(insn.reg32()).value();
-        ++m_secret_handshake_state;
-    } else if (m_secret_handshake_state == 3) {
-        m_secret_data[1] = gpr32(insn.reg32()).value();
-        ++m_secret_handshake_state;
-    } else if (m_secret_handshake_state == 4) {
-        m_secret_data[2] = gpr32(insn.reg32()).value();
-        m_secret_handshake_state = 0;
-        did_receive_secret_data();
-    }
 }
 
 template<typename T, bool cf>
@@ -2964,11 +2936,6 @@ void SoftCPU::SALC(const X86::Instruction&)
 {
     // FIXME: Respect shadow flags once they exists!
     set_al(shadow_wrap_as_initialized<u8>(cf() ? 0xff : 0x00));
-
-    if (m_secret_handshake_state < 2)
-        ++m_secret_handshake_state;
-    else
-        m_secret_handshake_state = 0;
 }
 
 template<typename T>

Userland/DevTools/UserspaceEmulator/SoftCPU.h

@@ -1105,8 +1105,6 @@ private:
 
     void update_code_cache();
 
-    void did_receive_secret_data();
-
 private:
     Emulator& m_emulator;
 
@@ -1154,9 +1152,6 @@ private:
 
     Region* m_cached_code_region { nullptr };
     u8* m_cached_code_base_ptr { nullptr };
-
-    u32 m_secret_handshake_state { 0 };
-    u32 m_secret_data[3];
 };
 
 ALWAYS_INLINE u8 SoftCPU::read8()

Userland/Libraries/LibC/malloc.cpp

@@ -47,21 +47,6 @@
 
 #define PAGE_ROUND_UP(x) ((((size_t)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1)))
 
-ALWAYS_INLINE static void ue_notify_malloc(const void* ptr, size_t size)
-{
-    send_secret_data_to_userspace_emulator(1, size, (FlatPtr)ptr);
-}
-
-ALWAYS_INLINE static void ue_notify_free(const void* ptr)
-{
-    send_secret_data_to_userspace_emulator(2, (FlatPtr)ptr, 0);
-}
-
-ALWAYS_INLINE static void ue_notify_realloc(const void* ptr, size_t size)
-{
-    send_secret_data_to_userspace_emulator(3, size, (FlatPtr)ptr);
-}
-
 static LibThread::Lock& malloc_lock()
 {
     static u32 lock_storage[sizeof(LibThread::Lock) / sizeof(u32)];
@@ -77,6 +62,24 @@ static bool s_scrub_free = true;
 static bool s_profiling = false;
 static bool s_in_userspace_emulator = false;
 
+ALWAYS_INLINE static void ue_notify_malloc(const void* ptr, size_t size)
+{
+    if (s_in_userspace_emulator)
+        syscall(SC_emuctl, 1, size, (FlatPtr)ptr);
+}
+
+ALWAYS_INLINE static void ue_notify_free(const void* ptr)
+{
+    if (s_in_userspace_emulator)
+        syscall(SC_emuctl, 2, (FlatPtr)ptr, 0);
+}
+
+ALWAYS_INLINE static void ue_notify_realloc(const void* ptr, size_t size)
+{
+    if (s_in_userspace_emulator)
+        syscall(SC_emuctl, 3, size, (FlatPtr)ptr);
+}
+
 struct MallocStats {
     size_t number_of_malloc_calls;
 
@@ -427,10 +430,7 @@ void __malloc_init()
 {
     new (&malloc_lock()) LibThread::Lock();
 
-#ifdef __serenity__
+    s_in_userspace_emulator = (int)syscall(SC_emuctl, 0) != -ENOSYS;
-    s_in_userspace_emulator = syscall(SC_emuctl) != ENOSYS;
-#endif
-
     if (s_in_userspace_emulator) {
         // Don't bother scrubbing memory if we're running in UE since it
         // keeps track of heap memory anyway.

Userland/Libraries/LibC/serenity.h

@@ -109,27 +109,4 @@ int serenity_readlink(const char* path, size_t path_length, char* buffer, size_t
 int getkeymap(char* name_buffer, size_t name_buffer_size, uint32_t* map, uint32_t* shift_map, uint32_t* alt_map, uint32_t* altgr_map, uint32_t* shift_altgr_map);
 int setkeymap(const char* name, const uint32_t* map, uint32_t* const shift_map, const uint32_t* alt_map, const uint32_t* altgr_map, const uint32_t* shift_altgr_map);
 
-#ifdef __i386__
-ALWAYS_INLINE void send_secret_data_to_userspace_emulator(uintptr_t data1, uintptr_t data2, uintptr_t data3)
-{
-    asm volatile(
-        ".byte 0xd6\n"
-        ".byte 0xd6\n" ::
-            : "eax");
-    asm volatile(
-        "push %%eax\n"
-        "push %%ecx\n"
-        "push %%edx\n"
-        "pop %%edx\n"
-        "pop %%ecx\n"
-        "pop %%eax\n" ::"a"(data1),
-        "c"(data2), "d"(data3)
-        : "memory");
-}
-#elif __x86_64__
-ALWAYS_INLINE void send_secret_data_to_userspace_emulator(uintptr_t, uintptr_t, uintptr_t)
-{
-}
-#endif
-
 __END_DECLS