RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-26 00:17:46 +00:00
Code Issues Activity Actions

Kernel: Protect processes' master TLS with a fine-grained spinlock

Browse source 
This moves it out of the scope of the big process lock, and allows us
to wean some syscalls off it, starting with sys$allocate_tls.
This commit is contained in:
Idan Horowitz 2023-12-16 12:37:28 +02:00 • committed by Andreas Kling
parent cd56ec6e5c
commit 6a4b93b3e0
7 changed files with 91 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Kernel/API/Syscall.h
View file

@ -48,7 +48,7 @@ enum class NeedsBigProcessLock {
S(accept4, NeedsBigProcessLock::No) \ S(accept4, NeedsBigProcessLock::No) \
S(adjtime, NeedsBigProcessLock::No) \ S(adjtime, NeedsBigProcessLock::No) \
S(alarm, NeedsBigProcessLock::No) \ S(alarm, NeedsBigProcessLock::No) \
S(allocate_tls, NeedsBigProcessLock::Yes) \ S(allocate_tls, NeedsBigProcessLock::No) \
S(anon_create, NeedsBigProcessLock::No) \ S(anon_create, NeedsBigProcessLock::No) \
S(annotate_mapping, NeedsBigProcessLock::No) \ S(annotate_mapping, NeedsBigProcessLock::No) \
S(bind, NeedsBigProcessLock::No) \ S(bind, NeedsBigProcessLock::No) \

27
Kernel/Syscalls/execve.cpp
View file

@ -431,9 +431,11 @@ Process::load(Memory::AddressSpace& new_space, NonnullRefPtr<OpenFileDescription
if (interpreter_description.is_null()) { if (interpreter_description.is_null()) {
auto load_result = TRY(load_elf_object(new_space, main_program_description, load_offset, ShouldAllocateTls::Yes, ShouldAllowSyscalls::No, minimum_stack_size)); auto load_result = TRY(load_elf_object(new_space, main_program_description, load_offset, ShouldAllocateTls::Yes, ShouldAllowSyscalls::No, minimum_stack_size));
m_master_tls_region = load_result.tls_region; m_master_tls.with([&load_result](auto& master_tls) {
m_master_tls_size = load_result.tls_size; master_tls.region = load_result.tls_region;
m_master_tls_alignment = load_result.tls_alignment; master_tls.size = load_result.tls_size;
master_tls.alignment = load_result.tls_alignment;
});
return load_result; return load_result;
} }
@ -490,25 +492,26 @@ ErrorOr<void> Process::do_exec(NonnullRefPtr<OpenFileDescription> main_program_d
auto allocated_space = TRY(Memory::AddressSpace::try_create(*this, nullptr)); auto allocated_space = TRY(Memory::AddressSpace::try_create(*this, nullptr));
OwnPtr<Memory::AddressSpace> old_space; OwnPtr<Memory::AddressSpace> old_space;
auto old_master_tls_region = m_master_tls_region; auto old_master_tls = m_master_tls.with([](auto& master_tls) {
auto old_master_tls_size = m_master_tls_size; auto old = master_tls;
auto old_master_tls_alignment = m_master_tls_alignment; master_tls.region = nullptr;
master_tls.size = 0;
master_tls.alignment = 0;
return old;
});
auto& new_space = m_space.with([&](auto& space) -> Memory::AddressSpace& { auto& new_space = m_space.with([&](auto& space) -> Memory::AddressSpace& {
old_space = move(space); old_space = move(space);
space = move(allocated_space); space = move(allocated_space);
return *space; return *space;
}); });
m_master_tls_region = nullptr;
m_master_tls_size = 0;
m_master_tls_alignment = 0;
ArmedScopeGuard space_guard([&]() { ArmedScopeGuard space_guard([&]() {
// If we failed at any point from now on we have to revert back to the old address space // If we failed at any point from now on we have to revert back to the old address space
m_space.with([&](auto& space) { m_space.with([&](auto& space) {
space = old_space.release_nonnull(); space = old_space.release_nonnull();
}); });
m_master_tls_region = old_master_tls_region; m_master_tls.with([&](auto& master_tls) {
m_master_tls_size = old_master_tls_size; master_tls = old_master_tls;
m_master_tls_alignment = old_master_tls_alignment; });
Memory::MemoryManager::enter_process_address_space(*this); Memory::MemoryManager::enter_process_address_space(*this);
}); });

32
Kernel/Syscalls/fork.cpp
View file

@ -158,21 +158,27 @@ ErrorOr<FlatPtr> Process::sys$fork(RegisterState& regs)
#endif #endif
TRY(address_space().with([&](auto& parent_space) { TRY(address_space().with([&](auto& parent_space) {
return child->address_space().with([&](auto& child_space) -> ErrorOr<void> { return m_master_tls.with([&](auto& parent_master_tls) -> ErrorOr<void> {
child_space->set_enforces_syscall_regions(parent_space->enforces_syscall_regions()); return child->address_space().with([&](auto& child_space) -> ErrorOr<void> {
for (auto& region : parent_space->region_tree().regions()) { child_space->set_enforces_syscall_regions(parent_space->enforces_syscall_regions());
dbgln_if(FORK_DEBUG, "fork: cloning Region '{}' @ {}", region.name(), region.vaddr()); for (auto& region : parent_space->region_tree().regions()) {
auto region_clone = TRY(region.try_clone()); dbgln_if(FORK_DEBUG, "fork: cloning Region '{}' @ {}", region.name(), region.vaddr());
TRY(region_clone->map(child_space->page_directory(), Memory::ShouldFlushTLB::No)); auto region_clone = TRY(region.try_clone());
TRY(child_space->region_tree().place_specifically(*region_clone, region.range())); TRY(region_clone->map(child_space->page_directory(), Memory::ShouldFlushTLB::No));
auto* child_region = region_clone.leak_ptr(); TRY(child_space->region_tree().place_specifically(*region_clone, region.range()));
auto* child_region = region_clone.leak_ptr();
if (&region == m_master_tls_region.unsafe_ptr()) { if (&region == parent_master_tls.region.unsafe_ptr()) {
child->m_master_tls_region = TRY(child_region->try_make_weak_ptr()); TRY(child->m_master_tls.with([&](auto& child_master_tls) -> ErrorOr<void> {
child->m_master_tls_size = m_master_tls_size; child_master_tls.region = TRY(child_region->try_make_weak_ptr());
child->m_master_tls_alignment = m_master_tls_alignment; child_master_tls.size = parent_master_tls.size;
child_master_tls.alignment = parent_master_tls.alignment;
return {};
}));
}
} }
} return {};
});
return {}; return {};
}); });
})); }));

62
Kernel/Syscalls/mmap.cpp
View file

@ -524,50 +524,52 @@ ErrorOr<FlatPtr> Process::sys$mremap(Userspace<Syscall::SC_mremap_params const*>
ErrorOr<FlatPtr> Process::sys$allocate_tls(Userspace<char const*> initial_data, size_t size) ErrorOr<FlatPtr> Process::sys$allocate_tls(Userspace<char const*> initial_data, size_t size)
{ {
VERIFY_PROCESS_BIG_LOCK_ACQUIRED(this); VERIFY_NO_PROCESS_BIG_LOCK(this);
TRY(require_promise(Pledge::stdio)); TRY(require_promise(Pledge::stdio));
if (!size || size % PAGE_SIZE != 0) if (!size || size % PAGE_SIZE != 0)
return EINVAL; return EINVAL;
if (!m_master_tls_region.is_null()) return m_master_tls.with([&](auto& master_tls) -> ErrorOr<FlatPtr> {
return EEXIST; if (!master_tls.region.is_null())
return EEXIST;
if (thread_count() != 1) if (thread_count() != 1)
return EFAULT; return EFAULT;
Thread* main_thread = nullptr; Thread* main_thread = nullptr;
bool multiple_threads = false; bool multiple_threads = false;
for_each_thread([&main_thread, &multiple_threads](auto& thread) { for_each_thread([&main_thread, &multiple_threads](auto& thread) {
if (main_thread) if (main_thread)
multiple_threads = true; multiple_threads = true;
main_thread = &thread; main_thread = &thread;
return IterationDecision::Break; return IterationDecision::Break;
}); });
VERIFY(main_thread); VERIFY(main_thread);
if (multiple_threads) if (multiple_threads)
return EINVAL; return EINVAL;
return address_space().with([&](auto& space) -> ErrorOr<FlatPtr> { return address_space().with([&](auto& space) -> ErrorOr<FlatPtr> {
auto* region = TRY(space->allocate_region(Memory::RandomizeVirtualAddress::Yes, {}, size, PAGE_SIZE, "Master TLS"sv, PROT_READ | PROT_WRITE)); auto* region = TRY(space->allocate_region(Memory::RandomizeVirtualAddress::Yes, {}, size, PAGE_SIZE, "Master TLS"sv, PROT_READ | PROT_WRITE));
m_master_tls_region = TRY(region->try_make_weak_ptr()); master_tls.region = TRY(region->try_make_weak_ptr());
m_master_tls_size = size; master_tls.size = size;
m_master_tls_alignment = PAGE_SIZE; master_tls.alignment = PAGE_SIZE;
{ {
Kernel::SmapDisabler disabler; Kernel::SmapDisabler disabler;
void* fault_at; void* fault_at;
if (!Kernel::safe_memcpy((char*)m_master_tls_region.unsafe_ptr()->vaddr().as_ptr(), (char*)initial_data.ptr(), size, fault_at)) if (!Kernel::safe_memcpy((char*)master_tls.region.unsafe_ptr()->vaddr().as_ptr(), (char*)initial_data.ptr(), size, fault_at))
return EFAULT; return EFAULT;
} }
TRY(main_thread->make_thread_specific_region({})); TRY(main_thread->make_thread_specific_region({}));
Processor::set_thread_specific_data(main_thread->thread_specific_data()); Processor::set_thread_specific_data(main_thread->thread_specific_data());
return m_master_tls_region.unsafe_ptr()->vaddr().get(); return master_tls.region.unsafe_ptr()->vaddr().get();
});
}); });
} }

10
Kernel/Tasks/Process.h
View file

@ -924,8 +924,6 @@ private:
Vector<NonnullOwnPtr<KString>> m_arguments; Vector<NonnullOwnPtr<KString>> m_arguments;
Vector<NonnullOwnPtr<KString>> m_environment; Vector<NonnullOwnPtr<KString>> m_environment;
LockWeakPtr<Memory::Region> m_master_tls_region;
IntrusiveListNode<Process> m_jail_process_list_node; IntrusiveListNode<Process> m_jail_process_list_node;
IntrusiveListNode<Process> m_all_processes_list_node; IntrusiveListNode<Process> m_all_processes_list_node;
@ -937,8 +935,12 @@ private:
SpinlockProtected<RefPtr<ProcessList>, LockRank::None> m_jail_process_list; SpinlockProtected<RefPtr<ProcessList>, LockRank::None> m_jail_process_list;
SpinlockProtected<RefPtr<Jail>, LockRank::Process> m_attached_jail {}; SpinlockProtected<RefPtr<Jail>, LockRank::Process> m_attached_jail {};
size_t m_master_tls_size { 0 }; struct MasterThreadLocalStorage {
size_t m_master_tls_alignment { 0 }; LockWeakPtr<Memory::Region> region;
size_t size { 0 };
size_t alignment { 0 };
};
SpinlockProtected<MasterThreadLocalStorage, LockRank::None> m_master_tls;
Mutex m_big_lock { "Process"sv, Mutex::MutexBehavior::BigLock }; Mutex m_big_lock { "Process"sv, Mutex::MutexBehavior::BigLock };
Mutex m_ptrace_lock { "ptrace"sv }; Mutex m_ptrace_lock { "ptrace"sv };

42
Kernel/Tasks/Thread.cpp
View file

@ -1360,37 +1360,31 @@ ErrorOr<NonnullOwnPtr<KString>> Thread::backtrace()
return KString::try_create(builder.string_view()); return KString::try_create(builder.string_view());
} }
size_t Thread::thread_specific_region_alignment() const
{
return max(process().m_master_tls_alignment, alignof(ThreadSpecificData));
}
size_t Thread::thread_specific_region_size() const
{
return align_up_to(process().m_master_tls_size, thread_specific_region_alignment()) + sizeof(ThreadSpecificData);
}
ErrorOr<void> Thread::make_thread_specific_region(Badge<Process>) ErrorOr<void> Thread::make_thread_specific_region(Badge<Process>)
{ {
// The process may not require a TLS region, or allocate TLS later with sys$allocate_tls (which is what dynamically loaded programs do) return process().m_master_tls.with([&](auto& master_tls) -> ErrorOr<void> {
if (!process().m_master_tls_region) // The process may not require a TLS region, or allocate TLS later with sys$allocate_tls (which is what dynamically loaded programs do)
return {}; if (!master_tls.region)
return {};
return process().address_space().with([&](auto& space) -> ErrorOr<void> { return process().address_space().with([&](auto& space) -> ErrorOr<void> {
auto* region = TRY(space->allocate_region(Memory::RandomizeVirtualAddress::Yes, {}, thread_specific_region_size(), PAGE_SIZE, "Thread-specific"sv, PROT_READ | PROT_WRITE)); auto region_alignment = max(master_tls.alignment, alignof(ThreadSpecificData));
auto region_size = align_up_to(master_tls.size, region_alignment) + sizeof(ThreadSpecificData);
auto* region = TRY(space->allocate_region(Memory::RandomizeVirtualAddress::Yes, {}, region_size, PAGE_SIZE, "Thread-specific"sv, PROT_READ | PROT_WRITE));
m_thread_specific_range = region->range(); m_thread_specific_range = region->range();
SmapDisabler disabler; SmapDisabler disabler;
auto* thread_specific_data = (ThreadSpecificData*)region->vaddr().offset(align_up_to(process().m_master_tls_size, thread_specific_region_alignment())).as_ptr(); auto* thread_specific_data = (ThreadSpecificData*)region->vaddr().offset(align_up_to(master_tls.size, region_alignment)).as_ptr();
auto* thread_local_storage = (u8*)((u8*)thread_specific_data) - align_up_to(process().m_master_tls_size, process().m_master_tls_alignment); auto* thread_local_storage = (u8*)((u8*)thread_specific_data) - align_up_to(master_tls.size, master_tls.size);
m_thread_specific_data = VirtualAddress(thread_specific_data); m_thread_specific_data = VirtualAddress(thread_specific_data);
thread_specific_data->self = thread_specific_data; thread_specific_data->self = thread_specific_data;
if (process().m_master_tls_size != 0) if (master_tls.size != 0)
memcpy(thread_local_storage, process().m_master_tls_region.unsafe_ptr()->vaddr().as_ptr(), process().m_master_tls_size); memcpy(thread_local_storage, master_tls.region.unsafe_ptr()->vaddr().as_ptr(), master_tls.size);
return {}; return {};
});
}); });
} }

2
Kernel/Tasks/Thread.h
View file

@ -788,8 +788,6 @@ public:
StringView state_string() const; StringView state_string() const;
VirtualAddress thread_specific_data() const { return m_thread_specific_data; } VirtualAddress thread_specific_data() const { return m_thread_specific_data; }
size_t thread_specific_region_size() const;
size_t thread_specific_region_alignment() const;
ALWAYS_INLINE void yield_if_stopped() ALWAYS_INLINE void yield_if_stopped()
{ {