diff options
Diffstat (limited to '')
-rw-r--r-- | src/core/hle/kernel/k_thread.cpp | 953 |
1 files changed, 551 insertions, 402 deletions
diff --git a/src/core/hle/kernel/k_thread.cpp b/src/core/hle/kernel/k_thread.cpp index 21207fe99..adb6ec581 100644 --- a/src/core/hle/kernel/k_thread.cpp +++ b/src/core/hle/kernel/k_thread.cpp @@ -4,6 +4,8 @@ #include <algorithm> #include <atomic> #include <cinttypes> +#include <condition_variable> +#include <mutex> #include <optional> #include <vector> @@ -29,36 +31,34 @@ #include "core/hle/kernel/k_thread_queue.h" #include "core/hle/kernel/k_worker_task_manager.h" #include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/svc.h" #include "core/hle/kernel/svc_results.h" #include "core/hle/kernel/svc_types.h" #include "core/hle/result.h" #include "core/memory.h" -#ifdef ARCHITECTURE_x86_64 -#include "core/arm/dynarmic/arm_dynarmic_32.h" -#endif - namespace { constexpr inline s32 TerminatingThreadPriority = Kernel::Svc::SystemThreadPriorityHighest - 1; -static void ResetThreadContext32(Core::ARM_Interface::ThreadContext32& context, u32 stack_top, +static void ResetThreadContext32(Kernel::KThread::ThreadContext32& context, u32 stack_top, u32 entry_point, u32 arg) { context = {}; context.cpu_registers[0] = arg; context.cpu_registers[15] = entry_point; context.cpu_registers[13] = stack_top; + context.fpscr = 0; } -static void ResetThreadContext64(Core::ARM_Interface::ThreadContext64& context, VAddr stack_top, - VAddr entry_point, u64 arg) { +static void ResetThreadContext64(Kernel::KThread::ThreadContext64& context, u64 stack_top, + u64 entry_point, u64 arg) { context = {}; context.cpu_registers[0] = arg; context.cpu_registers[18] = Kernel::KSystemControl::GenerateRandomU64() | 1; context.pc = entry_point; context.sp = stack_top; - // TODO(merry): Perform a hardware test to determine the below value. context.fpcr = 0; + context.fpsr = 0; } } // namespace @@ -75,14 +75,14 @@ struct ThreadLocalRegion { class ThreadQueueImplForKThreadSleep final : public KThreadQueueWithoutEndWait { public: - explicit ThreadQueueImplForKThreadSleep(KernelCore& kernel_) - : KThreadQueueWithoutEndWait(kernel_) {} + explicit ThreadQueueImplForKThreadSleep(KernelCore& kernel) + : KThreadQueueWithoutEndWait(kernel) {} }; class ThreadQueueImplForKThreadSetProperty final : public KThreadQueue { public: - explicit ThreadQueueImplForKThreadSetProperty(KernelCore& kernel_, KThread::WaiterList* wl) - : KThreadQueue(kernel_), m_wait_list(wl) {} + explicit ThreadQueueImplForKThreadSetProperty(KernelCore& kernel, KThread::WaiterList* wl) + : KThreadQueue(kernel), m_wait_list(wl) {} void CancelWait(KThread* waiting_thread, Result wait_result, bool cancel_timer_task) override { // Remove the thread from the wait list. @@ -93,17 +93,17 @@ public: } private: - KThread::WaiterList* m_wait_list; + KThread::WaiterList* m_wait_list{}; }; } // namespace -KThread::KThread(KernelCore& kernel_) - : KAutoObjectWithSlabHeapAndContainer{kernel_}, activity_pause_lock{kernel_} {} +KThread::KThread(KernelCore& kernel) + : KAutoObjectWithSlabHeapAndContainer{kernel}, m_activity_pause_lock{kernel} {} KThread::~KThread() = default; -Result KThread::Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack_top, s32 prio, - s32 virt_core, KProcess* owner, ThreadType type) { +Result KThread::Initialize(KThreadFunction func, uintptr_t arg, KProcessAddress user_stack_top, + s32 prio, s32 virt_core, KProcess* owner, ThreadType type) { // Assert parameters are valid. ASSERT((type == ThreadType::Main) || (type == ThreadType::Dummy) || (Svc::HighestThreadPriority <= prio && prio <= Svc::LowestThreadPriority)); @@ -115,7 +115,7 @@ Result KThread::Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack ASSERT(0 <= phys_core && phys_core < static_cast<s32>(Core::Hardware::NUM_CPU_CORES)); // First, clear the TLS address. - tls_address = {}; + m_tls_address = {}; // Next, assert things based on the type. switch (type) { @@ -139,110 +139,110 @@ Result KThread::Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack ASSERT_MSG(false, "KThread::Initialize: Unknown ThreadType {}", static_cast<u32>(type)); break; } - thread_type = type; + m_thread_type = type; // Set the ideal core ID and affinity mask. - virtual_ideal_core_id = virt_core; - physical_ideal_core_id = phys_core; - virtual_affinity_mask = 1ULL << virt_core; - physical_affinity_mask.SetAffinity(phys_core, true); + m_virtual_ideal_core_id = virt_core; + m_physical_ideal_core_id = phys_core; + m_virtual_affinity_mask = 1ULL << virt_core; + m_physical_affinity_mask.SetAffinity(phys_core, true); // Set the thread state. - thread_state = (type == ThreadType::Main || type == ThreadType::Dummy) - ? ThreadState::Runnable - : ThreadState::Initialized; + m_thread_state = (type == ThreadType::Main || type == ThreadType::Dummy) + ? ThreadState::Runnable + : ThreadState::Initialized; // Set TLS address. - tls_address = 0; + m_tls_address = 0; // Set parent and condvar tree. - parent = nullptr; - condvar_tree = nullptr; + m_parent = nullptr; + m_condvar_tree = nullptr; // Set sync booleans. - signaled = false; - termination_requested = false; - wait_cancelled = false; - cancellable = false; + m_signaled = false; + m_termination_requested = false; + m_wait_cancelled = false; + m_cancellable = false; // Set core ID and wait result. - core_id = phys_core; - wait_result = ResultNoSynchronizationObject; + m_core_id = phys_core; + m_wait_result = ResultNoSynchronizationObject; // Set priorities. - priority = prio; - base_priority = prio; + m_priority = prio; + m_base_priority = prio; // Initialize sleeping queue. - wait_queue = nullptr; + m_wait_queue = nullptr; // Set suspend flags. - suspend_request_flags = 0; - suspend_allowed_flags = static_cast<u32>(ThreadState::SuspendFlagMask); + m_suspend_request_flags = 0; + m_suspend_allowed_flags = static_cast<u32>(ThreadState::SuspendFlagMask); // We're neither debug attached, nor are we nesting our priority inheritance. - debug_attached = false; - priority_inheritance_count = 0; + m_debug_attached = false; + m_priority_inheritance_count = 0; // We haven't been scheduled, and we have done no light IPC. - schedule_count = -1; - last_scheduled_tick = 0; - light_ipc_data = nullptr; + m_schedule_count = -1; + m_last_scheduled_tick = 0; + m_light_ipc_data = nullptr; // We're not waiting for a lock, and we haven't disabled migration. - lock_owner = nullptr; - num_core_migration_disables = 0; + m_waiting_lock_info = nullptr; + m_num_core_migration_disables = 0; // We have no waiters, but we do have an entrypoint. - num_kernel_waiters = 0; + m_num_kernel_waiters = 0; // Set our current core id. - current_core_id = phys_core; + m_current_core_id = phys_core; // We haven't released our resource limit hint, and we've spent no time on the cpu. - resource_limit_release_hint = false; - cpu_time = 0; + m_resource_limit_release_hint = false; + m_cpu_time = 0; // Set debug context. - stack_top = user_stack_top; - argument = arg; + m_stack_top = user_stack_top; + m_argument = arg; // Clear our stack parameters. - std::memset(static_cast<void*>(std::addressof(GetStackParameters())), 0, + std::memset(static_cast<void*>(std::addressof(this->GetStackParameters())), 0, sizeof(StackParameters)); // Set parent, if relevant. if (owner != nullptr) { // Setup the TLS, if needed. if (type == ThreadType::User) { - R_TRY(owner->CreateThreadLocalRegion(std::addressof(tls_address))); + R_TRY(owner->CreateThreadLocalRegion(std::addressof(m_tls_address))); } - parent = owner; - parent->Open(); + m_parent = owner; + m_parent->Open(); } // Initialize thread context. - ResetThreadContext64(thread_context_64, user_stack_top, func, arg); - ResetThreadContext32(thread_context_32, static_cast<u32>(user_stack_top), - static_cast<u32>(func), static_cast<u32>(arg)); + ResetThreadContext64(m_thread_context_64, GetInteger(user_stack_top), GetInteger(func), arg); + ResetThreadContext32(m_thread_context_32, static_cast<u32>(GetInteger(user_stack_top)), + static_cast<u32>(GetInteger(func)), static_cast<u32>(arg)); // Setup the stack parameters. - StackParameters& sp = GetStackParameters(); + StackParameters& sp = this->GetStackParameters(); sp.cur_thread = this; sp.disable_count = 1; - SetInExceptionHandler(); + this->SetInExceptionHandler(); // Set thread ID. - thread_id = kernel.CreateNewThreadID(); + m_thread_id = m_kernel.CreateNewThreadID(); // We initialized! - initialized = true; + m_initialized = true; // Register ourselves with our parent process. - if (parent != nullptr) { - parent->RegisterThread(this); - if (parent->IsSuspended()) { + if (m_parent != nullptr) { + m_parent->RegisterThread(this); + if (m_parent->IsSuspended()) { RequestSuspend(SuspendType::Process); } } @@ -251,14 +251,14 @@ Result KThread::Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack } Result KThread::InitializeThread(KThread* thread, KThreadFunction func, uintptr_t arg, - VAddr user_stack_top, s32 prio, s32 core, KProcess* owner, - ThreadType type, std::function<void()>&& init_func) { + KProcessAddress user_stack_top, s32 prio, s32 core, + KProcess* owner, ThreadType type, + std::function<void()>&& init_func) { // Initialize the thread. R_TRY(thread->Initialize(func, arg, user_stack_top, prio, core, owner, type)); // Initialize emulation parameters. - thread->host_context = std::make_shared<Common::Fiber>(std::move(init_func)); - thread->is_single_core = !Settings::values.use_multi_core.GetValue(); + thread->m_host_context = std::make_shared<Common::Fiber>(std::move(init_func)); R_SUCCEED(); } @@ -268,7 +268,7 @@ Result KThread::InitializeDummyThread(KThread* thread, KProcess* owner) { R_TRY(thread->Initialize({}, {}, {}, DummyThreadPriority, 3, owner, ThreadType::Dummy)); // Initialize emulation parameters. - thread->stack_parameters.disable_count = 0; + thread->m_stack_parameters.disable_count = 0; R_SUCCEED(); } @@ -291,13 +291,32 @@ Result KThread::InitializeHighPriorityThread(Core::System& system, KThread* thre } Result KThread::InitializeUserThread(Core::System& system, KThread* thread, KThreadFunction func, - uintptr_t arg, VAddr user_stack_top, s32 prio, s32 virt_core, - KProcess* owner) { + uintptr_t arg, KProcessAddress user_stack_top, s32 prio, + s32 virt_core, KProcess* owner) { system.Kernel().GlobalSchedulerContext().AddThread(thread); R_RETURN(InitializeThread(thread, func, arg, user_stack_top, prio, virt_core, owner, ThreadType::User, system.GetCpuManager().GetGuestThreadFunc())); } +Result KThread::InitializeServiceThread(Core::System& system, KThread* thread, + std::function<void()>&& func, s32 prio, s32 virt_core, + KProcess* owner) { + system.Kernel().GlobalSchedulerContext().AddThread(thread); + std::function<void()> func2{[&system, func{std::move(func)}] { + // Similar to UserModeThreadStarter. + system.Kernel().CurrentScheduler()->OnThreadStart(); + + // Run the guest function. + func(); + + // Exit. + Svc::ExitThread(system); + }}; + + R_RETURN(InitializeThread(thread, {}, {}, {}, prio, virt_core, owner, ThreadType::HighPriority, + std::move(func2))); +} + void KThread::PostDestroy(uintptr_t arg) { KProcess* owner = reinterpret_cast<KProcess*>(arg & ~1ULL); const bool resource_limit_release_hint = (arg & 1); @@ -310,96 +329,110 @@ void KThread::PostDestroy(uintptr_t arg) { void KThread::Finalize() { // If the thread has an owner process, unregister it. - if (parent != nullptr) { - parent->UnregisterThread(this); + if (m_parent != nullptr) { + m_parent->UnregisterThread(this); } // If the thread has a local region, delete it. - if (tls_address != 0) { - ASSERT(parent->DeleteThreadLocalRegion(tls_address).IsSuccess()); + if (m_tls_address != 0) { + ASSERT(m_parent->DeleteThreadLocalRegion(m_tls_address).IsSuccess()); } // Release any waiters. { - ASSERT(lock_owner == nullptr); - KScopedSchedulerLock sl{kernel}; + ASSERT(m_waiting_lock_info == nullptr); + KScopedSchedulerLock sl{m_kernel}; + + // Check that we have no kernel waiters. + ASSERT(m_num_kernel_waiters == 0); - auto it = waiter_list.begin(); - while (it != waiter_list.end()) { - // Get the thread. - KThread* const waiter = std::addressof(*it); + auto it = m_held_lock_info_list.begin(); + while (it != m_held_lock_info_list.end()) { + // Get the lock info. + auto* const lock_info = std::addressof(*it); - // The thread shouldn't be a kernel waiter. - ASSERT(!IsKernelAddressKey(waiter->GetAddressKey())); + // The lock shouldn't have a kernel waiter. + ASSERT(!lock_info->GetIsKernelAddressKey()); - // Clear the lock owner. - waiter->SetLockOwner(nullptr); + // Remove all waiters. + while (lock_info->GetWaiterCount() != 0) { + // Get the front waiter. + KThread* const waiter = lock_info->GetHighestPriorityWaiter(); - // Erase the waiter from our list. - it = waiter_list.erase(it); + // Remove it from the lock. + if (lock_info->RemoveWaiter(waiter)) { + ASSERT(lock_info->GetWaiterCount() == 0); + } + + // Cancel the thread's wait. + waiter->CancelWait(ResultInvalidState, true); + } - // Cancel the thread's wait. - waiter->CancelWait(ResultInvalidState, true); + // Remove the held lock from our list. + it = m_held_lock_info_list.erase(it); + + // Free the lock info. + LockWithPriorityInheritanceInfo::Free(m_kernel, lock_info); } } // Release host emulation members. - host_context.reset(); + m_host_context.reset(); // Perform inherited finalization. KSynchronizationObject::Finalize(); } bool KThread::IsSignaled() const { - return signaled; + return m_signaled; } void KThread::OnTimer() { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); // If we're waiting, cancel the wait. - if (GetState() == ThreadState::Waiting) { - wait_queue->CancelWait(this, ResultTimedOut, false); + if (this->GetState() == ThreadState::Waiting) { + m_wait_queue->CancelWait(this, ResultTimedOut, false); } } void KThread::StartTermination() { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); // Release user exception and unpin, if relevant. - if (parent != nullptr) { - parent->ReleaseUserException(this); - if (parent->GetPinnedThread(GetCurrentCoreId(kernel)) == this) { - parent->UnpinCurrentThread(core_id); + if (m_parent != nullptr) { + m_parent->ReleaseUserException(this); + if (m_parent->GetPinnedThread(GetCurrentCoreId(m_kernel)) == this) { + m_parent->UnpinCurrentThread(m_core_id); } } // Set state to terminated. - SetState(ThreadState::Terminated); + this->SetState(ThreadState::Terminated); // Clear the thread's status as running in parent. - if (parent != nullptr) { - parent->ClearRunningThread(this); + if (m_parent != nullptr) { + m_parent->ClearRunningThread(this); } // Signal. - signaled = true; + m_signaled = true; KSynchronizationObject::NotifyAvailable(); // Clear previous thread in KScheduler. - KScheduler::ClearPreviousThread(kernel, this); + KScheduler::ClearPreviousThread(m_kernel, this); // Register terminated dpc flag. - RegisterDpc(DpcFlag::Terminated); + this->RegisterDpc(DpcFlag::Terminated); } void KThread::FinishTermination() { // Ensure that the thread is not executing on any core. - if (parent != nullptr) { + if (m_parent != nullptr) { for (std::size_t i = 0; i < static_cast<std::size_t>(Core::Hardware::NUM_CPU_CORES); ++i) { KThread* core_thread{}; do { - core_thread = kernel.Scheduler(i).GetSchedulerCurrentThread(); + core_thread = m_kernel.Scheduler(i).GetSchedulerCurrentThread(); } while (core_thread == this); } } @@ -414,182 +447,183 @@ void KThread::DoWorkerTaskImpl() { } void KThread::Pin(s32 current_core) { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); // Set ourselves as pinned. GetStackParameters().is_pinned = true; // Disable core migration. - ASSERT(num_core_migration_disables == 0); + ASSERT(m_num_core_migration_disables == 0); { - ++num_core_migration_disables; + ++m_num_core_migration_disables; // Save our ideal state to restore when we're unpinned. - original_physical_ideal_core_id = physical_ideal_core_id; - original_physical_affinity_mask = physical_affinity_mask; + m_original_physical_ideal_core_id = m_physical_ideal_core_id; + m_original_physical_affinity_mask = m_physical_affinity_mask; // Bind ourselves to this core. - const s32 active_core = GetActiveCore(); + const s32 active_core = this->GetActiveCore(); - SetActiveCore(current_core); - physical_ideal_core_id = current_core; - physical_affinity_mask.SetAffinityMask(1ULL << current_core); + this->SetActiveCore(current_core); + m_physical_ideal_core_id = current_core; + m_physical_affinity_mask.SetAffinityMask(1ULL << current_core); - if (active_core != current_core || physical_affinity_mask.GetAffinityMask() != - original_physical_affinity_mask.GetAffinityMask()) { - KScheduler::OnThreadAffinityMaskChanged(kernel, this, original_physical_affinity_mask, - active_core); + if (active_core != current_core || + m_physical_affinity_mask.GetAffinityMask() != + m_original_physical_affinity_mask.GetAffinityMask()) { + KScheduler::OnThreadAffinityMaskChanged(m_kernel, this, + m_original_physical_affinity_mask, active_core); } } // Disallow performing thread suspension. { // Update our allow flags. - suspend_allowed_flags &= ~(1 << (static_cast<u32>(SuspendType::Thread) + - static_cast<u32>(ThreadState::SuspendShift))); + m_suspend_allowed_flags &= ~(1 << (static_cast<u32>(SuspendType::Thread) + + static_cast<u32>(ThreadState::SuspendShift))); // Update our state. - UpdateState(); + this->UpdateState(); } // TODO(bunnei): Update our SVC access permissions. - ASSERT(parent != nullptr); + ASSERT(m_parent != nullptr); } void KThread::Unpin() { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); // Set ourselves as unpinned. - GetStackParameters().is_pinned = false; + this->GetStackParameters().is_pinned = false; // Enable core migration. - ASSERT(num_core_migration_disables == 1); + ASSERT(m_num_core_migration_disables == 1); { - num_core_migration_disables--; + m_num_core_migration_disables--; // Restore our original state. - const KAffinityMask old_mask = physical_affinity_mask; + const KAffinityMask old_mask = m_physical_affinity_mask; - physical_ideal_core_id = original_physical_ideal_core_id; - physical_affinity_mask = original_physical_affinity_mask; + m_physical_ideal_core_id = m_original_physical_ideal_core_id; + m_physical_affinity_mask = m_original_physical_affinity_mask; - if (physical_affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) { - const s32 active_core = GetActiveCore(); + if (m_physical_affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) { + const s32 active_core = this->GetActiveCore(); - if (!physical_affinity_mask.GetAffinity(active_core)) { - if (physical_ideal_core_id >= 0) { - SetActiveCore(physical_ideal_core_id); + if (!m_physical_affinity_mask.GetAffinity(active_core)) { + if (m_physical_ideal_core_id >= 0) { + this->SetActiveCore(m_physical_ideal_core_id); } else { - SetActiveCore(static_cast<s32>( + this->SetActiveCore(static_cast<s32>( Common::BitSize<u64>() - 1 - - std::countl_zero(physical_affinity_mask.GetAffinityMask()))); + std::countl_zero(m_physical_affinity_mask.GetAffinityMask()))); } } - KScheduler::OnThreadAffinityMaskChanged(kernel, this, old_mask, active_core); + KScheduler::OnThreadAffinityMaskChanged(m_kernel, this, old_mask, active_core); } } // Allow performing thread suspension (if termination hasn't been requested). - if (!IsTerminationRequested()) { + if (!this->IsTerminationRequested()) { // Update our allow flags. - suspend_allowed_flags |= (1 << (static_cast<u32>(SuspendType::Thread) + - static_cast<u32>(ThreadState::SuspendShift))); + m_suspend_allowed_flags |= (1 << (static_cast<u32>(SuspendType::Thread) + + static_cast<u32>(ThreadState::SuspendShift))); // Update our state. - UpdateState(); + this->UpdateState(); } // TODO(bunnei): Update our SVC access permissions. - ASSERT(parent != nullptr); + ASSERT(m_parent != nullptr); // Resume any threads that began waiting on us while we were pinned. - for (auto it = pinned_waiter_list.begin(); it != pinned_waiter_list.end(); ++it) { + for (auto it = m_pinned_waiter_list.begin(); it != m_pinned_waiter_list.end(); ++it) { it->EndWait(ResultSuccess); } } u16 KThread::GetUserDisableCount() const { - if (!IsUserThread()) { + if (!this->IsUserThread()) { // We only emulate TLS for user threads return {}; } - auto& memory = kernel.System().Memory(); - return memory.Read16(tls_address + offsetof(ThreadLocalRegion, disable_count)); + auto& memory = this->GetOwnerProcess()->GetMemory(); + return memory.Read16(m_tls_address + offsetof(ThreadLocalRegion, disable_count)); } void KThread::SetInterruptFlag() { - if (!IsUserThread()) { + if (!this->IsUserThread()) { // We only emulate TLS for user threads return; } - auto& memory = kernel.System().Memory(); - memory.Write16(tls_address + offsetof(ThreadLocalRegion, interrupt_flag), 1); + auto& memory = this->GetOwnerProcess()->GetMemory(); + memory.Write16(m_tls_address + offsetof(ThreadLocalRegion, interrupt_flag), 1); } void KThread::ClearInterruptFlag() { - if (!IsUserThread()) { + if (!this->IsUserThread()) { // We only emulate TLS for user threads return; } - auto& memory = kernel.System().Memory(); - memory.Write16(tls_address + offsetof(ThreadLocalRegion, interrupt_flag), 0); + auto& memory = this->GetOwnerProcess()->GetMemory(); + memory.Write16(m_tls_address + offsetof(ThreadLocalRegion, interrupt_flag), 0); } Result KThread::GetCoreMask(s32* out_ideal_core, u64* out_affinity_mask) { - KScopedSchedulerLock sl{kernel}; + KScopedSchedulerLock sl{m_kernel}; // Get the virtual mask. - *out_ideal_core = virtual_ideal_core_id; - *out_affinity_mask = virtual_affinity_mask; + *out_ideal_core = m_virtual_ideal_core_id; + *out_affinity_mask = m_virtual_affinity_mask; R_SUCCEED(); } Result KThread::GetPhysicalCoreMask(s32* out_ideal_core, u64* out_affinity_mask) { - KScopedSchedulerLock sl{kernel}; - ASSERT(num_core_migration_disables >= 0); + KScopedSchedulerLock sl{m_kernel}; + ASSERT(m_num_core_migration_disables >= 0); // Select between core mask and original core mask. - if (num_core_migration_disables == 0) { - *out_ideal_core = physical_ideal_core_id; - *out_affinity_mask = physical_affinity_mask.GetAffinityMask(); + if (m_num_core_migration_disables == 0) { + *out_ideal_core = m_physical_ideal_core_id; + *out_affinity_mask = m_physical_affinity_mask.GetAffinityMask(); } else { - *out_ideal_core = original_physical_ideal_core_id; - *out_affinity_mask = original_physical_affinity_mask.GetAffinityMask(); + *out_ideal_core = m_original_physical_ideal_core_id; + *out_affinity_mask = m_original_physical_affinity_mask.GetAffinityMask(); } R_SUCCEED(); } -Result KThread::SetCoreMask(s32 core_id_, u64 v_affinity_mask) { - ASSERT(parent != nullptr); +Result KThread::SetCoreMask(s32 core_id, u64 v_affinity_mask) { + ASSERT(m_parent != nullptr); ASSERT(v_affinity_mask != 0); - KScopedLightLock lk(activity_pause_lock); + KScopedLightLock lk(m_activity_pause_lock); // Set the core mask. u64 p_affinity_mask = 0; { - KScopedSchedulerLock sl(kernel); - ASSERT(num_core_migration_disables >= 0); + KScopedSchedulerLock sl(m_kernel); + ASSERT(m_num_core_migration_disables >= 0); // If we're updating, set our ideal virtual core. - if (core_id_ != Svc::IdealCoreNoUpdate) { - virtual_ideal_core_id = core_id_; + if (core_id != Svc::IdealCoreNoUpdate) { + m_virtual_ideal_core_id = core_id; } else { // Preserve our ideal core id. - core_id_ = virtual_ideal_core_id; - R_UNLESS(((1ULL << core_id_) & v_affinity_mask) != 0, ResultInvalidCombination); + core_id = m_virtual_ideal_core_id; + R_UNLESS(((1ULL << core_id) & v_affinity_mask) != 0, ResultInvalidCombination); } // Set our affinity mask. - virtual_affinity_mask = v_affinity_mask; + m_virtual_affinity_mask = v_affinity_mask; // Translate the virtual core to a physical core. - if (core_id_ >= 0) { - core_id_ = Core::Hardware::VirtualToPhysicalCoreMap[core_id_]; + if (core_id >= 0) { + core_id = Core::Hardware::VirtualToPhysicalCoreMap[core_id]; } // Translate the virtual affinity mask to a physical one. @@ -600,43 +634,43 @@ Result KThread::SetCoreMask(s32 core_id_, u64 v_affinity_mask) { } // If we haven't disabled migration, perform an affinity change. - if (num_core_migration_disables == 0) { - const KAffinityMask old_mask = physical_affinity_mask; + if (m_num_core_migration_disables == 0) { + const KAffinityMask old_mask = m_physical_affinity_mask; // Set our new ideals. - physical_ideal_core_id = core_id_; - physical_affinity_mask.SetAffinityMask(p_affinity_mask); + m_physical_ideal_core_id = core_id; + m_physical_affinity_mask.SetAffinityMask(p_affinity_mask); - if (physical_affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) { + if (m_physical_affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) { const s32 active_core = GetActiveCore(); - if (active_core >= 0 && !physical_affinity_mask.GetAffinity(active_core)) { + if (active_core >= 0 && !m_physical_affinity_mask.GetAffinity(active_core)) { const s32 new_core = static_cast<s32>( - physical_ideal_core_id >= 0 - ? physical_ideal_core_id + m_physical_ideal_core_id >= 0 + ? m_physical_ideal_core_id : Common::BitSize<u64>() - 1 - - std::countl_zero(physical_affinity_mask.GetAffinityMask())); + std::countl_zero(m_physical_affinity_mask.GetAffinityMask())); SetActiveCore(new_core); } - KScheduler::OnThreadAffinityMaskChanged(kernel, this, old_mask, active_core); + KScheduler::OnThreadAffinityMaskChanged(m_kernel, this, old_mask, active_core); } } else { // Otherwise, we edit the original affinity for restoration later. - original_physical_ideal_core_id = core_id_; - original_physical_affinity_mask.SetAffinityMask(p_affinity_mask); + m_original_physical_ideal_core_id = core_id; + m_original_physical_affinity_mask.SetAffinityMask(p_affinity_mask); } } // Update the pinned waiter list. - ThreadQueueImplForKThreadSetProperty wait_queue_(kernel, std::addressof(pinned_waiter_list)); + ThreadQueueImplForKThreadSetProperty wait_queue(m_kernel, std::addressof(m_pinned_waiter_list)); { bool retry_update{}; do { // Lock the scheduler. - KScopedSchedulerLock sl(kernel); + KScopedSchedulerLock sl(m_kernel); // Don't do any further management if our termination has been requested. - R_SUCCEED_IF(IsTerminationRequested()); + R_SUCCEED_IF(this->IsTerminationRequested()); // By default, we won't need to retry. retry_update = false; @@ -646,7 +680,7 @@ Result KThread::SetCoreMask(s32 core_id_, u64 v_affinity_mask) { s32 thread_core; for (thread_core = 0; thread_core < static_cast<s32>(Core::Hardware::NUM_CPU_CORES); ++thread_core) { - if (kernel.Scheduler(thread_core).GetSchedulerCurrentThread() == this) { + if (m_kernel.Scheduler(thread_core).GetSchedulerCurrentThread() == this) { thread_is_current = true; break; } @@ -656,14 +690,14 @@ Result KThread::SetCoreMask(s32 core_id_, u64 v_affinity_mask) { // new mask. if (thread_is_current && ((1ULL << thread_core) & p_affinity_mask) == 0) { // If the thread is pinned, we want to wait until it's not pinned. - if (GetStackParameters().is_pinned) { + if (this->GetStackParameters().is_pinned) { // Verify that the current thread isn't terminating. - R_UNLESS(!GetCurrentThread(kernel).IsTerminationRequested(), + R_UNLESS(!GetCurrentThread(m_kernel).IsTerminationRequested(), ResultTerminationRequested); // Wait until the thread isn't pinned any more. - pinned_waiter_list.push_back(GetCurrentThread(kernel)); - GetCurrentThread(kernel).BeginWait(std::addressof(wait_queue_)); + m_pinned_waiter_list.push_back(GetCurrentThread(m_kernel)); + GetCurrentThread(m_kernel).BeginWait(std::addressof(wait_queue)); } else { // If the thread isn't pinned, release the scheduler lock and retry until it's // not current. @@ -679,111 +713,137 @@ Result KThread::SetCoreMask(s32 core_id_, u64 v_affinity_mask) { void KThread::SetBasePriority(s32 value) { ASSERT(Svc::HighestThreadPriority <= value && value <= Svc::LowestThreadPriority); - KScopedSchedulerLock sl{kernel}; + KScopedSchedulerLock sl{m_kernel}; // Change our base priority. - base_priority = value; + m_base_priority = value; // Perform a priority restoration. - RestorePriority(kernel, this); + RestorePriority(m_kernel, this); +} + +KThread* KThread::GetLockOwner() const { + return m_waiting_lock_info != nullptr ? m_waiting_lock_info->GetOwner() : nullptr; +} + +void KThread::IncreaseBasePriority(s32 priority) { + ASSERT(Svc::HighestThreadPriority <= priority && priority <= Svc::LowestThreadPriority); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); + ASSERT(!this->GetStackParameters().is_pinned); + + // Set our base priority. + if (m_base_priority > priority) { + m_base_priority = priority; + + // Perform a priority restoration. + RestorePriority(m_kernel, this); + } } void KThread::RequestSuspend(SuspendType type) { - KScopedSchedulerLock sl{kernel}; + KScopedSchedulerLock sl{m_kernel}; // Note the request in our flags. - suspend_request_flags |= - (1u << (static_cast<u32>(ThreadState::SuspendShift) + static_cast<u32>(type))); + m_suspend_request_flags |= + (1U << (static_cast<u32>(ThreadState::SuspendShift) + static_cast<u32>(type))); // Try to perform the suspend. - TrySuspend(); + this->TrySuspend(); } void KThread::Resume(SuspendType type) { - KScopedSchedulerLock sl{kernel}; + KScopedSchedulerLock sl{m_kernel}; // Clear the request in our flags. - suspend_request_flags &= - ~(1u << (static_cast<u32>(ThreadState::SuspendShift) + static_cast<u32>(type))); + m_suspend_request_flags &= + ~(1U << (static_cast<u32>(ThreadState::SuspendShift) + static_cast<u32>(type))); // Update our state. this->UpdateState(); } void KThread::WaitCancel() { - KScopedSchedulerLock sl{kernel}; + KScopedSchedulerLock sl{m_kernel}; // Check if we're waiting and cancellable. - if (this->GetState() == ThreadState::Waiting && cancellable) { - wait_cancelled = false; - wait_queue->CancelWait(this, ResultCancelled, true); + if (this->GetState() == ThreadState::Waiting && m_cancellable) { + m_wait_cancelled = false; + m_wait_queue->CancelWait(this, ResultCancelled, true); } else { // Otherwise, note that we cancelled a wait. - wait_cancelled = true; + m_wait_cancelled = true; } } void KThread::TrySuspend() { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); - ASSERT(IsSuspendRequested()); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); + ASSERT(this->IsSuspendRequested()); // Ensure that we have no waiters. - if (GetNumKernelWaiters() > 0) { + if (this->GetNumKernelWaiters() > 0) { return; } - ASSERT(GetNumKernelWaiters() == 0); + ASSERT(this->GetNumKernelWaiters() == 0); // Perform the suspend. this->UpdateState(); } void KThread::UpdateState() { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); // Set our suspend flags in state. - const ThreadState old_state = thread_state.load(std::memory_order_relaxed); + const ThreadState old_state = m_thread_state.load(std::memory_order_relaxed); const auto new_state = static_cast<ThreadState>(this->GetSuspendFlags()) | (old_state & ThreadState::Mask); - thread_state.store(new_state, std::memory_order_relaxed); + m_thread_state.store(new_state, std::memory_order_relaxed); // Note the state change in scheduler. if (new_state != old_state) { - KScheduler::OnThreadStateChanged(kernel, this, old_state); + KScheduler::OnThreadStateChanged(m_kernel, this, old_state); } } void KThread::Continue() { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); // Clear our suspend flags in state. - const ThreadState old_state = thread_state.load(std::memory_order_relaxed); - thread_state.store(old_state & ThreadState::Mask, std::memory_order_relaxed); + const ThreadState old_state = m_thread_state.load(std::memory_order_relaxed); + m_thread_state.store(old_state & ThreadState::Mask, std::memory_order_relaxed); // Note the state change in scheduler. - KScheduler::OnThreadStateChanged(kernel, this, old_state); + KScheduler::OnThreadStateChanged(m_kernel, this, old_state); } -void KThread::WaitUntilSuspended() { - // Make sure we have a suspend requested. - ASSERT(IsSuspendRequested()); +void KThread::CloneFpuStatus() { + // We shouldn't reach here when starting kernel threads. + ASSERT(this->GetOwnerProcess() != nullptr); + ASSERT(this->GetOwnerProcess() == GetCurrentProcessPointer(m_kernel)); - // Loop until the thread is not executing on any core. - for (std::size_t i = 0; i < static_cast<std::size_t>(Core::Hardware::NUM_CPU_CORES); ++i) { - KThread* core_thread{}; - do { - core_thread = kernel.Scheduler(i).GetSchedulerCurrentThread(); - } while (core_thread == this); + if (this->GetOwnerProcess()->Is64BitProcess()) { + // Clone FPSR and FPCR. + ThreadContext64 cur_ctx{}; + m_kernel.System().CurrentArmInterface().SaveContext(cur_ctx); + + this->GetContext64().fpcr = cur_ctx.fpcr; + this->GetContext64().fpsr = cur_ctx.fpsr; + } else { + // Clone FPSCR. + ThreadContext32 cur_ctx{}; + m_kernel.System().CurrentArmInterface().SaveContext(cur_ctx); + + this->GetContext32().fpscr = cur_ctx.fpscr; } } Result KThread::SetActivity(Svc::ThreadActivity activity) { // Lock ourselves. - KScopedLightLock lk(activity_pause_lock); + KScopedLightLock lk(m_activity_pause_lock); // Set the activity. { // Lock the scheduler. - KScopedSchedulerLock sl(kernel); + KScopedSchedulerLock sl(m_kernel); // Verify our state. const auto cur_state = this->GetState(); @@ -810,13 +870,13 @@ Result KThread::SetActivity(Svc::ThreadActivity activity) { // If the thread is now paused, update the pinned waiter list. if (activity == Svc::ThreadActivity::Paused) { - ThreadQueueImplForKThreadSetProperty wait_queue_(kernel, - std::addressof(pinned_waiter_list)); + ThreadQueueImplForKThreadSetProperty wait_queue(m_kernel, + std::addressof(m_pinned_waiter_list)); - bool thread_is_current; + bool thread_is_current{}; do { // Lock the scheduler. - KScopedSchedulerLock sl(kernel); + KScopedSchedulerLock sl(m_kernel); // Don't do any further management if our termination has been requested. R_SUCCEED_IF(this->IsTerminationRequested()); @@ -827,17 +887,17 @@ Result KThread::SetActivity(Svc::ThreadActivity activity) { // Check whether the thread is pinned. if (this->GetStackParameters().is_pinned) { // Verify that the current thread isn't terminating. - R_UNLESS(!GetCurrentThread(kernel).IsTerminationRequested(), + R_UNLESS(!GetCurrentThread(m_kernel).IsTerminationRequested(), ResultTerminationRequested); // Wait until the thread isn't pinned any more. - pinned_waiter_list.push_back(GetCurrentThread(kernel)); - GetCurrentThread(kernel).BeginWait(std::addressof(wait_queue_)); + m_pinned_waiter_list.push_back(GetCurrentThread(m_kernel)); + GetCurrentThread(m_kernel).BeginWait(std::addressof(wait_queue)); } else { // Check if the thread is currently running. // If it is, we'll need to retry. for (auto i = 0; i < static_cast<s32>(Core::Hardware::NUM_CPU_CORES); ++i) { - if (kernel.Scheduler(i).GetSchedulerCurrentThread() == this) { + if (m_kernel.Scheduler(i).GetSchedulerCurrentThread() == this) { thread_is_current = true; break; } @@ -849,34 +909,32 @@ Result KThread::SetActivity(Svc::ThreadActivity activity) { R_SUCCEED(); } -Result KThread::GetThreadContext3(std::vector<u8>& out) { +Result KThread::GetThreadContext3(Common::ScratchBuffer<u8>& out) { // Lock ourselves. - KScopedLightLock lk{activity_pause_lock}; + KScopedLightLock lk{m_activity_pause_lock}; // Get the context. { // Lock the scheduler. - KScopedSchedulerLock sl{kernel}; + KScopedSchedulerLock sl{m_kernel}; // Verify that we're suspended. - R_UNLESS(IsSuspendRequested(SuspendType::Thread), ResultInvalidState); + R_UNLESS(this->IsSuspendRequested(SuspendType::Thread), ResultInvalidState); // If we're not terminating, get the thread's user context. - if (!IsTerminationRequested()) { - if (parent->Is64BitProcess()) { + if (!this->IsTerminationRequested()) { + if (m_parent->Is64BitProcess()) { // Mask away mode bits, interrupt bits, IL bit, and other reserved bits. auto context = GetContext64(); context.pstate &= 0xFF0FFE20; - - out.resize(sizeof(context)); - std::memcpy(out.data(), &context, sizeof(context)); + out.resize_destructive(sizeof(context)); + std::memcpy(out.data(), std::addressof(context), sizeof(context)); } else { // Mask away mode bits, interrupt bits, IL bit, and other reserved bits. auto context = GetContext32(); context.cpsr &= 0xFF0FFE20; - - out.resize(sizeof(context)); - std::memcpy(out.data(), &context, sizeof(context)); + out.resize_destructive(sizeof(context)); + std::memcpy(out.data(), std::addressof(context), sizeof(context)); } } } @@ -884,51 +942,89 @@ Result KThread::GetThreadContext3(std::vector<u8>& out) { R_SUCCEED(); } -void KThread::AddWaiterImpl(KThread* thread) { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); +void KThread::AddHeldLock(LockWithPriorityInheritanceInfo* lock_info) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); + + // Set ourselves as the lock's owner. + lock_info->SetOwner(this); + + // Add the lock to our held list. + m_held_lock_info_list.push_front(*lock_info); +} + +KThread::LockWithPriorityInheritanceInfo* KThread::FindHeldLock(KProcessAddress address_key, + bool is_kernel_address_key) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); - // Find the right spot to insert the waiter. - auto it = waiter_list.begin(); - while (it != waiter_list.end()) { - if (it->GetPriority() > thread->GetPriority()) { - break; + // Try to find an existing held lock. + for (auto& held_lock : m_held_lock_info_list) { + if (held_lock.GetAddressKey() == address_key && + held_lock.GetIsKernelAddressKey() == is_kernel_address_key) { + return std::addressof(held_lock); } - it++; } + return nullptr; +} + +void KThread::AddWaiterImpl(KThread* thread) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); + ASSERT(thread->GetConditionVariableTree() == nullptr); + + // Get the thread's address key. + const auto address_key = thread->GetAddressKey(); + const auto is_kernel_address_key = thread->GetIsKernelAddressKey(); + // Keep track of how many kernel waiters we have. - if (IsKernelAddressKey(thread->GetAddressKey())) { - ASSERT((num_kernel_waiters++) >= 0); - KScheduler::SetSchedulerUpdateNeeded(kernel); + if (is_kernel_address_key) { + ASSERT((m_num_kernel_waiters++) >= 0); + KScheduler::SetSchedulerUpdateNeeded(m_kernel); + } + + // Get the relevant lock info. + auto* lock_info = this->FindHeldLock(address_key, is_kernel_address_key); + if (lock_info == nullptr) { + // Create a new lock for the address key. + lock_info = + LockWithPriorityInheritanceInfo::Create(m_kernel, address_key, is_kernel_address_key); + + // Add the new lock to our list. + this->AddHeldLock(lock_info); } - // Insert the waiter. - waiter_list.insert(it, *thread); - thread->SetLockOwner(this); + // Add the thread as waiter to the lock info. + lock_info->AddWaiter(thread); } void KThread::RemoveWaiterImpl(KThread* thread) { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); // Keep track of how many kernel waiters we have. - if (IsKernelAddressKey(thread->GetAddressKey())) { - ASSERT((num_kernel_waiters--) > 0); - KScheduler::SetSchedulerUpdateNeeded(kernel); + if (thread->GetIsKernelAddressKey()) { + ASSERT((m_num_kernel_waiters--) > 0); + KScheduler::SetSchedulerUpdateNeeded(m_kernel); } + // Get the info for the lock the thread is waiting on. + auto* lock_info = thread->GetWaitingLockInfo(); + ASSERT(lock_info->GetOwner() == this); + // Remove the waiter. - waiter_list.erase(waiter_list.iterator_to(*thread)); - thread->SetLockOwner(nullptr); + if (lock_info->RemoveWaiter(thread)) { + m_held_lock_info_list.erase(m_held_lock_info_list.iterator_to(*lock_info)); + LockWithPriorityInheritanceInfo::Free(m_kernel, lock_info); + } } -void KThread::RestorePriority(KernelCore& kernel_ctx, KThread* thread) { - ASSERT(kernel_ctx.GlobalSchedulerContext().IsLocked()); +void KThread::RestorePriority(KernelCore& kernel, KThread* thread) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); - while (true) { + while (thread != nullptr) { // We want to inherit priority where possible. s32 new_priority = thread->GetBasePriority(); - if (thread->HasWaiters()) { - new_priority = std::min(new_priority, thread->waiter_list.front().GetPriority()); + for (const auto& held_lock : thread->m_held_lock_info_list) { + new_priority = + std::min(new_priority, held_lock.GetHighestPriorityWaiter()->GetPriority()); } // If the priority we would inherit is not different from ours, don't do anything. @@ -936,9 +1032,18 @@ void KThread::RestorePriority(KernelCore& kernel_ctx, KThread* thread) { return; } + // Get the owner of whatever lock this thread is waiting on. + KThread* const lock_owner = thread->GetLockOwner(); + + // If the thread is waiting on some lock, remove it as a waiter to prevent violating red + // black tree invariants. + if (lock_owner != nullptr) { + lock_owner->RemoveWaiterImpl(thread); + } + // Ensure we don't violate condition variable red black tree invariants. if (auto* cv_tree = thread->GetConditionVariableTree(); cv_tree != nullptr) { - BeforeUpdatePriority(kernel_ctx, cv_tree, thread); + BeforeUpdatePriority(kernel, cv_tree, thread); } // Change the priority. @@ -947,148 +1052,175 @@ void KThread::RestorePriority(KernelCore& kernel_ctx, KThread* thread) { // Restore the condition variable, if relevant. if (auto* cv_tree = thread->GetConditionVariableTree(); cv_tree != nullptr) { - AfterUpdatePriority(kernel_ctx, cv_tree, thread); + AfterUpdatePriority(kernel, cv_tree, thread); } - // Update the scheduler. - KScheduler::OnThreadPriorityChanged(kernel_ctx, thread, old_priority); - - // Keep the lock owner up to date. - KThread* lock_owner = thread->GetLockOwner(); - if (lock_owner == nullptr) { - return; + // If we removed the thread from some lock's waiting list, add it back. + if (lock_owner != nullptr) { + lock_owner->AddWaiterImpl(thread); } - // Update the thread in the lock owner's sorted list, and continue inheriting. - lock_owner->RemoveWaiterImpl(thread); - lock_owner->AddWaiterImpl(thread); + // Update the scheduler. + KScheduler::OnThreadPriorityChanged(kernel, thread, old_priority); + + // Continue inheriting priority. thread = lock_owner; } } void KThread::AddWaiter(KThread* thread) { - AddWaiterImpl(thread); - RestorePriority(kernel, this); + this->AddWaiterImpl(thread); + + // If the thread has a higher priority than us, we should inherit. + if (thread->GetPriority() < this->GetPriority()) { + RestorePriority(m_kernel, this); + } } void KThread::RemoveWaiter(KThread* thread) { - RemoveWaiterImpl(thread); - RestorePriority(kernel, this); + this->RemoveWaiterImpl(thread); + + // If our priority is the same as the thread's (and we've inherited), we may need to restore to + // lower priority. + if (this->GetPriority() == thread->GetPriority() && + this->GetPriority() < this->GetBasePriority()) { + RestorePriority(m_kernel, this); + } } -KThread* KThread::RemoveWaiterByKey(s32* out_num_waiters, VAddr key) { - ASSERT(kernel.GlobalSchedulerContext().IsLocked()); +KThread* KThread::RemoveWaiterByKey(bool* out_has_waiters, KProcessAddress key, + bool is_kernel_address_key_) { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); - s32 num_waiters{}; - KThread* next_lock_owner{}; - auto it = waiter_list.begin(); - while (it != waiter_list.end()) { - if (it->GetAddressKey() == key) { - KThread* thread = std::addressof(*it); + // Get the relevant lock info. + auto* lock_info = this->FindHeldLock(key, is_kernel_address_key_); + if (lock_info == nullptr) { + *out_has_waiters = false; + return nullptr; + } - // Keep track of how many kernel waiters we have. - if (IsKernelAddressKey(thread->GetAddressKey())) { - ASSERT((num_kernel_waiters--) > 0); - KScheduler::SetSchedulerUpdateNeeded(kernel); - } - it = waiter_list.erase(it); + // Remove the lock info from our held list. + m_held_lock_info_list.erase(m_held_lock_info_list.iterator_to(*lock_info)); - // Update the next lock owner. - if (next_lock_owner == nullptr) { - next_lock_owner = thread; - next_lock_owner->SetLockOwner(nullptr); - } else { - next_lock_owner->AddWaiterImpl(thread); - } - num_waiters++; - } else { - it++; + // Keep track of how many kernel waiters we have. + if (lock_info->GetIsKernelAddressKey()) { + m_num_kernel_waiters -= lock_info->GetWaiterCount(); + ASSERT(m_num_kernel_waiters >= 0); + KScheduler::SetSchedulerUpdateNeeded(m_kernel); + } + + ASSERT(lock_info->GetWaiterCount() > 0); + + // Remove the highest priority waiter from the lock to be the next owner. + KThread* next_lock_owner = lock_info->GetHighestPriorityWaiter(); + if (lock_info->RemoveWaiter(next_lock_owner)) { + // The new owner was the only waiter. + *out_has_waiters = false; + + // Free the lock info, since it has no waiters. + LockWithPriorityInheritanceInfo::Free(m_kernel, lock_info); + } else { + // There are additional waiters on the lock. + *out_has_waiters = true; + + // Add the lock to the new owner's held list. + next_lock_owner->AddHeldLock(lock_info); + + // Keep track of any kernel waiters for the new owner. + if (lock_info->GetIsKernelAddressKey()) { + next_lock_owner->m_num_kernel_waiters += lock_info->GetWaiterCount(); + ASSERT(next_lock_owner->m_num_kernel_waiters > 0); + + // NOTE: No need to set scheduler update needed, because we will have already done so + // when removing earlier. } } - // Do priority updates, if we have a next owner. - if (next_lock_owner) { - RestorePriority(kernel, this); - RestorePriority(kernel, next_lock_owner); + // If our priority is the same as the next owner's (and we've inherited), we may need to restore + // to lower priority. + if (this->GetPriority() == next_lock_owner->GetPriority() && + this->GetPriority() < this->GetBasePriority()) { + RestorePriority(m_kernel, this); + // NOTE: No need to restore priority on the next lock owner, because it was already the + // highest priority waiter on the lock. } - // Return output. - *out_num_waiters = num_waiters; + // Return the next lock owner. return next_lock_owner; } Result KThread::Run() { while (true) { - KScopedSchedulerLock lk{kernel}; + KScopedSchedulerLock lk{m_kernel}; // If either this thread or the current thread are requesting termination, note it. - R_UNLESS(!IsTerminationRequested(), ResultTerminationRequested); - R_UNLESS(!GetCurrentThread(kernel).IsTerminationRequested(), ResultTerminationRequested); + R_UNLESS(!this->IsTerminationRequested(), ResultTerminationRequested); + R_UNLESS(!GetCurrentThread(m_kernel).IsTerminationRequested(), ResultTerminationRequested); // Ensure our thread state is correct. - R_UNLESS(GetState() == ThreadState::Initialized, ResultInvalidState); + R_UNLESS(this->GetState() == ThreadState::Initialized, ResultInvalidState); // If the current thread has been asked to suspend, suspend it and retry. - if (GetCurrentThread(kernel).IsSuspended()) { - GetCurrentThread(kernel).UpdateState(); + if (GetCurrentThread(m_kernel).IsSuspended()) { + GetCurrentThread(m_kernel).UpdateState(); continue; } // If we're not a kernel thread and we've been asked to suspend, suspend ourselves. if (KProcess* owner = this->GetOwnerProcess(); owner != nullptr) { - if (IsUserThread() && IsSuspended()) { + if (this->IsUserThread() && this->IsSuspended()) { this->UpdateState(); } owner->IncrementRunningThreadCount(); } // Set our state and finish. - SetState(ThreadState::Runnable); + this->SetState(ThreadState::Runnable); R_SUCCEED(); } } void KThread::Exit() { - ASSERT(this == GetCurrentThreadPointer(kernel)); + ASSERT(this == GetCurrentThreadPointer(m_kernel)); // Release the thread resource hint, running thread count from parent. - if (parent != nullptr) { - parent->GetResourceLimit()->Release(Kernel::LimitableResource::ThreadCountMax, 0, 1); - resource_limit_release_hint = true; - parent->DecrementRunningThreadCount(); + if (m_parent != nullptr) { + m_parent->GetResourceLimit()->Release(Kernel::LimitableResource::ThreadCountMax, 0, 1); + m_resource_limit_release_hint = true; + m_parent->DecrementRunningThreadCount(); } // Perform termination. { - KScopedSchedulerLock sl{kernel}; + KScopedSchedulerLock sl{m_kernel}; // Disallow all suspension. - suspend_allowed_flags = 0; + m_suspend_allowed_flags = 0; this->UpdateState(); // Disallow all suspension. - suspend_allowed_flags = 0; + m_suspend_allowed_flags = 0; // Start termination. - StartTermination(); + this->StartTermination(); // Register the thread as a work task. - KWorkerTaskManager::AddTask(kernel, KWorkerTaskManager::WorkerType::Exit, this); + KWorkerTaskManager::AddTask(m_kernel, KWorkerTaskManager::WorkerType::Exit, this); } UNREACHABLE_MSG("KThread::Exit() would return"); } Result KThread::Terminate() { - ASSERT(this != GetCurrentThreadPointer(kernel)); + ASSERT(this != GetCurrentThreadPointer(m_kernel)); // Request the thread terminate if it hasn't already. if (const auto new_state = this->RequestTerminate(); new_state != ThreadState::Terminated) { // If the thread isn't terminated, wait for it to terminate. s32 index; KSynchronizationObject* objects[] = {this}; - R_TRY(KSynchronizationObject::Wait(kernel, std::addressof(index), objects, 1, + R_TRY(KSynchronizationObject::Wait(m_kernel, std::addressof(index), objects, 1, Svc::WaitInfinite)); } @@ -1096,22 +1228,22 @@ Result KThread::Terminate() { } ThreadState KThread::RequestTerminate() { - ASSERT(this != GetCurrentThreadPointer(kernel)); + ASSERT(this != GetCurrentThreadPointer(m_kernel)); - KScopedSchedulerLock sl{kernel}; + KScopedSchedulerLock sl{m_kernel}; // Determine if this is the first termination request. const bool first_request = [&]() -> bool { // Perform an atomic compare-and-swap from false to true. bool expected = false; - return termination_requested.compare_exchange_strong(expected, true); + return m_termination_requested.compare_exchange_strong(expected, true); }(); // If this is the first request, start termination procedure. if (first_request) { // If the thread is in initialized state, just change state to terminated. if (this->GetState() == ThreadState::Initialized) { - thread_state = ThreadState::Terminated; + m_thread_state = ThreadState::Terminated; return ThreadState::Terminated; } @@ -1125,27 +1257,25 @@ ThreadState KThread::RequestTerminate() { // If the thread is suspended, continue it. if (this->IsSuspended()) { - suspend_allowed_flags = 0; + m_suspend_allowed_flags = 0; this->UpdateState(); } // Change the thread's priority to be higher than any system thread's. - if (this->GetBasePriority() >= Svc::SystemThreadPriorityHighest) { - this->SetBasePriority(TerminatingThreadPriority); - } + this->IncreaseBasePriority(TerminatingThreadPriority); // If the thread is runnable, send a termination interrupt to other cores. if (this->GetState() == ThreadState::Runnable) { - if (const u64 core_mask = - physical_affinity_mask.GetAffinityMask() & ~(1ULL << GetCurrentCoreId(kernel)); + if (const u64 core_mask = m_physical_affinity_mask.GetAffinityMask() & + ~(1ULL << GetCurrentCoreId(m_kernel)); core_mask != 0) { - Kernel::KInterruptManager::SendInterProcessorInterrupt(kernel, core_mask); + Kernel::KInterruptManager::SendInterProcessorInterrupt(m_kernel, core_mask); } } // Wake up the thread. if (this->GetState() == ThreadState::Waiting) { - wait_queue->CancelWait(this, ResultTerminationRequested, true); + m_wait_queue->CancelWait(this, ResultTerminationRequested, true); } } @@ -1153,14 +1283,15 @@ ThreadState KThread::RequestTerminate() { } Result KThread::Sleep(s64 timeout) { - ASSERT(!kernel.GlobalSchedulerContext().IsLocked()); - ASSERT(this == GetCurrentThreadPointer(kernel)); + ASSERT(!KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); + ASSERT(this == GetCurrentThreadPointer(m_kernel)); ASSERT(timeout > 0); - ThreadQueueImplForKThreadSleep wait_queue_(kernel); + ThreadQueueImplForKThreadSleep wait_queue(m_kernel); + KHardwareTimer* timer{}; { // Setup the scheduling lock and sleep. - KScopedSchedulerLockAndSleep slp(kernel, this, timeout); + KScopedSchedulerLockAndSleep slp(m_kernel, std::addressof(timer), this, timeout); // Check if the thread should terminate. if (this->IsTerminationRequested()) { @@ -1169,102 +1300,107 @@ Result KThread::Sleep(s64 timeout) { } // Wait for the sleep to end. - this->BeginWait(std::addressof(wait_queue_)); - SetWaitReasonForDebugging(ThreadWaitReasonForDebugging::Sleep); + wait_queue.SetHardwareTimer(timer); + this->BeginWait(std::addressof(wait_queue)); + this->SetWaitReasonForDebugging(ThreadWaitReasonForDebugging::Sleep); } R_SUCCEED(); } void KThread::RequestDummyThreadWait() { - ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); ASSERT(this->IsDummyThread()); // We will block when the scheduler lock is released. - dummy_thread_runnable.store(false); + std::scoped_lock lock{m_dummy_thread_mutex}; + m_dummy_thread_runnable = false; } void KThread::DummyThreadBeginWait() { - if (!this->IsDummyThread() || kernel.IsPhantomModeForSingleCore()) { + if (!this->IsDummyThread() || m_kernel.IsPhantomModeForSingleCore()) { // Occurs in single core mode. return; } // Block until runnable is no longer false. - dummy_thread_runnable.wait(false); + std::unique_lock lock{m_dummy_thread_mutex}; + m_dummy_thread_cv.wait(lock, [this] { return m_dummy_thread_runnable; }); } void KThread::DummyThreadEndWait() { - ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); ASSERT(this->IsDummyThread()); // Wake up the waiting thread. - dummy_thread_runnable.store(true); - dummy_thread_runnable.notify_one(); + { + std::scoped_lock lock{m_dummy_thread_mutex}; + m_dummy_thread_runnable = true; + } + m_dummy_thread_cv.notify_one(); } void KThread::BeginWait(KThreadQueue* queue) { // Set our state as waiting. - SetState(ThreadState::Waiting); + this->SetState(ThreadState::Waiting); // Set our wait queue. - wait_queue = queue; + m_wait_queue = queue; } -void KThread::NotifyAvailable(KSynchronizationObject* signaled_object, Result wait_result_) { +void KThread::NotifyAvailable(KSynchronizationObject* signaled_object, Result wait_result) { // Lock the scheduler. - KScopedSchedulerLock sl(kernel); + KScopedSchedulerLock sl(m_kernel); // If we're waiting, notify our queue that we're available. - if (GetState() == ThreadState::Waiting) { - wait_queue->NotifyAvailable(this, signaled_object, wait_result_); + if (this->GetState() == ThreadState::Waiting) { + m_wait_queue->NotifyAvailable(this, signaled_object, wait_result); } } -void KThread::EndWait(Result wait_result_) { +void KThread::EndWait(Result wait_result) { // Lock the scheduler. - KScopedSchedulerLock sl(kernel); + KScopedSchedulerLock sl(m_kernel); // If we're waiting, notify our queue that we're available. - if (GetState() == ThreadState::Waiting) { - if (wait_queue == nullptr) { + if (this->GetState() == ThreadState::Waiting) { + if (m_wait_queue == nullptr) { // This should never happen, but avoid a hard crash below to get this logged. ASSERT_MSG(false, "wait_queue is nullptr!"); return; } - wait_queue->EndWait(this, wait_result_); + m_wait_queue->EndWait(this, wait_result); } } -void KThread::CancelWait(Result wait_result_, bool cancel_timer_task) { +void KThread::CancelWait(Result wait_result, bool cancel_timer_task) { // Lock the scheduler. - KScopedSchedulerLock sl(kernel); + KScopedSchedulerLock sl(m_kernel); // If we're waiting, notify our queue that we're available. - if (GetState() == ThreadState::Waiting) { - wait_queue->CancelWait(this, wait_result_, cancel_timer_task); + if (this->GetState() == ThreadState::Waiting) { + m_wait_queue->CancelWait(this, wait_result, cancel_timer_task); } } void KThread::SetState(ThreadState state) { - KScopedSchedulerLock sl{kernel}; + KScopedSchedulerLock sl{m_kernel}; // Clear debugging state - SetMutexWaitAddressForDebugging({}); - SetWaitReasonForDebugging({}); + this->SetWaitReasonForDebugging({}); - const ThreadState old_state = thread_state.load(std::memory_order_relaxed); - thread_state.store( + const ThreadState old_state = m_thread_state.load(std::memory_order_relaxed); + m_thread_state.store( static_cast<ThreadState>((old_state & ~ThreadState::Mask) | (state & ThreadState::Mask)), std::memory_order_relaxed); - if (thread_state.load(std::memory_order_relaxed) != old_state) { - KScheduler::OnThreadStateChanged(kernel, this, old_state); + if (m_thread_state.load(std::memory_order_relaxed) != old_state) { + KScheduler::OnThreadStateChanged(m_kernel, this, old_state); } } std::shared_ptr<Common::Fiber>& KThread::GetHostContext() { - return host_context; + return m_host_context; } void SetCurrentThread(KernelCore& kernel, KThread* thread) { @@ -1279,26 +1415,39 @@ KThread& GetCurrentThread(KernelCore& kernel) { return *GetCurrentThreadPointer(kernel); } +KProcess* GetCurrentProcessPointer(KernelCore& kernel) { + return GetCurrentThread(kernel).GetOwnerProcess(); +} + +KProcess& GetCurrentProcess(KernelCore& kernel) { + return *GetCurrentProcessPointer(kernel); +} + s32 GetCurrentCoreId(KernelCore& kernel) { return GetCurrentThread(kernel).GetCurrentCore(); } +Core::Memory::Memory& GetCurrentMemory(KernelCore& kernel) { + // TODO: per-process memory + return kernel.System().ApplicationMemory(); +} + KScopedDisableDispatch::~KScopedDisableDispatch() { // If we are shutting down the kernel, none of this is relevant anymore. - if (kernel.IsShuttingDown()) { + if (m_kernel.IsShuttingDown()) { return; } - if (GetCurrentThread(kernel).GetDisableDispatchCount() <= 1) { - auto* scheduler = kernel.CurrentScheduler(); + if (GetCurrentThread(m_kernel).GetDisableDispatchCount() <= 1) { + auto* scheduler = m_kernel.CurrentScheduler(); - if (scheduler && !kernel.IsPhantomModeForSingleCore()) { + if (scheduler && !m_kernel.IsPhantomModeForSingleCore()) { scheduler->RescheduleCurrentCore(); } else { - KScheduler::RescheduleCurrentHLEThread(kernel); + KScheduler::RescheduleCurrentHLEThread(m_kernel); } } else { - GetCurrentThread(kernel).EnableDispatch(); + GetCurrentThread(m_kernel).EnableDispatch(); } } |