summaryrefslogtreecommitdiffstats
path: root/src/core/hle/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'src/core/hle/kernel')
-rw-r--r--src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp21
-rw-r--r--src/core/hle/kernel/init/init_slab_setup.cpp8
-rw-r--r--src/core/hle/kernel/k_address_arbiter.cpp8
-rw-r--r--src/core/hle/kernel/k_address_space_info.cpp79
-rw-r--r--src/core/hle/kernel/k_condition_variable.cpp50
-rw-r--r--src/core/hle/kernel/k_light_lock.cpp8
-rw-r--r--src/core/hle/kernel/k_process.cpp6
-rw-r--r--src/core/hle/kernel/k_scheduler_lock.h11
-rw-r--r--src/core/hle/kernel/k_thread.cpp271
-rw-r--r--src/core/hle/kernel/k_thread.h175
-rw-r--r--src/core/hle/kernel/kernel.cpp93
-rw-r--r--src/core/hle/kernel/kernel.h67
-rw-r--r--src/core/hle/kernel/svc_types.h1
13 files changed, 510 insertions, 288 deletions
diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp
index c10b7bf30..5b8a248c8 100644
--- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp
+++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp
@@ -14,9 +14,12 @@ namespace Kernel::Board::Nintendo::Nx {
namespace impl {
-constexpr const std::size_t RequiredNonSecureSystemMemorySizeVi = 0x2238 * 4 * 1024;
-constexpr const std::size_t RequiredNonSecureSystemMemorySizeNvservices = 0x710 * 4 * 1024;
-constexpr const std::size_t RequiredNonSecureSystemMemorySizeMisc = 0x80 * 4 * 1024;
+using namespace Common::Literals;
+
+constexpr const std::size_t RequiredNonSecureSystemMemorySizeVi = 0x2280 * 4_KiB;
+constexpr const std::size_t RequiredNonSecureSystemMemorySizeViFatal = 0x200 * 4_KiB;
+constexpr const std::size_t RequiredNonSecureSystemMemorySizeNvservices = 0x704 * 4_KiB;
+constexpr const std::size_t RequiredNonSecureSystemMemorySizeMisc = 0x80 * 4_KiB;
} // namespace impl
@@ -24,6 +27,9 @@ constexpr const std::size_t RequiredNonSecureSystemMemorySize =
impl::RequiredNonSecureSystemMemorySizeVi + impl::RequiredNonSecureSystemMemorySizeNvservices +
impl::RequiredNonSecureSystemMemorySizeMisc;
+constexpr const std::size_t RequiredNonSecureSystemMemorySizeWithFatal =
+ RequiredNonSecureSystemMemorySize + impl::RequiredNonSecureSystemMemorySizeViFatal;
+
namespace {
using namespace Common::Literals;
@@ -120,10 +126,13 @@ size_t KSystemControl::Init::GetAppletPoolSize() {
size_t KSystemControl::Init::GetMinimumNonSecureSystemPoolSize() {
// Verify that our minimum is at least as large as Nintendo's.
- constexpr size_t MinimumSize = RequiredNonSecureSystemMemorySize;
- static_assert(MinimumSize >= 0x29C8000);
+ constexpr size_t MinimumSizeWithFatal = RequiredNonSecureSystemMemorySizeWithFatal;
+ static_assert(MinimumSizeWithFatal >= 0x2C04000);
+
+ constexpr size_t MinimumSizeWithoutFatal = RequiredNonSecureSystemMemorySize;
+ static_assert(MinimumSizeWithoutFatal >= 0x2A00000);
- return MinimumSize;
+ return MinimumSizeWithFatal;
}
namespace {
diff --git a/src/core/hle/kernel/init/init_slab_setup.cpp b/src/core/hle/kernel/init/init_slab_setup.cpp
index abdb5639f..5e4090e2b 100644
--- a/src/core/hle/kernel/init/init_slab_setup.cpp
+++ b/src/core/hle/kernel/init/init_slab_setup.cpp
@@ -33,6 +33,9 @@
namespace Kernel::Init {
+// For macro convenience.
+using KThreadLockInfo = KThread::LockWithPriorityInheritanceInfo;
+
#define SLAB_COUNT(CLASS) kernel.SlabResourceCounts().num_##CLASS
#define FOREACH_SLAB_TYPE(HANDLER, ...) \
@@ -54,7 +57,8 @@ namespace Kernel::Init {
HANDLER(KResourceLimit, (SLAB_COUNT(KResourceLimit)), ##__VA_ARGS__) \
HANDLER(KEventInfo, (SLAB_COUNT(KThread) + SLAB_COUNT(KDebug)), ##__VA_ARGS__) \
HANDLER(KDebug, (SLAB_COUNT(KDebug)), ##__VA_ARGS__) \
- HANDLER(KSecureSystemResource, (SLAB_COUNT(KProcess)), ##__VA_ARGS__)
+ HANDLER(KSecureSystemResource, (SLAB_COUNT(KProcess)), ##__VA_ARGS__) \
+ HANDLER(KThreadLockInfo, (SLAB_COUNT(KThread)), ##__VA_ARGS__)
namespace {
@@ -131,7 +135,7 @@ VAddr InitializeSlabHeap(Core::System& system, KMemoryLayout& memory_layout, VAd
}
size_t CalculateSlabHeapGapSize() {
- constexpr size_t KernelSlabHeapGapSize = 2_MiB - 320_KiB;
+ constexpr size_t KernelSlabHeapGapSize = 2_MiB - 356_KiB;
static_assert(KernelSlabHeapGapSize <= KernelSlabHeapGapsSizeMax);
return KernelSlabHeapGapSize;
}
diff --git a/src/core/hle/kernel/k_address_arbiter.cpp b/src/core/hle/kernel/k_address_arbiter.cpp
index a442a3b98..fb86451ea 100644
--- a/src/core/hle/kernel/k_address_arbiter.cpp
+++ b/src/core/hle/kernel/k_address_arbiter.cpp
@@ -29,7 +29,9 @@ bool DecrementIfLessThan(Core::System& system, s32* out, VAddr address, s32 valu
auto& monitor = system.Monitor();
const auto current_core = system.Kernel().CurrentPhysicalCoreIndex();
- // TODO(bunnei): We should disable interrupts here via KScopedInterruptDisable.
+ // NOTE: If scheduler lock is not held here, interrupt disable is required.
+ // KScopedInterruptDisable di;
+
// TODO(bunnei): We should call CanAccessAtomic(..) here.
// Load the value from the address.
@@ -59,7 +61,9 @@ bool UpdateIfEqual(Core::System& system, s32* out, VAddr address, s32 value, s32
auto& monitor = system.Monitor();
const auto current_core = system.Kernel().CurrentPhysicalCoreIndex();
- // TODO(bunnei): We should disable interrupts here via KScopedInterruptDisable.
+ // NOTE: If scheduler lock is not held here, interrupt disable is required.
+ // KScopedInterruptDisable di;
+
// TODO(bunnei): We should call CanAccessAtomic(..) here.
// Load the value from the address.
diff --git a/src/core/hle/kernel/k_address_space_info.cpp b/src/core/hle/kernel/k_address_space_info.cpp
index 3e612a207..97972ebae 100644
--- a/src/core/hle/kernel/k_address_space_info.cpp
+++ b/src/core/hle/kernel/k_address_space_info.cpp
@@ -23,86 +23,33 @@ constexpr std::array<KAddressSpaceInfo, 13> AddressSpaceInfos{{
{ .bit_width = 32, .address = Size_Invalid, .size = 1_GiB , .type = KAddressSpaceInfo::Type::Heap, },
{ .bit_width = 36, .address = 128_MiB , .size = 2_GiB - 128_MiB, .type = KAddressSpaceInfo::Type::MapSmall, },
{ .bit_width = 36, .address = 2_GiB , .size = 64_GiB - 2_GiB , .type = KAddressSpaceInfo::Type::MapLarge, },
- { .bit_width = 36, .address = Size_Invalid, .size = 6_GiB , .type = KAddressSpaceInfo::Type::Heap, },
+ { .bit_width = 36, .address = Size_Invalid, .size = 8_GiB , .type = KAddressSpaceInfo::Type::Heap, },
{ .bit_width = 36, .address = Size_Invalid, .size = 6_GiB , .type = KAddressSpaceInfo::Type::Alias, },
{ .bit_width = 39, .address = 128_MiB , .size = 512_GiB - 128_MiB, .type = KAddressSpaceInfo::Type::Map39Bit, },
{ .bit_width = 39, .address = Size_Invalid, .size = 64_GiB , .type = KAddressSpaceInfo::Type::MapSmall },
- { .bit_width = 39, .address = Size_Invalid, .size = 6_GiB , .type = KAddressSpaceInfo::Type::Heap, },
+ { .bit_width = 39, .address = Size_Invalid, .size = 8_GiB , .type = KAddressSpaceInfo::Type::Heap, },
{ .bit_width = 39, .address = Size_Invalid, .size = 64_GiB , .type = KAddressSpaceInfo::Type::Alias, },
{ .bit_width = 39, .address = Size_Invalid, .size = 2_GiB , .type = KAddressSpaceInfo::Type::Stack, },
}};
// clang-format on
-constexpr bool IsAllowedIndexForAddress(std::size_t index) {
- return index < AddressSpaceInfos.size() && AddressSpaceInfos[index].address != Size_Invalid;
-}
-
-using IndexArray =
- std::array<std::size_t, static_cast<std::size_t>(KAddressSpaceInfo::Type::Count)>;
-
-constexpr IndexArray AddressSpaceIndices32Bit{
- 0, 1, 0, 2, 0, 3,
-};
-
-constexpr IndexArray AddressSpaceIndices36Bit{
- 4, 5, 4, 6, 4, 7,
-};
-
-constexpr IndexArray AddressSpaceIndices39Bit{
- 9, 8, 8, 10, 12, 11,
-};
-
-constexpr bool IsAllowed32BitType(KAddressSpaceInfo::Type type) {
- return type < KAddressSpaceInfo::Type::Count && type != KAddressSpaceInfo::Type::Map39Bit &&
- type != KAddressSpaceInfo::Type::Stack;
-}
-
-constexpr bool IsAllowed36BitType(KAddressSpaceInfo::Type type) {
- return type < KAddressSpaceInfo::Type::Count && type != KAddressSpaceInfo::Type::Map39Bit &&
- type != KAddressSpaceInfo::Type::Stack;
-}
-
-constexpr bool IsAllowed39BitType(KAddressSpaceInfo::Type type) {
- return type < KAddressSpaceInfo::Type::Count && type != KAddressSpaceInfo::Type::MapLarge;
+const KAddressSpaceInfo& GetAddressSpaceInfo(size_t width, KAddressSpaceInfo::Type type) {
+ for (auto& info : AddressSpaceInfos) {
+ if (info.bit_width == width && info.type == type) {
+ return info;
+ }
+ }
+ UNREACHABLE_MSG("Could not find AddressSpaceInfo");
}
} // namespace
-u64 KAddressSpaceInfo::GetAddressSpaceStart(std::size_t width, Type type) {
- const std::size_t index{static_cast<std::size_t>(type)};
- switch (width) {
- case 32:
- ASSERT(IsAllowed32BitType(type));
- ASSERT(IsAllowedIndexForAddress(AddressSpaceIndices32Bit[index]));
- return AddressSpaceInfos[AddressSpaceIndices32Bit[index]].address;
- case 36:
- ASSERT(IsAllowed36BitType(type));
- ASSERT(IsAllowedIndexForAddress(AddressSpaceIndices36Bit[index]));
- return AddressSpaceInfos[AddressSpaceIndices36Bit[index]].address;
- case 39:
- ASSERT(IsAllowed39BitType(type));
- ASSERT(IsAllowedIndexForAddress(AddressSpaceIndices39Bit[index]));
- return AddressSpaceInfos[AddressSpaceIndices39Bit[index]].address;
- }
- ASSERT(false);
- return 0;
+uintptr_t KAddressSpaceInfo::GetAddressSpaceStart(size_t width, KAddressSpaceInfo::Type type) {
+ return GetAddressSpaceInfo(width, type).address;
}
-std::size_t KAddressSpaceInfo::GetAddressSpaceSize(std::size_t width, Type type) {
- const std::size_t index{static_cast<std::size_t>(type)};
- switch (width) {
- case 32:
- ASSERT(IsAllowed32BitType(type));
- return AddressSpaceInfos[AddressSpaceIndices32Bit[index]].size;
- case 36:
- ASSERT(IsAllowed36BitType(type));
- return AddressSpaceInfos[AddressSpaceIndices36Bit[index]].size;
- case 39:
- ASSERT(IsAllowed39BitType(type));
- return AddressSpaceInfos[AddressSpaceIndices39Bit[index]].size;
- }
- ASSERT(false);
- return 0;
+size_t KAddressSpaceInfo::GetAddressSpaceSize(size_t width, KAddressSpaceInfo::Type type) {
+ return GetAddressSpaceInfo(width, type).size;
}
} // namespace Kernel
diff --git a/src/core/hle/kernel/k_condition_variable.cpp b/src/core/hle/kernel/k_condition_variable.cpp
index 3f0be1c3f..f40cf92b1 100644
--- a/src/core/hle/kernel/k_condition_variable.cpp
+++ b/src/core/hle/kernel/k_condition_variable.cpp
@@ -111,36 +111,36 @@ Result KConditionVariable::SignalToAddress(VAddr addr) {
KScopedSchedulerLock sl(kernel);
// Remove waiter thread.
- s32 num_waiters{};
- KThread* next_owner_thread =
- owner_thread->RemoveWaiterByKey(std::addressof(num_waiters), addr);
+ bool has_waiters{};
+ KThread* const next_owner_thread =
+ owner_thread->RemoveUserWaiterByKey(std::addressof(has_waiters), addr);
// Determine the next tag.
u32 next_value{};
if (next_owner_thread != nullptr) {
next_value = next_owner_thread->GetAddressKeyValue();
- if (num_waiters > 1) {
+ if (has_waiters) {
next_value |= Svc::HandleWaitMask;
}
+ }
- // Write the value to userspace.
- Result result{ResultSuccess};
- if (WriteToUser(system, addr, std::addressof(next_value))) [[likely]] {
- result = ResultSuccess;
- } else {
- result = ResultInvalidCurrentMemory;
- }
+ // Synchronize memory before proceeding.
+ std::atomic_thread_fence(std::memory_order_seq_cst);
- // Signal the next owner thread.
- next_owner_thread->EndWait(result);
- return result;
+ // Write the value to userspace.
+ Result result{ResultSuccess};
+ if (WriteToUser(system, addr, std::addressof(next_value))) [[likely]] {
+ result = ResultSuccess;
} else {
- // Just write the value to userspace.
- R_UNLESS(WriteToUser(system, addr, std::addressof(next_value)),
- ResultInvalidCurrentMemory);
+ result = ResultInvalidCurrentMemory;
+ }
- return ResultSuccess;
+ // If necessary, signal the next owner thread.
+ if (next_owner_thread != nullptr) {
+ next_owner_thread->EndWait(result);
}
+
+ R_RETURN(result);
}
}
@@ -198,7 +198,9 @@ void KConditionVariable::SignalImpl(KThread* thread) {
u32 prev_tag{};
bool can_access{};
{
- // TODO(bunnei): We should disable interrupts here via KScopedInterruptDisable.
+ // NOTE: If scheduler lock is not held here, interrupt disable is required.
+ // KScopedInterruptDisable di;
+
// TODO(bunnei): We should call CanAccessAtomic(..) here.
can_access = true;
if (can_access) [[likely]] {
@@ -245,9 +247,11 @@ void KConditionVariable::Signal(u64 cv_key, s32 count) {
(it->GetConditionVariableKey() == cv_key)) {
KThread* target_thread = std::addressof(*it);
- this->SignalImpl(target_thread);
it = thread_tree.erase(it);
target_thread->ClearConditionVariable();
+
+ this->SignalImpl(target_thread);
+
++num_waiters;
}
@@ -277,16 +281,16 @@ Result KConditionVariable::Wait(VAddr addr, u64 key, u32 value, s64 timeout) {
// Update the value and process for the next owner.
{
// Remove waiter thread.
- s32 num_waiters{};
+ bool has_waiters{};
KThread* next_owner_thread =
- cur_thread->RemoveWaiterByKey(std::addressof(num_waiters), addr);
+ cur_thread->RemoveUserWaiterByKey(std::addressof(has_waiters), addr);
// Update for the next owner thread.
u32 next_value{};
if (next_owner_thread != nullptr) {
// Get the next tag value.
next_value = next_owner_thread->GetAddressKeyValue();
- if (num_waiters > 1) {
+ if (has_waiters) {
next_value |= Svc::HandleWaitMask;
}
diff --git a/src/core/hle/kernel/k_light_lock.cpp b/src/core/hle/kernel/k_light_lock.cpp
index d791acbe3..14cb615da 100644
--- a/src/core/hle/kernel/k_light_lock.cpp
+++ b/src/core/hle/kernel/k_light_lock.cpp
@@ -90,15 +90,15 @@ void KLightLock::UnlockSlowPath(uintptr_t _cur_thread) {
KScopedSchedulerLock sl(kernel);
// Get the next owner.
- s32 num_waiters;
- KThread* next_owner = owner_thread->RemoveWaiterByKey(
- std::addressof(num_waiters), reinterpret_cast<uintptr_t>(std::addressof(tag)));
+ bool has_waiters;
+ KThread* next_owner = owner_thread->RemoveKernelWaiterByKey(
+ std::addressof(has_waiters), reinterpret_cast<uintptr_t>(std::addressof(tag)));
// Pass the lock to the next owner.
uintptr_t next_tag = 0;
if (next_owner != nullptr) {
next_tag =
- reinterpret_cast<uintptr_t>(next_owner) | static_cast<uintptr_t>(num_waiters > 1);
+ reinterpret_cast<uintptr_t>(next_owner) | static_cast<uintptr_t>(has_waiters);
next_owner->EndWait(ResultSuccess);
diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp
index d9c1a0eb3..d44f6e921 100644
--- a/src/core/hle/kernel/k_process.cpp
+++ b/src/core/hle/kernel/k_process.cpp
@@ -156,9 +156,9 @@ bool KProcess::ReleaseUserException(KThread* thread) {
exception_thread = nullptr;
// Remove waiter thread.
- s32 num_waiters{};
- if (KThread* next = thread->RemoveWaiterByKey(
- std::addressof(num_waiters),
+ bool has_waiters{};
+ if (KThread* next = thread->RemoveKernelWaiterByKey(
+ std::addressof(has_waiters),
reinterpret_cast<uintptr_t>(std::addressof(exception_thread)));
next != nullptr) {
next->EndWait(ResultSuccess);
diff --git a/src/core/hle/kernel/k_scheduler_lock.h b/src/core/hle/kernel/k_scheduler_lock.h
index 129d60472..13463717f 100644
--- a/src/core/hle/kernel/k_scheduler_lock.h
+++ b/src/core/hle/kernel/k_scheduler_lock.h
@@ -31,22 +31,23 @@ public:
}
if (IsLockedByCurrentThread()) {
- // If we already own the lock, we can just increment the count.
+ // If we already own the lock, the lock count should be > 0.
+ // For debug, ensure this is true.
ASSERT(lock_count > 0);
- lock_count++;
} else {
// Otherwise, we want to disable scheduling and acquire the spinlock.
SchedulerType::DisableScheduling(kernel);
spin_lock.Lock();
- // For debug, ensure that our state is valid.
ASSERT(lock_count == 0);
ASSERT(owner_thread == nullptr);
- // Increment count, take ownership.
- lock_count = 1;
+ // Take ownership of the lock.
owner_thread = GetCurrentThreadPointer(kernel);
}
+
+ // Increment the lock count.
+ lock_count++;
}
void Unlock() {
diff --git a/src/core/hle/kernel/k_thread.cpp b/src/core/hle/kernel/k_thread.cpp
index 599d05947..8c403f5fd 100644
--- a/src/core/hle/kernel/k_thread.cpp
+++ b/src/core/hle/kernel/k_thread.cpp
@@ -191,7 +191,7 @@ Result KThread::Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack
light_ipc_data = nullptr;
// We're not waiting for a lock, and we haven't disabled migration.
- lock_owner = nullptr;
+ waiting_lock_info = nullptr;
num_core_migration_disables = 0;
// We have no waiters, but we do have an entrypoint.
@@ -341,25 +341,39 @@ void KThread::Finalize() {
// Release any waiters.
{
- ASSERT(lock_owner == nullptr);
+ ASSERT(waiting_lock_info == nullptr);
KScopedSchedulerLock sl{kernel};
- auto it = waiter_list.begin();
- while (it != waiter_list.end()) {
- // Get the thread.
- KThread* const waiter = std::addressof(*it);
+ // Check that we have no kernel waiters.
+ ASSERT(num_kernel_waiters == 0);
- // The thread shouldn't be a kernel waiter.
- ASSERT(!waiter->GetAddressKeyIsKernel());
+ auto it = held_lock_info_list.begin();
+ while (it != held_lock_info_list.end()) {
+ // Get the lock info.
+ auto* const lock_info = std::addressof(*it);
- // Clear the lock owner.
- waiter->SetLockOwner(nullptr);
+ // The lock shouldn't have a kernel waiter.
+ ASSERT(!lock_info->GetIsKernelAddressKey());
- // Erase the waiter from our list.
- it = waiter_list.erase(it);
+ // Remove all waiters.
+ while (lock_info->GetWaiterCount() != 0) {
+ // Get the front waiter.
+ KThread* const waiter = lock_info->GetHighestPriorityWaiter();
- // Cancel the thread's wait.
- waiter->CancelWait(ResultInvalidState, true);
+ // Remove it from the lock.
+ if (lock_info->RemoveWaiter(waiter)) {
+ ASSERT(lock_info->GetWaiterCount() == 0);
+ }
+
+ // Cancel the thread's wait.
+ waiter->CancelWait(ResultInvalidState, true);
+ }
+
+ // Remove the held lock from our list.
+ it = held_lock_info_list.erase(it);
+
+ // Free the lock info.
+ LockWithPriorityInheritanceInfo::Free(kernel, lock_info);
}
}
@@ -708,6 +722,24 @@ void KThread::SetBasePriority(s32 value) {
RestorePriority(kernel, this);
}
+KThread* KThread::GetLockOwner() const {
+ return waiting_lock_info != nullptr ? waiting_lock_info->GetOwner() : nullptr;
+}
+
+void KThread::IncreaseBasePriority(s32 priority_) {
+ ASSERT(Svc::HighestThreadPriority <= priority_ && priority_ <= Svc::LowestThreadPriority);
+ ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel));
+ ASSERT(!this->GetStackParameters().is_pinned);
+
+ // Set our base priority.
+ if (base_priority > priority_) {
+ base_priority = priority_;
+
+ // Perform a priority restoration.
+ RestorePriority(kernel, this);
+ }
+}
+
void KThread::RequestSuspend(SuspendType type) {
KScopedSchedulerLock sl{kernel};
@@ -891,51 +923,89 @@ Result KThread::GetThreadContext3(std::vector<u8>& out) {
R_SUCCEED();
}
-void KThread::AddWaiterImpl(KThread* thread) {
- ASSERT(kernel.GlobalSchedulerContext().IsLocked());
+void KThread::AddHeldLock(LockWithPriorityInheritanceInfo* lock_info) {
+ ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel));
+
+ // Set ourselves as the lock's owner.
+ lock_info->SetOwner(this);
- // Find the right spot to insert the waiter.
- auto it = waiter_list.begin();
- while (it != waiter_list.end()) {
- if (it->GetPriority() > thread->GetPriority()) {
- break;
+ // Add the lock to our held list.
+ held_lock_info_list.push_front(*lock_info);
+}
+
+KThread::LockWithPriorityInheritanceInfo* KThread::FindHeldLock(VAddr address_key_,
+ bool is_kernel_address_key_) {
+ ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel));
+
+ // Try to find an existing held lock.
+ for (auto& held_lock : held_lock_info_list) {
+ if (held_lock.GetAddressKey() == address_key_ &&
+ held_lock.GetIsKernelAddressKey() == is_kernel_address_key_) {
+ return std::addressof(held_lock);
}
- it++;
}
+ return nullptr;
+}
+
+void KThread::AddWaiterImpl(KThread* thread) {
+ ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel));
+ ASSERT(thread->GetConditionVariableTree() == nullptr);
+
+ // Get the thread's address key.
+ const auto address_key_ = thread->GetAddressKey();
+ const auto is_kernel_address_key_ = thread->GetIsKernelAddressKey();
+
// Keep track of how many kernel waiters we have.
- if (thread->GetAddressKeyIsKernel()) {
+ if (is_kernel_address_key_) {
ASSERT((num_kernel_waiters++) >= 0);
KScheduler::SetSchedulerUpdateNeeded(kernel);
}
- // Insert the waiter.
- waiter_list.insert(it, *thread);
- thread->SetLockOwner(this);
+ // Get the relevant lock info.
+ auto* lock_info = this->FindHeldLock(address_key_, is_kernel_address_key_);
+ if (lock_info == nullptr) {
+ // Create a new lock for the address key.
+ lock_info =
+ LockWithPriorityInheritanceInfo::Create(kernel, address_key_, is_kernel_address_key_);
+
+ // Add the new lock to our list.
+ this->AddHeldLock(lock_info);
+ }
+
+ // Add the thread as waiter to the lock info.
+ lock_info->AddWaiter(thread);
}
void KThread::RemoveWaiterImpl(KThread* thread) {
- ASSERT(kernel.GlobalSchedulerContext().IsLocked());
+ ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel));
// Keep track of how many kernel waiters we have.
- if (thread->GetAddressKeyIsKernel()) {
+ if (thread->GetIsKernelAddressKey()) {
ASSERT((num_kernel_waiters--) > 0);
KScheduler::SetSchedulerUpdateNeeded(kernel);
}
+ // Get the info for the lock the thread is waiting on.
+ auto* lock_info = thread->GetWaitingLockInfo();
+ ASSERT(lock_info->GetOwner() == this);
+
// Remove the waiter.
- waiter_list.erase(waiter_list.iterator_to(*thread));
- thread->SetLockOwner(nullptr);
+ if (lock_info->RemoveWaiter(thread)) {
+ held_lock_info_list.erase(held_lock_info_list.iterator_to(*lock_info));
+ LockWithPriorityInheritanceInfo::Free(kernel, lock_info);
+ }
}
-void KThread::RestorePriority(KernelCore& kernel_ctx, KThread* thread) {
- ASSERT(kernel_ctx.GlobalSchedulerContext().IsLocked());
+void KThread::RestorePriority(KernelCore& kernel, KThread* thread) {
+ ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel));
- while (true) {
+ while (thread != nullptr) {
// We want to inherit priority where possible.
s32 new_priority = thread->GetBasePriority();
- if (thread->HasWaiters()) {
- new_priority = std::min(new_priority, thread->waiter_list.front().GetPriority());
+ for (const auto& held_lock : thread->held_lock_info_list) {
+ new_priority =
+ std::min(new_priority, held_lock.GetHighestPriorityWaiter()->GetPriority());
}
// If the priority we would inherit is not different from ours, don't do anything.
@@ -943,9 +1013,18 @@ void KThread::RestorePriority(KernelCore& kernel_ctx, KThread* thread) {
return;
}
+ // Get the owner of whatever lock this thread is waiting on.
+ KThread* const lock_owner = thread->GetLockOwner();
+
+ // If the thread is waiting on some lock, remove it as a waiter to prevent violating red
+ // black tree invariants.
+ if (lock_owner != nullptr) {
+ lock_owner->RemoveWaiterImpl(thread);
+ }
+
// Ensure we don't violate condition variable red black tree invariants.
if (auto* cv_tree = thread->GetConditionVariableTree(); cv_tree != nullptr) {
- BeforeUpdatePriority(kernel_ctx, cv_tree, thread);
+ BeforeUpdatePriority(kernel, cv_tree, thread);
}
// Change the priority.
@@ -954,73 +1033,99 @@ void KThread::RestorePriority(KernelCore& kernel_ctx, KThread* thread) {
// Restore the condition variable, if relevant.
if (auto* cv_tree = thread->GetConditionVariableTree(); cv_tree != nullptr) {
- AfterUpdatePriority(kernel_ctx, cv_tree, thread);
+ AfterUpdatePriority(kernel, cv_tree, thread);
}
- // Update the scheduler.
- KScheduler::OnThreadPriorityChanged(kernel_ctx, thread, old_priority);
-
- // Keep the lock owner up to date.
- KThread* lock_owner = thread->GetLockOwner();
- if (lock_owner == nullptr) {
- return;
+ // If we removed the thread from some lock's waiting list, add it back.
+ if (lock_owner != nullptr) {
+ lock_owner->AddWaiterImpl(thread);
}
- // Update the thread in the lock owner's sorted list, and continue inheriting.
- lock_owner->RemoveWaiterImpl(thread);
- lock_owner->AddWaiterImpl(thread);
+ // Update the scheduler.
+ KScheduler::OnThreadPriorityChanged(kernel, thread, old_priority);
+
+ // Continue inheriting priority.
thread = lock_owner;
}
}
void KThread::AddWaiter(KThread* thread) {
- AddWaiterImpl(thread);
- RestorePriority(kernel, this);
+ this->AddWaiterImpl(thread);
+
+ // If the thread has a higher priority than us, we should inherit.
+ if (thread->GetPriority() < this->GetPriority()) {
+ RestorePriority(kernel, this);
+ }
}
void KThread::RemoveWaiter(KThread* thread) {
- RemoveWaiterImpl(thread);
- RestorePriority(kernel, this);
+ this->RemoveWaiterImpl(thread);
+
+ // If our priority is the same as the thread's (and we've inherited), we may need to restore to
+ // lower priority.
+ if (this->GetPriority() == thread->GetPriority() &&
+ this->GetPriority() < this->GetBasePriority()) {
+ RestorePriority(kernel, this);
+ }
}
-KThread* KThread::RemoveWaiterByKey(s32* out_num_waiters, VAddr key) {
- ASSERT(kernel.GlobalSchedulerContext().IsLocked());
+KThread* KThread::RemoveWaiterByKey(bool* out_has_waiters, VAddr key, bool is_kernel_address_key_) {
+ ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel));
- s32 num_waiters{};
- KThread* next_lock_owner{};
- auto it = waiter_list.begin();
- while (it != waiter_list.end()) {
- if (it->GetAddressKey() == key) {
- KThread* thread = std::addressof(*it);
-
- // Keep track of how many kernel waiters we have.
- if (thread->GetAddressKeyIsKernel()) {
- ASSERT((num_kernel_waiters--) > 0);
- KScheduler::SetSchedulerUpdateNeeded(kernel);
- }
- it = waiter_list.erase(it);
+ // Get the relevant lock info.
+ auto* lock_info = this->FindHeldLock(key, is_kernel_address_key_);
+ if (lock_info == nullptr) {
+ *out_has_waiters = false;
+ return nullptr;
+ }
- // Update the next lock owner.
- if (next_lock_owner == nullptr) {
- next_lock_owner = thread;
- next_lock_owner->SetLockOwner(nullptr);
- } else {
- next_lock_owner->AddWaiterImpl(thread);
- }
- num_waiters++;
- } else {
- it++;
+ // Remove the lock info from our held list.
+ held_lock_info_list.erase(held_lock_info_list.iterator_to(*lock_info));
+
+ // Keep track of how many kernel waiters we have.
+ if (lock_info->GetIsKernelAddressKey()) {
+ num_kernel_waiters -= lock_info->GetWaiterCount();
+ ASSERT(num_kernel_waiters >= 0);
+ KScheduler::SetSchedulerUpdateNeeded(kernel);
+ }
+
+ ASSERT(lock_info->GetWaiterCount() > 0);
+
+ // Remove the highest priority waiter from the lock to be the next owner.
+ KThread* next_lock_owner = lock_info->GetHighestPriorityWaiter();
+ if (lock_info->RemoveWaiter(next_lock_owner)) {
+ // The new owner was the only waiter.
+ *out_has_waiters = false;
+
+ // Free the lock info, since it has no waiters.
+ LockWithPriorityInheritanceInfo::Free(kernel, lock_info);
+ } else {
+ // There are additional waiters on the lock.
+ *out_has_waiters = true;
+
+ // Add the lock to the new owner's held list.
+ next_lock_owner->AddHeldLock(lock_info);
+
+ // Keep track of any kernel waiters for the new owner.
+ if (lock_info->GetIsKernelAddressKey()) {
+ next_lock_owner->num_kernel_waiters += lock_info->GetWaiterCount();
+ ASSERT(next_lock_owner->num_kernel_waiters > 0);
+
+ // NOTE: No need to set scheduler update needed, because we will have already done so
+ // when removing earlier.
}
}
- // Do priority updates, if we have a next owner.
- if (next_lock_owner) {
+ // If our priority is the same as the next owner's (and we've inherited), we may need to restore
+ // to lower priority.
+ if (this->GetPriority() == next_lock_owner->GetPriority() &&
+ this->GetPriority() < this->GetBasePriority()) {
RestorePriority(kernel, this);
- RestorePriority(kernel, next_lock_owner);
+ // NOTE: No need to restore priority on the next lock owner, because it was already the
+ // highest priority waiter on the lock.
}
- // Return output.
- *out_num_waiters = num_waiters;
+ // Return the next lock owner.
return next_lock_owner;
}
@@ -1137,9 +1242,7 @@ ThreadState KThread::RequestTerminate() {
}
// Change the thread's priority to be higher than any system thread's.
- if (this->GetBasePriority() >= Svc::SystemThreadPriorityHighest) {
- this->SetBasePriority(TerminatingThreadPriority);
- }
+ this->IncreaseBasePriority(TerminatingThreadPriority);
// If the thread is runnable, send a termination interrupt to other cores.
if (this->GetState() == ThreadState::Runnable) {
diff --git a/src/core/hle/kernel/k_thread.h b/src/core/hle/kernel/k_thread.h
index a04de21bc..bd125f5f1 100644
--- a/src/core/hle/kernel/k_thread.h
+++ b/src/core/hle/kernel/k_thread.h
@@ -339,13 +339,7 @@ public:
void SetInterruptFlag();
void ClearInterruptFlag();
- [[nodiscard]] KThread* GetLockOwner() const {
- return lock_owner;
- }
-
- void SetLockOwner(KThread* owner) {
- lock_owner = owner;
- }
+ KThread* GetLockOwner() const;
[[nodiscard]] const KAffinityMask& GetAffinityMask() const {
return physical_affinity_mask;
@@ -601,7 +595,13 @@ public:
[[nodiscard]] Result GetThreadContext3(std::vector<u8>& out);
- [[nodiscard]] KThread* RemoveWaiterByKey(s32* out_num_waiters, VAddr key);
+ [[nodiscard]] KThread* RemoveUserWaiterByKey(bool* out_has_waiters, VAddr key) {
+ return this->RemoveWaiterByKey(out_has_waiters, key, false);
+ }
+
+ [[nodiscard]] KThread* RemoveKernelWaiterByKey(bool* out_has_waiters, VAddr key) {
+ return this->RemoveWaiterByKey(out_has_waiters, key, true);
+ }
[[nodiscard]] VAddr GetAddressKey() const {
return address_key;
@@ -611,8 +611,8 @@ public:
return address_key_value;
}
- [[nodiscard]] bool GetAddressKeyIsKernel() const {
- return address_key_is_kernel;
+ [[nodiscard]] bool GetIsKernelAddressKey() const {
+ return is_kernel_address_key;
}
//! NB: intentional deviation from official kernel.
@@ -621,20 +621,17 @@ public:
// to cope with arbitrary host pointers making their way
// into things.
- void SetUserAddressKey(VAddr key) {
- address_key = key;
- address_key_is_kernel = false;
- }
-
void SetUserAddressKey(VAddr key, u32 val) {
+ ASSERT(waiting_lock_info == nullptr);
address_key = key;
address_key_value = val;
- address_key_is_kernel = false;
+ is_kernel_address_key = false;
}
void SetKernelAddressKey(VAddr key) {
+ ASSERT(waiting_lock_info == nullptr);
address_key = key;
- address_key_is_kernel = true;
+ is_kernel_address_key = true;
}
void ClearWaitQueue() {
@@ -646,10 +643,6 @@ public:
void EndWait(Result wait_result_);
void CancelWait(Result wait_result_, bool cancel_timer_task);
- [[nodiscard]] bool HasWaiters() const {
- return !waiter_list.empty();
- }
-
[[nodiscard]] s32 GetNumKernelWaiters() const {
return num_kernel_waiters;
}
@@ -679,6 +672,9 @@ public:
}
private:
+ [[nodiscard]] KThread* RemoveWaiterByKey(bool* out_has_waiters, VAddr key,
+ bool is_kernel_address_key);
+
static constexpr size_t PriorityInheritanceCountMax = 10;
union SyncObjectBuffer {
std::array<KSynchronizationObject*, Svc::ArgumentHandleCountMax> sync_objects{};
@@ -722,13 +718,14 @@ private:
};
void AddWaiterImpl(KThread* thread);
-
void RemoveWaiterImpl(KThread* thread);
+ static void RestorePriority(KernelCore& kernel, KThread* thread);
void StartTermination();
-
void FinishTermination();
+ void IncreaseBasePriority(s32 priority);
+
[[nodiscard]] Result Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack_top,
s32 prio, s32 virt_core, KProcess* owner, ThreadType type);
@@ -737,8 +734,6 @@ private:
s32 core, KProcess* owner, ThreadType type,
std::function<void()>&& init_func);
- static void RestorePriority(KernelCore& kernel_ctx, KThread* thread);
-
// For core KThread implementation
ThreadContext32 thread_context_32{};
ThreadContext64 thread_context_64{};
@@ -749,6 +744,127 @@ private:
&KThread::condvar_arbiter_tree_node>;
using ConditionVariableThreadTree =
ConditionVariableThreadTreeTraits::TreeType<ConditionVariableComparator>;
+
+private:
+ struct LockWithPriorityInheritanceComparator {
+ struct RedBlackKeyType {
+ s32 m_priority;
+
+ constexpr s32 GetPriority() const {
+ return m_priority;
+ }
+ };
+
+ template <typename T>
+ requires(std::same_as<T, KThread> || std::same_as<T, RedBlackKeyType>)
+ static constexpr int Compare(const T& lhs, const KThread& rhs) {
+ if (lhs.GetPriority() < rhs.GetPriority()) {
+ // Sort by priority.
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+ };
+ static_assert(std::same_as<Common::RedBlackKeyType<LockWithPriorityInheritanceComparator, void>,
+ LockWithPriorityInheritanceComparator::RedBlackKeyType>);
+
+ using LockWithPriorityInheritanceThreadTreeTraits =
+ Common::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<
+ &KThread::condvar_arbiter_tree_node>;
+ using LockWithPriorityInheritanceThreadTree =
+ ConditionVariableThreadTreeTraits::TreeType<LockWithPriorityInheritanceComparator>;
+
+public:
+ class LockWithPriorityInheritanceInfo : public KSlabAllocated<LockWithPriorityInheritanceInfo>,
+ public boost::intrusive::list_base_hook<> {
+ public:
+ explicit LockWithPriorityInheritanceInfo(KernelCore&) {}
+
+ static LockWithPriorityInheritanceInfo* Create(KernelCore& kernel, VAddr address_key,
+ bool is_kernel_address_key) {
+ // Create a new lock info.
+ auto* new_lock = LockWithPriorityInheritanceInfo::Allocate(kernel);
+ ASSERT(new_lock != nullptr);
+
+ // Set the new lock's address key.
+ new_lock->m_address_key = address_key;
+ new_lock->m_is_kernel_address_key = is_kernel_address_key;
+
+ return new_lock;
+ }
+
+ void SetOwner(KThread* new_owner) {
+ // Set new owner.
+ m_owner = new_owner;
+ }
+
+ void AddWaiter(KThread* waiter) {
+ // Insert the waiter.
+ m_tree.insert(*waiter);
+ m_waiter_count++;
+
+ waiter->SetWaitingLockInfo(this);
+ }
+
+ [[nodiscard]] bool RemoveWaiter(KThread* waiter) {
+ m_tree.erase(m_tree.iterator_to(*waiter));
+
+ waiter->SetWaitingLockInfo(nullptr);
+
+ return (--m_waiter_count) == 0;
+ }
+
+ KThread* GetHighestPriorityWaiter() {
+ return std::addressof(m_tree.front());
+ }
+ const KThread* GetHighestPriorityWaiter() const {
+ return std::addressof(m_tree.front());
+ }
+
+ LockWithPriorityInheritanceThreadTree& GetThreadTree() {
+ return m_tree;
+ }
+ const LockWithPriorityInheritanceThreadTree& GetThreadTree() const {
+ return m_tree;
+ }
+
+ VAddr GetAddressKey() const {
+ return m_address_key;
+ }
+ bool GetIsKernelAddressKey() const {
+ return m_is_kernel_address_key;
+ }
+ KThread* GetOwner() const {
+ return m_owner;
+ }
+ u32 GetWaiterCount() const {
+ return m_waiter_count;
+ }
+
+ private:
+ LockWithPriorityInheritanceThreadTree m_tree{};
+ VAddr m_address_key{};
+ KThread* m_owner{};
+ u32 m_waiter_count{};
+ bool m_is_kernel_address_key{};
+ };
+
+ void SetWaitingLockInfo(LockWithPriorityInheritanceInfo* lock) {
+ waiting_lock_info = lock;
+ }
+
+ LockWithPriorityInheritanceInfo* GetWaitingLockInfo() {
+ return waiting_lock_info;
+ }
+
+ void AddHeldLock(LockWithPriorityInheritanceInfo* lock_info);
+ LockWithPriorityInheritanceInfo* FindHeldLock(VAddr address_key, bool is_kernel_address_key);
+
+private:
+ using LockWithPriorityInheritanceInfoList =
+ boost::intrusive::list<LockWithPriorityInheritanceInfo>;
+
ConditionVariableThreadTree* condvar_tree{};
u64 condvar_key{};
u64 virtual_affinity_mask{};
@@ -765,9 +881,9 @@ private:
s64 last_scheduled_tick{};
std::array<QueueEntry, Core::Hardware::NUM_CPU_CORES> per_core_priority_queue_entry{};
KThreadQueue* wait_queue{};
- WaiterList waiter_list{};
+ LockWithPriorityInheritanceInfoList held_lock_info_list{};
+ LockWithPriorityInheritanceInfo* waiting_lock_info{};
WaiterList pinned_waiter_list{};
- KThread* lock_owner{};
u32 address_key_value{};
u32 suspend_request_flags{};
u32 suspend_allowed_flags{};
@@ -791,7 +907,7 @@ private:
bool debug_attached{};
s8 priority_inheritance_count{};
bool resource_limit_release_hint{};
- bool address_key_is_kernel{};
+ bool is_kernel_address_key{};
StackParameters stack_parameters{};
Common::SpinLock context_guard{};
@@ -814,10 +930,12 @@ public:
void SetConditionVariable(ConditionVariableThreadTree* tree, VAddr address, u64 cv_key,
u32 value) {
+ ASSERT(waiting_lock_info == nullptr);
condvar_tree = tree;
condvar_key = cv_key;
address_key = address;
address_key_value = value;
+ is_kernel_address_key = false;
}
void ClearConditionVariable() {
@@ -829,6 +947,7 @@ public:
}
void SetAddressArbiter(ConditionVariableThreadTree* tree, u64 address) {
+ ASSERT(waiting_lock_info == nullptr);
condvar_tree = tree;
condvar_key = address;
}
diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp
index ce94d3605..ef7057ff7 100644
--- a/src/core/hle/kernel/kernel.cpp
+++ b/src/core/hle/kernel/kernel.cpp
@@ -1318,4 +1318,97 @@ const Core::System& KernelCore::System() const {
return impl->system;
}
+struct KernelCore::SlabHeapContainer {
+ KSlabHeap<KClientSession> client_session;
+ KSlabHeap<KEvent> event;
+ KSlabHeap<KLinkedListNode> linked_list_node;
+ KSlabHeap<KPort> port;
+ KSlabHeap<KProcess> process;
+ KSlabHeap<KResourceLimit> resource_limit;
+ KSlabHeap<KSession> session;
+ KSlabHeap<KSharedMemory> shared_memory;
+ KSlabHeap<KSharedMemoryInfo> shared_memory_info;
+ KSlabHeap<KThread> thread;
+ KSlabHeap<KTransferMemory> transfer_memory;
+ KSlabHeap<KCodeMemory> code_memory;
+ KSlabHeap<KDeviceAddressSpace> device_address_space;
+ KSlabHeap<KPageBuffer> page_buffer;
+ KSlabHeap<KThreadLocalPage> thread_local_page;
+ KSlabHeap<KObjectName> object_name;
+ KSlabHeap<KSessionRequest> session_request;
+ KSlabHeap<KSecureSystemResource> secure_system_resource;
+ KSlabHeap<KThread::LockWithPriorityInheritanceInfo> lock_info;
+ KSlabHeap<KEventInfo> event_info;
+ KSlabHeap<KDebug> debug;
+};
+
+template <typename T>
+KSlabHeap<T>& KernelCore::SlabHeap() {
+ if constexpr (std::is_same_v<T, KClientSession>) {
+ return slab_heap_container->client_session;
+ } else if constexpr (std::is_same_v<T, KEvent>) {
+ return slab_heap_container->event;
+ } else if constexpr (std::is_same_v<T, KLinkedListNode>) {
+ return slab_heap_container->linked_list_node;
+ } else if constexpr (std::is_same_v<T, KPort>) {
+ return slab_heap_container->port;
+ } else if constexpr (std::is_same_v<T, KProcess>) {
+ return slab_heap_container->process;
+ } else if constexpr (std::is_same_v<T, KResourceLimit>) {
+ return slab_heap_container->resource_limit;
+ } else if constexpr (std::is_same_v<T, KSession>) {
+ return slab_heap_container->session;
+ } else if constexpr (std::is_same_v<T, KSharedMemory>) {
+ return slab_heap_container->shared_memory;
+ } else if constexpr (std::is_same_v<T, KSharedMemoryInfo>) {
+ return slab_heap_container->shared_memory_info;
+ } else if constexpr (std::is_same_v<T, KThread>) {
+ return slab_heap_container->thread;
+ } else if constexpr (std::is_same_v<T, KTransferMemory>) {
+ return slab_heap_container->transfer_memory;
+ } else if constexpr (std::is_same_v<T, KCodeMemory>) {
+ return slab_heap_container->code_memory;
+ } else if constexpr (std::is_same_v<T, KDeviceAddressSpace>) {
+ return slab_heap_container->device_address_space;
+ } else if constexpr (std::is_same_v<T, KPageBuffer>) {
+ return slab_heap_container->page_buffer;
+ } else if constexpr (std::is_same_v<T, KThreadLocalPage>) {
+ return slab_heap_container->thread_local_page;
+ } else if constexpr (std::is_same_v<T, KObjectName>) {
+ return slab_heap_container->object_name;
+ } else if constexpr (std::is_same_v<T, KSessionRequest>) {
+ return slab_heap_container->session_request;
+ } else if constexpr (std::is_same_v<T, KSecureSystemResource>) {
+ return slab_heap_container->secure_system_resource;
+ } else if constexpr (std::is_same_v<T, KThread::LockWithPriorityInheritanceInfo>) {
+ return slab_heap_container->lock_info;
+ } else if constexpr (std::is_same_v<T, KEventInfo>) {
+ return slab_heap_container->event_info;
+ } else if constexpr (std::is_same_v<T, KDebug>) {
+ return slab_heap_container->debug;
+ }
+}
+
+template KSlabHeap<KClientSession>& KernelCore::SlabHeap();
+template KSlabHeap<KEvent>& KernelCore::SlabHeap();
+template KSlabHeap<KLinkedListNode>& KernelCore::SlabHeap();
+template KSlabHeap<KPort>& KernelCore::SlabHeap();
+template KSlabHeap<KProcess>& KernelCore::SlabHeap();
+template KSlabHeap<KResourceLimit>& KernelCore::SlabHeap();
+template KSlabHeap<KSession>& KernelCore::SlabHeap();
+template KSlabHeap<KSharedMemory>& KernelCore::SlabHeap();
+template KSlabHeap<KSharedMemoryInfo>& KernelCore::SlabHeap();
+template KSlabHeap<KThread>& KernelCore::SlabHeap();
+template KSlabHeap<KTransferMemory>& KernelCore::SlabHeap();
+template KSlabHeap<KCodeMemory>& KernelCore::SlabHeap();
+template KSlabHeap<KDeviceAddressSpace>& KernelCore::SlabHeap();
+template KSlabHeap<KPageBuffer>& KernelCore::SlabHeap();
+template KSlabHeap<KThreadLocalPage>& KernelCore::SlabHeap();
+template KSlabHeap<KObjectName>& KernelCore::SlabHeap();
+template KSlabHeap<KSessionRequest>& KernelCore::SlabHeap();
+template KSlabHeap<KSecureSystemResource>& KernelCore::SlabHeap();
+template KSlabHeap<KThread::LockWithPriorityInheritanceInfo>& KernelCore::SlabHeap();
+template KSlabHeap<KEventInfo>& KernelCore::SlabHeap();
+template KSlabHeap<KDebug>& KernelCore::SlabHeap();
+
} // namespace Kernel
diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h
index 4449f6949..1b380a07b 100644
--- a/src/core/hle/kernel/kernel.h
+++ b/src/core/hle/kernel/kernel.h
@@ -305,49 +305,7 @@ public:
/// Gets the slab heap for the specified kernel object type.
template <typename T>
- KSlabHeap<T>& SlabHeap() {
- if constexpr (std::is_same_v<T, KClientSession>) {
- return slab_heap_container->client_session;
- } else if constexpr (std::is_same_v<T, KEvent>) {
- return slab_heap_container->event;
- } else if constexpr (std::is_same_v<T, KLinkedListNode>) {
- return slab_heap_container->linked_list_node;
- } else if constexpr (std::is_same_v<T, KPort>) {
- return slab_heap_container->port;
- } else if constexpr (std::is_same_v<T, KProcess>) {
- return slab_heap_container->process;
- } else if constexpr (std::is_same_v<T, KResourceLimit>) {
- return slab_heap_container->resource_limit;
- } else if constexpr (std::is_same_v<T, KSession>) {
- return slab_heap_container->session;
- } else if constexpr (std::is_same_v<T, KSharedMemory>) {
- return slab_heap_container->shared_memory;
- } else if constexpr (std::is_same_v<T, KSharedMemoryInfo>) {
- return slab_heap_container->shared_memory_info;
- } else if constexpr (std::is_same_v<T, KThread>) {
- return slab_heap_container->thread;
- } else if constexpr (std::is_same_v<T, KTransferMemory>) {
- return slab_heap_container->transfer_memory;
- } else if constexpr (std::is_same_v<T, KCodeMemory>) {
- return slab_heap_container->code_memory;
- } else if constexpr (std::is_same_v<T, KDeviceAddressSpace>) {
- return slab_heap_container->device_address_space;
- } else if constexpr (std::is_same_v<T, KPageBuffer>) {
- return slab_heap_container->page_buffer;
- } else if constexpr (std::is_same_v<T, KThreadLocalPage>) {
- return slab_heap_container->thread_local_page;
- } else if constexpr (std::is_same_v<T, KObjectName>) {
- return slab_heap_container->object_name;
- } else if constexpr (std::is_same_v<T, KSessionRequest>) {
- return slab_heap_container->session_request;
- } else if constexpr (std::is_same_v<T, KSecureSystemResource>) {
- return slab_heap_container->secure_system_resource;
- } else if constexpr (std::is_same_v<T, KEventInfo>) {
- return slab_heap_container->event_info;
- } else if constexpr (std::is_same_v<T, KDebug>) {
- return slab_heap_container->debug;
- }
- }
+ KSlabHeap<T>& SlabHeap();
/// Gets the current slab resource counts.
Init::KSlabResourceCounts& SlabResourceCounts();
@@ -393,28 +351,7 @@ private:
private:
/// Helper to encapsulate all slab heaps in a single heap allocated container
- struct SlabHeapContainer {
- KSlabHeap<KClientSession> client_session;
- KSlabHeap<KEvent> event;
- KSlabHeap<KLinkedListNode> linked_list_node;
- KSlabHeap<KPort> port;
- KSlabHeap<KProcess> process;
- KSlabHeap<KResourceLimit> resource_limit;
- KSlabHeap<KSession> session;
- KSlabHeap<KSharedMemory> shared_memory;
- KSlabHeap<KSharedMemoryInfo> shared_memory_info;
- KSlabHeap<KThread> thread;
- KSlabHeap<KTransferMemory> transfer_memory;
- KSlabHeap<KCodeMemory> code_memory;
- KSlabHeap<KDeviceAddressSpace> device_address_space;
- KSlabHeap<KPageBuffer> page_buffer;
- KSlabHeap<KThreadLocalPage> thread_local_page;
- KSlabHeap<KObjectName> object_name;
- KSlabHeap<KSessionRequest> session_request;
- KSlabHeap<KSecureSystemResource> secure_system_resource;
- KSlabHeap<KEventInfo> event_info;
- KSlabHeap<KDebug> debug;
- };
+ struct SlabHeapContainer;
std::unique_ptr<SlabHeapContainer> slab_heap_container;
};
diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h
index 542c13461..39355d9c4 100644
--- a/src/core/hle/kernel/svc_types.h
+++ b/src/core/hle/kernel/svc_types.h
@@ -151,6 +151,7 @@ enum class InfoType : u32 {
FreeThreadCount = 24,
ThreadTickCount = 25,
IsSvcPermitted = 26,
+ IoRegionHint = 27,
MesosphereMeta = 65000,
MesosphereCurrentProcess = 65001,
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-31 13:16:54 +0100