diff options
Diffstat (limited to 'src/core/hle/kernel')
40 files changed, 1110 insertions, 515 deletions
diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp index 8027bec00..7765e7848 100644 --- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp +++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp @@ -148,9 +148,9 @@ u64 GenerateUniformRange(u64 min, u64 max, F f) { } // Anonymous namespace u64 KSystemControl::GenerateRandomU64() { - static std::random_device device; - static std::mt19937 gen(device()); - static std::uniform_int_distribution<u64> distribution(1, std::numeric_limits<u64>::max()); + std::random_device device; + std::mt19937 gen(device()); + std::uniform_int_distribution<u64> distribution(1, std::numeric_limits<u64>::max()); return distribution(gen); } diff --git a/src/core/hle/kernel/hle_ipc.cpp b/src/core/hle/kernel/hle_ipc.cpp index e19544c54..5828ac923 100644 --- a/src/core/hle/kernel/hle_ipc.cpp +++ b/src/core/hle/kernel/hle_ipc.cpp @@ -17,7 +17,6 @@ #include "core/hle/kernel/k_auto_object.h" #include "core/hle/kernel/k_handle_table.h" #include "core/hle/kernel/k_process.h" -#include "core/hle/kernel/k_readable_event.h" #include "core/hle/kernel/k_server_session.h" #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/kernel.h" @@ -25,8 +24,15 @@ namespace Kernel { -SessionRequestHandler::SessionRequestHandler(KernelCore& kernel_, const char* service_name_) - : kernel{kernel_}, service_thread{kernel.CreateServiceThread(service_name_)} {} +SessionRequestHandler::SessionRequestHandler(KernelCore& kernel_, const char* service_name_, + ServiceThreadType thread_type) + : kernel{kernel_} { + if (thread_type == ServiceThreadType::CreateNew) { + service_thread = kernel.CreateServiceThread(service_name_); + } else { + service_thread = kernel.GetDefaultServiceThread(); + } +} SessionRequestHandler::~SessionRequestHandler() { kernel.ReleaseServiceThread(service_thread); @@ -45,7 +51,7 @@ bool SessionRequestManager::HasSessionRequestHandler(const HLERequestContext& co LOG_CRITICAL(IPC, "object_id {} is too big!", object_id); return false; } - return DomainHandler(object_id - 1) != nullptr; + return !DomainHandler(object_id - 1).expired(); } else { return session_handler != nullptr; } @@ -55,7 +61,7 @@ void SessionRequestHandler::ClientConnected(KServerSession* session) { session->ClientConnected(shared_from_this()); // Ensure our server session is tracked globally. - kernel.RegisterServerSession(session); + kernel.RegisterServerObject(session); } void SessionRequestHandler::ClientDisconnected(KServerSession* session) { diff --git a/src/core/hle/kernel/hle_ipc.h b/src/core/hle/kernel/hle_ipc.h index 754b41ff6..640146137 100644 --- a/src/core/hle/kernel/hle_ipc.h +++ b/src/core/hle/kernel/hle_ipc.h @@ -33,6 +33,11 @@ namespace Service { class ServiceFrameworkBase; } +enum class ServiceThreadType { + Default, + CreateNew, +}; + namespace Kernel { class Domain; @@ -57,7 +62,8 @@ enum class ThreadWakeupReason; */ class SessionRequestHandler : public std::enable_shared_from_this<SessionRequestHandler> { public: - SessionRequestHandler(KernelCore& kernel, const char* service_name_); + SessionRequestHandler(KernelCore& kernel_, const char* service_name_, + ServiceThreadType thread_type); virtual ~SessionRequestHandler(); /** @@ -94,6 +100,7 @@ protected: std::weak_ptr<ServiceThread> service_thread; }; +using SessionRequestHandlerWeakPtr = std::weak_ptr<SessionRequestHandler>; using SessionRequestHandlerPtr = std::shared_ptr<SessionRequestHandler>; /** @@ -139,7 +146,7 @@ public: } } - SessionRequestHandlerPtr DomainHandler(std::size_t index) const { + SessionRequestHandlerWeakPtr DomainHandler(std::size_t index) const { ASSERT_MSG(index < DomainHandlerCount(), "Unexpected handler index {}", index); return domain_handlers.at(index); } @@ -328,10 +335,10 @@ public: template <typename T> std::shared_ptr<T> GetDomainHandler(std::size_t index) const { - return std::static_pointer_cast<T>(manager->DomainHandler(index)); + return std::static_pointer_cast<T>(manager.lock()->DomainHandler(index).lock()); } - void SetSessionRequestManager(std::shared_ptr<SessionRequestManager> manager_) { + void SetSessionRequestManager(std::weak_ptr<SessionRequestManager> manager_) { manager = std::move(manager_); } @@ -374,7 +381,7 @@ private: u32 handles_offset{}; u32 domain_offset{}; - std::shared_ptr<SessionRequestManager> manager; + std::weak_ptr<SessionRequestManager> manager; KernelCore& kernel; Core::Memory::Memory& memory; diff --git a/src/core/hle/kernel/init/init_slab_setup.cpp b/src/core/hle/kernel/init/init_slab_setup.cpp index 36fc0944a..b0f773ee0 100644 --- a/src/core/hle/kernel/init/init_slab_setup.cpp +++ b/src/core/hle/kernel/init/init_slab_setup.cpp @@ -7,19 +7,23 @@ #include "common/common_funcs.h" #include "common/common_types.h" #include "core/core.h" +#include "core/device_memory.h" #include "core/hardware_properties.h" #include "core/hle/kernel/init/init_slab_setup.h" #include "core/hle/kernel/k_code_memory.h" #include "core/hle/kernel/k_event.h" #include "core/hle/kernel/k_memory_layout.h" #include "core/hle/kernel/k_memory_manager.h" +#include "core/hle/kernel/k_page_buffer.h" #include "core/hle/kernel/k_port.h" #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_session.h" #include "core/hle/kernel/k_shared_memory.h" +#include "core/hle/kernel/k_shared_memory_info.h" #include "core/hle/kernel/k_system_control.h" #include "core/hle/kernel/k_thread.h" +#include "core/hle/kernel/k_thread_local_page.h" #include "core/hle/kernel/k_transfer_memory.h" namespace Kernel::Init { @@ -32,9 +36,13 @@ namespace Kernel::Init { HANDLER(KEvent, (SLAB_COUNT(KEvent)), ##__VA_ARGS__) \ HANDLER(KPort, (SLAB_COUNT(KPort)), ##__VA_ARGS__) \ HANDLER(KSharedMemory, (SLAB_COUNT(KSharedMemory)), ##__VA_ARGS__) \ + HANDLER(KSharedMemoryInfo, (SLAB_COUNT(KSharedMemory) * 8), ##__VA_ARGS__) \ HANDLER(KTransferMemory, (SLAB_COUNT(KTransferMemory)), ##__VA_ARGS__) \ HANDLER(KCodeMemory, (SLAB_COUNT(KCodeMemory)), ##__VA_ARGS__) \ HANDLER(KSession, (SLAB_COUNT(KSession)), ##__VA_ARGS__) \ + HANDLER(KThreadLocalPage, \ + (SLAB_COUNT(KProcess) + (SLAB_COUNT(KProcess) + SLAB_COUNT(KThread)) / 8), \ + ##__VA_ARGS__) \ HANDLER(KResourceLimit, (SLAB_COUNT(KResourceLimit)), ##__VA_ARGS__) namespace { @@ -50,38 +58,46 @@ enum KSlabType : u32 { // Constexpr counts. constexpr size_t SlabCountKProcess = 80; constexpr size_t SlabCountKThread = 800; -constexpr size_t SlabCountKEvent = 700; +constexpr size_t SlabCountKEvent = 900; constexpr size_t SlabCountKInterruptEvent = 100; -constexpr size_t SlabCountKPort = 256 + 0x20; // Extra 0x20 ports over Nintendo for homebrew. +constexpr size_t SlabCountKPort = 384; constexpr size_t SlabCountKSharedMemory = 80; constexpr size_t SlabCountKTransferMemory = 200; constexpr size_t SlabCountKCodeMemory = 10; constexpr size_t SlabCountKDeviceAddressSpace = 300; -constexpr size_t SlabCountKSession = 933; +constexpr size_t SlabCountKSession = 1133; constexpr size_t SlabCountKLightSession = 100; constexpr size_t SlabCountKObjectName = 7; constexpr size_t SlabCountKResourceLimit = 5; constexpr size_t SlabCountKDebug = Core::Hardware::NUM_CPU_CORES; -constexpr size_t SlabCountKAlpha = 1; -constexpr size_t SlabCountKBeta = 6; +constexpr size_t SlabCountKIoPool = 1; +constexpr size_t SlabCountKIoRegion = 6; constexpr size_t SlabCountExtraKThread = 160; +/// Helper function to translate from the slab virtual address to the reserved location in physical +/// memory. +static PAddr TranslateSlabAddrToPhysical(KMemoryLayout& memory_layout, VAddr slab_addr) { + slab_addr -= memory_layout.GetSlabRegionAddress(); + return slab_addr + Core::DramMemoryMap::SlabHeapBase; +} + template <typename T> VAddr InitializeSlabHeap(Core::System& system, KMemoryLayout& memory_layout, VAddr address, size_t num_objects) { - // TODO(bunnei): This is just a place holder. We should initialize the appropriate KSlabHeap for - // kernel object type T with the backing kernel memory pointer once we emulate kernel memory. const size_t size = Common::AlignUp(sizeof(T) * num_objects, alignof(void*)); VAddr start = Common::AlignUp(address, alignof(T)); - // This is intentionally empty. Once KSlabHeap is fully implemented, we can replace this with - // the pointer to emulated memory to pass along. Until then, KSlabHeap will just allocate/free - // host memory. - void* backing_kernel_memory{}; + // This should use the virtual memory address passed in, but currently, we do not setup the + // kernel virtual memory layout. Instead, we simply map these at a region of physical memory + // that we reserve for the slab heaps. + // TODO(bunnei): Fix this once we support the kernel virtual memory layout. if (size > 0) { + void* backing_kernel_memory{ + system.DeviceMemory().GetPointer(TranslateSlabAddrToPhysical(memory_layout, start))}; + const KMemoryRegion* region = memory_layout.FindVirtual(start + size - 1); ASSERT(region != nullptr); ASSERT(region->IsDerivedFrom(KMemoryRegionType_KernelSlab)); @@ -91,6 +107,12 @@ VAddr InitializeSlabHeap(Core::System& system, KMemoryLayout& memory_layout, VAd return start + size; } +size_t CalculateSlabHeapGapSize() { + constexpr size_t KernelSlabHeapGapSize = 2_MiB - 296_KiB; + static_assert(KernelSlabHeapGapSize <= KernelSlabHeapGapsSizeMax); + return KernelSlabHeapGapSize; +} + } // namespace KSlabResourceCounts KSlabResourceCounts::CreateDefault() { @@ -109,8 +131,8 @@ KSlabResourceCounts KSlabResourceCounts::CreateDefault() { .num_KObjectName = SlabCountKObjectName, .num_KResourceLimit = SlabCountKResourceLimit, .num_KDebug = SlabCountKDebug, - .num_KAlpha = SlabCountKAlpha, - .num_KBeta = SlabCountKBeta, + .num_KIoPool = SlabCountKIoPool, + .num_KIoRegion = SlabCountKIoRegion, }; } @@ -136,11 +158,34 @@ size_t CalculateTotalSlabHeapSize(const KernelCore& kernel) { #undef ADD_SLAB_SIZE // Add the reserved size. - size += KernelSlabHeapGapsSize; + size += CalculateSlabHeapGapSize(); return size; } +void InitializeKPageBufferSlabHeap(Core::System& system) { + auto& kernel = system.Kernel(); + + const auto& counts = kernel.SlabResourceCounts(); + const size_t num_pages = + counts.num_KProcess + counts.num_KThread + (counts.num_KProcess + counts.num_KThread) / 8; + const size_t slab_size = num_pages * PageSize; + + // Reserve memory from the system resource limit. + ASSERT(kernel.GetSystemResourceLimit()->Reserve(LimitableResource::PhysicalMemory, slab_size)); + + // Allocate memory for the slab. + constexpr auto AllocateOption = KMemoryManager::EncodeOption( + KMemoryManager::Pool::System, KMemoryManager::Direction::FromFront); + const PAddr slab_address = + kernel.MemoryManager().AllocateAndOpenContinuous(num_pages, 1, AllocateOption); + ASSERT(slab_address != 0); + + // Initialize the slabheap. + KPageBuffer::InitializeSlabHeap(kernel, system.DeviceMemory().GetPointer(slab_address), + slab_size); +} + void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) { auto& kernel = system.Kernel(); @@ -160,13 +205,13 @@ void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) { } // Create an array to represent the gaps between the slabs. - const size_t total_gap_size = KernelSlabHeapGapsSize; + const size_t total_gap_size = CalculateSlabHeapGapSize(); std::array<size_t, slab_types.size()> slab_gaps; - for (size_t i = 0; i < slab_gaps.size(); i++) { + for (auto& slab_gap : slab_gaps) { // Note: This is an off-by-one error from Nintendo's intention, because GenerateRandomRange // is inclusive. However, Nintendo also has the off-by-one error, and it's "harmless", so we // will include it ourselves. - slab_gaps[i] = KSystemControl::GenerateRandomRange(0, total_gap_size); + slab_gap = KSystemControl::GenerateRandomRange(0, total_gap_size); } // Sort the array, so that we can treat differences between values as offsets to the starts of @@ -177,13 +222,21 @@ void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) { } } - for (size_t i = 0; i < slab_types.size(); i++) { + // Track the gaps, so that we can free them to the unused slab tree. + VAddr gap_start = address; + size_t gap_size = 0; + + for (size_t i = 0; i < slab_gaps.size(); i++) { // Add the random gap to the address. - address += (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1]; + const auto cur_gap = (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1]; + address += cur_gap; + gap_size += cur_gap; #define INITIALIZE_SLAB_HEAP(NAME, COUNT, ...) \ case KSlabType_##NAME: \ - address = InitializeSlabHeap<NAME>(system, memory_layout, address, COUNT); \ + if (COUNT > 0) { \ + address = InitializeSlabHeap<NAME>(system, memory_layout, address, COUNT); \ + } \ break; // Initialize the slabheap. @@ -192,7 +245,13 @@ void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) { FOREACH_SLAB_TYPE(INITIALIZE_SLAB_HEAP) // If we somehow get an invalid type, abort. default: - UNREACHABLE(); + UNREACHABLE_MSG("Unknown slab type: {}", slab_types[i]); + } + + // If we've hit the end of a gap, free it. + if (gap_start + gap_size != address) { + gap_start = address; + gap_size = 0; } } } diff --git a/src/core/hle/kernel/init/init_slab_setup.h b/src/core/hle/kernel/init/init_slab_setup.h index a8f7e0918..f54b67d02 100644 --- a/src/core/hle/kernel/init/init_slab_setup.h +++ b/src/core/hle/kernel/init/init_slab_setup.h @@ -32,12 +32,13 @@ struct KSlabResourceCounts { size_t num_KObjectName; size_t num_KResourceLimit; size_t num_KDebug; - size_t num_KAlpha; - size_t num_KBeta; + size_t num_KIoPool; + size_t num_KIoRegion; }; void InitializeSlabResourceCounts(KernelCore& kernel); size_t CalculateTotalSlabHeapSize(const KernelCore& kernel); +void InitializeKPageBufferSlabHeap(Core::System& system); void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout); } // namespace Kernel::Init diff --git a/src/core/hle/kernel/k_address_arbiter.cpp b/src/core/hle/kernel/k_address_arbiter.cpp index 1d1f5e5f8..8cdd0490f 100644 --- a/src/core/hle/kernel/k_address_arbiter.cpp +++ b/src/core/hle/kernel/k_address_arbiter.cpp @@ -115,7 +115,7 @@ ResultCode KAddressArbiter::Signal(VAddr addr, s32 count) { { KScopedSchedulerLock sl(kernel); - auto it = thread_tree.nfind_light({addr, -1}); + auto it = thread_tree.nfind_key({addr, -1}); while ((it != thread_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) { // End the thread's wait. @@ -148,7 +148,7 @@ ResultCode KAddressArbiter::SignalAndIncrementIfEqual(VAddr addr, s32 value, s32 return ResultInvalidState; } - auto it = thread_tree.nfind_light({addr, -1}); + auto it = thread_tree.nfind_key({addr, -1}); while ((it != thread_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) { // End the thread's wait. @@ -171,7 +171,7 @@ ResultCode KAddressArbiter::SignalAndModifyByWaitingCountIfEqual(VAddr addr, s32 { [[maybe_unused]] const KScopedSchedulerLock sl(kernel); - auto it = thread_tree.nfind_light({addr, -1}); + auto it = thread_tree.nfind_key({addr, -1}); // Determine the updated value. s32 new_value{}; if (count <= 0) { diff --git a/src/core/hle/kernel/k_auto_object.h b/src/core/hle/kernel/k_auto_object.h index 05779f2d5..423e8d8f5 100644 --- a/src/core/hle/kernel/k_auto_object.h +++ b/src/core/hle/kernel/k_auto_object.h @@ -89,9 +89,7 @@ public: explicit KAutoObject(KernelCore& kernel_) : kernel(kernel_) { RegisterWithKernel(); } - virtual ~KAutoObject() { - UnregisterWithKernel(); - } + virtual ~KAutoObject() = default; static KAutoObject* Create(KAutoObject* ptr); @@ -163,11 +161,12 @@ public: do { ASSERT(cur_ref_count > 0); } while (!m_ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count - 1, - std::memory_order_relaxed)); + std::memory_order_acq_rel)); // If ref count hits zero, destroy the object. if (cur_ref_count - 1 == 0) { this->Destroy(); + this->UnregisterWithKernel(); } } diff --git a/src/core/hle/kernel/k_class_token.h b/src/core/hle/kernel/k_class_token.h index 980010150..93fc8786d 100644 --- a/src/core/hle/kernel/k_class_token.h +++ b/src/core/hle/kernel/k_class_token.h @@ -4,8 +4,6 @@ #pragma once -#include <atomic> - #include "common/bit_util.h" #include "common/common_types.h" diff --git a/src/core/hle/kernel/k_code_memory.cpp b/src/core/hle/kernel/k_code_memory.cpp index 0b225e8e0..09eaf004c 100644 --- a/src/core/hle/kernel/k_code_memory.cpp +++ b/src/core/hle/kernel/k_code_memory.cpp @@ -5,7 +5,6 @@ #include "common/alignment.h" #include "common/common_types.h" #include "core/device_memory.h" -#include "core/hle/kernel/k_auto_object.h" #include "core/hle/kernel/k_code_memory.h" #include "core/hle/kernel/k_light_lock.h" #include "core/hle/kernel/k_memory_block.h" @@ -29,15 +28,21 @@ ResultCode KCodeMemory::Initialize(Core::DeviceMemory& device_memory, VAddr addr auto& page_table = m_owner->PageTable(); // Construct the page group. - m_page_group = KPageLinkedList(addr, Common::DivideUp(size, PageSize)); + m_page_group = + KPageLinkedList(page_table.GetPhysicalAddr(addr), Common::DivideUp(size, PageSize)); // Lock the memory. R_TRY(page_table.LockForCodeMemory(addr, size)) // Clear the memory. - for (const auto& block : m_page_group.Nodes()) { - std::memset(device_memory.GetPointer(block.GetAddress()), 0xFF, block.GetSize()); - } + // + // FIXME: this ends up clobbering address ranges outside the scope of the mapping within + // guest memory, and is not specifically required if the guest program is correctly + // written, so disable until this is further investigated. + // + // for (const auto& block : m_page_group.Nodes()) { + // std::memset(device_memory.GetPointer(block.GetAddress()), 0xFF, block.GetSize()); + // } // Set remaining tracking members. m_address = addr; diff --git a/src/core/hle/kernel/k_condition_variable.cpp b/src/core/hle/kernel/k_condition_variable.cpp index aadcc297a..aa29922d0 100644 --- a/src/core/hle/kernel/k_condition_variable.cpp +++ b/src/core/hle/kernel/k_condition_variable.cpp @@ -9,7 +9,6 @@ #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_scheduler.h" #include "core/hle/kernel/k_scoped_scheduler_lock_and_sleep.h" -#include "core/hle/kernel/k_synchronization_object.h" #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_thread_queue.h" #include "core/hle/kernel/kernel.h" @@ -244,7 +243,7 @@ void KConditionVariable::Signal(u64 cv_key, s32 count) { { KScopedSchedulerLock sl(kernel); - auto it = thread_tree.nfind_light({cv_key, -1}); + auto it = thread_tree.nfind_key({cv_key, -1}); while ((it != thread_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetConditionVariableKey() == cv_key)) { KThread* target_thread = std::addressof(*it); diff --git a/src/core/hle/kernel/k_handle_table.cpp b/src/core/hle/kernel/k_handle_table.cpp index cf95f0852..db7512ee7 100644 --- a/src/core/hle/kernel/k_handle_table.cpp +++ b/src/core/hle/kernel/k_handle_table.cpp @@ -63,7 +63,7 @@ bool KHandleTable::Remove(Handle handle) { return true; } -ResultCode KHandleTable::Add(Handle* out_handle, KAutoObject* obj, u16 type) { +ResultCode KHandleTable::Add(Handle* out_handle, KAutoObject* obj) { KScopedDisableDispatch dd(kernel); KScopedSpinLock lk(m_lock); @@ -75,7 +75,7 @@ ResultCode KHandleTable::Add(Handle* out_handle, KAutoObject* obj, u16 type) { const auto linear_id = this->AllocateLinearId(); const auto index = this->AllocateEntry(); - m_entry_infos[index].info = {.linear_id = linear_id, .type = type}; + m_entry_infos[index].linear_id = linear_id; m_objects[index] = obj; obj->Open(); @@ -116,7 +116,7 @@ void KHandleTable::Unreserve(Handle handle) { } } -void KHandleTable::Register(Handle handle, KAutoObject* obj, u16 type) { +void KHandleTable::Register(Handle handle, KAutoObject* obj) { KScopedDisableDispatch dd(kernel); KScopedSpinLock lk(m_lock); @@ -132,7 +132,7 @@ void KHandleTable::Register(Handle handle, KAutoObject* obj, u16 type) { // Set the entry. ASSERT(m_objects[index] == nullptr); - m_entry_infos[index].info = {.linear_id = static_cast<u16>(linear_id), .type = type}; + m_entry_infos[index].linear_id = static_cast<u16>(linear_id); m_objects[index] = obj; obj->Open(); diff --git a/src/core/hle/kernel/k_handle_table.h b/src/core/hle/kernel/k_handle_table.h index 87004a0f9..dd27689b6 100644 --- a/src/core/hle/kernel/k_handle_table.h +++ b/src/core/hle/kernel/k_handle_table.h @@ -42,7 +42,7 @@ public: m_free_head_index = -1; // Free all entries. - for (s32 i = 0; i < static_cast<s32>(m_table_size); ++i) { + for (s16 i = 0; i < static_cast<s16>(m_table_size); ++i) { m_objects[i] = nullptr; m_entry_infos[i].next_free_index = i - 1; m_free_head_index = i; @@ -104,17 +104,8 @@ public: ResultCode Reserve(Handle* out_handle); void Unreserve(Handle handle); - template <typename T> - ResultCode Add(Handle* out_handle, T* obj) { - static_assert(std::is_base_of_v<KAutoObject, T>); - return this->Add(out_handle, obj, obj->GetTypeObj().GetClassToken()); - } - - template <typename T> - void Register(Handle handle, T* obj) { - static_assert(std::is_base_of_v<KAutoObject, T>); - return this->Register(handle, obj, obj->GetTypeObj().GetClassToken()); - } + ResultCode Add(Handle* out_handle, KAutoObject* obj); + void Register(Handle handle, KAutoObject* obj); template <typename T> bool GetMultipleObjects(T** out, const Handle* handles, size_t num_handles) const { @@ -160,9 +151,6 @@ public: } private: - ResultCode Add(Handle* out_handle, KAutoObject* obj, u16 type); - void Register(Handle handle, KAutoObject* obj, u16 type); - s32 AllocateEntry() { ASSERT(m_count < m_table_size); @@ -179,7 +167,7 @@ private: ASSERT(m_count > 0); m_objects[index] = nullptr; - m_entry_infos[index].next_free_index = m_free_head_index; + m_entry_infos[index].next_free_index = static_cast<s16>(m_free_head_index); m_free_head_index = index; @@ -278,19 +266,13 @@ private: } union EntryInfo { - struct { - u16 linear_id; - u16 type; - } info; - s32 next_free_index; + u16 linear_id; + s16 next_free_index; constexpr u16 GetLinearId() const { - return info.linear_id; - } - constexpr u16 GetType() const { - return info.type; + return linear_id; } - constexpr s32 GetNextFreeIndex() const { + constexpr s16 GetNextFreeIndex() const { return next_free_index; } }; diff --git a/src/core/hle/kernel/k_memory_layout.h b/src/core/hle/kernel/k_memory_layout.h index bcddb0d62..0858827b6 100644 --- a/src/core/hle/kernel/k_memory_layout.h +++ b/src/core/hle/kernel/k_memory_layout.h @@ -57,11 +57,11 @@ constexpr std::size_t KernelPageTableHeapSize = GetMaximumOverheadSize(MainMemor constexpr std::size_t KernelInitialPageHeapSize = 128_KiB; constexpr std::size_t KernelSlabHeapDataSize = 5_MiB; -constexpr std::size_t KernelSlabHeapGapsSize = 2_MiB - 64_KiB; -constexpr std::size_t KernelSlabHeapSize = KernelSlabHeapDataSize + KernelSlabHeapGapsSize; +constexpr std::size_t KernelSlabHeapGapsSizeMax = 2_MiB - 64_KiB; +constexpr std::size_t KernelSlabHeapSize = KernelSlabHeapDataSize + KernelSlabHeapGapsSizeMax; // NOTE: This is calculated from KThread slab counts, assuming KThread size <= 0x860. -constexpr std::size_t KernelSlabHeapAdditionalSize = 416_KiB; +constexpr std::size_t KernelSlabHeapAdditionalSize = 0x68000; constexpr std::size_t KernelResourceSize = KernelPageTableHeapSize + KernelInitialPageHeapSize + KernelSlabHeapSize; diff --git a/src/core/hle/kernel/k_memory_manager.cpp b/src/core/hle/kernel/k_memory_manager.cpp index a2f18f643..17a1b76c7 100644 --- a/src/core/hle/kernel/k_memory_manager.cpp +++ b/src/core/hle/kernel/k_memory_manager.cpp @@ -15,7 +15,6 @@ #include "core/hle/kernel/k_page_linked_list.h" #include "core/hle/kernel/kernel.h" #include "core/hle/kernel/svc_results.h" -#include "core/memory.h" namespace Kernel { diff --git a/src/core/hle/kernel/k_page_buffer.cpp b/src/core/hle/kernel/k_page_buffer.cpp new file mode 100644 index 000000000..f7df4a9a8 --- /dev/null +++ b/src/core/hle/kernel/k_page_buffer.cpp @@ -0,0 +1,19 @@ +// Copyright 2022 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "common/alignment.h" +#include "common/assert.h" +#include "core/core.h" +#include "core/device_memory.h" +#include "core/hle/kernel/k_page_buffer.h" +#include "core/hle/kernel/memory_types.h" + +namespace Kernel { + +KPageBuffer* KPageBuffer::FromPhysicalAddress(Core::System& system, PAddr phys_addr) { + ASSERT(Common::IsAligned(phys_addr, PageSize)); + return reinterpret_cast<KPageBuffer*>(system.DeviceMemory().GetPointer(phys_addr)); +} + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_page_buffer.h b/src/core/hle/kernel/k_page_buffer.h new file mode 100644 index 000000000..1d11a4e27 --- /dev/null +++ b/src/core/hle/kernel/k_page_buffer.h @@ -0,0 +1,28 @@ +// Copyright 2022 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include <array> + +#include "common/common_types.h" +#include "core/hle/kernel/memory_types.h" +#include "core/hle/kernel/slab_helpers.h" + +namespace Kernel { + +class KPageBuffer final : public KSlabAllocated<KPageBuffer> { +public: + KPageBuffer() = default; + + static KPageBuffer* FromPhysicalAddress(Core::System& system, PAddr phys_addr); + +private: + [[maybe_unused]] alignas(PageSize) std::array<u8, PageSize> m_buffer{}; +}; + +static_assert(sizeof(KPageBuffer) == PageSize); +static_assert(alignof(KPageBuffer) == PageSize); + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_page_linked_list.h b/src/core/hle/kernel/k_page_linked_list.h index 0e2ae582a..869228322 100644 --- a/src/core/hle/kernel/k_page_linked_list.h +++ b/src/core/hle/kernel/k_page_linked_list.h @@ -89,6 +89,10 @@ public: return ResultSuccess; } + bool Empty() const { + return nodes.empty(); + } + private: std::list<Node> nodes; }; diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index 0602de1f7..47ea3c89c 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -346,7 +346,8 @@ ResultCode KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std:: return ResultSuccess; } -ResultCode KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size) { +ResultCode KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size, + ICacheInvalidationStrategy icache_invalidation_strategy) { // Validate the mapping request. R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), ResultInvalidMemoryRegion); @@ -396,7 +397,11 @@ ResultCode KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std bool reprotected_pages = false; SCOPE_EXIT({ if (reprotected_pages && any_code_pages) { - system.InvalidateCpuInstructionCacheRange(dst_address, size); + if (icache_invalidation_strategy == ICacheInvalidationStrategy::InvalidateRange) { + system.InvalidateCpuInstructionCacheRange(dst_address, size); + } else { + system.InvalidateCpuInstructionCaches(); + } } }); @@ -424,6 +429,120 @@ ResultCode KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std return ResultSuccess; } +VAddr KPageTable::FindFreeArea(VAddr region_start, std::size_t region_num_pages, + std::size_t num_pages, std::size_t alignment, std::size_t offset, + std::size_t guard_pages) { + VAddr address = 0; + + if (num_pages <= region_num_pages) { + if (this->IsAslrEnabled()) { + // Try to directly find a free area up to 8 times. + for (std::size_t i = 0; i < 8; i++) { + const std::size_t random_offset = + KSystemControl::GenerateRandomRange( + 0, (region_num_pages - num_pages - guard_pages) * PageSize / alignment) * + alignment; + const VAddr candidate = + Common::AlignDown((region_start + random_offset), alignment) + offset; + + KMemoryInfo info = this->QueryInfoImpl(candidate); + + if (info.state != KMemoryState::Free) { + continue; + } + if (region_start > candidate) { + continue; + } + if (info.GetAddress() + guard_pages * PageSize > candidate) { + continue; + } + + const VAddr candidate_end = candidate + (num_pages + guard_pages) * PageSize - 1; + if (candidate_end > info.GetLastAddress()) { + continue; + } + if (candidate_end > region_start + region_num_pages * PageSize - 1) { + continue; + } + + address = candidate; + break; + } + // Fall back to finding the first free area with a random offset. + if (address == 0) { + // NOTE: Nintendo does not account for guard pages here. + // This may theoretically cause an offset to be chosen that cannot be mapped. We + // will account for guard pages. + const std::size_t offset_pages = KSystemControl::GenerateRandomRange( + 0, region_num_pages - num_pages - guard_pages); + address = block_manager->FindFreeArea(region_start + offset_pages * PageSize, + region_num_pages - offset_pages, num_pages, + alignment, offset, guard_pages); + } + } + + // Find the first free area. + if (address == 0) { + address = block_manager->FindFreeArea(region_start, region_num_pages, num_pages, + alignment, offset, guard_pages); + } + } + + return address; +} + +ResultCode KPageTable::MakePageGroup(KPageLinkedList& pg, VAddr addr, size_t num_pages) { + ASSERT(this->IsLockedByCurrentThread()); + + const size_t size = num_pages * PageSize; + + // We're making a new group, not adding to an existing one. + R_UNLESS(pg.Empty(), ResultInvalidCurrentMemory); + + // Begin traversal. + Common::PageTable::TraversalContext context; + Common::PageTable::TraversalEntry next_entry; + R_UNLESS(page_table_impl.BeginTraversal(next_entry, context, addr), ResultInvalidCurrentMemory); + + // Prepare tracking variables. + PAddr cur_addr = next_entry.phys_addr; + size_t cur_size = next_entry.block_size - (cur_addr & (next_entry.block_size - 1)); + size_t tot_size = cur_size; + + // Iterate, adding to group as we go. + const auto& memory_layout = system.Kernel().MemoryLayout(); + while (tot_size < size) { + R_UNLESS(page_table_impl.ContinueTraversal(next_entry, context), + ResultInvalidCurrentMemory); + + if (next_entry.phys_addr != (cur_addr + cur_size)) { + const size_t cur_pages = cur_size / PageSize; + + R_UNLESS(IsHeapPhysicalAddress(memory_layout, cur_addr), ResultInvalidCurrentMemory); + R_TRY(pg.AddBlock(cur_addr, cur_pages)); + + cur_addr = next_entry.phys_addr; + cur_size = next_entry.block_size; + } else { + cur_size += next_entry.block_size; + } + + tot_size += next_entry.block_size; + } + + // Ensure we add the right amount for the last block. + if (tot_size > size) { + cur_size -= (tot_size - size); + } + + // Add the last block. + const size_t cur_pages = cur_size / PageSize; + R_UNLESS(IsHeapPhysicalAddress(memory_layout, cur_addr), ResultInvalidCurrentMemory); + R_TRY(pg.AddBlock(cur_addr, cur_pages)); + + return ResultSuccess; +} + ResultCode KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTable& src_page_table, VAddr src_addr) { KScopedLightLock lk(general_lock); @@ -449,6 +568,8 @@ ResultCode KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, block_manager->Update(dst_addr, num_pages, KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None); + system.InvalidateCpuInstructionCaches(); + return ResultSuccess; } @@ -1055,6 +1176,46 @@ ResultCode KPageTable::MapPages(VAddr address, KPageLinkedList& page_linked_list return ResultSuccess; } +ResultCode KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, + PAddr phys_addr, bool is_pa_valid, VAddr region_start, + std::size_t region_num_pages, KMemoryState state, + KMemoryPermission perm) { + ASSERT(Common::IsAligned(alignment, PageSize) && alignment >= PageSize); + + // Ensure this is a valid map request. + R_UNLESS(this->CanContain(region_start, region_num_pages * PageSize, state), + ResultInvalidCurrentMemory); + R_UNLESS(num_pages < region_num_pages, ResultOutOfMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Find a random address to map at. + VAddr addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, 0, + this->GetNumGuardPages()); + R_UNLESS(addr != 0, ResultOutOfMemory); + ASSERT(Common::IsAligned(addr, alignment)); + ASSERT(this->CanContain(addr, num_pages * PageSize, state)); + ASSERT(this->CheckMemoryState(addr, num_pages * PageSize, KMemoryState::All, KMemoryState::Free, + KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryAttribute::None) + .IsSuccess()); + + // Perform mapping operation. + if (is_pa_valid) { + R_TRY(this->Operate(addr, num_pages, perm, OperationType::Map, phys_addr)); + } else { + UNIMPLEMENTED(); + } + + // Update the blocks. + block_manager->Update(addr, num_pages, state, perm); + + // We successfully mapped the pages. + *out_addr = addr; + return ResultSuccess; +} + ResultCode KPageTable::UnmapPages(VAddr addr, const KPageLinkedList& page_linked_list) { ASSERT(this->IsLockedByCurrentThread()); @@ -1097,6 +1258,55 @@ ResultCode KPageTable::UnmapPages(VAddr addr, KPageLinkedList& page_linked_list, return ResultSuccess; } +ResultCode KPageTable::UnmapPages(VAddr address, std::size_t num_pages, KMemoryState state) { + // Check that the unmap is in range. + const std::size_t size = num_pages * PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check the memory state. + std::size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, + KMemoryState::All, state, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::All, + KMemoryAttribute::None)); + + // Perform the unmap. + R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); + + // Update the blocks. + block_manager->Update(address, num_pages, KMemoryState::Free, KMemoryPermission::None); + + return ResultSuccess; +} + +ResultCode KPageTable::MakeAndOpenPageGroup(KPageLinkedList* out, VAddr address, size_t num_pages, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr) { + // Ensure that the page group isn't null. + ASSERT(out != nullptr); + + // Make sure that the region we're mapping is valid for the table. + const size_t size = num_pages * PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check if state allows us to create the group. + R_TRY(this->CheckMemoryState(address, size, state_mask | KMemoryState::FlagReferenceCounted, + state | KMemoryState::FlagReferenceCounted, perm_mask, perm, + attr_mask, attr)); + + // Create a new page group for the region. + R_TRY(this->MakePageGroup(*out, address, num_pages)); + + return ResultSuccess; +} + ResultCode KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; @@ -1479,57 +1689,21 @@ ResultCode KPageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) } ResultCode KPageTable::LockForCodeMemory(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryPermission new_perm = KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite; - - KMemoryPermission old_perm{}; - - if (const ResultCode result{CheckMemoryState( - nullptr, &old_perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, - KMemoryState::FlagCanCodeMemory, KMemoryPermission::All, - KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None)}; - result.IsError()) { - return result; - } - - new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; - - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->ShareToDevice(permission); - }, - new_perm); - - return ResultSuccess; + return this->LockMemoryAndOpen( + nullptr, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, + KMemoryState::FlagCanCodeMemory, KMemoryPermission::All, KMemoryPermission::UserReadWrite, + KMemoryAttribute::All, KMemoryAttribute::None, + static_cast<KMemoryPermission>(KMemoryPermission::NotMapped | + KMemoryPermission::KernelReadWrite), + KMemoryAttribute::Locked); } ResultCode KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryPermission new_perm = KMemoryPermission::UserReadWrite; - - KMemoryPermission old_perm{}; - - if (const ResultCode result{CheckMemoryState( - nullptr, &old_perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, - KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::All, KMemoryAttribute::Locked)}; - result.IsError()) { - return result; - } - - new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; - - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->UnshareToDevice(permission); - }, - new_perm); - - return ResultSuccess; + return this->UnlockMemory(addr, size, KMemoryState::FlagCanCodeMemory, + KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::All, + KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, + KMemoryAttribute::Locked, nullptr); } ResultCode KPageTable::InitializeMemoryLayout(VAddr start, VAddr end) { @@ -1865,4 +2039,109 @@ ResultCode KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermissi return ResultSuccess; } +ResultCode KPageTable::LockMemoryAndOpen(KPageLinkedList* out_pg, PAddr* out_paddr, VAddr addr, + size_t size, KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr, + KMemoryPermission new_perm, KMemoryAttribute lock_attr) { + // Validate basic preconditions. + ASSERT((lock_attr & attr) == KMemoryAttribute::None); + ASSERT((lock_attr & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared)) == + KMemoryAttribute::None); + + // Validate the lock request. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check that the output page group is empty, if it exists. + if (out_pg) { + ASSERT(out_pg->GetNumPages() == 0); + } + + // Check the state. + KMemoryState old_state{}; + KMemoryPermission old_perm{}; + KMemoryAttribute old_attr{}; + size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(old_state), std::addressof(old_perm), + std::addressof(old_attr), std::addressof(num_allocator_blocks), + addr, size, state_mask | KMemoryState::FlagReferenceCounted, + state | KMemoryState::FlagReferenceCounted, perm_mask, perm, + attr_mask, attr)); + + // Get the physical address, if we're supposed to. + if (out_paddr != nullptr) { + ASSERT(this->GetPhysicalAddressLocked(out_paddr, addr)); + } + + // Make the page group, if we're supposed to. + if (out_pg != nullptr) { + R_TRY(this->MakePageGroup(*out_pg, addr, num_pages)); + } + + // Decide on new perm and attr. + new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; + KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr | lock_attr); + + // Update permission, if we need to. + if (new_perm != old_perm) { + R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); + } + + // Apply the memory block updates. + block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + + return ResultSuccess; +} + +ResultCode KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr, KMemoryPermission new_perm, + KMemoryAttribute lock_attr, const KPageLinkedList* pg) { + // Validate basic preconditions. + ASSERT((attr_mask & lock_attr) == lock_attr); + ASSERT((attr & lock_attr) == lock_attr); + + // Validate the unlock request. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check the state. + KMemoryState old_state{}; + KMemoryPermission old_perm{}; + KMemoryAttribute old_attr{}; + size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(old_state), std::addressof(old_perm), + std::addressof(old_attr), std::addressof(num_allocator_blocks), + addr, size, state_mask | KMemoryState::FlagReferenceCounted, + state | KMemoryState::FlagReferenceCounted, perm_mask, perm, + attr_mask, attr)); + + // Check the page group. + if (pg != nullptr) { + UNIMPLEMENTED_MSG("PageGroup support is unimplemented!"); + } + + // Decide on new perm and attr. + new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; + KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr & ~lock_attr); + + // Update permission, if we need to. + if (new_perm != old_perm) { + R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); + } + + // Apply the memory block updates. + block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + + return ResultSuccess; +} + } // namespace Kernel diff --git a/src/core/hle/kernel/k_page_table.h b/src/core/hle/kernel/k_page_table.h index e99abe36a..dd6022975 100644 --- a/src/core/hle/kernel/k_page_table.h +++ b/src/core/hle/kernel/k_page_table.h @@ -12,6 +12,7 @@ #include "core/file_sys/program_metadata.h" #include "core/hle/kernel/k_light_lock.h" #include "core/hle/kernel/k_memory_block.h" +#include "core/hle/kernel/k_memory_layout.h" #include "core/hle/kernel/k_memory_manager.h" #include "core/hle/result.h" @@ -25,6 +26,8 @@ class KMemoryBlockManager; class KPageTable final { public: + enum class ICacheInvalidationStrategy : u32 { InvalidateRange, InvalidateAll }; + YUZU_NON_COPYABLE(KPageTable); YUZU_NON_MOVEABLE(KPageTable); @@ -37,7 +40,8 @@ public: ResultCode MapProcessCode(VAddr addr, std::size_t pages_count, KMemoryState state, KMemoryPermission perm); ResultCode MapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size); - ResultCode UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size); + ResultCode UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size, + ICacheInvalidationStrategy icache_invalidation_strategy); ResultCode UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTable& src_page_table, VAddr src_addr); ResultCode MapPhysicalMemory(VAddr addr, std::size_t size); @@ -46,7 +50,14 @@ public: ResultCode UnmapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size); ResultCode MapPages(VAddr addr, KPageLinkedList& page_linked_list, KMemoryState state, KMemoryPermission perm); + ResultCode MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, + PAddr phys_addr, KMemoryState state, KMemoryPermission perm) { + return this->MapPages(out_addr, num_pages, alignment, phys_addr, true, + this->GetRegionAddress(state), this->GetRegionSize(state) / PageSize, + state, perm); + } ResultCode UnmapPages(VAddr addr, KPageLinkedList& page_linked_list, KMemoryState state); + ResultCode UnmapPages(VAddr address, std::size_t num_pages, KMemoryState state); ResultCode SetProcessMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission svc_perm); KMemoryInfo QueryInfo(VAddr addr); @@ -64,6 +75,10 @@ public: ResultCode UnlockForDeviceAddressSpace(VAddr addr, std::size_t size); ResultCode LockForCodeMemory(VAddr addr, std::size_t size); ResultCode UnlockForCodeMemory(VAddr addr, std::size_t size); + ResultCode MakeAndOpenPageGroup(KPageLinkedList* out, VAddr address, size_t num_pages, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr); Common::PageTable& PageTableImpl() { return page_table_impl; @@ -91,6 +106,9 @@ private: ResultCode InitializeMemoryLayout(VAddr start, VAddr end); ResultCode MapPages(VAddr addr, const KPageLinkedList& page_linked_list, KMemoryPermission perm); + ResultCode MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, + PAddr phys_addr, bool is_pa_valid, VAddr region_start, + std::size_t region_num_pages, KMemoryState state, KMemoryPermission perm); ResultCode UnmapPages(VAddr addr, const KPageLinkedList& page_linked_list); bool IsRegionMapped(VAddr address, u64 size); bool IsRegionContiguous(VAddr addr, u64 size) const; @@ -105,6 +123,9 @@ private: VAddr GetRegionAddress(KMemoryState state) const; std::size_t GetRegionSize(KMemoryState state) const; + VAddr FindFreeArea(VAddr region_start, std::size_t region_num_pages, std::size_t num_pages, + std::size_t alignment, std::size_t offset, std::size_t guard_pages); + ResultCode CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VAddr addr, std::size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, @@ -137,7 +158,7 @@ private: return CheckMemoryState(nullptr, nullptr, nullptr, out_blocks_needed, addr, size, state_mask, state, perm_mask, perm, attr_mask, attr, ignore_attr); } - ResultCode CheckMemoryState(VAddr addr, size_t size, KMemoryState state_mask, + ResultCode CheckMemoryState(VAddr addr, std::size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr, @@ -146,10 +167,37 @@ private: attr_mask, attr, ignore_attr); } + ResultCode LockMemoryAndOpen(KPageLinkedList* out_pg, PAddr* out_paddr, VAddr addr, size_t size, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr, + KMemoryPermission new_perm, KMemoryAttribute lock_attr); + ResultCode UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr, + KMemoryPermission new_perm, KMemoryAttribute lock_attr, + const KPageLinkedList* pg); + + ResultCode MakePageGroup(KPageLinkedList& pg, VAddr addr, size_t num_pages); + bool IsLockedByCurrentThread() const { return general_lock.IsLockedByCurrentThread(); } + bool IsHeapPhysicalAddress(const KMemoryLayout& layout, PAddr phys_addr) { + ASSERT(this->IsLockedByCurrentThread()); + + return layout.IsHeapPhysicalAddress(cached_physical_heap_region, phys_addr); + } + + bool GetPhysicalAddressLocked(PAddr* out, VAddr virt_addr) const { + ASSERT(this->IsLockedByCurrentThread()); + + *out = GetPhysicalAddr(virt_addr); + + return *out != 0; + } + mutable KLightLock general_lock; mutable KLightLock map_physical_memory_lock; @@ -210,7 +258,7 @@ public: constexpr VAddr GetAliasCodeRegionSize() const { return alias_code_region_end - alias_code_region_start; } - size_t GetNormalMemorySize() { + std::size_t GetNormalMemorySize() { KScopedLightLock lk(general_lock); return GetHeapSize() + mapped_physical_memory_size; } @@ -309,6 +357,7 @@ private: bool is_aslr_enabled{}; u32 heap_fill_value{}; + const KMemoryRegion* cached_physical_heap_region{}; KMemoryManager::Pool memory_pool{KMemoryManager::Pool::Application}; KMemoryManager::Direction allocation_option{KMemoryManager::Direction::FromFront}; diff --git a/src/core/hle/kernel/k_port.cpp b/src/core/hle/kernel/k_port.cpp index a8ba09c4a..ceb98709f 100644 --- a/src/core/hle/kernel/k_port.cpp +++ b/src/core/hle/kernel/k_port.cpp @@ -57,7 +57,12 @@ ResultCode KPort::EnqueueSession(KServerSession* session) { R_UNLESS(state == State::Normal, ResultPortClosed); server.EnqueueSession(session); - server.GetSessionRequestHandler()->ClientConnected(server.AcceptSession()); + + if (auto session_ptr = server.GetSessionRequestHandler().lock()) { + session_ptr->ClientConnected(server.AcceptSession()); + } else { + UNREACHABLE(); + } return ResultSuccess; } diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp index 9233261cd..490e31fc7 100644 --- a/src/core/hle/kernel/k_process.cpp +++ b/src/core/hle/kernel/k_process.cpp @@ -13,7 +13,6 @@ #include "common/scope_exit.h" #include "common/settings.h" #include "core/core.h" -#include "core/device_memory.h" #include "core/file_sys/program_metadata.h" #include "core/hle/kernel/code_set.h" #include "core/hle/kernel/k_memory_block_manager.h" @@ -24,7 +23,6 @@ #include "core/hle/kernel/k_scoped_resource_reservation.h" #include "core/hle/kernel/k_shared_memory.h" #include "core/hle/kernel/k_shared_memory_info.h" -#include "core/hle/kernel/k_slab_heap.h" #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/kernel.h" #include "core/hle/kernel/svc_results.h" @@ -70,58 +68,6 @@ void SetupMainThread(Core::System& system, KProcess& owner_process, u32 priority } } // Anonymous namespace -// Represents a page used for thread-local storage. -// -// Each TLS page contains slots that may be used by processes and threads. -// Every process and thread is created with a slot in some arbitrary page -// (whichever page happens to have an available slot). -class TLSPage { -public: - static constexpr std::size_t num_slot_entries = - Core::Memory::PAGE_SIZE / Core::Memory::TLS_ENTRY_SIZE; - - explicit TLSPage(VAddr address) : base_address{address} {} - - bool HasAvailableSlots() const { - return !is_slot_used.all(); - } - - VAddr GetBaseAddress() const { - return base_address; - } - - std::optional<VAddr> ReserveSlot() { - for (std::size_t i = 0; i < is_slot_used.size(); i++) { - if (is_slot_used[i]) { - continue; - } - - is_slot_used[i] = true; - return base_address + (i * Core::Memory::TLS_ENTRY_SIZE); - } - - return std::nullopt; - } - - void ReleaseSlot(VAddr address) { - // Ensure that all given addresses are consistent with how TLS pages - // are intended to be used when releasing slots. - ASSERT(IsWithinPage(address)); - ASSERT((address % Core::Memory::TLS_ENTRY_SIZE) == 0); - - const std::size_t index = (address - base_address) / Core::Memory::TLS_ENTRY_SIZE; - is_slot_used[index] = false; - } - -private: - bool IsWithinPage(VAddr address) const { - return base_address <= address && address < base_address + Core::Memory::PAGE_SIZE; - } - - VAddr base_address; - std::bitset<num_slot_entries> is_slot_used; -}; - ResultCode KProcess::Initialize(KProcess* process, Core::System& system, std::string process_name, ProcessType type, KResourceLimit* res_limit) { auto& kernel = system.Kernel(); @@ -404,7 +350,7 @@ ResultCode KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, } // Create TLS region - tls_region_address = CreateTLSRegion(); + R_TRY(this->CreateThreadLocalRegion(std::addressof(tls_region_address))); memory_reservation.Commit(); return handle_table.Initialize(capabilities.GetHandleTableSize()); @@ -444,7 +390,7 @@ void KProcess::PrepareForTermination() { stop_threads(kernel.System().GlobalSchedulerContext().GetThreadList()); - FreeTLSRegion(tls_region_address); + this->DeleteThreadLocalRegion(tls_region_address); tls_region_address = 0; if (resource_limit) { @@ -456,9 +402,6 @@ void KProcess::PrepareForTermination() { } void KProcess::Finalize() { - // Finalize the handle table and close any open handles. - handle_table.Finalize(); - // Free all shared memory infos. { auto it = shared_memory_list.begin(); @@ -483,67 +426,110 @@ void KProcess::Finalize() { resource_limit = nullptr; } + // Finalize the page table. + page_table.reset(); + // Perform inherited finalization. KAutoObjectWithSlabHeapAndContainer<KProcess, KWorkerTask>::Finalize(); } -/** - * Attempts to find a TLS page that contains a free slot for - * use by a thread. - * - * @returns If a page with an available slot is found, then an iterator - * pointing to the page is returned. Otherwise the end iterator - * is returned instead. - */ -static auto FindTLSPageWithAvailableSlots(std::vector<TLSPage>& tls_pages) { - return std::find_if(tls_pages.begin(), tls_pages.end(), - [](const auto& page) { return page.HasAvailableSlots(); }); -} +ResultCode KProcess::CreateThreadLocalRegion(VAddr* out) { + KThreadLocalPage* tlp = nullptr; + VAddr tlr = 0; -VAddr KProcess::CreateTLSRegion() { - KScopedSchedulerLock lock(kernel); - if (auto tls_page_iter{FindTLSPageWithAvailableSlots(tls_pages)}; - tls_page_iter != tls_pages.cend()) { - return *tls_page_iter->ReserveSlot(); - } + // See if we can get a region from a partially used TLP. + { + KScopedSchedulerLock sl{kernel}; - Page* const tls_page_ptr{kernel.GetUserSlabHeapPages().Allocate()}; - ASSERT(tls_page_ptr); + if (auto it = partially_used_tlp_tree.begin(); it != partially_used_tlp_tree.end()) { + tlr = it->Reserve(); + ASSERT(tlr != 0); - const VAddr start{page_table->GetKernelMapRegionStart()}; - const VAddr size{page_table->GetKernelMapRegionEnd() - start}; - const PAddr tls_map_addr{kernel.System().DeviceMemory().GetPhysicalAddr(tls_page_ptr)}; - const VAddr tls_page_addr{page_table - ->AllocateAndMapMemory(1, PageSize, true, start, size / PageSize, - KMemoryState::ThreadLocal, - KMemoryPermission::UserReadWrite, - tls_map_addr) - .ValueOr(0)}; + if (it->IsAllUsed()) { + tlp = std::addressof(*it); + partially_used_tlp_tree.erase(it); + fully_used_tlp_tree.insert(*tlp); + } - ASSERT(tls_page_addr); + *out = tlr; + return ResultSuccess; + } + } + + // Allocate a new page. + tlp = KThreadLocalPage::Allocate(kernel); + R_UNLESS(tlp != nullptr, ResultOutOfMemory); + auto tlp_guard = SCOPE_GUARD({ KThreadLocalPage::Free(kernel, tlp); }); + + // Initialize the new page. + R_TRY(tlp->Initialize(kernel, this)); - std::memset(tls_page_ptr, 0, PageSize); - tls_pages.emplace_back(tls_page_addr); + // Reserve a TLR. + tlr = tlp->Reserve(); + ASSERT(tlr != 0); - const auto reserve_result{tls_pages.back().ReserveSlot()}; - ASSERT(reserve_result.has_value()); + // Insert into our tree. + { + KScopedSchedulerLock sl{kernel}; + if (tlp->IsAllUsed()) { + fully_used_tlp_tree.insert(*tlp); + } else { + partially_used_tlp_tree.insert(*tlp); + } + } - return *reserve_result; + // We succeeded! + tlp_guard.Cancel(); + *out = tlr; + return ResultSuccess; } -void KProcess::FreeTLSRegion(VAddr tls_address) { - KScopedSchedulerLock lock(kernel); - const VAddr aligned_address = Common::AlignDown(tls_address, Core::Memory::PAGE_SIZE); - auto iter = - std::find_if(tls_pages.begin(), tls_pages.end(), [aligned_address](const auto& page) { - return page.GetBaseAddress() == aligned_address; - }); +ResultCode KProcess::DeleteThreadLocalRegion(VAddr addr) { + KThreadLocalPage* page_to_free = nullptr; - // Something has gone very wrong if we're freeing a region - // with no actual page available. - ASSERT(iter != tls_pages.cend()); + // Release the region. + { + KScopedSchedulerLock sl{kernel}; + + // Try to find the page in the partially used list. + auto it = partially_used_tlp_tree.find_key(Common::AlignDown(addr, PageSize)); + if (it == partially_used_tlp_tree.end()) { + // If we don't find it, it has to be in the fully used list. + it = fully_used_tlp_tree.find_key(Common::AlignDown(addr, PageSize)); + R_UNLESS(it != fully_used_tlp_tree.end(), ResultInvalidAddress); + + // Release the region. + it->Release(addr); + + // Move the page out of the fully used list. + KThreadLocalPage* tlp = std::addressof(*it); + fully_used_tlp_tree.erase(it); + if (tlp->IsAllFree()) { + page_to_free = tlp; + } else { + partially_used_tlp_tree.insert(*tlp); + } + } else { + // Release the region. + it->Release(addr); + + // Handle the all-free case. + KThreadLocalPage* tlp = std::addressof(*it); + if (tlp->IsAllFree()) { + partially_used_tlp_tree.erase(it); + page_to_free = tlp; + } + } + } + + // If we should free the page it was in, do so. + if (page_to_free != nullptr) { + page_to_free->Finalize(); - iter->ReleaseSlot(tls_address); + KThreadLocalPage::Free(kernel, page_to_free); + } + + return ResultSuccess; } void KProcess::LoadModule(CodeSet code_set, VAddr base_addr) { diff --git a/src/core/hle/kernel/k_process.h b/src/core/hle/kernel/k_process.h index cf1b67428..9f171e3da 100644 --- a/src/core/hle/kernel/k_process.h +++ b/src/core/hle/kernel/k_process.h @@ -8,13 +8,13 @@ #include <cstddef> #include <list> #include <string> -#include <vector> #include "common/common_types.h" #include "core/hle/kernel/k_address_arbiter.h" #include "core/hle/kernel/k_auto_object.h" #include "core/hle/kernel/k_condition_variable.h" #include "core/hle/kernel/k_handle_table.h" #include "core/hle/kernel/k_synchronization_object.h" +#include "core/hle/kernel/k_thread_local_page.h" #include "core/hle/kernel/k_worker_task.h" #include "core/hle/kernel/process_capability.h" #include "core/hle/kernel/slab_helpers.h" @@ -362,10 +362,10 @@ public: // Thread-local storage management // Marks the next available region as used and returns the address of the slot. - [[nodiscard]] VAddr CreateTLSRegion(); + [[nodiscard]] ResultCode CreateThreadLocalRegion(VAddr* out); // Frees a used TLS slot identified by the given address - void FreeTLSRegion(VAddr tls_address); + ResultCode DeleteThreadLocalRegion(VAddr addr); private: void PinThread(s32 core_id, KThread* thread) { @@ -413,13 +413,6 @@ private: /// The ideal CPU core for this process, threads are scheduled on this core by default. u8 ideal_core = 0; - /// The Thread Local Storage area is allocated as processes create threads, - /// each TLS area is 0x200 bytes, so one page (0x1000) is split up in 8 parts, and each part - /// holds the TLS for a specific thread. This vector contains which parts are in use for each - /// page as a bitmask. - /// This vector will grow as more pages are allocated for new threads. - std::vector<TLSPage> tls_pages; - /// Contains the parsed process capability descriptors. ProcessCapabilities capabilities; @@ -429,7 +422,7 @@ private: bool is_64bit_process = true; /// Total running time for the process in ticks. - u64 total_process_running_time_ticks = 0; + std::atomic<u64> total_process_running_time_ticks = 0; /// Per-process handle table for storing created object handles in. KHandleTable handle_table; @@ -482,6 +475,12 @@ private: KThread* exception_thread{}; KLightLock state_lock; + + using TLPTree = + Common::IntrusiveRedBlackTreeBaseTraits<KThreadLocalPage>::TreeType<KThreadLocalPage>; + using TLPIterator = TLPTree::iterator; + TLPTree fully_used_tlp_tree; + TLPTree partially_used_tlp_tree; }; } // namespace Kernel diff --git a/src/core/hle/kernel/k_scheduler.cpp b/src/core/hle/kernel/k_scheduler.cpp index c96520828..6c0bb1672 100644 --- a/src/core/hle/kernel/k_scheduler.cpp +++ b/src/core/hle/kernel/k_scheduler.cpp @@ -22,7 +22,6 @@ #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/kernel.h" #include "core/hle/kernel/physical_core.h" -#include "core/hle/kernel/time_manager.h" namespace Kernel { diff --git a/src/core/hle/kernel/k_scheduler_lock.h b/src/core/hle/kernel/k_scheduler_lock.h index 93c47f1b1..016e0a818 100644 --- a/src/core/hle/kernel/k_scheduler_lock.h +++ b/src/core/hle/kernel/k_scheduler_lock.h @@ -4,6 +4,7 @@ #pragma once +#include <atomic> #include "common/assert.h" #include "core/hle/kernel/k_spin_lock.h" #include "core/hle/kernel/k_thread.h" @@ -75,7 +76,7 @@ private: KernelCore& kernel; KAlignedSpinLock spin_lock{}; s32 lock_count{}; - KThread* owner_thread{}; + std::atomic<KThread*> owner_thread{}; }; } // namespace Kernel diff --git a/src/core/hle/kernel/k_server_port.cpp b/src/core/hle/kernel/k_server_port.cpp index 433fc98e1..e66c0c992 100644 --- a/src/core/hle/kernel/k_server_port.cpp +++ b/src/core/hle/kernel/k_server_port.cpp @@ -62,6 +62,12 @@ void KServerPort::Destroy() { // Close our reference to our parent. parent->Close(); + + // Release host emulation members. + session_handler.reset(); + + // Ensure that the global list tracking server objects does not hold on to a reference. + kernel.UnregisterServerObject(this); } bool KServerPort::IsSignaled() const { diff --git a/src/core/hle/kernel/k_server_port.h b/src/core/hle/kernel/k_server_port.h index 6302d5e61..2185736be 100644 --- a/src/core/hle/kernel/k_server_port.h +++ b/src/core/hle/kernel/k_server_port.h @@ -30,11 +30,11 @@ public: /// Whether or not this server port has an HLE handler available. bool HasSessionRequestHandler() const { - return session_handler != nullptr; + return !session_handler.expired(); } /// Gets the HLE handler for this port. - SessionRequestHandlerPtr GetSessionRequestHandler() const { + SessionRequestHandlerWeakPtr GetSessionRequestHandler() const { return session_handler; } @@ -42,7 +42,7 @@ public: * Sets the HLE handler template for the port. ServerSessions crated by connecting to this port * will inherit a reference to this handler. */ - void SetSessionHandler(SessionRequestHandlerPtr&& handler) { + void SetSessionHandler(SessionRequestHandlerWeakPtr&& handler) { session_handler = std::move(handler); } @@ -66,7 +66,7 @@ private: void CleanupSessions(); SessionList session_list; - SessionRequestHandlerPtr session_handler; + SessionRequestHandlerWeakPtr session_handler; KPort* parent{}; }; diff --git a/src/core/hle/kernel/k_server_session.cpp b/src/core/hle/kernel/k_server_session.cpp index 4d94eb9cf..7ac2ef254 100644 --- a/src/core/hle/kernel/k_server_session.cpp +++ b/src/core/hle/kernel/k_server_session.cpp @@ -27,10 +27,7 @@ namespace Kernel { KServerSession::KServerSession(KernelCore& kernel_) : KSynchronizationObject{kernel_} {} -KServerSession::~KServerSession() { - // Ensure that the global list tracking server sessions does not hold on to a reference. - kernel.UnregisterServerSession(this); -} +KServerSession::~KServerSession() = default; void KServerSession::Initialize(KSession* parent_session_, std::string&& name_, std::shared_ptr<SessionRequestManager> manager_) { @@ -49,6 +46,12 @@ void KServerSession::Destroy() { parent->OnServerClosed(); parent->Close(); + + // Release host emulation members. + manager.reset(); + + // Ensure that the global list tracking server objects does not hold on to a reference. + kernel.UnregisterServerObject(this); } void KServerSession::OnClientClosed() { @@ -98,7 +101,12 @@ ResultCode KServerSession::HandleDomainSyncRequest(Kernel::HLERequestContext& co UNREACHABLE(); return ResultSuccess; // Ignore error if asserts are off } - return manager->DomainHandler(object_id - 1)->HandleSyncRequest(*this, context); + if (auto strong_ptr = manager->DomainHandler(object_id - 1).lock()) { + return strong_ptr->HandleSyncRequest(*this, context); + } else { + UNREACHABLE(); + return ResultSuccess; + } case IPC::DomainMessageHeader::CommandType::CloseVirtualHandle: { LOG_DEBUG(IPC, "CloseVirtualHandle, object_id=0x{:08X}", object_id); diff --git a/src/core/hle/kernel/k_slab_heap.h b/src/core/hle/kernel/k_slab_heap.h index 05c0bec9c..5690cc757 100644 --- a/src/core/hle/kernel/k_slab_heap.h +++ b/src/core/hle/kernel/k_slab_heap.h @@ -16,39 +16,34 @@ class KernelCore; namespace impl { -class KSlabHeapImpl final { -public: +class KSlabHeapImpl { YUZU_NON_COPYABLE(KSlabHeapImpl); YUZU_NON_MOVEABLE(KSlabHeapImpl); +public: struct Node { Node* next{}; }; +public: constexpr KSlabHeapImpl() = default; - constexpr ~KSlabHeapImpl() = default; - void Initialize(std::size_t size) { - ASSERT(head == nullptr); - obj_size = size; - } - - constexpr std::size_t GetObjectSize() const { - return obj_size; + void Initialize() { + ASSERT(m_head == nullptr); } Node* GetHead() const { - return head; + return m_head; } void* Allocate() { - Node* ret = head.load(); + Node* ret = m_head.load(); do { if (ret == nullptr) { break; } - } while (!head.compare_exchange_weak(ret, ret->next)); + } while (!m_head.compare_exchange_weak(ret, ret->next)); return ret; } @@ -56,170 +51,157 @@ public: void Free(void* obj) { Node* node = static_cast<Node*>(obj); - Node* cur_head = head.load(); + Node* cur_head = m_head.load(); do { node->next = cur_head; - } while (!head.compare_exchange_weak(cur_head, node)); + } while (!m_head.compare_exchange_weak(cur_head, node)); } private: - std::atomic<Node*> head{}; - std::size_t obj_size{}; + std::atomic<Node*> m_head{}; }; } // namespace impl -class KSlabHeapBase { -public: +template <bool SupportDynamicExpansion> +class KSlabHeapBase : protected impl::KSlabHeapImpl { YUZU_NON_COPYABLE(KSlabHeapBase); YUZU_NON_MOVEABLE(KSlabHeapBase); - constexpr KSlabHeapBase() = default; - constexpr ~KSlabHeapBase() = default; +private: + size_t m_obj_size{}; + uintptr_t m_peak{}; + uintptr_t m_start{}; + uintptr_t m_end{}; - constexpr bool Contains(uintptr_t addr) const { - return start <= addr && addr < end; - } +private: + void UpdatePeakImpl(uintptr_t obj) { + static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free); + std::atomic_ref<uintptr_t> peak_ref(m_peak); - constexpr std::size_t GetSlabHeapSize() const { - return (end - start) / GetObjectSize(); + const uintptr_t alloc_peak = obj + this->GetObjectSize(); + uintptr_t cur_peak = m_peak; + do { + if (alloc_peak <= cur_peak) { + break; + } + } while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak)); } - constexpr std::size_t GetObjectSize() const { - return impl.GetObjectSize(); - } +public: + constexpr KSlabHeapBase() = default; - constexpr uintptr_t GetSlabHeapAddress() const { - return start; + bool Contains(uintptr_t address) const { + return m_start <= address && address < m_end; } - std::size_t GetObjectIndexImpl(const void* obj) const { - return (reinterpret_cast<uintptr_t>(obj) - start) / GetObjectSize(); + void Initialize(size_t obj_size, void* memory, size_t memory_size) { + // Ensure we don't initialize a slab using null memory. + ASSERT(memory != nullptr); + + // Set our object size. + m_obj_size = obj_size; + + // Initialize the base allocator. + KSlabHeapImpl::Initialize(); + + // Set our tracking variables. + const size_t num_obj = (memory_size / obj_size); + m_start = reinterpret_cast<uintptr_t>(memory); + m_end = m_start + num_obj * obj_size; + m_peak = m_start; + + // Free the objects. + u8* cur = reinterpret_cast<u8*>(m_end); + + for (size_t i = 0; i < num_obj; i++) { + cur -= obj_size; + KSlabHeapImpl::Free(cur); + } } - std::size_t GetPeakIndex() const { - return GetObjectIndexImpl(reinterpret_cast<const void*>(peak)); + size_t GetSlabHeapSize() const { + return (m_end - m_start) / this->GetObjectSize(); } - void* AllocateImpl() { - return impl.Allocate(); + size_t GetObjectSize() const { + return m_obj_size; } - void FreeImpl(void* obj) { - // Don't allow freeing an object that wasn't allocated from this heap - ASSERT(Contains(reinterpret_cast<uintptr_t>(obj))); + void* Allocate() { + void* obj = KSlabHeapImpl::Allocate(); - impl.Free(obj); + return obj; } - void InitializeImpl(std::size_t obj_size, void* memory, std::size_t memory_size) { - // Ensure we don't initialize a slab using null memory - ASSERT(memory != nullptr); - - // Initialize the base allocator - impl.Initialize(obj_size); + void Free(void* obj) { + // Don't allow freeing an object that wasn't allocated from this heap. + const bool contained = this->Contains(reinterpret_cast<uintptr_t>(obj)); + ASSERT(contained); + KSlabHeapImpl::Free(obj); + } - // Set our tracking variables - const std::size_t num_obj = (memory_size / obj_size); - start = reinterpret_cast<uintptr_t>(memory); - end = start + num_obj * obj_size; - peak = start; + size_t GetObjectIndex(const void* obj) const { + if constexpr (SupportDynamicExpansion) { + if (!this->Contains(reinterpret_cast<uintptr_t>(obj))) { + return std::numeric_limits<size_t>::max(); + } + } - // Free the objects - u8* cur = reinterpret_cast<u8*>(end); + return (reinterpret_cast<uintptr_t>(obj) - m_start) / this->GetObjectSize(); + } - for (std::size_t i{}; i < num_obj; i++) { - cur -= obj_size; - impl.Free(cur); - } + size_t GetPeakIndex() const { + return this->GetObjectIndex(reinterpret_cast<const void*>(m_peak)); } -private: - using Impl = impl::KSlabHeapImpl; + uintptr_t GetSlabHeapAddress() const { + return m_start; + } - Impl impl; - uintptr_t peak{}; - uintptr_t start{}; - uintptr_t end{}; + size_t GetNumRemaining() const { + // Only calculate the number of remaining objects under debug configuration. + return 0; + } }; template <typename T> -class KSlabHeap final : public KSlabHeapBase { -public: - enum class AllocationType { - Host, - Guest, - }; +class KSlabHeap final : public KSlabHeapBase<false> { +private: + using BaseHeap = KSlabHeapBase<false>; - explicit constexpr KSlabHeap(AllocationType allocation_type_ = AllocationType::Host) - : KSlabHeapBase(), allocation_type{allocation_type_} {} +public: + constexpr KSlabHeap() = default; - void Initialize(void* memory, std::size_t memory_size) { - if (allocation_type == AllocationType::Guest) { - InitializeImpl(sizeof(T), memory, memory_size); - } + void Initialize(void* memory, size_t memory_size) { + BaseHeap::Initialize(sizeof(T), memory, memory_size); } T* Allocate() { - switch (allocation_type) { - case AllocationType::Host: - // Fallback for cases where we do not yet support allocating guest memory from the slab - // heap, such as for kernel memory regions. - return new T; - - case AllocationType::Guest: - T* obj = static_cast<T*>(AllocateImpl()); - if (obj != nullptr) { - new (obj) T(); - } - return obj; - } + T* obj = static_cast<T*>(BaseHeap::Allocate()); - UNREACHABLE_MSG("Invalid AllocationType {}", allocation_type); - return nullptr; + if (obj != nullptr) [[likely]] { + std::construct_at(obj); + } + return obj; } - T* AllocateWithKernel(KernelCore& kernel) { - switch (allocation_type) { - case AllocationType::Host: - // Fallback for cases where we do not yet support allocating guest memory from the slab - // heap, such as for kernel memory regions. - return new T(kernel); + T* Allocate(KernelCore& kernel) { + T* obj = static_cast<T*>(BaseHeap::Allocate()); - case AllocationType::Guest: - T* obj = static_cast<T*>(AllocateImpl()); - if (obj != nullptr) { - new (obj) T(kernel); - } - return obj; + if (obj != nullptr) [[likely]] { + std::construct_at(obj, kernel); } - - UNREACHABLE_MSG("Invalid AllocationType {}", allocation_type); - return nullptr; + return obj; } void Free(T* obj) { - switch (allocation_type) { - case AllocationType::Host: - // Fallback for cases where we do not yet support allocating guest memory from the slab - // heap, such as for kernel memory regions. - delete obj; - return; - - case AllocationType::Guest: - FreeImpl(obj); - return; - } - - UNREACHABLE_MSG("Invalid AllocationType {}", allocation_type); + BaseHeap::Free(obj); } - constexpr std::size_t GetObjectIndex(const T* obj) const { - return GetObjectIndexImpl(obj); + size_t GetObjectIndex(const T* obj) const { + return BaseHeap::GetObjectIndex(obj); } - -private: - const AllocationType allocation_type; }; } // namespace Kernel diff --git a/src/core/hle/kernel/k_thread.cpp b/src/core/hle/kernel/k_thread.cpp index de3ffe0c7..d3bb1c871 100644 --- a/src/core/hle/kernel/k_thread.cpp +++ b/src/core/hle/kernel/k_thread.cpp @@ -14,9 +14,7 @@ #include "common/common_types.h" #include "common/fiber.h" #include "common/logging/log.h" -#include "common/scope_exit.h" #include "common/settings.h" -#include "common/thread_queue_list.h" #include "core/core.h" #include "core/cpu_manager.h" #include "core/hardware_properties.h" @@ -33,7 +31,6 @@ #include "core/hle/kernel/k_worker_task_manager.h" #include "core/hle/kernel/kernel.h" #include "core/hle/kernel/svc_results.h" -#include "core/hle/kernel/time_manager.h" #include "core/hle/result.h" #include "core/memory.h" @@ -210,7 +207,7 @@ ResultCode KThread::Initialize(KThreadFunction func, uintptr_t arg, VAddr user_s if (owner != nullptr) { // Setup the TLS, if needed. if (type == ThreadType::User) { - tls_address = owner->CreateTLSRegion(); + R_TRY(owner->CreateThreadLocalRegion(std::addressof(tls_address))); } parent = owner; @@ -305,7 +302,7 @@ void KThread::Finalize() { // If the thread has a local region, delete it. if (tls_address != 0) { - parent->FreeTLSRegion(tls_address); + ASSERT(parent->DeleteThreadLocalRegion(tls_address).IsSuccess()); } // Release any waiters. @@ -326,6 +323,9 @@ void KThread::Finalize() { } } + // Release host emulation members. + host_context.reset(); + // Perform inherited finalization. KSynchronizationObject::Finalize(); } @@ -723,7 +723,7 @@ void KThread::UpdateState() { ASSERT(kernel.GlobalSchedulerContext().IsLocked()); // Set our suspend flags in state. - const auto old_state = thread_state; + const ThreadState old_state = thread_state; const auto new_state = static_cast<ThreadState>(this->GetSuspendFlags()) | (old_state & ThreadState::Mask); thread_state = new_state; @@ -738,7 +738,7 @@ void KThread::Continue() { ASSERT(kernel.GlobalSchedulerContext().IsLocked()); // Clear our suspend flags in state. - const auto old_state = thread_state; + const ThreadState old_state = thread_state; thread_state = old_state & ThreadState::Mask; // Note the state change in scheduler. diff --git a/src/core/hle/kernel/k_thread.h b/src/core/hle/kernel/k_thread.h index d058db62c..d0fd85130 100644 --- a/src/core/hle/kernel/k_thread.h +++ b/src/core/hle/kernel/k_thread.h @@ -5,6 +5,7 @@ #pragma once #include <array> +#include <atomic> #include <span> #include <string> #include <utility> @@ -656,7 +657,7 @@ private: static_assert(sizeof(SyncObjectBuffer::sync_objects) == sizeof(SyncObjectBuffer::handles)); struct ConditionVariableComparator { - struct LightCompareType { + struct RedBlackKeyType { u64 cv_key{}; s32 priority{}; @@ -672,8 +673,8 @@ private: template <typename T> requires( std::same_as<T, KThread> || - std::same_as<T, LightCompareType>) static constexpr int Compare(const T& lhs, - const KThread& rhs) { + std::same_as<T, RedBlackKeyType>) static constexpr int Compare(const T& lhs, + const KThread& rhs) { const u64 l_key = lhs.GetConditionVariableKey(); const u64 r_key = rhs.GetConditionVariableKey(); @@ -751,7 +752,7 @@ private: KAffinityMask original_physical_affinity_mask{}; s32 original_physical_ideal_core_id{}; s32 num_core_migration_disables{}; - ThreadState thread_state{}; + std::atomic<ThreadState> thread_state{}; std::atomic<bool> termination_requested{}; bool wait_cancelled{}; bool cancellable{}; diff --git a/src/core/hle/kernel/k_thread_local_page.cpp b/src/core/hle/kernel/k_thread_local_page.cpp new file mode 100644 index 000000000..3fb277eba --- /dev/null +++ b/src/core/hle/kernel/k_thread_local_page.cpp @@ -0,0 +1,68 @@ +// Copyright 2022 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "common/scope_exit.h" +#include "core/core.h" + +#include "core/hle/kernel/k_memory_block.h" +#include "core/hle/kernel/k_page_buffer.h" +#include "core/hle/kernel/k_page_table.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/kernel/k_thread_local_page.h" +#include "core/hle/kernel/kernel.h" + +namespace Kernel { + +ResultCode KThreadLocalPage::Initialize(KernelCore& kernel, KProcess* process) { + // Set that this process owns us. + m_owner = process; + m_kernel = &kernel; + + // Allocate a new page. + KPageBuffer* page_buf = KPageBuffer::Allocate(kernel); + R_UNLESS(page_buf != nullptr, ResultOutOfMemory); + auto page_buf_guard = SCOPE_GUARD({ KPageBuffer::Free(kernel, page_buf); }); + + // Map the address in. + const auto phys_addr = kernel.System().DeviceMemory().GetPhysicalAddr(page_buf); + R_TRY(m_owner->PageTable().MapPages(std::addressof(m_virt_addr), 1, PageSize, phys_addr, + KMemoryState::ThreadLocal, + KMemoryPermission::UserReadWrite)); + + // We succeeded. + page_buf_guard.Cancel(); + + return ResultSuccess; +} + +ResultCode KThreadLocalPage::Finalize() { + // Get the physical address of the page. + const PAddr phys_addr = m_owner->PageTable().GetPhysicalAddr(m_virt_addr); + ASSERT(phys_addr); + + // Unmap the page. + R_TRY(m_owner->PageTable().UnmapPages(this->GetAddress(), 1, KMemoryState::ThreadLocal)); + + // Free the page. + KPageBuffer::Free(*m_kernel, KPageBuffer::FromPhysicalAddress(m_kernel->System(), phys_addr)); + + return ResultSuccess; +} + +VAddr KThreadLocalPage::Reserve() { + for (size_t i = 0; i < m_is_region_free.size(); i++) { + if (m_is_region_free[i]) { + m_is_region_free[i] = false; + return this->GetRegionAddress(i); + } + } + + return 0; +} + +void KThreadLocalPage::Release(VAddr addr) { + m_is_region_free[this->GetRegionIndex(addr)] = true; +} + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_thread_local_page.h b/src/core/hle/kernel/k_thread_local_page.h new file mode 100644 index 000000000..74b565a71 --- /dev/null +++ b/src/core/hle/kernel/k_thread_local_page.h @@ -0,0 +1,111 @@ +// Copyright 2022 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include <algorithm> +#include <array> + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/common_types.h" +#include "common/intrusive_red_black_tree.h" +#include "core/hle/kernel/memory_types.h" +#include "core/hle/kernel/slab_helpers.h" +#include "core/hle/result.h" + +namespace Kernel { + +class KernelCore; +class KProcess; + +class KThreadLocalPage final : public Common::IntrusiveRedBlackTreeBaseNode<KThreadLocalPage>, + public KSlabAllocated<KThreadLocalPage> { +public: + static constexpr size_t RegionsPerPage = PageSize / Svc::ThreadLocalRegionSize; + static_assert(RegionsPerPage > 0); + +public: + constexpr explicit KThreadLocalPage(VAddr addr = {}) : m_virt_addr(addr) { + m_is_region_free.fill(true); + } + + constexpr VAddr GetAddress() const { + return m_virt_addr; + } + + ResultCode Initialize(KernelCore& kernel, KProcess* process); + ResultCode Finalize(); + + VAddr Reserve(); + void Release(VAddr addr); + + bool IsAllUsed() const { + return std::ranges::all_of(m_is_region_free.begin(), m_is_region_free.end(), + [](bool is_free) { return !is_free; }); + } + + bool IsAllFree() const { + return std::ranges::all_of(m_is_region_free.begin(), m_is_region_free.end(), + [](bool is_free) { return is_free; }); + } + + bool IsAnyUsed() const { + return !this->IsAllFree(); + } + + bool IsAnyFree() const { + return !this->IsAllUsed(); + } + +public: + using RedBlackKeyType = VAddr; + + static constexpr RedBlackKeyType GetRedBlackKey(const RedBlackKeyType& v) { + return v; + } + static constexpr RedBlackKeyType GetRedBlackKey(const KThreadLocalPage& v) { + return v.GetAddress(); + } + + template <typename T> + requires(std::same_as<T, KThreadLocalPage> || + std::same_as<T, RedBlackKeyType>) static constexpr int Compare(const T& lhs, + const KThreadLocalPage& + rhs) { + const VAddr lval = GetRedBlackKey(lhs); + const VAddr rval = GetRedBlackKey(rhs); + + if (lval < rval) { + return -1; + } else if (lval == rval) { + return 0; + } else { + return 1; + } + } + +private: + constexpr VAddr GetRegionAddress(size_t i) const { + return this->GetAddress() + i * Svc::ThreadLocalRegionSize; + } + + constexpr bool Contains(VAddr addr) const { + return this->GetAddress() <= addr && addr < this->GetAddress() + PageSize; + } + + constexpr size_t GetRegionIndex(VAddr addr) const { + ASSERT(Common::IsAligned(addr, Svc::ThreadLocalRegionSize)); + ASSERT(this->Contains(addr)); + return (addr - this->GetAddress()) / Svc::ThreadLocalRegionSize; + } + +private: + VAddr m_virt_addr{}; + KProcess* m_owner{}; + KernelCore* m_kernel{}; + std::array<bool, RegionsPerPage> m_is_region_free{}; +}; + +} // namespace Kernel diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index 71bd466cf..d840d44e6 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -22,9 +22,7 @@ #include "core/arm/exclusive_monitor.h" #include "core/core.h" #include "core/core_timing.h" -#include "core/core_timing_util.h" #include "core/cpu_manager.h" -#include "core/device_memory.h" #include "core/hardware_properties.h" #include "core/hle/kernel/init/init_slab_setup.h" #include "core/hle/kernel/k_client_port.h" @@ -35,7 +33,6 @@ #include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_scheduler.h" #include "core/hle/kernel/k_shared_memory.h" -#include "core/hle/kernel/k_slab_heap.h" #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_worker_task_manager.h" #include "core/hle/kernel/kernel.h" @@ -52,7 +49,7 @@ namespace Kernel { struct KernelCore::Impl { explicit Impl(Core::System& system_, KernelCore& kernel_) - : time_manager{system_}, object_list_container{kernel_}, + : time_manager{system_}, service_threads_manager{1, "yuzu:ServiceThreadsManager"}, system{system_} {} void SetMulticore(bool is_multi) { @@ -60,9 +57,11 @@ struct KernelCore::Impl { } void Initialize(KernelCore& kernel) { + global_object_list_container = std::make_unique<KAutoObjectWithListContainer>(kernel); global_scheduler_context = std::make_unique<Kernel::GlobalSchedulerContext>(kernel); global_handle_table = std::make_unique<Kernel::KHandleTable>(kernel); global_handle_table->Initialize(KHandleTable::MaxTableSize); + default_service_thread = CreateServiceThread(kernel, "DefaultServiceThread"); is_phantom_mode_for_singlecore = false; @@ -76,7 +75,7 @@ struct KernelCore::Impl { // Initialize kernel memory and resources. InitializeSystemResourceLimit(kernel, system.CoreTiming()); InitializeMemoryLayout(); - InitializePageSlab(); + Init::InitializeKPageBufferSlabHeap(system); InitializeSchedulers(); InitializeSuspendThreads(); InitializePreemption(kernel); @@ -86,7 +85,7 @@ struct KernelCore::Impl { void InitializeCores() { for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) { - cores[core_id].Initialize(current_process->Is64BitProcess()); + cores[core_id].Initialize((*current_process).Is64BitProcess()); system.Memory().SetCurrentPageTable(*current_process, core_id); } } @@ -97,30 +96,17 @@ struct KernelCore::Impl { process_list.clear(); - // Close all open server ports. - std::unordered_set<KServerPort*> server_ports_; + // Close all open server sessions and ports. + std::unordered_set<KAutoObject*> server_objects_; { - std::lock_guard lk(server_ports_lock); - server_ports_ = server_ports; - server_ports.clear(); + std::scoped_lock lk(server_objects_lock); + server_objects_ = server_objects; + server_objects.clear(); } - for (auto* server_port : server_ports_) { - server_port->Close(); - } - // Close all open server sessions. - std::unordered_set<KServerSession*> server_sessions_; - { - std::lock_guard lk(server_sessions_lock); - server_sessions_ = server_sessions; - server_sessions.clear(); - } - for (auto* server_session : server_sessions_) { - server_session->Close(); + for (auto* server_object : server_objects_) { + server_object->Close(); } - // Ensure that the object list container is finalized and properly shutdown. - object_list_container.Finalize(); - // Ensures all service threads gracefully shutdown. ClearServiceThreads(); @@ -171,7 +157,7 @@ struct KernelCore::Impl { // Close kernel objects that were not freed on shutdown { - std::lock_guard lk(registered_in_use_objects_lock); + std::scoped_lock lk{registered_in_use_objects_lock}; if (registered_in_use_objects.size()) { for (auto& object : registered_in_use_objects) { object->Close(); @@ -182,23 +168,27 @@ struct KernelCore::Impl { // Shutdown all processes. if (current_process) { - current_process->Finalize(); + (*current_process).Finalize(); // current_process->Close(); // TODO: The current process should be destroyed based on accurate ref counting after // calling Close(). Adding a manual Destroy() call instead to avoid a memory leak. - current_process->Destroy(); + (*current_process).Destroy(); current_process = nullptr; } // Track kernel objects that were not freed on shutdown { - std::lock_guard lk(registered_objects_lock); + std::scoped_lock lk{registered_objects_lock}; if (registered_objects.size()) { - LOG_WARNING(Kernel, "{} kernel objects were dangling on shutdown!", - registered_objects.size()); + LOG_DEBUG(Kernel, "{} kernel objects were dangling on shutdown!", + registered_objects.size()); registered_objects.clear(); } } + + // Ensure that the object list container is finalized and properly shutdown. + global_object_list_container->Finalize(); + global_object_list_container.reset(); } void InitializePhysicalCores() { @@ -291,15 +281,16 @@ struct KernelCore::Impl { // Gets the dummy KThread for the caller, allocating a new one if this is the first time KThread* GetHostDummyThread() { - auto make_thread = [this]() { - KThread* thread = KThread::Create(system.Kernel()); + auto initialize = [this](KThread* thread) { ASSERT(KThread::InitializeDummyThread(thread).IsSuccess()); thread->SetName(fmt::format("DummyThread:{}", GetHostThreadId())); return thread; }; - thread_local KThread* saved_thread = make_thread(); - return saved_thread; + thread_local auto raw_thread = KThread(system.Kernel()); + thread_local auto thread = initialize(&raw_thread); + + return thread; } /// Registers a CPU core thread by allocating a host thread ID for it @@ -660,22 +651,6 @@ struct KernelCore::Impl { time_phys_addr, time_size, "Time:SharedMemory"); } - void InitializePageSlab() { - // Allocate slab heaps - user_slab_heap_pages = - std::make_unique<KSlabHeap<Page>>(KSlabHeap<Page>::AllocationType::Guest); - - // TODO(ameerj): This should be derived, not hardcoded within the kernel - constexpr u64 user_slab_heap_size{0x3de000}; - // Reserve slab heaps - ASSERT( - system_resource_limit->Reserve(LimitableResource::PhysicalMemory, user_slab_heap_size)); - // Initialize slab heap - user_slab_heap_pages->Initialize( - system.DeviceMemory().GetPointer(Core::DramMemoryMap::SlabHeapBase), - user_slab_heap_size); - } - KClientPort* CreateNamedServicePort(std::string name) { auto search = service_interface_factory.find(name); if (search == service_interface_factory.end()) { @@ -684,13 +659,20 @@ struct KernelCore::Impl { } KClientPort* port = &search->second(system.ServiceManager(), system); - { - std::lock_guard lk(server_ports_lock); - server_ports.insert(&port->GetParent()->GetServerPort()); - } + RegisterServerObject(&port->GetParent()->GetServerPort()); return port; } + void RegisterServerObject(KAutoObject* server_object) { + std::scoped_lock lk(server_objects_lock); + server_objects.insert(server_object); + } + + void UnregisterServerObject(KAutoObject* server_object) { + std::scoped_lock lk(server_objects_lock); + server_objects.erase(server_object); + } + std::weak_ptr<Kernel::ServiceThread> CreateServiceThread(KernelCore& kernel, const std::string& name) { auto service_thread = std::make_shared<Kernel::ServiceThread>(kernel, 1, name); @@ -703,6 +685,12 @@ struct KernelCore::Impl { void ReleaseServiceThread(std::weak_ptr<Kernel::ServiceThread> service_thread) { if (auto strong_ptr = service_thread.lock()) { + if (strong_ptr == default_service_thread.lock()) { + // Nothing to do here, the service is using default_service_thread, which will be + // released on shutdown. + return; + } + service_threads_manager.QueueWork( [this, strong_ptr{std::move(strong_ptr)}]() { service_threads.erase(strong_ptr); }); } @@ -712,8 +700,7 @@ struct KernelCore::Impl { service_threads_manager.QueueWork([this]() { service_threads.clear(); }); } - std::mutex server_ports_lock; - std::mutex server_sessions_lock; + std::mutex server_objects_lock; std::mutex registered_objects_lock; std::mutex registered_in_use_objects_lock; @@ -724,7 +711,7 @@ struct KernelCore::Impl { // Lists all processes that exist in the current session. std::vector<KProcess*> process_list; - KProcess* current_process{}; + std::atomic<KProcess*> current_process{}; std::unique_ptr<Kernel::GlobalSchedulerContext> global_scheduler_context; Kernel::TimeManager time_manager; @@ -737,14 +724,13 @@ struct KernelCore::Impl { // stores all the objects in place. std::unique_ptr<KHandleTable> global_handle_table; - KAutoObjectWithListContainer object_list_container; + std::unique_ptr<KAutoObjectWithListContainer> global_object_list_container; /// Map of named ports managed by the kernel, which can be retrieved using /// the ConnectToPort SVC. std::unordered_map<std::string, ServiceInterfaceFactory> service_interface_factory; NamedPortTable named_ports; - std::unordered_set<KServerPort*> server_ports; - std::unordered_set<KServerSession*> server_sessions; + std::unordered_set<KAutoObject*> server_objects; std::unordered_set<KAutoObject*> registered_objects; std::unordered_set<KAutoObject*> registered_in_use_objects; @@ -756,7 +742,6 @@ struct KernelCore::Impl { // Kernel memory management std::unique_ptr<KMemoryManager> memory_manager; - std::unique_ptr<KSlabHeap<Page>> user_slab_heap_pages; // Shared memory for services Kernel::KSharedMemory* hid_shared_mem{}; @@ -768,7 +753,8 @@ struct KernelCore::Impl { std::unique_ptr<KMemoryLayout> memory_layout; // Threads used for services - std::unordered_set<std::shared_ptr<Kernel::ServiceThread>> service_threads; + std::unordered_set<std::shared_ptr<ServiceThread>> service_threads; + std::weak_ptr<ServiceThread> default_service_thread; Common::ThreadWorker service_threads_manager; std::array<KThread*, Core::Hardware::NUM_CPU_CORES> suspend_threads; @@ -915,11 +901,11 @@ const Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() const { } KAutoObjectWithListContainer& KernelCore::ObjectListContainer() { - return impl->object_list_container; + return *impl->global_object_list_container; } const KAutoObjectWithListContainer& KernelCore::ObjectListContainer() const { - return impl->object_list_container; + return *impl->global_object_list_container; } void KernelCore::InvalidateAllInstructionCaches() { @@ -949,33 +935,31 @@ KClientPort* KernelCore::CreateNamedServicePort(std::string name) { return impl->CreateNamedServicePort(std::move(name)); } -void KernelCore::RegisterServerSession(KServerSession* server_session) { - std::lock_guard lk(impl->server_sessions_lock); - impl->server_sessions.insert(server_session); +void KernelCore::RegisterServerObject(KAutoObject* server_object) { + impl->RegisterServerObject(server_object); } -void KernelCore::UnregisterServerSession(KServerSession* server_session) { - std::lock_guard lk(impl->server_sessions_lock); - impl->server_sessions.erase(server_session); +void KernelCore::UnregisterServerObject(KAutoObject* server_object) { + impl->UnregisterServerObject(server_object); } void KernelCore::RegisterKernelObject(KAutoObject* object) { - std::lock_guard lk(impl->registered_objects_lock); + std::scoped_lock lk{impl->registered_objects_lock}; impl->registered_objects.insert(object); } void KernelCore::UnregisterKernelObject(KAutoObject* object) { - std::lock_guard lk(impl->registered_objects_lock); + std::scoped_lock lk{impl->registered_objects_lock}; impl->registered_objects.erase(object); } void KernelCore::RegisterInUseObject(KAutoObject* object) { - std::lock_guard lk(impl->registered_in_use_objects_lock); + std::scoped_lock lk{impl->registered_in_use_objects_lock}; impl->registered_in_use_objects.insert(object); } void KernelCore::UnregisterInUseObject(KAutoObject* object) { - std::lock_guard lk(impl->registered_in_use_objects_lock); + std::scoped_lock lk{impl->registered_in_use_objects_lock}; impl->registered_in_use_objects.erase(object); } @@ -1031,14 +1015,6 @@ const KMemoryManager& KernelCore::MemoryManager() const { return *impl->memory_manager; } -KSlabHeap<Page>& KernelCore::GetUserSlabHeapPages() { - return *impl->user_slab_heap_pages; -} - -const KSlabHeap<Page>& KernelCore::GetUserSlabHeapPages() const { - return *impl->user_slab_heap_pages; -} - Kernel::KSharedMemory& KernelCore::GetHidSharedMem() { return *impl->hid_shared_mem; } @@ -1112,6 +1088,10 @@ std::weak_ptr<Kernel::ServiceThread> KernelCore::CreateServiceThread(const std:: return impl->CreateServiceThread(*this, name); } +std::weak_ptr<Kernel::ServiceThread> KernelCore::GetDefaultServiceThread() const { + return impl->default_service_thread; +} + void KernelCore::ReleaseServiceThread(std::weak_ptr<Kernel::ServiceThread> service_thread) { impl->ReleaseServiceThread(service_thread); } diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h index c1254b18d..d709c368b 100644 --- a/src/core/hle/kernel/kernel.h +++ b/src/core/hle/kernel/kernel.h @@ -14,7 +14,6 @@ #include "core/hardware_properties.h" #include "core/hle/kernel/k_auto_object.h" #include "core/hle/kernel/k_slab_heap.h" -#include "core/hle/kernel/memory_types.h" #include "core/hle/kernel/svc_common.h" namespace Core { @@ -43,6 +42,7 @@ class KHandleTable; class KLinkedListNode; class KMemoryLayout; class KMemoryManager; +class KPageBuffer; class KPort; class KProcess; class KResourceLimit; @@ -52,6 +52,7 @@ class KSession; class KSharedMemory; class KSharedMemoryInfo; class KThread; +class KThreadLocalPage; class KTransferMemory; class KWorkerTaskManager; class KWritableEvent; @@ -194,13 +195,13 @@ public: /// Opens a port to a service previously registered with RegisterNamedService. KClientPort* CreateNamedServicePort(std::string name); - /// Registers a server session with the gobal emulation state, to be freed on shutdown. This is - /// necessary because we do not emulate processes for HLE sessions. - void RegisterServerSession(KServerSession* server_session); + /// Registers a server session or port with the gobal emulation state, to be freed on shutdown. + /// This is necessary because we do not emulate processes for HLE sessions and ports. + void RegisterServerObject(KAutoObject* server_object); - /// Unregisters a server session previously registered with RegisterServerSession when it was - /// destroyed during the current emulation session. - void UnregisterServerSession(KServerSession* server_session); + /// Unregisters a server session or port previously registered with RegisterServerSession when + /// it was destroyed during the current emulation session. + void UnregisterServerObject(KAutoObject* server_object); /// Registers all kernel objects with the global emulation state, this is purely for tracking /// leaks after emulation has been shutdown. @@ -239,12 +240,6 @@ public: /// Gets the virtual memory manager for the kernel. const KMemoryManager& MemoryManager() const; - /// Gets the slab heap allocated for user space pages. - KSlabHeap<Page>& GetUserSlabHeapPages(); - - /// Gets the slab heap allocated for user space pages. - const KSlabHeap<Page>& GetUserSlabHeapPages() const; - /// Gets the shared memory object for HID services. Kernel::KSharedMemory& GetHidSharedMem(); @@ -284,9 +279,11 @@ public: void ExitSVCProfile(); /** - * Creates an HLE service thread, which are used to execute service routines asynchronously. - * While these are allocated per ServerSession, these need to be owned and managed outside - * of ServerSession to avoid a circular dependency. + * Creates a host thread to execute HLE service requests, which are used to execute service + * routines asynchronously. While these are allocated per ServerSession, these need to be owned + * and managed outside of ServerSession to avoid a circular dependency. In general, most + * services can just use the default service thread, and not need their own host service thread. + * See GetDefaultServiceThread. * @param name String name for the ServerSession creating this thread, used for debug * purposes. * @returns The a weak pointer newly created service thread. @@ -294,6 +291,14 @@ public: std::weak_ptr<Kernel::ServiceThread> CreateServiceThread(const std::string& name); /** + * Gets the default host service thread, which executes HLE service requests. Unless service + * requests need to block on the host, the default service thread should be used in favor of + * creating a new service thread. + * @returns The a weak pointer for the default service thread. + */ + std::weak_ptr<Kernel::ServiceThread> GetDefaultServiceThread() const; + + /** * Releases a HLE service thread, instructing KernelCore to free it. This should be called when * the ServerSession associated with the thread is destroyed. * @param service_thread Service thread to release. @@ -336,6 +341,10 @@ public: return slab_heap_container->writeable_event; } else if constexpr (std::is_same_v<T, KCodeMemory>) { return slab_heap_container->code_memory; + } else if constexpr (std::is_same_v<T, KPageBuffer>) { + return slab_heap_container->page_buffer; + } else if constexpr (std::is_same_v<T, KThreadLocalPage>) { + return slab_heap_container->thread_local_page; } } @@ -397,6 +406,8 @@ private: KSlabHeap<KTransferMemory> transfer_memory; KSlabHeap<KWritableEvent> writeable_event; KSlabHeap<KCodeMemory> code_memory; + KSlabHeap<KPageBuffer> page_buffer; + KSlabHeap<KThreadLocalPage> thread_local_page; }; std::unique_ptr<SlabHeapContainer> slab_heap_container; diff --git a/src/core/hle/kernel/physical_core.cpp b/src/core/hle/kernel/physical_core.cpp index 7477668e4..18a5f40f8 100644 --- a/src/core/hle/kernel/physical_core.cpp +++ b/src/core/hle/kernel/physical_core.cpp @@ -58,6 +58,7 @@ bool PhysicalCore::IsInterrupted() const { void PhysicalCore::Interrupt() { guard->lock(); interrupts[core_index].SetInterrupt(true); + arm_interface->SignalInterrupt(); guard->unlock(); } diff --git a/src/core/hle/kernel/service_thread.cpp b/src/core/hle/kernel/service_thread.cpp index 4eb3a5988..52d25b837 100644 --- a/src/core/hle/kernel/service_thread.cpp +++ b/src/core/hle/kernel/service_thread.cpp @@ -49,12 +49,9 @@ ServiceThread::Impl::Impl(KernelCore& kernel, std::size_t num_threads, const std return; } + // Allocate a dummy guest thread for this host thread. kernel.RegisterHostThread(); - // Ensure the dummy thread allocated for this host thread is closed on exit. - auto* dummy_thread = kernel.GetCurrentEmuThread(); - SCOPE_EXIT({ dummy_thread->Close(); }); - while (true) { std::function<void()> task; diff --git a/src/core/hle/kernel/slab_helpers.h b/src/core/hle/kernel/slab_helpers.h index f1c11256e..dc1e48fc9 100644 --- a/src/core/hle/kernel/slab_helpers.h +++ b/src/core/hle/kernel/slab_helpers.h @@ -59,7 +59,7 @@ class KAutoObjectWithSlabHeapAndContainer : public Base { private: static Derived* Allocate(KernelCore& kernel) { - return kernel.SlabHeap<Derived>().AllocateWithKernel(kernel); + return kernel.SlabHeap<Derived>().Allocate(kernel); } static void Free(KernelCore& kernel, Derived* obj) { diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 839171e85..0c86435b5 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -1362,8 +1362,11 @@ static ResultCode MapProcessMemory(Core::System& system, VAddr dst_address, Hand ResultInvalidMemoryRegion); // Create a new page group. - KMemoryInfo kBlockInfo = dst_pt.QueryInfo(dst_address); - KPageLinkedList pg(kBlockInfo.GetAddress(), kBlockInfo.GetNumPages()); + KPageLinkedList pg; + R_TRY(src_pt.MakeAndOpenPageGroup( + std::addressof(pg), src_address, size / PageSize, KMemoryState::FlagCanMapProcess, + KMemoryState::FlagCanMapProcess, KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::All, KMemoryAttribute::None)); // Map the group. R_TRY(dst_pt.MapPages(dst_address, pg, KMemoryState::SharedCode, @@ -1408,8 +1411,8 @@ static ResultCode UnmapProcessMemory(Core::System& system, VAddr dst_address, Ha } static ResultCode CreateCodeMemory(Core::System& system, Handle* out, VAddr address, size_t size) { - LOG_TRACE(Kernel_SVC, "called, handle_out={}, address=0x{:X}, size=0x{:X}", - static_cast<void*>(out), address, size); + LOG_TRACE(Kernel_SVC, "called, address=0x{:X}, size=0x{:X}", address, size); + // Get kernel instance. auto& kernel = system.Kernel(); @@ -1664,7 +1667,7 @@ static ResultCode UnmapProcessCodeMemory(Core::System& system, Handle process_ha return ResultInvalidAddress; } - if (size == 0 || Common::Is4KBAligned(size)) { + if (size == 0 || !Common::Is4KBAligned(size)) { LOG_ERROR(Kernel_SVC, "Size is zero or not page-aligned (size=0x{:016X}).", size); return ResultInvalidSize; } @@ -1710,7 +1713,8 @@ static ResultCode UnmapProcessCodeMemory(Core::System& system, Handle process_ha return ResultInvalidMemoryRegion; } - return page_table.UnmapCodeMemory(dst_address, src_address, size); + return page_table.UnmapCodeMemory(dst_address, src_address, size, + KPageTable::ICacheInvalidationStrategy::InvalidateAll); } /// Exits the current process diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h index 365e22e4e..b2e9ec092 100644 --- a/src/core/hle/kernel/svc_types.h +++ b/src/core/hle/kernel/svc_types.h @@ -96,4 +96,6 @@ constexpr inline s32 IdealCoreNoUpdate = -3; constexpr inline s32 LowestThreadPriority = 63; constexpr inline s32 HighestThreadPriority = 0; +constexpr inline size_t ThreadLocalRegionSize = 0x200; + } // namespace Kernel::Svc diff --git a/src/core/hle/kernel/time_manager.cpp b/src/core/hle/kernel/time_manager.cpp index aa985d820..5b8fe8eae 100644 --- a/src/core/hle/kernel/time_manager.cpp +++ b/src/core/hle/kernel/time_manager.cpp @@ -24,7 +24,7 @@ TimeManager::TimeManager(Core::System& system_) : system{system_} { } void TimeManager::ScheduleTimeEvent(KThread* thread, s64 nanoseconds) { - std::lock_guard lock{mutex}; + std::scoped_lock lock{mutex}; if (nanoseconds > 0) { ASSERT(thread); ASSERT(thread->GetState() != ThreadState::Runnable); @@ -35,7 +35,7 @@ void TimeManager::ScheduleTimeEvent(KThread* thread, s64 nanoseconds) { } void TimeManager::UnscheduleTimeEvent(KThread* thread) { - std::lock_guard lock{mutex}; + std::scoped_lock lock{mutex}; system.CoreTiming().UnscheduleEvent(time_manager_event_type, reinterpret_cast<uintptr_t>(thread)); } |